Compare commits

..

3 commits

Author SHA1 Message Date
c6cd364b9e test: add music_charts test 2022-12-01 14:13:34 +01:00
88104753e0 feat: add Zz (global) country param 2022-12-01 13:31:36 +01:00
f20ea693a6 feat: add music charts 2022-12-01 12:56:58 +01:00
19 changed files with 83673 additions and 56 deletions

View file

@ -27,5 +27,5 @@ inspired by [NewPipe](https://github.com/TeamNewPipe/NewPipeExtractor).
- [X] **Radio**
- [X] **Track details** (lyrics, recommendations)
- [ ] **Moods**
- [ ] **Charts**
- [X] **Charts**
- [X] **New**

View file

@ -7,7 +7,10 @@ use std::{
use rustypipe::{
client::{ClientType, RustyPipe},
param::search_filter::{self, Entity, SearchFilter},
param::{
search_filter::{self, Entity, SearchFilter},
Country,
},
report::{Report, Reporter},
};
@ -57,6 +60,7 @@ pub async fn download_testfiles(project_root: &Path) {
music_radio_cont(&testfiles).await;
music_new_albums(&testfiles).await;
music_new_videos(&testfiles).await;
music_charts(&testfiles).await;
}
const CLIENT_TYPES: [ClientType; 5] = [
@ -827,3 +831,17 @@ async fn music_new_videos(testfiles: &Path) {
let rp = rp_testfile(&json_path);
rp.query().music_new_videos().await.unwrap();
}
async fn music_charts(testfiles: &Path) {
for (name, country) in [("global", Some(Country::Zz)), ("US", Some(Country::Us))] {
let mut json_path = testfiles.to_path_buf();
json_path.push("music_charts");
json_path.push(&format!("charts_{}.json", name));
if json_path.exists() {
continue;
}
let rp = rp_testfile(&json_path);
rp.query().music_charts(country).await.unwrap();
}
}

View file

@ -183,6 +183,7 @@ impl FromStr for Country {
let mut code_langs = r#"/// Available languages
#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[serde(rename_all = "lowercase")]
#[non_exhaustive]
pub enum Language {
"#
.to_owned();
@ -190,6 +191,7 @@ pub enum Language {
let mut code_countries = r#"/// Available countries
#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[serde(rename_all = "UPPERCASE")]
#[non_exhaustive]
pub enum Country {
"#
.to_owned();
@ -233,22 +235,23 @@ pub enum Country {
.collect::<String>();
// Language enum
let _ = write!(code_langs, " /// {}\n ", n);
write!(code_langs, " /// {}\n ", n).unwrap();
if c.contains('-') {
let _ = write!(code_langs, "#[serde(rename = \"{}\")]\n ", c);
write!(code_langs, "#[serde(rename = \"{}\")]\n ", c).unwrap();
}
code_langs += &enum_name;
code_langs += ",\n";
// Language array
let _ = writeln!(code_lang_array, " Language::{},", enum_name);
writeln!(code_lang_array, " Language::{},", enum_name).unwrap();
// Language names
let _ = writeln!(
writeln!(
code_lang_names,
" Language::{} => \"{}\",",
enum_name, n
);
)
.unwrap();
});
code_langs += "}\n";
@ -256,19 +259,26 @@ pub enum Country {
let enum_name = c[0..1].to_owned().to_uppercase() + &c[1..].to_owned().to_lowercase();
// Country enum
let _ = writeln!(code_countries, " /// {}", n);
let _ = writeln!(code_countries, " {},", enum_name);
writeln!(code_countries, " /// {}", n).unwrap();
writeln!(code_countries, " {},", enum_name).unwrap();
// Country array
let _ = writeln!(code_country_array, " Country::{},", enum_name);
writeln!(code_country_array, " Country::{},", enum_name).unwrap();
// Country names
let _ = writeln!(
writeln!(
code_country_names,
" Country::{} => \"{}\",",
enum_name, n
);
)
.unwrap();
});
// Add Country::Zz / Global
code_countries += " /// Global (can only be used for music charts)\n";
code_countries += " Zz,\n";
code_country_names += " Country::Zz => \"Global\",\n";
code_countries += "}\n";
code_lang_array += "];\n";

View file

@ -4,6 +4,7 @@ pub(crate) mod response;
mod channel;
mod music_artist;
mod music_charts;
mod music_details;
mod music_new;
mod music_playlist;
@ -420,7 +421,7 @@ impl RustyPipeBuilder {
///
/// **Info**: you can set this option for individual queries, too
pub fn country(mut self, country: Country) -> Self {
self.default_opts.country = country;
self.default_opts.country = validate_country(country);
self
}
@ -758,7 +759,7 @@ impl RustyPipeQuery {
/// Set the country parameter used when accessing the YouTube API.
/// This will change trends and recommended content.
pub fn country(mut self, country: Country) -> Self {
self.opts.country = country;
self.opts.country = validate_country(country);
self
}
@ -1193,6 +1194,15 @@ trait MapResponse<T> {
) -> Result<MapResult<T>, ExtractionError>;
}
fn validate_country(country: Country) -> Country {
if country == Country::Zz {
warn!("Country:Zz (Global) can only be used for fetching music charts, falling back to Country:Us");
Country::Us
} else {
country
}
}
#[cfg(test)]
mod tests {
use super::*;

View file

@ -192,9 +192,9 @@ fn map_artist_page(
mapper.map_response(shelf.contents);
}
response::music_item::ItemSection::MusicCarouselShelfRenderer { header, contents } => {
response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => {
let mut extendable_albums = false;
if let Some(h) = header {
if let Some(h) = shelf.header {
if let Some(button) = h
.music_carousel_shelf_basic_header_renderer
.more_content_button
@ -221,7 +221,7 @@ fn map_artist_page(
}
if !skip_extendables || !extendable_albums {
mapper.map_response(contents);
mapper.map_response(shelf.contents);
}
}
response::music_item::ItemSection::None => {}

171
src/client/music_charts.rs Normal file
View file

@ -0,0 +1,171 @@
use std::borrow::Cow;
use serde::Serialize;
use crate::{
error::{Error, ExtractionError},
model::{MusicCharts, TrackItem},
param::Country,
serializer::MapResult,
};
use super::{
response::{self, music_item::MusicListMapper, url_endpoint::MusicPageType},
ClientType, MapResponse, RustyPipeQuery, YTContext,
};
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct QCharts<'a> {
context: YTContext<'a>,
browse_id: &'a str,
params: &'a str,
#[serde(skip_serializing_if = "Option::is_none")]
form_data: Option<FormData>,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct FormData {
pub selected_values: [Country; 1],
}
impl RustyPipeQuery {
pub async fn music_charts(&self, country: Option<Country>) -> Result<MusicCharts, Error> {
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
let request_body = QCharts {
context,
browse_id: "FEmusic_charts",
params: "sgYPRkVtdXNpY19leHBsb3Jl",
form_data: country.map(|c| FormData {
selected_values: [c],
}),
};
self.execute_request::<response::MusicCharts, _, _>(
ClientType::DesktopMusic,
"music_charts",
"",
"browse",
&request_body,
)
.await
}
}
impl MapResponse<MusicCharts> for response::MusicCharts {
fn map_response(
self,
_id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<crate::serializer::MapResult<MusicCharts>, crate::error::ExtractionError> {
let countries = self
.framework_updates
.map(|fwu| {
fwu.entity_batch_update
.mutations
.into_iter()
.map(|x| x.payload.music_form_boolean_choice.opaque_token)
.collect()
})
.unwrap_or_default();
let mut top_playlist_id = None;
let mut trending_playlist_id = None;
let mut mapper_top = MusicListMapper::new(lang);
let mut mapper_trending = MusicListMapper::new(lang);
let mut mapper_other = MusicListMapper::new(lang);
self.contents
.single_column_browse_results_renderer
.contents
.into_iter()
.next()
.ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))?
.tab_renderer
.content
.section_list_renderer
.contents
.into_iter()
.for_each(|s| match s {
response::music_charts::ItemSection::MusicCarouselShelfRenderer(shelf) => {
match shelf.header.and_then(|h| {
h.music_carousel_shelf_basic_header_renderer
.more_content_button
.and_then(|btn| btn.button_renderer.navigation_endpoint.music_page())
}) {
Some((MusicPageType::Playlist, id)) => {
// Top music videos (first shelf with associated playlist)
if top_playlist_id.is_none() {
mapper_top.map_response(shelf.contents);
top_playlist_id = Some(id);
}
// Trending (second shelf with associated playlist)
else if trending_playlist_id.is_none() {
mapper_trending.map_response(shelf.contents);
trending_playlist_id = Some(id);
}
}
// Other sections (artists, playlists)
_ => {
mapper_other.map_response(shelf.contents);
}
}
}
response::music_charts::ItemSection::None => {}
});
let mapped_top = mapper_top.conv_items::<TrackItem>();
let mut mapped_trending = mapper_trending.conv_items::<TrackItem>();
let mut mapped_other = mapper_other.group_items();
let mut warnings = mapped_top.warnings;
warnings.append(&mut mapped_trending.warnings);
warnings.append(&mut mapped_other.warnings);
Ok(MapResult {
c: MusicCharts {
top_tracks: mapped_top.c,
trending_tracks: mapped_trending.c,
artists: mapped_other.c.artists,
playlists: mapped_other.c.playlists,
top_playlist_id,
trending_playlist_id,
available_countries: countries,
},
warnings,
})
}
}
#[cfg(test)]
mod tests {
use std::{fs::File, io::BufReader, path::Path};
use rstest::rstest;
use super::*;
use crate::param::Language;
#[rstest]
#[case::default("global")]
#[case::us("US")]
fn map_music_charts(#[case] name: &str) {
let filename = format!("testfiles/music_charts/charts_{}.json", name);
let json_path = Path::new(&filename);
let json_file = File::open(json_path).unwrap();
let charts: response::MusicCharts =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicCharts> = charts.map_response("", Language::En, None).unwrap();
assert!(
map_res.warnings.is_empty(),
"deserialization/mapping warnings: {:?}",
map_res.warnings
);
insta::assert_ron_snapshot!(format!("map_music_charts_{}", name), map_res.c);
}
}

View file

@ -315,22 +315,22 @@ impl MapResponse<MusicRelated> for response::MusicRelated {
.iter()
.find_map(|section| match section {
response::music_item::ItemSection::MusicShelfRenderer(_) => None,
response::music_item::ItemSection::MusicCarouselShelfRenderer {
header, ..
} => header.as_ref().and_then(|h| {
h.music_carousel_shelf_basic_header_renderer
.title
.0
.iter()
.find_map(|c| {
let artist = ArtistId::from(c.clone());
if artist.id.is_some() {
Some(artist)
} else {
None
}
})
}),
response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => {
shelf.header.as_ref().and_then(|h| {
h.music_carousel_shelf_basic_header_renderer
.title
.0
.iter()
.find_map(|c| {
let artist = ArtistId::from(c.clone());
if artist.id.is_some() {
Some(artist)
} else {
None
}
})
})
}
response::music_item::ItemSection::None => None,
});
@ -341,20 +341,18 @@ impl MapResponse<MusicRelated> for response::MusicRelated {
};
let mut sections = self.contents.section_list_renderer.contents.into_iter();
if let Some(response::music_item::ItemSection::MusicCarouselShelfRenderer {
contents,
..
}) = sections.next()
if let Some(response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf)) =
sections.next()
{
mapper_tracks.map_response(contents);
mapper_tracks.map_response(shelf.contents);
}
sections.for_each(|section| match section {
response::music_item::ItemSection::MusicShelfRenderer(shelf) => {
mapper.map_response(shelf.contents);
}
response::music_item::ItemSection::MusicCarouselShelfRenderer { contents, .. } => {
mapper.map_response(contents);
response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => {
mapper.map_response(shelf.contents);
}
response::music_item::ItemSection::None => {}
});

View file

@ -264,9 +264,9 @@ impl MapResponse<MusicAlbum> for response::MusicPlaylist {
for section in sections {
match section {
response::music_item::ItemSection::MusicShelfRenderer(sh) => shelf = Some(sh),
response::music_item::ItemSection::MusicCarouselShelfRenderer {
contents, ..
} => album_variants = Some(contents),
response::music_item::ItemSection::MusicCarouselShelfRenderer(sh) => {
album_variants = Some(sh.contents)
}
response::music_item::ItemSection::None => (),
}
}

View file

@ -143,11 +143,8 @@ impl MapResponse<Paginator<MusicItem>> for response::MusicContinuation {
mapper.map_response(shelf.contents);
continuations.append(&mut shelf.continuations);
}
response::music_item::ItemSection::MusicCarouselShelfRenderer {
contents,
..
} => {
mapper.map_response(contents);
response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => {
mapper.map_response(shelf.contents);
}
response::music_item::ItemSection::None => {}
}

View file

@ -1,5 +1,6 @@
pub(crate) mod channel;
pub(crate) mod music_artist;
pub(crate) mod music_charts;
pub(crate) mod music_details;
pub(crate) mod music_item;
pub(crate) mod music_new;
@ -16,6 +17,7 @@ pub(crate) mod video_item;
pub(crate) use channel::Channel;
pub(crate) use music_artist::MusicArtist;
pub(crate) use music_artist::MusicArtistAlbums;
pub(crate) use music_charts::MusicCharts;
pub(crate) use music_details::MusicDetails;
pub(crate) use music_details::MusicLyrics;
pub(crate) use music_details::MusicRelated;

View file

@ -0,0 +1,60 @@
use serde::Deserialize;
use serde_with::{rust::deserialize_ignore_any, serde_as, VecSkipError};
use crate::param::Country;
use super::{music_item::MusicCarouselShelf, ContentsRenderer, SectionList, Tab};
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct MusicCharts {
pub contents: Contents,
pub framework_updates: Option<FrameworkUpdates>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct Contents {
pub single_column_browse_results_renderer: ContentsRenderer<Tab<SectionList<ItemSection>>>,
}
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) enum ItemSection {
MusicCarouselShelfRenderer(Box<MusicCarouselShelf>),
#[serde(other, deserialize_with = "deserialize_ignore_any")]
None,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct FrameworkUpdates {
pub entity_batch_update: EntityBatchUpdate,
}
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct EntityBatchUpdate {
#[serde_as(as = "VecSkipError<_>")]
pub mutations: Vec<CountryOptionMutation>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct CountryOptionMutation {
pub payload: CountryOptionPayload,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct CountryOptionPayload {
pub music_form_boolean_choice: CountryOption,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct CountryOption {
pub opaque_token: Country,
}

View file

@ -25,11 +25,7 @@ use super::{
pub(crate) enum ItemSection {
#[serde(alias = "musicPlaylistShelfRenderer")]
MusicShelfRenderer(MusicShelf),
MusicCarouselShelfRenderer {
header: Option<MusicCarouselShelfHeader>,
#[serde_as(as = "VecLogError<_>")]
contents: MapResult<Vec<MusicResponseItem>>,
},
MusicCarouselShelfRenderer(MusicCarouselShelf),
#[serde(other, deserialize_with = "deserialize_ignore_any")]
None,
}
@ -52,6 +48,15 @@ pub(crate) struct MusicShelf {
pub bottom_endpoint: Option<BrowseEndpointWrap>,
}
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct MusicCarouselShelf {
pub header: Option<MusicCarouselShelfHeader>,
#[serde_as(as = "VecLogError<_>")]
pub contents: MapResult<Vec<MusicResponseItem>>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) enum MusicResponseItem {

View file

@ -10,12 +10,12 @@ pub use ordering::QualityOrd;
pub use paginator::Paginator;
use serde_with::serde_as;
use std::ops::Range;
use std::{collections::BTreeSet, ops::Range};
use serde::{Deserialize, Serialize};
use time::{Date, OffsetDateTime};
use crate::{error::Error, serializer::DateYmd, util};
use crate::{error::Error, param::Country, serializer::DateYmd, util};
use self::richtext::RichText;
@ -1250,7 +1250,7 @@ pub struct Lyrics {
pub footer: String,
}
/// YouTube Music related entities
/// YouTube Music entities related to a track
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[non_exhaustive]
pub struct MusicRelated {
@ -1265,3 +1265,23 @@ pub struct MusicRelated {
/// Related playlists
pub playlists: Vec<MusicPlaylistItem>,
}
/// YouTube Music charts
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[non_exhaustive]
pub struct MusicCharts {
/// List of top music videos
pub top_tracks: Vec<TrackItem>,
/// List of trending music videos
pub trending_tracks: Vec<TrackItem>,
/// List of top artists
pub artists: Vec<ArtistItem>,
/// List of playlists (charts by genre, currently only available in US)
pub playlists: Vec<MusicPlaylistItem>,
/// ID of the playlist containing top music videos
pub top_playlist_id: Option<String>,
/// ID of the playlist containing trending music videos
pub trending_playlist_id: Option<String>,
/// Set of available countries to fetch charts from
pub available_countries: BTreeSet<Country>,
}

View file

@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize};
/// Available languages
#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[serde(rename_all = "lowercase")]
#[non_exhaustive]
pub enum Language {
/// Afrikaans
Af,
@ -191,6 +192,7 @@ pub enum Language {
/// Available countries
#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[serde(rename_all = "UPPERCASE")]
#[non_exhaustive]
pub enum Country {
/// United Arab Emirates
Ae,
@ -410,6 +412,8 @@ pub enum Country {
Za,
/// Zimbabwe
Zw,
/// Global (can only be used for music charts)
Zz,
}
/// Array of all available languages
@ -820,6 +824,7 @@ impl Country {
Country::Ye => "Yemen",
Country::Za => "South Africa",
Country::Zw => "Zimbabwe",
Country::Zz => "Global",
}
}
}

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -4,6 +4,7 @@ use std::fmt::Display;
use fancy_regex::Regex;
use once_cell::sync::Lazy;
use rstest::rstest;
use rustypipe::param::Country;
use time::macros::date;
use time::OffsetDateTime;
@ -2050,6 +2051,35 @@ async fn music_radio_playlist_not_found() {
}
}
#[rstest]
#[case::de(
Country::De,
"PL4fGSI1pDJn4X-OicSCOy-dChXWdTgziQ",
"PL0sHkSjKd2rpxgOMD-vlUlIDqvQ5ChYJh"
)]
#[case::us(
Country::Us,
"PL4fGSI1pDJn69On1f-8NAvX_CYlx7QyZc",
"PLrEnWoR732-DtKgaDdnPkezM_nDidBU9H"
)]
#[tokio::test]
async fn music_charts(#[case] country: Country, #[case] plid_top: &str, #[case] plid_trend: &str) {
let rp = RustyPipe::builder().strict().build();
let charts = rp.query().music_charts(Some(country)).await.unwrap();
assert_eq!(charts.top_playlist_id.unwrap(), plid_top);
assert_eq!(charts.trending_playlist_id.unwrap(), plid_trend);
assert_gte(charts.top_tracks.len(), 40, "top tracks");
assert_gte(charts.artists.len(), 40, "top artists");
assert_gte(charts.trending_tracks.len(), 20, "trending tracks");
// Chart playlists only available in USA
if country == Country::Us {
assert_gte(charts.playlists.len(), 8, "charts playlists");
}
}
#[tokio::test]
async fn music_new_albums() {
let rp = RustyPipe::builder().strict().build();