test: add music_charts test

README.md

@@ -27,5 +27,5 @@ inspired by [NewPipe](https://github.com/TeamNewPipe/NewPipeExtractor).
 - [X] **Radio**
 - [X] **Track details** (lyrics, recommendations)
 - [ ] **Moods**
-- [ ] **Charts**
+- [X] **Charts**
 - [X] **New**

codegen/src/download_testfiles.rs

@@ -7,7 +7,10 @@ use std::{
 
 use rustypipe::{
     client::{ClientType, RustyPipe},
-    param::search_filter::{self, Entity, SearchFilter},
+    param::{
+        search_filter::{self, Entity, SearchFilter},
+        Country,
+    },
     report::{Report, Reporter},
 };
 
@@ -57,6 +60,7 @@ pub async fn download_testfiles(project_root: &Path) {
     music_radio_cont(&testfiles).await;
     music_new_albums(&testfiles).await;
     music_new_videos(&testfiles).await;
+    music_charts(&testfiles).await;
 }
 
 const CLIENT_TYPES: [ClientType; 5] = [
@@ -827,3 +831,17 @@ async fn music_new_videos(testfiles: &Path) {
     let rp = rp_testfile(&json_path);
     rp.query().music_new_videos().await.unwrap();
 }
+
+async fn music_charts(testfiles: &Path) {
+    for (name, country) in [("global", Some(Country::Zz)), ("US", Some(Country::Us))] {
+        let mut json_path = testfiles.to_path_buf();
+        json_path.push("music_charts");
+        json_path.push(&format!("charts_{}.json", name));
+        if json_path.exists() {
+            continue;
+        }
+
+        let rp = rp_testfile(&json_path);
+        rp.query().music_charts(country).await.unwrap();
+    }
+}

codegen/src/gen_locales.rs

@@ -183,6 +183,7 @@ impl FromStr for Country {
     let mut code_langs = r#"/// Available languages
 #[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
 #[serde(rename_all = "lowercase")]
+#[non_exhaustive]
 pub enum Language {
 "#
     .to_owned();
@@ -190,6 +191,7 @@ pub enum Language {
     let mut code_countries = r#"/// Available countries
 #[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
 #[serde(rename_all = "UPPERCASE")]
+#[non_exhaustive]
 pub enum Country {
 "#
     .to_owned();
@@ -233,22 +235,23 @@ pub enum Country {
             .collect::<String>();
 
         // Language enum
-        let _ = write!(code_langs, "    /// {}\n    ", n);
+        write!(code_langs, "    /// {}\n    ", n).unwrap();
         if c.contains('-') {
-            let _ = write!(code_langs, "#[serde(rename = \"{}\")]\n    ", c);
+            write!(code_langs, "#[serde(rename = \"{}\")]\n    ", c).unwrap();
         }
         code_langs += &enum_name;
         code_langs += ",\n";
 
         // Language array
-        let _ = writeln!(code_lang_array, "    Language::{},", enum_name);
+        writeln!(code_lang_array, "    Language::{},", enum_name).unwrap();
 
         // Language names
-        let _ = writeln!(
+        writeln!(
             code_lang_names,
             "            Language::{} => \"{}\",",
             enum_name, n
-        );
+        )
+        .unwrap();
     });
     code_langs += "}\n";
 
@@ -256,19 +259,26 @@ pub enum Country {
         let enum_name = c[0..1].to_owned().to_uppercase() + &c[1..].to_owned().to_lowercase();
 
         // Country enum
-        let _ = writeln!(code_countries, "    /// {}", n);
+        writeln!(code_countries, "    /// {}", n).unwrap();
-        let _ = writeln!(code_countries, "    {},", enum_name);
+        writeln!(code_countries, "    {},", enum_name).unwrap();
 
         // Country array
-        let _ = writeln!(code_country_array, "    Country::{},", enum_name);
+        writeln!(code_country_array, "    Country::{},", enum_name).unwrap();
 
         // Country names
-        let _ = writeln!(
+        writeln!(
             code_country_names,
             "            Country::{} => \"{}\",",
             enum_name, n
-        );
+        )
+        .unwrap();
     });
+
+    // Add Country::Zz / Global
+    code_countries += "    /// Global (can only be used for music charts)\n";
+    code_countries += "    Zz,\n";
+    code_country_names += "            Country::Zz => \"Global\",\n";
+
     code_countries += "}\n";
 
     code_lang_array += "];\n";

src/client/mod.rs

@@ -4,6 +4,7 @@ pub(crate) mod response;
 
 mod channel;
 mod music_artist;
+mod music_charts;
 mod music_details;
 mod music_new;
 mod music_playlist;
@@ -420,7 +421,7 @@ impl RustyPipeBuilder {
     ///
     /// **Info**: you can set this option for individual queries, too
     pub fn country(mut self, country: Country) -> Self {
-        self.default_opts.country = country;
+        self.default_opts.country = validate_country(country);
         self
     }
 
@@ -758,7 +759,7 @@ impl RustyPipeQuery {
     /// Set the country parameter used when accessing the YouTube API.
     /// This will change trends and recommended content.
     pub fn country(mut self, country: Country) -> Self {
-        self.opts.country = country;
+        self.opts.country = validate_country(country);
         self
     }
 
@@ -1193,6 +1194,15 @@ trait MapResponse<T> {
     ) -> Result<MapResult<T>, ExtractionError>;
 }
 
+fn validate_country(country: Country) -> Country {
+    if country == Country::Zz {
+        warn!("Country:Zz (Global) can only be used for fetching music charts, falling back to Country:Us");
+        Country::Us
+    } else {
+        country
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/client/music_artist.rs

@@ -192,9 +192,9 @@ fn map_artist_page(
 
                 mapper.map_response(shelf.contents);
             }
-            response::music_item::ItemSection::MusicCarouselShelfRenderer { header, contents } => {
+            response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => {
                 let mut extendable_albums = false;
-                if let Some(h) = header {
+                if let Some(h) = shelf.header {
                     if let Some(button) = h
                         .music_carousel_shelf_basic_header_renderer
                         .more_content_button
@@ -221,7 +221,7 @@ fn map_artist_page(
                 }
 
                 if !skip_extendables || !extendable_albums {
-                    mapper.map_response(contents);
+                    mapper.map_response(shelf.contents);
                 }
             }
             response::music_item::ItemSection::None => {}

src/client/music_charts.rs

@@ -0,0 +1,171 @@
+use std::borrow::Cow;
+
+use serde::Serialize;
+
+use crate::{
+    error::{Error, ExtractionError},
+    model::{MusicCharts, TrackItem},
+    param::Country,
+    serializer::MapResult,
+};
+
+use super::{
+    response::{self, music_item::MusicListMapper, url_endpoint::MusicPageType},
+    ClientType, MapResponse, RustyPipeQuery, YTContext,
+};
+
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct QCharts<'a> {
+    context: YTContext<'a>,
+    browse_id: &'a str,
+    params: &'a str,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    form_data: Option<FormData>,
+}
+
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct FormData {
+    pub selected_values: [Country; 1],
+}
+
+impl RustyPipeQuery {
+    pub async fn music_charts(&self, country: Option<Country>) -> Result<MusicCharts, Error> {
+        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
+        let request_body = QCharts {
+            context,
+            browse_id: "FEmusic_charts",
+            params: "sgYPRkVtdXNpY19leHBsb3Jl",
+            form_data: country.map(|c| FormData {
+                selected_values: [c],
+            }),
+        };
+
+        self.execute_request::<response::MusicCharts, _, _>(
+            ClientType::DesktopMusic,
+            "music_charts",
+            "",
+            "browse",
+            &request_body,
+        )
+        .await
+    }
+}
+
+impl MapResponse<MusicCharts> for response::MusicCharts {
+    fn map_response(
+        self,
+        _id: &str,
+        lang: crate::param::Language,
+        _deobf: Option<&crate::deobfuscate::Deobfuscator>,
+    ) -> Result<crate::serializer::MapResult<MusicCharts>, crate::error::ExtractionError> {
+        let countries = self
+            .framework_updates
+            .map(|fwu| {
+                fwu.entity_batch_update
+                    .mutations
+                    .into_iter()
+                    .map(|x| x.payload.music_form_boolean_choice.opaque_token)
+                    .collect()
+            })
+            .unwrap_or_default();
+
+        let mut top_playlist_id = None;
+        let mut trending_playlist_id = None;
+
+        let mut mapper_top = MusicListMapper::new(lang);
+        let mut mapper_trending = MusicListMapper::new(lang);
+        let mut mapper_other = MusicListMapper::new(lang);
+
+        self.contents
+            .single_column_browse_results_renderer
+            .contents
+            .into_iter()
+            .next()
+            .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))?
+            .tab_renderer
+            .content
+            .section_list_renderer
+            .contents
+            .into_iter()
+            .for_each(|s| match s {
+                response::music_charts::ItemSection::MusicCarouselShelfRenderer(shelf) => {
+                    match shelf.header.and_then(|h| {
+                        h.music_carousel_shelf_basic_header_renderer
+                            .more_content_button
+                            .and_then(|btn| btn.button_renderer.navigation_endpoint.music_page())
+                    }) {
+                        Some((MusicPageType::Playlist, id)) => {
+                            // Top music videos (first shelf with associated playlist)
+                            if top_playlist_id.is_none() {
+                                mapper_top.map_response(shelf.contents);
+                                top_playlist_id = Some(id);
+                            }
+                            // Trending (second shelf with associated playlist)
+                            else if trending_playlist_id.is_none() {
+                                mapper_trending.map_response(shelf.contents);
+                                trending_playlist_id = Some(id);
+                            }
+                        }
+                        // Other sections (artists, playlists)
+                        _ => {
+                            mapper_other.map_response(shelf.contents);
+                        }
+                    }
+                }
+                response::music_charts::ItemSection::None => {}
+            });
+
+        let mapped_top = mapper_top.conv_items::<TrackItem>();
+        let mut mapped_trending = mapper_trending.conv_items::<TrackItem>();
+        let mut mapped_other = mapper_other.group_items();
+
+        let mut warnings = mapped_top.warnings;
+        warnings.append(&mut mapped_trending.warnings);
+        warnings.append(&mut mapped_other.warnings);
+
+        Ok(MapResult {
+            c: MusicCharts {
+                top_tracks: mapped_top.c,
+                trending_tracks: mapped_trending.c,
+                artists: mapped_other.c.artists,
+                playlists: mapped_other.c.playlists,
+                top_playlist_id,
+                trending_playlist_id,
+                available_countries: countries,
+            },
+            warnings,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{fs::File, io::BufReader, path::Path};
+
+    use rstest::rstest;
+
+    use super::*;
+    use crate::param::Language;
+
+    #[rstest]
+    #[case::default("global")]
+    #[case::us("US")]
+    fn map_music_charts(#[case] name: &str) {
+        let filename = format!("testfiles/music_charts/charts_{}.json", name);
+        let json_path = Path::new(&filename);
+        let json_file = File::open(json_path).unwrap();
+
+        let charts: response::MusicCharts =
+            serde_json::from_reader(BufReader::new(json_file)).unwrap();
+        let map_res: MapResult<MusicCharts> = charts.map_response("", Language::En, None).unwrap();
+
+        assert!(
+            map_res.warnings.is_empty(),
+            "deserialization/mapping warnings: {:?}",
+            map_res.warnings
+        );
+        insta::assert_ron_snapshot!(format!("map_music_charts_{}", name), map_res.c);
+    }
+}

src/client/music_details.rs

@@ -315,22 +315,22 @@ impl MapResponse<MusicRelated> for response::MusicRelated {
             .iter()
             .find_map(|section| match section {
                 response::music_item::ItemSection::MusicShelfRenderer(_) => None,
-                response::music_item::ItemSection::MusicCarouselShelfRenderer {
+                response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => {
-                    header, ..
+                    shelf.header.as_ref().and_then(|h| {
-                } => header.as_ref().and_then(|h| {
+                        h.music_carousel_shelf_basic_header_renderer
-                    h.music_carousel_shelf_basic_header_renderer
+                            .title
-                        .title
+                            .0
-                        .0
+                            .iter()
-                        .iter()
+                            .find_map(|c| {
-                        .find_map(|c| {
+                                let artist = ArtistId::from(c.clone());
-                            let artist = ArtistId::from(c.clone());
+                                if artist.id.is_some() {
-                            if artist.id.is_some() {
+                                    Some(artist)
-                                Some(artist)
+                                } else {
-                            } else {
+                                    None
-                                None
+                                }
-                            }
+                            })
-                        })
+                    })
-                }),
+                }
                 response::music_item::ItemSection::None => None,
             });
 
@@ -341,20 +341,18 @@ impl MapResponse<MusicRelated> for response::MusicRelated {
         };
 
         let mut sections = self.contents.section_list_renderer.contents.into_iter();
-        if let Some(response::music_item::ItemSection::MusicCarouselShelfRenderer {
+        if let Some(response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf)) =
-            contents,
+            sections.next()
-            ..
-        }) = sections.next()
         {
-            mapper_tracks.map_response(contents);
+            mapper_tracks.map_response(shelf.contents);
         }
 
         sections.for_each(|section| match section {
             response::music_item::ItemSection::MusicShelfRenderer(shelf) => {
                 mapper.map_response(shelf.contents);
             }
-            response::music_item::ItemSection::MusicCarouselShelfRenderer { contents, .. } => {
+            response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => {
-                mapper.map_response(contents);
+                mapper.map_response(shelf.contents);
             }
             response::music_item::ItemSection::None => {}
         });

src/client/music_playlist.rs

@@ -264,9 +264,9 @@ impl MapResponse<MusicAlbum> for response::MusicPlaylist {
         for section in sections {
             match section {
                 response::music_item::ItemSection::MusicShelfRenderer(sh) => shelf = Some(sh),
-                response::music_item::ItemSection::MusicCarouselShelfRenderer {
+                response::music_item::ItemSection::MusicCarouselShelfRenderer(sh) => {
-                    contents, ..
+                    album_variants = Some(sh.contents)
-                } => album_variants = Some(contents),
+                }
                 response::music_item::ItemSection::None => (),
             }
         }

src/client/pagination.rs

@@ -143,11 +143,8 @@ impl MapResponse<Paginator<MusicItem>> for response::MusicContinuation {
                             mapper.map_response(shelf.contents);
                             continuations.append(&mut shelf.continuations);
                         }
-                        response::music_item::ItemSection::MusicCarouselShelfRenderer {
+                        response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => {
-                            contents,
+                            mapper.map_response(shelf.contents);
-                            ..
-                        } => {
-                            mapper.map_response(contents);
                         }
                         response::music_item::ItemSection::None => {}
                     }

src/client/response/mod.rs

@@ -1,5 +1,6 @@
 pub(crate) mod channel;
 pub(crate) mod music_artist;
+pub(crate) mod music_charts;
 pub(crate) mod music_details;
 pub(crate) mod music_item;
 pub(crate) mod music_new;
@@ -16,6 +17,7 @@ pub(crate) mod video_item;
 pub(crate) use channel::Channel;
 pub(crate) use music_artist::MusicArtist;
 pub(crate) use music_artist::MusicArtistAlbums;
+pub(crate) use music_charts::MusicCharts;
 pub(crate) use music_details::MusicDetails;
 pub(crate) use music_details::MusicLyrics;
 pub(crate) use music_details::MusicRelated;

src/client/response/music_charts.rs

@@ -0,0 +1,60 @@
+use serde::Deserialize;
+use serde_with::{rust::deserialize_ignore_any, serde_as, VecSkipError};
+
+use crate::param::Country;
+
+use super::{music_item::MusicCarouselShelf, ContentsRenderer, SectionList, Tab};
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct MusicCharts {
+    pub contents: Contents,
+    pub framework_updates: Option<FrameworkUpdates>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct Contents {
+    pub single_column_browse_results_renderer: ContentsRenderer<Tab<SectionList<ItemSection>>>,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) enum ItemSection {
+    MusicCarouselShelfRenderer(Box<MusicCarouselShelf>),
+    #[serde(other, deserialize_with = "deserialize_ignore_any")]
+    None,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct FrameworkUpdates {
+    pub entity_batch_update: EntityBatchUpdate,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct EntityBatchUpdate {
+    #[serde_as(as = "VecSkipError<_>")]
+    pub mutations: Vec<CountryOptionMutation>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct CountryOptionMutation {
+    pub payload: CountryOptionPayload,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct CountryOptionPayload {
+    pub music_form_boolean_choice: CountryOption,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct CountryOption {
+    pub opaque_token: Country,
+}

src/client/response/music_item.rs

@@ -25,11 +25,7 @@ use super::{
 pub(crate) enum ItemSection {
     #[serde(alias = "musicPlaylistShelfRenderer")]
     MusicShelfRenderer(MusicShelf),
-    MusicCarouselShelfRenderer {
+    MusicCarouselShelfRenderer(MusicCarouselShelf),
-        header: Option<MusicCarouselShelfHeader>,
-        #[serde_as(as = "VecLogError<_>")]
-        contents: MapResult<Vec<MusicResponseItem>>,
-    },
     #[serde(other, deserialize_with = "deserialize_ignore_any")]
     None,
 }
@@ -52,6 +48,15 @@ pub(crate) struct MusicShelf {
     pub bottom_endpoint: Option<BrowseEndpointWrap>,
 }
 
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct MusicCarouselShelf {
+    pub header: Option<MusicCarouselShelfHeader>,
+    #[serde_as(as = "VecLogError<_>")]
+    pub contents: MapResult<Vec<MusicResponseItem>>,
+}
+
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) enum MusicResponseItem {

src/model/mod.rs

@@ -10,12 +10,12 @@ pub use ordering::QualityOrd;
 pub use paginator::Paginator;
 use serde_with::serde_as;
 
-use std::ops::Range;
+use std::{collections::BTreeSet, ops::Range};
 
 use serde::{Deserialize, Serialize};
 use time::{Date, OffsetDateTime};
 
-use crate::{error::Error, serializer::DateYmd, util};
+use crate::{error::Error, param::Country, serializer::DateYmd, util};
 
 use self::richtext::RichText;
 
@@ -1250,7 +1250,7 @@ pub struct Lyrics {
     pub footer: String,
 }
 
-/// YouTube Music related entities
+/// YouTube Music entities related to a track
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[non_exhaustive]
 pub struct MusicRelated {
@@ -1265,3 +1265,23 @@ pub struct MusicRelated {
     /// Related playlists
     pub playlists: Vec<MusicPlaylistItem>,
 }
+
+/// YouTube Music charts
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[non_exhaustive]
+pub struct MusicCharts {
+    /// List of top music videos
+    pub top_tracks: Vec<TrackItem>,
+    /// List of trending music videos
+    pub trending_tracks: Vec<TrackItem>,
+    /// List of top artists
+    pub artists: Vec<ArtistItem>,
+    /// List of playlists (charts by genre, currently only available in US)
+    pub playlists: Vec<MusicPlaylistItem>,
+    /// ID of the playlist containing top music videos
+    pub top_playlist_id: Option<String>,
+    /// ID of the playlist containing trending music videos
+    pub trending_playlist_id: Option<String>,
+    /// Set of available countries to fetch charts from
+    pub available_countries: BTreeSet<Country>,
+}

src/param/locale.rs

@@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize};
 /// Available languages
 #[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
 #[serde(rename_all = "lowercase")]
+#[non_exhaustive]
 pub enum Language {
     /// Afrikaans
     Af,
@@ -191,6 +192,7 @@ pub enum Language {
 /// Available countries
 #[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
 #[serde(rename_all = "UPPERCASE")]
+#[non_exhaustive]
 pub enum Country {
     /// United Arab Emirates
     Ae,
@@ -410,6 +412,8 @@ pub enum Country {
     Za,
     /// Zimbabwe
     Zw,
+    /// Global (can only be used for music charts)
+    Zz,
 }
 
 /// Array of all available languages
@@ -820,6 +824,7 @@ impl Country {
             Country::Ye => "Yemen",
             Country::Za => "South Africa",
             Country::Zw => "Zimbabwe",
+            Country::Zz => "Global",
         }
     }
 }

tests/youtube.rs

@@ -4,6 +4,7 @@ use std::fmt::Display;
 use fancy_regex::Regex;
 use once_cell::sync::Lazy;
 use rstest::rstest;
+use rustypipe::param::Country;
 use time::macros::date;
 use time::OffsetDateTime;
 
@@ -2050,6 +2051,35 @@ async fn music_radio_playlist_not_found() {
     }
 }
 
+#[rstest]
+#[case::de(
+    Country::De,
+    "PL4fGSI1pDJn4X-OicSCOy-dChXWdTgziQ",
+    "PL0sHkSjKd2rpxgOMD-vlUlIDqvQ5ChYJh"
+)]
+#[case::us(
+    Country::Us,
+    "PL4fGSI1pDJn69On1f-8NAvX_CYlx7QyZc",
+    "PLrEnWoR732-DtKgaDdnPkezM_nDidBU9H"
+)]
+#[tokio::test]
+async fn music_charts(#[case] country: Country, #[case] plid_top: &str, #[case] plid_trend: &str) {
+    let rp = RustyPipe::builder().strict().build();
+    let charts = rp.query().music_charts(Some(country)).await.unwrap();
+
+    assert_eq!(charts.top_playlist_id.unwrap(), plid_top);
+    assert_eq!(charts.trending_playlist_id.unwrap(), plid_trend);
+
+    assert_gte(charts.top_tracks.len(), 40, "top tracks");
+    assert_gte(charts.artists.len(), 40, "top artists");
+    assert_gte(charts.trending_tracks.len(), 20, "trending tracks");
+
+    // Chart playlists only available in USA
+    if country == Country::Us {
+        assert_gte(charts.playlists.len(), 8, "charts playlists");
+    }
+}
+
 #[tokio::test]
 async fn music_new_albums() {
     let rp = RustyPipe::builder().strict().build();