chore: update client versions

fix: music_artist: fetch visitor data only once

src/client/mod.rs

@@ -25,7 +25,6 @@ mod channel_rss;
 use std::sync::Arc;
 use std::{borrow::Cow, fmt::Debug};
 
-use log::{debug, error, warn};
 use once_cell::sync::Lazy;
 use rand::Rng;
 use regex::Regex;
@@ -168,13 +167,13 @@ const YOUTUBE_MUSIC_HOME_URL: &str = "https://music.youtube.com/";
 
 const DISABLE_PRETTY_PRINT_PARAMETER: &str = "&prettyPrint=false";
 
-const DESKTOP_CLIENT_VERSION: &str = "2.20221011.00.00";
+const DESKTOP_CLIENT_VERSION: &str = "2.20230126.00.00";
 const DESKTOP_API_KEY: &str = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
 const TVHTML5_CLIENT_VERSION: &str = "2.0";
 const DESKTOP_MUSIC_API_KEY: &str = "AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30";
-const DESKTOP_MUSIC_CLIENT_VERSION: &str = "1.20221005.01.00";
+const DESKTOP_MUSIC_CLIENT_VERSION: &str = "1.20230123.01.01";
 
-const MOBILE_CLIENT_VERSION: &str = "17.39.35";
+const MOBILE_CLIENT_VERSION: &str = "18.03.33";
 const ANDROID_API_KEY: &str = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
 const IOS_API_KEY: &str = "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc";
 const IOS_DEVICE_MODEL: &str = "iPhone14,5";
@@ -323,6 +322,7 @@ impl RustyPipeBuilder {
             .user_agent(self.user_agent)
             .gzip(true)
             .brotli(true)
+            .redirect(reqwest::redirect::Policy::none())
             .build()
             .unwrap();
 
@@ -331,7 +331,7 @@ impl RustyPipeBuilder {
                 match serde_json::from_str::<CacheData>(&data) {
                     Ok(data) => data,
                     Err(e) => {
-                        error!("Could not deserialize cache. Error: {}", e);
+                        log::error!("Could not deserialize cache. Error: {}", e);
                         CacheData::default()
                     }
                 }
@@ -523,7 +523,7 @@ impl RustyPipe {
             };
 
             let ms = util::retry_delay(n, 1000, 60000, 3);
-            warn!("Retry attempt #{}. Error: {}. Waiting {} ms", n, emsg, ms);
+            log::warn!("Retry attempt #{}. Error: {}. Waiting {} ms", n, emsg, ms);
             tokio::time::sleep(std::time::Duration::from_millis(ms.into())).await;
 
             last_res = Some(res);
@@ -656,7 +656,7 @@ impl RustyPipe {
         match desktop_client.get() {
             Some(cdata) => cdata.version.to_owned(),
             None => {
-                debug!("getting desktop client version");
+                log::debug!("getting desktop client version");
                 match self.extract_desktop_client_version().await {
                     Ok(version) => {
                         *desktop_client = CacheEntry::from(ClientData {
@@ -667,7 +667,7 @@ impl RustyPipe {
                         version
                     }
                     Err(e) => {
-                        warn!("{}, falling back to hardcoded version", e);
+                        log::warn!("{}, falling back to hardcoded version", e);
                         DESKTOP_CLIENT_VERSION.to_owned()
                     }
                 }
@@ -688,7 +688,7 @@ impl RustyPipe {
         match music_client.get() {
             Some(cdata) => cdata.version.to_owned(),
             None => {
-                debug!("getting music client version");
+                log::debug!("getting music client version");
                 match self.extract_music_client_version().await {
                     Ok(version) => {
                         *music_client = CacheEntry::from(ClientData {
@@ -699,7 +699,7 @@ impl RustyPipe {
                         version
                     }
                     Err(e) => {
-                        warn!("{}, falling back to hardcoded version", e);
+                        log::warn!("{}, falling back to hardcoded version", e);
                         DESKTOP_MUSIC_CLIENT_VERSION.to_owned()
                     }
                 }
@@ -715,7 +715,7 @@ impl RustyPipe {
         match deobf.get() {
             Some(deobf) => Ok(Deobfuscator::from(deobf.to_owned())),
             None => {
-                debug!("getting deobfuscator");
+                log::debug!("getting deobfuscator");
                 let new_deobf = Deobfuscator::new(self.inner.http.clone()).await?;
                 *deobf = CacheEntry::from(new_deobf.get_data());
                 drop(deobf);
@@ -736,12 +736,13 @@ impl RustyPipe {
 
             match serde_json::to_string(&cdata) {
                 Ok(data) => storage.write(&data),
-                Err(e) => error!("Could not serialize cache. Error: {}", e),
+                Err(e) => log::error!("Could not serialize cache. Error: {}", e),
             }
         }
     }
 
     async fn get_ytm_visitor_data(&self) -> Result<String, Error> {
+        log::debug!("getting YTM visitor data");
         let resp = self.inner.http.get(YOUTUBE_MUSIC_HOME_URL).send().await?;
 
         resp.headers()
@@ -1040,6 +1041,8 @@ impl RustyPipeQuery {
         body: &B,
         deobf: Option<&Deobfuscator>,
     ) -> Result<M, Error> {
+        log::debug!("getting {}({})", operation, id);
+
         let request = self
             .request_builder(ctype, endpoint)
             .await
@@ -1212,7 +1215,7 @@ trait MapResponse<T> {
 
 fn validate_country(country: Country) -> Country {
     if country == Country::Zz {
-        warn!("Country:Zz (Global) can only be used for fetching music charts, falling back to Country:Us");
+        log::warn!("Country:Zz (Global) can only be used for fetching music charts, falling back to Country:Us");
         Country::Us
     } else {
         country
@@ -1223,10 +1226,11 @@ fn validate_country(country: Country) -> Country {
 mod tests {
     use super::*;
 
-    #[tokio::test]
+    #[test]
-    async fn t_get_ytm_visitor_data() {
+    fn t_get_ytm_visitor_data() {
         let rp = RustyPipe::new();
-        let visitor_data = rp.get_ytm_visitor_data().await.unwrap();
+        let visitor_data = tokio_test::block_on(rp.get_ytm_visitor_data()).unwrap();
+        dbg!(&visitor_data);
         assert!(visitor_data.ends_with("%3D"));
         assert_eq!(visitor_data.len(), 32)
     }

src/client/music_artist.rs

@@ -34,77 +34,83 @@ impl RustyPipeQuery {
         artist_id: S,
         all_albums: bool,
     ) -> Result<MusicArtist, Error> {
-        let res = self._music_artist(artist_id, all_albums).await;
+        let artist_id = artist_id.as_ref();
+        let visitor_data = match all_albums {
+            true => Some(self.get_ytm_visitor_data().await?),
+            false => None,
+        };
+
+        let res = self._music_artist(artist_id, visitor_data.as_deref()).await;
 
         if let Err(Error::Extraction(ExtractionError::Redirect(id))) = res {
-            self._music_artist(&id, all_albums).await.map(|x| *x)
+            log::debug!("music artist {} redirects to {}", artist_id, &id);
+            self._music_artist(&id, visitor_data.as_deref()).await
         } else {
-            res.map(|x| *x)
+            res
         }
     }
 
-    async fn _music_artist<S: AsRef<str>>(
+    async fn _music_artist(
         &self,
-        artist_id: S,
+        artist_id: &str,
-        all_albums: bool,
+        all_albums_vdata: Option<&str>,
-    ) -> Result<Box<MusicArtist>, Error> {
+    ) -> Result<MusicArtist, Error> {
-        let artist_id = artist_id.as_ref();
+        match all_albums_vdata {
+            Some(visitor_data) => {
+                let context = self
+                    .get_context(ClientType::DesktopMusic, true, Some(visitor_data))
+                    .await;
+                let request_body = QBrowse {
+                    context,
+                    browse_id: artist_id,
+                };
 
-        if all_albums {
+                let (mut artist, album_page_params) = self
-            let visitor_data = self.get_ytm_visitor_data().await?;
+                    .execute_request::<response::MusicArtist, _, _>(
-            let context = self
+                        ClientType::DesktopMusic,
-                .get_context(ClientType::DesktopMusic, true, Some(&visitor_data))
+                        "music_artist",
-                .await;
+                        artist_id,
-            let request_body = QBrowse {
+                        "browse",
-                context,
+                        &request_body,
-                browse_id: artist_id,
+                    )
-            };
+                    .await?;
 
-            let (mut artist, album_page_params) = self
+                let visitor_data = Rc::new(visitor_data);
-                .execute_request::<response::MusicArtist, _, _>(
+                let album_page_results = stream::iter(album_page_params)
+                    .map(|params| {
+                        let visitor_data = visitor_data.clone();
+                        async move {
+                            self.music_artist_album_page(artist_id, &params, &visitor_data)
+                                .await
+                        }
+                    })
+                    .buffer_unordered(2)
+                    .collect::<Vec<_>>()
+                    .await;
+
+                for res in album_page_results {
+                    let mut res = res?;
+                    artist.albums.append(&mut res);
+                }
+
+                Ok(artist)
+            }
+            None => {
+                let context = self.get_context(ClientType::DesktopMusic, true, None).await;
+                let request_body = QBrowse {
+                    context,
+                    browse_id: artist_id,
+                };
+
+                self.execute_request::<response::MusicArtist, _, _>(
                     ClientType::DesktopMusic,
                     "music_artist",
                     artist_id,
                     "browse",
                     &request_body,
                 )
-                .await?;
+                .await
-
-            let visitor_data = Rc::new(visitor_data);
-            let album_page_results = stream::iter(album_page_params)
-                .map(|params| {
-                    let visitor_data = visitor_data.clone();
-                    async move {
-                        self.music_artist_album_page(artist_id, &params, &visitor_data)
-                            .await
-                    }
-                })
-                .buffer_unordered(2)
-                .collect::<Vec<_>>()
-                .await;
-
-            for res in album_page_results {
-                let mut res = res?;
-                artist.albums.append(&mut res);
             }
-
-            Ok(artist.into())
-        } else {
-            let context = self.get_context(ClientType::DesktopMusic, true, None).await;
-            let request_body = QBrowse {
-                context,
-                browse_id: artist_id,
-            };
-
-            self.execute_request::<response::MusicArtist, _, _>(
-                ClientType::DesktopMusic,
-                "music_artist",
-                artist_id,
-                "browse",
-                &request_body,
-            )
-            .await
-            .map(|x: MusicArtist| x.into())
         }
     }
 
@@ -237,14 +243,24 @@ fn map_artist_page(
                         {
                             if let Some(cfg) = bep.browse_endpoint_context_supported_configs {
                                 match cfg.browse_endpoint_context_music_config.page_type {
+                                    // Music videos
                                     PageType::Playlist => {
                                         if videos_playlist_id.is_none() {
                                             videos_playlist_id = Some(bep.browse_id);
                                         }
                                     }
+                                    // Albums or playlists
                                     PageType::Artist => {
-                                        album_page_params.push(bep.params);
+                                        // Peek at the first item to determine type
-                                        extendable_albums = true;
+                                        if let Some(response::music_item::MusicResponseItem::MusicTwoRowItemRenderer(item)) = shelf.contents.c.first() {
+                                            if let Some(PageType::Album) = item.navigation_endpoint.browse_endpoint.as_ref().and_then(|be| {
+                                                be.browse_endpoint_context_supported_configs.as_ref().map(|config| {
+                                                        config.browse_endpoint_context_music_config.page_type
+                                                })}) {
+                                                    album_page_params.push(bep.params);
+                                                    extendable_albums = true;
+                                                }
+                                        }
                                     }
                                     _ => {}
                                 }

src/client/player.rs

@@ -676,7 +676,7 @@ mod tests {
         let (url, throttled) = map_res.c.unwrap();
 
         assert_eq!(url, "https://rr5---sn-h0jelnez.googlevideo.com/videoplayback?c=WEB&clen=3781277&dur=229.301&ei=vb7nYvH5BMK8gAfBj7ToBQ&expire=1659376413&fexp=24001373%2C24007246&fvip=5&gir=yes&id=o-AB_BABwrXZJN428ZwDxq5ScPn2AbcGODnRlTVhCQ3mj2&initcwndbps=1588750&ip=2003%3Ade%3Aaf06%3A6300%3Ac750%3A1b77%3Ac74a%3A80e3&itag=251&keepalive=yes&lmt=1655510291473933&lsig=AG3C_xAwRQIgCKCGJ1iu4wlaGXy3jcJyU3inh9dr1FIfqYOZEG_MdmACIQCbungkQYFk7EhD6K2YvLaHFMjKOFWjw001_tLb0lPDtg%3D%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=hH&mime=audio%2Fwebm&mm=31%2C26&mn=sn-h0jelnez%2Csn-4g5ednsl&ms=au%2Conr&mt=1659354538&mv=m&mvi=5&n=XzXGSfGusw6OCQ&ns=b_Mq_qlTFcSGlG9RpwpM9xQH&pl=37&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRQIhAPIsKd7-xi4xVHEC9gb__dU4hzfzsHEj9ytd3nt0gEceAiACJWBcw-wFEq9qir35bwKHJZxtQ9mOL7SKiVkLQNDa6A%3D%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-Khi831z8dTejFIRCvCEwx_6romtM&txp=4532434&vprv=1");
-        assert_eq!(throttled, false);
+        assert!(!throttled);
         assert!(
             map_res.warnings.is_empty(),
             "deserialization/mapping warnings: {:?}",

src/client/response/url_endpoint.rs

@@ -195,38 +195,37 @@ impl From<PageType> for MusicPageType {
 
 impl NavigationEndpoint {
     pub(crate) fn music_page(self) -> Option<(MusicPageType, String)> {
-        match self.browse_endpoint {
+        self.browse_endpoint
-            Some(browse) => match browse.browse_endpoint_context_supported_configs {
+            .and_then(|be| {
-                Some(config) => Some((
+                be.browse_endpoint_context_supported_configs.map(|config| {
-                    config.browse_endpoint_context_music_config.page_type.into(),
-                    browse.browse_id,
-                )),
-                None => None,
-            },
-            None => None,
-        }
-        .or_else(|| {
-            self.watch_endpoint.map(|watch| {
-                if watch
-                    .playlist_id
-                    .map(|plid| plid.starts_with("RDQM"))
-                    .unwrap_or_default()
-                {
-                    // Genre radios (e.g. "pop radio") will be skipped
-                    (MusicPageType::None, watch.video_id)
-                } else {
                     (
-                        MusicPageType::Track {
+                        config.browse_endpoint_context_music_config.page_type.into(),
-                            is_video: watch
+                        be.browse_id,
-                                .watch_endpoint_music_supported_configs
-                                .watch_endpoint_music_config
-                                .music_video_type
-                                == MusicVideoType::Video,
-                        },
-                        watch.video_id,
                     )
-                }
+                })
+            })
+            .or_else(|| {
+                self.watch_endpoint.map(|watch| {
+                    if watch
+                        .playlist_id
+                        .map(|plid| plid.starts_with("RDQM"))
+                        .unwrap_or_default()
+                    {
+                        // Genre radios (e.g. "pop radio") will be skipped
+                        (MusicPageType::None, watch.video_id)
+                    } else {
+                        (
+                            MusicPageType::Track {
+                                is_video: watch
+                                    .watch_endpoint_music_supported_configs
+                                    .watch_endpoint_music_config
+                                    .music_video_type
+                                    == MusicVideoType::Video,
+                            },
+                            watch.video_id,
+                        )
+                    }
+                })
             })
-        })
     }
 }

src/deobfuscate.rs

@@ -1,5 +1,4 @@
 use fancy_regex::Regex as FancyRegex;
-use log::debug;
 use once_cell::sync::Lazy;
 use regex::Regex;
 use reqwest::Client;
@@ -27,7 +26,7 @@ impl Deobfuscator {
         let js_url = get_player_js_url(&http).await?;
         let player_js = get_response(&http, &js_url).await?;
 
-        debug!("Downloaded player.js from {}", js_url);
+        log::debug!("downloaded player.js from {}", js_url);
 
         let sig_fn = get_sig_fn(&player_js)?;
         let nsig_fn = get_nsig_fn(&player_js)?;
@@ -389,18 +388,18 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
         assert_eq!(res, "nrkec0fwgTWolw");
     }
 
-    #[test(tokio::test)]
+    #[test]
-    async fn t_get_player_js_url() {
+    fn t_get_player_js_url() {
         let client = Client::new();
-        let url = get_player_js_url(&client).await.unwrap();
+        let url = tokio_test::block_on(get_player_js_url(&client)).unwrap();
         assert!(url.starts_with("https://www.youtube.com/s/player"));
         assert_eq!(url.len(), 73);
     }
 
-    #[test(tokio::test)]
+    #[test]
-    async fn t_update() {
+    fn t_update() {
         let client = Client::new();
-        let deobf = Deobfuscator::new(client).await.unwrap();
+        let deobf = tokio_test::block_on(Deobfuscator::new(client)).unwrap();
 
         let deobf_sig = deobf.deobfuscate_sig("GOqGOqGOq0QJ8wRAIgaryQHfplJ9xJSKFywyaSMHuuwZYsoMTAvRvfm51qIGECIA5061zWeyfMPX9hEl_U6f9J0tr7GTJMKyPf5XNrJb5fb5i").unwrap();
         println!("{}", deobf_sig);

src/serializer/text.rs

@@ -538,8 +538,8 @@ mod tests {
             txt: Vec<String>,
         }
 
-        let res_str = serde_json::from_str::<S>(&test_json).unwrap();
+        let res_str = serde_json::from_str::<S>(test_json).unwrap();
-        let res_vec = serde_json::from_str::<SVec>(&test_json).unwrap();
+        let res_vec = serde_json::from_str::<SVec>(test_json).unwrap();
 
         assert_eq!(res_str.txt, exp.join(""));
         assert_eq!(res_vec.txt, exp);
@@ -582,7 +582,7 @@ mod tests {
             }
         }"#;
 
-        let res = serde_json::from_str::<SLink>(&test_json).unwrap();
+        let res = serde_json::from_str::<SLink>(test_json).unwrap();
         insta::assert_debug_snapshot!(res, @r###"
         SLink {
             ln: Video {
@@ -617,7 +617,7 @@ mod tests {
             }
         }"#;
 
-        let res = serde_json::from_str::<SLink>(&test_json).unwrap();
+        let res = serde_json::from_str::<SLink>(test_json).unwrap();
         insta::assert_debug_snapshot!(res, @r###"
         SLink {
             ln: Browse {
@@ -651,7 +651,7 @@ mod tests {
             }
         }"#;
 
-        let res = serde_json::from_str::<SLink>(&test_json).unwrap();
+        let res = serde_json::from_str::<SLink>(test_json).unwrap();
         insta::assert_debug_snapshot!(res, @r###"
         SLink {
             ln: Browse {
@@ -675,7 +675,7 @@ mod tests {
             }
         }"#;
 
-        let res = serde_json::from_str::<SLink>(&test_json).unwrap();
+        let res = serde_json::from_str::<SLink>(test_json).unwrap();
         insta::assert_debug_snapshot!(res, @r###"
         SLink {
             ln: Text {
@@ -710,7 +710,7 @@ mod tests {
             }
         }"#;
 
-        let res = serde_json::from_str::<SLink>(&test_json).unwrap();
+        let res = serde_json::from_str::<SLink>(test_json).unwrap();
         insta::assert_debug_snapshot!(res, @r###"
         SLink {
             ln: Web {
@@ -759,7 +759,7 @@ mod tests {
             }
         }"#;
 
-        let res = serde_json::from_str::<SLinks>(&test_json).unwrap();
+        let res = serde_json::from_str::<SLinks>(test_json).unwrap();
         insta::assert_debug_snapshot!(res, @r###"
         SLinks {
             ln: TextComponents(
@@ -787,7 +787,7 @@ mod tests {
     fn t_links_empty() {
         let test_json = r#"{"ln": {}}"#;
 
-        let res = serde_json::from_str::<SLinks>(&test_json).unwrap();
+        let res = serde_json::from_str::<SLinks>(test_json).unwrap();
         assert!(res.ln.0.is_empty())
     }