Compare commits

...

4 commits

Author SHA1 Message Date
15dad0c428 chore: update client versions 2023-01-27 20:47:59 +01:00
06aa677ef7 tests: use tokio_test::block_on() instead of test macro 2023-01-27 20:47:37 +01:00
f94d8db4d0 feat: add logging for all operations
fix: music_artist: fetch visitor data only once
2023-01-27 19:49:16 +01:00
be741d28ca feat: dont fetch artist playlist section 2023-01-25 21:05:52 +01:00
7 changed files with 456 additions and 544 deletions

View file

@ -25,7 +25,6 @@ mod channel_rss;
use std::sync::Arc; use std::sync::Arc;
use std::{borrow::Cow, fmt::Debug}; use std::{borrow::Cow, fmt::Debug};
use log::{debug, error, warn};
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use rand::Rng; use rand::Rng;
use regex::Regex; use regex::Regex;
@ -168,13 +167,13 @@ const YOUTUBE_MUSIC_HOME_URL: &str = "https://music.youtube.com/";
const DISABLE_PRETTY_PRINT_PARAMETER: &str = "&prettyPrint=false"; const DISABLE_PRETTY_PRINT_PARAMETER: &str = "&prettyPrint=false";
const DESKTOP_CLIENT_VERSION: &str = "2.20221011.00.00"; const DESKTOP_CLIENT_VERSION: &str = "2.20230126.00.00";
const DESKTOP_API_KEY: &str = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; const DESKTOP_API_KEY: &str = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
const TVHTML5_CLIENT_VERSION: &str = "2.0"; const TVHTML5_CLIENT_VERSION: &str = "2.0";
const DESKTOP_MUSIC_API_KEY: &str = "AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30"; const DESKTOP_MUSIC_API_KEY: &str = "AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30";
const DESKTOP_MUSIC_CLIENT_VERSION: &str = "1.20221005.01.00"; const DESKTOP_MUSIC_CLIENT_VERSION: &str = "1.20230123.01.01";
const MOBILE_CLIENT_VERSION: &str = "17.39.35"; const MOBILE_CLIENT_VERSION: &str = "18.03.33";
const ANDROID_API_KEY: &str = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w"; const ANDROID_API_KEY: &str = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
const IOS_API_KEY: &str = "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc"; const IOS_API_KEY: &str = "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc";
const IOS_DEVICE_MODEL: &str = "iPhone14,5"; const IOS_DEVICE_MODEL: &str = "iPhone14,5";
@ -323,6 +322,7 @@ impl RustyPipeBuilder {
.user_agent(self.user_agent) .user_agent(self.user_agent)
.gzip(true) .gzip(true)
.brotli(true) .brotli(true)
.redirect(reqwest::redirect::Policy::none())
.build() .build()
.unwrap(); .unwrap();
@ -331,7 +331,7 @@ impl RustyPipeBuilder {
match serde_json::from_str::<CacheData>(&data) { match serde_json::from_str::<CacheData>(&data) {
Ok(data) => data, Ok(data) => data,
Err(e) => { Err(e) => {
error!("Could not deserialize cache. Error: {}", e); log::error!("Could not deserialize cache. Error: {}", e);
CacheData::default() CacheData::default()
} }
} }
@ -523,7 +523,7 @@ impl RustyPipe {
}; };
let ms = util::retry_delay(n, 1000, 60000, 3); let ms = util::retry_delay(n, 1000, 60000, 3);
warn!("Retry attempt #{}. Error: {}. Waiting {} ms", n, emsg, ms); log::warn!("Retry attempt #{}. Error: {}. Waiting {} ms", n, emsg, ms);
tokio::time::sleep(std::time::Duration::from_millis(ms.into())).await; tokio::time::sleep(std::time::Duration::from_millis(ms.into())).await;
last_res = Some(res); last_res = Some(res);
@ -656,7 +656,7 @@ impl RustyPipe {
match desktop_client.get() { match desktop_client.get() {
Some(cdata) => cdata.version.to_owned(), Some(cdata) => cdata.version.to_owned(),
None => { None => {
debug!("getting desktop client version"); log::debug!("getting desktop client version");
match self.extract_desktop_client_version().await { match self.extract_desktop_client_version().await {
Ok(version) => { Ok(version) => {
*desktop_client = CacheEntry::from(ClientData { *desktop_client = CacheEntry::from(ClientData {
@ -667,7 +667,7 @@ impl RustyPipe {
version version
} }
Err(e) => { Err(e) => {
warn!("{}, falling back to hardcoded version", e); log::warn!("{}, falling back to hardcoded version", e);
DESKTOP_CLIENT_VERSION.to_owned() DESKTOP_CLIENT_VERSION.to_owned()
} }
} }
@ -688,7 +688,7 @@ impl RustyPipe {
match music_client.get() { match music_client.get() {
Some(cdata) => cdata.version.to_owned(), Some(cdata) => cdata.version.to_owned(),
None => { None => {
debug!("getting music client version"); log::debug!("getting music client version");
match self.extract_music_client_version().await { match self.extract_music_client_version().await {
Ok(version) => { Ok(version) => {
*music_client = CacheEntry::from(ClientData { *music_client = CacheEntry::from(ClientData {
@ -699,7 +699,7 @@ impl RustyPipe {
version version
} }
Err(e) => { Err(e) => {
warn!("{}, falling back to hardcoded version", e); log::warn!("{}, falling back to hardcoded version", e);
DESKTOP_MUSIC_CLIENT_VERSION.to_owned() DESKTOP_MUSIC_CLIENT_VERSION.to_owned()
} }
} }
@ -715,7 +715,7 @@ impl RustyPipe {
match deobf.get() { match deobf.get() {
Some(deobf) => Ok(Deobfuscator::from(deobf.to_owned())), Some(deobf) => Ok(Deobfuscator::from(deobf.to_owned())),
None => { None => {
debug!("getting deobfuscator"); log::debug!("getting deobfuscator");
let new_deobf = Deobfuscator::new(self.inner.http.clone()).await?; let new_deobf = Deobfuscator::new(self.inner.http.clone()).await?;
*deobf = CacheEntry::from(new_deobf.get_data()); *deobf = CacheEntry::from(new_deobf.get_data());
drop(deobf); drop(deobf);
@ -736,12 +736,13 @@ impl RustyPipe {
match serde_json::to_string(&cdata) { match serde_json::to_string(&cdata) {
Ok(data) => storage.write(&data), Ok(data) => storage.write(&data),
Err(e) => error!("Could not serialize cache. Error: {}", e), Err(e) => log::error!("Could not serialize cache. Error: {}", e),
} }
} }
} }
async fn get_ytm_visitor_data(&self) -> Result<String, Error> { async fn get_ytm_visitor_data(&self) -> Result<String, Error> {
log::debug!("getting YTM visitor data");
let resp = self.inner.http.get(YOUTUBE_MUSIC_HOME_URL).send().await?; let resp = self.inner.http.get(YOUTUBE_MUSIC_HOME_URL).send().await?;
resp.headers() resp.headers()
@ -1040,6 +1041,8 @@ impl RustyPipeQuery {
body: &B, body: &B,
deobf: Option<&Deobfuscator>, deobf: Option<&Deobfuscator>,
) -> Result<M, Error> { ) -> Result<M, Error> {
log::debug!("getting {}({})", operation, id);
let request = self let request = self
.request_builder(ctype, endpoint) .request_builder(ctype, endpoint)
.await .await
@ -1212,7 +1215,7 @@ trait MapResponse<T> {
fn validate_country(country: Country) -> Country { fn validate_country(country: Country) -> Country {
if country == Country::Zz { if country == Country::Zz {
warn!("Country:Zz (Global) can only be used for fetching music charts, falling back to Country:Us"); log::warn!("Country:Zz (Global) can only be used for fetching music charts, falling back to Country:Us");
Country::Us Country::Us
} else { } else {
country country
@ -1223,10 +1226,11 @@ fn validate_country(country: Country) -> Country {
mod tests { mod tests {
use super::*; use super::*;
#[tokio::test] #[test]
async fn t_get_ytm_visitor_data() { fn t_get_ytm_visitor_data() {
let rp = RustyPipe::new(); let rp = RustyPipe::new();
let visitor_data = rp.get_ytm_visitor_data().await.unwrap(); let visitor_data = tokio_test::block_on(rp.get_ytm_visitor_data()).unwrap();
dbg!(&visitor_data);
assert!(visitor_data.ends_with("%3D")); assert!(visitor_data.ends_with("%3D"));
assert_eq!(visitor_data.len(), 32) assert_eq!(visitor_data.len(), 32)
} }

View file

@ -34,26 +34,31 @@ impl RustyPipeQuery {
artist_id: S, artist_id: S,
all_albums: bool, all_albums: bool,
) -> Result<MusicArtist, Error> { ) -> Result<MusicArtist, Error> {
let res = self._music_artist(artist_id, all_albums).await; let artist_id = artist_id.as_ref();
let visitor_data = match all_albums {
true => Some(self.get_ytm_visitor_data().await?),
false => None,
};
let res = self._music_artist(artist_id, visitor_data.as_deref()).await;
if let Err(Error::Extraction(ExtractionError::Redirect(id))) = res { if let Err(Error::Extraction(ExtractionError::Redirect(id))) = res {
self._music_artist(&id, all_albums).await.map(|x| *x) log::debug!("music artist {} redirects to {}", artist_id, &id);
self._music_artist(&id, visitor_data.as_deref()).await
} else { } else {
res.map(|x| *x) res
} }
} }
async fn _music_artist<S: AsRef<str>>( async fn _music_artist(
&self, &self,
artist_id: S, artist_id: &str,
all_albums: bool, all_albums_vdata: Option<&str>,
) -> Result<Box<MusicArtist>, Error> { ) -> Result<MusicArtist, Error> {
let artist_id = artist_id.as_ref(); match all_albums_vdata {
Some(visitor_data) => {
if all_albums {
let visitor_data = self.get_ytm_visitor_data().await?;
let context = self let context = self
.get_context(ClientType::DesktopMusic, true, Some(&visitor_data)) .get_context(ClientType::DesktopMusic, true, Some(visitor_data))
.await; .await;
let request_body = QBrowse { let request_body = QBrowse {
context, context,
@ -88,8 +93,9 @@ impl RustyPipeQuery {
artist.albums.append(&mut res); artist.albums.append(&mut res);
} }
Ok(artist.into()) Ok(artist)
} else { }
None => {
let context = self.get_context(ClientType::DesktopMusic, true, None).await; let context = self.get_context(ClientType::DesktopMusic, true, None).await;
let request_body = QBrowse { let request_body = QBrowse {
context, context,
@ -104,7 +110,7 @@ impl RustyPipeQuery {
&request_body, &request_body,
) )
.await .await
.map(|x: MusicArtist| x.into()) }
} }
} }
@ -237,15 +243,25 @@ fn map_artist_page(
{ {
if let Some(cfg) = bep.browse_endpoint_context_supported_configs { if let Some(cfg) = bep.browse_endpoint_context_supported_configs {
match cfg.browse_endpoint_context_music_config.page_type { match cfg.browse_endpoint_context_music_config.page_type {
// Music videos
PageType::Playlist => { PageType::Playlist => {
if videos_playlist_id.is_none() { if videos_playlist_id.is_none() {
videos_playlist_id = Some(bep.browse_id); videos_playlist_id = Some(bep.browse_id);
} }
} }
// Albums or playlists
PageType::Artist => { PageType::Artist => {
// Peek at the first item to determine type
if let Some(response::music_item::MusicResponseItem::MusicTwoRowItemRenderer(item)) = shelf.contents.c.first() {
if let Some(PageType::Album) = item.navigation_endpoint.browse_endpoint.as_ref().and_then(|be| {
be.browse_endpoint_context_supported_configs.as_ref().map(|config| {
config.browse_endpoint_context_music_config.page_type
})}) {
album_page_params.push(bep.params); album_page_params.push(bep.params);
extendable_albums = true; extendable_albums = true;
} }
}
}
_ => {} _ => {}
} }
} }

View file

@ -676,7 +676,7 @@ mod tests {
let (url, throttled) = map_res.c.unwrap(); let (url, throttled) = map_res.c.unwrap();
assert_eq!(url, "https://rr5---sn-h0jelnez.googlevideo.com/videoplayback?c=WEB&clen=3781277&dur=229.301&ei=vb7nYvH5BMK8gAfBj7ToBQ&expire=1659376413&fexp=24001373%2C24007246&fvip=5&gir=yes&id=o-AB_BABwrXZJN428ZwDxq5ScPn2AbcGODnRlTVhCQ3mj2&initcwndbps=1588750&ip=2003%3Ade%3Aaf06%3A6300%3Ac750%3A1b77%3Ac74a%3A80e3&itag=251&keepalive=yes&lmt=1655510291473933&lsig=AG3C_xAwRQIgCKCGJ1iu4wlaGXy3jcJyU3inh9dr1FIfqYOZEG_MdmACIQCbungkQYFk7EhD6K2YvLaHFMjKOFWjw001_tLb0lPDtg%3D%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=hH&mime=audio%2Fwebm&mm=31%2C26&mn=sn-h0jelnez%2Csn-4g5ednsl&ms=au%2Conr&mt=1659354538&mv=m&mvi=5&n=XzXGSfGusw6OCQ&ns=b_Mq_qlTFcSGlG9RpwpM9xQH&pl=37&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRQIhAPIsKd7-xi4xVHEC9gb__dU4hzfzsHEj9ytd3nt0gEceAiACJWBcw-wFEq9qir35bwKHJZxtQ9mOL7SKiVkLQNDa6A%3D%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-Khi831z8dTejFIRCvCEwx_6romtM&txp=4532434&vprv=1"); assert_eq!(url, "https://rr5---sn-h0jelnez.googlevideo.com/videoplayback?c=WEB&clen=3781277&dur=229.301&ei=vb7nYvH5BMK8gAfBj7ToBQ&expire=1659376413&fexp=24001373%2C24007246&fvip=5&gir=yes&id=o-AB_BABwrXZJN428ZwDxq5ScPn2AbcGODnRlTVhCQ3mj2&initcwndbps=1588750&ip=2003%3Ade%3Aaf06%3A6300%3Ac750%3A1b77%3Ac74a%3A80e3&itag=251&keepalive=yes&lmt=1655510291473933&lsig=AG3C_xAwRQIgCKCGJ1iu4wlaGXy3jcJyU3inh9dr1FIfqYOZEG_MdmACIQCbungkQYFk7EhD6K2YvLaHFMjKOFWjw001_tLb0lPDtg%3D%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=hH&mime=audio%2Fwebm&mm=31%2C26&mn=sn-h0jelnez%2Csn-4g5ednsl&ms=au%2Conr&mt=1659354538&mv=m&mvi=5&n=XzXGSfGusw6OCQ&ns=b_Mq_qlTFcSGlG9RpwpM9xQH&pl=37&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRQIhAPIsKd7-xi4xVHEC9gb__dU4hzfzsHEj9ytd3nt0gEceAiACJWBcw-wFEq9qir35bwKHJZxtQ9mOL7SKiVkLQNDa6A%3D%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-Khi831z8dTejFIRCvCEwx_6romtM&txp=4532434&vprv=1");
assert_eq!(throttled, false); assert!(!throttled);
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),
"deserialization/mapping warnings: {:?}", "deserialization/mapping warnings: {:?}",

View file

@ -195,16 +195,15 @@ impl From<PageType> for MusicPageType {
impl NavigationEndpoint { impl NavigationEndpoint {
pub(crate) fn music_page(self) -> Option<(MusicPageType, String)> { pub(crate) fn music_page(self) -> Option<(MusicPageType, String)> {
match self.browse_endpoint { self.browse_endpoint
Some(browse) => match browse.browse_endpoint_context_supported_configs { .and_then(|be| {
Some(config) => Some(( be.browse_endpoint_context_supported_configs.map(|config| {
(
config.browse_endpoint_context_music_config.page_type.into(), config.browse_endpoint_context_music_config.page_type.into(),
browse.browse_id, be.browse_id,
)), )
None => None, })
}, })
None => None,
}
.or_else(|| { .or_else(|| {
self.watch_endpoint.map(|watch| { self.watch_endpoint.map(|watch| {
if watch if watch

View file

@ -1,5 +1,4 @@
use fancy_regex::Regex as FancyRegex; use fancy_regex::Regex as FancyRegex;
use log::debug;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use regex::Regex; use regex::Regex;
use reqwest::Client; use reqwest::Client;
@ -27,7 +26,7 @@ impl Deobfuscator {
let js_url = get_player_js_url(&http).await?; let js_url = get_player_js_url(&http).await?;
let player_js = get_response(&http, &js_url).await?; let player_js = get_response(&http, &js_url).await?;
debug!("Downloaded player.js from {}", js_url); log::debug!("downloaded player.js from {}", js_url);
let sig_fn = get_sig_fn(&player_js)?; let sig_fn = get_sig_fn(&player_js)?;
let nsig_fn = get_nsig_fn(&player_js)?; let nsig_fn = get_nsig_fn(&player_js)?;
@ -389,18 +388,18 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
assert_eq!(res, "nrkec0fwgTWolw"); assert_eq!(res, "nrkec0fwgTWolw");
} }
#[test(tokio::test)] #[test]
async fn t_get_player_js_url() { fn t_get_player_js_url() {
let client = Client::new(); let client = Client::new();
let url = get_player_js_url(&client).await.unwrap(); let url = tokio_test::block_on(get_player_js_url(&client)).unwrap();
assert!(url.starts_with("https://www.youtube.com/s/player")); assert!(url.starts_with("https://www.youtube.com/s/player"));
assert_eq!(url.len(), 73); assert_eq!(url.len(), 73);
} }
#[test(tokio::test)] #[test]
async fn t_update() { fn t_update() {
let client = Client::new(); let client = Client::new();
let deobf = Deobfuscator::new(client).await.unwrap(); let deobf = tokio_test::block_on(Deobfuscator::new(client)).unwrap();
let deobf_sig = deobf.deobfuscate_sig("GOqGOqGOq0QJ8wRAIgaryQHfplJ9xJSKFywyaSMHuuwZYsoMTAvRvfm51qIGECIA5061zWeyfMPX9hEl_U6f9J0tr7GTJMKyPf5XNrJb5fb5i").unwrap(); let deobf_sig = deobf.deobfuscate_sig("GOqGOqGOq0QJ8wRAIgaryQHfplJ9xJSKFywyaSMHuuwZYsoMTAvRvfm51qIGECIA5061zWeyfMPX9hEl_U6f9J0tr7GTJMKyPf5XNrJb5fb5i").unwrap();
println!("{}", deobf_sig); println!("{}", deobf_sig);

View file

@ -538,8 +538,8 @@ mod tests {
txt: Vec<String>, txt: Vec<String>,
} }
let res_str = serde_json::from_str::<S>(&test_json).unwrap(); let res_str = serde_json::from_str::<S>(test_json).unwrap();
let res_vec = serde_json::from_str::<SVec>(&test_json).unwrap(); let res_vec = serde_json::from_str::<SVec>(test_json).unwrap();
assert_eq!(res_str.txt, exp.join("")); assert_eq!(res_str.txt, exp.join(""));
assert_eq!(res_vec.txt, exp); assert_eq!(res_vec.txt, exp);
@ -582,7 +582,7 @@ mod tests {
} }
}"#; }"#;
let res = serde_json::from_str::<SLink>(&test_json).unwrap(); let res = serde_json::from_str::<SLink>(test_json).unwrap();
insta::assert_debug_snapshot!(res, @r###" insta::assert_debug_snapshot!(res, @r###"
SLink { SLink {
ln: Video { ln: Video {
@ -617,7 +617,7 @@ mod tests {
} }
}"#; }"#;
let res = serde_json::from_str::<SLink>(&test_json).unwrap(); let res = serde_json::from_str::<SLink>(test_json).unwrap();
insta::assert_debug_snapshot!(res, @r###" insta::assert_debug_snapshot!(res, @r###"
SLink { SLink {
ln: Browse { ln: Browse {
@ -651,7 +651,7 @@ mod tests {
} }
}"#; }"#;
let res = serde_json::from_str::<SLink>(&test_json).unwrap(); let res = serde_json::from_str::<SLink>(test_json).unwrap();
insta::assert_debug_snapshot!(res, @r###" insta::assert_debug_snapshot!(res, @r###"
SLink { SLink {
ln: Browse { ln: Browse {
@ -675,7 +675,7 @@ mod tests {
} }
}"#; }"#;
let res = serde_json::from_str::<SLink>(&test_json).unwrap(); let res = serde_json::from_str::<SLink>(test_json).unwrap();
insta::assert_debug_snapshot!(res, @r###" insta::assert_debug_snapshot!(res, @r###"
SLink { SLink {
ln: Text { ln: Text {
@ -710,7 +710,7 @@ mod tests {
} }
}"#; }"#;
let res = serde_json::from_str::<SLink>(&test_json).unwrap(); let res = serde_json::from_str::<SLink>(test_json).unwrap();
insta::assert_debug_snapshot!(res, @r###" insta::assert_debug_snapshot!(res, @r###"
SLink { SLink {
ln: Web { ln: Web {
@ -759,7 +759,7 @@ mod tests {
} }
}"#; }"#;
let res = serde_json::from_str::<SLinks>(&test_json).unwrap(); let res = serde_json::from_str::<SLinks>(test_json).unwrap();
insta::assert_debug_snapshot!(res, @r###" insta::assert_debug_snapshot!(res, @r###"
SLinks { SLinks {
ln: TextComponents( ln: TextComponents(
@ -787,7 +787,7 @@ mod tests {
fn t_links_empty() { fn t_links_empty() {
let test_json = r#"{"ln": {}}"#; let test_json = r#"{"ln": {}}"#;
let res = serde_json::from_str::<SLinks>(&test_json).unwrap(); let res = serde_json::from_str::<SLinks>(test_json).unwrap();
assert!(res.ln.0.is_empty()) assert!(res.ln.0.is_empty())
} }

File diff suppressed because it is too large Load diff