Compare commits

...

4 commits

Author SHA1 Message Date
15dad0c428 chore: update client versions 2023-01-27 20:47:59 +01:00
06aa677ef7 tests: use tokio_test::block_on() instead of test macro 2023-01-27 20:47:37 +01:00
f94d8db4d0 feat: add logging for all operations
fix: music_artist: fetch visitor data only once
2023-01-27 19:49:16 +01:00
be741d28ca feat: dont fetch artist playlist section 2023-01-25 21:05:52 +01:00
7 changed files with 456 additions and 544 deletions

View file

@ -25,7 +25,6 @@ mod channel_rss;
use std::sync::Arc;
use std::{borrow::Cow, fmt::Debug};
use log::{debug, error, warn};
use once_cell::sync::Lazy;
use rand::Rng;
use regex::Regex;
@ -168,13 +167,13 @@ const YOUTUBE_MUSIC_HOME_URL: &str = "https://music.youtube.com/";
const DISABLE_PRETTY_PRINT_PARAMETER: &str = "&prettyPrint=false";
const DESKTOP_CLIENT_VERSION: &str = "2.20221011.00.00";
const DESKTOP_CLIENT_VERSION: &str = "2.20230126.00.00";
const DESKTOP_API_KEY: &str = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
const TVHTML5_CLIENT_VERSION: &str = "2.0";
const DESKTOP_MUSIC_API_KEY: &str = "AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30";
const DESKTOP_MUSIC_CLIENT_VERSION: &str = "1.20221005.01.00";
const DESKTOP_MUSIC_CLIENT_VERSION: &str = "1.20230123.01.01";
const MOBILE_CLIENT_VERSION: &str = "17.39.35";
const MOBILE_CLIENT_VERSION: &str = "18.03.33";
const ANDROID_API_KEY: &str = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
const IOS_API_KEY: &str = "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc";
const IOS_DEVICE_MODEL: &str = "iPhone14,5";
@ -323,6 +322,7 @@ impl RustyPipeBuilder {
.user_agent(self.user_agent)
.gzip(true)
.brotli(true)
.redirect(reqwest::redirect::Policy::none())
.build()
.unwrap();
@ -331,7 +331,7 @@ impl RustyPipeBuilder {
match serde_json::from_str::<CacheData>(&data) {
Ok(data) => data,
Err(e) => {
error!("Could not deserialize cache. Error: {}", e);
log::error!("Could not deserialize cache. Error: {}", e);
CacheData::default()
}
}
@ -523,7 +523,7 @@ impl RustyPipe {
};
let ms = util::retry_delay(n, 1000, 60000, 3);
warn!("Retry attempt #{}. Error: {}. Waiting {} ms", n, emsg, ms);
log::warn!("Retry attempt #{}. Error: {}. Waiting {} ms", n, emsg, ms);
tokio::time::sleep(std::time::Duration::from_millis(ms.into())).await;
last_res = Some(res);
@ -656,7 +656,7 @@ impl RustyPipe {
match desktop_client.get() {
Some(cdata) => cdata.version.to_owned(),
None => {
debug!("getting desktop client version");
log::debug!("getting desktop client version");
match self.extract_desktop_client_version().await {
Ok(version) => {
*desktop_client = CacheEntry::from(ClientData {
@ -667,7 +667,7 @@ impl RustyPipe {
version
}
Err(e) => {
warn!("{}, falling back to hardcoded version", e);
log::warn!("{}, falling back to hardcoded version", e);
DESKTOP_CLIENT_VERSION.to_owned()
}
}
@ -688,7 +688,7 @@ impl RustyPipe {
match music_client.get() {
Some(cdata) => cdata.version.to_owned(),
None => {
debug!("getting music client version");
log::debug!("getting music client version");
match self.extract_music_client_version().await {
Ok(version) => {
*music_client = CacheEntry::from(ClientData {
@ -699,7 +699,7 @@ impl RustyPipe {
version
}
Err(e) => {
warn!("{}, falling back to hardcoded version", e);
log::warn!("{}, falling back to hardcoded version", e);
DESKTOP_MUSIC_CLIENT_VERSION.to_owned()
}
}
@ -715,7 +715,7 @@ impl RustyPipe {
match deobf.get() {
Some(deobf) => Ok(Deobfuscator::from(deobf.to_owned())),
None => {
debug!("getting deobfuscator");
log::debug!("getting deobfuscator");
let new_deobf = Deobfuscator::new(self.inner.http.clone()).await?;
*deobf = CacheEntry::from(new_deobf.get_data());
drop(deobf);
@ -736,12 +736,13 @@ impl RustyPipe {
match serde_json::to_string(&cdata) {
Ok(data) => storage.write(&data),
Err(e) => error!("Could not serialize cache. Error: {}", e),
Err(e) => log::error!("Could not serialize cache. Error: {}", e),
}
}
}
async fn get_ytm_visitor_data(&self) -> Result<String, Error> {
log::debug!("getting YTM visitor data");
let resp = self.inner.http.get(YOUTUBE_MUSIC_HOME_URL).send().await?;
resp.headers()
@ -1040,6 +1041,8 @@ impl RustyPipeQuery {
body: &B,
deobf: Option<&Deobfuscator>,
) -> Result<M, Error> {
log::debug!("getting {}({})", operation, id);
let request = self
.request_builder(ctype, endpoint)
.await
@ -1212,7 +1215,7 @@ trait MapResponse<T> {
fn validate_country(country: Country) -> Country {
if country == Country::Zz {
warn!("Country:Zz (Global) can only be used for fetching music charts, falling back to Country:Us");
log::warn!("Country:Zz (Global) can only be used for fetching music charts, falling back to Country:Us");
Country::Us
} else {
country
@ -1223,10 +1226,11 @@ fn validate_country(country: Country) -> Country {
mod tests {
use super::*;
#[tokio::test]
async fn t_get_ytm_visitor_data() {
#[test]
fn t_get_ytm_visitor_data() {
let rp = RustyPipe::new();
let visitor_data = rp.get_ytm_visitor_data().await.unwrap();
let visitor_data = tokio_test::block_on(rp.get_ytm_visitor_data()).unwrap();
dbg!(&visitor_data);
assert!(visitor_data.ends_with("%3D"));
assert_eq!(visitor_data.len(), 32)
}

View file

@ -34,77 +34,83 @@ impl RustyPipeQuery {
artist_id: S,
all_albums: bool,
) -> Result<MusicArtist, Error> {
let res = self._music_artist(artist_id, all_albums).await;
let artist_id = artist_id.as_ref();
let visitor_data = match all_albums {
true => Some(self.get_ytm_visitor_data().await?),
false => None,
};
let res = self._music_artist(artist_id, visitor_data.as_deref()).await;
if let Err(Error::Extraction(ExtractionError::Redirect(id))) = res {
self._music_artist(&id, all_albums).await.map(|x| *x)
log::debug!("music artist {} redirects to {}", artist_id, &id);
self._music_artist(&id, visitor_data.as_deref()).await
} else {
res.map(|x| *x)
res
}
}
async fn _music_artist<S: AsRef<str>>(
async fn _music_artist(
&self,
artist_id: S,
all_albums: bool,
) -> Result<Box<MusicArtist>, Error> {
let artist_id = artist_id.as_ref();
artist_id: &str,
all_albums_vdata: Option<&str>,
) -> Result<MusicArtist, Error> {
match all_albums_vdata {
Some(visitor_data) => {
let context = self
.get_context(ClientType::DesktopMusic, true, Some(visitor_data))
.await;
let request_body = QBrowse {
context,
browse_id: artist_id,
};
if all_albums {
let visitor_data = self.get_ytm_visitor_data().await?;
let context = self
.get_context(ClientType::DesktopMusic, true, Some(&visitor_data))
.await;
let request_body = QBrowse {
context,
browse_id: artist_id,
};
let (mut artist, album_page_params) = self
.execute_request::<response::MusicArtist, _, _>(
ClientType::DesktopMusic,
"music_artist",
artist_id,
"browse",
&request_body,
)
.await?;
let (mut artist, album_page_params) = self
.execute_request::<response::MusicArtist, _, _>(
let visitor_data = Rc::new(visitor_data);
let album_page_results = stream::iter(album_page_params)
.map(|params| {
let visitor_data = visitor_data.clone();
async move {
self.music_artist_album_page(artist_id, &params, &visitor_data)
.await
}
})
.buffer_unordered(2)
.collect::<Vec<_>>()
.await;
for res in album_page_results {
let mut res = res?;
artist.albums.append(&mut res);
}
Ok(artist)
}
None => {
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
let request_body = QBrowse {
context,
browse_id: artist_id,
};
self.execute_request::<response::MusicArtist, _, _>(
ClientType::DesktopMusic,
"music_artist",
artist_id,
"browse",
&request_body,
)
.await?;
let visitor_data = Rc::new(visitor_data);
let album_page_results = stream::iter(album_page_params)
.map(|params| {
let visitor_data = visitor_data.clone();
async move {
self.music_artist_album_page(artist_id, &params, &visitor_data)
.await
}
})
.buffer_unordered(2)
.collect::<Vec<_>>()
.await;
for res in album_page_results {
let mut res = res?;
artist.albums.append(&mut res);
.await
}
Ok(artist.into())
} else {
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
let request_body = QBrowse {
context,
browse_id: artist_id,
};
self.execute_request::<response::MusicArtist, _, _>(
ClientType::DesktopMusic,
"music_artist",
artist_id,
"browse",
&request_body,
)
.await
.map(|x: MusicArtist| x.into())
}
}
@ -237,14 +243,24 @@ fn map_artist_page(
{
if let Some(cfg) = bep.browse_endpoint_context_supported_configs {
match cfg.browse_endpoint_context_music_config.page_type {
// Music videos
PageType::Playlist => {
if videos_playlist_id.is_none() {
videos_playlist_id = Some(bep.browse_id);
}
}
// Albums or playlists
PageType::Artist => {
album_page_params.push(bep.params);
extendable_albums = true;
// Peek at the first item to determine type
if let Some(response::music_item::MusicResponseItem::MusicTwoRowItemRenderer(item)) = shelf.contents.c.first() {
if let Some(PageType::Album) = item.navigation_endpoint.browse_endpoint.as_ref().and_then(|be| {
be.browse_endpoint_context_supported_configs.as_ref().map(|config| {
config.browse_endpoint_context_music_config.page_type
})}) {
album_page_params.push(bep.params);
extendable_albums = true;
}
}
}
_ => {}
}

View file

@ -676,7 +676,7 @@ mod tests {
let (url, throttled) = map_res.c.unwrap();
assert_eq!(url, "https://rr5---sn-h0jelnez.googlevideo.com/videoplayback?c=WEB&clen=3781277&dur=229.301&ei=vb7nYvH5BMK8gAfBj7ToBQ&expire=1659376413&fexp=24001373%2C24007246&fvip=5&gir=yes&id=o-AB_BABwrXZJN428ZwDxq5ScPn2AbcGODnRlTVhCQ3mj2&initcwndbps=1588750&ip=2003%3Ade%3Aaf06%3A6300%3Ac750%3A1b77%3Ac74a%3A80e3&itag=251&keepalive=yes&lmt=1655510291473933&lsig=AG3C_xAwRQIgCKCGJ1iu4wlaGXy3jcJyU3inh9dr1FIfqYOZEG_MdmACIQCbungkQYFk7EhD6K2YvLaHFMjKOFWjw001_tLb0lPDtg%3D%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=hH&mime=audio%2Fwebm&mm=31%2C26&mn=sn-h0jelnez%2Csn-4g5ednsl&ms=au%2Conr&mt=1659354538&mv=m&mvi=5&n=XzXGSfGusw6OCQ&ns=b_Mq_qlTFcSGlG9RpwpM9xQH&pl=37&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRQIhAPIsKd7-xi4xVHEC9gb__dU4hzfzsHEj9ytd3nt0gEceAiACJWBcw-wFEq9qir35bwKHJZxtQ9mOL7SKiVkLQNDa6A%3D%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-Khi831z8dTejFIRCvCEwx_6romtM&txp=4532434&vprv=1");
assert_eq!(throttled, false);
assert!(!throttled);
assert!(
map_res.warnings.is_empty(),
"deserialization/mapping warnings: {:?}",

View file

@ -195,38 +195,37 @@ impl From<PageType> for MusicPageType {
impl NavigationEndpoint {
pub(crate) fn music_page(self) -> Option<(MusicPageType, String)> {
match self.browse_endpoint {
Some(browse) => match browse.browse_endpoint_context_supported_configs {
Some(config) => Some((
config.browse_endpoint_context_music_config.page_type.into(),
browse.browse_id,
)),
None => None,
},
None => None,
}
.or_else(|| {
self.watch_endpoint.map(|watch| {
if watch
.playlist_id
.map(|plid| plid.starts_with("RDQM"))
.unwrap_or_default()
{
// Genre radios (e.g. "pop radio") will be skipped
(MusicPageType::None, watch.video_id)
} else {
self.browse_endpoint
.and_then(|be| {
be.browse_endpoint_context_supported_configs.map(|config| {
(
MusicPageType::Track {
is_video: watch
.watch_endpoint_music_supported_configs
.watch_endpoint_music_config
.music_video_type
== MusicVideoType::Video,
},
watch.video_id,
config.browse_endpoint_context_music_config.page_type.into(),
be.browse_id,
)
}
})
})
.or_else(|| {
self.watch_endpoint.map(|watch| {
if watch
.playlist_id
.map(|plid| plid.starts_with("RDQM"))
.unwrap_or_default()
{
// Genre radios (e.g. "pop radio") will be skipped
(MusicPageType::None, watch.video_id)
} else {
(
MusicPageType::Track {
is_video: watch
.watch_endpoint_music_supported_configs
.watch_endpoint_music_config
.music_video_type
== MusicVideoType::Video,
},
watch.video_id,
)
}
})
})
})
}
}

View file

@ -1,5 +1,4 @@
use fancy_regex::Regex as FancyRegex;
use log::debug;
use once_cell::sync::Lazy;
use regex::Regex;
use reqwest::Client;
@ -27,7 +26,7 @@ impl Deobfuscator {
let js_url = get_player_js_url(&http).await?;
let player_js = get_response(&http, &js_url).await?;
debug!("Downloaded player.js from {}", js_url);
log::debug!("downloaded player.js from {}", js_url);
let sig_fn = get_sig_fn(&player_js)?;
let nsig_fn = get_nsig_fn(&player_js)?;
@ -389,18 +388,18 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
assert_eq!(res, "nrkec0fwgTWolw");
}
#[test(tokio::test)]
async fn t_get_player_js_url() {
#[test]
fn t_get_player_js_url() {
let client = Client::new();
let url = get_player_js_url(&client).await.unwrap();
let url = tokio_test::block_on(get_player_js_url(&client)).unwrap();
assert!(url.starts_with("https://www.youtube.com/s/player"));
assert_eq!(url.len(), 73);
}
#[test(tokio::test)]
async fn t_update() {
#[test]
fn t_update() {
let client = Client::new();
let deobf = Deobfuscator::new(client).await.unwrap();
let deobf = tokio_test::block_on(Deobfuscator::new(client)).unwrap();
let deobf_sig = deobf.deobfuscate_sig("GOqGOqGOq0QJ8wRAIgaryQHfplJ9xJSKFywyaSMHuuwZYsoMTAvRvfm51qIGECIA5061zWeyfMPX9hEl_U6f9J0tr7GTJMKyPf5XNrJb5fb5i").unwrap();
println!("{}", deobf_sig);

View file

@ -538,8 +538,8 @@ mod tests {
txt: Vec<String>,
}
let res_str = serde_json::from_str::<S>(&test_json).unwrap();
let res_vec = serde_json::from_str::<SVec>(&test_json).unwrap();
let res_str = serde_json::from_str::<S>(test_json).unwrap();
let res_vec = serde_json::from_str::<SVec>(test_json).unwrap();
assert_eq!(res_str.txt, exp.join(""));
assert_eq!(res_vec.txt, exp);
@ -582,7 +582,7 @@ mod tests {
}
}"#;
let res = serde_json::from_str::<SLink>(&test_json).unwrap();
let res = serde_json::from_str::<SLink>(test_json).unwrap();
insta::assert_debug_snapshot!(res, @r###"
SLink {
ln: Video {
@ -617,7 +617,7 @@ mod tests {
}
}"#;
let res = serde_json::from_str::<SLink>(&test_json).unwrap();
let res = serde_json::from_str::<SLink>(test_json).unwrap();
insta::assert_debug_snapshot!(res, @r###"
SLink {
ln: Browse {
@ -651,7 +651,7 @@ mod tests {
}
}"#;
let res = serde_json::from_str::<SLink>(&test_json).unwrap();
let res = serde_json::from_str::<SLink>(test_json).unwrap();
insta::assert_debug_snapshot!(res, @r###"
SLink {
ln: Browse {
@ -675,7 +675,7 @@ mod tests {
}
}"#;
let res = serde_json::from_str::<SLink>(&test_json).unwrap();
let res = serde_json::from_str::<SLink>(test_json).unwrap();
insta::assert_debug_snapshot!(res, @r###"
SLink {
ln: Text {
@ -710,7 +710,7 @@ mod tests {
}
}"#;
let res = serde_json::from_str::<SLink>(&test_json).unwrap();
let res = serde_json::from_str::<SLink>(test_json).unwrap();
insta::assert_debug_snapshot!(res, @r###"
SLink {
ln: Web {
@ -759,7 +759,7 @@ mod tests {
}
}"#;
let res = serde_json::from_str::<SLinks>(&test_json).unwrap();
let res = serde_json::from_str::<SLinks>(test_json).unwrap();
insta::assert_debug_snapshot!(res, @r###"
SLinks {
ln: TextComponents(
@ -787,7 +787,7 @@ mod tests {
fn t_links_empty() {
let test_json = r#"{"ln": {}}"#;
let res = serde_json::from_str::<SLinks>(&test_json).unwrap();
let res = serde_json::from_str::<SLinks>(test_json).unwrap();
assert!(res.ln.0.is_empty())
}

File diff suppressed because it is too large Load diff