From dae8c1e775587f5f188877bd2bdb60cac09a786d Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Thu, 10 Nov 2022 23:26:05 +0100 Subject: [PATCH 1/2] fix: used borrowed str for QBrowse --- src/client/mod.rs | 2 +- src/client/music_artist.rs | 4 ++-- src/client/music_playlist.rs | 4 ++-- src/client/playlist.rs | 2 +- src/client/trends.rs | 4 ++-- tests/youtube.rs | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/client/mod.rs b/src/client/mod.rs index 497d1b0..4509c90 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -136,7 +136,7 @@ struct ThirdParty<'a> { #[serde(rename_all = "camelCase")] struct QBrowse<'a> { context: YTContext<'a>, - browse_id: String, + browse_id: &'a str, } #[derive(Debug, Serialize)] diff --git a/src/client/music_artist.rs b/src/client/music_artist.rs index 2039222..1ecbae1 100644 --- a/src/client/music_artist.rs +++ b/src/client/music_artist.rs @@ -42,7 +42,7 @@ impl RustyPipeQuery { .await; let request_body = QBrowse { context, - browse_id: artist_id.to_owned(), + browse_id: artist_id, }; let (mut artist, album_page_params) = self @@ -78,7 +78,7 @@ impl RustyPipeQuery { let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { context, - browse_id: artist_id.to_owned(), + browse_id: artist_id, }; self.execute_request::( diff --git a/src/client/music_playlist.rs b/src/client/music_playlist.rs index 87ed521..65d9c37 100644 --- a/src/client/music_playlist.rs +++ b/src/client/music_playlist.rs @@ -20,7 +20,7 @@ impl RustyPipeQuery { let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { context, - browse_id: "VL".to_owned() + playlist_id, + browse_id: &format!("VL{}", playlist_id), }; self.execute_request::( @@ -37,7 +37,7 @@ impl RustyPipeQuery { let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { context, - browse_id: album_id.to_owned(), + browse_id: album_id, }; self.execute_request::( diff --git a/src/client/playlist.rs b/src/client/playlist.rs index 7526e6f..ad00e27 100644 --- a/src/client/playlist.rs +++ b/src/client/playlist.rs @@ -18,7 +18,7 @@ impl RustyPipeQuery { let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QBrowse { context, - browse_id: "VL".to_owned() + playlist_id, + browse_id: &format!("VL{}", playlist_id), }; self.execute_request::( diff --git a/src/client/trends.rs b/src/client/trends.rs index 26a59a8..6d0f8a3 100644 --- a/src/client/trends.rs +++ b/src/client/trends.rs @@ -15,7 +15,7 @@ impl RustyPipeQuery { let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QBrowse { context, - browse_id: "FEwhat_to_watch".to_owned(), + browse_id: "FEwhat_to_watch", }; self.execute_request::( @@ -32,7 +32,7 @@ impl RustyPipeQuery { let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QBrowse { context, - browse_id: "FEtrending".to_owned(), + browse_id: "FEtrending", }; self.execute_request::( diff --git a/tests/youtube.rs b/tests/youtube.rs index 09eb248..694d905 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -1784,7 +1784,7 @@ async fn music_radio_playlist() { .music_radio_playlist("PL5dDx681T4bR7ZF1IuWzOv1omlRbE7PiJ") .await .unwrap(); - assert_next(tracks, &rp.query(), 20, 1).await; + assert_next(tracks, &rp.query(), 10, 1).await; } //#TESTUTIL From 2cd74b1da84fa5bcd7a05279c23a9f815ef89e03 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Fri, 11 Nov 2022 00:14:58 +0100 Subject: [PATCH 2/2] feat: add lyrics --- README.md | 4 +- codegen/src/download_testfiles.rs | 19 ++++ src/client/mod.rs | 8 ++ src/client/music_artist.rs | 6 +- src/client/music_details.rs | 63 ++++++++++- src/client/response/mod.rs | 7 ++ src/client/response/music_details.rs | 27 ++++- ...usic_details__tests__map_music_lyrics.snap | 8 ++ src/model/mod.rs | 7 ++ testfiles/music_details/lyrics.json | 103 ++++++++++++++++++ tests/snapshots/youtube__music_lyrics.snap | 8 ++ tests/youtube.rs | 12 ++ 12 files changed, 262 insertions(+), 10 deletions(-) create mode 100644 src/client/snapshots/rustypipe__client__music_details__tests__map_music_lyrics.snap create mode 100644 testfiles/music_details/lyrics.json create mode 100644 tests/snapshots/youtube__music_lyrics.snap diff --git a/README.md b/README.md index 404346a..a5d0974 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ inspired by [NewPipe](https://github.com/TeamNewPipe/NewPipeExtractor). - [X] **Artist** - [X] **Search** - [ ] **Search suggestions** -- [ ] **Radio** -- [ ] **Track details** +- [X] **Radio** +- [ ] **Track details** (lyrics, recommendations) - [ ] **Moods** - [ ] **Charts** - [ ] **New** diff --git a/codegen/src/download_testfiles.rs b/codegen/src/download_testfiles.rs index c618342..8eea853 100644 --- a/codegen/src/download_testfiles.rs +++ b/codegen/src/download_testfiles.rs @@ -50,6 +50,7 @@ pub async fn download_testfiles(project_root: &Path) { music_search_cont(&testfiles).await; music_artist(&testfiles).await; music_details(&testfiles).await; + music_lyrics(&testfiles).await; music_radio(&testfiles).await; music_radio_cont(&testfiles).await; } @@ -716,6 +717,24 @@ async fn music_details(testfiles: &Path) { } } +async fn music_lyrics(testfiles: &Path) { + let mut json_path = testfiles.to_path_buf(); + json_path.push("music_details"); + json_path.push("lyrics.json"); + if json_path.exists() { + return; + } + + let rp = RustyPipe::new(); + let res = rp.query().music_details("n4tK7LYFxI0").await.unwrap(); + + let rp = rp_testfile(&json_path); + rp.query() + .music_lyrics(&res.lyrics_id.unwrap()) + .await + .unwrap(); +} + async fn music_radio(testfiles: &Path) { for (name, id) in [("mv", "RDAMVMZeerrnuLi5E"), ("track", "RDAMVM7nigXQS1Xb0")] { let mut json_path = testfiles.to_path_buf(); diff --git a/src/client/mod.rs b/src/client/mod.rs index 4509c90..e3332f6 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -990,6 +990,14 @@ impl RustyPipeQuery { } } + /// Get a YouTube Music visitor data cookie, which is necessary for certain requests + async fn get_ytm_visitor_data(&self) -> Result { + match &self.opts.visitor_data { + Some(vd) => Ok(vd.to_owned()), + None => self.client.get_ytm_visitor_data().await, + } + } + /// Execute a request to the YouTube API, then deobfuscate and map the response. /// /// Creates a report in case of failure for easy debugging. diff --git a/src/client/music_artist.rs b/src/client/music_artist.rs index 1ecbae1..e8e2a65 100644 --- a/src/client/music_artist.rs +++ b/src/client/music_artist.rs @@ -32,11 +32,7 @@ impl RustyPipeQuery { all_albums: bool, ) -> Result { if all_albums { - let visitor_data = match &self.opts.visitor_data { - Some(vd) => vd.to_owned(), - None => self.client.get_ytm_visitor_data().await?, - }; - + let visitor_data = self.get_ytm_visitor_data().await?; let context = self .get_context(ClientType::DesktopMusic, true, Some(&visitor_data)) .await; diff --git a/src/client/music_details.rs b/src/client/music_details.rs index e30e9e7..4ab9b68 100644 --- a/src/client/music_details.rs +++ b/src/client/music_details.rs @@ -4,14 +4,14 @@ use serde::Serialize; use crate::{ error::{Error, ExtractionError}, - model::{Paginator, TrackDetails, TrackItem}, + model::{Lyrics, Paginator, TrackDetails, TrackItem}, param::Language, serializer::MapResult, }; use super::{ response::{self, music_item::map_queue_item}, - ClientType, MapResponse, RustyPipeQuery, YTContext, + ClientType, MapResponse, QBrowse, RustyPipeQuery, YTContext, }; #[derive(Debug, Serialize)] @@ -54,6 +54,23 @@ impl RustyPipeQuery { .await } + pub async fn music_lyrics(&self, lyrics_id: &str) -> Result { + let context = self.get_context(ClientType::DesktopMusic, true, None).await; + let request_body = QBrowse { + context, + browse_id: lyrics_id, + }; + + self.execute_request::( + ClientType::DesktopMusic, + "music_lyrics", + lyrics_id, + "browse", + &request_body, + ) + .await + } + pub async fn music_radio(&self, radio_id: &str) -> Result, Error> { let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QRadio { @@ -214,6 +231,31 @@ impl MapResponse> for response::MusicDetails { } } +impl MapResponse for response::MusicLyrics { + fn map_response( + self, + _id: &str, + _lang: Language, + _deobf: Option<&crate::deobfuscate::Deobfuscator>, + ) -> Result, ExtractionError> { + let lyrics = self + .contents + .section_list_renderer + .contents + .into_iter() + .find_map(|item| item.music_description_shelf_renderer) + .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))?; + + Ok(MapResult { + c: Lyrics { + body: lyrics.description, + footer: lyrics.footer, + }, + warnings: Vec::new(), + }) + } +} + #[cfg(test)] mod tests { use std::{fs::File, io::BufReader, path::Path}; @@ -264,4 +306,21 @@ mod tests { ); insta::assert_ron_snapshot!(format!("map_music_radio_{}", name), map_res.c); } + + #[test] + fn map_lyrics() { + let json_path = Path::new("testfiles/music_details/lyrics.json"); + let json_file = File::open(json_path).unwrap(); + + let lyrics: response::MusicLyrics = + serde_json::from_reader(BufReader::new(json_file)).unwrap(); + let map_res: MapResult = lyrics.map_response("", Language::En, None).unwrap(); + + assert!( + map_res.warnings.is_empty(), + "deserialization/mapping warnings: {:?}", + map_res.warnings + ); + insta::assert_ron_snapshot!(format!("map_music_lyrics"), map_res.c); + } } diff --git a/src/client/response/mod.rs b/src/client/response/mod.rs index 5a09fcb..4dc11e7 100644 --- a/src/client/response/mod.rs +++ b/src/client/response/mod.rs @@ -16,6 +16,7 @@ pub(crate) use channel::Channel; pub(crate) use music_artist::MusicArtist; pub(crate) use music_artist::MusicArtistAlbums; pub(crate) use music_details::MusicDetails; +pub(crate) use music_details::MusicLyrics; pub(crate) use music_item::MusicContinuation; pub(crate) use music_playlist::MusicPlaylist; pub(crate) use music_search::MusicSearch; @@ -62,6 +63,12 @@ pub(crate) struct Tab { pub tab_renderer: ContentRenderer, } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SectionList { + pub section_list_renderer: ContentsRenderer, +} + #[derive(Default, Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct ThumbnailsWrap { diff --git a/src/client/response/music_details.rs b/src/client/response/music_details.rs index 3cd24d9..e921d2e 100644 --- a/src/client/response/music_details.rs +++ b/src/client/response/music_details.rs @@ -1,6 +1,9 @@ use serde::Deserialize; +use serde_with::serde_as; -use super::{music_item::PlaylistPanelRenderer, ContentRenderer}; +use crate::serializer::text::Text; + +use super::{music_item::PlaylistPanelRenderer, ContentRenderer, SectionList}; /// Response model for YouTube Music track details #[derive(Debug, Deserialize)] @@ -91,3 +94,25 @@ pub(crate) struct TabContent { pub(crate) struct PlaylistPanel { pub playlist_panel_renderer: PlaylistPanelRenderer, } + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct MusicLyrics { + pub contents: SectionList, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct LyricsContents { + pub music_description_shelf_renderer: Option, +} + +#[serde_as] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct LyricsRenderer { + #[serde_as(as = "Text")] + pub description: String, + #[serde_as(as = "Text")] + pub footer: String, +} diff --git a/src/client/snapshots/rustypipe__client__music_details__tests__map_music_lyrics.snap b/src/client/snapshots/rustypipe__client__music_details__tests__map_music_lyrics.snap new file mode 100644 index 0000000..72b1752 --- /dev/null +++ b/src/client/snapshots/rustypipe__client__music_details__tests__map_music_lyrics.snap @@ -0,0 +1,8 @@ +--- +source: src/client/music_details.rs +expression: map_res.c +--- +Lyrics( + body: "Eyes, in the sky, gazing far into the night\nI raise my hand to the fire, but it\'s no use\n\'Cause you can\'t stop it from shining through\nIt\'s true\nBaby let the light shine through\nIf you believe it\'s true\nBaby won\'t you let the light shine through\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nWon\'t you let the light shine through\n\nEyes, in the sky, gazing far into the night\nI raise my hand to the fire, but it\'s no use\n\'Cause you can\'t stop it from shining through\nIt\'s true\nBaby let the light shine through\nIf you believe it\'s true\nBaby won\'t you let the light shine through\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you", + footer: "Source: Musixmatch", +) diff --git a/src/model/mod.rs b/src/model/mod.rs index 5d313cf..18b7741 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -1239,3 +1239,10 @@ pub struct TrackDetails { pub lyrics_id: Option, pub related_id: Option, } + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[non_exhaustive] +pub struct Lyrics { + pub body: String, + pub footer: String, +} diff --git a/testfiles/music_details/lyrics.json b/testfiles/music_details/lyrics.json new file mode 100644 index 0000000..72873aa --- /dev/null +++ b/testfiles/music_details/lyrics.json @@ -0,0 +1,103 @@ +{ + "contents": { + "sectionListRenderer": { + "contents": [ + { + "musicDescriptionShelfRenderer": { + "description": { + "runs": [ + { + "text": "Eyes, in the sky, gazing far into the night\nI raise my hand to the fire, but it's no use\n'Cause you can't stop it from shining through\nIt's true\nBaby let the light shine through\nIf you believe it's true\nBaby won't you let the light shine through\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nWon't you let the light shine through\n\nEyes, in the sky, gazing far into the night\nI raise my hand to the fire, but it's no use\n'Cause you can't stop it from shining through\nIt's true\nBaby let the light shine through\nIf you believe it's true\nBaby won't you let the light shine through\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you" + } + ] + }, + "footer": { + "runs": [ + { + "text": "Source: Musixmatch" + } + ] + }, + "maxCollapsedLines": 0, + "maxExpandedLines": 0, + "onShowCommands": [ + { + "clickTrackingParams": "CAIQ2fQEGAAiEwjpyODp3aT7AhUe0REIHV-EAKs=", + "logLyricEventCommand": { + "serializedLyricInfo": "Eg10X3E4eGptbTdDbEpMGAIiCDE4NTc2OTU0" + } + } + ], + "trackingParams": "CAIQ2fQEGAAiEwjpyODp3aT7AhUe0REIHV-EAKs=" + } + } + ], + "trackingParams": "CAEQui8iEwjpyODp3aT7AhUe0REIHV-EAKs=" + } + }, + "responseContext": { + "serviceTrackingParams": [ + { + "params": [ + { + "key": "has_unlimited_entitlement", + "value": "False" + }, + { + "key": "browse_id", + "value": "MPLYt_q8xjmm7ClJL" + }, + { + "key": "logged_in", + "value": "0" + }, + { + "key": "e", + "value": "1714259,23804281,23848211,23882503,23918597,23934970,23940248,23946420,23966208,23983296,23998056,24001373,24002022,24002025,24004644,24007246,24034168,24036948,24077241,24078244,24080738,24108447,24120819,24135310,24135692,24140247,24147965,24161116,24162920,24164186,24169501,24181174,24185614,24187043,24187377,24191629,24197450,24199724,24200839,24209350,24211178,24216167,24217535,24219713,24228638,24230619,24241378,24248091,24253729,24255165,24255543,24255545,24260783,24262346,24263796,24267564,24267570,24268142,24269410,24278596,24279196,24279852,24281671,24283556,24286005,24286017,24287327,24288043,24290971,24292955,24293803,24295708,24299747,24390374,24390675,24391018,24391537,24392450,24392500,24394549,24396819,24397910,24398998,24399052,24399916,24401557,24402891,24403118,24404641,24406605,24407199,24407665,24409401,24410273,24413557,24413558,24415139,24590921,39322504,39322574" + } + ], + "service": "GFEEDBACK" + }, + { + "params": [ + { + "key": "c", + "value": "WEB_REMIX" + }, + { + "key": "cver", + "value": "1.20221107.01.00" + }, + { + "key": "yt_li", + "value": "0" + }, + { + "key": "GetBrowseTrackLyricsPage_rid", + "value": "0x8d12ba6a166bab0c" + } + ], + "service": "CSI" + }, + { + "params": [ + { + "key": "client.version", + "value": "1.20000101" + }, + { + "key": "client.name", + "value": "WEB_REMIX" + }, + { + "key": "client.fexp", + "value": "24217535,24295708,24590921,24401557,24410273,24120819,24278596,24001373,24290971,24077241,24292955,23848211,24036948,24255545,24286005,24219713,39322504,24279196,24002022,24080738,24413557,23918597,24404641,24209350,39322574,24390675,23940248,24169501,24391018,24002025,24262346,24392500,24396819,24255165,24241378,24279852,24281671,24162920,1714259,24181174,24200839,24034168,24108447,24402891,24216167,24197450,24253729,23998056,24283556,24007246,24403118,24255543,24406605,24161116,24293803,24004644,24230619,24399052,24191629,24211178,24407199,24135310,23882503,24260783,24263796,24391537,24392450,23983296,24415139,24267564,24187043,24409401,24286017,24287327,24399916,23934970,24228638,24398998,24185614,24268142,24394549,24187377,24140247,24164186,24135692,24269410,24288043,24407665,24199724,24390374,24397910,23966208,24267570,24078244,24248091,24147965,24413558,23946420,23804281,24299747" + } + ], + "service": "ECATCHER" + } + ], + "visitorData": "Cgs3R19icmhVUkNYOCikibabBg%3D%3D" + }, + "trackingParams": "CAAQhGciEwjpyODp3aT7AhUe0REIHV-EAKs=" +} diff --git a/tests/snapshots/youtube__music_lyrics.snap b/tests/snapshots/youtube__music_lyrics.snap new file mode 100644 index 0000000..ecb76ac --- /dev/null +++ b/tests/snapshots/youtube__music_lyrics.snap @@ -0,0 +1,8 @@ +--- +source: tests/youtube.rs +expression: lyrics +--- +Lyrics( + body: "Eyes, in the sky, gazing far into the night\nI raise my hand to the fire, but it\'s no use\n\'Cause you can\'t stop it from shining through\nIt\'s true\nBaby let the light shine through\nIf you believe it\'s true\nBaby won\'t you let the light shine through\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nWon\'t you let the light shine through\n\nEyes, in the sky, gazing far into the night\nI raise my hand to the fire, but it\'s no use\n\'Cause you can\'t stop it from shining through\nIt\'s true\nBaby let the light shine through\nIf you believe it\'s true\nBaby won\'t you let the light shine through\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you", + footer: "Source: Musixmatch", +) diff --git a/tests/youtube.rs b/tests/youtube.rs index 694d905..9c9b99c 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -1769,6 +1769,18 @@ async fn music_details(#[case] name: &str, #[case] id: &str) { ); } +#[tokio::test] +async fn music_lyrics() { + let rp = RustyPipe::builder().strict().build(); + let track = rp.query().music_details("n4tK7LYFxI0").await.unwrap(); + let lyrics = rp + .query() + .music_lyrics(&track.lyrics_id.unwrap()) + .await + .unwrap(); + insta::assert_ron_snapshot!(lyrics); +} + #[tokio::test] async fn music_radio_track() { let rp = RustyPipe::builder().strict().build();