From 38bc12f695def431e1a6f6c7818fdda740dba137 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 30 Nov 2022 21:11:33 +0100 Subject: [PATCH 1/7] fix: accept music album playlist response without header --- src/client/music_playlist.rs | 107 +++++++++++++++++--------- src/client/response/music_playlist.rs | 2 +- src/model/mod.rs | 2 +- 3 files changed, 74 insertions(+), 37 deletions(-) diff --git a/src/client/music_playlist.rs b/src/client/music_playlist.rs index e672975..6db4c7e 100644 --- a/src/client/music_playlist.rs +++ b/src/client/music_playlist.rs @@ -2,7 +2,7 @@ use std::borrow::Cow; use crate::{ error::{Error, ExtractionError}, - model::{AlbumId, ChannelId, MusicAlbum, MusicPlaylist, Paginator}, + model::{AlbumId, ChannelId, MusicAlbum, MusicPlaylist, Paginator, TrackItem}, serializer::MapResult, util::{self, TryRemove}, }; @@ -65,8 +65,6 @@ impl MapResponse for response::MusicPlaylist { ) -> Result, ExtractionError> { // dbg!(&self); - let header = self.header.music_detail_header_renderer; - let mut content = self.contents.single_column_browse_results_renderer.contents; let mut music_contents = content .try_swap_remove(0) @@ -85,31 +83,15 @@ impl MapResponse for response::MusicPlaylist { "no sectionListRenderer content", )))?; - let playlist_id = shelf - .playlist_id - .ok_or(ExtractionError::InvalidData(Cow::Borrowed( - "no playlist id", - )))?; - - if playlist_id != id { - return Err(ExtractionError::WrongResult(format!( - "got wrong playlist id {}, expected {}", - playlist_id, id - ))); + if let Some(playlist_id) = shelf.playlist_id { + if playlist_id != id { + return Err(ExtractionError::WrongResult(format!( + "got wrong playlist id {}, expected {}", + playlist_id, id + ))); + } } - let from_ytm = header - .subtitle - .0 - .iter() - .any(|c| c.as_str() == util::YT_MUSIC_NAME); - - let channel = header - .subtitle - .0 - .into_iter() - .find_map(|c| ChannelId::try_from(c).ok()); - let mut mapper = MusicListMapper::new(lang); mapper.map_response(shelf.contents); let map_res = mapper.conv_items(); @@ -120,10 +102,12 @@ impl MapResponse for response::MusicPlaylist { .map(|cont| cont.next_continuation_data.continuation); let track_count = match ctoken { - Some(_) => header - .second_subtitle - .first() - .and_then(|txt| util::parse_numeric::(txt).ok()), + Some(_) => self.header.as_ref().and_then(|h| { + h.music_detail_header_renderer + .second_subtitle + .first() + .and_then(|txt| util::parse_numeric::(txt).ok()) + }), None => Some(map_res.c.len() as u64), }; @@ -132,13 +116,63 @@ impl MapResponse for response::MusicPlaylist { .try_swap_remove(0) .map(|c| c.next_continuation_data.continuation); + let (from_ytm, channel, name, thumbnail, description) = match self.header { + Some(header) => { + let h = header.music_detail_header_renderer; + + let from_ytm = h + .subtitle + .0 + .iter() + .any(|c| c.as_str() == util::YT_MUSIC_NAME); + let channel = h + .subtitle + .0 + .into_iter() + .find_map(|c| ChannelId::try_from(c).ok()); + + ( + from_ytm, + channel, + h.title, + h.thumbnail.into(), + h.description, + ) + } + None => { + // Album playlists fetched via the playlist method dont include a header + let (album, cover) = map_res + .c + .first() + .and_then(|t: &TrackItem| { + t.album.as_ref().map(|a| (a.clone(), t.cover.clone())) + }) + .ok_or(ExtractionError::InvalidData(Cow::Borrowed( + "playlist without header or album items", + )))?; + + if !map_res.c.iter().all(|t| { + t.album + .as_ref() + .map(|a| a.id == album.id) + .unwrap_or_default() + }) { + return Err(ExtractionError::InvalidData(Cow::Borrowed( + "album playlist containing items from different albums", + ))); + } + + (true, None, album.name, cover, None) + } + }; + Ok(MapResult { c: MusicPlaylist { - id: playlist_id, - name: header.title, - thumbnail: header.thumbnail.into(), + id: id.to_owned(), + name, + thumbnail, channel, - description: header.description, + description, track_count, from_ytm, tracks: Paginator::new_ext( @@ -170,7 +204,10 @@ impl MapResponse for response::MusicPlaylist { ) -> Result, ExtractionError> { // dbg!(&self); - let header = self.header.music_detail_header_renderer; + let header = self + .header + .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no header")))? + .music_detail_header_renderer; let mut content = self.contents.single_column_browse_results_renderer.contents; let sections = content diff --git a/src/client/response/music_playlist.rs b/src/client/response/music_playlist.rs index 6e2fb2e..6b8aa49 100644 --- a/src/client/response/music_playlist.rs +++ b/src/client/response/music_playlist.rs @@ -11,7 +11,7 @@ use super::{ContentsRenderer, Tab}; #[serde(rename_all = "camelCase")] pub(crate) struct MusicPlaylist { pub contents: Contents, - pub header: Header, + pub header: Option
, } #[derive(Debug, Deserialize)] diff --git a/src/model/mod.rs b/src/model/mod.rs index a037b19..2c8fbfd 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -1031,7 +1031,7 @@ pub struct ArtistId { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[non_exhaustive] pub struct AlbumItem { - /// Unique YouTube album ID (e.g. `OLAK5uy_nZpcQys48R0aNb046hV-n1OAHGE4reftQ`) + /// Unique YouTube album ID (e.g. `MPREb_T5s950Swfdy`) pub id: String, /// Album name pub name: String, From 01a2717c11874e9aeaa7430263ad5b0e542dd876 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 30 Nov 2022 22:43:50 +0100 Subject: [PATCH 2/7] fix: map track/mv type correctly for album items --- src/client/response/music_item.rs | 67 +++++++++++++------ src/client/response/url_endpoint.rs | 39 ++++++++++- ...t__tests__map_music_album_description.snap | 4 +- ...aylist__tests__map_music_album_single.snap | 2 +- ...ests__map_music_album_various_artists.snap | 2 +- src/model/richtext.rs | 2 +- ...text__tests__t_attributed_description.snap | 1 + src/serializer/text.rs | 11 ++- tests/snapshots/youtube__music_album_ep.snap | 2 +- .../youtube__music_album_single.snap | 2 +- .../youtube__music_album_various_artists.snap | 2 +- 11 files changed, 104 insertions(+), 30 deletions(-) diff --git a/src/client/response/music_item.rs b/src/client/response/music_item.rs index c505b02..d29ebd2 100644 --- a/src/client/response/music_item.rs +++ b/src/client/response/music_item.rs @@ -418,10 +418,12 @@ impl MusicListMapper { // List item MusicResponseItem::MusicResponsiveListItemRenderer(item) => { let mut columns = item.flex_columns.into_iter(); - let title = columns.next().map(|col| col.renderer.text.to_string()); + let c1 = columns.next(); let c2 = columns.next(); let c3 = columns.next(); + let title = c1.as_ref().map(|col| col.renderer.text.to_string()); + let first_tn = item .thumbnail .music_thumbnail_renderer @@ -433,27 +435,54 @@ impl MusicListMapper { .navigation_endpoint .and_then(|ne| ne.music_page()) .or_else(|| { - item.playlist_item_data - .map(|d| (MusicPageType::Track, d.video_id)) + c1.and_then(|c1| { + c1.renderer.text.0.into_iter().next().and_then(|t| match t { + crate::serializer::text::TextComponent::Video { + video_id, + is_video, + .. + } => Some((MusicPageType::Track { is_video }, video_id)), + crate::serializer::text::TextComponent::Browse { + page_type, + browse_id, + .. + } => Some((page_type.into(), browse_id)), + _ => None, + }) + }) + }) + .or_else(|| { + item.playlist_item_data.map(|d| { + ( + MusicPageType::Track { + is_video: self.album.is_none() + && !first_tn + .map(|tn| tn.height == tn.width) + .unwrap_or_default(), + }, + d.video_id, + ) + }) }) .or_else(|| { first_tn.and_then(|tn| { - util::video_id_from_thumbnail_url(&tn.url) - .map(|id| (MusicPageType::Track, id)) + util::video_id_from_thumbnail_url(&tn.url).map(|id| { + ( + MusicPageType::Track { + is_video: self.album.is_none() && tn.width != tn.height, + }, + id, + ) + }) }) }); match pt_id { // Track - Some((MusicPageType::Track, id)) => { + Some((MusicPageType::Track { is_video }, id)) => { let title = title.ok_or_else(|| format!("track {}: could not get title", id))?; - // Videos have rectangular thumbnails, YTM tracks have square covers - // Exception: there are no thumbnails on album items - let is_video = self.album.is_none() - && !first_tn.map(|tn| tn.height == tn.width).unwrap_or_default(); - let (artists_p, album_p, duration_p) = match item.flex_column_display_style { // Search result @@ -519,15 +548,14 @@ impl MusicListMapper { }), ), (_, false) => ( - album_p - .and_then(|p| { - p.0.into_iter().find_map(|c| AlbumId::try_from(c).ok()) - }) - .or_else(|| self.album.clone()), + album_p.and_then(|p| { + p.0.into_iter().find_map(|c| AlbumId::try_from(c).ok()) + }), None, ), (FlexColumnDisplayStyle::Default, true) => (None, None), }; + let album = album.or_else(|| self.album.clone()); let (mut artists, _) = map_artists(artists_p); @@ -640,7 +668,8 @@ impl MusicListMapper { // There may be broken YT channels from the artist search. They can be skipped. Ok(None) } - MusicPageType::Track => unreachable!(), + // Tracks were already handled above + MusicPageType::Track { .. } => unreachable!(), } } None => Err("could not determine item type".to_owned()), @@ -655,7 +684,7 @@ impl MusicListMapper { match item.navigation_endpoint.music_page() { Some((page_type, id)) => match page_type { - MusicPageType::Track => { + MusicPageType::Track { is_video } => { let artists = map_artists(subtitle_p1).0; self.items.push(MusicItem::Track(TrackItem { @@ -669,7 +698,7 @@ impl MusicListMapper { view_count: subtitle_p2.and_then(|c| { util::parse_large_numstr(c.first_str(), self.lang) }), - is_video: true, + is_video, track_nr: None, })); Ok(Some(MusicEntityType::Track)) diff --git a/src/client/response/url_endpoint.rs b/src/client/response/url_endpoint.rs index 09713e1..03d9043 100644 --- a/src/client/response/url_endpoint.rs +++ b/src/client/response/url_endpoint.rs @@ -35,6 +35,8 @@ pub(crate) struct WatchEndpoint { pub playlist_id: Option, #[serde(default)] pub start_time_seconds: u32, + #[serde(default)] + pub watch_endpoint_music_supported_configs: WatchEndpointConfigWrap, } #[derive(Debug)] @@ -118,6 +120,30 @@ pub(crate) struct WebCommandMetadata { pub web_page_type: PageType, } +#[derive(Default, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct WatchEndpointConfigWrap { + pub watch_endpoint_music_config: WatchEndpointConfig, +} + +#[serde_as] +#[derive(Default, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct WatchEndpointConfig { + #[serde(default)] + #[serde_as(deserialize_as = "DefaultOnError")] + pub music_video_type: MusicVideoType, +} + +#[derive(Default, Debug, Clone, Copy, Deserialize, PartialEq, Eq)] +pub(crate) enum MusicVideoType { + #[default] + #[serde(rename = "MUSIC_VIDEO_TYPE_OMV")] + Video, + #[serde(rename = "MUSIC_VIDEO_TYPE_ATV")] + Track, +} + #[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] pub(crate) enum PageType { #[serde( @@ -152,7 +178,7 @@ pub(crate) enum MusicPageType { Artist, Album, Playlist, - Track, + Track { is_video: bool }, None, } @@ -189,7 +215,16 @@ impl NavigationEndpoint { // Genre radios (e.g. "pop radio") will be skipped (MusicPageType::None, watch.video_id) } else { - (MusicPageType::Track, watch.video_id) + ( + MusicPageType::Track { + is_video: watch + .watch_endpoint_music_supported_configs + .watch_endpoint_music_config + .music_video_type + == MusicVideoType::Video, + }, + watch.video_id, + ) } }) }) diff --git a/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_description.snap b/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_description.snap index 8f2bb6c..19a4cd1 100644 --- a/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_description.snap +++ b/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_description.snap @@ -56,7 +56,7 @@ MusicAlbum( name: "25", )), view_count: None, - is_video: false, + is_video: true, track_nr: Some(1), ), TrackItem( @@ -76,7 +76,7 @@ MusicAlbum( name: "25", )), view_count: None, - is_video: false, + is_video: true, track_nr: Some(2), ), TrackItem( diff --git a/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_single.snap b/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_single.snap index fa429ce..3e86ad8 100644 --- a/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_single.snap +++ b/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_single.snap @@ -64,7 +64,7 @@ MusicAlbum( name: "Der Himmel reißt auf", )), view_count: None, - is_video: false, + is_video: true, track_nr: Some(1), ), ], diff --git a/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_various_artists.snap b/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_various_artists.snap index 83f0652..78bfbc7 100644 --- a/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_various_artists.snap +++ b/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_various_artists.snap @@ -51,7 +51,7 @@ MusicAlbum( name: "＜Queendom2＞ FINAL", )), view_count: None, - is_video: false, + is_video: true, track_nr: Some(1), ), TrackItem( diff --git a/src/model/richtext.rs b/src/model/richtext.rs index c4609be..1947e48 100644 --- a/src/model/richtext.rs +++ b/src/model/richtext.rs @@ -122,7 +122,7 @@ mod tests { text::TextComponent::Text { text: "🎧Listen and download aespa's debut single \"Black Mamba\": ".to_owned() }, text::TextComponent::Web { text: "https://smarturl.it/aespa_BlackMamba".to_owned(), url: "https://www.youtube.com/redirect?event=video_description&redir_token=QUFFLUhqbFY1QmpQamJPSms0Z1FnVTlQUS00ZFhBZnBJZ3xBQ3Jtc0tuRGJBanludGoyRnphb2dZWVd3cUNnS3dEd0FnNHFOZEY1NHBJaHFmLXpaWUJwX3ZucDZxVnpGeHNGX1FpMzFkZW9jQkI2Mi1wNGJ1UVFNN3h1MnN3R3JLMzdxU01nZ01POHBGcmxHU2puSUk1WHRzQQ&q=https%3A%2F%2Fsmarturl.it%2Faespa_BlackMamba&v=ZeerrnuLi5E".to_owned() }, text::TextComponent::Text { text: "\n🐍The Debut Stage ".to_owned() }, - text::TextComponent::Video { text: "https://youtu.be/Ky5RT5oGg0w".to_owned(), video_id: "Ky5RT5oGg0w".to_owned(), start_time: 0 }, + text::TextComponent::Video { text: "https://youtu.be/Ky5RT5oGg0w".to_owned(), video_id: "Ky5RT5oGg0w".to_owned(), start_time: 0, is_video: true }, text::TextComponent::Text { text: "\n\n🎟️ aespa Showcase SYNK in LA! Tickets now on sale: ".to_owned() }, text::TextComponent::Web { text: "https://www.ticketmaster.com/event/0A...".to_owned(), url: "https://www.youtube.com/redirect?event=video_description&redir_token=QUFFLUhqbFpUMEZiaXJWWkszaVZXaEM0emxWU1JQV3NoQXxBQ3Jtc0tuU2g4VWNPNE5UY3hoSWYtamFzX0h4bUVQLVJiRy1ubDZrTnh3MUpGdDNSaUo0ZlMyT3lUM28ycUVBdHJLMndGcDhla3BkOFpxSVFfOS1QdVJPVHBUTEV1LXpOV0J2QXdhV05lV210cEJtZUJMeHdaTQ&q=https%3A%2F%2Fwww.ticketmaster.com%2Fevent%2F0A005CCD9E871F6E&v=ZeerrnuLi5E".to_owned() }, text::TextComponent::Text { text: "\n\nSubscribe to aespa Official YouTube Channel!\n".to_owned() }, diff --git a/src/serializer/snapshots/rustypipe__serializer__text__tests__t_attributed_description.snap b/src/serializer/snapshots/rustypipe__serializer__text__tests__t_attributed_description.snap index 8ba3dfa..e937bcc 100644 --- a/src/serializer/snapshots/rustypipe__serializer__text__tests__t_attributed_description.snap +++ b/src/serializer/snapshots/rustypipe__serializer__text__tests__t_attributed_description.snap @@ -19,6 +19,7 @@ SAttributed { text: "aespa 에스파 'Black ...", video_id: "Ky5RT5oGg0w", start_time: 0, + is_video: true, }, Text { text: "\n\n🎟\u{fe0f} aespa Showcase SYNK in LA! Tickets now on sale: ", diff --git a/src/serializer/text.rs b/src/serializer/text.rs index 587daf0..abe8c80 100644 --- a/src/serializer/text.rs +++ b/src/serializer/text.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Deserializer}; use serde_with::{serde_as, DeserializeAs}; use crate::{ - client::response::url_endpoint::{NavigationEndpoint, PageType}, + client::response::url_endpoint::{MusicVideoType, NavigationEndpoint, PageType}, model::UrlTarget, util, }; @@ -94,6 +94,8 @@ pub(crate) enum TextComponent { text: String, video_id: String, start_time: u32, + /// True if the item is a video, false if it is a YTM track + is_video: bool, }, Browse { text: String, @@ -164,6 +166,11 @@ fn map_text_component(text: String, nav: NavigationEndpoint) -> TextComponent { text, video_id: w.video_id, start_time: w.start_time_seconds, + is_video: w + .watch_endpoint_music_supported_configs + .watch_endpoint_music_config + .music_video_type + == MusicVideoType::Video, }, None => match nav.browse_endpoint { Some(b) => TextComponent::Browse { @@ -365,6 +372,7 @@ impl From for crate::model::richtext::TextComponent { text, video_id, start_time, + .. } => Self::YouTube { text, target: UrlTarget::Video { @@ -581,6 +589,7 @@ mod tests { text: "DEEP", video_id: "wZIoIgz5mbs", start_time: 0, + is_video: true, }, } "###); diff --git a/tests/snapshots/youtube__music_album_ep.snap b/tests/snapshots/youtube__music_album_ep.snap index dd465da..e68573a 100644 --- a/tests/snapshots/youtube__music_album_ep.snap +++ b/tests/snapshots/youtube__music_album_ep.snap @@ -55,7 +55,7 @@ MusicAlbum( name: "Waldbrand", )), view_count: None, - is_video: false, + is_video: true, track_nr: Some(2), ), TrackItem( diff --git a/tests/snapshots/youtube__music_album_single.snap b/tests/snapshots/youtube__music_album_single.snap index befe8c2..61f7faa 100644 --- a/tests/snapshots/youtube__music_album_single.snap +++ b/tests/snapshots/youtube__music_album_single.snap @@ -43,7 +43,7 @@ MusicAlbum( name: "Der Himmel reißt auf", )), view_count: None, - is_video: false, + is_video: true, track_nr: Some(1), ), ], diff --git a/tests/snapshots/youtube__music_album_various_artists.snap b/tests/snapshots/youtube__music_album_various_artists.snap index 8da3db0..a3eef12 100644 --- a/tests/snapshots/youtube__music_album_various_artists.snap +++ b/tests/snapshots/youtube__music_album_various_artists.snap @@ -30,7 +30,7 @@ MusicAlbum( name: "＜Queendom2＞ FINAL", )), view_count: None, - is_video: false, + is_video: true, track_nr: Some(1), ), TrackItem( From a9aeb4dabee3a430ca16573d1f378a515029ed2a Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 30 Nov 2022 22:49:07 +0100 Subject: [PATCH 3/7] test: remove podcast episode search test --- tests/youtube.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/youtube.rs b/tests/youtube.rs index bd48b4a..4bf178f 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -1600,6 +1600,8 @@ async fn music_search_videos() { assert_next(res.items, rp.query(), 15, 2).await; } +/* +This podcast was removed from YouTube Music and I could not find another one #[tokio::test] async fn music_search_episode() { let rp = RustyPipe::builder().strict().build(); @@ -1624,6 +1626,7 @@ async fn music_search_episode() { ); assert!(!track.cover.is_empty(), "got no cover"); } +*/ #[rstest] #[case::single( From 2b91c76b85e6bb646c918cf265888285675e8bcc Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 30 Nov 2022 23:45:41 +0100 Subject: [PATCH 4/7] fix: replace MVs in albums with tracks from album playlist --- src/client/music_details.rs | 5 +- src/client/music_playlist.rs | 57 ++++++++++++++++--- tests/snapshots/youtube__music_album_ep.snap | 4 +- .../youtube__music_album_single.snap | 4 +- .../youtube__music_album_various_artists.snap | 4 +- 5 files changed, 59 insertions(+), 15 deletions(-) diff --git a/src/client/music_details.rs b/src/client/music_details.rs index 1a48e67..cd4717c 100644 --- a/src/client/music_details.rs +++ b/src/client/music_details.rs @@ -99,7 +99,10 @@ impl RustyPipeQuery { radio_id: S, ) -> Result, Error> { let radio_id = radio_id.as_ref(); - let context = self.get_context(ClientType::DesktopMusic, true, None).await; + let visitor_data = self.get_ytm_visitor_data().await?; + let context = self + .get_context(ClientType::DesktopMusic, true, Some(&visitor_data)) + .await; let request_body = QRadio { context, playlist_id: radio_id, diff --git a/src/client/music_playlist.rs b/src/client/music_playlist.rs index 6db4c7e..6fdd6ef 100644 --- a/src/client/music_playlist.rs +++ b/src/client/music_playlist.rs @@ -45,14 +45,55 @@ impl RustyPipeQuery { browse_id: album_id, }; - self.execute_request::( - ClientType::DesktopMusic, - "music_album", - album_id, - "browse", - &request_body, - ) - .await + let mut album = self + .execute_request::( + ClientType::DesktopMusic, + "music_album", + album_id, + "browse", + &request_body, + ) + .await?; + + // YouTube Music is replacing album tracks with their respective music videos. To get the original + // tracks, we have to fetch the album as a playlist and replace the offending track ids. + if let Some(playlist_id) = &album.playlist_id { + // Get a list of music videos in the album + let to_replace = album + .tracks + .iter() + .enumerate() + .filter_map(|(i, track)| { + if track.is_video { + Some((i, track.title.to_owned())) + } else { + None + } + }) + .collect::>(); + + if !to_replace.is_empty() { + let playlist = self.music_playlist(playlist_id).await?; + + for (i, title) in to_replace { + let found_track = playlist.tracks.items.iter().find_map(|track| { + if track.title == title && !track.is_video { + Some((track.id.to_owned(), track.duration)) + } else { + None + } + }); + if let Some((track_id, duration)) = found_track { + album.tracks[i].id = track_id; + if let Some(duration) = duration { + album.tracks[i].duration = Some(duration); + } + album.tracks[i].is_video = false; + } + } + } + } + Ok(album) } } diff --git a/tests/snapshots/youtube__music_album_ep.snap b/tests/snapshots/youtube__music_album_ep.snap index e68573a..26256f6 100644 --- a/tests/snapshots/youtube__music_album_ep.snap +++ b/tests/snapshots/youtube__music_album_ep.snap @@ -39,7 +39,7 @@ MusicAlbum( track_nr: Some(1), ), TrackItem( - id: "lhPOMUjV4rE", + id: "Jz-26iiDuYs", title: "Waldbrand", duration: Some(208), cover: [], @@ -55,7 +55,7 @@ MusicAlbum( name: "Waldbrand", )), view_count: None, - is_video: true, + is_video: false, track_nr: Some(2), ), TrackItem( diff --git a/tests/snapshots/youtube__music_album_single.snap b/tests/snapshots/youtube__music_album_single.snap index 61f7faa..aeba890 100644 --- a/tests/snapshots/youtube__music_album_single.snap +++ b/tests/snapshots/youtube__music_album_single.snap @@ -23,7 +23,7 @@ MusicAlbum( by_va: false, tracks: [ TrackItem( - id: "XX0epju-YvY", + id: "VU6lEv0PKAo", title: "Der Himmel reißt auf", duration: Some(183), cover: [], @@ -43,7 +43,7 @@ MusicAlbum( name: "Der Himmel reißt auf", )), view_count: None, - is_video: true, + is_video: false, track_nr: Some(1), ), ], diff --git a/tests/snapshots/youtube__music_album_various_artists.snap b/tests/snapshots/youtube__music_album_various_artists.snap index a3eef12..b04f5ea 100644 --- a/tests/snapshots/youtube__music_album_various_artists.snap +++ b/tests/snapshots/youtube__music_album_various_artists.snap @@ -14,7 +14,7 @@ MusicAlbum( by_va: true, tracks: [ TrackItem( - id: "8IqLxg0GqXc", + id: "Tzai7JXo45w", title: "Waka Boom (My Way) (feat. Lee Young Ji)", duration: Some(274), cover: [], @@ -30,7 +30,7 @@ MusicAlbum( name: "＜Queendom2＞ FINAL", )), view_count: None, - is_video: true, + is_video: false, track_nr: Some(1), ), TrackItem( From f236458f732514a967eb2b18ff2d49bd3dfee2f7 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 30 Nov 2022 23:50:57 +0100 Subject: [PATCH 5/7] tests: remove exact comment count assertion --- tests/youtube.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/youtube.rs b/tests/youtube.rs index 4bf178f..c700aab 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -792,12 +792,6 @@ async fn get_video_comments() { let n_comments = top_comments.count.unwrap(); assert_gte(n_comments, 700_000, "comments"); - // Comment count should be exact after fetching first page - assert!( - n_comments % 1000 != 0, - "estimated comment count: {}", - n_comments - ); let latest_comments = details .latest_comments From a741a61a30fa78a508004935b8fc14c757332a87 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 30 Nov 2022 23:57:39 +0100 Subject: [PATCH 6/7] refactor: use generic string params for client --- src/client/mod.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/client/mod.rs b/src/client/mod.rs index 3b82f4f..d54b31b 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -397,8 +397,8 @@ impl RustyPipeBuilder { /// /// **Default value**: `Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0` /// (Firefox ESR on Debian) - pub fn user_agent(mut self, user_agent: &str) -> Self { - self.user_agent = user_agent.to_owned(); + pub fn user_agent>(mut self, user_agent: S) -> Self { + self.user_agent = user_agent.into(); self } @@ -444,8 +444,8 @@ impl RustyPipeBuilder { } /// Set the default YouTube visitor data cookie - pub fn visitor_data(mut self, visitor_data: &str) -> Self { - self.default_opts.visitor_data = Some(visitor_data.to_owned()); + pub fn visitor_data>(mut self, visitor_data: S) -> Self { + self.default_opts.visitor_data = Some(visitor_data.into()); self } @@ -778,8 +778,8 @@ impl RustyPipeQuery { } /// Set the YouTube visitor data cookie - pub fn visitor_data(mut self, visitor_data: &str) -> Self { - self.opts.visitor_data = Some(visitor_data.to_owned()); + pub fn visitor_data>(mut self, visitor_data: S) -> Self { + self.opts.visitor_data = Some(visitor_data.into()); self } From e063c048217f5b68733ad38d006959d4975e46c7 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Thu, 1 Dec 2022 00:25:47 +0100 Subject: [PATCH 7/7] refactor!: move downloader to seperate crate --- Cargo.toml | 7 +-- cli/Cargo.toml | 13 +++-- cli/src/main.rs | 2 +- downloader/Cargo.toml | 31 ++++++++++++ src/download.rs => downloader/src/lib.rs | 60 +++++------------------- downloader/src/util.rs | 42 +++++++++++++++++ src/error.rs | 23 --------- src/lib.rs | 1 - 8 files changed, 96 insertions(+), 83 deletions(-) create mode 100644 downloader/Cargo.toml rename src/download.rs => downloader/src/lib.rs (91%) create mode 100644 downloader/src/util.rs diff --git a/Cargo.toml b/Cargo.toml index ea67285..1f2c729 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ keywords = ["youtube", "video", "music"] include = ["/src", "README.md", "LICENSE", "!snapshots"] [workspace] -members = [".", "codegen", "cli"] +members = [".", "codegen", "downloader", "cli"] [features] default = ["default-tls"] @@ -34,9 +34,8 @@ reqwest = { version = "0.11.11", default-features = false, features = [ "json", "gzip", "brotli", - "stream", ] } -tokio = { version = "1.20.0", features = ["macros", "time", "fs", "process"] } +tokio = { version = "1.20.0", features = ["macros", "time"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.82" serde_with = { version = "2.0.0", features = ["json"] } @@ -47,8 +46,6 @@ time = { version = "0.3.15", features = [ "serde-well-known", ] } futures = "0.3.21" -indicatif = "0.17.0" -filenamify = "0.1.0" ress = "0.11.4" phf = "0.11.1" base64 = "0.13.0" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 23b8ed6..53f895e 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -4,11 +4,16 @@ version = "0.1.0" edition = "2021" [dependencies] -rustypipe = {path = "../", default_features = false, features = ["rustls-tls-native-roots"]} -reqwest = {version = "0.11.11", default_features = false} -tokio = {version = "1.20.0", features = ["macros", "rt-multi-thread"]} +rustypipe = { path = "../", default_features = false, features = [ + "rustls-tls-native-roots", +] } +rustypipe-downloader = { path = "../downloader", default_features = false, features = [ + "rustls-tls-native-roots", +] } +reqwest = { version = "0.11.11", default_features = false } +tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] } indicatif = "0.17.0" futures = "0.3.21" anyhow = "1.0" clap = { version = "3.2.16", features = ["derive"] } -env_logger = "0.9.0" +env_logger = "0.10.0" diff --git a/cli/src/main.rs b/cli/src/main.rs index 0acac75..f6a3dc3 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -69,7 +69,7 @@ async fn download_single_video( } } - rustypipe::download::download_video( + rustypipe_downloader::download_video( &player_data, output_dir, output_fname, diff --git a/downloader/Cargo.toml b/downloader/Cargo.toml new file mode 100644 index 0000000..9f60764 --- /dev/null +++ b/downloader/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "rustypipe-downloader" +version = "0.1.0" +edition = "2021" + +[features] +# Reqwest TLS +default-tls = ["reqwest/default-tls", "rustypipe/default-tls"] +rustls-tls-webpki-roots = [ + "reqwest/rustls-tls-webpki-roots", + "rustypipe/rustls-tls-webpki-roots", +] +rustls-tls-native-roots = [ + "reqwest/rustls-tls-native-roots", + "rustypipe/rustls-tls-native-roots", +] + +[dependencies] +rustypipe = { path = "..", default-features = false } +once_cell = "1.12.0" +regex = "1.6.0" +thiserror = "1.0.36" +futures = "0.3.21" +indicatif = "0.17.0" +filenamify = "0.1.0" +log = "0.4.17" +reqwest = { version = "0.11.11", default-features = false, features = [ + "stream", +] } +rand = "0.8.5" +tokio = { version = "1.20.0", features = ["macros", "fs", "process"] } diff --git a/src/download.rs b/downloader/src/lib.rs similarity index 91% rename from src/download.rs rename to downloader/src/lib.rs index 238e9ab..34adc42 100644 --- a/src/download.rs +++ b/downloader/src/lib.rs @@ -1,26 +1,27 @@ -//! YouTube audio/video downloader +//! # YouTube audio/video downloader + +mod util; use std::{borrow::Cow, cmp::Ordering, ffi::OsString, ops::Range, path::PathBuf, time::Duration}; -use fancy_regex::Regex; use futures::stream::{self, StreamExt}; use indicatif::{ProgressBar, ProgressStyle}; use log::{debug, info}; use once_cell::sync::Lazy; use rand::Rng; +use regex::Regex; use reqwest::{header, Client}; +use rustypipe::{ + model::{AudioCodec, FileFormat, VideoCodec, VideoPlayer}, + param::StreamFilter, +}; use tokio::{ fs::{self, File}, io::AsyncWriteExt, process::Command, }; -use crate::{ - error::DownloadError, - model::{AudioCodec, FileFormat, VideoCodec, VideoPlayer}, - param::StreamFilter, - util, -}; +use util::DownloadError; type Result = core::result::Result; @@ -45,7 +46,7 @@ fn get_download_range(offset: u64, size: Option) -> Range { fn parse_cr_header(cr_header: &str) -> Result<(u64, u64)> { static PATTERN: Lazy = Lazy::new(|| Regex::new(r#"bytes (\d+)-(\d+)/(\d+)"#).unwrap()); - let captures = PATTERN.captures(cr_header).ok().flatten().ok_or_else(|| { + let captures = PATTERN.captures(cr_header).ok_or_else(|| { DownloadError::Progressive( format!( "Content-Range header '{}' does not match pattern", @@ -317,11 +318,9 @@ pub async fn download_video( Some(_) => "mp4", None => match audio { Some(audio) => match audio.codec { - AudioCodec::Unknown => { - return Err(DownloadError::Input("unknown audio codec".into())) - } AudioCodec::Mp4a => "m4a", AudioCodec::Opus => "opus", + _ => return Err(DownloadError::Input("unknown audio codec".into())), }, None => unreachable!(), }, @@ -473,40 +472,3 @@ async fn convert_streams>( } Ok(()) } - -/* -#[cfg(test)] -mod tests { - use crate::client::RustyTube; - - use super::*; - use indicatif::{ProgressDrawTarget, ProgressStyle}; - use reqwest::ClientBuilder; - - // #[test_log::test(tokio::test)] - #[tokio::test] - async fn t_download_video() { - let http = ClientBuilder::new() - .user_agent( - "Mozilla/5.0 (Windows NT 10.0; Win64; rv:107.0) Gecko/20100101 Firefox/107.0", - ) - .gzip(true) - .brotli(true) - .build() - .expect("unable to build the HTTP client"); - - // Indicatif setup - let pb = ProgressBar::new(0); - - let rt = RustyTube::new(); - let player_data = rt - .get_player("AbZH7XWDW_k", crate::client::ClientType::Desktop) - .await - .unwrap(); - - // download_video(&player_data, "tmp", "INVU", Some(1080), "ffmpeg", http, pb) - // .await - // .unwrap(); - } -} -*/ diff --git a/downloader/src/util.rs b/downloader/src/util.rs new file mode 100644 index 0000000..b6b6719 --- /dev/null +++ b/downloader/src/util.rs @@ -0,0 +1,42 @@ +use std::{borrow::Cow, collections::BTreeMap}; + +use reqwest::Url; + +/// Error from the video downloader +#[derive(thiserror::Error, Debug)] +#[non_exhaustive] +pub enum DownloadError { + /// Error from the HTTP client + #[error("http error: {0}")] + Http(#[from] reqwest::Error), + /// File IO error + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("FFmpeg error: {0}")] + Ffmpeg(Cow<'static, str>), + #[error("Progressive download error: {0}")] + Progressive(Cow<'static, str>), + #[error("input error: {0}")] + Input(Cow<'static, str>), + #[error("error: {0}")] + Other(Cow<'static, str>), +} + +/// Split an URL into its base string and parameter map +/// +/// Example: +/// +/// `example.com/api?k1=v1&k2=v2 => example.com/api; {k1: v1, k2: v2}` +pub fn url_to_params(url: &str) -> Result<(Url, BTreeMap), DownloadError> { + let mut parsed_url = Url::parse(url).map_err(|e| { + DownloadError::Other(format!("could not parse url `{}` err: {}", url, e).into()) + })?; + let url_params: BTreeMap = parsed_url + .query_pairs() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + parsed_url.set_query(None); + + Ok((parsed_url, url_params)) +} diff --git a/src/error.rs b/src/error.rs index ced5358..1ce1641 100644 --- a/src/error.rs +++ b/src/error.rs @@ -10,9 +10,6 @@ pub enum Error { /// Error from the deobfuscater #[error("deobfuscator error: {0}")] Deobfuscation(#[from] DeobfError), - /// Error from the video downloader - #[error("download error: {0}")] - Download(#[from] DownloadError), /// File IO error #[error(transparent)] Io(#[from] std::io::Error), @@ -45,26 +42,6 @@ pub enum DeobfError { Other(&'static str), } -/// Error from the video downloader -#[derive(thiserror::Error, Debug)] -#[non_exhaustive] -pub enum DownloadError { - /// Error from the HTTP client - #[error("http error: {0}")] - Http(#[from] reqwest::Error), - /// File IO error - #[error(transparent)] - Io(#[from] std::io::Error), - #[error("FFmpeg error: {0}")] - Ffmpeg(Cow<'static, str>), - #[error("Progressive download error: {0}")] - Progressive(Cow<'static, str>), - #[error("input error: {0}")] - Input(Cow<'static, str>), - #[error("error: {0}")] - Other(Cow<'static, str>), -} - /// Error extracting content from YouTube #[derive(thiserror::Error, Debug)] #[non_exhaustive] diff --git a/src/lib.rs b/src/lib.rs index f06ca4d..f406da5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,7 +14,6 @@ mod util; pub mod cache; pub mod client; -pub mod download; pub mod error; pub mod model; pub mod param;