diff --git a/cli/src/main.rs b/cli/src/main.rs index c96cd98..4cd1142 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -173,6 +173,7 @@ enum MusicSearchCategory { Videos, Artists, Albums, + Playlists, PlaylistsYtm, PlaylistsCommunity, } @@ -675,10 +676,15 @@ async fn main() { res.items.extend_limit(rp.query(), limit).await.unwrap(); print_data(&res, format, pretty); } + Some(MusicSearchCategory::Playlists) => { + let mut res = rp.query().music_search_playlists(&query).await.unwrap(); + res.items.extend_limit(rp.query(), limit).await.unwrap(); + print_data(&res, format, pretty); + } Some(MusicSearchCategory::PlaylistsYtm) => { let mut res = rp .query() - .music_search_playlists(&query, false) + .music_search_playlists_filter(&query, false) .await .unwrap(); res.items.extend_limit(rp.query(), limit).await.unwrap(); @@ -687,7 +693,7 @@ async fn main() { Some(MusicSearchCategory::PlaylistsCommunity) => { let mut res = rp .query() - .music_search_playlists(&query, true) + .music_search_playlists_filter(&query, true) .await .unwrap(); res.items.extend_limit(rp.query(), limit).await.unwrap(); diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index e3bfff8..6f48494 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -11,9 +11,7 @@ tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] } futures = "0.3.21" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.82" -serde_with = { version = "3.0.0", default-features = false, features = [ - "macros", -] } +serde_with = { version = "3.0.0", default-features = false, features = ["macros"] } anyhow = "1.0" log = "0.4.17" env_logger = "0.10.0" @@ -26,4 +24,3 @@ num_enum = "0.6.1" path_macro = "1.0.0" intl_pluralrules = "7.0.2" unic-langid = "0.9.1" -ordered_hash_map = { version = "0.2.0", features = ["serde"] } diff --git a/codegen/src/abtest.rs b/codegen/src/abtest.rs index cc9491a..9acf827 100644 --- a/codegen/src/abtest.rs +++ b/codegen/src/abtest.rs @@ -24,7 +24,6 @@ pub enum ABTest { TrendsPageHeaderRenderer = 5, DiscographyPage = 6, ShortDateFormat = 7, - PlaylistsForShorts = 9, } const TESTS_TO_RUN: [ABTest; 3] = [ @@ -95,7 +94,6 @@ pub async fn run_test( ABTest::TrendsPageHeaderRenderer => trends_page_header_renderer(&query).await, ABTest::DiscographyPage => discography_page(&query).await, ABTest::ShortDateFormat => short_date_format(&query).await, - ABTest::PlaylistsForShorts => playlists_for_shorts(&query).await, } .unwrap(); pb.inc(1); @@ -245,13 +243,3 @@ pub async fn short_date_format(rp: &RustyPipeQuery) -> Result { .unwrap_or_default() })) } - -pub async fn playlists_for_shorts(rp: &RustyPipeQuery) -> Result { - let playlist = rp.playlist("UUSHh8gHdtzO2tXd593_bjErWg").await?; - let v1 = playlist - .videos - .items - .first() - .ok_or_else(|| anyhow::anyhow!("no videos"))?; - Ok(v1.publish_date_txt.is_none()) -} diff --git a/codegen/src/collect_playlist_dates.rs b/codegen/src/collect_playlist_dates.rs index 21cce76..4a87d36 100644 --- a/codegen/src/collect_playlist_dates.rs +++ b/codegen/src/collect_playlist_dates.rs @@ -6,7 +6,6 @@ use std::{ }; use futures::{stream, StreamExt}; -use ordered_hash_map::OrderedHashMap; use path_macro::path; use rustypipe::{ client::RustyPipe, @@ -171,7 +170,7 @@ pub fn write_samples_to_dict() { dict_entry.months = BTreeMap::new(); if collect_nd_tokens { - dict_entry.timeago_nd_tokens = OrderedHashMap::new(); + dict_entry.timeago_nd_tokens = BTreeMap::new(); } for datestr_table in &datestr_tables { diff --git a/codegen/src/download_testfiles.rs b/codegen/src/download_testfiles.rs index 147db3e..0d1357f 100644 --- a/codegen/src/download_testfiles.rs +++ b/codegen/src/download_testfiles.rs @@ -619,7 +619,7 @@ async fn music_search_playlists() { let rp = rp_testfile(&json_path); rp.query() - .music_search_playlists("pop", community) + .music_search_playlists_filter("pop", community) .await .unwrap(); } diff --git a/codegen/src/model.rs b/codegen/src/model.rs index f7599cf..a5f9e11 100644 --- a/codegen/src/model.rs +++ b/codegen/src/model.rs @@ -1,6 +1,5 @@ use std::collections::BTreeMap; -use ordered_hash_map::OrderedHashMap; use rustypipe::{client::YTContext, model::AlbumType, param::Language}; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DefaultOnError, VecSkipError}; @@ -19,7 +18,7 @@ pub struct DictEntry { /// /// Identifiers: `Y`(ear), `M`(month), `W`(eek), `D`(ay), /// `h`(our), `m`(inute), `s`(econd) - pub timeago_tokens: OrderedHashMap, + pub timeago_tokens: BTreeMap, /// Order in which to parse numeric date components. Formatted as /// a string of date identifiers (Y, M, D). /// @@ -35,7 +34,7 @@ pub struct DictEntry { /// Tokens for parsing date strings with no digits (e.g. Today, Tomorrow) /// /// Format: Parsed token -> \[Quantity\] Identifier - pub timeago_nd_tokens: OrderedHashMap, + pub timeago_nd_tokens: BTreeMap, /// Are commas (instead of points) used as decimal separators? pub comma_decimal: bool, /// Tokens for parsing decimal prefixes (K, M, B, ...) @@ -50,10 +49,6 @@ pub struct DictEntry { /// /// Format: Parsed text -> Album type pub album_types: BTreeMap, - /// Names of item types (Song, Video, Artist, Playlist) - /// - /// Format: Parsed text -> Item type - pub item_types: BTreeMap, } /// Parsed TimeAgo string, contains amount and time unit. @@ -104,15 +99,6 @@ impl TimeUnit { } } -#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum ExtItemType { - Track, - Video, - Episode, - Playlist, - Artist, -} - #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct QBrowse<'a> { diff --git a/notes/AB_Tests.md b/notes/AB_Tests.md index f4294cd..b9ffbf0 100644 --- a/notes/AB_Tests.md +++ b/notes/AB_Tests.md @@ -360,8 +360,8 @@ YouTube changed the header renderer type on the trending page to a `pageHeaderRe - **Impact:** 🟡 Medium - **Endpoint:** browse (music artist) -YouTube merged the 2 sections for singles and albums on artist pages together. Now there -is only a _Top Releases_ section. +YouTube merged the 2 sections for singles and albums on artist pages together. Now +there is only a *Top Releases* section. YouTube also changed the way the full discography page is fetched, surprisingly making it easier for alternative clients. The discography page now has its own content ID in @@ -382,26 +382,5 @@ visitor data cookie to be set, as it was the case with the old system. - **Encountered on:** 28.05.2023 - **Impact:** 🟡 Medium -YouTube changed their date format from the long format (_21 hours ago_, _3 days ago_) to -a short format (_21h ago_, _3d ago_). - -## [9] Playlists for Shorts - -- **Encountered on:** 26.06.2023 -- **Impact:** 🟡 Medium -- **Endpoint:** browse (playlist) - -![A/B test 9 screenshot](./_img/ab_9.png) - -Original issue: https://github.com/TeamNewPipe/NewPipeExtractor/issues/10774 - -YouTube added a filter system for playlists, allowing users to only see shorts/full -videos. - -When shorts filter is enabled or when there are only shorts in a playlist, YouTube -return shorts UI elements instead of standard video ones, the ones that are also used -for shorts shelves in searches and suggestions and shorts in the corresponding channel -tab. - -Since the reel items dont include upload date information you can circumvent this new UI -by using the mobile client. But that may change in the future. +YouTube changed their date format from the long format (*21 hours ago*, *3 days ago*) to +a short format (*21h ago*, *3d ago*). diff --git a/notes/_img/ab_9.png b/notes/_img/ab_9.png deleted file mode 100644 index adca439..0000000 Binary files a/notes/_img/ab_9.png and /dev/null differ diff --git a/src/client/music_search.rs b/src/client/music_search.rs index f844cf9..3e2389d 100644 --- a/src/client/music_search.rs +++ b/src/client/music_search.rs @@ -40,6 +40,8 @@ enum Params { Albums, #[serde(rename = "EgWKAQIgAWoMEAMQBBAJEA4QChAF")] Artists, + #[serde(rename = "EgWKAQIoAWoMEAMQBBAJEA4QChAF")] + Playlists, #[serde(rename = "EgeKAQQoADgBagwQAxAEEAkQDhAKEAU%3D")] YtmPlaylists, #[serde(rename = "EgeKAQQoAEABagwQAxAEEAkQDhAKEAU%3D")] @@ -150,25 +152,44 @@ impl RustyPipeQuery { ) .await } + /// Search YouTube Music playlists - /// - /// Playlists are filtered whether they are created by users - /// (`community=true`) or by YouTube Music (`community=false`) pub async fn music_search_playlists>( &self, query: S, + ) -> Result, Error> { + self._music_search_playlists(query, Params::Playlists).await + } + + /// Search YouTube Music playlists that were created by users + /// (`community=true`) or by YouTube Music (`community=false`) + pub async fn music_search_playlists_filter>( + &self, + query: S, community: bool, + ) -> Result, Error> { + self._music_search_playlists( + query, + if community { + Params::CommunityPlaylists + } else { + Params::YtmPlaylists + }, + ) + .await + } + + async fn _music_search_playlists>( + &self, + query: S, + params: Params, ) -> Result, Error> { let query = query.as_ref(); let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QSearch { context, query, - params: Some(if community { - Params::CommunityPlaylists - } else { - Params::YtmPlaylists - }), + params: Some(params), }; self.execute_request::( diff --git a/src/client/response/playlist.rs b/src/client/response/playlist.rs index fbc76b4..e0fe342 100644 --- a/src/client/response/playlist.rs +++ b/src/client/response/playlist.rs @@ -29,7 +29,6 @@ pub(crate) struct ItemSection { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct PlaylistVideoListRenderer { - #[serde(alias = "richGridRenderer")] pub playlist_video_list_renderer: YouTubeListRenderer, } diff --git a/src/util/mod.rs b/src/util/mod.rs index 1a261d7..f148c6f 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -26,7 +26,7 @@ pub static VIDEO_ID_REGEX: Lazy = Lazy::new(|| Regex::new(r"^[A-Za-z0-9_- pub static CHANNEL_ID_REGEX: Lazy = Lazy::new(|| Regex::new(r"^UC[A-Za-z0-9_-]{22}$").unwrap()); pub static PLAYLIST_ID_REGEX: Lazy = - Lazy::new(|| Regex::new(r"^(?:PL|RD|OLAK|UU)[A-Za-z0-9_-]{5,50}$").unwrap()); + Lazy::new(|| Regex::new(r"^(?:PL|RD|OLAK|UU)[A-Za-z0-9_-]{16,50}$").unwrap()); pub static ALBUM_ID_REGEX: Lazy = Lazy::new(|| Regex::new(r"^MPREb_[A-Za-z0-9_-]{11}$").unwrap()); pub static VANITY_PATH_REGEX: Lazy = Lazy::new(|| { diff --git a/tests/youtube.rs b/tests/youtube.rs index 121870b..c1a86dd 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -1808,12 +1808,18 @@ fn music_search_artists_cont(rp: RustyPipe) { } #[rstest] -fn music_search_playlists(rp: RustyPipe, unlocalized: bool) { - let res = tokio_test::block_on( - rp.query() - .music_search_playlists("today's rock hits", false), - ) - .unwrap(); +#[case::ytm(false)] +#[case::default(true)] +fn music_search_playlists(#[case] with_community: bool, rp: RustyPipe, unlocalized: bool) { + let res = if with_community { + tokio_test::block_on(rp.query().music_search_playlists("today's rock hits")).unwrap() + } else { + tokio_test::block_on( + rp.query() + .music_search_playlists_filter("today's rock hits", false), + ) + .unwrap() + }; assert_eq!(res.corrected_query, None); let playlist = res @@ -1831,17 +1837,24 @@ fn music_search_playlists(rp: RustyPipe, unlocalized: bool) { assert_eq!(playlist.channel, None); assert!(playlist.from_ytm); - assert!( - res.items.items.iter().all(|p| p.from_ytm), - "community items found" - ) + if with_community { + assert!( + res.items.items.iter().any(|p| !p.from_ytm), + "no community items found" + ) + } else { + assert!( + res.items.items.iter().all(|p| p.from_ytm), + "community items found" + ) + } } #[rstest] fn music_search_playlists_community(rp: RustyPipe) { let res = tokio_test::block_on( rp.query() - .music_search_playlists("Best Pop Music Videos - Top Pop Hits Playlist", true), + .music_search_playlists_filter("Best Pop Music Videos - Top Pop Hits Playlist", true), ) .unwrap(); @@ -2228,10 +2241,14 @@ fn music_genre(#[case] id: &str, #[case] name: &str, rp: RustyPipe, unlocalized: assert!(!playlist.thumbnail.is_empty(), "got no cover"); if !playlist.from_ytm { - if let Some(channel) = playlist.channel.as_ref() { - validate::channel_id(&channel.id).unwrap(); - assert!(!channel.name.is_empty()); - } + assert!( + playlist.channel.is_some(), + "pl: {}, got no channel", + playlist.id + ); + let channel = playlist.channel.as_ref().unwrap(); + validate::channel_id(&channel.id).unwrap(); + assert!(!channel.name.is_empty()); } else { assert!(playlist.channel.is_none()); }