diff --git a/cli/src/main.rs b/cli/src/main.rs index 4cd1142..c96cd98 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -173,7 +173,6 @@ enum MusicSearchCategory { Videos, Artists, Albums, - Playlists, PlaylistsYtm, PlaylistsCommunity, } @@ -676,15 +675,10 @@ async fn main() { res.items.extend_limit(rp.query(), limit).await.unwrap(); print_data(&res, format, pretty); } - Some(MusicSearchCategory::Playlists) => { - let mut res = rp.query().music_search_playlists(&query).await.unwrap(); - res.items.extend_limit(rp.query(), limit).await.unwrap(); - print_data(&res, format, pretty); - } Some(MusicSearchCategory::PlaylistsYtm) => { let mut res = rp .query() - .music_search_playlists_filter(&query, false) + .music_search_playlists(&query, false) .await .unwrap(); res.items.extend_limit(rp.query(), limit).await.unwrap(); @@ -693,7 +687,7 @@ async fn main() { Some(MusicSearchCategory::PlaylistsCommunity) => { let mut res = rp .query() - .music_search_playlists_filter(&query, true) + .music_search_playlists(&query, true) .await .unwrap(); res.items.extend_limit(rp.query(), limit).await.unwrap(); diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 6f48494..e3bfff8 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -11,7 +11,9 @@ tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] } futures = "0.3.21" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.82" -serde_with = { version = "3.0.0", default-features = false, features = ["macros"] } +serde_with = { version = "3.0.0", default-features = false, features = [ + "macros", +] } anyhow = "1.0" log = "0.4.17" env_logger = "0.10.0" @@ -24,3 +26,4 @@ num_enum = "0.6.1" path_macro = "1.0.0" intl_pluralrules = "7.0.2" unic-langid = "0.9.1" +ordered_hash_map = { version = "0.2.0", features = ["serde"] } diff --git a/codegen/src/abtest.rs b/codegen/src/abtest.rs index 9acf827..cc9491a 100644 --- a/codegen/src/abtest.rs +++ b/codegen/src/abtest.rs @@ -24,6 +24,7 @@ pub enum ABTest { TrendsPageHeaderRenderer = 5, DiscographyPage = 6, ShortDateFormat = 7, + PlaylistsForShorts = 9, } const TESTS_TO_RUN: [ABTest; 3] = [ @@ -94,6 +95,7 @@ pub async fn run_test( ABTest::TrendsPageHeaderRenderer => trends_page_header_renderer(&query).await, ABTest::DiscographyPage => discography_page(&query).await, ABTest::ShortDateFormat => short_date_format(&query).await, + ABTest::PlaylistsForShorts => playlists_for_shorts(&query).await, } .unwrap(); pb.inc(1); @@ -243,3 +245,13 @@ pub async fn short_date_format(rp: &RustyPipeQuery) -> Result { .unwrap_or_default() })) } + +pub async fn playlists_for_shorts(rp: &RustyPipeQuery) -> Result { + let playlist = rp.playlist("UUSHh8gHdtzO2tXd593_bjErWg").await?; + let v1 = playlist + .videos + .items + .first() + .ok_or_else(|| anyhow::anyhow!("no videos"))?; + Ok(v1.publish_date_txt.is_none()) +} diff --git a/codegen/src/collect_playlist_dates.rs b/codegen/src/collect_playlist_dates.rs index 4a87d36..21cce76 100644 --- a/codegen/src/collect_playlist_dates.rs +++ b/codegen/src/collect_playlist_dates.rs @@ -6,6 +6,7 @@ use std::{ }; use futures::{stream, StreamExt}; +use ordered_hash_map::OrderedHashMap; use path_macro::path; use rustypipe::{ client::RustyPipe, @@ -170,7 +171,7 @@ pub fn write_samples_to_dict() { dict_entry.months = BTreeMap::new(); if collect_nd_tokens { - dict_entry.timeago_nd_tokens = BTreeMap::new(); + dict_entry.timeago_nd_tokens = OrderedHashMap::new(); } for datestr_table in &datestr_tables { diff --git a/codegen/src/download_testfiles.rs b/codegen/src/download_testfiles.rs index 0d1357f..147db3e 100644 --- a/codegen/src/download_testfiles.rs +++ b/codegen/src/download_testfiles.rs @@ -619,7 +619,7 @@ async fn music_search_playlists() { let rp = rp_testfile(&json_path); rp.query() - .music_search_playlists_filter("pop", community) + .music_search_playlists("pop", community) .await .unwrap(); } diff --git a/codegen/src/model.rs b/codegen/src/model.rs index a5f9e11..f7599cf 100644 --- a/codegen/src/model.rs +++ b/codegen/src/model.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; +use ordered_hash_map::OrderedHashMap; use rustypipe::{client::YTContext, model::AlbumType, param::Language}; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DefaultOnError, VecSkipError}; @@ -18,7 +19,7 @@ pub struct DictEntry { /// /// Identifiers: `Y`(ear), `M`(month), `W`(eek), `D`(ay), /// `h`(our), `m`(inute), `s`(econd) - pub timeago_tokens: BTreeMap, + pub timeago_tokens: OrderedHashMap, /// Order in which to parse numeric date components. Formatted as /// a string of date identifiers (Y, M, D). /// @@ -34,7 +35,7 @@ pub struct DictEntry { /// Tokens for parsing date strings with no digits (e.g. Today, Tomorrow) /// /// Format: Parsed token -> \[Quantity\] Identifier - pub timeago_nd_tokens: BTreeMap, + pub timeago_nd_tokens: OrderedHashMap, /// Are commas (instead of points) used as decimal separators? pub comma_decimal: bool, /// Tokens for parsing decimal prefixes (K, M, B, ...) @@ -49,6 +50,10 @@ pub struct DictEntry { /// /// Format: Parsed text -> Album type pub album_types: BTreeMap, + /// Names of item types (Song, Video, Artist, Playlist) + /// + /// Format: Parsed text -> Item type + pub item_types: BTreeMap, } /// Parsed TimeAgo string, contains amount and time unit. @@ -99,6 +104,15 @@ impl TimeUnit { } } +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum ExtItemType { + Track, + Video, + Episode, + Playlist, + Artist, +} + #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct QBrowse<'a> { diff --git a/notes/AB_Tests.md b/notes/AB_Tests.md index b9ffbf0..f4294cd 100644 --- a/notes/AB_Tests.md +++ b/notes/AB_Tests.md @@ -360,8 +360,8 @@ YouTube changed the header renderer type on the trending page to a `pageHeaderRe - **Impact:** 🟡 Medium - **Endpoint:** browse (music artist) -YouTube merged the 2 sections for singles and albums on artist pages together. Now -there is only a *Top Releases* section. +YouTube merged the 2 sections for singles and albums on artist pages together. Now there +is only a _Top Releases_ section. YouTube also changed the way the full discography page is fetched, surprisingly making it easier for alternative clients. The discography page now has its own content ID in @@ -382,5 +382,26 @@ visitor data cookie to be set, as it was the case with the old system. - **Encountered on:** 28.05.2023 - **Impact:** 🟡 Medium -YouTube changed their date format from the long format (*21 hours ago*, *3 days ago*) to -a short format (*21h ago*, *3d ago*). +YouTube changed their date format from the long format (_21 hours ago_, _3 days ago_) to +a short format (_21h ago_, _3d ago_). + +## [9] Playlists for Shorts + +- **Encountered on:** 26.06.2023 +- **Impact:** 🟡 Medium +- **Endpoint:** browse (playlist) + +![A/B test 9 screenshot](./_img/ab_9.png) + +Original issue: https://github.com/TeamNewPipe/NewPipeExtractor/issues/10774 + +YouTube added a filter system for playlists, allowing users to only see shorts/full +videos. + +When shorts filter is enabled or when there are only shorts in a playlist, YouTube +return shorts UI elements instead of standard video ones, the ones that are also used +for shorts shelves in searches and suggestions and shorts in the corresponding channel +tab. + +Since the reel items dont include upload date information you can circumvent this new UI +by using the mobile client. But that may change in the future. diff --git a/notes/_img/ab_9.png b/notes/_img/ab_9.png new file mode 100644 index 0000000..adca439 Binary files /dev/null and b/notes/_img/ab_9.png differ diff --git a/src/client/music_search.rs b/src/client/music_search.rs index 3e2389d..f844cf9 100644 --- a/src/client/music_search.rs +++ b/src/client/music_search.rs @@ -40,8 +40,6 @@ enum Params { Albums, #[serde(rename = "EgWKAQIgAWoMEAMQBBAJEA4QChAF")] Artists, - #[serde(rename = "EgWKAQIoAWoMEAMQBBAJEA4QChAF")] - Playlists, #[serde(rename = "EgeKAQQoADgBagwQAxAEEAkQDhAKEAU%3D")] YtmPlaylists, #[serde(rename = "EgeKAQQoAEABagwQAxAEEAkQDhAKEAU%3D")] @@ -152,44 +150,25 @@ impl RustyPipeQuery { ) .await } - /// Search YouTube Music playlists + /// + /// Playlists are filtered whether they are created by users + /// (`community=true`) or by YouTube Music (`community=false`) pub async fn music_search_playlists>( &self, query: S, - ) -> Result, Error> { - self._music_search_playlists(query, Params::Playlists).await - } - - /// Search YouTube Music playlists that were created by users - /// (`community=true`) or by YouTube Music (`community=false`) - pub async fn music_search_playlists_filter>( - &self, - query: S, community: bool, - ) -> Result, Error> { - self._music_search_playlists( - query, - if community { - Params::CommunityPlaylists - } else { - Params::YtmPlaylists - }, - ) - .await - } - - async fn _music_search_playlists>( - &self, - query: S, - params: Params, ) -> Result, Error> { let query = query.as_ref(); let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QSearch { context, query, - params: Some(params), + params: Some(if community { + Params::CommunityPlaylists + } else { + Params::YtmPlaylists + }), }; self.execute_request::( diff --git a/src/client/response/playlist.rs b/src/client/response/playlist.rs index e0fe342..fbc76b4 100644 --- a/src/client/response/playlist.rs +++ b/src/client/response/playlist.rs @@ -29,6 +29,7 @@ pub(crate) struct ItemSection { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct PlaylistVideoListRenderer { + #[serde(alias = "richGridRenderer")] pub playlist_video_list_renderer: YouTubeListRenderer, } diff --git a/src/util/mod.rs b/src/util/mod.rs index f148c6f..1a261d7 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -26,7 +26,7 @@ pub static VIDEO_ID_REGEX: Lazy = Lazy::new(|| Regex::new(r"^[A-Za-z0-9_- pub static CHANNEL_ID_REGEX: Lazy = Lazy::new(|| Regex::new(r"^UC[A-Za-z0-9_-]{22}$").unwrap()); pub static PLAYLIST_ID_REGEX: Lazy = - Lazy::new(|| Regex::new(r"^(?:PL|RD|OLAK|UU)[A-Za-z0-9_-]{16,50}$").unwrap()); + Lazy::new(|| Regex::new(r"^(?:PL|RD|OLAK|UU)[A-Za-z0-9_-]{5,50}$").unwrap()); pub static ALBUM_ID_REGEX: Lazy = Lazy::new(|| Regex::new(r"^MPREb_[A-Za-z0-9_-]{11}$").unwrap()); pub static VANITY_PATH_REGEX: Lazy = Lazy::new(|| { diff --git a/tests/youtube.rs b/tests/youtube.rs index c1a86dd..121870b 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -1808,18 +1808,12 @@ fn music_search_artists_cont(rp: RustyPipe) { } #[rstest] -#[case::ytm(false)] -#[case::default(true)] -fn music_search_playlists(#[case] with_community: bool, rp: RustyPipe, unlocalized: bool) { - let res = if with_community { - tokio_test::block_on(rp.query().music_search_playlists("today's rock hits")).unwrap() - } else { - tokio_test::block_on( - rp.query() - .music_search_playlists_filter("today's rock hits", false), - ) - .unwrap() - }; +fn music_search_playlists(rp: RustyPipe, unlocalized: bool) { + let res = tokio_test::block_on( + rp.query() + .music_search_playlists("today's rock hits", false), + ) + .unwrap(); assert_eq!(res.corrected_query, None); let playlist = res @@ -1837,24 +1831,17 @@ fn music_search_playlists(#[case] with_community: bool, rp: RustyPipe, unlocaliz assert_eq!(playlist.channel, None); assert!(playlist.from_ytm); - if with_community { - assert!( - res.items.items.iter().any(|p| !p.from_ytm), - "no community items found" - ) - } else { - assert!( - res.items.items.iter().all(|p| p.from_ytm), - "community items found" - ) - } + assert!( + res.items.items.iter().all(|p| p.from_ytm), + "community items found" + ) } #[rstest] fn music_search_playlists_community(rp: RustyPipe) { let res = tokio_test::block_on( rp.query() - .music_search_playlists_filter("Best Pop Music Videos - Top Pop Hits Playlist", true), + .music_search_playlists("Best Pop Music Videos - Top Pop Hits Playlist", true), ) .unwrap(); @@ -2241,14 +2228,10 @@ fn music_genre(#[case] id: &str, #[case] name: &str, rp: RustyPipe, unlocalized: assert!(!playlist.thumbnail.is_empty(), "got no cover"); if !playlist.from_ytm { - assert!( - playlist.channel.is_some(), - "pl: {}, got no channel", - playlist.id - ); - let channel = playlist.channel.as_ref().unwrap(); - validate::channel_id(&channel.id).unwrap(); - assert!(!channel.name.is_empty()); + if let Some(channel) = playlist.channel.as_ref() { + validate::channel_id(&channel.id).unwrap(); + assert!(!channel.name.is_empty()); + } } else { assert!(playlist.channel.is_none()); }