fix: playlist id regex

cli/src/main.rs

@@ -173,7 +173,6 @@ enum MusicSearchCategory {
     Videos,
     Artists,
     Albums,
-    Playlists,
     PlaylistsYtm,
     PlaylistsCommunity,
 }
@@ -676,15 +675,10 @@ async fn main() {
                 res.items.extend_limit(rp.query(), limit).await.unwrap();
                 print_data(&res, format, pretty);
             }
-            Some(MusicSearchCategory::Playlists) => {
-                let mut res = rp.query().music_search_playlists(&query).await.unwrap();
-                res.items.extend_limit(rp.query(), limit).await.unwrap();
-                print_data(&res, format, pretty);
-            }
             Some(MusicSearchCategory::PlaylistsYtm) => {
                 let mut res = rp
                     .query()
-                    .music_search_playlists_filter(&query, false)
+                    .music_search_playlists(&query, false)
                     .await
                     .unwrap();
                 res.items.extend_limit(rp.query(), limit).await.unwrap();
@@ -693,7 +687,7 @@ async fn main() {
             Some(MusicSearchCategory::PlaylistsCommunity) => {
                 let mut res = rp
                     .query()
-                    .music_search_playlists_filter(&query, true)
+                    .music_search_playlists(&query, true)
                     .await
                     .unwrap();
                 res.items.extend_limit(rp.query(), limit).await.unwrap();

codegen/Cargo.toml

@@ -11,7 +11,9 @@ tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] }
 futures = "0.3.21"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.82"
-serde_with = { version = "3.0.0", default-features = false, features = ["macros"] }
+serde_with = { version = "3.0.0", default-features = false, features = [
+    "macros",
+] }
 anyhow = "1.0"
 log = "0.4.17"
 env_logger = "0.10.0"
@@ -24,3 +26,4 @@ num_enum = "0.6.1"
 path_macro = "1.0.0"
 intl_pluralrules = "7.0.2"
 unic-langid = "0.9.1"
+ordered_hash_map = { version = "0.2.0", features = ["serde"] }

codegen/src/abtest.rs

@@ -24,6 +24,7 @@ pub enum ABTest {
     TrendsPageHeaderRenderer = 5,
     DiscographyPage = 6,
     ShortDateFormat = 7,
+    PlaylistsForShorts = 9,
 }
 
 const TESTS_TO_RUN: [ABTest; 3] = [
@@ -94,6 +95,7 @@ pub async fn run_test(
                     ABTest::TrendsPageHeaderRenderer => trends_page_header_renderer(&query).await,
                     ABTest::DiscographyPage => discography_page(&query).await,
                     ABTest::ShortDateFormat => short_date_format(&query).await,
+                    ABTest::PlaylistsForShorts => playlists_for_shorts(&query).await,
                 }
                 .unwrap();
                 pb.inc(1);
@@ -243,3 +245,13 @@ pub async fn short_date_format(rp: &RustyPipeQuery) -> Result<bool> {
             .unwrap_or_default()
     }))
 }
+
+pub async fn playlists_for_shorts(rp: &RustyPipeQuery) -> Result<bool> {
+    let playlist = rp.playlist("UUSHh8gHdtzO2tXd593_bjErWg").await?;
+    let v1 = playlist
+        .videos
+        .items
+        .first()
+        .ok_or_else(|| anyhow::anyhow!("no videos"))?;
+    Ok(v1.publish_date_txt.is_none())
+}

codegen/src/collect_playlist_dates.rs

@@ -6,6 +6,7 @@ use std::{
 };
 
 use futures::{stream, StreamExt};
+use ordered_hash_map::OrderedHashMap;
 use path_macro::path;
 use rustypipe::{
     client::RustyPipe,
@@ -170,7 +171,7 @@ pub fn write_samples_to_dict() {
         dict_entry.months = BTreeMap::new();
 
         if collect_nd_tokens {
-            dict_entry.timeago_nd_tokens = BTreeMap::new();
+            dict_entry.timeago_nd_tokens = OrderedHashMap::new();
         }
 
         for datestr_table in &datestr_tables {

codegen/src/download_testfiles.rs

@@ -619,7 +619,7 @@ async fn music_search_playlists() {
 
         let rp = rp_testfile(&json_path);
         rp.query()
-            .music_search_playlists_filter("pop", community)
+            .music_search_playlists("pop", community)
             .await
             .unwrap();
     }

codegen/src/model.rs

@@ -1,5 +1,6 @@
 use std::collections::BTreeMap;
 
+use ordered_hash_map::OrderedHashMap;
 use rustypipe::{client::YTContext, model::AlbumType, param::Language};
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DefaultOnError, VecSkipError};
@@ -18,7 +19,7 @@ pub struct DictEntry {
     ///
     /// Identifiers: `Y`(ear), `M`(month), `W`(eek), `D`(ay),
     /// `h`(our), `m`(inute), `s`(econd)
-    pub timeago_tokens: BTreeMap<String, String>,
+    pub timeago_tokens: OrderedHashMap<String, String>,
     /// Order in which to parse numeric date components. Formatted as
     /// a string of date identifiers (Y, M, D).
     ///
@@ -34,7 +35,7 @@ pub struct DictEntry {
     /// Tokens for parsing date strings with no digits (e.g. Today, Tomorrow)
     ///
     /// Format: Parsed token -> \[Quantity\] Identifier
-    pub timeago_nd_tokens: BTreeMap<String, String>,
+    pub timeago_nd_tokens: OrderedHashMap<String, String>,
     /// Are commas (instead of points) used as decimal separators?
     pub comma_decimal: bool,
     /// Tokens for parsing decimal prefixes (K, M, B, ...)
@@ -49,6 +50,10 @@ pub struct DictEntry {
     ///
     /// Format: Parsed text -> Album type
     pub album_types: BTreeMap<String, AlbumType>,
+    /// Names of item types (Song, Video, Artist, Playlist)
+    ///
+    /// Format: Parsed text -> Item type
+    pub item_types: BTreeMap<String, ExtItemType>,
 }
 
 /// Parsed TimeAgo string, contains amount and time unit.
@@ -99,6 +104,15 @@ impl TimeUnit {
     }
 }
 
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum ExtItemType {
+    Track,
+    Video,
+    Episode,
+    Playlist,
+    Artist,
+}
+
 #[derive(Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct QBrowse<'a> {

notes/AB_Tests.md

@@ -360,8 +360,8 @@ YouTube changed the header renderer type on the trending page to a `pageHeaderRe
 - **Impact:** 🟡 Medium
 - **Endpoint:** browse (music artist)
 
-YouTube merged the 2 sections for singles and albums on artist pages together. Now
-there is only a *Top Releases* section.
+YouTube merged the 2 sections for singles and albums on artist pages together. Now there
+is only a _Top Releases_ section.
 
 YouTube also changed the way the full discography page is fetched, surprisingly making
 it easier for alternative clients. The discography page now has its own content ID in
@@ -382,5 +382,26 @@ visitor data cookie to be set, as it was the case with the old system.
 - **Encountered on:** 28.05.2023
 - **Impact:** 🟡 Medium
 
-YouTube changed their date format from the long format (*21 hours ago*, *3 days ago*) to
-a short format (*21h ago*, *3d ago*).
+YouTube changed their date format from the long format (_21 hours ago_, _3 days ago_) to
+a short format (_21h ago_, _3d ago_).
+
+## [9] Playlists for Shorts
+
+- **Encountered on:** 26.06.2023
+- **Impact:** 🟡 Medium
+- **Endpoint:** browse (playlist)
+
+![A/B test 9 screenshot](./_img/ab_9.png)
+
+Original issue: https://github.com/TeamNewPipe/NewPipeExtractor/issues/10774
+
+YouTube added a filter system for playlists, allowing users to only see shorts/full
+videos.
+
+When shorts filter is enabled or when there are only shorts in a playlist, YouTube
+return shorts UI elements instead of standard video ones, the ones that are also used
+for shorts shelves in searches and suggestions and shorts in the corresponding channel
+tab.
+
+Since the reel items dont include upload date information you can circumvent this new UI
+by using the mobile client. But that may change in the future.

src/client/music_search.rs

@@ -40,8 +40,6 @@ enum Params {
     Albums,
     #[serde(rename = "EgWKAQIgAWoMEAMQBBAJEA4QChAF")]
     Artists,
-    #[serde(rename = "EgWKAQIoAWoMEAMQBBAJEA4QChAF")]
-    Playlists,
     #[serde(rename = "EgeKAQQoADgBagwQAxAEEAkQDhAKEAU%3D")]
     YtmPlaylists,
     #[serde(rename = "EgeKAQQoAEABagwQAxAEEAkQDhAKEAU%3D")]
@@ -152,44 +150,25 @@ impl RustyPipeQuery {
         )
         .await
     }
-
     /// Search YouTube Music playlists
+    ///
+    /// Playlists are filtered whether they are created by users
+    /// (`community=true`) or by YouTube Music (`community=false`)
     pub async fn music_search_playlists<S: AsRef<str>>(
         &self,
         query: S,
-    ) -> Result<MusicSearchFiltered<MusicPlaylistItem>, Error> {
-        self._music_search_playlists(query, Params::Playlists).await
-    }
-
-    /// Search YouTube Music playlists that were created by users
-    /// (`community=true`) or by YouTube Music (`community=false`)
-    pub async fn music_search_playlists_filter<S: AsRef<str>>(
-        &self,
-        query: S,
         community: bool,
-    ) -> Result<MusicSearchFiltered<MusicPlaylistItem>, Error> {
-        self._music_search_playlists(
-            query,
-            if community {
-                Params::CommunityPlaylists
-            } else {
-                Params::YtmPlaylists
-            },
-        )
-        .await
-    }
-
-    async fn _music_search_playlists<S: AsRef<str>>(
-        &self,
-        query: S,
-        params: Params,
     ) -> Result<MusicSearchFiltered<MusicPlaylistItem>, Error> {
         let query = query.as_ref();
         let context = self.get_context(ClientType::DesktopMusic, true, None).await;
         let request_body = QSearch {
             context,
             query,
-            params: Some(params),
+            params: Some(if community {
+                Params::CommunityPlaylists
+            } else {
+                Params::YtmPlaylists
+            }),
         };
 
         self.execute_request::<response::MusicSearch, _, _>(

src/client/response/playlist.rs

@@ -29,6 +29,7 @@ pub(crate) struct ItemSection {
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct PlaylistVideoListRenderer {
+    #[serde(alias = "richGridRenderer")]
     pub playlist_video_list_renderer: YouTubeListRenderer,
 }
 

src/util/mod.rs

@@ -26,7 +26,7 @@ pub static VIDEO_ID_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"^[A-Za-z0-9_-
 pub static CHANNEL_ID_REGEX: Lazy<Regex> =
     Lazy::new(|| Regex::new(r"^UC[A-Za-z0-9_-]{22}$").unwrap());
 pub static PLAYLIST_ID_REGEX: Lazy<Regex> =
-    Lazy::new(|| Regex::new(r"^(?:PL|RD|OLAK|UU)[A-Za-z0-9_-]{16,50}$").unwrap());
+    Lazy::new(|| Regex::new(r"^(?:PL|RD|OLAK|UU)[A-Za-z0-9_-]{5,50}$").unwrap());
 pub static ALBUM_ID_REGEX: Lazy<Regex> =
     Lazy::new(|| Regex::new(r"^MPREb_[A-Za-z0-9_-]{11}$").unwrap());
 pub static VANITY_PATH_REGEX: Lazy<Regex> = Lazy::new(|| {

tests/youtube.rs

@@ -1808,18 +1808,12 @@ fn music_search_artists_cont(rp: RustyPipe) {
 }
 
 #[rstest]
-#[case::ytm(false)]
-#[case::default(true)]
-fn music_search_playlists(#[case] with_community: bool, rp: RustyPipe, unlocalized: bool) {
-    let res = if with_community {
-        tokio_test::block_on(rp.query().music_search_playlists("today's rock hits")).unwrap()
-    } else {
-        tokio_test::block_on(
+fn music_search_playlists(rp: RustyPipe, unlocalized: bool) {
+    let res = tokio_test::block_on(
         rp.query()
-                .music_search_playlists_filter("today's rock hits", false),
+            .music_search_playlists("today's rock hits", false),
     )
-        .unwrap()
-    };
+    .unwrap();
 
     assert_eq!(res.corrected_query, None);
     let playlist = res
@@ -1837,24 +1831,17 @@ fn music_search_playlists(#[case] with_community: bool, rp: RustyPipe, unlocaliz
     assert_eq!(playlist.channel, None);
     assert!(playlist.from_ytm);
 
-    if with_community {
-        assert!(
-            res.items.items.iter().any(|p| !p.from_ytm),
-            "no community items found"
-        )
-    } else {
     assert!(
         res.items.items.iter().all(|p| p.from_ytm),
         "community items found"
     )
 }
-}
 
 #[rstest]
 fn music_search_playlists_community(rp: RustyPipe) {
     let res = tokio_test::block_on(
         rp.query()
-            .music_search_playlists_filter("Best Pop Music Videos - Top Pop Hits Playlist", true),
+            .music_search_playlists("Best Pop Music Videos - Top Pop Hits Playlist", true),
     )
     .unwrap();
 
@@ -2241,14 +2228,10 @@ fn music_genre(#[case] id: &str, #[case] name: &str, rp: RustyPipe, unlocalized:
                 assert!(!playlist.thumbnail.is_empty(), "got no cover");
 
                 if !playlist.from_ytm {
-                    assert!(
-                        playlist.channel.is_some(),
-                        "pl: {}, got no channel",
-                        playlist.id
-                    );
-                    let channel = playlist.channel.as_ref().unwrap();
+                    if let Some(channel) = playlist.channel.as_ref() {
                         validate::channel_id(&channel.id).unwrap();
                         assert!(!channel.name.is_empty());
+                    }
                 } else {
                     assert!(playlist.channel.is_none());
                 }