fix: parsing video details for DRM-restricted movies

Justfile

@@ -7,14 +7,6 @@ unittest:
 testyt:
     cargo test --features=rss --test youtube
 
-testyt10:
-    #!/usr/bin/env bash
-    set -e
-    for i in {1..10}; do \
-        echo "---TEST RUN $i---"; \
-        cargo test --features=rss --test youtube; \
-    done
-
 testintl:
     #!/usr/bin/env bash
     LANGUAGES=(
@@ -47,4 +39,4 @@ testfiles:
 
 report2yaml:
     mkdir -p rustypipe_reports/conv
-    for f in rustypipe_reports/*.json; do yq '.http_request.resp_body' $f | yq -o json -P > rustypipe_reports/conv/`basename $f .json`_body.json; yq e -Pi $f; mv $f rustypipe_reports/conv/`basename $f .json`.yaml; done;
+    for f in rustypipe_reports/*.json; do yq '.http_request.resp_body' $f | yq -o json -P > rustypipe_reports/conv/`basename $f .json`_body.json; yq e -Pi "del(.http_request.resp_body)" $f; mv $f rustypipe_reports/conv/`basename $f .json`.yaml; done;

README.md

@@ -154,3 +154,29 @@ Subscribers: 1780000
 [6Fv8bd9ICb4] Who owns this? (199s)
 ...
 ```
+
+## Development
+
+**Requirements:**
+
+- Current version of stable Rust
+- [`just`](https://github.com/casey/just) task runner
+- [`pre-commit`](https://pre-commit.com/)
+- yq (YAML processor)
+
+### Tasks
+
+**Testing**
+
+- `just test` Run unit+integration tests
+- `just unittest` Run unit tests
+- `just testyt` Run YouTube integration tests
+- `just testintl` Run YouTube integration tests for all supported languages (this takes
+  a long time and is therefore not run in CI)
+- `YT_LANG=de just testyt` Run YouTube integration tests for a specific language
+
+**Tools**
+
+- `just testfiles` Download missing testfiles for unit tests
+- `just report2yaml` Convert RustyPipe reports into a more readable yaml format
+  (requires `yq`)

cli/src/main.rs

@@ -19,6 +19,9 @@ use serde::Serialize;
 struct Cli {
     #[clap(subcommand)]
     command: Commands,
+    /// Always generate a report (used for debugging)
+    #[clap(long)]
+    report: bool,
 }
 
 #[derive(Subcommand)]
@@ -392,10 +395,11 @@ async fn main() {
     storage_dir.push("rustypipe");
     _ = std::fs::create_dir(&storage_dir);
 
-    let rp = RustyPipe::builder()
-        .storage_dir(storage_dir)
-        .build()
-        .unwrap();
+    let mut rp = RustyPipe::builder().storage_dir(storage_dir);
+    if cli.report {
+        rp = rp.report();
+    }
+    let rp = rp.build().unwrap();
 
     match cli.command {
         Commands::Download {

src/client/response/video_details.rs

@@ -77,8 +77,8 @@ pub(crate) enum VideoResultsItem {
         /// Like/Dislike button
         video_actions: VideoActions,
         /// Absolute textual date (e.g. `Dec 29, 2019`)
-        #[serde_as(as = "Text")]
-        date_text: String,
+        #[serde_as(as = "Option<Text>")]
+        date_text: Option<String>,
     },
     #[serde(rename_all = "camelCase")]
     VideoSecondaryInfoRenderer {

src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20220924_newdesc.snap

@@ -139,7 +139,7 @@ VideoDetails(
   view_count: 233243423,
   like_count: Some(4015532),
   publish_date: "[date]",
-  publish_date_txt: "Nov 17, 2020",
+  publish_date_txt: Some("Nov 17, 2020"),
   is_live: false,
   is_ccommons: false,
   chapters: [],

src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_new_continuation.snap

@@ -139,7 +139,7 @@ VideoDetails(
   view_count: 234258725,
   like_count: Some(4027586),
   publish_date: "[date]",
-  publish_date_txt: "Nov 17, 2020",
+  publish_date_txt: Some("Nov 17, 2020"),
   is_live: false,
   is_ccommons: false,
   chapters: [],

src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_rec_isr.snap

@@ -389,7 +389,7 @@ VideoDetails(
   view_count: 1251797,
   like_count: Some(58077),
   publish_date: "[date]",
-  publish_date_txt: "Sep 15, 2022",
+  publish_date_txt: Some("Sep 15, 2022"),
   is_live: false,
   is_ccommons: false,
   chapters: [

src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_agegate.snap

@@ -32,7 +32,7 @@ VideoDetails(
   view_count: 205,
   like_count: None,
   publish_date: "[date]",
-  publish_date_txt: "Jan 2, 2019",
+  publish_date_txt: Some("Jan 2, 2019"),
   is_live: false,
   is_ccommons: false,
   chapters: [],

src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_ccommons.snap

@@ -50,7 +50,7 @@ VideoDetails(
   view_count: 2493983,
   like_count: Some(52274),
   publish_date: "[date]",
-  publish_date_txt: "Dec 29, 2019",
+  publish_date_txt: Some("Dec 29, 2019"),
   is_live: false,
   is_ccommons: true,
   chapters: [],

src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_chapters.snap

@@ -371,7 +371,7 @@ VideoDetails(
   view_count: 971966,
   like_count: Some(48244),
   publish_date: "[date]",
-  publish_date_txt: "Sep 15, 2022",
+  publish_date_txt: Some("Sep 15, 2022"),
   is_live: false,
   is_ccommons: false,
   chapters: [

src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_live.snap

@@ -94,7 +94,7 @@ VideoDetails(
   view_count: 681,
   like_count: Some(872230),
   publish_date: "[date]",
-  publish_date_txt: "Started streaming on Sep 23, 2021",
+  publish_date_txt: Some("Started streaming on Sep 23, 2021"),
   is_live: true,
   is_ccommons: false,
   chapters: [],

src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_music.snap

@@ -36,7 +36,7 @@ VideoDetails(
   view_count: 20304,
   like_count: Some(146),
   publish_date: "[date]",
-  publish_date_txt: "Aug 6, 2020",
+  publish_date_txt: Some("Aug 6, 2020"),
   is_live: false,
   is_ccommons: false,
   chapters: [],

src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_mv.snap

@@ -139,7 +139,7 @@ VideoDetails(
   view_count: 232792465,
   like_count: Some(4010156),
   publish_date: "[date]",
-  publish_date_txt: "Nov 17, 2020",
+  publish_date_txt: Some("Nov 17, 2020"),
   is_live: false,
   is_ccommons: false,
   chapters: [],

src/client/video_details.rs

@@ -175,7 +175,9 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
                         // accessibility_data contains no digits if the like count is hidden,
                         // so we ignore parse errors here for now
                         like_btn.and_then(|btn| util::parse_numeric(&btn.accessibility_data).ok()),
-                        timeago::parse_textual_date_or_warn(lang, &date_text, &mut warnings),
+                        date_text.as_deref().and_then(|txt| {
+                            timeago::parse_textual_date_or_warn(lang, txt, &mut warnings)
+                        }),
                         date_text,
                         view_count
                             .map(|vc| vc.video_view_count_renderer.is_live)

src/model/mod.rs

@@ -567,7 +567,7 @@ pub struct VideoDetails {
     #[serde(with = "time::serde::rfc3339::option")]
     pub publish_date: Option<OffsetDateTime>,
     /// Textual video publishing date (e.g. `Aug 2, 2013`, depends on language)
-    pub publish_date_txt: String,
+    pub publish_date_txt: Option<String>,
     /// Is the video a livestream?
     pub is_live: bool,
     /// Is the video published under the Creative Commons BY 3.0 license?

tests/youtube.rs

@@ -1278,10 +1278,10 @@ fn trending(rp: RustyPipe) {
     false,
 )]
 #[case::short(
-    "RDCLAK5uy_kFQXdnqMaQCVx2wpUM4ZfbsGCDibZtkJk",
-    "Easy Pop",
+    "RDCLAK5uy_nLNY4ReQKH2kx5U23cyGMHql9ciHD9RSM",
+    "Presenting BLACKPINK (블랙핑크)",
     false,
-    Some("Stress-free tunes from classic rockers and newer artists.\nThis playlist is no longer being updated.".to_owned()),
+    Some("The most played hits and essential tracks. #blackpink #best #kpop".to_owned()),
     None,
     true
 )]
@@ -2181,7 +2181,7 @@ fn music_genres(rp: RustyPipe, unlocalized: bool) {
     let chill = genres
         .iter()
         .find(|g| g.id == "ggMPOg1uX1JOQWZFeDByc2Jm")
-        .unwrap();
+        .expect("genre: Chill");
     if unlocalized {
         assert_eq!(chill.name, "Chill");
     }
@@ -2189,8 +2189,8 @@ fn music_genres(rp: RustyPipe, unlocalized: bool) {
 
     let pop = genres
         .iter()
-        .find(|g| g.id == "ggMPOg1uX1lMbVZmbzl6NlJ3")
-        .unwrap();
+        .find(|g| g.id == "ggMPOg1uX1lMbVZmbzl6NlJ3" || g.id == "ggMPOg1uX1BmNzc2V2p0YXJ5")
+        .expect("genre: Pop");
     assert_eq!(pop.name, "Pop");
     assert!(!pop.is_mood);