diff --git a/Justfile b/Justfile index 7d7ca4d..c2f3fdf 100644 --- a/Justfile +++ b/Justfile @@ -7,14 +7,6 @@ unittest: testyt: cargo test --features=rss --test youtube -testyt10: - #!/usr/bin/env bash - set -e - for i in {1..10}; do \ - echo "---TEST RUN $i---"; \ - cargo test --features=rss --test youtube; \ - done - testintl: #!/usr/bin/env bash LANGUAGES=( @@ -47,4 +39,4 @@ testfiles: report2yaml: mkdir -p rustypipe_reports/conv - for f in rustypipe_reports/*.json; do yq '.http_request.resp_body' $f | yq -o json -P > rustypipe_reports/conv/`basename $f .json`_body.json; yq e -Pi $f; mv $f rustypipe_reports/conv/`basename $f .json`.yaml; done; + for f in rustypipe_reports/*.json; do yq '.http_request.resp_body' $f | yq -o json -P > rustypipe_reports/conv/`basename $f .json`_body.json; yq e -Pi "del(.http_request.resp_body)" $f; mv $f rustypipe_reports/conv/`basename $f .json`.yaml; done; diff --git a/README.md b/README.md index a237ebc..83bd294 100644 --- a/README.md +++ b/README.md @@ -154,3 +154,29 @@ Subscribers: 1780000 [6Fv8bd9ICb4] Who owns this? (199s) ... ``` + +## Development + +**Requirements:** + +- Current version of stable Rust +- [`just`](https://github.com/casey/just) task runner +- [`pre-commit`](https://pre-commit.com/) +- yq (YAML processor) + +### Tasks + +**Testing** + +- `just test` Run unit+integration tests +- `just unittest` Run unit tests +- `just testyt` Run YouTube integration tests +- `just testintl` Run YouTube integration tests for all supported languages (this takes + a long time and is therefore not run in CI) +- `YT_LANG=de just testyt` Run YouTube integration tests for a specific language + +**Tools** + +- `just testfiles` Download missing testfiles for unit tests +- `just report2yaml` Convert RustyPipe reports into a more readable yaml format + (requires `yq`) diff --git a/cli/src/main.rs b/cli/src/main.rs index c96cd98..ff53c77 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -19,6 +19,9 @@ use serde::Serialize; struct Cli { #[clap(subcommand)] command: Commands, + /// Always generate a report (used for debugging) + #[clap(long)] + report: bool, } #[derive(Subcommand)] @@ -392,10 +395,11 @@ async fn main() { storage_dir.push("rustypipe"); _ = std::fs::create_dir(&storage_dir); - let rp = RustyPipe::builder() - .storage_dir(storage_dir) - .build() - .unwrap(); + let mut rp = RustyPipe::builder().storage_dir(storage_dir); + if cli.report { + rp = rp.report(); + } + let rp = rp.build().unwrap(); match cli.command { Commands::Download { diff --git a/src/client/response/video_details.rs b/src/client/response/video_details.rs index f43878a..c9b4d2e 100644 --- a/src/client/response/video_details.rs +++ b/src/client/response/video_details.rs @@ -77,8 +77,8 @@ pub(crate) enum VideoResultsItem { /// Like/Dislike button video_actions: VideoActions, /// Absolute textual date (e.g. `Dec 29, 2019`) - #[serde_as(as = "Text")] - date_text: String, + #[serde_as(as = "Option")] + date_text: Option, }, #[serde(rename_all = "camelCase")] VideoSecondaryInfoRenderer { diff --git a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20220924_newdesc.snap b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20220924_newdesc.snap index fd92e05..a360227 100644 --- a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20220924_newdesc.snap +++ b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20220924_newdesc.snap @@ -139,7 +139,7 @@ VideoDetails( view_count: 233243423, like_count: Some(4015532), publish_date: "[date]", - publish_date_txt: "Nov 17, 2020", + publish_date_txt: Some("Nov 17, 2020"), is_live: false, is_ccommons: false, chapters: [], diff --git a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_new_continuation.snap b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_new_continuation.snap index 0841f39..68e704d 100644 --- a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_new_continuation.snap +++ b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_new_continuation.snap @@ -139,7 +139,7 @@ VideoDetails( view_count: 234258725, like_count: Some(4027586), publish_date: "[date]", - publish_date_txt: "Nov 17, 2020", + publish_date_txt: Some("Nov 17, 2020"), is_live: false, is_ccommons: false, chapters: [], diff --git a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_rec_isr.snap b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_rec_isr.snap index 10880e8..4cfe502 100644 --- a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_rec_isr.snap +++ b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_rec_isr.snap @@ -389,7 +389,7 @@ VideoDetails( view_count: 1251797, like_count: Some(58077), publish_date: "[date]", - publish_date_txt: "Sep 15, 2022", + publish_date_txt: Some("Sep 15, 2022"), is_live: false, is_ccommons: false, chapters: [ diff --git a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_agegate.snap b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_agegate.snap index ca890dd..970147c 100644 --- a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_agegate.snap +++ b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_agegate.snap @@ -32,7 +32,7 @@ VideoDetails( view_count: 205, like_count: None, publish_date: "[date]", - publish_date_txt: "Jan 2, 2019", + publish_date_txt: Some("Jan 2, 2019"), is_live: false, is_ccommons: false, chapters: [], diff --git a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_ccommons.snap b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_ccommons.snap index 1cc3cc7..9f51720 100644 --- a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_ccommons.snap +++ b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_ccommons.snap @@ -50,7 +50,7 @@ VideoDetails( view_count: 2493983, like_count: Some(52274), publish_date: "[date]", - publish_date_txt: "Dec 29, 2019", + publish_date_txt: Some("Dec 29, 2019"), is_live: false, is_ccommons: true, chapters: [], diff --git a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_chapters.snap b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_chapters.snap index cc9ce07..187bff9 100644 --- a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_chapters.snap +++ b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_chapters.snap @@ -371,7 +371,7 @@ VideoDetails( view_count: 971966, like_count: Some(48244), publish_date: "[date]", - publish_date_txt: "Sep 15, 2022", + publish_date_txt: Some("Sep 15, 2022"), is_live: false, is_ccommons: false, chapters: [ diff --git a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_live.snap b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_live.snap index 8236ef8..62979ec 100644 --- a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_live.snap +++ b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_live.snap @@ -94,7 +94,7 @@ VideoDetails( view_count: 681, like_count: Some(872230), publish_date: "[date]", - publish_date_txt: "Started streaming on Sep 23, 2021", + publish_date_txt: Some("Started streaming on Sep 23, 2021"), is_live: true, is_ccommons: false, chapters: [], diff --git a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_music.snap b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_music.snap index ce4edf3..2e65156 100644 --- a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_music.snap +++ b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_music.snap @@ -36,7 +36,7 @@ VideoDetails( view_count: 20304, like_count: Some(146), publish_date: "[date]", - publish_date_txt: "Aug 6, 2020", + publish_date_txt: Some("Aug 6, 2020"), is_live: false, is_ccommons: false, chapters: [], diff --git a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_mv.snap b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_mv.snap index 9071e3e..3442496 100644 --- a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_mv.snap +++ b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_mv.snap @@ -139,7 +139,7 @@ VideoDetails( view_count: 232792465, like_count: Some(4010156), publish_date: "[date]", - publish_date_txt: "Nov 17, 2020", + publish_date_txt: Some("Nov 17, 2020"), is_live: false, is_ccommons: false, chapters: [], diff --git a/src/client/video_details.rs b/src/client/video_details.rs index 92570cc..6d24583 100644 --- a/src/client/video_details.rs +++ b/src/client/video_details.rs @@ -175,7 +175,9 @@ impl MapResponse for response::VideoDetails { // accessibility_data contains no digits if the like count is hidden, // so we ignore parse errors here for now like_btn.and_then(|btn| util::parse_numeric(&btn.accessibility_data).ok()), - timeago::parse_textual_date_or_warn(lang, &date_text, &mut warnings), + date_text.as_deref().and_then(|txt| { + timeago::parse_textual_date_or_warn(lang, txt, &mut warnings) + }), date_text, view_count .map(|vc| vc.video_view_count_renderer.is_live) diff --git a/src/model/mod.rs b/src/model/mod.rs index 252cab6..ffbf652 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -567,7 +567,7 @@ pub struct VideoDetails { #[serde(with = "time::serde::rfc3339::option")] pub publish_date: Option, /// Textual video publishing date (e.g. `Aug 2, 2013`, depends on language) - pub publish_date_txt: String, + pub publish_date_txt: Option, /// Is the video a livestream? pub is_live: bool, /// Is the video published under the Creative Commons BY 3.0 license? diff --git a/tests/youtube.rs b/tests/youtube.rs index e3c9027..0f448de 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -1278,10 +1278,10 @@ fn trending(rp: RustyPipe) { false, )] #[case::short( - "RDCLAK5uy_kFQXdnqMaQCVx2wpUM4ZfbsGCDibZtkJk", - "Easy Pop", + "RDCLAK5uy_nLNY4ReQKH2kx5U23cyGMHql9ciHD9RSM", + "Presenting BLACKPINK (λΈ”λž™ν•‘ν¬)", false, - Some("Stress-free tunes from classic rockers and newer artists.\nThis playlist is no longer being updated.".to_owned()), + Some("The most played hits and essential tracks. #blackpink #best #kpop".to_owned()), None, true )] @@ -2181,7 +2181,7 @@ fn music_genres(rp: RustyPipe, unlocalized: bool) { let chill = genres .iter() .find(|g| g.id == "ggMPOg1uX1JOQWZFeDByc2Jm") - .unwrap(); + .expect("genre: Chill"); if unlocalized { assert_eq!(chill.name, "Chill"); } @@ -2189,8 +2189,8 @@ fn music_genres(rp: RustyPipe, unlocalized: bool) { let pop = genres .iter() - .find(|g| g.id == "ggMPOg1uX1lMbVZmbzl6NlJ3") - .unwrap(); + .find(|g| g.id == "ggMPOg1uX1lMbVZmbzl6NlJ3" || g.id == "ggMPOg1uX1BmNzc2V2p0YXJ5") + .expect("genre: Pop"); assert_eq!(pop.name, "Pop"); assert!(!pop.is_mood);