diff --git a/.gitignore b/.gitignore index 23e8123..e96be3d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /target /Cargo.lock -rusty-tube.json +rustypipe_reports +rustypipe_cache.json diff --git a/Cargo.toml b/Cargo.toml index 7ad4d82..255c669 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,10 +3,19 @@ name = "rustypipe" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [workspace] -members = [".", "cli"] +members = [".", "codegen", "cli"] + +[features] +default = ["default-tls"] + +# Reqwest TLS +default-tls = ["reqwest/default-tls"] +rustls-tls-webpki-roots = ["reqwest/rustls-tls-webpki-roots"] +rustls-tls-native-roots = ["reqwest/rustls-tls-native-roots"] + +# Error reports in yaml format +report-yaml = ["serde_yaml"] [dependencies] # quick-js = "0.4.1" @@ -17,13 +26,13 @@ anyhow = "1.0" thiserror = "1.0.31" url = "2.2.2" log = "0.4.17" -reqwest = {version = "0.11.11", default-features = false, features = ["json", "gzip", "brotli", "stream", "rustls-tls-native-roots"]} -tokio = {version = "1.20.0", features = ["macros", "fs", "process"]} +reqwest = {version = "0.11.11", default-features = false, features = ["json", "gzip", "brotli", "stream"]} +tokio = {version = "1.20.0", features = ["macros", "time", "fs", "process"]} serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.82" +serde_yaml = {version = "0.9.11", optional = true} serde_with = {version = "2.0.0", features = ["json"] } rand = "0.8.5" -async-trait = "0.1.56" chrono = {version = "0.4.19", features = ["serde"]} chronoutil = "0.2.3" futures = "0.3.21" @@ -37,6 +46,5 @@ env_logger = "0.9.0" test-log = "0.2.11" rstest = "0.15.0" temp_testdir = "0.2.3" -insta = {version = "1.17.1", features = ["redactions"]} +insta = {version = "1.17.1", features = ["yaml", "redactions"]} velcro = "0.5.3" -phf_codegen = "0.11.1" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 353a9be..23b8ed6 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -3,12 +3,10 @@ name = "rustypipe-cli" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] -rustypipe = {path = "../"} -reqwest = {version = "0.11.11", default_features = false, features = ["gzip", "brotli", "rustls-tls-native-roots"]} -tokio = {version = "1.20.0", features = ["rt-multi-thread"]} +rustypipe = {path = "../", default_features = false, features = ["rustls-tls-native-roots"]} +reqwest = {version = "0.11.11", default_features = false} +tokio = {version = "1.20.0", features = ["macros", "rt-multi-thread"]} indicatif = "0.17.0" futures = "0.3.21" anyhow = "1.0" diff --git a/cli/src/main.rs b/cli/src/main.rs index 01ee3c4..e6a77c0 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -6,7 +6,7 @@ use futures::stream::{self, StreamExt}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use reqwest::{Client, ClientBuilder}; use rustypipe::{ - client::{ClientType, RustyTube}, + client::{ClientType, RustyPipe}, model::stream_filter::Filter, }; @@ -46,7 +46,7 @@ async fn download_single_video( output_fname: Option, resolution: Option, ffmpeg: &str, - rt: &RustyTube, + rp: &RustyPipe, http: Client, multi: MultiProgress, main: Option, @@ -58,7 +58,8 @@ async fn download_single_video( pb.set_message(format!("Fetching player data for {}", video_title)); let res = async { - let player_data = rt + let player_data = rp + .query() .get_player(video_id.as_str(), ClientType::TvHtml5Embed) .await .context(format!( @@ -112,7 +113,7 @@ async fn download_video( .build() .expect("unable to build the HTTP client"); - let rt = RustyTube::new(); + let rp = RustyPipe::default(); // Indicatif setup let multi = MultiProgress::new(); @@ -124,7 +125,7 @@ async fn download_video( output_fname, resolution, "ffmpeg", - &rt, + &rp, http, multi, None, @@ -147,8 +148,8 @@ async fn download_playlist( .build() .expect("unable to build the HTTP client"); - let rt = RustyTube::new(); - let playlist = rt.get_playlist(id).await.unwrap(); + let rp = RustyPipe::default(); + let playlist = rp.query().get_playlist(id).await.unwrap(); // Indicatif setup let multi = MultiProgress::new(); @@ -173,7 +174,7 @@ async fn download_playlist( output_fname.to_owned(), resolution, "ffmpeg", - &rt, + &rp, http.clone(), multi.clone(), Some(main.clone()), diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml new file mode 100644 index 0000000..ccda07c --- /dev/null +++ b/codegen/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "rustypipe-codegen" +version = "0.1.0" +edition = "2021" + +[dependencies] +rustypipe = {path = "../"} +reqwest = "0.11.11" +tokio = {version = "1.20.0", features = ["macros", "rt-multi-thread"]} +futures = "0.3.21" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0.82" +serde_with = "2.0.0" +anyhow = "1.0" +log = "0.4.17" +env_logger = "0.9.0" +clap = { version = "3.2.16", features = ["derive"] } +phf_codegen = "0.11.1" +once_cell = "1.12.0" +fancy-regex = "0.10.0" diff --git a/src/codegen/collect_playlist_dates.rs b/codegen/src/collect_playlist_dates.rs similarity index 70% rename from src/codegen/collect_playlist_dates.rs rename to codegen/src/collect_playlist_dates.rs index fd071c9..23b7f69 100644 --- a/src/codegen/collect_playlist_dates.rs +++ b/codegen/src/collect_playlist_dates.rs @@ -1,5 +1,3 @@ -#![cfg(test)] - use std::{ collections::{BTreeMap, HashMap}, fs::File, @@ -8,14 +6,15 @@ use std::{ path::Path, }; +use futures::{stream, StreamExt}; +use rustypipe::{ + client::RustyPipe, + model::{locale::LANGUAGES, Language}, + timeago::{self, TimeAgo}, +}; use serde::{Deserialize, Serialize}; -use crate::{ - client::RustyTube, - model::{locale::LANGUAGES, Country, Language}, - timeago::{self, TimeAgo}, - util, -}; +use crate::util; type CollectedDates = BTreeMap>; @@ -38,20 +37,40 @@ enum DateCase { Dec, } -// #[test_log::test(tokio::test)] -async fn collect_dates() { - let json_path = Path::new("testfiles/date/playlist_samples.json").to_path_buf(); - if json_path.exists() { - return; - } +/// Collect 'Playlist updated' dates in every supported language +/// and write them to `testfiles/date/playlist_samples.json`. +/// +/// YouTube's API outputs the update date of playlists only in a +/// textual format (e.g. *Last updated on Jan 3, 2020*), which varies +/// by language. +/// +/// For recently updated playlists YouTube shows 'today', 'yesterday' +/// and 'x<=7 days ago' instead of the literal date. +/// +/// To parse these dates correctly we need to collect a sample set +/// in every language. +/// +/// This set includes +/// - one playlist updated today +/// - one playlist updated yesterday +/// - one playlist updated 2-7 days ago +/// - one playlist from every month. Note that there should not +/// be any dates which include the same number twice (e.g. 01.01.2020). +/// +/// Because the relative dates change with time, the first three playlists +/// should be checked and eventually changed before running the program. +pub async fn collect_dates(project_root: &Path, concurrency: usize) { + let mut json_path = project_root.to_path_buf(); + json_path.push("testfiles/date/playlist_samples.json"); + // These are the sample playlists let cases = [ ( DateCase::Today, "RDCLAK5uy_kj3rhiar1LINmyDcuFnXihEO0K1NQa2jI", ), - (DateCase::Yesterday, "PLmB6td997u3kUOrfFwkULZ910ho44oQSy"), - (DateCase::Ago, "PL7zsB-C3aNu2yRY2869T0zj1FhtRIu5am"), + (DateCase::Yesterday, "PL7zsB-C3aNu2yRY2869T0zj1FhtRIu5am"), + (DateCase::Ago, "PLmB6td997u3kUOrfFwkULZ910ho44oQSy"), (DateCase::Jan, "PL1J-6JOckZtFjcni6Xj1pLYglJp6JCpKD"), (DateCase::Feb, "PL1J-6JOckZtETrbzwZE7mRIIK6BzWNLAs"), (DateCase::Mar, "PL1J-6JOckZtG3AVdvBXhMO64mB2k3BtKi"), @@ -66,31 +85,42 @@ async fn collect_dates() { (DateCase::Dec, "PL1J-6JOckZtHo91uApeb10Qlf2XhkfM-9"), ]; - let mut collected_dates = CollectedDates::new(); + let rp = RustyPipe::new(); + let collected_dates = stream::iter(LANGUAGES) + .map(|lang| { + let rp = rp.clone(); + async move { + let mut map: BTreeMap = BTreeMap::new(); - for lang in LANGUAGES { - let rp = RustyTube::new_with_ua(lang, Country::Us, None); - let mut map: BTreeMap = BTreeMap::new(); + for (case, pl_id) in cases { + let playlist = rp.query().lang(lang).get_playlist(pl_id).await.unwrap(); + map.insert(case, playlist.last_update_txt.unwrap()); + } - for (case, pl_id) in cases { - let playlist = rp.get_playlist(pl_id).await.unwrap(); - map.insert(case, playlist.last_update_txt.unwrap()); - } - - collected_dates.insert(lang, map); - } + (lang, map) + } + }) + .buffer_unordered(concurrency) + .collect::>() + .await; let file = File::create(json_path).unwrap(); serde_json::to_writer_pretty(file, &collected_dates).unwrap(); } -// #[test] -fn write_samples_to_dict() { - let json_path = Path::new("testfiles/date/playlist_samples.json").to_path_buf(); +/// Attempt to parse the dates collected by `collect-playlist-dates` +/// and write the results to `dictionary.json`. +/// +/// The ND (no digit) tokens (today, tomorrow) of some languages cannot be +/// parsed automatically and require manual work. +pub fn write_samples_to_dict(project_root: &Path) { + let mut json_path = project_root.to_path_buf(); + json_path.push("testfiles/date/playlist_samples.json"); + let json_file = File::open(json_path).unwrap(); let collected_dates: CollectedDates = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let mut dict = super::read_dict(); + let mut dict = util::read_dict(project_root); let langs = dict.keys().map(|k| k.to_owned()).collect::>(); let months = [ @@ -134,7 +164,9 @@ fn write_samples_to_dict() { let dict_entry = dict.entry(lang).or_default(); let mut num_order = "".to_owned(); - let collect_nd_tokens = match lang { + let collect_nd_tokens = !matches!( + lang, + // ND tokens of these languages must be edited manually Language::Ja | Language::ZhCn | Language::ZhHk @@ -146,10 +178,9 @@ fn write_samples_to_dict() { | Language::Uz | Language::Te | Language::PtPt - // Singhalese YT translation is broken (today == tomorrow) - | Language::Si => false, - _ => true, - }; + // Singhalese YT translation has an error (today == tomorrow) + | Language::Si + ); dict_entry.months = BTreeMap::new(); @@ -164,7 +195,7 @@ fn write_samples_to_dict() { // Today/Yesterday { let mut parse = |string: &str, n: i8| { - timeago::filter_str(string) + util::filter_datestr(string) .split_whitespace() .for_each(|word| { td_words @@ -183,7 +214,7 @@ fn write_samples_to_dict() { // n days ago { let datestr = datestr_table.get(&DateCase::Ago).unwrap(); - let tago = timeago::parse_timeago(lang, &datestr); + let tago = timeago::parse_timeago(lang, datestr); assert_eq!( tago, Some(TimeAgo { @@ -201,7 +232,7 @@ fn write_samples_to_dict() { let datestr = datestr_table.get(m).unwrap(); // Get order of numbers - let nums = util::parse_numeric_vec::(&datestr); + let nums = util::parse_numeric_vec::(datestr); let date = dates[n]; let this_num_order = nums @@ -219,14 +250,14 @@ fn write_samples_to_dict() { }) .collect::(); - if num_order == "" { + if num_order.is_empty() { num_order = this_num_order; } else { assert_eq!(this_num_order, num_order, "lang: {}", lang); } // Insert words into the map - timeago::filter_str(&datestr) + util::filter_datestr(datestr) .split_whitespace() .for_each(|word| { month_words @@ -275,5 +306,5 @@ fn write_samples_to_dict() { dict_entry.date_order = num_order; } - super::write_dict(&dict); + util::write_dict(project_root, &dict); } diff --git a/codegen/src/download_testfiles.rs b/codegen/src/download_testfiles.rs new file mode 100644 index 0000000..58d4910 --- /dev/null +++ b/codegen/src/download_testfiles.rs @@ -0,0 +1,120 @@ +use std::{ + fs::File, + path::{Path, PathBuf}, +}; + +use rustypipe::{ + client::{ClientType, RustyPipe}, + report::{Report, Reporter}, +}; + +const CLIENT_TYPES: [ClientType; 5] = [ + ClientType::Desktop, + ClientType::DesktopMusic, + ClientType::TvHtml5Embed, + ClientType::Android, + ClientType::Ios, +]; + +/// Store pretty-printed response json +pub struct TestFileReporter { + path: PathBuf, +} + +impl TestFileReporter { + pub fn new>(path: P) -> Self { + Self { + path: path.as_ref().to_path_buf(), + } + } +} + +impl Reporter for TestFileReporter { + fn report(&self, report: &Report) { + let data = + serde_json::from_str::(&report.http_request.resp_body).unwrap(); + let file = File::create(&self.path).unwrap(); + serde_json::to_writer_pretty(file, &data).unwrap(); + + println!("Downloaded {}", self.path.display()); + } +} + +fn rp_testfile(json_path: &Path) -> RustyPipe { + let reporter = TestFileReporter::new(json_path); + RustyPipe::builder() + .reporter(Box::new(reporter)) + .report() + .strict() + .build() +} + +pub async fn download_testfiles(project_root: &Path) { + let mut testfiles = project_root.to_path_buf(); + testfiles.push("testfiles"); + + tokio::join!( + player(&testfiles), + player_model(&testfiles), + playlist(&testfiles) + ); +} + +async fn player(testfiles: &Path) { + let video_id = "pPvd8UxmSbQ"; + + for client_type in CLIENT_TYPES { + let mut json_path = testfiles.to_path_buf(); + json_path.push("player"); + json_path.push(format!("{:?}_video.json", client_type).to_lowercase()); + + if json_path.exists() { + continue; + } + + let rp = rp_testfile(&json_path); + rp.query().get_player(video_id, client_type).await.unwrap(); + } +} + +async fn player_model(testfiles: &Path) { + let rp = RustyPipe::builder().strict().build(); + + for (name, id) in [("multilanguage", "tVWWp1PqDus"), ("hdr", "LXb3EKWsInQ")] { + let mut json_path = testfiles.to_path_buf(); + json_path.push("player_model"); + json_path.push(format!("{}.json", name).to_lowercase()); + + if json_path.exists() { + continue; + } + + let player_data = rp + .query() + .get_player(id, ClientType::Desktop) + .await + .unwrap(); + let file = File::create(&json_path).unwrap(); + serde_json::to_writer_pretty(file, &player_data).unwrap(); + + println!("Downloaded {}", json_path.display()); + } +} + +async fn playlist(testfiles: &Path) { + for (name, id) in [ + ("short", "RDCLAK5uy_kFQXdnqMaQCVx2wpUM4ZfbsGCDibZtkJk"), + ("long", "PL5dDx681T4bR7ZF1IuWzOv1omlRbE7PiJ"), + ("nomusic", "PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe"), + ] { + let mut json_path = testfiles.to_path_buf(); + json_path.push("playlist"); + json_path.push(format!("playlist_{}.json", name)); + if json_path.exists() { + continue; + } + + let rp = rp_testfile(&json_path); + rp.query().get_playlist(id).await.unwrap(); + } +} diff --git a/src/codegen/gen_dictionary.rs b/codegen/src/gen_dictionary.rs similarity index 71% rename from src/codegen/gen_dictionary.rs rename to codegen/src/gen_dictionary.rs index be66a7c..ae86c0f 100644 --- a/src/codegen/gen_dictionary.rs +++ b/codegen/src/gen_dictionary.rs @@ -1,10 +1,13 @@ -#![cfg(test)] +use std::fmt::Write; +use std::path::Path; -use crate::{timeago::TimeUnit}; use fancy_regex::Regex; use once_cell::sync::Lazy; +use rustypipe::timeago::TimeUnit; -const TARGET_FILE: &str = "src/dictionary.rs"; +use crate::util; + +const TARGET_PATH: &str = "src/dictionary.rs"; fn parse_tu(tu: &str) -> (u8, Option) { static TU_PATTERN: Lazy = Lazy::new(|| Regex::new(r"^(\d*)(\w?)$").unwrap()); @@ -27,14 +30,13 @@ fn parse_tu(tu: &str) -> (u8, Option) { } } -// #[test] -fn generate_dictionary() { - let dict = super::read_dict(); +pub fn generate_dictionary(project_root: &Path) { + let dict = util::read_dict(project_root); let code_head = r#"// This file is automatically generated. DO NOT EDIT. use crate::{ model::Language, - timeago::{TaToken, TimeUnit, DateCmp}, + timeago::{DateCmp, TaToken, TimeUnit}, }; pub struct Entry { @@ -56,45 +58,45 @@ pub fn entry(lang: Language) -> Entry { // Match selector let mut selector = format!("Language::{:?}", lang); entry.equivalent.iter().for_each(|eq| { - selector += &format!(" | Language::{:?}", eq); + let _ = write!(selector, " | Language::{:?}", eq); }); // Timeago tokens let mut ta_tokens = phf_codegen::Map::<&str>::new(); entry.timeago_tokens.iter().for_each(|(txt, tu_str)| { - let (n, unit) = parse_tu(&tu_str); + let (n, unit) = parse_tu(tu_str); match unit { Some(unit) => ta_tokens.entry( - &txt, + txt, &format!("TaToken {{ n: {}, unit: Some(TimeUnit::{:?}) }}", n, unit), ), - None => ta_tokens.entry(&txt, &format!("TaToken {{ n: {}, unit: None }}", n)), + None => ta_tokens.entry(txt, &format!("TaToken {{ n: {}, unit: None }}", n)), }; }); // Months let mut months = phf_codegen::Map::<&str>::new(); entry.months.iter().for_each(|(txt, n_mon)| { - months.entry(&txt, &n_mon.to_string()); + months.entry(txt, &n_mon.to_string()); }); // Timeago(ND) tokens let mut ta_nd_tokens = phf_codegen::Map::<&str>::new(); entry.timeago_nd_tokens.iter().for_each(|(txt, tu_str)| { - let (n, unit) = parse_tu(&tu_str); + let (n, unit) = parse_tu(tu_str); match unit { Some(unit) => ta_nd_tokens.entry( - &txt, + txt, &format!("TaToken {{ n: {}, unit: Some(TimeUnit::{:?}) }}", n, unit), ), - None => ta_nd_tokens.entry(&txt, &format!("TaToken {{ n: {}, unit: None }}", n)), + None => ta_nd_tokens.entry(txt, &format!("TaToken {{ n: {}, unit: None }}", n)), }; }); // Date order let mut date_order = "&[".to_owned(); entry.date_order.chars().for_each(|c| { - date_order += &format!("DateCmp::{}, ", c); + let _ = write!(date_order, "DateCmp::{}, ", c); }); date_order = date_order.trim_end_matches([' ', ',']).to_owned() + "]"; @@ -102,15 +104,15 @@ pub fn entry(lang: Language) -> Entry { let code_ta_nd_tokens = &ta_nd_tokens.build().to_string().replace('\n', "\n "); let code_months = &months.build().to_string().replace('\n', "\n "); - code_timeago_tokens += &format!( - "{} => Entry {{\n by_char: {:?},\n timeago_tokens: {},\n date_order: {},\n months: {},\n timeago_nd_tokens: {},\n }},\n ", - selector, entry.by_char, code_ta_tokens, date_order, code_months, code_ta_nd_tokens - ); + let _ = write!(code_timeago_tokens, "{} => Entry {{\n by_char: {:?},\n timeago_tokens: {},\n date_order: {},\n months: {},\n timeago_nd_tokens: {},\n }},\n ", + selector, entry.by_char, code_ta_tokens, date_order, code_months, code_ta_nd_tokens); }); code_timeago_tokens = code_timeago_tokens.trim_end().to_owned() + "\n }\n}\n"; let code = format!("{}\n{}", code_head, code_timeago_tokens); - std::fs::write(TARGET_FILE, code).unwrap(); + let mut target_path = project_root.to_path_buf(); + target_path.push(TARGET_PATH); + std::fs::write(target_path, code).unwrap(); } diff --git a/src/codegen/gen_locales.rs b/codegen/src/gen_locales.rs similarity index 78% rename from src/codegen/gen_locales.rs rename to codegen/src/gen_locales.rs index 1225a05..55094b6 100644 --- a/src/codegen/gen_locales.rs +++ b/codegen/src/gen_locales.rs @@ -1,20 +1,13 @@ -#![cfg(test)] use std::collections::BTreeMap; +use std::fmt::Write; use std::path::Path; -use reqwest::Method; -use serde::{Deserialize, Serialize}; +use reqwest::header; +use reqwest::Client; +use serde::Deserialize; use serde_with::serde_as; use serde_with::VecSkipError; -use crate::client::{ClientType, ContextYT, RustyTube}; - -#[derive(Clone, Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct QLanguageMenu { - context: ContextYT, -} - #[serde_as] #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase")] @@ -122,12 +115,10 @@ struct LanguageItemWrap { compact_link_renderer: LanguageItem, } -#[serde_as] #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct LanguageItem { - #[serde_as(as = "crate::serializer::text::Text")] - title: String, + title: Text, service_endpoint: ServiceEndpoint, } @@ -144,9 +135,13 @@ struct LanguageCountryCommand { hl: String, } -// #[test_log::test(tokio::test)] -#[allow(dead_code)] -async fn generate_locales() { +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Text { + simple_text: String, +} + +pub async fn generate_locales(project_root: &Path) { let (languages, countries) = get_locales().await; let code_head = r#"// This file is automatically generated. DO NOT EDIT. @@ -186,18 +181,21 @@ impl FromStr for Country { } "#; - let mut code_langs = r#"#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] + let mut code_langs = + r#"#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] #[serde(rename_all = "lowercase")] pub enum Language { "#.to_owned(); - let mut code_countries = r#"#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] + let mut code_countries = + r#"#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] #[serde(rename_all = "UPPERCASE")] pub enum Country { "#.to_owned(); let mut code_lang_array = format!("pub const LANGUAGES: [Language; {}] = [\n", languages.len()); - let mut code_country_array = format!("pub const COUNTRIES: [Country; {}] = [\n", countries.len()); + let mut code_country_array = + format!("pub const COUNTRIES: [Country; {}] = [\n", countries.len()); let mut code_lang_names = r#"impl Language { pub fn name(&self) -> &str { @@ -223,18 +221,22 @@ pub enum Country { .collect::(); // Language enum - code_langs += &format!(" /// {}\n ", n); + let _ = write!(code_langs, " /// {}\n ", n); if c.contains('-') { - code_langs += &format!("#[serde(rename = \"{}\")]\n ", c); + let _ = write!(code_langs, "#[serde(rename = \"{}\")]\n ", c); } code_langs += &enum_name; code_langs += ",\n"; // Language array - code_lang_array += &format!(" Language::{},\n", enum_name); + let _ = writeln!(code_lang_array, " Language::{},", enum_name); // Language names - code_lang_names += &format!(" Language::{} => \"{}\",\n", enum_name, n); + let _ = writeln!( + code_lang_names, + " Language::{} => \"{}\",", + enum_name, n + ); }); code_langs += "}\n"; @@ -242,14 +244,18 @@ pub enum Country { let enum_name = c[0..1].to_owned().to_uppercase() + &c[1..].to_owned().to_lowercase(); // Country enum - code_countries += &format!(" /// {}\n", n); - code_countries += &format!(" {},\n", enum_name); + let _ = writeln!(code_countries, " /// {}", n); + let _ = writeln!(code_countries, " {},", enum_name); // Country array - code_country_array += &format!(" Country::{},\n", enum_name); + let _ = writeln!(code_country_array, " Country::{},", enum_name); // Country names - code_country_names += &format!(" Country::{} => \"{}\",\n", enum_name, n); + let _ = writeln!( + code_country_names, + " Country::{} => \"{}\",", + enum_name, n + ); }); code_countries += "}\n"; @@ -267,26 +273,23 @@ pub enum Country { code_country_array, code_lang_names, code_country_names, - code_foot, + code_foot ); - let locale_path = Path::new("src/model/locale.rs"); - std::fs::write(locale_path, code).unwrap(); + let mut target_path = project_root.to_path_buf(); + target_path.push("src/model/locale.rs"); + std::fs::write(target_path, code).unwrap(); } async fn get_locales() -> (BTreeMap, BTreeMap) { - let rt = RustyTube::new(); - let client = rt.get_ytclient(ClientType::Desktop); - let context = client.get_context(true).await; - - let request_body = QLanguageMenu { context }; - + let client = Client::new(); let resp = client - .request_builder(Method::POST, "account/account_menu") - .await - .json(&request_body) - .send() - .await + .post("https://www.youtube.com/youtubei/v1/account/account_menu?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8") + .header(header::CONTENT_TYPE, "application/json") + .body( + r##"{"context":{"client":{"clientName":"WEB","clientVersion":"2.20220914.06.00","platform":"DESKTOP","originalUrl":"https://www.youtube.com/","hl":"en","gl":"US"},"request":{"internalExperimentFlags":[],"useSsl":true},"user":{"lockedSafetyMode":false}}}"## + ) + .send().await .unwrap() .error_for_status() .unwrap(); @@ -344,8 +347,8 @@ fn map_language_section(section: &CompactLinkRendererWrap) -> BTreeMap>() + .collect() } diff --git a/codegen/src/main.rs b/codegen/src/main.rs new file mode 100644 index 0000000..fc1efe0 --- /dev/null +++ b/codegen/src/main.rs @@ -0,0 +1,50 @@ +mod collect_playlist_dates; +mod download_testfiles; +mod gen_dictionary; +mod gen_locales; +mod util; + +use std::path::PathBuf; + +use clap::{Parser, Subcommand}; + +#[derive(Parser)] +struct Cli { + #[clap(subcommand)] + command: Commands, + #[clap(short = 'd', default_value = "..")] + project_root: PathBuf, + #[clap(short, default_value = "8")] + concurrency: usize, +} + +#[derive(Subcommand)] +enum Commands { + CollectPlaylistDates, + WritePlaylistDates, + GenLocales, + GenDict, + DownloadTestfiles, +} + +#[tokio::main] +async fn main() { + env_logger::init(); + let cli = Cli::parse(); + + match cli.command { + Commands::CollectPlaylistDates => { + collect_playlist_dates::collect_dates(&cli.project_root, cli.concurrency).await; + } + Commands::WritePlaylistDates => { + collect_playlist_dates::write_samples_to_dict(&cli.project_root); + } + Commands::GenLocales => { + gen_locales::generate_locales(&cli.project_root).await; + } + Commands::GenDict => gen_dictionary::generate_dictionary(&cli.project_root), + Commands::DownloadTestfiles => { + download_testfiles::download_testfiles(&cli.project_root).await + } + }; +} diff --git a/codegen/src/util.rs b/codegen/src/util.rs new file mode 100644 index 0000000..5925322 --- /dev/null +++ b/codegen/src/util.rs @@ -0,0 +1,72 @@ +use std::{collections::BTreeMap, fs::File, io::BufReader, path::Path, str::FromStr}; + +use rustypipe::model::Language; +use serde::{Deserialize, Serialize}; + +const DICT_PATH: &str = "testfiles/date/dictionary.json"; + +type Dictionary = BTreeMap; + +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(default)] +pub struct DictEntry { + pub equivalent: Vec, + pub by_char: bool, + pub timeago_tokens: BTreeMap, + pub date_order: String, + pub months: BTreeMap, + pub timeago_nd_tokens: BTreeMap, +} + +pub fn read_dict(project_root: &Path) -> Dictionary { + let mut json_path = project_root.to_path_buf(); + json_path.push(DICT_PATH); + let json_file = File::open(json_path).unwrap(); + serde_json::from_reader(BufReader::new(json_file)).unwrap() +} + +pub fn write_dict(project_root: &Path, dict: &Dictionary) { + let mut json_path = project_root.to_path_buf(); + json_path.push(DICT_PATH); + let json_file = File::create(json_path).unwrap(); + serde_json::to_writer_pretty(json_file, dict).unwrap(); +} + +pub fn filter_datestr(string: &str) -> String { + string + .to_lowercase() + .chars() + .filter_map(|c| { + if c == '\u{200b}' || c.is_ascii_digit() { + None + } else if c == '-' { + Some(' ') + } else { + Some(c) + } + }) + .collect() +} + +/// Parse all numbers occurring in a string and reurn them as a vec +pub fn parse_numeric_vec(string: &str) -> Vec +where + F: FromStr, +{ + let mut numbers = vec![]; + + let mut buf = String::new(); + for c in string.chars() { + if c.is_ascii_digit() { + buf.push(c); + } else if !buf.is_empty() { + buf.parse::().map_or((), |n| numbers.push(n)); + buf.clear(); + } + } + if !buf.is_empty() { + buf.parse::().map_or((), |n| numbers.push(n)); + } + + numbers +} diff --git a/src/cache.rs b/src/cache.rs index 509f34b..e3339d3 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -1,364 +1,57 @@ use std::{ - fs::File, - future::Future, - io::BufReader, + fs, path::{Path, PathBuf}, - sync::Arc, }; -use anyhow::Result; -use chrono::{DateTime, Duration, Utc}; -use log::{error, info}; -use serde::{Deserialize, Serialize}; -use tokio::sync::Mutex; +use log::error; -#[derive(Default, Debug, Clone)] -pub struct Cache { - file: Option, - data: Arc>, +pub trait CacheStorage { + fn write(&self, data: &str); + fn read(&self) -> Option; } -#[derive(Default, Debug, Clone, Serialize, Deserialize)] -struct CacheData { - desktop_client: Option>, - music_client: Option>, - deobf: Option>, +pub struct FileStorage { + path: PathBuf, } -#[derive(Debug, Clone, Serialize, Deserialize)] -struct CacheEntry { - last_update: DateTime, - data: T, -} - -impl From for CacheEntry { - fn from(f: T) -> Self { +impl FileStorage { + pub fn new>(path: P) -> Self { Self { - last_update: Utc::now(), - data: f, + path: path.as_ref().to_path_buf(), } } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct ClientData { - pub version: String, +impl Default for FileStorage { + fn default() -> Self { + Self { + path: Path::new("rustypipe_cache.json").into(), + } + } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct DeobfData { - pub js_url: String, - pub sig_fn: String, - pub nsig_fn: String, - pub sts: String, -} - -impl Cache { - pub async fn get_desktop_client_data(&self, updater: F) -> Result - where - F: Future> + Send + 'static, - { - let mut cache = self.data.lock().await; - - if cache.desktop_client.is_none() - || cache.desktop_client.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24) - { - let cdata = updater.await?; - cache.desktop_client = Some(CacheEntry::from(cdata.clone())); - self.save(&cache); - Ok(cdata) - } else { - Ok(cache.desktop_client.as_ref().unwrap().data.clone()) - } +impl CacheStorage for FileStorage { + fn write(&self, data: &str) { + fs::write(&self.path, data).unwrap_or_else(|e| { + error!( + "Could not write cache to file `{}`. Error: {}", + self.path.to_string_lossy(), + e + ); + }); } - pub async fn get_music_client_data(&self, updater: F) -> Result - where - F: Future> + Send + 'static, - { - let mut cache = self.data.lock().await; - - if cache.music_client.is_none() - || cache.music_client.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24) - { - let cdata = updater.await?; - cache.music_client = Some(CacheEntry::from(cdata.clone())); - self.save(&cache); - Ok(cdata) - } else { - Ok(cache.music_client.as_ref().unwrap().data.clone()) - } - } - - pub async fn get_deobf_data(&self, updater: F) -> Result - where - F: Future> + Send + 'static, - { - let mut cache = self.data.lock().await; - if cache.deobf.is_none() - || cache.deobf.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24) - { - let deobf_data = updater.await?; - cache.deobf = Some(CacheEntry::from(deobf_data.clone())); - self.save(&cache); - Ok(deobf_data) - } else { - Ok(cache.deobf.as_ref().unwrap().data.clone()) - } - } - - pub async fn to_json(&self) -> Result { - let cache = self.data.lock().await; - Ok(serde_json::to_string(&cache.clone())?) - } - - pub async fn to_json_file>(&self, path: P) -> Result<()> { - let cache = self.data.lock().await; - Ok(serde_json::to_writer(&File::create(path)?, &cache.clone())?) - } - - pub fn from_json(json: &str) -> Self { - let data: CacheData = match serde_json::from_str(json) { - Ok(cd) => cd, + fn read(&self) -> Option { + match fs::read_to_string(&self.path) { + Ok(data) => Some(data), Err(e) => { error!( - "Could not load cache from json, falling back to default. Error: {}", + "Could not load cache from file `{}`. Error: {}", + self.path.to_string_lossy(), e ); - CacheData::default() + None } - }; - Cache { - data: Arc::new(Mutex::new(data)), - file: None, - } - } - - pub fn from_json_file>(path: P) -> Self { - let file = match File::open(path.as_ref()) { - Ok(file) => file, - Err(e) => { - if e.kind() == std::io::ErrorKind::NotFound { - info!( - "Cache json file at {} not found, will be created", - path.as_ref().to_string_lossy() - ) - } else { - error!( - "Could not open cache json file, falling back to default. Error: {}", - e - ); - } - return Cache { - file: Some(path.as_ref().to_path_buf()), - ..Default::default() - }; - } - }; - let data: CacheData = match serde_json::from_reader(BufReader::new(file)) { - Ok(data) => data, - Err(e) => { - error!( - "Could not load cache from json, falling back to default. Error: {}", - e - ); - return Cache { - file: Some(path.as_ref().to_path_buf()), - ..Default::default() - }; - } - }; - Cache { - data: Arc::new(Mutex::new(data)), - file: Some(path.as_ref().to_path_buf()), - } - } - - fn save(&self, cache: &CacheData) { - match self.file.as_ref() { - Some(file) => match File::create(file) { - Ok(file) => match serde_json::to_writer(file, cache) { - Ok(_) => {} - Err(e) => error!("Could not write cache to json. Error: {}", e), - }, - Err(e) => error!("Could not open cache json file. Error: {}", e), - }, - None => {} } } } - -#[cfg(test)] -mod tests { - use temp_testdir::TempDir; - - use super::*; - - #[tokio::test] - async fn test() { - let cache = Cache::default(); - - let desktop_c = cache - .get_desktop_client_data(async { - Ok(ClientData { - version: "1.2.3".to_owned(), - }) - }) - .await - .unwrap(); - - assert_eq!( - desktop_c, - ClientData { - version: "1.2.3".to_owned() - } - ); - - let music_c = cache - .get_music_client_data(async { - Ok(ClientData { - version: "4.5.6".to_owned(), - }) - }) - .await - .unwrap(); - - assert_eq!( - music_c, - ClientData { - version: "4.5.6".to_owned() - } - ); - - let deobf_data = cache - .get_deobf_data(async { - Ok(DeobfData { - js_url: - "https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js" - .to_owned(), - sig_fn: "t_sig_fn".to_owned(), - nsig_fn: "t_nsig_fn".to_owned(), - sts: "t_sts".to_owned(), - }) - }) - .await - .unwrap(); - - assert_eq!( - deobf_data, - DeobfData { - js_url: "https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js" - .to_owned(), - sig_fn: "t_sig_fn".to_owned(), - nsig_fn: "t_nsig_fn".to_owned(), - sts: "t_sts".to_owned(), - } - ); - - // Create a new cache from the first one's json - // and check if it returns the same cached data - let json = cache.to_json().await.unwrap(); - let new_cache = Cache::from_json(&json); - - assert_eq!( - new_cache - .get_desktop_client_data(async { - Ok(ClientData { - version: "".to_owned(), - }) - }) - .await - .unwrap(), - desktop_c - ); - - assert_eq!( - new_cache - .get_music_client_data(async { - Ok(ClientData { - version: "".to_owned(), - }) - }) - .await - .unwrap(), - music_c - ); - - assert_eq!( - new_cache - .get_deobf_data(async { - Ok(DeobfData { - js_url: "".to_owned(), - nsig_fn: "".to_owned(), - sig_fn: "".to_owned(), - sts: "".to_owned(), - }) - }) - .await - .unwrap(), - deobf_data - ); - } - - #[tokio::test] - async fn test_file() { - let temp = TempDir::default(); - let mut file_path = PathBuf::from(temp.as_ref()); - file_path.push("cache.json"); - - let cache = Cache::from_json_file(file_path.clone()); - - let cdata = cache - .get_desktop_client_data(async { - Ok(ClientData { - version: "1.2.3".to_owned(), - }) - }) - .await - .unwrap(); - - let deobf_data = cache - .get_deobf_data(async { - Ok(DeobfData { - js_url: - "https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js" - .to_owned(), - sig_fn: "t_sig_fn".to_owned(), - nsig_fn: "t_nsig_fn".to_owned(), - sts: "t_sts".to_owned(), - }) - }) - .await - .unwrap(); - - assert!(file_path.exists()); - let new_cache = Cache::from_json_file(file_path.clone()); - - assert_eq!( - new_cache - .get_desktop_client_data(async { - Ok(ClientData { - version: "".to_owned(), - }) - }) - .await - .unwrap(), - cdata - ); - - assert_eq!( - new_cache - .get_deobf_data(async { - Ok(DeobfData { - js_url: "".to_owned(), - nsig_fn: "".to_owned(), - sig_fn: "".to_owned(), - sts: "".to_owned(), - }) - }) - .await - .unwrap(), - deobf_data - ); - } -} diff --git a/src/client/channel.rs b/src/client/channel.rs deleted file mode 100644 index 3f1ec45..0000000 --- a/src/client/channel.rs +++ /dev/null @@ -1,41 +0,0 @@ -use anyhow::Result; -use reqwest::Method; -use serde::Serialize; - -use super::{response, ClientType, ContextYT, RustyTube}; - -#[derive(Clone, Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct QChannel { - context: ContextYT, - browse_id: String, - params: String, -} - -impl RustyTube { - async fn get_channel_response(&self, channel_id: &str) -> Result { - let client = self.get_ytclient(ClientType::Desktop); - let context = client.get_context(true).await; - - let request_body = QChannel { - context, - browse_id: channel_id.to_owned(), - params: "EgZ2aWRlb3PyBgQKAjoA".to_owned(), - }; - - let resp = client - .request_builder(Method::POST, "browse") - .await - .json(&request_body) - .send() - .await? - .error_for_status()?; - - Ok(resp.json::().await?) - } -} - -#[cfg(test)] -mod tests { - -} diff --git a/src/client/mod.rs b/src/client/mod.rs index adc1f88..4db6509 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1,34 +1,43 @@ pub mod player; pub mod playlist; -pub mod video; mod response; +use std::fmt::Debug; use std::sync::Arc; -use anyhow::{anyhow, Context, Result}; -use async_trait::async_trait; +use anyhow::{anyhow, bail, Context, Result}; +use chrono::{DateTime, Duration, Utc}; use fancy_regex::Regex; -use log::warn; +use log::{error, warn}; use once_cell::sync::Lazy; use rand::Rng; -use reqwest::{header, Client, ClientBuilder, Method, Request, RequestBuilder, Response}; -use serde::{Deserialize, Serialize}; +use reqwest::{ + header, Client, ClientBuilder, Method, Request, RequestBuilder, Response, StatusCode, +}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use tokio::sync::Mutex; use crate::{ - cache::{Cache, ClientData}, + cache::{CacheStorage, FileStorage}, + deobfuscate::{DeobfData, Deobfuscator}, model::{Country, Language}, + report::{FileReporter, Level, Report, Reporter}, + serializer::MapResult, util, }; -pub const CLIENT_TYPES: [ClientType; 5] = [ - ClientType::Desktop, - ClientType::DesktopMusic, - ClientType::TvHtml5Embed, - ClientType::Android, - ClientType::Ios, -]; - +/// Client types for accessing the YouTube API. +/// +/// There are multiple clients for accessing the YouTube API which have +/// slightly different features +/// +/// - **Desktop**: used by youtube.com +/// - **DesktopMusic**: used by music.youtube.com, can access special music data, +/// cannot access non-music content +/// - **TvHtml5Embed**: (probably) used by Smart TVs, can access age-restricted videos +/// - **Android**: used by the Android app, no obfuscated URLs, includes lower resolution audio streams +/// - **Ios**: used by the iOS app, no obfuscated URLs #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] #[serde(rename_all = "snake_case")] pub enum ClientType { @@ -39,15 +48,33 @@ pub enum ClientType { Ios, } +const CLIENT_TYPES: [ClientType; 5] = [ + ClientType::Desktop, + ClientType::DesktopMusic, + ClientType::TvHtml5Embed, + ClientType::Android, + ClientType::Ios, +]; + impl ClientType { - pub fn is_web(self) -> bool { - self == Self::Desktop || self == Self::DesktopMusic || self == Self::TvHtml5Embed + fn is_web(&self) -> bool { + match self { + ClientType::Desktop | ClientType::DesktopMusic | ClientType::TvHtml5Embed => true, + ClientType::Android | ClientType::Ios => false, + } } } +#[derive(Clone, Debug, Serialize)] +struct YTQuery { + context: YTContext, + #[serde(flatten)] + data: T, +} + #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] -pub struct ContextYT { +struct YTContext { client: ClientInfo, /// only used on desktop #[serde(skip_serializing_if = "Option::is_none")] @@ -93,7 +120,7 @@ impl Default for RequestYT { #[derive(Clone, Debug, Serialize, Default)] #[serde(rename_all = "camelCase")] struct User { - // TO DO: provide a way to enable restricted mode with: + // TODO: provide a way to enable restricted mode with: // "enableSafetyMode": true locked_safety_mode: bool, } @@ -104,8 +131,7 @@ struct ThirdParty { embed_url: String, } -const DEFAULT_UA: &str = - "Mozilla/5.0 (Windows NT 10.0; Win64; rv:107.0) Gecko/20100101 Firefox/107.0"; +const DEFAULT_UA: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0"; const CONSENT_COOKIE: &str = "CONSENT"; const CONSENT_COOKIE_YES: &str = "YES+yt.462272069.de+FX+"; @@ -130,635 +156,912 @@ const IOS_DEVICE_MODEL: &str = "iPhone14,5"; static CLIENT_VERSION_REGEXES: Lazy<[Regex; 1]> = Lazy::new(|| [Regex::new("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"").unwrap()]); -pub struct RustyTube { - localization: Arc, - cache: Cache, - desktop_client: Arc, - desktop_music_client: Arc, - android_client: Arc, - ios_client: Arc, - tvhtml5embed_client: Arc, +/// The RustyPipe client used to access YouTube's API +/// +/// RustyPipe includes an `Arc` internally, so if you are using the client +/// at multiple locations, you can just clone it. Note that options (lang/country/report) +/// are not shared between clones. +#[derive(Clone)] +pub struct RustyPipe { + inner: Arc, } -struct Localization { - language: Language, - content_country: Country, -} - -impl RustyTube { - #[must_use] - pub fn new() -> Self { - Self::new_with_ua( - Language::En, - Country::Us, - Some("rusty-tube.json".to_owned()), - ) - } - - #[must_use] - pub fn new_with_ua( - language: Language, - content_country: Country, - cache_file: Option, - ) -> Self { - let loc = Arc::new(Localization { - language, - content_country, - }); - - let cache = match cache_file.as_ref() { - Some(cache_file) => Cache::from_json_file(cache_file), - None => Cache::default(), - }; - - Self { - localization: loc.clone(), - cache: cache.clone(), - desktop_client: Arc::new(DesktopClient::new(loc.clone(), cache.clone())), - desktop_music_client: Arc::new(DesktopMusicClient::new(loc.clone(), cache)), - android_client: Arc::new(AndroidClient::new(loc.clone())), - ios_client: Arc::new(IosClient::new(loc.clone())), - tvhtml5embed_client: Arc::new(TvHtml5EmbedClient::new(loc)), - } - } - - pub(crate) fn get_ytclient(&self, client_type: ClientType) -> Arc { - match client_type { - ClientType::Desktop => self.desktop_client.clone(), - ClientType::DesktopMusic => self.desktop_music_client.clone(), - ClientType::TvHtml5Embed => self.tvhtml5embed_client.clone(), - ClientType::Android => self.android_client.clone(), - ClientType::Ios => self.ios_client.clone(), - } - } -} - -#[async_trait] -pub trait YTClient { - async fn get_context(&self, localized: bool) -> ContextYT; - async fn request_builder(&self, method: Method, url: &str) -> RequestBuilder; - fn http_client(&self) -> Client; - fn get_type(&self) -> ClientType; -} - -async fn exec_request(http: Client, request: Request) -> Result { - Ok(http.execute(request).await?.error_for_status()?) -} - -async fn exec_request_text(http: Client, request: Request) -> Result { - Ok(exec_request(http, request).await?.text().await?) -} - -pub struct DesktopClient { - localization: Arc, +struct RustyPipeRef { http: Client, - cache: Cache, + storage: Option>, + reporter: Option>, + n_retries: u32, + user_agent: String, consent_cookie: String, + cache: Mutex, + default_opts: RustyPipeOpts, } -#[async_trait] -impl YTClient for DesktopClient { - async fn get_context(&self, localized: bool) -> ContextYT { - ContextYT { - client: ClientInfo { - client_name: "WEB".to_owned(), - client_version: self.get_client_version().await, - client_screen: None, - device_model: None, - platform: "DESKTOP".to_owned(), - original_url: Some("https://www.youtube.com/".to_owned()), - hl: match localized { - true => self.localization.language, - false => Language::En, - }, - gl: match localized { - true => self.localization.content_country, - false => Country::Us, - }, - }, - request: Some(RequestYT::default()), - user: User::default(), - third_party: None, +#[derive(Clone)] +struct RustyPipeOpts { + lang: Language, + country: Country, + report: bool, + strict: bool, +} + +pub struct RustyPipeBuilder { + storage: Option>, + reporter: Option>, + n_retries: u32, + user_agent: String, + default_opts: RustyPipeOpts, +} + +#[derive(Clone)] +pub struct RustyPipeQuery { + client: RustyPipe, + opts: RustyPipeOpts, +} + +impl Default for RustyPipeOpts { + fn default() -> Self { + Self { + lang: Language::En, + country: Country::Us, + report: false, + strict: false, + } + } +} + +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +struct CacheData { + desktop_client: CacheEntry, + music_client: CacheEntry, + deobf: CacheEntry, +} + +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +enum CacheEntry { + #[default] + None, + Some { + last_update: DateTime, + data: T, + }, +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ClientData { + pub version: String, +} + +impl CacheEntry { + fn get(&self) -> Option<&T> { + match self { + CacheEntry::Some { last_update, data } => { + if last_update < &(Utc::now() - Duration::hours(24)) { + None + } else { + Some(data) + } + } + CacheEntry::None => None, + } + } +} + +impl From for CacheEntry { + fn from(f: T) -> Self { + Self::Some { + last_update: Utc::now(), + data: f, + } + } +} + +impl Default for RustyPipeBuilder { + fn default() -> Self { + Self::new() + } +} + +impl RustyPipeBuilder { + /// Constructs a new `RustyPipeBuilder`. + /// + /// This is the same as `RustyPipe::builder()` + pub fn new() -> Self { + RustyPipeBuilder { + default_opts: RustyPipeOpts::default(), + storage: Some(Box::new(FileStorage::default())), + reporter: Some(Box::new(FileReporter::default())), + n_retries: 3, + user_agent: DEFAULT_UA.to_owned(), } } - async fn request_builder(&self, method: Method, endpoint: &str) -> RequestBuilder { - self.http - .request( - method, - format!( - "{}{}?key={}{}", - YOUTUBEI_V1_URL, endpoint, DESKTOP_API_KEY, DISABLE_PRETTY_PRINT_PARAMETER - ), - ) - .header(header::ORIGIN, "https://www.youtube.com") - .header(header::REFERER, "https://www.youtube.com") - .header(header::COOKIE, self.consent_cookie.to_owned()) - .header("X-YouTube-Client-Name", "1") - .header("X-YouTube-Client-Version", self.get_client_version().await) - } - - fn http_client(&self) -> Client { - self.http.clone() - } - - fn get_type(&self) -> ClientType { - ClientType::Desktop - } -} - -impl DesktopClient { - fn new(localization: Arc, cache: Cache) -> Self { - let mut rng = rand::thread_rng(); - + /// Returns a new, configured RustyPipe instance. + pub fn build(self) -> RustyPipe { let http = ClientBuilder::new() - .user_agent(DEFAULT_UA) + .user_agent(self.user_agent.to_owned()) .gzip(true) .brotli(true) .build() - .expect("unable to build the HTTP client"); + .unwrap(); - Self { - localization, - http, - cache, - consent_cookie: format!( - "{}={}{}", - CONSENT_COOKIE, - CONSENT_COOKIE_YES, - rng.gen_range(100..1000) - ), - } - } - - async fn extract_client_version(http: Client, consent_cookie: &str) -> Result { - async fn extract_client_version_from_swjs( - http: Client, - consent_cookie: &str, - ) -> Result { - let swjs = exec_request_text( - http.clone(), - http.get("https://www.youtube.com/sw.js") - .header(header::ORIGIN, "https://www.youtube.com") - .header(header::REFERER, "https://www.youtube.com") - .header(header::COOKIE, consent_cookie) - .build() - .unwrap(), - ) - .await - .context("Failed to download sw.js")?; - - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) - .ok_or(anyhow!("Could not find desktop client version in sw.js")) - } - - async fn extract_client_version_from_html(http: Client) -> Result { - let html = exec_request_text( - http.clone(), - http.get("https://www.youtube.com/results?search_query=") - .build() - .unwrap(), - ) - .await - .context("Failed to get YT Desktop page")?; - - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(anyhow!( - "Could not find desktop client version on html page" - )) - } - - match extract_client_version_from_swjs(http.clone(), consent_cookie).await { - Ok(client_version) => Ok(client_version), - Err(_) => extract_client_version_from_html(http).await, - } - } - - async fn get_client_version(&self) -> String { - let http = self.http.clone(); - let consent_cookie = self.consent_cookie.clone(); - - let client_data = self - .cache - .get_desktop_client_data(async move { - let client_version = Self::extract_client_version(http, &consent_cookie).await?; - Ok(ClientData { - version: client_version, - }) - }) - .await; - - match client_data { - Ok(client_data) => client_data.version, - Err(e) => { - warn!("{}", e); - DESKTOP_CLIENT_VERSION.to_owned() + let cache = if let Some(storage) = &self.storage { + if let Some(data) = storage.read() { + match serde_json::from_str::(&data) { + Ok(data) => data, + Err(e) => { + error!("Could not deserialize cache. Error: {}", e); + CacheData::default() + } + } + } else { + CacheData::default() } - } - } -} + } else { + CacheData::default() + }; -pub struct AndroidClient { - localization: Arc, - http: Client, -} - -#[async_trait] -impl YTClient for AndroidClient { - async fn get_context(&self, localized: bool) -> ContextYT { - ContextYT { - client: ClientInfo { - client_name: "ANDROID".to_owned(), - client_version: MOBILE_CLIENT_VERSION.to_owned(), - client_screen: None, - device_model: None, - platform: "MOBILE".to_owned(), - original_url: None, - hl: match localized { - true => self.localization.language, - false => Language::En, - }, - gl: match localized { - true => self.localization.content_country, - false => Country::Us, - }, - }, - request: None, - user: User::default(), - third_party: None, - } - } - - async fn request_builder(&self, method: Method, endpoint: &str) -> RequestBuilder { - self.http - .request( - method, - format!( - "{}{}?key={}{}", - YOUTUBEI_V1_GAPIS_URL, - endpoint, - ANDROID_API_KEY, - DISABLE_PRETTY_PRINT_PARAMETER + RustyPipe { + inner: Arc::new(RustyPipeRef { + http, + storage: self.storage, + reporter: self.reporter, + n_retries: self.n_retries, + user_agent: self.user_agent, + consent_cookie: format!( + "{}={}{}", + CONSENT_COOKIE, + CONSENT_COOKIE_YES, + rand::thread_rng().gen_range(100..1000) ), - ) - .header("X-Goog-Api-Format-Version", "2") - } - - fn http_client(&self) -> Client { - self.http.clone() - } - - fn get_type(&self) -> ClientType { - ClientType::Android - } -} - -impl AndroidClient { - fn new(localization: Arc) -> Self { - let http = ClientBuilder::new() - .user_agent(format!( - "com.google.android.youtube/{} (Linux; U; Android 12; {}) gzip", - MOBILE_CLIENT_VERSION, localization.content_country - )) - .gzip(true) - .build() - .expect("unable to build the HTTP client"); - - Self { localization, http } - } -} - -pub struct IosClient { - localization: Arc, - http: Client, -} - -#[async_trait] -impl YTClient for IosClient { - async fn get_context(&self, localized: bool) -> ContextYT { - ContextYT { - client: ClientInfo { - client_name: "IOS".to_owned(), - client_version: MOBILE_CLIENT_VERSION.to_owned(), - client_screen: None, - device_model: Some(IOS_DEVICE_MODEL.to_owned()), - platform: "MOBILE".to_owned(), - original_url: None, - hl: match localized { - true => self.localization.language, - false => Language::En, - }, - gl: match localized { - true => self.localization.content_country, - false => Country::Us, - }, - }, - request: None, - user: User::default(), - third_party: None, - } - } - - async fn request_builder(&self, method: Method, endpoint: &str) -> RequestBuilder { - self.http - .request( - method, - format!( - "{}{}?key={}{}", - YOUTUBEI_V1_GAPIS_URL, endpoint, IOS_API_KEY, DISABLE_PRETTY_PRINT_PARAMETER - ), - ) - .header("X-Goog-Api-Format-Version", "2") - } - - fn http_client(&self) -> Client { - self.http.clone() - } - - fn get_type(&self) -> ClientType { - ClientType::Ios - } -} - -impl IosClient { - fn new(localization: Arc) -> Self { - let http = ClientBuilder::new() - .user_agent(format!( - "com.google.ios.youtube/{} ({}; U; CPU iOS 15_4 like Mac OS X; {})", - MOBILE_CLIENT_VERSION, IOS_DEVICE_MODEL, localization.content_country - )) - .gzip(true) - .build() - .expect("unable to build the HTTP client"); - - Self { localization, http } - } -} - -pub struct TvHtml5EmbedClient { - localization: Arc, - http: Client, -} - -#[async_trait] -impl YTClient for TvHtml5EmbedClient { - async fn get_context(&self, localized: bool) -> ContextYT { - ContextYT { - client: ClientInfo { - client_name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER".to_owned(), - client_version: TVHTML5_CLIENT_VERSION.to_owned(), - client_screen: Some("EMBED".to_owned()), - device_model: None, - platform: "TV".to_owned(), - original_url: None, - hl: match localized { - true => self.localization.language, - false => Language::En, - }, - gl: match localized { - true => self.localization.content_country, - false => Country::Us, - }, - }, - request: Some(RequestYT::default()), - user: User::default(), - third_party: Some(ThirdParty { - embed_url: "https://www.youtube.com/".to_owned(), + cache: Mutex::new(cache), + default_opts: RustyPipeOpts::default(), }), } } - async fn request_builder(&self, method: Method, endpoint: &str) -> RequestBuilder { - self.http - .request( - method, - format!( - "{}{}?key={}{}", - YOUTUBEI_V1_URL, endpoint, DESKTOP_API_KEY, DISABLE_PRETTY_PRINT_PARAMETER - ), - ) - .header(header::ORIGIN, "https://www.youtube.com") - .header(header::REFERER, "https://www.youtube.com") - .header("X-YouTube-Client-Name", "1") - .header("X-YouTube-Client-Version", TVHTML5_CLIENT_VERSION) + /// Add a `CacheStorage` backend for persisting cached information + /// (YouTube client versions, deobfuscation code) between + /// program executions. + /// + /// **Default value**: `FileStorage` in `rustypipe_cache.json` + pub fn storage(mut self, storage: Box) -> Self { + self.storage = Some(storage); + self } - fn http_client(&self) -> Client { - self.http.clone() + /// Disable cache storage + pub fn no_storage(mut self) -> Self { + self.storage = None; + self } - fn get_type(&self) -> ClientType { - ClientType::TvHtml5Embed + /// Add a `Reporter` to collect error details + /// + /// **Default value**: `FileReporter` creating reports in `./rustypipe_reports` + pub fn reporter(mut self, reporter: Box) -> Self { + self.reporter = Some(reporter); + self + } + + /// Disable the creation of report files in case of errors and warnings. + pub fn no_reporter(mut self) -> Self { + self.reporter = None; + self + } + + /// Set the number of retries for HTTP requests. + /// + /// If a HTTP requests fails and retries are enabled, + /// RustyPipe waits 1 second before the next attempt. + /// The waiting time is doubled for subsequent attempts (including a bit of + /// random jitter to be less predictable). + /// + /// **Default value**: 3 + pub fn n_retries(mut self, n_retries: u32) -> Self { + self.n_retries = n_retries; + self + } + + /// Set the user agent used for making requests to the web API. + /// + /// **Default value**: `Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0` + /// (Firefox ESR on Debian) + pub fn user_agent(mut self, user_agent: &str) -> Self { + self.user_agent = user_agent.to_owned(); + self + } + + /// Set the language parameter used when accessing the YouTube API. + /// This will change multilanguage video titles, descriptions and textual dates + /// + /// **Default value**: `Language::En` (English) + /// + /// **Info**: you can set this option for individual queries, too + pub fn lang(mut self, lang: Language) -> Self { + self.default_opts.lang = lang; + self + } + + /// Set the country parameter used when accessing the YouTube API. + /// This will change trends and recommended content. + /// + /// **Default value**: `Country::Us` (USA) + /// + /// **Info**: you can set this option for individual queries, too + pub fn country(mut self, country: Country) -> Self { + self.default_opts.country = country; + self + } + + /// Generate a report on every operation. + /// This should only be used for debugging. + /// + /// **Info**: you can set this option for individual queries, too + pub fn report(mut self) -> Self { + self.default_opts.report = true; + self + } + + /// Enable strict mode, causing operations to fail if there + /// are warnings during deserialization (e.g. invalid items). + /// This should only be used for testing. + /// + /// **Info**: you can set this option for individual queries, too + pub fn strict(mut self) -> Self { + self.default_opts.strict = true; + self } } -impl TvHtml5EmbedClient { - fn new(localization: Arc) -> Self { - let http = ClientBuilder::new() - .user_agent(DEFAULT_UA) - .gzip(true) - .brotli(true) - .build() - .expect("unable to build the HTTP client"); - - Self { localization, http } +impl Default for RustyPipe { + fn default() -> Self { + Self::new() } } -pub struct DesktopMusicClient { - localization: Arc, - http: Client, - cache: Cache, - consent_cookie: String, -} +impl RustyPipe { + /// Create a new RustyPipe instance with default settings. + /// + /// To create an instance with custom options, use `RustyPipeBuilder` instead. + pub fn new() -> Self { + RustyPipeBuilder::new().build() + } -#[async_trait] -impl YTClient for DesktopMusicClient { - async fn get_context(&self, localized: bool) -> ContextYT { - ContextYT { - client: ClientInfo { - client_name: "WEB_REMIX".to_owned(), - client_version: self.get_client_version().await, - client_screen: None, - device_model: None, - platform: "DESKTOP".to_owned(), - original_url: Some("https://music.youtube.com/".to_owned()), - hl: match localized { - true => self.localization.language, - false => Language::En, - }, - gl: match localized { - true => self.localization.content_country, - false => Country::Us, - }, - }, - request: Some(RequestYT::default()), - user: User::default(), - third_party: None, + /// Constructs a new `RustyPipeBuilder`. + /// + /// This is the same as `RustyPipeBuilder::new()` + pub fn builder() -> RustyPipeBuilder { + RustyPipeBuilder::new() + } + + /// Constructs a new `RustyPipeQuery`. + pub fn query(&self) -> RustyPipeQuery { + RustyPipeQuery { + client: self.clone(), + opts: self.inner.default_opts.clone(), } } - async fn request_builder(&self, method: Method, endpoint: &str) -> RequestBuilder { - self.http - .request( - method, - format!( - "{}{}?key={}{}", - YOUTUBE_MUSIC_V1_URL, - endpoint, - DESKTOP_MUSIC_API_KEY, - DISABLE_PRETTY_PRINT_PARAMETER - ), - ) - .header(header::ORIGIN, "https://music.youtube.com") - .header(header::REFERER, "https://music.youtube.com") - .header(header::COOKIE, self.consent_cookie.to_owned()) - .header("X-YouTube-Client-Name", "67") - .header("X-YouTube-Client-Version", self.get_client_version().await) - } + /// Execute the given http request. + async fn http_request(&self, request: Request) -> Result { + let mut last_res: Option> = None; + for n in 0..self.inner.n_retries { + let res = self.inner.http.execute(request.try_clone().unwrap()).await; + let emsg = match &res { + Ok(response) => { + let status = response.status(); + // Immediately return in case of success or unrecoverable status code + if status.is_success() || !status.is_server_error() { + return res; + } + status.to_string() + } + Err(e) => { + // Immediately return in case of unrecoverable error + if !e.is_timeout() && !e.is_connect() { + return res; + } + e.to_string() + } + }; - fn http_client(&self) -> Client { - self.http.clone() - } + let ms = util::retry_delay(n, 1000, 60000, 3); + warn!("Retry attempt #{}. Error: {}. Waiting {} ms", n, emsg, ms); + tokio::time::sleep(std::time::Duration::from_millis(ms.into())).await; - fn get_type(&self) -> ClientType { - ClientType::DesktopMusic - } -} - -impl DesktopMusicClient { - fn new(localization: Arc, cache: Cache) -> Self { - let mut rng = rand::thread_rng(); - - let http = ClientBuilder::new() - .user_agent(DEFAULT_UA) - .gzip(true) - .brotli(true) - .build() - .expect("unable to build the HTTP client"); - - Self { - localization, - http, - cache, - consent_cookie: format!( - "{}={}{}", - CONSENT_COOKIE, - CONSENT_COOKIE_YES, - rng.gen_range(100..1000) - ), + last_res = Some(res); } + last_res.unwrap() } - async fn extract_client_version(http: Client, consent_cookie: &str) -> Result { - async fn extract_client_version_from_swjs( - http: Client, - consent_cookie: &str, - ) -> Result { - let swjs = exec_request_text( - http.clone(), - http.get("https://music.youtube.com/sw.js") - .header(header::ORIGIN, "https://www.youtube.com") - .header(header::REFERER, "https://www.youtube.com") - .header(header::COOKIE, consent_cookie) - .build() - .unwrap(), - ) - .await - .context("Failed to download sw.js")?; + /// Execute the given http request, returning an error in case of a + /// non-successful status code. + async fn http_request_estatus(&self, request: Request) -> Result { + Ok(self.http_request(request).await?.error_for_status()?) + } + + /// Execute the given http request, returning the response body as a string. + async fn http_request_txt(&self, request: Request) -> Result { + Ok(self.http_request_estatus(request).await?.text().await?) + } + + /// Extract the current version of the YouTube desktop client from the website. + async fn extract_desktop_client_version(&self) -> Result { + let from_swjs = async { + let swjs = self + .http_request_txt( + self.inner + .http + .get("https://www.youtube.com/sw.js") + .header(header::ORIGIN, "https://www.youtube.com") + .header(header::REFERER, "https://www.youtube.com") + .header(header::COOKIE, self.inner.consent_cookie.to_owned()) + .build() + .unwrap(), + ) + .await + .context("Failed to download sw.js")?; util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) - .ok_or(anyhow!("Could not find music client version in sw.js")) - } + .ok_or_else(|| anyhow!("Could not find desktop client version in sw.js")) + }; - async fn extract_client_version_from_html(http: Client) -> Result { - let html = exec_request_text( - http.clone(), - http.get("https://music.youtube.com").build().unwrap(), - ) - .await - .context("Failed to get YT Music page")?; + let from_html = async { + let html = self + .http_request_txt( + self.inner + .http + .get("https://www.youtube.com/results?search_query=") + .build() + .unwrap(), + ) + .await + .context("Failed to get YT Desktop page")?; util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1) - .ok_or(anyhow!("Could not find music client version on html page")) - } + .ok_or_else(|| anyhow!("Could not find desktop client version on html page")) + }; - match extract_client_version_from_swjs(http.clone(), consent_cookie).await { + match from_swjs.await { Ok(client_version) => Ok(client_version), - Err(_) => extract_client_version_from_html(http).await, + Err(_) => from_html.await, } } - async fn get_client_version(&self) -> String { - let http = self.http.clone(); - let consent_cookie = self.consent_cookie.clone(); + /// Extract the current version of the YouTube Music desktop client from the website. + async fn extract_music_client_version(&self) -> Result { + let from_swjs = async { + let swjs = self + .http_request_txt( + self.inner + .http + .get("https://music.youtube.com/sw.js") + .header(header::ORIGIN, "https://music.youtube.com") + .header(header::REFERER, "https://music.youtube.com") + .header(header::COOKIE, self.inner.consent_cookie.to_owned()) + .build() + .unwrap(), + ) + .await + .context("Failed to download sw.js")?; - let client_data = self - .cache - .get_music_client_data(async move { - let client_version = Self::extract_client_version(http, &consent_cookie).await?; - Ok(ClientData { - version: client_version, - }) - }) - .await; + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) + .ok_or_else(|| anyhow!("Could not find desktop client version in sw.js")) + }; - match client_data { - Ok(client_data) => client_data.version, - Err(e) => { - warn!("{}", e); - DESKTOP_MUSIC_CLIENT_VERSION.to_owned() + let from_html = async { + let html = self + .http_request_txt( + self.inner + .http + .get("https://music.youtube.com") + .build() + .unwrap(), + ) + .await + .context("Failed to get YT Desktop page")?; + + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1) + .ok_or_else(|| anyhow!("Could not find desktop client version on html page")) + }; + + match from_swjs.await { + Ok(client_version) => Ok(client_version), + Err(_) => from_html.await, + } + } + + /// Get the current version of the YouTube web client from the following sources + /// + /// 1. from cache + /// 2. from YouTube's service worker script (`sw.js`) + /// 3. from the YouTube website + /// 4. fall back to the hardcoded version + async fn get_desktop_client_version(&self) -> String { + let mut cache = self.inner.cache.lock().await; + + match cache.desktop_client.get() { + Some(cdata) => cdata.version.to_owned(), + None => match self.extract_desktop_client_version().await { + Ok(version) => { + cache.desktop_client = CacheEntry::from(ClientData { + version: version.to_owned(), + }); + self.store_cache(&cache); + version + } + Err(e) => { + warn!("{}, falling back to hardcoded version", e); + DESKTOP_CLIENT_VERSION.to_owned() + } + }, + } + } + + /// Get the current version of the YouTube Music web client from the following sources + /// + /// 1. from cache + /// 2. from YouTube Music's service worker script (`sw.js`) + /// 3. from the YouTube Music website + /// 4. fall back to the hardcoded version + async fn get_music_client_version(&self) -> String { + let mut cache = self.inner.cache.lock().await; + + match cache.music_client.get() { + Some(cdata) => cdata.version.to_owned(), + None => match self.extract_music_client_version().await { + Ok(version) => { + cache.music_client = CacheEntry::from(ClientData { + version: version.to_owned(), + }); + self.store_cache(&cache); + version + } + Err(e) => { + warn!("{}, falling back to hardcoded version", e); + DESKTOP_MUSIC_CLIENT_VERSION.to_owned() + } + }, + } + } + + /// Instantiate a new deobfuscator from either cached or extracted YouTube JavaScript code. + async fn get_deobf(&self) -> Result { + let mut cache = self.inner.cache.lock().await; + + match cache.deobf.get() { + Some(deobf) => Ok(Deobfuscator::from(deobf.to_owned())), + None => { + let deobf = Deobfuscator::new(self.inner.http.clone()).await?; + cache.deobf = CacheEntry::from(deobf.get_data()); + self.store_cache(&cache); + Ok(deobf) + } + } + } + + /// Write the given cache data to the storage backend. + fn store_cache(&self, cache: &CacheData) { + if let Some(storage) = &self.inner.storage { + match serde_json::to_string(cache) { + Ok(data) => storage.write(&data), + Err(e) => error!("Could not serialize cache. Error: {}", e), } } } } -#[cfg(test)] -mod tests { - use super::*; - use test_log::test; +impl RustyPipeQuery { + /// Set the language parameter used when accessing the YouTube API + /// This will change multilanguage video titles, descriptions and textual dates + pub fn lang(mut self, lang: Language) -> Self { + self.opts.lang = lang; + self + } - static CLIENT_VERSION_REGEX: Lazy = - Lazy::new(|| Regex::new(r#"^\d+\.\d{8}\.\d{2}\.\d{2}"#).unwrap()); + /// Set the country parameter used when accessing the YouTube API. + /// This will change trends and recommended content. + pub fn country(mut self, country: Country) -> Self { + self.opts.country = country; + self + } - #[test(tokio::test)] - async fn t_extract_desktop_client_version() { - let rt = RustyTube::new(); - let client = rt.desktop_client; - let version = - DesktopClient::extract_client_version(client.http.clone(), &client.consent_cookie) - .await - .unwrap(); + /// Generate a report on every operation. + /// This should only be used for debugging. + pub fn report(mut self) -> Self { + self.opts.report = true; + self + } - assert!(CLIENT_VERSION_REGEX.is_match(&version).unwrap()); + /// Enable strict mode, causing operations to fail if there + /// are warnings during deserialization (e.g. invalid items). + /// This should only be used for testing. + pub fn strict(mut self) -> Self { + self.opts.strict = true; + self + } - // Client version changes often, - // notify during test so the hardcoded version can be updated - if version != DESKTOP_CLIENT_VERSION { - println!( - "INFO: YT Desktop Client was updated, new version: {}", - version - ); + /// Create a new context object, which is included in every request to + /// the YouTube API and contains language, country and device parameters. + /// + /// # Parameters + /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...) + /// - `localized`: Whether to include the configured language and country + async fn get_context(&self, ctype: ClientType, localized: bool) -> YTContext { + let hl = match localized { + true => self.opts.lang, + false => Language::En, + }; + let gl = match localized { + true => self.opts.country, + false => Country::Us, + }; + + match ctype { + ClientType::Desktop => YTContext { + client: ClientInfo { + client_name: "WEB".to_owned(), + client_version: self.client.get_desktop_client_version().await, + client_screen: None, + device_model: None, + platform: "DESKTOP".to_owned(), + original_url: Some("https://www.youtube.com/".to_owned()), + hl, + gl, + }, + request: Some(RequestYT::default()), + user: User::default(), + third_party: None, + }, + ClientType::DesktopMusic => YTContext { + client: ClientInfo { + client_name: "WEB_REMIX".to_owned(), + client_version: self.client.get_music_client_version().await, + client_screen: None, + device_model: None, + platform: "DESKTOP".to_owned(), + original_url: Some("https://music.youtube.com/".to_owned()), + hl, + gl, + }, + request: Some(RequestYT::default()), + user: User::default(), + third_party: None, + }, + ClientType::TvHtml5Embed => YTContext { + client: ClientInfo { + client_name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER".to_owned(), + client_version: TVHTML5_CLIENT_VERSION.to_owned(), + client_screen: Some("EMBED".to_owned()), + device_model: None, + platform: "TV".to_owned(), + original_url: None, + hl, + gl, + }, + request: Some(RequestYT::default()), + user: User::default(), + third_party: Some(ThirdParty { + embed_url: "https://www.youtube.com/".to_owned(), + }), + }, + ClientType::Android => YTContext { + client: ClientInfo { + client_name: "ANDROID".to_owned(), + client_version: MOBILE_CLIENT_VERSION.to_owned(), + client_screen: None, + device_model: None, + platform: "MOBILE".to_owned(), + original_url: None, + hl, + gl, + }, + request: None, + user: User::default(), + third_party: None, + }, + ClientType::Ios => YTContext { + client: ClientInfo { + client_name: "IOS".to_owned(), + client_version: MOBILE_CLIENT_VERSION.to_owned(), + client_screen: None, + device_model: Some(IOS_DEVICE_MODEL.to_owned()), + platform: "MOBILE".to_owned(), + original_url: None, + hl, + gl, + }, + request: None, + user: User::default(), + third_party: None, + }, } } - #[test(tokio::test)] - async fn t_extract_desktop_music_client_version() { - let rt = RustyTube::new(); - let client = rt.desktop_music_client; - let version = - DesktopMusicClient::extract_client_version(client.http.clone(), &client.consent_cookie) - .await - .unwrap(); - - assert!(CLIENT_VERSION_REGEX.is_match(&version).unwrap()); - - // Client version changes often, - // notify during test so the hardcoded version can be updated - if version != DESKTOP_MUSIC_CLIENT_VERSION { - println!( - "INFO: YT Desktop Music Client was updated, new version: {}", - version - ); + /// Create a new Reqwest HTTP request builder with the URL and headers required + /// for accessing the YouTube API + /// + /// # Parameters + /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...) + /// - `method`: HTTP method + /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/?key=...`) + async fn request_builder( + &self, + ctype: ClientType, + method: Method, + endpoint: &str, + ) -> RequestBuilder { + match ctype { + ClientType::Desktop => self + .client + .inner + .http + .request( + method, + format!( + "{}{}?key={}{}", + YOUTUBEI_V1_URL, endpoint, DESKTOP_API_KEY, DISABLE_PRETTY_PRINT_PARAMETER + ), + ) + .header(header::ORIGIN, "https://www.youtube.com") + .header(header::REFERER, "https://www.youtube.com") + .header(header::COOKIE, self.client.inner.consent_cookie.to_owned()) + .header("X-YouTube-Client-Name", "1") + .header( + "X-YouTube-Client-Version", + self.client.get_desktop_client_version().await, + ), + ClientType::DesktopMusic => self + .client + .inner + .http + .request( + method, + format!( + "{}{}?key={}{}", + YOUTUBE_MUSIC_V1_URL, + endpoint, + DESKTOP_MUSIC_API_KEY, + DISABLE_PRETTY_PRINT_PARAMETER + ), + ) + .header(header::ORIGIN, "https://music.youtube.com") + .header(header::REFERER, "https://music.youtube.com") + .header(header::COOKIE, self.client.inner.consent_cookie.to_owned()) + .header("X-YouTube-Client-Name", "67") + .header( + "X-YouTube-Client-Version", + self.client.get_music_client_version().await, + ), + ClientType::TvHtml5Embed => self + .client + .inner + .http + .request( + method, + format!( + "{}{}?key={}{}", + YOUTUBEI_V1_URL, endpoint, DESKTOP_API_KEY, DISABLE_PRETTY_PRINT_PARAMETER + ), + ) + .header(header::ORIGIN, "https://www.youtube.com") + .header(header::REFERER, "https://www.youtube.com") + .header("X-YouTube-Client-Name", "1") + .header("X-YouTube-Client-Version", TVHTML5_CLIENT_VERSION), + ClientType::Android => self + .client + .inner + .http + .request( + method, + format!( + "{}{}?key={}{}", + YOUTUBEI_V1_GAPIS_URL, + endpoint, + ANDROID_API_KEY, + DISABLE_PRETTY_PRINT_PARAMETER + ), + ) + .header( + header::USER_AGENT, + format!( + "com.google.android.youtube/{} (Linux; U; Android 12; {}) gzip", + MOBILE_CLIENT_VERSION, self.opts.country + ), + ) + .header("X-Goog-Api-Format-Version", "2"), + ClientType::Ios => self + .client + .inner + .http + .request( + method, + format!( + "{}{}?key={}{}", + YOUTUBEI_V1_GAPIS_URL, + endpoint, + IOS_API_KEY, + DISABLE_PRETTY_PRINT_PARAMETER + ), + ) + .header( + header::USER_AGENT, + format!( + "com.google.ios.youtube/{} ({}; U; CPU iOS 15_4 like Mac OS X; {})", + MOBILE_CLIENT_VERSION, IOS_DEVICE_MODEL, self.opts.country + ), + ) + .header("X-Goog-Api-Format-Version", "2"), } } + + /// Execute a request to the YouTube API, then deobfuscate and map the response. + /// + /// Creates a report in case of failure for easy debugging. + /// + /// # Parameters + /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...) + /// - `operation`: Name of the RustyPipe operation (only for reporting, e.g. `get_player`) + /// - `id`: ID of the requested entity (Video ID, Channel ID, ...). + /// The ID is included in reports and is also passed to the mapper for validating the response. + /// Set it to an empty string if you are not requesting an entity with an ID. + /// - `method`: HTTP method + /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/?key=...`) + /// - `body`: Serializable request body to be sent in json format + /// - `deobf`: Deobfuscator (is passed to the mapper to deobfuscate stream URLs). + #[allow(clippy::too_many_arguments)] + async fn execute_request_deobf< + R: DeserializeOwned + MapResponse + Debug, + M, + B: Serialize + ?Sized, + >( + &self, + ctype: ClientType, + operation: &str, + id: &str, + method: Method, + endpoint: &str, + body: &B, + deobf: Option<&Deobfuscator>, + ) -> Result { + let request = self + .request_builder(ctype, method.clone(), endpoint) + .await + .json(body) + .build()?; + + let request_url = request.url().to_string(); + let request_headers = request.headers().to_owned(); + + let response = self.client.inner.http.execute(request).await?; + + let status = response.status(); + let resp_str = response.text().await?; + + let create_report = |level: Level, error: Option, msgs: Vec| { + if let Some(reporter) = &self.client.inner.reporter { + let report = Report { + package: "rustypipe".to_owned(), + version: "0.1.0".to_owned(), + date: chrono::Local::now(), + level, + operation: format!("{}({})", operation, id), + error, + msgs, + deobf_data: deobf.map(Deobfuscator::get_data), + http_request: crate::report::HTTPRequest { + url: request_url, + method: method.to_string(), + req_header: request_headers + .iter() + .map(|(k, v)| { + (k.to_string(), v.to_str().unwrap_or_default().to_owned()) + }) + .collect(), + req_body: serde_json::to_string(body).unwrap_or_default(), + status: status.into(), + resp_body: resp_str.to_owned(), + }, + }; + + reporter.report(&report); + } + }; + + if status.is_client_error() || status.is_server_error() { + let e = anyhow!("Server responded with error code {}", status); + create_report(Level::ERR, Some(e.to_string()), vec![]); + return Err(e); + } + + match serde_json::from_str::(&resp_str) { + Ok(deserialized) => match deserialized.map_response(id, self.opts.lang, deobf) { + Ok(mapres) => { + if !mapres.warnings.is_empty() { + create_report( + Level::WRN, + Some("Warnings during deserialization/mapping".to_owned()), + mapres.warnings, + ); + + if self.opts.strict { + bail!("Warnings during deserialization/mapping"); + } + } else if self.opts.report { + create_report(Level::DBG, None, vec![]); + } + Ok(mapres.c) + } + Err(e) => { + let emsg = "Could not map reponse"; + create_report(Level::ERR, Some(emsg.to_owned()), vec![e.to_string()]); + Err(e).context(emsg) + } + }, + Err(e) => { + let emsg = "Could not deserialize response"; + create_report(Level::ERR, Some(emsg.to_owned()), vec![e.to_string()]); + Err(e).context(emsg) + } + } + } + + /// Execute a request to the YouTube API, then map the response. + /// + /// Creates a report in case of failure for easy debugging. + /// + /// # Parameters + /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...) + /// - `operation`: Name of the RustyPipe operation (only for reporting, e.g. `get_player`) + /// - `id`: ID of the requested entity (Video ID, Channel ID, ...). + /// The ID is included in reports and is also passed to the mapper for validating the response. + /// Set it to an empty string if you are not requesting an entity with an ID. + /// - `method`: HTTP method + /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/?key=...`) + /// - `body`: Serializable request body to be sent in json format + async fn execute_request< + R: DeserializeOwned + MapResponse + Debug, + M, + B: Serialize + ?Sized, + >( + &self, + ctype: ClientType, + operation: &str, + id: &str, + method: Method, + endpoint: &str, + body: &B, + ) -> Result { + self.execute_request_deobf::(ctype, operation, id, method, endpoint, body, None) + .await + } +} + +/// Implement this for YouTube API response structs that need to be mapped to +/// RustyPipe models. +trait MapResponse { + /// Map the YouTube API response structs to a RustyPipe model. + /// + /// Returns an error if crucial data required for the model could not be extracted. + /// + /// Returns a `MapResult` with warnings if there were issues with the deserializing/mapping, + /// but the resulting data is still usable. + /// + /// # Parameters + /// - `id`: The ID of the requested entity (Video ID, Channel ID, ...). If possible, assert + /// that the returned entity matches this ID and return an error instead. + /// - `lang`: Language of the request. Used for mapping localized information like dates. + /// - `deobf`: Deobfuscator (if passed to the `execute_request_deobf` method) + fn map_response( + self, + id: &str, + lang: Language, + deobf: Option<&Deobfuscator>, + ) -> Result>; +} + +#[cfg(test)] +mod tests { + // use super::*; } diff --git a/src/client/player.rs b/src/client/player.rs index 2044caa..c91f2b9 100644 --- a/src/client/player.rs +++ b/src/client/player.rs @@ -1,25 +1,33 @@ use std::{ borrow::Cow, collections::{BTreeMap, HashMap}, - sync::Arc, }; use anyhow::{anyhow, bail, Result}; use chrono::{Local, NaiveDateTime, NaiveTime, TimeZone}; use fancy_regex::Regex; -use log::{error, warn}; use once_cell::sync::Lazy; -use reqwest::Method; +use reqwest::{Method, Url}; use serde::Serialize; -use url::Url; -use super::{response, ClientType, ContextYT, RustyTube, YTClient}; -use crate::{client::response::player, deobfuscate::Deobfuscator, model::*, util}; +use crate::{ + deobfuscate::Deobfuscator, + model::{ + AudioCodec, AudioFormat, AudioStream, AudioTrack, Channel, Language, Subtitle, VideoCodec, + VideoFormat, VideoInfo, VideoPlayer, VideoStream, + }, + util, +}; + +use super::{ + response::{self, player}, + ClientType, MapResponse, MapResult, RustyPipeQuery, YTContext, +}; #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QPlayer { - context: ContextYT, + context: YTContext, /// Website playback context #[serde(skip_serializing_if = "Option::is_none")] playback_context: Option, @@ -49,58 +57,211 @@ struct QContentPlaybackContext { referer: String, } -impl RustyTube { - pub async fn get_player(&self, video_id: &str, client_type: ClientType) -> Result { - let client = self.get_ytclient(client_type); - let (context, deobf) = tokio::join!( - client.get_context(false), - Deobfuscator::from_fetched_info(client.http_client(), self.cache.clone()) - ); - let deobf = deobf?; - let request_body = build_request_body(client.clone(), &deobf, context, video_id); +impl RustyPipeQuery { + pub async fn get_player(self, video_id: &str, client_type: ClientType) -> Result { + let q1 = self.clone(); + let t_context = tokio::spawn(async move { q1.get_context(client_type, false).await }); + let q2 = self.client.clone(); + let t_deobf = tokio::spawn(async move { q2.get_deobf().await }); - let resp = client - .request_builder(Method::POST, "player") - .await - .json(&request_body) - .send() - .await? - .error_for_status()?; + let (context, deobf) = tokio::join!(t_context, t_deobf); + let context = context.unwrap(); + let deobf = deobf.unwrap()?; - let player_response = resp.json::().await?; - map_player_data(player_response, &deobf) + let request_body = if client_type.is_web() { + QPlayer { + context, + playback_context: Some(QPlaybackContext { + content_playback_context: QContentPlaybackContext { + signature_timestamp: deobf.get_sts(), + referer: format!("https://www.youtube.com/watch?v={}", video_id), + }, + }), + cpn: None, + video_id: video_id.to_owned(), + content_check_ok: true, + racy_check_ok: true, + } + } else { + QPlayer { + context, + playback_context: None, + cpn: Some(util::generate_content_playback_nonce()), + video_id: video_id.to_owned(), + content_check_ok: true, + racy_check_ok: true, + } + }; + + self.execute_request_deobf::( + client_type, + "get_player", + video_id, + Method::POST, + "player", + &request_body, + Some(&deobf), + ) + .await } } -fn build_request_body( - client: Arc, - deobf: &Deobfuscator, - context: ContextYT, - video_id: &str, -) -> QPlayer { - if client.get_type().is_web() { - QPlayer { - context, - playback_context: Some(QPlaybackContext { - content_playback_context: QContentPlaybackContext { - signature_timestamp: deobf.get_sts(), - referer: format!("https://www.youtube.com/watch?v={}", video_id), - }, - }), - cpn: None, - video_id: video_id.to_owned(), - content_check_ok: true, - racy_check_ok: true, +impl MapResponse for response::Player { + fn map_response( + self, + id: &str, + _lang: Language, + deobf: Option<&Deobfuscator>, + ) -> Result> { + let deobf = deobf.unwrap(); + let mut warnings = vec![]; + + // Check playability status + match self.playability_status { + response::player::PlayabilityStatus::Ok { live_streamability } => { + if live_streamability.is_some() { + bail!("Active livestreams are not supported") + } + } + response::player::PlayabilityStatus::Unplayable { reason } => { + bail!("Video is unplayable. Reason: {}", reason) + } + response::player::PlayabilityStatus::LoginRequired { reason } => { + bail!("Playback requires login. Reason: {}", reason) + } + response::player::PlayabilityStatus::LiveStreamOffline { reason } => { + bail!("Livestream is offline. Reason: {}", reason) + } + response::player::PlayabilityStatus::Error { reason } => { + bail!("Video was deleted. Reason: {}", reason) + } + }; + + let mut streaming_data = some_or_bail!( + self.streaming_data, + Err(anyhow!("No streaming data was returned")) + ); + let video_details = some_or_bail!( + self.video_details, + Err(anyhow!("No video details were returned")) + ); + let microformat = self.microformat.map(|m| m.player_microformat_renderer); + let (publish_date, category, tags, is_family_safe) = + microformat.map_or((None, None, None, None), |m| { + ( + Local + .from_local_datetime(&NaiveDateTime::new( + m.publish_date, + NaiveTime::from_hms(0, 0, 0), + )) + .single(), + Some(m.category), + m.tags, + Some(m.is_family_safe), + ) + }); + + if video_details.video_id != id { + bail!( + "got wrong video id {}, expected {}", + video_details.video_id, + id + ); } - } else { - QPlayer { - context, - playback_context: None, - cpn: Some(util::generate_content_playback_nonce()), - video_id: video_id.to_owned(), - content_check_ok: true, - racy_check_ok: true, + + let video_info = VideoInfo { + id: video_details.video_id, + title: video_details.title, + description: video_details.short_description, + length: video_details.length_seconds, + thumbnails: video_details.thumbnail.unwrap_or_default().into(), + channel: Channel { + id: video_details.channel_id, + name: video_details.author, + }, + publish_date, + view_count: video_details.view_count, + keywords: match video_details.keywords { + Some(keywords) => keywords, + None => tags.unwrap_or_default(), + }, + category, + is_live_content: video_details.is_live_content, + is_family_safe, + }; + + let mut formats = streaming_data.formats.c; + formats.append(&mut streaming_data.adaptive_formats.c); + + warnings.append(&mut streaming_data.formats.warnings); + warnings.append(&mut streaming_data.adaptive_formats.warnings); + + let mut last_nsig: [String; 2] = ["".to_owned(), "".to_owned()]; + + let mut video_streams: Vec = Vec::new(); + let mut video_only_streams: Vec = Vec::new(); + let mut audio_streams: Vec = Vec::new(); + + for f in formats { + if f.format_type == player::FormatType::FormatStreamTypeOtf { + continue; + } + + match (f.is_video(), f.is_audio()) { + (true, true) => { + let mut map_res = map_video_stream(f, deobf, &mut last_nsig); + warnings.append(&mut map_res.warnings); + if let Some(c) = map_res.c { + video_streams.push(c); + }; + } + (true, false) => { + let mut map_res = map_video_stream(f, deobf, &mut last_nsig); + warnings.append(&mut map_res.warnings); + if let Some(c) = map_res.c { + video_only_streams.push(c); + }; + } + (false, true) => { + let mut map_res = map_audio_stream(f, deobf, &mut last_nsig); + warnings.append(&mut map_res.warnings); + if let Some(c) = map_res.c { + audio_streams.push(c); + }; + } + (false, false) => warnings.push(format!("invalid stream: itag {}", f.itag)), + } } + + video_streams.sort(); + video_only_streams.sort(); + audio_streams.sort(); + + let mut subtitles = vec![]; + if let Some(captions) = self.captions { + for c in captions.player_captions_tracklist_renderer.caption_tracks { + let lang_auto = c.name.strip_suffix(" (auto-generated)"); + + subtitles.push(Subtitle { + url: c.base_url, + lang: c.language_code, + lang_name: lang_auto.unwrap_or(&c.name).to_owned(), + auto_generated: lang_auto.is_some(), + }) + } + } + + Ok(MapResult { + c: VideoPlayer { + info: video_info, + video_streams, + video_only_streams, + audio_streams, + subtitles, + expires_in_seconds: streaming_data.expires_in_seconds, + }, + warnings, + }) } } @@ -136,7 +297,7 @@ fn deobf_nsig( let nsig: String; match url_params.get("n") { Some(n) => { - nsig = if n.to_owned() == last_nsig[0] { + nsig = if n == &last_nsig[0] { last_nsig[1].to_owned() } else { let nsig = deobf.deobfuscate_nsig(n)?; @@ -157,108 +318,192 @@ fn map_url( signature_cipher: &Option, deobf: &Deobfuscator, last_nsig: &mut [String; 2], -) -> Option<(String, bool)> { +) -> MapResult> { let (url_base, mut url_params) = match url { - Some(url) => ok_or_bail!(util::url_to_params(url), None), + Some(url) => ok_or_bail!( + util::url_to_params(url), + MapResult { + c: None, + warnings: vec![format!("Could not parse url `{}`", url)] + } + ), None => match signature_cipher { Some(signature_cipher) => match cipher_to_url_params(signature_cipher, deobf) { Ok(res) => res, Err(e) => { - error!("Could not deobfuscate signatureCipher: {}", e); - return None; + return MapResult { + c: None, + warnings: vec![format!( + "Could not deobfuscate signatureCipher `{}`: {}", + signature_cipher, e + )], + }; } }, - None => return None, + None => { + return MapResult { + c: None, + warnings: vec!["stream contained neither url nor cipher".to_owned()], + } + } }, }; + let mut warnings = vec![]; let mut throttled = false; deobf_nsig(&mut url_params, deobf, last_nsig).unwrap_or_else(|e| { - warn!("Could not deobfuscate nsig: {}", e); + warnings.push(format!( + "Could not deobfuscate nsig (params: {:?}): {}", + url_params, e + )); throttled = true; }); - Some(( - ok_or_bail!( - Url::parse_with_params(url_base.as_str(), url_params.iter()), - None - ) - .to_string(), - throttled, - )) + MapResult { + c: Some(( + ok_or_bail!( + Url::parse_with_params(url_base.as_str(), url_params.iter()), + MapResult { + c: None, + warnings: vec![format!( + "url could not be joined. url: `{}` params: {:?}", + url_base, url_params + )], + } + ) + .to_string(), + throttled, + )), + warnings, + } } fn map_video_stream( - f: &player::Format, + f: player::Format, deobf: &Deobfuscator, last_nsig: &mut [String; 2], -) -> Option { - let (mtype, codecs) = some_or_bail!(parse_mime(&f.mime_type), None); - let (url, throttled) = - some_or_bail!(map_url(&f.url, &f.signature_cipher, deobf, last_nsig), None); +) -> MapResult> { + let (mtype, codecs) = some_or_bail!( + parse_mime(&f.mime_type), + MapResult { + c: None, + warnings: vec![format!( + "Invalid mime type `{}` in video format {:?}", + &f.mime_type, &f + )] + } + ); + let map_res = map_url(&f.url, &f.signature_cipher, deobf, last_nsig); - Some(VideoStream { - url, - itag: f.itag, - bitrate: f.bitrate, - average_bitrate: f.average_bitrate, - size: f.content_length, - index_range: f.index_range.clone(), - init_range: f.init_range.clone(), - width: some_or_bail!(f.width, None), - height: some_or_bail!(f.height, None), - fps: some_or_bail!(f.fps, None), - quality: some_or_bail!(f.quality_label.clone(), None), - hdr: f.color_info.clone().unwrap_or_default().primaries - == player::Primaries::ColorPrimariesBt2020, - mime: f.mime_type.to_owned(), - format: some_or_bail!(get_video_format(mtype), None), - codec: get_video_codec(codecs), - throttled, - }) + match map_res.c { + Some((url, throttled)) => MapResult { + c: Some(VideoStream { + url, + itag: f.itag, + bitrate: f.bitrate, + average_bitrate: f.average_bitrate.unwrap_or(f.bitrate), + size: f.content_length, + index_range: f.index_range, + init_range: f.init_range, + // Note that the format has already been verified using + // is_video(), so these unwraps are safe + width: f.width.unwrap(), + height: f.height.unwrap(), + fps: f.fps.unwrap(), + quality: f.quality_label.unwrap(), + hdr: f.color_info.unwrap_or_default().primaries + == player::Primaries::ColorPrimariesBt2020, + mime: f.mime_type.to_owned(), + format: some_or_bail!( + get_video_format(mtype), + MapResult { + c: None, + warnings: vec![format!("invalid video format. itag: {}", f.itag)] + } + ), + codec: get_video_codec(codecs), + throttled, + }), + warnings: map_res.warnings, + }, + None => MapResult { + c: None, + warnings: map_res.warnings, + }, + } } fn map_audio_stream( - f: &player::Format, + f: player::Format, deobf: &Deobfuscator, last_nsig: &mut [String; 2], -) -> Option { +) -> MapResult> { static LANG_PATTERN: Lazy = Lazy::new(|| Regex::new(r#"^([a-z]{2})\."#).unwrap()); - let (mtype, codecs) = some_or_bail!(parse_mime(&f.mime_type), None); - let (url, throttled) = - some_or_bail!(map_url(&f.url, &f.signature_cipher, deobf, last_nsig), None); + let (mtype, codecs) = some_or_bail!( + parse_mime(&f.mime_type), + MapResult { + c: None, + warnings: vec![format!( + "Invalid mime type `{}` in video format {:?}", + &f.mime_type, &f + )] + } + ); + let map_res = map_url(&f.url, &f.signature_cipher, deobf, last_nsig); - Some(AudioStream { - url, - itag: f.itag, - bitrate: f.bitrate, - average_bitrate: f.average_bitrate, - size: f.content_length, - index_range: f.index_range.to_owned(), - init_range: f.init_range.to_owned(), - mime: f.mime_type.to_owned(), - format: some_or_bail!(get_audio_format(mtype), None), - codec: get_audio_codec(codecs), - throttled, - track: f.audio_track.as_ref().map(|t| AudioTrack { - id: t.id.to_owned(), - lang: LANG_PATTERN - .captures(&t.id) - .ok() - .flatten() - .map(|m| m.get(1).unwrap().as_str().to_owned()), - lang_name: t.display_name.to_owned(), - is_default: t.audio_is_default, - }), - }) + match map_res.c { + Some((url, throttled)) => MapResult { + c: Some(AudioStream { + url, + itag: f.itag, + bitrate: f.bitrate, + average_bitrate: f.average_bitrate.unwrap_or(f.bitrate), + size: f.content_length.unwrap(), + index_range: f.index_range, + init_range: f.init_range, + mime: f.mime_type.to_owned(), + format: some_or_bail!( + get_audio_format(mtype), + MapResult { + c: None, + warnings: vec![format!("invalid audio format. itag: {}", f.itag)] + } + ), + codec: get_audio_codec(codecs), + throttled, + track: match f.audio_track { + Some(t) => { + let lang = LANG_PATTERN + .captures(&t.id) + .ok() + .flatten() + .map(|m| m.get(1).unwrap().as_str().to_owned()); + + Some(AudioTrack { + id: t.id, + lang, + lang_name: t.display_name, + is_default: t.audio_is_default, + }) + } + None => None, + }, + }), + warnings: map_res.warnings, + }, + None => MapResult { + c: None, + warnings: map_res.warnings, + }, + } } fn parse_mime(mime: &str) -> Option<(&str, Vec<&str>)> { static PATTERN: Lazy = Lazy::new(|| Regex::new(r#"(\w+/\w+);\scodecs="([a-zA-Z-0-9.,\s]*)""#).unwrap()); - let captures = some_or_bail!(PATTERN.captures(&mime).ok().flatten(), None); + let captures = some_or_bail!(PATTERN.captures(mime).ok().flatten(), None); Some(( captures.get(1).unwrap().as_str(), captures @@ -313,140 +558,11 @@ fn get_audio_codec(codecs: Vec<&str>) -> AudioCodec { AudioCodec::Unknown } -fn map_player_data(response: response::Player, deobf: &Deobfuscator) -> Result { - // Check playability status - match response.playability_status { - response::player::PlayabilityStatus::Ok { live_streamability } => { - if live_streamability.is_some() { - bail!("Active livestreams are not supported") - } - } - response::player::PlayabilityStatus::Unplayable { reason } => { - bail!("Video is unplayable. Reason: {}", reason) - } - response::player::PlayabilityStatus::LoginRequired { reason } => { - bail!("Playback requires login. Reason: {}", reason) - } - response::player::PlayabilityStatus::LiveStreamOffline { reason } => { - bail!("Livestream is offline. Reason: {}", reason) - } - response::player::PlayabilityStatus::Error { reason } => { - bail!("Video was deleted. Reason: {}", reason) - } - }; - - let streaming_data = some_or_bail!( - response.streaming_data, - Err(anyhow!("No streaming data was returned")) - ); - let video_details = some_or_bail!( - response.video_details, - Err(anyhow!("No video details were returned")) - ); - let microformat = response.microformat.map(|m| m.player_microformat_renderer); - - let video_info = VideoInfo { - id: video_details.video_id, - title: video_details.title, - description: video_details.short_description, - length: video_details.length_seconds, - thumbnails: video_details - .thumbnail - .unwrap_or_default() - .thumbnails - .iter() - .map(|t| Thumbnail { - url: t.url.to_owned(), - height: t.height, - width: t.width, - }) - .collect(), - channel: Channel { - id: video_details.channel_id, - name: video_details.author, - }, - publish_date: microformat.as_ref().map(|m| { - let ndt = NaiveDateTime::new(m.publish_date, NaiveTime::from_hms(0, 0, 0)); - Local.from_local_datetime(&ndt).unwrap() - }), - view_count: video_details.view_count, - keywords: video_details - .keywords - .or_else(|| microformat.as_ref().map_or(None, |mf| mf.tags.clone())) - .unwrap_or_default(), - category: microformat.as_ref().map(|m| m.category.to_owned()), - is_live_content: video_details.is_live_content, - is_family_safe: microformat.as_ref().map(|m| m.is_family_safe), - }; - - let mut formats = streaming_data.formats.clone(); - formats.append(&mut streaming_data.adaptive_formats.clone()); - - let mut last_nsig: [String; 2] = ["".to_owned(), "".to_owned()]; - - let mut video_streams: Vec = Vec::new(); - let mut video_only_streams: Vec = Vec::new(); - let mut audio_streams: Vec = Vec::new(); - - for f in formats { - if f.format_type == player::FormatType::FormatStreamTypeOtf { - continue; - } - - match (f.is_video(), f.is_audio()) { - (true, true) => match map_video_stream(&f, deobf, &mut last_nsig) { - Some(stream) => video_streams.push(stream), - None => {} - }, - (true, false) => match map_video_stream(&f, deobf, &mut last_nsig) { - Some(stream) => video_only_streams.push(stream), - None => {} - }, - (false, true) => match map_audio_stream(&f, deobf, &mut last_nsig) { - Some(stream) => audio_streams.push(stream), - None => {} - }, - (false, false) => {} - } - } - - video_streams.sort(); - video_only_streams.sort(); - audio_streams.sort(); - - let subtitles = response.captions.map_or(vec![], |captions| { - captions - .player_captions_tracklist_renderer - .caption_tracks - .iter() - .map(|caption| { - let lang_auto = caption.name.strip_suffix(" (auto-generated)"); - - Subtitle { - url: caption.base_url.to_owned(), - lang: caption.language_code.to_owned(), - lang_name: lang_auto.unwrap_or(&caption.name).to_owned(), - auto_generated: lang_auto.is_some(), - } - }) - .collect() - }); - - Ok(VideoPlayer { - info: video_info, - video_streams, - video_only_streams, - audio_streams, - subtitles, - expires_in_seconds: streaming_data.expires_in_seconds, - }) -} - #[cfg(test)] mod tests { use std::{fs::File, io::BufReader, path::Path}; - use crate::{cache::DeobfData, client::CLIENT_TYPES}; + use crate::{client::RustyPipe, deobfuscate::DeobfData}; use super::*; use rstest::rstest; @@ -460,59 +576,6 @@ mod tests { }) }); - #[test_log::test(tokio::test)] - async fn download_response_testfiles() { - let tf_dir = Path::new("testfiles/player"); - let video_id = "pPvd8UxmSbQ"; - - let rt = RustyTube::new(); - - for client_type in CLIENT_TYPES { - let mut json_path = tf_dir.to_path_buf(); - json_path.push(format!("{:?}_video.json", client_type).to_lowercase()); - if json_path.exists() { - continue; - } - - let client = rt.get_ytclient(client_type); - let context = client.get_context(false).await; - - let request_body = build_request_body(client.clone(), &DEOBFUSCATOR, context, video_id); - - let resp = client - .request_builder(Method::POST, "player") - .await - .json(&request_body) - .send() - .await - .unwrap() - .error_for_status() - .unwrap(); - - let mut file = File::create(json_path).unwrap(); - let mut content = std::io::Cursor::new(resp.bytes().await.unwrap()); - std::io::copy(&mut content, &mut file).unwrap(); - } - } - - #[test_log::test(tokio::test)] - async fn download_model_testfiles() { - let tf_dir = Path::new("testfiles/player_model"); - let rt = RustyTube::new(); - - for (name, id) in [("multilanguage", "tVWWp1PqDus"), ("hdr", "LXb3EKWsInQ")] { - let mut json_path = tf_dir.to_path_buf(); - json_path.push(format!("{}.json", name).to_lowercase()); - if json_path.exists() { - continue; - } - - let player_data = rt.get_player(id, ClientType::Desktop).await.unwrap(); - let file = File::create(json_path).unwrap(); - serde_json::to_writer_pretty(file, &player_data).unwrap(); - } - } - #[rstest] #[case::desktop("desktop")] #[case::desktop_music("desktopmusic")] @@ -525,10 +588,17 @@ mod tests { let json_file = File::open(json_path).unwrap(); let resp: response::Player = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let player_data = map_player_data(resp, &DEOBFUSCATOR).unwrap(); + let map_res = resp + .map_response("pPvd8UxmSbQ", Language::En, Some(&DEOBFUSCATOR)) + .unwrap(); + assert!( + map_res.warnings.is_empty(), + "deserialization/mapping warnings: {:?}", + map_res.warnings + ); let is_desktop = name == "desktop" || name == "desktopmusic"; - insta::assert_yaml_snapshot!(format!("map_player_data_{}", name), player_data, { + insta::assert_yaml_snapshot!(format!("map_player_data_{}", name), map_res.c, { ".info.publish_date" => insta::dynamic_redaction(move |value, _path| { if is_desktop { assert!(value.as_str().unwrap().starts_with("2019-05-30T00:00:00")); @@ -561,8 +631,12 @@ mod tests { #[case::ios(ClientType::Ios)] #[test_log::test(tokio::test)] async fn t_get_player(#[case] client_type: ClientType) { - let rt = RustyTube::new(); - let player_data = rt.get_player("n4tK7LYFxI0", client_type).await.unwrap(); + let rp = RustyPipe::builder().strict().build(); + let player_data = rp + .query() + .get_player("n4tK7LYFxI0", client_type) + .await + .unwrap(); // dbg!(&player_data); @@ -584,7 +658,12 @@ mod tests { assert_eq!(player_data.info.is_live_content, false); if client_type == ClientType::Desktop || client_type == ClientType::DesktopMusic { - assert!(player_data.info.publish_date.unwrap().to_string().starts_with("2013-05-05 00:00:00")); + assert!(player_data + .info + .publish_date + .unwrap() + .to_string() + .starts_with("2013-05-05 00:00:00")); assert_eq!(player_data.info.category.unwrap(), "Music"); assert_eq!(player_data.info.is_family_safe.unwrap(), true); } @@ -604,7 +683,7 @@ mod tests { // Bitrates may change between requests assert_approx(video.bitrate as f64, 1507068.0); assert_eq!(video.average_bitrate, 1345149); - assert_eq!(video.size, 43553412); + assert_eq!(video.size.unwrap(), 43553412); assert_eq!(video.width, 1280); assert_eq!(video.height, 720); assert_eq!(video.fps, 30); @@ -634,7 +713,7 @@ mod tests { assert_approx(video.bitrate as f64, 1340829.0); assert_approx(video.average_bitrate as f64, 1233444.0); - assert_approx(video.size as f64, 39936630.0); + assert_approx(video.size.unwrap() as f64, 39936630.0); assert_eq!(video.width, 1280); assert_eq!(video.height, 720); assert_eq!(video.fps, 30); @@ -661,15 +740,20 @@ mod tests { fn t_cipher_to_url() { let signature_cipher = "s=w%3DAe%3DA6aDNQLkViKS7LOm9QtxZJHKwb53riq9qEFw-ecBWJCAiA%3DcEg0tn3dty9jEHszfzh4Ud__bg9CEHVx4ix-7dKsIPAhIQRw8JQ0qOA&sp=sig&url=https://rr5---sn-h0jelnez.googlevideo.com/videoplayback%3Fexpire%3D1659376413%26ei%3Dvb7nYvH5BMK8gAfBj7ToBQ%26ip%3D2003%253Ade%253Aaf06%253A6300%253Ac750%253A1b77%253Ac74a%253A80e3%26id%3Do-AB_BABwrXZJN428ZwDxq5ScPn2AbcGODnRlTVhCQ3mj2%26itag%3D251%26source%3Dyoutube%26requiressl%3Dyes%26mh%3DhH%26mm%3D31%252C26%26mn%3Dsn-h0jelnez%252Csn-4g5ednsl%26ms%3Dau%252Conr%26mv%3Dm%26mvi%3D5%26pl%3D37%26initcwndbps%3D1588750%26spc%3DlT-Khi831z8dTejFIRCvCEwx_6romtM%26vprv%3D1%26mime%3Daudio%252Fwebm%26ns%3Db_Mq_qlTFcSGlG9RpwpM9xQH%26gir%3Dyes%26clen%3D3781277%26dur%3D229.301%26lmt%3D1655510291473933%26mt%3D1659354538%26fvip%3D5%26keepalive%3Dyes%26fexp%3D24001373%252C24007246%26c%3DWEB%26rbqsm%3Dfr%26txp%3D4532434%26n%3Dd2g6G2hVqWIXxedQ%26sparams%3Dexpire%252Cei%252Cip%252Cid%252Citag%252Csource%252Crequiressl%252Cspc%252Cvprv%252Cmime%252Cns%252Cgir%252Cclen%252Cdur%252Clmt%26lsparams%3Dmh%252Cmm%252Cmn%252Cms%252Cmv%252Cmvi%252Cpl%252Cinitcwndbps%26lsig%3DAG3C_xAwRQIgCKCGJ1iu4wlaGXy3jcJyU3inh9dr1FIfqYOZEG_MdmACIQCbungkQYFk7EhD6K2YvLaHFMjKOFWjw001_tLb0lPDtg%253D%253D"; let mut last_nsig: [String; 2] = ["".to_owned(), "".to_owned()]; - let (url, throttled) = map_url( + let map_res = map_url( &None, &Some(signature_cipher.to_owned()), &DEOBFUSCATOR, &mut last_nsig, - ) - .unwrap(); + ); + let (url, throttled) = map_res.c.unwrap(); assert_eq!(url, "https://rr5---sn-h0jelnez.googlevideo.com/videoplayback?c=WEB&clen=3781277&dur=229.301&ei=vb7nYvH5BMK8gAfBj7ToBQ&expire=1659376413&fexp=24001373%2C24007246&fvip=5&gir=yes&id=o-AB_BABwrXZJN428ZwDxq5ScPn2AbcGODnRlTVhCQ3mj2&initcwndbps=1588750&ip=2003%3Ade%3Aaf06%3A6300%3Ac750%3A1b77%3Ac74a%3A80e3&itag=251&keepalive=yes&lmt=1655510291473933&lsig=AG3C_xAwRQIgCKCGJ1iu4wlaGXy3jcJyU3inh9dr1FIfqYOZEG_MdmACIQCbungkQYFk7EhD6K2YvLaHFMjKOFWjw001_tLb0lPDtg%3D%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=hH&mime=audio%2Fwebm&mm=31%2C26&mn=sn-h0jelnez%2Csn-4g5ednsl&ms=au%2Conr&mt=1659354538&mv=m&mvi=5&n=XzXGSfGusw6OCQ&ns=b_Mq_qlTFcSGlG9RpwpM9xQH&pl=37&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRQIhAPIsKd7-xi4xVHEC9gb__dU4hzfzsHEj9ytd3nt0gEceAiACJWBcw-wFEq9qir35bwKHJZxtQ9mOL7SKiVkLQNDa6A%3D%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-Khi831z8dTejFIRCvCEwx_6romtM&txp=4532434&vprv=1"); assert_eq!(throttled, false); + assert!( + map_res.warnings.is_empty(), + "deserialization/mapping warnings: {:?}", + map_res.warnings + ); } } diff --git a/src/client/playlist.rs b/src/client/playlist.rs index 5b979a7..9056ab7 100644 --- a/src/client/playlist.rs +++ b/src/client/playlist.rs @@ -1,85 +1,68 @@ -use anyhow::{anyhow, Context, Result}; +use anyhow::{anyhow, bail, Result}; use reqwest::Method; use serde::Serialize; use crate::{ + deobfuscate::Deobfuscator, model::{Channel, Language, Playlist, Thumbnail, Video}, serializer::text::{PageType, TextLink}, timeago, util, }; -use super::{response, ClientType, ContextYT, RustyTube}; +use super::{response, ClientType, MapResponse, MapResult, RustyPipeQuery, YTContext}; #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QPlaylist { - context: ContextYT, + context: YTContext, browse_id: String, } #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QPlaylistCont { - context: ContextYT, + context: YTContext, continuation: String, } -impl RustyTube { - pub async fn get_playlist(&self, playlist_id: &str) -> Result { - let client = self.get_ytclient(ClientType::Desktop); - let context = client.get_context(true).await; - +impl RustyPipeQuery { + pub async fn get_playlist(self, playlist_id: &str) -> Result { + let context = self.get_context(ClientType::Desktop, true).await; let request_body = QPlaylist { context, browse_id: "VL".to_owned() + playlist_id, }; - let resp = client - .request_builder(Method::POST, "browse") - .await - .json(&request_body) - .send() - .await? - .error_for_status()?; - - let resp_body = resp.text().await?; - let playlist_response = - serde_json::from_str::(&resp_body).context(resp_body)?; - - map_playlist(&playlist_response, self.localization.language) + self.execute_request::( + ClientType::Desktop, + "get_playlist", + playlist_id, + Method::POST, + "browse", + &request_body, + ) + .await } - pub async fn get_playlist_cont(&self, playlist: &mut Playlist) -> Result<()> { + pub async fn get_playlist_cont(self, playlist: &mut Playlist) -> Result<()> { match &playlist.ctoken { Some(ctoken) => { - let client = self.get_ytclient(ClientType::Desktop); - let context = client.get_context(true).await; - + let context = self.get_context(ClientType::Desktop, true).await; let request_body = QPlaylistCont { context, continuation: ctoken.to_owned(), }; - let resp = client - .request_builder(Method::POST, "browse") - .await - .json(&request_body) - .send() - .await? - .error_for_status()?; - - let cont_response = resp.json::().await?; - - let action = some_or_bail!( - cont_response - .on_response_received_actions - .iter() - .find(|a| a.append_continuation_items_action.target_id == playlist.id), - Err(anyhow!("no continuation action")) - ); - - let (mut videos, ctoken) = - map_playlist_items(&action.append_continuation_items_action.continuation_items); + let (mut videos, ctoken) = self + .execute_request::( + ClientType::Desktop, + "get_playlist_cont", + &playlist.id, + Method::POST, + "browse", + &request_body, + ) + .await?; playlist.videos.append(&mut videos); playlist.ctoken = ctoken; @@ -95,149 +78,179 @@ impl RustyTube { } } -fn map_playlist(response: &response::Playlist, lang: Language) -> Result { - let video_items = &some_or_bail!( - some_or_bail!( +impl MapResponse for response::Playlist { + fn map_response( + self, + id: &str, + lang: Language, + _deobf: Option<&Deobfuscator>, + ) -> Result> { + let video_items = &some_or_bail!( some_or_bail!( - response - .contents - .two_column_browse_results_renderer - .contents - .get(0), - Err(anyhow!("twoColumnBrowseResultsRenderer empty")) + some_or_bail!( + self.contents + .two_column_browse_results_renderer + .contents + .get(0), + Err(anyhow!("twoColumnBrowseResultsRenderer empty")) + ) + .tab_renderer + .content + .section_list_renderer + .contents + .get(0), + Err(anyhow!("sectionListRenderer empty")) ) - .tab_renderer - .content - .section_list_renderer + .item_section_renderer .contents .get(0), - Err(anyhow!("sectionListRenderer empty")) + Err(anyhow!("itemSectionRenderer empty")) ) - .item_section_renderer - .contents - .get(0), - Err(anyhow!("itemSectionRenderer empty")) - ) - .playlist_video_list_renderer - .contents; + .playlist_video_list_renderer + .contents; - let (videos, ctoken) = map_playlist_items(video_items); + let (videos, ctoken) = map_playlist_items(&video_items.c); - let (thumbnails, last_update_txt) = match &response.sidebar { - Some(sidebar) => { - let primary = some_or_bail!( - sidebar.playlist_sidebar_renderer.items.get(0), - Err(anyhow!("no primary sidebar")) - ); + let (thumbnails, last_update_txt) = match &self.sidebar { + Some(sidebar) => { + let primary = some_or_bail!( + sidebar.playlist_sidebar_renderer.items.get(0), + Err(anyhow!("no primary sidebar")) + ); - ( - &primary - .playlist_sidebar_primary_info_renderer - .thumbnail_renderer - .playlist_video_thumbnail_renderer - .thumbnail - .thumbnails, - primary - .playlist_sidebar_primary_info_renderer - .stats - .get(2) - .map(|t| t.to_owned()), - ) - } - None => { - let header_banner = some_or_bail!( - &response + ( + &primary + .playlist_sidebar_primary_info_renderer + .thumbnail_renderer + .playlist_video_thumbnail_renderer + .thumbnail + .thumbnails, + primary + .playlist_sidebar_primary_info_renderer + .stats + .get(2) + .map(|t| t.to_owned()), + ) + } + None => { + let header_banner = some_or_bail!( + &self.header.playlist_header_renderer.playlist_header_banner, + Err(anyhow!("no thumbnail found")) + ); + + let last_update_txt = self .header .playlist_header_renderer - .playlist_header_banner, - Err(anyhow!("no thumbnail found")) - ); + .byline + .get(1) + .map(|b| b.playlist_byline_renderer.text.to_owned()); - let last_update_txt = response - .header - .playlist_header_renderer - .byline - .get(1) - .map(|b| b.playlist_byline_renderer.text.to_owned()); + ( + &header_banner + .hero_playlist_thumbnail_renderer + .thumbnail + .thumbnails, + last_update_txt, + ) + } + }; - ( - &header_banner - .hero_playlist_thumbnail_renderer - .thumbnail - .thumbnails, - last_update_txt, - ) + let thumbnails = thumbnails + .iter() + .map(|t| Thumbnail { + url: t.url.to_owned(), + width: t.width, + height: t.height, + }) + .collect::>(); + + let n_videos = match ctoken { + Some(_) => { + ok_or_bail!( + util::parse_numeric(&self.header.playlist_header_renderer.num_videos_text), + Err(anyhow!("no video count")) + ) + } + None => videos.len() as u32, + }; + + let playlist_id = self.header.playlist_header_renderer.playlist_id; + if playlist_id != id { + bail!("got wrong playlist id {}, expected {}", playlist_id, id); } - }; - let thumbnails = thumbnails - .iter() - .map(|t| Thumbnail { - url: t.url.to_owned(), - width: t.width, - height: t.height, - }) - .collect::>(); + let name = self.header.playlist_header_renderer.title; + let description = self.header.playlist_header_renderer.description_text; - let n_videos = match ctoken { - Some(_) => { - ok_or_bail!( - util::parse_numeric(&response.header.playlist_header_renderer.num_videos_text), - Err(anyhow!("no video count")) - ) - } - None => videos.len() as u32, - }; - - let id = response - .header - .playlist_header_renderer - .playlist_id - .to_owned(); - let name = response.header.playlist_header_renderer.title.to_owned(); - let description = response - .header - .playlist_header_renderer - .description_text - .to_owned(); - - let channel = match &response.header.playlist_header_renderer.owner_text { - Some(owner_text) => match owner_text { - TextLink::Browse { + let channel = match self.header.playlist_header_renderer.owner_text { + Some(TextLink::Browse { text, - page_type, + page_type: PageType::Channel, browse_id, - } => match page_type { - PageType::Channel => Some(Channel { - id: browse_id.to_owned(), - name: text.to_owned(), - }), - _ => None, - }, + }) => Some(Channel { + id: browse_id, + name: text, + }), _ => None, - }, - None => None, - }; + }; - Ok(Playlist { - id, - name, - videos, - n_videos, - ctoken, - thumbnails, - description, - channel, - last_update: match &last_update_txt { - Some(textual_date) => timeago::parse_textual_date_to_dt(lang, textual_date), + let mut warnings = video_items.warnings.to_owned(); + let last_update = match &last_update_txt { + Some(textual_date) => { + let parsed = timeago::parse_textual_date_to_dt(lang, textual_date); + if parsed.is_none() { + warnings.push(format!("could not parse textual date `{}`", textual_date)); + } + parsed + } None => None, - }, - last_update_txt, - }) + }; + + Ok(MapResult { + c: Playlist { + id: playlist_id, + name, + videos, + n_videos, + ctoken, + thumbnails, + description, + channel, + last_update, + last_update_txt, + }, + warnings, + }) + } +} + +impl MapResponse<(Vec