diff --git a/.gitignore b/.gitignore index e96be3d..23e8123 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ /target /Cargo.lock -rustypipe_reports -rustypipe_cache.json +rusty-tube.json diff --git a/Cargo.toml b/Cargo.toml index 255c669..7ad4d82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,19 +3,10 @@ name = "rustypipe" version = "0.1.0" edition = "2021" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + [workspace] -members = [".", "codegen", "cli"] - -[features] -default = ["default-tls"] - -# Reqwest TLS -default-tls = ["reqwest/default-tls"] -rustls-tls-webpki-roots = ["reqwest/rustls-tls-webpki-roots"] -rustls-tls-native-roots = ["reqwest/rustls-tls-native-roots"] - -# Error reports in yaml format -report-yaml = ["serde_yaml"] +members = [".", "cli"] [dependencies] # quick-js = "0.4.1" @@ -26,13 +17,13 @@ anyhow = "1.0" thiserror = "1.0.31" url = "2.2.2" log = "0.4.17" -reqwest = {version = "0.11.11", default-features = false, features = ["json", "gzip", "brotli", "stream"]} -tokio = {version = "1.20.0", features = ["macros", "time", "fs", "process"]} +reqwest = {version = "0.11.11", default-features = false, features = ["json", "gzip", "brotli", "stream", "rustls-tls-native-roots"]} +tokio = {version = "1.20.0", features = ["macros", "fs", "process"]} serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.82" -serde_yaml = {version = "0.9.11", optional = true} serde_with = {version = "2.0.0", features = ["json"] } rand = "0.8.5" +async-trait = "0.1.56" chrono = {version = "0.4.19", features = ["serde"]} chronoutil = "0.2.3" futures = "0.3.21" @@ -46,5 +37,6 @@ env_logger = "0.9.0" test-log = "0.2.11" rstest = "0.15.0" temp_testdir = "0.2.3" -insta = {version = "1.17.1", features = ["yaml", "redactions"]} +insta = {version = "1.17.1", features = ["redactions"]} velcro = "0.5.3" +phf_codegen = "0.11.1" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 23b8ed6..353a9be 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -3,10 +3,12 @@ name = "rustypipe-cli" version = "0.1.0" edition = "2021" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + [dependencies] -rustypipe = {path = "../", default_features = false, features = ["rustls-tls-native-roots"]} -reqwest = {version = "0.11.11", default_features = false} -tokio = {version = "1.20.0", features = ["macros", "rt-multi-thread"]} +rustypipe = {path = "../"} +reqwest = {version = "0.11.11", default_features = false, features = ["gzip", "brotli", "rustls-tls-native-roots"]} +tokio = {version = "1.20.0", features = ["rt-multi-thread"]} indicatif = "0.17.0" futures = "0.3.21" anyhow = "1.0" diff --git a/cli/src/main.rs b/cli/src/main.rs index e6a77c0..01ee3c4 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -6,7 +6,7 @@ use futures::stream::{self, StreamExt}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use reqwest::{Client, ClientBuilder}; use rustypipe::{ - client::{ClientType, RustyPipe}, + client::{ClientType, RustyTube}, model::stream_filter::Filter, }; @@ -46,7 +46,7 @@ async fn download_single_video( output_fname: Option, resolution: Option, ffmpeg: &str, - rp: &RustyPipe, + rt: &RustyTube, http: Client, multi: MultiProgress, main: Option, @@ -58,8 +58,7 @@ async fn download_single_video( pb.set_message(format!("Fetching player data for {}", video_title)); let res = async { - let player_data = rp - .query() + let player_data = rt .get_player(video_id.as_str(), ClientType::TvHtml5Embed) .await .context(format!( @@ -113,7 +112,7 @@ async fn download_video( .build() .expect("unable to build the HTTP client"); - let rp = RustyPipe::default(); + let rt = RustyTube::new(); // Indicatif setup let multi = MultiProgress::new(); @@ -125,7 +124,7 @@ async fn download_video( output_fname, resolution, "ffmpeg", - &rp, + &rt, http, multi, None, @@ -148,8 +147,8 @@ async fn download_playlist( .build() .expect("unable to build the HTTP client"); - let rp = RustyPipe::default(); - let playlist = rp.query().get_playlist(id).await.unwrap(); + let rt = RustyTube::new(); + let playlist = rt.get_playlist(id).await.unwrap(); // Indicatif setup let multi = MultiProgress::new(); @@ -174,7 +173,7 @@ async fn download_playlist( output_fname.to_owned(), resolution, "ffmpeg", - &rp, + &rt, http.clone(), multi.clone(), Some(main.clone()), diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml deleted file mode 100644 index ccda07c..0000000 --- a/codegen/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "rustypipe-codegen" -version = "0.1.0" -edition = "2021" - -[dependencies] -rustypipe = {path = "../"} -reqwest = "0.11.11" -tokio = {version = "1.20.0", features = ["macros", "rt-multi-thread"]} -futures = "0.3.21" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0.82" -serde_with = "2.0.0" -anyhow = "1.0" -log = "0.4.17" -env_logger = "0.9.0" -clap = { version = "3.2.16", features = ["derive"] } -phf_codegen = "0.11.1" -once_cell = "1.12.0" -fancy-regex = "0.10.0" diff --git a/codegen/src/download_testfiles.rs b/codegen/src/download_testfiles.rs deleted file mode 100644 index 58d4910..0000000 --- a/codegen/src/download_testfiles.rs +++ /dev/null @@ -1,120 +0,0 @@ -use std::{ - fs::File, - path::{Path, PathBuf}, -}; - -use rustypipe::{ - client::{ClientType, RustyPipe}, - report::{Report, Reporter}, -}; - -const CLIENT_TYPES: [ClientType; 5] = [ - ClientType::Desktop, - ClientType::DesktopMusic, - ClientType::TvHtml5Embed, - ClientType::Android, - ClientType::Ios, -]; - -/// Store pretty-printed response json -pub struct TestFileReporter { - path: PathBuf, -} - -impl TestFileReporter { - pub fn new>(path: P) -> Self { - Self { - path: path.as_ref().to_path_buf(), - } - } -} - -impl Reporter for TestFileReporter { - fn report(&self, report: &Report) { - let data = - serde_json::from_str::(&report.http_request.resp_body).unwrap(); - let file = File::create(&self.path).unwrap(); - serde_json::to_writer_pretty(file, &data).unwrap(); - - println!("Downloaded {}", self.path.display()); - } -} - -fn rp_testfile(json_path: &Path) -> RustyPipe { - let reporter = TestFileReporter::new(json_path); - RustyPipe::builder() - .reporter(Box::new(reporter)) - .report() - .strict() - .build() -} - -pub async fn download_testfiles(project_root: &Path) { - let mut testfiles = project_root.to_path_buf(); - testfiles.push("testfiles"); - - tokio::join!( - player(&testfiles), - player_model(&testfiles), - playlist(&testfiles) - ); -} - -async fn player(testfiles: &Path) { - let video_id = "pPvd8UxmSbQ"; - - for client_type in CLIENT_TYPES { - let mut json_path = testfiles.to_path_buf(); - json_path.push("player"); - json_path.push(format!("{:?}_video.json", client_type).to_lowercase()); - - if json_path.exists() { - continue; - } - - let rp = rp_testfile(&json_path); - rp.query().get_player(video_id, client_type).await.unwrap(); - } -} - -async fn player_model(testfiles: &Path) { - let rp = RustyPipe::builder().strict().build(); - - for (name, id) in [("multilanguage", "tVWWp1PqDus"), ("hdr", "LXb3EKWsInQ")] { - let mut json_path = testfiles.to_path_buf(); - json_path.push("player_model"); - json_path.push(format!("{}.json", name).to_lowercase()); - - if json_path.exists() { - continue; - } - - let player_data = rp - .query() - .get_player(id, ClientType::Desktop) - .await - .unwrap(); - let file = File::create(&json_path).unwrap(); - serde_json::to_writer_pretty(file, &player_data).unwrap(); - - println!("Downloaded {}", json_path.display()); - } -} - -async fn playlist(testfiles: &Path) { - for (name, id) in [ - ("short", "RDCLAK5uy_kFQXdnqMaQCVx2wpUM4ZfbsGCDibZtkJk"), - ("long", "PL5dDx681T4bR7ZF1IuWzOv1omlRbE7PiJ"), - ("nomusic", "PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe"), - ] { - let mut json_path = testfiles.to_path_buf(); - json_path.push("playlist"); - json_path.push(format!("playlist_{}.json", name)); - if json_path.exists() { - continue; - } - - let rp = rp_testfile(&json_path); - rp.query().get_playlist(id).await.unwrap(); - } -} diff --git a/codegen/src/main.rs b/codegen/src/main.rs deleted file mode 100644 index fc1efe0..0000000 --- a/codegen/src/main.rs +++ /dev/null @@ -1,50 +0,0 @@ -mod collect_playlist_dates; -mod download_testfiles; -mod gen_dictionary; -mod gen_locales; -mod util; - -use std::path::PathBuf; - -use clap::{Parser, Subcommand}; - -#[derive(Parser)] -struct Cli { - #[clap(subcommand)] - command: Commands, - #[clap(short = 'd', default_value = "..")] - project_root: PathBuf, - #[clap(short, default_value = "8")] - concurrency: usize, -} - -#[derive(Subcommand)] -enum Commands { - CollectPlaylistDates, - WritePlaylistDates, - GenLocales, - GenDict, - DownloadTestfiles, -} - -#[tokio::main] -async fn main() { - env_logger::init(); - let cli = Cli::parse(); - - match cli.command { - Commands::CollectPlaylistDates => { - collect_playlist_dates::collect_dates(&cli.project_root, cli.concurrency).await; - } - Commands::WritePlaylistDates => { - collect_playlist_dates::write_samples_to_dict(&cli.project_root); - } - Commands::GenLocales => { - gen_locales::generate_locales(&cli.project_root).await; - } - Commands::GenDict => gen_dictionary::generate_dictionary(&cli.project_root), - Commands::DownloadTestfiles => { - download_testfiles::download_testfiles(&cli.project_root).await - } - }; -} diff --git a/codegen/src/util.rs b/codegen/src/util.rs deleted file mode 100644 index 5925322..0000000 --- a/codegen/src/util.rs +++ /dev/null @@ -1,72 +0,0 @@ -use std::{collections::BTreeMap, fs::File, io::BufReader, path::Path, str::FromStr}; - -use rustypipe::model::Language; -use serde::{Deserialize, Serialize}; - -const DICT_PATH: &str = "testfiles/date/dictionary.json"; - -type Dictionary = BTreeMap; - -#[derive(Debug, Default, Serialize, Deserialize)] -#[serde(default)] -pub struct DictEntry { - pub equivalent: Vec, - pub by_char: bool, - pub timeago_tokens: BTreeMap, - pub date_order: String, - pub months: BTreeMap, - pub timeago_nd_tokens: BTreeMap, -} - -pub fn read_dict(project_root: &Path) -> Dictionary { - let mut json_path = project_root.to_path_buf(); - json_path.push(DICT_PATH); - let json_file = File::open(json_path).unwrap(); - serde_json::from_reader(BufReader::new(json_file)).unwrap() -} - -pub fn write_dict(project_root: &Path, dict: &Dictionary) { - let mut json_path = project_root.to_path_buf(); - json_path.push(DICT_PATH); - let json_file = File::create(json_path).unwrap(); - serde_json::to_writer_pretty(json_file, dict).unwrap(); -} - -pub fn filter_datestr(string: &str) -> String { - string - .to_lowercase() - .chars() - .filter_map(|c| { - if c == '\u{200b}' || c.is_ascii_digit() { - None - } else if c == '-' { - Some(' ') - } else { - Some(c) - } - }) - .collect() -} - -/// Parse all numbers occurring in a string and reurn them as a vec -pub fn parse_numeric_vec(string: &str) -> Vec -where - F: FromStr, -{ - let mut numbers = vec![]; - - let mut buf = String::new(); - for c in string.chars() { - if c.is_ascii_digit() { - buf.push(c); - } else if !buf.is_empty() { - buf.parse::().map_or((), |n| numbers.push(n)); - buf.clear(); - } - } - if !buf.is_empty() { - buf.parse::().map_or((), |n| numbers.push(n)); - } - - numbers -} diff --git a/src/cache.rs b/src/cache.rs index e3339d3..509f34b 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -1,57 +1,364 @@ use std::{ - fs, + fs::File, + future::Future, + io::BufReader, path::{Path, PathBuf}, + sync::Arc, }; -use log::error; +use anyhow::Result; +use chrono::{DateTime, Duration, Utc}; +use log::{error, info}; +use serde::{Deserialize, Serialize}; +use tokio::sync::Mutex; -pub trait CacheStorage { - fn write(&self, data: &str); - fn read(&self) -> Option; +#[derive(Default, Debug, Clone)] +pub struct Cache { + file: Option, + data: Arc>, } -pub struct FileStorage { - path: PathBuf, +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +struct CacheData { + desktop_client: Option>, + music_client: Option>, + deobf: Option>, } -impl FileStorage { - pub fn new>(path: P) -> Self { +#[derive(Debug, Clone, Serialize, Deserialize)] +struct CacheEntry { + last_update: DateTime, + data: T, +} + +impl From for CacheEntry { + fn from(f: T) -> Self { Self { - path: path.as_ref().to_path_buf(), + last_update: Utc::now(), + data: f, } } } -impl Default for FileStorage { - fn default() -> Self { - Self { - path: Path::new("rustypipe_cache.json").into(), - } - } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ClientData { + pub version: String, } -impl CacheStorage for FileStorage { - fn write(&self, data: &str) { - fs::write(&self.path, data).unwrap_or_else(|e| { - error!( - "Could not write cache to file `{}`. Error: {}", - self.path.to_string_lossy(), - e - ); - }); +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct DeobfData { + pub js_url: String, + pub sig_fn: String, + pub nsig_fn: String, + pub sts: String, +} + +impl Cache { + pub async fn get_desktop_client_data(&self, updater: F) -> Result + where + F: Future> + Send + 'static, + { + let mut cache = self.data.lock().await; + + if cache.desktop_client.is_none() + || cache.desktop_client.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24) + { + let cdata = updater.await?; + cache.desktop_client = Some(CacheEntry::from(cdata.clone())); + self.save(&cache); + Ok(cdata) + } else { + Ok(cache.desktop_client.as_ref().unwrap().data.clone()) + } } - fn read(&self) -> Option { - match fs::read_to_string(&self.path) { - Ok(data) => Some(data), + pub async fn get_music_client_data(&self, updater: F) -> Result + where + F: Future> + Send + 'static, + { + let mut cache = self.data.lock().await; + + if cache.music_client.is_none() + || cache.music_client.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24) + { + let cdata = updater.await?; + cache.music_client = Some(CacheEntry::from(cdata.clone())); + self.save(&cache); + Ok(cdata) + } else { + Ok(cache.music_client.as_ref().unwrap().data.clone()) + } + } + + pub async fn get_deobf_data(&self, updater: F) -> Result + where + F: Future> + Send + 'static, + { + let mut cache = self.data.lock().await; + if cache.deobf.is_none() + || cache.deobf.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24) + { + let deobf_data = updater.await?; + cache.deobf = Some(CacheEntry::from(deobf_data.clone())); + self.save(&cache); + Ok(deobf_data) + } else { + Ok(cache.deobf.as_ref().unwrap().data.clone()) + } + } + + pub async fn to_json(&self) -> Result { + let cache = self.data.lock().await; + Ok(serde_json::to_string(&cache.clone())?) + } + + pub async fn to_json_file>(&self, path: P) -> Result<()> { + let cache = self.data.lock().await; + Ok(serde_json::to_writer(&File::create(path)?, &cache.clone())?) + } + + pub fn from_json(json: &str) -> Self { + let data: CacheData = match serde_json::from_str(json) { + Ok(cd) => cd, Err(e) => { error!( - "Could not load cache from file `{}`. Error: {}", - self.path.to_string_lossy(), + "Could not load cache from json, falling back to default. Error: {}", e ); - None + CacheData::default() } + }; + Cache { + data: Arc::new(Mutex::new(data)), + file: None, + } + } + + pub fn from_json_file>(path: P) -> Self { + let file = match File::open(path.as_ref()) { + Ok(file) => file, + Err(e) => { + if e.kind() == std::io::ErrorKind::NotFound { + info!( + "Cache json file at {} not found, will be created", + path.as_ref().to_string_lossy() + ) + } else { + error!( + "Could not open cache json file, falling back to default. Error: {}", + e + ); + } + return Cache { + file: Some(path.as_ref().to_path_buf()), + ..Default::default() + }; + } + }; + let data: CacheData = match serde_json::from_reader(BufReader::new(file)) { + Ok(data) => data, + Err(e) => { + error!( + "Could not load cache from json, falling back to default. Error: {}", + e + ); + return Cache { + file: Some(path.as_ref().to_path_buf()), + ..Default::default() + }; + } + }; + Cache { + data: Arc::new(Mutex::new(data)), + file: Some(path.as_ref().to_path_buf()), + } + } + + fn save(&self, cache: &CacheData) { + match self.file.as_ref() { + Some(file) => match File::create(file) { + Ok(file) => match serde_json::to_writer(file, cache) { + Ok(_) => {} + Err(e) => error!("Could not write cache to json. Error: {}", e), + }, + Err(e) => error!("Could not open cache json file. Error: {}", e), + }, + None => {} } } } + +#[cfg(test)] +mod tests { + use temp_testdir::TempDir; + + use super::*; + + #[tokio::test] + async fn test() { + let cache = Cache::default(); + + let desktop_c = cache + .get_desktop_client_data(async { + Ok(ClientData { + version: "1.2.3".to_owned(), + }) + }) + .await + .unwrap(); + + assert_eq!( + desktop_c, + ClientData { + version: "1.2.3".to_owned() + } + ); + + let music_c = cache + .get_music_client_data(async { + Ok(ClientData { + version: "4.5.6".to_owned(), + }) + }) + .await + .unwrap(); + + assert_eq!( + music_c, + ClientData { + version: "4.5.6".to_owned() + } + ); + + let deobf_data = cache + .get_deobf_data(async { + Ok(DeobfData { + js_url: + "https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js" + .to_owned(), + sig_fn: "t_sig_fn".to_owned(), + nsig_fn: "t_nsig_fn".to_owned(), + sts: "t_sts".to_owned(), + }) + }) + .await + .unwrap(); + + assert_eq!( + deobf_data, + DeobfData { + js_url: "https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js" + .to_owned(), + sig_fn: "t_sig_fn".to_owned(), + nsig_fn: "t_nsig_fn".to_owned(), + sts: "t_sts".to_owned(), + } + ); + + // Create a new cache from the first one's json + // and check if it returns the same cached data + let json = cache.to_json().await.unwrap(); + let new_cache = Cache::from_json(&json); + + assert_eq!( + new_cache + .get_desktop_client_data(async { + Ok(ClientData { + version: "".to_owned(), + }) + }) + .await + .unwrap(), + desktop_c + ); + + assert_eq!( + new_cache + .get_music_client_data(async { + Ok(ClientData { + version: "".to_owned(), + }) + }) + .await + .unwrap(), + music_c + ); + + assert_eq!( + new_cache + .get_deobf_data(async { + Ok(DeobfData { + js_url: "".to_owned(), + nsig_fn: "".to_owned(), + sig_fn: "".to_owned(), + sts: "".to_owned(), + }) + }) + .await + .unwrap(), + deobf_data + ); + } + + #[tokio::test] + async fn test_file() { + let temp = TempDir::default(); + let mut file_path = PathBuf::from(temp.as_ref()); + file_path.push("cache.json"); + + let cache = Cache::from_json_file(file_path.clone()); + + let cdata = cache + .get_desktop_client_data(async { + Ok(ClientData { + version: "1.2.3".to_owned(), + }) + }) + .await + .unwrap(); + + let deobf_data = cache + .get_deobf_data(async { + Ok(DeobfData { + js_url: + "https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js" + .to_owned(), + sig_fn: "t_sig_fn".to_owned(), + nsig_fn: "t_nsig_fn".to_owned(), + sts: "t_sts".to_owned(), + }) + }) + .await + .unwrap(); + + assert!(file_path.exists()); + let new_cache = Cache::from_json_file(file_path.clone()); + + assert_eq!( + new_cache + .get_desktop_client_data(async { + Ok(ClientData { + version: "".to_owned(), + }) + }) + .await + .unwrap(), + cdata + ); + + assert_eq!( + new_cache + .get_deobf_data(async { + Ok(DeobfData { + js_url: "".to_owned(), + nsig_fn: "".to_owned(), + sig_fn: "".to_owned(), + sts: "".to_owned(), + }) + }) + .await + .unwrap(), + deobf_data + ); + } +} diff --git a/src/client/channel.rs b/src/client/channel.rs new file mode 100644 index 0000000..3f1ec45 --- /dev/null +++ b/src/client/channel.rs @@ -0,0 +1,41 @@ +use anyhow::Result; +use reqwest::Method; +use serde::Serialize; + +use super::{response, ClientType, ContextYT, RustyTube}; + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct QChannel { + context: ContextYT, + browse_id: String, + params: String, +} + +impl RustyTube { + async fn get_channel_response(&self, channel_id: &str) -> Result { + let client = self.get_ytclient(ClientType::Desktop); + let context = client.get_context(true).await; + + let request_body = QChannel { + context, + browse_id: channel_id.to_owned(), + params: "EgZ2aWRlb3PyBgQKAjoA".to_owned(), + }; + + let resp = client + .request_builder(Method::POST, "browse") + .await + .json(&request_body) + .send() + .await? + .error_for_status()?; + + Ok(resp.json::().await?) + } +} + +#[cfg(test)] +mod tests { + +} diff --git a/src/client/mod.rs b/src/client/mod.rs index 4db6509..adc1f88 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1,43 +1,34 @@ pub mod player; pub mod playlist; +pub mod video; mod response; -use std::fmt::Debug; use std::sync::Arc; -use anyhow::{anyhow, bail, Context, Result}; -use chrono::{DateTime, Duration, Utc}; +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; use fancy_regex::Regex; -use log::{error, warn}; +use log::warn; use once_cell::sync::Lazy; use rand::Rng; -use reqwest::{ - header, Client, ClientBuilder, Method, Request, RequestBuilder, Response, StatusCode, -}; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use tokio::sync::Mutex; +use reqwest::{header, Client, ClientBuilder, Method, Request, RequestBuilder, Response}; +use serde::{Deserialize, Serialize}; use crate::{ - cache::{CacheStorage, FileStorage}, - deobfuscate::{DeobfData, Deobfuscator}, + cache::{Cache, ClientData}, model::{Country, Language}, - report::{FileReporter, Level, Report, Reporter}, - serializer::MapResult, util, }; -/// Client types for accessing the YouTube API. -/// -/// There are multiple clients for accessing the YouTube API which have -/// slightly different features -/// -/// - **Desktop**: used by youtube.com -/// - **DesktopMusic**: used by music.youtube.com, can access special music data, -/// cannot access non-music content -/// - **TvHtml5Embed**: (probably) used by Smart TVs, can access age-restricted videos -/// - **Android**: used by the Android app, no obfuscated URLs, includes lower resolution audio streams -/// - **Ios**: used by the iOS app, no obfuscated URLs +pub const CLIENT_TYPES: [ClientType; 5] = [ + ClientType::Desktop, + ClientType::DesktopMusic, + ClientType::TvHtml5Embed, + ClientType::Android, + ClientType::Ios, +]; + #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] #[serde(rename_all = "snake_case")] pub enum ClientType { @@ -48,33 +39,15 @@ pub enum ClientType { Ios, } -const CLIENT_TYPES: [ClientType; 5] = [ - ClientType::Desktop, - ClientType::DesktopMusic, - ClientType::TvHtml5Embed, - ClientType::Android, - ClientType::Ios, -]; - impl ClientType { - fn is_web(&self) -> bool { - match self { - ClientType::Desktop | ClientType::DesktopMusic | ClientType::TvHtml5Embed => true, - ClientType::Android | ClientType::Ios => false, - } + pub fn is_web(self) -> bool { + self == Self::Desktop || self == Self::DesktopMusic || self == Self::TvHtml5Embed } } -#[derive(Clone, Debug, Serialize)] -struct YTQuery { - context: YTContext, - #[serde(flatten)] - data: T, -} - #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] -struct YTContext { +pub struct ContextYT { client: ClientInfo, /// only used on desktop #[serde(skip_serializing_if = "Option::is_none")] @@ -120,7 +93,7 @@ impl Default for RequestYT { #[derive(Clone, Debug, Serialize, Default)] #[serde(rename_all = "camelCase")] struct User { - // TODO: provide a way to enable restricted mode with: + // TO DO: provide a way to enable restricted mode with: // "enableSafetyMode": true locked_safety_mode: bool, } @@ -131,7 +104,8 @@ struct ThirdParty { embed_url: String, } -const DEFAULT_UA: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0"; +const DEFAULT_UA: &str = + "Mozilla/5.0 (Windows NT 10.0; Win64; rv:107.0) Gecko/20100101 Firefox/107.0"; const CONSENT_COOKIE: &str = "CONSENT"; const CONSENT_COOKIE_YES: &str = "YES+yt.462272069.de+FX+"; @@ -156,912 +130,635 @@ const IOS_DEVICE_MODEL: &str = "iPhone14,5"; static CLIENT_VERSION_REGEXES: Lazy<[Regex; 1]> = Lazy::new(|| [Regex::new("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"").unwrap()]); -/// The RustyPipe client used to access YouTube's API -/// -/// RustyPipe includes an `Arc` internally, so if you are using the client -/// at multiple locations, you can just clone it. Note that options (lang/country/report) -/// are not shared between clones. -#[derive(Clone)] -pub struct RustyPipe { - inner: Arc, +pub struct RustyTube { + localization: Arc, + cache: Cache, + desktop_client: Arc, + desktop_music_client: Arc, + android_client: Arc, + ios_client: Arc, + tvhtml5embed_client: Arc, } -struct RustyPipeRef { - http: Client, - storage: Option>, - reporter: Option>, - n_retries: u32, - user_agent: String, - consent_cookie: String, - cache: Mutex, - default_opts: RustyPipeOpts, +struct Localization { + language: Language, + content_country: Country, } -#[derive(Clone)] -struct RustyPipeOpts { - lang: Language, - country: Country, - report: bool, - strict: bool, -} - -pub struct RustyPipeBuilder { - storage: Option>, - reporter: Option>, - n_retries: u32, - user_agent: String, - default_opts: RustyPipeOpts, -} - -#[derive(Clone)] -pub struct RustyPipeQuery { - client: RustyPipe, - opts: RustyPipeOpts, -} - -impl Default for RustyPipeOpts { - fn default() -> Self { - Self { - lang: Language::En, - country: Country::Us, - report: false, - strict: false, - } - } -} - -#[derive(Default, Debug, Clone, Serialize, Deserialize)] -struct CacheData { - desktop_client: CacheEntry, - music_client: CacheEntry, - deobf: CacheEntry, -} - -#[derive(Default, Debug, Clone, Serialize, Deserialize)] -enum CacheEntry { - #[default] - None, - Some { - last_update: DateTime, - data: T, - }, -} - -#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct ClientData { - pub version: String, -} - -impl CacheEntry { - fn get(&self) -> Option<&T> { - match self { - CacheEntry::Some { last_update, data } => { - if last_update < &(Utc::now() - Duration::hours(24)) { - None - } else { - Some(data) - } - } - CacheEntry::None => None, - } - } -} - -impl From for CacheEntry { - fn from(f: T) -> Self { - Self::Some { - last_update: Utc::now(), - data: f, - } - } -} - -impl Default for RustyPipeBuilder { - fn default() -> Self { - Self::new() - } -} - -impl RustyPipeBuilder { - /// Constructs a new `RustyPipeBuilder`. - /// - /// This is the same as `RustyPipe::builder()` +impl RustyTube { + #[must_use] pub fn new() -> Self { - RustyPipeBuilder { - default_opts: RustyPipeOpts::default(), - storage: Some(Box::new(FileStorage::default())), - reporter: Some(Box::new(FileReporter::default())), - n_retries: 3, - user_agent: DEFAULT_UA.to_owned(), + Self::new_with_ua( + Language::En, + Country::Us, + Some("rusty-tube.json".to_owned()), + ) + } + + #[must_use] + pub fn new_with_ua( + language: Language, + content_country: Country, + cache_file: Option, + ) -> Self { + let loc = Arc::new(Localization { + language, + content_country, + }); + + let cache = match cache_file.as_ref() { + Some(cache_file) => Cache::from_json_file(cache_file), + None => Cache::default(), + }; + + Self { + localization: loc.clone(), + cache: cache.clone(), + desktop_client: Arc::new(DesktopClient::new(loc.clone(), cache.clone())), + desktop_music_client: Arc::new(DesktopMusicClient::new(loc.clone(), cache)), + android_client: Arc::new(AndroidClient::new(loc.clone())), + ios_client: Arc::new(IosClient::new(loc.clone())), + tvhtml5embed_client: Arc::new(TvHtml5EmbedClient::new(loc)), } } - /// Returns a new, configured RustyPipe instance. - pub fn build(self) -> RustyPipe { + pub(crate) fn get_ytclient(&self, client_type: ClientType) -> Arc { + match client_type { + ClientType::Desktop => self.desktop_client.clone(), + ClientType::DesktopMusic => self.desktop_music_client.clone(), + ClientType::TvHtml5Embed => self.tvhtml5embed_client.clone(), + ClientType::Android => self.android_client.clone(), + ClientType::Ios => self.ios_client.clone(), + } + } +} + +#[async_trait] +pub trait YTClient { + async fn get_context(&self, localized: bool) -> ContextYT; + async fn request_builder(&self, method: Method, url: &str) -> RequestBuilder; + fn http_client(&self) -> Client; + fn get_type(&self) -> ClientType; +} + +async fn exec_request(http: Client, request: Request) -> Result { + Ok(http.execute(request).await?.error_for_status()?) +} + +async fn exec_request_text(http: Client, request: Request) -> Result { + Ok(exec_request(http, request).await?.text().await?) +} + +pub struct DesktopClient { + localization: Arc, + http: Client, + cache: Cache, + consent_cookie: String, +} + +#[async_trait] +impl YTClient for DesktopClient { + async fn get_context(&self, localized: bool) -> ContextYT { + ContextYT { + client: ClientInfo { + client_name: "WEB".to_owned(), + client_version: self.get_client_version().await, + client_screen: None, + device_model: None, + platform: "DESKTOP".to_owned(), + original_url: Some("https://www.youtube.com/".to_owned()), + hl: match localized { + true => self.localization.language, + false => Language::En, + }, + gl: match localized { + true => self.localization.content_country, + false => Country::Us, + }, + }, + request: Some(RequestYT::default()), + user: User::default(), + third_party: None, + } + } + + async fn request_builder(&self, method: Method, endpoint: &str) -> RequestBuilder { + self.http + .request( + method, + format!( + "{}{}?key={}{}", + YOUTUBEI_V1_URL, endpoint, DESKTOP_API_KEY, DISABLE_PRETTY_PRINT_PARAMETER + ), + ) + .header(header::ORIGIN, "https://www.youtube.com") + .header(header::REFERER, "https://www.youtube.com") + .header(header::COOKIE, self.consent_cookie.to_owned()) + .header("X-YouTube-Client-Name", "1") + .header("X-YouTube-Client-Version", self.get_client_version().await) + } + + fn http_client(&self) -> Client { + self.http.clone() + } + + fn get_type(&self) -> ClientType { + ClientType::Desktop + } +} + +impl DesktopClient { + fn new(localization: Arc, cache: Cache) -> Self { + let mut rng = rand::thread_rng(); + let http = ClientBuilder::new() - .user_agent(self.user_agent.to_owned()) + .user_agent(DEFAULT_UA) .gzip(true) .brotli(true) .build() - .unwrap(); + .expect("unable to build the HTTP client"); - let cache = if let Some(storage) = &self.storage { - if let Some(data) = storage.read() { - match serde_json::from_str::(&data) { - Ok(data) => data, - Err(e) => { - error!("Could not deserialize cache. Error: {}", e); - CacheData::default() - } - } - } else { - CacheData::default() + Self { + localization, + http, + cache, + consent_cookie: format!( + "{}={}{}", + CONSENT_COOKIE, + CONSENT_COOKIE_YES, + rng.gen_range(100..1000) + ), + } + } + + async fn extract_client_version(http: Client, consent_cookie: &str) -> Result { + async fn extract_client_version_from_swjs( + http: Client, + consent_cookie: &str, + ) -> Result { + let swjs = exec_request_text( + http.clone(), + http.get("https://www.youtube.com/sw.js") + .header(header::ORIGIN, "https://www.youtube.com") + .header(header::REFERER, "https://www.youtube.com") + .header(header::COOKIE, consent_cookie) + .build() + .unwrap(), + ) + .await + .context("Failed to download sw.js")?; + + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) + .ok_or(anyhow!("Could not find desktop client version in sw.js")) + } + + async fn extract_client_version_from_html(http: Client) -> Result { + let html = exec_request_text( + http.clone(), + http.get("https://www.youtube.com/results?search_query=") + .build() + .unwrap(), + ) + .await + .context("Failed to get YT Desktop page")?; + + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(anyhow!( + "Could not find desktop client version on html page" + )) + } + + match extract_client_version_from_swjs(http.clone(), consent_cookie).await { + Ok(client_version) => Ok(client_version), + Err(_) => extract_client_version_from_html(http).await, + } + } + + async fn get_client_version(&self) -> String { + let http = self.http.clone(); + let consent_cookie = self.consent_cookie.clone(); + + let client_data = self + .cache + .get_desktop_client_data(async move { + let client_version = Self::extract_client_version(http, &consent_cookie).await?; + Ok(ClientData { + version: client_version, + }) + }) + .await; + + match client_data { + Ok(client_data) => client_data.version, + Err(e) => { + warn!("{}", e); + DESKTOP_CLIENT_VERSION.to_owned() } - } else { - CacheData::default() - }; + } + } +} - RustyPipe { - inner: Arc::new(RustyPipeRef { - http, - storage: self.storage, - reporter: self.reporter, - n_retries: self.n_retries, - user_agent: self.user_agent, - consent_cookie: format!( - "{}={}{}", - CONSENT_COOKIE, - CONSENT_COOKIE_YES, - rand::thread_rng().gen_range(100..1000) +pub struct AndroidClient { + localization: Arc, + http: Client, +} + +#[async_trait] +impl YTClient for AndroidClient { + async fn get_context(&self, localized: bool) -> ContextYT { + ContextYT { + client: ClientInfo { + client_name: "ANDROID".to_owned(), + client_version: MOBILE_CLIENT_VERSION.to_owned(), + client_screen: None, + device_model: None, + platform: "MOBILE".to_owned(), + original_url: None, + hl: match localized { + true => self.localization.language, + false => Language::En, + }, + gl: match localized { + true => self.localization.content_country, + false => Country::Us, + }, + }, + request: None, + user: User::default(), + third_party: None, + } + } + + async fn request_builder(&self, method: Method, endpoint: &str) -> RequestBuilder { + self.http + .request( + method, + format!( + "{}{}?key={}{}", + YOUTUBEI_V1_GAPIS_URL, + endpoint, + ANDROID_API_KEY, + DISABLE_PRETTY_PRINT_PARAMETER ), - cache: Mutex::new(cache), - default_opts: RustyPipeOpts::default(), + ) + .header("X-Goog-Api-Format-Version", "2") + } + + fn http_client(&self) -> Client { + self.http.clone() + } + + fn get_type(&self) -> ClientType { + ClientType::Android + } +} + +impl AndroidClient { + fn new(localization: Arc) -> Self { + let http = ClientBuilder::new() + .user_agent(format!( + "com.google.android.youtube/{} (Linux; U; Android 12; {}) gzip", + MOBILE_CLIENT_VERSION, localization.content_country + )) + .gzip(true) + .build() + .expect("unable to build the HTTP client"); + + Self { localization, http } + } +} + +pub struct IosClient { + localization: Arc, + http: Client, +} + +#[async_trait] +impl YTClient for IosClient { + async fn get_context(&self, localized: bool) -> ContextYT { + ContextYT { + client: ClientInfo { + client_name: "IOS".to_owned(), + client_version: MOBILE_CLIENT_VERSION.to_owned(), + client_screen: None, + device_model: Some(IOS_DEVICE_MODEL.to_owned()), + platform: "MOBILE".to_owned(), + original_url: None, + hl: match localized { + true => self.localization.language, + false => Language::En, + }, + gl: match localized { + true => self.localization.content_country, + false => Country::Us, + }, + }, + request: None, + user: User::default(), + third_party: None, + } + } + + async fn request_builder(&self, method: Method, endpoint: &str) -> RequestBuilder { + self.http + .request( + method, + format!( + "{}{}?key={}{}", + YOUTUBEI_V1_GAPIS_URL, endpoint, IOS_API_KEY, DISABLE_PRETTY_PRINT_PARAMETER + ), + ) + .header("X-Goog-Api-Format-Version", "2") + } + + fn http_client(&self) -> Client { + self.http.clone() + } + + fn get_type(&self) -> ClientType { + ClientType::Ios + } +} + +impl IosClient { + fn new(localization: Arc) -> Self { + let http = ClientBuilder::new() + .user_agent(format!( + "com.google.ios.youtube/{} ({}; U; CPU iOS 15_4 like Mac OS X; {})", + MOBILE_CLIENT_VERSION, IOS_DEVICE_MODEL, localization.content_country + )) + .gzip(true) + .build() + .expect("unable to build the HTTP client"); + + Self { localization, http } + } +} + +pub struct TvHtml5EmbedClient { + localization: Arc, + http: Client, +} + +#[async_trait] +impl YTClient for TvHtml5EmbedClient { + async fn get_context(&self, localized: bool) -> ContextYT { + ContextYT { + client: ClientInfo { + client_name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER".to_owned(), + client_version: TVHTML5_CLIENT_VERSION.to_owned(), + client_screen: Some("EMBED".to_owned()), + device_model: None, + platform: "TV".to_owned(), + original_url: None, + hl: match localized { + true => self.localization.language, + false => Language::En, + }, + gl: match localized { + true => self.localization.content_country, + false => Country::Us, + }, + }, + request: Some(RequestYT::default()), + user: User::default(), + third_party: Some(ThirdParty { + embed_url: "https://www.youtube.com/".to_owned(), }), } } - /// Add a `CacheStorage` backend for persisting cached information - /// (YouTube client versions, deobfuscation code) between - /// program executions. - /// - /// **Default value**: `FileStorage` in `rustypipe_cache.json` - pub fn storage(mut self, storage: Box) -> Self { - self.storage = Some(storage); - self - } - - /// Disable cache storage - pub fn no_storage(mut self) -> Self { - self.storage = None; - self - } - - /// Add a `Reporter` to collect error details - /// - /// **Default value**: `FileReporter` creating reports in `./rustypipe_reports` - pub fn reporter(mut self, reporter: Box) -> Self { - self.reporter = Some(reporter); - self - } - - /// Disable the creation of report files in case of errors and warnings. - pub fn no_reporter(mut self) -> Self { - self.reporter = None; - self - } - - /// Set the number of retries for HTTP requests. - /// - /// If a HTTP requests fails and retries are enabled, - /// RustyPipe waits 1 second before the next attempt. - /// The waiting time is doubled for subsequent attempts (including a bit of - /// random jitter to be less predictable). - /// - /// **Default value**: 3 - pub fn n_retries(mut self, n_retries: u32) -> Self { - self.n_retries = n_retries; - self - } - - /// Set the user agent used for making requests to the web API. - /// - /// **Default value**: `Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0` - /// (Firefox ESR on Debian) - pub fn user_agent(mut self, user_agent: &str) -> Self { - self.user_agent = user_agent.to_owned(); - self - } - - /// Set the language parameter used when accessing the YouTube API. - /// This will change multilanguage video titles, descriptions and textual dates - /// - /// **Default value**: `Language::En` (English) - /// - /// **Info**: you can set this option for individual queries, too - pub fn lang(mut self, lang: Language) -> Self { - self.default_opts.lang = lang; - self - } - - /// Set the country parameter used when accessing the YouTube API. - /// This will change trends and recommended content. - /// - /// **Default value**: `Country::Us` (USA) - /// - /// **Info**: you can set this option for individual queries, too - pub fn country(mut self, country: Country) -> Self { - self.default_opts.country = country; - self - } - - /// Generate a report on every operation. - /// This should only be used for debugging. - /// - /// **Info**: you can set this option for individual queries, too - pub fn report(mut self) -> Self { - self.default_opts.report = true; - self - } - - /// Enable strict mode, causing operations to fail if there - /// are warnings during deserialization (e.g. invalid items). - /// This should only be used for testing. - /// - /// **Info**: you can set this option for individual queries, too - pub fn strict(mut self) -> Self { - self.default_opts.strict = true; - self - } -} - -impl Default for RustyPipe { - fn default() -> Self { - Self::new() - } -} - -impl RustyPipe { - /// Create a new RustyPipe instance with default settings. - /// - /// To create an instance with custom options, use `RustyPipeBuilder` instead. - pub fn new() -> Self { - RustyPipeBuilder::new().build() - } - - /// Constructs a new `RustyPipeBuilder`. - /// - /// This is the same as `RustyPipeBuilder::new()` - pub fn builder() -> RustyPipeBuilder { - RustyPipeBuilder::new() - } - - /// Constructs a new `RustyPipeQuery`. - pub fn query(&self) -> RustyPipeQuery { - RustyPipeQuery { - client: self.clone(), - opts: self.inner.default_opts.clone(), - } - } - - /// Execute the given http request. - async fn http_request(&self, request: Request) -> Result { - let mut last_res: Option> = None; - for n in 0..self.inner.n_retries { - let res = self.inner.http.execute(request.try_clone().unwrap()).await; - let emsg = match &res { - Ok(response) => { - let status = response.status(); - // Immediately return in case of success or unrecoverable status code - if status.is_success() || !status.is_server_error() { - return res; - } - status.to_string() - } - Err(e) => { - // Immediately return in case of unrecoverable error - if !e.is_timeout() && !e.is_connect() { - return res; - } - e.to_string() - } - }; - - let ms = util::retry_delay(n, 1000, 60000, 3); - warn!("Retry attempt #{}. Error: {}. Waiting {} ms", n, emsg, ms); - tokio::time::sleep(std::time::Duration::from_millis(ms.into())).await; - - last_res = Some(res); - } - last_res.unwrap() - } - - /// Execute the given http request, returning an error in case of a - /// non-successful status code. - async fn http_request_estatus(&self, request: Request) -> Result { - Ok(self.http_request(request).await?.error_for_status()?) - } - - /// Execute the given http request, returning the response body as a string. - async fn http_request_txt(&self, request: Request) -> Result { - Ok(self.http_request_estatus(request).await?.text().await?) - } - - /// Extract the current version of the YouTube desktop client from the website. - async fn extract_desktop_client_version(&self) -> Result { - let from_swjs = async { - let swjs = self - .http_request_txt( - self.inner - .http - .get("https://www.youtube.com/sw.js") - .header(header::ORIGIN, "https://www.youtube.com") - .header(header::REFERER, "https://www.youtube.com") - .header(header::COOKIE, self.inner.consent_cookie.to_owned()) - .build() - .unwrap(), - ) - .await - .context("Failed to download sw.js")?; - - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) - .ok_or_else(|| anyhow!("Could not find desktop client version in sw.js")) - }; - - let from_html = async { - let html = self - .http_request_txt( - self.inner - .http - .get("https://www.youtube.com/results?search_query=") - .build() - .unwrap(), - ) - .await - .context("Failed to get YT Desktop page")?; - - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1) - .ok_or_else(|| anyhow!("Could not find desktop client version on html page")) - }; - - match from_swjs.await { - Ok(client_version) => Ok(client_version), - Err(_) => from_html.await, - } - } - - /// Extract the current version of the YouTube Music desktop client from the website. - async fn extract_music_client_version(&self) -> Result { - let from_swjs = async { - let swjs = self - .http_request_txt( - self.inner - .http - .get("https://music.youtube.com/sw.js") - .header(header::ORIGIN, "https://music.youtube.com") - .header(header::REFERER, "https://music.youtube.com") - .header(header::COOKIE, self.inner.consent_cookie.to_owned()) - .build() - .unwrap(), - ) - .await - .context("Failed to download sw.js")?; - - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) - .ok_or_else(|| anyhow!("Could not find desktop client version in sw.js")) - }; - - let from_html = async { - let html = self - .http_request_txt( - self.inner - .http - .get("https://music.youtube.com") - .build() - .unwrap(), - ) - .await - .context("Failed to get YT Desktop page")?; - - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1) - .ok_or_else(|| anyhow!("Could not find desktop client version on html page")) - }; - - match from_swjs.await { - Ok(client_version) => Ok(client_version), - Err(_) => from_html.await, - } - } - - /// Get the current version of the YouTube web client from the following sources - /// - /// 1. from cache - /// 2. from YouTube's service worker script (`sw.js`) - /// 3. from the YouTube website - /// 4. fall back to the hardcoded version - async fn get_desktop_client_version(&self) -> String { - let mut cache = self.inner.cache.lock().await; - - match cache.desktop_client.get() { - Some(cdata) => cdata.version.to_owned(), - None => match self.extract_desktop_client_version().await { - Ok(version) => { - cache.desktop_client = CacheEntry::from(ClientData { - version: version.to_owned(), - }); - self.store_cache(&cache); - version - } - Err(e) => { - warn!("{}, falling back to hardcoded version", e); - DESKTOP_CLIENT_VERSION.to_owned() - } - }, - } - } - - /// Get the current version of the YouTube Music web client from the following sources - /// - /// 1. from cache - /// 2. from YouTube Music's service worker script (`sw.js`) - /// 3. from the YouTube Music website - /// 4. fall back to the hardcoded version - async fn get_music_client_version(&self) -> String { - let mut cache = self.inner.cache.lock().await; - - match cache.music_client.get() { - Some(cdata) => cdata.version.to_owned(), - None => match self.extract_music_client_version().await { - Ok(version) => { - cache.music_client = CacheEntry::from(ClientData { - version: version.to_owned(), - }); - self.store_cache(&cache); - version - } - Err(e) => { - warn!("{}, falling back to hardcoded version", e); - DESKTOP_MUSIC_CLIENT_VERSION.to_owned() - } - }, - } - } - - /// Instantiate a new deobfuscator from either cached or extracted YouTube JavaScript code. - async fn get_deobf(&self) -> Result { - let mut cache = self.inner.cache.lock().await; - - match cache.deobf.get() { - Some(deobf) => Ok(Deobfuscator::from(deobf.to_owned())), - None => { - let deobf = Deobfuscator::new(self.inner.http.clone()).await?; - cache.deobf = CacheEntry::from(deobf.get_data()); - self.store_cache(&cache); - Ok(deobf) - } - } - } - - /// Write the given cache data to the storage backend. - fn store_cache(&self, cache: &CacheData) { - if let Some(storage) = &self.inner.storage { - match serde_json::to_string(cache) { - Ok(data) => storage.write(&data), - Err(e) => error!("Could not serialize cache. Error: {}", e), - } - } - } -} - -impl RustyPipeQuery { - /// Set the language parameter used when accessing the YouTube API - /// This will change multilanguage video titles, descriptions and textual dates - pub fn lang(mut self, lang: Language) -> Self { - self.opts.lang = lang; - self - } - - /// Set the country parameter used when accessing the YouTube API. - /// This will change trends and recommended content. - pub fn country(mut self, country: Country) -> Self { - self.opts.country = country; - self - } - - /// Generate a report on every operation. - /// This should only be used for debugging. - pub fn report(mut self) -> Self { - self.opts.report = true; - self - } - - /// Enable strict mode, causing operations to fail if there - /// are warnings during deserialization (e.g. invalid items). - /// This should only be used for testing. - pub fn strict(mut self) -> Self { - self.opts.strict = true; - self - } - - /// Create a new context object, which is included in every request to - /// the YouTube API and contains language, country and device parameters. - /// - /// # Parameters - /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...) - /// - `localized`: Whether to include the configured language and country - async fn get_context(&self, ctype: ClientType, localized: bool) -> YTContext { - let hl = match localized { - true => self.opts.lang, - false => Language::En, - }; - let gl = match localized { - true => self.opts.country, - false => Country::Us, - }; - - match ctype { - ClientType::Desktop => YTContext { - client: ClientInfo { - client_name: "WEB".to_owned(), - client_version: self.client.get_desktop_client_version().await, - client_screen: None, - device_model: None, - platform: "DESKTOP".to_owned(), - original_url: Some("https://www.youtube.com/".to_owned()), - hl, - gl, - }, - request: Some(RequestYT::default()), - user: User::default(), - third_party: None, - }, - ClientType::DesktopMusic => YTContext { - client: ClientInfo { - client_name: "WEB_REMIX".to_owned(), - client_version: self.client.get_music_client_version().await, - client_screen: None, - device_model: None, - platform: "DESKTOP".to_owned(), - original_url: Some("https://music.youtube.com/".to_owned()), - hl, - gl, - }, - request: Some(RequestYT::default()), - user: User::default(), - third_party: None, - }, - ClientType::TvHtml5Embed => YTContext { - client: ClientInfo { - client_name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER".to_owned(), - client_version: TVHTML5_CLIENT_VERSION.to_owned(), - client_screen: Some("EMBED".to_owned()), - device_model: None, - platform: "TV".to_owned(), - original_url: None, - hl, - gl, - }, - request: Some(RequestYT::default()), - user: User::default(), - third_party: Some(ThirdParty { - embed_url: "https://www.youtube.com/".to_owned(), - }), - }, - ClientType::Android => YTContext { - client: ClientInfo { - client_name: "ANDROID".to_owned(), - client_version: MOBILE_CLIENT_VERSION.to_owned(), - client_screen: None, - device_model: None, - platform: "MOBILE".to_owned(), - original_url: None, - hl, - gl, - }, - request: None, - user: User::default(), - third_party: None, - }, - ClientType::Ios => YTContext { - client: ClientInfo { - client_name: "IOS".to_owned(), - client_version: MOBILE_CLIENT_VERSION.to_owned(), - client_screen: None, - device_model: Some(IOS_DEVICE_MODEL.to_owned()), - platform: "MOBILE".to_owned(), - original_url: None, - hl, - gl, - }, - request: None, - user: User::default(), - third_party: None, - }, - } - } - - /// Create a new Reqwest HTTP request builder with the URL and headers required - /// for accessing the YouTube API - /// - /// # Parameters - /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...) - /// - `method`: HTTP method - /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/?key=...`) - async fn request_builder( - &self, - ctype: ClientType, - method: Method, - endpoint: &str, - ) -> RequestBuilder { - match ctype { - ClientType::Desktop => self - .client - .inner - .http - .request( - method, - format!( - "{}{}?key={}{}", - YOUTUBEI_V1_URL, endpoint, DESKTOP_API_KEY, DISABLE_PRETTY_PRINT_PARAMETER - ), - ) - .header(header::ORIGIN, "https://www.youtube.com") - .header(header::REFERER, "https://www.youtube.com") - .header(header::COOKIE, self.client.inner.consent_cookie.to_owned()) - .header("X-YouTube-Client-Name", "1") - .header( - "X-YouTube-Client-Version", - self.client.get_desktop_client_version().await, + async fn request_builder(&self, method: Method, endpoint: &str) -> RequestBuilder { + self.http + .request( + method, + format!( + "{}{}?key={}{}", + YOUTUBEI_V1_URL, endpoint, DESKTOP_API_KEY, DISABLE_PRETTY_PRINT_PARAMETER ), - ClientType::DesktopMusic => self - .client - .inner - .http - .request( - method, - format!( - "{}{}?key={}{}", - YOUTUBE_MUSIC_V1_URL, - endpoint, - DESKTOP_MUSIC_API_KEY, - DISABLE_PRETTY_PRINT_PARAMETER - ), - ) - .header(header::ORIGIN, "https://music.youtube.com") - .header(header::REFERER, "https://music.youtube.com") - .header(header::COOKIE, self.client.inner.consent_cookie.to_owned()) - .header("X-YouTube-Client-Name", "67") - .header( - "X-YouTube-Client-Version", - self.client.get_music_client_version().await, - ), - ClientType::TvHtml5Embed => self - .client - .inner - .http - .request( - method, - format!( - "{}{}?key={}{}", - YOUTUBEI_V1_URL, endpoint, DESKTOP_API_KEY, DISABLE_PRETTY_PRINT_PARAMETER - ), - ) - .header(header::ORIGIN, "https://www.youtube.com") - .header(header::REFERER, "https://www.youtube.com") - .header("X-YouTube-Client-Name", "1") - .header("X-YouTube-Client-Version", TVHTML5_CLIENT_VERSION), - ClientType::Android => self - .client - .inner - .http - .request( - method, - format!( - "{}{}?key={}{}", - YOUTUBEI_V1_GAPIS_URL, - endpoint, - ANDROID_API_KEY, - DISABLE_PRETTY_PRINT_PARAMETER - ), - ) - .header( - header::USER_AGENT, - format!( - "com.google.android.youtube/{} (Linux; U; Android 12; {}) gzip", - MOBILE_CLIENT_VERSION, self.opts.country - ), - ) - .header("X-Goog-Api-Format-Version", "2"), - ClientType::Ios => self - .client - .inner - .http - .request( - method, - format!( - "{}{}?key={}{}", - YOUTUBEI_V1_GAPIS_URL, - endpoint, - IOS_API_KEY, - DISABLE_PRETTY_PRINT_PARAMETER - ), - ) - .header( - header::USER_AGENT, - format!( - "com.google.ios.youtube/{} ({}; U; CPU iOS 15_4 like Mac OS X; {})", - MOBILE_CLIENT_VERSION, IOS_DEVICE_MODEL, self.opts.country - ), - ) - .header("X-Goog-Api-Format-Version", "2"), + ) + .header(header::ORIGIN, "https://www.youtube.com") + .header(header::REFERER, "https://www.youtube.com") + .header("X-YouTube-Client-Name", "1") + .header("X-YouTube-Client-Version", TVHTML5_CLIENT_VERSION) + } + + fn http_client(&self) -> Client { + self.http.clone() + } + + fn get_type(&self) -> ClientType { + ClientType::TvHtml5Embed + } +} + +impl TvHtml5EmbedClient { + fn new(localization: Arc) -> Self { + let http = ClientBuilder::new() + .user_agent(DEFAULT_UA) + .gzip(true) + .brotli(true) + .build() + .expect("unable to build the HTTP client"); + + Self { localization, http } + } +} + +pub struct DesktopMusicClient { + localization: Arc, + http: Client, + cache: Cache, + consent_cookie: String, +} + +#[async_trait] +impl YTClient for DesktopMusicClient { + async fn get_context(&self, localized: bool) -> ContextYT { + ContextYT { + client: ClientInfo { + client_name: "WEB_REMIX".to_owned(), + client_version: self.get_client_version().await, + client_screen: None, + device_model: None, + platform: "DESKTOP".to_owned(), + original_url: Some("https://music.youtube.com/".to_owned()), + hl: match localized { + true => self.localization.language, + false => Language::En, + }, + gl: match localized { + true => self.localization.content_country, + false => Country::Us, + }, + }, + request: Some(RequestYT::default()), + user: User::default(), + third_party: None, } } - /// Execute a request to the YouTube API, then deobfuscate and map the response. - /// - /// Creates a report in case of failure for easy debugging. - /// - /// # Parameters - /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...) - /// - `operation`: Name of the RustyPipe operation (only for reporting, e.g. `get_player`) - /// - `id`: ID of the requested entity (Video ID, Channel ID, ...). - /// The ID is included in reports and is also passed to the mapper for validating the response. - /// Set it to an empty string if you are not requesting an entity with an ID. - /// - `method`: HTTP method - /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/?key=...`) - /// - `body`: Serializable request body to be sent in json format - /// - `deobf`: Deobfuscator (is passed to the mapper to deobfuscate stream URLs). - #[allow(clippy::too_many_arguments)] - async fn execute_request_deobf< - R: DeserializeOwned + MapResponse + Debug, - M, - B: Serialize + ?Sized, - >( - &self, - ctype: ClientType, - operation: &str, - id: &str, - method: Method, - endpoint: &str, - body: &B, - deobf: Option<&Deobfuscator>, - ) -> Result { - let request = self - .request_builder(ctype, method.clone(), endpoint) + async fn request_builder(&self, method: Method, endpoint: &str) -> RequestBuilder { + self.http + .request( + method, + format!( + "{}{}?key={}{}", + YOUTUBE_MUSIC_V1_URL, + endpoint, + DESKTOP_MUSIC_API_KEY, + DISABLE_PRETTY_PRINT_PARAMETER + ), + ) + .header(header::ORIGIN, "https://music.youtube.com") + .header(header::REFERER, "https://music.youtube.com") + .header(header::COOKIE, self.consent_cookie.to_owned()) + .header("X-YouTube-Client-Name", "67") + .header("X-YouTube-Client-Version", self.get_client_version().await) + } + + fn http_client(&self) -> Client { + self.http.clone() + } + + fn get_type(&self) -> ClientType { + ClientType::DesktopMusic + } +} + +impl DesktopMusicClient { + fn new(localization: Arc, cache: Cache) -> Self { + let mut rng = rand::thread_rng(); + + let http = ClientBuilder::new() + .user_agent(DEFAULT_UA) + .gzip(true) + .brotli(true) + .build() + .expect("unable to build the HTTP client"); + + Self { + localization, + http, + cache, + consent_cookie: format!( + "{}={}{}", + CONSENT_COOKIE, + CONSENT_COOKIE_YES, + rng.gen_range(100..1000) + ), + } + } + + async fn extract_client_version(http: Client, consent_cookie: &str) -> Result { + async fn extract_client_version_from_swjs( + http: Client, + consent_cookie: &str, + ) -> Result { + let swjs = exec_request_text( + http.clone(), + http.get("https://music.youtube.com/sw.js") + .header(header::ORIGIN, "https://www.youtube.com") + .header(header::REFERER, "https://www.youtube.com") + .header(header::COOKIE, consent_cookie) + .build() + .unwrap(), + ) .await - .json(body) - .build()?; + .context("Failed to download sw.js")?; - let request_url = request.url().to_string(); - let request_headers = request.headers().to_owned(); - - let response = self.client.inner.http.execute(request).await?; - - let status = response.status(); - let resp_str = response.text().await?; - - let create_report = |level: Level, error: Option, msgs: Vec| { - if let Some(reporter) = &self.client.inner.reporter { - let report = Report { - package: "rustypipe".to_owned(), - version: "0.1.0".to_owned(), - date: chrono::Local::now(), - level, - operation: format!("{}({})", operation, id), - error, - msgs, - deobf_data: deobf.map(Deobfuscator::get_data), - http_request: crate::report::HTTPRequest { - url: request_url, - method: method.to_string(), - req_header: request_headers - .iter() - .map(|(k, v)| { - (k.to_string(), v.to_str().unwrap_or_default().to_owned()) - }) - .collect(), - req_body: serde_json::to_string(body).unwrap_or_default(), - status: status.into(), - resp_body: resp_str.to_owned(), - }, - }; - - reporter.report(&report); - } - }; - - if status.is_client_error() || status.is_server_error() { - let e = anyhow!("Server responded with error code {}", status); - create_report(Level::ERR, Some(e.to_string()), vec![]); - return Err(e); + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) + .ok_or(anyhow!("Could not find music client version in sw.js")) } - match serde_json::from_str::(&resp_str) { - Ok(deserialized) => match deserialized.map_response(id, self.opts.lang, deobf) { - Ok(mapres) => { - if !mapres.warnings.is_empty() { - create_report( - Level::WRN, - Some("Warnings during deserialization/mapping".to_owned()), - mapres.warnings, - ); + async fn extract_client_version_from_html(http: Client) -> Result { + let html = exec_request_text( + http.clone(), + http.get("https://music.youtube.com").build().unwrap(), + ) + .await + .context("Failed to get YT Music page")?; - if self.opts.strict { - bail!("Warnings during deserialization/mapping"); - } - } else if self.opts.report { - create_report(Level::DBG, None, vec![]); - } - Ok(mapres.c) - } - Err(e) => { - let emsg = "Could not map reponse"; - create_report(Level::ERR, Some(emsg.to_owned()), vec![e.to_string()]); - Err(e).context(emsg) - } - }, + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1) + .ok_or(anyhow!("Could not find music client version on html page")) + } + + match extract_client_version_from_swjs(http.clone(), consent_cookie).await { + Ok(client_version) => Ok(client_version), + Err(_) => extract_client_version_from_html(http).await, + } + } + + async fn get_client_version(&self) -> String { + let http = self.http.clone(); + let consent_cookie = self.consent_cookie.clone(); + + let client_data = self + .cache + .get_music_client_data(async move { + let client_version = Self::extract_client_version(http, &consent_cookie).await?; + Ok(ClientData { + version: client_version, + }) + }) + .await; + + match client_data { + Ok(client_data) => client_data.version, Err(e) => { - let emsg = "Could not deserialize response"; - create_report(Level::ERR, Some(emsg.to_owned()), vec![e.to_string()]); - Err(e).context(emsg) + warn!("{}", e); + DESKTOP_MUSIC_CLIENT_VERSION.to_owned() } } } - - /// Execute a request to the YouTube API, then map the response. - /// - /// Creates a report in case of failure for easy debugging. - /// - /// # Parameters - /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...) - /// - `operation`: Name of the RustyPipe operation (only for reporting, e.g. `get_player`) - /// - `id`: ID of the requested entity (Video ID, Channel ID, ...). - /// The ID is included in reports and is also passed to the mapper for validating the response. - /// Set it to an empty string if you are not requesting an entity with an ID. - /// - `method`: HTTP method - /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/?key=...`) - /// - `body`: Serializable request body to be sent in json format - async fn execute_request< - R: DeserializeOwned + MapResponse + Debug, - M, - B: Serialize + ?Sized, - >( - &self, - ctype: ClientType, - operation: &str, - id: &str, - method: Method, - endpoint: &str, - body: &B, - ) -> Result { - self.execute_request_deobf::(ctype, operation, id, method, endpoint, body, None) - .await - } -} - -/// Implement this for YouTube API response structs that need to be mapped to -/// RustyPipe models. -trait MapResponse { - /// Map the YouTube API response structs to a RustyPipe model. - /// - /// Returns an error if crucial data required for the model could not be extracted. - /// - /// Returns a `MapResult` with warnings if there were issues with the deserializing/mapping, - /// but the resulting data is still usable. - /// - /// # Parameters - /// - `id`: The ID of the requested entity (Video ID, Channel ID, ...). If possible, assert - /// that the returned entity matches this ID and return an error instead. - /// - `lang`: Language of the request. Used for mapping localized information like dates. - /// - `deobf`: Deobfuscator (if passed to the `execute_request_deobf` method) - fn map_response( - self, - id: &str, - lang: Language, - deobf: Option<&Deobfuscator>, - ) -> Result>; } #[cfg(test)] mod tests { - // use super::*; + use super::*; + use test_log::test; + + static CLIENT_VERSION_REGEX: Lazy = + Lazy::new(|| Regex::new(r#"^\d+\.\d{8}\.\d{2}\.\d{2}"#).unwrap()); + + #[test(tokio::test)] + async fn t_extract_desktop_client_version() { + let rt = RustyTube::new(); + let client = rt.desktop_client; + let version = + DesktopClient::extract_client_version(client.http.clone(), &client.consent_cookie) + .await + .unwrap(); + + assert!(CLIENT_VERSION_REGEX.is_match(&version).unwrap()); + + // Client version changes often, + // notify during test so the hardcoded version can be updated + if version != DESKTOP_CLIENT_VERSION { + println!( + "INFO: YT Desktop Client was updated, new version: {}", + version + ); + } + } + + #[test(tokio::test)] + async fn t_extract_desktop_music_client_version() { + let rt = RustyTube::new(); + let client = rt.desktop_music_client; + let version = + DesktopMusicClient::extract_client_version(client.http.clone(), &client.consent_cookie) + .await + .unwrap(); + + assert!(CLIENT_VERSION_REGEX.is_match(&version).unwrap()); + + // Client version changes often, + // notify during test so the hardcoded version can be updated + if version != DESKTOP_MUSIC_CLIENT_VERSION { + println!( + "INFO: YT Desktop Music Client was updated, new version: {}", + version + ); + } + } } diff --git a/src/client/player.rs b/src/client/player.rs index c91f2b9..2044caa 100644 --- a/src/client/player.rs +++ b/src/client/player.rs @@ -1,33 +1,25 @@ use std::{ borrow::Cow, collections::{BTreeMap, HashMap}, + sync::Arc, }; use anyhow::{anyhow, bail, Result}; use chrono::{Local, NaiveDateTime, NaiveTime, TimeZone}; use fancy_regex::Regex; +use log::{error, warn}; use once_cell::sync::Lazy; -use reqwest::{Method, Url}; +use reqwest::Method; use serde::Serialize; +use url::Url; -use crate::{ - deobfuscate::Deobfuscator, - model::{ - AudioCodec, AudioFormat, AudioStream, AudioTrack, Channel, Language, Subtitle, VideoCodec, - VideoFormat, VideoInfo, VideoPlayer, VideoStream, - }, - util, -}; - -use super::{ - response::{self, player}, - ClientType, MapResponse, MapResult, RustyPipeQuery, YTContext, -}; +use super::{response, ClientType, ContextYT, RustyTube, YTClient}; +use crate::{client::response::player, deobfuscate::Deobfuscator, model::*, util}; #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QPlayer { - context: YTContext, + context: ContextYT, /// Website playback context #[serde(skip_serializing_if = "Option::is_none")] playback_context: Option, @@ -57,211 +49,58 @@ struct QContentPlaybackContext { referer: String, } -impl RustyPipeQuery { - pub async fn get_player(self, video_id: &str, client_type: ClientType) -> Result { - let q1 = self.clone(); - let t_context = tokio::spawn(async move { q1.get_context(client_type, false).await }); - let q2 = self.client.clone(); - let t_deobf = tokio::spawn(async move { q2.get_deobf().await }); +impl RustyTube { + pub async fn get_player(&self, video_id: &str, client_type: ClientType) -> Result { + let client = self.get_ytclient(client_type); + let (context, deobf) = tokio::join!( + client.get_context(false), + Deobfuscator::from_fetched_info(client.http_client(), self.cache.clone()) + ); + let deobf = deobf?; + let request_body = build_request_body(client.clone(), &deobf, context, video_id); - let (context, deobf) = tokio::join!(t_context, t_deobf); - let context = context.unwrap(); - let deobf = deobf.unwrap()?; + let resp = client + .request_builder(Method::POST, "player") + .await + .json(&request_body) + .send() + .await? + .error_for_status()?; - let request_body = if client_type.is_web() { - QPlayer { - context, - playback_context: Some(QPlaybackContext { - content_playback_context: QContentPlaybackContext { - signature_timestamp: deobf.get_sts(), - referer: format!("https://www.youtube.com/watch?v={}", video_id), - }, - }), - cpn: None, - video_id: video_id.to_owned(), - content_check_ok: true, - racy_check_ok: true, - } - } else { - QPlayer { - context, - playback_context: None, - cpn: Some(util::generate_content_playback_nonce()), - video_id: video_id.to_owned(), - content_check_ok: true, - racy_check_ok: true, - } - }; - - self.execute_request_deobf::( - client_type, - "get_player", - video_id, - Method::POST, - "player", - &request_body, - Some(&deobf), - ) - .await + let player_response = resp.json::().await?; + map_player_data(player_response, &deobf) } } -impl MapResponse for response::Player { - fn map_response( - self, - id: &str, - _lang: Language, - deobf: Option<&Deobfuscator>, - ) -> Result> { - let deobf = deobf.unwrap(); - let mut warnings = vec![]; - - // Check playability status - match self.playability_status { - response::player::PlayabilityStatus::Ok { live_streamability } => { - if live_streamability.is_some() { - bail!("Active livestreams are not supported") - } - } - response::player::PlayabilityStatus::Unplayable { reason } => { - bail!("Video is unplayable. Reason: {}", reason) - } - response::player::PlayabilityStatus::LoginRequired { reason } => { - bail!("Playback requires login. Reason: {}", reason) - } - response::player::PlayabilityStatus::LiveStreamOffline { reason } => { - bail!("Livestream is offline. Reason: {}", reason) - } - response::player::PlayabilityStatus::Error { reason } => { - bail!("Video was deleted. Reason: {}", reason) - } - }; - - let mut streaming_data = some_or_bail!( - self.streaming_data, - Err(anyhow!("No streaming data was returned")) - ); - let video_details = some_or_bail!( - self.video_details, - Err(anyhow!("No video details were returned")) - ); - let microformat = self.microformat.map(|m| m.player_microformat_renderer); - let (publish_date, category, tags, is_family_safe) = - microformat.map_or((None, None, None, None), |m| { - ( - Local - .from_local_datetime(&NaiveDateTime::new( - m.publish_date, - NaiveTime::from_hms(0, 0, 0), - )) - .single(), - Some(m.category), - m.tags, - Some(m.is_family_safe), - ) - }); - - if video_details.video_id != id { - bail!( - "got wrong video id {}, expected {}", - video_details.video_id, - id - ); +fn build_request_body( + client: Arc, + deobf: &Deobfuscator, + context: ContextYT, + video_id: &str, +) -> QPlayer { + if client.get_type().is_web() { + QPlayer { + context, + playback_context: Some(QPlaybackContext { + content_playback_context: QContentPlaybackContext { + signature_timestamp: deobf.get_sts(), + referer: format!("https://www.youtube.com/watch?v={}", video_id), + }, + }), + cpn: None, + video_id: video_id.to_owned(), + content_check_ok: true, + racy_check_ok: true, } - - let video_info = VideoInfo { - id: video_details.video_id, - title: video_details.title, - description: video_details.short_description, - length: video_details.length_seconds, - thumbnails: video_details.thumbnail.unwrap_or_default().into(), - channel: Channel { - id: video_details.channel_id, - name: video_details.author, - }, - publish_date, - view_count: video_details.view_count, - keywords: match video_details.keywords { - Some(keywords) => keywords, - None => tags.unwrap_or_default(), - }, - category, - is_live_content: video_details.is_live_content, - is_family_safe, - }; - - let mut formats = streaming_data.formats.c; - formats.append(&mut streaming_data.adaptive_formats.c); - - warnings.append(&mut streaming_data.formats.warnings); - warnings.append(&mut streaming_data.adaptive_formats.warnings); - - let mut last_nsig: [String; 2] = ["".to_owned(), "".to_owned()]; - - let mut video_streams: Vec = Vec::new(); - let mut video_only_streams: Vec = Vec::new(); - let mut audio_streams: Vec = Vec::new(); - - for f in formats { - if f.format_type == player::FormatType::FormatStreamTypeOtf { - continue; - } - - match (f.is_video(), f.is_audio()) { - (true, true) => { - let mut map_res = map_video_stream(f, deobf, &mut last_nsig); - warnings.append(&mut map_res.warnings); - if let Some(c) = map_res.c { - video_streams.push(c); - }; - } - (true, false) => { - let mut map_res = map_video_stream(f, deobf, &mut last_nsig); - warnings.append(&mut map_res.warnings); - if let Some(c) = map_res.c { - video_only_streams.push(c); - }; - } - (false, true) => { - let mut map_res = map_audio_stream(f, deobf, &mut last_nsig); - warnings.append(&mut map_res.warnings); - if let Some(c) = map_res.c { - audio_streams.push(c); - }; - } - (false, false) => warnings.push(format!("invalid stream: itag {}", f.itag)), - } + } else { + QPlayer { + context, + playback_context: None, + cpn: Some(util::generate_content_playback_nonce()), + video_id: video_id.to_owned(), + content_check_ok: true, + racy_check_ok: true, } - - video_streams.sort(); - video_only_streams.sort(); - audio_streams.sort(); - - let mut subtitles = vec![]; - if let Some(captions) = self.captions { - for c in captions.player_captions_tracklist_renderer.caption_tracks { - let lang_auto = c.name.strip_suffix(" (auto-generated)"); - - subtitles.push(Subtitle { - url: c.base_url, - lang: c.language_code, - lang_name: lang_auto.unwrap_or(&c.name).to_owned(), - auto_generated: lang_auto.is_some(), - }) - } - } - - Ok(MapResult { - c: VideoPlayer { - info: video_info, - video_streams, - video_only_streams, - audio_streams, - subtitles, - expires_in_seconds: streaming_data.expires_in_seconds, - }, - warnings, - }) } } @@ -297,7 +136,7 @@ fn deobf_nsig( let nsig: String; match url_params.get("n") { Some(n) => { - nsig = if n == &last_nsig[0] { + nsig = if n.to_owned() == last_nsig[0] { last_nsig[1].to_owned() } else { let nsig = deobf.deobfuscate_nsig(n)?; @@ -318,192 +157,108 @@ fn map_url( signature_cipher: &Option, deobf: &Deobfuscator, last_nsig: &mut [String; 2], -) -> MapResult> { +) -> Option<(String, bool)> { let (url_base, mut url_params) = match url { - Some(url) => ok_or_bail!( - util::url_to_params(url), - MapResult { - c: None, - warnings: vec![format!("Could not parse url `{}`", url)] - } - ), + Some(url) => ok_or_bail!(util::url_to_params(url), None), None => match signature_cipher { Some(signature_cipher) => match cipher_to_url_params(signature_cipher, deobf) { Ok(res) => res, Err(e) => { - return MapResult { - c: None, - warnings: vec![format!( - "Could not deobfuscate signatureCipher `{}`: {}", - signature_cipher, e - )], - }; + error!("Could not deobfuscate signatureCipher: {}", e); + return None; } }, - None => { - return MapResult { - c: None, - warnings: vec!["stream contained neither url nor cipher".to_owned()], - } - } + None => return None, }, }; - let mut warnings = vec![]; let mut throttled = false; deobf_nsig(&mut url_params, deobf, last_nsig).unwrap_or_else(|e| { - warnings.push(format!( - "Could not deobfuscate nsig (params: {:?}): {}", - url_params, e - )); + warn!("Could not deobfuscate nsig: {}", e); throttled = true; }); - MapResult { - c: Some(( - ok_or_bail!( - Url::parse_with_params(url_base.as_str(), url_params.iter()), - MapResult { - c: None, - warnings: vec![format!( - "url could not be joined. url: `{}` params: {:?}", - url_base, url_params - )], - } - ) - .to_string(), - throttled, - )), - warnings, - } + Some(( + ok_or_bail!( + Url::parse_with_params(url_base.as_str(), url_params.iter()), + None + ) + .to_string(), + throttled, + )) } fn map_video_stream( - f: player::Format, + f: &player::Format, deobf: &Deobfuscator, last_nsig: &mut [String; 2], -) -> MapResult> { - let (mtype, codecs) = some_or_bail!( - parse_mime(&f.mime_type), - MapResult { - c: None, - warnings: vec![format!( - "Invalid mime type `{}` in video format {:?}", - &f.mime_type, &f - )] - } - ); - let map_res = map_url(&f.url, &f.signature_cipher, deobf, last_nsig); +) -> Option { + let (mtype, codecs) = some_or_bail!(parse_mime(&f.mime_type), None); + let (url, throttled) = + some_or_bail!(map_url(&f.url, &f.signature_cipher, deobf, last_nsig), None); - match map_res.c { - Some((url, throttled)) => MapResult { - c: Some(VideoStream { - url, - itag: f.itag, - bitrate: f.bitrate, - average_bitrate: f.average_bitrate.unwrap_or(f.bitrate), - size: f.content_length, - index_range: f.index_range, - init_range: f.init_range, - // Note that the format has already been verified using - // is_video(), so these unwraps are safe - width: f.width.unwrap(), - height: f.height.unwrap(), - fps: f.fps.unwrap(), - quality: f.quality_label.unwrap(), - hdr: f.color_info.unwrap_or_default().primaries - == player::Primaries::ColorPrimariesBt2020, - mime: f.mime_type.to_owned(), - format: some_or_bail!( - get_video_format(mtype), - MapResult { - c: None, - warnings: vec![format!("invalid video format. itag: {}", f.itag)] - } - ), - codec: get_video_codec(codecs), - throttled, - }), - warnings: map_res.warnings, - }, - None => MapResult { - c: None, - warnings: map_res.warnings, - }, - } + Some(VideoStream { + url, + itag: f.itag, + bitrate: f.bitrate, + average_bitrate: f.average_bitrate, + size: f.content_length, + index_range: f.index_range.clone(), + init_range: f.init_range.clone(), + width: some_or_bail!(f.width, None), + height: some_or_bail!(f.height, None), + fps: some_or_bail!(f.fps, None), + quality: some_or_bail!(f.quality_label.clone(), None), + hdr: f.color_info.clone().unwrap_or_default().primaries + == player::Primaries::ColorPrimariesBt2020, + mime: f.mime_type.to_owned(), + format: some_or_bail!(get_video_format(mtype), None), + codec: get_video_codec(codecs), + throttled, + }) } fn map_audio_stream( - f: player::Format, + f: &player::Format, deobf: &Deobfuscator, last_nsig: &mut [String; 2], -) -> MapResult> { +) -> Option { static LANG_PATTERN: Lazy = Lazy::new(|| Regex::new(r#"^([a-z]{2})\."#).unwrap()); - let (mtype, codecs) = some_or_bail!( - parse_mime(&f.mime_type), - MapResult { - c: None, - warnings: vec![format!( - "Invalid mime type `{}` in video format {:?}", - &f.mime_type, &f - )] - } - ); - let map_res = map_url(&f.url, &f.signature_cipher, deobf, last_nsig); + let (mtype, codecs) = some_or_bail!(parse_mime(&f.mime_type), None); + let (url, throttled) = + some_or_bail!(map_url(&f.url, &f.signature_cipher, deobf, last_nsig), None); - match map_res.c { - Some((url, throttled)) => MapResult { - c: Some(AudioStream { - url, - itag: f.itag, - bitrate: f.bitrate, - average_bitrate: f.average_bitrate.unwrap_or(f.bitrate), - size: f.content_length.unwrap(), - index_range: f.index_range, - init_range: f.init_range, - mime: f.mime_type.to_owned(), - format: some_or_bail!( - get_audio_format(mtype), - MapResult { - c: None, - warnings: vec![format!("invalid audio format. itag: {}", f.itag)] - } - ), - codec: get_audio_codec(codecs), - throttled, - track: match f.audio_track { - Some(t) => { - let lang = LANG_PATTERN - .captures(&t.id) - .ok() - .flatten() - .map(|m| m.get(1).unwrap().as_str().to_owned()); - - Some(AudioTrack { - id: t.id, - lang, - lang_name: t.display_name, - is_default: t.audio_is_default, - }) - } - None => None, - }, - }), - warnings: map_res.warnings, - }, - None => MapResult { - c: None, - warnings: map_res.warnings, - }, - } + Some(AudioStream { + url, + itag: f.itag, + bitrate: f.bitrate, + average_bitrate: f.average_bitrate, + size: f.content_length, + index_range: f.index_range.to_owned(), + init_range: f.init_range.to_owned(), + mime: f.mime_type.to_owned(), + format: some_or_bail!(get_audio_format(mtype), None), + codec: get_audio_codec(codecs), + throttled, + track: f.audio_track.as_ref().map(|t| AudioTrack { + id: t.id.to_owned(), + lang: LANG_PATTERN + .captures(&t.id) + .ok() + .flatten() + .map(|m| m.get(1).unwrap().as_str().to_owned()), + lang_name: t.display_name.to_owned(), + is_default: t.audio_is_default, + }), + }) } fn parse_mime(mime: &str) -> Option<(&str, Vec<&str>)> { static PATTERN: Lazy = Lazy::new(|| Regex::new(r#"(\w+/\w+);\scodecs="([a-zA-Z-0-9.,\s]*)""#).unwrap()); - let captures = some_or_bail!(PATTERN.captures(mime).ok().flatten(), None); + let captures = some_or_bail!(PATTERN.captures(&mime).ok().flatten(), None); Some(( captures.get(1).unwrap().as_str(), captures @@ -558,11 +313,140 @@ fn get_audio_codec(codecs: Vec<&str>) -> AudioCodec { AudioCodec::Unknown } +fn map_player_data(response: response::Player, deobf: &Deobfuscator) -> Result { + // Check playability status + match response.playability_status { + response::player::PlayabilityStatus::Ok { live_streamability } => { + if live_streamability.is_some() { + bail!("Active livestreams are not supported") + } + } + response::player::PlayabilityStatus::Unplayable { reason } => { + bail!("Video is unplayable. Reason: {}", reason) + } + response::player::PlayabilityStatus::LoginRequired { reason } => { + bail!("Playback requires login. Reason: {}", reason) + } + response::player::PlayabilityStatus::LiveStreamOffline { reason } => { + bail!("Livestream is offline. Reason: {}", reason) + } + response::player::PlayabilityStatus::Error { reason } => { + bail!("Video was deleted. Reason: {}", reason) + } + }; + + let streaming_data = some_or_bail!( + response.streaming_data, + Err(anyhow!("No streaming data was returned")) + ); + let video_details = some_or_bail!( + response.video_details, + Err(anyhow!("No video details were returned")) + ); + let microformat = response.microformat.map(|m| m.player_microformat_renderer); + + let video_info = VideoInfo { + id: video_details.video_id, + title: video_details.title, + description: video_details.short_description, + length: video_details.length_seconds, + thumbnails: video_details + .thumbnail + .unwrap_or_default() + .thumbnails + .iter() + .map(|t| Thumbnail { + url: t.url.to_owned(), + height: t.height, + width: t.width, + }) + .collect(), + channel: Channel { + id: video_details.channel_id, + name: video_details.author, + }, + publish_date: microformat.as_ref().map(|m| { + let ndt = NaiveDateTime::new(m.publish_date, NaiveTime::from_hms(0, 0, 0)); + Local.from_local_datetime(&ndt).unwrap() + }), + view_count: video_details.view_count, + keywords: video_details + .keywords + .or_else(|| microformat.as_ref().map_or(None, |mf| mf.tags.clone())) + .unwrap_or_default(), + category: microformat.as_ref().map(|m| m.category.to_owned()), + is_live_content: video_details.is_live_content, + is_family_safe: microformat.as_ref().map(|m| m.is_family_safe), + }; + + let mut formats = streaming_data.formats.clone(); + formats.append(&mut streaming_data.adaptive_formats.clone()); + + let mut last_nsig: [String; 2] = ["".to_owned(), "".to_owned()]; + + let mut video_streams: Vec = Vec::new(); + let mut video_only_streams: Vec = Vec::new(); + let mut audio_streams: Vec = Vec::new(); + + for f in formats { + if f.format_type == player::FormatType::FormatStreamTypeOtf { + continue; + } + + match (f.is_video(), f.is_audio()) { + (true, true) => match map_video_stream(&f, deobf, &mut last_nsig) { + Some(stream) => video_streams.push(stream), + None => {} + }, + (true, false) => match map_video_stream(&f, deobf, &mut last_nsig) { + Some(stream) => video_only_streams.push(stream), + None => {} + }, + (false, true) => match map_audio_stream(&f, deobf, &mut last_nsig) { + Some(stream) => audio_streams.push(stream), + None => {} + }, + (false, false) => {} + } + } + + video_streams.sort(); + video_only_streams.sort(); + audio_streams.sort(); + + let subtitles = response.captions.map_or(vec![], |captions| { + captions + .player_captions_tracklist_renderer + .caption_tracks + .iter() + .map(|caption| { + let lang_auto = caption.name.strip_suffix(" (auto-generated)"); + + Subtitle { + url: caption.base_url.to_owned(), + lang: caption.language_code.to_owned(), + lang_name: lang_auto.unwrap_or(&caption.name).to_owned(), + auto_generated: lang_auto.is_some(), + } + }) + .collect() + }); + + Ok(VideoPlayer { + info: video_info, + video_streams, + video_only_streams, + audio_streams, + subtitles, + expires_in_seconds: streaming_data.expires_in_seconds, + }) +} + #[cfg(test)] mod tests { use std::{fs::File, io::BufReader, path::Path}; - use crate::{client::RustyPipe, deobfuscate::DeobfData}; + use crate::{cache::DeobfData, client::CLIENT_TYPES}; use super::*; use rstest::rstest; @@ -576,6 +460,59 @@ mod tests { }) }); + #[test_log::test(tokio::test)] + async fn download_response_testfiles() { + let tf_dir = Path::new("testfiles/player"); + let video_id = "pPvd8UxmSbQ"; + + let rt = RustyTube::new(); + + for client_type in CLIENT_TYPES { + let mut json_path = tf_dir.to_path_buf(); + json_path.push(format!("{:?}_video.json", client_type).to_lowercase()); + if json_path.exists() { + continue; + } + + let client = rt.get_ytclient(client_type); + let context = client.get_context(false).await; + + let request_body = build_request_body(client.clone(), &DEOBFUSCATOR, context, video_id); + + let resp = client + .request_builder(Method::POST, "player") + .await + .json(&request_body) + .send() + .await + .unwrap() + .error_for_status() + .unwrap(); + + let mut file = File::create(json_path).unwrap(); + let mut content = std::io::Cursor::new(resp.bytes().await.unwrap()); + std::io::copy(&mut content, &mut file).unwrap(); + } + } + + #[test_log::test(tokio::test)] + async fn download_model_testfiles() { + let tf_dir = Path::new("testfiles/player_model"); + let rt = RustyTube::new(); + + for (name, id) in [("multilanguage", "tVWWp1PqDus"), ("hdr", "LXb3EKWsInQ")] { + let mut json_path = tf_dir.to_path_buf(); + json_path.push(format!("{}.json", name).to_lowercase()); + if json_path.exists() { + continue; + } + + let player_data = rt.get_player(id, ClientType::Desktop).await.unwrap(); + let file = File::create(json_path).unwrap(); + serde_json::to_writer_pretty(file, &player_data).unwrap(); + } + } + #[rstest] #[case::desktop("desktop")] #[case::desktop_music("desktopmusic")] @@ -588,17 +525,10 @@ mod tests { let json_file = File::open(json_path).unwrap(); let resp: response::Player = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res = resp - .map_response("pPvd8UxmSbQ", Language::En, Some(&DEOBFUSCATOR)) - .unwrap(); + let player_data = map_player_data(resp, &DEOBFUSCATOR).unwrap(); - assert!( - map_res.warnings.is_empty(), - "deserialization/mapping warnings: {:?}", - map_res.warnings - ); let is_desktop = name == "desktop" || name == "desktopmusic"; - insta::assert_yaml_snapshot!(format!("map_player_data_{}", name), map_res.c, { + insta::assert_yaml_snapshot!(format!("map_player_data_{}", name), player_data, { ".info.publish_date" => insta::dynamic_redaction(move |value, _path| { if is_desktop { assert!(value.as_str().unwrap().starts_with("2019-05-30T00:00:00")); @@ -631,12 +561,8 @@ mod tests { #[case::ios(ClientType::Ios)] #[test_log::test(tokio::test)] async fn t_get_player(#[case] client_type: ClientType) { - let rp = RustyPipe::builder().strict().build(); - let player_data = rp - .query() - .get_player("n4tK7LYFxI0", client_type) - .await - .unwrap(); + let rt = RustyTube::new(); + let player_data = rt.get_player("n4tK7LYFxI0", client_type).await.unwrap(); // dbg!(&player_data); @@ -658,12 +584,7 @@ mod tests { assert_eq!(player_data.info.is_live_content, false); if client_type == ClientType::Desktop || client_type == ClientType::DesktopMusic { - assert!(player_data - .info - .publish_date - .unwrap() - .to_string() - .starts_with("2013-05-05 00:00:00")); + assert!(player_data.info.publish_date.unwrap().to_string().starts_with("2013-05-05 00:00:00")); assert_eq!(player_data.info.category.unwrap(), "Music"); assert_eq!(player_data.info.is_family_safe.unwrap(), true); } @@ -683,7 +604,7 @@ mod tests { // Bitrates may change between requests assert_approx(video.bitrate as f64, 1507068.0); assert_eq!(video.average_bitrate, 1345149); - assert_eq!(video.size.unwrap(), 43553412); + assert_eq!(video.size, 43553412); assert_eq!(video.width, 1280); assert_eq!(video.height, 720); assert_eq!(video.fps, 30); @@ -713,7 +634,7 @@ mod tests { assert_approx(video.bitrate as f64, 1340829.0); assert_approx(video.average_bitrate as f64, 1233444.0); - assert_approx(video.size.unwrap() as f64, 39936630.0); + assert_approx(video.size as f64, 39936630.0); assert_eq!(video.width, 1280); assert_eq!(video.height, 720); assert_eq!(video.fps, 30); @@ -740,20 +661,15 @@ mod tests { fn t_cipher_to_url() { let signature_cipher = "s=w%3DAe%3DA6aDNQLkViKS7LOm9QtxZJHKwb53riq9qEFw-ecBWJCAiA%3DcEg0tn3dty9jEHszfzh4Ud__bg9CEHVx4ix-7dKsIPAhIQRw8JQ0qOA&sp=sig&url=https://rr5---sn-h0jelnez.googlevideo.com/videoplayback%3Fexpire%3D1659376413%26ei%3Dvb7nYvH5BMK8gAfBj7ToBQ%26ip%3D2003%253Ade%253Aaf06%253A6300%253Ac750%253A1b77%253Ac74a%253A80e3%26id%3Do-AB_BABwrXZJN428ZwDxq5ScPn2AbcGODnRlTVhCQ3mj2%26itag%3D251%26source%3Dyoutube%26requiressl%3Dyes%26mh%3DhH%26mm%3D31%252C26%26mn%3Dsn-h0jelnez%252Csn-4g5ednsl%26ms%3Dau%252Conr%26mv%3Dm%26mvi%3D5%26pl%3D37%26initcwndbps%3D1588750%26spc%3DlT-Khi831z8dTejFIRCvCEwx_6romtM%26vprv%3D1%26mime%3Daudio%252Fwebm%26ns%3Db_Mq_qlTFcSGlG9RpwpM9xQH%26gir%3Dyes%26clen%3D3781277%26dur%3D229.301%26lmt%3D1655510291473933%26mt%3D1659354538%26fvip%3D5%26keepalive%3Dyes%26fexp%3D24001373%252C24007246%26c%3DWEB%26rbqsm%3Dfr%26txp%3D4532434%26n%3Dd2g6G2hVqWIXxedQ%26sparams%3Dexpire%252Cei%252Cip%252Cid%252Citag%252Csource%252Crequiressl%252Cspc%252Cvprv%252Cmime%252Cns%252Cgir%252Cclen%252Cdur%252Clmt%26lsparams%3Dmh%252Cmm%252Cmn%252Cms%252Cmv%252Cmvi%252Cpl%252Cinitcwndbps%26lsig%3DAG3C_xAwRQIgCKCGJ1iu4wlaGXy3jcJyU3inh9dr1FIfqYOZEG_MdmACIQCbungkQYFk7EhD6K2YvLaHFMjKOFWjw001_tLb0lPDtg%253D%253D"; let mut last_nsig: [String; 2] = ["".to_owned(), "".to_owned()]; - let map_res = map_url( + let (url, throttled) = map_url( &None, &Some(signature_cipher.to_owned()), &DEOBFUSCATOR, &mut last_nsig, - ); - let (url, throttled) = map_res.c.unwrap(); + ) + .unwrap(); assert_eq!(url, "https://rr5---sn-h0jelnez.googlevideo.com/videoplayback?c=WEB&clen=3781277&dur=229.301&ei=vb7nYvH5BMK8gAfBj7ToBQ&expire=1659376413&fexp=24001373%2C24007246&fvip=5&gir=yes&id=o-AB_BABwrXZJN428ZwDxq5ScPn2AbcGODnRlTVhCQ3mj2&initcwndbps=1588750&ip=2003%3Ade%3Aaf06%3A6300%3Ac750%3A1b77%3Ac74a%3A80e3&itag=251&keepalive=yes&lmt=1655510291473933&lsig=AG3C_xAwRQIgCKCGJ1iu4wlaGXy3jcJyU3inh9dr1FIfqYOZEG_MdmACIQCbungkQYFk7EhD6K2YvLaHFMjKOFWjw001_tLb0lPDtg%3D%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=hH&mime=audio%2Fwebm&mm=31%2C26&mn=sn-h0jelnez%2Csn-4g5ednsl&ms=au%2Conr&mt=1659354538&mv=m&mvi=5&n=XzXGSfGusw6OCQ&ns=b_Mq_qlTFcSGlG9RpwpM9xQH&pl=37&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRQIhAPIsKd7-xi4xVHEC9gb__dU4hzfzsHEj9ytd3nt0gEceAiACJWBcw-wFEq9qir35bwKHJZxtQ9mOL7SKiVkLQNDa6A%3D%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-Khi831z8dTejFIRCvCEwx_6romtM&txp=4532434&vprv=1"); assert_eq!(throttled, false); - assert!( - map_res.warnings.is_empty(), - "deserialization/mapping warnings: {:?}", - map_res.warnings - ); } } diff --git a/src/client/playlist.rs b/src/client/playlist.rs index 9056ab7..5b979a7 100644 --- a/src/client/playlist.rs +++ b/src/client/playlist.rs @@ -1,68 +1,85 @@ -use anyhow::{anyhow, bail, Result}; +use anyhow::{anyhow, Context, Result}; use reqwest::Method; use serde::Serialize; use crate::{ - deobfuscate::Deobfuscator, model::{Channel, Language, Playlist, Thumbnail, Video}, serializer::text::{PageType, TextLink}, timeago, util, }; -use super::{response, ClientType, MapResponse, MapResult, RustyPipeQuery, YTContext}; +use super::{response, ClientType, ContextYT, RustyTube}; #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QPlaylist { - context: YTContext, + context: ContextYT, browse_id: String, } #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QPlaylistCont { - context: YTContext, + context: ContextYT, continuation: String, } -impl RustyPipeQuery { - pub async fn get_playlist(self, playlist_id: &str) -> Result { - let context = self.get_context(ClientType::Desktop, true).await; +impl RustyTube { + pub async fn get_playlist(&self, playlist_id: &str) -> Result { + let client = self.get_ytclient(ClientType::Desktop); + let context = client.get_context(true).await; + let request_body = QPlaylist { context, browse_id: "VL".to_owned() + playlist_id, }; - self.execute_request::( - ClientType::Desktop, - "get_playlist", - playlist_id, - Method::POST, - "browse", - &request_body, - ) - .await + let resp = client + .request_builder(Method::POST, "browse") + .await + .json(&request_body) + .send() + .await? + .error_for_status()?; + + let resp_body = resp.text().await?; + let playlist_response = + serde_json::from_str::(&resp_body).context(resp_body)?; + + map_playlist(&playlist_response, self.localization.language) } - pub async fn get_playlist_cont(self, playlist: &mut Playlist) -> Result<()> { + pub async fn get_playlist_cont(&self, playlist: &mut Playlist) -> Result<()> { match &playlist.ctoken { Some(ctoken) => { - let context = self.get_context(ClientType::Desktop, true).await; + let client = self.get_ytclient(ClientType::Desktop); + let context = client.get_context(true).await; + let request_body = QPlaylistCont { context, continuation: ctoken.to_owned(), }; - let (mut videos, ctoken) = self - .execute_request::( - ClientType::Desktop, - "get_playlist_cont", - &playlist.id, - Method::POST, - "browse", - &request_body, - ) - .await?; + let resp = client + .request_builder(Method::POST, "browse") + .await + .json(&request_body) + .send() + .await? + .error_for_status()?; + + let cont_response = resp.json::().await?; + + let action = some_or_bail!( + cont_response + .on_response_received_actions + .iter() + .find(|a| a.append_continuation_items_action.target_id == playlist.id), + Err(anyhow!("no continuation action")) + ); + + let (mut videos, ctoken) = + map_playlist_items(&action.append_continuation_items_action.continuation_items); playlist.videos.append(&mut videos); playlist.ctoken = ctoken; @@ -78,179 +95,149 @@ impl RustyPipeQuery { } } -impl MapResponse for response::Playlist { - fn map_response( - self, - id: &str, - lang: Language, - _deobf: Option<&Deobfuscator>, - ) -> Result> { - let video_items = &some_or_bail!( +fn map_playlist(response: &response::Playlist, lang: Language) -> Result { + let video_items = &some_or_bail!( + some_or_bail!( some_or_bail!( - some_or_bail!( - self.contents - .two_column_browse_results_renderer - .contents - .get(0), - Err(anyhow!("twoColumnBrowseResultsRenderer empty")) - ) - .tab_renderer - .content - .section_list_renderer - .contents - .get(0), - Err(anyhow!("sectionListRenderer empty")) + response + .contents + .two_column_browse_results_renderer + .contents + .get(0), + Err(anyhow!("twoColumnBrowseResultsRenderer empty")) ) - .item_section_renderer + .tab_renderer + .content + .section_list_renderer .contents .get(0), - Err(anyhow!("itemSectionRenderer empty")) + Err(anyhow!("sectionListRenderer empty")) ) - .playlist_video_list_renderer - .contents; + .item_section_renderer + .contents + .get(0), + Err(anyhow!("itemSectionRenderer empty")) + ) + .playlist_video_list_renderer + .contents; - let (videos, ctoken) = map_playlist_items(&video_items.c); + let (videos, ctoken) = map_playlist_items(video_items); - let (thumbnails, last_update_txt) = match &self.sidebar { - Some(sidebar) => { - let primary = some_or_bail!( - sidebar.playlist_sidebar_renderer.items.get(0), - Err(anyhow!("no primary sidebar")) - ); + let (thumbnails, last_update_txt) = match &response.sidebar { + Some(sidebar) => { + let primary = some_or_bail!( + sidebar.playlist_sidebar_renderer.items.get(0), + Err(anyhow!("no primary sidebar")) + ); - ( - &primary - .playlist_sidebar_primary_info_renderer - .thumbnail_renderer - .playlist_video_thumbnail_renderer - .thumbnail - .thumbnails, - primary - .playlist_sidebar_primary_info_renderer - .stats - .get(2) - .map(|t| t.to_owned()), - ) - } - None => { - let header_banner = some_or_bail!( - &self.header.playlist_header_renderer.playlist_header_banner, - Err(anyhow!("no thumbnail found")) - ); - - let last_update_txt = self + ( + &primary + .playlist_sidebar_primary_info_renderer + .thumbnail_renderer + .playlist_video_thumbnail_renderer + .thumbnail + .thumbnails, + primary + .playlist_sidebar_primary_info_renderer + .stats + .get(2) + .map(|t| t.to_owned()), + ) + } + None => { + let header_banner = some_or_bail!( + &response .header .playlist_header_renderer - .byline - .get(1) - .map(|b| b.playlist_byline_renderer.text.to_owned()); + .playlist_header_banner, + Err(anyhow!("no thumbnail found")) + ); - ( - &header_banner - .hero_playlist_thumbnail_renderer - .thumbnail - .thumbnails, - last_update_txt, - ) - } - }; + let last_update_txt = response + .header + .playlist_header_renderer + .byline + .get(1) + .map(|b| b.playlist_byline_renderer.text.to_owned()); - let thumbnails = thumbnails - .iter() - .map(|t| Thumbnail { - url: t.url.to_owned(), - width: t.width, - height: t.height, - }) - .collect::>(); - - let n_videos = match ctoken { - Some(_) => { - ok_or_bail!( - util::parse_numeric(&self.header.playlist_header_renderer.num_videos_text), - Err(anyhow!("no video count")) - ) - } - None => videos.len() as u32, - }; - - let playlist_id = self.header.playlist_header_renderer.playlist_id; - if playlist_id != id { - bail!("got wrong playlist id {}, expected {}", playlist_id, id); - } - - let name = self.header.playlist_header_renderer.title; - let description = self.header.playlist_header_renderer.description_text; - - let channel = match self.header.playlist_header_renderer.owner_text { - Some(TextLink::Browse { - text, - page_type: PageType::Channel, - browse_id, - }) => Some(Channel { - id: browse_id, - name: text, - }), - _ => None, - }; - - let mut warnings = video_items.warnings.to_owned(); - let last_update = match &last_update_txt { - Some(textual_date) => { - let parsed = timeago::parse_textual_date_to_dt(lang, textual_date); - if parsed.is_none() { - warnings.push(format!("could not parse textual date `{}`", textual_date)); - } - parsed - } - None => None, - }; - - Ok(MapResult { - c: Playlist { - id: playlist_id, - name, - videos, - n_videos, - ctoken, - thumbnails, - description, - channel, - last_update, + ( + &header_banner + .hero_playlist_thumbnail_renderer + .thumbnail + .thumbnails, last_update_txt, + ) + } + }; + + let thumbnails = thumbnails + .iter() + .map(|t| Thumbnail { + url: t.url.to_owned(), + width: t.width, + height: t.height, + }) + .collect::>(); + + let n_videos = match ctoken { + Some(_) => { + ok_or_bail!( + util::parse_numeric(&response.header.playlist_header_renderer.num_videos_text), + Err(anyhow!("no video count")) + ) + } + None => videos.len() as u32, + }; + + let id = response + .header + .playlist_header_renderer + .playlist_id + .to_owned(); + let name = response.header.playlist_header_renderer.title.to_owned(); + let description = response + .header + .playlist_header_renderer + .description_text + .to_owned(); + + let channel = match &response.header.playlist_header_renderer.owner_text { + Some(owner_text) => match owner_text { + TextLink::Browse { + text, + page_type, + browse_id, + } => match page_type { + PageType::Channel => Some(Channel { + id: browse_id.to_owned(), + name: text.to_owned(), + }), + _ => None, }, - warnings, - }) - } -} + _ => None, + }, + None => None, + }; -impl MapResponse<(Vec