From dda2211e04438d7e6ca73743d07569a5aef98304 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 14 Sep 2022 00:04:51 +0200 Subject: [PATCH 1/5] refactored cache --- .gitignore | 5 +- Cargo.toml | 9 +- src/cache.rs | 369 ++++------------------------------------ src/client2/mod.rs | 328 ++++++++++++++++++++++++++++++++--- src/client2/player.rs | 17 +- src/client2/playlist.rs | 6 +- src/deobfuscate.rs | 59 ++++--- src/report.rs | 12 +- 8 files changed, 396 insertions(+), 409 deletions(-) diff --git a/.gitignore b/.gitignore index cb98b06..e96be3d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ /target /Cargo.lock -RustyPipeReports -RustyPipeCache.json -rusty-tube.json +rustypipe_reports +rustypipe_cache.json diff --git a/Cargo.toml b/Cargo.toml index b529a84..2e4e6f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,12 +7,16 @@ edition = "2021" members = [".", "cli"] [features] -default = ["default-tls"] +default = ["default-tls", "yaml"] +# Reqwest TLS default-tls = ["reqwest/default-tls"] rustls-tls-webpki-roots = ["reqwest/rustls-tls-webpki-roots"] rustls-tls-native-roots = ["reqwest/rustls-tls-native-roots"] +# Error reports in yaml format +yaml = ["serde_yaml"] + [dependencies] # quick-js = "0.4.1" quick-js = { path = "../quickjs-rs" } @@ -26,10 +30,9 @@ reqwest = {version = "0.11.11", default-features = false, features = ["json", "g tokio = {version = "1.20.0", features = ["macros", "fs", "process"]} serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.82" -serde_yaml = "0.9.11" +serde_yaml = {version = "0.9.11", optional = true} serde_with = {version = "2.0.0", features = ["json"] } rand = "0.8.5" -async-trait = "0.1.56" chrono = {version = "0.4.19", features = ["serde"]} chronoutil = "0.2.3" futures = "0.3.21" diff --git a/src/cache.rs b/src/cache.rs index 509f34b..e3339d3 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -1,364 +1,57 @@ use std::{ - fs::File, - future::Future, - io::BufReader, + fs, path::{Path, PathBuf}, - sync::Arc, }; -use anyhow::Result; -use chrono::{DateTime, Duration, Utc}; -use log::{error, info}; -use serde::{Deserialize, Serialize}; -use tokio::sync::Mutex; +use log::error; -#[derive(Default, Debug, Clone)] -pub struct Cache { - file: Option, - data: Arc>, +pub trait CacheStorage { + fn write(&self, data: &str); + fn read(&self) -> Option; } -#[derive(Default, Debug, Clone, Serialize, Deserialize)] -struct CacheData { - desktop_client: Option>, - music_client: Option>, - deobf: Option>, +pub struct FileStorage { + path: PathBuf, } -#[derive(Debug, Clone, Serialize, Deserialize)] -struct CacheEntry { - last_update: DateTime, - data: T, -} - -impl From for CacheEntry { - fn from(f: T) -> Self { +impl FileStorage { + pub fn new>(path: P) -> Self { Self { - last_update: Utc::now(), - data: f, + path: path.as_ref().to_path_buf(), } } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct ClientData { - pub version: String, +impl Default for FileStorage { + fn default() -> Self { + Self { + path: Path::new("rustypipe_cache.json").into(), + } + } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct DeobfData { - pub js_url: String, - pub sig_fn: String, - pub nsig_fn: String, - pub sts: String, -} - -impl Cache { - pub async fn get_desktop_client_data(&self, updater: F) -> Result - where - F: Future> + Send + 'static, - { - let mut cache = self.data.lock().await; - - if cache.desktop_client.is_none() - || cache.desktop_client.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24) - { - let cdata = updater.await?; - cache.desktop_client = Some(CacheEntry::from(cdata.clone())); - self.save(&cache); - Ok(cdata) - } else { - Ok(cache.desktop_client.as_ref().unwrap().data.clone()) - } +impl CacheStorage for FileStorage { + fn write(&self, data: &str) { + fs::write(&self.path, data).unwrap_or_else(|e| { + error!( + "Could not write cache to file `{}`. Error: {}", + self.path.to_string_lossy(), + e + ); + }); } - pub async fn get_music_client_data(&self, updater: F) -> Result - where - F: Future> + Send + 'static, - { - let mut cache = self.data.lock().await; - - if cache.music_client.is_none() - || cache.music_client.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24) - { - let cdata = updater.await?; - cache.music_client = Some(CacheEntry::from(cdata.clone())); - self.save(&cache); - Ok(cdata) - } else { - Ok(cache.music_client.as_ref().unwrap().data.clone()) - } - } - - pub async fn get_deobf_data(&self, updater: F) -> Result - where - F: Future> + Send + 'static, - { - let mut cache = self.data.lock().await; - if cache.deobf.is_none() - || cache.deobf.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24) - { - let deobf_data = updater.await?; - cache.deobf = Some(CacheEntry::from(deobf_data.clone())); - self.save(&cache); - Ok(deobf_data) - } else { - Ok(cache.deobf.as_ref().unwrap().data.clone()) - } - } - - pub async fn to_json(&self) -> Result { - let cache = self.data.lock().await; - Ok(serde_json::to_string(&cache.clone())?) - } - - pub async fn to_json_file>(&self, path: P) -> Result<()> { - let cache = self.data.lock().await; - Ok(serde_json::to_writer(&File::create(path)?, &cache.clone())?) - } - - pub fn from_json(json: &str) -> Self { - let data: CacheData = match serde_json::from_str(json) { - Ok(cd) => cd, + fn read(&self) -> Option { + match fs::read_to_string(&self.path) { + Ok(data) => Some(data), Err(e) => { error!( - "Could not load cache from json, falling back to default. Error: {}", + "Could not load cache from file `{}`. Error: {}", + self.path.to_string_lossy(), e ); - CacheData::default() + None } - }; - Cache { - data: Arc::new(Mutex::new(data)), - file: None, - } - } - - pub fn from_json_file>(path: P) -> Self { - let file = match File::open(path.as_ref()) { - Ok(file) => file, - Err(e) => { - if e.kind() == std::io::ErrorKind::NotFound { - info!( - "Cache json file at {} not found, will be created", - path.as_ref().to_string_lossy() - ) - } else { - error!( - "Could not open cache json file, falling back to default. Error: {}", - e - ); - } - return Cache { - file: Some(path.as_ref().to_path_buf()), - ..Default::default() - }; - } - }; - let data: CacheData = match serde_json::from_reader(BufReader::new(file)) { - Ok(data) => data, - Err(e) => { - error!( - "Could not load cache from json, falling back to default. Error: {}", - e - ); - return Cache { - file: Some(path.as_ref().to_path_buf()), - ..Default::default() - }; - } - }; - Cache { - data: Arc::new(Mutex::new(data)), - file: Some(path.as_ref().to_path_buf()), - } - } - - fn save(&self, cache: &CacheData) { - match self.file.as_ref() { - Some(file) => match File::create(file) { - Ok(file) => match serde_json::to_writer(file, cache) { - Ok(_) => {} - Err(e) => error!("Could not write cache to json. Error: {}", e), - }, - Err(e) => error!("Could not open cache json file. Error: {}", e), - }, - None => {} } } } - -#[cfg(test)] -mod tests { - use temp_testdir::TempDir; - - use super::*; - - #[tokio::test] - async fn test() { - let cache = Cache::default(); - - let desktop_c = cache - .get_desktop_client_data(async { - Ok(ClientData { - version: "1.2.3".to_owned(), - }) - }) - .await - .unwrap(); - - assert_eq!( - desktop_c, - ClientData { - version: "1.2.3".to_owned() - } - ); - - let music_c = cache - .get_music_client_data(async { - Ok(ClientData { - version: "4.5.6".to_owned(), - }) - }) - .await - .unwrap(); - - assert_eq!( - music_c, - ClientData { - version: "4.5.6".to_owned() - } - ); - - let deobf_data = cache - .get_deobf_data(async { - Ok(DeobfData { - js_url: - "https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js" - .to_owned(), - sig_fn: "t_sig_fn".to_owned(), - nsig_fn: "t_nsig_fn".to_owned(), - sts: "t_sts".to_owned(), - }) - }) - .await - .unwrap(); - - assert_eq!( - deobf_data, - DeobfData { - js_url: "https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js" - .to_owned(), - sig_fn: "t_sig_fn".to_owned(), - nsig_fn: "t_nsig_fn".to_owned(), - sts: "t_sts".to_owned(), - } - ); - - // Create a new cache from the first one's json - // and check if it returns the same cached data - let json = cache.to_json().await.unwrap(); - let new_cache = Cache::from_json(&json); - - assert_eq!( - new_cache - .get_desktop_client_data(async { - Ok(ClientData { - version: "".to_owned(), - }) - }) - .await - .unwrap(), - desktop_c - ); - - assert_eq!( - new_cache - .get_music_client_data(async { - Ok(ClientData { - version: "".to_owned(), - }) - }) - .await - .unwrap(), - music_c - ); - - assert_eq!( - new_cache - .get_deobf_data(async { - Ok(DeobfData { - js_url: "".to_owned(), - nsig_fn: "".to_owned(), - sig_fn: "".to_owned(), - sts: "".to_owned(), - }) - }) - .await - .unwrap(), - deobf_data - ); - } - - #[tokio::test] - async fn test_file() { - let temp = TempDir::default(); - let mut file_path = PathBuf::from(temp.as_ref()); - file_path.push("cache.json"); - - let cache = Cache::from_json_file(file_path.clone()); - - let cdata = cache - .get_desktop_client_data(async { - Ok(ClientData { - version: "1.2.3".to_owned(), - }) - }) - .await - .unwrap(); - - let deobf_data = cache - .get_deobf_data(async { - Ok(DeobfData { - js_url: - "https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js" - .to_owned(), - sig_fn: "t_sig_fn".to_owned(), - nsig_fn: "t_nsig_fn".to_owned(), - sts: "t_sts".to_owned(), - }) - }) - .await - .unwrap(); - - assert!(file_path.exists()); - let new_cache = Cache::from_json_file(file_path.clone()); - - assert_eq!( - new_cache - .get_desktop_client_data(async { - Ok(ClientData { - version: "".to_owned(), - }) - }) - .await - .unwrap(), - cdata - ); - - assert_eq!( - new_cache - .get_deobf_data(async { - Ok(DeobfData { - js_url: "".to_owned(), - nsig_fn: "".to_owned(), - sig_fn: "".to_owned(), - sts: "".to_owned(), - }) - }) - .await - .unwrap(), - deobf_data - ); - } -} diff --git a/src/client2/mod.rs b/src/client2/mod.rs index 91b5622..356e59a 100644 --- a/src/client2/mod.rs +++ b/src/client2/mod.rs @@ -6,20 +6,35 @@ mod response; use std::fmt::Debug; use std::sync::Arc; -use anyhow::{anyhow, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; +use chrono::{DateTime, Duration, Utc}; use fancy_regex::Regex; +use log::{error, warn}; use once_cell::sync::Lazy; use rand::Rng; -use reqwest::{header, Client, ClientBuilder, Method, RequestBuilder}; +use reqwest::{header, Client, ClientBuilder, Method, Request, RequestBuilder, Response}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use tokio::sync::Mutex; use crate::{ - cache::Cache, - deobfuscate::Deobfuscator, + cache::{CacheStorage, FileStorage}, + deobfuscate::{DeobfData, Deobfuscator}, model::{Country, Language}, - report::{Level, Report, Reporter, YamlFileReporter}, + report::{JsonFileReporter, Level, Report, Reporter}, + util, }; +/// Client types for accessing the YouTube API. +/// +/// There are multiple clients for accessing the YouTube API which have +/// slightly different features +/// +/// - **Desktop**: used by youtube.com +/// - **DesktopMusic**: used by music.youtube.com, can access special music data, +/// cannot access non-music content +/// - **TvHtml5Embed**: (probably) used by Smart TVs, can access age-restricted videos +/// - **Android**: used by the Android app, no obfuscated URLs, includes lower resolution audio streams +/// - **Ios**: used by the iOS app, no obfuscated URLs #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] #[serde(rename_all = "snake_case")] pub enum ClientType { @@ -95,7 +110,7 @@ impl Default for RequestYT { #[derive(Clone, Debug, Serialize, Default)] #[serde(rename_all = "camelCase")] struct User { - // TO DO: provide a way to enable restricted mode with: + // TODO: provide a way to enable restricted mode with: // "enableSafetyMode": true locked_safety_mode: bool, } @@ -131,6 +146,11 @@ const IOS_DEVICE_MODEL: &str = "iPhone14,5"; static CLIENT_VERSION_REGEXES: Lazy<[Regex; 1]> = Lazy::new(|| [Regex::new("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"").unwrap()]); +/// The RustyPipe client used to access YouTube's API +/// +/// RustyPipe includes an `Arc` internally, so if you are using the client +/// at multiple locations, you can just clone it. Note that options (lang/country/report) +/// are not shared between clones. #[derive(Clone)] pub struct RustyPipe { inner: Arc, @@ -139,10 +159,11 @@ pub struct RustyPipe { struct RustyPipeRef { http: Client, - cache: Cache, + storage: Option>, reporter: Option>, user_agent: String, consent_cookie: String, + cache: Mutex, } #[derive(Clone)] @@ -150,13 +171,14 @@ struct RustyPipeOpts { lang: Language, country: Country, report: bool, + strict: bool, } impl Default for RustyPipe { fn default() -> Self { Self::new( - Some(Cache::from_json_file("RustyPipeCache.json")), - Some(Box::new(YamlFileReporter::default())), + Some(Box::new(FileStorage::default())), + Some(Box::new(JsonFileReporter::default())), None, ) } @@ -168,17 +190,64 @@ impl Default for RustyPipeOpts { lang: Language::En, country: Country::Us, report: false, + strict: false, + } + } +} + +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +struct CacheData { + desktop_client: CacheEntry, + music_client: CacheEntry, + deobf: CacheEntry, +} + +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +enum CacheEntry { + #[default] + None, + Some { + last_update: DateTime, + data: T, + }, +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ClientData { + pub version: String, +} + +impl CacheEntry { + fn get(&self) -> Option<&T> { + match self { + CacheEntry::Some { last_update, data } => { + if last_update < &(Utc::now() - Duration::hours(24)) { + None + } else { + Some(data) + } + } + CacheEntry::None => None, + } + } +} + +impl From for CacheEntry { + fn from(f: T) -> Self { + Self::Some { + last_update: Utc::now(), + data: f, } } } impl RustyPipe { + /// Create a new RustyPipe instance pub fn new( - cache: Option, + storage: Option>, reporter: Option>, user_agent: Option, ) -> Self { - let cache = cache.unwrap_or_else(|| Cache::default()); let user_agent = user_agent.unwrap_or(DEFAULT_UA.to_owned()); let http = ClientBuilder::new() @@ -188,10 +257,26 @@ impl RustyPipe { .build() .expect("unable to build the HTTP client"); + let cache = if let Some(storage) = &storage { + if let Some(data) = storage.read() { + match serde_json::from_str::(&data) { + Ok(data) => data, + Err(e) => { + error!("Could not deserialize cache. Error: {}", e); + CacheData::default() + } + } + } else { + CacheData::default() + } + } else { + CacheData::default() + }; + RustyPipe { inner: Arc::new(RustyPipeRef { http, - cache, + storage, reporter, user_agent, consent_cookie: format!( @@ -200,26 +285,53 @@ impl RustyPipe { CONSENT_COOKIE_YES, rand::thread_rng().gen_range(100..1000) ), + cache: Mutex::new(cache), }), opts: RustyPipeOpts::default(), } } + /// Create a new RustyPipe instance configured for testing + #[cfg(test)] + #[cfg(feature = "yaml")] + pub fn new_test() -> Self { + Self::new( + Some(Box::new(FileStorage::default())), + Some(Box::new(crate::report::YamlFileReporter::default())), + None, + ) + .strict(true) + } + + /// Set the language parameter used when accessing the YouTube API + /// This will change multilanguage video titles, descriptions and textual dates pub fn lang(mut self, lang: Language) -> Self { self.opts.lang = lang; self } + /// Set the country parameter used when accessing the YouTube API. + /// This will change trends and recommended content. pub fn country(mut self, country: Country) -> Self { self.opts.country = country; self } + /// Generate a report on every operation. + /// This should only be used for debugging. pub fn report(mut self, report: bool) -> Self { self.opts.report = report; self } + /// Enable strict mode, causing operations to fail if there + /// are warnings during deserialization (e.g. invalid items). + /// This should only be used for testing. + pub fn strict(mut self, strict: bool) -> Self { + self.opts.strict = strict; + self + } + async fn get_context(&self, ctype: ClientType, localized: bool) -> ContextYT { let hl = match localized { true => self.opts.lang, @@ -234,7 +346,7 @@ impl RustyPipe { ClientType::Desktop => ContextYT { client: ClientInfo { client_name: "WEB".to_owned(), - client_version: DESKTOP_CLIENT_VERSION.to_owned(), + client_version: self.get_desktop_client_version().await, client_screen: None, device_model: None, platform: "DESKTOP".to_owned(), @@ -249,7 +361,7 @@ impl RustyPipe { ClientType::DesktopMusic => ContextYT { client: ClientInfo { client_name: "WEB_REMIX".to_owned(), - client_version: DESKTOP_MUSIC_CLIENT_VERSION.to_owned(), + client_version: self.get_music_client_version().await, client_screen: None, device_model: None, platform: "DESKTOP".to_owned(), @@ -332,7 +444,7 @@ impl RustyPipe { .header(header::REFERER, "https://www.youtube.com") .header(header::COOKIE, self.inner.consent_cookie.to_owned()) .header("X-YouTube-Client-Name", "1") - .header("X-YouTube-Client-Version", DESKTOP_CLIENT_VERSION), + .header("X-YouTube-Client-Version", self.get_desktop_client_version().await), ClientType::DesktopMusic => self .inner .http @@ -350,7 +462,7 @@ impl RustyPipe { .header(header::REFERER, "https://music.youtube.com") .header(header::COOKIE, self.inner.consent_cookie.to_owned()) .header("X-YouTube-Client-Name", "67") - .header("X-YouTube-Client-Version", DESKTOP_MUSIC_CLIENT_VERSION), + .header("X-YouTube-Client-Version", self.get_music_client_version().await), ClientType::TvHtml5Embed => self .inner .http @@ -410,7 +522,7 @@ impl RustyPipe { } } - async fn execute_request< + async fn execute_request_deobf< R: DeserializeOwned + MapResponse + Debug, M, B: Serialize + ?Sized, @@ -448,6 +560,7 @@ impl RustyPipe { operation: operation.to_owned(), error, msgs, + deobf_data: deobf.map(Deobfuscator::get_data), http_request: crate::report::HTTPRequest { url: request_url, method: method.to_string(), @@ -482,6 +595,10 @@ impl RustyPipe { Some("Warnings during deserialization/mapping".to_owned()), mapres.warnings, ); + + if self.opts.strict { + bail!("Warnings during deserialization/mapping"); + } } else if self.opts.report { create_report(Level::DBG, None, vec![]); } @@ -500,6 +617,176 @@ impl RustyPipe { } } } + + async fn execute_request< + R: DeserializeOwned + MapResponse + Debug, + M, + B: Serialize + ?Sized, + >( + &self, + ctype: ClientType, + operation: &str, + method: Method, + endpoint: &str, + id: &str, + body: &B, + ) -> Result { + self.execute_request_deobf::(ctype, operation, method, endpoint, id, body, None) + .await + } + + async fn get_desktop_client_version(&self) -> String { + let mut cache = self.inner.cache.lock().await; + + match cache.desktop_client.get() { + Some(cdata) => cdata.version.to_owned(), + None => match self.extract_desktop_client_version().await { + Ok(version) => { + cache.desktop_client = CacheEntry::from(ClientData { + version: version.to_owned(), + }); + self.write_cache(&cache); + version + } + Err(e) => { + warn!("{}, falling back to hardcoded version", e); + DESKTOP_CLIENT_VERSION.to_owned() + } + }, + } + } + + async fn get_music_client_version(&self) -> String { + let mut cache = self.inner.cache.lock().await; + + match cache.music_client.get() { + Some(cdata) => cdata.version.to_owned(), + None => match self.extract_music_client_version().await { + Ok(version) => { + cache.music_client = CacheEntry::from(ClientData { + version: version.to_owned(), + }); + self.write_cache(&cache); + version + } + Err(e) => { + warn!("{}, falling back to hardcoded version", e); + DESKTOP_MUSIC_CLIENT_VERSION.to_owned() + } + }, + } + } + + async fn get_deobf(&self) -> Result { + let mut cache = self.inner.cache.lock().await; + let deobf = Deobfuscator::new(self.inner.http.clone()).await?; + cache.deobf = CacheEntry::from(deobf.get_data()); + self.write_cache(&cache); + Ok(deobf) + } + + async fn extract_desktop_client_version(&self) -> Result { + let from_swjs = async { + let swjs = self + .exec_request_text( + self.inner + .http + .get("https://www.youtube.com/sw.js") + .header(header::ORIGIN, "https://www.youtube.com") + .header(header::REFERER, "https://www.youtube.com") + .header(header::COOKIE, self.inner.consent_cookie.to_owned()) + .build() + .unwrap(), + ) + .await + .context("Failed to download sw.js")?; + + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) + .ok_or(anyhow!("Could not find desktop client version in sw.js")) + }; + + let from_html = async { + let html = self + .exec_request_text( + self.inner + .http + .get("https://www.youtube.com/results?search_query=") + .build() + .unwrap(), + ) + .await + .context("Failed to get YT Desktop page")?; + + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(anyhow!( + "Could not find desktop client version on html page" + )) + }; + + match from_swjs.await { + Ok(client_version) => Ok(client_version), + Err(_) => from_html.await, + } + } + + async fn extract_music_client_version(&self) -> Result { + let from_swjs = async { + let swjs = self + .exec_request_text( + self.inner + .http + .get("https://music.youtube.com/sw.js") + .header(header::ORIGIN, "https://music.youtube.com") + .header(header::REFERER, "https://music.youtube.com") + .header(header::COOKIE, self.inner.consent_cookie.to_owned()) + .build() + .unwrap(), + ) + .await + .context("Failed to download sw.js")?; + + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) + .ok_or(anyhow!("Could not find desktop client version in sw.js")) + }; + + let from_html = async { + let html = self + .exec_request_text( + self.inner + .http + .get("https://music.youtube.com") + .build() + .unwrap(), + ) + .await + .context("Failed to get YT Desktop page")?; + + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(anyhow!( + "Could not find desktop client version on html page" + )) + }; + + match from_swjs.await { + Ok(client_version) => Ok(client_version), + Err(_) => from_html.await, + } + } + + async fn exec_request(&self, request: Request) -> Result { + Ok(self.inner.http.execute(request).await?.error_for_status()?) + } + + async fn exec_request_text(&self, request: Request) -> Result { + Ok(self.exec_request(request).await?.text().await?) + } + + fn write_cache(&self, cache: &CacheData) { + if let Some(storage) = &self.inner.storage { + match serde_json::to_string(cache) { + Ok(data) => storage.write(&data), + Err(e) => error!("Could not serialize cache. Error: {}", e), + } + } + } } trait MapResponse { @@ -525,10 +812,3 @@ where self.c.fmt(f) } } - -/* -#[cfg(test)] -mod tests { - use super::*; -} -*/ diff --git a/src/client2/player.rs b/src/client2/player.rs index c907389..402b8ba 100644 --- a/src/client2/player.rs +++ b/src/client2/player.rs @@ -59,10 +59,10 @@ struct QContentPlaybackContext { impl RustyPipe { pub async fn get_player(&self, video_id: &str, client_type: ClientType) -> Result { - let (context, deobf) = tokio::join!( - self.get_context(client_type, false), - Deobfuscator::from_fetched_info(self.inner.http.clone(), self.inner.cache.clone()) - ); + let (context, deobf) = tokio::join!(self.get_context(client_type, false), self.get_deobf()); + // let context = self.get_context(client_type, false).await; + // let deobf = self.get_deobf().await; + let deobf = deobf?; let request_body = if client_type.is_web() { @@ -90,7 +90,7 @@ impl RustyPipe { } }; - self.execute_request::( + self.execute_request_deobf::( client_type, "get_player", Method::POST, @@ -575,10 +575,11 @@ fn get_audio_codec(codecs: Vec<&str>) -> AudioCodec { } #[cfg(test)] +#[cfg(feature = "yaml")] mod tests { use std::{fs::File, io::BufReader, path::Path}; - use crate::{cache::DeobfData, client2::CLIENT_TYPES, report::TestFileReporter}; + use crate::{deobfuscate::DeobfData, client2::CLIENT_TYPES, report::TestFileReporter}; use super::*; use rstest::rstest; @@ -613,7 +614,7 @@ mod tests { #[test_log::test(tokio::test)] async fn download_model_testfiles() { let tf_dir = Path::new("testfiles/player_model"); - let rp = RustyPipe::default(); + let rp = RustyPipe::new_test(); for (name, id) in [("multilanguage", "tVWWp1PqDus"), ("hdr", "LXb3EKWsInQ")] { let mut json_path = tf_dir.to_path_buf(); @@ -683,7 +684,7 @@ mod tests { #[case::ios(ClientType::Ios)] #[test_log::test(tokio::test)] async fn t_get_player(#[case] client_type: ClientType) { - let rp = RustyPipe::default(); + let rp = RustyPipe::new_test(); let player_data = rp.get_player("n4tK7LYFxI0", client_type).await.unwrap(); // dbg!(&player_data); diff --git a/src/client2/playlist.rs b/src/client2/playlist.rs index e92f1fa..8e3214a 100644 --- a/src/client2/playlist.rs +++ b/src/client2/playlist.rs @@ -40,7 +40,6 @@ impl RustyPipe { "browse", playlist_id, &request_body, - None, ) .await } @@ -62,7 +61,6 @@ impl RustyPipe { "browse", &playlist.id, &request_body, - None, ) .await?; @@ -350,7 +348,7 @@ mod tests { #[case] description: Option, #[case] channel: Option, ) { - let rp = RustyPipe::default(); + let rp = RustyPipe::new_test(); let playlist = rp.get_playlist(id).await.unwrap(); assert_eq!(playlist.id, id); @@ -412,7 +410,7 @@ mod tests { #[test_log::test(tokio::test)] async fn t_playlist_cont() { - let rp = RustyPipe::default(); + let rp = RustyPipe::new_test(); let mut playlist = rp .get_playlist("PLbZIPy20-1pN7mqjckepWF78ndb6ci_qi") .await diff --git a/src/deobfuscate.rs b/src/deobfuscate.rs index 0227b37..e85d012 100644 --- a/src/deobfuscate.rs +++ b/src/deobfuscate.rs @@ -3,43 +3,47 @@ use fancy_regex::Regex; use log::debug; use once_cell::sync::Lazy; use reqwest::Client; +use serde::{Serialize, Deserialize}; use std::result::Result::Ok; -use crate::cache::{Cache, DeobfData}; use crate::util; pub struct Deobfuscator { data: DeobfData, } +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct DeobfData { + pub js_url: String, + pub sig_fn: String, + pub nsig_fn: String, + pub sts: String, +} + impl Deobfuscator { - pub async fn from_fetched_info(http: Client, cache: Cache) -> Result { - let data = cache - .get_deobf_data(async move { - let js_url = get_player_js_url(&http) - .await - .context("Failed to retrieve player.js URL")?; + pub async fn new(http: Client) -> Result { + let js_url = get_player_js_url(&http) + .await + .context("Failed to retrieve player.js URL")?; - let player_js = get_response(&http, &js_url) - .await - .context("Failed to download player.js")?; + let player_js = get_response(&http, &js_url) + .await + .context("Failed to download player.js")?; - debug!("Downloaded player.js from {}", js_url); + debug!("Downloaded player.js from {}", js_url); - let sig_fn = get_sig_fn(&player_js)?; - let nsig_fn = get_nsig_fn(&player_js)?; - let sts = get_sts(&player_js)?; + let sig_fn = get_sig_fn(&player_js)?; + let nsig_fn = get_nsig_fn(&player_js)?; + let sts = get_sts(&player_js)?; - Ok(DeobfData { - js_url, - nsig_fn, - sig_fn, - sts, - }) - }) - .await?; - - Ok(Self { data }) + Ok(Self { + data: DeobfData { + js_url, + nsig_fn, + sig_fn, + sts, + }, + }) } pub fn deobfuscate_sig(&self, sig: &str) -> Result { @@ -53,6 +57,10 @@ impl Deobfuscator { pub fn get_sts(&self) -> String { self.data.sts.to_owned() } + + pub fn get_data(&self) -> DeobfData { + self.data.to_owned() + } } impl From for Deobfuscator { @@ -472,8 +480,7 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c #[test(tokio::test)] async fn t_update() { let client = Client::new(); - let cache = Cache::default(); - let deobf = Deobfuscator::from_fetched_info(client, cache) + let deobf = Deobfuscator::new(client) .await .unwrap(); diff --git a/src/report.rs b/src/report.rs index bd5ea16..85939d9 100644 --- a/src/report.rs +++ b/src/report.rs @@ -9,6 +9,8 @@ use chrono::{DateTime, Local}; use log::error; use serde::{Deserialize, Serialize}; +use crate::deobfuscate::DeobfData; + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Report { /// Rust package name (`rustypipe`) @@ -25,9 +27,9 @@ pub struct Report { pub error: Option, /// Detailed error/warning messages pub msgs: Vec, - // /// Deobfuscation data (only for player requests) - // #[serde(skip_serializing_if = "Option::is_none")] - // pub deobf_data: Option, + /// Deobfuscation data (only for player requests) + #[serde(skip_serializing_if = "Option::is_none")] + pub deobf_data: Option, /// HTTP request data pub http_request: HTTPRequest, } @@ -96,10 +98,12 @@ impl Reporter for JsonFileReporter { } } +#[cfg(feature="yaml")] pub struct YamlFileReporter { path: PathBuf, } +#[cfg(feature="yaml")] impl YamlFileReporter { pub fn new>(path: P) -> Self { Self { @@ -114,6 +118,7 @@ impl YamlFileReporter { } } +#[cfg(feature="yaml")] impl Default for YamlFileReporter { fn default() -> Self { Self { @@ -122,6 +127,7 @@ impl Default for YamlFileReporter { } } +#[cfg(feature="yaml")] impl Reporter for YamlFileReporter { fn report(&self, report: &Report) { self._report(report) From b52fd7349ba129c3fbe7d814b2f57e10bee93423 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 14 Sep 2022 23:55:44 +0200 Subject: [PATCH 2/5] refactored client API (query builder) use VecLogError for player streams --- cli/Cargo.toml | 2 +- src/client2/mod.rs | 295 +++++++++++++++++++------------ src/client2/player.rs | 55 ++++-- src/client2/playlist.rs | 22 ++- src/client2/response/channel.rs | 7 +- src/client2/response/mod.rs | 10 +- src/client2/response/player.rs | 18 +- src/client2/response/playlist.rs | 23 +-- src/deobfuscate.rs | 8 +- src/report.rs | 10 +- src/serializer/mod.rs | 4 +- 11 files changed, 277 insertions(+), 177 deletions(-) diff --git a/cli/Cargo.toml b/cli/Cargo.toml index b2a3462..23b8ed6 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" [dependencies] rustypipe = {path = "../", default_features = false, features = ["rustls-tls-native-roots"]} reqwest = {version = "0.11.11", default_features = false} -tokio = {version = "1.20.0", features = ["rt-multi-thread"]} +tokio = {version = "1.20.0", features = ["macros", "rt-multi-thread"]} indicatif = "0.17.0" futures = "0.3.21" anyhow = "1.0" diff --git a/src/client2/mod.rs b/src/client2/mod.rs index 356e59a..f7ee912 100644 --- a/src/client2/mod.rs +++ b/src/client2/mod.rs @@ -154,13 +154,12 @@ static CLIENT_VERSION_REGEXES: Lazy<[Regex; 1]> = #[derive(Clone)] pub struct RustyPipe { inner: Arc, - opts: RustyPipeOpts, } struct RustyPipeRef { http: Client, - storage: Option>, - reporter: Option>, + storage: Option>, + reporter: Option>, user_agent: String, consent_cookie: String, cache: Mutex, @@ -174,6 +173,12 @@ struct RustyPipeOpts { strict: bool, } +#[derive(Clone)] +pub struct RustyPipeQuery { + client: RustyPipe, + opts: RustyPipeOpts, +} + impl Default for RustyPipe { fn default() -> Self { Self::new( @@ -244,8 +249,8 @@ impl From for CacheEntry { impl RustyPipe { /// Create a new RustyPipe instance pub fn new( - storage: Option>, - reporter: Option>, + storage: Option>, + reporter: Option>, user_agent: Option, ) -> Self { let user_agent = user_agent.unwrap_or(DEFAULT_UA.to_owned()); @@ -287,7 +292,6 @@ impl RustyPipe { ), cache: Mutex::new(cache), }), - opts: RustyPipeOpts::default(), } } @@ -300,9 +304,35 @@ impl RustyPipe { Some(Box::new(crate::report::YamlFileReporter::default())), None, ) - .strict(true) } + pub fn query(&self) -> RustyPipeQuery { + RustyPipeQuery { + client: self.clone(), + opts: RustyPipeOpts { + lang: Language::En, + country: Country::Us, + report: false, + strict: false, + }, + } + } + + #[cfg(test)] + pub fn test_query(&self) -> RustyPipeQuery { + RustyPipeQuery { + client: self.clone(), + opts: RustyPipeOpts { + lang: Language::En, + country: Country::Us, + report: false, + strict: true, + }, + } + } +} + +impl RustyPipeQuery { /// Set the language parameter used when accessing the YouTube API /// This will change multilanguage video titles, descriptions and textual dates pub fn lang(mut self, lang: Language) -> Self { @@ -431,6 +461,7 @@ impl RustyPipe { ) -> RequestBuilder { match ctype { ClientType::Desktop => self + .client .inner .http .request( @@ -442,10 +473,14 @@ impl RustyPipe { ) .header(header::ORIGIN, "https://www.youtube.com") .header(header::REFERER, "https://www.youtube.com") - .header(header::COOKIE, self.inner.consent_cookie.to_owned()) + .header(header::COOKIE, self.client.inner.consent_cookie.to_owned()) .header("X-YouTube-Client-Name", "1") - .header("X-YouTube-Client-Version", self.get_desktop_client_version().await), + .header( + "X-YouTube-Client-Version", + self.get_desktop_client_version().await, + ), ClientType::DesktopMusic => self + .client .inner .http .request( @@ -460,10 +495,14 @@ impl RustyPipe { ) .header(header::ORIGIN, "https://music.youtube.com") .header(header::REFERER, "https://music.youtube.com") - .header(header::COOKIE, self.inner.consent_cookie.to_owned()) + .header(header::COOKIE, self.client.inner.consent_cookie.to_owned()) .header("X-YouTube-Client-Name", "67") - .header("X-YouTube-Client-Version", self.get_music_client_version().await), + .header( + "X-YouTube-Client-Version", + self.get_music_client_version().await, + ), ClientType::TvHtml5Embed => self + .client .inner .http .request( @@ -478,6 +517,7 @@ impl RustyPipe { .header("X-YouTube-Client-Name", "1") .header("X-YouTube-Client-Version", TVHTML5_CLIENT_VERSION), ClientType::Android => self + .client .inner .http .request( @@ -499,6 +539,7 @@ impl RustyPipe { ) .header("X-Goog-Api-Format-Version", "2"), ClientType::Ios => self + .client .inner .http .request( @@ -545,13 +586,13 @@ impl RustyPipe { let request_url = request.url().to_string(); let request_headers = request.headers().to_owned(); - let response = self.inner.http.execute(request).await?; + let response = self.client.inner.http.execute(request).await?; let status = response.status(); let resp_str = response.text().await?; let create_report = |level: Level, error: Option, msgs: Vec| { - if let Some(reporter) = &self.inner.reporter { + if let Some(reporter) = &self.client.inner.reporter { let report = Report { package: "rustypipe".to_owned(), version: "0.1.0".to_owned(), @@ -636,11 +677,16 @@ impl RustyPipe { } async fn get_desktop_client_version(&self) -> String { - let mut cache = self.inner.cache.lock().await; + let mut cache = self.client.inner.cache.lock().await; match cache.desktop_client.get() { Some(cdata) => cdata.version.to_owned(), - None => match self.extract_desktop_client_version().await { + None => match extract_desktop_client_version( + self.client.inner.http.clone(), + self.client.inner.consent_cookie.to_owned(), + ) + .await + { Ok(version) => { cache.desktop_client = CacheEntry::from(ClientData { version: version.to_owned(), @@ -657,11 +703,16 @@ impl RustyPipe { } async fn get_music_client_version(&self) -> String { - let mut cache = self.inner.cache.lock().await; + let mut cache = self.client.inner.cache.lock().await; match cache.music_client.get() { Some(cdata) => cdata.version.to_owned(), - None => match self.extract_music_client_version().await { + None => match extract_music_client_version( + self.client.inner.http.clone(), + self.client.inner.consent_cookie.to_owned(), + ) + .await + { Ok(version) => { cache.music_client = CacheEntry::from(ClientData { version: version.to_owned(), @@ -678,109 +729,21 @@ impl RustyPipe { } async fn get_deobf(&self) -> Result { - let mut cache = self.inner.cache.lock().await; - let deobf = Deobfuscator::new(self.inner.http.clone()).await?; - cache.deobf = CacheEntry::from(deobf.get_data()); - self.write_cache(&cache); - Ok(deobf) - } + let mut cache = self.client.inner.cache.lock().await; - async fn extract_desktop_client_version(&self) -> Result { - let from_swjs = async { - let swjs = self - .exec_request_text( - self.inner - .http - .get("https://www.youtube.com/sw.js") - .header(header::ORIGIN, "https://www.youtube.com") - .header(header::REFERER, "https://www.youtube.com") - .header(header::COOKIE, self.inner.consent_cookie.to_owned()) - .build() - .unwrap(), - ) - .await - .context("Failed to download sw.js")?; - - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) - .ok_or(anyhow!("Could not find desktop client version in sw.js")) - }; - - let from_html = async { - let html = self - .exec_request_text( - self.inner - .http - .get("https://www.youtube.com/results?search_query=") - .build() - .unwrap(), - ) - .await - .context("Failed to get YT Desktop page")?; - - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(anyhow!( - "Could not find desktop client version on html page" - )) - }; - - match from_swjs.await { - Ok(client_version) => Ok(client_version), - Err(_) => from_html.await, + match cache.deobf.get() { + Some(deobf) => Ok(Deobfuscator::from(deobf.to_owned())), + None => { + let deobf = Deobfuscator::new(self.client.inner.http.clone()).await?; + cache.deobf = CacheEntry::from(deobf.get_data()); + self.write_cache(&cache); + Ok(deobf) + } } } - async fn extract_music_client_version(&self) -> Result { - let from_swjs = async { - let swjs = self - .exec_request_text( - self.inner - .http - .get("https://music.youtube.com/sw.js") - .header(header::ORIGIN, "https://music.youtube.com") - .header(header::REFERER, "https://music.youtube.com") - .header(header::COOKIE, self.inner.consent_cookie.to_owned()) - .build() - .unwrap(), - ) - .await - .context("Failed to download sw.js")?; - - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) - .ok_or(anyhow!("Could not find desktop client version in sw.js")) - }; - - let from_html = async { - let html = self - .exec_request_text( - self.inner - .http - .get("https://music.youtube.com") - .build() - .unwrap(), - ) - .await - .context("Failed to get YT Desktop page")?; - - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(anyhow!( - "Could not find desktop client version on html page" - )) - }; - - match from_swjs.await { - Ok(client_version) => Ok(client_version), - Err(_) => from_html.await, - } - } - - async fn exec_request(&self, request: Request) -> Result { - Ok(self.inner.http.execute(request).await?.error_for_status()?) - } - - async fn exec_request_text(&self, request: Request) -> Result { - Ok(self.exec_request(request).await?.text().await?) - } - fn write_cache(&self, cache: &CacheData) { - if let Some(storage) = &self.inner.storage { + if let Some(storage) = &self.client.inner.storage { match serde_json::to_string(cache) { Ok(data) => storage.write(&data), Err(e) => error!("Could not serialize cache. Error: {}", e), @@ -789,6 +752,90 @@ impl RustyPipe { } } +async fn extract_desktop_client_version(http: Client, consent_cookie: String) -> Result { + let from_swjs = async { + let swjs = exec_request_text( + http.clone(), + http.get("https://www.youtube.com/sw.js") + .header(header::ORIGIN, "https://www.youtube.com") + .header(header::REFERER, "https://www.youtube.com") + .header(header::COOKIE, consent_cookie) + .build() + .unwrap(), + ) + .await + .context("Failed to download sw.js")?; + + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) + .ok_or(anyhow!("Could not find desktop client version in sw.js")) + }; + + let from_html = async { + let html = exec_request_text( + http.clone(), + http.get("https://www.youtube.com/results?search_query=") + .build() + .unwrap(), + ) + .await + .context("Failed to get YT Desktop page")?; + + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(anyhow!( + "Could not find desktop client version on html page" + )) + }; + + match from_swjs.await { + Ok(client_version) => Ok(client_version), + Err(_) => from_html.await, + } +} + +async fn extract_music_client_version(http: Client, consent_cookie: String) -> Result { + let from_swjs = async { + let swjs = exec_request_text( + http.clone(), + http.get("https://music.youtube.com/sw.js") + .header(header::ORIGIN, "https://music.youtube.com") + .header(header::REFERER, "https://music.youtube.com") + .header(header::COOKIE, consent_cookie) + .build() + .unwrap(), + ) + .await + .context("Failed to download sw.js")?; + + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) + .ok_or(anyhow!("Could not find desktop client version in sw.js")) + }; + + let from_html = async { + let html = exec_request_text( + http.clone(), + http.get("https://music.youtube.com").build().unwrap(), + ) + .await + .context("Failed to get YT Desktop page")?; + + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(anyhow!( + "Could not find desktop client version on html page" + )) + }; + + match from_swjs.await { + Ok(client_version) => Ok(client_version), + Err(_) => from_html.await, + } +} + +async fn exec_request(http: Client, request: Request) -> Result { + Ok(http.execute(request).await?.error_for_status()?) +} + +async fn exec_request_text(http: Client, request: Request) -> Result { + Ok(exec_request(http, request).await?.text().await?) +} + trait MapResponse { fn map_response( self, @@ -812,3 +859,21 @@ where self.c.fmt(f) } } + +impl Default for MapResult +where + T: Default, +{ + fn default() -> Self { + Self { + c: Default::default(), + warnings: Vec::new(), + } + } +} + +#[cfg(test)] +#[cfg(feature = "yaml")] +mod tests { + // use super::*; +} diff --git a/src/client2/player.rs b/src/client2/player.rs index 402b8ba..27571c9 100644 --- a/src/client2/player.rs +++ b/src/client2/player.rs @@ -21,7 +21,7 @@ use crate::{ use super::{ response::{self, player}, - ClientType, ContextYT, MapResponse, MapResult, RustyPipe, + ClientType, ContextYT, MapResponse, MapResult, RustyPipeQuery, }; #[derive(Clone, Debug, Serialize)] @@ -57,13 +57,21 @@ struct QContentPlaybackContext { referer: String, } -impl RustyPipe { - pub async fn get_player(&self, video_id: &str, client_type: ClientType) -> Result { - let (context, deobf) = tokio::join!(self.get_context(client_type, false), self.get_deobf()); - // let context = self.get_context(client_type, false).await; - // let deobf = self.get_deobf().await; +impl RustyPipeQuery { + pub async fn get_player(self, video_id: &str, client_type: ClientType) -> Result { + // let (context, deobf) = tokio::join!(self.get_context(client_type, false), self.get_deobf()); + // let deobf = deobf?; - let deobf = deobf?; + let q1 = self.clone(); + let t_context = tokio::spawn(async move { q1.get_context(client_type, false).await }); + let q2 = self.clone(); + let t_deobf = tokio::spawn(async move { q2.get_deobf().await }); + // let context = t_context.await.unwrap(); + // let deobf = t_deobf.await.unwrap()?; + + let (context, deobf) = tokio::join!(t_context, t_deobf); + let context = context.unwrap(); + let deobf = deobf.unwrap()?; let request_body = if client_type.is_web() { QPlayer { @@ -187,8 +195,11 @@ impl MapResponse for response::Player { is_family_safe, }; - let mut formats = streaming_data.formats; - formats.append(&mut streaming_data.adaptive_formats); + let mut formats = streaming_data.formats.c; + formats.append(&mut streaming_data.adaptive_formats.c); + + warnings.append(&mut streaming_data.formats.warnings); + warnings.append(&mut streaming_data.adaptive_formats.warnings); let mut last_nsig: [String; 2] = ["".to_owned(), "".to_owned()]; @@ -579,7 +590,11 @@ fn get_audio_codec(codecs: Vec<&str>) -> AudioCodec { mod tests { use std::{fs::File, io::BufReader, path::Path}; - use crate::{deobfuscate::DeobfData, client2::CLIENT_TYPES, report::TestFileReporter}; + use crate::{ + client2::{RustyPipe, CLIENT_TYPES}, + deobfuscate::DeobfData, + report::TestFileReporter, + }; use super::*; use rstest::rstest; @@ -606,8 +621,12 @@ mod tests { } let reporter = TestFileReporter::new(json_path); - let rp = RustyPipe::new(None, Some(Box::new(reporter)), None).report(true); - rp.get_player(video_id, client_type).await.unwrap(); + let rp = RustyPipe::new(None, Some(Box::new(reporter)), None); + rp.test_query() + .report(true) + .get_player(video_id, client_type) + .await + .unwrap(); } } @@ -623,7 +642,11 @@ mod tests { continue; } - let player_data = rp.get_player(id, ClientType::Desktop).await.unwrap(); + let player_data = rp + .test_query() + .get_player(id, ClientType::Desktop) + .await + .unwrap(); let file = File::create(json_path).unwrap(); serde_json::to_writer_pretty(file, &player_data).unwrap(); } @@ -685,7 +708,11 @@ mod tests { #[test_log::test(tokio::test)] async fn t_get_player(#[case] client_type: ClientType) { let rp = RustyPipe::new_test(); - let player_data = rp.get_player("n4tK7LYFxI0", client_type).await.unwrap(); + let player_data = rp + .test_query() + .get_player("n4tK7LYFxI0", client_type) + .await + .unwrap(); // dbg!(&player_data); diff --git a/src/client2/playlist.rs b/src/client2/playlist.rs index 8e3214a..2fd0ade 100644 --- a/src/client2/playlist.rs +++ b/src/client2/playlist.rs @@ -9,7 +9,7 @@ use crate::{ timeago, util, }; -use super::{response, ClientType, ContextYT, MapResponse, MapResult, RustyPipe}; +use super::{response, ClientType, ContextYT, MapResponse, MapResult, RustyPipeQuery}; #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] @@ -25,8 +25,8 @@ struct QPlaylistCont { continuation: String, } -impl RustyPipe { - pub async fn get_playlist(&self, playlist_id: &str) -> Result { +impl RustyPipeQuery { + pub async fn get_playlist(self, playlist_id: &str) -> Result { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QPlaylist { context, @@ -44,7 +44,7 @@ impl RustyPipe { .await } - pub async fn get_playlist_cont(&self, playlist: &mut Playlist) -> Result<()> { + pub async fn get_playlist_cont(self, playlist: &mut Playlist) -> Result<()> { match &playlist.ctoken { Some(ctoken) => { let context = self.get_context(ClientType::Desktop, true).await; @@ -308,7 +308,7 @@ mod tests { use rstest::rstest; - use crate::report::TestFileReporter; + use crate::{client2::RustyPipe, report::TestFileReporter}; use super::*; @@ -349,7 +349,7 @@ mod tests { #[case] channel: Option, ) { let rp = RustyPipe::new_test(); - let playlist = rp.get_playlist(id).await.unwrap(); + let playlist = rp.test_query().get_playlist(id).await.unwrap(); assert_eq!(playlist.id, id); assert_eq!(playlist.name, name); @@ -380,8 +380,8 @@ mod tests { } let reporter = TestFileReporter::new(json_path); - let rp = RustyPipe::new(None, Some(Box::new(reporter)), None).report(true); - rp.get_playlist(id).await.unwrap(); + let rp = RustyPipe::new(None, Some(Box::new(reporter)), None); + rp.test_query().report(true).get_playlist(id).await.unwrap(); } } @@ -412,12 +412,16 @@ mod tests { async fn t_playlist_cont() { let rp = RustyPipe::new_test(); let mut playlist = rp + .test_query() .get_playlist("PLbZIPy20-1pN7mqjckepWF78ndb6ci_qi") .await .unwrap(); while playlist.ctoken.is_some() { - rp.get_playlist_cont(&mut playlist).await.unwrap(); + rp.test_query() + .get_playlist_cont(&mut playlist) + .await + .unwrap(); } assert!(playlist.videos.len() > 100); diff --git a/src/client2/response/channel.rs b/src/client2/response/channel.rs index ce7a6db..b4796c3 100644 --- a/src/client2/response/channel.rs +++ b/src/client2/response/channel.rs @@ -4,6 +4,7 @@ use serde_with::VecSkipError; use super::TimeOverlay; use super::{ContentRenderer, ContentsRenderer, Thumbnails, VideoListItem}; +use crate::serializer::text::Text; #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase")] @@ -63,11 +64,11 @@ pub struct GridRenderer { pub struct ChannelVideo { pub video_id: String, pub thumbnail: Thumbnails, - #[serde_as(as = "crate::serializer::text::Text")] + #[serde_as(as = "Text")] pub title: String, - #[serde_as(as = "Option")] + #[serde_as(as = "Option")] pub published_time_text: Option, - #[serde_as(as = "crate::serializer::text::Text")] + #[serde_as(as = "Text")] pub view_count_text: String, #[serde_as(as = "VecSkipError<_>")] pub thumbnail_overlays: Vec, diff --git a/src/client2/response/mod.rs b/src/client2/response/mod.rs index 09597c8..65641b8 100644 --- a/src/client2/response/mod.rs +++ b/src/client2/response/mod.rs @@ -16,7 +16,7 @@ pub use video::VideoRecommendations; use serde::Deserialize; use serde_with::{serde_as, DefaultOnError, VecSkipError}; -use crate::serializer::text::TextLink; +use crate::serializer::text::{Text, TextLink, TextLinks}; #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase")] @@ -94,10 +94,10 @@ pub struct VideoOwner { #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct VideoOwnerRenderer { - #[serde_as(as = "crate::serializer::text::TextLink")] + #[serde_as(as = "TextLink")] pub title: TextLink, pub thumbnail: Thumbnails, - #[serde_as(as = "Option")] + #[serde_as(as = "Option")] pub subscriber_count_text: Option, #[serde(default)] #[serde_as(as = "VecSkipError<_>")] @@ -133,7 +133,7 @@ pub struct TimeOverlay { #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct TimeOverlayRenderer { - #[serde_as(as = "crate::serializer::text::Text")] + #[serde_as(as = "Text")] pub text: String, #[serde(default)] #[serde_as(deserialize_as = "DefaultOnError")] @@ -199,7 +199,7 @@ pub struct MusicColumn { #[serde_as] #[derive(Clone, Debug, Deserialize)] pub struct MusicColumnRenderer { - #[serde_as(as = "crate::serializer::text::TextLinks")] + #[serde_as(as = "TextLinks")] pub text: Vec, } diff --git a/src/client2/response/player.rs b/src/client2/response/player.rs index 36b20f6..eb6a1b7 100644 --- a/src/client2/response/player.rs +++ b/src/client2/response/player.rs @@ -3,9 +3,11 @@ use std::ops::Range; use chrono::NaiveDate; use serde::Deserialize; use serde_with::serde_as; -use serde_with::{json::JsonString, DefaultOnError, VecSkipError}; +use serde_with::{json::JsonString, DefaultOnError}; use super::Thumbnails; +use crate::client2::MapResult; +use crate::serializer::{text::Text, VecLogError}; #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase")] @@ -45,11 +47,11 @@ pub struct StreamingData { #[serde_as(as = "JsonString")] pub expires_in_seconds: u32, #[serde(default)] - #[serde_as(as = "VecSkipError<_>")] - pub formats: Vec, + #[serde_as(as = "VecLogError<_>")] + pub formats: MapResult>, #[serde(default)] - #[serde_as(as = "VecSkipError<_>")] - pub adaptive_formats: Vec, + #[serde_as(as = "VecLogError<_>")] + pub adaptive_formats: MapResult>, /// Only on livestreams pub dash_manifest_url: Option, /// Only on livestreams @@ -73,9 +75,9 @@ pub struct Format { pub width: Option, pub height: Option, - #[serde_as(as = "Option")] + #[serde_as(as = "Option")] pub index_range: Option>, - #[serde_as(as = "Option")] + #[serde_as(as = "Option")] pub init_range: Option>, #[serde_as(as = "JsonString")] @@ -188,7 +190,7 @@ pub struct PlayerCaptionsTracklistRenderer { #[serde(rename_all = "camelCase")] pub struct CaptionTrack { pub base_url: String, - #[serde_as(as = "crate::serializer::text::Text")] + #[serde_as(as = "Text")] pub name: String, pub language_code: String, } diff --git a/src/client2/response/playlist.rs b/src/client2/response/playlist.rs index 2ab8357..8b9118b 100644 --- a/src/client2/response/playlist.rs +++ b/src/client2/response/playlist.rs @@ -3,7 +3,8 @@ use serde_with::serde_as; use serde_with::{json::JsonString, DefaultOnError, VecSkipError}; use crate::client2::MapResult; -use crate::serializer::text::TextLink; +use crate::serializer::text::{Text, TextLink}; +use crate::serializer::VecLogError; use super::{ContentRenderer, ContentsRenderer, Thumbnails, ThumbnailsWrap, VideoListItem}; @@ -57,7 +58,7 @@ pub struct PlaylistVideoListRenderer { #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct PlaylistVideoList { - #[serde_as(as = "crate::serializer::VecLogError<_>")] + #[serde_as(as = "VecLogError<_>")] pub contents: MapResult>>, } @@ -67,10 +68,10 @@ pub struct PlaylistVideoList { pub struct PlaylistVideo { pub video_id: String, pub thumbnail: Thumbnails, - #[serde_as(as = "crate::serializer::text::Text")] + #[serde_as(as = "Text")] pub title: String, #[serde(rename = "shortBylineText")] - #[serde_as(as = "crate::serializer::text::TextLink")] + #[serde_as(as = "TextLink")] pub channel: TextLink, #[serde_as(as = "JsonString")] pub length_seconds: u32, @@ -87,14 +88,14 @@ pub struct Header { #[serde(rename_all = "camelCase")] pub struct HeaderRenderer { pub playlist_id: String, - #[serde_as(as = "crate::serializer::text::Text")] + #[serde_as(as = "Text")] pub title: String, #[serde(default)] - #[serde_as(as = "DefaultOnError>")] + #[serde_as(as = "DefaultOnError>")] pub description_text: Option, - #[serde_as(as = "crate::serializer::text::Text")] + #[serde_as(as = "Text")] pub num_videos_text: String, - #[serde_as(as = "Option")] + #[serde_as(as = "Option")] pub owner_text: Option, // Alternative layout @@ -119,7 +120,7 @@ pub struct Byline { #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct BylineRenderer { - #[serde_as(as = "crate::serializer::text::Text")] + #[serde_as(as = "Text")] pub text: String, } @@ -151,7 +152,7 @@ pub struct SidebarPrimaryInfoRenderer { // - `"495", " videos"` // - `"3,310,996 views"` // - `"Last updated on ", "Aug 7, 2022"` - #[serde_as(as = "Vec")] + #[serde_as(as = "Vec")] pub stats: Vec, } @@ -173,7 +174,7 @@ pub struct OnResponseReceivedAction { #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct AppendAction { - #[serde_as(as = "crate::serializer::VecLogError<_>")] + #[serde_as(as = "VecLogError<_>")] pub continuation_items: MapResult>>, pub target_id: String, } diff --git a/src/deobfuscate.rs b/src/deobfuscate.rs index e85d012..2428421 100644 --- a/src/deobfuscate.rs +++ b/src/deobfuscate.rs @@ -3,7 +3,7 @@ use fancy_regex::Regex; use log::debug; use once_cell::sync::Lazy; use reqwest::Client; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; use std::result::Result::Ok; use crate::util; @@ -43,7 +43,7 @@ impl Deobfuscator { sig_fn, sts, }, - }) + }) } pub fn deobfuscate_sig(&self, sig: &str) -> Result { @@ -480,9 +480,7 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c #[test(tokio::test)] async fn t_update() { let client = Client::new(); - let deobf = Deobfuscator::new(client) - .await - .unwrap(); + let deobf = Deobfuscator::new(client).await.unwrap(); let deobf_sig = deobf.deobfuscate_sig("GOqGOqGOq0QJ8wRAIgaryQHfplJ9xJSKFywyaSMHuuwZYsoMTAvRvfm51qIGECIA5061zWeyfMPX9hEl_U6f9J0tr7GTJMKyPf5XNrJb5fb5i").unwrap(); println!("{}", deobf_sig); diff --git a/src/report.rs b/src/report.rs index 85939d9..2c6df25 100644 --- a/src/report.rs +++ b/src/report.rs @@ -86,7 +86,7 @@ impl JsonFileReporter { impl Default for JsonFileReporter { fn default() -> Self { Self { - path: Path::new("RustyPipeReports").to_path_buf(), + path: Path::new("rustypipe_reports").to_path_buf(), } } } @@ -98,12 +98,12 @@ impl Reporter for JsonFileReporter { } } -#[cfg(feature="yaml")] +#[cfg(feature = "yaml")] pub struct YamlFileReporter { path: PathBuf, } -#[cfg(feature="yaml")] +#[cfg(feature = "yaml")] impl YamlFileReporter { pub fn new>(path: P) -> Self { Self { @@ -118,7 +118,7 @@ impl YamlFileReporter { } } -#[cfg(feature="yaml")] +#[cfg(feature = "yaml")] impl Default for YamlFileReporter { fn default() -> Self { Self { @@ -127,7 +127,7 @@ impl Default for YamlFileReporter { } } -#[cfg(feature="yaml")] +#[cfg(feature = "yaml")] impl Reporter for YamlFileReporter { fn report(&self, report: &Report) { self._report(report) diff --git a/src/serializer/mod.rs b/src/serializer/mod.rs index fad2f1e..045160a 100644 --- a/src/serializer/mod.rs +++ b/src/serializer/mod.rs @@ -1,5 +1,7 @@ -pub mod range; pub mod text; + +mod range; mod vec_log_err; +pub use range::Range; pub use vec_log_err::VecLogError; From 63d2a0fb360cd84e8a19791486ecd083f8f1dd42 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Thu, 15 Sep 2022 13:06:45 +0200 Subject: [PATCH 3/5] fix: accept video streams without content length/average bitrate --- cli/src/main.rs | 3 +- src/client2/player.rs | 55 +++++-------------- src/client2/response/player.rs | 10 ++-- ...layer__tests__map_player_data_android.snap | 20 ++++++- src/model/mod.rs | 2 +- src/report.rs | 2 +- 6 files changed, 42 insertions(+), 50 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 7f581d6..0c120f8 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -59,6 +59,7 @@ async fn download_single_video( let res = async { let player_data = rp + .query() .get_player(video_id.as_str(), ClientType::TvHtml5Embed) .await .context(format!( @@ -148,7 +149,7 @@ async fn download_playlist( .expect("unable to build the HTTP client"); let rp = RustyPipe::default(); - let playlist = rp.get_playlist(id).await.unwrap(); + let playlist = rp.query().get_playlist(id).await.unwrap(); // Indicatif setup let multi = MultiProgress::new(); diff --git a/src/client2/player.rs b/src/client2/player.rs index 27571c9..079ef26 100644 --- a/src/client2/player.rs +++ b/src/client2/player.rs @@ -59,15 +59,10 @@ struct QContentPlaybackContext { impl RustyPipeQuery { pub async fn get_player(self, video_id: &str, client_type: ClientType) -> Result { - // let (context, deobf) = tokio::join!(self.get_context(client_type, false), self.get_deobf()); - // let deobf = deobf?; - let q1 = self.clone(); let t_context = tokio::spawn(async move { q1.get_context(client_type, false).await }); let q2 = self.clone(); let t_deobf = tokio::spawn(async move { q2.get_deobf().await }); - // let context = t_context.await.unwrap(); - // let deobf = t_deobf.await.unwrap()?; let (context, deobf) = tokio::join!(t_context, t_deobf); let context = context.unwrap(); @@ -234,7 +229,7 @@ impl MapResponse for response::Player { audio_streams.push(c); }; } - (false, false) => warnings.push(format!("invalid format: {}", f.itag)), + (false, false) => warnings.push(format!("invalid stream: itag {}", f.itag)), } } @@ -406,38 +401,16 @@ fn map_video_stream( url, itag: f.itag, bitrate: f.bitrate, - average_bitrate: f.average_bitrate, + average_bitrate: f.average_bitrate.unwrap_or(f.bitrate), size: f.content_length, index_range: f.index_range, init_range: f.init_range, - width: some_or_bail!( - f.width, - MapResult { - c: None, - warnings: map_res.warnings - } - ), - height: some_or_bail!( - f.height, - MapResult { - c: None, - warnings: map_res.warnings - } - ), - fps: some_or_bail!( - f.fps, - MapResult { - c: None, - warnings: map_res.warnings - } - ), - quality: some_or_bail!( - f.quality_label, - MapResult { - c: None, - warnings: map_res.warnings - } - ), + // Note that the format has already been verified using + // is_video(), so these unwraps are safe + width: f.width.unwrap(), + height: f.height.unwrap(), + fps: f.fps.unwrap(), + quality: f.quality_label.unwrap(), hdr: f.color_info.unwrap_or_default().primaries == player::Primaries::ColorPrimariesBt2020, mime: f.mime_type.to_owned(), @@ -445,7 +418,7 @@ fn map_video_stream( get_video_format(mtype), MapResult { c: None, - warnings: vec![format!("no valid format in video format")] + warnings: vec![format!("invalid video format. itag: {}", f.itag)] } ), codec: get_video_codec(codecs), @@ -485,8 +458,8 @@ fn map_audio_stream( url, itag: f.itag, bitrate: f.bitrate, - average_bitrate: f.average_bitrate, - size: f.content_length, + average_bitrate: f.average_bitrate.unwrap_or(f.bitrate), + size: f.content_length.unwrap(), index_range: f.index_range, init_range: f.init_range, mime: f.mime_type.to_owned(), @@ -494,7 +467,7 @@ fn map_audio_stream( get_audio_format(mtype), MapResult { c: None, - warnings: vec![format!("invalid format in audio format {}", f.itag)] + warnings: vec![format!("invalid audio format. itag: {}", f.itag)] } ), codec: get_audio_codec(codecs), @@ -759,7 +732,7 @@ mod tests { // Bitrates may change between requests assert_approx(video.bitrate as f64, 1507068.0); assert_eq!(video.average_bitrate, 1345149); - assert_eq!(video.size, 43553412); + assert_eq!(video.size.unwrap(), 43553412); assert_eq!(video.width, 1280); assert_eq!(video.height, 720); assert_eq!(video.fps, 30); @@ -789,7 +762,7 @@ mod tests { assert_approx(video.bitrate as f64, 1340829.0); assert_approx(video.average_bitrate as f64, 1233444.0); - assert_approx(video.size as f64, 39936630.0); + assert_approx(video.size.unwrap() as f64, 39936630.0); assert_eq!(video.width, 1280); assert_eq!(video.height, 720); assert_eq!(video.fps, 30); diff --git a/src/client2/response/player.rs b/src/client2/response/player.rs index eb6a1b7..501c0d2 100644 --- a/src/client2/response/player.rs +++ b/src/client2/response/player.rs @@ -80,15 +80,15 @@ pub struct Format { #[serde_as(as = "Option")] pub init_range: Option>, - #[serde_as(as = "JsonString")] - pub content_length: u64, + #[serde_as(as = "Option")] + pub content_length: Option, #[serde(default)] #[serde_as(deserialize_as = "DefaultOnError")] pub quality: Option, pub fps: Option, pub quality_label: Option, - pub average_bitrate: u32, + pub average_bitrate: Option, pub color_info: Option, // Audio only @@ -106,7 +106,9 @@ pub struct Format { impl Format { pub fn is_audio(&self) -> bool { - self.audio_quality.is_some() && self.audio_sample_rate.is_some() + self.content_length.is_some() + && self.audio_quality.is_some() + && self.audio_sample_rate.is_some() } pub fn is_video(&self) -> bool { diff --git a/src/client2/snapshots/rustypipe__client2__player__tests__map_player_data_android.snap b/src/client2/snapshots/rustypipe__client2__player__tests__map_player_data_android.snap index 845ccda..4cc0a00 100644 --- a/src/client2/snapshots/rustypipe__client2__player__tests__map_player_data_android.snap +++ b/src/client2/snapshots/rustypipe__client2__player__tests__map_player_data_android.snap @@ -1,6 +1,6 @@ --- -source: src/client/player.rs -expression: player_data +source: src/client2/player.rs +expression: map_res.c --- info: id: pPvd8UxmSbQ @@ -184,6 +184,22 @@ video_only_streams: format: mp4 codec: av01 throttled: false + - url: "https://rr5---sn-h0jeenek.googlevideo.com/videoplayback?c=ANDROID&dur=163.096&ei=q1jpYtOPEYSBgQeHmqbwAQ&expire=1659481355&fexp=24001373%2C24007246&fvip=4&id=o-AEDMTCojVtwpIKOdhBaxEHE5s322qnAJHGqa2r1F46BM&initcwndbps=1527500&ip=2003%3Ade%3Aaf0e%3A2f00%3Ade47%3A297%3Aa6db%3A774e&itag=22&lmt=1580005750956837&lsig=AG3C_xAwRgIhAOiL-qJ04sA8FSOkEJfOYl3gFe4SzwYu_rAf3DMLHYigAiEA0Upi1HqqIu7NH_LTDL0jT1R5TTozQypL5FiSP9RoqtU%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=mQ&mime=video%2Fmp4&mm=31%2C29&mn=sn-h0jeenek%2Csn-h0jelnez&ms=au%2Crdu&mt=1659459429&mv=m&mvi=5&pl=37&ratebypass=yes&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRAIgFlQZgR63Yz9UgY9gVqiyGDVkZmSmACRP3-MmKN7CRzQCIAMHAwZbHmWL1qNH4Nu3A0pXZwErXMVPzMIt-PyxeZqa&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cratebypass%2Cdur%2Clmt&txp=2211222&vprv=1" + itag: 22 + bitrate: 1574434 + average_bitrate: 1574434 + size: ~ + index_range: ~ + init_range: ~ + width: 1280 + height: 720 + fps: 30 + quality: 720p + hdr: false + mime: "video/mp4; codecs=\"avc1.64001F, mp4a.40.2\"" + format: mp4 + codec: avc1 + throttled: false - url: "https://rr5---sn-h0jeenek.googlevideo.com/videoplayback?c=ANDROID&clen=22365208&dur=163.046&ei=q1jpYtOPEYSBgQeHmqbwAQ&expire=1659481355&fexp=24001373%2C24007246&fvip=4&gir=yes&id=o-AEDMTCojVtwpIKOdhBaxEHE5s322qnAJHGqa2r1F46BM&initcwndbps=1527500&ip=2003%3Ade%3Aaf0e%3A2f00%3Ade47%3A297%3Aa6db%3A774e&itag=398&keepalive=yes&lmt=1608048380553749&lsig=AG3C_xAwRgIhAOiL-qJ04sA8FSOkEJfOYl3gFe4SzwYu_rAf3DMLHYigAiEA0Upi1HqqIu7NH_LTDL0jT1R5TTozQypL5FiSP9RoqtU%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=mQ&mime=video%2Fmp4&mm=31%2C29&mn=sn-h0jeenek%2Csn-h0jelnez&ms=au%2Crdu&mt=1659459429&mv=m&mvi=5&otfp=1&pl=37&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRAIgR6KqCOoig_FMl2tWKa7qHSmCjIZa9S7ABzEI16qdO2sCIFXccwql4bqV9CHlqXY4tgxyMFUsp7vW4XUjxs3AyG6H&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cgir%2Cclen%2Cotfp%2Cdur%2Clmt&txp=1311222&vprv=1" itag: 398 bitrate: 1348419 diff --git a/src/model/mod.rs b/src/model/mod.rs index 7853896..a7f2f53 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -59,7 +59,7 @@ pub struct VideoStream { pub itag: u32, pub bitrate: u32, pub average_bitrate: u32, - pub size: u64, + pub size: Option, pub index_range: Option>, pub init_range: Option>, pub width: u32, diff --git a/src/report.rs b/src/report.rs index 2c6df25..7406f6a 100644 --- a/src/report.rs +++ b/src/report.rs @@ -122,7 +122,7 @@ impl YamlFileReporter { impl Default for YamlFileReporter { fn default() -> Self { Self { - path: Path::new("RustyPipeReports").to_path_buf(), + path: Path::new("rustypipe_reports").to_path_buf(), } } } From 05f609e2479fad4f1be4b43f4b3e96a17f116139 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Thu, 15 Sep 2022 16:05:06 +0200 Subject: [PATCH 4/5] fix: clippy warnings --- src/client2/mod.rs | 17 ++++----- src/client2/player.rs | 4 +- src/client2/playlist.rs | 69 +++++++++++++++-------------------- src/client2/response/mod.rs | 16 ++++---- src/client2/response/video.rs | 2 + src/deobfuscate.rs | 4 +- src/download.rs | 20 +++++----- src/lib.rs | 1 + src/serializer/text.rs | 6 +-- src/timeago.rs | 69 ++++++++++++++++------------------- src/util.rs | 7 ++-- 11 files changed, 97 insertions(+), 118 deletions(-) diff --git a/src/client2/mod.rs b/src/client2/mod.rs index f7ee912..ec66071 100644 --- a/src/client2/mod.rs +++ b/src/client2/mod.rs @@ -253,7 +253,7 @@ impl RustyPipe { reporter: Option>, user_agent: Option, ) -> Self { - let user_agent = user_agent.unwrap_or(DEFAULT_UA.to_owned()); + let user_agent = user_agent.unwrap_or_else(|| DEFAULT_UA.to_owned()); let http = ClientBuilder::new() .user_agent(user_agent.to_owned()) @@ -563,6 +563,7 @@ impl RustyPipeQuery { } } + #[allow(clippy::too_many_arguments)] async fn execute_request_deobf< R: DeserializeOwned + MapResponse + Debug, M, @@ -767,7 +768,7 @@ async fn extract_desktop_client_version(http: Client, consent_cookie: String) -> .context("Failed to download sw.js")?; util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) - .ok_or(anyhow!("Could not find desktop client version in sw.js")) + .ok_or_else(|| anyhow!("Could not find desktop client version in sw.js")) }; let from_html = async { @@ -780,9 +781,8 @@ async fn extract_desktop_client_version(http: Client, consent_cookie: String) -> .await .context("Failed to get YT Desktop page")?; - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(anyhow!( - "Could not find desktop client version on html page" - )) + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1) + .ok_or_else(|| anyhow!("Could not find desktop client version on html page")) }; match from_swjs.await { @@ -806,7 +806,7 @@ async fn extract_music_client_version(http: Client, consent_cookie: String) -> R .context("Failed to download sw.js")?; util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) - .ok_or(anyhow!("Could not find desktop client version in sw.js")) + .ok_or_else(|| anyhow!("Could not find desktop client version in sw.js")) }; let from_html = async { @@ -817,9 +817,8 @@ async fn extract_music_client_version(http: Client, consent_cookie: String) -> R .await .context("Failed to get YT Desktop page")?; - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(anyhow!( - "Could not find desktop client version on html page" - )) + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1) + .ok_or_else(|| anyhow!("Could not find desktop client version on html page")) }; match from_swjs.await { diff --git a/src/client2/player.rs b/src/client2/player.rs index 079ef26..b920971 100644 --- a/src/client2/player.rs +++ b/src/client2/player.rs @@ -297,7 +297,7 @@ fn deobf_nsig( let nsig: String; match url_params.get("n") { Some(n) => { - nsig = if n.to_owned() == last_nsig[0] { + nsig = if n == &last_nsig[0] { last_nsig[1].to_owned() } else { let nsig = deobf.deobfuscate_nsig(n)?; @@ -503,7 +503,7 @@ fn parse_mime(mime: &str) -> Option<(&str, Vec<&str>)> { static PATTERN: Lazy = Lazy::new(|| Regex::new(r#"(\w+/\w+);\scodecs="([a-zA-Z-0-9.,\s]*)""#).unwrap()); - let captures = some_or_bail!(PATTERN.captures(&mime).ok().flatten(), None); + let captures = some_or_bail!(PATTERN.captures(mime).ok().flatten(), None); Some(( captures.get(1).unwrap().as_str(), captures diff --git a/src/client2/playlist.rs b/src/client2/playlist.rs index 2fd0ade..dcd1023 100644 --- a/src/client2/playlist.rs +++ b/src/client2/playlist.rs @@ -183,21 +183,15 @@ impl MapResponse for response::Playlist { let description = self.header.playlist_header_renderer.description_text; let channel = match self.header.playlist_header_renderer.owner_text { - Some(owner_text) => match owner_text { - TextLink::Browse { - text, - page_type, - browse_id, - } => match page_type { - PageType::Channel => Some(Channel { - id: browse_id, - name: text, - }), - _ => None, - }, - _ => None, - }, - None => None, + Some(TextLink::Browse { + text, + page_type: PageType::Channel, + browse_id, + }) => Some(Channel { + id: browse_id, + name: text, + }), + _ => None, }; let mut warnings = video_items.warnings.to_owned(); @@ -256,7 +250,7 @@ impl MapResponse<(Vec