From 62853d50bfaf6ffd693d26773947e0734e45a074 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 12 Oct 2022 21:17:54 +0200 Subject: [PATCH 1/2] tests: add tests for YT internal channels, remove RSS default feature --- Cargo.toml | 2 +- src/client/channel.rs | 12 ++-- src/client/mod.rs | 1 - src/client/response/mod.rs | 5 +- src/error.rs | 6 +- tests/youtube.rs | 124 +++++++++++++++++++++++-------------- 6 files changed, 90 insertions(+), 60 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index be45ff9..f3457e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ include = ["/src", "README.md", "LICENSE", "!snapshots"] members = [".", "codegen", "cli"] [features] -default = ["default-tls", "rss"] +default = ["default-tls"] all = ["rss", "html"] rss = ["quick-xml"] diff --git a/src/client/channel.rs b/src/client/channel.rs index d8a69fd..90a36d9 100644 --- a/src/client/channel.rs +++ b/src/client/channel.rs @@ -383,11 +383,13 @@ fn map_channel( id: &str, lang: Language, ) -> Result, ExtractionError> { - let header = header.ok_or(ExtractionError::NoData)?; + let header = + header.ok_or_else(|| ExtractionError::ContentUnavailable("channel not found".into()))?; let metadata = metadata - .ok_or(ExtractionError::NoData)? + .ok_or_else(|| ExtractionError::ContentUnavailable("channel not found".into()))? .channel_metadata_renderer; - let microformat = microformat.ok_or(ExtractionError::NoData)?; + let microformat = microformat + .ok_or_else(|| ExtractionError::ContentUnavailable("channel not found".into()))?; if metadata.external_id != id { return Err(ExtractionError::WrongResult(format!( @@ -463,7 +465,9 @@ fn map_channel_content( Some(contents) => { let tabs = contents.two_column_browse_results_renderer.tabs; if tabs.is_empty() { - return Err(ExtractionError::NoData); + return Err(ExtractionError::ContentUnavailable( + "channel not found".into(), + )); } let (channel_content, target_id) = tabs diff --git a/src/client/mod.rs b/src/client/mod.rs index 1249fe5..397fa7c 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1095,7 +1095,6 @@ impl RustyPipeQuery { ExtractionError::VideoUnavailable(_, _) | ExtractionError::VideoAgeRestricted | ExtractionError::ContentUnavailable(_) - | ExtractionError::NoData | ExtractionError::Retry => (), _ => create_report(Level::ERR, Some(e.to_string()), Vec::new()), } diff --git a/src/client/response/mod.rs b/src/client/response/mod.rs index 7e6e628..9c34515 100644 --- a/src/client/response/mod.rs +++ b/src/client/response/mod.rs @@ -506,9 +506,10 @@ pub fn alerts_to_err(alerts: Option>) -> ExtractionError { .into_iter() .map(|a| a.alert_renderer.text) .collect::>() - .join(" "), + .join(" ") + .into(), ), - None => ExtractionError::InvalidData("no contents".into()), + None => ExtractionError::ContentUnavailable("content not found".into()), } } diff --git a/src/error.rs b/src/error.rs index 3b81c1c..94bdfb0 100644 --- a/src/error.rs +++ b/src/error.rs @@ -75,10 +75,8 @@ pub enum ExtractionError { VideoUnavailable(&'static str, String), #[error("Video is age restricted")] VideoAgeRestricted, - #[error("Content is not available. Reason (from YT): {0}")] - ContentUnavailable(String), - #[error("Got no data from YouTube")] - NoData, + #[error("Content is not available. Reason: {0}")] + ContentUnavailable(Cow<'static, str>), #[error("deserialization error: {0}")] Deserialization(#[from] serde_json::Error), #[error("got invalid data from YT: {0}")] diff --git a/tests/youtube.rs b/tests/youtube.rs index 668da38..9a92272 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -1,4 +1,4 @@ -use chrono::{Datelike, Timelike}; +use chrono::Datelike; use rstest::rstest; use rustypipe::client::{ClientType, RustyPipe}; @@ -218,10 +218,14 @@ async fn playlist_not_found() { .await .unwrap_err(); - assert!(matches!( - err, - Error::Extraction(ExtractionError::ContentUnavailable(_)) - )); + assert!( + matches!( + err, + Error::Extraction(ExtractionError::ContentUnavailable(_)) + ), + "got: {}", + err + ); } //#VIDEO DETAILS @@ -324,6 +328,7 @@ async fn get_video_details_music() { assert!(!details.is_live); assert!(!details.is_ccommons); + assert!(!details.recommended.items.is_empty()); assert!(!details.recommended.is_exhausted()); // Comments are disabled for this video @@ -381,6 +386,7 @@ async fn get_video_details_ccommons() { assert!(!details.is_live); assert!(details.is_ccommons); + assert!(!details.recommended.items.is_empty()); assert!(!details.recommended.is_exhausted()); assert!( @@ -520,6 +526,7 @@ async fn get_video_details_chapters() { "###); } + assert!(!details.recommended.items.is_empty()); assert!(!details.recommended.is_exhausted()); assert!( @@ -579,6 +586,7 @@ async fn get_video_details_live() { assert!(details.is_live); assert!(!details.is_ccommons); + assert!(!details.recommended.items.is_empty()); assert!(!details.recommended.is_exhausted()); // No comments because livestream @@ -637,10 +645,14 @@ async fn get_video_details_not_found() { let rp = RustyPipe::builder().strict().build(); let err = rp.query().video_details("abcdefgLi5X").await.unwrap_err(); - assert!(matches!( - err, - Error::Extraction(ExtractionError::ContentUnavailable(_)) - )) + assert!( + matches!( + err, + Error::Extraction(ExtractionError::ContentUnavailable(_)) + ), + "got: {}", + err + ) } #[tokio::test] @@ -837,13 +849,15 @@ fn assert_channel_eevblog(channel: &Channel) { #[rstest] #[case::artist("UC_vmjW5e1xEHhYjY2a0kK1A", "Oonagh - Topic", false, false)] #[case::shorts("UCh8gHdtzO2tXd593_bjErWg", "Doobydobap", true, true)] -#[case::live( +#[case::livestream( "UChs0pSaEoNLV4mevBFGaoKA", "The Good Life Radio x Sensual Musique", true, true )] #[case::music("UC-9-kyTW8ZkZNDHQJ6FgpwQ", "Music", false, false)] +#[case::live("UC4R8DWoMoI7CAwX8_LjQHig", "Live", false, false)] +#[case::news("UCYfdidRxbB8Qhf0Nx7ioOYw", "News", false, false)] #[tokio::test] async fn channel_more( #[case] id: &str, @@ -882,58 +896,72 @@ async fn channel_more( } #[rstest] -#[case::gaming("UCOpNcN46UbXVtpKMrmU4Abg", false)] -#[case::not_found("UCOpNcN46UbXVtpKMrmU4Abx", true)] +#[case::not_exist("UCOpNcN46UbXVtpKMrmU4Abx")] +#[case::gaming("UCOpNcN46UbXVtpKMrmU4Abg")] +#[case::movies("UCuJcl0Ju-gPDoksRjK1ya-w")] +#[case::sports("UCEgdi0XIXXZ-qJOFPf4JSKw")] +#[case::learning("UCtFRv9O2AHqOZjjynzrv-xg")] #[tokio::test] -async fn channel_error(#[case] id: &str, #[case] not_found: bool) { +async fn channel_not_found(#[case] id: &str) { let rp = RustyPipe::builder().strict().build(); let err = rp.query().channel_videos(&id).await.unwrap_err(); - if not_found { - assert!(matches!( + assert!( + matches!( err, Error::Extraction(ExtractionError::ContentUnavailable(_)) - )); - } else { - assert!(matches!(err, Error::Extraction(ExtractionError::NoData))); - } + ), + "got: {}", + err + ); } //#CHANNEL_RSS -#[tokio::test] -async fn get_channel_rss() { - let rp = RustyPipe::builder().strict().build(); - let channel = rp - .query() - .channel_rss("UCHnyfMqiRRG1u-2MsSQLbXA") - .await - .unwrap(); +#[cfg(feature = "rss")] +mod channel_rss { + use super::*; - assert_eq!(channel.id, "UCHnyfMqiRRG1u-2MsSQLbXA"); - assert_eq!(channel.name, "Veritasium"); - assert_eq!(channel.create_date.year(), 2010); - assert_eq!(channel.create_date.month(), 7); - assert_eq!(channel.create_date.day(), 21); - assert_eq!(channel.create_date.hour(), 7); - assert_eq!(channel.create_date.minute(), 18); + use chrono::Timelike; - assert!(!channel.videos.is_empty()); -} + #[tokio::test] + async fn get_channel_rss() { + let rp = RustyPipe::builder().strict().build(); + let channel = rp + .query() + .channel_rss("UCHnyfMqiRRG1u-2MsSQLbXA") + .await + .unwrap(); -#[tokio::test] -async fn get_channel_rss_not_found() { - let rp = RustyPipe::builder().strict().build(); - let err = rp - .query() - .channel_rss("UCHnyfMqiRRG1u-2MsSQLbXZ") - .await - .unwrap_err(); + assert_eq!(channel.id, "UCHnyfMqiRRG1u-2MsSQLbXA"); + assert_eq!(channel.name, "Veritasium"); + assert_eq!(channel.create_date.year(), 2010); + assert_eq!(channel.create_date.month(), 7); + assert_eq!(channel.create_date.day(), 21); + assert_eq!(channel.create_date.hour(), 7); + assert_eq!(channel.create_date.minute(), 18); - assert!(matches!( - err, - Error::Extraction(ExtractionError::ContentUnavailable(_)) - )); + assert!(!channel.videos.is_empty()); + } + + #[tokio::test] + async fn get_channel_rss_not_found() { + let rp = RustyPipe::builder().strict().build(); + let err = rp + .query() + .channel_rss("UCHnyfMqiRRG1u-2MsSQLbXZ") + .await + .unwrap_err(); + + assert!( + matches!( + err, + Error::Extraction(ExtractionError::ContentUnavailable(_)) + ), + "got: {}", + err + ); + } } //#SEARCH From 01b9c8e310dc1fce4d934d823748f8cf5d77c048 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 12 Oct 2022 22:29:19 +0200 Subject: [PATCH 2/2] fix: clean stuff up --- src/client/channel.rs | 10 +- src/client/channel_rss.rs | 2 +- src/client/mod.rs | 25 ++- src/client/pagination.rs | 319 +++++++----------------------------- src/client/response/mod.rs | 94 ++++++++++- src/client/search.rs | 88 +--------- src/client/video_details.rs | 6 +- src/error.rs | 2 - src/report.rs | 6 +- src/timeago.rs | 12 -- src/util/mod.rs | 14 +- 11 files changed, 185 insertions(+), 393 deletions(-) diff --git a/src/client/channel.rs b/src/client/channel.rs index 90a36d9..e28eb55 100644 --- a/src/client/channel.rs +++ b/src/client/channel.rs @@ -39,7 +39,7 @@ enum Params { impl RustyPipeQuery { pub async fn channel_videos( - &self, + self, channel_id: &str, ) -> Result>, Error> { self.channel_videos_ordered(channel_id, ChannelOrder::default()) @@ -47,7 +47,7 @@ impl RustyPipeQuery { } pub async fn channel_videos_ordered( - &self, + self, channel_id: &str, order: ChannelOrder, ) -> Result>, Error> { @@ -73,7 +73,7 @@ impl RustyPipeQuery { } pub async fn channel_videos_continuation( - &self, + self, ctoken: &str, ) -> Result, Error> { let context = self.get_context(ClientType::Desktop, true).await; @@ -93,7 +93,7 @@ impl RustyPipeQuery { } pub async fn channel_playlists( - &self, + self, channel_id: &str, ) -> Result>, Error> { let context = self.get_context(ClientType::Desktop, true).await; @@ -114,7 +114,7 @@ impl RustyPipeQuery { } pub async fn channel_playlists_continuation( - &self, + self, ctoken: &str, ) -> Result, Error> { let context = self.get_context(ClientType::Desktop, true).await; diff --git a/src/client/channel_rss.rs b/src/client/channel_rss.rs index 0cb9322..e7d3307 100644 --- a/src/client/channel_rss.rs +++ b/src/client/channel_rss.rs @@ -9,7 +9,7 @@ use crate::{ use super::{response, RustyPipeQuery}; impl RustyPipeQuery { - pub async fn channel_rss(&self, channel_id: &str) -> Result { + pub async fn channel_rss(self, channel_id: &str) -> Result { let url = format!( "https://www.youtube.com/feeds/videos.xml?channel_id={}", channel_id diff --git a/src/client/mod.rs b/src/client/mod.rs index 397fa7c..f7b76ff 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -27,7 +27,7 @@ use tokio::sync::RwLock; use crate::{ cache::{CacheStorage, FileStorage}, deobfuscate::{DeobfData, Deobfuscator}, - error::{Error, ExtractionError, Result}, + error::{Error, ExtractionError}, param::{Country, Language}, report::{FileReporter, Level, Report, Reporter}, serializer::MapResult, @@ -467,10 +467,7 @@ impl RustyPipe { } /// Execute the given http request. - async fn http_request( - &self, - request: Request, - ) -> core::result::Result { + async fn http_request(&self, request: Request) -> Result { let mut last_res = None; for n in 0..self.inner.n_http_retries { let res = self.inner.http.execute(request.try_clone().unwrap()).await; @@ -504,7 +501,7 @@ impl RustyPipe { /// Execute the given http request, returning an error in case of a /// non-successful status code. - async fn http_request_estatus(&self, request: Request) -> Result { + async fn http_request_estatus(&self, request: Request) -> Result { let res = self.http_request(request).await?; let status = res.status(); @@ -516,12 +513,12 @@ impl RustyPipe { } /// Execute the given http request, returning the response body as a string. - async fn http_request_txt(&self, request: Request) -> Result { + async fn http_request_txt(&self, request: Request) -> Result { Ok(self.http_request_estatus(request).await?.text().await?) } /// Extract the current version of the YouTube desktop client from the website. - async fn extract_desktop_client_version(&self) -> Result { + async fn extract_desktop_client_version(&self) -> Result { let from_swjs = async { let swjs = self .http_request_txt( @@ -568,7 +565,7 @@ impl RustyPipe { } /// Extract the current version of the YouTube Music desktop client from the website. - async fn extract_music_client_version(&self) -> Result { + async fn extract_music_client_version(&self) -> Result { let from_swjs = async { let swjs = self .http_request_txt( @@ -679,7 +676,7 @@ impl RustyPipe { } /// Instantiate a new deobfuscator from either cached or extracted YouTube JavaScript code. - async fn get_deobf(&self) -> Result { + async fn get_deobf(&self) -> Result { // Write lock here to prevent concurrent tasks from fetching the same data let mut deobf = self.inner.cache.deobf.write().await; @@ -960,7 +957,7 @@ impl RustyPipeQuery { endpoint: &str, body: &B, deobf: Option<&Deobfuscator>, - ) -> Result { + ) -> Result { for n in 0..self.client.inner.n_query_retries.saturating_sub(1) { let res = self ._try_execute_request_deobf::( @@ -1016,7 +1013,7 @@ impl RustyPipeQuery { body: &B, deobf: Option<&Deobfuscator>, report: bool, - ) -> Result { + ) -> Result { let request = self .request_builder(ctype, endpoint) .await @@ -1132,7 +1129,7 @@ impl RustyPipeQuery { id: &str, endpoint: &str, body: &B, - ) -> Result { + ) -> Result { self.execute_request_deobf::(ctype, operation, id, endpoint, body, None) .await } @@ -1158,7 +1155,7 @@ trait MapResponse { id: &str, lang: Language, deobf: Option<&Deobfuscator>, - ) -> core::result::Result, crate::error::ExtractionError>; + ) -> Result, ExtractionError>; } #[cfg(test)] diff --git a/src/client/pagination.rs b/src/client/pagination.rs index 3ebd019..2835a72 100644 --- a/src/client/pagination.rs +++ b/src/client/pagination.rs @@ -1,271 +1,72 @@ -use crate::error::Result; - +use crate::error::Error; use crate::model::{ ChannelPlaylist, ChannelVideo, Comment, Paginator, PlaylistVideo, RecommendedVideo, SearchItem, }; use super::RustyPipeQuery; -impl Paginator { - pub async fn next(&self, query: RustyPipeQuery) -> Result> { - Ok(match &self.ctoken { - Some(ctoken) => Some(query.playlist_continuation(ctoken).await?), - None => None, - }) - } - - pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { - match self.next(query).await { - Ok(Some(paginator)) => { - let mut items = paginator.items; - self.items.append(&mut items); - self.ctoken = paginator.ctoken; - Ok(true) +macro_rules! paginator { + ($entity_type:ty, $cont_function:path) => { + impl Paginator<$entity_type> { + pub async fn next(&self, query: RustyPipeQuery) -> Result, Error> { + Ok(match &self.ctoken { + Some(ctoken) => Some($cont_function(query, ctoken).await?), + None => None, + }) } - Ok(None) => Ok(false), - Err(e) => Err(e), - } - } - pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> { - for _ in 0..n_pages { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} + pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { + match self.next(query).await { + Ok(Some(paginator)) => { + let mut items = paginator.items; + self.items.append(&mut items); + self.ctoken = paginator.ctoken; + Ok(true) + } + Ok(None) => Ok(false), + Err(e) => Err(e), + } + } + + pub async fn extend_pages( + &mut self, + query: RustyPipeQuery, + n_pages: usize, + ) -> Result<(), Error> { + for _ in 0..n_pages { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } + + pub async fn extend_limit( + &mut self, + query: RustyPipeQuery, + n_items: usize, + ) -> Result<(), Error> { + while self.items.len() < n_items { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) } } - Ok(()) - } - - pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> { - while self.items.len() < n_items { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } + }; } -impl Paginator { - pub async fn next(&self, query: RustyPipeQuery) -> Result> { - Ok(match &self.ctoken { - Some(ctoken) => Some(query.video_recommendations(ctoken).await?), - None => None, - }) - } - - pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { - match self.next(query).await { - Ok(Some(paginator)) => { - let mut items = paginator.items; - self.items.append(&mut items); - self.ctoken = paginator.ctoken; - Ok(true) - } - Ok(None) => Ok(false), - Err(e) => Err(e), - } - } - - pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> { - for _ in 0..n_pages { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } - - pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> { - while self.items.len() < n_items { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } -} - -impl Paginator { - pub async fn next(&self, query: RustyPipeQuery) -> Result> { - Ok(match &self.ctoken { - Some(ctoken) => Some(query.channel_videos_continuation(ctoken).await?), - None => None, - }) - } - - pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { - match self.next(query).await { - Ok(Some(paginator)) => { - let mut items = paginator.items; - self.items.append(&mut items); - self.ctoken = paginator.ctoken; - Ok(true) - } - Ok(None) => Ok(false), - Err(e) => Err(e), - } - } - - pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> { - for _ in 0..n_pages { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } - - pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> { - while self.items.len() < n_items { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } -} - -impl Paginator { - pub async fn next(&self, query: RustyPipeQuery) -> Result> { - Ok(match &self.ctoken { - Some(ctoken) => Some(query.channel_playlists_continuation(ctoken).await?), - None => None, - }) - } - - pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { - match self.next(query).await { - Ok(Some(paginator)) => { - let mut items = paginator.items; - self.items.append(&mut items); - self.ctoken = paginator.ctoken; - Ok(true) - } - Ok(None) => Ok(false), - Err(e) => Err(e), - } - } - - pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> { - for _ in 0..n_pages { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } - - pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> { - while self.items.len() < n_items { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } -} - -impl Paginator { - pub async fn next(&self, query: RustyPipeQuery) -> Result> { - Ok(match &self.ctoken { - Some(ctoken) => Some(query.video_comments(ctoken).await?), - None => None, - }) - } - - pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { - match self.next(query).await { - Ok(Some(paginator)) => { - let mut items = paginator.items; - self.items.append(&mut items); - self.ctoken = paginator.ctoken; - Ok(true) - } - Ok(None) => Ok(false), - Err(e) => Err(e), - } - } - - pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> { - for _ in 0..n_pages { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } - - pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> { - while self.items.len() < n_items { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } -} - -impl Paginator { - pub async fn next(&self, query: RustyPipeQuery) -> Result> { - Ok(match &self.ctoken { - Some(ctoken) => Some(query.search_continuation(ctoken).await?), - None => None, - }) - } - - pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { - match self.next(query).await { - Ok(Some(paginator)) => { - let mut items = paginator.items; - self.items.append(&mut items); - self.ctoken = paginator.ctoken; - Ok(true) - } - Ok(None) => Ok(false), - Err(e) => Err(e), - } - } - - pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> { - for _ in 0..n_pages { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } - - pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> { - while self.items.len() < n_items { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } -} +paginator!(PlaylistVideo, RustyPipeQuery::playlist_continuation); +paginator!(RecommendedVideo, RustyPipeQuery::video_recommendations); +paginator!(Comment, RustyPipeQuery::video_comments); +paginator!(ChannelVideo, RustyPipeQuery::channel_videos_continuation); +paginator!( + ChannelPlaylist, + RustyPipeQuery::channel_playlists_continuation +); +paginator!(SearchItem, RustyPipeQuery::search_continuation); diff --git a/src/client/response/mod.rs b/src/client/response/mod.rs index 9c34515..204b466 100644 --- a/src/client/response/mod.rs +++ b/src/client/response/mod.rs @@ -38,6 +38,10 @@ use crate::serializer::{ use crate::timeago; use crate::util::{self, TryRemove}; +use self::search::ChannelRenderer; +use self::search::PlaylistRenderer; +use self::search::VideoRenderer; + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ContentRenderer { @@ -518,7 +522,7 @@ pub trait FromWLang { } pub trait TryFromWLang: Sized { - fn from_w_lang(from: T, lang: Language) -> core::result::Result; + fn from_w_lang(from: T, lang: Language) -> Result; } impl FromWLang for model::ChannelVideo { @@ -578,7 +582,7 @@ impl TryFromWLang for model::RecommendedVideo { fn from_w_lang( video: CompactVideoRenderer, lang: Language, - ) -> core::result::Result { + ) -> Result { let channel = model::ChannelId::try_from(video.channel)?; Ok(Self { @@ -609,3 +613,89 @@ impl TryFromWLang for model::RecommendedVideo { }) } } + +impl TryFromWLang for model::SearchVideo { + fn from_w_lang(video: VideoRenderer, lang: Language) -> Result { + let channel = model::ChannelId::try_from(video.channel)?; + let mut metadata_snippets = video.detailed_metadata_snippets; + + Ok(Self { + id: video.video_id, + title: video.title, + length: video + .length_text + .and_then(|txt| util::parse_video_length(&txt)), + thumbnail: video.thumbnail.into(), + channel: model::ChannelTag { + id: channel.id, + name: channel.name, + avatar: video + .channel_thumbnail_supported_renderers + .channel_thumbnail_with_link_renderer + .thumbnail + .into(), + verification: video.owner_badges.into(), + subscriber_count: None, + }, + publish_date: video + .published_time_text + .as_ref() + .and_then(|txt| timeago::parse_timeago_to_dt(lang, txt)), + publish_date_txt: video.published_time_text, + view_count: video + .view_count_text + .and_then(|txt| util::parse_numeric(&txt).ok()) + .unwrap_or_default(), + is_live: video.thumbnail_overlays.is_live(), + is_short: video.thumbnail_overlays.is_short(), + short_description: metadata_snippets + .try_swap_remove(0) + .map(|s| s.snippet_text) + .unwrap_or_default(), + }) + } +} + +impl From for model::SearchPlaylist { + fn from(playlist: PlaylistRenderer) -> Self { + let mut thumbnails = playlist.thumbnails; + + Self { + id: playlist.playlist_id, + name: playlist.title, + thumbnail: thumbnails.try_swap_remove(0).unwrap_or_default().into(), + video_count: playlist.video_count, + first_videos: playlist + .videos + .into_iter() + .map(|v| model::SearchPlaylistVideo { + id: v.child_video_renderer.video_id, + title: v.child_video_renderer.title, + length: v + .child_video_renderer + .length_text + .and_then(|txt| util::parse_video_length(&txt)), + }) + .collect(), + } + } +} + +impl From for model::SearchChannel { + fn from(channel: ChannelRenderer) -> Self { + Self { + id: channel.channel_id, + name: channel.title, + avatar: channel.thumbnail.into(), + verification: channel.owner_badges.into(), + subscriber_count: channel + .subscriber_count_text + .and_then(|txt| util::parse_numeric(&txt).ok()), + video_count: channel + .video_count_text + .and_then(|txt| util::parse_numeric(&txt).ok()) + .unwrap_or_default(), + short_description: channel.description_snippet, + } + } +} diff --git a/src/client/search.rs b/src/client/search.rs index c6a3500..249c89f 100644 --- a/src/client/search.rs +++ b/src/client/search.rs @@ -3,17 +3,13 @@ use serde::Serialize; use crate::{ deobfuscate::Deobfuscator, error::{Error, ExtractionError}, - model::{ - ChannelId, ChannelTag, Paginator, SearchChannel, SearchItem, SearchPlaylist, - SearchPlaylistVideo, SearchResult, SearchVideo, - }, + model::{Paginator, SearchItem, SearchResult, SearchVideo}, param::{search_filter::SearchFilter, Language}, - timeago, - util::{self, TryRemove}, + util::TryRemove, }; use super::{ - response::{self, IsLive, IsShort}, + response::{self, TryFromWLang}, ClientType, MapResponse, MapResult, QContinuation, RustyPipeQuery, YTContext, }; @@ -183,86 +179,20 @@ fn map_search_items( let mapped_items = items .into_iter() .filter_map(|item| match item { - response::search::SearchItem::VideoRenderer(mut video) => { - match ChannelId::try_from(video.channel) { - Ok(channel) => Some(SearchItem::Video(SearchVideo { - id: video.video_id, - title: video.title, - length: video - .length_text - .and_then(|txt| util::parse_video_length_or_warn(&txt, &mut warnings)), - thumbnail: video.thumbnail.into(), - channel: ChannelTag { - id: channel.id, - name: channel.name, - avatar: video - .channel_thumbnail_supported_renderers - .channel_thumbnail_with_link_renderer - .thumbnail - .into(), - verification: video.owner_badges.into(), - subscriber_count: None, - }, - publish_date: video.published_time_text.as_ref().and_then(|txt| { - timeago::parse_timeago_or_warn(lang, txt, &mut warnings) - }), - publish_date_txt: video.published_time_text, - view_count: video - .view_count_text - .and_then(|txt| util::parse_numeric(&txt).ok()) - .unwrap_or_default(), - is_live: video.thumbnail_overlays.is_live(), - is_short: video.thumbnail_overlays.is_short(), - short_description: video - .detailed_metadata_snippets - .try_swap_remove(0) - .map(|s| s.snippet_text) - .unwrap_or_default(), - })), + response::search::SearchItem::VideoRenderer(video) => { + match SearchVideo::from_w_lang(video, lang) { + Ok(video) => Some(SearchItem::Video(video)), Err(e) => { warnings.push(e.to_string()); None } } } - response::search::SearchItem::PlaylistRenderer(mut playlist) => { - Some(SearchItem::Playlist(SearchPlaylist { - id: playlist.playlist_id, - name: playlist.title, - thumbnail: playlist - .thumbnails - .try_swap_remove(0) - .unwrap_or_default() - .into(), - video_count: playlist.video_count, - first_videos: playlist - .videos - .into_iter() - .map(|v| SearchPlaylistVideo { - id: v.child_video_renderer.video_id, - title: v.child_video_renderer.title, - length: v.child_video_renderer.length_text.and_then(|txt| { - util::parse_video_length_or_warn(&txt, &mut warnings) - }), - }) - .collect(), - })) + response::search::SearchItem::PlaylistRenderer(playlist) => { + Some(SearchItem::Playlist(playlist.into())) } response::search::SearchItem::ChannelRenderer(channel) => { - Some(SearchItem::Channel(SearchChannel { - id: channel.channel_id, - name: channel.title, - avatar: channel.thumbnail.into(), - verification: channel.owner_badges.into(), - subscriber_count: channel - .subscriber_count_text - .and_then(|txt| util::parse_numeric_or_warn(&txt, &mut warnings)), - video_count: channel - .video_count_text - .and_then(|txt| util::parse_numeric(&txt).ok()) - .unwrap_or_default(), - short_description: channel.description_snippet, - })) + Some(SearchItem::Channel(channel.into())) } response::search::SearchItem::ShowingResultsForRenderer { corrected_query } => { c_query = Some(corrected_query); diff --git a/src/client/video_details.rs b/src/client/video_details.rs index 731d1d8..5d55af2 100644 --- a/src/client/video_details.rs +++ b/src/client/video_details.rs @@ -516,11 +516,7 @@ fn map_comment( }), _ => None, }, - publish_date: timeago::parse_timeago_or_warn( - lang, - &c.published_time_text, - &mut warnings, - ), + publish_date: timeago::parse_timeago_to_dt(lang, &c.published_time_text), publish_date_txt: c.published_time_text, like_count: util::parse_numeric_or_warn( &c.action_buttons diff --git a/src/error.rs b/src/error.rs index 94bdfb0..212c144 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,7 +1,5 @@ use std::borrow::Cow; -pub(crate) type Result = core::result::Result; - /// Custom error type for the RustyPipe library #[derive(thiserror::Error, Debug)] #[non_exhaustive] diff --git a/src/report.rs b/src/report.rs index cf29592..a7ab00a 100644 --- a/src/report.rs +++ b/src/report.rs @@ -11,7 +11,7 @@ use log::error; use serde::{Deserialize, Serialize}; use crate::deobfuscate::DeobfData; -use crate::error::{Error, Result}; +use crate::error::Error; #[derive(Debug, Clone, Serialize, Deserialize)] #[non_exhaustive] @@ -96,7 +96,7 @@ impl FileReporter { } } - fn _report(&self, report: &Report) -> Result<()> { + fn _report(&self, report: &Report) -> Result<(), Error> { let report_path = get_report_path(&self.path, report, "json")?; serde_json::to_writer_pretty(&File::create(report_path)?, &report) .map_err(|e| Error::Other(format!("could not serialize report. err: {}", e).into()))?; @@ -119,7 +119,7 @@ impl Reporter for FileReporter { } } -fn get_report_path(root: &Path, report: &Report, ext: &str) -> Result { +fn get_report_path(root: &Path, report: &Report, ext: &str) -> Result { if !root.is_dir() { std::fs::create_dir_all(root)?; } diff --git a/src/timeago.rs b/src/timeago.rs index c6ed972..fbd648e 100644 --- a/src/timeago.rs +++ b/src/timeago.rs @@ -184,18 +184,6 @@ pub fn parse_timeago_to_dt(lang: Language, textual_date: &str) -> Option, -) -> Option> { - let res = parse_timeago_to_dt(lang, textual_date); - if res.is_none() { - warnings.push(format!("could not parse timeago `{}`", textual_date)); - } - res -} - /// Parse a textual date (e.g. "29 minutes ago" or "Jul 2, 2014") into a ParsedDate object. /// /// Returns None if the date could not be parsed. diff --git a/src/util/mod.rs b/src/util/mod.rs index 20452a1..d87db0a 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -15,7 +15,7 @@ use once_cell::sync::Lazy; use rand::Rng; use url::Url; -use crate::{error::Error, error::Result, param::Language}; +use crate::{error::Error, param::Language}; const CONTENT_PLAYBACK_NONCE_ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; @@ -57,7 +57,7 @@ pub fn generate_content_playback_nonce() -> String { /// Example: /// /// `example.com/api?k1=v1&k2=v2 => example.com/api; {k1: v1, k2: v2}` -pub fn url_to_params(url: &str) -> Result<(String, BTreeMap)> { +pub fn url_to_params(url: &str) -> Result<(String, BTreeMap), Error> { let mut parsed_url = Url::parse(url) .map_err(|e| Error::Other(format!("could not parse url `{}` err: {}", url, e).into()))?; let url_params: BTreeMap = parsed_url @@ -77,7 +77,7 @@ pub fn urlencode(string: &str) -> String { } /// Parse a string after removing all non-numeric characters -pub fn parse_numeric(string: &str) -> core::result::Result +pub fn parse_numeric(string: &str) -> Result where F: FromStr, { @@ -147,14 +147,6 @@ where res.ok() } -pub fn parse_video_length_or_warn(text: &str, warnings: &mut Vec) -> Option { - let res = parse_video_length(text); - if res.is_none() { - warnings.push(format!("could not parse video length `{}`", text)); - } - res -} - pub fn retry_delay( n_past_retries: u32, min_retry_interval: u32,