diff --git a/Cargo.toml b/Cargo.toml index f3457e4..be45ff9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ include = ["/src", "README.md", "LICENSE", "!snapshots"] members = [".", "codegen", "cli"] [features] -default = ["default-tls"] +default = ["default-tls", "rss"] all = ["rss", "html"] rss = ["quick-xml"] diff --git a/src/client/channel.rs b/src/client/channel.rs index e28eb55..d8a69fd 100644 --- a/src/client/channel.rs +++ b/src/client/channel.rs @@ -39,7 +39,7 @@ enum Params { impl RustyPipeQuery { pub async fn channel_videos( - self, + &self, channel_id: &str, ) -> Result>, Error> { self.channel_videos_ordered(channel_id, ChannelOrder::default()) @@ -47,7 +47,7 @@ impl RustyPipeQuery { } pub async fn channel_videos_ordered( - self, + &self, channel_id: &str, order: ChannelOrder, ) -> Result>, Error> { @@ -73,7 +73,7 @@ impl RustyPipeQuery { } pub async fn channel_videos_continuation( - self, + &self, ctoken: &str, ) -> Result, Error> { let context = self.get_context(ClientType::Desktop, true).await; @@ -93,7 +93,7 @@ impl RustyPipeQuery { } pub async fn channel_playlists( - self, + &self, channel_id: &str, ) -> Result>, Error> { let context = self.get_context(ClientType::Desktop, true).await; @@ -114,7 +114,7 @@ impl RustyPipeQuery { } pub async fn channel_playlists_continuation( - self, + &self, ctoken: &str, ) -> Result, Error> { let context = self.get_context(ClientType::Desktop, true).await; @@ -383,13 +383,11 @@ fn map_channel( id: &str, lang: Language, ) -> Result, ExtractionError> { - let header = - header.ok_or_else(|| ExtractionError::ContentUnavailable("channel not found".into()))?; + let header = header.ok_or(ExtractionError::NoData)?; let metadata = metadata - .ok_or_else(|| ExtractionError::ContentUnavailable("channel not found".into()))? + .ok_or(ExtractionError::NoData)? .channel_metadata_renderer; - let microformat = microformat - .ok_or_else(|| ExtractionError::ContentUnavailable("channel not found".into()))?; + let microformat = microformat.ok_or(ExtractionError::NoData)?; if metadata.external_id != id { return Err(ExtractionError::WrongResult(format!( @@ -465,9 +463,7 @@ fn map_channel_content( Some(contents) => { let tabs = contents.two_column_browse_results_renderer.tabs; if tabs.is_empty() { - return Err(ExtractionError::ContentUnavailable( - "channel not found".into(), - )); + return Err(ExtractionError::NoData); } let (channel_content, target_id) = tabs diff --git a/src/client/channel_rss.rs b/src/client/channel_rss.rs index e7d3307..0cb9322 100644 --- a/src/client/channel_rss.rs +++ b/src/client/channel_rss.rs @@ -9,7 +9,7 @@ use crate::{ use super::{response, RustyPipeQuery}; impl RustyPipeQuery { - pub async fn channel_rss(self, channel_id: &str) -> Result { + pub async fn channel_rss(&self, channel_id: &str) -> Result { let url = format!( "https://www.youtube.com/feeds/videos.xml?channel_id={}", channel_id diff --git a/src/client/mod.rs b/src/client/mod.rs index f7b76ff..1249fe5 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -27,7 +27,7 @@ use tokio::sync::RwLock; use crate::{ cache::{CacheStorage, FileStorage}, deobfuscate::{DeobfData, Deobfuscator}, - error::{Error, ExtractionError}, + error::{Error, ExtractionError, Result}, param::{Country, Language}, report::{FileReporter, Level, Report, Reporter}, serializer::MapResult, @@ -467,7 +467,10 @@ impl RustyPipe { } /// Execute the given http request. - async fn http_request(&self, request: Request) -> Result { + async fn http_request( + &self, + request: Request, + ) -> core::result::Result { let mut last_res = None; for n in 0..self.inner.n_http_retries { let res = self.inner.http.execute(request.try_clone().unwrap()).await; @@ -501,7 +504,7 @@ impl RustyPipe { /// Execute the given http request, returning an error in case of a /// non-successful status code. - async fn http_request_estatus(&self, request: Request) -> Result { + async fn http_request_estatus(&self, request: Request) -> Result { let res = self.http_request(request).await?; let status = res.status(); @@ -513,12 +516,12 @@ impl RustyPipe { } /// Execute the given http request, returning the response body as a string. - async fn http_request_txt(&self, request: Request) -> Result { + async fn http_request_txt(&self, request: Request) -> Result { Ok(self.http_request_estatus(request).await?.text().await?) } /// Extract the current version of the YouTube desktop client from the website. - async fn extract_desktop_client_version(&self) -> Result { + async fn extract_desktop_client_version(&self) -> Result { let from_swjs = async { let swjs = self .http_request_txt( @@ -565,7 +568,7 @@ impl RustyPipe { } /// Extract the current version of the YouTube Music desktop client from the website. - async fn extract_music_client_version(&self) -> Result { + async fn extract_music_client_version(&self) -> Result { let from_swjs = async { let swjs = self .http_request_txt( @@ -676,7 +679,7 @@ impl RustyPipe { } /// Instantiate a new deobfuscator from either cached or extracted YouTube JavaScript code. - async fn get_deobf(&self) -> Result { + async fn get_deobf(&self) -> Result { // Write lock here to prevent concurrent tasks from fetching the same data let mut deobf = self.inner.cache.deobf.write().await; @@ -957,7 +960,7 @@ impl RustyPipeQuery { endpoint: &str, body: &B, deobf: Option<&Deobfuscator>, - ) -> Result { + ) -> Result { for n in 0..self.client.inner.n_query_retries.saturating_sub(1) { let res = self ._try_execute_request_deobf::( @@ -1013,7 +1016,7 @@ impl RustyPipeQuery { body: &B, deobf: Option<&Deobfuscator>, report: bool, - ) -> Result { + ) -> Result { let request = self .request_builder(ctype, endpoint) .await @@ -1092,6 +1095,7 @@ impl RustyPipeQuery { ExtractionError::VideoUnavailable(_, _) | ExtractionError::VideoAgeRestricted | ExtractionError::ContentUnavailable(_) + | ExtractionError::NoData | ExtractionError::Retry => (), _ => create_report(Level::ERR, Some(e.to_string()), Vec::new()), } @@ -1129,7 +1133,7 @@ impl RustyPipeQuery { id: &str, endpoint: &str, body: &B, - ) -> Result { + ) -> Result { self.execute_request_deobf::(ctype, operation, id, endpoint, body, None) .await } @@ -1155,7 +1159,7 @@ trait MapResponse { id: &str, lang: Language, deobf: Option<&Deobfuscator>, - ) -> Result, ExtractionError>; + ) -> core::result::Result, crate::error::ExtractionError>; } #[cfg(test)] diff --git a/src/client/pagination.rs b/src/client/pagination.rs index 2835a72..3ebd019 100644 --- a/src/client/pagination.rs +++ b/src/client/pagination.rs @@ -1,72 +1,271 @@ -use crate::error::Error; +use crate::error::Result; + use crate::model::{ ChannelPlaylist, ChannelVideo, Comment, Paginator, PlaylistVideo, RecommendedVideo, SearchItem, }; use super::RustyPipeQuery; -macro_rules! paginator { - ($entity_type:ty, $cont_function:path) => { - impl Paginator<$entity_type> { - pub async fn next(&self, query: RustyPipeQuery) -> Result, Error> { - Ok(match &self.ctoken { - Some(ctoken) => Some($cont_function(query, ctoken).await?), - None => None, - }) - } +impl Paginator { + pub async fn next(&self, query: RustyPipeQuery) -> Result> { + Ok(match &self.ctoken { + Some(ctoken) => Some(query.playlist_continuation(ctoken).await?), + None => None, + }) + } - pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { - match self.next(query).await { - Ok(Some(paginator)) => { - let mut items = paginator.items; - self.items.append(&mut items); - self.ctoken = paginator.ctoken; - Ok(true) - } - Ok(None) => Ok(false), - Err(e) => Err(e), - } + pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { + match self.next(query).await { + Ok(Some(paginator)) => { + let mut items = paginator.items; + self.items.append(&mut items); + self.ctoken = paginator.ctoken; + Ok(true) } + Ok(None) => Ok(false), + Err(e) => Err(e), + } + } - pub async fn extend_pages( - &mut self, - query: RustyPipeQuery, - n_pages: usize, - ) -> Result<(), Error> { - for _ in 0..n_pages { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } - - pub async fn extend_limit( - &mut self, - query: RustyPipeQuery, - n_items: usize, - ) -> Result<(), Error> { - while self.items.len() < n_items { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) + pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> { + for _ in 0..n_pages { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} } } - }; + Ok(()) + } + + pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> { + while self.items.len() < n_items { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } } -paginator!(PlaylistVideo, RustyPipeQuery::playlist_continuation); -paginator!(RecommendedVideo, RustyPipeQuery::video_recommendations); -paginator!(Comment, RustyPipeQuery::video_comments); -paginator!(ChannelVideo, RustyPipeQuery::channel_videos_continuation); -paginator!( - ChannelPlaylist, - RustyPipeQuery::channel_playlists_continuation -); -paginator!(SearchItem, RustyPipeQuery::search_continuation); +impl Paginator { + pub async fn next(&self, query: RustyPipeQuery) -> Result> { + Ok(match &self.ctoken { + Some(ctoken) => Some(query.video_recommendations(ctoken).await?), + None => None, + }) + } + + pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { + match self.next(query).await { + Ok(Some(paginator)) => { + let mut items = paginator.items; + self.items.append(&mut items); + self.ctoken = paginator.ctoken; + Ok(true) + } + Ok(None) => Ok(false), + Err(e) => Err(e), + } + } + + pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> { + for _ in 0..n_pages { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } + + pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> { + while self.items.len() < n_items { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } +} + +impl Paginator { + pub async fn next(&self, query: RustyPipeQuery) -> Result> { + Ok(match &self.ctoken { + Some(ctoken) => Some(query.channel_videos_continuation(ctoken).await?), + None => None, + }) + } + + pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { + match self.next(query).await { + Ok(Some(paginator)) => { + let mut items = paginator.items; + self.items.append(&mut items); + self.ctoken = paginator.ctoken; + Ok(true) + } + Ok(None) => Ok(false), + Err(e) => Err(e), + } + } + + pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> { + for _ in 0..n_pages { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } + + pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> { + while self.items.len() < n_items { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } +} + +impl Paginator { + pub async fn next(&self, query: RustyPipeQuery) -> Result> { + Ok(match &self.ctoken { + Some(ctoken) => Some(query.channel_playlists_continuation(ctoken).await?), + None => None, + }) + } + + pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { + match self.next(query).await { + Ok(Some(paginator)) => { + let mut items = paginator.items; + self.items.append(&mut items); + self.ctoken = paginator.ctoken; + Ok(true) + } + Ok(None) => Ok(false), + Err(e) => Err(e), + } + } + + pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> { + for _ in 0..n_pages { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } + + pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> { + while self.items.len() < n_items { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } +} + +impl Paginator { + pub async fn next(&self, query: RustyPipeQuery) -> Result> { + Ok(match &self.ctoken { + Some(ctoken) => Some(query.video_comments(ctoken).await?), + None => None, + }) + } + + pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { + match self.next(query).await { + Ok(Some(paginator)) => { + let mut items = paginator.items; + self.items.append(&mut items); + self.ctoken = paginator.ctoken; + Ok(true) + } + Ok(None) => Ok(false), + Err(e) => Err(e), + } + } + + pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> { + for _ in 0..n_pages { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } + + pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> { + while self.items.len() < n_items { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } +} + +impl Paginator { + pub async fn next(&self, query: RustyPipeQuery) -> Result> { + Ok(match &self.ctoken { + Some(ctoken) => Some(query.search_continuation(ctoken).await?), + None => None, + }) + } + + pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { + match self.next(query).await { + Ok(Some(paginator)) => { + let mut items = paginator.items; + self.items.append(&mut items); + self.ctoken = paginator.ctoken; + Ok(true) + } + Ok(None) => Ok(false), + Err(e) => Err(e), + } + } + + pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> { + for _ in 0..n_pages { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } + + pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> { + while self.items.len() < n_items { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } +} diff --git a/src/client/response/mod.rs b/src/client/response/mod.rs index 204b466..7e6e628 100644 --- a/src/client/response/mod.rs +++ b/src/client/response/mod.rs @@ -38,10 +38,6 @@ use crate::serializer::{ use crate::timeago; use crate::util::{self, TryRemove}; -use self::search::ChannelRenderer; -use self::search::PlaylistRenderer; -use self::search::VideoRenderer; - #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ContentRenderer { @@ -510,10 +506,9 @@ pub fn alerts_to_err(alerts: Option>) -> ExtractionError { .into_iter() .map(|a| a.alert_renderer.text) .collect::>() - .join(" ") - .into(), + .join(" "), ), - None => ExtractionError::ContentUnavailable("content not found".into()), + None => ExtractionError::InvalidData("no contents".into()), } } @@ -522,7 +517,7 @@ pub trait FromWLang { } pub trait TryFromWLang: Sized { - fn from_w_lang(from: T, lang: Language) -> Result; + fn from_w_lang(from: T, lang: Language) -> core::result::Result; } impl FromWLang for model::ChannelVideo { @@ -582,7 +577,7 @@ impl TryFromWLang for model::RecommendedVideo { fn from_w_lang( video: CompactVideoRenderer, lang: Language, - ) -> Result { + ) -> core::result::Result { let channel = model::ChannelId::try_from(video.channel)?; Ok(Self { @@ -613,89 +608,3 @@ impl TryFromWLang for model::RecommendedVideo { }) } } - -impl TryFromWLang for model::SearchVideo { - fn from_w_lang(video: VideoRenderer, lang: Language) -> Result { - let channel = model::ChannelId::try_from(video.channel)?; - let mut metadata_snippets = video.detailed_metadata_snippets; - - Ok(Self { - id: video.video_id, - title: video.title, - length: video - .length_text - .and_then(|txt| util::parse_video_length(&txt)), - thumbnail: video.thumbnail.into(), - channel: model::ChannelTag { - id: channel.id, - name: channel.name, - avatar: video - .channel_thumbnail_supported_renderers - .channel_thumbnail_with_link_renderer - .thumbnail - .into(), - verification: video.owner_badges.into(), - subscriber_count: None, - }, - publish_date: video - .published_time_text - .as_ref() - .and_then(|txt| timeago::parse_timeago_to_dt(lang, txt)), - publish_date_txt: video.published_time_text, - view_count: video - .view_count_text - .and_then(|txt| util::parse_numeric(&txt).ok()) - .unwrap_or_default(), - is_live: video.thumbnail_overlays.is_live(), - is_short: video.thumbnail_overlays.is_short(), - short_description: metadata_snippets - .try_swap_remove(0) - .map(|s| s.snippet_text) - .unwrap_or_default(), - }) - } -} - -impl From for model::SearchPlaylist { - fn from(playlist: PlaylistRenderer) -> Self { - let mut thumbnails = playlist.thumbnails; - - Self { - id: playlist.playlist_id, - name: playlist.title, - thumbnail: thumbnails.try_swap_remove(0).unwrap_or_default().into(), - video_count: playlist.video_count, - first_videos: playlist - .videos - .into_iter() - .map(|v| model::SearchPlaylistVideo { - id: v.child_video_renderer.video_id, - title: v.child_video_renderer.title, - length: v - .child_video_renderer - .length_text - .and_then(|txt| util::parse_video_length(&txt)), - }) - .collect(), - } - } -} - -impl From for model::SearchChannel { - fn from(channel: ChannelRenderer) -> Self { - Self { - id: channel.channel_id, - name: channel.title, - avatar: channel.thumbnail.into(), - verification: channel.owner_badges.into(), - subscriber_count: channel - .subscriber_count_text - .and_then(|txt| util::parse_numeric(&txt).ok()), - video_count: channel - .video_count_text - .and_then(|txt| util::parse_numeric(&txt).ok()) - .unwrap_or_default(), - short_description: channel.description_snippet, - } - } -} diff --git a/src/client/search.rs b/src/client/search.rs index 249c89f..c6a3500 100644 --- a/src/client/search.rs +++ b/src/client/search.rs @@ -3,13 +3,17 @@ use serde::Serialize; use crate::{ deobfuscate::Deobfuscator, error::{Error, ExtractionError}, - model::{Paginator, SearchItem, SearchResult, SearchVideo}, + model::{ + ChannelId, ChannelTag, Paginator, SearchChannel, SearchItem, SearchPlaylist, + SearchPlaylistVideo, SearchResult, SearchVideo, + }, param::{search_filter::SearchFilter, Language}, - util::TryRemove, + timeago, + util::{self, TryRemove}, }; use super::{ - response::{self, TryFromWLang}, + response::{self, IsLive, IsShort}, ClientType, MapResponse, MapResult, QContinuation, RustyPipeQuery, YTContext, }; @@ -179,20 +183,86 @@ fn map_search_items( let mapped_items = items .into_iter() .filter_map(|item| match item { - response::search::SearchItem::VideoRenderer(video) => { - match SearchVideo::from_w_lang(video, lang) { - Ok(video) => Some(SearchItem::Video(video)), + response::search::SearchItem::VideoRenderer(mut video) => { + match ChannelId::try_from(video.channel) { + Ok(channel) => Some(SearchItem::Video(SearchVideo { + id: video.video_id, + title: video.title, + length: video + .length_text + .and_then(|txt| util::parse_video_length_or_warn(&txt, &mut warnings)), + thumbnail: video.thumbnail.into(), + channel: ChannelTag { + id: channel.id, + name: channel.name, + avatar: video + .channel_thumbnail_supported_renderers + .channel_thumbnail_with_link_renderer + .thumbnail + .into(), + verification: video.owner_badges.into(), + subscriber_count: None, + }, + publish_date: video.published_time_text.as_ref().and_then(|txt| { + timeago::parse_timeago_or_warn(lang, txt, &mut warnings) + }), + publish_date_txt: video.published_time_text, + view_count: video + .view_count_text + .and_then(|txt| util::parse_numeric(&txt).ok()) + .unwrap_or_default(), + is_live: video.thumbnail_overlays.is_live(), + is_short: video.thumbnail_overlays.is_short(), + short_description: video + .detailed_metadata_snippets + .try_swap_remove(0) + .map(|s| s.snippet_text) + .unwrap_or_default(), + })), Err(e) => { warnings.push(e.to_string()); None } } } - response::search::SearchItem::PlaylistRenderer(playlist) => { - Some(SearchItem::Playlist(playlist.into())) + response::search::SearchItem::PlaylistRenderer(mut playlist) => { + Some(SearchItem::Playlist(SearchPlaylist { + id: playlist.playlist_id, + name: playlist.title, + thumbnail: playlist + .thumbnails + .try_swap_remove(0) + .unwrap_or_default() + .into(), + video_count: playlist.video_count, + first_videos: playlist + .videos + .into_iter() + .map(|v| SearchPlaylistVideo { + id: v.child_video_renderer.video_id, + title: v.child_video_renderer.title, + length: v.child_video_renderer.length_text.and_then(|txt| { + util::parse_video_length_or_warn(&txt, &mut warnings) + }), + }) + .collect(), + })) } response::search::SearchItem::ChannelRenderer(channel) => { - Some(SearchItem::Channel(channel.into())) + Some(SearchItem::Channel(SearchChannel { + id: channel.channel_id, + name: channel.title, + avatar: channel.thumbnail.into(), + verification: channel.owner_badges.into(), + subscriber_count: channel + .subscriber_count_text + .and_then(|txt| util::parse_numeric_or_warn(&txt, &mut warnings)), + video_count: channel + .video_count_text + .and_then(|txt| util::parse_numeric(&txt).ok()) + .unwrap_or_default(), + short_description: channel.description_snippet, + })) } response::search::SearchItem::ShowingResultsForRenderer { corrected_query } => { c_query = Some(corrected_query); diff --git a/src/client/video_details.rs b/src/client/video_details.rs index 5d55af2..731d1d8 100644 --- a/src/client/video_details.rs +++ b/src/client/video_details.rs @@ -516,7 +516,11 @@ fn map_comment( }), _ => None, }, - publish_date: timeago::parse_timeago_to_dt(lang, &c.published_time_text), + publish_date: timeago::parse_timeago_or_warn( + lang, + &c.published_time_text, + &mut warnings, + ), publish_date_txt: c.published_time_text, like_count: util::parse_numeric_or_warn( &c.action_buttons diff --git a/src/error.rs b/src/error.rs index 212c144..3b81c1c 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,5 +1,7 @@ use std::borrow::Cow; +pub(crate) type Result = core::result::Result; + /// Custom error type for the RustyPipe library #[derive(thiserror::Error, Debug)] #[non_exhaustive] @@ -73,8 +75,10 @@ pub enum ExtractionError { VideoUnavailable(&'static str, String), #[error("Video is age restricted")] VideoAgeRestricted, - #[error("Content is not available. Reason: {0}")] - ContentUnavailable(Cow<'static, str>), + #[error("Content is not available. Reason (from YT): {0}")] + ContentUnavailable(String), + #[error("Got no data from YouTube")] + NoData, #[error("deserialization error: {0}")] Deserialization(#[from] serde_json::Error), #[error("got invalid data from YT: {0}")] diff --git a/src/report.rs b/src/report.rs index a7ab00a..cf29592 100644 --- a/src/report.rs +++ b/src/report.rs @@ -11,7 +11,7 @@ use log::error; use serde::{Deserialize, Serialize}; use crate::deobfuscate::DeobfData; -use crate::error::Error; +use crate::error::{Error, Result}; #[derive(Debug, Clone, Serialize, Deserialize)] #[non_exhaustive] @@ -96,7 +96,7 @@ impl FileReporter { } } - fn _report(&self, report: &Report) -> Result<(), Error> { + fn _report(&self, report: &Report) -> Result<()> { let report_path = get_report_path(&self.path, report, "json")?; serde_json::to_writer_pretty(&File::create(report_path)?, &report) .map_err(|e| Error::Other(format!("could not serialize report. err: {}", e).into()))?; @@ -119,7 +119,7 @@ impl Reporter for FileReporter { } } -fn get_report_path(root: &Path, report: &Report, ext: &str) -> Result { +fn get_report_path(root: &Path, report: &Report, ext: &str) -> Result { if !root.is_dir() { std::fs::create_dir_all(root)?; } diff --git a/src/timeago.rs b/src/timeago.rs index fbd648e..c6ed972 100644 --- a/src/timeago.rs +++ b/src/timeago.rs @@ -184,6 +184,18 @@ pub fn parse_timeago_to_dt(lang: Language, textual_date: &str) -> Option, +) -> Option> { + let res = parse_timeago_to_dt(lang, textual_date); + if res.is_none() { + warnings.push(format!("could not parse timeago `{}`", textual_date)); + } + res +} + /// Parse a textual date (e.g. "29 minutes ago" or "Jul 2, 2014") into a ParsedDate object. /// /// Returns None if the date could not be parsed. diff --git a/src/util/mod.rs b/src/util/mod.rs index d87db0a..20452a1 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -15,7 +15,7 @@ use once_cell::sync::Lazy; use rand::Rng; use url::Url; -use crate::{error::Error, param::Language}; +use crate::{error::Error, error::Result, param::Language}; const CONTENT_PLAYBACK_NONCE_ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; @@ -57,7 +57,7 @@ pub fn generate_content_playback_nonce() -> String { /// Example: /// /// `example.com/api?k1=v1&k2=v2 => example.com/api; {k1: v1, k2: v2}` -pub fn url_to_params(url: &str) -> Result<(String, BTreeMap), Error> { +pub fn url_to_params(url: &str) -> Result<(String, BTreeMap)> { let mut parsed_url = Url::parse(url) .map_err(|e| Error::Other(format!("could not parse url `{}` err: {}", url, e).into()))?; let url_params: BTreeMap = parsed_url @@ -77,7 +77,7 @@ pub fn urlencode(string: &str) -> String { } /// Parse a string after removing all non-numeric characters -pub fn parse_numeric(string: &str) -> Result +pub fn parse_numeric(string: &str) -> core::result::Result where F: FromStr, { @@ -147,6 +147,14 @@ where res.ok() } +pub fn parse_video_length_or_warn(text: &str, warnings: &mut Vec) -> Option { + let res = parse_video_length(text); + if res.is_none() { + warnings.push(format!("could not parse video length `{}`", text)); + } + res +} + pub fn retry_delay( n_past_retries: u32, min_retry_interval: u32, diff --git a/tests/youtube.rs b/tests/youtube.rs index 9a92272..668da38 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -1,4 +1,4 @@ -use chrono::Datelike; +use chrono::{Datelike, Timelike}; use rstest::rstest; use rustypipe::client::{ClientType, RustyPipe}; @@ -218,14 +218,10 @@ async fn playlist_not_found() { .await .unwrap_err(); - assert!( - matches!( - err, - Error::Extraction(ExtractionError::ContentUnavailable(_)) - ), - "got: {}", - err - ); + assert!(matches!( + err, + Error::Extraction(ExtractionError::ContentUnavailable(_)) + )); } //#VIDEO DETAILS @@ -328,7 +324,6 @@ async fn get_video_details_music() { assert!(!details.is_live); assert!(!details.is_ccommons); - assert!(!details.recommended.items.is_empty()); assert!(!details.recommended.is_exhausted()); // Comments are disabled for this video @@ -386,7 +381,6 @@ async fn get_video_details_ccommons() { assert!(!details.is_live); assert!(details.is_ccommons); - assert!(!details.recommended.items.is_empty()); assert!(!details.recommended.is_exhausted()); assert!( @@ -526,7 +520,6 @@ async fn get_video_details_chapters() { "###); } - assert!(!details.recommended.items.is_empty()); assert!(!details.recommended.is_exhausted()); assert!( @@ -586,7 +579,6 @@ async fn get_video_details_live() { assert!(details.is_live); assert!(!details.is_ccommons); - assert!(!details.recommended.items.is_empty()); assert!(!details.recommended.is_exhausted()); // No comments because livestream @@ -645,14 +637,10 @@ async fn get_video_details_not_found() { let rp = RustyPipe::builder().strict().build(); let err = rp.query().video_details("abcdefgLi5X").await.unwrap_err(); - assert!( - matches!( - err, - Error::Extraction(ExtractionError::ContentUnavailable(_)) - ), - "got: {}", - err - ) + assert!(matches!( + err, + Error::Extraction(ExtractionError::ContentUnavailable(_)) + )) } #[tokio::test] @@ -849,15 +837,13 @@ fn assert_channel_eevblog(channel: &Channel) { #[rstest] #[case::artist("UC_vmjW5e1xEHhYjY2a0kK1A", "Oonagh - Topic", false, false)] #[case::shorts("UCh8gHdtzO2tXd593_bjErWg", "Doobydobap", true, true)] -#[case::livestream( +#[case::live( "UChs0pSaEoNLV4mevBFGaoKA", "The Good Life Radio x Sensual Musique", true, true )] #[case::music("UC-9-kyTW8ZkZNDHQJ6FgpwQ", "Music", false, false)] -#[case::live("UC4R8DWoMoI7CAwX8_LjQHig", "Live", false, false)] -#[case::news("UCYfdidRxbB8Qhf0Nx7ioOYw", "News", false, false)] #[tokio::test] async fn channel_more( #[case] id: &str, @@ -896,72 +882,58 @@ async fn channel_more( } #[rstest] -#[case::not_exist("UCOpNcN46UbXVtpKMrmU4Abx")] -#[case::gaming("UCOpNcN46UbXVtpKMrmU4Abg")] -#[case::movies("UCuJcl0Ju-gPDoksRjK1ya-w")] -#[case::sports("UCEgdi0XIXXZ-qJOFPf4JSKw")] -#[case::learning("UCtFRv9O2AHqOZjjynzrv-xg")] +#[case::gaming("UCOpNcN46UbXVtpKMrmU4Abg", false)] +#[case::not_found("UCOpNcN46UbXVtpKMrmU4Abx", true)] #[tokio::test] -async fn channel_not_found(#[case] id: &str) { +async fn channel_error(#[case] id: &str, #[case] not_found: bool) { let rp = RustyPipe::builder().strict().build(); let err = rp.query().channel_videos(&id).await.unwrap_err(); - assert!( - matches!( + if not_found { + assert!(matches!( err, Error::Extraction(ExtractionError::ContentUnavailable(_)) - ), - "got: {}", - err - ); + )); + } else { + assert!(matches!(err, Error::Extraction(ExtractionError::NoData))); + } } //#CHANNEL_RSS -#[cfg(feature = "rss")] -mod channel_rss { - use super::*; +#[tokio::test] +async fn get_channel_rss() { + let rp = RustyPipe::builder().strict().build(); + let channel = rp + .query() + .channel_rss("UCHnyfMqiRRG1u-2MsSQLbXA") + .await + .unwrap(); - use chrono::Timelike; + assert_eq!(channel.id, "UCHnyfMqiRRG1u-2MsSQLbXA"); + assert_eq!(channel.name, "Veritasium"); + assert_eq!(channel.create_date.year(), 2010); + assert_eq!(channel.create_date.month(), 7); + assert_eq!(channel.create_date.day(), 21); + assert_eq!(channel.create_date.hour(), 7); + assert_eq!(channel.create_date.minute(), 18); - #[tokio::test] - async fn get_channel_rss() { - let rp = RustyPipe::builder().strict().build(); - let channel = rp - .query() - .channel_rss("UCHnyfMqiRRG1u-2MsSQLbXA") - .await - .unwrap(); + assert!(!channel.videos.is_empty()); +} - assert_eq!(channel.id, "UCHnyfMqiRRG1u-2MsSQLbXA"); - assert_eq!(channel.name, "Veritasium"); - assert_eq!(channel.create_date.year(), 2010); - assert_eq!(channel.create_date.month(), 7); - assert_eq!(channel.create_date.day(), 21); - assert_eq!(channel.create_date.hour(), 7); - assert_eq!(channel.create_date.minute(), 18); +#[tokio::test] +async fn get_channel_rss_not_found() { + let rp = RustyPipe::builder().strict().build(); + let err = rp + .query() + .channel_rss("UCHnyfMqiRRG1u-2MsSQLbXZ") + .await + .unwrap_err(); - assert!(!channel.videos.is_empty()); - } - - #[tokio::test] - async fn get_channel_rss_not_found() { - let rp = RustyPipe::builder().strict().build(); - let err = rp - .query() - .channel_rss("UCHnyfMqiRRG1u-2MsSQLbXZ") - .await - .unwrap_err(); - - assert!( - matches!( - err, - Error::Extraction(ExtractionError::ContentUnavailable(_)) - ), - "got: {}", - err - ); - } + assert!(matches!( + err, + Error::Extraction(ExtractionError::ContentUnavailable(_)) + )); } //#SEARCH