From 6066813cba3d673e7780b315ad426784eeed111c Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 21 Sep 2022 17:44:23 +0200 Subject: [PATCH 1/2] chore: remove check-added-large-files lint --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a99c63..7d7459e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,6 @@ repos: rev: v4.3.0 hooks: - id: end-of-file-fixer - - id: check-added-large-files - id: check-json - repo: https://github.com/cathiele/pre-commit-rust From 86a348f210297680fa0005286c395c2e082d8f94 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 21 Sep 2022 18:16:21 +0200 Subject: [PATCH 2/2] feat: add video details pagination, tests --- src/client/mod.rs | 1 + src/client/pagination.rs | 137 ++++++++++++++++++++++++++++++++++++ src/client/playlist.rs | 44 ------------ src/client/video_details.rs | 60 +++++++++++----- 4 files changed, 182 insertions(+), 60 deletions(-) create mode 100644 src/client/pagination.rs diff --git a/src/client/mod.rs b/src/client/mod.rs index 424a8b0..b7cbf66 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1,3 +1,4 @@ +pub mod pagination; pub mod player; pub mod playlist; pub mod video_details; diff --git a/src/client/pagination.rs b/src/client/pagination.rs new file mode 100644 index 0000000..32dbcb8 --- /dev/null +++ b/src/client/pagination.rs @@ -0,0 +1,137 @@ +use anyhow::Result; + +use crate::model::{Comment, Paginator, PlaylistVideo, RecommendedVideo}; + +use super::RustyPipeQuery; + +impl Paginator { + pub async fn next(&self, query: RustyPipeQuery) -> Result> { + Ok(match &self.ctoken { + Some(ctoken) => Some(query.playlist_continuation(ctoken).await?), + None => None, + }) + } + + pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { + match self.next(query).await { + Ok(Some(paginator)) => { + let mut items = paginator.items; + self.items.append(&mut items); + self.ctoken = paginator.ctoken; + Ok(true) + } + Ok(None) => Ok(false), + Err(e) => Err(e), + } + } + + pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> { + for _ in 0..n_pages { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } + + pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> { + while self.items.len() < n_items { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } +} + +impl Paginator { + pub async fn next(&self, query: RustyPipeQuery) -> Result> { + Ok(match &self.ctoken { + Some(ctoken) => Some(query.video_recommendations(ctoken).await?), + None => None, + }) + } + + pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { + match self.next(query).await { + Ok(Some(paginator)) => { + let mut items = paginator.items; + self.items.append(&mut items); + self.ctoken = paginator.ctoken; + Ok(true) + } + Ok(None) => Ok(false), + Err(e) => Err(e), + } + } + + pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> { + for _ in 0..n_pages { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } + + pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> { + while self.items.len() < n_items { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } +} + +impl Paginator { + pub async fn next(&self, query: RustyPipeQuery) -> Result> { + Ok(match &self.ctoken { + Some(ctoken) => Some(query.video_comments(ctoken).await?), + None => None, + }) + } + + pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { + match self.next(query).await { + Ok(Some(paginator)) => { + let mut items = paginator.items; + self.items.append(&mut items); + self.ctoken = paginator.ctoken; + Ok(true) + } + Ok(None) => Ok(false), + Err(e) => Err(e), + } + } + + pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> { + for _ in 0..n_pages { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } + + pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> { + while self.items.len() < n_items { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } +} diff --git a/src/client/playlist.rs b/src/client/playlist.rs index 874ea6e..a83e130 100644 --- a/src/client/playlist.rs +++ b/src/client/playlist.rs @@ -237,50 +237,6 @@ fn map_playlist_items( (videos, ctoken) } -impl Paginator { - pub async fn next(&self, query: RustyPipeQuery) -> Result> { - Ok(match &self.ctoken { - Some(ctoken) => Some(query.playlist_continuation(ctoken).await?), - None => None, - }) - } - - pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { - match self.next(query).await { - Ok(Some(paginator)) => { - let mut items = paginator.items; - self.items.append(&mut items); - self.ctoken = paginator.ctoken; - Ok(true) - } - Ok(None) => Ok(false), - Err(e) => Err(e), - } - } - - pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> { - for _ in 0..n_pages { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } - - pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> { - while self.items.len() < n_items { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } -} - #[cfg(test)] mod tests { use std::{fs::File, io::BufReader, path::Path}; diff --git a/src/client/video_details.rs b/src/client/video_details.rs index 6f4119d..cbd1331 100644 --- a/src/client/video_details.rs +++ b/src/client/video_details.rs @@ -1162,31 +1162,59 @@ mod tests { assert!(details.recommended.items.is_empty()); } - /* - #[test_log::test(tokio::test)] + #[tokio::test] async fn get_video_recommendations() { let rp = RustyPipe::builder().strict().build(); let details = rp.query().video_details("ZeerrnuLi5E").await.unwrap(); - let rec = rp - .query() - .video_recommendations(&details.recommended.ctoken.unwrap()) - .await - .unwrap(); + let next_recommendations = details.recommended.next(rp.query()).await.unwrap().unwrap(); + dbg!(&next_recommendations); - dbg!(&rec); + assert!( + next_recommendations.items.len() > 10, + "expected > 10 next recommendations, got {}", + next_recommendations.items.len() + ); + assert!(!next_recommendations.is_exhausted()); } - #[test_log::test(tokio::test)] + #[tokio::test] async fn get_video_comments() { let rp = RustyPipe::builder().strict().build(); let details = rp.query().video_details("ZeerrnuLi5E").await.unwrap(); - let rec = rp - .query() - .video_comments(&details.top_comments.ctoken.unwrap()) - .await - .unwrap(); - dbg!(&rec); + let top_comments = details + .top_comments + .next(rp.query()) + .await + .unwrap() + .unwrap(); + assert!( + top_comments.items.len() > 10, + "expected > 10 next comments, got {}", + top_comments.items.len() + ); + assert!(!top_comments.is_exhausted()); + + let n_comments = top_comments.count.unwrap(); + assert!( + n_comments > 700000, + "expected > 700k comments, got {}", + n_comments + ); + // Comment count should be exact after fetching first page + assert!(n_comments % 1000 != 0); + + let latest_comments = details + .latest_comments + .next(rp.query()) + .await + .unwrap() + .unwrap(); + assert!( + latest_comments.items.len() > 10, + "expected > 10 next comments, got {}", + latest_comments.items.len() + ); + assert!(!latest_comments.is_exhausted()); } - */ }