Compare commits
2 commits
bbaa6cdb90
...
0bc9496865
Author | SHA1 | Date | |
---|---|---|---|
0bc9496865 | |||
77960170bb |
13 changed files with 71576 additions and 153 deletions
6
Justfile
6
Justfile
|
@ -1,3 +1,9 @@
|
||||||
|
test:
|
||||||
|
cargo test -F all
|
||||||
|
|
||||||
|
testfiles:
|
||||||
|
cargo run -p rustypipe-codegen -- -d . download-testfiles
|
||||||
|
|
||||||
report2yaml:
|
report2yaml:
|
||||||
mkdir -p rustypipe_reports/conv
|
mkdir -p rustypipe_reports/conv
|
||||||
for f in rustypipe_reports/*.json; do yq '.http_request.resp_body' $f | yq -o json -P > rustypipe_reports/conv/`basename $f .json`_body.json; yq e -Pi $f; mv $f rustypipe_reports/conv/`basename $f .json`.yaml; done;
|
for f in rustypipe_reports/*.json; do yq '.http_request.resp_body' $f | yq -o json -P > rustypipe_reports/conv/`basename $f .json`_body.json; yq e -Pi $f; mv $f rustypipe_reports/conv/`basename $f .json`.yaml; done;
|
||||||
|
|
|
@ -13,8 +13,8 @@ inspired by [NewPipe](https://github.com/TeamNewPipe/NewPipeExtractor).
|
||||||
- [X] **Channel** (videos, playlists, info)
|
- [X] **Channel** (videos, playlists, info)
|
||||||
- [X] **ChannelRSS**
|
- [X] **ChannelRSS**
|
||||||
- [X] **Search** (with filters)
|
- [X] **Search** (with filters)
|
||||||
- [ ] **Search suggestions**
|
- [X] **Search suggestions**
|
||||||
- [ ] **Trending**
|
- [X] **Trending**
|
||||||
- [ ] **URL resolver**
|
- [ ] **URL resolver**
|
||||||
|
|
||||||
### YouTube Music
|
### YouTube Music
|
||||||
|
|
|
@ -29,6 +29,8 @@ pub async fn download_testfiles(project_root: &Path) {
|
||||||
search_cont(&testfiles).await;
|
search_cont(&testfiles).await;
|
||||||
search_playlists(&testfiles).await;
|
search_playlists(&testfiles).await;
|
||||||
search_empty(&testfiles).await;
|
search_empty(&testfiles).await;
|
||||||
|
startpage(&testfiles).await;
|
||||||
|
trending(&testfiles).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
const CLIENT_TYPES: [ClientType; 5] = [
|
const CLIENT_TYPES: [ClientType; 5] = [
|
||||||
|
@ -366,3 +368,27 @@ async fn search_empty(testfiles: &Path) {
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn startpage(testfiles: &Path) {
|
||||||
|
let mut json_path = testfiles.to_path_buf();
|
||||||
|
json_path.push("trends");
|
||||||
|
json_path.push("startpage.json");
|
||||||
|
if json_path.exists() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let rp = rp_testfile(&json_path);
|
||||||
|
rp.query().startpage().await.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn trending(testfiles: &Path) {
|
||||||
|
let mut json_path = testfiles.to_path_buf();
|
||||||
|
json_path.push("trends");
|
||||||
|
json_path.push("trending.json");
|
||||||
|
if json_path.exists() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let rp = rp_testfile(&json_path);
|
||||||
|
rp.query().trending().await.unwrap();
|
||||||
|
}
|
||||||
|
|
|
@ -6,6 +6,7 @@ mod player;
|
||||||
mod playlist;
|
mod playlist;
|
||||||
mod response;
|
mod response;
|
||||||
mod search;
|
mod search;
|
||||||
|
mod trends;
|
||||||
mod video_details;
|
mod video_details;
|
||||||
|
|
||||||
#[cfg(feature = "rss")]
|
#[cfg(feature = "rss")]
|
||||||
|
@ -123,6 +124,13 @@ struct ThirdParty {
|
||||||
embed_url: String,
|
embed_url: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
struct QBrowse {
|
||||||
|
context: YTContext,
|
||||||
|
browse_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
struct QContinuation<'a> {
|
struct QContinuation<'a> {
|
||||||
|
|
|
@ -1,7 +1,5 @@
|
||||||
use std::convert::TryFrom;
|
use std::convert::TryFrom;
|
||||||
|
|
||||||
use serde::Serialize;
|
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
deobfuscate::Deobfuscator,
|
deobfuscate::Deobfuscator,
|
||||||
error::{Error, ExtractionError},
|
error::{Error, ExtractionError},
|
||||||
|
@ -11,21 +9,12 @@ use crate::{
|
||||||
util::{self, TryRemove},
|
util::{self, TryRemove},
|
||||||
};
|
};
|
||||||
|
|
||||||
use super::{
|
use super::{response, ClientType, MapResponse, MapResult, QBrowse, QContinuation, RustyPipeQuery};
|
||||||
response, ClientType, MapResponse, MapResult, QContinuation, RustyPipeQuery, YTContext,
|
|
||||||
};
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
struct QPlaylist {
|
|
||||||
context: YTContext,
|
|
||||||
browse_id: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl RustyPipeQuery {
|
impl RustyPipeQuery {
|
||||||
pub async fn playlist(self, playlist_id: &str) -> Result<Playlist, Error> {
|
pub async fn playlist(self, playlist_id: &str) -> Result<Playlist, Error> {
|
||||||
let context = self.get_context(ClientType::Desktop, true).await;
|
let context = self.get_context(ClientType::Desktop, true).await;
|
||||||
let request_body = QPlaylist {
|
let request_body = QBrowse {
|
||||||
context,
|
context,
|
||||||
browse_id: "VL".to_owned() + playlist_id,
|
browse_id: "VL".to_owned() + playlist_id,
|
||||||
};
|
};
|
||||||
|
|
|
@ -3,17 +3,19 @@ pub mod player;
|
||||||
pub mod playlist;
|
pub mod playlist;
|
||||||
pub mod playlist_music;
|
pub mod playlist_music;
|
||||||
pub mod search;
|
pub mod search;
|
||||||
|
pub mod trends;
|
||||||
pub mod video_details;
|
pub mod video_details;
|
||||||
|
|
||||||
pub use channel::Channel;
|
pub use channel::Channel;
|
||||||
pub use channel::ChannelCont;
|
pub use channel::ChannelCont;
|
||||||
use chrono::TimeZone;
|
|
||||||
pub use player::Player;
|
pub use player::Player;
|
||||||
pub use playlist::Playlist;
|
pub use playlist::Playlist;
|
||||||
pub use playlist::PlaylistCont;
|
pub use playlist::PlaylistCont;
|
||||||
pub use playlist_music::PlaylistMusic;
|
pub use playlist_music::PlaylistMusic;
|
||||||
pub use search::Search;
|
pub use search::Search;
|
||||||
pub use search::SearchCont;
|
pub use search::SearchCont;
|
||||||
|
pub use trends::Startpage;
|
||||||
|
pub use trends::Trending;
|
||||||
pub use video_details::VideoComments;
|
pub use video_details::VideoComments;
|
||||||
pub use video_details::VideoDetails;
|
pub use video_details::VideoDetails;
|
||||||
pub use video_details::VideoRecommendations;
|
pub use video_details::VideoRecommendations;
|
||||||
|
@ -23,6 +25,7 @@ pub mod channel_rss;
|
||||||
#[cfg(feature = "rss")]
|
#[cfg(feature = "rss")]
|
||||||
pub use channel_rss::ChannelRss;
|
pub use channel_rss::ChannelRss;
|
||||||
|
|
||||||
|
use chrono::TimeZone;
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use serde_with::{json::JsonString, serde_as, DefaultOnError, VecSkipError};
|
use serde_with::{json::JsonString, serde_as, DefaultOnError, VecSkipError};
|
||||||
|
|
||||||
|
@ -36,12 +39,9 @@ use crate::serializer::{
|
||||||
VecLogError,
|
VecLogError,
|
||||||
};
|
};
|
||||||
use crate::timeago;
|
use crate::timeago;
|
||||||
|
use crate::util::MappingError;
|
||||||
use crate::util::{self, TryRemove};
|
use crate::util::{self, TryRemove};
|
||||||
|
|
||||||
use self::search::ChannelRenderer;
|
|
||||||
use self::search::PlaylistRenderer;
|
|
||||||
use self::search::VideoRenderer;
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
pub struct ContentRenderer<T> {
|
pub struct ContentRenderer<T> {
|
||||||
|
@ -93,6 +93,8 @@ pub enum VideoListItem {
|
||||||
/// Playlist on channel page
|
/// Playlist on channel page
|
||||||
GridPlaylistRenderer(GridPlaylistRenderer),
|
GridPlaylistRenderer(GridPlaylistRenderer),
|
||||||
|
|
||||||
|
/// Video on startpage
|
||||||
|
///
|
||||||
/// Seems to be currently A/B tested on the channel page,
|
/// Seems to be currently A/B tested on the channel page,
|
||||||
/// as of 11.10.2022
|
/// as of 11.10.2022
|
||||||
RichItemRenderer { content: RichItem },
|
RichItemRenderer { content: RichItem },
|
||||||
|
@ -174,6 +176,67 @@ pub struct CompactVideoRenderer {
|
||||||
pub thumbnail_overlays: Vec<TimeOverlay>,
|
pub thumbnail_overlays: Vec<TimeOverlay>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Video displayed in search results
|
||||||
|
#[serde_as]
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct VideoRenderer {
|
||||||
|
pub video_id: String,
|
||||||
|
pub thumbnail: Thumbnails,
|
||||||
|
#[serde_as(as = "Text")]
|
||||||
|
pub title: String,
|
||||||
|
#[serde(rename = "shortBylineText")]
|
||||||
|
pub channel: Option<TextComponent>,
|
||||||
|
pub channel_thumbnail_supported_renderers: Option<ChannelThumbnailSupportedRenderers>,
|
||||||
|
#[serde_as(as = "Option<Text>")]
|
||||||
|
pub published_time_text: Option<String>,
|
||||||
|
#[serde_as(as = "Option<Text>")]
|
||||||
|
pub length_text: Option<String>,
|
||||||
|
/// Contains `No views` if the view count is zero
|
||||||
|
#[serde_as(as = "Option<Text>")]
|
||||||
|
pub view_count_text: Option<String>,
|
||||||
|
/// Channel verification badge
|
||||||
|
#[serde(default)]
|
||||||
|
#[serde_as(as = "VecSkipError<_>")]
|
||||||
|
pub owner_badges: Vec<ChannelBadge>,
|
||||||
|
/// Contains Short/Live tag
|
||||||
|
#[serde_as(as = "VecSkipError<_>")]
|
||||||
|
pub thumbnail_overlays: Vec<TimeOverlay>,
|
||||||
|
/// Abbreviated video description (on startpage)
|
||||||
|
#[serde_as(as = "Option<Text>")]
|
||||||
|
pub description_snippet: Option<String>,
|
||||||
|
/// Contains abbreviated video description (on search page)
|
||||||
|
#[serde_as(as = "Option<VecSkipError<_>>")]
|
||||||
|
pub detailed_metadata_snippets: Option<Vec<DetailedMetadataSnippet>>,
|
||||||
|
/// Release date for upcoming videos
|
||||||
|
pub upcoming_event_data: Option<UpcomingEventData>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Playlist displayed in search results
|
||||||
|
#[serde_as]
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct PlaylistRenderer {
|
||||||
|
pub playlist_id: String,
|
||||||
|
#[serde_as(as = "Text")]
|
||||||
|
pub title: String,
|
||||||
|
/// The first item of this list contains the playlist thumbnail,
|
||||||
|
/// subsequent items contain very small thumbnails of the next playlist videos
|
||||||
|
pub thumbnails: Vec<Thumbnails>,
|
||||||
|
#[serde_as(as = "JsonString")]
|
||||||
|
pub video_count: u64,
|
||||||
|
#[serde(rename = "shortBylineText")]
|
||||||
|
pub channel: TextComponent,
|
||||||
|
/// Channel verification badge
|
||||||
|
#[serde(default)]
|
||||||
|
#[serde_as(as = "VecSkipError<_>")]
|
||||||
|
pub owner_badges: Vec<ChannelBadge>,
|
||||||
|
/// First 2 videos
|
||||||
|
#[serde(default)]
|
||||||
|
#[serde_as(as = "VecSkipError<_>")]
|
||||||
|
pub videos: Vec<ChildVideoRendererWrap>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Video displayed in a playlist
|
/// Video displayed in a playlist
|
||||||
#[serde_as]
|
#[serde_as]
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
|
@ -189,6 +252,32 @@ pub struct PlaylistVideoRenderer {
|
||||||
pub length_seconds: u32,
|
pub length_seconds: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Channel displayed in search results
|
||||||
|
#[serde_as]
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct ChannelRenderer {
|
||||||
|
pub channel_id: String,
|
||||||
|
#[serde_as(as = "Text")]
|
||||||
|
pub title: String,
|
||||||
|
pub thumbnail: Thumbnails,
|
||||||
|
/// Abbreviated channel description
|
||||||
|
///
|
||||||
|
/// Not present if the channel has no description
|
||||||
|
#[serde(default)]
|
||||||
|
#[serde_as(as = "Text")]
|
||||||
|
pub description_snippet: String,
|
||||||
|
/// Not present if the channel has no videos
|
||||||
|
#[serde_as(as = "Option<Text>")]
|
||||||
|
pub video_count_text: Option<String>,
|
||||||
|
#[serde_as(as = "Option<Text>")]
|
||||||
|
pub subscriber_count_text: Option<String>,
|
||||||
|
/// Channel verification badge
|
||||||
|
#[serde(default)]
|
||||||
|
#[serde_as(as = "VecSkipError<_>")]
|
||||||
|
pub owner_badges: Vec<ChannelBadge>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Playlist displayed on a channel page
|
/// Playlist displayed on a channel page
|
||||||
#[serde_as]
|
#[serde_as]
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
|
@ -204,8 +293,9 @@ pub struct GridPlaylistRenderer {
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
|
#[allow(clippy::large_enum_variant)]
|
||||||
pub enum RichItem {
|
pub enum RichItem {
|
||||||
VideoRenderer(GridVideoRenderer),
|
VideoRenderer(VideoRenderer),
|
||||||
PlaylistRenderer(GridPlaylistRenderer),
|
PlaylistRenderer(GridPlaylistRenderer),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -358,6 +448,43 @@ pub struct AlertRenderer {
|
||||||
pub text: String,
|
pub text: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct ChannelThumbnailSupportedRenderers {
|
||||||
|
pub channel_thumbnail_with_link_renderer: ChannelThumbnailWithLinkRenderer,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct ChannelThumbnailWithLinkRenderer {
|
||||||
|
pub thumbnail: Thumbnails,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[serde_as]
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct DetailedMetadataSnippet {
|
||||||
|
#[serde_as(as = "Text")]
|
||||||
|
pub snippet_text: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct ChildVideoRendererWrap {
|
||||||
|
pub child_video_renderer: ChildVideoRenderer,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[serde_as]
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct ChildVideoRenderer {
|
||||||
|
pub video_id: String,
|
||||||
|
#[serde_as(as = "Text")]
|
||||||
|
pub title: String,
|
||||||
|
#[serde_as(as = "Option<Text>")]
|
||||||
|
pub length_text: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
// YouTube Music
|
// YouTube Music
|
||||||
|
|
||||||
#[serde_as]
|
#[serde_as]
|
||||||
|
@ -567,6 +694,48 @@ impl FromWLang<GridVideoRenderer> for model::ChannelVideo {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl FromWLang<VideoRenderer> for model::ChannelVideo {
|
||||||
|
fn from_w_lang(video: VideoRenderer, lang: Language) -> Self {
|
||||||
|
let mut toverlays = video.thumbnail_overlays;
|
||||||
|
let is_live = toverlays.is_live();
|
||||||
|
let is_short = toverlays.is_short();
|
||||||
|
let to = toverlays.try_swap_remove(0);
|
||||||
|
|
||||||
|
Self {
|
||||||
|
id: video.video_id,
|
||||||
|
title: video.title,
|
||||||
|
// Time text is `LIVE` for livestreams, so we ignore parse errors
|
||||||
|
length: to.and_then(|to| {
|
||||||
|
util::parse_video_length(&to.thumbnail_overlay_time_status_renderer.text)
|
||||||
|
}),
|
||||||
|
thumbnail: video.thumbnail.into(),
|
||||||
|
publish_date: video
|
||||||
|
.upcoming_event_data
|
||||||
|
.as_ref()
|
||||||
|
.map(|upc| {
|
||||||
|
chrono::Local.from_utc_datetime(&chrono::NaiveDateTime::from_timestamp(
|
||||||
|
upc.start_time,
|
||||||
|
0,
|
||||||
|
))
|
||||||
|
})
|
||||||
|
.or_else(|| {
|
||||||
|
video
|
||||||
|
.published_time_text
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|txt| timeago::parse_timeago_to_dt(lang, txt))
|
||||||
|
}),
|
||||||
|
publish_date_txt: video.published_time_text,
|
||||||
|
view_count: video
|
||||||
|
.view_count_text
|
||||||
|
.and_then(|txt| util::parse_numeric(&txt).ok())
|
||||||
|
.unwrap_or_default(),
|
||||||
|
is_live,
|
||||||
|
is_short,
|
||||||
|
is_upcoming: video.upcoming_event_data.is_some(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl From<GridPlaylistRenderer> for model::ChannelPlaylist {
|
impl From<GridPlaylistRenderer> for model::ChannelPlaylist {
|
||||||
fn from(playlist: GridPlaylistRenderer) -> Self {
|
fn from(playlist: GridPlaylistRenderer) -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
@ -616,8 +785,14 @@ impl TryFromWLang<CompactVideoRenderer> for model::RecommendedVideo {
|
||||||
|
|
||||||
impl TryFromWLang<VideoRenderer> for model::SearchVideo {
|
impl TryFromWLang<VideoRenderer> for model::SearchVideo {
|
||||||
fn from_w_lang(video: VideoRenderer, lang: Language) -> Result<Self, util::MappingError> {
|
fn from_w_lang(video: VideoRenderer, lang: Language) -> Result<Self, util::MappingError> {
|
||||||
let channel = model::ChannelId::try_from(video.channel)?;
|
let channel = model::ChannelId::try_from(
|
||||||
let mut metadata_snippets = video.detailed_metadata_snippets;
|
video
|
||||||
|
.channel
|
||||||
|
.ok_or_else(|| MappingError("no video channel".into()))?,
|
||||||
|
)?;
|
||||||
|
let channel_thumbnail = video
|
||||||
|
.channel_thumbnail_supported_renderers
|
||||||
|
.ok_or_else(|| MappingError("no video channel thumbnail".into()))?;
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
id: video.video_id,
|
id: video.video_id,
|
||||||
|
@ -629,8 +804,7 @@ impl TryFromWLang<VideoRenderer> for model::SearchVideo {
|
||||||
channel: model::ChannelTag {
|
channel: model::ChannelTag {
|
||||||
id: channel.id,
|
id: channel.id,
|
||||||
name: channel.name,
|
name: channel.name,
|
||||||
avatar: video
|
avatar: channel_thumbnail
|
||||||
.channel_thumbnail_supported_renderers
|
|
||||||
.channel_thumbnail_with_link_renderer
|
.channel_thumbnail_with_link_renderer
|
||||||
.thumbnail
|
.thumbnail
|
||||||
.into(),
|
.into(),
|
||||||
|
@ -648,9 +822,10 @@ impl TryFromWLang<VideoRenderer> for model::SearchVideo {
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
is_live: video.thumbnail_overlays.is_live(),
|
is_live: video.thumbnail_overlays.is_live(),
|
||||||
is_short: video.thumbnail_overlays.is_short(),
|
is_short: video.thumbnail_overlays.is_short(),
|
||||||
short_description: metadata_snippets
|
short_description: video
|
||||||
.try_swap_remove(0)
|
.detailed_metadata_snippets
|
||||||
.map(|s| s.snippet_text)
|
.and_then(|mut snippets| snippets.try_swap_remove(0).map(|s| s.snippet_text))
|
||||||
|
.or(video.description_snippet)
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,12 +3,11 @@ use serde_with::json::JsonString;
|
||||||
use serde_with::{serde_as, VecSkipError};
|
use serde_with::{serde_as, VecSkipError};
|
||||||
|
|
||||||
use crate::serializer::ignore_any;
|
use crate::serializer::ignore_any;
|
||||||
use crate::serializer::{
|
use crate::serializer::{text::Text, MapResult, VecLogError};
|
||||||
text::{Text, TextComponent},
|
|
||||||
MapResult, VecLogError,
|
|
||||||
};
|
|
||||||
|
|
||||||
use super::{ChannelBadge, ContentsRenderer, ContinuationEndpoint, Thumbnails, TimeOverlay};
|
use super::{
|
||||||
|
ChannelRenderer, ContentsRenderer, ContinuationEndpoint, PlaylistRenderer, VideoRenderer,
|
||||||
|
};
|
||||||
|
|
||||||
#[serde_as]
|
#[serde_as]
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
|
@ -99,122 +98,3 @@ pub enum SearchItem {
|
||||||
#[serde(other, deserialize_with = "ignore_any")]
|
#[serde(other, deserialize_with = "ignore_any")]
|
||||||
None,
|
None,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Video displayed in search results
|
|
||||||
#[serde_as]
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct VideoRenderer {
|
|
||||||
pub video_id: String,
|
|
||||||
pub thumbnail: Thumbnails,
|
|
||||||
#[serde_as(as = "Text")]
|
|
||||||
pub title: String,
|
|
||||||
#[serde(rename = "shortBylineText")]
|
|
||||||
pub channel: TextComponent,
|
|
||||||
pub channel_thumbnail_supported_renderers: ChannelThumbnailSupportedRenderers,
|
|
||||||
#[serde_as(as = "Option<Text>")]
|
|
||||||
pub published_time_text: Option<String>,
|
|
||||||
#[serde_as(as = "Option<Text>")]
|
|
||||||
pub length_text: Option<String>,
|
|
||||||
/// Contains `No views` if the view count is zero
|
|
||||||
#[serde_as(as = "Option<Text>")]
|
|
||||||
pub view_count_text: Option<String>,
|
|
||||||
/// Channel verification badge
|
|
||||||
#[serde(default)]
|
|
||||||
#[serde_as(as = "VecSkipError<_>")]
|
|
||||||
pub owner_badges: Vec<ChannelBadge>,
|
|
||||||
/// Contains Short/Live tag
|
|
||||||
#[serde_as(as = "VecSkipError<_>")]
|
|
||||||
pub thumbnail_overlays: Vec<TimeOverlay>,
|
|
||||||
#[serde(default)]
|
|
||||||
#[serde_as(as = "VecSkipError<_>")]
|
|
||||||
pub detailed_metadata_snippets: Vec<DetailedMetadataSnippet>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Playlist displayed in search results
|
|
||||||
#[serde_as]
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct PlaylistRenderer {
|
|
||||||
pub playlist_id: String,
|
|
||||||
#[serde_as(as = "Text")]
|
|
||||||
pub title: String,
|
|
||||||
/// The first item of this list contains the playlist thumbnail,
|
|
||||||
/// subsequent items contain very small thumbnails of the next playlist videos
|
|
||||||
pub thumbnails: Vec<Thumbnails>,
|
|
||||||
#[serde_as(as = "JsonString")]
|
|
||||||
pub video_count: u64,
|
|
||||||
#[serde(rename = "shortBylineText")]
|
|
||||||
pub channel: TextComponent,
|
|
||||||
/// Channel verification badge
|
|
||||||
#[serde(default)]
|
|
||||||
#[serde_as(as = "VecSkipError<_>")]
|
|
||||||
pub owner_badges: Vec<ChannelBadge>,
|
|
||||||
/// First 2 videos
|
|
||||||
#[serde(default)]
|
|
||||||
#[serde_as(as = "VecSkipError<_>")]
|
|
||||||
pub videos: Vec<ChildVideoRendererWrap>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Channel displayed in search results
|
|
||||||
#[serde_as]
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct ChannelRenderer {
|
|
||||||
pub channel_id: String,
|
|
||||||
#[serde_as(as = "Text")]
|
|
||||||
pub title: String,
|
|
||||||
pub thumbnail: Thumbnails,
|
|
||||||
/// Abbreviated channel description
|
|
||||||
///
|
|
||||||
/// Not present if the channel has no description
|
|
||||||
#[serde(default)]
|
|
||||||
#[serde_as(as = "Text")]
|
|
||||||
pub description_snippet: String,
|
|
||||||
/// Not present if the channel has no videos
|
|
||||||
#[serde_as(as = "Option<Text>")]
|
|
||||||
pub video_count_text: Option<String>,
|
|
||||||
#[serde_as(as = "Option<Text>")]
|
|
||||||
pub subscriber_count_text: Option<String>,
|
|
||||||
/// Channel verification badge
|
|
||||||
#[serde(default)]
|
|
||||||
#[serde_as(as = "VecSkipError<_>")]
|
|
||||||
pub owner_badges: Vec<ChannelBadge>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct ChannelThumbnailSupportedRenderers {
|
|
||||||
pub channel_thumbnail_with_link_renderer: ChannelThumbnailWithLinkRenderer,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct ChannelThumbnailWithLinkRenderer {
|
|
||||||
pub thumbnail: Thumbnails,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct ChildVideoRendererWrap {
|
|
||||||
pub child_video_renderer: ChildVideoRenderer,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[serde_as]
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct ChildVideoRenderer {
|
|
||||||
pub video_id: String,
|
|
||||||
#[serde_as(as = "Text")]
|
|
||||||
pub title: String,
|
|
||||||
#[serde_as(as = "Option<Text>")]
|
|
||||||
pub length_text: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[serde_as]
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct DetailedMetadataSnippet {
|
|
||||||
#[serde_as(as = "Text")]
|
|
||||||
pub snippet_text: String,
|
|
||||||
}
|
|
||||||
|
|
103
src/client/response/trends.rs
Normal file
103
src/client/response/trends.rs
Normal file
|
@ -0,0 +1,103 @@
|
||||||
|
use serde::Deserialize;
|
||||||
|
use serde_with::{serde_as, VecSkipError};
|
||||||
|
|
||||||
|
use crate::serializer::{ignore_any, MapResult, VecLogError};
|
||||||
|
|
||||||
|
use super::{ContentRenderer, ContentsRenderer, VideoListItem, VideoRenderer};
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct Startpage {
|
||||||
|
pub contents: Contents<BrowseResultsStartpage>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct Contents<T> {
|
||||||
|
pub two_column_browse_results_renderer: T,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[serde_as]
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct BrowseResultsStartpage {
|
||||||
|
#[serde_as(as = "VecSkipError<_>")]
|
||||||
|
pub tabs: Vec<Tab<StartpageTabContent>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[serde_as]
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct BrowseResultsTrends {
|
||||||
|
#[serde_as(as = "VecSkipError<_>")]
|
||||||
|
pub tabs: Vec<Tab<TrendingTabContent>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct Tab<T> {
|
||||||
|
pub tab_renderer: ContentRenderer<T>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct StartpageTabContent {
|
||||||
|
pub rich_grid_renderer: RichGridRenderer,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[serde_as]
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct RichGridRenderer {
|
||||||
|
#[serde_as(as = "VecLogError<_>")]
|
||||||
|
pub contents: MapResult<Vec<VideoListItem>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct Trending {
|
||||||
|
pub contents: Contents<BrowseResultsTrends>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct TrendingTabContent {
|
||||||
|
pub section_list_renderer: ContentsRenderer<ItemSectionRenderer>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct ItemSectionRenderer {
|
||||||
|
pub item_section_renderer: ContentsRenderer<ShelfRenderer>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct ShelfRenderer {
|
||||||
|
pub shelf_renderer: ContentRenderer<ShelfContents>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct ShelfContents {
|
||||||
|
pub expanded_shelf_contents_renderer: Option<ShelfContentsRenderer>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[serde_as]
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct ShelfContentsRenderer {
|
||||||
|
#[serde_as(as = "VecLogError<_>")]
|
||||||
|
pub items: MapResult<Vec<TrendingListItem>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[serde_as]
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
#[allow(clippy::large_enum_variant)]
|
||||||
|
pub enum TrendingListItem {
|
||||||
|
VideoRenderer(VideoRenderer),
|
||||||
|
|
||||||
|
#[serde(other, deserialize_with = "ignore_any")]
|
||||||
|
None,
|
||||||
|
}
|
|
@ -1,4 +1,4 @@
|
||||||
use serde::Serialize;
|
use serde::{de::IgnoredAny, Serialize};
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
deobfuscate::Deobfuscator,
|
deobfuscate::Deobfuscator,
|
||||||
|
@ -79,6 +79,32 @@ impl RustyPipeQuery {
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn search_suggestion(self, query: &str) -> Result<Vec<String>, Error> {
|
||||||
|
let url = url::Url::parse_with_params("https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&gs_rn=64&gs_ri=youtube&ds=yt&cp=1&gs_id=4&xhr=t&xssi=t",
|
||||||
|
&[("hl", self.opts.lang.to_string()), ("gl", self.opts.country.to_string()), ("q", query.to_string())]
|
||||||
|
).map_err(|_| Error::Other("could not build url".into()))?;
|
||||||
|
|
||||||
|
let response = self
|
||||||
|
.client
|
||||||
|
.http_request_txt(self.client.inner.http.get(url).build()?)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let trimmed = response.get(5..).ok_or_else(|| {
|
||||||
|
Error::Extraction(ExtractionError::InvalidData(
|
||||||
|
"could not get string slice".into(),
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let parsed = serde_json::from_str::<(
|
||||||
|
IgnoredAny,
|
||||||
|
Vec<(String, IgnoredAny, IgnoredAny)>,
|
||||||
|
IgnoredAny,
|
||||||
|
)>(trimmed)
|
||||||
|
.map_err(|e| Error::Extraction(ExtractionError::InvalidData(e.to_string().into())))?;
|
||||||
|
|
||||||
|
Ok(parsed.1.into_iter().map(|item| item.0).collect())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MapResponse<SearchResult> for response::Search {
|
impl MapResponse<SearchResult> for response::Search {
|
||||||
|
|
148
src/client/trends.rs
Normal file
148
src/client/trends.rs
Normal file
|
@ -0,0 +1,148 @@
|
||||||
|
use crate::{
|
||||||
|
error::{Error, ExtractionError},
|
||||||
|
model::{Paginator, SearchVideo},
|
||||||
|
serializer::MapResult,
|
||||||
|
util::TryRemove,
|
||||||
|
};
|
||||||
|
|
||||||
|
use super::{
|
||||||
|
response::{self, TryFromWLang},
|
||||||
|
ClientType, MapResponse, QBrowse, RustyPipeQuery,
|
||||||
|
};
|
||||||
|
|
||||||
|
impl RustyPipeQuery {
|
||||||
|
pub async fn startpage(self) -> Result<Paginator<SearchVideo>, Error> {
|
||||||
|
let context = self.get_context(ClientType::Desktop, true).await;
|
||||||
|
let request_body = QBrowse {
|
||||||
|
context,
|
||||||
|
browse_id: "FEwhat_to_watch".to_owned(),
|
||||||
|
};
|
||||||
|
|
||||||
|
self.execute_request::<response::Startpage, _, _>(
|
||||||
|
ClientType::Desktop,
|
||||||
|
"startpage",
|
||||||
|
"",
|
||||||
|
"browse",
|
||||||
|
&request_body,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn trending(self) -> Result<Vec<SearchVideo>, Error> {
|
||||||
|
let context = self.get_context(ClientType::Desktop, true).await;
|
||||||
|
let request_body = QBrowse {
|
||||||
|
context,
|
||||||
|
browse_id: "FEtrending".to_owned(),
|
||||||
|
};
|
||||||
|
|
||||||
|
self.execute_request::<response::Trending, _, _>(
|
||||||
|
ClientType::Desktop,
|
||||||
|
"trends",
|
||||||
|
"",
|
||||||
|
"browse",
|
||||||
|
&request_body,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MapResponse<Paginator<SearchVideo>> for response::Startpage {
|
||||||
|
fn map_response(
|
||||||
|
self,
|
||||||
|
_id: &str,
|
||||||
|
lang: crate::param::Language,
|
||||||
|
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
||||||
|
) -> Result<MapResult<Paginator<SearchVideo>>, ExtractionError> {
|
||||||
|
let mut contents = self.contents.two_column_browse_results_renderer.tabs;
|
||||||
|
let grid = contents
|
||||||
|
.try_swap_remove(0)
|
||||||
|
.ok_or_else(|| ExtractionError::InvalidData("no contents".into()))?
|
||||||
|
.tab_renderer
|
||||||
|
.content
|
||||||
|
.rich_grid_renderer
|
||||||
|
.contents;
|
||||||
|
|
||||||
|
let mut warnings = grid.warnings;
|
||||||
|
let mut ctoken = None;
|
||||||
|
let items = grid
|
||||||
|
.c
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|item| match item {
|
||||||
|
response::VideoListItem::RichItemRenderer {
|
||||||
|
content: response::RichItem::VideoRenderer(video),
|
||||||
|
} => match SearchVideo::from_w_lang(video, lang) {
|
||||||
|
Ok(video) => Some(video),
|
||||||
|
Err(e) => {
|
||||||
|
warnings.push(e.to_string());
|
||||||
|
None
|
||||||
|
}
|
||||||
|
},
|
||||||
|
response::VideoListItem::ContinuationItemRenderer {
|
||||||
|
continuation_endpoint,
|
||||||
|
} => {
|
||||||
|
ctoken = Some(continuation_endpoint.continuation_command.token);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Ok(MapResult {
|
||||||
|
c: Paginator::new(None, items, ctoken),
|
||||||
|
warnings,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MapResponse<Vec<SearchVideo>> for response::Trending {
|
||||||
|
fn map_response(
|
||||||
|
self,
|
||||||
|
_id: &str,
|
||||||
|
lang: crate::param::Language,
|
||||||
|
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
||||||
|
) -> Result<MapResult<Vec<SearchVideo>>, ExtractionError> {
|
||||||
|
let mut contents = self.contents.two_column_browse_results_renderer.tabs;
|
||||||
|
let sections = contents
|
||||||
|
.try_swap_remove(0)
|
||||||
|
.ok_or_else(|| ExtractionError::InvalidData("no contents".into()))?
|
||||||
|
.tab_renderer
|
||||||
|
.content
|
||||||
|
.section_list_renderer
|
||||||
|
.contents;
|
||||||
|
|
||||||
|
let mut items = Vec::new();
|
||||||
|
let mut warnings = Vec::new();
|
||||||
|
|
||||||
|
for mut section in sections {
|
||||||
|
let shelf = section
|
||||||
|
.item_section_renderer
|
||||||
|
.contents
|
||||||
|
.try_swap_remove(0)
|
||||||
|
.and_then(|shelf| {
|
||||||
|
shelf
|
||||||
|
.shelf_renderer
|
||||||
|
.content
|
||||||
|
.expanded_shelf_contents_renderer
|
||||||
|
});
|
||||||
|
|
||||||
|
if let Some(mut shelf) = shelf {
|
||||||
|
warnings.append(&mut shelf.items.warnings);
|
||||||
|
|
||||||
|
for item in shelf.items.c {
|
||||||
|
if let response::trends::TrendingListItem::VideoRenderer(video) = item {
|
||||||
|
match SearchVideo::from_w_lang(video, lang) {
|
||||||
|
Ok(video) => {
|
||||||
|
items.push(video);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
warnings.push(e.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(MapResult { c: items, warnings })
|
||||||
|
}
|
||||||
|
}
|
21776
testfiles/trends/startpage.json
Normal file
21776
testfiles/trends/startpage.json
Normal file
File diff suppressed because it is too large
Load diff
49239
testfiles/trends/trending.json
Normal file
49239
testfiles/trends/trending.json
Normal file
File diff suppressed because it is too large
Load diff
|
@ -1185,6 +1185,53 @@ async fn search_empty() {
|
||||||
assert!(result.items.is_empty());
|
assert!(result.items.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn search_suggestion() {
|
||||||
|
let rp = RustyPipe::builder().strict().build();
|
||||||
|
let result = rp.query().search_suggestion("hunger ga").await.unwrap();
|
||||||
|
|
||||||
|
assert!(result.contains(&"hunger games".to_owned()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn search_suggestion_empty() {
|
||||||
|
let rp = RustyPipe::builder().strict().build();
|
||||||
|
let result = rp
|
||||||
|
.query()
|
||||||
|
.search_suggestion("fjew327%4ifjelwfvnewg49")
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(result.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
//#TRENDS
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn startpage() {
|
||||||
|
let rp = RustyPipe::builder().strict().build();
|
||||||
|
let result = rp.query().startpage().await.unwrap();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
result.items.len() > 20,
|
||||||
|
"expected > 20 items, got {}",
|
||||||
|
result.items.len()
|
||||||
|
);
|
||||||
|
assert!(!result.is_exhausted());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn trending() {
|
||||||
|
let rp = RustyPipe::builder().strict().build();
|
||||||
|
let result = rp.query().trending().await.unwrap();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
result.len() > 50,
|
||||||
|
"expected > 50 items, got {}",
|
||||||
|
result.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
//#TESTUTIL
|
//#TESTUTIL
|
||||||
|
|
||||||
/// Assert equality within 10% margin
|
/// Assert equality within 10% margin
|
||||||
|
|
Loading…
Reference in a new issue