Compare commits
No commits in common. "0bc949686515b7e3bd9eb28d978927d3c6fdb057" and "bbaa6cdb902528200708b950a8858cb350ec3796" have entirely different histories.
0bc9496865
...
bbaa6cdb90
13 changed files with 153 additions and 71576 deletions
6
Justfile
6
Justfile
|
@ -1,9 +1,3 @@
|
||||||
test:
|
|
||||||
cargo test -F all
|
|
||||||
|
|
||||||
testfiles:
|
|
||||||
cargo run -p rustypipe-codegen -- -d . download-testfiles
|
|
||||||
|
|
||||||
report2yaml:
|
report2yaml:
|
||||||
mkdir -p rustypipe_reports/conv
|
mkdir -p rustypipe_reports/conv
|
||||||
for f in rustypipe_reports/*.json; do yq '.http_request.resp_body' $f | yq -o json -P > rustypipe_reports/conv/`basename $f .json`_body.json; yq e -Pi $f; mv $f rustypipe_reports/conv/`basename $f .json`.yaml; done;
|
for f in rustypipe_reports/*.json; do yq '.http_request.resp_body' $f | yq -o json -P > rustypipe_reports/conv/`basename $f .json`_body.json; yq e -Pi $f; mv $f rustypipe_reports/conv/`basename $f .json`.yaml; done;
|
||||||
|
|
|
@ -13,8 +13,8 @@ inspired by [NewPipe](https://github.com/TeamNewPipe/NewPipeExtractor).
|
||||||
- [X] **Channel** (videos, playlists, info)
|
- [X] **Channel** (videos, playlists, info)
|
||||||
- [X] **ChannelRSS**
|
- [X] **ChannelRSS**
|
||||||
- [X] **Search** (with filters)
|
- [X] **Search** (with filters)
|
||||||
- [X] **Search suggestions**
|
- [ ] **Search suggestions**
|
||||||
- [X] **Trending**
|
- [ ] **Trending**
|
||||||
- [ ] **URL resolver**
|
- [ ] **URL resolver**
|
||||||
|
|
||||||
### YouTube Music
|
### YouTube Music
|
||||||
|
|
|
@ -29,8 +29,6 @@ pub async fn download_testfiles(project_root: &Path) {
|
||||||
search_cont(&testfiles).await;
|
search_cont(&testfiles).await;
|
||||||
search_playlists(&testfiles).await;
|
search_playlists(&testfiles).await;
|
||||||
search_empty(&testfiles).await;
|
search_empty(&testfiles).await;
|
||||||
startpage(&testfiles).await;
|
|
||||||
trending(&testfiles).await;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const CLIENT_TYPES: [ClientType; 5] = [
|
const CLIENT_TYPES: [ClientType; 5] = [
|
||||||
|
@ -368,27 +366,3 @@ async fn search_empty(testfiles: &Path) {
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn startpage(testfiles: &Path) {
|
|
||||||
let mut json_path = testfiles.to_path_buf();
|
|
||||||
json_path.push("trends");
|
|
||||||
json_path.push("startpage.json");
|
|
||||||
if json_path.exists() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
let rp = rp_testfile(&json_path);
|
|
||||||
rp.query().startpage().await.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn trending(testfiles: &Path) {
|
|
||||||
let mut json_path = testfiles.to_path_buf();
|
|
||||||
json_path.push("trends");
|
|
||||||
json_path.push("trending.json");
|
|
||||||
if json_path.exists() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
let rp = rp_testfile(&json_path);
|
|
||||||
rp.query().trending().await.unwrap();
|
|
||||||
}
|
|
||||||
|
|
|
@ -6,7 +6,6 @@ mod player;
|
||||||
mod playlist;
|
mod playlist;
|
||||||
mod response;
|
mod response;
|
||||||
mod search;
|
mod search;
|
||||||
mod trends;
|
|
||||||
mod video_details;
|
mod video_details;
|
||||||
|
|
||||||
#[cfg(feature = "rss")]
|
#[cfg(feature = "rss")]
|
||||||
|
@ -124,13 +123,6 @@ struct ThirdParty {
|
||||||
embed_url: String,
|
embed_url: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
struct QBrowse {
|
|
||||||
context: YTContext,
|
|
||||||
browse_id: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
struct QContinuation<'a> {
|
struct QContinuation<'a> {
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
use std::convert::TryFrom;
|
use std::convert::TryFrom;
|
||||||
|
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
deobfuscate::Deobfuscator,
|
deobfuscate::Deobfuscator,
|
||||||
error::{Error, ExtractionError},
|
error::{Error, ExtractionError},
|
||||||
|
@ -9,12 +11,21 @@ use crate::{
|
||||||
util::{self, TryRemove},
|
util::{self, TryRemove},
|
||||||
};
|
};
|
||||||
|
|
||||||
use super::{response, ClientType, MapResponse, MapResult, QBrowse, QContinuation, RustyPipeQuery};
|
use super::{
|
||||||
|
response, ClientType, MapResponse, MapResult, QContinuation, RustyPipeQuery, YTContext,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
struct QPlaylist {
|
||||||
|
context: YTContext,
|
||||||
|
browse_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
impl RustyPipeQuery {
|
impl RustyPipeQuery {
|
||||||
pub async fn playlist(self, playlist_id: &str) -> Result<Playlist, Error> {
|
pub async fn playlist(self, playlist_id: &str) -> Result<Playlist, Error> {
|
||||||
let context = self.get_context(ClientType::Desktop, true).await;
|
let context = self.get_context(ClientType::Desktop, true).await;
|
||||||
let request_body = QBrowse {
|
let request_body = QPlaylist {
|
||||||
context,
|
context,
|
||||||
browse_id: "VL".to_owned() + playlist_id,
|
browse_id: "VL".to_owned() + playlist_id,
|
||||||
};
|
};
|
||||||
|
|
|
@ -3,19 +3,17 @@ pub mod player;
|
||||||
pub mod playlist;
|
pub mod playlist;
|
||||||
pub mod playlist_music;
|
pub mod playlist_music;
|
||||||
pub mod search;
|
pub mod search;
|
||||||
pub mod trends;
|
|
||||||
pub mod video_details;
|
pub mod video_details;
|
||||||
|
|
||||||
pub use channel::Channel;
|
pub use channel::Channel;
|
||||||
pub use channel::ChannelCont;
|
pub use channel::ChannelCont;
|
||||||
|
use chrono::TimeZone;
|
||||||
pub use player::Player;
|
pub use player::Player;
|
||||||
pub use playlist::Playlist;
|
pub use playlist::Playlist;
|
||||||
pub use playlist::PlaylistCont;
|
pub use playlist::PlaylistCont;
|
||||||
pub use playlist_music::PlaylistMusic;
|
pub use playlist_music::PlaylistMusic;
|
||||||
pub use search::Search;
|
pub use search::Search;
|
||||||
pub use search::SearchCont;
|
pub use search::SearchCont;
|
||||||
pub use trends::Startpage;
|
|
||||||
pub use trends::Trending;
|
|
||||||
pub use video_details::VideoComments;
|
pub use video_details::VideoComments;
|
||||||
pub use video_details::VideoDetails;
|
pub use video_details::VideoDetails;
|
||||||
pub use video_details::VideoRecommendations;
|
pub use video_details::VideoRecommendations;
|
||||||
|
@ -25,7 +23,6 @@ pub mod channel_rss;
|
||||||
#[cfg(feature = "rss")]
|
#[cfg(feature = "rss")]
|
||||||
pub use channel_rss::ChannelRss;
|
pub use channel_rss::ChannelRss;
|
||||||
|
|
||||||
use chrono::TimeZone;
|
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use serde_with::{json::JsonString, serde_as, DefaultOnError, VecSkipError};
|
use serde_with::{json::JsonString, serde_as, DefaultOnError, VecSkipError};
|
||||||
|
|
||||||
|
@ -39,9 +36,12 @@ use crate::serializer::{
|
||||||
VecLogError,
|
VecLogError,
|
||||||
};
|
};
|
||||||
use crate::timeago;
|
use crate::timeago;
|
||||||
use crate::util::MappingError;
|
|
||||||
use crate::util::{self, TryRemove};
|
use crate::util::{self, TryRemove};
|
||||||
|
|
||||||
|
use self::search::ChannelRenderer;
|
||||||
|
use self::search::PlaylistRenderer;
|
||||||
|
use self::search::VideoRenderer;
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
pub struct ContentRenderer<T> {
|
pub struct ContentRenderer<T> {
|
||||||
|
@ -93,8 +93,6 @@ pub enum VideoListItem {
|
||||||
/// Playlist on channel page
|
/// Playlist on channel page
|
||||||
GridPlaylistRenderer(GridPlaylistRenderer),
|
GridPlaylistRenderer(GridPlaylistRenderer),
|
||||||
|
|
||||||
/// Video on startpage
|
|
||||||
///
|
|
||||||
/// Seems to be currently A/B tested on the channel page,
|
/// Seems to be currently A/B tested on the channel page,
|
||||||
/// as of 11.10.2022
|
/// as of 11.10.2022
|
||||||
RichItemRenderer { content: RichItem },
|
RichItemRenderer { content: RichItem },
|
||||||
|
@ -176,67 +174,6 @@ pub struct CompactVideoRenderer {
|
||||||
pub thumbnail_overlays: Vec<TimeOverlay>,
|
pub thumbnail_overlays: Vec<TimeOverlay>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Video displayed in search results
|
|
||||||
#[serde_as]
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct VideoRenderer {
|
|
||||||
pub video_id: String,
|
|
||||||
pub thumbnail: Thumbnails,
|
|
||||||
#[serde_as(as = "Text")]
|
|
||||||
pub title: String,
|
|
||||||
#[serde(rename = "shortBylineText")]
|
|
||||||
pub channel: Option<TextComponent>,
|
|
||||||
pub channel_thumbnail_supported_renderers: Option<ChannelThumbnailSupportedRenderers>,
|
|
||||||
#[serde_as(as = "Option<Text>")]
|
|
||||||
pub published_time_text: Option<String>,
|
|
||||||
#[serde_as(as = "Option<Text>")]
|
|
||||||
pub length_text: Option<String>,
|
|
||||||
/// Contains `No views` if the view count is zero
|
|
||||||
#[serde_as(as = "Option<Text>")]
|
|
||||||
pub view_count_text: Option<String>,
|
|
||||||
/// Channel verification badge
|
|
||||||
#[serde(default)]
|
|
||||||
#[serde_as(as = "VecSkipError<_>")]
|
|
||||||
pub owner_badges: Vec<ChannelBadge>,
|
|
||||||
/// Contains Short/Live tag
|
|
||||||
#[serde_as(as = "VecSkipError<_>")]
|
|
||||||
pub thumbnail_overlays: Vec<TimeOverlay>,
|
|
||||||
/// Abbreviated video description (on startpage)
|
|
||||||
#[serde_as(as = "Option<Text>")]
|
|
||||||
pub description_snippet: Option<String>,
|
|
||||||
/// Contains abbreviated video description (on search page)
|
|
||||||
#[serde_as(as = "Option<VecSkipError<_>>")]
|
|
||||||
pub detailed_metadata_snippets: Option<Vec<DetailedMetadataSnippet>>,
|
|
||||||
/// Release date for upcoming videos
|
|
||||||
pub upcoming_event_data: Option<UpcomingEventData>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Playlist displayed in search results
|
|
||||||
#[serde_as]
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct PlaylistRenderer {
|
|
||||||
pub playlist_id: String,
|
|
||||||
#[serde_as(as = "Text")]
|
|
||||||
pub title: String,
|
|
||||||
/// The first item of this list contains the playlist thumbnail,
|
|
||||||
/// subsequent items contain very small thumbnails of the next playlist videos
|
|
||||||
pub thumbnails: Vec<Thumbnails>,
|
|
||||||
#[serde_as(as = "JsonString")]
|
|
||||||
pub video_count: u64,
|
|
||||||
#[serde(rename = "shortBylineText")]
|
|
||||||
pub channel: TextComponent,
|
|
||||||
/// Channel verification badge
|
|
||||||
#[serde(default)]
|
|
||||||
#[serde_as(as = "VecSkipError<_>")]
|
|
||||||
pub owner_badges: Vec<ChannelBadge>,
|
|
||||||
/// First 2 videos
|
|
||||||
#[serde(default)]
|
|
||||||
#[serde_as(as = "VecSkipError<_>")]
|
|
||||||
pub videos: Vec<ChildVideoRendererWrap>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Video displayed in a playlist
|
/// Video displayed in a playlist
|
||||||
#[serde_as]
|
#[serde_as]
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
|
@ -252,32 +189,6 @@ pub struct PlaylistVideoRenderer {
|
||||||
pub length_seconds: u32,
|
pub length_seconds: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Channel displayed in search results
|
|
||||||
#[serde_as]
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct ChannelRenderer {
|
|
||||||
pub channel_id: String,
|
|
||||||
#[serde_as(as = "Text")]
|
|
||||||
pub title: String,
|
|
||||||
pub thumbnail: Thumbnails,
|
|
||||||
/// Abbreviated channel description
|
|
||||||
///
|
|
||||||
/// Not present if the channel has no description
|
|
||||||
#[serde(default)]
|
|
||||||
#[serde_as(as = "Text")]
|
|
||||||
pub description_snippet: String,
|
|
||||||
/// Not present if the channel has no videos
|
|
||||||
#[serde_as(as = "Option<Text>")]
|
|
||||||
pub video_count_text: Option<String>,
|
|
||||||
#[serde_as(as = "Option<Text>")]
|
|
||||||
pub subscriber_count_text: Option<String>,
|
|
||||||
/// Channel verification badge
|
|
||||||
#[serde(default)]
|
|
||||||
#[serde_as(as = "VecSkipError<_>")]
|
|
||||||
pub owner_badges: Vec<ChannelBadge>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Playlist displayed on a channel page
|
/// Playlist displayed on a channel page
|
||||||
#[serde_as]
|
#[serde_as]
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
|
@ -293,9 +204,8 @@ pub struct GridPlaylistRenderer {
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
#[allow(clippy::large_enum_variant)]
|
|
||||||
pub enum RichItem {
|
pub enum RichItem {
|
||||||
VideoRenderer(VideoRenderer),
|
VideoRenderer(GridVideoRenderer),
|
||||||
PlaylistRenderer(GridPlaylistRenderer),
|
PlaylistRenderer(GridPlaylistRenderer),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -448,43 +358,6 @@ pub struct AlertRenderer {
|
||||||
pub text: String,
|
pub text: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct ChannelThumbnailSupportedRenderers {
|
|
||||||
pub channel_thumbnail_with_link_renderer: ChannelThumbnailWithLinkRenderer,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct ChannelThumbnailWithLinkRenderer {
|
|
||||||
pub thumbnail: Thumbnails,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[serde_as]
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct DetailedMetadataSnippet {
|
|
||||||
#[serde_as(as = "Text")]
|
|
||||||
pub snippet_text: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct ChildVideoRendererWrap {
|
|
||||||
pub child_video_renderer: ChildVideoRenderer,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[serde_as]
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct ChildVideoRenderer {
|
|
||||||
pub video_id: String,
|
|
||||||
#[serde_as(as = "Text")]
|
|
||||||
pub title: String,
|
|
||||||
#[serde_as(as = "Option<Text>")]
|
|
||||||
pub length_text: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
// YouTube Music
|
// YouTube Music
|
||||||
|
|
||||||
#[serde_as]
|
#[serde_as]
|
||||||
|
@ -694,48 +567,6 @@ impl FromWLang<GridVideoRenderer> for model::ChannelVideo {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl FromWLang<VideoRenderer> for model::ChannelVideo {
|
|
||||||
fn from_w_lang(video: VideoRenderer, lang: Language) -> Self {
|
|
||||||
let mut toverlays = video.thumbnail_overlays;
|
|
||||||
let is_live = toverlays.is_live();
|
|
||||||
let is_short = toverlays.is_short();
|
|
||||||
let to = toverlays.try_swap_remove(0);
|
|
||||||
|
|
||||||
Self {
|
|
||||||
id: video.video_id,
|
|
||||||
title: video.title,
|
|
||||||
// Time text is `LIVE` for livestreams, so we ignore parse errors
|
|
||||||
length: to.and_then(|to| {
|
|
||||||
util::parse_video_length(&to.thumbnail_overlay_time_status_renderer.text)
|
|
||||||
}),
|
|
||||||
thumbnail: video.thumbnail.into(),
|
|
||||||
publish_date: video
|
|
||||||
.upcoming_event_data
|
|
||||||
.as_ref()
|
|
||||||
.map(|upc| {
|
|
||||||
chrono::Local.from_utc_datetime(&chrono::NaiveDateTime::from_timestamp(
|
|
||||||
upc.start_time,
|
|
||||||
0,
|
|
||||||
))
|
|
||||||
})
|
|
||||||
.or_else(|| {
|
|
||||||
video
|
|
||||||
.published_time_text
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|txt| timeago::parse_timeago_to_dt(lang, txt))
|
|
||||||
}),
|
|
||||||
publish_date_txt: video.published_time_text,
|
|
||||||
view_count: video
|
|
||||||
.view_count_text
|
|
||||||
.and_then(|txt| util::parse_numeric(&txt).ok())
|
|
||||||
.unwrap_or_default(),
|
|
||||||
is_live,
|
|
||||||
is_short,
|
|
||||||
is_upcoming: video.upcoming_event_data.is_some(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<GridPlaylistRenderer> for model::ChannelPlaylist {
|
impl From<GridPlaylistRenderer> for model::ChannelPlaylist {
|
||||||
fn from(playlist: GridPlaylistRenderer) -> Self {
|
fn from(playlist: GridPlaylistRenderer) -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
@ -785,14 +616,8 @@ impl TryFromWLang<CompactVideoRenderer> for model::RecommendedVideo {
|
||||||
|
|
||||||
impl TryFromWLang<VideoRenderer> for model::SearchVideo {
|
impl TryFromWLang<VideoRenderer> for model::SearchVideo {
|
||||||
fn from_w_lang(video: VideoRenderer, lang: Language) -> Result<Self, util::MappingError> {
|
fn from_w_lang(video: VideoRenderer, lang: Language) -> Result<Self, util::MappingError> {
|
||||||
let channel = model::ChannelId::try_from(
|
let channel = model::ChannelId::try_from(video.channel)?;
|
||||||
video
|
let mut metadata_snippets = video.detailed_metadata_snippets;
|
||||||
.channel
|
|
||||||
.ok_or_else(|| MappingError("no video channel".into()))?,
|
|
||||||
)?;
|
|
||||||
let channel_thumbnail = video
|
|
||||||
.channel_thumbnail_supported_renderers
|
|
||||||
.ok_or_else(|| MappingError("no video channel thumbnail".into()))?;
|
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
id: video.video_id,
|
id: video.video_id,
|
||||||
|
@ -804,7 +629,8 @@ impl TryFromWLang<VideoRenderer> for model::SearchVideo {
|
||||||
channel: model::ChannelTag {
|
channel: model::ChannelTag {
|
||||||
id: channel.id,
|
id: channel.id,
|
||||||
name: channel.name,
|
name: channel.name,
|
||||||
avatar: channel_thumbnail
|
avatar: video
|
||||||
|
.channel_thumbnail_supported_renderers
|
||||||
.channel_thumbnail_with_link_renderer
|
.channel_thumbnail_with_link_renderer
|
||||||
.thumbnail
|
.thumbnail
|
||||||
.into(),
|
.into(),
|
||||||
|
@ -822,10 +648,9 @@ impl TryFromWLang<VideoRenderer> for model::SearchVideo {
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
is_live: video.thumbnail_overlays.is_live(),
|
is_live: video.thumbnail_overlays.is_live(),
|
||||||
is_short: video.thumbnail_overlays.is_short(),
|
is_short: video.thumbnail_overlays.is_short(),
|
||||||
short_description: video
|
short_description: metadata_snippets
|
||||||
.detailed_metadata_snippets
|
.try_swap_remove(0)
|
||||||
.and_then(|mut snippets| snippets.try_swap_remove(0).map(|s| s.snippet_text))
|
.map(|s| s.snippet_text)
|
||||||
.or(video.description_snippet)
|
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,12 +3,13 @@ use serde_with::json::JsonString;
|
||||||
use serde_with::{serde_as, VecSkipError};
|
use serde_with::{serde_as, VecSkipError};
|
||||||
|
|
||||||
use crate::serializer::ignore_any;
|
use crate::serializer::ignore_any;
|
||||||
use crate::serializer::{text::Text, MapResult, VecLogError};
|
use crate::serializer::{
|
||||||
|
text::{Text, TextComponent},
|
||||||
use super::{
|
MapResult, VecLogError,
|
||||||
ChannelRenderer, ContentsRenderer, ContinuationEndpoint, PlaylistRenderer, VideoRenderer,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use super::{ChannelBadge, ContentsRenderer, ContinuationEndpoint, Thumbnails, TimeOverlay};
|
||||||
|
|
||||||
#[serde_as]
|
#[serde_as]
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
|
@ -98,3 +99,122 @@ pub enum SearchItem {
|
||||||
#[serde(other, deserialize_with = "ignore_any")]
|
#[serde(other, deserialize_with = "ignore_any")]
|
||||||
None,
|
None,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Video displayed in search results
|
||||||
|
#[serde_as]
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct VideoRenderer {
|
||||||
|
pub video_id: String,
|
||||||
|
pub thumbnail: Thumbnails,
|
||||||
|
#[serde_as(as = "Text")]
|
||||||
|
pub title: String,
|
||||||
|
#[serde(rename = "shortBylineText")]
|
||||||
|
pub channel: TextComponent,
|
||||||
|
pub channel_thumbnail_supported_renderers: ChannelThumbnailSupportedRenderers,
|
||||||
|
#[serde_as(as = "Option<Text>")]
|
||||||
|
pub published_time_text: Option<String>,
|
||||||
|
#[serde_as(as = "Option<Text>")]
|
||||||
|
pub length_text: Option<String>,
|
||||||
|
/// Contains `No views` if the view count is zero
|
||||||
|
#[serde_as(as = "Option<Text>")]
|
||||||
|
pub view_count_text: Option<String>,
|
||||||
|
/// Channel verification badge
|
||||||
|
#[serde(default)]
|
||||||
|
#[serde_as(as = "VecSkipError<_>")]
|
||||||
|
pub owner_badges: Vec<ChannelBadge>,
|
||||||
|
/// Contains Short/Live tag
|
||||||
|
#[serde_as(as = "VecSkipError<_>")]
|
||||||
|
pub thumbnail_overlays: Vec<TimeOverlay>,
|
||||||
|
#[serde(default)]
|
||||||
|
#[serde_as(as = "VecSkipError<_>")]
|
||||||
|
pub detailed_metadata_snippets: Vec<DetailedMetadataSnippet>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Playlist displayed in search results
|
||||||
|
#[serde_as]
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct PlaylistRenderer {
|
||||||
|
pub playlist_id: String,
|
||||||
|
#[serde_as(as = "Text")]
|
||||||
|
pub title: String,
|
||||||
|
/// The first item of this list contains the playlist thumbnail,
|
||||||
|
/// subsequent items contain very small thumbnails of the next playlist videos
|
||||||
|
pub thumbnails: Vec<Thumbnails>,
|
||||||
|
#[serde_as(as = "JsonString")]
|
||||||
|
pub video_count: u64,
|
||||||
|
#[serde(rename = "shortBylineText")]
|
||||||
|
pub channel: TextComponent,
|
||||||
|
/// Channel verification badge
|
||||||
|
#[serde(default)]
|
||||||
|
#[serde_as(as = "VecSkipError<_>")]
|
||||||
|
pub owner_badges: Vec<ChannelBadge>,
|
||||||
|
/// First 2 videos
|
||||||
|
#[serde(default)]
|
||||||
|
#[serde_as(as = "VecSkipError<_>")]
|
||||||
|
pub videos: Vec<ChildVideoRendererWrap>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Channel displayed in search results
|
||||||
|
#[serde_as]
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct ChannelRenderer {
|
||||||
|
pub channel_id: String,
|
||||||
|
#[serde_as(as = "Text")]
|
||||||
|
pub title: String,
|
||||||
|
pub thumbnail: Thumbnails,
|
||||||
|
/// Abbreviated channel description
|
||||||
|
///
|
||||||
|
/// Not present if the channel has no description
|
||||||
|
#[serde(default)]
|
||||||
|
#[serde_as(as = "Text")]
|
||||||
|
pub description_snippet: String,
|
||||||
|
/// Not present if the channel has no videos
|
||||||
|
#[serde_as(as = "Option<Text>")]
|
||||||
|
pub video_count_text: Option<String>,
|
||||||
|
#[serde_as(as = "Option<Text>")]
|
||||||
|
pub subscriber_count_text: Option<String>,
|
||||||
|
/// Channel verification badge
|
||||||
|
#[serde(default)]
|
||||||
|
#[serde_as(as = "VecSkipError<_>")]
|
||||||
|
pub owner_badges: Vec<ChannelBadge>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct ChannelThumbnailSupportedRenderers {
|
||||||
|
pub channel_thumbnail_with_link_renderer: ChannelThumbnailWithLinkRenderer,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct ChannelThumbnailWithLinkRenderer {
|
||||||
|
pub thumbnail: Thumbnails,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct ChildVideoRendererWrap {
|
||||||
|
pub child_video_renderer: ChildVideoRenderer,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[serde_as]
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct ChildVideoRenderer {
|
||||||
|
pub video_id: String,
|
||||||
|
#[serde_as(as = "Text")]
|
||||||
|
pub title: String,
|
||||||
|
#[serde_as(as = "Option<Text>")]
|
||||||
|
pub length_text: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[serde_as]
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct DetailedMetadataSnippet {
|
||||||
|
#[serde_as(as = "Text")]
|
||||||
|
pub snippet_text: String,
|
||||||
|
}
|
||||||
|
|
|
@ -1,103 +0,0 @@
|
||||||
use serde::Deserialize;
|
|
||||||
use serde_with::{serde_as, VecSkipError};
|
|
||||||
|
|
||||||
use crate::serializer::{ignore_any, MapResult, VecLogError};
|
|
||||||
|
|
||||||
use super::{ContentRenderer, ContentsRenderer, VideoListItem, VideoRenderer};
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct Startpage {
|
|
||||||
pub contents: Contents<BrowseResultsStartpage>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct Contents<T> {
|
|
||||||
pub two_column_browse_results_renderer: T,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[serde_as]
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct BrowseResultsStartpage {
|
|
||||||
#[serde_as(as = "VecSkipError<_>")]
|
|
||||||
pub tabs: Vec<Tab<StartpageTabContent>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[serde_as]
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct BrowseResultsTrends {
|
|
||||||
#[serde_as(as = "VecSkipError<_>")]
|
|
||||||
pub tabs: Vec<Tab<TrendingTabContent>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct Tab<T> {
|
|
||||||
pub tab_renderer: ContentRenderer<T>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct StartpageTabContent {
|
|
||||||
pub rich_grid_renderer: RichGridRenderer,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[serde_as]
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct RichGridRenderer {
|
|
||||||
#[serde_as(as = "VecLogError<_>")]
|
|
||||||
pub contents: MapResult<Vec<VideoListItem>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct Trending {
|
|
||||||
pub contents: Contents<BrowseResultsTrends>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct TrendingTabContent {
|
|
||||||
pub section_list_renderer: ContentsRenderer<ItemSectionRenderer>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct ItemSectionRenderer {
|
|
||||||
pub item_section_renderer: ContentsRenderer<ShelfRenderer>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct ShelfRenderer {
|
|
||||||
pub shelf_renderer: ContentRenderer<ShelfContents>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct ShelfContents {
|
|
||||||
pub expanded_shelf_contents_renderer: Option<ShelfContentsRenderer>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[serde_as]
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct ShelfContentsRenderer {
|
|
||||||
#[serde_as(as = "VecLogError<_>")]
|
|
||||||
pub items: MapResult<Vec<TrendingListItem>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[serde_as]
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
#[allow(clippy::large_enum_variant)]
|
|
||||||
pub enum TrendingListItem {
|
|
||||||
VideoRenderer(VideoRenderer),
|
|
||||||
|
|
||||||
#[serde(other, deserialize_with = "ignore_any")]
|
|
||||||
None,
|
|
||||||
}
|
|
|
@ -1,4 +1,4 @@
|
||||||
use serde::{de::IgnoredAny, Serialize};
|
use serde::Serialize;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
deobfuscate::Deobfuscator,
|
deobfuscate::Deobfuscator,
|
||||||
|
@ -79,32 +79,6 @@ impl RustyPipeQuery {
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn search_suggestion(self, query: &str) -> Result<Vec<String>, Error> {
|
|
||||||
let url = url::Url::parse_with_params("https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&gs_rn=64&gs_ri=youtube&ds=yt&cp=1&gs_id=4&xhr=t&xssi=t",
|
|
||||||
&[("hl", self.opts.lang.to_string()), ("gl", self.opts.country.to_string()), ("q", query.to_string())]
|
|
||||||
).map_err(|_| Error::Other("could not build url".into()))?;
|
|
||||||
|
|
||||||
let response = self
|
|
||||||
.client
|
|
||||||
.http_request_txt(self.client.inner.http.get(url).build()?)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let trimmed = response.get(5..).ok_or_else(|| {
|
|
||||||
Error::Extraction(ExtractionError::InvalidData(
|
|
||||||
"could not get string slice".into(),
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let parsed = serde_json::from_str::<(
|
|
||||||
IgnoredAny,
|
|
||||||
Vec<(String, IgnoredAny, IgnoredAny)>,
|
|
||||||
IgnoredAny,
|
|
||||||
)>(trimmed)
|
|
||||||
.map_err(|e| Error::Extraction(ExtractionError::InvalidData(e.to_string().into())))?;
|
|
||||||
|
|
||||||
Ok(parsed.1.into_iter().map(|item| item.0).collect())
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MapResponse<SearchResult> for response::Search {
|
impl MapResponse<SearchResult> for response::Search {
|
||||||
|
|
|
@ -1,148 +0,0 @@
|
||||||
use crate::{
|
|
||||||
error::{Error, ExtractionError},
|
|
||||||
model::{Paginator, SearchVideo},
|
|
||||||
serializer::MapResult,
|
|
||||||
util::TryRemove,
|
|
||||||
};
|
|
||||||
|
|
||||||
use super::{
|
|
||||||
response::{self, TryFromWLang},
|
|
||||||
ClientType, MapResponse, QBrowse, RustyPipeQuery,
|
|
||||||
};
|
|
||||||
|
|
||||||
impl RustyPipeQuery {
|
|
||||||
pub async fn startpage(self) -> Result<Paginator<SearchVideo>, Error> {
|
|
||||||
let context = self.get_context(ClientType::Desktop, true).await;
|
|
||||||
let request_body = QBrowse {
|
|
||||||
context,
|
|
||||||
browse_id: "FEwhat_to_watch".to_owned(),
|
|
||||||
};
|
|
||||||
|
|
||||||
self.execute_request::<response::Startpage, _, _>(
|
|
||||||
ClientType::Desktop,
|
|
||||||
"startpage",
|
|
||||||
"",
|
|
||||||
"browse",
|
|
||||||
&request_body,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn trending(self) -> Result<Vec<SearchVideo>, Error> {
|
|
||||||
let context = self.get_context(ClientType::Desktop, true).await;
|
|
||||||
let request_body = QBrowse {
|
|
||||||
context,
|
|
||||||
browse_id: "FEtrending".to_owned(),
|
|
||||||
};
|
|
||||||
|
|
||||||
self.execute_request::<response::Trending, _, _>(
|
|
||||||
ClientType::Desktop,
|
|
||||||
"trends",
|
|
||||||
"",
|
|
||||||
"browse",
|
|
||||||
&request_body,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl MapResponse<Paginator<SearchVideo>> for response::Startpage {
|
|
||||||
fn map_response(
|
|
||||||
self,
|
|
||||||
_id: &str,
|
|
||||||
lang: crate::param::Language,
|
|
||||||
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
|
||||||
) -> Result<MapResult<Paginator<SearchVideo>>, ExtractionError> {
|
|
||||||
let mut contents = self.contents.two_column_browse_results_renderer.tabs;
|
|
||||||
let grid = contents
|
|
||||||
.try_swap_remove(0)
|
|
||||||
.ok_or_else(|| ExtractionError::InvalidData("no contents".into()))?
|
|
||||||
.tab_renderer
|
|
||||||
.content
|
|
||||||
.rich_grid_renderer
|
|
||||||
.contents;
|
|
||||||
|
|
||||||
let mut warnings = grid.warnings;
|
|
||||||
let mut ctoken = None;
|
|
||||||
let items = grid
|
|
||||||
.c
|
|
||||||
.into_iter()
|
|
||||||
.filter_map(|item| match item {
|
|
||||||
response::VideoListItem::RichItemRenderer {
|
|
||||||
content: response::RichItem::VideoRenderer(video),
|
|
||||||
} => match SearchVideo::from_w_lang(video, lang) {
|
|
||||||
Ok(video) => Some(video),
|
|
||||||
Err(e) => {
|
|
||||||
warnings.push(e.to_string());
|
|
||||||
None
|
|
||||||
}
|
|
||||||
},
|
|
||||||
response::VideoListItem::ContinuationItemRenderer {
|
|
||||||
continuation_endpoint,
|
|
||||||
} => {
|
|
||||||
ctoken = Some(continuation_endpoint.continuation_command.token);
|
|
||||||
None
|
|
||||||
}
|
|
||||||
_ => None,
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
Ok(MapResult {
|
|
||||||
c: Paginator::new(None, items, ctoken),
|
|
||||||
warnings,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl MapResponse<Vec<SearchVideo>> for response::Trending {
|
|
||||||
fn map_response(
|
|
||||||
self,
|
|
||||||
_id: &str,
|
|
||||||
lang: crate::param::Language,
|
|
||||||
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
|
||||||
) -> Result<MapResult<Vec<SearchVideo>>, ExtractionError> {
|
|
||||||
let mut contents = self.contents.two_column_browse_results_renderer.tabs;
|
|
||||||
let sections = contents
|
|
||||||
.try_swap_remove(0)
|
|
||||||
.ok_or_else(|| ExtractionError::InvalidData("no contents".into()))?
|
|
||||||
.tab_renderer
|
|
||||||
.content
|
|
||||||
.section_list_renderer
|
|
||||||
.contents;
|
|
||||||
|
|
||||||
let mut items = Vec::new();
|
|
||||||
let mut warnings = Vec::new();
|
|
||||||
|
|
||||||
for mut section in sections {
|
|
||||||
let shelf = section
|
|
||||||
.item_section_renderer
|
|
||||||
.contents
|
|
||||||
.try_swap_remove(0)
|
|
||||||
.and_then(|shelf| {
|
|
||||||
shelf
|
|
||||||
.shelf_renderer
|
|
||||||
.content
|
|
||||||
.expanded_shelf_contents_renderer
|
|
||||||
});
|
|
||||||
|
|
||||||
if let Some(mut shelf) = shelf {
|
|
||||||
warnings.append(&mut shelf.items.warnings);
|
|
||||||
|
|
||||||
for item in shelf.items.c {
|
|
||||||
if let response::trends::TrendingListItem::VideoRenderer(video) = item {
|
|
||||||
match SearchVideo::from_w_lang(video, lang) {
|
|
||||||
Ok(video) => {
|
|
||||||
items.push(video);
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
warnings.push(e.to_string());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(MapResult { c: items, warnings })
|
|
||||||
}
|
|
||||||
}
|
|
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
@ -1185,53 +1185,6 @@ async fn search_empty() {
|
||||||
assert!(result.items.is_empty());
|
assert!(result.items.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn search_suggestion() {
|
|
||||||
let rp = RustyPipe::builder().strict().build();
|
|
||||||
let result = rp.query().search_suggestion("hunger ga").await.unwrap();
|
|
||||||
|
|
||||||
assert!(result.contains(&"hunger games".to_owned()));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn search_suggestion_empty() {
|
|
||||||
let rp = RustyPipe::builder().strict().build();
|
|
||||||
let result = rp
|
|
||||||
.query()
|
|
||||||
.search_suggestion("fjew327%4ifjelwfvnewg49")
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
assert!(result.is_empty());
|
|
||||||
}
|
|
||||||
|
|
||||||
//#TRENDS
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn startpage() {
|
|
||||||
let rp = RustyPipe::builder().strict().build();
|
|
||||||
let result = rp.query().startpage().await.unwrap();
|
|
||||||
|
|
||||||
assert!(
|
|
||||||
result.items.len() > 20,
|
|
||||||
"expected > 20 items, got {}",
|
|
||||||
result.items.len()
|
|
||||||
);
|
|
||||||
assert!(!result.is_exhausted());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn trending() {
|
|
||||||
let rp = RustyPipe::builder().strict().build();
|
|
||||||
let result = rp.query().trending().await.unwrap();
|
|
||||||
|
|
||||||
assert!(
|
|
||||||
result.len() > 50,
|
|
||||||
"expected > 50 items, got {}",
|
|
||||||
result.len()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
//#TESTUTIL
|
//#TESTUTIL
|
||||||
|
|
||||||
/// Assert equality within 10% margin
|
/// Assert equality within 10% margin
|
||||||
|
|
Loading…
Reference in a new issue