Compare commits

..

8 commits

25 changed files with 336 additions and 171 deletions

View file

@ -10,7 +10,7 @@ keywords = ["youtube", "video", "music"]
include = ["/src", "README.md", "LICENSE", "!snapshots"] include = ["/src", "README.md", "LICENSE", "!snapshots"]
[workspace] [workspace]
members = [".", "codegen", "cli"] members = [".", "codegen", "downloader", "cli"]
[features] [features]
default = ["default-tls"] default = ["default-tls"]
@ -34,9 +34,8 @@ reqwest = { version = "0.11.11", default-features = false, features = [
"json", "json",
"gzip", "gzip",
"brotli", "brotli",
"stream",
] } ] }
tokio = { version = "1.20.0", features = ["macros", "time", "fs", "process"] } tokio = { version = "1.20.0", features = ["macros", "time"] }
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0.82" serde_json = "1.0.82"
serde_with = { version = "2.0.0", features = ["json"] } serde_with = { version = "2.0.0", features = ["json"] }
@ -47,8 +46,6 @@ time = { version = "0.3.15", features = [
"serde-well-known", "serde-well-known",
] } ] }
futures = "0.3.21" futures = "0.3.21"
indicatif = "0.17.0"
filenamify = "0.1.0"
ress = "0.11.4" ress = "0.11.4"
phf = "0.11.1" phf = "0.11.1"
base64 = "0.13.0" base64 = "0.13.0"

View file

@ -4,11 +4,16 @@ version = "0.1.0"
edition = "2021" edition = "2021"
[dependencies] [dependencies]
rustypipe = {path = "../", default_features = false, features = ["rustls-tls-native-roots"]} rustypipe = { path = "../", default_features = false, features = [
"rustls-tls-native-roots",
] }
rustypipe-downloader = { path = "../downloader", default_features = false, features = [
"rustls-tls-native-roots",
] }
reqwest = { version = "0.11.11", default_features = false } reqwest = { version = "0.11.11", default_features = false }
tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] } tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] }
indicatif = "0.17.0" indicatif = "0.17.0"
futures = "0.3.21" futures = "0.3.21"
anyhow = "1.0" anyhow = "1.0"
clap = { version = "3.2.16", features = ["derive"] } clap = { version = "3.2.16", features = ["derive"] }
env_logger = "0.9.0" env_logger = "0.10.0"

View file

@ -69,7 +69,7 @@ async fn download_single_video(
} }
} }
rustypipe::download::download_video( rustypipe_downloader::download_video(
&player_data, &player_data,
output_dir, output_dir,
output_fname, output_fname,

31
downloader/Cargo.toml Normal file
View file

@ -0,0 +1,31 @@
[package]
name = "rustypipe-downloader"
version = "0.1.0"
edition = "2021"
[features]
# Reqwest TLS
default-tls = ["reqwest/default-tls", "rustypipe/default-tls"]
rustls-tls-webpki-roots = [
"reqwest/rustls-tls-webpki-roots",
"rustypipe/rustls-tls-webpki-roots",
]
rustls-tls-native-roots = [
"reqwest/rustls-tls-native-roots",
"rustypipe/rustls-tls-native-roots",
]
[dependencies]
rustypipe = { path = "..", default-features = false }
once_cell = "1.12.0"
regex = "1.6.0"
thiserror = "1.0.36"
futures = "0.3.21"
indicatif = "0.17.0"
filenamify = "0.1.0"
log = "0.4.17"
reqwest = { version = "0.11.11", default-features = false, features = [
"stream",
] }
rand = "0.8.5"
tokio = { version = "1.20.0", features = ["macros", "fs", "process"] }

View file

@ -1,26 +1,27 @@
//! YouTube audio/video downloader //! # YouTube audio/video downloader
mod util;
use std::{borrow::Cow, cmp::Ordering, ffi::OsString, ops::Range, path::PathBuf, time::Duration}; use std::{borrow::Cow, cmp::Ordering, ffi::OsString, ops::Range, path::PathBuf, time::Duration};
use fancy_regex::Regex;
use futures::stream::{self, StreamExt}; use futures::stream::{self, StreamExt};
use indicatif::{ProgressBar, ProgressStyle}; use indicatif::{ProgressBar, ProgressStyle};
use log::{debug, info}; use log::{debug, info};
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use rand::Rng; use rand::Rng;
use regex::Regex;
use reqwest::{header, Client}; use reqwest::{header, Client};
use rustypipe::{
model::{AudioCodec, FileFormat, VideoCodec, VideoPlayer},
param::StreamFilter,
};
use tokio::{ use tokio::{
fs::{self, File}, fs::{self, File},
io::AsyncWriteExt, io::AsyncWriteExt,
process::Command, process::Command,
}; };
use crate::{ use util::DownloadError;
error::DownloadError,
model::{AudioCodec, FileFormat, VideoCodec, VideoPlayer},
param::StreamFilter,
util,
};
type Result<T> = core::result::Result<T, DownloadError>; type Result<T> = core::result::Result<T, DownloadError>;
@ -45,7 +46,7 @@ fn get_download_range(offset: u64, size: Option<u64>) -> Range<u64> {
fn parse_cr_header(cr_header: &str) -> Result<(u64, u64)> { fn parse_cr_header(cr_header: &str) -> Result<(u64, u64)> {
static PATTERN: Lazy<Regex> = Lazy::new(|| Regex::new(r#"bytes (\d+)-(\d+)/(\d+)"#).unwrap()); static PATTERN: Lazy<Regex> = Lazy::new(|| Regex::new(r#"bytes (\d+)-(\d+)/(\d+)"#).unwrap());
let captures = PATTERN.captures(cr_header).ok().flatten().ok_or_else(|| { let captures = PATTERN.captures(cr_header).ok_or_else(|| {
DownloadError::Progressive( DownloadError::Progressive(
format!( format!(
"Content-Range header '{}' does not match pattern", "Content-Range header '{}' does not match pattern",
@ -317,11 +318,9 @@ pub async fn download_video(
Some(_) => "mp4", Some(_) => "mp4",
None => match audio { None => match audio {
Some(audio) => match audio.codec { Some(audio) => match audio.codec {
AudioCodec::Unknown => {
return Err(DownloadError::Input("unknown audio codec".into()))
}
AudioCodec::Mp4a => "m4a", AudioCodec::Mp4a => "m4a",
AudioCodec::Opus => "opus", AudioCodec::Opus => "opus",
_ => return Err(DownloadError::Input("unknown audio codec".into())),
}, },
None => unreachable!(), None => unreachable!(),
}, },
@ -473,40 +472,3 @@ async fn convert_streams<P: Into<PathBuf>>(
} }
Ok(()) Ok(())
} }
/*
#[cfg(test)]
mod tests {
use crate::client::RustyTube;
use super::*;
use indicatif::{ProgressDrawTarget, ProgressStyle};
use reqwest::ClientBuilder;
// #[test_log::test(tokio::test)]
#[tokio::test]
async fn t_download_video() {
let http = ClientBuilder::new()
.user_agent(
"Mozilla/5.0 (Windows NT 10.0; Win64; rv:107.0) Gecko/20100101 Firefox/107.0",
)
.gzip(true)
.brotli(true)
.build()
.expect("unable to build the HTTP client");
// Indicatif setup
let pb = ProgressBar::new(0);
let rt = RustyTube::new();
let player_data = rt
.get_player("AbZH7XWDW_k", crate::client::ClientType::Desktop)
.await
.unwrap();
// download_video(&player_data, "tmp", "INVU", Some(1080), "ffmpeg", http, pb)
// .await
// .unwrap();
}
}
*/

42
downloader/src/util.rs Normal file
View file

@ -0,0 +1,42 @@
use std::{borrow::Cow, collections::BTreeMap};
use reqwest::Url;
/// Error from the video downloader
#[derive(thiserror::Error, Debug)]
#[non_exhaustive]
pub enum DownloadError {
/// Error from the HTTP client
#[error("http error: {0}")]
Http(#[from] reqwest::Error),
/// File IO error
#[error(transparent)]
Io(#[from] std::io::Error),
#[error("FFmpeg error: {0}")]
Ffmpeg(Cow<'static, str>),
#[error("Progressive download error: {0}")]
Progressive(Cow<'static, str>),
#[error("input error: {0}")]
Input(Cow<'static, str>),
#[error("error: {0}")]
Other(Cow<'static, str>),
}
/// Split an URL into its base string and parameter map
///
/// Example:
///
/// `example.com/api?k1=v1&k2=v2 => example.com/api; {k1: v1, k2: v2}`
pub fn url_to_params(url: &str) -> Result<(Url, BTreeMap<String, String>), DownloadError> {
let mut parsed_url = Url::parse(url).map_err(|e| {
DownloadError::Other(format!("could not parse url `{}` err: {}", url, e).into())
})?;
let url_params: BTreeMap<String, String> = parsed_url
.query_pairs()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect();
parsed_url.set_query(None);
Ok((parsed_url, url_params))
}

View file

@ -397,8 +397,8 @@ impl RustyPipeBuilder {
/// ///
/// **Default value**: `Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0` /// **Default value**: `Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0`
/// (Firefox ESR on Debian) /// (Firefox ESR on Debian)
pub fn user_agent(mut self, user_agent: &str) -> Self { pub fn user_agent<S: Into<String>>(mut self, user_agent: S) -> Self {
self.user_agent = user_agent.to_owned(); self.user_agent = user_agent.into();
self self
} }
@ -444,8 +444,8 @@ impl RustyPipeBuilder {
} }
/// Set the default YouTube visitor data cookie /// Set the default YouTube visitor data cookie
pub fn visitor_data(mut self, visitor_data: &str) -> Self { pub fn visitor_data<S: Into<String>>(mut self, visitor_data: S) -> Self {
self.default_opts.visitor_data = Some(visitor_data.to_owned()); self.default_opts.visitor_data = Some(visitor_data.into());
self self
} }
@ -778,8 +778,8 @@ impl RustyPipeQuery {
} }
/// Set the YouTube visitor data cookie /// Set the YouTube visitor data cookie
pub fn visitor_data(mut self, visitor_data: &str) -> Self { pub fn visitor_data<S: Into<String>>(mut self, visitor_data: S) -> Self {
self.opts.visitor_data = Some(visitor_data.to_owned()); self.opts.visitor_data = Some(visitor_data.into());
self self
} }

View file

@ -99,7 +99,10 @@ impl RustyPipeQuery {
radio_id: S, radio_id: S,
) -> Result<Paginator<TrackItem>, Error> { ) -> Result<Paginator<TrackItem>, Error> {
let radio_id = radio_id.as_ref(); let radio_id = radio_id.as_ref();
let context = self.get_context(ClientType::DesktopMusic, true, None).await; let visitor_data = self.get_ytm_visitor_data().await?;
let context = self
.get_context(ClientType::DesktopMusic, true, Some(&visitor_data))
.await;
let request_body = QRadio { let request_body = QRadio {
context, context,
playlist_id: radio_id, playlist_id: radio_id,

View file

@ -2,7 +2,7 @@ use std::borrow::Cow;
use crate::{ use crate::{
error::{Error, ExtractionError}, error::{Error, ExtractionError},
model::{AlbumId, ChannelId, MusicAlbum, MusicPlaylist, Paginator}, model::{AlbumId, ChannelId, MusicAlbum, MusicPlaylist, Paginator, TrackItem},
serializer::MapResult, serializer::MapResult,
util::{self, TryRemove}, util::{self, TryRemove},
}; };
@ -45,14 +45,55 @@ impl RustyPipeQuery {
browse_id: album_id, browse_id: album_id,
}; };
self.execute_request::<response::MusicPlaylist, _, _>( let mut album = self
.execute_request::<response::MusicPlaylist, MusicAlbum, _>(
ClientType::DesktopMusic, ClientType::DesktopMusic,
"music_album", "music_album",
album_id, album_id,
"browse", "browse",
&request_body, &request_body,
) )
.await .await?;
// YouTube Music is replacing album tracks with their respective music videos. To get the original
// tracks, we have to fetch the album as a playlist and replace the offending track ids.
if let Some(playlist_id) = &album.playlist_id {
// Get a list of music videos in the album
let to_replace = album
.tracks
.iter()
.enumerate()
.filter_map(|(i, track)| {
if track.is_video {
Some((i, track.title.to_owned()))
} else {
None
}
})
.collect::<Vec<_>>();
if !to_replace.is_empty() {
let playlist = self.music_playlist(playlist_id).await?;
for (i, title) in to_replace {
let found_track = playlist.tracks.items.iter().find_map(|track| {
if track.title == title && !track.is_video {
Some((track.id.to_owned(), track.duration))
} else {
None
}
});
if let Some((track_id, duration)) = found_track {
album.tracks[i].id = track_id;
if let Some(duration) = duration {
album.tracks[i].duration = Some(duration);
}
album.tracks[i].is_video = false;
}
}
}
}
Ok(album)
} }
} }
@ -65,8 +106,6 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
) -> Result<MapResult<MusicPlaylist>, ExtractionError> { ) -> Result<MapResult<MusicPlaylist>, ExtractionError> {
// dbg!(&self); // dbg!(&self);
let header = self.header.music_detail_header_renderer;
let mut content = self.contents.single_column_browse_results_renderer.contents; let mut content = self.contents.single_column_browse_results_renderer.contents;
let mut music_contents = content let mut music_contents = content
.try_swap_remove(0) .try_swap_remove(0)
@ -85,30 +124,14 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
"no sectionListRenderer content", "no sectionListRenderer content",
)))?; )))?;
let playlist_id = shelf if let Some(playlist_id) = shelf.playlist_id {
.playlist_id
.ok_or(ExtractionError::InvalidData(Cow::Borrowed(
"no playlist id",
)))?;
if playlist_id != id { if playlist_id != id {
return Err(ExtractionError::WrongResult(format!( return Err(ExtractionError::WrongResult(format!(
"got wrong playlist id {}, expected {}", "got wrong playlist id {}, expected {}",
playlist_id, id playlist_id, id
))); )));
} }
}
let from_ytm = header
.subtitle
.0
.iter()
.any(|c| c.as_str() == util::YT_MUSIC_NAME);
let channel = header
.subtitle
.0
.into_iter()
.find_map(|c| ChannelId::try_from(c).ok());
let mut mapper = MusicListMapper::new(lang); let mut mapper = MusicListMapper::new(lang);
mapper.map_response(shelf.contents); mapper.map_response(shelf.contents);
@ -120,10 +143,12 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
.map(|cont| cont.next_continuation_data.continuation); .map(|cont| cont.next_continuation_data.continuation);
let track_count = match ctoken { let track_count = match ctoken {
Some(_) => header Some(_) => self.header.as_ref().and_then(|h| {
h.music_detail_header_renderer
.second_subtitle .second_subtitle
.first() .first()
.and_then(|txt| util::parse_numeric::<u64>(txt).ok()), .and_then(|txt| util::parse_numeric::<u64>(txt).ok())
}),
None => Some(map_res.c.len() as u64), None => Some(map_res.c.len() as u64),
}; };
@ -132,13 +157,63 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
.try_swap_remove(0) .try_swap_remove(0)
.map(|c| c.next_continuation_data.continuation); .map(|c| c.next_continuation_data.continuation);
let (from_ytm, channel, name, thumbnail, description) = match self.header {
Some(header) => {
let h = header.music_detail_header_renderer;
let from_ytm = h
.subtitle
.0
.iter()
.any(|c| c.as_str() == util::YT_MUSIC_NAME);
let channel = h
.subtitle
.0
.into_iter()
.find_map(|c| ChannelId::try_from(c).ok());
(
from_ytm,
channel,
h.title,
h.thumbnail.into(),
h.description,
)
}
None => {
// Album playlists fetched via the playlist method dont include a header
let (album, cover) = map_res
.c
.first()
.and_then(|t: &TrackItem| {
t.album.as_ref().map(|a| (a.clone(), t.cover.clone()))
})
.ok_or(ExtractionError::InvalidData(Cow::Borrowed(
"playlist without header or album items",
)))?;
if !map_res.c.iter().all(|t| {
t.album
.as_ref()
.map(|a| a.id == album.id)
.unwrap_or_default()
}) {
return Err(ExtractionError::InvalidData(Cow::Borrowed(
"album playlist containing items from different albums",
)));
}
(true, None, album.name, cover, None)
}
};
Ok(MapResult { Ok(MapResult {
c: MusicPlaylist { c: MusicPlaylist {
id: playlist_id, id: id.to_owned(),
name: header.title, name,
thumbnail: header.thumbnail.into(), thumbnail,
channel, channel,
description: header.description, description,
track_count, track_count,
from_ytm, from_ytm,
tracks: Paginator::new_ext( tracks: Paginator::new_ext(
@ -170,7 +245,10 @@ impl MapResponse<MusicAlbum> for response::MusicPlaylist {
) -> Result<MapResult<MusicAlbum>, ExtractionError> { ) -> Result<MapResult<MusicAlbum>, ExtractionError> {
// dbg!(&self); // dbg!(&self);
let header = self.header.music_detail_header_renderer; let header = self
.header
.ok_or(ExtractionError::InvalidData(Cow::Borrowed("no header")))?
.music_detail_header_renderer;
let mut content = self.contents.single_column_browse_results_renderer.contents; let mut content = self.contents.single_column_browse_results_renderer.contents;
let sections = content let sections = content

View file

@ -418,10 +418,12 @@ impl MusicListMapper {
// List item // List item
MusicResponseItem::MusicResponsiveListItemRenderer(item) => { MusicResponseItem::MusicResponsiveListItemRenderer(item) => {
let mut columns = item.flex_columns.into_iter(); let mut columns = item.flex_columns.into_iter();
let title = columns.next().map(|col| col.renderer.text.to_string()); let c1 = columns.next();
let c2 = columns.next(); let c2 = columns.next();
let c3 = columns.next(); let c3 = columns.next();
let title = c1.as_ref().map(|col| col.renderer.text.to_string());
let first_tn = item let first_tn = item
.thumbnail .thumbnail
.music_thumbnail_renderer .music_thumbnail_renderer
@ -433,27 +435,54 @@ impl MusicListMapper {
.navigation_endpoint .navigation_endpoint
.and_then(|ne| ne.music_page()) .and_then(|ne| ne.music_page())
.or_else(|| { .or_else(|| {
item.playlist_item_data c1.and_then(|c1| {
.map(|d| (MusicPageType::Track, d.video_id)) c1.renderer.text.0.into_iter().next().and_then(|t| match t {
crate::serializer::text::TextComponent::Video {
video_id,
is_video,
..
} => Some((MusicPageType::Track { is_video }, video_id)),
crate::serializer::text::TextComponent::Browse {
page_type,
browse_id,
..
} => Some((page_type.into(), browse_id)),
_ => None,
})
})
})
.or_else(|| {
item.playlist_item_data.map(|d| {
(
MusicPageType::Track {
is_video: self.album.is_none()
&& !first_tn
.map(|tn| tn.height == tn.width)
.unwrap_or_default(),
},
d.video_id,
)
})
}) })
.or_else(|| { .or_else(|| {
first_tn.and_then(|tn| { first_tn.and_then(|tn| {
util::video_id_from_thumbnail_url(&tn.url) util::video_id_from_thumbnail_url(&tn.url).map(|id| {
.map(|id| (MusicPageType::Track, id)) (
MusicPageType::Track {
is_video: self.album.is_none() && tn.width != tn.height,
},
id,
)
})
}) })
}); });
match pt_id { match pt_id {
// Track // Track
Some((MusicPageType::Track, id)) => { Some((MusicPageType::Track { is_video }, id)) => {
let title = let title =
title.ok_or_else(|| format!("track {}: could not get title", id))?; title.ok_or_else(|| format!("track {}: could not get title", id))?;
// Videos have rectangular thumbnails, YTM tracks have square covers
// Exception: there are no thumbnails on album items
let is_video = self.album.is_none()
&& !first_tn.map(|tn| tn.height == tn.width).unwrap_or_default();
let (artists_p, album_p, duration_p) = match item.flex_column_display_style let (artists_p, album_p, duration_p) = match item.flex_column_display_style
{ {
// Search result // Search result
@ -519,15 +548,14 @@ impl MusicListMapper {
}), }),
), ),
(_, false) => ( (_, false) => (
album_p album_p.and_then(|p| {
.and_then(|p| {
p.0.into_iter().find_map(|c| AlbumId::try_from(c).ok()) p.0.into_iter().find_map(|c| AlbumId::try_from(c).ok())
}) }),
.or_else(|| self.album.clone()),
None, None,
), ),
(FlexColumnDisplayStyle::Default, true) => (None, None), (FlexColumnDisplayStyle::Default, true) => (None, None),
}; };
let album = album.or_else(|| self.album.clone());
let (mut artists, _) = map_artists(artists_p); let (mut artists, _) = map_artists(artists_p);
@ -640,7 +668,8 @@ impl MusicListMapper {
// There may be broken YT channels from the artist search. They can be skipped. // There may be broken YT channels from the artist search. They can be skipped.
Ok(None) Ok(None)
} }
MusicPageType::Track => unreachable!(), // Tracks were already handled above
MusicPageType::Track { .. } => unreachable!(),
} }
} }
None => Err("could not determine item type".to_owned()), None => Err("could not determine item type".to_owned()),
@ -655,7 +684,7 @@ impl MusicListMapper {
match item.navigation_endpoint.music_page() { match item.navigation_endpoint.music_page() {
Some((page_type, id)) => match page_type { Some((page_type, id)) => match page_type {
MusicPageType::Track => { MusicPageType::Track { is_video } => {
let artists = map_artists(subtitle_p1).0; let artists = map_artists(subtitle_p1).0;
self.items.push(MusicItem::Track(TrackItem { self.items.push(MusicItem::Track(TrackItem {
@ -669,7 +698,7 @@ impl MusicListMapper {
view_count: subtitle_p2.and_then(|c| { view_count: subtitle_p2.and_then(|c| {
util::parse_large_numstr(c.first_str(), self.lang) util::parse_large_numstr(c.first_str(), self.lang)
}), }),
is_video: true, is_video,
track_nr: None, track_nr: None,
})); }));
Ok(Some(MusicEntityType::Track)) Ok(Some(MusicEntityType::Track))

View file

@ -11,7 +11,7 @@ use super::{ContentsRenderer, Tab};
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub(crate) struct MusicPlaylist { pub(crate) struct MusicPlaylist {
pub contents: Contents, pub contents: Contents,
pub header: Header, pub header: Option<Header>,
} }
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]

View file

@ -35,6 +35,8 @@ pub(crate) struct WatchEndpoint {
pub playlist_id: Option<String>, pub playlist_id: Option<String>,
#[serde(default)] #[serde(default)]
pub start_time_seconds: u32, pub start_time_seconds: u32,
#[serde(default)]
pub watch_endpoint_music_supported_configs: WatchEndpointConfigWrap,
} }
#[derive(Debug)] #[derive(Debug)]
@ -118,6 +120,30 @@ pub(crate) struct WebCommandMetadata {
pub web_page_type: PageType, pub web_page_type: PageType,
} }
#[derive(Default, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct WatchEndpointConfigWrap {
pub watch_endpoint_music_config: WatchEndpointConfig,
}
#[serde_as]
#[derive(Default, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct WatchEndpointConfig {
#[serde(default)]
#[serde_as(deserialize_as = "DefaultOnError")]
pub music_video_type: MusicVideoType,
}
#[derive(Default, Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
pub(crate) enum MusicVideoType {
#[default]
#[serde(rename = "MUSIC_VIDEO_TYPE_OMV")]
Video,
#[serde(rename = "MUSIC_VIDEO_TYPE_ATV")]
Track,
}
#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] #[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
pub(crate) enum PageType { pub(crate) enum PageType {
#[serde( #[serde(
@ -152,7 +178,7 @@ pub(crate) enum MusicPageType {
Artist, Artist,
Album, Album,
Playlist, Playlist,
Track, Track { is_video: bool },
None, None,
} }
@ -189,7 +215,16 @@ impl NavigationEndpoint {
// Genre radios (e.g. "pop radio") will be skipped // Genre radios (e.g. "pop radio") will be skipped
(MusicPageType::None, watch.video_id) (MusicPageType::None, watch.video_id)
} else { } else {
(MusicPageType::Track, watch.video_id) (
MusicPageType::Track {
is_video: watch
.watch_endpoint_music_supported_configs
.watch_endpoint_music_config
.music_video_type
== MusicVideoType::Video,
},
watch.video_id,
)
} }
}) })
}) })

View file

@ -56,7 +56,7 @@ MusicAlbum(
name: "25", name: "25",
)), )),
view_count: None, view_count: None,
is_video: false, is_video: true,
track_nr: Some(1), track_nr: Some(1),
), ),
TrackItem( TrackItem(
@ -76,7 +76,7 @@ MusicAlbum(
name: "25", name: "25",
)), )),
view_count: None, view_count: None,
is_video: false, is_video: true,
track_nr: Some(2), track_nr: Some(2),
), ),
TrackItem( TrackItem(

View file

@ -64,7 +64,7 @@ MusicAlbum(
name: "Der Himmel reißt auf", name: "Der Himmel reißt auf",
)), )),
view_count: None, view_count: None,
is_video: false, is_video: true,
track_nr: Some(1), track_nr: Some(1),
), ),
], ],

View file

@ -51,7 +51,7 @@ MusicAlbum(
name: "Queendom2 FINAL", name: "Queendom2 FINAL",
)), )),
view_count: None, view_count: None,
is_video: false, is_video: true,
track_nr: Some(1), track_nr: Some(1),
), ),
TrackItem( TrackItem(

View file

@ -10,9 +10,6 @@ pub enum Error {
/// Error from the deobfuscater /// Error from the deobfuscater
#[error("deobfuscator error: {0}")] #[error("deobfuscator error: {0}")]
Deobfuscation(#[from] DeobfError), Deobfuscation(#[from] DeobfError),
/// Error from the video downloader
#[error("download error: {0}")]
Download(#[from] DownloadError),
/// File IO error /// File IO error
#[error(transparent)] #[error(transparent)]
Io(#[from] std::io::Error), Io(#[from] std::io::Error),
@ -45,26 +42,6 @@ pub enum DeobfError {
Other(&'static str), Other(&'static str),
} }
/// Error from the video downloader
#[derive(thiserror::Error, Debug)]
#[non_exhaustive]
pub enum DownloadError {
/// Error from the HTTP client
#[error("http error: {0}")]
Http(#[from] reqwest::Error),
/// File IO error
#[error(transparent)]
Io(#[from] std::io::Error),
#[error("FFmpeg error: {0}")]
Ffmpeg(Cow<'static, str>),
#[error("Progressive download error: {0}")]
Progressive(Cow<'static, str>),
#[error("input error: {0}")]
Input(Cow<'static, str>),
#[error("error: {0}")]
Other(Cow<'static, str>),
}
/// Error extracting content from YouTube /// Error extracting content from YouTube
#[derive(thiserror::Error, Debug)] #[derive(thiserror::Error, Debug)]
#[non_exhaustive] #[non_exhaustive]

View file

@ -14,7 +14,6 @@ mod util;
pub mod cache; pub mod cache;
pub mod client; pub mod client;
pub mod download;
pub mod error; pub mod error;
pub mod model; pub mod model;
pub mod param; pub mod param;

View file

@ -1031,7 +1031,7 @@ pub struct ArtistId {
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[non_exhaustive] #[non_exhaustive]
pub struct AlbumItem { pub struct AlbumItem {
/// Unique YouTube album ID (e.g. `OLAK5uy_nZpcQys48R0aNb046hV-n1OAHGE4reftQ`) /// Unique YouTube album ID (e.g. `MPREb_T5s950Swfdy`)
pub id: String, pub id: String,
/// Album name /// Album name
pub name: String, pub name: String,

View file

@ -122,7 +122,7 @@ mod tests {
text::TextComponent::Text { text: "🎧Listen and download aespa's debut single \"Black Mamba\": ".to_owned() }, text::TextComponent::Text { text: "🎧Listen and download aespa's debut single \"Black Mamba\": ".to_owned() },
text::TextComponent::Web { text: "https://smarturl.it/aespa_BlackMamba".to_owned(), url: "https://www.youtube.com/redirect?event=video_description&redir_token=QUFFLUhqbFY1QmpQamJPSms0Z1FnVTlQUS00ZFhBZnBJZ3xBQ3Jtc0tuRGJBanludGoyRnphb2dZWVd3cUNnS3dEd0FnNHFOZEY1NHBJaHFmLXpaWUJwX3ZucDZxVnpGeHNGX1FpMzFkZW9jQkI2Mi1wNGJ1UVFNN3h1MnN3R3JLMzdxU01nZ01POHBGcmxHU2puSUk1WHRzQQ&q=https%3A%2F%2Fsmarturl.it%2Faespa_BlackMamba&v=ZeerrnuLi5E".to_owned() }, text::TextComponent::Web { text: "https://smarturl.it/aespa_BlackMamba".to_owned(), url: "https://www.youtube.com/redirect?event=video_description&redir_token=QUFFLUhqbFY1QmpQamJPSms0Z1FnVTlQUS00ZFhBZnBJZ3xBQ3Jtc0tuRGJBanludGoyRnphb2dZWVd3cUNnS3dEd0FnNHFOZEY1NHBJaHFmLXpaWUJwX3ZucDZxVnpGeHNGX1FpMzFkZW9jQkI2Mi1wNGJ1UVFNN3h1MnN3R3JLMzdxU01nZ01POHBGcmxHU2puSUk1WHRzQQ&q=https%3A%2F%2Fsmarturl.it%2Faespa_BlackMamba&v=ZeerrnuLi5E".to_owned() },
text::TextComponent::Text { text: "\n🐍The Debut Stage ".to_owned() }, text::TextComponent::Text { text: "\n🐍The Debut Stage ".to_owned() },
text::TextComponent::Video { text: "https://youtu.be/Ky5RT5oGg0w".to_owned(), video_id: "Ky5RT5oGg0w".to_owned(), start_time: 0 }, text::TextComponent::Video { text: "https://youtu.be/Ky5RT5oGg0w".to_owned(), video_id: "Ky5RT5oGg0w".to_owned(), start_time: 0, is_video: true },
text::TextComponent::Text { text: "\n\n🎟️ aespa Showcase SYNK in LA! Tickets now on sale: ".to_owned() }, text::TextComponent::Text { text: "\n\n🎟️ aespa Showcase SYNK in LA! Tickets now on sale: ".to_owned() },
text::TextComponent::Web { text: "https://www.ticketmaster.com/event/0A...".to_owned(), url: "https://www.youtube.com/redirect?event=video_description&redir_token=QUFFLUhqbFpUMEZiaXJWWkszaVZXaEM0emxWU1JQV3NoQXxBQ3Jtc0tuU2g4VWNPNE5UY3hoSWYtamFzX0h4bUVQLVJiRy1ubDZrTnh3MUpGdDNSaUo0ZlMyT3lUM28ycUVBdHJLMndGcDhla3BkOFpxSVFfOS1QdVJPVHBUTEV1LXpOV0J2QXdhV05lV210cEJtZUJMeHdaTQ&q=https%3A%2F%2Fwww.ticketmaster.com%2Fevent%2F0A005CCD9E871F6E&v=ZeerrnuLi5E".to_owned() }, text::TextComponent::Web { text: "https://www.ticketmaster.com/event/0A...".to_owned(), url: "https://www.youtube.com/redirect?event=video_description&redir_token=QUFFLUhqbFpUMEZiaXJWWkszaVZXaEM0emxWU1JQV3NoQXxBQ3Jtc0tuU2g4VWNPNE5UY3hoSWYtamFzX0h4bUVQLVJiRy1ubDZrTnh3MUpGdDNSaUo0ZlMyT3lUM28ycUVBdHJLMndGcDhla3BkOFpxSVFfOS1QdVJPVHBUTEV1LXpOV0J2QXdhV05lV210cEJtZUJMeHdaTQ&q=https%3A%2F%2Fwww.ticketmaster.com%2Fevent%2F0A005CCD9E871F6E&v=ZeerrnuLi5E".to_owned() },
text::TextComponent::Text { text: "\n\nSubscribe to aespa Official YouTube Channel!\n".to_owned() }, text::TextComponent::Text { text: "\n\nSubscribe to aespa Official YouTube Channel!\n".to_owned() },

View file

@ -19,6 +19,7 @@ SAttributed {
text: "aespa 에스파 'Black ...", text: "aespa 에스파 'Black ...",
video_id: "Ky5RT5oGg0w", video_id: "Ky5RT5oGg0w",
start_time: 0, start_time: 0,
is_video: true,
}, },
Text { Text {
text: "\n\n🎟\u{fe0f} aespa Showcase SYNK in LA! Tickets now on sale: ", text: "\n\n🎟\u{fe0f} aespa Showcase SYNK in LA! Tickets now on sale: ",

View file

@ -6,7 +6,7 @@ use serde::{Deserialize, Deserializer};
use serde_with::{serde_as, DeserializeAs}; use serde_with::{serde_as, DeserializeAs};
use crate::{ use crate::{
client::response::url_endpoint::{NavigationEndpoint, PageType}, client::response::url_endpoint::{MusicVideoType, NavigationEndpoint, PageType},
model::UrlTarget, model::UrlTarget,
util, util,
}; };
@ -94,6 +94,8 @@ pub(crate) enum TextComponent {
text: String, text: String,
video_id: String, video_id: String,
start_time: u32, start_time: u32,
/// True if the item is a video, false if it is a YTM track
is_video: bool,
}, },
Browse { Browse {
text: String, text: String,
@ -164,6 +166,11 @@ fn map_text_component(text: String, nav: NavigationEndpoint) -> TextComponent {
text, text,
video_id: w.video_id, video_id: w.video_id,
start_time: w.start_time_seconds, start_time: w.start_time_seconds,
is_video: w
.watch_endpoint_music_supported_configs
.watch_endpoint_music_config
.music_video_type
== MusicVideoType::Video,
}, },
None => match nav.browse_endpoint { None => match nav.browse_endpoint {
Some(b) => TextComponent::Browse { Some(b) => TextComponent::Browse {
@ -365,6 +372,7 @@ impl From<TextComponent> for crate::model::richtext::TextComponent {
text, text,
video_id, video_id,
start_time, start_time,
..
} => Self::YouTube { } => Self::YouTube {
text, text,
target: UrlTarget::Video { target: UrlTarget::Video {
@ -581,6 +589,7 @@ mod tests {
text: "DEEP", text: "DEEP",
video_id: "wZIoIgz5mbs", video_id: "wZIoIgz5mbs",
start_time: 0, start_time: 0,
is_video: true,
}, },
} }
"###); "###);

View file

@ -39,7 +39,7 @@ MusicAlbum(
track_nr: Some(1), track_nr: Some(1),
), ),
TrackItem( TrackItem(
id: "lhPOMUjV4rE", id: "Jz-26iiDuYs",
title: "Waldbrand", title: "Waldbrand",
duration: Some(208), duration: Some(208),
cover: [], cover: [],

View file

@ -23,7 +23,7 @@ MusicAlbum(
by_va: false, by_va: false,
tracks: [ tracks: [
TrackItem( TrackItem(
id: "XX0epju-YvY", id: "VU6lEv0PKAo",
title: "Der Himmel reißt auf", title: "Der Himmel reißt auf",
duration: Some(183), duration: Some(183),
cover: [], cover: [],

View file

@ -14,7 +14,7 @@ MusicAlbum(
by_va: true, by_va: true,
tracks: [ tracks: [
TrackItem( TrackItem(
id: "8IqLxg0GqXc", id: "Tzai7JXo45w",
title: "Waka Boom (My Way) (feat. Lee Young Ji)", title: "Waka Boom (My Way) (feat. Lee Young Ji)",
duration: Some(274), duration: Some(274),
cover: [], cover: [],

View file

@ -792,12 +792,6 @@ async fn get_video_comments() {
let n_comments = top_comments.count.unwrap(); let n_comments = top_comments.count.unwrap();
assert_gte(n_comments, 700_000, "comments"); assert_gte(n_comments, 700_000, "comments");
// Comment count should be exact after fetching first page
assert!(
n_comments % 1000 != 0,
"estimated comment count: {}",
n_comments
);
let latest_comments = details let latest_comments = details
.latest_comments .latest_comments
@ -1600,6 +1594,8 @@ async fn music_search_videos() {
assert_next(res.items, rp.query(), 15, 2).await; assert_next(res.items, rp.query(), 15, 2).await;
} }
/*
This podcast was removed from YouTube Music and I could not find another one
#[tokio::test] #[tokio::test]
async fn music_search_episode() { async fn music_search_episode() {
let rp = RustyPipe::builder().strict().build(); let rp = RustyPipe::builder().strict().build();
@ -1624,6 +1620,7 @@ async fn music_search_episode() {
); );
assert!(!track.cover.is_empty(), "got no cover"); assert!(!track.cover.is_empty(), "got no cover");
} }
*/
#[rstest] #[rstest]
#[case::single( #[case::single(