diff --git a/Cargo.toml b/Cargo.toml index 1f2c729..ea67285 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ keywords = ["youtube", "video", "music"] include = ["/src", "README.md", "LICENSE", "!snapshots"] [workspace] -members = [".", "codegen", "downloader", "cli"] +members = [".", "codegen", "cli"] [features] default = ["default-tls"] @@ -34,8 +34,9 @@ reqwest = { version = "0.11.11", default-features = false, features = [ "json", "gzip", "brotli", + "stream", ] } -tokio = { version = "1.20.0", features = ["macros", "time"] } +tokio = { version = "1.20.0", features = ["macros", "time", "fs", "process"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.82" serde_with = { version = "2.0.0", features = ["json"] } @@ -46,6 +47,8 @@ time = { version = "0.3.15", features = [ "serde-well-known", ] } futures = "0.3.21" +indicatif = "0.17.0" +filenamify = "0.1.0" ress = "0.11.4" phf = "0.11.1" base64 = "0.13.0" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 53f895e..23b8ed6 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -4,16 +4,11 @@ version = "0.1.0" edition = "2021" [dependencies] -rustypipe = { path = "../", default_features = false, features = [ - "rustls-tls-native-roots", -] } -rustypipe-downloader = { path = "../downloader", default_features = false, features = [ - "rustls-tls-native-roots", -] } -reqwest = { version = "0.11.11", default_features = false } -tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] } +rustypipe = {path = "../", default_features = false, features = ["rustls-tls-native-roots"]} +reqwest = {version = "0.11.11", default_features = false} +tokio = {version = "1.20.0", features = ["macros", "rt-multi-thread"]} indicatif = "0.17.0" futures = "0.3.21" anyhow = "1.0" clap = { version = "3.2.16", features = ["derive"] } -env_logger = "0.10.0" +env_logger = "0.9.0" diff --git a/cli/src/main.rs b/cli/src/main.rs index f6a3dc3..0acac75 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -69,7 +69,7 @@ async fn download_single_video( } } - rustypipe_downloader::download_video( + rustypipe::download::download_video( &player_data, output_dir, output_fname, diff --git a/downloader/Cargo.toml b/downloader/Cargo.toml deleted file mode 100644 index 9f60764..0000000 --- a/downloader/Cargo.toml +++ /dev/null @@ -1,31 +0,0 @@ -[package] -name = "rustypipe-downloader" -version = "0.1.0" -edition = "2021" - -[features] -# Reqwest TLS -default-tls = ["reqwest/default-tls", "rustypipe/default-tls"] -rustls-tls-webpki-roots = [ - "reqwest/rustls-tls-webpki-roots", - "rustypipe/rustls-tls-webpki-roots", -] -rustls-tls-native-roots = [ - "reqwest/rustls-tls-native-roots", - "rustypipe/rustls-tls-native-roots", -] - -[dependencies] -rustypipe = { path = "..", default-features = false } -once_cell = "1.12.0" -regex = "1.6.0" -thiserror = "1.0.36" -futures = "0.3.21" -indicatif = "0.17.0" -filenamify = "0.1.0" -log = "0.4.17" -reqwest = { version = "0.11.11", default-features = false, features = [ - "stream", -] } -rand = "0.8.5" -tokio = { version = "1.20.0", features = ["macros", "fs", "process"] } diff --git a/downloader/src/util.rs b/downloader/src/util.rs deleted file mode 100644 index b6b6719..0000000 --- a/downloader/src/util.rs +++ /dev/null @@ -1,42 +0,0 @@ -use std::{borrow::Cow, collections::BTreeMap}; - -use reqwest::Url; - -/// Error from the video downloader -#[derive(thiserror::Error, Debug)] -#[non_exhaustive] -pub enum DownloadError { - /// Error from the HTTP client - #[error("http error: {0}")] - Http(#[from] reqwest::Error), - /// File IO error - #[error(transparent)] - Io(#[from] std::io::Error), - #[error("FFmpeg error: {0}")] - Ffmpeg(Cow<'static, str>), - #[error("Progressive download error: {0}")] - Progressive(Cow<'static, str>), - #[error("input error: {0}")] - Input(Cow<'static, str>), - #[error("error: {0}")] - Other(Cow<'static, str>), -} - -/// Split an URL into its base string and parameter map -/// -/// Example: -/// -/// `example.com/api?k1=v1&k2=v2 => example.com/api; {k1: v1, k2: v2}` -pub fn url_to_params(url: &str) -> Result<(Url, BTreeMap), DownloadError> { - let mut parsed_url = Url::parse(url).map_err(|e| { - DownloadError::Other(format!("could not parse url `{}` err: {}", url, e).into()) - })?; - let url_params: BTreeMap = parsed_url - .query_pairs() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(); - - parsed_url.set_query(None); - - Ok((parsed_url, url_params)) -} diff --git a/src/client/mod.rs b/src/client/mod.rs index d54b31b..3b82f4f 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -397,8 +397,8 @@ impl RustyPipeBuilder { /// /// **Default value**: `Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0` /// (Firefox ESR on Debian) - pub fn user_agent>(mut self, user_agent: S) -> Self { - self.user_agent = user_agent.into(); + pub fn user_agent(mut self, user_agent: &str) -> Self { + self.user_agent = user_agent.to_owned(); self } @@ -444,8 +444,8 @@ impl RustyPipeBuilder { } /// Set the default YouTube visitor data cookie - pub fn visitor_data>(mut self, visitor_data: S) -> Self { - self.default_opts.visitor_data = Some(visitor_data.into()); + pub fn visitor_data(mut self, visitor_data: &str) -> Self { + self.default_opts.visitor_data = Some(visitor_data.to_owned()); self } @@ -778,8 +778,8 @@ impl RustyPipeQuery { } /// Set the YouTube visitor data cookie - pub fn visitor_data>(mut self, visitor_data: S) -> Self { - self.opts.visitor_data = Some(visitor_data.into()); + pub fn visitor_data(mut self, visitor_data: &str) -> Self { + self.opts.visitor_data = Some(visitor_data.to_owned()); self } diff --git a/src/client/music_details.rs b/src/client/music_details.rs index cd4717c..1a48e67 100644 --- a/src/client/music_details.rs +++ b/src/client/music_details.rs @@ -99,10 +99,7 @@ impl RustyPipeQuery { radio_id: S, ) -> Result, Error> { let radio_id = radio_id.as_ref(); - let visitor_data = self.get_ytm_visitor_data().await?; - let context = self - .get_context(ClientType::DesktopMusic, true, Some(&visitor_data)) - .await; + let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QRadio { context, playlist_id: radio_id, diff --git a/src/client/music_playlist.rs b/src/client/music_playlist.rs index 6fdd6ef..e672975 100644 --- a/src/client/music_playlist.rs +++ b/src/client/music_playlist.rs @@ -2,7 +2,7 @@ use std::borrow::Cow; use crate::{ error::{Error, ExtractionError}, - model::{AlbumId, ChannelId, MusicAlbum, MusicPlaylist, Paginator, TrackItem}, + model::{AlbumId, ChannelId, MusicAlbum, MusicPlaylist, Paginator}, serializer::MapResult, util::{self, TryRemove}, }; @@ -45,55 +45,14 @@ impl RustyPipeQuery { browse_id: album_id, }; - let mut album = self - .execute_request::( - ClientType::DesktopMusic, - "music_album", - album_id, - "browse", - &request_body, - ) - .await?; - - // YouTube Music is replacing album tracks with their respective music videos. To get the original - // tracks, we have to fetch the album as a playlist and replace the offending track ids. - if let Some(playlist_id) = &album.playlist_id { - // Get a list of music videos in the album - let to_replace = album - .tracks - .iter() - .enumerate() - .filter_map(|(i, track)| { - if track.is_video { - Some((i, track.title.to_owned())) - } else { - None - } - }) - .collect::>(); - - if !to_replace.is_empty() { - let playlist = self.music_playlist(playlist_id).await?; - - for (i, title) in to_replace { - let found_track = playlist.tracks.items.iter().find_map(|track| { - if track.title == title && !track.is_video { - Some((track.id.to_owned(), track.duration)) - } else { - None - } - }); - if let Some((track_id, duration)) = found_track { - album.tracks[i].id = track_id; - if let Some(duration) = duration { - album.tracks[i].duration = Some(duration); - } - album.tracks[i].is_video = false; - } - } - } - } - Ok(album) + self.execute_request::( + ClientType::DesktopMusic, + "music_album", + album_id, + "browse", + &request_body, + ) + .await } } @@ -106,6 +65,8 @@ impl MapResponse for response::MusicPlaylist { ) -> Result, ExtractionError> { // dbg!(&self); + let header = self.header.music_detail_header_renderer; + let mut content = self.contents.single_column_browse_results_renderer.contents; let mut music_contents = content .try_swap_remove(0) @@ -124,15 +85,31 @@ impl MapResponse for response::MusicPlaylist { "no sectionListRenderer content", )))?; - if let Some(playlist_id) = shelf.playlist_id { - if playlist_id != id { - return Err(ExtractionError::WrongResult(format!( - "got wrong playlist id {}, expected {}", - playlist_id, id - ))); - } + let playlist_id = shelf + .playlist_id + .ok_or(ExtractionError::InvalidData(Cow::Borrowed( + "no playlist id", + )))?; + + if playlist_id != id { + return Err(ExtractionError::WrongResult(format!( + "got wrong playlist id {}, expected {}", + playlist_id, id + ))); } + let from_ytm = header + .subtitle + .0 + .iter() + .any(|c| c.as_str() == util::YT_MUSIC_NAME); + + let channel = header + .subtitle + .0 + .into_iter() + .find_map(|c| ChannelId::try_from(c).ok()); + let mut mapper = MusicListMapper::new(lang); mapper.map_response(shelf.contents); let map_res = mapper.conv_items(); @@ -143,12 +120,10 @@ impl MapResponse for response::MusicPlaylist { .map(|cont| cont.next_continuation_data.continuation); let track_count = match ctoken { - Some(_) => self.header.as_ref().and_then(|h| { - h.music_detail_header_renderer - .second_subtitle - .first() - .and_then(|txt| util::parse_numeric::(txt).ok()) - }), + Some(_) => header + .second_subtitle + .first() + .and_then(|txt| util::parse_numeric::(txt).ok()), None => Some(map_res.c.len() as u64), }; @@ -157,63 +132,13 @@ impl MapResponse for response::MusicPlaylist { .try_swap_remove(0) .map(|c| c.next_continuation_data.continuation); - let (from_ytm, channel, name, thumbnail, description) = match self.header { - Some(header) => { - let h = header.music_detail_header_renderer; - - let from_ytm = h - .subtitle - .0 - .iter() - .any(|c| c.as_str() == util::YT_MUSIC_NAME); - let channel = h - .subtitle - .0 - .into_iter() - .find_map(|c| ChannelId::try_from(c).ok()); - - ( - from_ytm, - channel, - h.title, - h.thumbnail.into(), - h.description, - ) - } - None => { - // Album playlists fetched via the playlist method dont include a header - let (album, cover) = map_res - .c - .first() - .and_then(|t: &TrackItem| { - t.album.as_ref().map(|a| (a.clone(), t.cover.clone())) - }) - .ok_or(ExtractionError::InvalidData(Cow::Borrowed( - "playlist without header or album items", - )))?; - - if !map_res.c.iter().all(|t| { - t.album - .as_ref() - .map(|a| a.id == album.id) - .unwrap_or_default() - }) { - return Err(ExtractionError::InvalidData(Cow::Borrowed( - "album playlist containing items from different albums", - ))); - } - - (true, None, album.name, cover, None) - } - }; - Ok(MapResult { c: MusicPlaylist { - id: id.to_owned(), - name, - thumbnail, + id: playlist_id, + name: header.title, + thumbnail: header.thumbnail.into(), channel, - description, + description: header.description, track_count, from_ytm, tracks: Paginator::new_ext( @@ -245,10 +170,7 @@ impl MapResponse for response::MusicPlaylist { ) -> Result, ExtractionError> { // dbg!(&self); - let header = self - .header - .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no header")))? - .music_detail_header_renderer; + let header = self.header.music_detail_header_renderer; let mut content = self.contents.single_column_browse_results_renderer.contents; let sections = content diff --git a/src/client/response/music_item.rs b/src/client/response/music_item.rs index d29ebd2..c505b02 100644 --- a/src/client/response/music_item.rs +++ b/src/client/response/music_item.rs @@ -418,12 +418,10 @@ impl MusicListMapper { // List item MusicResponseItem::MusicResponsiveListItemRenderer(item) => { let mut columns = item.flex_columns.into_iter(); - let c1 = columns.next(); + let title = columns.next().map(|col| col.renderer.text.to_string()); let c2 = columns.next(); let c3 = columns.next(); - let title = c1.as_ref().map(|col| col.renderer.text.to_string()); - let first_tn = item .thumbnail .music_thumbnail_renderer @@ -435,54 +433,27 @@ impl MusicListMapper { .navigation_endpoint .and_then(|ne| ne.music_page()) .or_else(|| { - c1.and_then(|c1| { - c1.renderer.text.0.into_iter().next().and_then(|t| match t { - crate::serializer::text::TextComponent::Video { - video_id, - is_video, - .. - } => Some((MusicPageType::Track { is_video }, video_id)), - crate::serializer::text::TextComponent::Browse { - page_type, - browse_id, - .. - } => Some((page_type.into(), browse_id)), - _ => None, - }) - }) - }) - .or_else(|| { - item.playlist_item_data.map(|d| { - ( - MusicPageType::Track { - is_video: self.album.is_none() - && !first_tn - .map(|tn| tn.height == tn.width) - .unwrap_or_default(), - }, - d.video_id, - ) - }) + item.playlist_item_data + .map(|d| (MusicPageType::Track, d.video_id)) }) .or_else(|| { first_tn.and_then(|tn| { - util::video_id_from_thumbnail_url(&tn.url).map(|id| { - ( - MusicPageType::Track { - is_video: self.album.is_none() && tn.width != tn.height, - }, - id, - ) - }) + util::video_id_from_thumbnail_url(&tn.url) + .map(|id| (MusicPageType::Track, id)) }) }); match pt_id { // Track - Some((MusicPageType::Track { is_video }, id)) => { + Some((MusicPageType::Track, id)) => { let title = title.ok_or_else(|| format!("track {}: could not get title", id))?; + // Videos have rectangular thumbnails, YTM tracks have square covers + // Exception: there are no thumbnails on album items + let is_video = self.album.is_none() + && !first_tn.map(|tn| tn.height == tn.width).unwrap_or_default(); + let (artists_p, album_p, duration_p) = match item.flex_column_display_style { // Search result @@ -548,14 +519,15 @@ impl MusicListMapper { }), ), (_, false) => ( - album_p.and_then(|p| { - p.0.into_iter().find_map(|c| AlbumId::try_from(c).ok()) - }), + album_p + .and_then(|p| { + p.0.into_iter().find_map(|c| AlbumId::try_from(c).ok()) + }) + .or_else(|| self.album.clone()), None, ), (FlexColumnDisplayStyle::Default, true) => (None, None), }; - let album = album.or_else(|| self.album.clone()); let (mut artists, _) = map_artists(artists_p); @@ -668,8 +640,7 @@ impl MusicListMapper { // There may be broken YT channels from the artist search. They can be skipped. Ok(None) } - // Tracks were already handled above - MusicPageType::Track { .. } => unreachable!(), + MusicPageType::Track => unreachable!(), } } None => Err("could not determine item type".to_owned()), @@ -684,7 +655,7 @@ impl MusicListMapper { match item.navigation_endpoint.music_page() { Some((page_type, id)) => match page_type { - MusicPageType::Track { is_video } => { + MusicPageType::Track => { let artists = map_artists(subtitle_p1).0; self.items.push(MusicItem::Track(TrackItem { @@ -698,7 +669,7 @@ impl MusicListMapper { view_count: subtitle_p2.and_then(|c| { util::parse_large_numstr(c.first_str(), self.lang) }), - is_video, + is_video: true, track_nr: None, })); Ok(Some(MusicEntityType::Track)) diff --git a/src/client/response/music_playlist.rs b/src/client/response/music_playlist.rs index 6b8aa49..6e2fb2e 100644 --- a/src/client/response/music_playlist.rs +++ b/src/client/response/music_playlist.rs @@ -11,7 +11,7 @@ use super::{ContentsRenderer, Tab}; #[serde(rename_all = "camelCase")] pub(crate) struct MusicPlaylist { pub contents: Contents, - pub header: Option
, + pub header: Header, } #[derive(Debug, Deserialize)] diff --git a/src/client/response/url_endpoint.rs b/src/client/response/url_endpoint.rs index 03d9043..09713e1 100644 --- a/src/client/response/url_endpoint.rs +++ b/src/client/response/url_endpoint.rs @@ -35,8 +35,6 @@ pub(crate) struct WatchEndpoint { pub playlist_id: Option, #[serde(default)] pub start_time_seconds: u32, - #[serde(default)] - pub watch_endpoint_music_supported_configs: WatchEndpointConfigWrap, } #[derive(Debug)] @@ -120,30 +118,6 @@ pub(crate) struct WebCommandMetadata { pub web_page_type: PageType, } -#[derive(Default, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct WatchEndpointConfigWrap { - pub watch_endpoint_music_config: WatchEndpointConfig, -} - -#[serde_as] -#[derive(Default, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct WatchEndpointConfig { - #[serde(default)] - #[serde_as(deserialize_as = "DefaultOnError")] - pub music_video_type: MusicVideoType, -} - -#[derive(Default, Debug, Clone, Copy, Deserialize, PartialEq, Eq)] -pub(crate) enum MusicVideoType { - #[default] - #[serde(rename = "MUSIC_VIDEO_TYPE_OMV")] - Video, - #[serde(rename = "MUSIC_VIDEO_TYPE_ATV")] - Track, -} - #[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] pub(crate) enum PageType { #[serde( @@ -178,7 +152,7 @@ pub(crate) enum MusicPageType { Artist, Album, Playlist, - Track { is_video: bool }, + Track, None, } @@ -215,16 +189,7 @@ impl NavigationEndpoint { // Genre radios (e.g. "pop radio") will be skipped (MusicPageType::None, watch.video_id) } else { - ( - MusicPageType::Track { - is_video: watch - .watch_endpoint_music_supported_configs - .watch_endpoint_music_config - .music_video_type - == MusicVideoType::Video, - }, - watch.video_id, - ) + (MusicPageType::Track, watch.video_id) } }) }) diff --git a/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_description.snap b/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_description.snap index 19a4cd1..8f2bb6c 100644 --- a/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_description.snap +++ b/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_description.snap @@ -56,7 +56,7 @@ MusicAlbum( name: "25", )), view_count: None, - is_video: true, + is_video: false, track_nr: Some(1), ), TrackItem( @@ -76,7 +76,7 @@ MusicAlbum( name: "25", )), view_count: None, - is_video: true, + is_video: false, track_nr: Some(2), ), TrackItem( diff --git a/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_single.snap b/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_single.snap index 3e86ad8..fa429ce 100644 --- a/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_single.snap +++ b/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_single.snap @@ -64,7 +64,7 @@ MusicAlbum( name: "Der Himmel reißt auf", )), view_count: None, - is_video: true, + is_video: false, track_nr: Some(1), ), ], diff --git a/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_various_artists.snap b/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_various_artists.snap index 78bfbc7..83f0652 100644 --- a/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_various_artists.snap +++ b/src/client/snapshots/rustypipe__client__music_playlist__tests__map_music_album_various_artists.snap @@ -51,7 +51,7 @@ MusicAlbum( name: "＜Queendom2＞ FINAL", )), view_count: None, - is_video: true, + is_video: false, track_nr: Some(1), ), TrackItem( diff --git a/downloader/src/lib.rs b/src/download.rs similarity index 91% rename from downloader/src/lib.rs rename to src/download.rs index 34adc42..238e9ab 100644 --- a/downloader/src/lib.rs +++ b/src/download.rs @@ -1,27 +1,26 @@ -//! # YouTube audio/video downloader - -mod util; +//! YouTube audio/video downloader use std::{borrow::Cow, cmp::Ordering, ffi::OsString, ops::Range, path::PathBuf, time::Duration}; +use fancy_regex::Regex; use futures::stream::{self, StreamExt}; use indicatif::{ProgressBar, ProgressStyle}; use log::{debug, info}; use once_cell::sync::Lazy; use rand::Rng; -use regex::Regex; use reqwest::{header, Client}; -use rustypipe::{ - model::{AudioCodec, FileFormat, VideoCodec, VideoPlayer}, - param::StreamFilter, -}; use tokio::{ fs::{self, File}, io::AsyncWriteExt, process::Command, }; -use util::DownloadError; +use crate::{ + error::DownloadError, + model::{AudioCodec, FileFormat, VideoCodec, VideoPlayer}, + param::StreamFilter, + util, +}; type Result = core::result::Result; @@ -46,7 +45,7 @@ fn get_download_range(offset: u64, size: Option) -> Range { fn parse_cr_header(cr_header: &str) -> Result<(u64, u64)> { static PATTERN: Lazy = Lazy::new(|| Regex::new(r#"bytes (\d+)-(\d+)/(\d+)"#).unwrap()); - let captures = PATTERN.captures(cr_header).ok_or_else(|| { + let captures = PATTERN.captures(cr_header).ok().flatten().ok_or_else(|| { DownloadError::Progressive( format!( "Content-Range header '{}' does not match pattern", @@ -318,9 +317,11 @@ pub async fn download_video( Some(_) => "mp4", None => match audio { Some(audio) => match audio.codec { + AudioCodec::Unknown => { + return Err(DownloadError::Input("unknown audio codec".into())) + } AudioCodec::Mp4a => "m4a", AudioCodec::Opus => "opus", - _ => return Err(DownloadError::Input("unknown audio codec".into())), }, None => unreachable!(), }, @@ -472,3 +473,40 @@ async fn convert_streams>( } Ok(()) } + +/* +#[cfg(test)] +mod tests { + use crate::client::RustyTube; + + use super::*; + use indicatif::{ProgressDrawTarget, ProgressStyle}; + use reqwest::ClientBuilder; + + // #[test_log::test(tokio::test)] + #[tokio::test] + async fn t_download_video() { + let http = ClientBuilder::new() + .user_agent( + "Mozilla/5.0 (Windows NT 10.0; Win64; rv:107.0) Gecko/20100101 Firefox/107.0", + ) + .gzip(true) + .brotli(true) + .build() + .expect("unable to build the HTTP client"); + + // Indicatif setup + let pb = ProgressBar::new(0); + + let rt = RustyTube::new(); + let player_data = rt + .get_player("AbZH7XWDW_k", crate::client::ClientType::Desktop) + .await + .unwrap(); + + // download_video(&player_data, "tmp", "INVU", Some(1080), "ffmpeg", http, pb) + // .await + // .unwrap(); + } +} +*/ diff --git a/src/error.rs b/src/error.rs index 1ce1641..ced5358 100644 --- a/src/error.rs +++ b/src/error.rs @@ -10,6 +10,9 @@ pub enum Error { /// Error from the deobfuscater #[error("deobfuscator error: {0}")] Deobfuscation(#[from] DeobfError), + /// Error from the video downloader + #[error("download error: {0}")] + Download(#[from] DownloadError), /// File IO error #[error(transparent)] Io(#[from] std::io::Error), @@ -42,6 +45,26 @@ pub enum DeobfError { Other(&'static str), } +/// Error from the video downloader +#[derive(thiserror::Error, Debug)] +#[non_exhaustive] +pub enum DownloadError { + /// Error from the HTTP client + #[error("http error: {0}")] + Http(#[from] reqwest::Error), + /// File IO error + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("FFmpeg error: {0}")] + Ffmpeg(Cow<'static, str>), + #[error("Progressive download error: {0}")] + Progressive(Cow<'static, str>), + #[error("input error: {0}")] + Input(Cow<'static, str>), + #[error("error: {0}")] + Other(Cow<'static, str>), +} + /// Error extracting content from YouTube #[derive(thiserror::Error, Debug)] #[non_exhaustive] diff --git a/src/lib.rs b/src/lib.rs index f406da5..f06ca4d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,7 @@ mod util; pub mod cache; pub mod client; +pub mod download; pub mod error; pub mod model; pub mod param; diff --git a/src/model/mod.rs b/src/model/mod.rs index 2c8fbfd..a037b19 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -1031,7 +1031,7 @@ pub struct ArtistId { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[non_exhaustive] pub struct AlbumItem { - /// Unique YouTube album ID (e.g. `MPREb_T5s950Swfdy`) + /// Unique YouTube album ID (e.g. `OLAK5uy_nZpcQys48R0aNb046hV-n1OAHGE4reftQ`) pub id: String, /// Album name pub name: String, diff --git a/src/model/richtext.rs b/src/model/richtext.rs index 1947e48..c4609be 100644 --- a/src/model/richtext.rs +++ b/src/model/richtext.rs @@ -122,7 +122,7 @@ mod tests { text::TextComponent::Text { text: "🎧Listen and download aespa's debut single \"Black Mamba\": ".to_owned() }, text::TextComponent::Web { text: "https://smarturl.it/aespa_BlackMamba".to_owned(), url: "https://www.youtube.com/redirect?event=video_description&redir_token=QUFFLUhqbFY1QmpQamJPSms0Z1FnVTlQUS00ZFhBZnBJZ3xBQ3Jtc0tuRGJBanludGoyRnphb2dZWVd3cUNnS3dEd0FnNHFOZEY1NHBJaHFmLXpaWUJwX3ZucDZxVnpGeHNGX1FpMzFkZW9jQkI2Mi1wNGJ1UVFNN3h1MnN3R3JLMzdxU01nZ01POHBGcmxHU2puSUk1WHRzQQ&q=https%3A%2F%2Fsmarturl.it%2Faespa_BlackMamba&v=ZeerrnuLi5E".to_owned() }, text::TextComponent::Text { text: "\n🐍The Debut Stage ".to_owned() }, - text::TextComponent::Video { text: "https://youtu.be/Ky5RT5oGg0w".to_owned(), video_id: "Ky5RT5oGg0w".to_owned(), start_time: 0, is_video: true }, + text::TextComponent::Video { text: "https://youtu.be/Ky5RT5oGg0w".to_owned(), video_id: "Ky5RT5oGg0w".to_owned(), start_time: 0 }, text::TextComponent::Text { text: "\n\n🎟️ aespa Showcase SYNK in LA! Tickets now on sale: ".to_owned() }, text::TextComponent::Web { text: "https://www.ticketmaster.com/event/0A...".to_owned(), url: "https://www.youtube.com/redirect?event=video_description&redir_token=QUFFLUhqbFpUMEZiaXJWWkszaVZXaEM0emxWU1JQV3NoQXxBQ3Jtc0tuU2g4VWNPNE5UY3hoSWYtamFzX0h4bUVQLVJiRy1ubDZrTnh3MUpGdDNSaUo0ZlMyT3lUM28ycUVBdHJLMndGcDhla3BkOFpxSVFfOS1QdVJPVHBUTEV1LXpOV0J2QXdhV05lV210cEJtZUJMeHdaTQ&q=https%3A%2F%2Fwww.ticketmaster.com%2Fevent%2F0A005CCD9E871F6E&v=ZeerrnuLi5E".to_owned() }, text::TextComponent::Text { text: "\n\nSubscribe to aespa Official YouTube Channel!\n".to_owned() }, diff --git a/src/serializer/snapshots/rustypipe__serializer__text__tests__t_attributed_description.snap b/src/serializer/snapshots/rustypipe__serializer__text__tests__t_attributed_description.snap index e937bcc..8ba3dfa 100644 --- a/src/serializer/snapshots/rustypipe__serializer__text__tests__t_attributed_description.snap +++ b/src/serializer/snapshots/rustypipe__serializer__text__tests__t_attributed_description.snap @@ -19,7 +19,6 @@ SAttributed { text: "aespa 에스파 'Black ...", video_id: "Ky5RT5oGg0w", start_time: 0, - is_video: true, }, Text { text: "\n\n🎟\u{fe0f} aespa Showcase SYNK in LA! Tickets now on sale: ", diff --git a/src/serializer/text.rs b/src/serializer/text.rs index abe8c80..587daf0 100644 --- a/src/serializer/text.rs +++ b/src/serializer/text.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Deserializer}; use serde_with::{serde_as, DeserializeAs}; use crate::{ - client::response::url_endpoint::{MusicVideoType, NavigationEndpoint, PageType}, + client::response::url_endpoint::{NavigationEndpoint, PageType}, model::UrlTarget, util, }; @@ -94,8 +94,6 @@ pub(crate) enum TextComponent { text: String, video_id: String, start_time: u32, - /// True if the item is a video, false if it is a YTM track - is_video: bool, }, Browse { text: String, @@ -166,11 +164,6 @@ fn map_text_component(text: String, nav: NavigationEndpoint) -> TextComponent { text, video_id: w.video_id, start_time: w.start_time_seconds, - is_video: w - .watch_endpoint_music_supported_configs - .watch_endpoint_music_config - .music_video_type - == MusicVideoType::Video, }, None => match nav.browse_endpoint { Some(b) => TextComponent::Browse { @@ -372,7 +365,6 @@ impl From for crate::model::richtext::TextComponent { text, video_id, start_time, - .. } => Self::YouTube { text, target: UrlTarget::Video { @@ -589,7 +581,6 @@ mod tests { text: "DEEP", video_id: "wZIoIgz5mbs", start_time: 0, - is_video: true, }, } "###); diff --git a/tests/snapshots/youtube__music_album_ep.snap b/tests/snapshots/youtube__music_album_ep.snap index 26256f6..dd465da 100644 --- a/tests/snapshots/youtube__music_album_ep.snap +++ b/tests/snapshots/youtube__music_album_ep.snap @@ -39,7 +39,7 @@ MusicAlbum( track_nr: Some(1), ), TrackItem( - id: "Jz-26iiDuYs", + id: "lhPOMUjV4rE", title: "Waldbrand", duration: Some(208), cover: [], diff --git a/tests/snapshots/youtube__music_album_single.snap b/tests/snapshots/youtube__music_album_single.snap index aeba890..befe8c2 100644 --- a/tests/snapshots/youtube__music_album_single.snap +++ b/tests/snapshots/youtube__music_album_single.snap @@ -23,7 +23,7 @@ MusicAlbum( by_va: false, tracks: [ TrackItem( - id: "VU6lEv0PKAo", + id: "XX0epju-YvY", title: "Der Himmel reißt auf", duration: Some(183), cover: [], diff --git a/tests/snapshots/youtube__music_album_various_artists.snap b/tests/snapshots/youtube__music_album_various_artists.snap index b04f5ea..8da3db0 100644 --- a/tests/snapshots/youtube__music_album_various_artists.snap +++ b/tests/snapshots/youtube__music_album_various_artists.snap @@ -14,7 +14,7 @@ MusicAlbum( by_va: true, tracks: [ TrackItem( - id: "Tzai7JXo45w", + id: "8IqLxg0GqXc", title: "Waka Boom (My Way) (feat. Lee Young Ji)", duration: Some(274), cover: [], diff --git a/tests/youtube.rs b/tests/youtube.rs index c700aab..bd48b4a 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -792,6 +792,12 @@ async fn get_video_comments() { let n_comments = top_comments.count.unwrap(); assert_gte(n_comments, 700_000, "comments"); + // Comment count should be exact after fetching first page + assert!( + n_comments % 1000 != 0, + "estimated comment count: {}", + n_comments + ); let latest_comments = details .latest_comments @@ -1594,8 +1600,6 @@ async fn music_search_videos() { assert_next(res.items, rp.query(), 15, 2).await; } -/* -This podcast was removed from YouTube Music and I could not find another one #[tokio::test] async fn music_search_episode() { let rp = RustyPipe::builder().strict().build(); @@ -1620,7 +1624,6 @@ async fn music_search_episode() { ); assert!(!track.cover.is_empty(), "got no cover"); } -*/ #[rstest] #[case::single(