Compare commits

..

No commits in common. "2e06d9c572fd0c99515b59c897f439698dbb07e5" and "3458924018c6b1d943f20e1fc0a3f22e62f4b483" have entirely different histories.

5 changed files with 93 additions and 535 deletions

View file

@ -4,8 +4,12 @@ version = "0.1.0"
edition = "2021" edition = "2021"
[dependencies] [dependencies]
rustypipe = { path = "../" } rustypipe = { path = "../", default_features = false, features = [
rustypipe-downloader = { path = "../downloader" } "rustls-tls-native-roots",
] }
rustypipe-downloader = { path = "../downloader", default_features = false, features = [
"rustls-tls-native-roots",
] }
reqwest = { version = "0.11.11", default_features = false } reqwest = { version = "0.11.11", default_features = false }
tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] } tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] }
indicatif = "0.17.0" indicatif = "0.17.0"
@ -13,6 +17,3 @@ futures = "0.3.21"
anyhow = "1.0" anyhow = "1.0"
clap = { version = "4.0.29", features = ["derive"] } clap = { version = "4.0.29", features = ["derive"] }
env_logger = "0.10.0" env_logger = "0.10.0"
serde = "1.0"
serde_json = "1.0.82"
serde_yaml = "0.9.19"

View file

@ -1,220 +1,45 @@
use std::path::PathBuf; use std::path::PathBuf;
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use clap::{Parser, Subcommand, ValueEnum}; use clap::{Parser, Subcommand};
use futures::stream::{self, StreamExt}; use futures::stream::{self, StreamExt};
use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
use reqwest::{Client, ClientBuilder}; use reqwest::{Client, ClientBuilder};
use rustypipe::{ use rustypipe::{client::RustyPipe, param::StreamFilter};
client::RustyPipe,
model::{UrlTarget, VideoId},
param::{search_filter, StreamFilter},
};
use serde::Serialize;
#[derive(Parser)] #[derive(Parser)]
#[clap(author, version, about, long_about = None)] #[clap(author, version, about, long_about = None)]
struct Cli { struct Cli {
#[clap(subcommand)] #[clap(subcommand)]
command: Commands, command: Commands,
#[clap(short, value_parser, default_value = ".", global = true)]
output: PathBuf,
#[clap(long, value_parser, global = true)]
resolution: Option<u32>,
#[clap(short, long, value_parser, default_value = "8", global = true)]
parallel: usize,
} }
#[derive(Subcommand)] #[derive(Subcommand)]
enum Commands { enum Commands {
/// Download a video, playlist, album or channel /// Download a playlist
#[clap(alias = "dl")] Playlist {
Download { /// Playlist ID
/// ID or URL #[clap(value_parser)]
id: String, id: String,
/// Output path
#[clap(short, default_value = ".")]
output: PathBuf,
/// Video resolution (e.g. 720, 1080). Set to 0 for audio-only.
#[clap(short, long)]
resolution: Option<u32>,
/// Number of videos downloaded in parallel
#[clap(short, long, default_value_t = 8)]
parallel: usize,
/// Limit the number of videos to download
#[clap(long, default_value_t = 1000)]
limit: usize,
}, },
/// Extract video, playlist, album or channel data /// Download a video
Get { Video {
/// ID or URL /// Video ID
#[clap(value_parser)]
id: String, id: String,
/// Output format
#[clap(long, value_parser, default_value = "json")]
format: Format,
/// Pretty-print output
#[clap(long)]
pretty: bool,
/// Limit the number of items to fetch
#[clap(long, default_value_t = 20)]
limit: usize,
/// Channel tab
#[clap(long, default_value = "videos")]
tab: ChannelTab,
/// Use YouTube Music
#[clap(long)]
music: bool,
/// Get comments
#[clap(long)]
comments: Option<CommentsOrder>,
/// Get lyrics
#[clap(long)]
lyrics: bool,
}, },
/// Search YouTube
Search {
/// Search query
query: String,
/// Output format
#[clap(long, value_parser, default_value = "json")]
format: Format,
/// Pretty-print output
#[clap(long)]
pretty: bool,
/// Limit the number of items to fetch
#[clap(long, default_value_t = 20)]
limit: usize,
/// Filter results by item type
#[clap(long)]
item_type: Option<SearchItemType>,
/// Filter results by video length
#[clap(long)]
length: Option<SearchLength>,
/// Filter results by upload date
#[clap(long)]
date: Option<SearchUploadDate>,
/// Sort search resulus
#[clap(long)]
order: Option<SearchOrder>,
/// YouTube Music search filter
#[clap(long)]
music: Option<MusicSearchCategory>,
},
}
#[derive(Copy, Clone, ValueEnum)]
enum Format {
Json,
Yaml,
}
#[derive(Copy, Clone, ValueEnum)]
enum ChannelTab {
Videos,
Shorts,
Live,
Info,
}
#[derive(Copy, Clone, ValueEnum)]
enum CommentsOrder {
Top,
Latest,
}
#[derive(Copy, Clone, ValueEnum)]
enum SearchItemType {
Video,
Channel,
Playlist,
}
#[derive(Copy, Clone, ValueEnum)]
enum SearchLength {
/// < 4min
Short,
/// 4-20min
Medium,
/// > 20min
Long,
}
#[derive(Copy, Clone, ValueEnum)]
enum SearchUploadDate {
/// 1 hour old or newer
Hour,
/// 1 day old or newer
Day,
/// 1 week old or newer
Week,
/// 1 month old or newer
Month,
/// 1 year old or newer
Year,
}
#[derive(Copy, Clone, ValueEnum)]
enum SearchOrder {
/// Sort by Like/Dislike ratio
Rating,
/// Sort by upload date
Date,
/// Sort by view count
Views,
}
#[derive(Copy, Clone, ValueEnum)]
enum MusicSearchCategory {
All,
Tracks,
Videos,
Artists,
Albums,
Playlists,
PlaylistsYtm,
PlaylistsCommunity,
}
impl From<SearchItemType> for search_filter::ItemType {
fn from(value: SearchItemType) -> Self {
match value {
SearchItemType::Video => search_filter::ItemType::Video,
SearchItemType::Channel => search_filter::ItemType::Channel,
SearchItemType::Playlist => search_filter::ItemType::Playlist,
}
}
}
impl From<SearchLength> for search_filter::Length {
fn from(value: SearchLength) -> Self {
match value {
SearchLength::Short => search_filter::Length::Short,
SearchLength::Medium => search_filter::Length::Medium,
SearchLength::Long => search_filter::Length::Long,
}
}
}
impl From<SearchUploadDate> for search_filter::UploadDate {
fn from(value: SearchUploadDate) -> Self {
match value {
SearchUploadDate::Hour => search_filter::UploadDate::Hour,
SearchUploadDate::Day => search_filter::UploadDate::Day,
SearchUploadDate::Week => search_filter::UploadDate::Week,
SearchUploadDate::Month => search_filter::UploadDate::Month,
SearchUploadDate::Year => search_filter::UploadDate::Year,
}
}
}
impl From<SearchOrder> for search_filter::Order {
fn from(value: SearchOrder) -> Self {
match value {
SearchOrder::Rating => search_filter::Order::Rating,
SearchOrder::Date => search_filter::Order::Date,
SearchOrder::Views => search_filter::Order::Views,
}
}
} }
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
async fn download_single_video( async fn download_single_video(
video_id: &str, video_id: String,
video_title: &str, video_title: String,
output_dir: &str, output_dir: &str,
output_fname: Option<String>, output_fname: Option<String>,
resolution: Option<u32>, resolution: Option<u32>,
@ -232,7 +57,7 @@ async fn download_single_video(
let res = async { let res = async {
let player_data = rp let player_data = rp
.query() .query()
.player(video_id) .player(video_id.as_str())
.await .await
.context(format!("Failed to fetch player data for video {video_id}"))?; .context(format!("Failed to fetch player data for video {video_id}"))?;
@ -269,22 +94,7 @@ async fn download_single_video(
res res
} }
fn print_data<T: Serialize>(data: &T, format: Format, pretty: bool) {
let stdout = std::io::stdout().lock();
match format {
Format::Json => {
if pretty {
serde_json::to_writer_pretty(stdout, data).unwrap()
} else {
serde_json::to_writer(stdout, data).unwrap()
}
}
Format::Yaml => serde_yaml::to_writer(stdout, data).unwrap(),
};
}
async fn download_video( async fn download_video(
rp: &RustyPipe,
id: &str, id: &str,
output_dir: &str, output_dir: &str,
output_fname: Option<String>, output_fname: Option<String>,
@ -297,17 +107,19 @@ async fn download_video(
.build() .build()
.expect("unable to build the HTTP client"); .expect("unable to build the HTTP client");
let rp = RustyPipe::default();
// Indicatif setup // Indicatif setup
let multi = MultiProgress::new(); let multi = MultiProgress::new();
download_single_video( download_single_video(
id, id.to_owned(),
id, id.to_owned(),
output_dir, output_dir,
output_fname, output_fname,
resolution, resolution,
"ffmpeg", "ffmpeg",
rp, &rp,
http, http,
multi, multi,
None, None,
@ -316,9 +128,8 @@ async fn download_video(
.unwrap_or_else(|e| println!("ERROR: {e:?}")); .unwrap_or_else(|e| println!("ERROR: {e:?}"));
} }
async fn download_videos( async fn download_playlist(
rp: &RustyPipe, id: &str,
videos: &[VideoId],
output_dir: &str, output_dir: &str,
output_fname: Option<String>, output_fname: Option<String>,
resolution: Option<u32>, resolution: Option<u32>,
@ -331,10 +142,18 @@ async fn download_videos(
.build() .build()
.expect("unable to build the HTTP client"); .expect("unable to build the HTTP client");
let rp = RustyPipe::default();
let mut playlist = rp.query().playlist(id).await.unwrap();
playlist
.videos
.extend_pages(&rp.query(), usize::MAX)
.await
.unwrap();
// Indicatif setup // Indicatif setup
let multi = MultiProgress::new(); let multi = MultiProgress::new();
let main = multi.add(ProgressBar::new( let main = multi.add(ProgressBar::new(
videos.len().try_into().unwrap_or_default(), playlist.videos.items.len().try_into().unwrap_or_default(),
)); ));
main.set_style( main.set_style(
@ -345,16 +164,16 @@ async fn download_videos(
); );
main.tick(); main.tick();
stream::iter(videos) stream::iter(playlist.videos.items)
.map(|video| { .map(|video| {
download_single_video( download_single_video(
&video.id, video.id,
&video.name, video.name,
output_dir, output_dir,
output_fname.to_owned(), output_fname.to_owned(),
resolution, resolution,
"ffmpeg", "ffmpeg",
rp, &rp,
http.clone(), http.clone(),
multi.clone(), multi.clone(),
Some(main.clone()), Some(main.clone()),
@ -378,19 +197,9 @@ async fn main() {
let cli = Cli::parse(); let cli = Cli::parse();
let rp = RustyPipe::new();
match cli.command {
Commands::Download {
id,
output,
resolution,
parallel,
limit,
} => {
// Cases: Existing folder, non-existing file with existing parent folder, // Cases: Existing folder, non-existing file with existing parent folder,
// Error cases: non-existing parent folder, existing file // Error cases: non-existing parent folder, existing file
let output_path = std::fs::canonicalize(output).unwrap(); let output_path = std::fs::canonicalize(cli.output).unwrap();
if output_path.is_file() { if output_path.is_file() {
println!("Output file already exists"); println!("Output file already exists");
return; return;
@ -419,253 +228,12 @@ async fn main() {
) )
}; };
let target = rp.query().resolve_string(&id, false).await.unwrap(); match cli.command {
match target { Commands::Playlist { id } => {
UrlTarget::Video { id, .. } => { download_playlist(&id, &output_dir, output_fname, cli.resolution, cli.parallel).await
download_video(&rp, &id, &output_dir, output_fname, resolution).await;
} }
UrlTarget::Channel { id } => { Commands::Video { id } => {
let mut channel = rp.query().channel_videos(id).await.unwrap(); download_video(&id, &output_dir, output_fname, cli.resolution).await
channel
.content
.extend_limit(&rp.query(), limit)
.await
.unwrap();
let videos: Vec<VideoId> = channel
.content
.items
.into_iter()
.take(limit)
.map(VideoId::from)
.collect();
download_videos(
&rp,
&videos,
&output_dir,
output_fname,
resolution,
parallel,
)
.await;
} }
UrlTarget::Playlist { id } => {
let mut playlist = rp.query().playlist(id).await.unwrap();
playlist
.videos
.extend_limit(&rp.query(), limit)
.await
.unwrap();
let videos: Vec<VideoId> = playlist
.videos
.items
.into_iter()
.take(limit)
.map(VideoId::from)
.collect();
download_videos(
&rp,
&videos,
&output_dir,
output_fname,
resolution,
parallel,
)
.await;
}
UrlTarget::Album { id } => {
let album = rp.query().music_album(id).await.unwrap();
let videos: Vec<VideoId> = album
.tracks
.into_iter()
.take(limit)
.map(VideoId::from)
.collect();
download_videos(
&rp,
&videos,
&output_dir,
output_fname,
resolution,
parallel,
)
.await;
}
}
}
Commands::Get {
id,
format,
pretty,
limit,
tab,
music,
comments,
lyrics,
} => {
let target = rp.query().resolve_string(&id, false).await.unwrap();
match target {
UrlTarget::Video { id, .. } => {
if lyrics {
let details = rp.query().music_details(&id).await.unwrap();
match details.lyrics_id {
Some(lyrics_id) => {
let lyrics = rp.query().music_lyrics(lyrics_id).await.unwrap();
print_data(&lyrics, format, pretty);
}
None => eprintln!("no lyrics found"),
}
} else if music {
let details = rp.query().music_details(&id).await.unwrap();
print_data(&details, format, pretty);
} else {
let mut details = rp.query().video_details(&id).await.unwrap();
match comments {
Some(CommentsOrder::Top) => {
details
.top_comments
.extend_limit(rp.query(), limit)
.await
.unwrap();
}
Some(CommentsOrder::Latest) => {
details
.latest_comments
.extend_limit(rp.query(), limit)
.await
.unwrap();
}
None => {}
}
print_data(&details, format, pretty);
}
}
UrlTarget::Channel { id } => {
if music {
let artist = rp.query().music_artist(&id, true).await.unwrap();
print_data(&artist, format, pretty);
} else {
match tab {
ChannelTab::Videos => {
let mut channel = rp.query().channel_videos(&id).await.unwrap();
channel
.content
.extend_limit(rp.query(), limit)
.await
.unwrap();
print_data(&channel, format, pretty);
}
ChannelTab::Shorts => {
let mut channel = rp.query().channel_shorts(&id).await.unwrap();
channel
.content
.extend_limit(rp.query(), limit)
.await
.unwrap();
print_data(&channel, format, pretty);
}
ChannelTab::Live => {
let mut channel =
rp.query().channel_livestreams(&id).await.unwrap();
channel
.content
.extend_limit(rp.query(), limit)
.await
.unwrap();
print_data(&channel, format, pretty);
}
ChannelTab::Info => {
let channel = rp.query().channel_info(&id).await.unwrap();
print_data(&channel, format, pretty);
}
}
}
}
UrlTarget::Playlist { id } => {
if music {
let playlist = rp.query().music_playlist(&id).await.unwrap();
print_data(&playlist, format, pretty);
} else {
let playlist = rp.query().playlist(&id).await.unwrap();
print_data(&playlist, format, pretty);
}
}
UrlTarget::Album { id } => {
let album = rp.query().music_album(&id).await.unwrap();
print_data(&album, format, pretty);
}
}
}
Commands::Search {
query,
format,
pretty,
limit,
item_type,
length,
date,
order,
music,
} => match music {
None => {
let filter = search_filter::SearchFilter::new()
.item_type_opt(item_type.map(search_filter::ItemType::from))
.length_opt(length.map(search_filter::Length::from))
.date_opt(date.map(search_filter::UploadDate::from))
.sort_opt(order.map(search_filter::Order::from));
let mut res = rp.query().search_filter(&query, &filter).await.unwrap();
res.items.extend_limit(rp.query(), limit).await.unwrap();
print_data(&res, format, pretty);
}
Some(MusicSearchCategory::All) => {
let res = rp.query().music_search(&query).await.unwrap();
print_data(&res, format, pretty);
}
Some(MusicSearchCategory::Tracks) => {
let mut res = rp.query().music_search_tracks(&query).await.unwrap();
res.items.extend_limit(rp.query(), limit).await.unwrap();
print_data(&res, format, pretty);
}
Some(MusicSearchCategory::Videos) => {
let mut res = rp.query().music_search_videos(&query).await.unwrap();
res.items.extend_limit(rp.query(), limit).await.unwrap();
print_data(&res, format, pretty);
}
Some(MusicSearchCategory::Artists) => {
let mut res = rp.query().music_search_artists(&query).await.unwrap();
res.items.extend_limit(rp.query(), limit).await.unwrap();
print_data(&res, format, pretty);
}
Some(MusicSearchCategory::Albums) => {
let mut res = rp.query().music_search_albums(&query).await.unwrap();
res.items.extend_limit(rp.query(), limit).await.unwrap();
print_data(&res, format, pretty);
}
Some(MusicSearchCategory::Playlists) => {
let mut res = rp.query().music_search_playlists(&query).await.unwrap();
res.items.extend_limit(rp.query(), limit).await.unwrap();
print_data(&res, format, pretty);
}
Some(MusicSearchCategory::PlaylistsYtm) => {
let mut res = rp
.query()
.music_search_playlists_filter(&query, false)
.await
.unwrap();
res.items.extend_limit(rp.query(), limit).await.unwrap();
print_data(&res, format, pretty);
}
Some(MusicSearchCategory::PlaylistsCommunity) => {
let mut res = rp
.query()
.music_search_playlists_filter(&query, true)
.await
.unwrap();
res.items.extend_limit(rp.query(), limit).await.unwrap();
print_data(&res, format, pretty);
}
},
}; };
} }

View file

@ -29,8 +29,7 @@ struct QVideo<'a> {
impl RustyPipeQuery { impl RustyPipeQuery {
/// Get the metadata for a video /// Get the metadata for a video
pub async fn video_details<S: AsRef<str>>(&self, video_id: S) -> Result<VideoDetails, Error> { pub async fn video_details(&self, video_id: &str) -> Result<VideoDetails, Error> {
let video_id = video_id.as_ref();
let context = self.get_context(ClientType::Desktop, true, None).await; let context = self.get_context(ClientType::Desktop, true, None).await;
let request_body = QVideo { let request_body = QVideo {
context, context,
@ -50,12 +49,11 @@ impl RustyPipeQuery {
} }
/// Get the comments for a video using the continuation token obtained from `rusty_pipe_query.video_details()` /// Get the comments for a video using the continuation token obtained from `rusty_pipe_query.video_details()`
pub async fn video_comments<S: AsRef<str>>( pub async fn video_comments(
&self, &self,
ctoken: S, ctoken: &str,
visitor_data: Option<&str>, visitor_data: Option<&str>,
) -> Result<Paginator<Comment>, Error> { ) -> Result<Paginator<Comment>, Error> {
let ctoken = ctoken.as_ref();
let context = self let context = self
.get_context(ClientType::Desktop, true, visitor_data) .get_context(ClientType::Desktop, true, visitor_data)
.await; .await;

View file

@ -1,7 +1,7 @@
use super::{ use super::{
AlbumItem, ArtistId, ArtistItem, Channel, ChannelId, ChannelItem, ChannelRssVideo, ChannelTag, AlbumItem, ArtistId, ArtistItem, Channel, ChannelId, ChannelItem, ChannelTag, MusicArtist,
MusicArtist, MusicItem, MusicPlaylistItem, PlaylistItem, PlaylistVideo, TrackItem, VideoId, MusicItem, MusicPlaylistItem, PlaylistItem, PlaylistVideo, TrackItem, VideoId, VideoItem,
VideoItem, YouTubeItem, YouTubeItem,
}; };
/// Trait for casting generic YouTube/YouTube music items to a specific kind. /// Trait for casting generic YouTube/YouTube music items to a specific kind.
@ -168,15 +168,6 @@ impl From<PlaylistVideo> for VideoId {
} }
} }
impl From<ChannelRssVideo> for VideoId {
fn from(video: ChannelRssVideo) -> Self {
Self {
id: video.id,
name: video.name,
}
}
}
impl From<TrackItem> for VideoId { impl From<TrackItem> for VideoId {
fn from(track: TrackItem) -> Self { fn from(track: TrackItem) -> Self {
Self { Self {

View file

@ -133,15 +133,15 @@ impl SearchFilter {
self self
} }
/// Filter videos by item type /// Filter videos by entity type
pub fn item_type(mut self, item_type: ItemType) -> Self { pub fn item_type(mut self, entity: ItemType) -> Self {
self.item_type = Some(item_type); self.item_type = Some(entity);
self self
} }
/// Filter videos by item type /// Filter videos by entity type
pub fn item_type_opt(mut self, item_type: Option<ItemType>) -> Self { pub fn item_type_opt(mut self, entity: Option<ItemType>) -> Self {
self.item_type = item_type; self.item_type = entity;
self self
} }
@ -175,8 +175,8 @@ impl SearchFilter {
if let Some(date) = self.date { if let Some(date) = self.date {
filters.varint(1, date as u64); filters.varint(1, date as u64);
} }
if let Some(item_type) = self.item_type { if let Some(entity) = self.item_type {
filters.varint(2, item_type as u64); filters.varint(2, entity as u64);
} }
if let Some(length) = self.length { if let Some(length) = self.length {
filters.varint(3, length as u64); filters.varint(3, length as u64);