Compare commits

..

No commits in common. "2e06d9c572fd0c99515b59c897f439698dbb07e5" and "3458924018c6b1d943f20e1fc0a3f22e62f4b483" have entirely different histories.

5 changed files with 93 additions and 535 deletions

View file

@ -4,8 +4,12 @@ version = "0.1.0"
edition = "2021"
[dependencies]
rustypipe = { path = "../" }
rustypipe-downloader = { path = "../downloader" }
rustypipe = { path = "../", default_features = false, features = [
"rustls-tls-native-roots",
] }
rustypipe-downloader = { path = "../downloader", default_features = false, features = [
"rustls-tls-native-roots",
] }
reqwest = { version = "0.11.11", default_features = false }
tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] }
indicatif = "0.17.0"
@ -13,6 +17,3 @@ futures = "0.3.21"
anyhow = "1.0"
clap = { version = "4.0.29", features = ["derive"] }
env_logger = "0.10.0"
serde = "1.0"
serde_json = "1.0.82"
serde_yaml = "0.9.19"

View file

@ -1,220 +1,45 @@
use std::path::PathBuf;
use anyhow::{Context, Result};
use clap::{Parser, Subcommand, ValueEnum};
use clap::{Parser, Subcommand};
use futures::stream::{self, StreamExt};
use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
use reqwest::{Client, ClientBuilder};
use rustypipe::{
client::RustyPipe,
model::{UrlTarget, VideoId},
param::{search_filter, StreamFilter},
};
use serde::Serialize;
use rustypipe::{client::RustyPipe, param::StreamFilter};
#[derive(Parser)]
#[clap(author, version, about, long_about = None)]
struct Cli {
#[clap(subcommand)]
command: Commands,
#[clap(short, value_parser, default_value = ".", global = true)]
output: PathBuf,
#[clap(long, value_parser, global = true)]
resolution: Option<u32>,
#[clap(short, long, value_parser, default_value = "8", global = true)]
parallel: usize,
}
#[derive(Subcommand)]
enum Commands {
/// Download a video, playlist, album or channel
#[clap(alias = "dl")]
Download {
/// ID or URL
/// Download a playlist
Playlist {
/// Playlist ID
#[clap(value_parser)]
id: String,
/// Output path
#[clap(short, default_value = ".")]
output: PathBuf,
/// Video resolution (e.g. 720, 1080). Set to 0 for audio-only.
#[clap(short, long)]
resolution: Option<u32>,
/// Number of videos downloaded in parallel
#[clap(short, long, default_value_t = 8)]
parallel: usize,
/// Limit the number of videos to download
#[clap(long, default_value_t = 1000)]
limit: usize,
},
/// Extract video, playlist, album or channel data
Get {
/// ID or URL
/// Download a video
Video {
/// Video ID
#[clap(value_parser)]
id: String,
/// Output format
#[clap(long, value_parser, default_value = "json")]
format: Format,
/// Pretty-print output
#[clap(long)]
pretty: bool,
/// Limit the number of items to fetch
#[clap(long, default_value_t = 20)]
limit: usize,
/// Channel tab
#[clap(long, default_value = "videos")]
tab: ChannelTab,
/// Use YouTube Music
#[clap(long)]
music: bool,
/// Get comments
#[clap(long)]
comments: Option<CommentsOrder>,
/// Get lyrics
#[clap(long)]
lyrics: bool,
},
/// Search YouTube
Search {
/// Search query
query: String,
/// Output format
#[clap(long, value_parser, default_value = "json")]
format: Format,
/// Pretty-print output
#[clap(long)]
pretty: bool,
/// Limit the number of items to fetch
#[clap(long, default_value_t = 20)]
limit: usize,
/// Filter results by item type
#[clap(long)]
item_type: Option<SearchItemType>,
/// Filter results by video length
#[clap(long)]
length: Option<SearchLength>,
/// Filter results by upload date
#[clap(long)]
date: Option<SearchUploadDate>,
/// Sort search resulus
#[clap(long)]
order: Option<SearchOrder>,
/// YouTube Music search filter
#[clap(long)]
music: Option<MusicSearchCategory>,
},
}
#[derive(Copy, Clone, ValueEnum)]
enum Format {
Json,
Yaml,
}
#[derive(Copy, Clone, ValueEnum)]
enum ChannelTab {
Videos,
Shorts,
Live,
Info,
}
#[derive(Copy, Clone, ValueEnum)]
enum CommentsOrder {
Top,
Latest,
}
#[derive(Copy, Clone, ValueEnum)]
enum SearchItemType {
Video,
Channel,
Playlist,
}
#[derive(Copy, Clone, ValueEnum)]
enum SearchLength {
/// < 4min
Short,
/// 4-20min
Medium,
/// > 20min
Long,
}
#[derive(Copy, Clone, ValueEnum)]
enum SearchUploadDate {
/// 1 hour old or newer
Hour,
/// 1 day old or newer
Day,
/// 1 week old or newer
Week,
/// 1 month old or newer
Month,
/// 1 year old or newer
Year,
}
#[derive(Copy, Clone, ValueEnum)]
enum SearchOrder {
/// Sort by Like/Dislike ratio
Rating,
/// Sort by upload date
Date,
/// Sort by view count
Views,
}
#[derive(Copy, Clone, ValueEnum)]
enum MusicSearchCategory {
All,
Tracks,
Videos,
Artists,
Albums,
Playlists,
PlaylistsYtm,
PlaylistsCommunity,
}
impl From<SearchItemType> for search_filter::ItemType {
fn from(value: SearchItemType) -> Self {
match value {
SearchItemType::Video => search_filter::ItemType::Video,
SearchItemType::Channel => search_filter::ItemType::Channel,
SearchItemType::Playlist => search_filter::ItemType::Playlist,
}
}
}
impl From<SearchLength> for search_filter::Length {
fn from(value: SearchLength) -> Self {
match value {
SearchLength::Short => search_filter::Length::Short,
SearchLength::Medium => search_filter::Length::Medium,
SearchLength::Long => search_filter::Length::Long,
}
}
}
impl From<SearchUploadDate> for search_filter::UploadDate {
fn from(value: SearchUploadDate) -> Self {
match value {
SearchUploadDate::Hour => search_filter::UploadDate::Hour,
SearchUploadDate::Day => search_filter::UploadDate::Day,
SearchUploadDate::Week => search_filter::UploadDate::Week,
SearchUploadDate::Month => search_filter::UploadDate::Month,
SearchUploadDate::Year => search_filter::UploadDate::Year,
}
}
}
impl From<SearchOrder> for search_filter::Order {
fn from(value: SearchOrder) -> Self {
match value {
SearchOrder::Rating => search_filter::Order::Rating,
SearchOrder::Date => search_filter::Order::Date,
SearchOrder::Views => search_filter::Order::Views,
}
}
}
#[allow(clippy::too_many_arguments)]
async fn download_single_video(
video_id: &str,
video_title: &str,
video_id: String,
video_title: String,
output_dir: &str,
output_fname: Option<String>,
resolution: Option<u32>,
@ -232,7 +57,7 @@ async fn download_single_video(
let res = async {
let player_data = rp
.query()
.player(video_id)
.player(video_id.as_str())
.await
.context(format!("Failed to fetch player data for video {video_id}"))?;
@ -269,22 +94,7 @@ async fn download_single_video(
res
}
fn print_data<T: Serialize>(data: &T, format: Format, pretty: bool) {
let stdout = std::io::stdout().lock();
match format {
Format::Json => {
if pretty {
serde_json::to_writer_pretty(stdout, data).unwrap()
} else {
serde_json::to_writer(stdout, data).unwrap()
}
}
Format::Yaml => serde_yaml::to_writer(stdout, data).unwrap(),
};
}
async fn download_video(
rp: &RustyPipe,
id: &str,
output_dir: &str,
output_fname: Option<String>,
@ -297,17 +107,19 @@ async fn download_video(
.build()
.expect("unable to build the HTTP client");
let rp = RustyPipe::default();
// Indicatif setup
let multi = MultiProgress::new();
download_single_video(
id,
id,
id.to_owned(),
id.to_owned(),
output_dir,
output_fname,
resolution,
"ffmpeg",
rp,
&rp,
http,
multi,
None,
@ -316,9 +128,8 @@ async fn download_video(
.unwrap_or_else(|e| println!("ERROR: {e:?}"));
}
async fn download_videos(
rp: &RustyPipe,
videos: &[VideoId],
async fn download_playlist(
id: &str,
output_dir: &str,
output_fname: Option<String>,
resolution: Option<u32>,
@ -331,10 +142,18 @@ async fn download_videos(
.build()
.expect("unable to build the HTTP client");
let rp = RustyPipe::default();
let mut playlist = rp.query().playlist(id).await.unwrap();
playlist
.videos
.extend_pages(&rp.query(), usize::MAX)
.await
.unwrap();
// Indicatif setup
let multi = MultiProgress::new();
let main = multi.add(ProgressBar::new(
videos.len().try_into().unwrap_or_default(),
playlist.videos.items.len().try_into().unwrap_or_default(),
));
main.set_style(
@ -345,16 +164,16 @@ async fn download_videos(
);
main.tick();
stream::iter(videos)
stream::iter(playlist.videos.items)
.map(|video| {
download_single_video(
&video.id,
&video.name,
video.id,
video.name,
output_dir,
output_fname.to_owned(),
resolution,
"ffmpeg",
rp,
&rp,
http.clone(),
multi.clone(),
Some(main.clone()),
@ -378,294 +197,43 @@ async fn main() {
let cli = Cli::parse();
let rp = RustyPipe::new();
// Cases: Existing folder, non-existing file with existing parent folder,
// Error cases: non-existing parent folder, existing file
let output_path = std::fs::canonicalize(cli.output).unwrap();
if output_path.is_file() {
println!("Output file already exists");
return;
}
let (output_dir, output_fname) = if output_path.is_dir() {
(output_path.to_string_lossy().to_string(), None)
} else {
let output_dir_parent = output_path.parent().unwrap();
if !output_dir_parent.is_dir() {
println!(
"Parent folder {} does not exist",
output_dir_parent.to_string_lossy()
);
return;
}
(
output_dir_parent.to_string_lossy().to_string(),
Some(
output_path
.file_name()
.unwrap()
.to_string_lossy()
.to_string(),
),
)
};
match cli.command {
Commands::Download {
id,
output,
resolution,
parallel,
limit,
} => {
// Cases: Existing folder, non-existing file with existing parent folder,
// Error cases: non-existing parent folder, existing file
let output_path = std::fs::canonicalize(output).unwrap();
if output_path.is_file() {
println!("Output file already exists");
return;
}
let (output_dir, output_fname) = if output_path.is_dir() {
(output_path.to_string_lossy().to_string(), None)
} else {
let output_dir_parent = output_path.parent().unwrap();
if !output_dir_parent.is_dir() {
println!(
"Parent folder {} does not exist",
output_dir_parent.to_string_lossy()
);
return;
}
(
output_dir_parent.to_string_lossy().to_string(),
Some(
output_path
.file_name()
.unwrap()
.to_string_lossy()
.to_string(),
),
)
};
let target = rp.query().resolve_string(&id, false).await.unwrap();
match target {
UrlTarget::Video { id, .. } => {
download_video(&rp, &id, &output_dir, output_fname, resolution).await;
}
UrlTarget::Channel { id } => {
let mut channel = rp.query().channel_videos(id).await.unwrap();
channel
.content
.extend_limit(&rp.query(), limit)
.await
.unwrap();
let videos: Vec<VideoId> = channel
.content
.items
.into_iter()
.take(limit)
.map(VideoId::from)
.collect();
download_videos(
&rp,
&videos,
&output_dir,
output_fname,
resolution,
parallel,
)
.await;
}
UrlTarget::Playlist { id } => {
let mut playlist = rp.query().playlist(id).await.unwrap();
playlist
.videos
.extend_limit(&rp.query(), limit)
.await
.unwrap();
let videos: Vec<VideoId> = playlist
.videos
.items
.into_iter()
.take(limit)
.map(VideoId::from)
.collect();
download_videos(
&rp,
&videos,
&output_dir,
output_fname,
resolution,
parallel,
)
.await;
}
UrlTarget::Album { id } => {
let album = rp.query().music_album(id).await.unwrap();
let videos: Vec<VideoId> = album
.tracks
.into_iter()
.take(limit)
.map(VideoId::from)
.collect();
download_videos(
&rp,
&videos,
&output_dir,
output_fname,
resolution,
parallel,
)
.await;
}
}
Commands::Playlist { id } => {
download_playlist(&id, &output_dir, output_fname, cli.resolution, cli.parallel).await
}
Commands::Get {
id,
format,
pretty,
limit,
tab,
music,
comments,
lyrics,
} => {
let target = rp.query().resolve_string(&id, false).await.unwrap();
match target {
UrlTarget::Video { id, .. } => {
if lyrics {
let details = rp.query().music_details(&id).await.unwrap();
match details.lyrics_id {
Some(lyrics_id) => {
let lyrics = rp.query().music_lyrics(lyrics_id).await.unwrap();
print_data(&lyrics, format, pretty);
}
None => eprintln!("no lyrics found"),
}
} else if music {
let details = rp.query().music_details(&id).await.unwrap();
print_data(&details, format, pretty);
} else {
let mut details = rp.query().video_details(&id).await.unwrap();
match comments {
Some(CommentsOrder::Top) => {
details
.top_comments
.extend_limit(rp.query(), limit)
.await
.unwrap();
}
Some(CommentsOrder::Latest) => {
details
.latest_comments
.extend_limit(rp.query(), limit)
.await
.unwrap();
}
None => {}
}
print_data(&details, format, pretty);
}
}
UrlTarget::Channel { id } => {
if music {
let artist = rp.query().music_artist(&id, true).await.unwrap();
print_data(&artist, format, pretty);
} else {
match tab {
ChannelTab::Videos => {
let mut channel = rp.query().channel_videos(&id).await.unwrap();
channel
.content
.extend_limit(rp.query(), limit)
.await
.unwrap();
print_data(&channel, format, pretty);
}
ChannelTab::Shorts => {
let mut channel = rp.query().channel_shorts(&id).await.unwrap();
channel
.content
.extend_limit(rp.query(), limit)
.await
.unwrap();
print_data(&channel, format, pretty);
}
ChannelTab::Live => {
let mut channel =
rp.query().channel_livestreams(&id).await.unwrap();
channel
.content
.extend_limit(rp.query(), limit)
.await
.unwrap();
print_data(&channel, format, pretty);
}
ChannelTab::Info => {
let channel = rp.query().channel_info(&id).await.unwrap();
print_data(&channel, format, pretty);
}
}
}
}
UrlTarget::Playlist { id } => {
if music {
let playlist = rp.query().music_playlist(&id).await.unwrap();
print_data(&playlist, format, pretty);
} else {
let playlist = rp.query().playlist(&id).await.unwrap();
print_data(&playlist, format, pretty);
}
}
UrlTarget::Album { id } => {
let album = rp.query().music_album(&id).await.unwrap();
print_data(&album, format, pretty);
}
}
Commands::Video { id } => {
download_video(&id, &output_dir, output_fname, cli.resolution).await
}
Commands::Search {
query,
format,
pretty,
limit,
item_type,
length,
date,
order,
music,
} => match music {
None => {
let filter = search_filter::SearchFilter::new()
.item_type_opt(item_type.map(search_filter::ItemType::from))
.length_opt(length.map(search_filter::Length::from))
.date_opt(date.map(search_filter::UploadDate::from))
.sort_opt(order.map(search_filter::Order::from));
let mut res = rp.query().search_filter(&query, &filter).await.unwrap();
res.items.extend_limit(rp.query(), limit).await.unwrap();
print_data(&res, format, pretty);
}
Some(MusicSearchCategory::All) => {
let res = rp.query().music_search(&query).await.unwrap();
print_data(&res, format, pretty);
}
Some(MusicSearchCategory::Tracks) => {
let mut res = rp.query().music_search_tracks(&query).await.unwrap();
res.items.extend_limit(rp.query(), limit).await.unwrap();
print_data(&res, format, pretty);
}
Some(MusicSearchCategory::Videos) => {
let mut res = rp.query().music_search_videos(&query).await.unwrap();
res.items.extend_limit(rp.query(), limit).await.unwrap();
print_data(&res, format, pretty);
}
Some(MusicSearchCategory::Artists) => {
let mut res = rp.query().music_search_artists(&query).await.unwrap();
res.items.extend_limit(rp.query(), limit).await.unwrap();
print_data(&res, format, pretty);
}
Some(MusicSearchCategory::Albums) => {
let mut res = rp.query().music_search_albums(&query).await.unwrap();
res.items.extend_limit(rp.query(), limit).await.unwrap();
print_data(&res, format, pretty);
}
Some(MusicSearchCategory::Playlists) => {
let mut res = rp.query().music_search_playlists(&query).await.unwrap();
res.items.extend_limit(rp.query(), limit).await.unwrap();
print_data(&res, format, pretty);
}
Some(MusicSearchCategory::PlaylistsYtm) => {
let mut res = rp
.query()
.music_search_playlists_filter(&query, false)
.await
.unwrap();
res.items.extend_limit(rp.query(), limit).await.unwrap();
print_data(&res, format, pretty);
}
Some(MusicSearchCategory::PlaylistsCommunity) => {
let mut res = rp
.query()
.music_search_playlists_filter(&query, true)
.await
.unwrap();
res.items.extend_limit(rp.query(), limit).await.unwrap();
print_data(&res, format, pretty);
}
},
};
}

View file

@ -29,8 +29,7 @@ struct QVideo<'a> {
impl RustyPipeQuery {
/// Get the metadata for a video
pub async fn video_details<S: AsRef<str>>(&self, video_id: S) -> Result<VideoDetails, Error> {
let video_id = video_id.as_ref();
pub async fn video_details(&self, video_id: &str) -> Result<VideoDetails, Error> {
let context = self.get_context(ClientType::Desktop, true, None).await;
let request_body = QVideo {
context,
@ -50,12 +49,11 @@ impl RustyPipeQuery {
}
/// Get the comments for a video using the continuation token obtained from `rusty_pipe_query.video_details()`
pub async fn video_comments<S: AsRef<str>>(
pub async fn video_comments(
&self,
ctoken: S,
ctoken: &str,
visitor_data: Option<&str>,
) -> Result<Paginator<Comment>, Error> {
let ctoken = ctoken.as_ref();
let context = self
.get_context(ClientType::Desktop, true, visitor_data)
.await;

View file

@ -1,7 +1,7 @@
use super::{
AlbumItem, ArtistId, ArtistItem, Channel, ChannelId, ChannelItem, ChannelRssVideo, ChannelTag,
MusicArtist, MusicItem, MusicPlaylistItem, PlaylistItem, PlaylistVideo, TrackItem, VideoId,
VideoItem, YouTubeItem,
AlbumItem, ArtistId, ArtistItem, Channel, ChannelId, ChannelItem, ChannelTag, MusicArtist,
MusicItem, MusicPlaylistItem, PlaylistItem, PlaylistVideo, TrackItem, VideoId, VideoItem,
YouTubeItem,
};
/// Trait for casting generic YouTube/YouTube music items to a specific kind.
@ -168,15 +168,6 @@ impl From<PlaylistVideo> for VideoId {
}
}
impl From<ChannelRssVideo> for VideoId {
fn from(video: ChannelRssVideo) -> Self {
Self {
id: video.id,
name: video.name,
}
}
}
impl From<TrackItem> for VideoId {
fn from(track: TrackItem) -> Self {
Self {

View file

@ -133,15 +133,15 @@ impl SearchFilter {
self
}
/// Filter videos by item type
pub fn item_type(mut self, item_type: ItemType) -> Self {
self.item_type = Some(item_type);
/// Filter videos by entity type
pub fn item_type(mut self, entity: ItemType) -> Self {
self.item_type = Some(entity);
self
}
/// Filter videos by item type
pub fn item_type_opt(mut self, item_type: Option<ItemType>) -> Self {
self.item_type = item_type;
/// Filter videos by entity type
pub fn item_type_opt(mut self, entity: Option<ItemType>) -> Self {
self.item_type = entity;
self
}
@ -175,8 +175,8 @@ impl SearchFilter {
if let Some(date) = self.date {
filters.varint(1, date as u64);
}
if let Some(item_type) = self.item_type {
filters.varint(2, item_type as u64);
if let Some(entity) = self.item_type {
filters.varint(2, entity as u64);
}
if let Some(length) = self.length {
filters.varint(3, length as u64);