From 00454d0524b434966a92ce6dce92149a15004358 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 15 Jan 2025 18:14:51 +0100 Subject: [PATCH 01/12] chore: update pre-commit hooks --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c77c173..c6dc7ee 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v5.0.0 hooks: - id: end-of-file-fixer From bd7572828b99deba2d0909d190c38b1043c1952b Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 10 Feb 2025 00:25:25 +0100 Subject: [PATCH 02/12] feat: improve NormalisationData parser --- crates/downloader/src/lib.rs | 4 ++-- crates/spotifyio/Cargo.toml | 1 + crates/spotifyio/src/normalisation.rs | 25 ++++++++++++++----------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/crates/downloader/src/lib.rs b/crates/downloader/src/lib.rs index b1bc9ae..9c13520 100644 --- a/crates/downloader/src/lib.rs +++ b/crates/downloader/src/lib.rs @@ -662,8 +662,8 @@ impl SpotifyDownloader { drop(file); std::fs::rename(&tpath_tmp, dest)?; - norm_data = spotify_header - .and_then(|h| NormalisationData::parse_from_ogg(Cursor::new(&h)).ok()); + let h = spotify_header.ok_or(SpotifyError::failed_precondition("no header"))?; + norm_data = Some(NormalisationData::parse_from_ogg(&mut Cursor::new(&h))?); } AudioKeyVariant::Widevine(key) => { let decryptor = self diff --git a/crates/spotifyio/Cargo.toml b/crates/spotifyio/Cargo.toml index 447125b..282d970 100644 --- a/crates/spotifyio/Cargo.toml +++ b/crates/spotifyio/Cargo.toml @@ -75,6 +75,7 @@ governor = { version = "0.8", default-features = false, features = [ "jitter", ] } async-stream = "0.3.0" +ogg_pager = "0.7.0" protobuf.workspace = true spotifyio-protocol.workspace = true diff --git a/crates/spotifyio/src/normalisation.rs b/crates/spotifyio/src/normalisation.rs index f65fbb3..4d0a4be 100644 --- a/crates/spotifyio/src/normalisation.rs +++ b/crates/spotifyio/src/normalisation.rs @@ -1,4 +1,4 @@ -use std::io::{Read, Seek, SeekFrom}; +use std::io::{Cursor, Read, Seek}; use byteorder::{LittleEndian, ReadBytesExt}; @@ -20,20 +20,23 @@ pub struct NormalisationData { } impl NormalisationData { - pub fn parse_from_ogg(mut file: T) -> Result { - const SPOTIFY_NORMALIZATION_HEADER_START_OFFSET: u64 = 144; - let newpos = file.seek(SeekFrom::Start(SPOTIFY_NORMALIZATION_HEADER_START_OFFSET))?; - if newpos != SPOTIFY_NORMALIZATION_HEADER_START_OFFSET { + /// Parse normalisation data from a Spotify OGG header (first 167 bytes) + pub fn parse_from_ogg(file: &mut R) -> Result { + let packets = ogg_pager::Packets::read_count(file, 1) + .map_err(|e| Error::failed_precondition(format!("invalid ogg file: {e}")))?; + let packet = packets.get(0).unwrap(); + if packet.len() != 139 { return Err(Error::failed_precondition(format!( - "NormalisationData::parse_from_ogg seeking to {} but position is now {}", - SPOTIFY_NORMALIZATION_HEADER_START_OFFSET, newpos + "ogg header len={}, expected=139", + packet.len() ))); } + let mut rdr = Cursor::new(&packet[116..]); - let track_gain_db = file.read_f32::()? as f64; - let track_peak = file.read_f32::()? as f64; - let album_gain_db = file.read_f32::()? as f64; - let album_peak = file.read_f32::()? as f64; + let track_gain_db = rdr.read_f32::()? as f64; + let track_peak = rdr.read_f32::()? as f64; + let album_gain_db = rdr.read_f32::()? as f64; + let album_peak = rdr.read_f32::()? as f64; Ok(Self { track_gain_db, From 0e232580695c3150aaa7f8dc8391ee08d413ecc4 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 10 Mar 2025 15:35:19 +0100 Subject: [PATCH 03/12] fix: correct timeout, f32 normalization data --- crates/downloader/src/lib.rs | 3 +-- crates/spotifyio/src/normalisation.rs | 19 ++++++++----------- crates/spotifyio/src/pool.rs | 8 ++++---- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/crates/downloader/src/lib.rs b/crates/downloader/src/lib.rs index 9c13520..113df1c 100644 --- a/crates/downloader/src/lib.rs +++ b/crates/downloader/src/lib.rs @@ -645,8 +645,7 @@ impl SpotifyDownloader { let mut first = true; let mut cipher: Aes128Ctr = - Aes128Ctr::new_from_slices(&audio_key.0, &spotifyio::util::AUDIO_AESIV) - .unwrap(); + Aes128Ctr::new(&audio_key.0.into(), &spotifyio::util::AUDIO_AESIV.into()); let mut spotify_header = None; while let Some(item) = stream.next().await { diff --git a/crates/spotifyio/src/normalisation.rs b/crates/spotifyio/src/normalisation.rs index 4d0a4be..f8744ca 100644 --- a/crates/spotifyio/src/normalisation.rs +++ b/crates/spotifyio/src/normalisation.rs @@ -6,17 +6,14 @@ use crate::Error; /// Audio metadata for volume normalization /// -/// Spotify provides these as `f32`, but audio metadata can contain up to `f64`. -/// Also, this negates the need for casting during sample processing. -/// /// More information about ReplayGain and how to apply this infomation to audio files /// can be found here: . #[derive(Clone, Copy, Debug)] pub struct NormalisationData { - pub track_gain_db: f64, - pub track_peak: f64, - pub album_gain_db: f64, - pub album_peak: f64, + pub track_gain_db: f32, + pub track_peak: f32, + pub album_gain_db: f32, + pub album_peak: f32, } impl NormalisationData { @@ -33,10 +30,10 @@ impl NormalisationData { } let mut rdr = Cursor::new(&packet[116..]); - let track_gain_db = rdr.read_f32::()? as f64; - let track_peak = rdr.read_f32::()? as f64; - let album_gain_db = rdr.read_f32::()? as f64; - let album_peak = rdr.read_f32::()? as f64; + let track_gain_db = rdr.read_f32::()?; + let track_peak = rdr.read_f32::()?; + let album_gain_db = rdr.read_f32::()?; + let album_peak = rdr.read_f32::()?; Ok(Self { track_gain_db, diff --git a/crates/spotifyio/src/pool.rs b/crates/spotifyio/src/pool.rs index b46b636..735569e 100644 --- a/crates/spotifyio/src/pool.rs +++ b/crates/spotifyio/src/pool.rs @@ -86,7 +86,7 @@ impl ClientPool { fn next_pos(&self, timeout: Option) -> Result<(usize, Duration), PoolError> { let mut robin = self.robin.lock(); let mut i = *robin; - let mut max_to = Duration::ZERO; + let mut min_to = Duration::MAX; for _ in 0..((self.len() - 1).max(1)) { let entry = &self.entries[i]; @@ -100,7 +100,7 @@ impl ClientPool { Err(not_until) => { let wait_time = not_until.wait_time_from(entry.limiter.clock().now()); if timeout.is_some_and(|to| wait_time > to) { - max_to = max_to.max(wait_time); + min_to = min_to.min(wait_time); } else { *robin = self.wrapping_inc(i); return Ok((i, wait_time)); @@ -111,10 +111,10 @@ impl ClientPool { i = self.wrapping_inc(i); } // If the pool is empty or all entries were skipped - Err(if max_to == Duration::ZERO { + Err(if min_to == Duration::MAX { PoolError::Empty } else { - PoolError::Timeout(max_to) + PoolError::Timeout(min_to) }) } From 10faa32495533cf19efbd8012b830bb3d362e281 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 10 Mar 2025 16:22:29 +0100 Subject: [PATCH 04/12] fix: parsing get_user_following with artists and users --- crates/model/src/idtypes.rs | 109 ++++++++++++++++++++++++++++++ crates/spotifyio/src/gql_model.rs | 32 ++++++++- crates/spotifyio/src/spclient.rs | 22 ++++-- 3 files changed, 154 insertions(+), 9 deletions(-) diff --git a/crates/model/src/idtypes.rs b/crates/model/src/idtypes.rs index 708ce69..ec705b4 100644 --- a/crates/model/src/idtypes.rs +++ b/crates/model/src/idtypes.rs @@ -843,6 +843,115 @@ impl<'de> Deserialize<'de> for PlayableId<'static> { } } +#[enum_dispatch(Id)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum UserlikeId<'a> { + User(UserId<'a>), + Artist(ArtistId<'a>), +} +impl IdBase62 for UserlikeId<'_> {} + +// These don't work with `enum_dispatch`, unfortunately. +impl<'a> UserlikeId<'a> { + #[must_use] + pub fn as_ref(&'a self) -> Self { + match self { + UserlikeId::User(x) => UserlikeId::User(x.as_ref()), + UserlikeId::Artist(x) => UserlikeId::Artist(x.as_ref()), + } + } + + #[must_use] + pub fn into_static(self) -> UserlikeId<'static> { + match self { + UserlikeId::User(x) => UserlikeId::User(x.into_static()), + UserlikeId::Artist(x) => UserlikeId::Artist(x.into_static()), + } + } + + #[must_use] + pub fn clone_static(&'a self) -> UserlikeId<'static> { + match self { + UserlikeId::User(x) => UserlikeId::User(x.clone_static()), + UserlikeId::Artist(x) => UserlikeId::Artist(x.clone_static()), + } + } + + /// Parse Spotify URI from string slice + /// + /// Spotify URI must be in one of the following formats: + /// `spotify:{type}:{id}` or `spotify/{type}/{id}`. + /// Where `{type}` is one of `artist`, `album`, `track`, + /// `playlist`, `user`, `show`, or `episode`, and `{id}` is a + /// non-empty valid string. + /// + /// Examples: `spotify:album:6IcGNaXFRf5Y1jc7QsE9O2`, + /// `spotify/track/4y4VO05kYgUTo2bzbox1an`. + /// + /// # Errors + /// + /// - `IdError::InvalidPrefix` - if `uri` is not started with + /// `spotify:` or `spotify/`, + /// - `IdError::InvalidType` - if type part of an `uri` is not a + /// valid Spotify type `T`, + /// - `IdError::InvalidId` - if id part of an `uri` is not a + /// valid id, + /// - `IdError::InvalidFormat` - if it can't be splitted into + /// type and id parts. + pub fn from_uri(uri: &'a str) -> Result { + let (tpe, id) = parse_uri(uri)?; + match tpe { + SpotifyType::User => Ok(Self::User(UserId::from_id(id)?)), + SpotifyType::Artist => Ok(Self::Artist(ArtistId::from_id(id)?)), + _ => Err(IdError::InvalidType), + } + } +} + +/// Displaying the ID shows its URI +impl std::fmt::Display for UserlikeId<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", self.uri()) + } +} + +impl Serialize for UserlikeId<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.uri()) + } +} + +impl<'de> Deserialize<'de> for UserlikeId<'static> { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct UriVisitor; + + impl serde::de::Visitor<'_> for UriVisitor { + type Value = UserlikeId<'static>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("URI for PlayableId") + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + UserlikeId::from_uri(v) + .map(UserlikeId::into_static) + .map_err(serde::de::Error::custom) + } + } + + deserializer.deserialize_str(UriVisitor) + } +} + #[cfg(test)] mod test { use super::*; diff --git a/crates/spotifyio/src/gql_model.rs b/crates/spotifyio/src/gql_model.rs index 35292e9..699df29 100644 --- a/crates/spotifyio/src/gql_model.rs +++ b/crates/spotifyio/src/gql_model.rs @@ -8,6 +8,7 @@ use time::OffsetDateTime; use spotifyio_model::{ AlbumId, ArtistId, ConcertId, PlaylistId, PrereleaseId, SongwriterId, TrackId, UserId, + UserlikeId, }; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -526,8 +527,8 @@ pub struct PublicPlaylistItem { } #[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct UserProfilesWrap { - pub profiles: Vec, +pub(crate) struct UserProfilesWrap { + pub profiles: Vec, } /// May be an artist or an user @@ -540,6 +541,16 @@ pub struct FollowerItem { pub image_url: Option, } +/// May be an artist or an user +#[derive(Debug, Clone, Serialize, Deserialize)] +#[non_exhaustive] +pub(crate) struct FollowerItemUserlike { + pub uri: UserlikeId<'static>, + pub name: Option, + pub followers_count: Option, + pub image_url: Option, +} + /// Seektable for AAC tracks #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Seektable { @@ -621,3 +632,20 @@ impl From> for Option { value.data.into_option() } } + +impl TryFrom for FollowerItem { + type Error = (); + + fn try_from(value: FollowerItemUserlike) -> Result { + if let UserlikeId::User(uri) = value.uri { + Ok(Self { + uri, + name: value.name, + followers_count: value.followers_count, + image_url: value.image_url, + }) + } else { + Err(()) + } + } +} diff --git a/crates/spotifyio/src/spclient.rs b/crates/spotifyio/src/spclient.rs index 74dd472..14b226d 100644 --- a/crates/spotifyio/src/spclient.rs +++ b/crates/spotifyio/src/spclient.rs @@ -33,9 +33,9 @@ use crate::{ error::ErrorKind, gql_model::{ ArtistGql, ArtistGqlWrap, Concert, ConcertGql, ConcertGqlWrap, ConcertOption, FollowerItem, - GqlPlaylistItem, GqlSearchResult, GqlWrap, Lyrics, LyricsWrap, PlaylistWrap, - PrereleaseItem, PrereleaseLookup, SearchItemType, SearchResultWrap, Seektable, - TrackCredits, UserPlaylists, UserProfile, UserProfilesWrap, + FollowerItemUserlike, GqlPlaylistItem, GqlSearchResult, GqlWrap, Lyrics, LyricsWrap, + PlaylistWrap, PrereleaseItem, PrereleaseLookup, SearchItemType, SearchResultWrap, + Seektable, TrackCredits, UserPlaylists, UserProfile, UserProfilesWrap, }, model::{ AlbumId, AlbumType, ArtistId, AudioAnalysis, AudioFeatures, AudioFeaturesPayload, Category, @@ -1200,23 +1200,31 @@ impl SpClient { pub async fn get_user_followers(&self, id: UserId<'_>) -> Result, Error> { debug!("getting user followers {id}"); let res = self - .request_get_json::(&format!( + .request_get_json::>(&format!( "/user-profile-view/v3/profile/{}/followers?market=from_token", id.id() )) .await?; - Ok(res.profiles) + Ok(res + .profiles + .into_iter() + .filter_map(|p| p.try_into().ok()) + .collect()) } pub async fn get_user_following(&self, id: UserId<'_>) -> Result, Error> { debug!("getting user following {id}"); let res = self - .request_get_json::(&format!( + .request_get_json::>(&format!( "/user-profile-view/v3/profile/{}/following?market=from_token", id.id() )) .await?; - Ok(res.profiles) + Ok(res + .profiles + .into_iter() + .filter_map(|p| p.try_into().ok()) + .collect()) } // PUBLIC SPOTIFY API - FROM RSPOTIFY From 4ee9079044655aada74e2113f085495383d30712 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 10 Mar 2025 16:34:09 +0100 Subject: [PATCH 05/12] feat: write multiple artists to tags --- crates/downloader/src/lib.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/crates/downloader/src/lib.rs b/crates/downloader/src/lib.rs index 113df1c..4e25e48 100644 --- a/crates/downloader/src/lib.rs +++ b/crates/downloader/src/lib.rs @@ -20,8 +20,8 @@ use itertools::Itertools; use lofty::{config::WriteOptions, prelude::*, tag::Tag}; use model::{ album_type_str, convert_date, date_to_string, get_covers, parse_country_codes, - AlternativeTrack, ArtistItem, ArtistWithRole, AudioFiles, AudioItem, EpisodeUniqueFields, - TrackUniqueFields, UnavailabilityReason, UniqueFields, + AlternativeTrack, ArtistItem, ArtistRole, ArtistWithRole, AudioFiles, AudioItem, + EpisodeUniqueFields, TrackUniqueFields, UnavailabilityReason, UniqueFields, }; use once_cell::sync::Lazy; use path_macro::path; @@ -815,7 +815,7 @@ impl SpotifyDownloader { &track_fields.album_name, track_fields.number, track_fields.disc_number, - &artist.name, + &track_fields.artists, &artists_json, album_id_str, Some(&album_artist.id), @@ -1314,7 +1314,7 @@ fn tag_file( album: &str, track_nr: u32, disc_nr: u32, - artist: &str, + artists: &[ArtistWithRole], artists_json: &str, album_id: &str, album_artist_id: Option<&str>, @@ -1340,7 +1340,17 @@ fn tag_file( tag.set_title(name.to_owned()); tag.set_album(album.to_owned()); tag.set_track(track_nr); - tag.set_artist(artist.to_owned()); + + for artist in artists { + let k = match artist.role { + ArtistRole::ArtistRoleRemixer => ItemKey::Remixer, + ArtistRole::ArtistRoleComposer => ItemKey::Composer, + ArtistRole::ArtistRoleConductor => ItemKey::Conductor, + _ => ItemKey::TrackArtist, + }; + tag.insert_text(k, artist.name.to_owned()); + } + tag.insert_text(ItemKey::AlbumArtist, album_artist.to_owned()); if let Some(date) = release_date { tag.insert_text(ItemKey::RecordingDate, fix_release_date(date)); From ced1a146360fd5b42d1ad3554f91eaf54985a7d4 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 10 Mar 2025 16:54:47 +0100 Subject: [PATCH 06/12] feat: tag multiple artist names --- crates/downloader/src/lib.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/crates/downloader/src/lib.rs b/crates/downloader/src/lib.rs index 4e25e48..6158ea4 100644 --- a/crates/downloader/src/lib.rs +++ b/crates/downloader/src/lib.rs @@ -43,6 +43,8 @@ pub mod model; type Aes128Ctr = ctr::Ctr128BE; +const DOT_SEPARATOR: &str = " • "; + #[derive(Debug, thiserror::Error)] pub enum Error { #[error("Spotify: {0}")] @@ -1341,7 +1343,18 @@ fn tag_file( tag.set_album(album.to_owned()); tag.set_track(track_nr); + let mut artist_names = Vec::new(); + let mut feat_names = Vec::new(); + for artist in artists { + match artist.role { + ArtistRole::ArtistRoleFeaturedArtist => feat_names.push(artist.name.to_owned()), + ArtistRole::ArtistRoleComposer + | ArtistRole::ArtistRoleConductor + | ArtistRole::ArtistRoleRemixer => {} + _ => artist_names.push(artist.name.to_owned()), + } + let k = match artist.role { ArtistRole::ArtistRoleRemixer => ItemKey::Remixer, ArtistRole::ArtistRoleComposer => ItemKey::Composer, @@ -1351,6 +1364,13 @@ fn tag_file( tag.insert_text(k, artist.name.to_owned()); } + let mut artist_names_str = artist_names.join(DOT_SEPARATOR); + if !feat_names.is_empty() { + artist_names_str += " feat. "; + artist_names_str += &feat_names.join(DOT_SEPARATOR); + } + + tag.set_artist(artist_names_str); tag.insert_text(ItemKey::AlbumArtist, album_artist.to_owned()); if let Some(date) = release_date { tag.insert_text(ItemKey::RecordingDate, fix_release_date(date)); From cc99ff610e82c38dffb086e0918f964aaaad60b0 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 10 Mar 2025 16:55:29 +0100 Subject: [PATCH 07/12] bump downloader version to 0.4.0 --- crates/downloader/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/downloader/Cargo.toml b/crates/downloader/Cargo.toml index 438e6b0..124dfdc 100644 --- a/crates/downloader/Cargo.toml +++ b/crates/downloader/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "spotifyio-downloader" description = "CLI for downloading music from Spotify" -version = "0.3.1" +version = "0.4.0" edition.workspace = true authors.workspace = true license.workspace = true From f38a1702a3d9d01fa3d445e8e1f95a774d6eb8c2 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Tue, 11 Mar 2025 03:01:36 +0100 Subject: [PATCH 08/12] fix: set multiple TrackArtists tags --- crates/downloader/src/lib.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/downloader/src/lib.rs b/crates/downloader/src/lib.rs index 6158ea4..af18ae5 100644 --- a/crates/downloader/src/lib.rs +++ b/crates/downloader/src/lib.rs @@ -17,7 +17,11 @@ use aes::cipher::{KeyIvInit, StreamCipher}; use futures_util::{stream, StreamExt, TryStreamExt}; use indicatif::{MultiProgress, ProgressBar}; use itertools::Itertools; -use lofty::{config::WriteOptions, prelude::*, tag::Tag}; +use lofty::{ + config::WriteOptions, + prelude::*, + tag::{ItemValue, Tag, TagItem}, +}; use model::{ album_type_str, convert_date, date_to_string, get_covers, parse_country_codes, AlternativeTrack, ArtistItem, ArtistRole, ArtistWithRole, AudioFiles, AudioItem, @@ -1359,9 +1363,9 @@ fn tag_file( ArtistRole::ArtistRoleRemixer => ItemKey::Remixer, ArtistRole::ArtistRoleComposer => ItemKey::Composer, ArtistRole::ArtistRoleConductor => ItemKey::Conductor, - _ => ItemKey::TrackArtist, + _ => ItemKey::TrackArtists, }; - tag.insert_text(k, artist.name.to_owned()); + tag.push(TagItem::new(k, ItemValue::Text(artist.name.to_owned()))); } let mut artist_names_str = artist_names.join(DOT_SEPARATOR); From b2ba8a2fe35b84b4bee16142dc892265852b5d57 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Tue, 11 Mar 2025 14:30:18 +0100 Subject: [PATCH 09/12] fix: log timeout with info level if longer than 10s --- crates/spotifyio/src/pool.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/spotifyio/src/pool.rs b/crates/spotifyio/src/pool.rs index 735569e..783225b 100644 --- a/crates/spotifyio/src/pool.rs +++ b/crates/spotifyio/src/pool.rs @@ -151,7 +151,11 @@ impl ClientPool { let entry = &self.entries[i]; if !wait_time.is_zero() { let wait_with_jitter = self.jitter + wait_time; - tracing::debug!("pool: waiting for {wait_with_jitter:?}"); + if wait_with_jitter > Duration::from_secs(10) { + tracing::info!("waiting for {}s", wait_with_jitter.as_secs()); + } else { + tracing::debug!("waiting for {}s", wait_with_jitter.as_secs()); + }; tokio::time::sleep(wait_with_jitter).await; } Ok(&entry.entry) From d2aa57a0cf378ff5f68dbe87a94321e4c93c641b Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Sun, 18 May 2025 21:13:43 +0200 Subject: [PATCH 10/12] chore: update dependencies --- crates/model/Cargo.toml | 2 +- crates/spotifyio/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/model/Cargo.toml b/crates/model/Cargo.toml index ae1a727..7dddc6f 100644 --- a/crates/model/Cargo.toml +++ b/crates/model/Cargo.toml @@ -16,7 +16,7 @@ categories.workspace = true enum_dispatch = "0.3.8" serde = { version = "1", features = ["derive"] } serde_json = "1" -strum = { version = "0.26.1", features = ["derive"] } +strum = { version = "0.27", features = ["derive"] } thiserror = "2" time = { version = "0.3.21", features = ["serde-well-known"] } data-encoding = "2.5" diff --git a/crates/spotifyio/Cargo.toml b/crates/spotifyio/Cargo.toml index 282d970..0f87ed5 100644 --- a/crates/spotifyio/Cargo.toml +++ b/crates/spotifyio/Cargo.toml @@ -69,7 +69,7 @@ pin-project-lite = "0.2" quick-xml = "0.37" urlencoding = "2.1.0" parking_lot = "0.12.0" -governor = { version = "0.8", default-features = false, features = [ +governor = { version = "0.10", default-features = false, features = [ "std", "quanta", "jitter", From 3d66d2ee8c383740cf20f24a45586bd519cceac8 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 19 May 2025 01:57:00 +0200 Subject: [PATCH 11/12] feat: add playlist downloader --- crates/downloader/Cargo.toml | 2 +- crates/downloader/src/lib.rs | 109 ++++++++++++++++-- crates/downloader/src/main.rs | 31 ++++- crates/downloader/src/model/mod.rs | 8 ++ .../spotifyio/src/connection/proxytunnel.rs | 11 +- 5 files changed, 139 insertions(+), 22 deletions(-) diff --git a/crates/downloader/Cargo.toml b/crates/downloader/Cargo.toml index 124dfdc..d00c9eb 100644 --- a/crates/downloader/Cargo.toml +++ b/crates/downloader/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "spotifyio-downloader" description = "CLI for downloading music from Spotify" -version = "0.4.0" +version = "0.5.0" edition.workspace = true authors.workspace = true license.workspace = true diff --git a/crates/downloader/src/lib.rs b/crates/downloader/src/lib.rs index af18ae5..1462fc5 100644 --- a/crates/downloader/src/lib.rs +++ b/crates/downloader/src/lib.rs @@ -24,14 +24,17 @@ use lofty::{ }; use model::{ album_type_str, convert_date, date_to_string, get_covers, parse_country_codes, - AlternativeTrack, ArtistItem, ArtistRole, ArtistWithRole, AudioFiles, AudioItem, + AlternativeTrack, ArtistItem, ArtistRole, ArtistWithRole, AudioFiles, AudioItem, DlAudioItem, EpisodeUniqueFields, TrackUniqueFields, UnavailabilityReason, UniqueFields, }; use once_cell::sync::Lazy; use path_macro::path; use serde::Serialize; use spotifyio::{ - model::{AlbumId, ArtistId, EpisodeId, FileId, Id, IdConstruct, IdError, PlayableId, TrackId}, + model::{ + AlbumId, ArtistId, EpisodeId, FileId, Id, IdConstruct, IdError, PlayableId, PlaylistId, + TrackId, + }, pb::AudioFileFormat, AudioKey, CdnUrl, Error as SpotifyError, NormalisationData, NotModifiedRes, PoolConfig, PoolError, Quota, Session, SessionConfig, SpotifyIoPool, @@ -695,7 +698,7 @@ impl SpotifyDownloader { &self, track_id: TrackId<'_>, force: bool, - ) -> Result, Error> { + ) -> Result, Error> { let pid = PlayableId::Track(track_id.as_ref()); let input_id_str = track_id.id().to_owned(); // Check if track was already downloaded @@ -907,7 +910,11 @@ impl SpotifyDownloader { .execute(&self.i.pool) .await?; } - Ok(Some(sp_item)) + Ok(Some(DlAudioItem { + item: sp_item, + subdir_path: subpath, + dl_path: tpath, + })) } /// Download a Spotify track and log the error if one occurs @@ -916,15 +923,19 @@ impl SpotifyDownloader { /// This is the case if the download is either /// - successful /// - unavailable - pub async fn download_track_log(&self, track_id: TrackId<'_>, force: bool) -> bool { + pub async fn download_track_log( + &self, + track_id: TrackId<'_>, + force: bool, + ) -> (bool, Option) { for _ in 0..3 { match self.download_track(track_id.as_ref(), force).await { Ok(sp_item) => { if let Some(sp_item) = &sp_item { - if let UniqueFields::Track(sp_track) = &sp_item.unique_fields { + if let UniqueFields::Track(sp_track) = &sp_item.item.unique_fields { self.i.dl_tracks.write().unwrap().tracks.push(format!( "{} - {}", - sp_item.name, + sp_item.item.name, sp_track .artists .first() @@ -933,7 +944,7 @@ impl SpotifyDownloader { )); } } - return true; + return (true, sp_item); } Err(e) => { tracing::error!("[{track_id}] {e}"); @@ -945,14 +956,14 @@ impl SpotifyDownloader { .push(format!("[{track_id}]: {e}")); if matches!(e, Error::Unavailable(_)) { self.mark_track_unavailable(track_id).await.unwrap(); - return true; + return (true, None); } else if !matches!(e, Error::AudioKey(_)) { - return false; + return (false, None); } } } } - false + (false, None) } pub async fn download_artist( @@ -1053,7 +1064,7 @@ impl SpotifyDownloader { .for_each_concurrent(4, |id| { let success = success.clone(); async move { - let ok = self.download_track_log(id.as_ref(), false).await; + let ok = self.download_track_log(id.as_ref(), false).await.0; if !ok { success.store(false, Ordering::SeqCst); } @@ -1074,6 +1085,80 @@ impl SpotifyDownloader { Ok(success) } + pub async fn download_playlist(&self, playlist_id: PlaylistId<'_>) -> Result<(), Error> { + let playlist_id_str = playlist_id.id(); + let playlist = self + .i + .sp + .spclient()? + .pb_playlist(playlist_id.as_ref(), None) + .await? + .data; + + let track_ids = playlist + .contents + .items + .iter() + .filter_map(|itm| TrackId::from_uri(itm.uri()).ok()) + .collect::>(); + + let success = Arc::new(AtomicBool::new(true)); + + let playlist_data = stream::iter(track_ids.iter()) + .map(|id| { + let success = success.clone(); + let id_str = id.id(); + async move { + let (ok, item) = self.download_track_log(id.as_ref(), false).await; + if ok { + if let Some(item) = item { + return Ok(Some(item.subdir_path)); + } else { + let row = sqlx::query!("select path from tracks where id=$1", id_str) + .fetch_optional(&self.i.pool) + .await?; + if let Some(subdir_path) = row.and_then(|r| r.path) { + let audio_path = path!(self.i.base_dir / subdir_path); + if audio_path.is_file() { + return Ok(Some(path!(subdir_path))); + } else { + tracing::error!( + "[{id}] audio file not found: {}", + audio_path.to_string_lossy() + ); + success.store(false, Ordering::SeqCst); + } + } else { + tracing::error!("[{id}] audio file not found in db"); + success.store(false, Ordering::SeqCst); + } + } + } else { + success.store(false, Ordering::SeqCst); + } + Ok::<_, Error>(None) + } + }) + .buffered(4) + .try_collect::>() + .await? + .into_iter() + .filter_map(|x| x.and_then(|x| path!(".." / x).to_str().map(str::to_owned))) + .join("\n"); + + let pl_dir = path!(self.i.base_dir / "__Playlists"); + std::fs::create_dir_all(&pl_dir)?; + let pl_path = path!( + pl_dir + / better_filenamify( + playlist.attributes.name(), + Some(&format!(" [{playlist_id_str}].m3u")) + ) + ); + std::fs::write(pl_path, playlist_data)?; + Ok(()) + } + pub async fn download_new(&self, update_age_h: u32) -> Result<(), Error> { let date_thr = (OffsetDateTime::now_utc() - time::Duration::hours(update_age_h.into())) .unix_timestamp(); diff --git a/crates/downloader/src/main.rs b/crates/downloader/src/main.rs index 0cf0a85..274e347 100644 --- a/crates/downloader/src/main.rs +++ b/crates/downloader/src/main.rs @@ -13,7 +13,7 @@ use clap::{Parser, Subcommand}; use futures_util::{stream, StreamExt, TryStreamExt}; use indicatif::{MultiProgress, ProgressBar}; use spotifyio::{ - model::{AlbumId, ArtistId, Id, IdConstruct, SearchResult, SearchType, TrackId}, + model::{AlbumId, ArtistId, Id, IdConstruct, PlaylistId, SearchResult, SearchType, TrackId}, ApplicationCache, AuthCredentials, AuthenticationType, Quota, Session, SessionConfig, }; use spotifyio_downloader::{Error, SpotifyDownloader, SpotifyDownloaderConfig}; @@ -113,6 +113,10 @@ enum Commands { #[clap(long)] force: bool, }, + /// Download a playlist + DlPlaylist { + playlist: Vec, + }, ScrapeAudioFeatures, } @@ -337,7 +341,7 @@ async fn download(cli: Cli, multi: MultiProgress) -> Result<(), Error> { let dl = dl.clone(); let success = success.clone(); async move { - let ok = dl.download_track_log(id.as_ref(), force).await; + let ok = dl.download_track_log(id.as_ref(), force).await.0; if !ok { success.store(false, Ordering::SeqCst); } @@ -350,6 +354,29 @@ async fn download(cli: Cli, multi: MultiProgress) -> Result<(), Error> { )); } } + Commands::DlPlaylist { playlist } => { + let playlist_ids = parse_url_ids::(&playlist)?; + let success = Arc::new(AtomicBool::new(true)); + stream::iter(playlist_ids) + .map(Ok::<_, Error>) + .try_for_each_concurrent(8, |id| { + let dl = dl.clone(); + let success = success.clone(); + async move { + if let Err(e) = dl.download_playlist(id.as_ref()).await { + tracing::error!("Error downloading playlist [{id}]: {e}"); + success.store(false, Ordering::SeqCst); + } + Ok(()) + } + }) + .await?; + if !success.load(Ordering::SeqCst) { + return Err(Error::Other( + "not all playlists downloaded successfully".into(), + )); + } + } Commands::ScrapeAudioFeatures => { let n = sqlx::query_scalar!(r#"select count(*) from tracks where audio_features is null"#) diff --git a/crates/downloader/src/model/mod.rs b/crates/downloader/src/model/mod.rs index 074e15f..fd08077 100644 --- a/crates/downloader/src/model/mod.rs +++ b/crates/downloader/src/model/mod.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use serde::Serialize; use spotifyio::{ model::{AlbumId, ArtistId, FileId, IdConstruct, PlayableId, TrackId}, @@ -40,6 +42,12 @@ pub struct AudioItem { pub unique_fields: UniqueFields, } +pub struct DlAudioItem { + pub item: AudioItem, + pub subdir_path: PathBuf, + pub dl_path: PathBuf, +} + #[derive(Debug, Clone)] pub struct TrackUniqueFields { pub artists: Vec, diff --git a/crates/spotifyio/src/connection/proxytunnel.rs b/crates/spotifyio/src/connection/proxytunnel.rs index af51bbb..9e8cd3b 100644 --- a/crates/spotifyio/src/connection/proxytunnel.rs +++ b/crates/spotifyio/src/connection/proxytunnel.rs @@ -22,7 +22,7 @@ pub async fn proxy_connect( loop { let bytes_read = proxy_connection.read(&mut buffer[offset..]).await?; if bytes_read == 0 { - return Err(io::Error::new(io::ErrorKind::Other, "Early EOF from proxy")); + return Err(io::Error::other("Early EOF from proxy")); } offset += bytes_read; @@ -31,7 +31,7 @@ pub async fn proxy_connect( let status = response .parse(&buffer[..offset]) - .map_err(|err| io::Error::new(io::ErrorKind::Other, err))?; + .map_err(io::Error::other)?; if status.is_complete() { return match response.code { @@ -39,12 +39,9 @@ pub async fn proxy_connect( Some(code) => { let reason = response.reason.unwrap_or("no reason"); let msg = format!("Proxy responded with {code}: {reason}"); - Err(io::Error::new(io::ErrorKind::Other, msg)) + Err(io::Error::other(msg)) } - None => Err(io::Error::new( - io::ErrorKind::Other, - "Malformed response from proxy", - )), + None => Err(io::Error::other("Malformed response from proxy")), }; } From daf91278baa7984f00eec24189e67b29952f951e Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Sat, 24 May 2025 18:37:34 +0200 Subject: [PATCH 12/12] chore(deps): update crate oauth2 to 5.0.0 --- crates/spotifyio/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/spotifyio/Cargo.toml b/crates/spotifyio/Cargo.toml index 0f87ed5..71bdfec 100644 --- a/crates/spotifyio/Cargo.toml +++ b/crates/spotifyio/Cargo.toml @@ -62,7 +62,7 @@ rand = "0.8" rsa = "0.9.2" httparse = "1.7" base64 = "0.22" -oauth2 = { version = "5.0.0-rc.1", default-features = false, features = [ +oauth2 = { version = "5.0.0", default-features = false, features = [ "reqwest", ], optional = true } pin-project-lite = "0.2"