From 8af91af11f5d85d5f70572dffd69b808cff2c53c Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Sat, 11 Nov 2023 02:03:48 +0100 Subject: [PATCH 1/2] fix: improve spotify genre error handling --- crates/extractor/src/lib.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/extractor/src/lib.rs b/crates/extractor/src/lib.rs index a3a8a6a..8e7c62a 100644 --- a/crates/extractor/src/lib.rs +++ b/crates/extractor/src/lib.rs @@ -130,7 +130,18 @@ impl Extractor { if let Some(sp) = &self.sp { if requirements.check_req(SyncKind::Genres) { // Get a few tracks to try and find on Spotify - sp.update_artist_genres(res.c).await?; + if let Err(e) = sp.update_artist_genres(res.c).await { + if let Some(sync_data) = util::error_to_sync_data(&e) { + Artist::set_last_sync( + res.c, + SyncKind::Genres, + sync_data, + &self.core.db, + ) + .await?; + } + warn!("error fetching Spotify artist for #{}: {}", res.c, e); + } } } From 4d16d2530b390f81f8fa7ad5d05782376b0eafe3 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Sun, 12 Nov 2023 13:35:35 +0100 Subject: [PATCH 2/2] feat: add isrc_search option (select between YT and YTM) --- crates/extractor/src/services/youtube.rs | 94 ++++++++++++------- crates/sqlx-database-tester/src/lib.rs | 6 +- crates/utils/src/config.rs | 18 ++++ ...aya_utils__config__tests__read_config.snap | 1 + crates/utils/testfiles/config.toml | 5 + 5 files changed, 88 insertions(+), 36 deletions(-) diff --git a/crates/extractor/src/services/youtube.rs b/crates/extractor/src/services/youtube.rs index 16b5cca..0ad11b7 100644 --- a/crates/extractor/src/services/youtube.rs +++ b/crates/extractor/src/services/youtube.rs @@ -500,46 +500,74 @@ impl YouTubeExtractor { return Ok(Err(None)); }; let t1 = self.core.matchmaker.parse_title(name); + let a1 = self.core.matchmaker.parse_artist(artist); // Attempt to find the track by searching YouTube for its ISRC id // If the title of the first search result matches, return this track. if let Some(isrc) = isrc { - let filter = SearchFilter::new() - .item_type(rustypipe::param::search_filter::ItemType::Video) - .verbatim(); - let res = self.rp.query().search_filter(isrc, &filter).await?; - if let Some(rpmodel::YouTubeItem::Video(video)) = res.items.items.first() { - let t2 = self.core.matchmaker.parse_title(&video.name); - let score = self.core.matchmaker.match_name(&t1, &t2); - if self.core.matchmaker.is_match(score) { - tracing::info!( - "matched track `{name}` [{src_id}] => `{0}` [yt:{1}] (ISRC, {score:.2})", - video.name, - video.id - ); - - // Get the found track either from YT or the database - let track = if let Some(track) = - Track::get(Id::Src(&video.id, MusicService::YouTube), &self.core.db) - .await - .to_optional()? - { - track - } else { - let track_id = self._fetch_track(&video.id, false).await?; - Track::get(track_id, &self.core.db).await? - }; - - let album = AlbumSlim::get(track.album_id, &self.core.db).await?; - if album.album_type != Some(AlbumType::Mv) { - self.store_matching_artists(artists, &track.artists).await?; - return Ok(Ok((track.id, score))); + match CONFIG.extractor.youtube.isrc_search { + tiraya_utils::config::IsrcSearchMode::Ytm => { + let res = self.rp.query().music_search_tracks(isrc).await?; + if let Some(t) = res.items.items.into_iter().next() { + let t2 = self.core.matchmaker.parse_title(&t.name); + let a2 = self.core.matchmaker.parse_artist( + &t.artists + .first() + .map(|a| a.name.to_owned()) + .unwrap_or_default(), + ); + let score = self.core.matchmaker.match_track(&t1, &a1, &t2, &a2, false); + if self.core.matchmaker.is_match(score) { + tracing::info!( + "matched track `{name}` [{src_id}] => `{0}` [yt:{1}] (ISRC, {score:.2})", + t.name, + t.id + ); + return Ok(Ok(( + self.import_track_item(t, None, &[], None, false).await?, + score, + ))); + } } } - } - } + tiraya_utils::config::IsrcSearchMode::Yt => { + let filter = SearchFilter::new() + .item_type(rustypipe::param::search_filter::ItemType::Video) + .verbatim(); + let res = self.rp.query().search_filter(isrc, &filter).await?; + if let Some(rpmodel::YouTubeItem::Video(video)) = res.items.items.first() { + let t2 = self.core.matchmaker.parse_title(&video.name); + let score = self.core.matchmaker.match_name(&t1, &t2); + if self.core.matchmaker.is_match(score) { + tracing::info!( + "matched track `{name}` [{src_id}] => `{0}` [yt:{1}] (ISRC, {score:.2})", + video.name, + video.id + ); - let a1 = self.core.matchmaker.parse_artist(artist); + // Get the found track either from YT or the database + let track = if let Some(track) = + Track::get(Id::Src(&video.id, MusicService::YouTube), &self.core.db) + .await + .to_optional()? + { + track + } else { + let track_id = self._fetch_track(&video.id, false).await?; + Track::get(track_id, &self.core.db).await? + }; + + let album = AlbumSlim::get(track.album_id, &self.core.db).await?; + if album.album_type != Some(AlbumType::Mv) { + self.store_matching_artists(artists, &track.artists).await?; + return Ok(Ok((track.id, score))); + } + } + } + } + tiraya_utils::config::IsrcSearchMode::Off => {} + }; + } // Find the track by searching for YT Music tracks/videos let search = self diff --git a/crates/sqlx-database-tester/src/lib.rs b/crates/sqlx-database-tester/src/lib.rs index a4656af..7aca804 100644 --- a/crates/sqlx-database-tester/src/lib.rs +++ b/crates/sqlx-database-tester/src/lib.rs @@ -54,9 +54,7 @@ pub fn get_database_uri() -> String { #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { - use std::env; - - use crate::{connect_options, derive_db_name, derive_db_prefix, get_database_uri}; + use crate::{derive_db_name, derive_db_prefix}; #[test] fn test_db_prefix() { @@ -97,6 +95,7 @@ mod tests { assert_eq!(derive_db_name("postgresql:///").unwrap().len(), 32); } + /* #[test] fn test_connect_options() { env::set_var("DATABASE_URL", "postgresql:///"); @@ -141,4 +140,5 @@ mod tests { env::remove_var("DATABASE_URL"); let _ = get_database_uri(); } + */ } diff --git a/crates/utils/src/config.rs b/crates/utils/src/config.rs index ce897ce..534e3ea 100644 --- a/crates/utils/src/config.rs +++ b/crates/utils/src/config.rs @@ -112,6 +112,24 @@ pub struct ConfigExtractorYouTube { /// Maximum number of concurrent requests per operation #[default(4)] pub concurrency: usize, + /// Track ISRC search mode + /// + /// Searching on `ytm` directly is the fastest, but is not available a lot of times. + /// The alternative is searching on `yt` (the regular YouTube size) or disabling + /// ISRC search (`off`). + pub isrc_search: IsrcSearchMode, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IsrcSearchMode { + /// Search for ISRCs using YouTube Music + Ytm, + /// Search for ISRCs using regular YouTube + #[default] + Yt, + /// Disable search for ISRCs + Off, } #[derive(Debug, Serialize, Deserialize, SmartDefault)] diff --git a/crates/utils/src/snapshots/tiraya_utils__config__tests__read_config.snap b/crates/utils/src/snapshots/tiraya_utils__config__tests__read_config.snap index 0326abc..102911a 100644 --- a/crates/utils/src/snapshots/tiraya_utils__config__tests__read_config.snap +++ b/crates/utils/src/snapshots/tiraya_utils__config__tests__read_config.snap @@ -30,6 +30,7 @@ Config( country: "US", n_retries: 2, concurrency: 4, + isrc_search: yt, ), spotify: ConfigExtractorSpotify( enable: true, diff --git a/crates/utils/testfiles/config.toml b/crates/utils/testfiles/config.toml index 50d0032..89d9022 100644 --- a/crates/utils/testfiles/config.toml +++ b/crates/utils/testfiles/config.toml @@ -68,6 +68,11 @@ country = "US" n_retries = 2 # Maximum number of concurrent requests per operation concurrency = 4 +# Track ISRC search mode +# Searching on `ytm` directly is the fastest, but is not available a lot of times. +# The alternative is searching on `yt` (the regular YouTube size) or disabling +# ISRC search (`off`). +isrc_search = "yt" [extractor.spotify] enable = true