Compare commits

...

2 commits

Author SHA1 Message Date
b17c58f523 feat: add Spotify search 2023-11-18 03:19:02 +01:00
097e3e0612 feat: add search endpoint 2023-11-18 02:34:22 +01:00
26 changed files with 706 additions and 51 deletions

3
Cargo.lock generated
View file

@ -2473,7 +2473,7 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
[[package]]
name = "rustypipe"
version = "0.1.0"
source = "git+https://code.thetadev.de/ThetaDev/rustypipe.git#48ccfc5c067a55d7a727da61aadc0830981ec6d4"
source = "git+https://code.thetadev.de/ThetaDev/rustypipe.git?rev=8458d878e7#8458d878e712e19e38844ee2e7dc73ce4cd5aa4a"
dependencies = [
"base64 0.21.5",
"fancy-regex",
@ -3382,6 +3382,7 @@ dependencies = [
"tiraya-utils",
"tokio",
"tracing",
"url",
"uuid",
]

View file

@ -96,7 +96,7 @@ reqwest = { version = "0.11.11", default-features = false, features = [
] }
rustypipe = { git = "https://code.thetadev.de/ThetaDev/rustypipe.git", features = [
"rss",
] }
], rev = "8458d878e7" }
rspotify = { version = "0.12.0", default-features = false, features = [
"client-reqwest",
"reqwest-rustls-tls",

View file

@ -3,6 +3,8 @@ use serde::{Deserialize, Serialize};
#[cfg(feature = "utoipa")]
use utoipa::ToSchema;
use crate::{AlbumSlim, ArtistSlim, PlaylistSlim, TrackSlim, UserSlim};
/// 2-letter codes identifying a specific streaming service
#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[cfg_attr(feature = "utoipa", derive(ToSchema))]
@ -92,3 +94,30 @@ pub enum SyncKind {
/// Synchronization of the artist's genres
Genres,
}
/// Search for items of a specific type
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[cfg_attr(feature = "utoipa", derive(ToSchema))]
#[serde(rename_all = "snake_case")]
#[allow(missing_docs)]
pub enum SearchFilterType {
Artist,
Album,
Track,
Video,
Playlist,
User,
}
/// Tiraya item of any type
#[derive(Debug, Clone, Serialize, Deserialize)]
#[cfg_attr(feature = "utoipa", derive(ToSchema))]
#[serde(rename_all = "snake_case")]
#[allow(missing_docs)]
pub enum TirayaItem {
Artist(ArtistSlim),
Album(AlbumSlim),
Track(TrackSlim),
Playlist(PlaylistSlim),
User(UserSlim),
}

View file

@ -11,6 +11,7 @@ mod error;
mod genre;
mod id;
mod playlist;
mod search;
mod serializer;
mod track;
mod user;
@ -23,5 +24,6 @@ pub use error::*;
pub use genre::{Genre, TreeGenre};
pub use id::*;
pub use playlist::*;
pub use search::*;
pub use track::*;
pub use user::*;

View file

@ -0,0 +1,19 @@
use serde::{Deserialize, Serialize};
#[cfg(feature = "utoipa")]
use utoipa::ToSchema;
use crate::TirayaItem;
/// Search result model
#[derive(Debug, Clone, Serialize, Deserialize)]
#[cfg_attr(feature = "utoipa", derive(ToSchema))]
pub struct SearchResult {
/// Found items
pub items: Vec<TirayaItem>,
/// Continuation token
///
/// The continuation token has to be included in the next request to fetch
/// subsequent results
pub ctoken: Option<String>,
}

View file

@ -11,8 +11,9 @@ use spotify_genrebase::GenreDb;
use sqlx::PgPool;
use tiraya_api_model::{
Album, AlbumSlim, AlbumTag, AlbumType, ApiError, ApiErrorKind, Artist, ArtistSlim, ArtistTag,
DatePrecision, Genre, MusicService, Playlist, PlaylistEntry, PlaylistSlim, PlaylistType, TId,
Track, TrackPlaybackInfo, TrackSlim, TreeGenre, User, UserSlim, UserType,
DatePrecision, Genre, MusicService, Playlist, PlaylistEntry, PlaylistSlim, PlaylistType,
SearchFilterType, SearchResult, TId, Track, TrackPlaybackInfo, TrackSlim, TreeGenre, User,
UserSlim, UserType,
};
use tiraya_extractor::Extractor;
use tiraya_proxy::Proxy;
@ -54,11 +55,13 @@ use utoipa::OpenApi;
UserSlim,
Genre,
TreeGenre,
SearchResult,
TId,
MusicService,
DatePrecision,
AlbumType,
UserType,
SearchFilterType,
ApiError,
ApiErrorKind,
))
@ -141,6 +144,7 @@ pub async fn serve() -> Result<(), anyhow::Error> {
"/user/:id/playlists",
routing::get(routes::user::get_user_playlists),
)
.route("/search", routing::get(routes::search::search))
.fallback(|| async {
crate::error::ApiError::NotFound("API endpoint not found".into())
}),

View file

@ -3,7 +3,7 @@ use std::collections::HashMap;
use itertools::Itertools;
use spotify_genrebase::GenreDb;
use tiraya_api_model::{Artist, Genre};
use tiraya_api_model::{Artist, Genre, SearchResult, TirayaItem};
use tiraya_db::models as tdb;
use tiraya_utils::ImageKind;
@ -14,12 +14,20 @@ pub fn map_artist(
lang: Option<&str>,
) -> Artist {
let id = tiraya_api_model::TId::new(a.src_id, a.service.into());
let image_url = a
.image_date
.map(|dt| tiraya_utils::image_url_local(id.as_ref(), ImageKind::Artist, dt, false));
let header_image_url = a
.header_image_date
.map(|dt| tiraya_utils::image_url_local(id.as_ref(), ImageKind::ArtistHeader, dt, false));
let image_url = tiraya_utils::map_image_url(
a.image_url,
a.image_date,
id.as_ref(),
ImageKind::Artist,
false,
);
let header_image_url = tiraya_utils::map_image_url(
a.header_image_url,
a.header_image_date,
id.as_ref(),
ImageKind::ArtistHeader,
false,
);
Artist {
id,
@ -92,6 +100,36 @@ fn capitalize(s: &str) -> String {
.join(" ")
}
pub fn map_search_filter_type(
filter: tiraya_api_model::SearchFilterType,
) -> tiraya_extractor::model::SearchFilterType {
match filter {
tiraya_api_model::SearchFilterType::Artist => {
tiraya_extractor::model::SearchFilterType::Artist
}
tiraya_api_model::SearchFilterType::Album => {
tiraya_extractor::model::SearchFilterType::Album
}
tiraya_api_model::SearchFilterType::Track => {
tiraya_extractor::model::SearchFilterType::Track
}
tiraya_api_model::SearchFilterType::Video => {
tiraya_extractor::model::SearchFilterType::Video
}
tiraya_api_model::SearchFilterType::Playlist => {
tiraya_extractor::model::SearchFilterType::Playlist
}
tiraya_api_model::SearchFilterType::User => tiraya_extractor::model::SearchFilterType::User,
}
}
pub fn map_search_result(v: tiraya_extractor::model::SearchResult) -> SearchResult {
SearchResult {
items: v.items.into_iter().map(TirayaItem::from).collect(),
ctoken: v.ctoken,
}
}
#[cfg(test)]
mod tests {
use tiraya_utils::config::CONFIG;

View file

@ -2,5 +2,6 @@ pub mod album;
pub mod artist;
pub mod image;
pub mod playlist;
pub mod search;
pub mod track;
pub mod user;

View file

@ -0,0 +1,53 @@
use axum::extract::State;
use serde::Deserialize;
use tiraya_api_model::{MusicService, SearchFilterType, SearchResult};
use tiraya_db::models::{self as tdb};
use crate::{
error::ApiError,
extract::{Json, Query},
mapper, ApiState,
};
#[derive(Default, Deserialize)]
#[serde(default)]
pub struct SearchQuery {
query: String,
filter_srv: Option<MusicService>,
filter_type: Option<SearchFilterType>,
ctoken: Option<String>,
}
/// Search
#[utoipa::path(
get,
path = "/search",
params(
("query" = String, Query, description = "Search query"),
("filter_srv" = Option<MusicService>, Query, description = "Search a specific service"),
("filter_type" = Option<SearchFilterType>, Query, description = "Filter items of a specific type"),
("ctoken" = Option<String>, Query, description = "Continuation token"),
),
tag = "Search",
responses(
(status = 200, description = "Returns the search result", body = SearchResult),
(status = 400, description = "Bad request", body = ApiError),
(status = 500, description = "Internal error", body = ApiError),
)
)]
pub async fn search(
State(state): State<ApiState>,
Query(query): Query<SearchQuery>,
) -> Result<Json<SearchResult>, ApiError> {
let res = state
.xtr
.search(
&query.query,
query.filter_type.map(mapper::map_search_filter_type),
query.filter_srv.map(tdb::MusicService::from),
query.ctoken,
)
.await?;
Ok(Json(mapper::map_search_result(res)))
}

View file

@ -616,9 +616,13 @@ impl From<AlbumSlimRow> for AlbumSlim {
impl From<Album> for tiraya_api_model::Album {
fn from(b: Album) -> Self {
let id = tiraya_api_model::TId::new(b.src_id, b.service.into());
let image_url = b
.image_date
.map(|dt| tiraya_utils::image_url_local(id.as_ref(), ImageKind::Album, dt, false));
let image_url = tiraya_utils::map_image_url(
b.image_url,
b.image_date,
id.as_ref(),
ImageKind::Album,
false,
);
Self {
id,
@ -644,9 +648,13 @@ impl From<Album> for tiraya_api_model::Album {
impl From<AlbumSlim> for tiraya_api_model::AlbumSlim {
fn from(b: AlbumSlim) -> Self {
let id = tiraya_api_model::TId::new(b.src_id, b.service.into());
let image_url = b
.image_date
.map(|dt| tiraya_utils::image_url_local(id.as_ref(), ImageKind::Album, dt, false));
let image_url = tiraya_utils::map_image_url(
b.image_url,
b.image_date,
id.as_ref(),
ImageKind::Album,
false,
);
Self {
id,
@ -666,9 +674,13 @@ impl From<AlbumSlim> for tiraya_api_model::AlbumSlim {
impl From<AlbumTag> for tiraya_api_model::AlbumTag {
fn from(b: AlbumTag) -> Self {
let id = tiraya_api_model::TId::new(b.src_id, b.service.into());
let image_url = b
.image_date
.map(|dt| tiraya_utils::image_url_local(id.as_ref(), ImageKind::Album, dt, false));
let image_url = tiraya_utils::map_image_url(
b.image_url,
b.image_date,
id.as_ref(),
ImageKind::Album,
false,
);
Self {
id,

View file

@ -938,9 +938,13 @@ impl From<ArtistSyncEntry> for tiraya_api_model::SyncEntry {
impl From<ArtistSlim> for tiraya_api_model::ArtistSlim {
fn from(a: ArtistSlim) -> Self {
let id = tiraya_api_model::TId::new(a.src_id, a.service.into());
let image_url = a
.image_date
.map(|dt| tiraya_utils::image_url_local(id.as_ref(), ImageKind::Artist, dt, false));
let image_url = tiraya_utils::map_image_url(
a.image_url,
a.image_date,
id.as_ref(),
ImageKind::Artist,
false,
);
Self {
id,

View file

@ -1,6 +1,8 @@
use serde::{Deserialize, Serialize};
use time::{Duration, OffsetDateTime};
use super::{AlbumSlim, ArtistSlim, PlaylistSlim, TrackSlim, UserSlim};
#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, sqlx::Type)]
#[sqlx(type_name = "music_service")]
pub enum MusicService {
@ -32,6 +34,17 @@ impl From<MusicService> for tiraya_api_model::MusicService {
}
}
impl From<tiraya_api_model::MusicService> for MusicService {
fn from(value: tiraya_api_model::MusicService) -> Self {
match value {
tiraya_api_model::MusicService::Tiraya => Self::Tiraya,
tiraya_api_model::MusicService::YouTube => Self::YouTube,
tiraya_api_model::MusicService::Spotify => Self::Spotify,
tiraya_api_model::MusicService::Musixmatch => Self::Musixmatch,
}
}
}
#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, sqlx::Type)]
#[serde(rename_all = "snake_case")]
#[sqlx(type_name = "album_type", rename_all = "snake_case")]
@ -195,6 +208,28 @@ impl SyncError {
}
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TirayaItem {
Artist(ArtistSlim),
Album(AlbumSlim),
Track(TrackSlim),
Playlist(PlaylistSlim),
User(UserSlim),
}
impl From<TirayaItem> for tiraya_api_model::TirayaItem {
fn from(value: TirayaItem) -> Self {
match value {
TirayaItem::Artist(a) => Self::Artist(a.into()),
TirayaItem::Album(b) => Self::Album(b.into()),
TirayaItem::Track(t) => Self::Track(t.into()),
TirayaItem::Playlist(p) => Self::Playlist(p.into()),
TirayaItem::User(u) => Self::User(u.into()),
}
}
}
#[cfg(test)]
mod tests {
use time::macros::datetime;

View file

@ -15,7 +15,7 @@ pub use artist::{
};
pub use enums::{
AlbumType, DatePrecision, MusicService, PlaylistImgType, PlaylistType, SyncData, SyncError,
SyncKind, UserType,
SyncKind, TirayaItem, UserType,
};
pub use id::{Id, IdLike, IdOwned, InternalId, TId, TIdOwned};
pub use playlist::{

View file

@ -723,9 +723,13 @@ where p.src_id=$1 and p.service=$2"#,
impl From<PlaylistSlim> for tiraya_api_model::PlaylistSlim {
fn from(p: PlaylistSlim) -> Self {
let id = tiraya_api_model::TId::new(p.src_id, p.service.into());
let image_url = p
.image_date
.map(|dt| tiraya_utils::image_url_local(id.as_ref(), ImageKind::Playlist, dt, false));
let image_url = tiraya_utils::map_image_url(
p.image_url,
p.image_date,
id.as_ref(),
ImageKind::Playlist,
false,
);
Self {
id,
name: p.name,

View file

@ -377,9 +377,13 @@ where src_id=$1 and service=$2"#,
impl From<User> for tiraya_api_model::User {
fn from(u: User) -> Self {
let id = tiraya_api_model::TId::new(u.src_id, u.service.into());
let image_url = u
.image_date
.map(|dt| tiraya_utils::image_url_local(id.as_ref(), ImageKind::User, dt, false));
let image_url = tiraya_utils::map_image_url(
u.image_url,
u.image_date,
id.as_ref(),
ImageKind::User,
false,
);
Self {
id,

View file

@ -31,6 +31,7 @@ time.workspace = true
tokio.workspace = true
tracing.workspace = true
uuid.workspace = true
url.workspace = true
tiraya-db.workspace = true
tiraya-utils.workspace = true

View file

@ -15,6 +15,8 @@ pub enum ExtractorError {
Database(#[from] tiraya_db::error::DatabaseError),
#[error("cannot fetch {typ} from {srv}")]
Unsupported { typ: EntityType, srv: MusicService },
#[error("invalid input: {0}")]
Input(Cow<'static, str>),
#[error("got invalid data: [{id}] {msg}")]
InvalidData {
id: TIdOwned,
@ -129,7 +131,9 @@ impl ErrorStatus for ExtractorError {
ExtractorError::Unavailable { .. } | ExtractorError::NoMatch { .. } => {
StatusCode::NOT_FOUND
}
ExtractorError::Unsupported { .. } => StatusCode::BAD_REQUEST,
ExtractorError::Unsupported { .. } | ExtractorError::Input(_) => {
StatusCode::BAD_REQUEST
}
ExtractorError::Database(e) => e.status(),
_ => StatusCode::INTERNAL_SERVER_ERROR,
}
@ -141,7 +145,7 @@ impl ErrorStatus for ExtractorError {
| ExtractorError::NoMatch { .. }
| ExtractorError::InvalidData { .. }
| ExtractorError::Source(_) => ApiErrorKind::Src,
ExtractorError::Unsupported { .. } => ApiErrorKind::User,
ExtractorError::Unsupported { .. } | ExtractorError::Input(_) => ApiErrorKind::User,
ExtractorError::Database(e) => e.kind(),
_ => ApiErrorKind::Other,
}

View file

@ -1,13 +1,12 @@
#![warn(clippy::dbg_macro, clippy::todo)]
pub mod error;
mod model;
pub mod model;
mod services;
mod util;
use error::TIdParseError;
use model::SyncRequirements;
pub use model::{GetResult, GetStatus};
use model::{GetResult, GetStatus, SearchFilterType, SearchResult, SyncRequirements};
use services::SpotifyExtractor;
use short_uuid::{Base58Flickr, Base62};
use tiraya_utils::{config::CONFIG, EntityType};
@ -49,6 +48,7 @@ impl std::ops::Deref for Extractor {
}
}
pub(crate) const IMG_SIZE_SEARCH: u32 = 64;
const MSG_NOT_EXIST: &str = "does not exist";
impl Extractor {
@ -542,6 +542,25 @@ impl Extractor {
}
Ok(())
}
#[tracing::instrument(skip(self))]
pub async fn search(
&self,
query: &str,
type_filter: Option<SearchFilterType>,
service_filter: Option<MusicService>,
ctoken: Option<String>,
) -> Result<SearchResult, ExtractorError> {
let srv = service_filter.unwrap_or(MusicService::YouTube);
match srv {
// MusicService::Tiraya => todo!(),
MusicService::YouTube => self.yt.search(query, type_filter, ctoken).await,
MusicService::Spotify => self.sp()?.search(query, type_filter, ctoken).await,
_ => Err(ExtractorError::Input(
format!("search not available for {srv}").into(),
)),
}
}
}
/// Parse and validate a Tiraya entity id (TId)

View file

@ -3,7 +3,7 @@ use std::collections::{HashMap, HashSet};
use quick_cache::sync::Cache;
use sqlx::{Pool, Postgres};
use time::{Duration, OffsetDateTime};
use tiraya_db::models::{ArtistSyncData, SyncData, SyncKind, TIdOwned};
use tiraya_db::models::{ArtistSyncData, SyncData, SyncKind, TIdOwned, TirayaItem};
use tiraya_utils::{config::CONFIG, EntityType};
use crate::{error::ExtractorError, util::matchmaker::Matchmaker};
@ -17,14 +17,30 @@ pub struct ExtractorCore {
pub artist_playlist_excluded_types: HashSet<usize>,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum SearchFilterType {
Artist,
Album,
Track,
Video,
Playlist,
User,
}
#[derive(Debug, Default)]
pub struct SearchResult {
pub items: Vec<TirayaItem>,
pub ctoken: Option<String>,
}
#[derive(Default)]
pub struct SyncRequirements {
pub(crate) struct SyncRequirements {
pub id: Option<i32>,
pub kinds: HashMap<SyncKind, SyncRequirement>,
}
/// Whether an item needs to be synchronized
pub enum SyncRequirement {
pub(crate) enum SyncRequirement {
/// Data is up to date
Current,
/// Data is outdated and has to be updated
@ -32,7 +48,7 @@ pub enum SyncRequirement {
}
#[derive(Debug)]
pub struct SyncLastUpdate {
pub(crate) struct SyncLastUpdate {
pub id: i32,
pub state: LastUpdateState,
}
@ -163,7 +179,7 @@ impl<T> GetResult<T> {
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub enum LastUpdateState {
pub(crate) enum LastUpdateState {
#[default]
Empty,
Version(String),

View file

@ -52,6 +52,14 @@ pub(crate) trait ServiceExtractor {
src_id: &str,
last_update: Option<&SyncLastUpdate>,
) -> Result<GetResult<i32>, ExtractorError>;
/// Search the streaming service for content
async fn search(
&self,
query: &str,
type_filter: Option<SearchFilterType>,
ctoken: Option<String>,
) -> Result<SearchResult, ExtractorError>;
}
*/

View file

@ -1,3 +1,5 @@
use std::{collections::HashSet, sync::Arc};
use futures::{stream::TryStreamExt, StreamExt};
use path_macro::path;
use rspotify::{
@ -8,19 +10,21 @@ use rspotify::{
prelude::{BaseClient, Id},
ClientCredsSpotify,
};
use std::{collections::HashSet, sync::Arc};
use time::OffsetDateTime;
use tiraya_db::models::{
Artist, ArtistTag, ArtistUpdate, MusicService, Playlist, PlaylistEntry, PlaylistImgType,
PlaylistNew, PlaylistType, SyncData, SyncError, SyncKind, TId, TIdOwned, Track,
TrackMatchAlbum, TrackMatchMeta, User, UserNew, UserType,
PlaylistNew, PlaylistSlim, PlaylistType, SyncData, SyncError, SyncKind, TId, TIdOwned,
TirayaItem, Track, TrackMatchAlbum, TrackMatchMeta, User, UserNew, UserType,
};
use tiraya_utils::{config::CONFIG, EntityType};
use tracing::{debug, info};
use url::Url;
use crate::{
error::ExtractorError,
model::{ExtractorCore, GetResult, LastUpdateState, SyncLastUpdate},
model::{
ExtractorCore, GetResult, LastUpdateState, SearchFilterType, SearchResult, SyncLastUpdate,
},
};
use super::YouTubeExtractor;
@ -347,6 +351,79 @@ impl SpotifyExtractor {
Ok(())
}
pub async fn search(
&self,
query: &str,
type_filter: Option<SearchFilterType>,
ctoken: Option<String>,
) -> Result<SearchResult, ExtractorError> {
self.load_token().await?;
if !matches!(type_filter, Some(SearchFilterType::Playlist) | None) {
return Err(ExtractorError::Input(
"Spotify only supports searching playlists".into(),
));
}
let offset = match ctoken {
Some(ctoken) => Some(ctoken.parse().map_err(|_| {
ExtractorError::Input("Spotify ctoken must be numeric offset".into())
})?),
None => None,
};
let res = self
.sp
.search(
query,
rspotify::model::SearchType::Playlist,
Some(self.market),
None,
Some(20),
offset,
)
.await?;
if let rspotify::model::SearchResult::Playlists(playlists) = res {
let items = playlists
.items
.into_iter()
.map(|p| {
TirayaItem::Playlist(PlaylistSlim {
src_id: p.id.id().to_owned(),
service: MusicService::Spotify,
name: p.name,
image_url: p
.images
.into_iter()
.min_by_key(|img| {
img.width
.map(|w| w.abs_diff(crate::IMG_SIZE_SEARCH))
.unwrap_or(u32::MAX)
})
.map(|img| img.url),
image_date: None,
owner_src_id: Some(p.owner.id.id().to_owned()),
owner_service: Some(MusicService::Spotify),
owner_name: p.owner.display_name,
})
})
.collect();
let ctoken = playlists.next.and_then(|url_str| {
let url = Url::parse(&url_str).ok()?;
tiraya_utils::url_param(&url, "offset")
});
Ok(SearchResult { items, ctoken })
} else {
Err(ExtractorError::InvalidData {
id: TIdOwned(query.to_owned(), MusicService::Spotify),
msg: "Spotify did not return playlists".into(),
})
}
}
/// Find the Spotify ID of a local artist
async fn match_local_artist(&self, id: i32) -> Result<Option<String>, ExtractorError> {
// Get a few tracks to try and find on Spotify

View file

@ -4,13 +4,16 @@ use std::sync::Arc;
use futures::{StreamExt, TryStreamExt};
use path_macro::path;
use rustypipe::model as rpmodel;
use rustypipe::model::{self as rpmodel};
use rustypipe::param::search_filter::MusicSearchFilter;
use rustypipe::{
client::RustyPipe, model::richtext::ToPlaintext, param::search_filter::SearchFilter,
report::FileReporter,
};
use time::{Date, OffsetDateTime, PrimitiveDateTime, Time};
use tiraya_db::models::{AlbumSlim, SyncKind};
use tiraya_db::models::{
AlbumSlim, AlbumTag, ArtistSlim, PlaylistSlim, SyncKind, TirayaItem, TrackSlim,
};
use tiraya_db::{
error::OptionalRes,
models::{
@ -22,6 +25,7 @@ use tiraya_db::{
};
use tiraya_utils::{config::CONFIG, EntityType};
use crate::model::{SearchFilterType, SearchResult};
use crate::util::ArtistSrcidName;
use crate::{
error::{ExtractorError, MapSrcError},
@ -30,6 +34,8 @@ use crate::{
};
const YTM_CHANNEL_SUFFIX: &str = " - Topic";
const YTM_USER: TId = TId("music", MusicService::YouTube);
const YTM_USER_NAME: &str = "YouTube Music";
/// Tiraya extractor for YouTube Music
#[derive(Clone)]
@ -419,6 +425,294 @@ impl YouTubeExtractor {
Ok(GetResult::fetched(user_id))
}
pub async fn search(
&self,
query: &str,
type_filter: Option<SearchFilterType>,
ctoken: Option<String>,
) -> Result<SearchResult, ExtractorError> {
if let Some(ctoken) = ctoken {
if let Some(type_filter) = type_filter {
return self.search_continuation(type_filter, ctoken).await;
} else {
return Err(ExtractorError::Input(
"unfiltered YTM searches have no continuation".into(),
));
}
}
match type_filter {
Some(SearchFilterType::Video) => {
let flt = rustypipe::param::search_filter::SearchFilter::new()
.item_type(rustypipe::param::search_filter::ItemType::Video);
let res = self
.rp
.query()
.search_filter::<rpmodel::VideoItem, _>(query, &flt)
.await?;
Ok(SearchResult {
items: res
.items
.items
.into_iter()
.map(map_video_item)
.map(TirayaItem::Track)
.collect(),
ctoken: res.items.ctoken,
})
}
Some(SearchFilterType::User) => Err(ExtractorError::Unsupported {
typ: EntityType::User,
srv: MusicService::YouTube,
}),
Some(type_filter) => {
let filters = match type_filter {
SearchFilterType::Artist => vec![MusicSearchFilter::Artists],
SearchFilterType::Album => vec![MusicSearchFilter::Albums],
SearchFilterType::Track => vec![MusicSearchFilter::Tracks],
SearchFilterType::Playlist => vec![
MusicSearchFilter::YtmPlaylists,
MusicSearchFilter::CommunityPlaylists,
],
SearchFilterType::User | SearchFilterType::Video => unreachable!(),
};
let mut search_res = SearchResult::default();
let results = futures::stream::iter(filters)
.map(|filter| async move {
self.rp
.query()
.music_search::<rpmodel::MusicItem, _>(query, Some(filter))
.await
})
.boxed()
.buffered(CONFIG.extractor.youtube.concurrency)
.try_collect::<Vec<_>>()
.await?;
for res in results {
search_res
.items
.extend(res.items.items.into_iter().map(map_tiraya_item));
store_ctoken(res.items.ctoken, &mut search_res.ctoken);
}
Ok(search_res)
}
None => {
let res = self.rp.query().music_search_main(query).await?;
Ok(SearchResult {
items: res.items.items.into_iter().map(map_tiraya_item).collect(),
ctoken: res.items.ctoken,
})
}
}
}
async fn search_continuation(
&self,
type_filter: SearchFilterType,
ctoken: String,
) -> Result<SearchResult, ExtractorError> {
if type_filter == SearchFilterType::Video {
let res = self
.rp
.query()
.continuation::<rpmodel::VideoItem, _>(
&ctoken,
rpmodel::paginator::ContinuationEndpoint::Search,
None,
)
.await?;
Ok(SearchResult {
items: res
.items
.into_iter()
.map(map_video_item)
.map(TirayaItem::Track)
.collect(),
ctoken: res.ctoken,
})
} else {
let mut search_res = SearchResult::default();
let ctokens = ctoken
.split('|')
.filter(|t| !t.is_empty())
.collect::<Vec<_>>();
if ctokens.len() > 3 {
return Err(ExtractorError::Input(
"more than 3 continuation tokens".into(),
));
}
let results = futures::stream::iter(ctokens)
.map(|tkn| async move {
self.rp
.query()
.continuation::<rustypipe::model::MusicItem, _>(
tkn,
rpmodel::paginator::ContinuationEndpoint::MusicSearch,
None,
)
.await
})
.boxed()
.buffered(CONFIG.extractor.youtube.concurrency)
.try_collect::<Vec<_>>()
.await?;
for pag in results {
search_res
.items
.extend(pag.items.into_iter().map(map_tiraya_item));
store_ctoken(pag.ctoken, &mut search_res.ctoken);
}
Ok(search_res)
}
}
}
fn map_artist_slim(a: rpmodel::ArtistItem) -> ArtistSlim {
ArtistSlim {
src_id: a.id,
service: MusicService::YouTube,
name: a.name,
image_url: util::yt_image_closest(&a.avatar, crate::IMG_SIZE_SEARCH),
image_date: None,
}
}
fn map_artist_tag(a: rpmodel::ArtistId) -> ArtistTag {
ArtistTag {
id: a.id.map(|id| TIdOwned(id, MusicService::YouTube)),
name: a.name,
}
}
fn map_album_slim(b: rpmodel::AlbumItem) -> AlbumSlim {
AlbumSlim {
src_id: b.id,
service: MusicService::YouTube,
name: b.name,
artists: b.artists.into_iter().map(map_artist_tag).collect(),
release_date: None,
album_type: Some(util::map_album_type(b.album_type)),
image_url: util::yt_image_closest(&b.cover, crate::IMG_SIZE_SEARCH),
image_date: None,
}
}
fn map_track_slim(t: rpmodel::TrackItem) -> TrackSlim {
let image_url = util::yt_image_closest(&t.cover, crate::IMG_SIZE_SEARCH);
TrackSlim {
album: match t.album {
Some(b) => AlbumTag {
src_id: b.id,
service: MusicService::YouTube,
name: b.name,
release_date: None,
album_type: None,
image_url,
image_date: None,
},
None => AlbumTag {
src_id: t.id.clone(),
service: MusicService::YouTube,
name: t.name.clone(),
release_date: None,
album_type: if t.is_video {
Some(AlbumType::Mv)
} else {
None
},
image_url,
image_date: None,
},
},
src_id: t.id,
service: MusicService::YouTube,
name: t.name,
duration: t.duration.and_then(|d| d.try_into().ok()),
artists: t.artists.into_iter().map(map_artist_tag).collect(),
album_pos: None,
}
}
fn map_playlist_slim(p: rpmodel::MusicPlaylistItem) -> PlaylistSlim {
let (owner_src_id, owner_name) = if p.from_ytm {
(Some(YTM_USER.0.to_owned()), Some(YTM_USER_NAME.to_owned()))
} else if let Some(ch) = p.channel {
(Some(ch.id), Some(ch.name))
} else {
(None, None)
};
PlaylistSlim {
src_id: p.id,
service: MusicService::YouTube,
name: p.name,
image_url: util::yt_image_closest(&p.thumbnail, crate::IMG_SIZE_SEARCH),
image_date: None,
owner_service: Some(MusicService::YouTube).filter(|_| owner_src_id.is_some()),
owner_src_id,
owner_name,
}
}
fn map_video_item(v: rpmodel::VideoItem) -> TrackSlim {
TrackSlim {
src_id: v.id.clone(),
service: MusicService::YouTube,
name: v.name.clone(),
duration: v.length.and_then(|d| d.try_into().ok()),
artists: v
.channel
.map(|ch| {
vec![ArtistTag {
id: Some(TIdOwned(ch.id, MusicService::YouTube)),
name: ch.name,
}]
})
.unwrap_or_default(),
album: AlbumTag {
src_id: v.id,
service: MusicService::YouTube,
name: v.name,
release_date: None,
album_type: Some(AlbumType::Mv),
image_url: util::yt_image_closest(&v.thumbnail, crate::IMG_SIZE_SEARCH),
image_date: None,
},
album_pos: None,
}
}
fn map_tiraya_item(v: rpmodel::MusicItem) -> TirayaItem {
match v {
rpmodel::MusicItem::Track(t) => TirayaItem::Track(map_track_slim(t)),
rpmodel::MusicItem::Album(b) => TirayaItem::Album(map_album_slim(b)),
rpmodel::MusicItem::Artist(a) => TirayaItem::Artist(map_artist_slim(a)),
rpmodel::MusicItem::Playlist(p) => TirayaItem::Playlist(map_playlist_slim(p)),
}
}
fn store_ctoken(ctoken: Option<String>, stored_tkn: &mut Option<String>) {
match stored_tkn {
Some(stored_tkn) => {
if let Some(ctoken) = ctoken {
stored_tkn.push('|');
stored_tkn.push_str(&ctoken);
}
}
None => {
*stored_tkn = ctoken;
}
};
}
// Matcher
@ -573,10 +867,11 @@ impl YouTubeExtractor {
let search = self
.rp
.query()
.music_search(format!("{name} {artist}"))
.music_search::<rpmodel::TrackItem, _>(format!("{name} {artist}"), None)
.await?;
let best_match = search
.tracks
.items
.items
.into_iter()
.map(|t| {
let t2 = self.core.matchmaker.parse_title(&t.name);
@ -1320,7 +1615,7 @@ impl YouTubeExtractor {
Ok(if from_ytm {
Some(
self.core
.import_user_id(TId("music", MusicService::YouTube), "YouTube Music", true)
.import_user_id(YTM_USER, YTM_USER_NAME, true)
.await?,
)
} else if let Some(channel) = channel {

View file

@ -58,6 +58,14 @@ pub fn yt_image_url(images: &[rpmodel::Thumbnail], format: ImgFormat) -> Option<
})
}
/// Get the image from a list of YT images which is closest to the given width
pub fn yt_image_closest(images: &[rpmodel::Thumbnail], width: u32) -> Option<String> {
images
.iter()
.min_by_key(|img| img.width.abs_diff(width))
.map(|img| img.url.to_owned())
}
/// Convert an extractor error to a SyncData object if the error should be stored in the database
pub fn error_to_sync_data(error: &ExtractorError) -> Option<SyncData> {
match error {

View file

@ -2,7 +2,7 @@ use tiraya_db::{
models::{Album, Artist, MusicService, Playlist, TId, User},
testutil,
};
use tiraya_extractor::{Extractor, GetStatus};
use tiraya_extractor::{model::GetStatus, Extractor};
fn _is_send<T: Send>(_t: T) {}

View file

@ -28,7 +28,7 @@ const IMAGE_FORMAT: image::ImageFormat = image::ImageFormat::WebP;
const SIZE_LG: u32 = 640;
const SIZE_MD: u32 = 300;
const SIZE_SM: u32 = 64;
const SIZE_SM: u32 = 60;
const WIDTH_HEADER_LG: u32 = 1920;
const WIDTH_HEADER_MD: u32 = 1080;

View file

@ -5,6 +5,8 @@ use serde::{Deserialize, Serialize};
mod cfg_params;
mod signed_url;
pub use signed_url::{image_url_local, image_url_proxy, validate_image_url, SignatureError};
use time::PrimitiveDateTime;
use url::Url;
pub mod config;
pub mod traits;
@ -86,3 +88,22 @@ pub fn cache_immutable_private() -> headers::CacheControl {
.with_private()
.with_immutable()
}
pub fn map_image_url(
url: Option<String>,
date: Option<PrimitiveDateTime>,
tid: tiraya_api_model::TId<'_>,
image_kind: ImageKind,
private: bool,
) -> Option<String> {
match date {
Some(date) => Some(image_url_local(tid, image_kind, date, private)),
None => url.map(|url| image_url_proxy(&url)),
}
}
pub fn url_param(url: &Url, param: &str) -> Option<String> {
url.query_pairs()
.find(|(k, _)| k == param)
.map(|(_, v)| v.to_string())
}