Tiraya/Tiraya
1
0
Fork 0
Code Issues Pull requests Projects Releases Activity

Compare commits

base: Tiraya:f9ade790274c5a2d0c4c6d3fb992bf6195adfc6f
Branches Tags
Tiraya:main
..
compare: Tiraya:86a4bd9762d955bbd3de965826a6ecb2af749c73
Branches Tags
Tiraya:main

3 commits
f9ade79027 ... 86a4bd9762

Author SHA1 Message Date
ThetaDev
86a4bd9762 fix: faster playlist imports, small matchmaker fix 2023-10-07 12:42:35 +02:00
ThetaDev
4597ffab16 fix: allow for redirected tracks 2023-10-06 17:24:56 +02:00
ThetaDev
fcc621c046 feat: finish Spotify extractor 2023-10-06 03:16:26 +02:00
20 changed files with 935 additions and 317 deletions
Download patch file Download diff file Expand all files Collapse all files

269
Cargo.lock generated
View file

@ -76,9 +76,9 @@ dependencies = [
[[package]] [[package]]
name = "anstream" name = "anstream"
version = "0.5.0" version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44"
dependencies = [ dependencies = [
"anstyle", "anstyle",
"anstyle-parse", "anstyle-parse",
@ -90,15 +90,15 @@ dependencies = [
[[package]] [[package]]
name = "anstyle" name = "anstyle"
version = "1.0.3" version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"
[[package]] [[package]]
name = "anstyle-parse" name = "anstyle-parse"
version = "0.2.1" version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140"
dependencies = [ dependencies = [
"utf8parse", "utf8parse",
] ]
@ -114,9 +114,9 @@ dependencies = [
[[package]] [[package]]
name = "anstyle-wincon" name = "anstyle-wincon"
version = "2.1.0" version = "3.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628"
dependencies = [ dependencies = [
"anstyle", "anstyle",
"windows-sys 0.48.0", "windows-sys 0.48.0",
@ -161,7 +161,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.37", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -172,7 +172,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.37", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -264,9 +264,9 @@ dependencies = [
[[package]] [[package]]
name = "brotli" name = "brotli"
version = "3.3.4" version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f"
dependencies = [ dependencies = [
"alloc-no-stdlib", "alloc-no-stdlib",
"alloc-stdlib", "alloc-stdlib",
@ -275,9 +275,9 @@ dependencies = [
[[package]] [[package]]
name = "brotli-decompressor" name = "brotli-decompressor"
version = "2.3.4" version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" checksum = "da74e2b81409b1b743f8f0c62cc6254afefb8b8e50bbfe3735550f7aeefa3448"
dependencies = [ dependencies = [
"alloc-no-stdlib", "alloc-no-stdlib",
"alloc-stdlib", "alloc-stdlib",
@ -291,9 +291,9 @@ checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
[[package]] [[package]]
name = "byteorder" name = "byteorder"
version = "1.4.3" version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]] [[package]]
name = "bytes" name = "bytes"
@ -333,9 +333,9 @@ dependencies = [
[[package]] [[package]]
name = "clap" name = "clap"
version = "4.4.5" version = "4.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "824956d0dca8334758a5b7f7e50518d66ea319330cbceedcf76905c2f6ab30e3" checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956"
dependencies = [ dependencies = [
"clap_builder", "clap_builder",
"clap_derive", "clap_derive",
@ -343,9 +343,9 @@ dependencies = [
[[package]] [[package]]
name = "clap_builder" name = "clap_builder"
version = "4.4.5" version = "4.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "122ec64120a49b4563ccaedcbea7818d069ed8e9aa6d829b82d8a4128936b2ab" checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45"
dependencies = [ dependencies = [
"anstream", "anstream",
"anstyle", "anstyle",
@ -362,7 +362,7 @@ dependencies = [
"heck", "heck",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.37", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -527,7 +527,7 @@ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"strsim", "strsim",
"syn 2.0.37", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -549,7 +549,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
dependencies = [ dependencies = [
"darling_core 0.20.3", "darling_core 0.20.3",
"quote", "quote",
"syn 2.0.37", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -629,7 +629,7 @@ dependencies = [
"once_cell", "once_cell",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.37", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -640,9 +640,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]] [[package]]
name = "errno" name = "errno"
version = "0.3.3" version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480"
dependencies = [ dependencies = [
"errno-dragonfly", "errno-dragonfly",
"libc", "libc",
@ -688,9 +688,9 @@ dependencies = [
[[package]] [[package]]
name = "fastrand" name = "fastrand"
version = "2.0.0" version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
[[package]] [[package]]
name = "finl_unicode" name = "finl_unicode"
@ -710,13 +710,12 @@ dependencies = [
[[package]] [[package]]
name = "flume" name = "flume"
version = "0.10.14" version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-sink", "futures-sink",
"pin-project",
"spin 0.9.8", "spin 0.9.8",
] ]
@ -817,7 +816,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.37", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -910,9 +909,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.14.0" version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12"
dependencies = [ dependencies = [
"ahash", "ahash",
"allocator-api2", "allocator-api2",
@ -924,7 +923,7 @@ version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7"
dependencies = [ dependencies = [
"hashbrown 0.14.0", "hashbrown 0.14.1",
] ]
[[package]] [[package]]
@ -1117,19 +1116,19 @@ dependencies = [
[[package]] [[package]]
name = "indexmap" name = "indexmap"
version = "2.0.0" version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897"
dependencies = [ dependencies = [
"equivalent", "equivalent",
"hashbrown 0.14.0", "hashbrown 0.14.1",
] ]
[[package]] [[package]]
name = "insta" name = "insta"
version = "1.32.0" version = "1.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3e02c584f4595792d09509a94cdb92a3cef7592b1eb2d9877ee6f527062d0ea" checksum = "1aa511b2e298cd49b1856746f6bb73e17036bcd66b25f5e92cdcdbec9bd75686"
dependencies = [ dependencies = [
"console", "console",
"lazy_static", "lazy_static",
@ -1234,9 +1233,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]] [[package]]
name = "linux-raw-sys" name = "linux-raw-sys"
version = "0.4.7" version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db"
[[package]] [[package]]
name = "lock_api" name = "lock_api"
@ -1267,18 +1266,19 @@ dependencies = [
[[package]] [[package]]
name = "md-5" name = "md-5"
version = "0.10.5" version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
dependencies = [ dependencies = [
"cfg-if",
"digest", "digest",
] ]
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.6.3" version = "2.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"
[[package]] [[package]]
name = "mime" name = "mime"
@ -1455,7 +1455,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.37", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -1560,9 +1560,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
[[package]] [[package]]
name = "pest" name = "pest"
version = "2.7.3" version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7a4d085fd991ac8d5b05a147b437791b4260b76326baf0fc60cf7c9c27ecd33" checksum = "c022f1e7b65d6a24c0dbbd5fb344c66881bc01f3e5ae74a1c8100f2f985d98a4"
dependencies = [ dependencies = [
"memchr", "memchr",
"thiserror", "thiserror",
@ -1571,9 +1571,9 @@ dependencies = [
[[package]] [[package]]
name = "pest_derive" name = "pest_derive"
version = "2.7.3" version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2bee7be22ce7918f641a33f08e3f43388c7656772244e2bbb2477f44cc9021a" checksum = "35513f630d46400a977c4cb58f78e1bfbe01434316e60c37d27b9ad6139c66d8"
dependencies = [ dependencies = [
"pest", "pest",
"pest_generator", "pest_generator",
@ -1581,22 +1581,22 @@ dependencies = [
[[package]] [[package]]
name = "pest_generator" name = "pest_generator"
version = "2.7.3" version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1511785c5e98d79a05e8a6bc34b4ac2168a0e3e92161862030ad84daa223141" checksum = "bc9fc1b9e7057baba189b5c626e2d6f40681ae5b6eb064dc7c7834101ec8123a"
dependencies = [ dependencies = [
"pest", "pest",
"pest_meta", "pest_meta",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.37", "syn 2.0.38",
] ]
[[package]] [[package]]
name = "pest_meta" name = "pest_meta"
version = "2.7.3" version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b42f0394d3123e33353ca5e1e89092e533d2cc490389f2bd6131c43c634ebc5f" checksum = "1df74e9e7ec4053ceb980e7c0c8bd3594e977fde1af91daba9c928e8e8c6708d"
dependencies = [ dependencies = [
"once_cell", "once_cell",
"pest", "pest",
@ -1633,7 +1633,7 @@ dependencies = [
"phf_shared", "phf_shared",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.37", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -1645,26 +1645,6 @@ dependencies = [
"siphasher 0.3.11", "siphasher 0.3.11",
] ]
[[package]]
name = "pin-project"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.37",
]
[[package]] [[package]]
name = "pin-project-lite" name = "pin-project-lite"
version = "0.2.13" version = "0.2.13"
@ -1747,7 +1727,7 @@ checksum = "f69f8d22fa3f34f3083d9a4375c038732c7a7e964de1beb81c544da92dfc40b8"
dependencies = [ dependencies = [
"ahash", "ahash",
"equivalent", "equivalent",
"hashbrown 0.14.0", "hashbrown 0.14.1",
"parking_lot", "parking_lot",
] ]
@ -1807,9 +1787,9 @@ dependencies = [
[[package]] [[package]]
name = "regex" name = "regex"
version = "1.9.5" version = "1.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff"
dependencies = [ dependencies = [
"aho-corasick", "aho-corasick",
"memchr", "memchr",
@ -1819,9 +1799,9 @@ dependencies = [
[[package]] [[package]]
name = "regex-automata" name = "regex-automata"
version = "0.3.8" version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9"
dependencies = [ dependencies = [
"aho-corasick", "aho-corasick",
"memchr", "memchr",
@ -1842,9 +1822,9 @@ checksum = "c707298afce11da2efef2f600116fa93ffa7a032b5d7b628aa17711ec81383ca"
[[package]] [[package]]
name = "reqwest" name = "reqwest"
version = "0.11.20" version = "0.11.22"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b"
dependencies = [ dependencies = [
"async-compression", "async-compression",
"base64 0.21.4", "base64 0.21.4",
@ -1871,6 +1851,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"serde_urlencoded", "serde_urlencoded",
"system-configuration",
"tokio", "tokio",
"tokio-native-tls", "tokio-native-tls",
"tokio-rustls", "tokio-rustls",
@ -2024,7 +2005,7 @@ dependencies = [
"regex", "regex",
"relative-path", "relative-path",
"rustc_version", "rustc_version",
"syn 2.0.37", "syn 2.0.38",
"unicode-ident", "unicode-ident",
] ]
@ -2045,9 +2026,9 @@ dependencies = [
[[package]] [[package]]
name = "rustix" name = "rustix"
version = "0.38.14" version = "0.38.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" checksum = "f25469e9ae0f3d0047ca8b93fc56843f38e6774f0914a107ff8b41be8be8e0b7"
dependencies = [ dependencies = [
"bitflags 2.4.0", "bitflags 2.4.0",
"errno", "errno",
@ -2096,7 +2077,7 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
[[package]] [[package]]
name = "rustypipe" name = "rustypipe"
version = "0.1.0" version = "0.1.0"
source = "git+https://code.thetadev.de/ThetaDev/rustypipe.git#d6de428549bfe7881e5ac52857d4bc1fa2db7f5d" source = "git+https://code.thetadev.de/ThetaDev/rustypipe.git#e247b0c5d9874313d8a350142694bc90db805a11"
dependencies = [ dependencies = [
"base64 0.21.4", "base64 0.21.4",
"fancy-regex", "fancy-regex",
@ -2186,9 +2167,9 @@ dependencies = [
[[package]] [[package]]
name = "semver" name = "semver"
version = "1.0.18" version = "1.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0"
[[package]] [[package]]
name = "serde" name = "serde"
@ -2207,7 +2188,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.37", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -2275,7 +2256,7 @@ dependencies = [
"darling 0.20.3", "darling 0.20.3",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.37", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -2291,9 +2272,9 @@ dependencies = [
[[package]] [[package]]
name = "sha2" name = "sha2"
version = "0.10.7" version = "0.10.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"cpufeatures", "cpufeatures",
@ -2302,9 +2283,9 @@ dependencies = [
[[package]] [[package]]
name = "sharded-slab" name = "sharded-slab"
version = "0.1.4" version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
dependencies = [ dependencies = [
"lazy_static", "lazy_static",
] ]
@ -2360,7 +2341,7 @@ checksum = "0eb01866308440fc64d6c44d9e86c5cc17adfe33c4d6eed55da9145044d0ffc1"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.37", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -2421,9 +2402,9 @@ dependencies = [
[[package]] [[package]]
name = "sqlx" name = "sqlx"
version = "0.7.1" version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e58421b6bc416714d5115a2ca953718f6c621a51b68e4f4922aea5a4391a721" checksum = "0e50c216e3624ec8e7ecd14c6a6a6370aad6ee5d8cfc3ab30b5162eeeef2ed33"
dependencies = [ dependencies = [
"sqlx-core", "sqlx-core",
"sqlx-macros", "sqlx-macros",
@ -2434,9 +2415,9 @@ dependencies = [
[[package]] [[package]]
name = "sqlx-core" name = "sqlx-core"
version = "0.7.1" version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53" checksum = "8d6753e460c998bbd4cd8c6f0ed9a64346fcca0723d6e75e52fdc351c5d2169d"
dependencies = [ dependencies = [
"ahash", "ahash",
"atoi", "atoi",
@ -2454,7 +2435,7 @@ dependencies = [
"futures-util", "futures-util",
"hashlink", "hashlink",
"hex", "hex",
"indexmap 2.0.0", "indexmap 2.0.2",
"log", "log",
"memchr", "memchr",
"once_cell", "once_cell",
@ -2502,9 +2483,9 @@ dependencies = [
[[package]] [[package]]
name = "sqlx-macros" name = "sqlx-macros"
version = "0.7.1" version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "208e3165167afd7f3881b16c1ef3f2af69fa75980897aac8874a0696516d12c2" checksum = "9a793bb3ba331ec8359c1853bd39eed32cdd7baaf22c35ccf5c92a7e8d1189ec"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -2515,9 +2496,9 @@ dependencies = [
[[package]] [[package]]
name = "sqlx-macros-core" name = "sqlx-macros-core"
version = "0.7.1" version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a4a8336d278c62231d87f24e8a7a74898156e34c1c18942857be2acb29c7dfc" checksum = "0a4ee1e104e00dedb6aa5ffdd1343107b0a4702e862a84320ee7cc74782d96fc"
dependencies = [ dependencies = [
"dotenvy", "dotenvy",
"either", "either",
@ -2541,9 +2522,9 @@ dependencies = [
[[package]] [[package]]
name = "sqlx-mysql" name = "sqlx-mysql"
version = "0.7.1" version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ca69bf415b93b60b80dc8fda3cb4ef52b2336614d8da2de5456cc942a110482" checksum = "864b869fdf56263f4c95c45483191ea0af340f9f3e3e7b4d57a61c7c87a970db"
dependencies = [ dependencies = [
"atoi", "atoi",
"base64 0.21.4", "base64 0.21.4",
@ -2585,9 +2566,9 @@ dependencies = [
[[package]] [[package]]
name = "sqlx-postgres" name = "sqlx-postgres"
version = "0.7.1" version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0db2df1b8731c3651e204629dd55e52adbae0462fa1bdcbed56a2302c18181e" checksum = "eb7ae0e6a97fb3ba33b23ac2671a5ce6e3cabe003f451abd5a56e7951d975624"
dependencies = [ dependencies = [
"atoi", "atoi",
"base64 0.21.4", "base64 0.21.4",
@ -2626,9 +2607,9 @@ dependencies = [
[[package]] [[package]]
name = "sqlx-sqlite" name = "sqlx-sqlite"
version = "0.7.1" version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4c21bf34c7cae5b283efb3ac1bcc7670df7561124dc2f8bdc0b59be40f79a2" checksum = "d59dc83cf45d89c555a577694534fcd1b55c545a816c816ce51f20bbe56a4f3f"
dependencies = [ dependencies = [
"atoi", "atoi",
"flume", "flume",
@ -2684,7 +2665,7 @@ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"rustversion", "rustversion",
"syn 2.0.37", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -2706,15 +2687,36 @@ dependencies = [
[[package]] [[package]]
name = "syn" name = "syn"
version = "2.0.37" version = "2.0.38"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "system-configuration"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
dependencies = [
"bitflags 1.3.2",
"core-foundation",
"system-configuration-sys",
]
[[package]]
name = "system-configuration-sys"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]] [[package]]
name = "tempfile" name = "tempfile"
version = "3.8.0" version = "3.8.0"
@ -2743,22 +2745,22 @@ dependencies = [
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.48" version = "1.0.49"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4"
dependencies = [ dependencies = [
"thiserror-impl", "thiserror-impl",
] ]
[[package]] [[package]]
name = "thiserror-impl" name = "thiserror-impl"
version = "1.0.48" version = "1.0.49"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.37", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -2773,9 +2775,9 @@ dependencies = [
[[package]] [[package]]
name = "time" name = "time"
version = "0.3.28" version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe"
dependencies = [ dependencies = [
"deranged", "deranged",
"itoa", "itoa",
@ -2786,15 +2788,15 @@ dependencies = [
[[package]] [[package]]
name = "time-core" name = "time-core"
version = "0.1.1" version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
[[package]] [[package]]
name = "time-macros" name = "time-macros"
version = "0.2.14" version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20"
dependencies = [ dependencies = [
"time-core", "time-core",
] ]
@ -2860,6 +2862,7 @@ dependencies = [
"rspotify", "rspotify",
"rstest", "rstest",
"rustypipe", "rustypipe",
"serde_plain",
"siphasher 1.0.0", "siphasher 1.0.0",
"sqlx", "sqlx",
"sqlx-database-tester", "sqlx-database-tester",
@ -2912,7 +2915,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.37", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -2999,7 +3002,7 @@ version = "0.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338" checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338"
dependencies = [ dependencies = [
"indexmap 2.0.0", "indexmap 2.0.2",
"serde", "serde",
"serde_spanned", "serde_spanned",
"toml_datetime", "toml_datetime",
@ -3033,7 +3036,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.37", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -3231,7 +3234,7 @@ dependencies = [
"once_cell", "once_cell",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.37", "syn 2.0.38",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -3265,7 +3268,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.37", "syn 2.0.38",
"wasm-bindgen-backend", "wasm-bindgen-backend",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -3481,9 +3484,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]] [[package]]
name = "winnow" name = "winnow"
version = "0.5.15" version = "0.5.16"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" checksum = "037711d82167854aff2018dfd193aa0fef5370f456732f0d5a0c59b0f1b4b907"
dependencies = [ dependencies = [
"memchr", "memchr",
] ]

4
crates/db/src/models/album.rs
View file

@ -146,6 +146,10 @@ impl Album {
Id::Db(self.id) Id::Db(self.id)
} }
pub fn src_id(&self) -> SrcId {
SrcId(&self.src_id, self.service)
}
pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError> pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError>
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,

4
crates/db/src/models/artist.rs
View file

@ -99,6 +99,10 @@ impl Artist {
Id::Db(self.id) Id::Db(self.id)
} }
pub fn src_id(&self) -> SrcId {
SrcId(&self.src_id, self.service)
}
pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError> pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError>
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy, E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy,

8
crates/db/src/models/playlist.rs
View file

@ -102,6 +102,10 @@ impl Playlist {
Id::Db(self.id) Id::Db(self.id)
} }
pub fn src_id(&self) -> SrcId {
SrcId(&self.src_id, self.service)
}
pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError> pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError>
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,
@ -631,6 +635,10 @@ impl PlaylistUpdate<'_> {
} }
impl PlaylistSlim { impl PlaylistSlim {
pub fn src_id(&self) -> SrcId {
SrcId(&self.src_id, self.service)
}
pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError> pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError>
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,

8
crates/db/src/models/track.rs
View file

@ -138,6 +138,10 @@ impl Track {
Id::Db(self.id) Id::Db(self.id)
} }
pub fn src_id(&self) -> SrcId {
SrcId(&self.src_id, self.service)
}
pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError> pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError>
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy, E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy,
@ -511,6 +515,10 @@ impl TrackUpdate<'_> {
} }
impl TrackSlim { impl TrackSlim {
pub fn src_id(&self) -> SrcId {
SrcId(&self.src_id, self.service)
}
pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError> pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError>
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy, E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy,

10
crates/db/src/models/user.rs
View file

@ -55,6 +55,10 @@ impl User {
Id::Db(self.id) Id::Db(self.id)
} }
pub fn src_id(&self) -> SrcId {
SrcId(&self.src_id, self.service)
}
pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError> pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError>
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,
@ -316,6 +320,12 @@ impl UserUpdate<'_> {
} }
} }
impl UserItem {
pub fn src_id(&self) -> SrcId {
SrcId(&self.src_id, self.service)
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;

1
crates/extractor/Cargo.toml
View file

@ -21,6 +21,7 @@ quick_cache.workspace = true
regex.workspace = true regex.workspace = true
rspotify.workspace = true rspotify.workspace = true
rustypipe.workspace = true rustypipe.workspace = true
serde_plain.workspace = true
siphasher.workspace = true siphasher.workspace = true
sqlx.workspace = true sqlx.workspace = true
strsim.workspace = true strsim.workspace = true

2
crates/extractor/src/error.rs
View file

@ -17,6 +17,8 @@ pub enum ExtractorError {
}, },
#[error("could not find a match for track [{id}]")] #[error("could not find a match for track [{id}]")]
NoMatch { id: SrcIdOwned }, NoMatch { id: SrcIdOwned },
#[error("no id found")]
NoId,
#[error("{0}")] #[error("{0}")]
Other(Cow<'static, str>), Other(Cow<'static, str>),
} }

37
crates/extractor/src/lib.rs
View file

@ -1,5 +1,4 @@
#![warn(clippy::dbg_macro, clippy::todo)] #![warn(clippy::dbg_macro, clippy::todo)]
#![allow(dead_code)]
pub mod error; pub mod error;
mod model; mod model;
@ -8,6 +7,7 @@ mod util;
pub use model::GetStatus; pub use model::GetStatus;
use services::SpotifyExtractor; use services::SpotifyExtractor;
use tiraya_utils::config::CONFIG;
use std::{collections::HashSet, sync::Arc}; use std::{collections::HashSet, sync::Arc};
@ -28,13 +28,6 @@ use crate::{
services::YouTubeExtractor, services::YouTubeExtractor,
}; };
const ARTIST_STALE: Duration = Duration::hours(24);
const PLAYLIST_STALE: Duration = Duration::hours(24);
const USER_STALE: Duration = Duration::hours(24);
pub(crate) const CONCURRENCY: usize = 4;
pub(crate) const DB_CONCURRENCY: usize = 8;
pub(crate) const ITEM_LIMIT: usize = 2000;
pub struct Extractor(Arc<ExtractorInner>); pub struct Extractor(Arc<ExtractorInner>);
pub struct ExtractorInner { pub struct ExtractorInner {
@ -57,7 +50,11 @@ impl Extractor {
let yt = YouTubeExtractor::new(core.clone())?; let yt = YouTubeExtractor::new(core.clone())?;
Ok(Self( Ok(Self(
ExtractorInner { ExtractorInner {
sp: SpotifyExtractor::new(core.clone(), yt.clone())?, sp: if CONFIG.extractor.spotify.enable {
Some(SpotifyExtractor::new(core.clone(), yt.clone())?)
} else {
None
},
yt, yt,
core, core,
} }
@ -71,6 +68,12 @@ impl Extractor {
.await .await
.to_optional() .to_optional()
.unwrap(); .unwrap();
// Get srcid from fetched artist if available (could be an alias)
let srcid = artist
.as_ref()
.map(|a| a.src_id().to_owned())
.unwrap_or_else(|| id.to_owned());
let last_update = if let Some(artist) = artist { let last_update = if let Some(artist) = artist {
self.core.artist_cache.insert(id.to_owned(), artist.id); self.core.artist_cache.insert(id.to_owned(), artist.id);
@ -80,7 +83,7 @@ impl Extractor {
let age = OffsetDateTime::now_utc() - last_sync_at.assume_utc(); let age = OffsetDateTime::now_utc() - last_sync_at.assume_utc();
// Check if artist in DB is still fresh // Check if artist in DB is still fresh
if age < ARTIST_STALE { if age < Duration::hours(CONFIG.extractor.artist_stale_h.into()) {
return Ok(GetResult::stored(artist)); return Ok(GetResult::stored(artist));
} else { } else {
match last_sync_data { match last_sync_data {
@ -112,7 +115,8 @@ impl Extractor {
None None
}; };
self.update_artist(id, last_update.as_ref()).await self.update_artist(srcid.as_srcid(), last_update.as_ref())
.await
} }
async fn update_artist( async fn update_artist(
@ -122,6 +126,7 @@ impl Extractor {
) -> Result<GetResult<Artist>, ExtractorError> { ) -> Result<GetResult<Artist>, ExtractorError> {
let res = match id.1 { let res = match id.1 {
MusicService::YouTube => self.yt.update_artist(id.0, last_update).await, MusicService::YouTube => self.yt.update_artist(id.0, last_update).await,
MusicService::Spotify => self.sp()?.update_artist(id.0).await,
_ => { _ => {
return Err(ExtractorError::Unsupported { return Err(ExtractorError::Unsupported {
typ: EntityType::Artist, typ: EntityType::Artist,
@ -172,7 +177,7 @@ impl Extractor {
} }
} }
}) })
.buffer_unordered(DB_CONCURRENCY) .buffer_unordered(CONFIG.core.db_concurrency)
.collect::<Vec<_>>() .collect::<Vec<_>>()
.await; .await;
@ -211,6 +216,7 @@ impl Extractor {
let res = match id.1 { let res = match id.1 {
MusicService::YouTube => self.yt.fetch_album(id.0).await?, MusicService::YouTube => self.yt.fetch_album(id.0).await?,
MusicService::Spotify => self.sp()?.fetch_album(id.0).await?,
_ => { _ => {
return Err(ExtractorError::Unsupported { return Err(ExtractorError::Unsupported {
typ: EntityType::Album, typ: EntityType::Album,
@ -265,7 +271,7 @@ impl Extractor {
let age = OffsetDateTime::now_utc() - last_sync_at.assume_utc(); let age = OffsetDateTime::now_utc() - last_sync_at.assume_utc();
// Check if playlist< in DB is still fresh // Check if playlist< in DB is still fresh
if age < PLAYLIST_STALE { if age < Duration::hours(CONFIG.extractor.playlist_stale_h.into()) {
return Ok(GetResult::stored(playlist)); return Ok(GetResult::stored(playlist));
} else { } else {
match last_sync_data { match last_sync_data {
@ -364,8 +370,8 @@ impl Extractor {
{ {
let age = OffsetDateTime::now_utc() - last_sync_at.assume_utc(); let age = OffsetDateTime::now_utc() - last_sync_at.assume_utc();
// Check if playlist< in DB is still fresh // Check if playlist in DB is still fresh
if age < USER_STALE { if age < Duration::hours(CONFIG.extractor.user_stale_h.into()) {
return Ok(GetResult::stored(user)); return Ok(GetResult::stored(user));
} else { } else {
match last_sync_data { match last_sync_data {
@ -410,6 +416,7 @@ impl Extractor {
) -> Result<GetResult<User>, ExtractorError> { ) -> Result<GetResult<User>, ExtractorError> {
let res = match id.1 { let res = match id.1 {
MusicService::YouTube => self.yt.update_user(id.0, last_update).await, MusicService::YouTube => self.yt.update_user(id.0, last_update).await,
MusicService::Spotify => self.sp()?.update_user(id.0, last_update).await,
_ => { _ => {
return Err(ExtractorError::Unsupported { return Err(ExtractorError::Unsupported {
typ: EntityType::User, typ: EntityType::User,

6
crates/extractor/src/services/mod.rs
View file

@ -8,6 +8,12 @@ use crate::{error::ExtractorError, model::ExtractorCore};
/* /*
/// Tiraya extractor implementation for a specific music streaming service /// Tiraya extractor implementation for a specific music streaming service
///
/// Terminology for the individual methods:
/// - **update** Check if an item was updated on the source, download and import the
/// changes if necessary
/// - **fetch** Download and import the item from source if it is not present in the database
/// - **import** Take a data object from a service and store it in the database
pub(crate) trait ServiceExtractor { pub(crate) trait ServiceExtractor {
/// Create a new instance of the service extractor /// Create a new instance of the service extractor
fn new(core: Arc<ExtractorCore>) -> Result<Self, ExtractorError>; fn new(core: Arc<ExtractorCore>) -> Result<Self, ExtractorError>;

344
crates/extractor/src/services/spotify.rs
View file

@ -1,21 +1,25 @@
use futures::stream::{StreamExt, TryStreamExt}; use futures::{stream::TryStreamExt, StreamExt};
use path_macro::path; use path_macro::path;
use rspotify::{ use rspotify::{
model::{FullTrack, PlayableItem, PlaylistId, PublicUser, TrackId}, model::{
AlbumId, ArtistId, FullTrack, Market, PlayableItem, PlaylistId, PublicUser,
SimplifiedPlaylist, TrackId, UserId,
},
prelude::{BaseClient, Id}, prelude::{BaseClient, Id},
ClientCredsSpotify, ClientCredsSpotify,
}; };
use std::{collections::HashSet, sync::Arc}; use std::{collections::HashSet, sync::Arc};
use time::OffsetDateTime; use time::OffsetDateTime;
use tiraya_db::models::{ use tiraya_db::models::{
MusicService, Playlist, PlaylistEntry, PlaylistImgType, PlaylistNew, PlaylistType, SrcId, Artist, MusicService, Playlist, PlaylistEntry, PlaylistImgType, PlaylistNew, PlaylistType,
SrcIdOwned, SrcId, SrcIdOwned, Track, User, UserNew, UserType,
}; };
use tiraya_utils::config::CONFIG; use tiraya_utils::config::CONFIG;
use crate::{ use crate::{
error::ExtractorError, error::ExtractorError,
model::{ExtractorCore, GetResult, LastUpdateState, SyncLastUpdate}, model::{ExtractorCore, GetResult, LastUpdateState, SyncLastUpdate},
util::ArtistIdName,
}; };
use super::YouTubeExtractor; use super::YouTubeExtractor;
@ -25,32 +29,41 @@ pub struct SpotifyExtractor {
core: Arc<ExtractorCore>, core: Arc<ExtractorCore>,
sp: ClientCredsSpotify, sp: ClientCredsSpotify,
yt: YouTubeExtractor, yt: YouTubeExtractor,
market: Market,
} }
impl SpotifyExtractor { const PAGE_LIMIT: u32 = 50;
pub fn new(
core: Arc<ExtractorCore>,
yt: YouTubeExtractor,
) -> Result<Option<Self>, ExtractorError> {
if let Some(cfg) = &CONFIG.extractor.spotify {
let sp = ClientCredsSpotify::with_config(
rspotify::Credentials {
id: cfg.client_id.to_owned(),
secret: Some(cfg.client_secret.to_owned()),
},
rspotify::Config {
cache_path: path!(CONFIG.extractor.data_dir / "spotify_token_cache.json"),
token_cached: true,
..Default::default()
},
);
Ok(Some(Self { core, sp, yt })) impl SpotifyExtractor {
} else { pub fn new(core: Arc<ExtractorCore>, yt: YouTubeExtractor) -> Result<Self, ExtractorError> {
Ok(None) let sp = ClientCredsSpotify::with_config(
} rspotify::Credentials {
id: CONFIG.extractor.spotify.client_id.to_owned(),
secret: Some(CONFIG.extractor.spotify.client_secret.to_owned()),
},
rspotify::Config {
cache_path: path!(CONFIG.extractor.data_dir / "spotify_token_cache.json"),
token_cached: true,
pagination_chunks: PAGE_LIMIT,
..Default::default()
},
);
let market_str = &CONFIG.extractor.spotify.market;
let market = Market::Country(serde_plain::from_str(market_str).map_err(|_| {
ExtractorError::Other(format!("invalid spotify market `{market_str}`").into())
})?);
Ok(Self {
core,
sp,
yt,
market,
})
} }
/// Since Rspotify does not initialize the token by itself it needs to be loaded
/// before making a request
async fn load_token(&self) -> Result<(), ExtractorError> { async fn load_token(&self) -> Result<(), ExtractorError> {
let mut token = self.sp.token.lock().await.unwrap(); let mut token = self.sp.token.lock().await.unwrap();
if token.is_none() { if token.is_none() {
@ -67,14 +80,93 @@ impl SpotifyExtractor {
Ok(()) Ok(())
} }
pub async fn update_artist(&self, src_id: &str) -> Result<GetResult<i32>, ExtractorError> {
// Info: since we are matching Spotify artists with YTM artists, this function
// will only be called on the initial request of the artist, not on updates
// (hence no last_update parameter).
self.load_token().await?;
let artist_id = ArtistId::from_id(src_id)?;
let mut top_tracks = self
.sp
.artist_top_tracks(artist_id, Some(self.market))
.await?;
let mut get_track = || -> Result<FullTrack, ExtractorError> {
let i = top_tracks
.iter()
.enumerate()
.find(|(_, t)| {
t.artists.len() == 1
&& t.artists[0]
.id
.as_ref()
.map(|id| id.id() == src_id)
.unwrap_or_default()
})
.or_else(|| {
top_tracks.iter().enumerate().find(|(_, t)| {
t.artists.iter().any(|a| {
a.id.as_ref()
.map(|id| id.id() == src_id)
.unwrap_or_default()
})
})
})
.ok_or_else(|| ExtractorError::InvalidData {
id: SrcIdOwned(src_id.to_owned(), MusicService::Spotify),
msg: "no top tracks found".into(),
})?
.0;
Ok(top_tracks.remove(i))
};
let a_id = SrcId(src_id, MusicService::Spotify);
for _ in 0..3 {
let track = get_track()?;
match self.import_track(track).await {
Ok(_) => {
if let Some(a_id) = Artist::get_id(a_id, &self.core.db).await? {
let src_id = Artist::get_src_id(a_id, &self.core.db)
.await?
.ok_or(ExtractorError::NoId)?;
if src_id.1 != MusicService::YouTube {
return Err(ExtractorError::Other("artist id not from yt".into()));
}
return self.yt.update_artist(&src_id.0, None).await;
}
}
Err(ExtractorError::NoMatch { .. }) => {}
Err(e) => return Err(e),
}
}
Err(ExtractorError::NoMatch {
id: a_id.to_owned(),
})
}
pub async fn fetch_album(&self, src_id: &str) -> Result<(i32, bool), ExtractorError> {
self.load_token().await?;
let album = self.sp.album(AlbumId::from_id(src_id)?, None).await?;
let artist = album
.artists
.into_iter()
.next()
.map(|artist| artist.name)
.unwrap_or_default();
let album_id = self
.yt
.match_album(SrcId(src_id, MusicService::Spotify), &album.name, &artist)
.await?;
Ok((album_id, false))
}
pub async fn fetch_track(&self, src_id: &str) -> Result<i32, ExtractorError> { pub async fn fetch_track(&self, src_id: &str) -> Result<i32, ExtractorError> {
self.load_token().await?; self.load_token().await?;
let track = self.sp.track(TrackId::from_id(src_id)?, None).await?; let track = self.sp.track(TrackId::from_id(src_id)?, None).await?;
self.import_track(track) self.import_track(track).await
.await?
.ok_or_else(|| ExtractorError::NoMatch {
id: SrcIdOwned(src_id.to_owned(), MusicService::Spotify),
})
} }
pub async fn update_playlist( pub async fn update_playlist(
@ -101,7 +193,7 @@ impl SpotifyExtractor {
.into_iter() .into_iter()
.max_by_key(|img| img.height.unwrap_or_default()) .max_by_key(|img| img.height.unwrap_or_default())
.map(|img| img.url); .map(|img| img.url);
let owner_id = self.import_user(&playlist.owner).await?; let owner_id = self.import_user(playlist.owner).await?;
let n_playlist = PlaylistNew { let n_playlist = PlaylistNew {
src_id, src_id,
@ -130,6 +222,7 @@ impl SpotifyExtractor {
None None
}) })
}) })
.take(CONFIG.extractor.playlist_item_limit_matchmaker)
.try_collect::<Vec<_>>() .try_collect::<Vec<_>>()
.await?; .await?;
@ -169,32 +262,167 @@ impl SpotifyExtractor {
Ok(GetResult::fetched(playlist_id)) Ok(GetResult::fetched(playlist_id))
} }
async fn import_user(&self, user: &PublicUser) -> Result<i32, ExtractorError> { pub async fn update_user(
&self,
src_id: &str,
last_update: Option<&SyncLastUpdate>,
) -> Result<GetResult<i32>, ExtractorError> {
self.load_token().await?;
let sp_user_id = UserId::from_id(src_id)?;
// First page of playlists
let playlists_p1 = self
.sp
.user_playlists_manual(sp_user_id.clone_static(), Some(PAGE_LIMIT), Some(0))
.await?;
let current_state = playlists_p1
.items
.first()
.map(|pl| LastUpdateState::Version(pl.id.id().to_owned()))
.unwrap_or_default();
if let Some(last_update) = last_update {
if last_update.state.is_current(&current_state) {
User::set_last_sync(last_update.id, current_state.into(), &self.core.db).await?;
return Ok(GetResult::stored(last_update.id));
}
}
let user = self.sp.user(sp_user_id.clone_static()).await?;
let user_id = self.import_user(user).await?;
self.import_playlists(playlists_p1.items).await?;
if playlists_p1.next.is_some() {
let mut offset = PAGE_LIMIT;
while offset < CONFIG.extractor.user_playlist_limit as u32 {
let playlists = self
.sp
.user_playlists_manual(
sp_user_id.clone_static(),
Some(PAGE_LIMIT),
Some(offset),
)
.await?;
self.import_playlists(playlists.items).await?;
if playlists.next.is_none() {
break;
}
offset += PAGE_LIMIT;
}
}
User::set_last_sync(user_id, current_state.into(), &self.core.db).await?;
Ok(GetResult::fetched(user_id))
}
async fn import_user(&self, user: PublicUser) -> Result<i32, ExtractorError> {
let id_owned = SrcIdOwned(user.id.id().to_owned(), MusicService::Spotify);
let image_url = user
.images
.into_iter()
.max_by_key(|img| img.height.unwrap_or_default())
.map(|img| img.url);
let name = user.display_name.as_deref().unwrap_or(user.id.id());
self.core self.core
.import_user_id( .user_cache
SrcId(user.id.id(), MusicService::Spotify), .get_or_insert_async(&id_owned, async {
user.display_name.as_deref().unwrap_or(user.id.id()), let user = UserNew {
false, src_id: &id_owned.0,
) service: id_owned.1,
name,
user_type: UserType::Remote,
image_url: image_url.as_deref(),
..Default::default()
};
let user_id = user.upsert(&self.core.db).await?;
tracing::debug!("imported user id [{}] {}", id_owned, name);
Ok(user_id)
})
.await .await
} }
async fn import_track(&self, track: FullTrack) -> Result<Option<i32>, ExtractorError> { async fn import_playlist(&self, playlist: SimplifiedPlaylist) -> Result<i32, ExtractorError> {
if let Some(id) = Playlist::get_id(
SrcId(playlist.id.id(), MusicService::Spotify),
&self.core.db,
)
.await?
{
return Ok(id);
}
let image_url = playlist
.images
.into_iter()
.max_by_key(|img| img.height.unwrap_or_default())
.map(|img| img.url);
let owner_id = self.import_user(playlist.owner).await?;
let playlist_n = PlaylistNew {
src_id: playlist.id.id(),
service: MusicService::Spotify,
name: &playlist.name,
owner_id: Some(owner_id),
playlist_type: PlaylistType::Remote,
image_url: image_url.as_deref(),
image_type: Some(PlaylistImgType::Custom).filter(|_| image_url.is_some()),
..Default::default()
};
let playlist_id = playlist_n.upsert(&self.core.db).await?;
tracing::debug!(
"imported playlist item [sp:{}] {}",
playlist.id.id(),
playlist.name
);
Ok(playlist_id)
}
async fn import_playlists(
&self,
playlists: impl IntoIterator<Item = SimplifiedPlaylist>,
) -> Result<(), ExtractorError> {
futures::stream::iter(playlists.into_iter().map(Ok))
.try_for_each_concurrent(CONFIG.core.db_concurrency, |pl| async {
self.import_playlist(pl).await?;
Ok(())
})
.await
}
async fn import_track(&self, track: FullTrack) -> Result<i32, ExtractorError> {
let src_id = SrcId(
track
.id
.as_ref()
.ok_or(ExtractorError::Other("no Spotify track id".into()))?
.id(),
MusicService::Spotify,
);
if let Some(track_id) = Track::get_id(src_id, &self.core.db).await? {
return Ok(track_id);
}
let artists = track
.artists
.into_iter()
.filter_map(|a| {
a.id.as_ref().map(|a_id| ArtistIdName {
id: a_id.id().to_owned(),
name: a.name,
})
})
.collect::<Vec<_>>();
self.yt self.yt
.match_track( .match_track(
SrcId( src_id,
track
.id
.ok_or(ExtractorError::Other("no Spotify track id".into()))?
.id(),
MusicService::Spotify,
),
&track.name, &track.name,
track &artists,
.artists
.first()
.map(|a| a.name.as_str())
.unwrap_or_default(),
track.external_ids.get("isrc").map(|s| s.as_str()), track.external_ids.get("isrc").map(|s| s.as_str()),
) )
.await .await
@ -203,11 +431,17 @@ impl SpotifyExtractor {
async fn import_tracks( async fn import_tracks(
&self, &self,
tracks: impl IntoIterator<Item = FullTrack>, tracks: impl IntoIterator<Item = FullTrack>,
) -> Result<Vec<Option<i32>>, ExtractorError> { ) -> Result<(), ExtractorError> {
futures::stream::iter(tracks) futures::stream::iter(tracks.into_iter().map(Ok))
.map(|track| async { self.import_track(track).await }) .try_for_each_concurrent(CONFIG.extractor.youtube.concurrency, |track| async move {
.buffered(CONFIG.extractor.youtube.concurrency) match self.import_track(track).await {
.try_collect() Ok(_) => Ok(()),
Err(e) => match e {
ExtractorError::NoMatch { .. } => Ok(()),
_ => Err(e),
},
}
})
.await .await
} }
} }

388
crates/extractor/src/services/youtube.rs
View file

@ -3,29 +3,29 @@ use std::str::FromStr;
use std::sync::Arc; use std::sync::Arc;
use futures::{StreamExt, TryStreamExt}; use futures::{StreamExt, TryStreamExt};
use path_macro::path;
use rustypipe::model as rpmodel; use rustypipe::model as rpmodel;
use rustypipe::param::search_filter::SearchFilter; use rustypipe::{
use rustypipe::report::FileReporter; client::RustyPipe, model::richtext::ToPlaintext, param::search_filter::SearchFilter,
use rustypipe::{client::RustyPipe, model::richtext::ToPlaintext}; report::FileReporter,
};
use time::{Date, OffsetDateTime, PrimitiveDateTime, Time}; use time::{Date, OffsetDateTime, PrimitiveDateTime, Time};
use tiraya_db::models::{PlaylistEntry, User};
use tiraya_db::{ use tiraya_db::{
error::OptionalRes, error::OptionalRes,
models::{ models::{
Album, AlbumNew, AlbumType, AlbumUpdate, Artist, ArtistNew, DatePrecision, Id, Album, AlbumNew, AlbumType, AlbumUpdate, Artist, ArtistId, ArtistNew, DatePrecision, Id,
MusicService, Playlist, PlaylistImgType, PlaylistNew, PlaylistType, SrcId, SrcIdOwned, MusicService, Playlist, PlaylistEntry, PlaylistImgType, PlaylistNew, PlaylistType, SrcId,
Track, TrackNew, TrackUpdate, UserNew, UserType, SrcIdOwned, Track, TrackNew, TrackUpdate, User, UserNew, UserType,
}, },
}; };
use tiraya_utils::config::CONFIG; use tiraya_utils::config::CONFIG;
use crate::util::ImgFormat; use crate::util::ArtistSrcidName;
use crate::ITEM_LIMIT;
use crate::{ use crate::{
error::ExtractorError, error::ExtractorError,
model::{ExtractorCore, GetResult, LastUpdateState, SyncLastUpdate}, model::{ExtractorCore, GetResult, LastUpdateState, SyncLastUpdate},
util::{self, ArtistIdName}, util::{self, ArtistIdName, ImgFormat},
CONCURRENCY, DB_CONCURRENCY, GetStatus,
}; };
const YTM_CHANNEL_SUFFIX: &str = " - Topic"; const YTM_CHANNEL_SUFFIX: &str = " - Topic";
@ -50,7 +50,9 @@ impl YouTubeExtractor {
&CONFIG.extractor.youtube.country, &CONFIG.extractor.youtube.country,
)?) )?)
.storage_dir(&CONFIG.extractor.data_dir) .storage_dir(&CONFIG.extractor.data_dir)
.reporter(Box::new(FileReporter::new(&CONFIG.extractor.data_dir))) .reporter(Box::new(FileReporter::new(path!(
CONFIG.extractor.data_dir / "rustypipe_reports"
))))
.n_http_retries(CONFIG.extractor.youtube.n_retries) .n_http_retries(CONFIG.extractor.youtube.n_retries)
.build()?, .build()?,
}) })
@ -125,8 +127,8 @@ impl YouTubeExtractor {
// Insert all albums // Insert all albums
futures::stream::iter(artist.albums) futures::stream::iter(artist.albums)
.map(Ok) .map(Ok)
.try_for_each_concurrent(DB_CONCURRENCY, |album| async { .try_for_each_concurrent(CONFIG.core.db_concurrency, |album| async {
self.import_album_item(album).await self.import_album_item(album).await.map(|_| ())
}) })
.await?; .await?;
@ -200,8 +202,8 @@ impl YouTubeExtractor {
// Insert album variants# // Insert album variants#
let has_variants = !album.variants.is_empty(); let has_variants = !album.variants.is_empty();
futures::stream::iter(album.variants.into_iter().map(Ok::<_, ExtractorError>)) futures::stream::iter(album.variants.into_iter().map(Ok::<_, ExtractorError>))
.try_for_each_concurrent(DB_CONCURRENCY, |album| async { .try_for_each_concurrent(CONFIG.core.db_concurrency, |album| async {
self.import_album_item(album).await self.import_album_item(album).await.map(|_| ())
}) })
.await?; .await?;
@ -221,12 +223,39 @@ impl YouTubeExtractor {
} }
pub async fn fetch_track(&self, src_id: &str) -> Result<i32, ExtractorError> { pub async fn fetch_track(&self, src_id: &str) -> Result<i32, ExtractorError> {
self._fetch_track(src_id, true).await
}
async fn _fetch_track(&self, src_id: &str, with_details: bool) -> Result<i32, ExtractorError> {
let query = self.rp.query(); let query = self.rp.query();
let (track, details) = let (track, details) = if with_details {
tokio::join!(query.music_details(src_id), self.get_track_details(src_id)); let (track, details) =
let track = track?.track; tokio::join!(query.music_details(src_id), self.get_track_details(src_id));
self.import_track_item(track, None, &[], details, false) (track?.track, details)
.await } else {
(query.music_details(src_id).await?.track, None)
};
let redirected_id = if track.id != src_id {
Some(track.id.clone())
} else {
None
};
let track_id = self
.import_track_item(track, None, &[], details, false)
.await?;
if let Some(redirected_id) = redirected_id {
tracing::info!(
"track [yt:{src_id}] got redirected to [yt:{redirected_id}], added alias"
);
Track::add_alias(
track_id,
SrcId(&redirected_id, MusicService::YouTube),
&self.core.db,
)
.await?;
}
Ok(track_id)
} }
pub async fn update_playlist( pub async fn update_playlist(
@ -273,7 +302,9 @@ impl YouTubeExtractor {
let playlist_id = n_playlist.upsert(&self.core.db).await?; let playlist_id = n_playlist.upsert(&self.core.db).await?;
let mut m_tracks = m_playlist.tracks; let mut m_tracks = m_playlist.tracks;
m_tracks.extend_limit(&query, ITEM_LIMIT).await?; m_tracks
.extend_limit(&query, CONFIG.extractor.playlist_item_limit)
.await?;
let new_entries = m_tracks let new_entries = m_tracks
.items .items
@ -346,7 +377,7 @@ impl YouTubeExtractor {
channel channel
.content .content
.extend_limit(self.rp.query(), ITEM_LIMIT) .extend_limit(self.rp.query(), CONFIG.extractor.user_playlist_limit)
.await?; .await?;
let image_url = util::yt_image_url(&channel.avatar, ImgFormat::Avatar); let image_url = util::yt_image_url(&channel.avatar, ImgFormat::Avatar);
@ -381,38 +412,40 @@ impl YouTubeExtractor {
pub async fn match_track( pub async fn match_track(
&self, &self,
src_id: SrcId<'_>, src_id: SrcId<'_>,
title: &str, name: &str,
artist: &str, artists: &[ArtistIdName],
isrc: Option<&str>, isrc: Option<&str>,
) -> Result<Option<i32>, ExtractorError> { ) -> Result<i32, ExtractorError> {
if let Some(track_id) = Track::get_id(src_id, &self.core.db).await? { let track_id = self.get_matching_track(src_id, name, artists, isrc).await?;
return Ok(Some(track_id));
}
if let Some((track_id, _)) = self.get_matching_track(title, artist, isrc).await? { Track::add_alias(track_id, src_id, &self.core.db).await?;
Track::add_alias(track_id, src_id, &self.core.db).await?; if let Some(isrc) = isrc.and_then(util::normalize_isrc) {
if let Some(isrc) = isrc { let t_upd = TrackUpdate {
let t_upd = TrackUpdate { isrc: Some(Some(&isrc)),
isrc: Some(Some(isrc)), ..Default::default()
..Default::default() };
}; t_upd.update(Id::Db(track_id), &self.core.db).await?;
t_upd.update(Id::Db(track_id), &self.core.db).await?;
}
tracing::info!("matched track {title} [{src_id}]");
Ok(Some(track_id))
} else {
Ok(None)
} }
Ok(track_id)
} }
async fn get_matching_track( async fn get_matching_track(
&self, &self,
title: &str, src_id: SrcId<'_>,
artist: &str, name: &str,
artists: &[ArtistIdName],
isrc: Option<&str>, isrc: Option<&str>,
) -> Result<Option<(i32, f32)>, ExtractorError> { ) -> Result<i32, ExtractorError> {
let t1 = self.core.matchmaker.parse_title(title); let artist = &artists
.first()
.ok_or_else(|| ExtractorError::NoMatch {
id: src_id.to_owned(),
})?
.name;
let t1 = self.core.matchmaker.parse_title(name);
// Attempt to find the track by searching YouTube for its ISRC id
// If the title of the first search result matches, return this track.
if let Some(isrc) = isrc { if let Some(isrc) = isrc {
let filter = SearchFilter::new() let filter = SearchFilter::new()
.item_type(rustypipe::param::search_filter::ItemType::Video) .item_type(rustypipe::param::search_filter::ItemType::Video)
@ -422,17 +455,38 @@ impl YouTubeExtractor {
let t2 = self.core.matchmaker.parse_title(&video.name); let t2 = self.core.matchmaker.parse_title(&video.name);
let score = self.core.matchmaker.match_name(&t1, &t2); let score = self.core.matchmaker.match_name(&t1, &t2);
if self.core.matchmaker.is_match(score) { if self.core.matchmaker.is_match(score) {
return Ok(Some((self.fetch_track(&video.id).await?, score))); tracing::info!(
"matched track `{name}` [{src_id}] => `{0}` [yt:{1}] (ISRC, {score:.2})",
video.name,
video.id
);
// Get the found track either from YT or the database
let track = if let Some(track) =
Track::get(Id::Src(&video.id, MusicService::YouTube), &self.core.db)
.await
.to_optional()?
{
track
} else {
let track_id = self._fetch_track(&video.id, false).await?;
Track::get(Id::Db(track_id), &self.core.db).await?
};
self.store_matching_artists(artists, src_id.1, &track.artists)
.await?;
return Ok(track.id);
} }
} }
} }
let a1 = self.core.matchmaker.parse_artist(artist); let a1 = self.core.matchmaker.parse_artist(artist);
// Find the track by searching for YT Music tracks/videos
let search = self let search = self
.rp .rp
.query() .query()
.music_search(format!("{title} {artist}")) .music_search(format!("{name} {artist}"))
.await?; .await?;
let best_match = search let best_match = search
.tracks .tracks
@ -454,15 +508,170 @@ impl YouTubeExtractor {
.max_by(|a, b| a.1.total_cmp(&b.1)); .max_by(|a, b| a.1.total_cmp(&b.1));
if let Some((track, score)) = best_match { if let Some((track, score)) = best_match {
if self.core.matchmaker.is_match(score) { if self.core.matchmaker.is_match(score) {
return Ok(Some(( tracing::info!(
self.import_track_item(track, None, &[], None, false) "matched track `{name}` [{src_id}] => `{0}` [yt:{1}] (search, {score:.2})",
.await?, track.name,
score, track.id
))); );
return self.import_track_item(track, None, &[], None, false).await;
} }
tracing::info!(
"could not match track `{name}` [{src_id}] => `{0}` [yt:{1}] (search, {score:.2})",
track.name,
track.id
);
} else {
tracing::info!("could not match track `{name}` [{src_id}] (search)");
} }
Ok(None) Err(ExtractorError::NoMatch {
id: src_id.to_owned(),
})
}
/// Try to match the artists from a different music service to the ones of a track
/// already stored in the database.
///
/// Returns the artist id matched to the first given artist
async fn store_matching_artists(
&self,
artists_to_match: &[ArtistIdName],
service: MusicService,
yt_artists: &[ArtistId],
) -> Result<(), ExtractorError> {
let mut yt_artists = yt_artists
.iter()
.filter_map(|a| {
a.id.as_ref().map(|id| {
(
ArtistSrcidName {
id: id.as_srcid(),
name: &a.name,
},
self.core.matchmaker.parse_artist(&a.name),
)
})
})
.collect::<Vec<_>>();
for atm in artists_to_match {
let atm_srcid = SrcIdOwned(atm.id.to_owned(), service);
// Check if the artist has already been matched
let atm_id_res = self
.core
.artist_cache
.get_or_insert_async(&atm_srcid, async {
Artist::get_id(atm_srcid.as_srcid(), &self.core.db)
.await?
.ok_or(ExtractorError::NoId)
})
.await;
match atm_id_res {
Ok(atm_id) => {
// Artist is already matched
let yt_src_id = Artist::get_src_id(atm_id, &self.core.db)
.await?
.ok_or(ExtractorError::NoId)?;
if let Some((idx, _)) = yt_artists
.iter()
.enumerate()
.find(|(_, (a, _))| a.id == yt_src_id)
{
yt_artists.swap_remove(idx);
}
}
Err(ExtractorError::NoId) => {
// Match artist
let a1 = self.core.matchmaker.parse_artist(&atm.name);
let (ib, score) = yt_artists
.iter()
.map(|(_, a2)| self.core.matchmaker.match_name(&a1, a2))
.enumerate()
.max_by(|(_, a), (_, b)| a.total_cmp(b))
.unwrap_or_default();
if score > 0.8 {
let yta = yt_artists.swap_remove(ib).0;
let yta_id = Artist::get_id(yta.id, &self.core.db)
.await?
.ok_or(ExtractorError::NoId)?;
Artist::add_alias(yta_id, atm_srcid.as_srcid(), &self.core.db).await?;
tracing::info!(
"matched artist `{}` [{}] => `{}` [{}] ({score:.2})",
yta.name,
yta.id,
atm.name,
atm_srcid
);
self.core.artist_cache.insert(atm_srcid, yta_id);
}
}
Err(e) => return Err(e),
}
}
Ok(())
}
/// Try to match an album from a different streaming service to a YouTube track
pub async fn match_album(
&self,
src_id: SrcId<'_>,
name: &str,
artist: &str,
) -> Result<i32, ExtractorError> {
let albums = self
.rp
.query()
.music_search_albums(&format!("{name} {artist}"))
.await?
.items
.items;
let t1 = self.core.matchmaker.parse_title(name);
let a1 = self.core.matchmaker.parse_artist(name);
let best_match = albums
.into_iter()
.map(|album| {
let t2 = self.core.matchmaker.parse_title(&album.name);
let a2 = self.core.matchmaker.parse_title(
&album
.artists
.first()
.map(|a| a.name.to_owned())
.unwrap_or_default(),
);
let score = self.core.matchmaker.match_track(&t1, &a1, &t2, &a2, false);
(album, score)
})
.max_by(|a, b| a.1.total_cmp(&b.1));
if let Some((album, score)) = best_match {
if self.core.matchmaker.is_match(score) {
tracing::info!(
"matched album `{name}` [{src_id}] => `{0}` [yt:{1}] (search, {score:.2})",
album.name,
album.id
);
let album_id = album.id.clone();
let import_res = self.import_album_item(album).await?;
if import_res.status == GetStatus::Fetched {
return self.fetch_album(&album_id).await.map(|res| res.0);
} else {
return Ok(import_res.c);
}
}
tracing::info!(
"could not match album `{name}` [{src_id}] => `{0}` [yt:{1}] (search, {score:.2})",
album.name,
album.id
);
} else {
tracing::info!("could not match album `{name}` [{src_id}] (search)");
}
Err(ExtractorError::NoMatch {
id: src_id.to_owned(),
})
} }
} }
@ -515,22 +724,7 @@ impl YouTubeExtractor {
let release_date_precision = let release_date_precision =
details.as_ref().and_then(|d| d.release_date).map(|d| d.1); details.as_ref().and_then(|d| d.release_date).map(|d| d.1);
if track.is_video { if let Some(t_album) = track.album {
// Create a pseudo-album for music videos
let n_album = AlbumNew {
src_id: &track.id,
service: MusicService::YouTube,
name: &track.name,
release_date,
release_date_precision,
album_type: Some(AlbumType::Mv),
by_va: track.by_va,
image_url: image_url.as_deref(),
dirty: false,
..Default::default()
};
n_album.upsert(&self.core.db).await?
} else if let Some(t_album) = track.album {
let n_album = AlbumNew { let n_album = AlbumNew {
src_id: &t_album.id, src_id: &t_album.id,
service: MusicService::YouTube, service: MusicService::YouTube,
@ -542,10 +736,24 @@ impl YouTubeExtractor {
}; };
n_album.upsert(&self.core.db).await? n_album.upsert(&self.core.db).await?
} else { } else {
return Err(ExtractorError::InvalidData { if !track.is_video {
id: SrcIdOwned(track.id, MusicService::YouTube), tracing::warn!("track [yt:{}] has no album, but is no MV", track.id);
msg: "unknown album".into(), }
});
// Create a pseudo-album for music videos
let n_album = AlbumNew {
src_id: &track.id,
service: MusicService::YouTube,
name: &track.name,
release_date,
release_date_precision,
album_type: Some(AlbumType::Mv).filter(|_| track.is_video),
by_va: track.by_va,
image_url: image_url.as_deref(),
dirty: false,
..Default::default()
};
n_album.upsert(&self.core.db).await?
} }
} }
}; };
@ -588,7 +796,7 @@ impl YouTubeExtractor {
self.import_track_item(track, album_id, album_artists, None, hidden) self.import_track_item(track, album_id, album_artists, None, hidden)
.await .await
}) })
.buffered(DB_CONCURRENCY) .buffered(CONFIG.core.db_concurrency)
.try_collect() .try_collect()
.await .await
} }
@ -619,7 +827,7 @@ impl YouTubeExtractor {
) -> Result<Vec<i32>, ExtractorError> { ) -> Result<Vec<i32>, ExtractorError> {
futures::stream::iter(artists) futures::stream::iter(artists)
.map(|aid| async { self.import_artist_id(aid).await }) .map(|aid| async { self.import_artist_id(aid).await })
.buffered(CONCURRENCY) .buffered(CONFIG.core.db_concurrency)
.try_collect::<Vec<_>>() .try_collect::<Vec<_>>()
.await .await
.map_err(ExtractorError::from) .map_err(ExtractorError::from)
@ -653,20 +861,22 @@ impl YouTubeExtractor {
) -> Result<Vec<i32>, ExtractorError> { ) -> Result<Vec<i32>, ExtractorError> {
futures::stream::iter(artists) futures::stream::iter(artists)
.map(|artist| async { self.import_artist_item(artist).await }) .map(|artist| async { self.import_artist_item(artist).await })
.buffered(CONCURRENCY) .buffered(CONFIG.core.db_concurrency)
.try_collect::<Vec<_>>() .try_collect::<Vec<_>>()
.await .await
.map_err(ExtractorError::from) .map_err(ExtractorError::from)
} }
/// Import a YT Music album item (album from artist page) /// Import a YT Music album item (album from artist page)
async fn import_album_item(&self, album: rpmodel::AlbumItem) -> Result<(), ExtractorError> { async fn import_album_item(
&self,
album: rpmodel::AlbumItem,
) -> Result<GetResult<i32>, ExtractorError> {
// Return if the album was already imported // Return if the album was already imported
if Album::get_id_clean(SrcId(&album.id, MusicService::YouTube), &self.core.db) if let Some(album_id) =
.await? Album::get_id_clean(SrcId(&album.id, MusicService::YouTube), &self.core.db).await?
.is_some()
{ {
return Ok(()); return Ok(GetResult::stored(album_id));
} }
let (artists, ul_artists) = self let (artists, ul_artists) = self
@ -695,7 +905,7 @@ impl YouTubeExtractor {
tx.commit().await?; tx.commit().await?;
tracing::debug!("imported album item [yt:{}] {}", album.id, album.name); tracing::debug!("imported album item [yt:{}] {}", album.id, album.name);
Ok(()) Ok(GetResult::fetched(album_id))
} }
/// Import a YT Music playlist item /// Import a YT Music playlist item
@ -735,7 +945,7 @@ impl YouTubeExtractor {
) -> Result<Vec<i32>, ExtractorError> { ) -> Result<Vec<i32>, ExtractorError> {
futures::stream::iter(playlists) futures::stream::iter(playlists)
.map(|playlists| async { self.import_playlist_item(playlists).await }) .map(|playlists| async { self.import_playlist_item(playlists).await })
.buffered(CONCURRENCY) .buffered(CONFIG.core.db_concurrency)
.try_collect::<Vec<_>>() .try_collect::<Vec<_>>()
.await .await
.map_err(ExtractorError::from) .map_err(ExtractorError::from)
@ -779,8 +989,8 @@ impl YouTubeExtractor {
playlists: impl IntoIterator<Item = rpmodel::PlaylistItem>, playlists: impl IntoIterator<Item = rpmodel::PlaylistItem>,
) -> Result<Vec<i32>, ExtractorError> { ) -> Result<Vec<i32>, ExtractorError> {
futures::stream::iter(playlists) futures::stream::iter(playlists)
.map(|playlists| async { self.import_playlist_item_yt(playlists).await }) .map(|pl| async { self.import_playlist_item_yt(pl).await })
.buffered(CONCURRENCY) .buffered(CONFIG.core.db_concurrency)
.try_collect::<Vec<_>>() .try_collect::<Vec<_>>()
.await .await
.map_err(ExtractorError::from) .map_err(ExtractorError::from)
@ -988,11 +1198,13 @@ mod tests {
.match_track( .match_track(
SrcId("5awNIWVrh2ISfvPd5IUZNh", MusicService::Spotify), SrcId("5awNIWVrh2ISfvPd5IUZNh", MusicService::Spotify),
"PTT (Paint The Town)", "PTT (Paint The Town)",
"LOONA", &[ArtistIdName {
id: "52zMTJCKluDlFwMQWmccY7".to_owned(),
name: "LOONA".to_owned(),
}],
Some("KRA382152284"), Some("KRA382152284"),
) )
.await .await
.unwrap()
.unwrap(); .unwrap();
let track = Track::get(Id::Db(track_id), &pool).await.unwrap(); let track = Track::get(Id::Db(track_id), &pool).await.unwrap();
insta::assert_ron_snapshot!(track, { insta::assert_ron_snapshot!(track, {

48
crates/extractor/src/util/matchmaker.rs
View file

@ -163,10 +163,7 @@ impl Matchmaker {
} }
fn parse(&self, s: &str, artist: bool) -> ParsedName { fn parse(&self, s: &str, artist: bool) -> ParsedName {
let mut s = s.to_ascii_lowercase(); let s = s.to_ascii_lowercase().replace("feat.", "|");
if !artist {
s = s.replace("feat.", "|")
}
let mut parts = Vec::new(); let mut parts = Vec::new();
let mut n_parts = 0; let mut n_parts = 0;
@ -192,18 +189,12 @@ impl Matchmaker {
if brace_level == 0 { if brace_level == 0 {
do_split = true; do_split = true;
} }
} else if ((artist || space) && SEPARATORS.contains(&c)) } else if (space && SEPARATORS.contains(&c)) || SEPARATORS_NOSPACE.contains(&c) {
|| SEPARATORS_NOSPACE.contains(&c)
{
do_split = true; do_split = true;
} else if c.is_whitespace() { } else if c.is_whitespace() {
if artist { // Dont create double spaces
do_split = true; if !space && !buf.is_empty() {
} else { buf += " ";
// Dont create double spaces
if !space && !buf.is_empty() {
buf += " ";
}
} }
} else { } else {
buf.push(c); buf.push(c);
@ -556,4 +547,33 @@ mod tests {
"'{a}' does not match '{b}': score {score} < 0.5\npA: {parsed_a:?}\npB: {parsed_b:?}" "'{a}' does not match '{b}': score {score} < 0.5\npA: {parsed_a:?}\npB: {parsed_b:?}"
); );
} }
#[rstest]
#[case("Fantasy", "Max-Antoine", "Fantasy", "Max-Antoine Meisters", false)]
#[case(
"Armada",
"Two Steps From Hell",
"Two Steps From Hell - Armada",
"ThePrimeAres2",
true
)]
fn match_tracks(
#[case] t1: &str,
#[case] a1: &str,
#[case] t2: &str,
#[case] a2: &str,
#[case] is_video: bool,
) {
let mm = Matchmaker::new(KEYWORDS.values(), MATCH_THR);
let parsed_t1 = mm.parse_title(t1);
let parsed_a1 = mm.parse_artist(a1);
let parsed_t2 = mm.parse_title(t2);
let parsed_a2 = mm.parse_artist(a2);
let score = mm.match_track(&parsed_t1, &parsed_a1, &parsed_t2, &parsed_a2, is_video);
assert!(
mm.is_match(score),
"'{t1}' does not match '{t2}': score {score} < 0.5\npA: {parsed_t1:?}\npB: {parsed_t2:?}"
);
}
} }

30
crates/extractor/src/util/mod.rs
View file

@ -9,10 +9,15 @@ use regex::Regex;
use rustypipe::model as rpmodel; use rustypipe::model as rpmodel;
use siphasher::sip128::{Hasher128, SipHasher}; use siphasher::sip128::{Hasher128, SipHasher};
use time::{Date, Duration, OffsetDateTime}; use time::{Date, Duration, OffsetDateTime};
use tiraya_db::models::{AlbumType, DatePrecision, SyncData, SyncError}; use tiraya_db::models::{AlbumType, DatePrecision, SrcId, SyncData, SyncError};
use crate::error::ExtractorSourceError; use crate::error::ExtractorSourceError;
pub struct ArtistSrcidName<'a> {
pub id: SrcId<'a>,
pub name: &'a str,
}
#[derive(Clone)] #[derive(Clone)]
pub struct ArtistIdName { pub struct ArtistIdName {
pub id: String, pub id: String,
@ -177,6 +182,21 @@ impl AlbumHasher {
} }
} }
pub fn normalize_isrc(isrc: &str) -> Option<String> {
static ISRC_RE: Lazy<Regex> =
Lazy::new(|| Regex::new("^[A-z]{2}[- ]?[A-z0-9]{3}[- ]?[0-9]{2}[- ]?[0-9]{5}$").unwrap());
if ISRC_RE.is_match(isrc) {
Some(
isrc.chars()
.filter(|c| *c != ' ' && *c != '-')
.map(|c| c.to_ascii_uppercase())
.collect(),
)
} else {
None
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@ -245,4 +265,12 @@ mod tests {
let hash = hasher.finish(); let hash = hasher.finish();
assert_eq!(hash, expect, "got {hash:x?}"); assert_eq!(hash, expect, "got {hash:x?}");
} }
#[rstest]
#[case("NL 1AP 19 35969", Some("NL1AP1935969"))]
#[case("nl 1AP 19 35969", Some("NL1AP1935969"))]
#[case("DE-ZC6-2342630", Some("DEZC62342630"))]
fn t_normalize_isrc(#[case] isrc: &str, #[case] expect: Option<&str>) {
assert_eq!(normalize_isrc(isrc).as_deref(), expect);
}
} }

11
crates/extractor/tests/tests.rs
View file

@ -204,7 +204,7 @@ mod sp {
/* /*
#[sqlx_database_tester::test(pool(variable = "pool", migrations = "../db/migrations"))] #[sqlx_database_tester::test(pool(variable = "pool", migrations = "../db/migrations"))]
async fn update_playlist() { async fn update_playlist() {
let xtr = xtr(pool.clone()).await; let xtr = Extractor::new(pool.clone()).unwrap();
// 7Cdbc4geOdYRdsne9iOH49 // 7Cdbc4geOdYRdsne9iOH49
@ -215,5 +215,14 @@ mod sp {
let playlist_tracks = Playlist::get_tracks(res.c, &pool).await.unwrap().0; let playlist_tracks = Playlist::get_tracks(res.c, &pool).await.unwrap().0;
dbg!(&playlist_tracks); dbg!(&playlist_tracks);
}
#[sqlx_database_tester::test(pool(variable = "pool", migrations = "../db/migrations"))]
async fn update_user() {
setup_log();
let xtr = Extractor::new(pool.clone()).unwrap();
xtr.get_user(SrcId("1251642561", MusicService::Spotify)).await.unwrap();
}*/ }*/
} }

38
crates/testbed/src/main.rs
View file

@ -2,7 +2,7 @@ use std::{str::FromStr, time::Instant};
use clap::{Parser, Subcommand}; use clap::{Parser, Subcommand};
use sqlx::PgPool; use sqlx::PgPool;
use tiraya_db::models::{Playlist, SrcIdOwned}; use tiraya_db::models::{Playlist, SrcIdOwned, User};
use tiraya_extractor::{Extractor, GetStatus}; use tiraya_extractor::{Extractor, GetStatus};
#[derive(Parser)] #[derive(Parser)]
@ -14,7 +14,10 @@ struct Cli {
#[derive(Subcommand)] #[derive(Subcommand)]
enum Commands { enum Commands {
Artist { id: String }, Artist { id: String },
Album { id: String },
Track { id: String },
Playlist { id: String }, Playlist { id: String },
User { id: String },
} }
#[tokio::main] #[tokio::main]
@ -40,21 +43,50 @@ async fn run() {
if artist_res.status == GetStatus::Fetched { if artist_res.status == GetStatus::Fetched {
ext.fetch_artist_albums(artist_res.c.id).await.unwrap(); ext.fetch_artist_albums(artist_res.c.id).await.unwrap();
} }
dbg!(artist_res.c); dbg!(artist_res.c);
} }
Commands::Album { id } => {
let src_id = SrcIdOwned::from_str(&id).unwrap();
let album = ext.get_album(src_id.as_srcid()).await.unwrap().c;
dbg!(album);
}
Commands::Track { id } => {
let src_id = SrcIdOwned::from_str(&id).unwrap();
let track = ext.get_track(src_id.as_srcid()).await.unwrap().c;
dbg!(track);
}
Commands::Playlist { id } => { Commands::Playlist { id } => {
let src_id = SrcIdOwned::from_str(&id).unwrap(); let src_id = SrcIdOwned::from_str(&id).unwrap();
let playlist = ext.get_playlist(src_id.as_srcid()).await.unwrap(); let playlist = ext.get_playlist(src_id.as_srcid()).await.unwrap();
let tracks = Playlist::get_tracks(playlist.c.id, &pool).await.unwrap().0; let tracks = Playlist::get_tracks(playlist.c.id, &pool).await.unwrap().0;
for (i, e) in tracks.iter().enumerate() { for (i, e) in tracks.iter().enumerate() {
if let Some(track) = &e.track { if let Some(track) = &e.track {
println!("{} [{}] {}", i + 1, track.src_id, track.name); println!(
"{} [{}] {} - {}",
i + 1,
track.src_id,
track.name,
track
.artists
.first()
.map(|a| a.name.as_str())
.unwrap_or_default()
);
} else { } else {
println!("{} n/a", i + 1) println!("{} n/a", i + 1)
} }
} }
} }
Commands::User { id } => {
let src_id = SrcIdOwned::from_str(&id).unwrap();
let user = ext.get_user(src_id.as_srcid()).await.unwrap().c;
let playlists = User::playlists(user.id, &pool).await.unwrap();
dbg!(user);
for (i, playlist) in playlists.iter().enumerate() {
println!("{} {}", i + 1, playlist.name)
}
}
} }
println!("Time: {}ms", t_start.elapsed().as_millis()); println!("Time: {}ms", t_start.elapsed().as_millis());
} }

21
crates/utils/src/config.rs
View file

@ -41,6 +41,18 @@ pub struct ConfigExtractor {
/// # Number of hours after a playlist is considered stale and needs to be updated from the source /// # Number of hours after a playlist is considered stale and needs to be updated from the source
#[default(24)] #[default(24)]
pub playlist_stale_h: u32, pub playlist_stale_h: u32,
/// # Number of hours after a user is considered stale and needs to be updated from the source
#[default(24)]
pub user_stale_h: u32,
/// Maximum number of playlist items to be extracted
#[default(2000)]
pub playlist_item_limit: usize,
/// Maximum number of playlist items to be extracted and matched to YTM tracks
#[default(500)]
pub playlist_item_limit_matchmaker: usize,
/// Maximum number of user playlists to be extracted
#[default(200)]
pub user_playlist_limit: usize,
/// Directory for caching service tokens and other extractor data /// Directory for caching service tokens and other extractor data
#[default("extractor_data")] #[default("extractor_data")]
pub data_dir: PathBuf, pub data_dir: PathBuf,
@ -60,7 +72,7 @@ pub struct ConfigExtractor {
pub match_threshold: f32, pub match_threshold: f32,
pub youtube: ConfigExtractorYouTube, pub youtube: ConfigExtractorYouTube,
pub spotify: Option<ConfigExtractorSpotify>, pub spotify: ConfigExtractorSpotify,
} }
#[derive(Debug, Serialize, Deserialize, SmartDefault)] #[derive(Debug, Serialize, Deserialize, SmartDefault)]
@ -80,13 +92,18 @@ pub struct ConfigExtractorYouTube {
pub concurrency: usize, pub concurrency: usize,
} }
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize, Deserialize, SmartDefault)]
#[serde(default)]
pub struct ConfigExtractorSpotify { pub struct ConfigExtractorSpotify {
pub enable: bool,
/// Spotify client credentials /// Spotify client credentials
/// ///
/// Setup under <https://developer.spotify.com/dashboard> /// Setup under <https://developer.spotify.com/dashboard>
pub client_id: String, pub client_id: String,
pub client_secret: String, pub client_secret: String,
/// Spotify content country
#[default("US")]
pub market: String,
} }
const DEFAULT_CONFIG_FILE: &str = "config.toml"; const DEFAULT_CONFIG_FILE: &str = "config.toml";

2
crates/utils/src/lib.rs
View file

@ -1 +1,3 @@
#![warn(clippy::dbg_macro, clippy::todo)]
pub mod config; pub mod config;

8
crates/utils/src/snapshots/tiraya_utils__config__tests__read_config.snap
View file

@ -1,6 +1,5 @@
--- ---
source: crates/utils/src/config.rs source: crates/utils/src/config.rs
assertion_line: 117
expression: cfg expression: cfg
--- ---
Config( Config(
@ -17,15 +16,18 @@ Config(
artist_playlist_excluded_types: [ artist_playlist_excluded_types: [
"inst", "inst",
], ],
match_threshold: 0.5,
youtube: ConfigExtractorYouTube( youtube: ConfigExtractorYouTube(
language: "en", language: "en",
country: "US", country: "US",
n_retries: 2, n_retries: 2,
concurrency: 4, concurrency: 4,
), ),
spotify: Some(ConfigExtractorSpotify( spotify: ConfigExtractorSpotify(
enable: true,
client_id: "abc123", client_id: "abc123",
client_secret: "supersecret", client_secret: "supersecret",
)), market: None,
),
), ),
) )

13
crates/utils/testfiles/config.toml
View file

@ -11,10 +11,16 @@ db_concurrency = 8
# Music data extractor settings # Music data extractor settings
[extractor] [extractor]
# Number of hours after an item is considered stale # Number of hours after an item is considered stale and needs to be updated from the source
# and needs to be updated from the source
artist_stale_h = 24 artist_stale_h = 24
playlist_stale_h = 24 playlist_stale_h = 24
user_stale_h = 24
# Maximum number of playlist items to be extracted
playlist_item_limit = 2000
# Maximum number of playlist items to be extracted and matched to YTM tracks
playlist_item_limit_matchmaker = 500
# Maximum number of user playlists to be extracted
user_playlist_limit = 200
# Directory for caching service tokens and other extractor data # Directory for caching service tokens and other extractor data
data_dir = "extractor_data" data_dir = "extractor_data"
# Set of track types (keys from `track_type_keywords`) excluded from artist playlists # Set of track types (keys from `track_type_keywords`) excluded from artist playlists
@ -54,7 +60,10 @@ n_retries = 2
concurrency = 4 concurrency = 4
[extractor.spotify] [extractor.spotify]
enable = true
# Spotify client credentials # Spotify client credentials
# Setup under https://developer.spotify.com/dashboard # Setup under https://developer.spotify.com/dashboard
client_id = "abc123" client_id = "abc123"
client_secret = "supersecret" client_secret = "supersecret"
# Spotify content country (Default: country from user account)
# market = "US"