Tiraya/Tiraya
1
0
Fork 0
Code Issues Pull requests Projects Releases Activity

Compare commits

base: Tiraya:86a4bd9762d955bbd3de965826a6ecb2af749c73
Branches Tags
Tiraya:main
..
compare: Tiraya:f9ade790274c5a2d0c4c6d3fb992bf6195adfc6f
Branches Tags
Tiraya:main

No commits in common. "86a4bd9762d955bbd3de965826a6ecb2af749c73" and "f9ade790274c5a2d0c4c6d3fb992bf6195adfc6f" have entirely different histories.
86a4bd9762 ... f9ade79027

20 changed files with 317 additions and 935 deletions
Download patch file Download diff file Expand all files Collapse all files

269
Cargo.lock generated
View file

@ -76,9 +76,9 @@ dependencies = [
[[package]] [[package]]
name = "anstream" name = "anstream"
version = "0.6.4" version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c"
dependencies = [ dependencies = [
"anstyle", "anstyle",
"anstyle-parse", "anstyle-parse",
@ -90,15 +90,15 @@ dependencies = [
[[package]] [[package]]
name = "anstyle" name = "anstyle"
version = "1.0.4" version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46"
[[package]] [[package]]
name = "anstyle-parse" name = "anstyle-parse"
version = "0.2.2" version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333"
dependencies = [ dependencies = [
"utf8parse", "utf8parse",
] ]
@ -114,9 +114,9 @@ dependencies = [
[[package]] [[package]]
name = "anstyle-wincon" name = "anstyle-wincon"
version = "3.0.1" version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd"
dependencies = [ dependencies = [
"anstyle", "anstyle",
"windows-sys 0.48.0", "windows-sys 0.48.0",
@ -161,7 +161,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.37",
] ]
[[package]] [[package]]
@ -172,7 +172,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.37",
] ]
[[package]] [[package]]
@ -264,9 +264,9 @@ dependencies = [
[[package]] [[package]]
name = "brotli" name = "brotli"
version = "3.4.0" version = "3.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68"
dependencies = [ dependencies = [
"alloc-no-stdlib", "alloc-no-stdlib",
"alloc-stdlib", "alloc-stdlib",
@ -275,9 +275,9 @@ dependencies = [
[[package]] [[package]]
name = "brotli-decompressor" name = "brotli-decompressor"
version = "2.5.0" version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da74e2b81409b1b743f8f0c62cc6254afefb8b8e50bbfe3735550f7aeefa3448" checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744"
dependencies = [ dependencies = [
"alloc-no-stdlib", "alloc-no-stdlib",
"alloc-stdlib", "alloc-stdlib",
@ -291,9 +291,9 @@ checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
[[package]] [[package]]
name = "byteorder" name = "byteorder"
version = "1.5.0" version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]] [[package]]
name = "bytes" name = "bytes"
@ -333,9 +333,9 @@ dependencies = [
[[package]] [[package]]
name = "clap" name = "clap"
version = "4.4.6" version = "4.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" checksum = "824956d0dca8334758a5b7f7e50518d66ea319330cbceedcf76905c2f6ab30e3"
dependencies = [ dependencies = [
"clap_builder", "clap_builder",
"clap_derive", "clap_derive",
@ -343,9 +343,9 @@ dependencies = [
[[package]] [[package]]
name = "clap_builder" name = "clap_builder"
version = "4.4.6" version = "4.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" checksum = "122ec64120a49b4563ccaedcbea7818d069ed8e9aa6d829b82d8a4128936b2ab"
dependencies = [ dependencies = [
"anstream", "anstream",
"anstyle", "anstyle",
@ -362,7 +362,7 @@ dependencies = [
"heck", "heck",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.37",
] ]
[[package]] [[package]]
@ -527,7 +527,7 @@ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"strsim", "strsim",
"syn 2.0.38", "syn 2.0.37",
] ]
[[package]] [[package]]
@ -549,7 +549,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
dependencies = [ dependencies = [
"darling_core 0.20.3", "darling_core 0.20.3",
"quote", "quote",
"syn 2.0.38", "syn 2.0.37",
] ]
[[package]] [[package]]
@ -629,7 +629,7 @@ dependencies = [
"once_cell", "once_cell",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.37",
] ]
[[package]] [[package]]
@ -640,9 +640,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]] [[package]]
name = "errno" name = "errno"
version = "0.3.4" version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480" checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd"
dependencies = [ dependencies = [
"errno-dragonfly", "errno-dragonfly",
"libc", "libc",
@ -688,9 +688,9 @@ dependencies = [
[[package]] [[package]]
name = "fastrand" name = "fastrand"
version = "2.0.1" version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764"
[[package]] [[package]]
name = "finl_unicode" name = "finl_unicode"
@ -710,12 +710,13 @@ dependencies = [
[[package]] [[package]]
name = "flume" name = "flume"
version = "0.11.0" version = "0.10.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-sink", "futures-sink",
"pin-project",
"spin 0.9.8", "spin 0.9.8",
] ]
@ -816,7 +817,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.37",
] ]
[[package]] [[package]]
@ -909,9 +910,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.14.1" version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
dependencies = [ dependencies = [
"ahash", "ahash",
"allocator-api2", "allocator-api2",
@ -923,7 +924,7 @@ version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7"
dependencies = [ dependencies = [
"hashbrown 0.14.1", "hashbrown 0.14.0",
] ]
[[package]] [[package]]
@ -1116,19 +1117,19 @@ dependencies = [
[[package]] [[package]]
name = "indexmap" name = "indexmap"
version = "2.0.2" version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
dependencies = [ dependencies = [
"equivalent", "equivalent",
"hashbrown 0.14.1", "hashbrown 0.14.0",
] ]
[[package]] [[package]]
name = "insta" name = "insta"
version = "1.33.0" version = "1.32.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aa511b2e298cd49b1856746f6bb73e17036bcd66b25f5e92cdcdbec9bd75686" checksum = "a3e02c584f4595792d09509a94cdb92a3cef7592b1eb2d9877ee6f527062d0ea"
dependencies = [ dependencies = [
"console", "console",
"lazy_static", "lazy_static",
@ -1233,9 +1234,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]] [[package]]
name = "linux-raw-sys" name = "linux-raw-sys"
version = "0.4.8" version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db" checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128"
[[package]] [[package]]
name = "lock_api" name = "lock_api"
@ -1266,19 +1267,18 @@ dependencies = [
[[package]] [[package]]
name = "md-5" name = "md-5"
version = "0.10.6" version = "0.10.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca"
dependencies = [ dependencies = [
"cfg-if",
"digest", "digest",
] ]
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.6.4" version = "2.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
[[package]] [[package]]
name = "mime" name = "mime"
@ -1455,7 +1455,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.37",
] ]
[[package]] [[package]]
@ -1560,9 +1560,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
[[package]] [[package]]
name = "pest" name = "pest"
version = "2.7.4" version = "2.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c022f1e7b65d6a24c0dbbd5fb344c66881bc01f3e5ae74a1c8100f2f985d98a4" checksum = "d7a4d085fd991ac8d5b05a147b437791b4260b76326baf0fc60cf7c9c27ecd33"
dependencies = [ dependencies = [
"memchr", "memchr",
"thiserror", "thiserror",
@ -1571,9 +1571,9 @@ dependencies = [
[[package]] [[package]]
name = "pest_derive" name = "pest_derive"
version = "2.7.4" version = "2.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35513f630d46400a977c4cb58f78e1bfbe01434316e60c37d27b9ad6139c66d8" checksum = "a2bee7be22ce7918f641a33f08e3f43388c7656772244e2bbb2477f44cc9021a"
dependencies = [ dependencies = [
"pest", "pest",
"pest_generator", "pest_generator",
@ -1581,22 +1581,22 @@ dependencies = [
[[package]] [[package]]
name = "pest_generator" name = "pest_generator"
version = "2.7.4" version = "2.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc9fc1b9e7057baba189b5c626e2d6f40681ae5b6eb064dc7c7834101ec8123a" checksum = "d1511785c5e98d79a05e8a6bc34b4ac2168a0e3e92161862030ad84daa223141"
dependencies = [ dependencies = [
"pest", "pest",
"pest_meta", "pest_meta",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.37",
] ]
[[package]] [[package]]
name = "pest_meta" name = "pest_meta"
version = "2.7.4" version = "2.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1df74e9e7ec4053ceb980e7c0c8bd3594e977fde1af91daba9c928e8e8c6708d" checksum = "b42f0394d3123e33353ca5e1e89092e533d2cc490389f2bd6131c43c634ebc5f"
dependencies = [ dependencies = [
"once_cell", "once_cell",
"pest", "pest",
@ -1633,7 +1633,7 @@ dependencies = [
"phf_shared", "phf_shared",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.37",
] ]
[[package]] [[package]]
@ -1645,6 +1645,26 @@ dependencies = [
"siphasher 0.3.11", "siphasher 0.3.11",
] ]
[[package]]
name = "pin-project"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.37",
]
[[package]] [[package]]
name = "pin-project-lite" name = "pin-project-lite"
version = "0.2.13" version = "0.2.13"
@ -1727,7 +1747,7 @@ checksum = "f69f8d22fa3f34f3083d9a4375c038732c7a7e964de1beb81c544da92dfc40b8"
dependencies = [ dependencies = [
"ahash", "ahash",
"equivalent", "equivalent",
"hashbrown 0.14.1", "hashbrown 0.14.0",
"parking_lot", "parking_lot",
] ]
@ -1787,9 +1807,9 @@ dependencies = [
[[package]] [[package]]
name = "regex" name = "regex"
version = "1.9.6" version = "1.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47"
dependencies = [ dependencies = [
"aho-corasick", "aho-corasick",
"memchr", "memchr",
@ -1799,9 +1819,9 @@ dependencies = [
[[package]] [[package]]
name = "regex-automata" name = "regex-automata"
version = "0.3.9" version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"
dependencies = [ dependencies = [
"aho-corasick", "aho-corasick",
"memchr", "memchr",
@ -1822,9 +1842,9 @@ checksum = "c707298afce11da2efef2f600116fa93ffa7a032b5d7b628aa17711ec81383ca"
[[package]] [[package]]
name = "reqwest" name = "reqwest"
version = "0.11.22" version = "0.11.20"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1"
dependencies = [ dependencies = [
"async-compression", "async-compression",
"base64 0.21.4", "base64 0.21.4",
@ -1851,7 +1871,6 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"serde_urlencoded", "serde_urlencoded",
"system-configuration",
"tokio", "tokio",
"tokio-native-tls", "tokio-native-tls",
"tokio-rustls", "tokio-rustls",
@ -2005,7 +2024,7 @@ dependencies = [
"regex", "regex",
"relative-path", "relative-path",
"rustc_version", "rustc_version",
"syn 2.0.38", "syn 2.0.37",
"unicode-ident", "unicode-ident",
] ]
@ -2026,9 +2045,9 @@ dependencies = [
[[package]] [[package]]
name = "rustix" name = "rustix"
version = "0.38.17" version = "0.38.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f25469e9ae0f3d0047ca8b93fc56843f38e6774f0914a107ff8b41be8be8e0b7" checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f"
dependencies = [ dependencies = [
"bitflags 2.4.0", "bitflags 2.4.0",
"errno", "errno",
@ -2077,7 +2096,7 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
[[package]] [[package]]
name = "rustypipe" name = "rustypipe"
version = "0.1.0" version = "0.1.0"
source = "git+https://code.thetadev.de/ThetaDev/rustypipe.git#e247b0c5d9874313d8a350142694bc90db805a11" source = "git+https://code.thetadev.de/ThetaDev/rustypipe.git#d6de428549bfe7881e5ac52857d4bc1fa2db7f5d"
dependencies = [ dependencies = [
"base64 0.21.4", "base64 0.21.4",
"fancy-regex", "fancy-regex",
@ -2167,9 +2186,9 @@ dependencies = [
[[package]] [[package]]
name = "semver" name = "semver"
version = "1.0.19" version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918"
[[package]] [[package]]
name = "serde" name = "serde"
@ -2188,7 +2207,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.37",
] ]
[[package]] [[package]]
@ -2256,7 +2275,7 @@ dependencies = [
"darling 0.20.3", "darling 0.20.3",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.37",
] ]
[[package]] [[package]]
@ -2272,9 +2291,9 @@ dependencies = [
[[package]] [[package]]
name = "sha2" name = "sha2"
version = "0.10.8" version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"cpufeatures", "cpufeatures",
@ -2283,9 +2302,9 @@ dependencies = [
[[package]] [[package]]
name = "sharded-slab" name = "sharded-slab"
version = "0.1.7" version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31"
dependencies = [ dependencies = [
"lazy_static", "lazy_static",
] ]
@ -2341,7 +2360,7 @@ checksum = "0eb01866308440fc64d6c44d9e86c5cc17adfe33c4d6eed55da9145044d0ffc1"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.37",
] ]
[[package]] [[package]]
@ -2402,9 +2421,9 @@ dependencies = [
[[package]] [[package]]
name = "sqlx" name = "sqlx"
version = "0.7.2" version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e50c216e3624ec8e7ecd14c6a6a6370aad6ee5d8cfc3ab30b5162eeeef2ed33" checksum = "8e58421b6bc416714d5115a2ca953718f6c621a51b68e4f4922aea5a4391a721"
dependencies = [ dependencies = [
"sqlx-core", "sqlx-core",
"sqlx-macros", "sqlx-macros",
@ -2415,9 +2434,9 @@ dependencies = [
[[package]] [[package]]
name = "sqlx-core" name = "sqlx-core"
version = "0.7.2" version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d6753e460c998bbd4cd8c6f0ed9a64346fcca0723d6e75e52fdc351c5d2169d" checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53"
dependencies = [ dependencies = [
"ahash", "ahash",
"atoi", "atoi",
@ -2435,7 +2454,7 @@ dependencies = [
"futures-util", "futures-util",
"hashlink", "hashlink",
"hex", "hex",
"indexmap 2.0.2", "indexmap 2.0.0",
"log", "log",
"memchr", "memchr",
"once_cell", "once_cell",
@ -2483,9 +2502,9 @@ dependencies = [
[[package]] [[package]]
name = "sqlx-macros" name = "sqlx-macros"
version = "0.7.2" version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a793bb3ba331ec8359c1853bd39eed32cdd7baaf22c35ccf5c92a7e8d1189ec" checksum = "208e3165167afd7f3881b16c1ef3f2af69fa75980897aac8874a0696516d12c2"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -2496,9 +2515,9 @@ dependencies = [
[[package]] [[package]]
name = "sqlx-macros-core" name = "sqlx-macros-core"
version = "0.7.2" version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a4ee1e104e00dedb6aa5ffdd1343107b0a4702e862a84320ee7cc74782d96fc" checksum = "8a4a8336d278c62231d87f24e8a7a74898156e34c1c18942857be2acb29c7dfc"
dependencies = [ dependencies = [
"dotenvy", "dotenvy",
"either", "either",
@ -2522,9 +2541,9 @@ dependencies = [
[[package]] [[package]]
name = "sqlx-mysql" name = "sqlx-mysql"
version = "0.7.2" version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "864b869fdf56263f4c95c45483191ea0af340f9f3e3e7b4d57a61c7c87a970db" checksum = "8ca69bf415b93b60b80dc8fda3cb4ef52b2336614d8da2de5456cc942a110482"
dependencies = [ dependencies = [
"atoi", "atoi",
"base64 0.21.4", "base64 0.21.4",
@ -2566,9 +2585,9 @@ dependencies = [
[[package]] [[package]]
name = "sqlx-postgres" name = "sqlx-postgres"
version = "0.7.2" version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb7ae0e6a97fb3ba33b23ac2671a5ce6e3cabe003f451abd5a56e7951d975624" checksum = "a0db2df1b8731c3651e204629dd55e52adbae0462fa1bdcbed56a2302c18181e"
dependencies = [ dependencies = [
"atoi", "atoi",
"base64 0.21.4", "base64 0.21.4",
@ -2607,9 +2626,9 @@ dependencies = [
[[package]] [[package]]
name = "sqlx-sqlite" name = "sqlx-sqlite"
version = "0.7.2" version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d59dc83cf45d89c555a577694534fcd1b55c545a816c816ce51f20bbe56a4f3f" checksum = "be4c21bf34c7cae5b283efb3ac1bcc7670df7561124dc2f8bdc0b59be40f79a2"
dependencies = [ dependencies = [
"atoi", "atoi",
"flume", "flume",
@ -2665,7 +2684,7 @@ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"rustversion", "rustversion",
"syn 2.0.38", "syn 2.0.37",
] ]
[[package]] [[package]]
@ -2687,36 +2706,15 @@ dependencies = [
[[package]] [[package]]
name = "syn" name = "syn"
version = "2.0.38" version = "2.0.37"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "system-configuration"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
dependencies = [
"bitflags 1.3.2",
"core-foundation",
"system-configuration-sys",
]
[[package]]
name = "system-configuration-sys"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]] [[package]]
name = "tempfile" name = "tempfile"
version = "3.8.0" version = "3.8.0"
@ -2745,22 +2743,22 @@ dependencies = [
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.49" version = "1.0.48"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7"
dependencies = [ dependencies = [
"thiserror-impl", "thiserror-impl",
] ]
[[package]] [[package]]
name = "thiserror-impl" name = "thiserror-impl"
version = "1.0.49" version = "1.0.48"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.37",
] ]
[[package]] [[package]]
@ -2775,9 +2773,9 @@ dependencies = [
[[package]] [[package]]
name = "time" name = "time"
version = "0.3.29" version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48"
dependencies = [ dependencies = [
"deranged", "deranged",
"itoa", "itoa",
@ -2788,15 +2786,15 @@ dependencies = [
[[package]] [[package]]
name = "time-core" name = "time-core"
version = "0.1.2" version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
[[package]] [[package]]
name = "time-macros" name = "time-macros"
version = "0.2.15" version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572"
dependencies = [ dependencies = [
"time-core", "time-core",
] ]
@ -2862,7 +2860,6 @@ dependencies = [
"rspotify", "rspotify",
"rstest", "rstest",
"rustypipe", "rustypipe",
"serde_plain",
"siphasher 1.0.0", "siphasher 1.0.0",
"sqlx", "sqlx",
"sqlx-database-tester", "sqlx-database-tester",
@ -2915,7 +2912,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.37",
] ]
[[package]] [[package]]
@ -3002,7 +2999,7 @@ version = "0.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338" checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338"
dependencies = [ dependencies = [
"indexmap 2.0.2", "indexmap 2.0.0",
"serde", "serde",
"serde_spanned", "serde_spanned",
"toml_datetime", "toml_datetime",
@ -3036,7 +3033,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.37",
] ]
[[package]] [[package]]
@ -3234,7 +3231,7 @@ dependencies = [
"once_cell", "once_cell",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.37",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -3268,7 +3265,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.37",
"wasm-bindgen-backend", "wasm-bindgen-backend",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -3484,9 +3481,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]] [[package]]
name = "winnow" name = "winnow"
version = "0.5.16" version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "037711d82167854aff2018dfd193aa0fef5370f456732f0d5a0c59b0f1b4b907" checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc"
dependencies = [ dependencies = [
"memchr", "memchr",
] ]

4
crates/db/src/models/album.rs
View file

@ -146,10 +146,6 @@ impl Album {
Id::Db(self.id) Id::Db(self.id)
} }
pub fn src_id(&self) -> SrcId {
SrcId(&self.src_id, self.service)
}
pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError> pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError>
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,

4
crates/db/src/models/artist.rs
View file

@ -99,10 +99,6 @@ impl Artist {
Id::Db(self.id) Id::Db(self.id)
} }
pub fn src_id(&self) -> SrcId {
SrcId(&self.src_id, self.service)
}
pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError> pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError>
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy, E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy,

8
crates/db/src/models/playlist.rs
View file

@ -102,10 +102,6 @@ impl Playlist {
Id::Db(self.id) Id::Db(self.id)
} }
pub fn src_id(&self) -> SrcId {
SrcId(&self.src_id, self.service)
}
pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError> pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError>
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,
@ -635,10 +631,6 @@ impl PlaylistUpdate<'_> {
} }
impl PlaylistSlim { impl PlaylistSlim {
pub fn src_id(&self) -> SrcId {
SrcId(&self.src_id, self.service)
}
pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError> pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError>
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,

8
crates/db/src/models/track.rs
View file

@ -138,10 +138,6 @@ impl Track {
Id::Db(self.id) Id::Db(self.id)
} }
pub fn src_id(&self) -> SrcId {
SrcId(&self.src_id, self.service)
}
pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError> pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError>
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy, E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy,
@ -515,10 +511,6 @@ impl TrackUpdate<'_> {
} }
impl TrackSlim { impl TrackSlim {
pub fn src_id(&self) -> SrcId {
SrcId(&self.src_id, self.service)
}
pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError> pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError>
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy, E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy,

10
crates/db/src/models/user.rs
View file

@ -55,10 +55,6 @@ impl User {
Id::Db(self.id) Id::Db(self.id)
} }
pub fn src_id(&self) -> SrcId {
SrcId(&self.src_id, self.service)
}
pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError> pub async fn get<'a, E>(id: Id<'_>, exec: E) -> Result<Self, DatabaseError>
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,
@ -320,12 +316,6 @@ impl UserUpdate<'_> {
} }
} }
impl UserItem {
pub fn src_id(&self) -> SrcId {
SrcId(&self.src_id, self.service)
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;

1
crates/extractor/Cargo.toml
View file

@ -21,7 +21,6 @@ quick_cache.workspace = true
regex.workspace = true regex.workspace = true
rspotify.workspace = true rspotify.workspace = true
rustypipe.workspace = true rustypipe.workspace = true
serde_plain.workspace = true
siphasher.workspace = true siphasher.workspace = true
sqlx.workspace = true sqlx.workspace = true
strsim.workspace = true strsim.workspace = true

2
crates/extractor/src/error.rs
View file

@ -17,8 +17,6 @@ pub enum ExtractorError {
}, },
#[error("could not find a match for track [{id}]")] #[error("could not find a match for track [{id}]")]
NoMatch { id: SrcIdOwned }, NoMatch { id: SrcIdOwned },
#[error("no id found")]
NoId,
#[error("{0}")] #[error("{0}")]
Other(Cow<'static, str>), Other(Cow<'static, str>),
} }

37
crates/extractor/src/lib.rs
View file

@ -1,4 +1,5 @@
#![warn(clippy::dbg_macro, clippy::todo)] #![warn(clippy::dbg_macro, clippy::todo)]
#![allow(dead_code)]
pub mod error; pub mod error;
mod model; mod model;
@ -7,7 +8,6 @@ mod util;
pub use model::GetStatus; pub use model::GetStatus;
use services::SpotifyExtractor; use services::SpotifyExtractor;
use tiraya_utils::config::CONFIG;
use std::{collections::HashSet, sync::Arc}; use std::{collections::HashSet, sync::Arc};
@ -28,6 +28,13 @@ use crate::{
services::YouTubeExtractor, services::YouTubeExtractor,
}; };
const ARTIST_STALE: Duration = Duration::hours(24);
const PLAYLIST_STALE: Duration = Duration::hours(24);
const USER_STALE: Duration = Duration::hours(24);
pub(crate) const CONCURRENCY: usize = 4;
pub(crate) const DB_CONCURRENCY: usize = 8;
pub(crate) const ITEM_LIMIT: usize = 2000;
pub struct Extractor(Arc<ExtractorInner>); pub struct Extractor(Arc<ExtractorInner>);
pub struct ExtractorInner { pub struct ExtractorInner {
@ -50,11 +57,7 @@ impl Extractor {
let yt = YouTubeExtractor::new(core.clone())?; let yt = YouTubeExtractor::new(core.clone())?;
Ok(Self( Ok(Self(
ExtractorInner { ExtractorInner {
sp: if CONFIG.extractor.spotify.enable { sp: SpotifyExtractor::new(core.clone(), yt.clone())?,
Some(SpotifyExtractor::new(core.clone(), yt.clone())?)
} else {
None
},
yt, yt,
core, core,
} }
@ -68,12 +71,6 @@ impl Extractor {
.await .await
.to_optional() .to_optional()
.unwrap(); .unwrap();
// Get srcid from fetched artist if available (could be an alias)
let srcid = artist
.as_ref()
.map(|a| a.src_id().to_owned())
.unwrap_or_else(|| id.to_owned());
let last_update = if let Some(artist) = artist { let last_update = if let Some(artist) = artist {
self.core.artist_cache.insert(id.to_owned(), artist.id); self.core.artist_cache.insert(id.to_owned(), artist.id);
@ -83,7 +80,7 @@ impl Extractor {
let age = OffsetDateTime::now_utc() - last_sync_at.assume_utc(); let age = OffsetDateTime::now_utc() - last_sync_at.assume_utc();
// Check if artist in DB is still fresh // Check if artist in DB is still fresh
if age < Duration::hours(CONFIG.extractor.artist_stale_h.into()) { if age < ARTIST_STALE {
return Ok(GetResult::stored(artist)); return Ok(GetResult::stored(artist));
} else { } else {
match last_sync_data { match last_sync_data {
@ -115,8 +112,7 @@ impl Extractor {
None None
}; };
self.update_artist(srcid.as_srcid(), last_update.as_ref()) self.update_artist(id, last_update.as_ref()).await
.await
} }
async fn update_artist( async fn update_artist(
@ -126,7 +122,6 @@ impl Extractor {
) -> Result<GetResult<Artist>, ExtractorError> { ) -> Result<GetResult<Artist>, ExtractorError> {
let res = match id.1 { let res = match id.1 {
MusicService::YouTube => self.yt.update_artist(id.0, last_update).await, MusicService::YouTube => self.yt.update_artist(id.0, last_update).await,
MusicService::Spotify => self.sp()?.update_artist(id.0).await,
_ => { _ => {
return Err(ExtractorError::Unsupported { return Err(ExtractorError::Unsupported {
typ: EntityType::Artist, typ: EntityType::Artist,
@ -177,7 +172,7 @@ impl Extractor {
} }
} }
}) })
.buffer_unordered(CONFIG.core.db_concurrency) .buffer_unordered(DB_CONCURRENCY)
.collect::<Vec<_>>() .collect::<Vec<_>>()
.await; .await;
@ -216,7 +211,6 @@ impl Extractor {
let res = match id.1 { let res = match id.1 {
MusicService::YouTube => self.yt.fetch_album(id.0).await?, MusicService::YouTube => self.yt.fetch_album(id.0).await?,
MusicService::Spotify => self.sp()?.fetch_album(id.0).await?,
_ => { _ => {
return Err(ExtractorError::Unsupported { return Err(ExtractorError::Unsupported {
typ: EntityType::Album, typ: EntityType::Album,
@ -271,7 +265,7 @@ impl Extractor {
let age = OffsetDateTime::now_utc() - last_sync_at.assume_utc(); let age = OffsetDateTime::now_utc() - last_sync_at.assume_utc();
// Check if playlist< in DB is still fresh // Check if playlist< in DB is still fresh
if age < Duration::hours(CONFIG.extractor.playlist_stale_h.into()) { if age < PLAYLIST_STALE {
return Ok(GetResult::stored(playlist)); return Ok(GetResult::stored(playlist));
} else { } else {
match last_sync_data { match last_sync_data {
@ -370,8 +364,8 @@ impl Extractor {
{ {
let age = OffsetDateTime::now_utc() - last_sync_at.assume_utc(); let age = OffsetDateTime::now_utc() - last_sync_at.assume_utc();
// Check if playlist in DB is still fresh // Check if playlist< in DB is still fresh
if age < Duration::hours(CONFIG.extractor.user_stale_h.into()) { if age < USER_STALE {
return Ok(GetResult::stored(user)); return Ok(GetResult::stored(user));
} else { } else {
match last_sync_data { match last_sync_data {
@ -416,7 +410,6 @@ impl Extractor {
) -> Result<GetResult<User>, ExtractorError> { ) -> Result<GetResult<User>, ExtractorError> {
let res = match id.1 { let res = match id.1 {
MusicService::YouTube => self.yt.update_user(id.0, last_update).await, MusicService::YouTube => self.yt.update_user(id.0, last_update).await,
MusicService::Spotify => self.sp()?.update_user(id.0, last_update).await,
_ => { _ => {
return Err(ExtractorError::Unsupported { return Err(ExtractorError::Unsupported {
typ: EntityType::User, typ: EntityType::User,

6
crates/extractor/src/services/mod.rs
View file

@ -8,12 +8,6 @@ use crate::{error::ExtractorError, model::ExtractorCore};
/* /*
/// Tiraya extractor implementation for a specific music streaming service /// Tiraya extractor implementation for a specific music streaming service
///
/// Terminology for the individual methods:
/// - **update** Check if an item was updated on the source, download and import the
/// changes if necessary
/// - **fetch** Download and import the item from source if it is not present in the database
/// - **import** Take a data object from a service and store it in the database
pub(crate) trait ServiceExtractor { pub(crate) trait ServiceExtractor {
/// Create a new instance of the service extractor /// Create a new instance of the service extractor
fn new(core: Arc<ExtractorCore>) -> Result<Self, ExtractorError>; fn new(core: Arc<ExtractorCore>) -> Result<Self, ExtractorError>;

342
crates/extractor/src/services/spotify.rs
View file

@ -1,25 +1,21 @@
use futures::{stream::TryStreamExt, StreamExt}; use futures::stream::{StreamExt, TryStreamExt};
use path_macro::path; use path_macro::path;
use rspotify::{ use rspotify::{
model::{ model::{FullTrack, PlayableItem, PlaylistId, PublicUser, TrackId},
AlbumId, ArtistId, FullTrack, Market, PlayableItem, PlaylistId, PublicUser,
SimplifiedPlaylist, TrackId, UserId,
},
prelude::{BaseClient, Id}, prelude::{BaseClient, Id},
ClientCredsSpotify, ClientCredsSpotify,
}; };
use std::{collections::HashSet, sync::Arc}; use std::{collections::HashSet, sync::Arc};
use time::OffsetDateTime; use time::OffsetDateTime;
use tiraya_db::models::{ use tiraya_db::models::{
Artist, MusicService, Playlist, PlaylistEntry, PlaylistImgType, PlaylistNew, PlaylistType, MusicService, Playlist, PlaylistEntry, PlaylistImgType, PlaylistNew, PlaylistType, SrcId,
SrcId, SrcIdOwned, Track, User, UserNew, UserType, SrcIdOwned,
}; };
use tiraya_utils::config::CONFIG; use tiraya_utils::config::CONFIG;
use crate::{ use crate::{
error::ExtractorError, error::ExtractorError,
model::{ExtractorCore, GetResult, LastUpdateState, SyncLastUpdate}, model::{ExtractorCore, GetResult, LastUpdateState, SyncLastUpdate},
util::ArtistIdName,
}; };
use super::YouTubeExtractor; use super::YouTubeExtractor;
@ -29,41 +25,32 @@ pub struct SpotifyExtractor {
core: Arc<ExtractorCore>, core: Arc<ExtractorCore>,
sp: ClientCredsSpotify, sp: ClientCredsSpotify,
yt: YouTubeExtractor, yt: YouTubeExtractor,
market: Market,
} }
const PAGE_LIMIT: u32 = 50;
impl SpotifyExtractor { impl SpotifyExtractor {
pub fn new(core: Arc<ExtractorCore>, yt: YouTubeExtractor) -> Result<Self, ExtractorError> { pub fn new(
let sp = ClientCredsSpotify::with_config( core: Arc<ExtractorCore>,
rspotify::Credentials { yt: YouTubeExtractor,
id: CONFIG.extractor.spotify.client_id.to_owned(), ) -> Result<Option<Self>, ExtractorError> {
secret: Some(CONFIG.extractor.spotify.client_secret.to_owned()), if let Some(cfg) = &CONFIG.extractor.spotify {
}, let sp = ClientCredsSpotify::with_config(
rspotify::Config { rspotify::Credentials {
cache_path: path!(CONFIG.extractor.data_dir / "spotify_token_cache.json"), id: cfg.client_id.to_owned(),
token_cached: true, secret: Some(cfg.client_secret.to_owned()),
pagination_chunks: PAGE_LIMIT, },
..Default::default() rspotify::Config {
}, cache_path: path!(CONFIG.extractor.data_dir / "spotify_token_cache.json"),
); token_cached: true,
..Default::default()
},
);
let market_str = &CONFIG.extractor.spotify.market; Ok(Some(Self { core, sp, yt }))
let market = Market::Country(serde_plain::from_str(market_str).map_err(|_| { } else {
ExtractorError::Other(format!("invalid spotify market `{market_str}`").into()) Ok(None)
})?); }
Ok(Self {
core,
sp,
yt,
market,
})
} }
/// Since Rspotify does not initialize the token by itself it needs to be loaded
/// before making a request
async fn load_token(&self) -> Result<(), ExtractorError> { async fn load_token(&self) -> Result<(), ExtractorError> {
let mut token = self.sp.token.lock().await.unwrap(); let mut token = self.sp.token.lock().await.unwrap();
if token.is_none() { if token.is_none() {
@ -80,93 +67,14 @@ impl SpotifyExtractor {
Ok(()) Ok(())
} }
pub async fn update_artist(&self, src_id: &str) -> Result<GetResult<i32>, ExtractorError> {
// Info: since we are matching Spotify artists with YTM artists, this function
// will only be called on the initial request of the artist, not on updates
// (hence no last_update parameter).
self.load_token().await?;
let artist_id = ArtistId::from_id(src_id)?;
let mut top_tracks = self
.sp
.artist_top_tracks(artist_id, Some(self.market))
.await?;
let mut get_track = || -> Result<FullTrack, ExtractorError> {
let i = top_tracks
.iter()
.enumerate()
.find(|(_, t)| {
t.artists.len() == 1
&& t.artists[0]
.id
.as_ref()
.map(|id| id.id() == src_id)
.unwrap_or_default()
})
.or_else(|| {
top_tracks.iter().enumerate().find(|(_, t)| {
t.artists.iter().any(|a| {
a.id.as_ref()
.map(|id| id.id() == src_id)
.unwrap_or_default()
})
})
})
.ok_or_else(|| ExtractorError::InvalidData {
id: SrcIdOwned(src_id.to_owned(), MusicService::Spotify),
msg: "no top tracks found".into(),
})?
.0;
Ok(top_tracks.remove(i))
};
let a_id = SrcId(src_id, MusicService::Spotify);
for _ in 0..3 {
let track = get_track()?;
match self.import_track(track).await {
Ok(_) => {
if let Some(a_id) = Artist::get_id(a_id, &self.core.db).await? {
let src_id = Artist::get_src_id(a_id, &self.core.db)
.await?
.ok_or(ExtractorError::NoId)?;
if src_id.1 != MusicService::YouTube {
return Err(ExtractorError::Other("artist id not from yt".into()));
}
return self.yt.update_artist(&src_id.0, None).await;
}
}
Err(ExtractorError::NoMatch { .. }) => {}
Err(e) => return Err(e),
}
}
Err(ExtractorError::NoMatch {
id: a_id.to_owned(),
})
}
pub async fn fetch_album(&self, src_id: &str) -> Result<(i32, bool), ExtractorError> {
self.load_token().await?;
let album = self.sp.album(AlbumId::from_id(src_id)?, None).await?;
let artist = album
.artists
.into_iter()
.next()
.map(|artist| artist.name)
.unwrap_or_default();
let album_id = self
.yt
.match_album(SrcId(src_id, MusicService::Spotify), &album.name, &artist)
.await?;
Ok((album_id, false))
}
pub async fn fetch_track(&self, src_id: &str) -> Result<i32, ExtractorError> { pub async fn fetch_track(&self, src_id: &str) -> Result<i32, ExtractorError> {
self.load_token().await?; self.load_token().await?;
let track = self.sp.track(TrackId::from_id(src_id)?, None).await?; let track = self.sp.track(TrackId::from_id(src_id)?, None).await?;
self.import_track(track).await self.import_track(track)
.await?
.ok_or_else(|| ExtractorError::NoMatch {
id: SrcIdOwned(src_id.to_owned(), MusicService::Spotify),
})
} }
pub async fn update_playlist( pub async fn update_playlist(
@ -193,7 +101,7 @@ impl SpotifyExtractor {
.into_iter() .into_iter()
.max_by_key(|img| img.height.unwrap_or_default()) .max_by_key(|img| img.height.unwrap_or_default())
.map(|img| img.url); .map(|img| img.url);
let owner_id = self.import_user(playlist.owner).await?; let owner_id = self.import_user(&playlist.owner).await?;
let n_playlist = PlaylistNew { let n_playlist = PlaylistNew {
src_id, src_id,
@ -222,7 +130,6 @@ impl SpotifyExtractor {
None None
}) })
}) })
.take(CONFIG.extractor.playlist_item_limit_matchmaker)
.try_collect::<Vec<_>>() .try_collect::<Vec<_>>()
.await?; .await?;
@ -262,167 +169,32 @@ impl SpotifyExtractor {
Ok(GetResult::fetched(playlist_id)) Ok(GetResult::fetched(playlist_id))
} }
pub async fn update_user( async fn import_user(&self, user: &PublicUser) -> Result<i32, ExtractorError> {
&self,
src_id: &str,
last_update: Option<&SyncLastUpdate>,
) -> Result<GetResult<i32>, ExtractorError> {
self.load_token().await?;
let sp_user_id = UserId::from_id(src_id)?;
// First page of playlists
let playlists_p1 = self
.sp
.user_playlists_manual(sp_user_id.clone_static(), Some(PAGE_LIMIT), Some(0))
.await?;
let current_state = playlists_p1
.items
.first()
.map(|pl| LastUpdateState::Version(pl.id.id().to_owned()))
.unwrap_or_default();
if let Some(last_update) = last_update {
if last_update.state.is_current(&current_state) {
User::set_last_sync(last_update.id, current_state.into(), &self.core.db).await?;
return Ok(GetResult::stored(last_update.id));
}
}
let user = self.sp.user(sp_user_id.clone_static()).await?;
let user_id = self.import_user(user).await?;
self.import_playlists(playlists_p1.items).await?;
if playlists_p1.next.is_some() {
let mut offset = PAGE_LIMIT;
while offset < CONFIG.extractor.user_playlist_limit as u32 {
let playlists = self
.sp
.user_playlists_manual(
sp_user_id.clone_static(),
Some(PAGE_LIMIT),
Some(offset),
)
.await?;
self.import_playlists(playlists.items).await?;
if playlists.next.is_none() {
break;
}
offset += PAGE_LIMIT;
}
}
User::set_last_sync(user_id, current_state.into(), &self.core.db).await?;
Ok(GetResult::fetched(user_id))
}
async fn import_user(&self, user: PublicUser) -> Result<i32, ExtractorError> {
let id_owned = SrcIdOwned(user.id.id().to_owned(), MusicService::Spotify);
let image_url = user
.images
.into_iter()
.max_by_key(|img| img.height.unwrap_or_default())
.map(|img| img.url);
let name = user.display_name.as_deref().unwrap_or(user.id.id());
self.core self.core
.user_cache .import_user_id(
.get_or_insert_async(&id_owned, async { SrcId(user.id.id(), MusicService::Spotify),
let user = UserNew { user.display_name.as_deref().unwrap_or(user.id.id()),
src_id: &id_owned.0, false,
service: id_owned.1, )
name,
user_type: UserType::Remote,
image_url: image_url.as_deref(),
..Default::default()
};
let user_id = user.upsert(&self.core.db).await?;
tracing::debug!("imported user id [{}] {}", id_owned, name);
Ok(user_id)
})
.await .await
} }
async fn import_playlist(&self, playlist: SimplifiedPlaylist) -> Result<i32, ExtractorError> { async fn import_track(&self, track: FullTrack) -> Result<Option<i32>, ExtractorError> {
if let Some(id) = Playlist::get_id(
SrcId(playlist.id.id(), MusicService::Spotify),
&self.core.db,
)
.await?
{
return Ok(id);
}
let image_url = playlist
.images
.into_iter()
.max_by_key(|img| img.height.unwrap_or_default())
.map(|img| img.url);
let owner_id = self.import_user(playlist.owner).await?;
let playlist_n = PlaylistNew {
src_id: playlist.id.id(),
service: MusicService::Spotify,
name: &playlist.name,
owner_id: Some(owner_id),
playlist_type: PlaylistType::Remote,
image_url: image_url.as_deref(),
image_type: Some(PlaylistImgType::Custom).filter(|_| image_url.is_some()),
..Default::default()
};
let playlist_id = playlist_n.upsert(&self.core.db).await?;
tracing::debug!(
"imported playlist item [sp:{}] {}",
playlist.id.id(),
playlist.name
);
Ok(playlist_id)
}
async fn import_playlists(
&self,
playlists: impl IntoIterator<Item = SimplifiedPlaylist>,
) -> Result<(), ExtractorError> {
futures::stream::iter(playlists.into_iter().map(Ok))
.try_for_each_concurrent(CONFIG.core.db_concurrency, |pl| async {
self.import_playlist(pl).await?;
Ok(())
})
.await
}
async fn import_track(&self, track: FullTrack) -> Result<i32, ExtractorError> {
let src_id = SrcId(
track
.id
.as_ref()
.ok_or(ExtractorError::Other("no Spotify track id".into()))?
.id(),
MusicService::Spotify,
);
if let Some(track_id) = Track::get_id(src_id, &self.core.db).await? {
return Ok(track_id);
}
let artists = track
.artists
.into_iter()
.filter_map(|a| {
a.id.as_ref().map(|a_id| ArtistIdName {
id: a_id.id().to_owned(),
name: a.name,
})
})
.collect::<Vec<_>>();
self.yt self.yt
.match_track( .match_track(
src_id, SrcId(
track
.id
.ok_or(ExtractorError::Other("no Spotify track id".into()))?
.id(),
MusicService::Spotify,
),
&track.name, &track.name,
&artists, track
.artists
.first()
.map(|a| a.name.as_str())
.unwrap_or_default(),
track.external_ids.get("isrc").map(|s| s.as_str()), track.external_ids.get("isrc").map(|s| s.as_str()),
) )
.await .await
@ -431,17 +203,11 @@ impl SpotifyExtractor {
async fn import_tracks( async fn import_tracks(
&self, &self,
tracks: impl IntoIterator<Item = FullTrack>, tracks: impl IntoIterator<Item = FullTrack>,
) -> Result<(), ExtractorError> { ) -> Result<Vec<Option<i32>>, ExtractorError> {
futures::stream::iter(tracks.into_iter().map(Ok)) futures::stream::iter(tracks)
.try_for_each_concurrent(CONFIG.extractor.youtube.concurrency, |track| async move { .map(|track| async { self.import_track(track).await })
match self.import_track(track).await { .buffered(CONFIG.extractor.youtube.concurrency)
Ok(_) => Ok(()), .try_collect()
Err(e) => match e {
ExtractorError::NoMatch { .. } => Ok(()),
_ => Err(e),
},
}
})
.await .await
} }
} }

390
crates/extractor/src/services/youtube.rs
View file

@ -3,29 +3,29 @@ use std::str::FromStr;
use std::sync::Arc; use std::sync::Arc;
use futures::{StreamExt, TryStreamExt}; use futures::{StreamExt, TryStreamExt};
use path_macro::path;
use rustypipe::model as rpmodel; use rustypipe::model as rpmodel;
use rustypipe::{ use rustypipe::param::search_filter::SearchFilter;
client::RustyPipe, model::richtext::ToPlaintext, param::search_filter::SearchFilter, use rustypipe::report::FileReporter;
report::FileReporter, use rustypipe::{client::RustyPipe, model::richtext::ToPlaintext};
};
use time::{Date, OffsetDateTime, PrimitiveDateTime, Time}; use time::{Date, OffsetDateTime, PrimitiveDateTime, Time};
use tiraya_db::models::{PlaylistEntry, User};
use tiraya_db::{ use tiraya_db::{
error::OptionalRes, error::OptionalRes,
models::{ models::{
Album, AlbumNew, AlbumType, AlbumUpdate, Artist, ArtistId, ArtistNew, DatePrecision, Id, Album, AlbumNew, AlbumType, AlbumUpdate, Artist, ArtistNew, DatePrecision, Id,
MusicService, Playlist, PlaylistEntry, PlaylistImgType, PlaylistNew, PlaylistType, SrcId, MusicService, Playlist, PlaylistImgType, PlaylistNew, PlaylistType, SrcId, SrcIdOwned,
SrcIdOwned, Track, TrackNew, TrackUpdate, User, UserNew, UserType, Track, TrackNew, TrackUpdate, UserNew, UserType,
}, },
}; };
use tiraya_utils::config::CONFIG; use tiraya_utils::config::CONFIG;
use crate::util::ArtistSrcidName; use crate::util::ImgFormat;
use crate::ITEM_LIMIT;
use crate::{ use crate::{
error::ExtractorError, error::ExtractorError,
model::{ExtractorCore, GetResult, LastUpdateState, SyncLastUpdate}, model::{ExtractorCore, GetResult, LastUpdateState, SyncLastUpdate},
util::{self, ArtistIdName, ImgFormat}, util::{self, ArtistIdName},
GetStatus, CONCURRENCY, DB_CONCURRENCY,
}; };
const YTM_CHANNEL_SUFFIX: &str = " - Topic"; const YTM_CHANNEL_SUFFIX: &str = " - Topic";
@ -50,9 +50,7 @@ impl YouTubeExtractor {
&CONFIG.extractor.youtube.country, &CONFIG.extractor.youtube.country,
)?) )?)
.storage_dir(&CONFIG.extractor.data_dir) .storage_dir(&CONFIG.extractor.data_dir)
.reporter(Box::new(FileReporter::new(path!( .reporter(Box::new(FileReporter::new(&CONFIG.extractor.data_dir)))
CONFIG.extractor.data_dir / "rustypipe_reports"
))))
.n_http_retries(CONFIG.extractor.youtube.n_retries) .n_http_retries(CONFIG.extractor.youtube.n_retries)
.build()?, .build()?,
}) })
@ -127,8 +125,8 @@ impl YouTubeExtractor {
// Insert all albums // Insert all albums
futures::stream::iter(artist.albums) futures::stream::iter(artist.albums)
.map(Ok) .map(Ok)
.try_for_each_concurrent(CONFIG.core.db_concurrency, |album| async { .try_for_each_concurrent(DB_CONCURRENCY, |album| async {
self.import_album_item(album).await.map(|_| ()) self.import_album_item(album).await
}) })
.await?; .await?;
@ -202,8 +200,8 @@ impl YouTubeExtractor {
// Insert album variants# // Insert album variants#
let has_variants = !album.variants.is_empty(); let has_variants = !album.variants.is_empty();
futures::stream::iter(album.variants.into_iter().map(Ok::<_, ExtractorError>)) futures::stream::iter(album.variants.into_iter().map(Ok::<_, ExtractorError>))
.try_for_each_concurrent(CONFIG.core.db_concurrency, |album| async { .try_for_each_concurrent(DB_CONCURRENCY, |album| async {
self.import_album_item(album).await.map(|_| ()) self.import_album_item(album).await
}) })
.await?; .await?;
@ -223,39 +221,12 @@ impl YouTubeExtractor {
} }
pub async fn fetch_track(&self, src_id: &str) -> Result<i32, ExtractorError> { pub async fn fetch_track(&self, src_id: &str) -> Result<i32, ExtractorError> {
self._fetch_track(src_id, true).await
}
async fn _fetch_track(&self, src_id: &str, with_details: bool) -> Result<i32, ExtractorError> {
let query = self.rp.query(); let query = self.rp.query();
let (track, details) = if with_details { let (track, details) =
let (track, details) = tokio::join!(query.music_details(src_id), self.get_track_details(src_id));
tokio::join!(query.music_details(src_id), self.get_track_details(src_id)); let track = track?.track;
(track?.track, details) self.import_track_item(track, None, &[], details, false)
} else { .await
(query.music_details(src_id).await?.track, None)
};
let redirected_id = if track.id != src_id {
Some(track.id.clone())
} else {
None
};
let track_id = self
.import_track_item(track, None, &[], details, false)
.await?;
if let Some(redirected_id) = redirected_id {
tracing::info!(
"track [yt:{src_id}] got redirected to [yt:{redirected_id}], added alias"
);
Track::add_alias(
track_id,
SrcId(&redirected_id, MusicService::YouTube),
&self.core.db,
)
.await?;
}
Ok(track_id)
} }
pub async fn update_playlist( pub async fn update_playlist(
@ -302,9 +273,7 @@ impl YouTubeExtractor {
let playlist_id = n_playlist.upsert(&self.core.db).await?; let playlist_id = n_playlist.upsert(&self.core.db).await?;
let mut m_tracks = m_playlist.tracks; let mut m_tracks = m_playlist.tracks;
m_tracks m_tracks.extend_limit(&query, ITEM_LIMIT).await?;
.extend_limit(&query, CONFIG.extractor.playlist_item_limit)
.await?;
let new_entries = m_tracks let new_entries = m_tracks
.items .items
@ -377,7 +346,7 @@ impl YouTubeExtractor {
channel channel
.content .content
.extend_limit(self.rp.query(), CONFIG.extractor.user_playlist_limit) .extend_limit(self.rp.query(), ITEM_LIMIT)
.await?; .await?;
let image_url = util::yt_image_url(&channel.avatar, ImgFormat::Avatar); let image_url = util::yt_image_url(&channel.avatar, ImgFormat::Avatar);
@ -412,40 +381,38 @@ impl YouTubeExtractor {
pub async fn match_track( pub async fn match_track(
&self, &self,
src_id: SrcId<'_>, src_id: SrcId<'_>,
name: &str, title: &str,
artists: &[ArtistIdName], artist: &str,
isrc: Option<&str>, isrc: Option<&str>,
) -> Result<i32, ExtractorError> { ) -> Result<Option<i32>, ExtractorError> {
let track_id = self.get_matching_track(src_id, name, artists, isrc).await?; if let Some(track_id) = Track::get_id(src_id, &self.core.db).await? {
return Ok(Some(track_id));
Track::add_alias(track_id, src_id, &self.core.db).await?; }
if let Some(isrc) = isrc.and_then(util::normalize_isrc) {
let t_upd = TrackUpdate { if let Some((track_id, _)) = self.get_matching_track(title, artist, isrc).await? {
isrc: Some(Some(&isrc)), Track::add_alias(track_id, src_id, &self.core.db).await?;
..Default::default() if let Some(isrc) = isrc {
}; let t_upd = TrackUpdate {
t_upd.update(Id::Db(track_id), &self.core.db).await?; isrc: Some(Some(isrc)),
..Default::default()
};
t_upd.update(Id::Db(track_id), &self.core.db).await?;
}
tracing::info!("matched track {title} [{src_id}]");
Ok(Some(track_id))
} else {
Ok(None)
} }
Ok(track_id)
} }
async fn get_matching_track( async fn get_matching_track(
&self, &self,
src_id: SrcId<'_>, title: &str,
name: &str, artist: &str,
artists: &[ArtistIdName],
isrc: Option<&str>, isrc: Option<&str>,
) -> Result<i32, ExtractorError> { ) -> Result<Option<(i32, f32)>, ExtractorError> {
let artist = &artists let t1 = self.core.matchmaker.parse_title(title);
.first()
.ok_or_else(|| ExtractorError::NoMatch {
id: src_id.to_owned(),
})?
.name;
let t1 = self.core.matchmaker.parse_title(name);
// Attempt to find the track by searching YouTube for its ISRC id
// If the title of the first search result matches, return this track.
if let Some(isrc) = isrc { if let Some(isrc) = isrc {
let filter = SearchFilter::new() let filter = SearchFilter::new()
.item_type(rustypipe::param::search_filter::ItemType::Video) .item_type(rustypipe::param::search_filter::ItemType::Video)
@ -455,38 +422,17 @@ impl YouTubeExtractor {
let t2 = self.core.matchmaker.parse_title(&video.name); let t2 = self.core.matchmaker.parse_title(&video.name);
let score = self.core.matchmaker.match_name(&t1, &t2); let score = self.core.matchmaker.match_name(&t1, &t2);
if self.core.matchmaker.is_match(score) { if self.core.matchmaker.is_match(score) {
tracing::info!( return Ok(Some((self.fetch_track(&video.id).await?, score)));
"matched track `{name}` [{src_id}] => `{0}` [yt:{1}] (ISRC, {score:.2})",
video.name,
video.id
);
// Get the found track either from YT or the database
let track = if let Some(track) =
Track::get(Id::Src(&video.id, MusicService::YouTube), &self.core.db)
.await
.to_optional()?
{
track
} else {
let track_id = self._fetch_track(&video.id, false).await?;
Track::get(Id::Db(track_id), &self.core.db).await?
};
self.store_matching_artists(artists, src_id.1, &track.artists)
.await?;
return Ok(track.id);
} }
} }
} }
let a1 = self.core.matchmaker.parse_artist(artist); let a1 = self.core.matchmaker.parse_artist(artist);
// Find the track by searching for YT Music tracks/videos
let search = self let search = self
.rp .rp
.query() .query()
.music_search(format!("{name} {artist}")) .music_search(format!("{title} {artist}"))
.await?; .await?;
let best_match = search let best_match = search
.tracks .tracks
@ -508,170 +454,15 @@ impl YouTubeExtractor {
.max_by(|a, b| a.1.total_cmp(&b.1)); .max_by(|a, b| a.1.total_cmp(&b.1));
if let Some((track, score)) = best_match { if let Some((track, score)) = best_match {
if self.core.matchmaker.is_match(score) { if self.core.matchmaker.is_match(score) {
tracing::info!( return Ok(Some((
"matched track `{name}` [{src_id}] => `{0}` [yt:{1}] (search, {score:.2})", self.import_track_item(track, None, &[], None, false)
track.name, .await?,
track.id score,
); )));
return self.import_track_item(track, None, &[], None, false).await;
}
tracing::info!(
"could not match track `{name}` [{src_id}] => `{0}` [yt:{1}] (search, {score:.2})",
track.name,
track.id
);
} else {
tracing::info!("could not match track `{name}` [{src_id}] (search)");
}
Err(ExtractorError::NoMatch {
id: src_id.to_owned(),
})
}
/// Try to match the artists from a different music service to the ones of a track
/// already stored in the database.
///
/// Returns the artist id matched to the first given artist
async fn store_matching_artists(
&self,
artists_to_match: &[ArtistIdName],
service: MusicService,
yt_artists: &[ArtistId],
) -> Result<(), ExtractorError> {
let mut yt_artists = yt_artists
.iter()
.filter_map(|a| {
a.id.as_ref().map(|id| {
(
ArtistSrcidName {
id: id.as_srcid(),
name: &a.name,
},
self.core.matchmaker.parse_artist(&a.name),
)
})
})
.collect::<Vec<_>>();
for atm in artists_to_match {
let atm_srcid = SrcIdOwned(atm.id.to_owned(), service);
// Check if the artist has already been matched
let atm_id_res = self
.core
.artist_cache
.get_or_insert_async(&atm_srcid, async {
Artist::get_id(atm_srcid.as_srcid(), &self.core.db)
.await?
.ok_or(ExtractorError::NoId)
})
.await;
match atm_id_res {
Ok(atm_id) => {
// Artist is already matched
let yt_src_id = Artist::get_src_id(atm_id, &self.core.db)
.await?
.ok_or(ExtractorError::NoId)?;
if let Some((idx, _)) = yt_artists
.iter()
.enumerate()
.find(|(_, (a, _))| a.id == yt_src_id)
{
yt_artists.swap_remove(idx);
}
}
Err(ExtractorError::NoId) => {
// Match artist
let a1 = self.core.matchmaker.parse_artist(&atm.name);
let (ib, score) = yt_artists
.iter()
.map(|(_, a2)| self.core.matchmaker.match_name(&a1, a2))
.enumerate()
.max_by(|(_, a), (_, b)| a.total_cmp(b))
.unwrap_or_default();
if score > 0.8 {
let yta = yt_artists.swap_remove(ib).0;
let yta_id = Artist::get_id(yta.id, &self.core.db)
.await?
.ok_or(ExtractorError::NoId)?;
Artist::add_alias(yta_id, atm_srcid.as_srcid(), &self.core.db).await?;
tracing::info!(
"matched artist `{}` [{}] => `{}` [{}] ({score:.2})",
yta.name,
yta.id,
atm.name,
atm_srcid
);
self.core.artist_cache.insert(atm_srcid, yta_id);
}
}
Err(e) => return Err(e),
} }
} }
Ok(())
}
/// Try to match an album from a different streaming service to a YouTube track Ok(None)
pub async fn match_album(
&self,
src_id: SrcId<'_>,
name: &str,
artist: &str,
) -> Result<i32, ExtractorError> {
let albums = self
.rp
.query()
.music_search_albums(&format!("{name} {artist}"))
.await?
.items
.items;
let t1 = self.core.matchmaker.parse_title(name);
let a1 = self.core.matchmaker.parse_artist(name);
let best_match = albums
.into_iter()
.map(|album| {
let t2 = self.core.matchmaker.parse_title(&album.name);
let a2 = self.core.matchmaker.parse_title(
&album
.artists
.first()
.map(|a| a.name.to_owned())
.unwrap_or_default(),
);
let score = self.core.matchmaker.match_track(&t1, &a1, &t2, &a2, false);
(album, score)
})
.max_by(|a, b| a.1.total_cmp(&b.1));
if let Some((album, score)) = best_match {
if self.core.matchmaker.is_match(score) {
tracing::info!(
"matched album `{name}` [{src_id}] => `{0}` [yt:{1}] (search, {score:.2})",
album.name,
album.id
);
let album_id = album.id.clone();
let import_res = self.import_album_item(album).await?;
if import_res.status == GetStatus::Fetched {
return self.fetch_album(&album_id).await.map(|res| res.0);
} else {
return Ok(import_res.c);
}
}
tracing::info!(
"could not match album `{name}` [{src_id}] => `{0}` [yt:{1}] (search, {score:.2})",
album.name,
album.id
);
} else {
tracing::info!("could not match album `{name}` [{src_id}] (search)");
}
Err(ExtractorError::NoMatch {
id: src_id.to_owned(),
})
} }
} }
@ -724,7 +515,22 @@ impl YouTubeExtractor {
let release_date_precision = let release_date_precision =
details.as_ref().and_then(|d| d.release_date).map(|d| d.1); details.as_ref().and_then(|d| d.release_date).map(|d| d.1);
if let Some(t_album) = track.album { if track.is_video {
// Create a pseudo-album for music videos
let n_album = AlbumNew {
src_id: &track.id,
service: MusicService::YouTube,
name: &track.name,
release_date,
release_date_precision,
album_type: Some(AlbumType::Mv),
by_va: track.by_va,
image_url: image_url.as_deref(),
dirty: false,
..Default::default()
};
n_album.upsert(&self.core.db).await?
} else if let Some(t_album) = track.album {
let n_album = AlbumNew { let n_album = AlbumNew {
src_id: &t_album.id, src_id: &t_album.id,
service: MusicService::YouTube, service: MusicService::YouTube,
@ -736,24 +542,10 @@ impl YouTubeExtractor {
}; };
n_album.upsert(&self.core.db).await? n_album.upsert(&self.core.db).await?
} else { } else {
if !track.is_video { return Err(ExtractorError::InvalidData {
tracing::warn!("track [yt:{}] has no album, but is no MV", track.id); id: SrcIdOwned(track.id, MusicService::YouTube),
} msg: "unknown album".into(),
});
// Create a pseudo-album for music videos
let n_album = AlbumNew {
src_id: &track.id,
service: MusicService::YouTube,
name: &track.name,
release_date,
release_date_precision,
album_type: Some(AlbumType::Mv).filter(|_| track.is_video),
by_va: track.by_va,
image_url: image_url.as_deref(),
dirty: false,
..Default::default()
};
n_album.upsert(&self.core.db).await?
} }
} }
}; };
@ -796,7 +588,7 @@ impl YouTubeExtractor {
self.import_track_item(track, album_id, album_artists, None, hidden) self.import_track_item(track, album_id, album_artists, None, hidden)
.await .await
}) })
.buffered(CONFIG.core.db_concurrency) .buffered(DB_CONCURRENCY)
.try_collect() .try_collect()
.await .await
} }
@ -827,7 +619,7 @@ impl YouTubeExtractor {
) -> Result<Vec<i32>, ExtractorError> { ) -> Result<Vec<i32>, ExtractorError> {
futures::stream::iter(artists) futures::stream::iter(artists)
.map(|aid| async { self.import_artist_id(aid).await }) .map(|aid| async { self.import_artist_id(aid).await })
.buffered(CONFIG.core.db_concurrency) .buffered(CONCURRENCY)
.try_collect::<Vec<_>>() .try_collect::<Vec<_>>()
.await .await
.map_err(ExtractorError::from) .map_err(ExtractorError::from)
@ -861,22 +653,20 @@ impl YouTubeExtractor {
) -> Result<Vec<i32>, ExtractorError> { ) -> Result<Vec<i32>, ExtractorError> {
futures::stream::iter(artists) futures::stream::iter(artists)
.map(|artist| async { self.import_artist_item(artist).await }) .map(|artist| async { self.import_artist_item(artist).await })
.buffered(CONFIG.core.db_concurrency) .buffered(CONCURRENCY)
.try_collect::<Vec<_>>() .try_collect::<Vec<_>>()
.await .await
.map_err(ExtractorError::from) .map_err(ExtractorError::from)
} }
/// Import a YT Music album item (album from artist page) /// Import a YT Music album item (album from artist page)
async fn import_album_item( async fn import_album_item(&self, album: rpmodel::AlbumItem) -> Result<(), ExtractorError> {
&self,
album: rpmodel::AlbumItem,
) -> Result<GetResult<i32>, ExtractorError> {
// Return if the album was already imported // Return if the album was already imported
if let Some(album_id) = if Album::get_id_clean(SrcId(&album.id, MusicService::YouTube), &self.core.db)
Album::get_id_clean(SrcId(&album.id, MusicService::YouTube), &self.core.db).await? .await?
.is_some()
{ {
return Ok(GetResult::stored(album_id)); return Ok(());
} }
let (artists, ul_artists) = self let (artists, ul_artists) = self
@ -905,7 +695,7 @@ impl YouTubeExtractor {
tx.commit().await?; tx.commit().await?;
tracing::debug!("imported album item [yt:{}] {}", album.id, album.name); tracing::debug!("imported album item [yt:{}] {}", album.id, album.name);
Ok(GetResult::fetched(album_id)) Ok(())
} }
/// Import a YT Music playlist item /// Import a YT Music playlist item
@ -945,7 +735,7 @@ impl YouTubeExtractor {
) -> Result<Vec<i32>, ExtractorError> { ) -> Result<Vec<i32>, ExtractorError> {
futures::stream::iter(playlists) futures::stream::iter(playlists)
.map(|playlists| async { self.import_playlist_item(playlists).await }) .map(|playlists| async { self.import_playlist_item(playlists).await })
.buffered(CONFIG.core.db_concurrency) .buffered(CONCURRENCY)
.try_collect::<Vec<_>>() .try_collect::<Vec<_>>()
.await .await
.map_err(ExtractorError::from) .map_err(ExtractorError::from)
@ -989,8 +779,8 @@ impl YouTubeExtractor {
playlists: impl IntoIterator<Item = rpmodel::PlaylistItem>, playlists: impl IntoIterator<Item = rpmodel::PlaylistItem>,
) -> Result<Vec<i32>, ExtractorError> { ) -> Result<Vec<i32>, ExtractorError> {
futures::stream::iter(playlists) futures::stream::iter(playlists)
.map(|pl| async { self.import_playlist_item_yt(pl).await }) .map(|playlists| async { self.import_playlist_item_yt(playlists).await })
.buffered(CONFIG.core.db_concurrency) .buffered(CONCURRENCY)
.try_collect::<Vec<_>>() .try_collect::<Vec<_>>()
.await .await
.map_err(ExtractorError::from) .map_err(ExtractorError::from)
@ -1198,13 +988,11 @@ mod tests {
.match_track( .match_track(
SrcId("5awNIWVrh2ISfvPd5IUZNh", MusicService::Spotify), SrcId("5awNIWVrh2ISfvPd5IUZNh", MusicService::Spotify),
"PTT (Paint The Town)", "PTT (Paint The Town)",
&[ArtistIdName { "LOONA",
id: "52zMTJCKluDlFwMQWmccY7".to_owned(),
name: "LOONA".to_owned(),
}],
Some("KRA382152284"), Some("KRA382152284"),
) )
.await .await
.unwrap()
.unwrap(); .unwrap();
let track = Track::get(Id::Db(track_id), &pool).await.unwrap(); let track = Track::get(Id::Db(track_id), &pool).await.unwrap();
insta::assert_ron_snapshot!(track, { insta::assert_ron_snapshot!(track, {

48
crates/extractor/src/util/matchmaker.rs
View file

@ -163,7 +163,10 @@ impl Matchmaker {
} }
fn parse(&self, s: &str, artist: bool) -> ParsedName { fn parse(&self, s: &str, artist: bool) -> ParsedName {
let s = s.to_ascii_lowercase().replace("feat.", "|"); let mut s = s.to_ascii_lowercase();
if !artist {
s = s.replace("feat.", "|")
}
let mut parts = Vec::new(); let mut parts = Vec::new();
let mut n_parts = 0; let mut n_parts = 0;
@ -189,12 +192,18 @@ impl Matchmaker {
if brace_level == 0 { if brace_level == 0 {
do_split = true; do_split = true;
} }
} else if (space && SEPARATORS.contains(&c)) || SEPARATORS_NOSPACE.contains(&c) { } else if ((artist || space) && SEPARATORS.contains(&c))
|| SEPARATORS_NOSPACE.contains(&c)
{
do_split = true; do_split = true;
} else if c.is_whitespace() { } else if c.is_whitespace() {
// Dont create double spaces if artist {
if !space && !buf.is_empty() { do_split = true;
buf += " "; } else {
// Dont create double spaces
if !space && !buf.is_empty() {
buf += " ";
}
} }
} else { } else {
buf.push(c); buf.push(c);
@ -547,33 +556,4 @@ mod tests {
"'{a}' does not match '{b}': score {score} < 0.5\npA: {parsed_a:?}\npB: {parsed_b:?}" "'{a}' does not match '{b}': score {score} < 0.5\npA: {parsed_a:?}\npB: {parsed_b:?}"
); );
} }
#[rstest]
#[case("Fantasy", "Max-Antoine", "Fantasy", "Max-Antoine Meisters", false)]
#[case(
"Armada",
"Two Steps From Hell",
"Two Steps From Hell - Armada",
"ThePrimeAres2",
true
)]
fn match_tracks(
#[case] t1: &str,
#[case] a1: &str,
#[case] t2: &str,
#[case] a2: &str,
#[case] is_video: bool,
) {
let mm = Matchmaker::new(KEYWORDS.values(), MATCH_THR);
let parsed_t1 = mm.parse_title(t1);
let parsed_a1 = mm.parse_artist(a1);
let parsed_t2 = mm.parse_title(t2);
let parsed_a2 = mm.parse_artist(a2);
let score = mm.match_track(&parsed_t1, &parsed_a1, &parsed_t2, &parsed_a2, is_video);
assert!(
mm.is_match(score),
"'{t1}' does not match '{t2}': score {score} < 0.5\npA: {parsed_t1:?}\npB: {parsed_t2:?}"
);
}
} }

30
crates/extractor/src/util/mod.rs
View file

@ -9,15 +9,10 @@ use regex::Regex;
use rustypipe::model as rpmodel; use rustypipe::model as rpmodel;
use siphasher::sip128::{Hasher128, SipHasher}; use siphasher::sip128::{Hasher128, SipHasher};
use time::{Date, Duration, OffsetDateTime}; use time::{Date, Duration, OffsetDateTime};
use tiraya_db::models::{AlbumType, DatePrecision, SrcId, SyncData, SyncError}; use tiraya_db::models::{AlbumType, DatePrecision, SyncData, SyncError};
use crate::error::ExtractorSourceError; use crate::error::ExtractorSourceError;
pub struct ArtistSrcidName<'a> {
pub id: SrcId<'a>,
pub name: &'a str,
}
#[derive(Clone)] #[derive(Clone)]
pub struct ArtistIdName { pub struct ArtistIdName {
pub id: String, pub id: String,
@ -182,21 +177,6 @@ impl AlbumHasher {
} }
} }
pub fn normalize_isrc(isrc: &str) -> Option<String> {
static ISRC_RE: Lazy<Regex> =
Lazy::new(|| Regex::new("^[A-z]{2}[- ]?[A-z0-9]{3}[- ]?[0-9]{2}[- ]?[0-9]{5}$").unwrap());
if ISRC_RE.is_match(isrc) {
Some(
isrc.chars()
.filter(|c| *c != ' ' && *c != '-')
.map(|c| c.to_ascii_uppercase())
.collect(),
)
} else {
None
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@ -265,12 +245,4 @@ mod tests {
let hash = hasher.finish(); let hash = hasher.finish();
assert_eq!(hash, expect, "got {hash:x?}"); assert_eq!(hash, expect, "got {hash:x?}");
} }
#[rstest]
#[case("NL 1AP 19 35969", Some("NL1AP1935969"))]
#[case("nl 1AP 19 35969", Some("NL1AP1935969"))]
#[case("DE-ZC6-2342630", Some("DEZC62342630"))]
fn t_normalize_isrc(#[case] isrc: &str, #[case] expect: Option<&str>) {
assert_eq!(normalize_isrc(isrc).as_deref(), expect);
}
} }

11
crates/extractor/tests/tests.rs
View file

@ -204,7 +204,7 @@ mod sp {
/* /*
#[sqlx_database_tester::test(pool(variable = "pool", migrations = "../db/migrations"))] #[sqlx_database_tester::test(pool(variable = "pool", migrations = "../db/migrations"))]
async fn update_playlist() { async fn update_playlist() {
let xtr = Extractor::new(pool.clone()).unwrap(); let xtr = xtr(pool.clone()).await;
// 7Cdbc4geOdYRdsne9iOH49 // 7Cdbc4geOdYRdsne9iOH49
@ -215,14 +215,5 @@ mod sp {
let playlist_tracks = Playlist::get_tracks(res.c, &pool).await.unwrap().0; let playlist_tracks = Playlist::get_tracks(res.c, &pool).await.unwrap().0;
dbg!(&playlist_tracks); dbg!(&playlist_tracks);
}
#[sqlx_database_tester::test(pool(variable = "pool", migrations = "../db/migrations"))]
async fn update_user() {
setup_log();
let xtr = Extractor::new(pool.clone()).unwrap();
xtr.get_user(SrcId("1251642561", MusicService::Spotify)).await.unwrap();
}*/ }*/
} }

38
crates/testbed/src/main.rs
View file

@ -2,7 +2,7 @@ use std::{str::FromStr, time::Instant};
use clap::{Parser, Subcommand}; use clap::{Parser, Subcommand};
use sqlx::PgPool; use sqlx::PgPool;
use tiraya_db::models::{Playlist, SrcIdOwned, User}; use tiraya_db::models::{Playlist, SrcIdOwned};
use tiraya_extractor::{Extractor, GetStatus}; use tiraya_extractor::{Extractor, GetStatus};
#[derive(Parser)] #[derive(Parser)]
@ -14,10 +14,7 @@ struct Cli {
#[derive(Subcommand)] #[derive(Subcommand)]
enum Commands { enum Commands {
Artist { id: String }, Artist { id: String },
Album { id: String },
Track { id: String },
Playlist { id: String }, Playlist { id: String },
User { id: String },
} }
#[tokio::main] #[tokio::main]
@ -43,50 +40,21 @@ async fn run() {
if artist_res.status == GetStatus::Fetched { if artist_res.status == GetStatus::Fetched {
ext.fetch_artist_albums(artist_res.c.id).await.unwrap(); ext.fetch_artist_albums(artist_res.c.id).await.unwrap();
} }
dbg!(artist_res.c); dbg!(artist_res.c);
} }
Commands::Album { id } => {
let src_id = SrcIdOwned::from_str(&id).unwrap();
let album = ext.get_album(src_id.as_srcid()).await.unwrap().c;
dbg!(album);
}
Commands::Track { id } => {
let src_id = SrcIdOwned::from_str(&id).unwrap();
let track = ext.get_track(src_id.as_srcid()).await.unwrap().c;
dbg!(track);
}
Commands::Playlist { id } => { Commands::Playlist { id } => {
let src_id = SrcIdOwned::from_str(&id).unwrap(); let src_id = SrcIdOwned::from_str(&id).unwrap();
let playlist = ext.get_playlist(src_id.as_srcid()).await.unwrap(); let playlist = ext.get_playlist(src_id.as_srcid()).await.unwrap();
let tracks = Playlist::get_tracks(playlist.c.id, &pool).await.unwrap().0; let tracks = Playlist::get_tracks(playlist.c.id, &pool).await.unwrap().0;
for (i, e) in tracks.iter().enumerate() { for (i, e) in tracks.iter().enumerate() {
if let Some(track) = &e.track { if let Some(track) = &e.track {
println!( println!("{} [{}] {}", i + 1, track.src_id, track.name);
"{} [{}] {} - {}",
i + 1,
track.src_id,
track.name,
track
.artists
.first()
.map(|a| a.name.as_str())
.unwrap_or_default()
);
} else { } else {
println!("{} n/a", i + 1) println!("{} n/a", i + 1)
} }
} }
} }
Commands::User { id } => {
let src_id = SrcIdOwned::from_str(&id).unwrap();
let user = ext.get_user(src_id.as_srcid()).await.unwrap().c;
let playlists = User::playlists(user.id, &pool).await.unwrap();
dbg!(user);
for (i, playlist) in playlists.iter().enumerate() {
println!("{} {}", i + 1, playlist.name)
}
}
} }
println!("Time: {}ms", t_start.elapsed().as_millis()); println!("Time: {}ms", t_start.elapsed().as_millis());
} }

21
crates/utils/src/config.rs
View file

@ -41,18 +41,6 @@ pub struct ConfigExtractor {
/// # Number of hours after a playlist is considered stale and needs to be updated from the source /// # Number of hours after a playlist is considered stale and needs to be updated from the source
#[default(24)] #[default(24)]
pub playlist_stale_h: u32, pub playlist_stale_h: u32,
/// # Number of hours after a user is considered stale and needs to be updated from the source
#[default(24)]
pub user_stale_h: u32,
/// Maximum number of playlist items to be extracted
#[default(2000)]
pub playlist_item_limit: usize,
/// Maximum number of playlist items to be extracted and matched to YTM tracks
#[default(500)]
pub playlist_item_limit_matchmaker: usize,
/// Maximum number of user playlists to be extracted
#[default(200)]
pub user_playlist_limit: usize,
/// Directory for caching service tokens and other extractor data /// Directory for caching service tokens and other extractor data
#[default("extractor_data")] #[default("extractor_data")]
pub data_dir: PathBuf, pub data_dir: PathBuf,
@ -72,7 +60,7 @@ pub struct ConfigExtractor {
pub match_threshold: f32, pub match_threshold: f32,
pub youtube: ConfigExtractorYouTube, pub youtube: ConfigExtractorYouTube,
pub spotify: ConfigExtractorSpotify, pub spotify: Option<ConfigExtractorSpotify>,
} }
#[derive(Debug, Serialize, Deserialize, SmartDefault)] #[derive(Debug, Serialize, Deserialize, SmartDefault)]
@ -92,18 +80,13 @@ pub struct ConfigExtractorYouTube {
pub concurrency: usize, pub concurrency: usize,
} }
#[derive(Debug, Serialize, Deserialize, SmartDefault)] #[derive(Debug, Serialize, Deserialize)]
#[serde(default)]
pub struct ConfigExtractorSpotify { pub struct ConfigExtractorSpotify {
pub enable: bool,
/// Spotify client credentials /// Spotify client credentials
/// ///
/// Setup under <https://developer.spotify.com/dashboard> /// Setup under <https://developer.spotify.com/dashboard>
pub client_id: String, pub client_id: String,
pub client_secret: String, pub client_secret: String,
/// Spotify content country
#[default("US")]
pub market: String,
} }
const DEFAULT_CONFIG_FILE: &str = "config.toml"; const DEFAULT_CONFIG_FILE: &str = "config.toml";

2
crates/utils/src/lib.rs
View file

@ -1,3 +1 @@
#![warn(clippy::dbg_macro, clippy::todo)]
pub mod config; pub mod config;

8
crates/utils/src/snapshots/tiraya_utils__config__tests__read_config.snap
View file

@ -1,5 +1,6 @@
--- ---
source: crates/utils/src/config.rs source: crates/utils/src/config.rs
assertion_line: 117
expression: cfg expression: cfg
--- ---
Config( Config(
@ -16,18 +17,15 @@ Config(
artist_playlist_excluded_types: [ artist_playlist_excluded_types: [
"inst", "inst",
], ],
match_threshold: 0.5,
youtube: ConfigExtractorYouTube( youtube: ConfigExtractorYouTube(
language: "en", language: "en",
country: "US", country: "US",
n_retries: 2, n_retries: 2,
concurrency: 4, concurrency: 4,
), ),
spotify: ConfigExtractorSpotify( spotify: Some(ConfigExtractorSpotify(
enable: true,
client_id: "abc123", client_id: "abc123",
client_secret: "supersecret", client_secret: "supersecret",
market: None, )),
),
), ),
) )

13
crates/utils/testfiles/config.toml
View file

@ -11,16 +11,10 @@ db_concurrency = 8
# Music data extractor settings # Music data extractor settings
[extractor] [extractor]
# Number of hours after an item is considered stale and needs to be updated from the source # Number of hours after an item is considered stale
# and needs to be updated from the source
artist_stale_h = 24 artist_stale_h = 24
playlist_stale_h = 24 playlist_stale_h = 24
user_stale_h = 24
# Maximum number of playlist items to be extracted
playlist_item_limit = 2000
# Maximum number of playlist items to be extracted and matched to YTM tracks
playlist_item_limit_matchmaker = 500
# Maximum number of user playlists to be extracted
user_playlist_limit = 200
# Directory for caching service tokens and other extractor data # Directory for caching service tokens and other extractor data
data_dir = "extractor_data" data_dir = "extractor_data"
# Set of track types (keys from `track_type_keywords`) excluded from artist playlists # Set of track types (keys from `track_type_keywords`) excluded from artist playlists
@ -60,10 +54,7 @@ n_retries = 2
concurrency = 4 concurrency = 4
[extractor.spotify] [extractor.spotify]
enable = true
# Spotify client credentials # Spotify client credentials
# Setup under https://developer.spotify.com/dashboard # Setup under https://developer.spotify.com/dashboard
client_id = "abc123" client_id = "abc123"
client_secret = "supersecret" client_secret = "supersecret"
# Spotify content country (Default: country from user account)
# market = "US"