Compare commits
No commits in common. "24e706cb1914435c1d7af363cf5d5e908f642889" and "4ff2d82bd303bba8f85552028ed6a5e53fd72ea4" have entirely different histories.
24e706cb19
...
4ff2d82bd3
26 changed files with 342 additions and 725 deletions
1
.env.dev
1
.env.dev
|
|
@ -1,2 +1 @@
|
|||
DATABASE_URL="postgres://postgres:1234@localhost/tiraya"
|
||||
RUST_LOG="tiraya_extractor=info"
|
||||
|
|
|
|||
200
Cargo.lock
generated
200
Cargo.lock
generated
|
|
@ -31,9 +31,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.1"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab"
|
||||
checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
|
@ -374,7 +374,7 @@ dependencies = [
|
|||
"proc-macro2",
|
||||
"quote",
|
||||
"strsim",
|
||||
"syn 2.0.37",
|
||||
"syn 2.0.36",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -396,7 +396,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
|
|||
dependencies = [
|
||||
"darling_core 0.20.3",
|
||||
"quote",
|
||||
"syn 2.0.37",
|
||||
"syn 2.0.36",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -659,7 +659,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
|
|||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.37",
|
||||
"syn 2.0.36",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -719,12 +719,6 @@ version = "0.28.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.3.21"
|
||||
|
|
@ -780,9 +774,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.3.3"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
|
||||
checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
|
||||
|
||||
[[package]]
|
||||
name = "hex"
|
||||
|
|
@ -938,9 +932,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "insta"
|
||||
version = "1.32.0"
|
||||
version = "1.31.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a3e02c584f4595792d09509a94cdb92a3cef7592b1eb2d9877ee6f527062d0ea"
|
||||
checksum = "a0770b0a3d4c70567f0d58331f3088b0e4c4f56c9b8d764efe654b4a5d46de3a"
|
||||
dependencies = [
|
||||
"console",
|
||||
"lazy_static",
|
||||
|
|
@ -1148,16 +1142,6 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nu-ansi-term"
|
||||
version = "0.46.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
|
||||
dependencies = [
|
||||
"overload",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-bigint-dig"
|
||||
version = "0.8.4"
|
||||
|
|
@ -1254,7 +1238,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
|
|||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.37",
|
||||
"syn 2.0.36",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1283,12 +1267,6 @@ dependencies = [
|
|||
"range-ext",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "overload"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.12.1"
|
||||
|
|
@ -1370,7 +1348,7 @@ dependencies = [
|
|||
"pest_meta",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.37",
|
||||
"syn 2.0.36",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1419,7 +1397,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
|
|||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.37",
|
||||
"syn 2.0.36",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1488,9 +1466,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.30.0"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956"
|
||||
checksum = "81b9228215d82c7b61490fec1de287136b5de6f5700f6e58ea9ad61a7964ca51"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"serde",
|
||||
|
|
@ -1591,12 +1569,6 @@ version = "0.7.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
|
||||
|
||||
[[package]]
|
||||
name = "relative-path"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c707298afce11da2efef2f600116fa93ffa7a032b5d7b628aa17711ec81383ca"
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.11.20"
|
||||
|
|
@ -1694,53 +1666,17 @@ dependencies = [
|
|||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rstest"
|
||||
version = "0.18.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97eeab2f3c0a199bc4be135c36c924b6590b88c377d416494288c14f2db30199"
|
||||
dependencies = [
|
||||
"rstest_macros",
|
||||
"rustc_version",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rstest_macros"
|
||||
version = "0.18.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"glob",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"regex",
|
||||
"relative-path",
|
||||
"rustc_version",
|
||||
"syn 2.0.37",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
|
||||
|
||||
[[package]]
|
||||
name = "rustc_version"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
|
||||
dependencies = [
|
||||
"semver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.38.14"
|
||||
version = "0.38.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f"
|
||||
checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662"
|
||||
dependencies = [
|
||||
"bitflags 2.4.0",
|
||||
"errno",
|
||||
|
|
@ -1771,9 +1707,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.101.6"
|
||||
version = "0.101.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe"
|
||||
checksum = "45a27e3b59326c16e23d30aeb7a36a24cc0d29e71d68ff611cdfb4a01d013bed"
|
||||
dependencies = [
|
||||
"ring",
|
||||
"untrusted",
|
||||
|
|
@ -1782,11 +1718,12 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "rustypipe"
|
||||
version = "0.1.0"
|
||||
source = "git+https://code.thetadev.de/ThetaDev/rustypipe.git#127596687b0f5a29be65adfe80c40059edc4cc50"
|
||||
source = "git+https://code.thetadev.de/ThetaDev/rustypipe.git#abd3317a10535203adab0bda236c4ace66435b63"
|
||||
dependencies = [
|
||||
"base64 0.21.4",
|
||||
"fancy-regex",
|
||||
"futures",
|
||||
"log",
|
||||
"once_cell",
|
||||
"phf",
|
||||
"quick-js-dtp",
|
||||
|
|
@ -1802,7 +1739,6 @@ dependencies = [
|
|||
"thiserror",
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"url",
|
||||
"urlencoding",
|
||||
]
|
||||
|
|
@ -1870,12 +1806,6 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.188"
|
||||
|
|
@ -1893,7 +1823,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
|
|||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.37",
|
||||
"syn 2.0.36",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1952,14 +1882,14 @@ dependencies = [
|
|||
"darling 0.20.3",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.37",
|
||||
"syn 2.0.36",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1"
|
||||
version = "0.10.6"
|
||||
version = "0.10.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
|
||||
checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
|
|
@ -1977,15 +1907,6 @@ dependencies = [
|
|||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sharded-slab"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signature"
|
||||
version = "2.1.0"
|
||||
|
|
@ -2025,9 +1946,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.11.1"
|
||||
version = "1.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a"
|
||||
checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
|
|
@ -2350,9 +2271,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.37"
|
||||
version = "2.0.36"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8"
|
||||
checksum = "91e02e55d62894af2a08aca894c6577281f76769ba47c94d5756bec8ac6e7373"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -2397,12 +2318,12 @@ name = "testbed"
|
|||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"dotenvy",
|
||||
"env_logger",
|
||||
"rustypipe",
|
||||
"sqlx",
|
||||
"tiraya-db",
|
||||
"tiraya-extractor",
|
||||
"tokio",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2422,17 +2343,7 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35"
|
|||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.37",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "1.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"syn 2.0.36",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2508,10 +2419,10 @@ dependencies = [
|
|||
"env_logger",
|
||||
"futures",
|
||||
"hex-literal",
|
||||
"log",
|
||||
"once_cell",
|
||||
"quick_cache",
|
||||
"regex",
|
||||
"rstest",
|
||||
"rustypipe",
|
||||
"siphasher 1.0.0",
|
||||
"sqlx",
|
||||
|
|
@ -2521,7 +2432,6 @@ dependencies = [
|
|||
"time",
|
||||
"tiraya-db",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2549,7 +2459,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
|
|||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.37",
|
||||
"syn 2.0.36",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2575,9 +2485,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.7.9"
|
||||
version = "0.7.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d"
|
||||
checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
|
|
@ -2614,7 +2524,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
|
|||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.37",
|
||||
"syn 2.0.36",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2624,32 +2534,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"valuable",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-log"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"log",
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-subscriber"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77"
|
||||
dependencies = [
|
||||
"nu-ansi-term",
|
||||
"sharded-slab",
|
||||
"smallvec",
|
||||
"thread_local",
|
||||
"tracing-core",
|
||||
"tracing-log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2742,12 +2626,6 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "valuable"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
|
||||
|
||||
[[package]]
|
||||
name = "vcpkg"
|
||||
version = "0.2.15"
|
||||
|
|
@ -2806,7 +2684,7 @@ dependencies = [
|
|||
"once_cell",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.37",
|
||||
"syn 2.0.36",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
|
|
@ -2840,7 +2718,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
|
|||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.37",
|
||||
"syn 2.0.36",
|
||||
"wasm-bindgen-backend",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
|
@ -2894,9 +2772,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
|||
|
||||
[[package]]
|
||||
name = "winapi-util"
|
||||
version = "0.1.6"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596"
|
||||
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -17,8 +17,6 @@ thiserror = "1.0.36"
|
|||
anyhow = "1.0.71"
|
||||
dotenvy = "0.15.7"
|
||||
log = "0.4.17"
|
||||
tracing = "0.1.37"
|
||||
tracing-subscriber = "0.3.17"
|
||||
env_logger = "0.10.0"
|
||||
path_macro = "1.0.0"
|
||||
hex-literal = "0.4.1"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "select t.id, t.src_id, t.service as \"service: _\", t.name, t.duration,\n t.size, t.loudness, t.album_id, t.album_pos, t.ul_artists, t.isrc, t.description, t.created_at, t.updated_at,\n t.primary_track, t.downloaded_at, t.last_streamed_at, t.n_streams,\n jsonb_agg(json_build_object('id', a.src_id, 'sv', a.service, 'n', a.name) order by art.seq)\n filter (where a.src_id is not null) as \"artists: _\"\nfrom tracks t\n left join artists_tracks art on art.track_id = t.id\n left join artists a on a.id = art.artist_id\n join track_aliases ta on ta.track_id=t.id\nwhere ta.src_id=$1 and ta.service=$2\ngroup by t.id",
|
||||
"query": "select t.id, t.src_id, t.service as \"service: _\", t.name, t.duration, t.duration_ms,\n t.size, t.loudness, t.album_id, t.album_pos, t.ul_artists, t.isrc, t.description, t.created_at, t.updated_at,\n t.primary_track, t.downloaded_at, t.last_streamed_at, t.n_streams,\n jsonb_agg(json_build_object('id', a.src_id, 'sv', a.service, 'n', a.name) order by art.seq)\n filter (where a.src_id is not null) as \"artists: _\"\nfrom tracks t\n left join artists_tracks art on art.track_id = t.id\n left join artists a on a.id = art.artist_id\nwhere t.src_id=$1 and t.service=$2\ngroup by t.id",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
|
|
@ -42,71 +42,76 @@
|
|||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "duration_ms",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "size",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"ordinal": 7,
|
||||
"name": "loudness",
|
||||
"type_info": "Float4"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"ordinal": 8,
|
||||
"name": "album_id",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 8,
|
||||
"ordinal": 9,
|
||||
"name": "album_pos",
|
||||
"type_info": "Int2"
|
||||
},
|
||||
{
|
||||
"ordinal": 9,
|
||||
"ordinal": 10,
|
||||
"name": "ul_artists",
|
||||
"type_info": "TextArray"
|
||||
},
|
||||
{
|
||||
"ordinal": 10,
|
||||
"ordinal": 11,
|
||||
"name": "isrc",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 11,
|
||||
"ordinal": 12,
|
||||
"name": "description",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 12,
|
||||
"ordinal": 13,
|
||||
"name": "created_at",
|
||||
"type_info": "Timestamp"
|
||||
},
|
||||
{
|
||||
"ordinal": 13,
|
||||
"ordinal": 14,
|
||||
"name": "updated_at",
|
||||
"type_info": "Timestamp"
|
||||
},
|
||||
{
|
||||
"ordinal": 14,
|
||||
"ordinal": 15,
|
||||
"name": "primary_track",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 15,
|
||||
"ordinal": 16,
|
||||
"name": "downloaded_at",
|
||||
"type_info": "Timestamp"
|
||||
},
|
||||
{
|
||||
"ordinal": 16,
|
||||
"ordinal": 17,
|
||||
"name": "last_streamed_at",
|
||||
"type_info": "Timestamp"
|
||||
},
|
||||
{
|
||||
"ordinal": 17,
|
||||
"ordinal": 18,
|
||||
"name": "n_streams",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 18,
|
||||
"ordinal": 19,
|
||||
"name": "artists: _",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
|
|
@ -135,6 +140,7 @@
|
|||
false,
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
|
|
@ -151,5 +157,5 @@
|
|||
null
|
||||
]
|
||||
},
|
||||
"hash": "240c8386d19e97338971891b871bc2f5ed7a6edcdf6305d5a1bbbccc19793057"
|
||||
"hash": "1075ed43809289b5b81d7cda88d66a270ff6085a0e280cc70e5a2a3c45fccf12"
|
||||
}
|
||||
46
crates/db/.sqlx/query-147c40380556124f815737fece63fa626ec7b9a214c371bf89feaa90824a629e.json
generated
Normal file
46
crates/db/.sqlx/query-147c40380556124f815737fece63fa626ec7b9a214c371bf89feaa90824a629e.json
generated
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "insert into tracks (src_id, service, name, duration, duration_ms,\n size, loudness, album_id, album_pos, ul_artists, isrc, description, primary_track)\nvalues ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)\non conflict (src_id, service) do update set\n name = excluded.name,\n duration = case when tracks.duration_ms and not excluded.duration_ms\n then tracks.duration else coalesce(excluded.duration, tracks.duration) end,\n duration_ms = excluded.duration_ms or tracks.duration_ms,\n size = coalesce(excluded.size, tracks.size),\n loudness = coalesce(excluded.loudness, tracks.loudness),\n album_id = excluded.album_id,\n album_pos = coalesce(excluded.album_pos, tracks.album_pos),\n ul_artists = coalesce(excluded.ul_artists, tracks.ul_artists),\n isrc = coalesce(excluded.isrc, tracks.isrc),\n description = coalesce(excluded.description, tracks.description),\n primary_track = coalesce(excluded.primary_track, tracks.primary_track)\nreturning id",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "id",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
{
|
||||
"Custom": {
|
||||
"name": "music_service",
|
||||
"kind": {
|
||||
"Enum": [
|
||||
"yt",
|
||||
"ty",
|
||||
"sp",
|
||||
"mx"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"Text",
|
||||
"Int4",
|
||||
"Bool",
|
||||
"Int8",
|
||||
"Float4",
|
||||
"Int4",
|
||||
"Int2",
|
||||
"TextArray",
|
||||
"Varchar",
|
||||
"Text",
|
||||
"Bool"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "147c40380556124f815737fece63fa626ec7b9a214c371bf89feaa90824a629e"
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "select t.id, t.src_id, t.service as \"service: _\", t.name, t.duration,\n t.size, t.loudness, t.album_id, t.album_pos, t.ul_artists, t.isrc, t.description, t.created_at, t.updated_at,\n t.primary_track, t.downloaded_at, t.last_streamed_at, t.n_streams,\n jsonb_agg(json_build_object('id', a.src_id, 'sv', a.service, 'n', a.name) order by art.seq)\n filter (where a.src_id is not null) as \"artists: _\"\nfrom tracks t\n left join artists_tracks art on art.track_id = t.id\n left join artists a on a.id = art.artist_id\nwhere t.src_id=$1 and t.service=$2\ngroup by t.id",
|
||||
"query": "select t.id, t.src_id, t.service as \"service: _\", t.name, t.duration, t.duration_ms,\n t.size, t.loudness, t.album_id, t.album_pos, t.ul_artists, t.isrc, t.description, t.created_at, t.updated_at,\n t.primary_track, t.downloaded_at, t.last_streamed_at, t.n_streams,\n jsonb_agg(json_build_object('id', a.src_id, 'sv', a.service, 'n', a.name) order by art.seq)\n filter (where a.src_id is not null) as \"artists: _\"\nfrom tracks t\n left join artists_tracks art on art.track_id = t.id\n left join artists a on a.id = art.artist_id\n join track_aliases ta on ta.track_id=t.id\nwhere ta.src_id=$1 and ta.service=$2\ngroup by t.id",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
|
|
@ -42,71 +42,76 @@
|
|||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "duration_ms",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "size",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"ordinal": 7,
|
||||
"name": "loudness",
|
||||
"type_info": "Float4"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"ordinal": 8,
|
||||
"name": "album_id",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 8,
|
||||
"ordinal": 9,
|
||||
"name": "album_pos",
|
||||
"type_info": "Int2"
|
||||
},
|
||||
{
|
||||
"ordinal": 9,
|
||||
"ordinal": 10,
|
||||
"name": "ul_artists",
|
||||
"type_info": "TextArray"
|
||||
},
|
||||
{
|
||||
"ordinal": 10,
|
||||
"ordinal": 11,
|
||||
"name": "isrc",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 11,
|
||||
"ordinal": 12,
|
||||
"name": "description",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 12,
|
||||
"ordinal": 13,
|
||||
"name": "created_at",
|
||||
"type_info": "Timestamp"
|
||||
},
|
||||
{
|
||||
"ordinal": 13,
|
||||
"ordinal": 14,
|
||||
"name": "updated_at",
|
||||
"type_info": "Timestamp"
|
||||
},
|
||||
{
|
||||
"ordinal": 14,
|
||||
"ordinal": 15,
|
||||
"name": "primary_track",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 15,
|
||||
"ordinal": 16,
|
||||
"name": "downloaded_at",
|
||||
"type_info": "Timestamp"
|
||||
},
|
||||
{
|
||||
"ordinal": 16,
|
||||
"ordinal": 17,
|
||||
"name": "last_streamed_at",
|
||||
"type_info": "Timestamp"
|
||||
},
|
||||
{
|
||||
"ordinal": 17,
|
||||
"ordinal": 18,
|
||||
"name": "n_streams",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 18,
|
||||
"ordinal": 19,
|
||||
"name": "artists: _",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
|
|
@ -135,6 +140,7 @@
|
|||
false,
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
|
|
@ -151,5 +157,5 @@
|
|||
null
|
||||
]
|
||||
},
|
||||
"hash": "b74d96a7a07be63547f6292beb8e2ed946f04184555c3092d1a502d2ba7622bb"
|
||||
"hash": "22bd3df069a5e7fa8c7882a2b38c2d2e4be1b49a61ea77c4ac2770064cdc3323"
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "select t.id, t.src_id, t.service as \"service: _\", t.name, t.duration,\n t.size, t.loudness, t.album_id, t.album_pos, t.ul_artists, t.isrc, t.description, t.created_at, t.updated_at,\n t.primary_track, t.downloaded_at, t.last_streamed_at, t.n_streams,\n jsonb_agg(json_build_object('id', a.src_id, 'sv', a.service, 'n', a.name) order by art.seq)\n filter (where a.src_id is not null) as \"artists: _\"\nfrom tracks t\n left join artists_tracks art on art.track_id = t.id\n left join artists a on a.id = art.artist_id\nwhere t.id=$1\ngroup by t.id",
|
||||
"query": "select t.id, t.src_id, t.service as \"service: _\", t.name, t.duration, t.duration_ms,\n t.size, t.loudness, t.album_id, t.album_pos, t.ul_artists, t.isrc, t.description, t.created_at, t.updated_at,\n t.primary_track, t.downloaded_at, t.last_streamed_at, t.n_streams,\n jsonb_agg(json_build_object('id', a.src_id, 'sv', a.service, 'n', a.name) order by art.seq)\n filter (where a.src_id is not null) as \"artists: _\"\nfrom tracks t\n left join artists_tracks art on art.track_id = t.id\n left join artists a on a.id = art.artist_id\nwhere t.id=$1\ngroup by t.id",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
|
|
@ -42,71 +42,76 @@
|
|||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "duration_ms",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "size",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"ordinal": 7,
|
||||
"name": "loudness",
|
||||
"type_info": "Float4"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"ordinal": 8,
|
||||
"name": "album_id",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 8,
|
||||
"ordinal": 9,
|
||||
"name": "album_pos",
|
||||
"type_info": "Int2"
|
||||
},
|
||||
{
|
||||
"ordinal": 9,
|
||||
"ordinal": 10,
|
||||
"name": "ul_artists",
|
||||
"type_info": "TextArray"
|
||||
},
|
||||
{
|
||||
"ordinal": 10,
|
||||
"ordinal": 11,
|
||||
"name": "isrc",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 11,
|
||||
"ordinal": 12,
|
||||
"name": "description",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 12,
|
||||
"ordinal": 13,
|
||||
"name": "created_at",
|
||||
"type_info": "Timestamp"
|
||||
},
|
||||
{
|
||||
"ordinal": 13,
|
||||
"ordinal": 14,
|
||||
"name": "updated_at",
|
||||
"type_info": "Timestamp"
|
||||
},
|
||||
{
|
||||
"ordinal": 14,
|
||||
"ordinal": 15,
|
||||
"name": "primary_track",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 15,
|
||||
"ordinal": 16,
|
||||
"name": "downloaded_at",
|
||||
"type_info": "Timestamp"
|
||||
},
|
||||
{
|
||||
"ordinal": 16,
|
||||
"ordinal": 17,
|
||||
"name": "last_streamed_at",
|
||||
"type_info": "Timestamp"
|
||||
},
|
||||
{
|
||||
"ordinal": 17,
|
||||
"ordinal": 18,
|
||||
"name": "n_streams",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 18,
|
||||
"ordinal": 19,
|
||||
"name": "artists: _",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
|
|
@ -122,6 +127,7 @@
|
|||
false,
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
|
|
@ -138,5 +144,5 @@
|
|||
null
|
||||
]
|
||||
},
|
||||
"hash": "ebd526014759ba1316f26ba42a0b66aef00cae6c86a6d5f4747a99414d42c2b1"
|
||||
"hash": "dbb478aab35ec0b538845d633a84399984847f42b62fd980c022385a8c6e69e7"
|
||||
}
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "insert into tracks (src_id, service, name, duration,\n size, loudness, album_id, album_pos, ul_artists, isrc, description, primary_track)\nvalues ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)\non conflict (src_id, service) do update set\n name = excluded.name,\n duration = coalesce(excluded.duration, tracks.duration),\n size = coalesce(excluded.size, tracks.size),\n loudness = coalesce(excluded.loudness, tracks.loudness),\n album_id = excluded.album_id,\n album_pos = coalesce(excluded.album_pos, tracks.album_pos),\n ul_artists = coalesce(excluded.ul_artists, tracks.ul_artists),\n isrc = coalesce(excluded.isrc, tracks.isrc),\n description = coalesce(excluded.description, tracks.description),\n primary_track = coalesce(excluded.primary_track, tracks.primary_track)\nreturning id",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "id",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
{
|
||||
"Custom": {
|
||||
"name": "music_service",
|
||||
"kind": {
|
||||
"Enum": [
|
||||
"yt",
|
||||
"ty",
|
||||
"sp",
|
||||
"mx"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"Text",
|
||||
"Int4",
|
||||
"Int8",
|
||||
"Float4",
|
||||
"Int4",
|
||||
"Int2",
|
||||
"TextArray",
|
||||
"Varchar",
|
||||
"Text",
|
||||
"Bool"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "f3e06fbfd2fc3bff86b80c754109bbbd11d8c2decf4d72760c68a2f6d9d22c67"
|
||||
}
|
||||
|
|
@ -23,6 +23,7 @@ CREATE TABLE tracks (
|
|||
service music_service NOT NULL,
|
||||
name text NOT NULL,
|
||||
duration integer,
|
||||
duration_ms bool NOT NULL DEFAULT false,
|
||||
size bigint,
|
||||
loudness float4,
|
||||
album_pos smallint,
|
||||
|
|
@ -42,7 +43,8 @@ COMMENT ON COLUMN tracks.id IS E'Internal track ID';
|
|||
COMMENT ON COLUMN tracks.src_id IS E'Track ID from the source';
|
||||
COMMENT ON COLUMN tracks.service IS E'Service providing the track';
|
||||
COMMENT ON COLUMN tracks.name IS E'Track name';
|
||||
COMMENT ON COLUMN tracks.duration IS E'Duration of the track in seconds';
|
||||
COMMENT ON COLUMN tracks.duration IS E'Duration of the track in milliseconds';
|
||||
COMMENT ON COLUMN tracks.duration_ms IS E'True if the duration is in millisecond resolution';
|
||||
COMMENT ON COLUMN tracks.size IS E'File size in bytes';
|
||||
COMMENT ON COLUMN tracks.loudness IS E'Track loudness in dB';
|
||||
COMMENT ON COLUMN tracks.album_pos IS E'Position of the track in its album';
|
||||
|
|
@ -329,7 +331,7 @@ CREATE TRIGGER albums_set_updated_at
|
|||
EXECUTE PROCEDURE set_updated_at();
|
||||
|
||||
CREATE TRIGGER tracks_set_updated_at
|
||||
BEFORE UPDATE OF id,src_id,service,name,duration,size,loudness,album_pos,album_id,ul_artists,isrc,description
|
||||
BEFORE UPDATE OF id,src_id,service,name,duration,duration_ms,size,loudness,album_pos,album_id,ul_artists,isrc,description
|
||||
ON tracks
|
||||
FOR EACH ROW
|
||||
EXECUTE PROCEDURE set_updated_at();
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use time::{Date, PrimitiveDateTime};
|
|||
|
||||
use super::{
|
||||
artist::{ArtistId, ArtistJsonb},
|
||||
map_artists, AlbumType, DatePrecision, Id, MusicService, SrcId, SrcIdOwned, TrackSlim,
|
||||
map_artists, AlbumType, Artist, DatePrecision, Id, MusicService, SrcId, SrcIdOwned, TrackSlim,
|
||||
TrackSlimRow,
|
||||
};
|
||||
use crate::{
|
||||
|
|
@ -254,10 +254,11 @@ group by b.id"#,
|
|||
|
||||
pub async fn add_artists(
|
||||
id: i32,
|
||||
artists: &[i32],
|
||||
artists: &[Id<'_>],
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||
) -> Result<(), DatabaseError> {
|
||||
for artist_id in artists {
|
||||
let artist_id = Artist::resolve_id(*artist_id, tx).await?;
|
||||
sqlx::query!(
|
||||
r#"insert into artists_albums (album_id, artist_id) values ($1, $2)
|
||||
on conflict (album_id, artist_id) do nothing"#,
|
||||
|
|
@ -272,7 +273,7 @@ on conflict (album_id, artist_id) do nothing"#,
|
|||
|
||||
pub async fn set_artists(
|
||||
id: i32,
|
||||
artists: &[i32],
|
||||
artists: &[Id<'_>],
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||
) -> Result<(), DatabaseError> {
|
||||
sqlx::query!(r#"delete from artists_albums where album_id=$1"#, id)
|
||||
|
|
@ -639,7 +640,7 @@ mod tests {
|
|||
album_hash: Some(&hex!("badeaffe")),
|
||||
..Default::default()
|
||||
};
|
||||
let album_artists = [ids::ARTIST_ID_LEA, ids::ARTIST_ID_CYRIL];
|
||||
let album_artists = [ids::ARTIST_LEA, ids::ARTIST_CYRIL];
|
||||
|
||||
// Create
|
||||
let mut c_tx = pool.begin().await.unwrap();
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ expression: tracks_iwwus
|
|||
src_id: "hWFarQmaQAQ",
|
||||
service: yt,
|
||||
name: "Immer wenn wir uns sehn (\"Das schönste Mädchen der Welt\", Soundtrack)",
|
||||
duration: Some(186),
|
||||
duration: Some(186000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ expression: tracks_vakuum
|
|||
src_id: "2txScm52-QI",
|
||||
service: yt,
|
||||
name: "Die Segel sind gesetzt",
|
||||
duration: Some(186),
|
||||
duration: Some(186000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -29,7 +29,7 @@ expression: tracks_vakuum
|
|||
src_id: "oZKv47vyqQU",
|
||||
service: yt,
|
||||
name: "Monster",
|
||||
duration: Some(224),
|
||||
duration: Some(224000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -51,7 +51,7 @@ expression: tracks_vakuum
|
|||
src_id: "7WXlMU9ItnA",
|
||||
service: yt,
|
||||
name: "Dach",
|
||||
duration: Some(231),
|
||||
duration: Some(231000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -73,7 +73,7 @@ expression: tracks_vakuum
|
|||
src_id: "ySChj_9rT5Y",
|
||||
service: yt,
|
||||
name: "Kennst du das",
|
||||
duration: Some(191),
|
||||
duration: Some(191000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -95,7 +95,7 @@ expression: tracks_vakuum
|
|||
src_id: "revpIT2HiNs",
|
||||
service: yt,
|
||||
name: "Wohin willst du",
|
||||
duration: Some(255),
|
||||
duration: Some(255000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -117,7 +117,7 @@ expression: tracks_vakuum
|
|||
src_id: "LeEgBsYfjLU",
|
||||
service: yt,
|
||||
name: "Vakuum",
|
||||
duration: Some(220),
|
||||
duration: Some(220000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -139,7 +139,7 @@ expression: tracks_vakuum
|
|||
src_id: "-i5XjMkQN8M",
|
||||
service: yt,
|
||||
name: "Melodie",
|
||||
duration: Some(251),
|
||||
duration: Some(251000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -161,7 +161,7 @@ expression: tracks_vakuum
|
|||
src_id: "DhlIZkoPsxg",
|
||||
service: yt,
|
||||
name: "Du & Ich",
|
||||
duration: Some(238),
|
||||
duration: Some(238000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -183,7 +183,7 @@ expression: tracks_vakuum
|
|||
src_id: "LCoomBMOkgU",
|
||||
service: yt,
|
||||
name: "Schwerelos",
|
||||
duration: Some(198),
|
||||
duration: Some(198000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -205,7 +205,7 @@ expression: tracks_vakuum
|
|||
src_id: "c6Ot-Z3HEBo",
|
||||
service: yt,
|
||||
name: "Lichtermeer",
|
||||
duration: Some(164),
|
||||
duration: Some(164000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -227,7 +227,7 @@ expression: tracks_vakuum
|
|||
src_id: "ybm_4hQG0ok",
|
||||
service: yt,
|
||||
name: "Nachtzug",
|
||||
duration: Some(290),
|
||||
duration: Some(290000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -249,7 +249,7 @@ expression: tracks_vakuum
|
|||
src_id: "DJKmtK5PmSY",
|
||||
service: yt,
|
||||
name: "Rückenwind",
|
||||
duration: Some(263),
|
||||
duration: Some(263000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ expression: tracks
|
|||
src_id: "LeEgBsYfjLU",
|
||||
service: yt,
|
||||
name: "Vakuum",
|
||||
duration: Some(220),
|
||||
duration: Some(220000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -29,7 +29,7 @@ expression: tracks
|
|||
src_id: "LCoomBMOkgU",
|
||||
service: yt,
|
||||
name: "Schwerelos",
|
||||
duration: Some(198),
|
||||
duration: Some(198000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -51,7 +51,7 @@ expression: tracks
|
|||
src_id: "c6Ot-Z3HEBo",
|
||||
service: yt,
|
||||
name: "Lichtermeer",
|
||||
duration: Some(164),
|
||||
duration: Some(164000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -73,7 +73,7 @@ expression: tracks
|
|||
src_id: "hWFarQmaQAQ",
|
||||
service: yt,
|
||||
name: "Immer wenn wir uns sehn (\"Das schönste Mädchen der Welt\", Soundtrack)",
|
||||
duration: Some(186),
|
||||
duration: Some(186000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -99,7 +99,7 @@ expression: tracks
|
|||
src_id: "2txScm52-QI",
|
||||
service: yt,
|
||||
name: "Die Segel sind gesetzt",
|
||||
duration: Some(186),
|
||||
duration: Some(186000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ expression: tracks
|
|||
src_id: "2txScm52-QI",
|
||||
service: yt,
|
||||
name: "Die Segel sind gesetzt",
|
||||
duration: Some(186),
|
||||
duration: Some(186000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -29,7 +29,7 @@ expression: tracks
|
|||
src_id: "oZKv47vyqQU",
|
||||
service: yt,
|
||||
name: "Monster",
|
||||
duration: Some(224),
|
||||
duration: Some(224000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -51,7 +51,7 @@ expression: tracks
|
|||
src_id: "7WXlMU9ItnA",
|
||||
service: yt,
|
||||
name: "Dach",
|
||||
duration: Some(231),
|
||||
duration: Some(231000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -73,7 +73,7 @@ expression: tracks
|
|||
src_id: "ySChj_9rT5Y",
|
||||
service: yt,
|
||||
name: "Kennst du das",
|
||||
duration: Some(191),
|
||||
duration: Some(191000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -95,7 +95,7 @@ expression: tracks
|
|||
src_id: "revpIT2HiNs",
|
||||
service: yt,
|
||||
name: "Wohin willst du",
|
||||
duration: Some(255),
|
||||
duration: Some(255000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -117,7 +117,7 @@ expression: tracks
|
|||
src_id: "LeEgBsYfjLU",
|
||||
service: yt,
|
||||
name: "Vakuum",
|
||||
duration: Some(220),
|
||||
duration: Some(220000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -139,7 +139,7 @@ expression: tracks
|
|||
src_id: "-i5XjMkQN8M",
|
||||
service: yt,
|
||||
name: "Melodie",
|
||||
duration: Some(251),
|
||||
duration: Some(251000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -161,7 +161,7 @@ expression: tracks
|
|||
src_id: "DhlIZkoPsxg",
|
||||
service: yt,
|
||||
name: "Du & Ich",
|
||||
duration: Some(238),
|
||||
duration: Some(238000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -183,7 +183,7 @@ expression: tracks
|
|||
src_id: "LCoomBMOkgU",
|
||||
service: yt,
|
||||
name: "Schwerelos",
|
||||
duration: Some(198),
|
||||
duration: Some(198000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -205,7 +205,7 @@ expression: tracks
|
|||
src_id: "c6Ot-Z3HEBo",
|
||||
service: yt,
|
||||
name: "Lichtermeer",
|
||||
duration: Some(164),
|
||||
duration: Some(164000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -227,7 +227,7 @@ expression: tracks
|
|||
src_id: "ybm_4hQG0ok",
|
||||
service: yt,
|
||||
name: "Nachtzug",
|
||||
duration: Some(290),
|
||||
duration: Some(290000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -249,7 +249,7 @@ expression: tracks
|
|||
src_id: "DJKmtK5PmSY",
|
||||
service: yt,
|
||||
name: "Rückenwind",
|
||||
duration: Some(263),
|
||||
duration: Some(263000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
@ -271,7 +271,7 @@ expression: tracks
|
|||
src_id: "hWFarQmaQAQ",
|
||||
service: yt,
|
||||
name: "Immer wenn wir uns sehn (\"Das schönste Mädchen der Welt\", Soundtrack)",
|
||||
duration: Some(186),
|
||||
duration: Some(186000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ expression: tracks
|
|||
src_id: "WSBUeFdXiSs",
|
||||
service: yt,
|
||||
name: "Leicht",
|
||||
duration: Some(206),
|
||||
duration: Some(206000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC-2mb3G26qV676d-iXbOTVQ"),
|
||||
|
|
@ -36,7 +36,7 @@ expression: tracks
|
|||
src_id: "OCgE2GSL1Pk",
|
||||
service: yt,
|
||||
name: "Smoke Signals",
|
||||
duration: Some(197),
|
||||
duration: Some(197000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UCQ6yypykkyPLM5FVhOm4Eog"),
|
||||
|
|
@ -62,7 +62,7 @@ expression: tracks
|
|||
src_id: "6485PhOtHzY",
|
||||
service: yt,
|
||||
name: "Lieblingsmensch",
|
||||
duration: Some(190),
|
||||
duration: Some(190000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UCIh4j8fXWf2U0ro0qnGU8Mg"),
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ Track(
|
|||
service: yt,
|
||||
name: "empty",
|
||||
duration: None,
|
||||
duration_ms: false,
|
||||
artists: [],
|
||||
size: None,
|
||||
loudness: None,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ Track(
|
|||
src_id: "g0iRiJ_ck48",
|
||||
service: yt,
|
||||
name: "Aulë und Yavanna",
|
||||
duration: Some(216),
|
||||
duration: Some(216000),
|
||||
duration_ms: false,
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ TrackSlim(
|
|||
src_id: "g0iRiJ_ck48",
|
||||
service: yt,
|
||||
name: "Aulë und Yavanna",
|
||||
duration: Some(216),
|
||||
duration: Some(216000),
|
||||
artists: [
|
||||
ArtistId(
|
||||
id: Some("yt:UC_MxOdawj_BStPs4CKBYD0Q"),
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ use time::{Date, PrimitiveDateTime};
|
|||
use super::{
|
||||
album::AlbumId,
|
||||
artist::{ArtistId, ArtistJsonb},
|
||||
map_artists, AlbumType, Id, MusicService, SrcId, SrcIdOwned,
|
||||
map_artists, AlbumType, Artist, Id, MusicService, SrcId, SrcIdOwned,
|
||||
};
|
||||
use crate::{
|
||||
error::{DatabaseError, OptionalRes},
|
||||
|
|
@ -20,6 +20,7 @@ pub struct Track {
|
|||
pub service: MusicService,
|
||||
pub name: String,
|
||||
pub duration: Option<i32>,
|
||||
pub duration_ms: bool,
|
||||
pub artists: Vec<ArtistId>,
|
||||
pub size: Option<i64>,
|
||||
pub loudness: Option<f32>,
|
||||
|
|
@ -42,6 +43,7 @@ struct TrackRow {
|
|||
service: MusicService,
|
||||
name: String,
|
||||
duration: Option<i32>,
|
||||
duration_ms: bool,
|
||||
size: Option<i64>,
|
||||
loudness: Option<f32>,
|
||||
album_id: i32,
|
||||
|
|
@ -65,6 +67,7 @@ pub struct TrackNew<'a> {
|
|||
pub service: MusicService,
|
||||
pub name: &'a str,
|
||||
pub duration: Option<i32>,
|
||||
pub duration_ms: bool,
|
||||
pub size: Option<i64>,
|
||||
pub loudness: Option<f32>,
|
||||
pub album_id: i32,
|
||||
|
|
@ -80,6 +83,7 @@ pub struct TrackNew<'a> {
|
|||
pub struct TrackUpdate<'a> {
|
||||
pub name: Option<&'a str>,
|
||||
pub duration: Option<Option<i32>>,
|
||||
pub duration_ms: Option<bool>,
|
||||
pub size: Option<Option<i64>>,
|
||||
pub loudness: Option<Option<f32>>,
|
||||
pub album_id: Option<i32>,
|
||||
|
|
@ -148,7 +152,7 @@ impl Track {
|
|||
Id::Db(id) => {
|
||||
sqlx::query_as!(
|
||||
TrackRow,
|
||||
r#"select t.id, t.src_id, t.service as "service: _", t.name, t.duration,
|
||||
r#"select t.id, t.src_id, t.service as "service: _", t.name, t.duration, t.duration_ms,
|
||||
t.size, t.loudness, t.album_id, t.album_pos, t.ul_artists, t.isrc, t.description, t.created_at, t.updated_at,
|
||||
t.primary_track, t.downloaded_at, t.last_streamed_at, t.n_streams,
|
||||
jsonb_agg(json_build_object('id', a.src_id, 'sv', a.service, 'n', a.name) order by art.seq)
|
||||
|
|
@ -166,7 +170,7 @@ group by t.id"#,
|
|||
Id::Src(src_id, srv) => {
|
||||
let res = sqlx::query_as!(
|
||||
TrackRow,
|
||||
r#"select t.id, t.src_id, t.service as "service: _", t.name, t.duration,
|
||||
r#"select t.id, t.src_id, t.service as "service: _", t.name, t.duration, t.duration_ms,
|
||||
t.size, t.loudness, t.album_id, t.album_pos, t.ul_artists, t.isrc, t.description, t.created_at, t.updated_at,
|
||||
t.primary_track, t.downloaded_at, t.last_streamed_at, t.n_streams,
|
||||
jsonb_agg(json_build_object('id', a.src_id, 'sv', a.service, 'n', a.name) order by art.seq)
|
||||
|
|
@ -188,7 +192,7 @@ group by t.id"#,
|
|||
None => {
|
||||
sqlx::query_as!(
|
||||
TrackRow,
|
||||
r#"select t.id, t.src_id, t.service as "service: _", t.name, t.duration,
|
||||
r#"select t.id, t.src_id, t.service as "service: _", t.name, t.duration, t.duration_ms,
|
||||
t.size, t.loudness, t.album_id, t.album_pos, t.ul_artists, t.isrc, t.description, t.created_at, t.updated_at,
|
||||
t.primary_track, t.downloaded_at, t.last_streamed_at, t.n_streams,
|
||||
jsonb_agg(json_build_object('id', a.src_id, 'sv', a.service, 'n', a.name) order by art.seq)
|
||||
|
|
@ -280,7 +284,7 @@ where ta.src_id=$1 and ta.service=$2"#,
|
|||
|
||||
pub async fn set_artists(
|
||||
id: i32,
|
||||
artists: &[i32],
|
||||
artists: &[Id<'_>],
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||
) -> Result<(), DatabaseError> {
|
||||
sqlx::query!(r#"delete from artists_tracks where track_id=$1"#, id)
|
||||
|
|
@ -288,6 +292,7 @@ where ta.src_id=$1 and ta.service=$2"#,
|
|||
.await?;
|
||||
|
||||
for artist_id in artists {
|
||||
let artist_id = Artist::resolve_id(*artist_id, tx).await?;
|
||||
sqlx::query!(
|
||||
r#"insert into artists_tracks (track_id, artist_id) values ($1, $2)"#,
|
||||
id,
|
||||
|
|
@ -347,12 +352,14 @@ impl TrackNew<'_> {
|
|||
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
|
||||
{
|
||||
let res = sqlx::query!(
|
||||
r#"insert into tracks (src_id, service, name, duration,
|
||||
r#"insert into tracks (src_id, service, name, duration, duration_ms,
|
||||
size, loudness, album_id, album_pos, ul_artists, isrc, description, primary_track)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
||||
on conflict (src_id, service) do update set
|
||||
name = excluded.name,
|
||||
duration = coalesce(excluded.duration, tracks.duration),
|
||||
duration = case when tracks.duration_ms and not excluded.duration_ms
|
||||
then tracks.duration else coalesce(excluded.duration, tracks.duration) end,
|
||||
duration_ms = excluded.duration_ms or tracks.duration_ms,
|
||||
size = coalesce(excluded.size, tracks.size),
|
||||
loudness = coalesce(excluded.loudness, tracks.loudness),
|
||||
album_id = excluded.album_id,
|
||||
|
|
@ -366,6 +373,7 @@ returning id"#,
|
|||
self.service as MusicService,
|
||||
self.name,
|
||||
self.duration,
|
||||
self.duration_ms,
|
||||
self.size,
|
||||
self.loudness,
|
||||
self.album_id,
|
||||
|
|
@ -402,6 +410,14 @@ impl TrackUpdate<'_> {
|
|||
query.push_bind(duration);
|
||||
n += 1;
|
||||
}
|
||||
if let Some(duration_ms) = &self.duration_ms {
|
||||
if n != 0 {
|
||||
query.push(", ");
|
||||
}
|
||||
query.push("duration_ms=");
|
||||
query.push_bind(duration_ms);
|
||||
n += 1;
|
||||
}
|
||||
if let Some(size) = &self.size {
|
||||
if n != 0 {
|
||||
query.push(", ");
|
||||
|
|
@ -583,6 +599,7 @@ impl From<TrackRow> for Track {
|
|||
service: value.service,
|
||||
name: value.name,
|
||||
duration: value.duration,
|
||||
duration_ms: value.duration_ms,
|
||||
artists: map_artists(value.artists, value.ul_artists),
|
||||
size: value.size,
|
||||
loudness: value.loudness,
|
||||
|
|
@ -637,7 +654,8 @@ mod tests {
|
|||
src_id: "g0iRiJ_ck48",
|
||||
service: MusicService::YouTube,
|
||||
name: "Aulë und Yavanna",
|
||||
duration: Some(216),
|
||||
duration: Some(216000),
|
||||
duration_ms: false,
|
||||
size: Some(3_439_414),
|
||||
loudness: Some(6.1513805),
|
||||
album_id: 1,
|
||||
|
|
@ -647,7 +665,7 @@ mod tests {
|
|||
description: Some("Hello World"),
|
||||
primary_track: Some(true),
|
||||
};
|
||||
let track_artists = [ids::ARTIST_ID_LEA, ids::ARTIST_ID_CYRIL];
|
||||
let track_artists = [ids::ARTIST_LEA, ids::ARTIST_CYRIL];
|
||||
|
||||
// Create
|
||||
let mut c_tx = pool.begin().await.unwrap();
|
||||
|
|
@ -686,6 +704,7 @@ mod tests {
|
|||
let clear = TrackUpdate {
|
||||
name: Some("empty"),
|
||||
duration: Some(None),
|
||||
duration_ms: Some(false),
|
||||
size: Some(None),
|
||||
loudness: Some(None),
|
||||
album_id: None,
|
||||
|
|
|
|||
38
crates/db/testdata/base.sql
vendored
38
crates/db/testdata/base.sql
vendored
|
|
@ -36,25 +36,25 @@ INSERT INTO artists_albums (artist_id,album_id) VALUES
|
|||
(8,6),
|
||||
(9,7);
|
||||
|
||||
INSERT INTO tracks (src_id,service,"name",duration,"size",loudness,album_pos,album_id,ul_artists,isrc,created_at,updated_at,primary_track,downloaded_at,last_streamed_at,n_streams) VALUES
|
||||
('2txScm52-QI','yt','Die Segel sind gesetzt',186,NULL,NULL,1,1,'{}',NULL,'2023-08-30 22:40:30.15406','2023-08-30 22:49:15.955139',NULL,NULL,NULL,0),
|
||||
('oZKv47vyqQU','yt','Monster',224,NULL,NULL,2,1,'{}',NULL,'2023-08-30 22:41:22.905632','2023-08-30 22:49:15.957959',NULL,NULL,NULL,0),
|
||||
('7WXlMU9ItnA','yt','Dach',231,NULL,NULL,3,1,'{}',NULL,'2023-08-30 22:42:00.093845','2023-08-30 22:49:15.959538',NULL,NULL,NULL,0),
|
||||
('ySChj_9rT5Y','yt','Kennst du das',191,NULL,NULL,4,1,'{}',NULL,'2023-08-30 22:42:57.462304','2023-08-30 22:49:15.961008',NULL,NULL,NULL,0),
|
||||
('revpIT2HiNs','yt','Wohin willst du',255,NULL,NULL,5,1,'{}',NULL,'2023-08-30 22:43:39.594609','2023-08-30 22:49:15.962497',NULL,NULL,NULL,0),
|
||||
('LeEgBsYfjLU','yt','Vakuum',220,NULL,NULL,6,1,'{}',NULL,'2023-08-30 22:44:14.749107','2023-08-30 22:49:15.963926',NULL,NULL,NULL,0),
|
||||
('-i5XjMkQN8M','yt','Melodie',251,NULL,NULL,7,1,'{}',NULL,'2023-08-30 22:44:44.585095','2023-08-30 22:49:15.96526',NULL,NULL,NULL,0),
|
||||
('DhlIZkoPsxg','yt','Du & Ich',238,NULL,NULL,8,1,'{}',NULL,'2023-08-30 22:45:20.711259','2023-08-30 22:49:15.966532',NULL,NULL,NULL,0),
|
||||
('LCoomBMOkgU','yt','Schwerelos',198,NULL,NULL,9,1,'{}',NULL,'2023-08-30 22:46:13.885768','2023-08-30 22:49:15.967865',NULL,NULL,NULL,0),
|
||||
('c6Ot-Z3HEBo','yt','Lichtermeer',164,NULL,NULL,10,1,'{}',NULL,'2023-08-30 22:46:41.10766','2023-08-30 22:49:15.969191',NULL,NULL,NULL,0),
|
||||
('ybm_4hQG0ok','yt','Nachtzug',290,NULL,NULL,11,1,'{}',NULL,'2023-08-30 22:47:28.033803','2023-08-30 22:49:15.97051',NULL,NULL,NULL,0),
|
||||
('DJKmtK5PmSY','yt','Rückenwind',263,NULL,NULL,12,1,'{}',NULL,'2023-08-30 22:47:54.822762','2023-08-30 22:49:15.971789',NULL,NULL,NULL,0),
|
||||
('hWFarQmaQAQ','yt','Immer wenn wir uns sehn ("Das schönste Mädchen der Welt", Soundtrack)',186,NULL,NULL,1,2,'{}',NULL,'2023-08-30 23:00:09.653056','2023-08-30 23:00:09.653056',NULL,NULL,NULL,0),
|
||||
('tXb7WTkhE1c','yt','Das schönste Mädchen der Welt ("Das schönste Mädchen der Welt", Soundtrack)',142,NULL,NULL,1,3,'{}',NULL,'2023-08-30 23:00:09.653056','2023-08-30 23:00:09.653056',NULL,NULL,NULL,0),
|
||||
('ZeerrnuLi5E','yt','Black Mamba',229,NULL,NULL,1,4,'{}',NULL,'2023-09-09 22:53:50.360159','2023-09-09 22:53:50.360159',NULL,NULL,NULL,0),
|
||||
('OCgE2GSL1Pk','yt','Smoke Signals',197,NULL,NULL,NULL,5,'{}',NULL,'2023-09-09 22:53:50.360159','2023-09-09 22:53:50.360159',NULL,NULL,NULL,0),
|
||||
('6485PhOtHzY','yt','Lieblingsmensch',190,NULL,NULL,1,6,'{}',NULL,'2023-09-09 22:53:50.360159','2023-09-09 22:53:50.360159',NULL,NULL,NULL,0),
|
||||
('WSBUeFdXiSs','yt','Leicht',206,NULL,NULL,NULL,7,'{}',NULL,'2023-09-09 22:53:50.360159','2023-09-09 22:53:50.360159',NULL,NULL,NULL,0);
|
||||
INSERT INTO tracks (src_id,service,"name",duration,duration_ms,"size",loudness,album_pos,album_id,ul_artists,isrc,created_at,updated_at,primary_track,downloaded_at,last_streamed_at,n_streams) VALUES
|
||||
('2txScm52-QI','yt','Die Segel sind gesetzt',186000,false,NULL,NULL,1,1,'{}',NULL,'2023-08-30 22:40:30.15406','2023-08-30 22:49:15.955139',NULL,NULL,NULL,0),
|
||||
('oZKv47vyqQU','yt','Monster',224000,false,NULL,NULL,2,1,'{}',NULL,'2023-08-30 22:41:22.905632','2023-08-30 22:49:15.957959',NULL,NULL,NULL,0),
|
||||
('7WXlMU9ItnA','yt','Dach',231000,false,NULL,NULL,3,1,'{}',NULL,'2023-08-30 22:42:00.093845','2023-08-30 22:49:15.959538',NULL,NULL,NULL,0),
|
||||
('ySChj_9rT5Y','yt','Kennst du das',191000,false,NULL,NULL,4,1,'{}',NULL,'2023-08-30 22:42:57.462304','2023-08-30 22:49:15.961008',NULL,NULL,NULL,0),
|
||||
('revpIT2HiNs','yt','Wohin willst du',255000,false,NULL,NULL,5,1,'{}',NULL,'2023-08-30 22:43:39.594609','2023-08-30 22:49:15.962497',NULL,NULL,NULL,0),
|
||||
('LeEgBsYfjLU','yt','Vakuum',220000,false,NULL,NULL,6,1,'{}',NULL,'2023-08-30 22:44:14.749107','2023-08-30 22:49:15.963926',NULL,NULL,NULL,0),
|
||||
('-i5XjMkQN8M','yt','Melodie',251000,false,NULL,NULL,7,1,'{}',NULL,'2023-08-30 22:44:44.585095','2023-08-30 22:49:15.96526',NULL,NULL,NULL,0),
|
||||
('DhlIZkoPsxg','yt','Du & Ich',238000,false,NULL,NULL,8,1,'{}',NULL,'2023-08-30 22:45:20.711259','2023-08-30 22:49:15.966532',NULL,NULL,NULL,0),
|
||||
('LCoomBMOkgU','yt','Schwerelos',198000,false,NULL,NULL,9,1,'{}',NULL,'2023-08-30 22:46:13.885768','2023-08-30 22:49:15.967865',NULL,NULL,NULL,0),
|
||||
('c6Ot-Z3HEBo','yt','Lichtermeer',164000,false,NULL,NULL,10,1,'{}',NULL,'2023-08-30 22:46:41.10766','2023-08-30 22:49:15.969191',NULL,NULL,NULL,0),
|
||||
('ybm_4hQG0ok','yt','Nachtzug',290000,false,NULL,NULL,11,1,'{}',NULL,'2023-08-30 22:47:28.033803','2023-08-30 22:49:15.97051',NULL,NULL,NULL,0),
|
||||
('DJKmtK5PmSY','yt','Rückenwind',263000,false,NULL,NULL,12,1,'{}',NULL,'2023-08-30 22:47:54.822762','2023-08-30 22:49:15.971789',NULL,NULL,NULL,0),
|
||||
('hWFarQmaQAQ','yt','Immer wenn wir uns sehn ("Das schönste Mädchen der Welt", Soundtrack)',186000,false,NULL,NULL,1,2,'{}',NULL,'2023-08-30 23:00:09.653056','2023-08-30 23:00:09.653056',NULL,NULL,NULL,0),
|
||||
('tXb7WTkhE1c','yt','Das schönste Mädchen der Welt ("Das schönste Mädchen der Welt", Soundtrack)',142000,false,NULL,NULL,1,3,'{}',NULL,'2023-08-30 23:00:09.653056','2023-08-30 23:00:09.653056',NULL,NULL,NULL,0),
|
||||
('ZeerrnuLi5E','yt','Black Mamba',229000,false,NULL,NULL,1,4,'{}',NULL,'2023-09-09 22:53:50.360159','2023-09-09 22:53:50.360159',NULL,NULL,NULL,0),
|
||||
('OCgE2GSL1Pk','yt','Smoke Signals',197000,false,NULL,NULL,NULL,5,'{}',NULL,'2023-09-09 22:53:50.360159','2023-09-09 22:53:50.360159',NULL,NULL,NULL,0),
|
||||
('6485PhOtHzY','yt','Lieblingsmensch',190000,false,NULL,NULL,1,6,'{}',NULL,'2023-09-09 22:53:50.360159','2023-09-09 22:53:50.360159',NULL,NULL,NULL,0),
|
||||
('WSBUeFdXiSs','yt','Leicht',206000,false,NULL,NULL,NULL,7,'{}',NULL,'2023-09-09 22:53:50.360159','2023-09-09 22:53:50.360159',NULL,NULL,NULL,0);
|
||||
|
||||
INSERT INTO artists_tracks (artist_id,track_id) VALUES
|
||||
(1,1),
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ futures.workspace = true
|
|||
time.workspace = true
|
||||
once_cell.workspace = true
|
||||
regex.workspace = true
|
||||
tracing.workspace = true
|
||||
log.workspace = true
|
||||
quick_cache.workspace = true
|
||||
siphasher.workspace = true
|
||||
hex-literal.workspace = true
|
||||
|
|
@ -25,6 +25,5 @@ tiraya-db.workspace = true
|
|||
[dev-dependencies]
|
||||
tokio.workspace = true
|
||||
sqlx-database-tester.workspace = true
|
||||
rstest.workspace = true
|
||||
env_logger.workspace = true
|
||||
test-log.workspace = true
|
||||
|
|
|
|||
1
crates/extractor/rustypipe_cache.json
Normal file
1
crates/extractor/rustypipe_cache.json
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"desktop_client":null,"music_client":{"last_update":"2023-09-20T17:47:33Z","data":{"version":"1.20230913.01.00-canary_control"}},"deobf":null}
|
||||
|
|
@ -3,20 +3,22 @@
|
|||
pub mod error;
|
||||
mod util;
|
||||
|
||||
use std::{borrow::Cow, sync::Arc};
|
||||
use std::{borrow::Cow, hash::Hasher, sync::Arc};
|
||||
|
||||
use error::ExtractorError;
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
use hex_literal::hex;
|
||||
use quick_cache::sync::Cache;
|
||||
use rustypipe::{client::RustyPipe, model::richtext::ToPlaintext};
|
||||
use rustypipe::client::RustyPipe;
|
||||
use siphasher::sip128::{Hasher128, SipHasher};
|
||||
use sqlx::{Pool, Postgres};
|
||||
use time::{Date, Duration, OffsetDateTime};
|
||||
use tiraya_db::{
|
||||
error::OptionalRes,
|
||||
models::{
|
||||
Album, AlbumNew, AlbumType, AlbumUpdate, Artist, ArtistNew, DatePrecision, EntityType, Id,
|
||||
MusicService, Playlist, PlaylistImgType, PlaylistNew, PlaylistType, SrcId, SrcIdOwned,
|
||||
SyncData, Track, TrackNew, TrackUpdate,
|
||||
Album, AlbumNew, AlbumType, Artist, ArtistNew, DatePrecision, EntityType, Id, MusicService,
|
||||
Playlist, PlaylistImgType, PlaylistNew, PlaylistType, SrcId, SrcIdOwned, SyncData, Track,
|
||||
TrackNew,
|
||||
},
|
||||
};
|
||||
|
||||
|
|
@ -35,7 +37,6 @@ const ARTIST_STALE: Duration = Duration::hours(24);
|
|||
const CONCURRENCY: usize = 4;
|
||||
const DB_CONCURRENCY: usize = 8;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct SyncLastUpdate {
|
||||
id: i32,
|
||||
state: LastUpdateState,
|
||||
|
|
@ -90,7 +91,6 @@ impl Extractor {
|
|||
)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub async fn get_artist(&self, id: SrcId<'_>) -> Result<GetResult<Artist>, ExtractorError> {
|
||||
let artist = Artist::get(id.id(), &self.db).await.to_optional().unwrap();
|
||||
let last_update = if let Some(artist) = artist {
|
||||
|
|
@ -143,7 +143,6 @@ impl Extractor {
|
|||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn update_yt_artist(
|
||||
&self,
|
||||
id: &str,
|
||||
|
|
@ -194,7 +193,7 @@ impl Extractor {
|
|||
|
||||
let top_track_ids =
|
||||
futures::stream::iter(artist.tracks.into_iter().filter(|t| !t.is_video))
|
||||
.map(|track| async { self.import_yt_track(track, None, &[]).await })
|
||||
.map(|track| async move { self.import_yt_track(track, None).await })
|
||||
.buffered(CONCURRENCY)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
|
@ -203,14 +202,14 @@ impl Extractor {
|
|||
.filter_map(|id| match id {
|
||||
Ok(id) => Some(id),
|
||||
Err(e) => {
|
||||
tracing::error!("could not import artist track: {}", e);
|
||||
log::error!("could not import artist track: {}", e);
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let related_artist_ids = futures::stream::iter(artist.similar_artists)
|
||||
.map(|artist| async { self.import_yt_artist_item(artist).await })
|
||||
.map(|artist| async move { self.import_yt_artist_item(artist).await })
|
||||
.buffered(DB_CONCURRENCY)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
|
@ -219,14 +218,14 @@ impl Extractor {
|
|||
.filter_map(|id| match id {
|
||||
Ok(id) => Some(id),
|
||||
Err(e) => {
|
||||
tracing::error!("could not import related artist: {}", e);
|
||||
log::error!("could not import related artist: {}", e);
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let related_playlist_ids = futures::stream::iter(artist.playlists)
|
||||
.map(|pl| async { self.import_yt_playlist_item(pl).await })
|
||||
.map(|pl| async move { self.import_yt_playlist_item(pl).await })
|
||||
.buffered(DB_CONCURRENCY)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
|
@ -235,7 +234,7 @@ impl Extractor {
|
|||
.filter_map(|id| match id {
|
||||
Ok(id) => Some(id),
|
||||
Err(e) => {
|
||||
tracing::error!("could not import artist playlist: {}", e);
|
||||
log::error!("could not import artist playlist: {}", e);
|
||||
None
|
||||
}
|
||||
})
|
||||
|
|
@ -277,7 +276,7 @@ impl Extractor {
|
|||
// Insert all albums
|
||||
futures::stream::iter(artist.albums)
|
||||
.map(Ok)
|
||||
.try_for_each_concurrent(DB_CONCURRENCY, |album| async {
|
||||
.try_for_each_concurrent(DB_CONCURRENCY, |album| async move {
|
||||
self.import_yt_album_item(album).await
|
||||
})
|
||||
.await?;
|
||||
|
|
@ -289,32 +288,35 @@ impl Extractor {
|
|||
};
|
||||
Artist::set_last_sync(artist_id, this_update_state.into(), &self.db).await?;
|
||||
|
||||
tracing::info!("fetched artist [yt:{}] {}", artist.id, artist.name);
|
||||
Ok(GetResult::fetched(artist_id))
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub async fn fetch_yt_artist_albums(&self, id: i32) -> Result<(), ExtractorError> {
|
||||
for _ in 0..2 {
|
||||
let dirty_albums = Artist::dirty_album_ids(id, &self.db).await?;
|
||||
let has_more = futures::stream::iter(dirty_albums)
|
||||
.map(|id| async move {
|
||||
match self.fetch_yt_album(&id.src_id).await {
|
||||
Ok(more) => more.1,
|
||||
Err(e) => {
|
||||
tracing::error!("could not import album [yt:{}]: {}", id.src_id, e);
|
||||
false
|
||||
}
|
||||
pub async fn update_yt_artist_albums(&self, id: i32) -> Result<(), ExtractorError> {
|
||||
let dirty_albums = Artist::dirty_album_ids(id, &self.db).await?;
|
||||
let more_albums = futures::stream::iter(dirty_albums)
|
||||
.map(|id| async move {
|
||||
match self.import_yt_album(&id.src_id).await {
|
||||
Ok(more) => more,
|
||||
Err(e) => {
|
||||
log::error!("could not import album [yt:{}]: {}", id.src_id, e);
|
||||
Vec::new()
|
||||
}
|
||||
})
|
||||
.buffer_unordered(DB_CONCURRENCY)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
if has_more.iter().all(|x| !x) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
tracing::info!("updated artist albums for #{id}");
|
||||
}
|
||||
})
|
||||
.buffer_unordered(DB_CONCURRENCY)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
futures::stream::iter(more_albums.into_iter().flatten())
|
||||
.for_each_concurrent(DB_CONCURRENCY, |album| async move {
|
||||
match self.import_yt_album(&album.id).await {
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
log::error!("could not import album [yt:{}]: {}", album.id, e);
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -333,7 +335,6 @@ impl Extractor {
|
|||
&self,
|
||||
track: rpmodel::TrackItem,
|
||||
album_id: Option<i32>,
|
||||
album_artists: &[ArtistIdName],
|
||||
) -> Result<i32, ExtractorError> {
|
||||
if album_id.is_none() {
|
||||
if let Some(id) =
|
||||
|
|
@ -343,9 +344,7 @@ impl Extractor {
|
|||
}
|
||||
}
|
||||
|
||||
let (artists, ul_artists) = self
|
||||
.split_yt_artists(track.artists, track.artist_id, album_artists)
|
||||
.await;
|
||||
let (artists, ul_artists) = util::split_yt_artists(track.artists);
|
||||
|
||||
let album_id = match album_id {
|
||||
Some(album_id) => album_id,
|
||||
|
|
@ -389,7 +388,8 @@ impl Extractor {
|
|||
src_id: &track.id,
|
||||
service: MusicService::YouTube,
|
||||
name: &track.name,
|
||||
duration: track.duration.and_then(|v| v.try_into().ok()),
|
||||
duration: track.duration.and_then(|v| (v * 1000).try_into().ok()),
|
||||
duration_ms: false,
|
||||
album_id,
|
||||
album_pos: track.track_nr.and_then(|v| v.try_into().ok()),
|
||||
ul_artists: ul_artists.as_deref(),
|
||||
|
|
@ -400,42 +400,39 @@ impl Extractor {
|
|||
Track::set_artists(track_id, &artist_ids, &mut tx).await?;
|
||||
tx.commit().await?;
|
||||
|
||||
tracing::debug!("imported track [yt:{}] {}", track.id, track.name);
|
||||
Ok(track_id)
|
||||
}
|
||||
|
||||
/// Import a list of YT Music artist ids
|
||||
async fn import_yt_artist_ids(
|
||||
&self,
|
||||
artists: &[ArtistIdName],
|
||||
) -> Result<Vec<i32>, ExtractorError> {
|
||||
) -> Result<Vec<Id<'_>>, ExtractorError> {
|
||||
futures::stream::iter(artists)
|
||||
.map(|aid| async { self.import_yt_artist_id(aid).await })
|
||||
.map(|aid| async move { self.import_yt_artist_id(aid).await.map(Id::Db) })
|
||||
.buffered(CONCURRENCY)
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.map_err(ExtractorError::from)
|
||||
}
|
||||
|
||||
/// Import a YT Music artist id (artist with ID and name)
|
||||
async fn import_yt_artist_id(&self, aid: &ArtistIdName) -> Result<i32, ExtractorError> {
|
||||
let id_owned = SrcIdOwned(aid.id.to_owned(), MusicService::YouTube);
|
||||
self.artist_cache
|
||||
.get_or_insert_async(&id_owned, async {
|
||||
.get_or_insert_async(&id_owned, async move {
|
||||
let artist = ArtistNew {
|
||||
src_id: &aid.id,
|
||||
service: MusicService::YouTube,
|
||||
name: &aid.name,
|
||||
..Default::default()
|
||||
};
|
||||
let artist_id = artist.upsert_recessive(&self.db).await?;
|
||||
tracing::debug!("imported artist id [yt:{}] {}", aid.id, aid.name);
|
||||
Ok(artist_id)
|
||||
artist
|
||||
.upsert_recessive(&self.db)
|
||||
.await
|
||||
.map_err(ExtractorError::from)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Import a YT Music artist item (artist with name, image and subscriber count)
|
||||
async fn import_yt_artist_item(
|
||||
&self,
|
||||
artist: rpmodel::ArtistItem,
|
||||
|
|
@ -452,13 +449,11 @@ impl Extractor {
|
|||
..Default::default()
|
||||
};
|
||||
let artist_id = n_artist.upsert_recessive(&self.db).await?;
|
||||
tracing::debug!("imported artist item [yt:{}] {}", artist.id, artist.name);
|
||||
self.artist_cache
|
||||
.insert(SrcIdOwned(artist.id, MusicService::YouTube), artist_id);
|
||||
Ok(artist_id)
|
||||
}
|
||||
|
||||
/// Import a YT Music album item (album from artist page)
|
||||
async fn import_yt_album_item(&self, album: rpmodel::AlbumItem) -> Result<(), ExtractorError> {
|
||||
// Return if the album was already imported
|
||||
if Album::get_id_clean(SrcId(&album.id, MusicService::YouTube), &self.db)
|
||||
|
|
@ -468,9 +463,7 @@ impl Extractor {
|
|||
return Ok(());
|
||||
}
|
||||
|
||||
let (artists, ul_artists) = self
|
||||
.split_yt_artists(album.artists, album.artist_id, &[])
|
||||
.await;
|
||||
let (artists, ul_artists) = util::split_yt_artists(album.artists);
|
||||
let artist_ids = self.import_yt_artist_ids(&artists).await?;
|
||||
|
||||
let image_url = util::get_image_url(&album.cover, false);
|
||||
|
|
@ -493,11 +486,9 @@ impl Extractor {
|
|||
Album::add_artists(album_id, &artist_ids, &mut tx).await?;
|
||||
tx.commit().await?;
|
||||
|
||||
tracing::debug!("imported album item [yt:{}] {}", album.id, album.name);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Import a YT Music playlist item
|
||||
async fn import_yt_playlist_item(
|
||||
&self,
|
||||
pl: rpmodel::MusicPlaylistItem,
|
||||
|
|
@ -528,46 +519,30 @@ impl Extractor {
|
|||
image_type: Some(PlaylistImgType::Custom),
|
||||
..Default::default()
|
||||
};
|
||||
let playlist_id = playlist_n.upsert(&self.db).await?;
|
||||
|
||||
tracing::debug!("imported playlist item [yt:{}] {}", pl.id, pl.name);
|
||||
Ok(playlist_id)
|
||||
playlist_n
|
||||
.upsert(&self.db)
|
||||
.await
|
||||
.map_err(ExtractorError::from)
|
||||
}
|
||||
|
||||
/// Fetch and import a YT Music album and return its ID and whether the album contains variants
|
||||
async fn fetch_yt_album(&self, id: &str) -> Result<(i32, bool), ExtractorError> {
|
||||
// Return if the album was already imported
|
||||
if let Some(album_id) =
|
||||
Album::get_id_clean(SrcId(id, MusicService::YouTube), &self.db).await?
|
||||
{
|
||||
return Ok((album_id, false));
|
||||
}
|
||||
|
||||
async fn import_yt_album(&self, id: &str) -> Result<Vec<rpmodel::AlbumItem>, ExtractorError> {
|
||||
let album = self.rp.query().music_album(id).await?;
|
||||
|
||||
let (artists, ul_artists) = self
|
||||
.split_yt_artists(album.artists, album.artist_id, &[])
|
||||
.await;
|
||||
let (artists, ul_artists) = util::split_yt_artists(album.artists);
|
||||
let image_url = util::get_image_url(&album.cover, false);
|
||||
|
||||
let artist_ids = self.import_yt_artist_ids(&artists).await?;
|
||||
|
||||
// Get album hash
|
||||
let mut hasher = util::AlbumHasher::new();
|
||||
let mut hasher = SipHasher::new_with_key(&hex!("e0060fd1ea207d8f43d2bf9bcae63f65"));
|
||||
for track in &album.tracks {
|
||||
hasher.add_track(&track.name, track.duration.unwrap_or_default());
|
||||
hasher.write(track.name.as_bytes());
|
||||
hasher.write_u32(track.duration.unwrap_or_default());
|
||||
}
|
||||
let album_hash = hasher.finish();
|
||||
|
||||
let hidden = if let Some(id) = artist_ids.first() {
|
||||
!Album::ids_from_hash(*id, &album_hash, &self.db)
|
||||
.await?
|
||||
.is_empty()
|
||||
} else {
|
||||
false
|
||||
};
|
||||
let album_hash = hasher.finish128().as_bytes();
|
||||
|
||||
// Insert album
|
||||
// TODO: accurate release date
|
||||
let album_n = AlbumNew {
|
||||
src_id: &album.id,
|
||||
service: MusicService::YouTube,
|
||||
|
|
@ -580,7 +555,6 @@ impl Extractor {
|
|||
ul_artists: ul_artists.as_deref(),
|
||||
by_va: album.by_va,
|
||||
image_url: image_url.as_deref(),
|
||||
hidden,
|
||||
album_hash: Some(&album_hash),
|
||||
..Default::default()
|
||||
};
|
||||
|
|
@ -589,176 +563,18 @@ impl Extractor {
|
|||
Album::set_artists(album_id, &artist_ids, &mut tx).await?;
|
||||
tx.commit().await?;
|
||||
|
||||
let first_track_id = album.tracks.first().map(|t| t.id.to_owned());
|
||||
|
||||
// Insert tracks
|
||||
futures::stream::iter(album.tracks.into_iter().map(Ok::<_, ExtractorError>))
|
||||
.try_for_each_concurrent(DB_CONCURRENCY, |track| async {
|
||||
self.import_yt_track(track, Some(album_id), &artists)
|
||||
.await?;
|
||||
.try_for_each_concurrent(DB_CONCURRENCY, |track| async move {
|
||||
self.import_yt_track(track, Some(album_id)).await?;
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Insert album variants
|
||||
let has_variants = !album.variants.is_empty();
|
||||
futures::stream::iter(album.variants.into_iter().map(Ok::<_, ExtractorError>))
|
||||
.try_for_each_concurrent(DB_CONCURRENCY, |album| async {
|
||||
self.import_yt_album_item(album).await
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Fetch more accurate album date
|
||||
if let Some(first_track_id) = first_track_id {
|
||||
self.fetch_track_details(album_id, album.year, &first_track_id, true)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// Mark album clean
|
||||
Album::mark_dirty(album_id, false, &self.db).await?;
|
||||
|
||||
tracing::info!("imported album [yt:{}] {}", album.id, album.name);
|
||||
Ok((album_id, has_variants))
|
||||
}
|
||||
|
||||
/// Fetch the description text and upload date of a YT video and store it in the database.
|
||||
async fn fetch_track_details(
|
||||
&self,
|
||||
album_id: i32,
|
||||
release_year: Option<u16>,
|
||||
yt_id: &str,
|
||||
ytm_track: bool,
|
||||
) -> Result<(), ExtractorError> {
|
||||
let details = self.rp.query().video_details(yt_id).await?;
|
||||
let description = details.description.to_plaintext();
|
||||
|
||||
let release_date_detail = if ytm_track {
|
||||
util::extract_yt_release_date(&description, details.publish_date)
|
||||
} else {
|
||||
details.publish_date.map(|d| (d.date(), DatePrecision::Day))
|
||||
};
|
||||
|
||||
let release_date = release_date_detail.filter(|d| match release_year {
|
||||
Some(year) => d.0.year() == i32::from(year),
|
||||
None => true,
|
||||
});
|
||||
|
||||
let t_upd = TrackUpdate {
|
||||
description: Some(Some(&description)),
|
||||
..Default::default()
|
||||
};
|
||||
t_upd
|
||||
.update(Id::Src(yt_id, MusicService::YouTube), &self.db)
|
||||
.await?;
|
||||
|
||||
if let Some(release_date) = release_date {
|
||||
let b_upd = AlbumUpdate {
|
||||
release_date: Some(Some(release_date.0)),
|
||||
release_date_precision: Some(Some(release_date.1)),
|
||||
..Default::default()
|
||||
};
|
||||
b_upd.update(Id::Db(album_id), &self.db).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get the name of a YouTube artist with the given ID (either from DB or the RSS feed)
|
||||
async fn get_yt_artist_name(&self, id: &str) -> Result<String, ExtractorError> {
|
||||
let artist = Artist::get(Id::Src(id, MusicService::YouTube), &self.db)
|
||||
.await
|
||||
.to_optional()?;
|
||||
if let Some(artist) = artist {
|
||||
Ok(artist.name)
|
||||
} else {
|
||||
let feed = self.rp.query().channel_rss(id).await?;
|
||||
let name = feed.name.strip_suffix(" - Topic").unwrap_or(&feed.name);
|
||||
self.import_yt_artist_id(&ArtistIdName {
|
||||
id: id.to_owned(),
|
||||
name: name.to_owned(),
|
||||
})
|
||||
.await?;
|
||||
Ok(name.to_owned())
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert track/album artists scraped from YouTube into the format used by Tiraya
|
||||
///
|
||||
/// YouTube Music's metadata is incomplete and there are a lot of tracks with
|
||||
/// unlinked artists (the user is only shown a list of artists that are not
|
||||
/// clickable links).
|
||||
///
|
||||
/// The YouTube interface also includes the "Go to artist" button in the dropdown
|
||||
/// menu which may be present even if there are only unlinked artists shown.
|
||||
///
|
||||
/// So we can apply a few tricks to improve the metadata:
|
||||
///
|
||||
/// - If the artist ID from the dropdown is not included in the list of artists,
|
||||
/// fetch its name and add it to the beginning
|
||||
/// - Add the album artists (if given) to the list of artists if they appear in the
|
||||
/// list of unlinked artists
|
||||
///
|
||||
/// Example albums for testing:
|
||||
/// - https://music.youtube.com/browse/MPREb_GXN2zsSMUKJ
|
||||
/// - https://music.youtube.com/browse/MPREb_98weK02o4DO
|
||||
/// - https://music.youtube.com/browse/MPREb_1pFivcTlTls
|
||||
async fn split_yt_artists(
|
||||
&self,
|
||||
artists: Vec<rpmodel::ArtistId>,
|
||||
artist_id: Option<String>,
|
||||
album_artists: &[ArtistIdName],
|
||||
) -> (Vec<ArtistIdName>, Option<Vec<String>>) {
|
||||
let mut ul_artists = Vec::new();
|
||||
let mut add_aid = true;
|
||||
|
||||
let mut artists = artists
|
||||
.into_iter()
|
||||
.filter_map(|a| match a.id {
|
||||
Some(id) => {
|
||||
if Some(&id) == artist_id.as_ref() {
|
||||
add_aid = false;
|
||||
}
|
||||
Some(ArtistIdName { id, name: a.name })
|
||||
}
|
||||
None => {
|
||||
ul_artists.push(a.name);
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
// Artist from artist_id field is not contained in artists list
|
||||
if add_aid {
|
||||
if let Some(artist_id) = artist_id {
|
||||
if let Ok(name) = self.get_yt_artist_name(&artist_id).await {
|
||||
util::extract_ul_artist(&mut ul_artists, &name);
|
||||
artists.insert(
|
||||
0,
|
||||
ArtistIdName {
|
||||
id: artist_id,
|
||||
name,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try to match album artists(s)
|
||||
if !ul_artists.is_empty() {
|
||||
for aa in album_artists {
|
||||
if util::extract_ul_artist(&mut ul_artists, &aa.name) {
|
||||
artists.insert(0, aa.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(
|
||||
artists,
|
||||
if ul_artists.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(ul_artists)
|
||||
},
|
||||
)
|
||||
Ok(album.variants)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -777,7 +593,7 @@ mod tests {
|
|||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
// #[test_tracing::test]
|
||||
// #[test_log::test]
|
||||
async fn import_album() {
|
||||
sqlx_database_tester::dotenv::dotenv().unwrap();
|
||||
let url = std::env::var("DATABASE_URL").unwrap();
|
||||
|
|
@ -790,7 +606,7 @@ mod tests {
|
|||
.unwrap();
|
||||
if artist_res.fetched {
|
||||
extractor
|
||||
.fetch_yt_artist_albums(artist_res.c.id)
|
||||
.update_yt_artist_albums(artist_res.c.id)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,19 +1,39 @@
|
|||
use std::{borrow::Cow, hash::Hasher};
|
||||
use std::borrow::Cow;
|
||||
|
||||
use hex_literal::hex;
|
||||
use once_cell::sync::Lazy;
|
||||
use regex::Regex;
|
||||
use rustypipe::model as rpmodel;
|
||||
use siphasher::sip128::{Hasher128, SipHasher};
|
||||
use time::{Date, Duration, OffsetDateTime};
|
||||
use tiraya_db::models::{AlbumType, DatePrecision, SyncData, SyncError};
|
||||
use tiraya_db::models::{AlbumType, SyncData, SyncError};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ArtistIdName {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
pub fn split_yt_artists(
|
||||
artists: Vec<rpmodel::ArtistId>,
|
||||
) -> (Vec<ArtistIdName>, Option<Vec<String>>) {
|
||||
let mut ul_artists = Vec::new();
|
||||
let artists = artists
|
||||
.into_iter()
|
||||
.filter_map(|a| match a.id {
|
||||
Some(id) => Some(ArtistIdName { id, name: a.name }),
|
||||
None => {
|
||||
ul_artists.push(a.name);
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
(
|
||||
artists,
|
||||
if ul_artists.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(ul_artists)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
static YTM_IMAGE_REGEX: Lazy<Regex> =
|
||||
Lazy::new(|| Regex::new(r"^https://[a-z\d]+.googleusercontent.com/[\w\d_/-]+").unwrap());
|
||||
|
||||
|
|
@ -67,96 +87,11 @@ pub fn map_album_type(album_type: rpmodel::AlbumType) -> AlbumType {
|
|||
}
|
||||
}
|
||||
|
||||
static RELEASE_DATE_REGEX: Lazy<Regex> =
|
||||
Lazy::new(|| Regex::new(r"Released on: (\d{4}-\d{2}-\d{2})").unwrap());
|
||||
const YMD_FORMAT: &[time::format_description::FormatItem] =
|
||||
time::macros::format_description!("[year]-[month]-[day]");
|
||||
|
||||
pub fn extract_yt_release_date(
|
||||
description: &str,
|
||||
upload_date: Option<OffsetDateTime>,
|
||||
) -> Option<(Date, DatePrecision)> {
|
||||
RELEASE_DATE_REGEX
|
||||
.captures(description)
|
||||
.and_then(|cap| {
|
||||
let raw_date = &cap[1];
|
||||
Date::parse(raw_date, YMD_FORMAT).ok()
|
||||
})
|
||||
.map(|release_date| {
|
||||
if let Some(upload_date) = upload_date {
|
||||
// Prefer the video upload date if it lies within 4 days of the release date
|
||||
let upload_date = upload_date.date();
|
||||
let diff = (upload_date - release_date).abs();
|
||||
if diff < Duration::days(4) {
|
||||
return (upload_date, DatePrecision::Day);
|
||||
}
|
||||
}
|
||||
(
|
||||
release_date,
|
||||
// If a date component is unknown, YT will show a 1 in that place
|
||||
if release_date.day() == 1 {
|
||||
if release_date.month() == time::Month::January {
|
||||
DatePrecision::Year
|
||||
} else {
|
||||
DatePrecision::Month
|
||||
}
|
||||
} else {
|
||||
DatePrecision::Day
|
||||
},
|
||||
)
|
||||
})
|
||||
.or_else(|| upload_date.map(|d| (d.date(), DatePrecision::Day)))
|
||||
}
|
||||
|
||||
/// Try to extract the given artist name from a list of unlinked artists
|
||||
///
|
||||
/// Returns true if the artist was extracted
|
||||
pub fn extract_ul_artist(ul_artists: &mut [String], name: &str) -> bool {
|
||||
// TODO: add support for other languages
|
||||
static END_RE: Lazy<Regex> = Lazy::new(|| Regex::new(",? *(and|&)? *$").unwrap());
|
||||
static START_RE: Lazy<Regex> = Lazy::new(|| Regex::new("^,? *(and|&)? *").unwrap());
|
||||
|
||||
for ul_a in ul_artists.iter_mut() {
|
||||
if let Some((a, b)) = ul_a.split_once(name) {
|
||||
// Trim end of first part
|
||||
if a.is_empty() {
|
||||
*ul_a = START_RE.replace(b, "").to_string()
|
||||
} else {
|
||||
*ul_a = END_RE.replace(a, "").to_string() + b;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub struct AlbumHasher(SipHasher);
|
||||
|
||||
impl AlbumHasher {
|
||||
pub fn new() -> Self {
|
||||
Self(SipHasher::new_with_key(&hex!(
|
||||
"e0060fd1ea207d8f43d2bf9bcae63f65"
|
||||
)))
|
||||
}
|
||||
|
||||
pub fn add_track(&mut self, name: &str, duration: u32) {
|
||||
self.0.write(name.as_bytes());
|
||||
self.0.write_u32(duration);
|
||||
}
|
||||
|
||||
pub fn finish(&self) -> [u8; 16] {
|
||||
self.0.finish128().as_bytes()
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
use rstest::rstest;
|
||||
use time::macros::{date, datetime};
|
||||
|
||||
/*
|
||||
fn get_image_url_album() {
|
||||
let images = [rpmodel::Thumbnail {
|
||||
url: "https://lh3.googleusercontent.com/46FajCeiWBIdYQmkVkDvNVpprs-ihk9hVulFc8v-pUPEfgUU2uzGmc45vO-sZCYQiEasgo21cbengDYIAQ=w226-h226-l90-rj",
|
||||
|
|
@ -164,57 +99,5 @@ mod tests {
|
|||
height: 226,
|
||||
}];
|
||||
}
|
||||
*/
|
||||
|
||||
#[rstest]
|
||||
#[case("Released on: 2016-04-22", Some(datetime!(2016-05-10 0:0 UTC)), Some((date!(2016-04-22), DatePrecision::Day)))]
|
||||
#[case("", Some(datetime!(2016-05-10 0:0 UTC)), Some((date!(2016-05-10), DatePrecision::Day)))]
|
||||
#[case("Released on: 2016-04-22", Some(datetime!(2016-4-20 0:0 UTC)), Some((date!(2016-04-20), DatePrecision::Day)))]
|
||||
#[case("Released on: 2016-04-20", Some(datetime!(2016-4-22 0:0 UTC)), Some((date!(2016-04-22), DatePrecision::Day)))]
|
||||
#[case("Released on: 2016-04-01", Some(datetime!(2016-4-22 0:0 UTC)), Some((date!(2016-04-01), DatePrecision::Month)))]
|
||||
#[case("Released on: 2016-01-01", Some(datetime!(2016-4-22 0:0 UTC)), Some((date!(2016-01-01), DatePrecision::Year)))]
|
||||
#[case("Released on: 2016-04-20", None, Some((date!(2016-04-20), DatePrecision::Day)))]
|
||||
#[case("", None, None)]
|
||||
fn t_extract_yt_release_date(
|
||||
#[case] description: &str,
|
||||
#[case] upload_date: Option<OffsetDateTime>,
|
||||
#[case] expect: Option<(Date, DatePrecision)>,
|
||||
) {
|
||||
let res = extract_yt_release_date(description, upload_date);
|
||||
assert_eq!(res, expect);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(&["Aurelio Fierro, Furio Rendine, Vincenzo de Crescenzo"], "Aurelio Fierro", &["Furio Rendine, Vincenzo de Crescenzo"])]
|
||||
#[case(&["Miley Cyrus, Swae Lee, & Mike WiLL Made-It"], "Swae Lee", &["Miley Cyrus, & Mike WiLL Made-It"])]
|
||||
#[case(&["Miley Cyrus, Swae Lee, & Mike WiLL Made-It"], "Mike WiLL Made-It", &["Miley Cyrus, Swae Lee"])]
|
||||
#[case(&["Miley Cyrus, Swae Lee, & Mike WiLL Made-It"], "foobar", &["Miley Cyrus, Swae Lee, & Mike WiLL Made-It"])]
|
||||
#[case(&["Miley Cyrus"], "Miley Cyrus", &[""])]
|
||||
fn t_extract_ul_artist(
|
||||
#[case] ul_artists: &[&str],
|
||||
#[case] name: &str,
|
||||
#[case] expect: &[&str],
|
||||
) {
|
||||
let mut art = ul_artists.iter().map(|s| s.to_string()).collect::<Vec<_>>();
|
||||
assert_eq!(extract_ul_artist(&mut art, name), ul_artists != expect);
|
||||
assert_eq!(art, expect);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(&[("Lieblingsmensch", 190)], hex!("290deeaedd84dbdb71d2f62979eeedc8"))]
|
||||
#[case(&[
|
||||
("Ich wache auf", 221),
|
||||
("Waldbrand", 208),
|
||||
("Verlernt", 223),
|
||||
("In Farbe", 221),
|
||||
("Stadt im Hinterland", 197)
|
||||
], hex!("ffbd4cad41df8e99f3d0a3d629f5a5d5"))]
|
||||
fn album_hash(#[case] tracks: &[(&str, u32)], #[case] expect: [u8; 16]) {
|
||||
let mut hasher = AlbumHasher::new();
|
||||
for t in tracks {
|
||||
hasher.add_track(t.0, t.1);
|
||||
}
|
||||
let hash = hasher.finish();
|
||||
assert_eq!(hash, expect, "got {hash:x?}");
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -17,4 +17,4 @@ rustypipe.workspace = true
|
|||
sqlx.workspace = true
|
||||
dotenvy.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
env_logger.workspace = true
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ use tiraya_extractor::Extractor;
|
|||
#[tokio::main]
|
||||
async fn main() {
|
||||
dotenvy::dotenv().unwrap();
|
||||
tracing_subscriber::fmt::init();
|
||||
env_logger::init();
|
||||
|
||||
let url = std::env::var("DATABASE_URL").unwrap();
|
||||
let pool = PgPool::connect(&url).await.unwrap();
|
||||
|
|
@ -20,7 +20,7 @@ async fn main() {
|
|||
.await
|
||||
.unwrap();
|
||||
if artist_res.fetched {
|
||||
ext.fetch_yt_artist_albums(artist_res.c.id).await.unwrap();
|
||||
ext.update_yt_artist_albums(artist_res.c.id).await.unwrap();
|
||||
}
|
||||
|
||||
dbg!(artist_res.c);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue