Compare commits

..

8 commits

88 changed files with 701 additions and 47444 deletions

View file

@ -28,22 +28,17 @@ jobs:
run: |
TARGET=$(rustc --version --verbose | grep "host:" | sed -e 's/^host: //')
cd ~
curl -SsL -o rustypipe-botguard.tar.xz "https://codeberg.org/ThetaDev/rustypipe-botguard/releases/download/v0.1.1/rustypipe-botguard-v0.1.1-${TARGET}.tar.xz"
curl -SsL -o rustypipe-botguard.tar.xz "https://codeberg.org/ThetaDev/rustypipe-botguard/releases/download/v0.1.0/rustypipe-botguard-v0.1.0-${TARGET}.tar.xz"
cd /usr/local/bin
sudo tar -xJf ~/rustypipe-botguard.tar.xz
rm ~/rustypipe-botguard.tar.xz
rustypipe-botguard --version
- name: 📎 Clippy
run: |
cargo clippy --all --tests --features=rss,userdata,indicatif,audiotag -- -D warnings
cargo clippy --package=rustypipe --tests -- -D warnings
cargo clippy --package=rustypipe-downloader -- -D warnings
cargo clippy --package=rustypipe-cli -- -D warnings
cargo clippy --package=rustypipe-cli --features=timezone -- -D warnings
run: cargo clippy --all --tests --features=rss,indicatif,audiotag -- -D warnings
- name: 🧪 Test
run: cargo nextest run --config-file ~/.config/nextest.toml --profile ci --retries 2 --features rss,userdata --workspace -- --skip 'user_data::'
run: cargo nextest run --config-file ~/.config/nextest.toml --profile ci --retries 2 --features rss --workspace -- --skip 'cookie_auth::'
env:
ALL_PROXY: "http://warpproxy:8124"

View file

@ -1,69 +0,0 @@
name: Release CLI
on:
push:
tags:
- "rustypipe-cli/v*.*.*"
jobs:
Release:
runs-on: cimaster-latest
steps:
- name: 📦 Checkout repository
uses: actions/checkout@v4
- name: Setup cross compilation
run: |
rustup target add x86_64-pc-windows-msvc x86_64-apple-darwin aarch64-apple-darwin
cargo install cargo-xwin
# https://wapl.es/rust/2019/02/17/rust-cross-compile-linux-to-macos.html/
sudo apt-get install -y llvm clang cmake
cd ~
git clone https://github.com/tpoechtrager/osxcross
cd osxcross
wget -nc "https://github.com/joseluisq/macosx-sdks/releases/download/12.3/MacOSX12.3.sdk.tar.xz"
mv MacOSX12.3.sdk.tar.xz tarballs/
UNATTENDED=yes OSX_VERSION_MIN=12.3 ./build.sh
OSXCROSS_BIN="$(pwd)/target/bin"
echo "CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=$(find "$OSXCROSS_BIN" -name "x86_64-apple-darwin*-clang")" >> $GITHUB_ENV
echo "CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS=-Car=$(find "$OSXCROSS_BIN" -name "x86_64-apple-darwin*-ar"),-Clink-arg=-undefined,-Clink-arg=dynamic_lookup" >> $GITHUB_ENV
echo "CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=$(find "$OSXCROSS_BIN" -name "aarch64-apple-darwin*-clang")" >> $GITHUB_ENV
echo "CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS=-Car=$(find "$OSXCROSS_BIN" -name "aarch64-apple-darwin*-ar"),-Clink-arg=-undefined,-Clink-arg=dynamic_lookup" >> $GITHUB_ENV
- name: ⚒️ Build application
run: |
export PATH="$PATH:$HOME/osxcross/target/bin"
CRATE="rustypipe-cli"
PKG_CONFIG_SYSROOT_DIR=/usr/x86_64-linux-gnu cargo build --release --package=$CRATE --target x86_64-unknown-linux-gnu
PKG_CONFIG_SYSROOT_DIR=/usr/aarch64-linux-gnu cargo build --release --package=$CRATE --target aarch64-unknown-linux-gnu
CC="$CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER" CXX="$CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER++" cargo build --release --package=$CRATE --target x86_64-apple-darwin
CC="$CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER" CXX="$CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER++" cargo build --release --package=$CRATE --target aarch64-apple-darwin
cargo xwin build --release --package=$CRATE --target x86_64-pc-windows-msvc
- name: Prepare release
run: |
CRATE="rustypipe-cli"
BIN="rustypipe"
echo "CRATE=$CRATE" >> "$GITHUB_ENV"
echo "CRATE_VERSION=$(echo '${{ github.ref_name }}' | awk 'BEGIN{RS="/"} NR==2{print}')" >> "$GITHUB_ENV"
CL_PATH="cli/CHANGELOG.md"
{
echo 'CHANGELOG<<END_OF_FILE'
awk 'BEGIN{RS="(^|\n)## [^\n]+\n*"} NR==2 { print }' "$CL_PATH"
echo END_OF_FILE
} >> "$GITHUB_ENV"
mkdir dist
for arch in x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu x86_64-apple-darwin aarch64-apple-darwin; do
tar -cJf "dist/${BIN}-${CRATE_VERSION}-${arch}.tar.xz" -C target/${arch}/release "${BIN}"
done
(cd target/x86_64-pc-windows-msvc/release && zip -9 "../../../dist/${BIN}-${CRATE_VERSION}-x86_64-pc-windows-msvc.zip" "${BIN}.exe")
- name: 🎉 Publish release
uses: https://gitea.com/actions/release-action@main
with:
title: "${{ env.CRATE }} ${{ env.CRATE_VERSION }}"
body: "${{ env.CHANGELOG }}"
files: dist/*

View file

@ -10,8 +10,4 @@ repos:
hooks:
- id: cargo-fmt
- id: cargo-clippy
name: cargo-clippy rustypipe
args: ["--package=rustypipe", "--tests", "--", "-D", "warnings"]
- id: cargo-clippy
name: cargo-clippy workspace
args: ["--all", "--tests", "--features=rss,userdata,indicatif,audiotag", "--", "-D", "warnings"]
args: ["--all", "--tests", "--features=rss,indicatif,audiotag", "--", "-D", "warnings"]

10
.woodpecker.yml Normal file
View file

@ -0,0 +1,10 @@
steps:
test:
image: rust:latest
environment:
- CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
commands:
- rustup component add rustfmt clippy
- cargo fmt --all --check
- cargo clippy --all --features=rss -- -D warnings
- cargo test --features=rss --workspace

View file

@ -3,93 +3,6 @@
All notable changes to this project will be documented in this file.
## [v0.11.3](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.11.2..rustypipe/v0.11.3) - 2025-04-03
### 🐛 Bug Fixes
- Deobfuscator: global variable extraction fixed - ([ac44e95](https://codeberg.org/ThetaDev/rustypipe/commit/ac44e95a88d95f9d2d1ec672f86ca9d31d6991b9))
- Deobfuscator: small simplification - ([189ba81](https://codeberg.org/ThetaDev/rustypipe/commit/189ba81a42e6c09f6af4d2768c449c22b864101e))
- Deobfuscator: handle global functions as well - ([939a7ae](https://codeberg.org/ThetaDev/rustypipe/commit/939a7aea61a3eee4c1e67bfbfc835f0ce3934171))
- Handle music playlist/album not found - ([ea80717](https://codeberg.org/ThetaDev/rustypipe/commit/ea80717f692b2c45b5063c362c9fa8ebca5a3471))
- Switch client if no adaptive stream URLs were returned - ([187bf1c](https://codeberg.org/ThetaDev/rustypipe/commit/187bf1c9a0e846bff205e0d71a19c5a1ce7b1943))
- Handle music artist not found - ([daf3d03](https://codeberg.org/ThetaDev/rustypipe/commit/daf3d035be38b59aef1ae205ac91c2bbdda2fe66))
### ⚙️ Miscellaneous Tasks
- *(deps)* Update rust crate rand to 0.9.0 - ([af415dd](https://codeberg.org/ThetaDev/rustypipe/commit/af415ddf8f94f00edb918f271d8e6336503e9faf))
## [v0.11.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.11.1..rustypipe/v0.11.2) - 2025-03-24
### 🐛 Bug Fixes
- A/B test 22: commandExecutorCommand for playlist continuations - ([e8acbfb](https://codeberg.org/ThetaDev/rustypipe/commit/e8acbfbbcf5d31b5ac34410ddf334e5534e3762f))
- Extract deobf data with global strings variable - ([4ce6746](https://codeberg.org/ThetaDev/rustypipe/commit/4ce6746be538564e79f7e3c67d7a91aaa53f48ea))
- Handle player returning no adaptive stream URLs - ([07db7b1](https://codeberg.org/ThetaDev/rustypipe/commit/07db7b1166e912e1554f98f2ae20c2c356fed38f))
## [v0.11.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.11.0..rustypipe/v0.11.1) - 2025-03-16
### 🐛 Bug Fixes
- Simplify get_player_from_clients logic - ([c04b606](https://codeberg.org/ThetaDev/rustypipe/commit/c04b60604d2628bf8f0e3de453c243adbb966e57))
- Desktop client: generate PO token from user_syncid when authenticated - ([8342cae](https://codeberg.org/ThetaDev/rustypipe/commit/8342caeb0f566a38060a6ec69f3ca65b9a2afcd6))
- Always skip failed clients - ([63a6f50](https://codeberg.org/ThetaDev/rustypipe/commit/63a6f50a8b5ad6bb984282335c1481ae3cd2fe83))
### ⚙️ Miscellaneous Tasks
- *(deps)* Update rust crate rstest to 0.25.0 - ([9ed1306](https://codeberg.org/ThetaDev/rustypipe/commit/9ed1306f3aaeb993c409997ddfbc47499e4f4d22))
## [v0.11.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.10.0..rustypipe/v0.11.0) - 2025-02-26
### 🚀 Features
- Add original album track count, fix fetching albums with more than 200 tracks - ([544782f](https://codeberg.org/ThetaDev/rustypipe/commit/544782f8de728cda0aca9a1cb95837cdfbd001f1))
### 🐛 Bug Fixes
- A/B test 21: music album recommendations - ([6737512](https://codeberg.org/ThetaDev/rustypipe/commit/6737512f5f67c8cd05d4552dd0e0f24381035b35))
## [v0.10.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.9.0..rustypipe/v0.10.0) - 2025-02-09
### 🚀 Features
- Add visitor data cache, remove random visitor data - ([b12f4c5](https://codeberg.org/ThetaDev/rustypipe/commit/b12f4c5d821a9189d7ed8410ad860824b6d052ef))
- Add support for rustypipe-botguard to get PO tokens - ([b90a252](https://codeberg.org/ThetaDev/rustypipe/commit/b90a252a5e1bf05a5294168b0ec16a73cbb88f42))
- Add session po token cache - ([b72b501](https://codeberg.org/ThetaDev/rustypipe/commit/b72b501b6dbcf4333b24cd80e7c8c61b0c21ec91))
- Check rustypipe-botguard-api version - ([8385b87](https://codeberg.org/ThetaDev/rustypipe/commit/8385b87c63677f32a240679a78702f53072e517a))
- Rewrite request attempt system, retry with different visitor data - ([dfd03ed](https://codeberg.org/ThetaDev/rustypipe/commit/dfd03edfadff2657e9cfbf04e5d313ba409520ac))
- Log failed player fetch attempts with player_from_clients - ([8e35358](https://codeberg.org/ThetaDev/rustypipe/commit/8e35358c8941301f6ebf7646a11ab22711082569))
- Add timezone query option - ([3a2370b](https://codeberg.org/ThetaDev/rustypipe/commit/3a2370b97ca3d0f40d72d66a23295557317d29fb))
- [**breaking**] Add userdata feature for all personal data queries (playback history, subscriptions) - ([65cb424](https://codeberg.org/ThetaDev/rustypipe/commit/65cb4244c6ab547f53d0cb12af802c4189188c86))
- Add RustyPipe::version_botguard fn, detect rustypipe-botguard in current dir, add botguard version to report - ([1d755b7](https://codeberg.org/ThetaDev/rustypipe/commit/1d755b76bf4569f7d0bb90a65494ac8e7aae499a))
### 🐛 Bug Fixes
- Parsing history dates - ([af7dc10](https://codeberg.org/ThetaDev/rustypipe/commit/af7dc1016322a87dd8fec0b739939c2b12b6f400))
- A/V streams incorrectly recognized as video-only - ([2b891ca](https://codeberg.org/ThetaDev/rustypipe/commit/2b891ca0788f91f16dbb9203191cb3d2092ecc74))
- Update iOS client - ([e915416](https://codeberg.org/ThetaDev/rustypipe/commit/e91541629d6c944c1001f5883e3c1264aeeb3969))
- A/B test 20: music continuation item renderer - ([9c67f8f](https://codeberg.org/ThetaDev/rustypipe/commit/9c67f8f85bef8214848dc9d17bff6cff252e015e))
- Include whole request body in report - ([15245c1](https://codeberg.org/ThetaDev/rustypipe/commit/15245c18b584e42523762b94fcc7284d483660a0))
- Extracting nsig fn when outside variable starts with $ - ([eda16e3](https://codeberg.org/ThetaDev/rustypipe/commit/eda16e378730a3b57c4982a626df1622a93c574a))
- Retry updating deobf data after a RustyPipe update - ([50ab1f7](https://codeberg.org/ThetaDev/rustypipe/commit/50ab1f7a5d8aeaa3720264b4a4b27805bb0e8121))
- Allow player data to be fetched without botguard - ([29c854b](https://codeberg.org/ThetaDev/rustypipe/commit/29c854b20d7a6677415b1744e7ba7ecd4f594ea5))
- Output full request body in reports, clean up `get_player_po_token` - ([a0d850f](https://codeberg.org/ThetaDev/rustypipe/commit/a0d850f8e01428a73bbd66397d0dbf797b45958f))
- Correct timezone offset for parsed dates, add timezone_local option - ([a5a7be5](https://codeberg.org/ThetaDev/rustypipe/commit/a5a7be5b4e0a0b73d7e1dc802ebd7bd48dafc76d))
- Use localzone crate to get local tz - ([5acbf0e](https://codeberg.org/ThetaDev/rustypipe/commit/5acbf0e456b1f10707e0a56125d993a8129eee3a))
- Only use cached potokens with min. 10min lifetime - ([0c94267](https://codeberg.org/ThetaDev/rustypipe/commit/0c94267d0371b2b26c7b5c9abfa156d5cde2153e))
### 📚 Documentation
- Add Botguard info to README - ([9957add](https://codeberg.org/ThetaDev/rustypipe/commit/9957add2b5d6391b2c1869d2019fd7dd91b8cd41))
### ⚙️ Miscellaneous Tasks
- *(deps)* Update rust crate rquickjs to 0.9.0 (#33) - ([2c8ac41](https://codeberg.org/ThetaDev/rustypipe/commit/2c8ac410aa535d83f8bcc7181f81914b13bceb77))
## [v0.9.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.8.0..rustypipe/v0.9.0) - 2025-01-16
### 🚀 Features

View file

@ -1,6 +1,6 @@
[package]
name = "rustypipe"
version = "0.11.3"
version = "0.9.0"
rust-version = "1.67.1"
edition.workspace = true
authors.workspace = true
@ -40,7 +40,7 @@ serde_with = { version = "3.0.0", default-features = false, features = [
] }
serde_plain = "1.0.0"
sha1 = "0.10.0"
rand = "0.9.0"
rand = "0.8.0"
time = { version = "0.3.37", features = [
"macros",
"serde-human-readable",
@ -67,15 +67,15 @@ dirs = "6.0.0"
filenamify = "0.1.0"
# Testing
rstest = "0.25.0"
rstest = "0.24.0"
tokio-test = "0.4.2"
insta = { version = "1.17.1", features = ["ron", "redactions"] }
path_macro = "1.0.0"
tracing-test = "0.2.5"
# Included crates
rustypipe = { path = ".", version = "0.11.3", default-features = false }
rustypipe-downloader = { path = "./downloader", version = "0.3.1", default-features = false, features = [
rustypipe = { path = ".", version = "0.9.0", default-features = false }
rustypipe-downloader = { path = "./downloader", version = "0.2.1", default-features = false, features = [
"indicatif",
"audiotag",
] }
@ -84,7 +84,6 @@ rustypipe-downloader = { path = "./downloader", version = "0.3.1", default-featu
default = ["default-tls"]
rss = ["dep:quick-xml"]
userdata = []
# Reqwest TLS options
default-tls = ["reqwest/default-tls"]
@ -127,6 +126,6 @@ tracing-test.workspace = true
[package.metadata.docs.rs]
# To build locally:
# RUSTDOCFLAGS="--cfg docsrs" cargo +nightly doc --features rss,userdata --no-deps --open
features = ["rss", "userdata"]
# RUSTDOCFLAGS="--cfg docsrs" cargo +nightly doc --features rss --no-deps --open
features = ["rss"]
rustdoc-args = ["--cfg", "docsrs"]

View file

@ -1,19 +1,19 @@
test:
# cargo test --features=rss,userdata
cargo nextest run --workspace --features=rss,userdata --no-fail-fast --retries 1 -- --skip 'user_data::'
# cargo test --features=rss
cargo nextest run --workspace --features=rss --no-fail-fast --retries 1 -- --skip 'cookie_auth::'
unittest:
cargo nextest run --features=rss,userdata --no-fail-fast --lib
cargo nextest run --features=rss --no-fail-fast --lib
testyt:
cargo nextest run --features=rss,userdata --no-fail-fast --retries 1 --test youtube -- --skip 'user_data::'
cargo nextest run --features=rss --no-fail-fast --retries 1 --test youtube -- --skip 'cookie_auth::'
testyt-cookie:
cargo nextest run --features=rss,userdata --no-fail-fast --retries 1 --test youtube
cargo nextest run --features=rss --no-fail-fast --retries 1 --test youtube
testyt-localized:
YT_LANG=th cargo nextest run --features=rss,userdata --no-fail-fast --retries 1 --test youtube -- \
--skip 'user_data::' --skip 'search_suggestion' --skip 'isrc_search_languages'
YT_LANG=th cargo nextest run --features=rss --no-fail-fast --retries 1 --test youtube -- \
--skip 'cookie_auth::' --skip 'search_suggestion' --skip 'isrc_search_languages'
testintl:
#!/usr/bin/env bash
@ -33,7 +33,7 @@ testintl:
echo "---TESTS FOR $YT_LANG ---"
if YT_LANG="$YT_LANG" cargo nextest run --no-fail-fast --retries 1 --test-threads 4 --test youtube -- \
--skip 'user_data::' --skip 'search_suggestion' --skip 'isrc_search_languages' --skip 'resolve_'; then
--skip 'cookie_auth::' --skip 'search_suggestion' --skip 'isrc_search_languages' --skip 'resolve_'; then
echo "--- $YT_LANG COMPLETED ---"
else
echo "--- $YT_LANG FAILED ---"

View file

@ -181,19 +181,6 @@ Subscribers: 1780000
...
```
## Crate features
Some features of RustyPipe are gated behind features to avoid compiling unneeded
dependencies.
- `rss` Fetch a channel's RSS feed, which is faster than fetching the channel page
- `userdata` Add functions to fetch YouTube user data (watch history, subscriptions,
music library)
You can also choose the TLS library used for making web requests using the same features
as the reqwest crate (`default-tls`, `native-tls`, `native-tls-alpn`,
`native-tls-vendored`, `rustls-tls-webpki-roots`, `rustls-tls-native-roots`).
## Cache storage
The RustyPipe cache holds the current version numbers for all clients, the JavaScript
@ -226,21 +213,6 @@ RustyPipe reports come in 3 severity levels:
incomplete)
- ERR (entire response could not be deserialized/parsed, RustyPipe returned an error)
## PO tokens
Since August 2024 YouTube requires PO tokens to access streams from web-based clients
(Desktop, Mobile). Otherwise streams will return a 403 error.
Generating PO tokens requires a simulated browser environment, which would be too large
to include in RustyPipe directly.
Therefore, the PO token generation is handled by a seperate CLI application
([rustypipe-botguard](https://codeberg.org/ThetaDev/rustypipe-botguard)) which is called
by the RustyPipe crate. RustyPipe automatically detects the rustypipe-botguard binary if
it is located in PATH or the current working directory. If your rustypipe-botguard
binary is located at a different path, you can specify it with the `.botguard_bin(path)`
option.
## Authentication
RustyPipe supports authenticating with your YouTube account to access

BIN
bg_snapshot.bin Normal file

Binary file not shown.

View file

@ -3,52 +3,6 @@
All notable changes to this project will be documented in this file.
## [v0.7.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.7.1..rustypipe-cli/v0.7.2) - 2025-03-16
### ⚙️ Miscellaneous Tasks
- *(deps)* Update rustypipe to 0.11.1
- *(deps)* Update rustypipe-downloader to 0.3.1
- *(deps)* Update rust crate rstest to 0.25.0 - ([9ed1306](https://codeberg.org/ThetaDev/rustypipe/commit/9ed1306f3aaeb993c409997ddfbc47499e4f4d22))
## [v0.7.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.7.0..rustypipe-cli/v0.7.1) - 2025-02-26
### ⚙️ Miscellaneous Tasks
- *(deps)* Update rustypipe to 0.11.0 - ([035c07f](https://codeberg.org/ThetaDev/rustypipe/commit/035c07f170aa293bcc626f27998c2b2b28660881))
## [v0.7.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.6.0..rustypipe-cli/v0.7.0) - 2025-02-09
### 🚀 Features
- Add support for rustypipe-botguard to get PO tokens - ([b90a252](https://codeberg.org/ThetaDev/rustypipe/commit/b90a252a5e1bf05a5294168b0ec16a73cbb88f42))
- [**breaking**] Remove manual PO token options from downloader/cli, add new rustypipe-botguard options - ([cddb32f](https://codeberg.org/ThetaDev/rustypipe/commit/cddb32f190276265258c6ab45b3d43a8891c4b39))
- Add session po token cache - ([b72b501](https://codeberg.org/ThetaDev/rustypipe/commit/b72b501b6dbcf4333b24cd80e7c8c61b0c21ec91))
- Add timezone query option - ([3a2370b](https://codeberg.org/ThetaDev/rustypipe/commit/3a2370b97ca3d0f40d72d66a23295557317d29fb))
- Add --timezone-local CLI option - ([4f2bb47](https://codeberg.org/ThetaDev/rustypipe/commit/4f2bb47ab42ae0c68a64f3b3c2831fa7850b6f56))
- Add verbose flag - ([629b590](https://codeberg.org/ThetaDev/rustypipe/commit/629b5905da653c6fe0f3c6b5814dd2f49030e7ed))
### 🐛 Bug Fixes
- Parsing mixed-case language codes like zh-CN - ([9c73ed4](https://codeberg.org/ThetaDev/rustypipe/commit/9c73ed4b3008cb093c0fa7fd94fd9f1ba8cd3627))
### 🚜 Refactor
- [**breaking**] Add client_type field to DownloadError, rename cli option po-token-cache to pot-cache - ([594e675](https://codeberg.org/ThetaDev/rustypipe/commit/594e675b39efc5fbcdbd5e920a4d2cdee64f718e))
- Rename rustypipe-cli binary to rustypipe - ([c1a872e](https://codeberg.org/ThetaDev/rustypipe/commit/c1a872e1c14ea0956053bd7c65f6875b1cb3bc55))
### 📚 Documentation
- Add Botguard info to README - ([9957add](https://codeberg.org/ThetaDev/rustypipe/commit/9957add2b5d6391b2c1869d2019fd7dd91b8cd41))
### ⚙️ Miscellaneous Tasks
- *(deps)* Update rustypipe to 0.10.0
- *(deps)* Update rust crate rquickjs to 0.9.0 (#33) - ([2c8ac41](https://codeberg.org/ThetaDev/rustypipe/commit/2c8ac410aa535d83f8bcc7181f81914b13bceb77))
## [v0.6.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.5.0..rustypipe-cli/v0.6.0) - 2025-01-16
### 🚀 Features

View file

@ -1,6 +1,6 @@
[package]
name = "rustypipe-cli"
version = "0.7.2"
version = "0.6.0"
rust-version = "1.70.0"
edition.workspace = true
authors.workspace = true
@ -42,7 +42,7 @@ rustls-tls-native-roots = [
]
[dependencies]
rustypipe = { workspace = true, features = ["rss", "userdata"] }
rustypipe = { workspace = true, features = ["rss"] }
rustypipe-downloader.workspace = true
reqwest.workspace = true
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
@ -64,7 +64,3 @@ dirs.workspace = true
anstream = "0.6.15"
owo-colors = "4.0.0"
const_format = "0.2.33"
[[bin]]
name = "rustypipe"
path = "src/main.rs"

View file

@ -8,19 +8,7 @@ The RustyPipe CLI is a powerful YouTube client for the command line. It allows y
access most of the features of the RustyPipe crate: getting data from YouTube and
downloading videos.
## Installation
You can download a compiled version of RustyPipe here:
<https://codeberg.org/ThetaDev/rustypipe/releases>
Alternatively, you can compile it yourself by installing [Rust](https://rustup.rs/) and
running `cargo install rustypipe-cli`.
To be able to access streams from web-based clients (Desktop, Mobile) you need to
download [rustypipe-botguard](https://codeberg.org/ThetaDev/rustypipe-botguard/releases)
and place the binary either in the PATH or the current working directory.
For downloading videos you also need to have ffmpeg installed.
The following subcommands are included:
## `get`: Fetch information
@ -139,8 +127,8 @@ Fetch a list of all the items saved in your YouTube/YouTube Music profile.
- **Proxy:** RustyPipe respects the environment variables `HTTP_PROXY`, `HTTPS_PROXY`
and `ALL_PROXY`
- **Logging:** Enable debug logging with the `-v` (verbose) flag. If you want more
fine-grained control, use the `RUST_LOG` environment variable.
- **Logging:** You can change the log level with the `RUST_LOG` environment variable, it
is set to `info` by default
- **Visitor data:** A custom visitor data ID can be used with the `--vdata` flag
- **Authentication:** Use the commands `rustypipe login` and `rustypipe login --cookie`
to log into your Google account using either OAuth or YouTube cookies. With the
@ -152,7 +140,6 @@ Fetch a list of all the items saved in your YouTube/YouTube Music profile.
Europe/Berlin, Australia/Sydney)
**Note:** this requires building rustypipe-cli with the `timezone` feature
- `--local-tz` Use the local timezone instead of UTC
- `--report` Generate a report on every request and store it in a `rustypipe_reports`
folder in the current directory

View file

@ -80,9 +80,6 @@ struct Cli {
/// Enable caching for session-bound PO tokens
#[clap(long, global = true)]
pot_cache: bool,
/// Enable debug logging
#[clap(short, long, global = true)]
verbose: bool,
}
#[derive(Parser)]
@ -881,15 +878,12 @@ async fn run() -> anyhow::Result<()> {
let cli = Cli::parse();
let multi = MultiProgress::new();
let mut env_filter = EnvFilter::builder()
.with_default_directive(LevelFilter::INFO.into())
.from_env_lossy();
if cli.verbose {
env_filter = env_filter.add_directive("rustypipe=debug".parse().unwrap());
}
tracing_subscriber::fmt::SubscriberBuilder::default()
.with_env_filter(env_filter)
.with_env_filter(
EnvFilter::builder()
.with_default_directive(LevelFilter::INFO.into())
.from_env_lossy(),
)
.with_writer(ProgWriter(multi.clone()))
.init();

View file

@ -9,7 +9,7 @@ repository.workspace = true
publish = false
[dependencies]
rustypipe = { path = "../", features = ["userdata"] }
rustypipe = { path = "../" }
reqwest.workspace = true
tokio = { workspace = true, features = ["rt-multi-thread"] }
futures-util.workspace = true

View file

@ -40,15 +40,12 @@ pub enum ABTest {
MusicPlaylistFacepile = 18,
MusicAlbumGroupsReordered = 19,
MusicContinuationItemRenderer = 20,
AlbumRecommends = 21,
CommandExecutorCommand = 22,
}
/// List of active A/B tests that are run when none is manually specified
const TESTS_TO_RUN: &[ABTest] = &[
ABTest::MusicAlbumGroupsReordered,
ABTest::AlbumRecommends,
ABTest::CommandExecutorCommand,
ABTest::MusicContinuationItemRenderer,
];
#[derive(Debug, Serialize, Deserialize)]
@ -124,8 +121,6 @@ pub async fn run_test(
ABTest::MusicContinuationItemRenderer => {
music_continuation_item_renderer(&query).await
}
ABTest::AlbumRecommends => album_recommends(&query).await,
ABTest::CommandExecutorCommand => command_executor_command(&query).await,
}
.unwrap();
pb.inc(1);
@ -448,33 +443,3 @@ pub async fn music_continuation_item_renderer(rp: &RustyPipeQuery) -> Result<boo
.await?;
Ok(res.contains("\"continuationItemRenderer\""))
}
pub async fn album_recommends(rp: &RustyPipeQuery) -> Result<bool> {
let id = "MPREb_u1I69lSAe5v";
let res = rp
.raw(
ClientType::DesktopMusic,
"browse",
&QBrowse {
browse_id: id,
params: None,
},
)
.await?;
Ok(res.contains("\"musicCarouselShelfRenderer\""))
}
pub async fn command_executor_command(rp: &RustyPipeQuery) -> Result<bool> {
let id = "VLPLbZIPy20-1pN7mqjckepWF78ndb6ci_qi";
let res = rp
.raw(
ClientType::Desktop,
"browse",
&QBrowse {
browse_id: id,
params: None,
},
)
.await?;
Ok(res.contains("\"commandExecutorCommand\""))
}

View file

@ -1,130 +0,0 @@
use std::{collections::BTreeMap, fs::File, io::BufReader};
use path_macro::path;
use rustypipe::{
client::{ClientType, RustyPipe},
param::{Language, LANGUAGES},
};
use serde::Deserialize;
use serde_with::rust::deserialize_ignore_any;
use crate::{
model::{QBrowse, SectionList, TextRuns},
util::{self, DICT_DIR},
};
pub async fn collect_album_versions_titles() {
let json_path = path!(*DICT_DIR / "other_versions_titles.json");
let mut res = BTreeMap::new();
let rp = RustyPipe::new();
for lang in LANGUAGES {
let query = QBrowse {
browse_id: "MPREb_nlBWQROfvjo",
params: None,
};
let raw_resp = rp
.query()
.lang(lang)
.raw(ClientType::DesktopMusic, "browse", &query)
.await
.unwrap();
let data = serde_json::from_str::<AlbumData>(&raw_resp).unwrap();
let title = data
.contents
.two_column_browse_results_renderer
.secondary_contents
.section_list_renderer
.contents
.into_iter()
.find_map(|x| match x {
ItemSection::MusicCarouselShelfRenderer(music_carousel_shelf) => {
Some(music_carousel_shelf)
}
ItemSection::None => None,
})
.expect("other versions")
.header
.expect("header")
.music_carousel_shelf_basic_header_renderer
.title
.runs
.into_iter()
.next()
.unwrap()
.text;
println!("{lang}: {title}");
res.insert(lang, title);
}
let file = File::create(json_path).unwrap();
serde_json::to_writer_pretty(file, &res).unwrap();
}
pub fn write_samples_to_dict() {
let json_path = path!(*DICT_DIR / "other_versions_titles.json");
let json_file = File::open(json_path).unwrap();
let collected: BTreeMap<Language, String> =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let mut dict = util::read_dict();
let langs = dict.keys().copied().collect::<Vec<_>>();
for lang in langs {
let dict_entry = dict.entry(lang).or_default();
let e = collected.get(&lang).unwrap();
assert_eq!(e, e.trim());
dict_entry.album_versions_title = e.to_owned();
for lang in &dict_entry.equivalent {
let ee = collected.get(lang).unwrap();
if ee != e {
panic!("equivalent lang conflict, lang: {lang}");
}
}
}
util::write_dict(dict);
}
#[derive(Debug, Deserialize)]
struct AlbumData {
contents: AlbumDataContents,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct AlbumDataContents {
two_column_browse_results_renderer: X1,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct X1 {
secondary_contents: SectionList<ItemSection>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
enum ItemSection {
MusicCarouselShelfRenderer(MusicCarouselShelf),
#[serde(other, deserialize_with = "deserialize_ignore_any")]
None,
}
#[derive(Debug, Deserialize)]
struct MusicCarouselShelf {
header: Option<MusicCarouselShelfHeader>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct MusicCarouselShelfHeader {
music_carousel_shelf_basic_header_renderer: MusicCarouselShelfHeaderRenderer,
}
#[derive(Debug, Deserialize)]
struct MusicCarouselShelfHeaderRenderer {
title: TextRuns,
}

View file

@ -204,6 +204,8 @@ pub fn parse_video_durations() {
parse(&mut words, lang, dict_entry.by_char, txt, *d);
}
// dbg!(&words);
for (k, v) in words {
if let Some(v) = v {
dict_entry.timeago_tokens.insert(k, v.to_string());

View file

@ -39,6 +39,9 @@ pub async fn download_testfiles() {
search_playlists().await;
search_empty().await;
trending().await;
history().await;
subscriptions().await;
subscription_feed().await;
music_playlist().await;
music_playlist_cont().await;
@ -62,12 +65,6 @@ pub async fn download_testfiles() {
music_charts().await;
music_genres().await;
music_genre().await;
// User data
history().await;
subscriptions().await;
subscription_feed().await;
music_history().await;
music_saved_artists().await;
music_saved_albums().await;
@ -467,7 +464,7 @@ async fn trending() {
}
async fn history() {
let json_path = path!(*TESTFILES_DIR / "userdata" / "history.json");
let json_path = path!(*TESTFILES_DIR / "history" / "history.json");
if json_path.exists() {
return;
}
@ -477,7 +474,7 @@ async fn history() {
}
async fn subscriptions() {
let json_path = path!(*TESTFILES_DIR / "userdata" / "subscriptions.json");
let json_path = path!(*TESTFILES_DIR / "history" / "subscriptions.json");
if json_path.exists() {
return;
}
@ -487,7 +484,7 @@ async fn subscriptions() {
}
async fn subscription_feed() {
let json_path = path!(*TESTFILES_DIR / "userdata" / "subscription_feed.json");
let json_path = path!(*TESTFILES_DIR / "history" / "subscription_feed.json");
if json_path.exists() {
return;
}
@ -819,7 +816,7 @@ async fn music_genre() {
}
async fn music_history() {
let json_path = path!(*TESTFILES_DIR / "music_userdata" / "music_history.json");
let json_path = path!(*TESTFILES_DIR / "music_history" / "music_history.json");
if json_path.exists() {
return;
}
@ -829,7 +826,7 @@ async fn music_history() {
}
async fn music_saved_artists() {
let json_path = path!(*TESTFILES_DIR / "music_userdata" / "saved_artists.json");
let json_path = path!(*TESTFILES_DIR / "music_history" / "saved_artists.json");
if json_path.exists() {
return;
}
@ -839,7 +836,7 @@ async fn music_saved_artists() {
}
async fn music_saved_albums() {
let json_path = path!(*TESTFILES_DIR / "music_userdata" / "saved_albums.json");
let json_path = path!(*TESTFILES_DIR / "music_history" / "saved_albums.json");
if json_path.exists() {
return;
}
@ -849,7 +846,7 @@ async fn music_saved_albums() {
}
async fn music_saved_tracks() {
let json_path = path!(*TESTFILES_DIR / "music_userdata" / "saved_tracks.json");
let json_path = path!(*TESTFILES_DIR / "music_history" / "saved_tracks.json");
if json_path.exists() {
return;
}
@ -859,7 +856,7 @@ async fn music_saved_tracks() {
}
async fn music_saved_playlists() {
let json_path = path!(*TESTFILES_DIR / "music_userdata" / "saved_playlists.json");
let json_path = path!(*TESTFILES_DIR / "music_history" / "saved_playlists.json");
if json_path.exists() {
return;
}

View file

@ -90,8 +90,6 @@ pub(crate) struct Entry {
pub chan_prefix: &'static str,
/// Channel name suffix on playlist pages
pub chan_suffix: &'static str,
/// "Other versions" title on album pages
pub album_versions_title: &'static str,
}
"#;
@ -180,8 +178,8 @@ pub(crate) fn entry(lang: Language) -> Entry {
.to_string()
.replace('\n', "\n ");
write!(code_timeago_tokens, "{} => Entry {{\n timeago_tokens: {},\n month_before_day: {:?},\n months: {},\n timeago_nd_tokens: {},\n comma_decimal: {:?},\n number_tokens: {},\n number_nd_tokens: {},\n album_types: {},\n chan_prefix: {:?},\n chan_suffix: {:?},\n album_versions_title: {:?},\n }},\n ",
selector, code_ta_tokens, entry.month_before_day, code_months, code_ta_nd_tokens, entry.comma_decimal, code_number_tokens, code_number_nd_tokens, code_album_types, entry.chan_prefix, entry.chan_suffix, entry.album_versions_title).unwrap();
write!(code_timeago_tokens, "{} => Entry {{\n timeago_tokens: {},\n month_before_day: {:?},\n months: {},\n timeago_nd_tokens: {},\n comma_decimal: {:?},\n number_tokens: {},\n number_nd_tokens: {},\n album_types: {},\n chan_prefix: {:?},\n chan_suffix: {:?},\n }},\n ",
selector, code_ta_tokens, entry.month_before_day, code_months, code_ta_nd_tokens, entry.comma_decimal, code_number_tokens, code_number_nd_tokens, code_album_types, entry.chan_prefix, entry.chan_suffix).unwrap();
}
code_timeago_tokens = code_timeago_tokens.trim_end().to_owned() + "\n }\n}\n";

View file

@ -2,7 +2,6 @@
mod abtest;
mod collect_album_types;
mod collect_album_versions_titles;
mod collect_chan_prefixes;
mod collect_history_dates;
mod collect_large_numbers;
@ -35,14 +34,12 @@ enum Commands {
CollectHistoryDates,
CollectMusicHistoryDates,
CollectChanPrefixes,
CollectAlbumVersionsTitles,
ParsePlaylistDates,
ParseHistoryDates,
ParseLargeNumbers,
ParseAlbumTypes,
ParseVideoDurations,
ParseChanPrefixes,
ParseAlbumVersionsTitles,
GenLocales,
GenDict,
DownloadTestfiles,
@ -61,25 +58,28 @@ async fn main() {
match cli.command {
Commands::CollectPlaylistDates => {
collect_playlist_dates::collect_dates(cli.concurrency).await
collect_playlist_dates::collect_dates(cli.concurrency).await;
}
Commands::CollectLargeNumbers => {
collect_large_numbers::collect_large_numbers(cli.concurrency).await
collect_large_numbers::collect_large_numbers(cli.concurrency).await;
}
Commands::CollectAlbumTypes => {
collect_album_types::collect_album_types(cli.concurrency).await
collect_album_types::collect_album_types(cli.concurrency).await;
}
Commands::CollectVideoDurations => {
collect_video_durations::collect_video_durations(cli.concurrency).await
collect_video_durations::collect_video_durations(cli.concurrency).await;
}
Commands::CollectVideoDates => {
collect_video_dates::collect_video_dates(cli.concurrency).await
collect_video_dates::collect_video_dates(cli.concurrency).await;
}
Commands::CollectHistoryDates => collect_history_dates::collect_dates().await,
Commands::CollectMusicHistoryDates => collect_history_dates::collect_dates_music().await,
Commands::CollectChanPrefixes => collect_chan_prefixes::collect_chan_prefixes().await,
Commands::CollectAlbumVersionsTitles => {
collect_album_versions_titles::collect_album_versions_titles().await
Commands::CollectHistoryDates => {
collect_history_dates::collect_dates().await;
}
Commands::CollectMusicHistoryDates => {
collect_history_dates::collect_dates_music().await;
}
Commands::CollectChanPrefixes => {
collect_chan_prefixes::collect_chan_prefixes().await;
}
Commands::ParsePlaylistDates => collect_playlist_dates::write_samples_to_dict(),
Commands::ParseHistoryDates => collect_history_dates::write_samples_to_dict(),
@ -87,10 +87,9 @@ async fn main() {
Commands::ParseAlbumTypes => collect_album_types::write_samples_to_dict(),
Commands::ParseVideoDurations => collect_video_durations::parse_video_durations(),
Commands::ParseChanPrefixes => collect_chan_prefixes::write_samples_to_dict(),
Commands::ParseAlbumVersionsTitles => {
collect_album_versions_titles::write_samples_to_dict()
Commands::GenLocales => {
gen_locales::generate_locales().await;
}
Commands::GenLocales => gen_locales::generate_locales().await,
Commands::GenDict => gen_dictionary::generate_dictionary(),
Commands::DownloadTestfiles => download_testfiles::download_testfiles().await,
Commands::AbTest { id, n } => {

View file

@ -61,8 +61,6 @@ pub struct DictEntry {
pub chan_prefix: String,
/// Channel name suffix on playlist pages
pub chan_suffix: String,
/// "Other versions" title on album pages
pub album_versions_title: String,
}
/// Parsed TimeAgo string, contains amount and time unit.

View file

@ -3,37 +3,6 @@
All notable changes to this project will be documented in this file.
## [v0.3.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.3.0..rustypipe-downloader/v0.3.1) - 2024-12-20
### ⚙️ Miscellaneous Tasks
- *(deps)* Update rustypipe to 0.11.0
## [v0.3.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.7..rustypipe-downloader/v0.3.0) - 2025-02-09
### 🚀 Features
- [**breaking**] Remove manual PO token options from downloader in favor of rustypipe-botguard - ([cddb32f](https://codeberg.org/ThetaDev/rustypipe/commit/cddb32f190276265258c6ab45b3d43a8891c4b39))
### 🐛 Bug Fixes
- Ensure downloader futures are send - ([812ff4c](https://codeberg.org/ThetaDev/rustypipe/commit/812ff4c5bafffc5708a6d5066f1ebadb6d9fc958))
- Download audio with dolby codec - ([9234005](https://codeberg.org/ThetaDev/rustypipe/commit/92340056f868007beccb64e9e26eb39abc40f7aa))
### 🚜 Refactor
- [**breaking**] Add client_type field to DownloadError, rename cli option po-token-cache to pot-cache - ([594e675](https://codeberg.org/ThetaDev/rustypipe/commit/594e675b39efc5fbcdbd5e920a4d2cdee64f718e))
### 📚 Documentation
- Add Botguard info to README - ([9957add](https://codeberg.org/ThetaDev/rustypipe/commit/9957add2b5d6391b2c1869d2019fd7dd91b8cd41))
### ⚙️ Miscellaneous Tasks
- *(deps)* Update rustypipe to 0.10.0
## [v0.2.7](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.6..rustypipe-downloader/v0.2.7) - 2025-01-16
### 🚀 Features

View file

@ -1,6 +1,6 @@
[package]
name = "rustypipe-downloader"
version = "0.3.1"
version = "0.2.7"
rust-version = "1.67.1"
edition.workspace = true
authors.workspace = true
@ -51,7 +51,7 @@ image = { version = "0.25.0", optional = true, default-features = false, feature
"jpeg",
"webp",
] }
smartcrop2 = { version = "0.4.0", optional = true }
smartcrop2 = { version = "0.3.1", optional = true }
[dev-dependencies]
path_macro.workspace = true

View file

@ -1033,11 +1033,7 @@ impl DownloadQuery {
image::load_from_memory(&img_bts)?
};
let crop = smartcrop::find_best_crop_no_borders(
&img,
NonZeroU32::MIN,
NonZeroU32::MIN,
)
let crop = smartcrop::find_best_crop(&img, NonZeroU32::MIN, NonZeroU32::MIN)
.map_err(|e| DownloadError::AudioTag(format!("image crop: {e}").into()))?
.crop;
img = img.crop_imm(crop.x, crop.y, crop.width, crop.height);
@ -1067,8 +1063,8 @@ impl DownloadQuery {
}
fn get_download_range(offset: u64, size: Option<u64>) -> Range<u64> {
let mut rng = rand::rng();
let chunk_size = rng.random_range(CHUNK_SIZE_MIN..CHUNK_SIZE_MAX);
let mut rng = rand::thread_rng();
let chunk_size = rng.gen_range(CHUNK_SIZE_MIN..CHUNK_SIZE_MAX);
let mut chunk_end = offset + chunk_size;
if let Some(size) = size {
@ -1201,8 +1197,6 @@ async fn download_single_file(
}
}
tracing::debug!("downloading {} to {}", url, output.to_string_lossy());
let mut file = fs::OpenOptions::new()
.append(true)
.create(true)

View file

@ -3,13 +3,13 @@
When YouTube introduces a new feature, it does so gradually. When a user creates a new
session, YouTube decided randomly which new features should be enabled.
YouTube sessions are identified by the visitor data ID. This cookie is sent with every
API request using the `context.client.visitor_data` JSON parameter. It is also returned
in the `responseContext.visitorData` response parameter and stored as the `__SECURE-YEC`
cookie.
YouTube sessions are identified by the visitor data ID. This cookie is sent with
every API request using the `context.client.visitor_data` JSON parameter. It is also
returned in the `responseContext.visitorData` response parameter and stored as the
`__SECURE-YEC` cookie.
By sending the same visitor data ID, A/B tests can be reproduced, which is important for
testing alternative YouTube clients.
By sending the same visitor data ID, A/B tests can be reproduced, which is important
for testing alternative YouTube clients.
This page lists all A/B tests that were encountered while maintaining the RustyPipe
client.
@ -1030,7 +1030,7 @@ commandContext missing).
- **Encountered on:** 13.01.2025
- **Impact:** 🟢 Low
- **Endpoint:** browse (YTM)
- **Status:** Frequent (59%)
- **Status:** Common (10%)
YouTube Music used to group artist albums into 2 rows: "Albums" and "Singles".
@ -1042,7 +1042,7 @@ omitted for albums in their group, while singles and EPs have a label with their
- **Encountered on:** 25.01.2025
- **Impact:** 🟢 Low
- **Endpoint:** browse (YTM)
- **Status:** Stabilized
- **Status:** Common (4%)
YouTube Music now uses a `continuationItemRenderer` for music playlists instead of
putting the continuations in a separate attribute of the MusicShelf.
@ -1052,52 +1052,3 @@ items.
YouTube Music now also sends a random 16-character string as a `clientScreenNonce` in
the request context. This is not mandatory though.
## [21] Music album recommendations
- **Encountered on:** 26.02.2025
- **Impact:** 🟢 Low
- **Endpoint:** browse (YTM)
- **Status:** Common (15%)
![A/B test 21 screenshot](./_img/ab_21.png)
YouTube Music has added "Recommended" and "More from \<Artist\>" carousels to album
pages. The difficulty is distinguishing them reliably for parsing the album variants.
The current solution is adding the "Other versions" title in all languages to the
dictionary and comparing it.
## [22] commandExecutorCommand for continuations
- **Encountered on:** 16.03.2025
- **Impact:** 🟢 Low
- **Endpoint:** browse (YTM)
- **Status:** Experimental (1%)
YouTube playlists may use a commandExecutorCommand which holds a list of commands: the
`continuationCommand` that needs to be extracted as well as a `playlistVotingRefreshPopupCommand`.
```json
{
"continuationItemRenderer": {
"continuationEndpoint": {
"commandExecutorCommand": {
"commands": [
{
"playlistVotingRefreshPopupCommand": {
"command": {}
}
},
{
"continuationCommand": {
"request": "CONTINUATION_REQUEST_TYPE_BROWSE",
"token": "4qmFsgKBARIkVkxQTGJaSVB5MjAtMXBON21xamNrZXBXRjc4bmRiNmNpX3FpGjRDQUY2SGxCVU9rTklTV2xGUkVreVVtdEZOVTVFU1hsU2FrWkRVa1JKZWs1NldRJTNEJTNEmgIiUExiWklQeTIwLTFwTjdtcWpja2VwV0Y3OG5kYjZjaV9xaQ%3D%3D"
}
}
]
}
}
}
}
```

Binary file not shown.

Before

Width:  |  Height:  |  Size: 290 KiB

View file

@ -3,7 +3,7 @@ use std::fmt::Debug;
use crate::{
error::{Error, ExtractionError},
model::ChannelRss,
report::Report,
report::{Report, RustyPipeInfo},
util,
};
@ -45,7 +45,7 @@ impl RustyPipeQuery {
Err(e) => {
if let Some(reporter) = &self.client.inner.reporter {
let report = Report {
info: self.rp_info(),
info: RustyPipeInfo::new(Some(self.opts.lang)),
level: crate::report::Level::ERR,
operation: "channel_rss",
error: Some(e.to_string()),

View file

@ -7,7 +7,7 @@ use crate::{
error::{Error, ExtractionError},
model::{
paginator::{ContinuationEndpoint, Paginator},
ChannelItem, HistoryItem, Playlist, PlaylistItem, VideoItem,
ChannelItem, HistoryItem, PlaylistItem, VideoItem,
},
serializer::MapResult,
};
@ -148,28 +148,6 @@ impl RustyPipeQuery {
)
.await
}
/// Get all liked videos of the logged-in user
///
/// Requires authentication cookies.
pub async fn liked_videos(&self) -> Result<Playlist, Error> {
self.clone()
.authenticated()
.playlist("LL")
.await
.map_err(crate::util::map_internal_playlist_err)
}
/// Get the "Watch later" playlist of the logged-in user
///
/// Requires authentication cookies.
pub async fn watch_later(&self) -> Result<Playlist, Error> {
self.clone()
.authenticated()
.playlist("WL")
.await
.map_err(crate::util::map_internal_playlist_err)
}
}
impl MapResponse<Paginator<HistoryItem<VideoItem>>> for response::History {
@ -207,9 +185,11 @@ impl MapResponse<Paginator<HistoryItem<VideoItem>>> for response::History {
&mut map_res,
);
}
response::YouTubeListItem::ContinuationItemRenderer(ep) => {
response::YouTubeListItem::ContinuationItemRenderer {
continuation_endpoint,
} => {
if ctoken.is_none() {
ctoken = ep.continuation_endpoint.into_token();
ctoken = Some(continuation_endpoint.continuation_command.token);
}
}
_ => {}
@ -278,7 +258,7 @@ mod tests {
#[test]
fn map_history() {
let json_path = path!(*TESTFILES / "userdata" / "history.json");
let json_path = path!(*TESTFILES / "history" / "history.json");
let json_file = File::open(json_path).unwrap();
let history: response::History =
@ -298,7 +278,7 @@ mod tests {
#[test]
fn map_subscription_feed() {
let json_path = path!(*TESTFILES / "userdata" / "subscription_feed.json");
let json_path = path!(*TESTFILES / "history" / "subscription_feed.json");
let json_file = File::open(json_path).unwrap();
let history: response::History =

View file

@ -3,10 +3,12 @@
pub(crate) mod response;
mod channel;
mod history;
mod music_artist;
mod music_charts;
mod music_details;
mod music_genres;
mod music_history;
mod music_new;
mod music_playlist;
mod music_search;
@ -18,13 +20,6 @@ mod trends;
mod url_resolver;
mod video_details;
#[cfg(feature = "userdata")]
#[cfg_attr(docsrs, doc(cfg(feature = "userdata")))]
mod music_userdata;
#[cfg(feature = "userdata")]
#[cfg_attr(docsrs, doc(cfg(feature = "userdata")))]
mod userdata;
#[cfg(feature = "rss")]
#[cfg_attr(docsrs, doc(cfg(feature = "rss")))]
mod channel_rss;
@ -293,10 +288,8 @@ struct OauthToken {
#[derive(Debug, Clone, Serialize, Deserialize)]
struct AuthCookie {
cookie: String,
#[serde(alias = "account_syncid", skip_serializing_if = "Option::is_none")]
channel_syncid: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
user_syncid: Option<String>,
account_syncid: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
session_index: Option<String>,
}
@ -321,9 +314,8 @@ impl AuthCookie {
fn new(cookie: String) -> Self {
Self {
cookie,
channel_syncid: None,
account_syncid: None,
session_index: None,
user_syncid: None,
}
}
}
@ -365,8 +357,6 @@ const OAUTH_CLIENT_ID: &str =
const OAUTH_CLIENT_SECRET: &str = "SboVhoG9s0rNafixCSGGKXAT";
const OAUTH_SCOPES: &str = "http://gdata.youtube.com https://www.googleapis.com/auth/youtube";
const BOTGUARD_API_VERSION: &str = "1";
static CLIENT_VERSION_REGEX: Lazy<Regex> =
Lazy::new(|| Regex::new(r#""INNERTUBE_CONTEXT_CLIENT_VERSION":"([\w\d\._-]+?)""#).unwrap());
@ -384,7 +374,7 @@ struct RustyPipeRef {
http: Client,
storage: Option<Box<dyn CacheStorage>>,
reporter: Option<Box<dyn Reporter>>,
n_http_retries: u32,
n_request_attempts: u32,
cache: CacheHolder,
default_opts: RustyPipeOpts,
user_agent: Cow<'static, str>,
@ -408,19 +398,17 @@ struct RustyPipeOpts {
pub struct RustyPipeBuilder {
storage: DefaultOpt<Box<dyn CacheStorage>>,
reporter: DefaultOpt<Box<dyn Reporter>>,
n_http_retries: u32,
n_request_attempts: u32,
timeout: DefaultOpt<Duration>,
user_agent: Option<String>,
default_opts: RustyPipeOpts,
storage_dir: Option<PathBuf>,
botguard_bin: DefaultOpt<OsString>,
snapshot_file: Option<PathBuf>,
po_token_cache: bool,
}
struct BotguardCfg {
program: OsString,
version: String,
snapshot_file: PathBuf,
po_token_cache: bool,
}
@ -448,6 +436,13 @@ impl<T> DefaultOpt<T> {
DefaultOpt::Default => Some(f()),
}
}
fn or_default_opt<F: FnOnce() -> Option<T>>(self, f: F) -> Option<T> {
match self {
DefaultOpt::Some(x) => Some(x),
DefaultOpt::None => None,
DefaultOpt::Default => f(),
}
}
}
/// # RustyPipe query
@ -508,26 +503,6 @@ impl<T> DefaultOpt<T> {
/// - [`music_new_albums`](RustyPipeQuery::music_new_albums)
/// - [`music_new_videos`](RustyPipeQuery::music_new_videos)
///
/// ### User data (🔒 Feature `userdata`)
///
/// - **Playback history**
/// - [`history`](RustyPipeQuery::history)
/// - [`history_search`](RustyPipeQuery::history_search)
/// - [`music_history`](RustyPipeQuery::music_history)
/// - **YouTube library**
/// - [`liked_videos`](RustyPipeQuery::liked_videos)
/// - [`watch_later`](RustyPipeQuery::watch_later)
/// - [`saved_playlists`](RustyPipeQuery::saved_playlists)
/// - **Music library**
/// - [`music_saved_artists`](RustyPipeQuery::music_saved_artists)
/// - [`music_saved_albums`](RustyPipeQuery::music_saved_albums)
/// - [`music_saved_tracks`](RustyPipeQuery::music_saved_tracks)
/// - [`music_saved_playlists`](RustyPipeQuery::music_saved_playlists)
/// - [`music_liked_tracks`](RustyPipeQuery::music_liked_tracks)
/// - **Subscriptions**
/// - [`subscriptions`](RustyPipeQuery::subscriptions)
/// - [`subscription_feed`](RustyPipeQuery::subscription_feed)
///
/// ## Options
///
/// You can set the language, country and visitor data ID for individual requests.
@ -680,11 +655,10 @@ impl RustyPipeBuilder {
storage: DefaultOpt::Default,
reporter: DefaultOpt::Default,
timeout: DefaultOpt::Default,
n_http_retries: 2,
n_request_attempts: 2,
user_agent: None,
storage_dir: None,
botguard_bin: DefaultOpt::Default,
snapshot_file: None,
po_token_cache: false,
}
}
@ -750,31 +724,27 @@ impl RustyPipeBuilder {
let visitor_data_cache = VisitorDataCache::new(http.clone(), 50, 20);
let botguard = match self.botguard_bin {
DefaultOpt::Some(botguard_bin) => Some(detect_botguard_bin(botguard_bin)?),
DefaultOpt::None => None,
DefaultOpt::Default => detect_botguard_bin("./rustypipe-botguard".into())
.or_else(|_| detect_botguard_bin("rustypipe-botguard".into()))
.map_err(|e| tracing::debug!("could not detect rustypipe-botguard: {e}"))
.ok(),
let botguard_bin = self.botguard_bin.or_default_opt(|| {
let n = OsString::from("rustypipe-botguard");
let out = std::process::Command::new(&n)
.arg("--version")
.output()
.ok()?;
if !out.status.success() {
return None;
}
.map(|(program, version)| {
tracing::debug!(
"rustypipe-botguard: using {} at {}",
version,
program.to_string_lossy()
);
BotguardCfg {
program: program.to_owned(),
version,
snapshot_file: self.snapshot_file.unwrap_or_else(|| {
let mut snapshot_file = storage_dir.clone();
snapshot_file.push("bg_snapshot.bin");
snapshot_file
}),
po_token_cache: self.po_token_cache,
let output = String::from_utf8_lossy(&out.stdout);
let pat = "rustypipe-botguard-api ";
let pos = output.find(pat)? + pat.len();
let pos_end = output[pos..]
.char_indices()
.find(|(_, c)| !c.is_ascii_digit())
.map(|(p, _)| p + pos)
.unwrap_or(output.len());
if &output[pos..pos_end] != "1" {
return None;
}
Some(n)
});
Ok(RustyPipe {
@ -782,11 +752,11 @@ impl RustyPipeBuilder {
http,
storage,
reporter: self.reporter.or_default(|| {
let mut report_dir = storage_dir;
let mut report_dir = storage_dir.clone();
report_dir.push(DEFAULT_REPORT_DIR);
Box::new(FileReporter::new(report_dir))
}),
n_http_retries: self.n_http_retries,
n_request_attempts: self.n_request_attempts,
cache: CacheHolder {
clients: cache_clients,
deobf: AsyncRwLock::new(cdata.deobf),
@ -796,7 +766,15 @@ impl RustyPipeBuilder {
default_opts: self.default_opts,
user_agent,
visitor_data_cache,
botguard,
botguard: botguard_bin.map(|program| {
let mut snapshot_file = storage_dir;
snapshot_file.push("bg_snapshot.bin");
BotguardCfg {
program,
snapshot_file,
po_token_cache: self.po_token_cache,
}
}),
}),
})
}
@ -865,9 +843,9 @@ impl RustyPipeBuilder {
self
}
/// Set the maximum number of retries for YouTube requests.
/// Set the maximum number of attempts for HTTP requests (at least 1).
///
/// If a request fails because of a serverside error and retries are enabled,
/// If a HTTP requests fails because of a serverside error and retries are enabled,
/// RustyPipe waits 1 second before the next attempt.
///
/// The wait time is doubled for subsequent attempts (including a bit of
@ -875,8 +853,8 @@ impl RustyPipeBuilder {
///
/// **Default value**: 2
#[must_use]
pub fn n_http_retries(mut self, n_retries: u32) -> Self {
self.n_http_retries = n_retries.max(1);
pub fn n_request_attempts(mut self, n_retries: u32) -> Self {
self.n_request_attempts = n_retries.max(1);
self
}
@ -1032,18 +1010,6 @@ impl RustyPipeBuilder {
self
}
/// Set the path where the rustypipe-botguard snapshot file is stored
///
/// After solving a Botguard challenge, rustypipe-botguard stores its
/// JavaScript environment in a snapshot file, so it can quickly generate additional tokens.
///
/// By default the snapshot is stored in the storage_dir (Filename: bg_snapshot.bin).
#[must_use]
pub fn botguard_snapshot_file<P: Into<PathBuf>>(mut self, snapshot_file: P) -> Self {
self.snapshot_file = Some(snapshot_file.into());
self
}
/// Enable caching for session-bound PO tokens
///
/// By default, RustyPipe calls Botguard for every player request to fetch both a
@ -1094,7 +1060,7 @@ impl RustyPipe {
/// Execute the given http request.
async fn http_request(&self, request: &Request) -> Result<Response, reqwest::Error> {
let mut last_resp = None;
for n in 0..=self.inner.n_http_retries {
for n in 0..=self.inner.n_request_attempts {
let resp = self.inner.http.execute(request.try_clone().unwrap()).await;
let err = match resp {
@ -1120,7 +1086,7 @@ impl RustyPipe {
};
// Retry in case of a recoverable status code (server err, too many requests)
if n != self.inner.n_http_retries {
if n != self.inner.n_request_attempts {
let ms = util::retry_delay(n, 1000, 60000, 3);
tracing::warn!(
"Retry attempt #{}. Error: {}. Waiting {} ms",
@ -1594,17 +1560,6 @@ impl RustyPipe {
.ok_or(Error::Auth(AuthError::NoLogin))
}
fn user_auth_datasync_id(&self) -> Result<String, Error> {
self.inner
.cache
.auth_cookie
.read()
.unwrap()
.as_ref()
.and_then(|c| c.user_syncid.as_ref().map(|id| id.to_owned()))
.ok_or(Error::Auth(AuthError::NoLogin))
}
/// Set the user authentication cookie
///
/// The cookie is used for authenticated requests with browser-based clients
@ -1699,17 +1654,17 @@ impl RustyPipe {
))?;
// datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
// and just "user_syncid||" for primary channel.
let (p1, p2) =
// and just "user_syncid||" for primary channel. We only want the channel_syncid
let (channel_syncid, user_syncid) =
datasync_id
.split_once("||")
.ok_or(Error::Extraction(ExtractionError::InvalidData(
"datasyncId does not contain || seperator".into(),
)))?;
(auth_cookie.channel_syncid, auth_cookie.user_syncid) = if p2.is_empty() {
(None, Some(p1.to_owned()))
auth_cookie.account_syncid = if user_syncid.is_empty() {
None
} else {
(Some(p1.to_owned()), Some(p2.to_owned()))
Some(channel_syncid.to_owned())
};
auth_cookie.session_index = Some(
@ -1719,11 +1674,6 @@ impl RustyPipe {
);
Ok(())
}
/// Get the version string (e.g. `rustypipe-botguard 0.1.1`) of the used botguard binary
pub async fn version_botguard(&self) -> Option<String> {
self.inner.botguard.as_ref().map(|bg| bg.version.to_owned())
}
}
impl RustyPipeQuery {
@ -2143,7 +2093,7 @@ impl RustyPipeQuery {
if let Some(session_index) = auth_cookie.session_index {
r = r.header("X-Goog-AuthUser", session_index);
}
if let Some(account_syncid) = auth_cookie.channel_syncid {
if let Some(account_syncid) = auth_cookie.account_syncid {
r = r.header("X-Goog-PageId", account_syncid);
}
cookie = Some(auth_cookie.cookie);
@ -2202,7 +2152,7 @@ impl RustyPipeQuery {
self.client.inner.visitor_data_cache.remove(visitor_data);
}
/// Generate PO tokens
/// Get PO tokens
async fn get_po_tokens(&self, idents: &[&str]) -> Result<(Vec<String>, OffsetDateTime), Error> {
let bg = self
.client
@ -2241,11 +2191,9 @@ impl RustyPipeQuery {
}
let mut valid_until = None;
let mut from_snapshot = false;
for word in words {
if let Some((k, v)) = word.split_once('=') {
match k {
"valid_until" => {
if k == "valid_until" {
valid_until = Some(
v.parse::<i64>()
.ok()
@ -2255,27 +2203,16 @@ impl RustyPipeQuery {
))?,
);
}
"from_snapshot" => {
from_snapshot = v.eq_ignore_ascii_case("true") || v == "1";
}
_ => {}
}
}
}
let valid_until =
valid_until.unwrap_or_else(|| OffsetDateTime::now_utc() + time::Duration::hours(12));
tracing::debug!(
"generated PO token (valid_until {}, from_snapshot={}, took {}ms)",
valid_until,
from_snapshot,
start.elapsed().as_millis()
);
Ok((tokens, valid_until))
tracing::debug!("generated PO token (took {:?})", start.elapsed());
Ok((
tokens,
valid_until.unwrap_or_else(|| OffsetDateTime::now_utc() + time::Duration::hours(12)),
))
}
/// Get a session-bound PO token (either from cache or newly generated)
async fn get_session_po_token(&self, visitor_data: &str) -> Result<PoToken, Error> {
if let Some(po_token) = self.client.inner.visitor_data_cache.get_pot(visitor_data) {
return Ok(po_token);
@ -2289,7 +2226,7 @@ impl RustyPipeQuery {
Ok(po_token)
}
/// Get a PO token (Proof-of-origin token)
/// Get a Proof-of-origin token
///
/// PO tokens are used by the web-based YouTube clients for requesting player data and video streams.
///
@ -2303,22 +2240,6 @@ impl RustyPipeQuery {
})
}
/// Get a new RustyPipeInfo object for reports
fn rp_info(&self) -> RustyPipeInfo<'_> {
RustyPipeInfo::new(
Some(self.opts.lang),
self.client
.inner
.botguard
.as_ref()
.map(|bg| bg.version.as_str()),
)
}
/// Execute a request to the YouTube API, then deobfuscate and map the response.
///
/// Runs a single attempt, returns Ok with a erroneous RequestResult in case of a
/// HTTP or mapping error so it can be retried/reported.
async fn execute_request_attempt<
R: DeserializeOwned + MapResponse<M> + Debug,
M,
@ -2410,10 +2331,6 @@ impl RustyPipeQuery {
})
}
/// Execute a request to the YouTube API, then deobfuscate and map the response.
///
/// Runs up to n_request_attempts, returns Ok with a erroneous RequestResult in case of a
/// HTTP or mapping error so it can be reported.
async fn execute_request_inner<
R: DeserializeOwned + MapResponse<M> + Debug,
M,
@ -2427,7 +2344,7 @@ impl RustyPipeQuery {
ctx_src: &MapRespOptions<'_>,
) -> Result<RequestResult<M>, Error> {
let mut last_resp = None;
for n in 0..=self.client.inner.n_http_retries {
for n in 0..=self.client.inner.n_request_attempts {
let resp = self
.execute_request_attempt::<R, M, B>(ctype, id, endpoint, body, ctx_src)
.await?;
@ -2445,7 +2362,7 @@ impl RustyPipeQuery {
// Remove the used visitor data from cache if the request resulted in a recoverable error
self.remove_visitor_data(&resp.visitor_data);
if n != self.client.inner.n_http_retries {
if n != self.client.inner.n_request_attempts {
let ms = util::retry_delay(n, 1000, 60000, 3);
tracing::warn!(
"Retry attempt #{}. Error: {}. Waiting {} ms",
@ -2520,7 +2437,7 @@ impl RustyPipeQuery {
if level > Level::DBG || self.opts.report {
if let Some(reporter) = &self.client.inner.reporter {
let report = Report {
info: self.rp_info(),
info: RustyPipeInfo::new(Some(self.opts.lang)),
level,
operation: &format!("{operation}({id})"),
error,
@ -2720,46 +2637,6 @@ fn local_tz_offset() -> (String, i16) {
}
}
/// Check if a valid Botguard binary is available at the given location
fn detect_botguard_bin(program: OsString) -> Result<(OsString, String), Error> {
let out = std::process::Command::new(&program)
.arg("--version")
.output()
.map_err(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
Error::Other("rustypipe-botguard binary not found".into())
} else {
Error::Other(format!("error calling rustypipe-botguard {e}").into())
}
})?;
if !out.status.success() {
return Err(Error::Extraction(ExtractionError::Botguard(
format!("version check failed with status {}", out.status).into(),
)));
}
let output = String::from_utf8_lossy(&out.stdout);
let pat = "rustypipe-botguard-api ";
let pos = output.find(pat).ok_or(Error::Other(
"no rustypipe-botguard-api version returned".into(),
))? + pat.len();
let pos_end = output[pos..]
.char_indices()
.find(|(_, c)| !c.is_ascii_digit())
.map(|(p, _)| p + pos)
.unwrap_or(output.len());
let api_version = &output[pos..pos_end];
if api_version != BOTGUARD_API_VERSION {
return Err(Error::Other(
format!(
"incompatible rustypipe-botguard-api version {api_version}, expected {BOTGUARD_API_VERSION}"
)
.into(),
));
}
let version = output[..pos].lines().next().unwrap_or_default().to_owned();
Ok((program, version))
}
#[cfg(test)]
mod tests {
use super::*;

View file

@ -154,24 +154,9 @@ fn map_artist_page(
ctx: &MapRespCtx<'_>,
skip_extendables: bool,
) -> Result<MapResult<(MusicArtist, bool)>, ExtractionError> {
let contents = match res.contents {
Some(c) => c,
None => {
if res.microformat.microformat_data_renderer.noindex {
return Err(ExtractionError::NotFound {
id: ctx.id.to_owned(),
msg: "no contents".into(),
});
} else {
return Err(ExtractionError::InvalidData("no contents".into()));
}
}
};
// dbg!(&res);
let header = res
.header
.ok_or(ExtractionError::InvalidData("no header".into()))?
.music_immersive_header_renderer;
let header = res.header.music_immersive_header_renderer;
if let Some(share) = header.share_endpoint {
let pb = share.share_entity_endpoint.serialized_share_entity;
@ -188,7 +173,8 @@ fn map_artist_page(
}
}
let sections = contents
let sections = res
.contents
.single_column_browse_results_renderer
.contents
.into_iter()
@ -352,6 +338,8 @@ impl MapResponse<FirstAlbumPage> for response::MusicArtistAlbums {
self,
ctx: &MapRespCtx<'_>,
) -> Result<MapResult<FirstAlbumPage>, ExtractionError> {
// dbg!(&self);
let Some(header) = self.header else {
return Err(ExtractionError::NotFound {
id: ctx.id.into(),

View file

@ -105,6 +105,8 @@ impl MapResponse<Vec<MusicGenreItem>> for response::MusicGenres {
impl MapResponse<MusicGenre> for response::MusicGenre {
fn map_response(self, ctx: &MapRespCtx<'_>) -> Result<MapResult<MusicGenre>, ExtractionError> {
// dbg!(&self);
let content = self
.contents
.single_column_browse_results_renderer

View file

@ -8,7 +8,7 @@ use crate::{
error::{Error, ExtractionError},
model::{
paginator::{ContinuationEndpoint, Paginator},
AlbumItem, ArtistItem, HistoryItem, MusicPlaylist, MusicPlaylistItem, TrackItem,
AlbumItem, ArtistItem, HistoryItem, MusicPlaylistItem, TrackItem,
},
serializer::MapResult,
};
@ -127,20 +127,6 @@ impl RustyPipeQuery {
)
.await
}
/// Get all liked YouTube Music tracks of the logged-in user
///
/// The difference to [`RustyPipeQuery::music_saved_tracks`] is that this function only returns
/// tracks that were explicitly liked by the user.
///
/// Requires authentication cookies.
pub async fn music_liked_tracks(&self) -> Result<MusicPlaylist, Error> {
self.clone()
.authenticated()
.music_playlist("LM")
.await
.map_err(crate::util::map_internal_playlist_err)
}
}
impl MapResponse<Paginator<HistoryItem<TrackItem>>> for response::MusicHistory {
@ -209,7 +195,7 @@ mod tests {
#[test]
fn map_history() {
let json_path = path!(*TESTFILES / "music_userdata" / "music_history.json");
let json_path = path!(*TESTFILES / "music_history" / "music_history.json");
let json_file = File::open(json_path).unwrap();
let history: response::MusicHistory =

View file

@ -9,7 +9,7 @@ use crate::{
AlbumId, ChannelId, MusicAlbum, MusicPlaylist, TrackItem, TrackType,
},
serializer::{text::TextComponents, MapResult},
util::{self, dictionary, TryRemove, DOT_SEPARATOR},
util::{self, TryRemove, DOT_SEPARATOR},
};
use self::response::url_endpoint::MusicPageType;
@ -95,21 +95,11 @@ impl RustyPipeQuery {
})
.collect::<Vec<_>>();
let last_tn = album
.tracks
.last()
.and_then(|t| t.track_nr)
.unwrap_or_default();
if !to_replace.is_empty() || last_tn < album.track_count {
tracing::debug!(
"fetching album playlist ({} tracks, {} to replace)",
album.track_count,
to_replace.len()
);
if !to_replace.is_empty() {
let mut playlist = self.music_playlist(playlist_id).await?;
playlist
.tracks
.extend_limit(&self, album.track_count.into())
.extend_limit(&self, album.tracks.len())
.await?;
for (i, title) in to_replace {
@ -128,22 +118,24 @@ impl RustyPipeQuery {
album.tracks[i].track_type = TrackType::Track;
}
}
// Extend the list of album tracks with the ones from the playlist if the playlist returned more tracks
// This is the case for albums with more than 200 tracks (e.g. audiobooks)
if album.tracks.len() < playlist.tracks.items.len() {
let mut tn = last_tn;
for mut t in playlist.tracks.items.into_iter().skip(album.tracks.len()) {
tn += 1;
t.album = album.tracks.first().and_then(|t| t.album.clone());
t.track_nr = Some(tn);
album.tracks.push(t);
}
}
}
}
Ok(album)
}
/// Get all liked YouTube Music tracks of the logged-in user
///
/// The difference to [`RustyPipeQuery::music_saved_tracks`] is that this function only returns
/// tracks that were explicitly liked by the user.
///
/// Requires authentication cookies.
pub async fn music_liked_tracks(&self) -> Result<MusicPlaylist, Error> {
self.clone()
.authenticated()
.music_playlist("LM")
.await
.map_err(util::map_internal_playlist_err)
}
}
impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
@ -151,21 +143,9 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
self,
ctx: &MapRespCtx<'_>,
) -> Result<MapResult<MusicPlaylist>, ExtractionError> {
let contents = match self.contents {
Some(c) => c,
None => {
if self.microformat.microformat_data_renderer.noindex {
return Err(ExtractionError::NotFound {
id: ctx.id.to_owned(),
msg: "no contents".into(),
});
} else {
return Err(ExtractionError::InvalidData("no contents".into()));
}
}
};
// dbg!(&self);
let (header, music_contents) = match contents {
let (header, music_contents) = match self.contents {
response::music_playlist::Contents::SingleColumnBrowseResultsRenderer(c) => (
self.header,
c.contents
@ -352,21 +332,9 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
impl MapResponse<MusicAlbum> for response::MusicPlaylist {
fn map_response(self, ctx: &MapRespCtx<'_>) -> Result<MapResult<MusicAlbum>, ExtractionError> {
let contents = match self.contents {
Some(c) => c,
None => {
if self.microformat.microformat_data_renderer.noindex {
return Err(ExtractionError::NotFound {
id: ctx.id.to_owned(),
msg: "no contents".into(),
});
} else {
return Err(ExtractionError::InvalidData("no contents".into()));
}
}
};
// dbg!(&self);
let (header, sections) = match contents {
let (header, sections) = match self.contents {
response::music_playlist::Contents::SingleColumnBrowseResultsRenderer(c) => (
self.header,
c.contents
@ -406,19 +374,8 @@ impl MapResponse<MusicAlbum> for response::MusicPlaylist {
match section {
response::music_item::ItemSection::MusicShelfRenderer(sh) => shelf = Some(sh),
response::music_item::ItemSection::MusicCarouselShelfRenderer(sh) => {
if sh
.header
.map(|h| {
h.music_carousel_shelf_basic_header_renderer
.title
.first_str()
== dictionary::entry(ctx.lang).album_versions_title
})
.unwrap_or_default()
{
album_variants = Some(sh.contents);
}
}
_ => (),
}
}
@ -482,12 +439,10 @@ impl MapResponse<MusicAlbum> for response::MusicPlaylist {
}
}
let playlist_id = self
.microformat
.microformat_data_renderer
let playlist_id = self.microformat.and_then(|mf| {
mf.microformat_data_renderer
.url_canonical
.and_then(|x| {
x.strip_prefix("https://music.youtube.com/playlist?list=")
.strip_prefix("https://music.youtube.com/playlist?list=")
.map(str::to_owned)
});
let (playlist_id, artist_id) = header
@ -516,14 +471,6 @@ impl MapResponse<MusicAlbum> for response::MusicPlaylist {
.unwrap_or_default();
let artist_id = artist_id.or_else(|| artists.first().and_then(|a| a.id.clone()));
let second_subtitle_parts = header
.second_subtitle
.split(|p| p == DOT_SEPARATOR)
.collect::<Vec<_>>();
let track_count = second_subtitle_parts
.get(usize::from(second_subtitle_parts.len() > 2))
.and_then(|txt| util::parse_numeric::<u16>(&txt[0]).ok());
let mut mapper = MusicListMapper::with_album(
ctx.lang,
artists.clone(),
@ -558,7 +505,6 @@ impl MapResponse<MusicAlbum> for response::MusicPlaylist {
album_type,
year,
by_va,
track_count: track_count.unwrap_or(tracks_res.c.len() as u16),
tracks: tracks_res.c,
variants: variants_res.c,
},
@ -609,8 +555,8 @@ mod tests {
#[case::single("single", "MPREb_bHfHGoy7vuv")]
#[case::description("description", "MPREb_PiyfuVl6aYd")]
#[case::unavailable("unavailable", "MPREb_AzuWg8qAVVl")]
#[case::unavailable("unavailable", "MPREb_AzuWg8qAVVl")]
#[case::two_columns("20240228_twoColumns", "MPREb_bHfHGoy7vuv")]
#[case::recommends("20250225_recommends", "MPREb_u1I69lSAe5v")]
fn map_music_album(#[case] name: &str, #[case] id: &str) {
let json_path = path!(*TESTFILES / "music_playlist" / format!("album_{name}.json"));
let json_file = File::open(json_path).unwrap();

View file

@ -155,6 +155,8 @@ impl<T: FromYtItem> MapResponse<MusicSearchResult<T>> for response::MusicSearch
self,
ctx: &MapRespCtx<'_>,
) -> Result<MapResult<MusicSearchResult<T>>, ExtractionError> {
// dbg!(&self);
let tabs = self.contents.tabbed_search_results_renderer.contents;
let sections = tabs
.into_iter()

View file

@ -6,15 +6,12 @@ use crate::model::{
traits::FromYtItem,
Comment, MusicItem, YouTubeItem,
};
use crate::model::{HistoryItem, TrackItem, VideoItem};
use crate::serializer::MapResult;
#[cfg(feature = "userdata")]
use crate::model::{HistoryItem, TrackItem, VideoItem};
use self::response::YouTubeListItem;
use super::response::{
music_item::{map_queue_item, MusicListMapper, PlaylistPanelVideo},
YouTubeListItem,
};
use super::response::music_item::{map_queue_item, MusicListMapper, PlaylistPanelVideo};
use super::{
response, ClientType, MapRespCtx, MapRespOptions, MapResponse, QContinuation, RustyPipeQuery,
};
@ -228,7 +225,6 @@ impl MapResponse<Paginator<MusicItem>> for response::MusicContinuation {
}
}
#[cfg(feature = "userdata")]
impl MapResponse<Paginator<HistoryItem<VideoItem>>> for response::Continuation {
fn map_response(
self,
@ -249,9 +245,11 @@ impl MapResponse<Paginator<HistoryItem<VideoItem>>> for response::Continuation {
&mut map_res,
);
}
response::YouTubeListItem::ContinuationItemRenderer(ep) => {
response::YouTubeListItem::ContinuationItemRenderer {
continuation_endpoint,
} => {
if ctoken.is_none() {
ctoken = ep.continuation_endpoint.into_token();
ctoken = Some(continuation_endpoint.continuation_command.token);
}
}
_ => {}
@ -272,7 +270,6 @@ impl MapResponse<Paginator<HistoryItem<VideoItem>>> for response::Continuation {
}
}
#[cfg(feature = "userdata")]
impl MapResponse<Paginator<HistoryItem<TrackItem>>> for response::MusicContinuation {
fn map_response(
self,
@ -425,8 +422,6 @@ impl Paginator<Comment> {
}
}
#[cfg(feature = "userdata")]
#[cfg_attr(docsrs, doc(cfg(feature = "userdata")))]
impl Paginator<HistoryItem<VideoItem>> {
/// Get the next page from the paginator (or `None` if the paginator is exhausted)
pub async fn next<Q: AsRef<RustyPipeQuery>>(&self, query: Q) -> Result<Option<Self>, Error> {
@ -442,8 +437,6 @@ impl Paginator<HistoryItem<VideoItem>> {
}
}
#[cfg(feature = "userdata")]
#[cfg_attr(docsrs, doc(cfg(feature = "userdata")))]
impl Paginator<HistoryItem<TrackItem>> {
/// Get the next page from the paginator (or `None` if the paginator is exhausted)
pub async fn next<Q: AsRef<RustyPipeQuery>>(&self, query: Q) -> Result<Option<Self>, Error> {
@ -540,11 +533,7 @@ macro_rules! paginator {
}
paginator!(Comment);
#[cfg(feature = "userdata")]
#[cfg_attr(docsrs, doc(cfg(feature = "userdata")))]
paginator!(HistoryItem<VideoItem>);
#[cfg(feature = "userdata")]
#[cfg_attr(docsrs, doc(cfg(feature = "userdata")))]
paginator!(HistoryItem<TrackItem>);
#[cfg(test)]
@ -631,7 +620,7 @@ mod tests {
}
#[rstest]
#[case::subscriptions("subscriptions", path!("userdata" / "subscriptions.json"))]
#[case::subscriptions("subscriptions", path!("history" / "subscriptions.json"))]
fn map_continuation_channels(#[case] name: &str, #[case] path: PathBuf) {
let json_path = path!(*TESTFILES / path);
let json_file = File::open(json_path).unwrap();
@ -655,7 +644,7 @@ mod tests {
#[case::playlist_tracks("playlist_tracks", path!("music_playlist" / "playlist_cont.json"))]
#[case::search_tracks("search_tracks", path!("music_search" / "tracks_cont.json"))]
#[case::radio_tracks("radio_tracks", path!("music_details" / "radio_cont.json"))]
#[case::saved_tracks("saved_tracks", path!("music_userdata" / "saved_tracks.json"))]
#[case::saved_tracks("saved_tracks", path!("music_history" / "saved_tracks.json"))]
fn map_continuation_tracks(#[case] name: &str, #[case] path: PathBuf) {
let json_path = path!(*TESTFILES / path);
let json_file = File::open(json_path).unwrap();
@ -676,7 +665,7 @@ mod tests {
}
#[rstest]
#[case::saved_artists("saved_artists", path!("music_userdata" / "saved_artists.json"))]
#[case::saved_artists("saved_artists", path!("music_history" / "saved_artists.json"))]
fn map_continuation_artists(#[case] name: &str, #[case] path: PathBuf) {
let json_path = path!(*TESTFILES / path);
let json_file = File::open(json_path).unwrap();
@ -697,7 +686,7 @@ mod tests {
}
#[rstest]
#[case::saved_albums("saved_albums", path!("music_userdata" / "saved_albums.json"))]
#[case::saved_albums("saved_albums", path!("music_history" / "saved_albums.json"))]
fn map_continuation_albums(#[case] name: &str, #[case] path: PathBuf) {
let json_path = path!(*TESTFILES / path);
let json_file = File::open(json_path).unwrap();
@ -719,7 +708,7 @@ mod tests {
#[rstest]
#[case::playlist_related("playlist_related", path!("music_playlist" / "playlist_related.json"))]
#[case::saved_playlists("saved_playlists", path!("music_userdata" / "saved_playlists.json"))]
#[case::saved_playlists("saved_playlists", path!("music_history" / "saved_playlists.json"))]
fn map_continuation_music_playlists(#[case] name: &str, #[case] path: PathBuf) {
let json_path = path!(*TESTFILES / path);
let json_file = File::open(json_path).unwrap();

View file

@ -1,6 +1,6 @@
use std::{
borrow::Cow,
collections::{BTreeMap, HashMap, HashSet},
collections::{BTreeMap, HashMap},
fmt::Debug,
};
@ -104,29 +104,42 @@ impl RustyPipeQuery {
) -> Result<VideoPlayer, Error> {
let video_id = video_id.as_ref();
let mut last_e = None;
let mut query = Cow::Borrowed(self);
let mut clients_iter = clients.iter().peekable();
let mut failed_clients = HashSet::new();
while let Some(client) = clients_iter.next() {
if query.opts.auth == Some(true) && !self.auth_enabled(*client) {
if self.opts.auth == Some(true) && !self.auth_enabled(*client) {
// If no client has auth enabled, return NoLogin error instead of "no clients"
if last_e.is_none() {
last_e = Some(Error::Auth(AuthError::NoLogin));
}
continue;
}
if failed_clients.contains(client) {
continue;
}
let res = query.player_from_client(video_id, *client).await;
let res = self.player_from_client(video_id, *client).await;
match res {
Ok(res) => return Ok(res),
Err(Error::Extraction(e)) => {
if e.use_login() && query.opts.auth.is_none() {
clients_iter = clients.iter().peekable();
query = Cow::Owned(self.clone().authenticated());
if e.use_login() {
if let Some(c) = self.auth_enabled_client(clients) {
tracing::info!("{e}; fetching player with login");
match self
.clone()
.authenticated()
.player_from_client(video_id, c)
.await
{
Ok(res) => return Ok(res),
Err(Error::Extraction(e)) => {
if !e.switch_client() {
return Err(Error::Extraction(e));
}
}
Err(e) => return Err(e),
}
} else {
return Err(Error::Extraction(e));
}
} else if !e.switch_client() {
return Err(Error::Extraction(e));
}
@ -134,7 +147,6 @@ impl RustyPipeQuery {
tracing::warn!("error fetching player with {client:?} client: {e}; retrying with {next_client:?} client");
}
last_e = Some(Error::Extraction(e));
failed_clients.insert(*client);
}
Err(e) => return Err(e),
}
@ -144,27 +156,22 @@ impl RustyPipeQuery {
async fn get_player_po_token(&self, video_id: &str) -> Result<PlayerPoToken, Error> {
if let Some(bg) = &self.client.inner.botguard {
let (ident, visitor_data) = if self.opts.auth == Some(true) {
(self.client.user_auth_datasync_id()?, None)
} else {
let visitor_data = self.get_visitor_data(false).await?;
(visitor_data.to_owned(), Some(visitor_data))
};
if bg.po_token_cache {
let session_token = self.get_session_po_token(&ident).await?;
let session_token = self.get_session_po_token(&visitor_data).await?;
Ok(PlayerPoToken {
visitor_data,
visitor_data: Some(visitor_data),
session_po_token: Some(session_token),
content_po_token: None,
})
} else {
let (po_tokens, valid_until) = self.get_po_tokens(&[video_id, &ident]).await?;
let (po_tokens, valid_until) =
self.get_po_tokens(&[video_id, &visitor_data]).await?;
let mut po_tokens = po_tokens.into_iter();
let po_token = po_tokens.next().unwrap();
let session_po_token = po_tokens.next().unwrap();
Ok(PlayerPoToken {
visitor_data,
visitor_data: Some(visitor_data),
session_po_token: Some(PoToken {
po_token: session_po_token,
valid_until,
@ -184,11 +191,6 @@ impl RustyPipeQuery {
video_id: S,
client_type: ClientType,
) -> Result<VideoPlayer, Error> {
if self.opts.auth == Some(true) {
tracing::info!("fetching {client_type:?} player with login");
} else {
tracing::debug!("fetching {client_type:?} player");
}
let video_id = video_id.as_ref();
let (deobf, player_po) = tokio::try_join!(
@ -383,21 +385,6 @@ impl MapResponse<VideoPlayer> for response::Player {
video_details.video_id, ctx.id
)));
}
// Sometimes YouTube Desktop does not output any URLs for adaptive streams.
// Since this is currently rare, it is best to retry the request in this case.
if !is_live
&& !streaming_data.adaptive_formats.c.is_empty()
&& streaming_data
.adaptive_formats
.c
.iter()
.all(|f| f.url.is_none() && f.signature_cipher.is_none())
{
return Err(ExtractionError::Unavailable {
reason: UnavailabilityReason::TryAgain,
msg: "no adaptive stream URLs".to_owned(),
});
}
let video_info = VideoPlayerDetails {
id: video_details.video_id,
@ -640,7 +627,7 @@ impl<'a> StreamsMapper<'a> {
fn deobf(&self) -> Result<&Deobfuscator, DeobfError> {
self.deobf
.as_ref()
.ok_or(DeobfError::Other("no deobfuscator".into()))
.ok_or(DeobfError::Other("no deobfuscator"))
}
fn cipher_to_url_params(

View file

@ -33,6 +33,28 @@ impl RustyPipeQuery {
)
.await
}
/// Get all liked videos of the logged-in user
///
/// Requires authentication cookies.
pub async fn liked_videos(&self) -> Result<Playlist, Error> {
self.clone()
.authenticated()
.playlist("LL")
.await
.map_err(util::map_internal_playlist_err)
}
/// Get the "Watch later" playlist of the logged-in user
///
/// Requires authentication cookies.
pub async fn watch_later(&self) -> Result<Playlist, Error> {
self.clone()
.authenticated()
.playlist("WL")
.await
.map_err(util::map_internal_playlist_err)
}
}
impl MapResponse<Playlist> for response::Playlist {
@ -257,7 +279,6 @@ mod tests {
#[case::nomusic("nomusic", "PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe")]
#[case::live("live", "UULVvqRdlKsE5Q8mf8YXbdIJLw")]
#[case::pageheader("20241011_pageheader", "PLT2w2oBf1TZKyvY_M6JsASs73m-wjLzH5")]
#[case::cmdexecutor("20250316_cmdexecutor", "PLbZIPy20-1pN7mqjckepWF78ndb6ci_qi")]
fn map_playlist_data(#[case] name: &str, #[case] id: &str) {
let json_path = path!(*TESTFILES / "playlist" / format!("playlist_{name}.json"));
let json_file = File::open(json_path).unwrap();

View file

@ -1,8 +1,10 @@
pub(crate) mod channel;
pub(crate) mod history;
pub(crate) mod music_artist;
pub(crate) mod music_charts;
pub(crate) mod music_details;
pub(crate) mod music_genres;
pub(crate) mod music_history;
pub(crate) mod music_item;
pub(crate) mod music_new;
pub(crate) mod music_playlist;
@ -17,6 +19,7 @@ pub(crate) mod video_item;
pub(crate) use channel::Channel;
pub(crate) use channel::ChannelAbout;
pub(crate) use history::History;
pub(crate) use music_artist::MusicArtist;
pub(crate) use music_artist::MusicArtistAlbums;
pub(crate) use music_charts::MusicCharts;
@ -25,6 +28,7 @@ pub(crate) use music_details::MusicLyrics;
pub(crate) use music_details::MusicRelated;
pub(crate) use music_genres::MusicGenre;
pub(crate) use music_genres::MusicGenres;
pub(crate) use music_history::MusicHistory;
pub(crate) use music_item::MusicContinuation;
pub(crate) use music_new::MusicNew;
pub(crate) use music_playlist::MusicPlaylist;
@ -47,15 +51,6 @@ pub(crate) mod channel_rss;
#[cfg(feature = "rss")]
pub(crate) use channel_rss::ChannelRss;
#[cfg(feature = "userdata")]
pub(crate) mod history;
#[cfg(feature = "userdata")]
pub(crate) use history::History;
#[cfg(feature = "userdata")]
pub(crate) mod music_history;
#[cfg(feature = "userdata")]
pub(crate) use music_history::MusicHistory;
use std::borrow::Cow;
use std::collections::HashMap;
use std::marker::PhantomData;
@ -152,16 +147,9 @@ pub(crate) struct ContinuationItemRenderer {
pub continuation_endpoint: ContinuationEndpoint,
}
#[derive(Debug, Deserialize)]
#[serde(untagged)]
pub(crate) enum ContinuationEndpoint {
ContinuationCommand(ContinuationCommandWrap),
CommandExecutorCommand(CommandExecutorCommandWrap),
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ContinuationCommandWrap {
pub(crate) struct ContinuationEndpoint {
pub continuation_command: ContinuationCommand,
}
@ -171,34 +159,7 @@ pub(crate) struct ContinuationCommand {
pub token: String,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct CommandExecutorCommandWrap {
pub command_executor_command: CommandExecutorCommand,
}
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct CommandExecutorCommand {
#[serde_as(as = "VecSkipError<_>")]
commands: Vec<ContinuationCommandWrap>,
}
impl ContinuationEndpoint {
pub fn into_token(self) -> Option<String> {
match self {
Self::ContinuationCommand(cmd) => Some(cmd.continuation_command.token),
Self::CommandExecutorCommand(cmd) => cmd
.command_executor_command
.commands
.into_iter()
.next()
.map(|c| c.continuation_command.token),
}
}
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct Icon {

View file

@ -5,8 +5,7 @@ use crate::serializer::text::Text;
use super::{
music_item::{
Button, Grid, ItemSection, MusicMicroformat, MusicThumbnailRenderer, SimpleHeader,
SingleColumnBrowseResult,
Button, Grid, ItemSection, MusicThumbnailRenderer, SimpleHeader, SingleColumnBrowseResult,
},
SectionList, Tab,
};
@ -15,10 +14,8 @@ use super::{
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct MusicArtist {
pub contents: Option<SingleColumnBrowseResult<Tab<SectionList<ItemSection>>>>,
pub header: Option<Header>,
#[serde(default)]
pub microformat: MusicMicroformat,
pub contents: SingleColumnBrowseResult<Tab<SectionList<ItemSection>>>,
pub header: Header,
}
#[derive(Debug, Deserialize)]

View file

@ -1,10 +1,11 @@
use serde::Deserialize;
use serde_with::{rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError};
use time::UtcOffset;
use crate::{
model::{
self, traits::FromYtItem, AlbumId, AlbumItem, AlbumType, ArtistId, ArtistItem, ChannelId,
MusicItem, MusicItemType, MusicPlaylistItem, TrackItem, UserItem,
HistoryItem, MusicItem, MusicItemType, MusicPlaylistItem, TrackItem, UserItem,
},
param::Language,
serializer::{
@ -22,11 +23,6 @@ use super::{
SimpleHeaderRenderer, Thumbnails, ThumbnailsWrap,
};
#[cfg(feature = "userdata")]
use crate::model::HistoryItem;
#[cfg(feature = "userdata")]
use time::UtcOffset;
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) enum ItemSection {
@ -44,7 +40,6 @@ pub(crate) enum ItemSection {
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct MusicShelf {
#[cfg(feature = "userdata")]
#[serde_as(as = "Option<Text>")]
pub title: Option<String>,
/// Playlist ID (only for playlists)
@ -433,22 +428,6 @@ pub(crate) enum TrackBadge {
LiveBadgeRenderer {},
}
#[serde_as]
#[derive(Default, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct MusicMicroformat {
#[serde_as(as = "DefaultOnError")]
pub microformat_data_renderer: MicroformatData,
}
#[derive(Default, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct MicroformatData {
pub url_canonical: Option<String>,
#[serde(default)]
pub noindex: bool,
}
/*
#MAPPER
*/
@ -546,9 +525,7 @@ impl MusicListMapper {
MusicResponseItem::ContinuationItemRenderer {
continuation_endpoint,
} => {
if self.ctoken.is_none() {
self.ctoken = continuation_endpoint.into_token();
}
self.ctoken = Some(continuation_endpoint.continuation_command.token);
Ok(None)
}
}
@ -1293,7 +1270,6 @@ impl MusicListMapper {
}
}
#[cfg(feature = "userdata")]
pub fn conv_history_items(
self,
date_txt: Option<String>,

View file

@ -5,21 +5,22 @@ use crate::serializer::text::{AttributedText, Text, TextComponents};
use super::{
music_item::{
Button, ItemSection, MusicContentsRenderer, MusicItemMenuEntry, MusicMicroformat,
MusicThumbnailRenderer,
Button, ItemSection, MusicContentsRenderer, MusicItemMenuEntry, MusicThumbnailRenderer,
},
url_endpoint::OnTapWrap,
ContentsRenderer, SectionList, Tab,
};
/// Response model for YouTube Music playlists and albums
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct MusicPlaylist {
pub contents: Option<Contents>,
pub contents: Contents,
pub header: Option<Header>,
#[serde(default)]
pub microformat: MusicMicroformat,
#[serde_as(as = "DefaultOnError")]
pub microformat: Option<Microformat>,
}
#[serde_as]
@ -161,3 +162,15 @@ pub(crate) struct AvatarStackViewModel {
pub(crate) struct AvatarStackRendererContext {
pub command_context: Option<OnTapWrap>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct Microformat {
pub microformat_data_renderer: MicroformatData,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct MicroformatData {
pub url_canonical: String,
}

View file

@ -530,14 +530,15 @@ pub(crate) enum ContinuationItemVariants {
}
impl ContinuationItemVariants {
pub fn into_token(self) -> Option<String> {
pub fn token(self) -> String {
match self {
ContinuationItemVariants::Ep {
continuation_endpoint,
} => continuation_endpoint,
ContinuationItemVariants::Btn { button } => button.button_renderer.command,
}
.into_token()
.continuation_command
.token
}
}

View file

@ -2,11 +2,14 @@ use serde::Deserialize;
use serde_with::{
rust::deserialize_ignore_any, serde_as, DefaultOnError, DisplayFromStr, VecSkipError,
};
use time::OffsetDateTime;
use time::{OffsetDateTime, UtcOffset};
use super::{ChannelBadge, ContentImage, ContinuationItemRenderer, PhMetadataView, Thumbnails};
use super::{
ChannelBadge, ContentImage, ContinuationEndpoint, PhMetadataView, SimpleHeaderRenderer,
Thumbnails,
};
use crate::{
model::{Channel, ChannelItem, ChannelTag, PlaylistItem, VideoItem, YouTubeItem},
model::{Channel, ChannelItem, ChannelTag, HistoryItem, PlaylistItem, VideoItem, YouTubeItem},
param::Language,
serializer::{
text::{AttributedText, Text, TextComponent},
@ -15,11 +18,6 @@ use crate::{
util::{self, timeago, TryRemove},
};
#[cfg(feature = "userdata")]
use crate::{client::response::SimpleHeaderRenderer, model::HistoryItem};
#[cfg(feature = "userdata")]
use time::UtcOffset;
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
@ -37,9 +35,12 @@ pub(crate) enum YouTubeListItem {
LockupViewModel(LockupViewModel),
/// Continuation items are located at the end of a list
/// Continauation items are located at the end of a list
/// and contain the continuation token for progressive loading
ContinuationItemRenderer(ContinuationItemRenderer),
#[serde(rename_all = "camelCase")]
ContinuationItemRenderer {
continuation_endpoint: ContinuationEndpoint,
},
/// Corrected search query
#[serde(rename_all = "camelCase")]
@ -65,7 +66,6 @@ pub(crate) enum YouTubeListItem {
/// GridRenderer: contains videos on channel page
#[serde(alias = "expandedShelfContentsRenderer", alias = "gridRenderer")]
ItemSectionRenderer {
#[cfg(feature = "userdata")]
header: Option<ItemSectionHeader>,
#[serde(alias = "items")]
contents: MapResult<Vec<YouTubeListItem>>,
@ -298,7 +298,6 @@ pub(crate) struct YouTubeListRenderer {
pub contents: MapResult<Vec<YouTubeListItem>>,
}
#[cfg(feature = "userdata")]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ItemSectionHeader {
@ -835,11 +834,9 @@ impl YouTubeListMapper<YouTubeItem> {
self.items.push(mapped);
}
}
YouTubeListItem::ContinuationItemRenderer(r) => {
if self.ctoken.is_none() {
self.ctoken = r.continuation_endpoint.into_token();
}
}
YouTubeListItem::ContinuationItemRenderer {
continuation_endpoint,
} => self.ctoken = Some(continuation_endpoint.continuation_command.token),
YouTubeListItem::ShowingResultsForRenderer { corrected_query } => {
self.corrected_query = Some(corrected_query);
}
@ -885,11 +882,9 @@ impl YouTubeListMapper<VideoItem> {
self.items.push(mapped);
}
}
YouTubeListItem::ContinuationItemRenderer(r) => {
if self.ctoken.is_none() {
self.ctoken = r.continuation_endpoint.into_token();
}
}
YouTubeListItem::ContinuationItemRenderer {
continuation_endpoint,
} => self.ctoken = Some(continuation_endpoint.continuation_command.token),
YouTubeListItem::ShowingResultsForRenderer { corrected_query } => {
self.corrected_query = Some(corrected_query);
}
@ -909,7 +904,6 @@ impl YouTubeListMapper<VideoItem> {
res.c.into_iter().for_each(|item| self.map_item(item));
}
#[cfg(feature = "userdata")]
pub(crate) fn conv_history_items(
self,
date_txt: Option<String>,
@ -939,11 +933,9 @@ impl YouTubeListMapper<PlaylistItem> {
self.items.push(mapped);
}
}
YouTubeListItem::ContinuationItemRenderer(r) => {
if self.ctoken.is_none() {
self.ctoken = r.continuation_endpoint.into_token();
}
}
YouTubeListItem::ContinuationItemRenderer {
continuation_endpoint,
} => self.ctoken = Some(continuation_endpoint.continuation_command.token),
YouTubeListItem::ShowingResultsForRenderer { corrected_query } => {
self.corrected_query = Some(corrected_query);
}

View file

@ -43,7 +43,6 @@ MusicAlbum(
album_type: single,
year: Some(2020),
by_va: false,
track_count: 1,
tracks: [
TrackItem(
id: "XX0epju-YvY",

View file

@ -1,151 +0,0 @@
---
source: src/client/music_playlist.rs
expression: map_res.c
---
MusicAlbum(
id: "MPREb_u1I69lSAe5v",
playlist_id: Some("OLAK5uy_lGP_zv0vJDUlecQDzugUJmjcF7pvyVNyY"),
name: "Waldbrand",
cover: [
Thumbnail(
url: "https://lh3.googleusercontent.com/IYxE8yTIpFUu0OayA5SaxFEn6zQ7T21hpkvI8CODY9NEH1XIhyoUhGohkZuaK-xSu22BC4wjp6srNjIW=w60-h60-l90-rj",
width: 60,
height: 60,
),
Thumbnail(
url: "https://lh3.googleusercontent.com/IYxE8yTIpFUu0OayA5SaxFEn6zQ7T21hpkvI8CODY9NEH1XIhyoUhGohkZuaK-xSu22BC4wjp6srNjIW=w120-h120-l90-rj",
width: 120,
height: 120,
),
Thumbnail(
url: "https://lh3.googleusercontent.com/IYxE8yTIpFUu0OayA5SaxFEn6zQ7T21hpkvI8CODY9NEH1XIhyoUhGohkZuaK-xSu22BC4wjp6srNjIW=w226-h226-l90-rj",
width: 226,
height: 226,
),
Thumbnail(
url: "https://lh3.googleusercontent.com/IYxE8yTIpFUu0OayA5SaxFEn6zQ7T21hpkvI8CODY9NEH1XIhyoUhGohkZuaK-xSu22BC4wjp6srNjIW=w544-h544-l90-rj",
width: 544,
height: 544,
),
],
artists: [
ArtistId(
id: Some("UCpJyCbFbdTrx0M90HCNBHFQ"),
name: "Madeline Juno",
),
],
artist_id: Some("UCpJyCbFbdTrx0M90HCNBHFQ"),
description: None,
album_type: ep,
year: Some(2016),
by_va: false,
track_count: 5,
tracks: [
TrackItem(
id: "aGd3VKSOTxY",
name: "Ich wache auf",
duration: Some(222),
cover: [],
artists: [
ArtistId(
id: Some("UCpJyCbFbdTrx0M90HCNBHFQ"),
name: "Madeline Juno",
),
],
artist_id: Some("UCpJyCbFbdTrx0M90HCNBHFQ"),
album: Some(AlbumId(
id: "MPREb_u1I69lSAe5v",
name: "Waldbrand",
)),
view_count: Some(208000),
track_type: track,
track_nr: Some(1),
by_va: false,
),
TrackItem(
id: "lhPOMUjV4rE",
name: "Waldbrand",
duration: Some(209),
cover: [],
artists: [
ArtistId(
id: Some("UCpJyCbFbdTrx0M90HCNBHFQ"),
name: "Madeline Juno",
),
],
artist_id: Some("UCpJyCbFbdTrx0M90HCNBHFQ"),
album: Some(AlbumId(
id: "MPREb_u1I69lSAe5v",
name: "Waldbrand",
)),
view_count: Some(6000000),
track_type: video,
track_nr: Some(2),
by_va: false,
),
TrackItem(
id: "Bu26uFtpt58",
name: "Verlernt",
duration: Some(224),
cover: [],
artists: [
ArtistId(
id: Some("UCpJyCbFbdTrx0M90HCNBHFQ"),
name: "Madeline Juno",
),
],
artist_id: Some("UCpJyCbFbdTrx0M90HCNBHFQ"),
album: Some(AlbumId(
id: "MPREb_u1I69lSAe5v",
name: "Waldbrand",
)),
view_count: Some(418000),
track_type: track,
track_nr: Some(3),
by_va: false,
),
TrackItem(
id: "RgwNqqiVqdY",
name: "In Farbe",
duration: Some(222),
cover: [],
artists: [
ArtistId(
id: Some("UCpJyCbFbdTrx0M90HCNBHFQ"),
name: "Madeline Juno",
),
],
artist_id: Some("UCpJyCbFbdTrx0M90HCNBHFQ"),
album: Some(AlbumId(
id: "MPREb_u1I69lSAe5v",
name: "Waldbrand",
)),
view_count: Some(127000),
track_type: track,
track_nr: Some(4),
by_va: false,
),
TrackItem(
id: "2TuOh30XbCI",
name: "Stadt im Hinterland",
duration: Some(198),
cover: [],
artists: [
ArtistId(
id: Some("UCpJyCbFbdTrx0M90HCNBHFQ"),
name: "Madeline Juno",
),
],
artist_id: Some("UCpJyCbFbdTrx0M90HCNBHFQ"),
album: Some(AlbumId(
id: "MPREb_u1I69lSAe5v",
name: "Waldbrand",
)),
view_count: Some(79000),
track_type: track,
track_nr: Some(5),
by_va: false,
),
],
variants: [],
)

View file

@ -43,7 +43,6 @@ MusicAlbum(
album_type: album,
year: Some(2015),
by_va: false,
track_count: 11,
tracks: [
TrackItem(
id: "YQHsXMglC9A",

View file

@ -39,7 +39,6 @@ MusicAlbum(
album_type: album,
year: Some(2016),
by_va: false,
track_count: 18,
tracks: [
TrackItem(
id: "g0iRiJ_ck48",

View file

@ -43,7 +43,6 @@ MusicAlbum(
album_type: single,
year: Some(2020),
by_va: false,
track_count: 1,
tracks: [
TrackItem(
id: "XX0epju-YvY",

View file

@ -34,7 +34,6 @@ MusicAlbum(
album_type: album,
year: Some(2019),
by_va: true,
track_count: 18,
tracks: [
TrackItem(
id: "JWeJHN5P-E8",

View file

@ -34,7 +34,6 @@ MusicAlbum(
album_type: single,
year: Some(2022),
by_va: true,
track_count: 6,
tracks: [
TrackItem(
id: "8IqLxg0GqXc",

View file

@ -208,10 +208,11 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
)
});
let comment_ctoken = comment_ctoken_section.and_then(|s| {
let comment_ctoken = comment_ctoken_section.map(|s| {
s.continuation_item_renderer
.continuation_endpoint
.into_token()
.continuation_command
.token
});
let (owner, description, is_ccommons) = match secondary_info {
@ -332,7 +333,7 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
.sub_menu_items;
items
.try_swap_remove(1)
.and_then(|c| c.service_endpoint.into_token())
.map(|c| c.service_endpoint.continuation_command.token)
});
Ok(MapResult {
@ -452,9 +453,7 @@ impl MapResponse<Paginator<Comment>> for response::VideoComments {
}
}
response::video_details::CommentListItem::ContinuationItemRenderer(cont) => {
if ctoken.is_none() {
ctoken = cont.into_token();
}
ctoken = Some(cont.token());
}
response::video_details::CommentListItem::CommentsHeaderRenderer { count_text } => {
comment_count = count_text
@ -521,9 +520,7 @@ fn map_replies(
))
}
response::video_details::CommentListItem::ContinuationItemRenderer(cont) => {
if reply_ctoken.is_none() {
reply_ctoken = cont.into_token();
}
reply_ctoken = Some(cont.token());
None
}
_ => None,

View file

@ -3,7 +3,7 @@ use std::collections::HashMap;
use once_cell::sync::Lazy;
use regex::Regex;
use reqwest::Client;
use ress::tokens::{Keyword, Punct, Token};
use ress::tokens::Token;
use rquickjs::{Context, Runtime};
use serde::{Deserialize, Serialize};
@ -39,7 +39,7 @@ impl DeobfData {
if let Err(e) = &res {
if let Some(reporter) = reporter {
let report = Report {
info: RustyPipeInfo::new(None, None),
info: RustyPipeInfo::new(None),
level: Level::ERR,
operation: "extract_deobf",
error: Some(e.to_string()),
@ -106,7 +106,7 @@ impl Deobfuscator {
.with(|ctx| call_fn(&ctx, DEOBF_NSIG_FUNC_NAME, nsig))?;
tracing::trace!("deobf nsig: {nsig} -> {res}");
if res.starts_with("enhanced_except_") || res.ends_with(nsig) {
return Err(DeobfError::Other("nsig fn returned an exception".into()));
return Err(DeobfError::Other("nsig fn returned an exception"));
}
Ok(res)
}
@ -134,21 +134,55 @@ fn caller_function(mapped_name: &str, fn_name: &str) -> String {
}
fn get_sig_fn(player_js: &str) -> Result<String, DeobfError> {
let name = get_sig_fn_name(player_js)?;
let code = extract_js_fn(player_js, &name)?;
let js_fn = format!("{}{}", code, caller_function(DEOBF_SIG_FUNC_NAME, &name));
let dfunc_name = get_sig_fn_name(player_js)?;
let function_pattern_str = format!(
r#"({}=function\([\w]+\)\{{.+?\}})"#,
dfunc_name.replace('$', "\\$")
);
let function_pattern = Regex::new(&function_pattern_str)
.map_err(|_| DeobfError::Other("could not parse sig fn pattern regex"))?;
let deobfuscate_function = format!(
"var {};",
&function_pattern
.captures(player_js)
.ok_or(DeobfError::Extraction("sig fn"))?[1]
);
let helper_object_name_pattern = Regex::new(r";([\w\$]{2,3})\...\(").unwrap();
let helper_object_name = helper_object_name_pattern
.captures(&deobfuscate_function)
.ok_or(DeobfError::Extraction("sig fn helper object name"))?
.get(1)
.unwrap()
.as_str();
let helper_pattern_str = format!(
r#"(var {}=\{{.+?\}}\}};)"#,
helper_object_name.replace('$', "\\$")
);
let helper_pattern = Regex::new(&helper_pattern_str)
.map_err(|_| DeobfError::Other("could not parse helper pattern regex"))?;
let player_js_nonl = player_js.replace('\n', "");
let helper_object = &helper_pattern
.captures(&player_js_nonl)
.ok_or(DeobfError::Extraction("sig fn helper object"))?[1];
let js_fn = helper_object.to_owned()
+ &deobfuscate_function
+ &caller_function(DEOBF_SIG_FUNC_NAME, &dfunc_name);
tracing::trace!("sig_fn: {js_fn}");
verify_fn(&js_fn, DEOBF_SIG_FUNC_NAME)?;
tracing::debug!("successfully extracted sig fn `{name}`");
tracing::debug!("successfully extracted sig fn `{dfunc_name}`");
Ok(js_fn)
}
fn get_nsig_fn_names(player_js: &str) -> impl Iterator<Item = String> + '_ {
static FUNCTION_NAME_REGEX: Lazy<Regex> = Lazy::new(|| {
// x.get( OR index.m3u8 OR delete x.y.file .. y=functionName[array_num](z) .. x.set(
Regex::new(r#"(?:[\w$]\.get\(|index\.m3u8|delete [\w$]+\.[\w$]+\.file).+[a-zA-Z]=([\w$]{2,})(?:\[(\d+)\])?\([a-zA-Z0-9]\).+[a-zA-Z0-9]\.set\("#)
// x.get( .. y=functionName[array_num](z) .. x.set(
Regex::new(r#"(?:[\w$]\.get\(|index\.m3u8).+[a-zA-Z]=([\w$]{2,})(?:\[(\d+)\])?\([a-zA-Z0-9]\).+[a-zA-Z0-9]\.set\("#)
.unwrap()
});
@ -172,71 +206,26 @@ fn get_nsig_fn_names(player_js: &str) -> impl Iterator<Item = String> + '_ {
})
}
fn extract_js_fn(js: &str, name: &str) -> Result<String, DeobfError> {
let function_base_re = Regex::new(&format!(r#"{}\s*=\s*function\("#, regex::escape(name)))
.map_err(|e| DeobfError::Other(format!("parsing regex for {name}: {e}").into()))?;
let offset = function_base_re
.find(js)
.ok_or(DeobfError::Extraction("could not find function base"))?
.start();
fn extract_js_fn(js: &str, offset: usize, name: &str) -> Result<String, DeobfError> {
let scan = ress::Scanner::new(&js[offset..]);
let mut state = 0;
let mut level = 0;
#[derive(Default, Clone, PartialEq, Eq)]
struct Level {
brace: isize,
paren: isize,
bracket: isize,
}
let mut level = Level::default();
let mut start = 0usize;
let mut end = 0usize;
let mut start = 0;
let mut end = 0;
let mut period_before = false;
let mut function_before = false;
let mut idents: HashMap<String, bool> = HashMap::new();
// Set if the current statement is a variable/function param definition
// First value is the brace level, second is true if we are on the right hand side of an assignment
let mut var_def_stmt: Option<(Level, bool)> = None;
let mut last_ident = None;
let mut idents: HashMap<String, usize> = HashMap::new();
let global_objects = [
"globalThis",
"NaN",
"undefined",
"Infinity",
"Object",
"Function",
"Boolean",
"Symbol",
"Error",
"Number",
"BigInt",
"Math",
"Date",
"String",
"RegExp",
"Array",
"Map",
"Set",
"eval",
"isFinite",
"isNaN",
"parseFloat",
"parseInt",
"decodeURI",
"decodeURIComponent",
"encodeURI",
"encodeURIComponent",
"escape",
"unescape",
"NaN", "Infinity", "Object", "Function", "Boolean", "Symbol", "Error", "Number", "BigInt",
"Math", "Date", "String", "RegExp", "Array", "Map", "Set",
];
for item in scan {
let it = item?;
let token = it.token;
match state {
// Looking for fn name
0 => {
@ -247,116 +236,47 @@ fn extract_js_fn(js: &str, name: &str) -> Result<String, DeobfError> {
}
// Looking for equals
1 => {
if token.matches_punct(Punct::Equal) {
if token.matches_punct(ress::tokens::Punct::Equal) {
state = 2;
} else {
state = 0;
}
}
2 => {
match &token {
Token::Punct(punct) => {
let var_def_this_lvl = || {
var_def_stmt
.as_ref()
.map(|(x, _)| x == &level)
.unwrap_or_default()
};
// Looking for begin/end braces
if token.matches_punct(ress::tokens::Punct::OpenBrace) {
level += 1;
} else if token.matches_punct(ress::tokens::Punct::CloseBrace) {
level -= 1;
match punct {
Punct::OpenBrace => {
level.brace += 1;
}
Punct::CloseBrace => {
if var_def_this_lvl() {
var_def_stmt = None;
}
level.brace -= 1;
if level.brace == 0 {
if level == 0 {
end = it.span.end;
state = 3;
break;
}
}
Punct::OpenParen => {
level.paren += 1;
}
Punct::CloseParen => {
if var_def_this_lvl() {
var_def_stmt = None;
}
level.paren -= 1;
}
Punct::OpenBracket => {
level.bracket += 1;
}
Punct::CloseBracket => {
if var_def_this_lvl() {
var_def_stmt = None;
}
level.bracket -= 1;
}
Punct::SemiColon => {
if var_def_this_lvl() {
var_def_stmt = None;
// Looking for variable names
if let Token::Ident(id) = &token {
if !period_before {
let id_str = id.to_string();
if !global_objects.contains(&id_str.as_str()) {
last_ident = Some(id.to_string());
}
}
Punct::Comma => {
if let Some((lvl, rhs)) = &mut var_def_stmt {
if lvl == &level {
*rhs = false;
}
}
}
Punct::Equal => {
if let Some((lvl, rhs)) = &mut var_def_stmt {
if lvl == &level {
*rhs = true;
}
}
}
_ => {}
}
}
Token::Keyword(kw) => match kw {
Keyword::Var(_) | Keyword::Let(_) | Keyword::Const(_) => {
var_def_stmt = Some((level.clone(), false));
}
Keyword::Function(_) => {
let mut l = level.clone();
l.paren += 1;
var_def_stmt = Some((l, false));
}
_ => {}
},
Token::Ident(id) => {
// Ignore object attributes and 1char long local vars
if !period_before
&& id.as_ref().len() > 1
&& !global_objects.contains(&id.as_ref())
} else if last_ident.is_some()
&& !token.matches_punct(ress::tokens::Punct::OpenParen)
{
// If we are on the left hand side of a variable definition statement
// or after "function", mark the variable name as defined
if var_def_stmt
.as_ref()
.map(|(lvl, rhs)| lvl == &level && !rhs)
.unwrap_or_default()
|| function_before
{
idents.insert(id.to_string(), true);
let n = idents.entry(last_ident.unwrap()).or_default();
*n += 1;
last_ident = None;
} else {
idents.entry(id.to_string()).or_default();
}
}
}
_ => {}
last_ident = None;
}
}
_ => break,
};
period_before = token.matches_punct(Punct::Period);
function_before = matches!(&token, Token::Keyword(Keyword::Function(_)));
period_before = token.matches_punct(ress::tokens::Punct::Period);
}
if state != 3 {
@ -367,10 +287,9 @@ fn extract_js_fn(js: &str, name: &str) -> Result<String, DeobfError> {
let mut code = format!("var {};", &js[fn_range.clone()]);
let rt = rquickjs::Runtime::new()?;
for (ident, _) in idents.into_iter().filter(|(_, v)| !v) {
let var_pattern_str = format!(r#"(^|[^\w$\.]){}\s*=[^=]"#, regex::escape(&ident));
let re = Regex::new(&var_pattern_str)
.map_err(|e| DeobfError::Other(format!("parsing regex for {ident}: {e}").into()))?;
for (ident, _) in idents.into_iter().filter(|(_, v)| *v == 1) {
let var_pattern_str = format!(r#"(^|[^\w$]){}\s*=[^=]"#, regex::escape(&ident));
let re = Regex::new(&var_pattern_str).unwrap();
let found_variable = re
.captures_iter(js)
.filter(|cap| {
@ -428,13 +347,13 @@ fn extract_js_var(js: &str) -> Option<&str> {
if let Token::Punct(p) = &token {
match p {
Punct::OpenBrace => braces.push(b'{'),
Punct::OpenBracket => braces.push(b'['),
Punct::OpenParen => braces.push(b'('),
Punct::CloseBrace => close_brace(&mut braces, b'{')?,
Punct::CloseBracket => close_brace(&mut braces, b'[')?,
Punct::CloseParen => close_brace(&mut braces, b'(')?,
Punct::Comma | Punct::SemiColon => {
ress::tokens::Punct::OpenBrace => braces.push(b'{'),
ress::tokens::Punct::OpenBracket => braces.push(b'['),
ress::tokens::Punct::OpenParen => braces.push(b'('),
ress::tokens::Punct::CloseBrace => close_brace(&mut braces, b'{')?,
ress::tokens::Punct::CloseBracket => close_brace(&mut braces, b'[')?,
ress::tokens::Punct::CloseParen => close_brace(&mut braces, b'(')?,
ress::tokens::Punct::Comma | ress::tokens::Punct::SemiColon => {
if braces.is_empty() {
end = it.span.start;
break;
@ -469,19 +388,23 @@ fn verify_fn(js_fn: &str, fn_name: &str) -> Result<(), DeobfError> {
})?;
if res.is_empty() {
return Err(DeobfError::Other(
"deobfuscation fn returned empty string".into(),
));
return Err(DeobfError::Other("deobfuscation fn returned empty string"));
}
if res.starts_with("enhanced_except_") || res.ends_with(&testinp) {
return Err(DeobfError::Other("nsig fn returned an exception".into()));
return Err(DeobfError::Other("nsig fn returned an exception"));
}
Ok(())
}
fn get_nsig_fn(player_js: &str) -> Result<String, DeobfError> {
let extract_fn = |name: &str| -> Result<String, DeobfError> {
let code = extract_js_fn(player_js, name)?;
let function_base = format!("{name}=function");
let offset = player_js
.find(&function_base)
.ok_or(DeobfError::Extraction("could not find function base"))?;
let code = extract_js_fn(player_js, offset, name)?;
let js_fn = format!("{}{}", code, caller_function(DEOBF_NSIG_FUNC_NAME, name));
tracing::trace!("nsig_fn: {js_fn}");
verify_fn(&js_fn, DEOBF_NSIG_FUNC_NAME)?;
@ -549,9 +472,7 @@ mod tests {
std::fs::read_to_string(js_path).unwrap()
});
const SIG_DEOBF_FUNC: &str = r#"var qB={w8:function(a){a.reverse()},
EC:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c},
Np:function(a,b){a.splice(0,b)}}; var Rva=function(a){a=a.split("");qB.Np(a,3);qB.w8(a,41);qB.EC(a,55);qB.Np(a,3);qB.w8(a,33);qB.Np(a,3);qB.EC(a,48);qB.EC(a,17);qB.EC(a,43);return a.join("")};var deobf_sig=Rva;"#;
const SIG_DEOBF_FUNC: &str = r#"var qB={w8:function(a){a.reverse()},EC:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c},Np:function(a,b){a.splice(0,b)}};var Rva=function(a){a=a.split("");qB.Np(a,3);qB.w8(a,41);qB.EC(a,55);qB.Np(a,3);qB.w8(a,33);qB.Np(a,3);qB.EC(a,48);qB.EC(a,17);qB.EC(a,43);return a.join("")};var deobf_sig=Rva;"#;
const NSIG_DEOBF_FUNC: &str = r#"var Vo=function(a){var b=a.split(""),c=[function(d,e,f){var h=f.length;d.forEach(function(l,m,n){this.push(n[m]=f[(f.indexOf(l)-f.indexOf(this[m])+m+h--)%f.length])},e.split(""))},
928409064,-595856984,1403221911,653089124,-168714481,-1883008765,158931990,1346921902,361518508,1403221911,-362174697,-233641452,function(){for(var d=64,e=[];++d-e.length-32;){switch(d){case 91:d=44;continue;case 123:d=65;break;case 65:d-=18;continue;case 58:d=96;continue;case 46:d=95}e.push(String.fromCharCode(d))}return e},
b,158931990,791141857,-907319795,-1776185924,1595027902,-829736173,function(d,e){e=(e%d.length+d.length)%d.length;d.splice(0,1,d.splice(e,1,d[0])[0])},
@ -604,7 +525,7 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
#[test]
fn t_extract_js_fn() {
let base_js = "Wka = function(d){let x=10/2;return /,,[/,913,/](,)}/}let a = 42;";
let res = extract_js_fn(base_js, "Wka").unwrap();
let res = extract_js_fn(base_js, 0, "Wka").unwrap();
assert_eq!(
res,
"var Wka = function(d){let x=10/2;return /,,[/,913,/](,)}/};"
@ -615,7 +536,7 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
fn t_extract_js_fn_eviljs() {
// Evil JavaScript code containing braces within strings and regular expressions
let base_js = "Wka = function(d){var x = [/,,/,913,/(,)}/,\"abcdef}\\\"\",];var y = 10/2/1;return x[1][y];}//some={}random-padding+;";
let res = extract_js_fn(base_js, "Wka").unwrap();
let res = extract_js_fn(base_js, 0, "Wka").unwrap();
assert_eq!(
res,
"var Wka = function(d){var x = [/,,/,913,/(,)}/,\"abcdef}\\\"\",];var y = 10/2/1;return x[1][y];};"
@ -624,43 +545,33 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
#[test]
fn t_extract_js_fn_outside_vars() {
let base_js = "let a1 = 42;foo();var b1=11;var da=77;bar();Wka = function(da){var xy=1+2+a1*b1;return xy;}";
let res = extract_js_fn(base_js, "Wka").unwrap();
let base_js = "let a = 42;foo();var b=11;bar();Wka = function(d){var x=1+2+a*b;return x;}";
let res = extract_js_fn(base_js, 0, "Wka").unwrap();
// order of variables is non-reproducible
assert!(
res == "var a1 = 42; var b1=11; var Wka = function(da){var xy=1+2+a1*b1;return xy;};"
|| res == "var b1=11; var a1 = 42; var Wka = function(da){var xy=1+2+a1*b1;return xy;};",
res == "var a = 42; var b=11; var Wka = function(d){var x=1+2+a*b;return x;};"
|| res == "var b=11; var a = 42; var Wka = function(d){var x=1+2+a*b;return x;};",
"got {res}"
);
}
#[test]
fn t_extract_js_fn_outside_vars2() {
let base_js = "{let a1 = {v1:1,v2:2}}foo();Wka = function(d){var x=1+2+a1.v1;return x;}";
let res = extract_js_fn(base_js, "Wka").unwrap();
let base_js = "{let a = {v1:1,v2:2}}foo();Wka = function(d){var x=1+2+a.v1;return x;}";
let res = extract_js_fn(base_js, 0, "Wka").unwrap();
assert_eq!(
res,
"var a1 = {v1:1,v2:2}; var Wka = function(d){var x=1+2+a1.v1;return x;};"
"var a = {v1:1,v2:2}; var Wka = function(d){var x=1+2+a.v1;return x;};"
);
}
#[test]
fn t_extract_js_fn_outside_vars3() {
let base_js = "Wka = function(d){var x=1+2+a1[0];return x;};let a1=[1,2,3]";
let res = extract_js_fn(base_js, "Wka").unwrap();
let base_js = "Wka = function(d){var x=1+2+a[0];return x;};let a=[1,2,3]";
let res = extract_js_fn(base_js, 0, "Wka").unwrap();
assert_eq!(
res,
"var a1=[1,2,3]; var Wka = function(d){var x=1+2+a1[0];return x;};"
);
}
#[test]
fn t_extract_js_fn_outside_vars4() {
let base_js = "let a0=123456;let a1=function(a){return a};let Wka = function(d){var x=1+2+a1();return x;}";
let res = extract_js_fn(base_js, "Wka").unwrap();
assert_eq!(
res,
"var a1=function(a){return a}; var Wka = function(d){var x=1+2+a1();return x;};"
"var a=[1,2,3]; var Wka = function(d){var x=1+2+a[0];return x;};"
);
}
@ -714,86 +625,65 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
}
// Test cases from https://github.com/yt-dlp/yt-dlp/blob/master/test/test_youtube_signature.py
#[rstest]
#[case("6ed0d907", "AOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL2QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0")]
#[case("3bb1f723", "MyOSJXtKI3m-uME_jv7-pT12gOFC02RFkGoqWpzE0Cs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA")]
#[case("2f1832d2", "0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xxAj7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJ2OySqa0q")]
#[tokio::test]
#[traced_test]
async fn sig_tests() {
let cases = [
("6ed0d907", "AOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL2QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0"),
("3bb1f723", "MyOSJXtKI3m-uME_jv7-pT12gOFC02RFkGoqWpzE0Cs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA"),
("2f1832d2", "0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xxAj7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJ2OySqa0q"),
("643afba4", "AAOAOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7vgpDL0QwbdV06sCIEzpWqMGkFR20CFOS21Tp-7vj_EMu-m37KtXJoOy1"),
("363db69b", "0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpz2ICs6EVdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA"),
];
for (js_hash, exp_sig) in cases {
let span = tracing::span!(tracing::Level::ERROR, "sig_test", js_hash);
let _enter = span.enter();
async fn sig_tests(#[case] js_hash: &str, #[case] exp_sig: &str) {
let (js_url, js_path) = player_js_file(js_hash).await;
let player_js = std::fs::read_to_string(js_path).unwrap();
let deobf_data = DeobfData::extract_fns(&js_url, &player_js).unwrap();
let deobf = Deobfuscator::new(&deobf_data).unwrap();
let deobf_sig = deobf.deobfuscate_sig("2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA").unwrap();
assert_eq!(deobf_sig, exp_sig, "[{js_hash}]");
}
assert_eq!(deobf_sig, exp_sig, "js: {js_hash}");
}
#[rstest]
#[case("7862ca1f", "X_LCxVDjAavgE5t", "yxJ1dM6iz5ogUg")]
#[case("9216d1f7", "SLp9F5bwjAdhE9F-", "gWnb9IK2DJ8Q1w")]
#[case("f8cb7a3b", "oBo2h5euWy6osrUt", "ivXHpm7qJjJN")]
#[case("2dfe380c", "oBo2h5euWy6osrUt", "3DIBbn3qdQ")]
#[case("f1ca6900", "cu3wyu6LQn2hse", "jvxetvmlI9AN9Q")]
#[case("8040e515", "wvOFaY-yjgDuIEg5", "HkfBFDHmgw4rsw")]
#[case("e06dea74", "AiuodmaDDYw8d3y4bf", "ankd8eza2T6Qmw")]
#[case("5dd88d1d", "kSxKFLeqzv_ZyHSAt", "n8gS8oRlHOxPFA")]
#[case("324f67b9", "xdftNy7dh9QGnhW", "22qLGxrmX8F1rA")]
#[case("4c3f79c5", "TDCstCG66tEAO5pR9o", "dbxNtZ14c-yWyw")]
#[case("c81bbb4a", "gre3EcLurNY2vqp94", "Z9DfGxWP115WTg")]
#[case("1f7d5369", "batNX7sYqIJdkJ", "IhOkL_zxbkOZBw")]
#[case("009f1d77", "5dwFHw8aFWQUQtffRq", "audescmLUzI3jw")]
#[case("dc0c6770", "5EHDMgYLV6HPGk_Mu-kk", "n9lUJLHbxUI0GQ")]
#[case("113ca41c", "cgYl-tlYkhjT7A", "hI7BBr2zUgcmMg")]
#[case("c57c113c", "M92UUMHa8PdvPd3wyM", "3hPqLJsiNZx7yA")]
#[case("5a3b6271", "B2j7f_UPT4rfje85Lu_e", "m5DmNymaGQ5RdQ")]
#[case("7a062b77", "NRcE3y3mVtm_cV-W", "VbsCYUATvqlt5w")]
#[case("dac945fd", "o8BkRxXhuYsBCWi6RplPdP", "3Lx32v_hmzTm6A")]
#[case("6f20102c", "lE8DhoDmKqnmJJ", "pJTTX6XyJP2BYw")]
#[case("cfa9e7cb", "aCi3iElgd2kq0bxVbQ", "QX1y8jGb2IbZ0w")]
#[case("8c7583ff", "1wWCVpRR96eAmMI87L", "KSkWAVv1ZQxC3A")]
#[case("b7910ca8", "_hXMCwMt9qE310D", "LoZMgkkofRMCZQ")]
#[case("590f65a6", "1tm7-g_A9zsI8_Lay_", "xI4Vem4Put_rOg")]
#[case("b22ef6e7", "b6HcntHGkvBLk_FRf", "kNPW6A7FyP2l8A")]
#[case("3400486c", "lL46g3XifCKUZn1Xfw", "z767lhet6V2Skl")]
#[case("20dfca59", "-fLCxedkAk4LUTK2", "O8kfRq1y1eyHGw")]
#[case("b12cc44b", "keLa5R2U00sR9SQK", "N1OGyujjEwMnLw")]
#[case("3bb1f723", "gK15nzVyaXE9RsMP3z", "ZFFWFLPWx9DEgQ")]
#[case("2f1832d2", "YWt1qdbe8SAfkoPHW5d", "RrRjWQOJmBiP")]
#[case("19d2ae9d", "YWt1qdbe8SAfkoPHW5d", "CS6dVTYzpZrAZ5TD")]
#[tokio::test]
#[traced_test]
async fn nsig_tests() {
let cases = [
("7862ca1f", "X_LCxVDjAavgE5t", "yxJ1dM6iz5ogUg"),
("9216d1f7", "SLp9F5bwjAdhE9F-", "gWnb9IK2DJ8Q1w"),
("f8cb7a3b", "oBo2h5euWy6osrUt", "ivXHpm7qJjJN"),
("2dfe380c", "oBo2h5euWy6osrUt", "3DIBbn3qdQ"),
("f1ca6900", "cu3wyu6LQn2hse", "jvxetvmlI9AN9Q"),
("8040e515", "wvOFaY-yjgDuIEg5", "HkfBFDHmgw4rsw"),
("e06dea74", "AiuodmaDDYw8d3y4bf", "ankd8eza2T6Qmw"),
("5dd88d1d", "kSxKFLeqzv_ZyHSAt", "n8gS8oRlHOxPFA"),
("324f67b9", "xdftNy7dh9QGnhW", "22qLGxrmX8F1rA"),
("4c3f79c5", "TDCstCG66tEAO5pR9o", "dbxNtZ14c-yWyw"),
("c81bbb4a", "gre3EcLurNY2vqp94", "Z9DfGxWP115WTg"),
("1f7d5369", "batNX7sYqIJdkJ", "IhOkL_zxbkOZBw"),
("009f1d77", "5dwFHw8aFWQUQtffRq", "audescmLUzI3jw"),
("dc0c6770", "5EHDMgYLV6HPGk_Mu-kk", "n9lUJLHbxUI0GQ"),
("113ca41c", "cgYl-tlYkhjT7A", "hI7BBr2zUgcmMg"),
("c57c113c", "M92UUMHa8PdvPd3wyM", "3hPqLJsiNZx7yA"),
("5a3b6271", "B2j7f_UPT4rfje85Lu_e", "m5DmNymaGQ5RdQ"),
("7a062b77", "NRcE3y3mVtm_cV-W", "VbsCYUATvqlt5w"),
("dac945fd", "o8BkRxXhuYsBCWi6RplPdP", "3Lx32v_hmzTm6A"),
("6f20102c", "lE8DhoDmKqnmJJ", "pJTTX6XyJP2BYw"),
("cfa9e7cb", "aCi3iElgd2kq0bxVbQ", "QX1y8jGb2IbZ0w"),
("8c7583ff", "1wWCVpRR96eAmMI87L", "KSkWAVv1ZQxC3A"),
("b7910ca8", "_hXMCwMt9qE310D", "LoZMgkkofRMCZQ"),
("590f65a6", "1tm7-g_A9zsI8_Lay_", "xI4Vem4Put_rOg"),
("b22ef6e7", "b6HcntHGkvBLk_FRf", "kNPW6A7FyP2l8A"),
("3400486c", "lL46g3XifCKUZn1Xfw", "z767lhet6V2Skl"),
("20dfca59", "-fLCxedkAk4LUTK2", "O8kfRq1y1eyHGw"),
("b12cc44b", "keLa5R2U00sR9SQK", "N1OGyujjEwMnLw"),
("3bb1f723", "gK15nzVyaXE9RsMP3z", "ZFFWFLPWx9DEgQ"),
("2f1832d2", "YWt1qdbe8SAfkoPHW5d", "RrRjWQOJmBiP"),
("19d2ae9d", "YWt1qdbe8SAfkoPHW5d", "CS6dVTYzpZrAZ5TD"),
("e7567ecf", "Sy4aDGc0VpYRR9ew_", "5UPOT1VhoZxNLQ"),
("d50f54ef", "Ha7507LzRmH3Utygtj", "XFTb2HoeOE5MHg"),
("074a8365", "Ha7507LzRmH3Utygtj", "ufTsrE0IVYrkl8v"),
("643afba4", "N5uAlLqm0eg1GyHO", "dCBQOejdq5s-ww"),
("69f581a5", "-qIP447rVlTTwaZjY", "KNcGOksBAvwqQg"),
("363db69b", "eWYu5d5YeY_4LyEDc", "XJQqf-N7Xra3gg"),
];
for (js_hash, nsig_in, exp_nsig) in cases {
let span = tracing::span!(tracing::Level::ERROR, "nsig_test", js_hash);
let _enter = span.enter();
async fn nsig_tests(#[case] js_hash: &str, #[case] nsig_in: &str, #[case] expect: &str) {
let (js_url, js_path) = player_js_file(js_hash).await;
let player_js = std::fs::read_to_string(js_path).unwrap();
let deobf_data = DeobfData::extract_fns(&js_url, &player_js).expect(js_hash);
let deobf = Deobfuscator::new(&deobf_data).expect(js_hash);
let deobf_data = DeobfData::extract_fns(&js_url, &player_js).unwrap();
let deobf = Deobfuscator::new(&deobf_data).unwrap();
let deobf_nsig = deobf.deobfuscate_nsig(nsig_in).expect(js_hash);
assert_eq!(deobf_nsig, exp_nsig, "[{js_hash}]");
}
let deobf_nsig = deobf.deobfuscate_nsig(nsig_in).unwrap();
assert_eq!(deobf_nsig, expect, "js: {js_hash}");
}
#[tokio::test]

View file

@ -151,8 +151,6 @@ pub enum AuthError {
}
pub(crate) mod internal {
use std::borrow::Cow;
use super::{Error, ExtractionError};
/// Error that occurred during the initialization
@ -170,7 +168,7 @@ pub(crate) mod internal {
Extraction(&'static str),
/// Unspecified error
#[error("error: {0}")]
Other(Cow<'static, str>),
Other(&'static str),
}
impl From<DeobfError> for Error {

View file

@ -38,7 +38,7 @@ pub struct Thumbnail {
pub enum UrlTarget {
/// YouTube video
///
/// Example: <https://youtube.com/watch?v=ZeerrnuLi5E>
/// Example: <youtube.com/watch?v=ZeerrnuLi5E>
Video {
/// Unique YouTube video ID
id: String,
@ -1234,8 +1234,6 @@ pub struct MusicAlbum {
pub year: Option<u16>,
/// Is the album by 'Various artists'?
pub by_va: bool,
/// Number of album tracks
pub track_count: u16,
/// Album tracks
pub tracks: Vec<TrackItem>,
/// Album variants

View file

@ -70,8 +70,6 @@ pub struct RustyPipeInfo<'a> {
/// YouTube content language
#[serde(skip_serializing_if = "Option::is_none")]
pub language: Option<Language>,
/// RustyPipe Botguard version (`rustypipe-botguard 0.1.1`)
pub botguard_version: Option<&'a str>,
}
/// Reported HTTP request data
@ -106,14 +104,13 @@ pub enum Level {
ERR,
}
impl<'a> RustyPipeInfo<'a> {
pub(crate) fn new(language: Option<Language>, botguard_version: Option<&'a str>) -> Self {
impl RustyPipeInfo<'_> {
pub(crate) fn new(language: Option<Language>) -> Self {
Self {
package: env!("CARGO_PKG_NAME"),
version: crate::VERSION,
date: util::now_sec(),
language,
botguard_version,
}
}
}

View file

@ -53,8 +53,6 @@ pub(crate) struct Entry {
pub chan_prefix: &'static str,
/// Channel name suffix on playlist pages
pub chan_suffix: &'static str,
/// "Other versions" title on album pages
pub album_versions_title: &'static str,
}
#[rustfmt::skip]
@ -185,7 +183,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "deur",
chan_suffix: "",
album_versions_title: "Ander weergawes",
},
Language::Am => Entry {
timeago_tokens: ::phf::Map {
@ -313,7 +310,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "",
album_versions_title: "ሌሎች ስሪቶች",
},
Language::Ar => Entry {
timeago_tokens: ::phf::Map {
@ -449,7 +445,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "بواسطة",
chan_suffix: "",
album_versions_title: "إصدارات أخرى",
},
Language::As => Entry {
timeago_tokens: ::phf::Map {
@ -572,7 +567,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "ৰ দ\u{9cd}\u{9be}\u{9be}",
album_versions_title: "অন\u{9cd}য সংস\u{9cd}কৰণ",
},
Language::Az => Entry {
timeago_tokens: ::phf::Map {
@ -688,7 +682,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "by",
chan_suffix: "",
album_versions_title: "Digər versiyalar",
},
Language::Be => Entry {
timeago_tokens: ::phf::Map {
@ -836,7 +829,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "ад",
chan_suffix: "",
album_versions_title: "Іншыя версіі",
},
Language::Bg => Entry {
timeago_tokens: ::phf::Map {
@ -953,7 +945,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "от",
chan_suffix: "",
album_versions_title: "Други версии",
},
Language::Bn => Entry {
timeago_tokens: ::phf::Map {
@ -1071,7 +1062,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: ",",
chan_suffix: "\u{9cd}\u{9be}\u{9be}",
album_versions_title: "অন\u{9cd}য সংস\u{9cd}করণগ\u{9c1}লি",
},
Language::Bs => Entry {
timeago_tokens: ::phf::Map {
@ -1211,7 +1201,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "od",
chan_suffix: "",
album_versions_title: "Druge verzije",
},
Language::Ca => Entry {
timeago_tokens: ::phf::Map {
@ -1336,7 +1325,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "de:",
chan_suffix: "",
album_versions_title: "Altres versions",
},
Language::Cs => Entry {
timeago_tokens: ::phf::Map {
@ -1467,7 +1455,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "autor:",
chan_suffix: "",
album_versions_title: "Jiné verze",
},
Language::Da => Entry {
timeago_tokens: ::phf::Map {
@ -1592,7 +1579,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "af",
chan_suffix: "",
album_versions_title: "Andre versioner",
},
Language::De => Entry {
timeago_tokens: ::phf::Map {
@ -1714,7 +1700,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "von",
chan_suffix: "",
album_versions_title: "Weitere Versionen",
},
Language::El => Entry {
timeago_tokens: ::phf::Map {
@ -1845,7 +1830,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "από το χρήστη",
chan_suffix: "",
album_versions_title: "Άλλες εκτελέσεις",
},
Language::En | Language::EnGb | Language::EnIn => Entry {
timeago_tokens: ::phf::Map {
@ -1987,7 +1971,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "by",
chan_suffix: "",
album_versions_title: "Other versions",
},
Language::Es => Entry {
timeago_tokens: ::phf::Map {
@ -2115,7 +2098,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "de",
chan_suffix: "",
album_versions_title: "Otras versiones",
},
Language::EsUs | Language::Es419 => Entry {
timeago_tokens: ::phf::Map {
@ -2244,7 +2226,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "de",
chan_suffix: "",
album_versions_title: "Otras versiones",
},
Language::Et => Entry {
timeago_tokens: ::phf::Map {
@ -2370,7 +2351,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "kanalilt",
chan_suffix: "",
album_versions_title: "Teised versioonid",
},
Language::Eu => Entry {
timeago_tokens: ::phf::Map {
@ -2487,7 +2467,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "egilea:",
chan_suffix: "",
album_versions_title: "Beste bertsio batzuk",
},
Language::Fa => Entry {
timeago_tokens: ::phf::Map {
@ -2595,7 +2574,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "توسط",
chan_suffix: "",
album_versions_title: "نسخه\u{200c}های دیگر",
},
Language::Fi => Entry {
timeago_tokens: ::phf::Map {
@ -2715,7 +2693,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "tekijä:",
chan_suffix: "",
album_versions_title: "Muut versiot",
},
Language::Fil => Entry {
timeago_tokens: ::phf::Map {
@ -2833,7 +2810,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "ni/ng",
chan_suffix: "",
album_versions_title: "Iba pang bersyon",
},
Language::Fr | Language::FrCa => Entry {
timeago_tokens: ::phf::Map {
@ -2965,7 +2941,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "de",
chan_suffix: "",
album_versions_title: "Autres versions",
},
Language::Gl => Entry {
timeago_tokens: ::phf::Map {
@ -3090,7 +3065,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "de",
chan_suffix: "",
album_versions_title: "Outras versións",
},
Language::Gu => Entry {
timeago_tokens: ::phf::Map {
@ -3196,7 +3170,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "\u{acd}વારા",
album_versions_title: "અન\u{acd}ય વર\u{acd}ઝન",
},
Language::Hi => Entry {
timeago_tokens: ::phf::Map {
@ -3313,7 +3286,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "\u{947}\u{93c}रिए",
album_versions_title: "अन\u{94d}य वर\u{94d}शन",
},
Language::Hr => Entry {
timeago_tokens: ::phf::Map {
@ -3453,7 +3425,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "omogućio kanal",
chan_suffix: "",
album_versions_title: "Druge verzije",
},
Language::Hu => Entry {
timeago_tokens: ::phf::Map {
@ -3583,7 +3554,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "csatornától",
album_versions_title: "Más verziók",
},
Language::Hy => Entry {
timeago_tokens: ::phf::Map {
@ -3706,7 +3676,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "հեղինակ՝",
chan_suffix: "",
album_versions_title: "Այլ տարբերակներ",
},
Language::Id => Entry {
timeago_tokens: ::phf::Map {
@ -3825,7 +3794,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "oleh",
chan_suffix: "",
album_versions_title: "Versi lainnya",
},
Language::Is => Entry {
timeago_tokens: ::phf::Map {
@ -3960,7 +3928,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "eftir",
chan_suffix: "",
album_versions_title: "Aðrar útgáfur",
},
Language::It => Entry {
timeago_tokens: ::phf::Map {
@ -4093,7 +4060,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "di",
chan_suffix: "",
album_versions_title: "Altre versioni",
},
Language::Iw => Entry {
timeago_tokens: ::phf::Map {
@ -4232,7 +4198,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "מאת",
chan_suffix: "",
album_versions_title: "גרסאות אחרות",
},
Language::Ja => Entry {
timeago_tokens: ::phf::Map {
@ -4313,7 +4278,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "作成者:",
chan_suffix: "",
album_versions_title: "他のバージョン",
},
Language::Ka => Entry {
timeago_tokens: ::phf::Map {
@ -4436,7 +4400,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "-ის მიერ",
album_versions_title: "სხვა ვერსიები",
},
Language::Kk => Entry {
timeago_tokens: ::phf::Map {
@ -4560,7 +4523,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "қосқан",
chan_suffix: "",
album_versions_title: "Басқа нұсқалары",
},
Language::Km => Entry {
timeago_tokens: ::phf::Map {
@ -4661,7 +4623,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "ដោយ",
chan_suffix: "",
album_versions_title: "\u{17d2}រភេទផ\u{17d2}សេងៗ",
},
Language::Kn => Entry {
timeago_tokens: ::phf::Map {
@ -4788,7 +4749,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "ಚಾನಲ\u{ccd}\u{200c}\u{cbf}ಂದ",
album_versions_title: "ಇತರ ಆವೃತ\u{ccd}\u{cbf}ಗಳು",
},
Language::Ko => Entry {
timeago_tokens: ::phf::Map {
@ -4872,7 +4832,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "게시자:",
chan_suffix: "",
album_versions_title: "다른 버전",
},
Language::Ky => Entry {
timeago_tokens: ::phf::Map {
@ -4991,7 +4950,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "каналы аркылуу",
album_versions_title: "Башка версиялар",
},
Language::Lo => Entry {
timeago_tokens: ::phf::Map {
@ -5118,7 +5076,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "ໂດຍ",
chan_suffix: "",
album_versions_title: "ເວ\u{eb5}\u{eb1}ນອ\u{eb7}\u{ec8}ນໆ",
},
Language::Lt => Entry {
timeago_tokens: ::phf::Map {
@ -5253,7 +5210,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "pridėjo",
chan_suffix: "",
album_versions_title: "Kitos versijos",
},
Language::Lv => Entry {
timeago_tokens: ::phf::Map {
@ -5388,7 +5344,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "autors:",
chan_suffix: "",
album_versions_title: "Citas versijas",
},
Language::Mk => Entry {
timeago_tokens: ::phf::Map {
@ -5516,7 +5471,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "од",
chan_suffix: "",
album_versions_title: "Други верзии",
},
Language::Ml => Entry {
timeago_tokens: ::phf::Map {
@ -5631,7 +5585,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "\u{d41}ഖേന",
album_versions_title: "മറ\u{d4d}\u{d4d} പതിപ\u{d4d}\u{d41}കൾ",
},
Language::Mn => Entry {
timeago_tokens: ::phf::Map {
@ -5736,7 +5689,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "сувгийн нэр:",
chan_suffix: "",
album_versions_title: "Бусад хувилбар",
},
Language::Mr => Entry {
timeago_tokens: ::phf::Map {
@ -5861,7 +5813,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "\u{94d}वार\u{947}",
album_versions_title: "इतर आव\u{943}\u{94d}\u{94d}या",
},
Language::Ms => Entry {
timeago_tokens: ::phf::Map {
@ -5975,7 +5926,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "oleh",
chan_suffix: "",
album_versions_title: "Versi lain",
},
Language::My => Entry {
timeago_tokens: ::phf::Map {
@ -6096,7 +6046,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "\u{103e}",
album_versions_title: "အခြား ဗားရ\u{103e}\u{103a}းများ",
},
Language::Ne => Entry {
timeago_tokens: ::phf::Map {
@ -6200,7 +6149,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "\u{94d}वारा",
album_versions_title: "अन\u{94d}य स\u{902}\u{94d}करणहर\u{942}",
},
Language::Nl => Entry {
timeago_tokens: ::phf::Map {
@ -6323,7 +6271,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "door",
chan_suffix: "",
album_versions_title: "Andere versies",
},
Language::No => Entry {
timeago_tokens: ::phf::Map {
@ -6452,7 +6399,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "av",
chan_suffix: "",
album_versions_title: "Andre versjoner",
},
Language::Or => Entry {
timeago_tokens: ::phf::Map {
@ -6568,7 +6514,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "\u{b4d}\u{b3e}\u{b3e}",
album_versions_title: "ଅନ\u{b4d}ୟ ସଂସ\u{b4d}କରଣଗ\u{b41}\u{b3c}\u{b3f}",
},
Language::Pa => Entry {
timeago_tokens: ::phf::Map {
@ -6684,7 +6629,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "\u{a71}\u{a4b}\u{a02}",
album_versions_title: "\u{a4b}ਰ ਵਰਜਨ",
},
Language::Pl => Entry {
timeago_tokens: ::phf::Map {
@ -6830,7 +6774,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "autor:",
chan_suffix: "",
album_versions_title: "Inne wersje",
},
Language::Pt => Entry {
timeago_tokens: ::phf::Map {
@ -6960,7 +6903,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "por",
chan_suffix: "",
album_versions_title: "Outras versões",
},
Language::PtPt => Entry {
timeago_tokens: ::phf::Map {
@ -7073,7 +7015,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "de",
chan_suffix: "",
album_versions_title: "Outras versões",
},
Language::Ro => Entry {
timeago_tokens: ::phf::Map {
@ -7202,7 +7143,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "de",
chan_suffix: "",
album_versions_title: "Alte versiuni",
},
Language::Ru => Entry {
timeago_tokens: ::phf::Map {
@ -7346,7 +7286,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "",
album_versions_title: "Другие версии",
},
Language::Si => Entry {
timeago_tokens: ::phf::Map {
@ -7458,7 +7397,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "\u{dd2}\u{dd2}\u{dca}",
album_versions_title: "අනෙක\u{dd4}\u{dca} අන\u{dd4}\u{dcf}දයන\u{dca}",
},
Language::Sk => Entry {
timeago_tokens: ::phf::Map {
@ -7589,7 +7527,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "Autori:",
chan_suffix: "",
album_versions_title: "Ďalšie verzie",
},
Language::Sl => Entry {
timeago_tokens: ::phf::Map {
@ -7739,7 +7676,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "kanal",
chan_suffix: "",
album_versions_title: "Druge različice",
},
Language::Sq => Entry {
timeago_tokens: ::phf::Map {
@ -7860,7 +7796,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "nga",
chan_suffix: "",
album_versions_title: "Versione të tjera",
},
Language::Sr => Entry {
timeago_tokens: ::phf::Map {
@ -7991,7 +7926,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "са канала",
chan_suffix: "",
album_versions_title: "Друге верзије",
},
Language::SrLatn => Entry {
timeago_tokens: ::phf::Map {
@ -8122,7 +8056,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "sa kanala",
chan_suffix: "",
album_versions_title: "Druge verzije",
},
Language::Sv => Entry {
timeago_tokens: ::phf::Map {
@ -8245,7 +8178,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "från",
chan_suffix: "",
album_versions_title: "Andra versioner",
},
Language::Sw => Entry {
timeago_tokens: ::phf::Map {
@ -8359,7 +8291,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "kutoka",
chan_suffix: "",
album_versions_title: "Matoleo mengine",
},
Language::Ta => Entry {
timeago_tokens: ::phf::Map {
@ -8490,7 +8421,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "வழங\u{bcd}கியவர\u{bcd}:",
chan_suffix: "",
album_versions_title: "பிற பதிப\u{bcd}புகள\u{bcd}",
},
Language::Te => Entry {
timeago_tokens: ::phf::Map {
@ -8617,7 +8547,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "\u{c3e}\u{c46}\u{c4d}\u{c4d}\u{c3e}\u{c3e}",
album_versions_title: "ఇతర వ\u{c46}\u{c4d}షన\u{c4d}\u{200c}లు",
},
Language::Th => Entry {
timeago_tokens: ::phf::Map {
@ -8748,7 +8677,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "โดย",
chan_suffix: "",
album_versions_title: "เวอร\u{e4c}\u{e31}นอ\u{e37}\u{e48}นๆ",
},
Language::Tr => Entry {
timeago_tokens: ::phf::Map {
@ -8869,7 +8797,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "tarafından",
album_versions_title: "Diğer versiyonlar",
},
Language::Uk => Entry {
timeago_tokens: ::phf::Map {
@ -9018,7 +8945,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "власник:",
chan_suffix: "",
album_versions_title: "Інші версії",
},
Language::Ur => Entry {
timeago_tokens: ::phf::Map {
@ -9144,7 +9070,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "منجانب",
chan_suffix: "",
album_versions_title: "دیگر ورژنز",
},
Language::Uz => Entry {
timeago_tokens: ::phf::Map {
@ -9259,7 +9184,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "muallif:",
chan_suffix: "",
album_versions_title: "Boshqa versiyalari",
},
Language::Vi => Entry {
timeago_tokens: ::phf::Map {
@ -9341,7 +9265,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "của",
chan_suffix: "",
album_versions_title: "Các phiên bản khác",
},
Language::ZhCn => Entry {
timeago_tokens: ::phf::Map {
@ -9439,7 +9362,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "创建者:",
chan_suffix: "",
album_versions_title: "其他版本",
},
Language::ZhHk => Entry {
timeago_tokens: ::phf::Map {
@ -9521,7 +9443,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "來自",
chan_suffix: "",
album_versions_title: "其他版本",
},
Language::ZhTw => Entry {
timeago_tokens: ::phf::Map {
@ -9602,7 +9523,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "",
chan_suffix: "建立",
album_versions_title: "其他版本",
},
Language::Zu => Entry {
timeago_tokens: ::phf::Map {
@ -9738,7 +9658,6 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
chan_prefix: "ka-",
chan_suffix: "",
album_versions_title: "Ezinye izinguqulo",
},
}
}

View file

@ -21,7 +21,7 @@ use regex::Regex;
use url::Url;
use crate::{
error::Error,
error::{AuthError, Error, ExtractionError},
param::{Country, Language, COUNTRIES},
serializer::text::TextComponent,
};
@ -75,10 +75,10 @@ pub fn get_cg_from_fancy_regexes(regexes: &[&str], text: &str, cg_name: &str) ->
/// Generate a random string with given length and byte charset.
fn random_string(charset: &[u8], length: usize) -> String {
let mut result = String::with_capacity(length);
let mut rng = rand::rng();
let mut rng = rand::thread_rng();
for _ in 0..length {
result.push(char::from(charset[rng.random_range(0..charset.len())]));
result.push(char::from(charset[rng.gen_range(0..charset.len())]));
}
result
@ -90,14 +90,14 @@ pub fn generate_content_playback_nonce() -> String {
}
pub fn random_uuid() -> String {
let mut rng = rand::rng();
let mut rng = rand::thread_rng();
format!(
"{:08x}-{:04x}-{:04x}-{:04x}-{:012x}",
rng.random::<u32>(),
rng.random::<u16>(),
rng.random::<u16>(),
rng.random::<u16>(),
rng.random::<u64>() & 0xffff_ffff_ffff,
rng.gen::<u32>(),
rng.gen::<u16>(),
rng.gen::<u16>(),
rng.gen::<u16>(),
rng.gen::<u64>() & 0xffff_ffff_ffff,
)
}
@ -229,7 +229,7 @@ pub fn retry_delay(
backoff_base: u32,
) -> u32 {
let unjittered_delay = backoff_base.checked_pow(n_past_retries).unwrap_or(u32::MAX);
let jitter_factor = rand::rng().random_range(800..1500);
let jitter_factor = rand::thread_rng().gen_range(800..1500);
let jittered_delay = unjittered_delay
.checked_mul(jitter_factor)
.unwrap_or(u32::MAX);
@ -581,10 +581,9 @@ where
///
/// If no user is logged in, YouTube returns a "NotFound" error. This has to be corrected
/// into a NoLogin error.
#[cfg(feature = "userdata")]
pub fn map_internal_playlist_err(e: Error) -> Error {
if let Error::Extraction(crate::error::ExtractionError::NotFound { .. }) = e {
Error::Auth(crate::error::AuthError::NoLogin)
if let Error::Extraction(ExtractionError::NotFound { .. }) = e {
Error::Auth(AuthError::NoLogin)
} else {
e
}

View file

@ -347,7 +347,6 @@ pub fn parse_textual_date_to_dt(
/// Parse a textual date (e.g. "29 minutes ago" "Jul 2, 2014") into a Date object.
///
/// Returns None if the date could not be parsed.
#[cfg(feature = "userdata")]
pub fn parse_textual_date_to_d(
lang: Language,
utc_offset: UtcOffset,

View file

@ -148,8 +148,8 @@ impl VisitorDataCache {
{
let vds = self.inner.visitor_data.read().unwrap();
if !vds.is_empty() {
let mut rng = rand::rng();
let vd = vds[rng.random_range(0..vds.len())].to_owned();
let mut rng = rand::thread_rng();
let vd = vds[rng.gen_range(0..vds.len())].to_owned();
tracing::debug!("visitor data {vd} picked from cache");
return Ok(vd);
}
@ -181,7 +181,7 @@ impl VisitorDataCache {
pub fn get_pot(&self, visitor_data: &str) -> Option<PoToken> {
let pots = self.inner.session_potoken.read().unwrap();
if let Some(entry) = pots.get(visitor_data) {
if entry.valid_until > OffsetDateTime::now_utc() + time::Duration::minutes(10) {
if entry.valid_until > OffsetDateTime::now_utc() {
return Some(entry.clone());
}
}
@ -245,21 +245,12 @@ mod tests {
for _ in 0..4 {
cache.get().await.unwrap();
}
for _ in 0..3 {
tokio::time::sleep(Duration::from_millis(1000)).await;
{
let vd = cache.inner.visitor_data.read().unwrap();
if !vd.contains(&v1) {
break;
}
}
}
{
let vd = cache.inner.visitor_data.read().unwrap();
assert!(!vd.contains(&v1), "first token still present");
}
assert_eq!(cache.get_pot(&v1), None);
}
}

File diff suppressed because it is too large Load diff

View file

@ -1,85 +0,0 @@
{
"af": "Ander weergawes",
"am": "ሌሎች ስሪቶች",
"ar": "إصدارات أخرى",
"as": "অন্য সংস্কৰণ",
"az": "Digər versiyalar",
"be": "Іншыя версіі",
"bg": "Други версии",
"bn": "অন্য সংস্করণগুলি",
"bs": "Druge verzije",
"ca": "Altres versions",
"cs": "Jiné verze",
"da": "Andre versioner",
"de": "Weitere Versionen",
"el": "Άλλες εκτελέσεις",
"en": "Other versions",
"en-GB": "Other versions",
"en-IN": "Other versions",
"es": "Otras versiones",
"es-419": "Otras versiones",
"es-US": "Otras versiones",
"et": "Teised versioonid",
"eu": "Beste bertsio batzuk",
"fa": "نسخه‌های دیگر",
"fi": "Muut versiot",
"fil": "Iba pang bersyon",
"fr": "Autres versions",
"fr-CA": "Autres versions",
"gl": "Outras versións",
"gu": "અન્ય વર્ઝન",
"hi": "अन्य वर्शन",
"hr": "Druge verzije",
"hu": "Más verziók",
"hy": "Այլ տարբերակներ",
"id": "Versi lainnya",
"is": "Aðrar útgáfur",
"it": "Altre versioni",
"iw": "גרסאות אחרות",
"ja": "他のバージョン",
"ka": "სხვა ვერსიები",
"kk": "Басқа нұсқалары",
"km": "ប្រភេទផ្សេងៗ",
"kn": "ಇತರ ಆವೃತ್ತಿಗಳು",
"ko": "다른 버전",
"ky": "Башка версиялар",
"lo": "ເວີຊັນອື່ນໆ",
"lt": "Kitos versijos",
"lv": "Citas versijas",
"mk": "Други верзии",
"ml": "മറ്റ് പതിപ്പുകൾ",
"mn": "Бусад хувилбар",
"mr": "इतर आवृत्त्या",
"ms": "Versi lain",
"my": "အခြား ဗားရှင်းများ",
"ne": "अन्य संस्करणहरू",
"nl": "Andere versies",
"no": "Andre versjoner",
"or": "ଅନ୍ୟ ସଂସ୍କରଣଗୁଡ଼ିକ",
"pa": "ਹੋਰ ਵਰਜਨ",
"pl": "Inne wersje",
"pt": "Outras versões",
"pt-PT": "Outras versões",
"ro": "Alte versiuni",
"ru": "Другие версии",
"si": "අනෙකුත් අනුවාදයන්",
"sk": "Ďalšie verzie",
"sl": "Druge različice",
"sq": "Versione të tjera",
"sr": "Друге верзије",
"sr-Latn": "Druge verzije",
"sv": "Andra versioner",
"sw": "Matoleo mengine",
"ta": "பிற பதிப்புகள்",
"te": "ఇతర వెర్షన్‌లు",
"th": "เวอร์ชันอื่นๆ",
"tr": "Diğer versiyonlar",
"uk": "Інші версії",
"ur": "دیگر ورژنز",
"uz": "Boshqa versiyalari",
"vi": "Các phiên bản khác",
"zh-CN": "其他版本",
"zh-HK": "其他版本",
"zh-TW": "其他版本",
"zu": "Ezinye izinguqulo"
}

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -18,7 +18,6 @@ MusicAlbum(
album_type: ep,
year: Some(2016),
by_va: false,
track_count: 5,
tracks: [
TrackItem(
id: "aGd3VKSOTxY",

View file

@ -13,7 +13,6 @@ MusicAlbum(
album_type: album,
year: Some(2024),
by_va: true,
track_count: 14,
tracks: [
TrackItem(
id: "ilNEztApdjI",

View file

@ -26,7 +26,6 @@ MusicAlbum(
album_type: single,
year: None,
by_va: false,
track_count: 1,
tracks: [
TrackItem(
id: "1Sz3lUVGBSM",

View file

@ -36,7 +36,6 @@ MusicAlbum(
album_type: album,
year: Some(2011),
by_va: false,
track_count: 15,
tracks: [
TrackItem(
id: "js0moD0CIRQ",

View file

@ -22,7 +22,6 @@ MusicAlbum(
album_type: show,
year: Some(2015),
by_va: false,
track_count: 27,
tracks: [
TrackItem(
id: "ZIjGPc6vG0Y",

View file

@ -22,7 +22,6 @@ MusicAlbum(
album_type: single,
year: Some(2020),
by_va: false,
track_count: 1,
tracks: [
TrackItem(
id: "VU6lEv0PKAo",

View file

@ -26,7 +26,6 @@ MusicAlbum(
album_type: album,
year: Some(2019),
by_va: false,
track_count: 18,
tracks: [
TrackItem(
id: "R3VIKRtzAdE",

View file

@ -13,7 +13,6 @@ MusicAlbum(
album_type: single,
year: Some(2022),
by_va: true,
track_count: 6,
tracks: [
TrackItem(
id: "Tzai7JXo45w",

View file

@ -42,8 +42,8 @@ MusicArtist(
by_va: false,
),
AlbumItem(
id: "MPREb_HrCgErOdgCv",
name: "Freiheit",
id: "MPREb_6PEkIQE7sWY",
name: "An deiner Seite (Online Version)",
cover: "[cover]",
artists: [
ArtistId(
@ -52,8 +52,8 @@ MusicArtist(
),
],
artist_id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
album_type: album,
year: Some(2004),
album_type: ep,
year: Some(2008),
by_va: false,
),
AlbumItem(
@ -87,8 +87,8 @@ MusicArtist(
by_va: false,
),
AlbumItem(
id: "MPREb_Oq0WKqNwSVY",
name: "Das 2. Gebot",
id: "MPREb_QEClJsuO9xM",
name: "So wie Du warst",
cover: "[cover]",
artists: [
ArtistId(
@ -97,8 +97,8 @@ MusicArtist(
),
],
artist_id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
album_type: album,
year: Some(2003),
album_type: single,
year: Some(2012),
by_va: false,
),
AlbumItem(
@ -251,21 +251,6 @@ MusicArtist(
year: Some(2015),
by_va: false,
),
AlbumItem(
id: "MPREb_ohcGTZrqKPZ",
name: "Zelluloid",
cover: "[cover]",
artists: [
ArtistId(
id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
name: "Unheilig",
),
],
artist_id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
album_type: album,
year: Some(2004),
by_va: false,
),
AlbumItem(
id: "MPREb_pWpeXxATZYb",
name: "Wir sind alle wie eins",
@ -281,6 +266,21 @@ MusicArtist(
year: Some(2014),
by_va: false,
),
AlbumItem(
id: "MPREb_rHhaDLqalbT",
name: "Winter (EP)",
cover: "[cover]",
artists: [
ArtistId(
id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
name: "Unheilig",
),
],
artist_id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
album_type: ep,
year: Some(2010),
by_va: false,
),
AlbumItem(
id: "MPREb_saXgTKNPaSu",
name: "Zeit zu gehen",

View file

@ -146,6 +146,21 @@ MusicArtist(
year: Some(2015),
by_va: false,
),
AlbumItem(
id: "MPREb_ghrNI6BJSM8",
name: "Friends And Family",
cover: "[cover]",
artists: [
ArtistId(
id: Some("UCFKUUtHjT4iq3p0JJA13SOA"),
name: "Every Time I Die",
),
],
artist_id: Some("UCFKUUtHjT4iq3p0JJA13SOA"),
album_type: album,
year: Some(2017),
by_va: false,
),
AlbumItem(
id: "MPREb_h0UZr2ALQXf",
name: "From Parts Unknown (Deluxe Edition)",

View file

@ -5,7 +5,7 @@ use std::fmt::Display;
use std::str::FromStr;
use rstest::{fixture, rstest};
use rustypipe::model::TrackType;
use rustypipe::model::{HistoryItem, TrackItem, TrackType, VideoItem};
use rustypipe::param::{AlbumOrder, LANGUAGES};
use time::{macros::date, OffsetDateTime};
@ -30,7 +30,7 @@ use rustypipe::validate;
#[case::desktop(ClientType::Desktop)]
#[case::tv(ClientType::Tv)]
#[case::mobile(ClientType::Mobile)]
// #[case::android(ClientType::Android)] Removed since it requires Android device attestation
#[case::android(ClientType::Android)]
#[case::ios(ClientType::Ios)]
#[tokio::test]
async fn get_player_from_client(#[case] client_type: ClientType, rp: RustyPipe) {
@ -40,6 +40,8 @@ async fn get_player_from_client(#[case] client_type: ClientType, rp: RustyPipe)
.await
.unwrap();
// dbg!(&player_data);
assert_eq!(player_data.details.id, "n4tK7LYFxI0");
assert_eq!(player_data.details.duration, 259);
assert!(!player_data.details.thumbnail.is_empty());
@ -480,6 +482,8 @@ async fn playlist_not_found(rp: RustyPipe) {
async fn get_video_details(rp: RustyPipe) {
let details = rp.query().video_details("ZeerrnuLi5E").await.unwrap();
// dbg!(&details);
assert_eq!(details.id, "ZeerrnuLi5E");
assert_eq!(details.name, "aespa 에스파 'Black Mamba' MV");
let desc = details.description.to_plaintext();
@ -515,6 +519,8 @@ async fn get_video_details(rp: RustyPipe) {
async fn get_video_details_music(rp: RustyPipe) {
let details = rp.query().video_details("XuM2onMGvTI").await.unwrap();
// dbg!(&details);
assert_eq!(details.id, "XuM2onMGvTI");
assert_eq!(details.name, "Gäa");
let desc = details.description.to_plaintext();
@ -551,6 +557,8 @@ async fn get_video_details_music(rp: RustyPipe) {
async fn get_video_details_ccommons(rp: RustyPipe) {
let details = rp.query().video_details("0rb9CfOvojk").await.unwrap();
// dbg!(&details);
assert_eq!(details.id, "0rb9CfOvojk");
assert_eq!(
details.name,
@ -589,6 +597,8 @@ async fn get_video_details_ccommons(rp: RustyPipe) {
async fn get_video_details_chapters(rp: RustyPipe) {
let details = rp.query().video_details("nFDBxBUfE74").await.unwrap();
// dbg!(&details);
assert_eq!(details.id, "nFDBxBUfE74");
assert_eq!(details.name, "The Prepper PC");
let desc = details.description.to_plaintext();
@ -707,6 +717,8 @@ async fn get_video_details_chapters(rp: RustyPipe) {
async fn get_video_details_live(rp: RustyPipe) {
let details = rp.query().video_details("jfKfPfyJRdk").await.unwrap();
// dbg!(&details);
assert_eq!(details.id, "jfKfPfyJRdk");
assert_eq!(
details.name,
@ -747,6 +759,8 @@ async fn get_video_details_live(rp: RustyPipe) {
async fn get_video_details_agelimit(rp: RustyPipe) {
let details = rp.query().video_details("ZDKQmBWTRnw").await.unwrap();
// dbg!(&details);
assert_eq!(details.id, "ZDKQmBWTRnw");
assert_eq!(
details.name,
@ -850,6 +864,7 @@ async fn channel_videos(rp: RustyPipe) {
.await
.unwrap();
// dbg!(&channel);
assert_channel_eevblog(&channel);
assert!(
@ -875,6 +890,7 @@ async fn channel_shorts(rp: RustyPipe) {
.await
.unwrap();
// dbg!(&channel);
assert_eq!(channel.id, "UCh8gHdtzO2tXd593_bjErWg");
assert_eq!(channel.name, "Doobydobap");
assert_eq!(channel.handle.as_deref(), Some("@Doobydobap"));
@ -903,6 +919,7 @@ async fn channel_livestreams(rp: RustyPipe) {
.await
.unwrap();
// dbg!(&channel);
assert_channel_eevblog(&channel);
assert!(
@ -2119,13 +2136,11 @@ async fn music_search_artists(rp: RustyPipe, unlocalized: bool) {
#[rstest]
#[tokio::test]
async fn music_search_artists_cont(rp: RustyPipe) {
let res = rp.query().music_search_artists("girls").await.unwrap();
let res = rp.query().music_search_artists("boys").await.unwrap();
assert_eq!(res.corrected_query, None);
if !res.items.is_exhausted() {
assert_next(res.items, rp.query(), 15, 2, true).await;
}
}
#[rstest]
#[tokio::test]
@ -2596,7 +2611,7 @@ async fn music_genres(rp: RustyPipe, unlocalized: bool) {
}
#[rstest]
#[case::party("ggMPOg1uX2w1aW1CRDFTSUNo", "Party")]
#[case::chill("ggMPOg1uX1JOQWZFeDByc2Jm", "Chill")]
#[case::pop("ggMPOg1uX1lMbVZmbzl6NlJ3", "Pop")]
#[tokio::test]
async fn music_genre(#[case] id: &str, #[case] name: &str, rp: RustyPipe, unlocalized: bool) {
@ -2640,7 +2655,7 @@ async fn music_genre(#[case] id: &str, #[case] name: &str, rp: RustyPipe, unloca
let subgenres = check_music_genre(genre, id, name, unlocalized);
if name == "Party" {
if name == "Chill" {
assert_gte(subgenres.len(), 2, "subgenres");
}
@ -2713,12 +2728,9 @@ async fn isrc_search_languages(rp: RustyPipe) {
}
}
#[cfg(feature = "userdata")]
mod user_data {
mod cookie_auth {
use super::*;
use rustypipe::model::{HistoryItem, TrackItem, VideoItem};
#[rstest]
#[tokio::test]
async fn history(rp: RustyPipe) {
@ -2802,30 +2814,6 @@ mod user_data {
let tracks = rp.query().music_liked_tracks().await.unwrap();
assert_next_items(tracks.tracks, rp.query(), 5).await;
}
/// Assert that the history paginator produces at least n items
async fn assert_next_history<Q: AsRef<RustyPipeQuery>>(
paginator: Paginator<HistoryItem<VideoItem>>,
query: Q,
n_items: usize,
) {
let mut p = paginator;
let query = query.as_ref();
p.extend_limit(query, n_items).await.unwrap();
assert_gte(p.items.len(), n_items, "items");
}
/// Assert that the music history paginator produces at least n items
async fn assert_next_music_history<Q: AsRef<RustyPipeQuery>>(
paginator: Paginator<HistoryItem<TrackItem>>,
query: Q,
n_items: usize,
) {
let mut p = paginator;
let query = query.as_ref();
p.extend_limit(query, n_items).await.unwrap();
assert_gte(p.items.len(), n_items, "items");
}
}
#[rstest]
@ -2931,11 +2919,7 @@ async fn assert_next<T: FromYtItem, Q: AsRef<RustyPipeQuery>>(
}
for i in 0..n_pages {
match p.next(query).await.unwrap() {
Some(np) => p = np,
None => panic!("paginator exhausted after {i} pages"),
}
p = p.next(query).await.unwrap().expect("paginator exhausted");
assert_gte(
p.items.len(),
min_items,
@ -2956,6 +2940,30 @@ async fn assert_next_items<T: FromYtItem, Q: AsRef<RustyPipeQuery>>(
assert_gte(p.items.len(), n_items, "items");
}
/// Assert that the history paginator produces at least n items
async fn assert_next_history<Q: AsRef<RustyPipeQuery>>(
paginator: Paginator<HistoryItem<VideoItem>>,
query: Q,
n_items: usize,
) {
let mut p = paginator;
let query = query.as_ref();
p.extend_limit(query, n_items).await.unwrap();
assert_gte(p.items.len(), n_items, "items");
}
/// Assert that the music history paginator produces at least n items
async fn assert_next_music_history<Q: AsRef<RustyPipeQuery>>(
paginator: Paginator<HistoryItem<TrackItem>>,
query: Q,
n_items: usize,
) {
let mut p = paginator;
let query = query.as_ref();
p.extend_limit(query, n_items).await.unwrap();
assert_gte(p.items.len(), n_items, "items");
}
#[track_caller]
fn assert_frameset(frameset: &Frameset) {
assert_gte(frameset.frame_height, 20, "frame height");
@ -3017,6 +3025,10 @@ async fn all_send_and_sync() {
rp.query()
.drm_license("", rustypipe::model::DrmSystem::Widevine, "", "", &[]),
);
send_and_sync(rp.query().history());
send_and_sync(rp.query().history_continuation("", None));
send_and_sync(rp.query().history_search(""));
send_and_sync(rp.query().liked_videos());
send_and_sync(rp.query().music_album(""));
send_and_sync(rp.query().music_artist("", false));
send_and_sync(rp.query().music_artist_albums("", None, None));
@ -3025,6 +3037,9 @@ async fn all_send_and_sync() {
send_and_sync(rp.query().music_details(""));
send_and_sync(rp.query().music_genre(""));
send_and_sync(rp.query().music_genres());
send_and_sync(rp.query().music_history());
send_and_sync(rp.query().music_history_continuation("", None));
send_and_sync(rp.query().music_liked_tracks());
send_and_sync(rp.query().music_lyrics(""));
send_and_sync(rp.query().music_new_albums());
send_and_sync(rp.query().music_new_videos());
@ -3033,6 +3048,10 @@ async fn all_send_and_sync() {
send_and_sync(rp.query().music_radio_playlist(""));
send_and_sync(rp.query().music_radio_track(""));
send_and_sync(rp.query().music_related(""));
send_and_sync(rp.query().music_saved_albums());
send_and_sync(rp.query().music_saved_artists());
send_and_sync(rp.query().music_saved_playlists());
send_and_sync(rp.query().music_saved_tracks());
send_and_sync(rp.query().music_search::<MusicItem, _>("", None));
send_and_sync(rp.query().music_search_albums(""));
send_and_sync(rp.query().music_search_artists(""));
@ -3049,32 +3068,17 @@ async fn all_send_and_sync() {
send_and_sync(rp.query().raw(ClientType::Desktop, "", ""));
send_and_sync(rp.query().resolve_string("", false));
send_and_sync(rp.query().resolve_url("", false));
send_and_sync(rp.query().saved_playlists());
send_and_sync(rp.query().search::<YouTubeItem, _>(""));
send_and_sync(
rp.query()
.search_filter::<YouTubeItem, _>("", &SearchFilter::default()),
);
send_and_sync(rp.query().search_suggestion(""));
send_and_sync(rp.query().subscription_feed());
send_and_sync(rp.query().subscriptions());
send_and_sync(rp.query().trending());
send_and_sync(rp.query().video_comments("", None));
send_and_sync(rp.query().video_details(""));
#[cfg(feature = "userdata")]
{
send_and_sync(rp.query().history());
send_and_sync(rp.query().history_continuation("", None));
send_and_sync(rp.query().history_search(""));
send_and_sync(rp.query().liked_videos());
send_and_sync(rp.query().watch_later());
send_and_sync(rp.query().music_history());
send_and_sync(rp.query().music_history_continuation("", None));
send_and_sync(rp.query().music_saved_albums());
send_and_sync(rp.query().music_saved_artists());
send_and_sync(rp.query().music_saved_playlists());
send_and_sync(rp.query().music_saved_tracks());
send_and_sync(rp.query().saved_playlists());
send_and_sync(rp.query().subscription_feed());
send_and_sync(rp.query().subscriptions());
send_and_sync(rp.query().music_liked_tracks());
}
}