diff --git a/.forgejo/workflows/ci.yaml b/.forgejo/workflows/ci.yaml deleted file mode 100644 index 6c53208..0000000 --- a/.forgejo/workflows/ci.yaml +++ /dev/null @@ -1,68 +0,0 @@ -name: CI -on: - push: - branches: ["main"] - pull_request: - -jobs: - Test: - runs-on: cimaster-latest - services: - warpproxy: - image: thetadev256/warpproxy - env: - WARP_DEVICE_ID: ${{ secrets.WARP_DEVICE_ID }} - WARP_ACCESS_TOKEN: ${{ secrets.WARP_ACCESS_TOKEN }} - WARP_LICENSE_KEY: ${{ secrets.WARP_LICENSE_KEY }} - WARP_PRIVATE_KEY: ${{ secrets.WARP_PRIVATE_KEY }} - steps: - - name: 📦 Checkout repository - uses: actions/checkout@v4 - - - name: 🦀 Setup Rust cache - uses: https://github.com/Swatinem/rust-cache@v2 - with: - cache-on-failure: "true" - - - name: Download rustypipe-botguard - run: | - TARGET=$(rustc --version --verbose | grep "host:" | sed -e 's/^host: //') - cd ~ - curl -SsL -o rustypipe-botguard.tar.xz "https://codeberg.org/ThetaDev/rustypipe-botguard/releases/download/v0.1.1/rustypipe-botguard-v0.1.1-${TARGET}.tar.xz" - cd /usr/local/bin - sudo tar -xJf ~/rustypipe-botguard.tar.xz - rm ~/rustypipe-botguard.tar.xz - rustypipe-botguard --version - - - name: 📎 Clippy - run: | - cargo clippy --all --tests --features=rss,userdata,indicatif,audiotag -- -D warnings - cargo clippy --package=rustypipe --tests -- -D warnings - cargo clippy --package=rustypipe-downloader -- -D warnings - cargo clippy --package=rustypipe-cli -- -D warnings - cargo clippy --package=rustypipe-cli --features=timezone -- -D warnings - - - name: 🧪 Test - run: cargo nextest run --config-file ~/.config/nextest.toml --profile ci --retries 2 --features rss,userdata --workspace -- --skip 'user_data::' - env: - ALL_PROXY: "http://warpproxy:8124" - - - name: Move test report - if: always() - run: mv target/nextest/ci/junit.xml junit.xml || true - - - name: 💌 Upload test report - if: always() - uses: https://code.forgejo.org/forgejo/upload-artifact@v4 - with: - name: test - path: | - junit.xml - rustypipe_reports - - - name: 🔗 Artifactview PR comment - if: ${{ always() && github.event_name == 'pull_request' }} - run: | - if [[ "$GITEA_ACTIONS" == "true" ]]; then RUN_NUMBER="$GITHUB_RUN_NUMBER"; else RUN_NUMBER="$GITHUB_RUN_ID"; fi - curl -SsL --fail-with-body -w "\n" -X POST https://av.thetadev.de/.well-known/api/prComment -H "Content-Type: application/json" \ - --data '{"url": "'"$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$RUN_NUMBER"'", "pr": ${{ github.event.number }}, "artifact_titles": {"test":"🧪 Test report"}, "artifact_paths": {"test":"/junit.xml?viewer=1"}}' diff --git a/.forgejo/workflows/release-cli.yaml b/.forgejo/workflows/release-cli.yaml deleted file mode 100644 index 6268ead..0000000 --- a/.forgejo/workflows/release-cli.yaml +++ /dev/null @@ -1,69 +0,0 @@ -name: Release CLI -on: - push: - tags: - - "rustypipe-cli/v*.*.*" - -jobs: - Release: - runs-on: cimaster-latest - steps: - - name: 📦 Checkout repository - uses: actions/checkout@v4 - - - name: Setup cross compilation - run: | - rustup target add x86_64-pc-windows-msvc x86_64-apple-darwin aarch64-apple-darwin - cargo install cargo-xwin - - # https://wapl.es/rust/2019/02/17/rust-cross-compile-linux-to-macos.html/ - sudo apt-get install -y llvm clang cmake - cd ~ - git clone https://github.com/tpoechtrager/osxcross - cd osxcross - wget -nc "https://github.com/joseluisq/macosx-sdks/releases/download/12.3/MacOSX12.3.sdk.tar.xz" - mv MacOSX12.3.sdk.tar.xz tarballs/ - UNATTENDED=yes OSX_VERSION_MIN=12.3 ./build.sh - OSXCROSS_BIN="$(pwd)/target/bin" - - echo "CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=$(find "$OSXCROSS_BIN" -name "x86_64-apple-darwin*-clang")" >> $GITHUB_ENV - echo "CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS=-Car=$(find "$OSXCROSS_BIN" -name "x86_64-apple-darwin*-ar"),-Clink-arg=-undefined,-Clink-arg=dynamic_lookup" >> $GITHUB_ENV - echo "CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=$(find "$OSXCROSS_BIN" -name "aarch64-apple-darwin*-clang")" >> $GITHUB_ENV - echo "CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS=-Car=$(find "$OSXCROSS_BIN" -name "aarch64-apple-darwin*-ar"),-Clink-arg=-undefined,-Clink-arg=dynamic_lookup" >> $GITHUB_ENV - - - name: ⚒️ Build application - run: | - export PATH="$PATH:$HOME/osxcross/target/bin" - CRATE="rustypipe-cli" - PKG_CONFIG_SYSROOT_DIR=/usr/x86_64-linux-gnu cargo build --release --package=$CRATE --target x86_64-unknown-linux-gnu - PKG_CONFIG_SYSROOT_DIR=/usr/aarch64-linux-gnu cargo build --release --package=$CRATE --target aarch64-unknown-linux-gnu - CC="$CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER" CXX="$CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER++" cargo build --release --package=$CRATE --target x86_64-apple-darwin - CC="$CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER" CXX="$CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER++" cargo build --release --package=$CRATE --target aarch64-apple-darwin - cargo xwin build --release --package=$CRATE --target x86_64-pc-windows-msvc - - - name: Prepare release - run: | - CRATE="rustypipe-cli" - BIN="rustypipe" - echo "CRATE=$CRATE" >> "$GITHUB_ENV" - echo "CRATE_VERSION=$(echo '${{ github.ref_name }}' | awk 'BEGIN{RS="/"} NR==2{print}')" >> "$GITHUB_ENV" - CL_PATH="cli/CHANGELOG.md" - { - echo 'CHANGELOG<> "$GITHUB_ENV" - - mkdir dist - - for arch in x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu x86_64-apple-darwin aarch64-apple-darwin; do - tar -cJf "dist/${BIN}-${CRATE_VERSION}-${arch}.tar.xz" -C target/${arch}/release "${BIN}" - done - (cd target/x86_64-pc-windows-msvc/release && zip -9 "../../../dist/${BIN}-${CRATE_VERSION}-x86_64-pc-windows-msvc.zip" "${BIN}.exe") - - - name: 🎉 Publish release - uses: https://gitea.com/actions/release-action@main - with: - title: "${{ env.CRATE }} ${{ env.CRATE_VERSION }}" - body: "${{ env.CHANGELOG }}" - files: dist/* diff --git a/.forgejo/workflows/release.yaml b/.forgejo/workflows/release.yaml deleted file mode 100644 index 95d30cf..0000000 --- a/.forgejo/workflows/release.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: Release -on: - push: - tags: - - "*/v*.*.*" - -jobs: - Release: - runs-on: cimaster-latest - steps: - - name: 📦 Checkout repository - uses: actions/checkout@v4 - - - name: Get variables - run: | - CRATE=$(echo '${{ github.ref_name }}' | awk 'BEGIN{RS="/"} NR==1{print}') - echo "CRATE=$CRATE" >> "$GITHUB_ENV" - echo "CRATE_VERSION=$(echo '${{ github.ref_name }}' | awk 'BEGIN{RS="/"} NR==2{print}')" >> "$GITHUB_ENV" - CL_PATH="CHANGELOG.md" - if [[ "$CRATE" != "rustypipe" ]]; then pfx="rustypipe-"; CL_PATH="${CRATE#"$pfx"}/$CL_PATH"; fi - { - echo 'CHANGELOG<> "$GITHUB_ENV" - - - name: 📤 Publish crate on crates.io - run: cargo publish --token ${{ secrets.CARGO_TOKEN }} --package "${{ env.CRATE }}" - - - name: 🎉 Publish release - uses: https://gitea.com/actions/release-action@main - with: - title: "${{ env.CRATE }} ${{ env.CRATE_VERSION }}" - body: "${{ env.CHANGELOG }}" diff --git a/.forgejo/workflows/renovate.yaml.bak b/.forgejo/workflows/renovate.yaml.bak deleted file mode 100644 index 013b185..0000000 --- a/.forgejo/workflows/renovate.yaml.bak +++ /dev/null @@ -1,63 +0,0 @@ -name: renovate - -on: - push: - branches: ["main"] - paths: - - ".forgejo/workflows/renovate.yaml" - - "renovate.json" - schedule: - - cron: "0 0 * * *" - workflow_dispatch: - -env: - RENOVATE_REPOSITORIES: ${{ github.repository }} - -jobs: - renovate: - runs-on: docker - container: - image: renovate/renovate:39 - - steps: - - name: Load renovate repo cache - uses: actions/cache/restore@v4 - with: - path: | - .tmp/cache/renovate/repository - .tmp/cache/renovate/renovate-cache-sqlite - .tmp/osv - key: repo-cache-${{ github.run_id }} - restore-keys: | - repo-cache- - - - name: Run renovate - run: renovate - env: - LOG_LEVEL: debug - RENOVATE_BASE_DIR: ${{ github.workspace }}/.tmp - RENOVATE_ENDPOINT: ${{ github.server_url }} - RENOVATE_PLATFORM: gitea - RENOVATE_REPOSITORY_CACHE: 'enabled' - RENOVATE_TOKEN: ${{ secrets.FORGEJO_CI_BOT_TOKEN }} - GITHUB_COM_TOKEN: ${{ secrets.GH_PUBLIC_TOKEN }} - RENOVATE_GIT_AUTHOR: 'Renovate Bot ' - - RENOVATE_X_SQLITE_PACKAGE_CACHE: true - - GIT_AUTHOR_NAME: 'Renovate Bot' - GIT_AUTHOR_EMAIL: 'forgejo-renovate-action@forgejo.org' - GIT_COMMITTER_NAME: 'Renovate Bot' - GIT_COMMITTER_EMAIL: 'forgejo-renovate-action@forgejo.org' - - OSV_OFFLINE_ROOT_DIR: ${{ github.workspace }}/.tmp/osv - - - name: Save renovate repo cache - if: always() && env.RENOVATE_DRY_RUN != 'full' - uses: actions/cache/save@v4 - with: - path: | - .tmp/cache/renovate/repository - .tmp/cache/renovate/renovate-cache-sqlite - .tmp/osv - key: repo-cache-${{ github.run_id }} diff --git a/.gitignore b/.gitignore index 3e0f26a..4e1b2c5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,4 @@ *.snap.new rustypipe_reports -rustypipe_cache*.json -bg_snapshot.bin +rustypipe_cache.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9a0cbb3..5c65c90 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v4.3.0 hooks: - id: end-of-file-fixer - id: check-json @@ -10,8 +10,4 @@ repos: hooks: - id: cargo-fmt - id: cargo-clippy - name: cargo-clippy rustypipe - args: ["--package=rustypipe", "--tests", "--", "-D", "warnings"] - - id: cargo-clippy - name: cargo-clippy workspace - args: ["--all", "--tests", "--features=rss,userdata,indicatif,audiotag", "--", "-D", "warnings"] + args: ["--all", "--features=rss", "--", "-D", "warnings"] diff --git a/.woodpecker.yml b/.woodpecker.yml new file mode 100644 index 0000000..47541f7 --- /dev/null +++ b/.woodpecker.yml @@ -0,0 +1,10 @@ +pipeline: + test: + image: rust:latest + environment: + - CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse + commands: + - rustup component add rustfmt clippy + - cargo fmt --all --check + - cargo clippy --all --features=rss -- -D warnings + - cargo test --features=rss --workspace diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index b44ec1f..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,396 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - - -## [v0.11.4](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.11.3..rustypipe/v0.11.4) - 2025-04-23 - -### 🚀 Features - -- Player: handle VPN ban and captcha required error messages - ([be6da5e](https://codeberg.org/ThetaDev/rustypipe/commit/be6da5e7e3558ef39773bf45bcb8afbf006bacec)) - -### 🐛 Bug Fixes - -- Deobfuscator: handle 1-char long global variables, find nsig fn (player 6450230e) - ([d675987](https://codeberg.org/ThetaDev/rustypipe/commit/d675987654972c6aa4cc2b291d25bc49fa60173e)) - - -## [v0.11.3](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.11.2..rustypipe/v0.11.3) - 2025-04-03 - -### 🐛 Bug Fixes - -- Deobfuscator: global variable extraction fixed - ([ac44e95](https://codeberg.org/ThetaDev/rustypipe/commit/ac44e95a88d95f9d2d1ec672f86ca9d31d6991b9)) -- Deobfuscator: small simplification - ([189ba81](https://codeberg.org/ThetaDev/rustypipe/commit/189ba81a42e6c09f6af4d2768c449c22b864101e)) -- Deobfuscator: handle global functions as well - ([939a7ae](https://codeberg.org/ThetaDev/rustypipe/commit/939a7aea61a3eee4c1e67bfbfc835f0ce3934171)) -- Handle music playlist/album not found - ([ea80717](https://codeberg.org/ThetaDev/rustypipe/commit/ea80717f692b2c45b5063c362c9fa8ebca5a3471)) -- Switch client if no adaptive stream URLs were returned - ([187bf1c](https://codeberg.org/ThetaDev/rustypipe/commit/187bf1c9a0e846bff205e0d71a19c5a1ce7b1943)) -- Handle music artist not found - ([daf3d03](https://codeberg.org/ThetaDev/rustypipe/commit/daf3d035be38b59aef1ae205ac91c2bbdda2fe66)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rust crate rand to 0.9.0 - ([af415dd](https://codeberg.org/ThetaDev/rustypipe/commit/af415ddf8f94f00edb918f271d8e6336503e9faf)) - - -## [v0.11.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.11.1..rustypipe/v0.11.2) - 2025-03-24 - -### 🐛 Bug Fixes - -- A/B test 22: commandExecutorCommand for playlist continuations - ([e8acbfb](https://codeberg.org/ThetaDev/rustypipe/commit/e8acbfbbcf5d31b5ac34410ddf334e5534e3762f)) -- Extract deobf data with global strings variable - ([4ce6746](https://codeberg.org/ThetaDev/rustypipe/commit/4ce6746be538564e79f7e3c67d7a91aaa53f48ea)) -- Handle player returning no adaptive stream URLs - ([07db7b1](https://codeberg.org/ThetaDev/rustypipe/commit/07db7b1166e912e1554f98f2ae20c2c356fed38f)) - - -## [v0.11.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.11.0..rustypipe/v0.11.1) - 2025-03-16 - -### 🐛 Bug Fixes - -- Simplify get_player_from_clients logic - ([c04b606](https://codeberg.org/ThetaDev/rustypipe/commit/c04b60604d2628bf8f0e3de453c243adbb966e57)) -- Desktop client: generate PO token from user_syncid when authenticated - ([8342cae](https://codeberg.org/ThetaDev/rustypipe/commit/8342caeb0f566a38060a6ec69f3ca65b9a2afcd6)) -- Always skip failed clients - ([63a6f50](https://codeberg.org/ThetaDev/rustypipe/commit/63a6f50a8b5ad6bb984282335c1481ae3cd2fe83)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rust crate rstest to 0.25.0 - ([9ed1306](https://codeberg.org/ThetaDev/rustypipe/commit/9ed1306f3aaeb993c409997ddfbc47499e4f4d22)) - - -## [v0.11.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.10.0..rustypipe/v0.11.0) - 2025-02-26 - -### 🚀 Features - -- Add original album track count, fix fetching albums with more than 200 tracks - ([544782f](https://codeberg.org/ThetaDev/rustypipe/commit/544782f8de728cda0aca9a1cb95837cdfbd001f1)) - -### 🐛 Bug Fixes - -- A/B test 21: music album recommendations - ([6737512](https://codeberg.org/ThetaDev/rustypipe/commit/6737512f5f67c8cd05d4552dd0e0f24381035b35)) - - -## [v0.10.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.9.0..rustypipe/v0.10.0) - 2025-02-09 - -### 🚀 Features - -- Add visitor data cache, remove random visitor data - ([b12f4c5](https://codeberg.org/ThetaDev/rustypipe/commit/b12f4c5d821a9189d7ed8410ad860824b6d052ef)) -- Add support for rustypipe-botguard to get PO tokens - ([b90a252](https://codeberg.org/ThetaDev/rustypipe/commit/b90a252a5e1bf05a5294168b0ec16a73cbb88f42)) -- Add session po token cache - ([b72b501](https://codeberg.org/ThetaDev/rustypipe/commit/b72b501b6dbcf4333b24cd80e7c8c61b0c21ec91)) -- Check rustypipe-botguard-api version - ([8385b87](https://codeberg.org/ThetaDev/rustypipe/commit/8385b87c63677f32a240679a78702f53072e517a)) -- Rewrite request attempt system, retry with different visitor data - ([dfd03ed](https://codeberg.org/ThetaDev/rustypipe/commit/dfd03edfadff2657e9cfbf04e5d313ba409520ac)) -- Log failed player fetch attempts with player_from_clients - ([8e35358](https://codeberg.org/ThetaDev/rustypipe/commit/8e35358c8941301f6ebf7646a11ab22711082569)) -- Add timezone query option - ([3a2370b](https://codeberg.org/ThetaDev/rustypipe/commit/3a2370b97ca3d0f40d72d66a23295557317d29fb)) -- [**breaking**] Add userdata feature for all personal data queries (playback history, subscriptions) - ([65cb424](https://codeberg.org/ThetaDev/rustypipe/commit/65cb4244c6ab547f53d0cb12af802c4189188c86)) -- Add RustyPipe::version_botguard fn, detect rustypipe-botguard in current dir, add botguard version to report - ([1d755b7](https://codeberg.org/ThetaDev/rustypipe/commit/1d755b76bf4569f7d0bb90a65494ac8e7aae499a)) - -### 🐛 Bug Fixes - -- Parsing history dates - ([af7dc10](https://codeberg.org/ThetaDev/rustypipe/commit/af7dc1016322a87dd8fec0b739939c2b12b6f400)) -- A/V streams incorrectly recognized as video-only - ([2b891ca](https://codeberg.org/ThetaDev/rustypipe/commit/2b891ca0788f91f16dbb9203191cb3d2092ecc74)) -- Update iOS client - ([e915416](https://codeberg.org/ThetaDev/rustypipe/commit/e91541629d6c944c1001f5883e3c1264aeeb3969)) -- A/B test 20: music continuation item renderer - ([9c67f8f](https://codeberg.org/ThetaDev/rustypipe/commit/9c67f8f85bef8214848dc9d17bff6cff252e015e)) -- Include whole request body in report - ([15245c1](https://codeberg.org/ThetaDev/rustypipe/commit/15245c18b584e42523762b94fcc7284d483660a0)) -- Extracting nsig fn when outside variable starts with $ - ([eda16e3](https://codeberg.org/ThetaDev/rustypipe/commit/eda16e378730a3b57c4982a626df1622a93c574a)) -- Retry updating deobf data after a RustyPipe update - ([50ab1f7](https://codeberg.org/ThetaDev/rustypipe/commit/50ab1f7a5d8aeaa3720264b4a4b27805bb0e8121)) -- Allow player data to be fetched without botguard - ([29c854b](https://codeberg.org/ThetaDev/rustypipe/commit/29c854b20d7a6677415b1744e7ba7ecd4f594ea5)) -- Output full request body in reports, clean up `get_player_po_token` - ([a0d850f](https://codeberg.org/ThetaDev/rustypipe/commit/a0d850f8e01428a73bbd66397d0dbf797b45958f)) -- Correct timezone offset for parsed dates, add timezone_local option - ([a5a7be5](https://codeberg.org/ThetaDev/rustypipe/commit/a5a7be5b4e0a0b73d7e1dc802ebd7bd48dafc76d)) -- Use localzone crate to get local tz - ([5acbf0e](https://codeberg.org/ThetaDev/rustypipe/commit/5acbf0e456b1f10707e0a56125d993a8129eee3a)) -- Only use cached potokens with min. 10min lifetime - ([0c94267](https://codeberg.org/ThetaDev/rustypipe/commit/0c94267d0371b2b26c7b5c9abfa156d5cde2153e)) - -### 📚 Documentation - -- Add Botguard info to README - ([9957add](https://codeberg.org/ThetaDev/rustypipe/commit/9957add2b5d6391b2c1869d2019fd7dd91b8cd41)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rust crate rquickjs to 0.9.0 (#33) - ([2c8ac41](https://codeberg.org/ThetaDev/rustypipe/commit/2c8ac410aa535d83f8bcc7181f81914b13bceb77)) - - -## [v0.9.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.8.0..rustypipe/v0.9.0) - 2025-01-16 - -### 🚀 Features - -- Add functions to fetch a user's history and subscriptions - ([14e3995](https://codeberg.org/ThetaDev/rustypipe/commit/14e399594f97a1228a8c2991a14dd8745af1beb7)) -- Add history item dates, extend timeago parser - ([320a8c2](https://codeberg.org/ThetaDev/rustypipe/commit/320a8c2c24217ad5697f0424c4f994bbbe31f3aa)) -- Add session headers when using cookie auth - ([3c95b52](https://codeberg.org/ThetaDev/rustypipe/commit/3c95b52ceaf0df2d67ee0d2f2ac658f666f29836)) -- Add cookies.txt parser, add cookie auth + history cmds to CLI - ([cf498e4](https://codeberg.org/ThetaDev/rustypipe/commit/cf498e4a8f9318b0197bc3f0cbaf7043c53adb9d)) -- Add method to get saved_playlists - ([27f64fc](https://codeberg.org/ThetaDev/rustypipe/commit/27f64fc412e833d5bd19ad72913aae19358e98b9)) -- Extract player DRM data - ([2af4001](https://codeberg.org/ThetaDev/rustypipe/commit/2af4001c75f2ff4f7c891aa59ac22c2c6b7902a2)) -- Add Dolby audio codecs (ac-3, ec-3) - ([a7f8c78](https://codeberg.org/ThetaDev/rustypipe/commit/a7f8c789b1a34710274c4630e027ef868397aea2)) -- Add DRM and audio channel number filtering to StreamFilter - ([d5abee2](https://codeberg.org/ThetaDev/rustypipe/commit/d5abee275300ab1bc10fc8d6c35a4e3813fd2bd4)) -- Set cache file permissions to 600 - ([dee8a99](https://codeberg.org/ThetaDev/rustypipe/commit/dee8a99e7a8d071c987709a01f02ee8fecf2d776)) - -### 🐛 Bug Fixes - -- Dont leak authorization and cookie header in reports - ([75fce91](https://codeberg.org/ThetaDev/rustypipe/commit/75fce91353c02cd498f27d21b08261c23ea03d70)) -- Require new time crate version which added Month::length - ([ec7a195](https://codeberg.org/ThetaDev/rustypipe/commit/ec7a195c98f39346c4c8db875212c3843580450e)) -- Parsing numbers (it), dates (kn) - ([63f86b6](https://codeberg.org/ThetaDev/rustypipe/commit/63f86b6e186aa1d2dcaf7e9169ccebb2265e5905)) -- Accept user-specific playlist ids (LL, WL) - ([97c3f30](https://codeberg.org/ThetaDev/rustypipe/commit/97c3f30d180d3e62b7e19f22d191d7fd7614daca)) -- Only use auth-enabled clients for fetching player with auth option enabled - ([2b2b4af](https://codeberg.org/ThetaDev/rustypipe/commit/2b2b4af0b26cdd0d4bf2218d3f527abd88658abf)) -- A/B test 19: Music artist album groups reordered - ([5daad1b](https://codeberg.org/ThetaDev/rustypipe/commit/5daad1b700e8dcf1f3e803db1685f08f27794898)) -- Switch to rquickjs crate for deobfuscator - ([75c3746](https://codeberg.org/ThetaDev/rustypipe/commit/75c3746890f3428f3314b7b10c9ec816ad275836)) -- Player_from_clients method not send/sync - ([9c512c3](https://codeberg.org/ThetaDev/rustypipe/commit/9c512c3c4dbec0fc3b973536733d61ba61125a92)) - -### 📚 Documentation - -- Update README - ([0432477](https://codeberg.org/ThetaDev/rustypipe/commit/0432477451ecd5f64145d65239c721f4e44826c0)) -- Fix README - ([11442df](https://codeberg.org/ThetaDev/rustypipe/commit/11442dfd369599396357f5b7a7a4268a7b537f57)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rust crate rstest to 0.24.0 (#20) - ([ab19034](https://codeberg.org/ThetaDev/rustypipe/commit/ab19034ab19baf090e83eada056559676ffdadce)) -- *(deps)* Update rust crate dirs to v6 (#24) - ([6a60425](https://codeberg.org/ThetaDev/rustypipe/commit/6a604252b1af7a9388db5dc170f737069cc31051)) -- Update pre-commit hooks - ([7cd9246](https://codeberg.org/ThetaDev/rustypipe/commit/7cd9246260493d7839018cb39a2dfb4dded8b343)) - - -## [v0.8.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.7.2..rustypipe/v0.8.0) - 2024-12-20 - -### 🚀 Features - -- Log warning when generating report - ([258f18a](https://codeberg.org/ThetaDev/rustypipe/commit/258f18a99d848ae7e6808beddad054037a3b3799)) -- Add auto-dubbed audio tracks, improved StreamFilter - ([1d1ae17](https://codeberg.org/ThetaDev/rustypipe/commit/1d1ae17ffc16724667d43142aa57abda2e6468e4)) - -### 🐛 Bug Fixes - -- Replace deprecated call to `time::util::days_in_year_month` - ([69ef6ae](https://codeberg.org/ThetaDev/rustypipe/commit/69ef6ae51e9b09a9b9c06057e717bf6f054c9803)) -- Nsig fn extra variable extraction - ([8014741](https://codeberg.org/ThetaDev/rustypipe/commit/80147413ee3190bb530f8f6b02738bcc787a6444)) -- Deobf function extraction, allow $ in variable names - ([8cadbc1](https://codeberg.org/ThetaDev/rustypipe/commit/8cadbc1a4c865d085e30249dba0f353472456a32)) -- Remove leading zero-width-space from comments, ensure space after links - ([162959c](https://codeberg.org/ThetaDev/rustypipe/commit/162959ca4513a03496776fae905b4bf20c79899c)) -- Update client versions, enable Opus audio with iOS client - ([1b60c97](https://codeberg.org/ThetaDev/rustypipe/commit/1b60c97a183b9d74b92df14b5b113c61aba1be7f)) -- Extract transcript from comment voice replies - ([30f60c3](https://codeberg.org/ThetaDev/rustypipe/commit/30f60c30f9d87d39585db93c1c9e274f48d688ba)) -- Error 400 when fetching player with login - ([5ce84c4](https://codeberg.org/ThetaDev/rustypipe/commit/5ce84c44a6844f692258066c83e04df875e0aa91)) - -### ⚙️ Miscellaneous Tasks - -- Update user agent - ([53e5846](https://codeberg.org/ThetaDev/rustypipe/commit/53e5846286e8db920622152c2a0a57ddc7c41d25)) - - -## [v0.7.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.7.1..rustypipe/v0.7.2) - 2024-12-13 - -### 🐛 Bug Fixes - -- Replace futures dependency with futures-util - ([5c39bf4](https://codeberg.org/ThetaDev/rustypipe/commit/5c39bf4842b13d37a4277ea5506e15c179892ce5)) -- Lifetime-related lints - ([c4feff3](https://codeberg.org/ThetaDev/rustypipe/commit/c4feff37a5989097b575c43d89c26427d92d77b9)) -- Limit retry attempts to fetch client versions and deobf data - ([44ae456](https://codeberg.org/ThetaDev/rustypipe/commit/44ae456d2c654679837da8ec44932c44b1b01195)) -- Deobfuscation function extraction - ([f5437aa](https://codeberg.org/ThetaDev/rustypipe/commit/f5437aa127b2b7c5a08839643e30ea1ec989d30b)) - - -## [v0.7.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.7.0..rustypipe/v0.7.1) - 2024-11-25 - -### 🐛 Bug Fixes - -- Disable Android client - ([a846b72](https://codeberg.org/ThetaDev/rustypipe/commit/a846b729e3519e3d5e62bdf028d9b48a7f8ea2ce)) -- A/B test 18: music playlist facepile avatar model - ([6c8108c](https://codeberg.org/ThetaDev/rustypipe/commit/6c8108c94acf9ca2336381bdca7c97b24a809521)) - -### ⚙️ Miscellaneous Tasks - -- Add docs badge to README - ([706e881](https://codeberg.org/ThetaDev/rustypipe/commit/706e88134c0e94ce7d880735e9d31b3ff531a4f9)) - - -## [v0.7.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.6.0..rustypipe/v0.7.0) - 2024-11-10 - -### 🚀 Features - -- Allow searching for YTM users - ([50010b7](https://codeberg.org/ThetaDev/rustypipe/commit/50010b7b0856d3ce05fe7a9d5989e526089bc2ef)) -- [**breaking**] Replace `TrackItem::is_video` attr with TrackType enum; serde lowercase AlbumType enum for consistency - ([044094a](https://codeberg.org/ThetaDev/rustypipe/commit/044094a4b70f05c46a459fa1597e23f4224b7b0b)) - -### 🐛 Bug Fixes - -- Fetch unlocalized player data to interpret errors correctly; regression introduced with v0.6.0 - ([0919cbd](https://codeberg.org/ThetaDev/rustypipe/commit/0919cbd0dfe28ea00610c67a694e5f319e80635f)) -- A/B test 17: channel playlists lockupViewModel - ([342119d](https://codeberg.org/ThetaDev/rustypipe/commit/342119dba6f3dc2152eef1fc9841264a9e56b9f0)) -- [**breaking**] Serde: lowercase Verification enum - ([badb3ae](https://codeberg.org/ThetaDev/rustypipe/commit/badb3aef8249315909160b8ff73df3019f07cf97)) -- Parsing videos using LockupViewModel (Music video recommendations) - ([870ff79](https://codeberg.org/ThetaDev/rustypipe/commit/870ff79ee07dfab1f4f2be3a401cd5320ed587da)) -- Parsing lockup playlists with "MIX" instead of view count - ([ac8fbc3](https://codeberg.org/ThetaDev/rustypipe/commit/ac8fbc3e679819189e2791c323975acaf1b43035)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rust crate thiserror to v2 (#16) - ([e1e1687](https://codeberg.org/ThetaDev/rustypipe/commit/e1e1687605603686ac5fd5deeb6aa8fecaf92494)) - - -## [v0.6.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.5.0..rustypipe/v0.6.0) - 2024-10-28 - -### 🚀 Features - -- [**breaking**] Remove TvHtml5Embed client as it got disabled - ([9e835c8](https://codeberg.org/ThetaDev/rustypipe/commit/9e835c8f38a3dd28c65561b2f9bb7a0f530c24f1)) -- [**breaking**] Generate random visitorData, remove `RustyPipeQuery::get_context` and `YTContext<'a>` from public API - ([7c4f44d](https://codeberg.org/ThetaDev/rustypipe/commit/7c4f44d09c4d813efff9e7d1059ddacd226b9e9d)) -- Add OAuth user login to access age-restricted videos - ([1cc3f9a](https://codeberg.org/ThetaDev/rustypipe/commit/1cc3f9ad74908d33e247ba6243103bfc22540164)) -- Add user_auth_logout method - ([9e2fe61](https://codeberg.org/ThetaDev/rustypipe/commit/9e2fe61267846ce216e0c498d8fa9ee672e03cbf)) -- Revoke OAuth token when logging out - ([62f8a92](https://codeberg.org/ThetaDev/rustypipe/commit/62f8a9210c23e1f02c711a2294af8766ca6b70e2)) - -### 🐛 Bug Fixes - -- Skip serializing empty cache entries - ([be18d89](https://codeberg.org/ThetaDev/rustypipe/commit/be18d89ea65e35ddcf0f31bea3360e5db209fb9f)) -- Fetch artist albums continuation - ([b589061](https://codeberg.org/ThetaDev/rustypipe/commit/b589061a40245637b4fe619a26892291d87d25e6)) -- Update channel order tokens - ([79a6281](https://codeberg.org/ThetaDev/rustypipe/commit/79a62816ff62d94e5c706f45b1ce5971e5e58a81)) -- Handle auth errors - ([512223f](https://codeberg.org/ThetaDev/rustypipe/commit/512223fd83fb1ba2ba7ad96ed050a70bb7ec294d)) -- Use same visitor data for fetching artist album continuations - ([7b0499f](https://codeberg.org/ThetaDev/rustypipe/commit/7b0499f6b7cbf6ac4b83695adadfebb3f30349c7)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rust crate fancy-regex to 0.14.0 (#14) - ([94194e0](https://codeberg.org/ThetaDev/rustypipe/commit/94194e019c46ca49c343086e80e8eb75c52f4bc6)) -- *(deps)* Update rust crate quick-xml to 0.37.0 (#15) - ([0662b5c](https://codeberg.org/ThetaDev/rustypipe/commit/0662b5ccfccc922b28629f11ea52c3eb35f9efd2)) - - -## [v0.5.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.4.0..rustypipe/v0.5.0) - 2024-10-13 - -### 🚀 Features - -- Add mobile client - ([71d3ec6](https://codeberg.org/ThetaDev/rustypipe/commit/71d3ec65ddafa966ef6b41cf4eb71687ba4b594c)) - -### 🐛 Bug Fixes - -- Prioritize visitor_data argument before opts - ([ace0fae](https://codeberg.org/ThetaDev/rustypipe/commit/ace0fae1005217cd396000176e7c01682eae026f)) -- Ignore live tracks in YTM searches - ([f3f2e1d](https://codeberg.org/ThetaDev/rustypipe/commit/f3f2e1d3ca1e9c838c682356bb5a7ded6951c8e5)) -- A/B test 16 (pageHeaderRenderer on playlist pages) - ([e65f145](https://codeberg.org/ThetaDev/rustypipe/commit/e65f14556f3003fa59fee3f9f1410fb5ddf63219)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rust crate rstest to 0.23.0 (#12) - ([96776e9](https://codeberg.org/ThetaDev/rustypipe/commit/96776e98d76fa1d31d5f84dbceafbe8f9dfd9085)) - - -## [v0.4.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.3.0..rustypipe/v0.4.0) - 2024-09-10 - -### 🚀 Features - -- Add RustyPipe version constant - ([7a019f5](https://codeberg.org/ThetaDev/rustypipe/commit/7a019f5706e19f7fe9f2e16e3b94d7b98cc8aca9)) - -### 🐛 Bug Fixes - -- Show docs.rs feature flags - ([67a231d](https://codeberg.org/ThetaDev/rustypipe/commit/67a231d6d1b6427f500667729a59032f2b28cc65)) -- A/B test 15 (parsing channel shortsLockupViewModel) - ([7972df0](https://codeberg.org/ThetaDev/rustypipe/commit/7972df0df498edd7801e25037b9b2456367f9204)) - -### 📚 Documentation - -- Fix license badge URL - ([4a253e1](https://codeberg.org/ThetaDev/rustypipe/commit/4a253e1a47317e9999e6ad31ac5c411956a0986a)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rust crate tokio to 1.20.4 [security] (#10) - ([a445e51](https://codeberg.org/ThetaDev/rustypipe/commit/a445e51b54a9afc44cd9657260a0b3d2abddbfa6)) - - -## [v0.3.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.2.1..rustypipe/v0.3.0) - 2024-08-18 - -### 🚀 Features - -- Add client_type to VideoPlayer, simplify MapResponse trait - ([90540c6](https://codeberg.org/ThetaDev/rustypipe/commit/90540c6aaad658d4ce24ed41450d8509bac711bd)) -- Add http_client method to RustyPipe and user_agent method to RustyPipeQuery - ([3d6de53](https://codeberg.org/ThetaDev/rustypipe/commit/3d6de5354599ea691351e0ca161154e53f2e0b41)) -- Add channel_id and channel_name getters to YtEntity trait - ([bbbe9b4](https://codeberg.org/ThetaDev/rustypipe/commit/bbbe9b4b322c6b5b30764772e282c6823aeea524)) -- [**breaking**] Make StreamFilter use Vec internally, remove lifetime - ([821984b](https://codeberg.org/ThetaDev/rustypipe/commit/821984bbd51d65cf96b1d14087417ef968eaf9b2)) -- Overhauled downloader - ([11a0038](https://codeberg.org/ThetaDev/rustypipe/commit/11a00383502917cd98245c3da349107289ba3aa9)) -- Add player_from_clients function to specify client order - ([72b5dfe](https://codeberg.org/ThetaDev/rustypipe/commit/72b5dfec69ec25445b94cb0976662416a5df56ef)) -- [**breaking**] Add TV client - ([e608811](https://codeberg.org/ThetaDev/rustypipe/commit/e608811e5f5615416241e67561671330097092cb)) -- Downloader: add audio tagging - ([1e1315a](https://codeberg.org/ThetaDev/rustypipe/commit/1e1315a8378bd0ad25b5f1614e83dabc4a0b40d5)) -- Add audiotag+indicatif features to downloader - ([97fb057](https://codeberg.org/ThetaDev/rustypipe/commit/97fb0578b5c4954a596d8dee0c4b6e1d773a9300)) -- Add YtEntity trait to YouTubeItem and MusicItem - ([114a86a](https://codeberg.org/ThetaDev/rustypipe/commit/114a86a3823a175875aa2aeb31a61a6799ef13bc)) -- Change default player client order - ([97904d7](https://codeberg.org/ThetaDev/rustypipe/commit/97904d77374c2c937a49dc7905759c2d8e8ef9ae)) -- [**breaking**] Update channel model, addd handle + video_count, remove tv/mobile banner - ([e671570](https://codeberg.org/ThetaDev/rustypipe/commit/e6715700d950912031d5fbc1263f8770b6ffc49c)) -- [**breaking**] Add handle to ChannelItem, remove video_count - ([1cffb27](https://codeberg.org/ThetaDev/rustypipe/commit/1cffb27cc0b64929f9627f5839df2d73b81988a4)) -- [**breaking**] Remove startpage - ([3599aca](https://codeberg.org/ThetaDev/rustypipe/commit/3599acafef1a21fa6f8dea97902eb4a3fb048c14)) - -### 🐛 Bug Fixes - -- [**breaking**] Extracting nsig function, remove field `throttled` from Video/Audio stream model - ([dd0565b](https://codeberg.org/ThetaDev/rustypipe/commit/dd0565ba98acb3289ed220fd2a3aaf86bb8b0788)) -- Make nsig_fn regex more generic - ([fb7af3b](https://codeberg.org/ThetaDev/rustypipe/commit/fb7af3b96698b452b6b24d1e094ba13a245cb83c)) -- Improve deobfuscator (support multiple nsig name matches, error if mapping all streams fails) - ([8152ce6](https://codeberg.org/ThetaDev/rustypipe/commit/8152ce6b088b57be9b8419b754aca93805e5f34d)) -- Nsig fn extraction - ([3c83e11](https://codeberg.org/ThetaDev/rustypipe/commit/3c83e11e753f8eb6efea5d453a7c819c487b3464)) -- Add var to deobf fn assignment - ([c6bd03f](https://codeberg.org/ThetaDev/rustypipe/commit/c6bd03fb70871ae1b764be18f88e86e71818fc56)) -- Make Verification enum exhaustive - ([d053ac3](https://codeberg.org/ThetaDev/rustypipe/commit/d053ac3eba810a7241df91f2f50bcbe1fd968c86)) -- Extraction error message - ([d36ba59](https://codeberg.org/ThetaDev/rustypipe/commit/d36ba595dab0bbaef1012ebfa8930fc0e6bf8167)) -- Set tracing instrumentation level to Error - ([9da3b25](https://codeberg.org/ThetaDev/rustypipe/commit/9da3b25be2b2577f7bd0282c09d10d368ac8b73f)) -- Detect ip-ban error message - ([da39c64](https://codeberg.org/ThetaDev/rustypipe/commit/da39c64f302bc2edc4214bbe25a0a9eb54063b09)) -- Player_from_clients: fall back to TvHtml5Embed client - ([d0ae796](https://codeberg.org/ThetaDev/rustypipe/commit/d0ae7961ba91d56c8b9a8d1c545875e869b818f5)) -- Parsing channels without banner - ([5a6b2c3](https://codeberg.org/ThetaDev/rustypipe/commit/5a6b2c3a621f6b20c1324ea8b9c03426e3d8018b)) -- Get TV client version - ([ee3ae40](https://codeberg.org/ThetaDev/rustypipe/commit/ee3ae40395263c5989784c7e00038ff13bc1151a)) - -### ⚙️ Miscellaneous Tasks - -- Renovate: disable approveMajorUpdates - ([4743f9d](https://codeberg.org/ThetaDev/rustypipe/commit/4743f9d8e101b58ad6a43548495da9f4f381b9f4)) -- Renovate: disable scheduleDaily - ([015bd6f](https://codeberg.org/ThetaDev/rustypipe/commit/015bd6fcbf04163565fcb190b163ecfdb5664e11)) -- Renovate: enable automerge - ([882abc5](https://codeberg.org/ThetaDev/rustypipe/commit/882abc53ca894229ee78ec0edaa723d9ea61bbcb)) -- *(deps)* Update rust crate quick-xml to 0.36.0 (#8) - ([b6bc05c](https://codeberg.org/ThetaDev/rustypipe/commit/b6bc05c1f39da9a846b2e3d1d24bcbccb031203b)) -- *(deps)* Update rust crate rstest to 0.22.0 (#9) - ([abb7832](https://codeberg.org/ThetaDev/rustypipe/commit/abb783219aba4b492c1dff03c2148acf1f51a55d)) -- Change repo URL to Codeberg - ([1793331](https://codeberg.org/ThetaDev/rustypipe/commit/17933315d947f76d5fe1aa52abf7ea24c3ce6381)) -- Adjust dependency versions - ([70c6f8c](https://codeberg.org/ThetaDev/rustypipe/commit/70c6f8c3b97baefd316fff90cc727524516657af)) - -### Todo - -- Update metadata - ([8692ca8](https://codeberg.org/ThetaDev/rustypipe/commit/8692ca81d972d0d2acf6fb4da79b9e0f5ebf4daf)) - - -## [v0.2.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.2.0..rustypipe/v0.2.1) - 2024-07-01 - -### 🐛 Bug Fixes - -- *(deps)* Update quick-xml to v0.35.0 - ([298e4de](https://codeberg.org/ThetaDev/rustypipe/commit/298e4def93d1595fba91be103f014aa645a08937)) - - -## [v0.2.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.1.3..rustypipe/v0.2.0) - 2024-06-27 - -### 🚀 Features - -- Add text formatting (bold/italic/strikethrough) - ([b8825f9](https://codeberg.org/ThetaDev/rustypipe/commit/b8825f9199365c873a4f0edd98a435e986b8daa2)) -- Prefix chip-style web links (social media) with the service name - ([6c41ef2](https://codeberg.org/ThetaDev/rustypipe/commit/6c41ef2fb2531e10a12c271e2d48504510a3b0bf)) -- Make get_visitor_data() public - ([da1d1bd](https://codeberg.org/ThetaDev/rustypipe/commit/da1d1bd2a0b214da10436ae221c90a0f88697b9a)) -- Add UnavailabilityReason: IpBan - ([401d4e8](https://codeberg.org/ThetaDev/rustypipe/commit/401d4e8255b1e86444319fed6d114dfbd0f80bbd)) -- Add YtEntity trait - ([792e3b3](https://codeberg.org/ThetaDev/rustypipe/commit/792e3b31e0101087a167935baad39a2e3b4296d0)) - -### 🐛 Bug Fixes - -- Remove Innertube API keys, update android player params - ([a8fb337](https://codeberg.org/ThetaDev/rustypipe/commit/a8fb337fae9cb0112e0152f9a0a19ebae49c2a4d)) -- Parsing error when no `music_related` content available - ([8fbd6b9](https://codeberg.org/ThetaDev/rustypipe/commit/8fbd6b95b6f01108b46f53fe60a56b0c561e40c1)) -- Parsing audiobook type in European Portuguese - ([041ce2d](https://codeberg.org/ThetaDev/rustypipe/commit/041ce2d08f6021c88e8890034f551f7e01b2f012)) -- Renovate ci token - ([e0759eb](https://codeberg.org/ThetaDev/rustypipe/commit/e0759ebce32a5520245bb2c0cb920734b04ee7dc)) - -### 🚜 Refactor - -- [**breaking**] Rename VideoItem/VideoPlayerDetails.length to duration for consistency - ([94e8d24](https://codeberg.org/ThetaDev/rustypipe/commit/94e8d24c6848b8bfca70dd03a7d89547ba9d6051)) - -### 📚 Documentation - -- Add logo - ([6646078](https://codeberg.org/ThetaDev/rustypipe/commit/66460789449be0d5984cbdb6ec372e69323b7a88)) - -### ⚙️ Miscellaneous Tasks - -- Changelog: fix incorrect version URLs - ([97b6f07](https://codeberg.org/ThetaDev/rustypipe/commit/97b6f07399e80e00a6c015d013e744568be125dd)) -- Update rstest to v0.19.0 - ([50fd1f0](https://codeberg.org/ThetaDev/rustypipe/commit/50fd1f08caf39c1298654e06059cc393543e925b)) -- Introduce MSRV - ([5dbb288](https://codeberg.org/ThetaDev/rustypipe/commit/5dbb288a496d53a299effa2026f5258af7b1f176)) -- Fix clippy lints - ([45b9f2a](https://codeberg.org/ThetaDev/rustypipe/commit/45b9f2a627b4e7075ba0b1c5f16efcc19aef7922)) -- Vscode: enable rss feature by default - ([e75ffbb](https://codeberg.org/ThetaDev/rustypipe/commit/e75ffbb5da6198086385ea96383ab9d0791592a5)) -- Configure Renovate (#3) - ([44c2deb](https://codeberg.org/ThetaDev/rustypipe/commit/44c2debea61f70c24ad6d827987e85e2132ed3d1)) -- *(deps)* Update rust crate tokio to 1.20.4 [security] (#4) - ([ce3ec34](https://codeberg.org/ThetaDev/rustypipe/commit/ce3ec34337b8acac41410ea39264aab7423d5801)) -- *(deps)* Update rust crate quick-xml to 0.34.0 (#5) - ([1e8a1af](https://codeberg.org/ThetaDev/rustypipe/commit/1e8a1af08c873cee7feadf63c2eff62753a78f64)) -- *(deps)* Update rust crate rstest to 0.21.0 (#7) - ([c3af918](https://codeberg.org/ThetaDev/rustypipe/commit/c3af918ba53c6230c0e4aef822a0cb2cf120bf3f)) - -## [v0.1.3](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.1.2..rustypipe/v0.1.3) - 2024-04-01 - -### 🐛 Bug Fixes - -- Parse new comment model (A/B#14 frameworkUpdates) - ([b0331f7](https://codeberg.org/ThetaDev/rustypipe/commit/b0331f7250f5d7d61a45209150739d2cb08b4280)) - -### ◀️ Revert - -- "fix: improve VecLogErr messages" (leads to infinite loop) - ([348c852](https://codeberg.org/ThetaDev/rustypipe/commit/348c8523fe847f2f6ce98317375a7ab65e778ed2)) - - -## [v0.1.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.1.1..rustypipe/v0.1.2) - 2024-03-26 - -### 🐛 Bug Fixes - -- Correctly parse subscriber count with new channel header - ([180dd98](https://codeberg.org/ThetaDev/rustypipe/commit/180dd9891a14b4da9f130a73d73aecc3822fce2f)) - - -## [v0.1.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.1.0..rustypipe/v0.1.1) - 2024-03-26 - -### 🐛 Bug Fixes - -- Specify internal dependency versions - ([6598a23](https://codeberg.org/ThetaDev/rustypipe/commit/6598a23d0699e6fe298275a67e0146a19c422c88)) -- Move package attributes to workspace - ([e4b204e](https://codeberg.org/ThetaDev/rustypipe/commit/e4b204eae65f450471be0890b0198d2f30714b3b)) -- Parsing music details with video description tab - ([a81c3e8](https://codeberg.org/ThetaDev/rustypipe/commit/a81c3e83366fdf72d01dd3ee00fb2e831f7aaa26)) - -### ⚙️ Miscellaneous Tasks - -- Changes to release command - ([0bcced1](https://codeberg.org/ThetaDev/rustypipe/commit/0bcced1db377198a54c9c7d03b8d038125a2bfe4)) -- Update user agent (FF 115.0) - ([be314d5](https://codeberg.org/ThetaDev/rustypipe/commit/be314d57ea1d99bfdc80649351ee3e7845541238)) -- Fix release script (unquoted include paths) - ([78ba9cb](https://codeberg.org/ThetaDev/rustypipe/commit/78ba9cb34c6bba3aba177583b242d3f76ea9847d)) - - -## [v0.1.0](https://codeberg.org/ThetaDev/rustypipe/commits/tag/rustypipe/v0.1.0) - 2024-03-22 - -Initial release - - diff --git a/Cargo.toml b/Cargo.toml index ae8bbec..22e8514 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,90 +1,22 @@ [package] name = "rustypipe" -version = "0.11.4" -rust-version = "1.67.1" -edition.workspace = true -authors.workspace = true -license.workspace = true -repository.workspace = true -keywords.workspace = true -categories.workspace = true -description = "Client for the public YouTube / YouTube Music API (Innertube), inspired by NewPipe" - -include = ["/src", "README.md", "CHANGELOG.md", "LICENSE", "!snapshots"] - -[workspace] -members = [".", "codegen", "downloader", "cli"] - -[workspace.package] +version = "0.1.0" edition = "2021" -authors = ["ThetaDev "] +authors = ["ThetaDev "] license = "GPL-3.0" -repository = "https://codeberg.org/ThetaDev/rustypipe" +description = "Client for the public YouTube / YouTube Music API (Innertube), inspired by NewPipe" keywords = ["youtube", "video", "music"] categories = ["api-bindings", "multimedia"] -[workspace.dependencies] -rquickjs = "0.9.0" -once_cell = "1.12.0" -regex = "1.6.0" -fancy-regex = "0.14.0" -thiserror = "2.0.0" -url = "2.2.0" -reqwest = { version = "0.12.0", default-features = false } -tokio = "1.20.4" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0.82" -serde_with = { version = "3.0.0", default-features = false, features = [ - "alloc", - "macros", -] } -serde_plain = "1.0.0" -sha1 = "0.10.0" -rand = "0.9.0" -time = { version = "0.3.37", features = [ - "macros", - "serde-human-readable", - "serde-well-known", - "local-offset", -] } -futures-util = "0.3.31" -ress = "0.11.0" -phf = "0.11.0" -phf_codegen = "0.11.0" -data-encoding = "2.0.0" -urlencoding = "2.1.0" -quick-xml = { version = "0.37.0", features = ["serialize"] } -tracing = { version = "0.1.0", features = ["log"] } -localzone = "0.3.1" +include = ["/src", "README.md", "LICENSE", "!snapshots"] -# CLI -indicatif = "0.17.0" -anyhow = "1.0" -clap = { version = "4.0.0", features = ["derive"] } -tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } -serde_yaml = "0.9.0" -dirs = "6.0.0" -filenamify = "0.1.0" - -# Testing -rstest = "0.25.0" -tokio-test = "0.4.2" -insta = { version = "1.17.1", features = ["ron", "redactions"] } -path_macro = "1.0.0" -tracing-test = "0.2.5" - -# Included crates -rustypipe = { path = ".", version = "0.11.4", default-features = false } -rustypipe-downloader = { path = "./downloader", version = "0.3.1", default-features = false, features = [ - "indicatif", - "audiotag", -] } +[workspace] +members = [".", "codegen", "downloader", "postprocessor", "cli"] [features] default = ["default-tls"] -rss = ["dep:quick-xml"] -userdata = [] +rss = ["quick-xml"] # Reqwest TLS options default-tls = ["reqwest/default-tls"] @@ -95,38 +27,45 @@ rustls-tls-webpki-roots = ["reqwest/rustls-tls-webpki-roots"] rustls-tls-native-roots = ["reqwest/rustls-tls-native-roots"] [dependencies] -rquickjs.workspace = true -once_cell.workspace = true -regex.workspace = true -fancy-regex.workspace = true -thiserror.workspace = true -url.workspace = true -reqwest = { workspace = true, features = ["json", "gzip", "brotli"] } -tokio = { workspace = true, features = ["macros", "time", "process"] } -serde.workspace = true -serde_json.workspace = true -serde_with.workspace = true -serde_plain.workspace = true -sha1.workspace = true -rand.workspace = true -time.workspace = true -ress.workspace = true -phf.workspace = true -data-encoding.workspace = true -urlencoding.workspace = true -tracing.workspace = true -localzone.workspace = true -quick-xml = { workspace = true, optional = true } +quick-js-dtp = { version = "0.4.1", default-features = false, features = [ + "patch-dateparser", +] } +once_cell = "1.12.0" +regex = "1.6.0" +fancy-regex = "0.11.0" +thiserror = "1.0.36" +url = "2.2.2" +log = "0.4.17" +reqwest = { version = "0.11.11", default-features = false, features = [ + "json", + "gzip", + "brotli", +] } +tokio = { version = "1.20.0", features = ["macros", "time"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0.82" +serde_with = { version = "3.0.0", default-features = false, features = [ + "macros", + "json", +] } +serde_plain = "1.0.1" +rand = "0.8.5" +time = { version = "0.3.15", features = [ + "macros", + "serde", + "serde-well-known", +] } +futures = "0.3.21" +ress = "0.11.4" +phf = "0.11.1" +base64 = "0.21.0" +urlencoding = "2.1.2" +quick-xml = { version = "0.28.1", features = ["serialize"], optional = true } [dev-dependencies] -rstest.workspace = true -tokio-test.workspace = true -insta.workspace = true -path_macro.workspace = true -tracing-test.workspace = true - -[package.metadata.docs.rs] -# To build locally: -# RUSTDOCFLAGS="--cfg docsrs" cargo +nightly doc --features rss,userdata --no-deps --open -features = ["rss", "userdata"] -rustdoc-args = ["--cfg", "docsrs"] +env_logger = "0.10.0" +test-log = "0.2.11" +rstest = "0.17.0" +tokio-test = "0.4.2" +insta = { version = "1.17.1", features = ["ron", "redactions"] } +path_macro = "1.0.0" diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md deleted file mode 100644 index 5cfc9b6..0000000 --- a/DEVELOPMENT.md +++ /dev/null @@ -1,26 +0,0 @@ -## Development - -**Requirements:** - -- Current version of stable Rust -- [`just`](https://github.com/casey/just) task runner -- [`nextest`](https://nexte.st) test runner -- [`pre-commit`](https://pre-commit.com/) -- yq (YAML processor) - -### Tasks - -**Testing** - -- `just test` Run unit+integration tests -- `just unittest` Run unit tests -- `just testyt` Run YouTube integration tests -- `just testintl` Run YouTube integration tests for all supported languages (this takes - a long time and is therefore not run in CI) -- `YT_LANG=de just testyt` Run YouTube integration tests for a specific language - -**Tools** - -- `just testfiles` Download missing testfiles for unit tests -- `just report2yaml` Convert RustyPipe reports into a more readable yaml format - (requires `yq`) diff --git a/Justfile b/Justfile index d8bd7aa..7d7ca4d 100644 --- a/Justfile +++ b/Justfile @@ -1,19 +1,19 @@ test: - # cargo test --features=rss,userdata - cargo nextest run --workspace --features=rss,userdata --no-fail-fast --retries 1 -- --skip 'user_data::' + cargo test --features=rss unittest: - cargo nextest run --features=rss,userdata --no-fail-fast --lib + cargo test --features=rss --lib testyt: - cargo nextest run --features=rss,userdata --no-fail-fast --retries 1 --test youtube -- --skip 'user_data::' + cargo test --features=rss --test youtube -testyt-cookie: - cargo nextest run --features=rss,userdata --no-fail-fast --retries 1 --test youtube - -testyt-localized: - YT_LANG=th cargo nextest run --features=rss,userdata --no-fail-fast --retries 1 --test youtube -- \ - --skip 'user_data::' --skip 'search_suggestion' --skip 'isrc_search_languages' +testyt10: + #!/usr/bin/env bash + set -e + for i in {1..10}; do \ + echo "---TEST RUN $i---"; \ + cargo test --features=rss --test youtube; \ + done testintl: #!/usr/bin/env bash @@ -32,8 +32,7 @@ testintl: for YT_LANG in "${LANGUAGES[@]}"; do echo "---TESTS FOR $YT_LANG ---" - if YT_LANG="$YT_LANG" cargo nextest run --no-fail-fast --retries 1 --test-threads 4 --test youtube -- \ - --skip 'user_data::' --skip 'search_suggestion' --skip 'isrc_search_languages' --skip 'resolve_'; then + if YT_LANG="$YT_LANG" cargo test --test youtube -- --test-threads 4 --skip resolve; then echo "--- $YT_LANG COMPLETED ---" else echo "--- $YT_LANG FAILED ---" @@ -48,45 +47,4 @@ testfiles: report2yaml: mkdir -p rustypipe_reports/conv - for f in rustypipe_reports/*.json; do yq '.http_request.resp_body' $f | yq -o json -P > rustypipe_reports/conv/`basename $f .json`_body.json; yq e -Pi "del(.http_request.resp_body)" $f; mv $f rustypipe_reports/conv/`basename $f .json`.yaml; done; - -release crate="rustypipe": - #!/usr/bin/env bash - set -e - - CRATE="{{crate}}" - CHANGELOG="CHANGELOG.md" - - if [ "$CRATE" = "rustypipe" ]; then - INCLUDES="--exclude-path 'notes/**' --exclude-path 'cli/**' --exclude-path 'downloader/**'" - else - if [ ! -d "$CRATE" ]; then - echo "$CRATE does not exist."; exit 1 - fi - INCLUDES="--include-path README.md --include-path LICENSE --include-path Cargo.toml --include-path '$CRATE/**'" - CHANGELOG="$CRATE/$CHANGELOG" - CRATE="rustypipe-$CRATE" # Add crate name prefix - fi - - VERSION=$(cargo pkgid --package "$CRATE" | tr '#@' '\n' | tail -n 1) - TAG="${CRATE}/v${VERSION}" - echo "Releasing $TAG:" - - if git rev-parse "$TAG" >/dev/null 2>&1; then echo "version tag $TAG already exists"; exit 1; fi - - CLIFF_ARGS="--tag '${TAG}' --tag-pattern '${CRATE}/v*' --unreleased $INCLUDES" - echo "git-cliff $CLIFF_ARGS" - if [ -f "$CHANGELOG" ]; then - eval "git-cliff $CLIFF_ARGS --prepend '$CHANGELOG'" - else - eval "git-cliff $CLIFF_ARGS --output '$CHANGELOG'" - fi - - editor "$CHANGELOG" - - git add . - git commit -m "chore(release): release $CRATE v$VERSION" - - awk 'BEGIN{RS="(^|\n)## [^\n]+\n*"} NR==2 { print }' "$CHANGELOG" | git tag -as -F - --cleanup whitespace "$TAG" - - echo "🚀 Run 'git push origin $TAG' to publish" + for f in rustypipe_reports/*.json; do yq '.http_request.resp_body' $f | yq -o json -P > rustypipe_reports/conv/`basename $f .json`_body.json; yq e -Pi $f; mv $f rustypipe_reports/conv/`basename $f .json`.yaml; done; diff --git a/README.md b/README.md index 767680f..a237ebc 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,9 @@ -# ![RustyPipe](https://codeberg.org/ThetaDev/rustypipe/raw/branch/main/notes/logo.svg) +# RustyPipe -[![Current crates.io version](https://img.shields.io/crates/v/rustypipe.svg)](https://crates.io/crates/rustypipe) -[![License](https://img.shields.io/badge/License-GPL--3-blue.svg?style=flat)](https://opensource.org/licenses/GPL-3.0) -[![Docs](https://img.shields.io/docsrs/rustypipe/latest?style=flat)](https://docs.rs/rustypipe) -[![CI status](https://codeberg.org/ThetaDev/rustypipe/actions/workflows/ci.yaml/badge.svg?style=flat&label=CI)](https://codeberg.org/ThetaDev/rustypipe/actions/?workflow=ci.yaml) +[![CI status](https://ci.thetadev.de/api/badges/ThetaDev/rustypipe/status.svg)](https://ci.thetadev.de/ThetaDev/rustypipe) -RustyPipe is a fully featured Rust client for the public YouTube / YouTube Music API -(Innertube), inspired by [NewPipe](https://github.com/TeamNewPipe/NewPipeExtractor). +Client for the public YouTube / YouTube Music API (Innertube), inspired by +[NewPipe](https://github.com/TeamNewPipe/NewPipeExtractor). ## Features @@ -21,8 +18,6 @@ RustyPipe is a fully featured Rust client for the public YouTube / YouTube Music - **Search suggestions** - **Trending** - **URL resolver** -- **Subscriptions** -- **Playback history** ### YouTube Music @@ -36,35 +31,14 @@ RustyPipe is a fully featured Rust client for the public YouTube / YouTube Music - **Moods/Genres** - **Charts** - **New** (albums, music videos) -- **Saved items** -- **Playback history** ## Getting started -The RustyPipe library works as follows: at first you have to instantiate a RustyPipe -client. You can either create it with default options or use the `RustyPipe::builder()` -to customize it. - -For fetching data you have to start with a new RustyPipe query object (`rp.query()`). -The query object holds options for an individual query (e.g. content language or -country). You can adjust these options with setter methods. Finally call your query -method to fetch the data you need. - -All query methods are async, you need the tokio runtime to execute them. - -```rust ignore -let rp = RustyPipe::new(); -let rp = RustyPipe::builder().storage_dir("/app/data").build().unwrap(); -let channel = rp.query().lang(Language::De).channel_videos("UCl2mFZoRqjw_ELax4Yisf6w").await.unwrap(); -``` - -Here are a few examples to get you started: - ### Cargo.toml ```toml [dependencies] -rustypipe = "0.1.3" +rustypipe = "0.1.0" tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] } ``` @@ -180,106 +154,3 @@ Subscribers: 1780000 [6Fv8bd9ICb4] Who owns this? (199s) ... ``` - -## Crate features - -Some features of RustyPipe are gated behind features to avoid compiling unneeded -dependencies. - -- `rss` Fetch a channel's RSS feed, which is faster than fetching the channel page -- `userdata` Add functions to fetch YouTube user data (watch history, subscriptions, - music library) - -You can also choose the TLS library used for making web requests using the same features -as the reqwest crate (`default-tls`, `native-tls`, `native-tls-alpn`, -`native-tls-vendored`, `rustls-tls-webpki-roots`, `rustls-tls-native-roots`). - -## Cache storage - -The RustyPipe cache holds the current version numbers for all clients, the JavaScript -code used to deobfuscate video URLs and the authentication token/cookies. Never share -the contents of the cache if you are using authentication. - -By default the cache is written to a JSON file named `rustypipe_cache.json` in the -current working directory. This path can be changed with the `storage_dir` option of the -RustyPipeBuilder. The RustyPipe CLI stores its cache in the userdata folder. The full -path on Linux is `~/.local/share/rustypipe/rustypipe_cache.json`. - -You can integrate your own cache storage backend (e.g. database storage) by implementing -the `CacheStorage` trait. - -## Reports - -RustyPipe has a builtin error reporting system. If a YouTube response cannot be -deserialized or parsed, the original response data along with some request metadata is -written to a JSON file in the folder `rustypipe_reports`, located in RustyPipe's storage -directory (current folder by default, `~/.local/share/rustypipe` for the CLI). - -When submitting a bug report to the RustyPipe project, you can share this report to help -resolve the issue. - -RustyPipe reports come in 3 severity levels: - -- DBG (no error occurred, report creation was enabled by the `RustyPipeQuery::report` - query option) -- WRN (parts of the response could not be deserialized/parsed, response data may be - incomplete) -- ERR (entire response could not be deserialized/parsed, RustyPipe returned an error) - -## PO tokens - -Since August 2024 YouTube requires PO tokens to access streams from web-based clients -(Desktop, Mobile). Otherwise streams will return a 403 error. - -Generating PO tokens requires a simulated browser environment, which would be too large -to include in RustyPipe directly. - -Therefore, the PO token generation is handled by a seperate CLI application -([rustypipe-botguard](https://codeberg.org/ThetaDev/rustypipe-botguard)) which is called -by the RustyPipe crate. RustyPipe automatically detects the rustypipe-botguard binary if -it is located in PATH or the current working directory. If your rustypipe-botguard -binary is located at a different path, you can specify it with the `.botguard_bin(path)` -option. - -## Authentication - -RustyPipe supports authenticating with your YouTube account to access -age-restricted/private videos and user information. There are 2 supported authentication -methods: OAuth and cookies. - -To execute a query with authentication, use the `.authenticated()` query option. This -option is enabled by default for queries that always require authentication like -fetching user data. RustyPipe may automatically use authentication in case a video is -age-restricted or your IP address is banned by YouTube. If you never want to use -authentication, set the `.unauthenticated()` query option. - -### OAuth - -OAuth is the authentication method used by the YouTube TV client. It is more -user-friendly than extracting cookies, however it only works with the TV client. This -means that you can only fetch videos and not access any user data. - -To login using OAuth, you first have to get a new device code using the -`rp.user_auth_get_code()` function. You can then enter the code on - and log in with your Google account. After generating the -code, you can call the `rp.user_auth_wait_for_login()` function which waits until the -user has logged in and stores the authentication token in the cache. - -### Cookies - -Authenticating with cookies allows you to use the functionality of the YouTube/YouTube -Music Desktop client. You can fetch your subscribed channels, playlists and your music -collection. You can also fetch videos using the Desktop client, including private -videos, as long as you have access to them. - -To authenticate with cookies you have to log into YouTube in a fresh browser session -(open Incognito/Private mode). Then extract the cookies from the developer tools or by -using browser plugins like "Get cookies.txt LOCALLY" -([Firefox](https://addons.mozilla.org/de/firefox/addon/get-cookies-txt-locally/)) -([Chromium](https://chromewebstore.google.com/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc)). -Close the browser window after extracting the cookies to prevent YouTube from rotating -the cookies. - -You can then add the cookies to your RustyPipe client using the `user_auth_set_cookie` -or `user_auth_set_cookie_txt` function. The cookies are stored in the cache file. To log -out, use the function `user_auth_remove_cookie`. diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md deleted file mode 100644 index efe7ee5..0000000 --- a/cli/CHANGELOG.md +++ /dev/null @@ -1,207 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - - -## [v0.7.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.7.1..rustypipe-cli/v0.7.2) - 2025-03-16 - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rustypipe to 0.11.1 -- *(deps)* Update rustypipe-downloader to 0.3.1 -- *(deps)* Update rust crate rstest to 0.25.0 - ([9ed1306](https://codeberg.org/ThetaDev/rustypipe/commit/9ed1306f3aaeb993c409997ddfbc47499e4f4d22)) - - -## [v0.7.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.7.0..rustypipe-cli/v0.7.1) - 2025-02-26 - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rustypipe to 0.11.0 - ([035c07f](https://codeberg.org/ThetaDev/rustypipe/commit/035c07f170aa293bcc626f27998c2b2b28660881)) - - -## [v0.7.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.6.0..rustypipe-cli/v0.7.0) - 2025-02-09 - -### 🚀 Features - -- Add support for rustypipe-botguard to get PO tokens - ([b90a252](https://codeberg.org/ThetaDev/rustypipe/commit/b90a252a5e1bf05a5294168b0ec16a73cbb88f42)) -- [**breaking**] Remove manual PO token options from downloader/cli, add new rustypipe-botguard options - ([cddb32f](https://codeberg.org/ThetaDev/rustypipe/commit/cddb32f190276265258c6ab45b3d43a8891c4b39)) -- Add session po token cache - ([b72b501](https://codeberg.org/ThetaDev/rustypipe/commit/b72b501b6dbcf4333b24cd80e7c8c61b0c21ec91)) -- Add timezone query option - ([3a2370b](https://codeberg.org/ThetaDev/rustypipe/commit/3a2370b97ca3d0f40d72d66a23295557317d29fb)) -- Add --timezone-local CLI option - ([4f2bb47](https://codeberg.org/ThetaDev/rustypipe/commit/4f2bb47ab42ae0c68a64f3b3c2831fa7850b6f56)) -- Add verbose flag - ([629b590](https://codeberg.org/ThetaDev/rustypipe/commit/629b5905da653c6fe0f3c6b5814dd2f49030e7ed)) - -### 🐛 Bug Fixes - -- Parsing mixed-case language codes like zh-CN - ([9c73ed4](https://codeberg.org/ThetaDev/rustypipe/commit/9c73ed4b3008cb093c0fa7fd94fd9f1ba8cd3627)) - -### 🚜 Refactor - -- [**breaking**] Add client_type field to DownloadError, rename cli option po-token-cache to pot-cache - ([594e675](https://codeberg.org/ThetaDev/rustypipe/commit/594e675b39efc5fbcdbd5e920a4d2cdee64f718e)) -- Rename rustypipe-cli binary to rustypipe - ([c1a872e](https://codeberg.org/ThetaDev/rustypipe/commit/c1a872e1c14ea0956053bd7c65f6875b1cb3bc55)) - -### 📚 Documentation - -- Add Botguard info to README - ([9957add](https://codeberg.org/ThetaDev/rustypipe/commit/9957add2b5d6391b2c1869d2019fd7dd91b8cd41)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rustypipe to 0.10.0 -- *(deps)* Update rust crate rquickjs to 0.9.0 (#33) - ([2c8ac41](https://codeberg.org/ThetaDev/rustypipe/commit/2c8ac410aa535d83f8bcc7181f81914b13bceb77)) - - -## [v0.6.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.5.0..rustypipe-cli/v0.6.0) - 2025-01-16 - -### 🚀 Features - -- Add functions to fetch a user's history and subscriptions - ([14e3995](https://codeberg.org/ThetaDev/rustypipe/commit/14e399594f97a1228a8c2991a14dd8745af1beb7)) -- Add history item dates, extend timeago parser - ([320a8c2](https://codeberg.org/ThetaDev/rustypipe/commit/320a8c2c24217ad5697f0424c4f994bbbe31f3aa)) -- Add cookies.txt parser, add cookie auth + history cmds to CLI - ([cf498e4](https://codeberg.org/ThetaDev/rustypipe/commit/cf498e4a8f9318b0197bc3f0cbaf7043c53adb9d)) -- Add CLI commands to fetch user library and YTM releases/charts - ([a1b43ad](https://codeberg.org/ThetaDev/rustypipe/commit/a1b43ad70a66cfcbaba8ef302ac8699f243e56e7)) -- Export subscriptions as OPML / NewPipe JSON - ([c90d966](https://codeberg.org/ThetaDev/rustypipe/commit/c90d966b17eab24e957d980695888a459707055c)) - -### 📚 Documentation - -- Update README - ([0432477](https://codeberg.org/ThetaDev/rustypipe/commit/0432477451ecd5f64145d65239c721f4e44826c0)) -- Fix README - ([11442df](https://codeberg.org/ThetaDev/rustypipe/commit/11442dfd369599396357f5b7a7a4268a7b537f57)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rust crate rstest to 0.24.0 (#20) - ([ab19034](https://codeberg.org/ThetaDev/rustypipe/commit/ab19034ab19baf090e83eada056559676ffdadce)) -- *(deps)* Update rust crate dirs to v6 (#24) - ([6a60425](https://codeberg.org/ThetaDev/rustypipe/commit/6a604252b1af7a9388db5dc170f737069cc31051)) - - -## [v0.5.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.4.0..rustypipe-cli/v0.5.0) - 2024-12-20 - -### 🚀 Features - -- Get comment replies, rich text formatting - ([dceba44](https://codeberg.org/ThetaDev/rustypipe/commit/dceba442fe1a1d5d8d2a6d9422ff699593131f6d)) - -### 🐛 Bug Fixes - -- Replace futures dependency with futures-util - ([5c39bf4](https://codeberg.org/ThetaDev/rustypipe/commit/5c39bf4842b13d37a4277ea5506e15c179892ce5)) -- Error 400 when fetching player with login - ([5ce84c4](https://codeberg.org/ThetaDev/rustypipe/commit/5ce84c44a6844f692258066c83e04df875e0aa91)) - -### ⚙️ Miscellaneous Tasks - -- Add docs badge to README - ([706e881](https://codeberg.org/ThetaDev/rustypipe/commit/706e88134c0e94ce7d880735e9d31b3ff531a4f9)) -- *(deps)* Update rustypipe to 0.8.0 - - -## [v0.4.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.3.0..rustypipe-cli/v0.4.0) - 2024-11-10 - -### 🚀 Features - -- Allow searching for YTM users - ([50010b7](https://codeberg.org/ThetaDev/rustypipe/commit/50010b7b0856d3ce05fe7a9d5989e526089bc2ef)) -- [**breaking**] Replace `TrackItem::is_video` attr with TrackType enum; serde lowercase AlbumType enum for consistency - ([044094a](https://codeberg.org/ThetaDev/rustypipe/commit/044094a4b70f05c46a459fa1597e23f4224b7b0b)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rust crate thiserror to v2 (#16) - ([e1e1687](https://codeberg.org/ThetaDev/rustypipe/commit/e1e1687605603686ac5fd5deeb6aa8fecaf92494)) - - -## [v0.3.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.2.2..rustypipe-cli/v0.3.0) - 2024-10-28 - -### 🚀 Features - -- [**breaking**] Remove TvHtml5Embed client as it got disabled - ([9e835c8](https://codeberg.org/ThetaDev/rustypipe/commit/9e835c8f38a3dd28c65561b2f9bb7a0f530c24f1)) -- Add OAuth user login to access age-restricted videos - ([1cc3f9a](https://codeberg.org/ThetaDev/rustypipe/commit/1cc3f9ad74908d33e247ba6243103bfc22540164)) -- Revoke OAuth token when logging out - ([62f8a92](https://codeberg.org/ThetaDev/rustypipe/commit/62f8a9210c23e1f02c711a2294af8766ca6b70e2)) - - -## [v0.2.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.2.1..rustypipe-cli/v0.2.2) - 2024-10-13 - -### 🚀 Features - -- Add mobile client - ([71d3ec6](https://codeberg.org/ThetaDev/rustypipe/commit/71d3ec65ddafa966ef6b41cf4eb71687ba4b594c)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rust crate rstest to 0.23.0 (#12) - ([96776e9](https://codeberg.org/ThetaDev/rustypipe/commit/96776e98d76fa1d31d5f84dbceafbe8f9dfd9085)) -- *(deps)* Update rustypipe to 0.5.0 - - -## [v0.2.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.2.0..rustypipe-cli/v0.2.1) - 2024-09-10 - -### 🚀 Features - -- Add RustyPipe version constant - ([7a019f5](https://codeberg.org/ThetaDev/rustypipe/commit/7a019f5706e19f7fe9f2e16e3b94d7b98cc8aca9)) - -### 📚 Documentation - -- Fix license badge URL - ([4a253e1](https://codeberg.org/ThetaDev/rustypipe/commit/4a253e1a47317e9999e6ad31ac5c411956a0986a)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rust crate tokio to 1.20.4 [security] (#10) - ([a445e51](https://codeberg.org/ThetaDev/rustypipe/commit/a445e51b54a9afc44cd9657260a0b3d2abddbfa6)) - - -## [v0.2.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.1.1..rustypipe-cli/v0.2.0) - 2024-08-18 - -### 🚀 Features - -- Overhauled downloader - ([11a0038](https://codeberg.org/ThetaDev/rustypipe/commit/11a00383502917cd98245c3da349107289ba3aa9)) -- [**breaking**] Add TV client - ([e608811](https://codeberg.org/ThetaDev/rustypipe/commit/e608811e5f5615416241e67561671330097092cb)) -- Downloader: add audio tagging - ([1e1315a](https://codeberg.org/ThetaDev/rustypipe/commit/1e1315a8378bd0ad25b5f1614e83dabc4a0b40d5)) -- Downloader: add download_track fn, improve path templates - ([e1e4fb2](https://codeberg.org/ThetaDev/rustypipe/commit/e1e4fb29c190fec07f17c59ec88bef4f1c2a76a1)) -- Add audiotag+indicatif features to downloader - ([97fb057](https://codeberg.org/ThetaDev/rustypipe/commit/97fb0578b5c4954a596d8dee0c4b6e1d773a9300)) -- Add plaintext output to CLI - ([91b020e](https://codeberg.org/ThetaDev/rustypipe/commit/91b020efd498eff6e0f354a1de39439e252a79dd)) -- Add potoken option to downloader - ([904f821](https://codeberg.org/ThetaDev/rustypipe/commit/904f8215d84c810b04e4d2134718e786a4803ad2)) -- Print error message - ([8f16e5b](https://codeberg.org/ThetaDev/rustypipe/commit/8f16e5ba6eec3fd6aba1bb6a19571c65fb69ce0e)) -- Add list of clients to downloader - ([5e646af](https://codeberg.org/ThetaDev/rustypipe/commit/5e646afd1edc6c0101501311527ea56d3bad5fd2)) -- Retry with different client after 403 error - ([d875b54](https://codeberg.org/ThetaDev/rustypipe/commit/d875b5442de9822ba7ddc6f05789f56a8962808c)) -- Add option to fetch RSS feed - ([03c4d3c](https://codeberg.org/ThetaDev/rustypipe/commit/03c4d3c392386e06f2673f0e0783e22d10087989)) -- [**breaking**] Update channel model, addd handle + video_count, remove tv/mobile banner - ([e671570](https://codeberg.org/ThetaDev/rustypipe/commit/e6715700d950912031d5fbc1263f8770b6ffc49c)) - -### 🐛 Bug Fixes - -- *(deps)* Update quick-xml to v0.35.0 - ([298e4de](https://codeberg.org/ThetaDev/rustypipe/commit/298e4def93d1595fba91be103f014aa645a08937)) -- Improve deobfuscator (support multiple nsig name matches, error if mapping all streams fails) - ([8152ce6](https://codeberg.org/ThetaDev/rustypipe/commit/8152ce6b088b57be9b8419b754aca93805e5f34d)) -- Cli: print video ID when logging errors - ([2c7a3fb](https://codeberg.org/ThetaDev/rustypipe/commit/2c7a3fb5cc153ff0b8b5e79234ae497d916e471c)) -- Use anstream + owo-color for colorful CLI output - ([e8324cf](https://codeberg.org/ThetaDev/rustypipe/commit/e8324cf3b065cb977adbc9529b1ef5ee18c3dd47)) -- Use native tls by default for CLI - ([f37432a](https://codeberg.org/ThetaDev/rustypipe/commit/f37432a48c1f93cab5f7942f791daf7b27cb1565)) -- Detect ip-ban error message - ([da39c64](https://codeberg.org/ThetaDev/rustypipe/commit/da39c64f302bc2edc4214bbe25a0a9eb54063b09)) -- Dont store cache in current dir with --report option - ([6009de7](https://codeberg.org/ThetaDev/rustypipe/commit/6009de7bddc6031f2af17005c473c17934327c02)) -- Show docs.rs feature flags - ([67a231d](https://codeberg.org/ThetaDev/rustypipe/commit/67a231d6d1b6427f500667729a59032f2b28cc65)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rust crate quick-xml to 0.36.0 (#8) - ([b6bc05c](https://codeberg.org/ThetaDev/rustypipe/commit/b6bc05c1f39da9a846b2e3d1d24bcbccb031203b)) -- *(deps)* Update rust crate rstest to 0.22.0 (#9) - ([abb7832](https://codeberg.org/ThetaDev/rustypipe/commit/abb783219aba4b492c1dff03c2148acf1f51a55d)) -- Change repo URL to Codeberg - ([1793331](https://codeberg.org/ThetaDev/rustypipe/commit/17933315d947f76d5fe1aa52abf7ea24c3ce6381)) -- Adjust dependency versions - ([70c6f8c](https://codeberg.org/ThetaDev/rustypipe/commit/70c6f8c3b97baefd316fff90cc727524516657af)) - -### Todo - -- Update metadata - ([8692ca8](https://codeberg.org/ThetaDev/rustypipe/commit/8692ca81d972d0d2acf6fb4da79b9e0f5ebf4daf)) - - -## [v0.1.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.1.0..rustypipe-cli/v0.1.1) - 2024-06-27 - -### 🚀 Features - -- CLI: setting player type - ([16e0e28](https://codeberg.org/ThetaDev/rustypipe/commit/16e0e28c4866bb69d8e4c06eef94176f329a1c27)) - -### 🐛 Bug Fixes - -- Clippy warning - ([8420c2f](https://codeberg.org/ThetaDev/rustypipe/commit/8420c2f8dbd2791b524ceca2e19fb68e5b918bfa)) - -### 📚 Documentation - -- Add logo - ([6646078](https://codeberg.org/ThetaDev/rustypipe/commit/66460789449be0d5984cbdb6ec372e69323b7a88)) - -### ⚙️ Miscellaneous Tasks - -- Changelog: fix incorrect version URLs - ([97b6f07](https://codeberg.org/ThetaDev/rustypipe/commit/97b6f07399e80e00a6c015d013e744568be125dd)) -- Update rstest to v0.19.0 - ([50fd1f0](https://codeberg.org/ThetaDev/rustypipe/commit/50fd1f08caf39c1298654e06059cc393543e925b)) -- Introduce MSRV - ([5dbb288](https://codeberg.org/ThetaDev/rustypipe/commit/5dbb288a496d53a299effa2026f5258af7b1f176)) -- Fix clippy lints - ([45b9f2a](https://codeberg.org/ThetaDev/rustypipe/commit/45b9f2a627b4e7075ba0b1c5f16efcc19aef7922)) -- *(deps)* Update rust crate tokio to 1.20.4 [security] (#4) - ([ce3ec34](https://codeberg.org/ThetaDev/rustypipe/commit/ce3ec34337b8acac41410ea39264aab7423d5801)) -- *(deps)* Update rust crate quick-xml to 0.34.0 (#5) - ([1e8a1af](https://codeberg.org/ThetaDev/rustypipe/commit/1e8a1af08c873cee7feadf63c2eff62753a78f64)) -- *(deps)* Update rust crate rstest to 0.21.0 (#7) - ([c3af918](https://codeberg.org/ThetaDev/rustypipe/commit/c3af918ba53c6230c0e4aef822a0cb2cf120bf3f)) -- Update rustypipe to 0.2.0 - -## [v0.1.0](https://codeberg.org/ThetaDev/rustypipe/commits/tag/rustypipe-cli/v0.1.0) - 2024-03-22 - -Initial release - - diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 175f4cd..5dab51e 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,18 +1,15 @@ [package] name = "rustypipe-cli" -version = "0.7.2" -rust-version = "1.70.0" -edition.workspace = true -authors.workspace = true -license.workspace = true -repository.workspace = true -keywords.workspace = true -categories.workspace = true +version = "0.1.0" +edition = "2021" +authors = ["ThetaDev "] +license = "GPL-3.0" description = "CLI for RustyPipe - download videos and extract data from YouTube / YouTube Music" +keywords = ["youtube", "video", "music"] +categories = ["multimedia"] [features] -default = ["native-tls"] -timezone = ["dep:time", "dep:time-tz"] +default = ["rustls-tls-native-roots"] # Reqwest TLS options native-tls = [ @@ -42,29 +39,16 @@ rustls-tls-native-roots = [ ] [dependencies] -rustypipe = { workspace = true, features = ["rss", "userdata"] } -rustypipe-downloader.workspace = true -reqwest.workspace = true -tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } -futures-util.workspace = true -serde.workspace = true -serde_json.workspace = true -quick-xml.workspace = true -time = { workspace = true, optional = true } -time-tz = { version = "2.0.0", optional = true } - -indicatif.workspace = true -anyhow.workspace = true -clap.workspace = true -tracing.workspace = true -tracing-subscriber.workspace = true -serde_yaml.workspace = true -dirs.workspace = true - -anstream = "0.6.15" -owo-colors = "4.0.0" -const_format = "0.2.33" - -[[bin]] -name = "rustypipe" -path = "src/main.rs" +rustypipe = { path = "../", default-features = false } +rustypipe-downloader = { path = "../downloader", default-features = false } +reqwest = { version = "0.11.11", default_features = false } +tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] } +indicatif = "0.17.0" +futures = "0.3.21" +anyhow = "1.0" +clap = { version = "4.0.29", features = ["derive"] } +env_logger = "0.10.0" +serde = "1.0" +serde_json = "1.0.82" +serde_yaml = "0.9.19" +dirs = "5.0.0" diff --git a/cli/README.md b/cli/README.md deleted file mode 100644 index 7474f0a..0000000 --- a/cli/README.md +++ /dev/null @@ -1,174 +0,0 @@ -# ![RustyPipe](https://codeberg.org/ThetaDev/rustypipe/raw/branch/main/notes/logo.svg) CLI - -[![Current crates.io version](https://img.shields.io/crates/v/rustypipe-cli.svg)](https://crates.io/crates/rustypipe-cli) -[![License](https://img.shields.io/badge/License-GPL--3-blue.svg?style=flat)](https://opensource.org/licenses/GPL-3.0) -[![CI status](https://codeberg.org/ThetaDev/rustypipe/actions/workflows/ci.yaml/badge.svg?style=flat&label=CI)](https://codeberg.org/ThetaDev/rustypipe/actions/?workflow=ci.yaml) - -The RustyPipe CLI is a powerful YouTube client for the command line. It allows you to -access most of the features of the RustyPipe crate: getting data from YouTube and -downloading videos. - -## Installation - -You can download a compiled version of RustyPipe here: - - -Alternatively, you can compile it yourself by installing [Rust](https://rustup.rs/) and -running `cargo install rustypipe-cli`. - -To be able to access streams from web-based clients (Desktop, Mobile) you need to -download [rustypipe-botguard](https://codeberg.org/ThetaDev/rustypipe-botguard/releases) -and place the binary either in the PATH or the current working directory. - -For downloading videos you also need to have ffmpeg installed. - -## `get`: Fetch information - -You can call the get command with any YouTube entity ID or URL and RustyPipe will fetch -the associated metadata. It can fetch channels, playlists, albums and videos. - -**Usage:** `rustypipe get UC2TXq_t06Hjdr2g_KdKpHQg` - -- `-l`, `--limit` Limit the number of list items to fetch -- `-t`, `--tab` Channel tab (options: **videos**, shorts, live, playlists, info) -- `-m, --music` Use the YouTube Music API -- `--rss`Fetch the RSS feed of a channel -- `--comments` Get comments (options: top, latest) -- `--lyrics` Get the lyrics for YTM tracks -- `--player` Get the player data instead of the video details when fetching videos -- `-c`, `--client-type` YT clients used to fetch player data (options: desktop, tv, - tv-embed, android, ios; if multiple clients are specified, they are attempted in - order) - -## `search`: Search YouTube - -With the search command you can search the entire YouTube platform or individual -channels. YouTube Music search is also supported. - -Note that search filters are only supported when searching YouTube. They have no effect -when searching YTM or individual channels. - -**Usage:** `rustypipe search "query"` - -### Options - -- `-l`, `--limit` Limit the number of list items to fetch - -- `--item-type` Filter results by item type -- `--length` Filter results by video length -- `--date` Filter results by upload date (options: hour, day, week, month, year) -- `--order` Sort search results (options: rating, date, views) -- `--channel` Channel ID for searching channel videos -- `-m`, `--music` Search YouTube Music in the given category (options: all, tracks, - videos, artists, albums, playlists-ytm, playlists-community) - -## `dl`: Download videos - -The downloader can download individual videos, playlists, albums and channels. Multiple -videos can be downloaded in parallel for improved performance. - -**Usage:** `rustypipe dl eRsGyueVLvQ` - -### Options - -- `-o`, `--output` Download to the given directory -- `--output-file` Download to the given file -- `--template` Download to a path determined by a template - -- `-r`, `--resolution` Video resolution (e.g. 720, 1080). Set to 0 for audio-only -- `-a`, `--audio` Download only the audio track and write track metadata + album cover -- `-p`, `--parallel` Number of videos downloaded in parallel (default: 8) -- `-m`, `--music` Use YouTube Music for downloading playlists -- `-l`, `--limit` Limit the number of videos to download (default: 1000) -- `-c`, `--client-type` YT clients used to fetch player data (options: desktop, tv, - tv-embed, android, ios; if multiple clients are specified, they are attempted in - order) - -## `vdata`: Get visitor data - -You can use the vdata command to get a new visitor data ID. This feature may come in -handy for testing and reproducing A/B tests. - -## `releases` Get YouTube Music new releases - -Get a list of new albums or music videos on YouTube Music - -**Usage:** `rustypipe releases` or `rustypipe releases --videos` - -## `charts`: Get YouTube Music charts - -Get a list of the most popular tracks and artists for a given country - -**Usage:** `rustypipe charts DE` - -## `history`: Get YouTube playback history - -Get a list of recently played videos or tracks - -### Options - -- `-l`, `--limit` Limit the number of list items to fetch -- `--search` Search the playback history (unavailable on YouTube Music) -- `-m`, `--music` Get the YouTube Music playback history - -## `subscriptions`: Get subscribed channels - -You can use the RustyPipe CLI to get a list of the channels you subscribed to. With the -`--format` flag you can export then in different formats, including OPML and NewPipe -JSON. - -With the `--feed` option you can output a list of the latest videos from your -subscription feed instead. - -### Options - -- `-l`, `--limit` Limit the number of list items to fetch -- `-m`, `--music` Get a list of subscribed YouTube Music artists -- `--feed` Output YouTube Music subscription feed - -## `playlists`, `albums`, `tracks`: Get your YouTube library - -Fetch a list of all the items saved in your YouTube/YouTube Music profile. - -### Options - -- `-l`, `--limit` Limit the number of list items to fetch -- `-m`, `--music` (only for playlists): Get your YouTube Music playlists - -## Global options - -- **Proxy:** RustyPipe respects the environment variables `HTTP_PROXY`, `HTTPS_PROXY` - and `ALL_PROXY` -- **Logging:** Enable debug logging with the `-v` (verbose) flag. If you want more - fine-grained control, use the `RUST_LOG` environment variable. -- **Visitor data:** A custom visitor data ID can be used with the `--vdata` flag -- **Authentication:** Use the commands `rustypipe login` and `rustypipe login --cookie` - to log into your Google account using either OAuth or YouTube cookies. With the - `--auth` flag you can use authentication for any request. -- `--lang` Change the YouTube content language -- `--country` Change the YouTube content country -- `--tz` Use a specific - [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) (e.g. - Europe/Berlin, Australia/Sydney) - - **Note:** this requires building rustypipe-cli with the `timezone` feature - -- `--local-tz` Use the local timezone instead of UTC -- `--report` Generate a report on every request and store it in a `rustypipe_reports` - folder in the current directory -- `--cache-file` Change the RustyPipe cache file location (Default: - `~/.local/share/rustypipe/rustypipe_cache.json`) -- `--report-dir` Change the RustyPipe report directory location (Default: - `~/.local/share/rustypipe/rustypipe_reports`) -- `--botguard-bin` Use a - [rustypipe-botguard](https://codeberg.org/ThetaDev/rustypipe-botguard) binary from the - given path for generating PO tokens -- `--no-botguard` Disable Botguard, only download videos using clients that dont require - it -- `--pot-cache` Enable caching for session-bound PO tokens - -### Output format - -By default, the CLI outputs YouTube data in a human-readable text format. If you want to -store the data or process it with a script, you should choose a machine readable output -format. You can choose both JSON and YAML with the `-f, --format` flag. diff --git a/cli/src/main.rs b/cli/src/main.rs index b226221..2a1ba78 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,126 +1,24 @@ -#![doc = include_str!("../README.md")] #![warn(clippy::todo, clippy::dbg_macro)] -use std::{ - ffi::OsString, - path::PathBuf, - str::FromStr, - sync::{atomic::AtomicUsize, Arc}, - time::Duration, -}; +use std::{path::PathBuf, time::Duration}; +use anyhow::{Context, Result}; use clap::{Parser, Subcommand, ValueEnum}; -use const_format::formatcp; -use futures_util::stream::{self, StreamExt}; +use futures::stream::{self, StreamExt}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; -use owo_colors::OwoColorize; +use reqwest::{Client, ClientBuilder}; use rustypipe::{ - cache::FileStorage, - client::{ClientType, RustyPipe}, - model::{ - richtext::{RichText, ToPlaintext}, - traits::YtEntity, - ArtistId, AudioCodec, Comment, MusicSearchResult, TrackItem, TrackType, UrlTarget, - Verification, YouTubeItem, - }, - param::{search_filter, ChannelVideoTab, Country, Language, StreamFilter}, - report::FileReporter, -}; -use rustypipe_downloader::{ - DownloadError, DownloadQuery, DownloadVideo, Downloader, DownloaderBuilder, + client::RustyPipe, + model::{UrlTarget, VideoId}, + param::{search_filter, ChannelVideoTab, StreamFilter}, }; use serde::Serialize; -use tracing::level_filters::LevelFilter; -use tracing_subscriber::{fmt::MakeWriter, EnvFilter}; #[derive(Parser)] -#[clap( - author, - version = formatcp!("{}\nrustypipe {}", env!("CARGO_PKG_VERSION"), rustypipe::VERSION), - about, - long_about = None -)] +#[clap(author, version, about, long_about = None)] struct Cli { #[clap(subcommand)] command: Commands, - /// Always generate a report (used for debugging) - #[clap(long, global = true)] - report: bool, - /// YouTube visitor data ID - #[clap(long, global = true)] - vdata: Option, - /// YouTube content language - #[clap(long, global = true)] - lang: Option, - /// YouTube content country - #[clap(long, global = true)] - country: Option, - /// Use a specific timezone (e.g. Europe/Berlin, Australia/Sydney) - #[cfg(feature = "timezone")] - #[clap(long, global = true)] - tz: Option, - /// Use local timezone - #[clap(long, global = true)] - tz_local: bool, - /// Use authentication - #[clap(long, global = true)] - auth: bool, - #[clap(long, global = true)] - /// RustyPipe cache file - cache_file: Option, - /// RustyPipe report folder - #[clap(long, global = true)] - report_dir: Option, - /// Path to rustypipe-botguard binary - #[clap(long, global = true)] - botguard_bin: Option, - /// Disable Botguard - #[clap(long, global = true)] - no_botguard: bool, - /// Enable caching for session-bound PO tokens - #[clap(long, global = true)] - pot_cache: bool, - /// Enable debug logging - #[clap(short, long, global = true)] - verbose: bool, -} - -#[derive(Parser)] -#[group(multiple = false)] -struct DownloadTarget { - /// Download to the given directory - #[clap(short, long)] - output: Option, - /// Download to the given file - #[clap(long)] - output_file: Option, - /// Download to a path determined by a template - #[clap(long)] - template: Option, -} - -impl DownloadTarget { - fn assert_dir(&self) { - if self.output_file.is_some() { - panic!("Cannot download multiple videos to a single file") - } else if let Some(template) = &self.template { - if !template.contains("{id}") && !template.contains("{title}") { - panic!("Template must contain {{id}} or {{title}} variables") - } - } - } - - fn apply(&self, q: DownloadQuery) -> DownloadQuery { - if let Some(output_file) = &self.output_file { - q.to_file(output_file) - } else if let Some(output) = &self.output { - q.to_dir(output) - } else if let Some(template) = &self.template { - q.to_template(template) - } else { - q - } - } } #[derive(Subcommand)] @@ -130,74 +28,60 @@ enum Commands { Download { /// ID or URL id: String, - #[clap(flatten)] - target: DownloadTarget, + /// Output path + #[clap(short, default_value = ".")] + output: PathBuf, /// Video resolution (e.g. 720, 1080). Set to 0 for audio-only. #[clap(short, long)] resolution: Option, - /// Download only the audio track and write track information - #[clap(short, long)] - audio: bool, /// Number of videos downloaded in parallel #[clap(short, long, default_value_t = 8)] parallel: usize, - /// Use YouTube Music for downloading playlists - #[clap(short, long)] - music: bool, /// Limit the number of videos to download - #[clap(short, long, default_value_t = 1000)] + #[clap(long, default_value_t = 1000)] limit: usize, - /// YT Client used to fetch player data - #[clap(short, long)] - client_type: Option>, }, /// Extract video, playlist, album or channel data Get { /// ID or URL id: String, /// Output format - #[clap(short, long, value_parser)] - format: Option, + #[clap(long, value_parser, default_value = "json")] + format: Format, /// Pretty-print output #[clap(long)] pretty: bool, /// Limit the number of items to fetch - #[clap(short, long, default_value_t = 20)] + #[clap(long, default_value_t = 20)] limit: usize, /// Channel tab - #[clap(short, long, default_value = "videos")] + #[clap(long, default_value = "videos")] tab: ChannelTab, /// Use YouTube Music - #[clap(short, long)] - music: bool, - /// Fetch the RSS feed of a channel #[clap(long)] - rss: bool, + music: bool, /// Get comments #[clap(long)] comments: Option, - /// Get lyrics for YTM tracks + /// Get lyrics #[clap(long)] lyrics: bool, - /// Get the player data instead of the video details + /// Get the player #[clap(long)] player: bool, - /// YT Client used to fetch player data - #[clap(short, long)] - client_type: Option>, }, /// Search YouTube Search { /// Search query query: String, /// Output format - #[clap(short, long, value_parser)] - format: Option, + #[clap(long, value_parser, default_value = "json")] + format: Format, /// Pretty-print output #[clap(long)] pretty: bool, /// Limit the number of items to fetch - #[clap(short, long, default_value_t = 20)] + #[clap(long, default_value_t = 20)] limit: usize, /// Filter results by item type #[clap(long)] @@ -214,144 +98,19 @@ enum Commands { /// Channel ID for searching channel videos #[clap(long)] channel: Option, - /// Search YouTube Music in the given category - #[clap(short, long)] + /// YouTube Music search filter + #[clap(long)] music: Option, }, - /// Get your playback history - History { - /// Output format - #[clap(short, long, value_parser)] - format: Option, - /// Pretty-print output - #[clap(long)] - pretty: bool, - /// Limit the number of items to fetch - #[clap(short, long, default_value_t = 20)] - limit: usize, - /// Use YouTube Music - #[clap(short, long)] - music: bool, - /// Search YouTube playback history - #[clap(long)] - search: Option, - }, - /// Get the channels you subscribed to - Subscriptions { - /// Output format - #[clap(short, long, value_parser)] - format: Option, - /// Pretty-print output - #[clap(long)] - pretty: bool, - /// Limit the number of items to fetch - #[clap(short, long, default_value_t = 20)] - limit: usize, - /// Use YouTube Music - #[clap(short, long)] - music: bool, - /// Output YouTube subscription feed - #[clap(long)] - feed: bool, - }, - /// Get the playlists from your library - Playlists { - /// Output format - #[clap(short, long, value_parser)] - format: Option, - /// Pretty-print output - #[clap(long)] - pretty: bool, - /// Limit the number of items to fetch - #[clap(short, long, default_value_t = 20)] - limit: usize, - /// Use YouTube Music - #[clap(short, long)] - music: bool, - }, - /// Get the albums from your library - Albums { - /// Output format - #[clap(short, long, value_parser)] - format: Option, - /// Pretty-print output - #[clap(long)] - pretty: bool, - /// Limit the number of items to fetch - #[clap(short, long, default_value_t = 20)] - limit: usize, - }, - /// Get the tracks from your library - Tracks { - /// Output format - #[clap(short, long, value_parser)] - format: Option, - /// Pretty-print output - #[clap(long)] - pretty: bool, - /// Limit the number of items to fetch - #[clap(short, long, default_value_t = 20)] - limit: usize, - }, - /// Get the latest music releases - Releases { - /// Get latest music videos - #[clap(long)] - videos: bool, - /// Output format - #[clap(short, long, value_parser)] - format: Option, - /// Pretty-print output - #[clap(long)] - pretty: bool, - }, - /// Get YouTube Music charts - Charts { - /// Chart country - country: Option, - /// Output format - #[clap(short, long, value_parser)] - format: Option, - /// Pretty-print output - #[clap(long)] - pretty: bool, - }, - /// Get a YouTube visitor data ID - Vdata, - /// Log in using your Google account - Login { - /// Log in using YouTube cookies (otherwise OAuth is used) - #[clap(long)] - cookie: bool, - /// Path to cookie.txt - #[clap(long)] - cookies_txt: Option, - }, - /// Log out from your Google account - Logout { - /// Remove stored YouTube cookies (otherwise OAuth is used) - #[clap(long)] - cookie: bool, - }, } -#[derive(Default, Copy, Clone, ValueEnum)] +#[derive(Copy, Clone, ValueEnum)] enum Format { - #[default] Json, Yaml, } -#[derive(Debug, Default, Copy, Clone, ValueEnum)] -enum SubscriptionFormat { - #[default] - Json, - Yaml, - Newpipe, - Opml, -} - -#[derive(Debug, Copy, Clone, ValueEnum)] +#[derive(Copy, Clone, ValueEnum)] enum ChannelTab { Videos, Shorts, @@ -407,39 +166,16 @@ enum SearchOrder { Views, } -#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)] +#[derive(Copy, Clone, ValueEnum)] enum MusicSearchCategory { All, Tracks, Videos, Artists, Albums, + Playlists, PlaylistsYtm, PlaylistsCommunity, - Users, -} - -#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)] -enum ClientTypeArg { - Desktop, - Mobile, - Tv, - Android, - Ios, -} - -#[derive(Serialize)] -struct NewpipeSubscriptions { - app_version: &'static str, - app_version_int: u16, - subscriptions: Vec, -} - -#[derive(Serialize)] -struct NewpipeSubscription { - service_id: u16, - url: String, - name: String, } impl From for search_filter::ItemType { @@ -484,26 +220,62 @@ impl From for search_filter::Order { } } -impl From for ClientType { - fn from(value: ClientTypeArg) -> Self { - match value { - ClientTypeArg::Desktop => Self::Desktop, - ClientTypeArg::Mobile => Self::Mobile, - ClientTypeArg::Tv => Self::Tv, - ClientTypeArg::Android => Self::Android, - ClientTypeArg::Ios => Self::Ios, - } - } -} +#[allow(clippy::too_many_arguments)] +async fn download_single_video( + video_id: &str, + video_title: &str, + output_dir: &str, + output_fname: Option, + resolution: Option, + ffmpeg: &str, + rp: &RustyPipe, + http: Client, + multi: MultiProgress, + main: Option, +) -> Result<()> { + let pb = multi.add(ProgressBar::new(1)); + pb.set_style(ProgressStyle::with_template("{msg}\n{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({bytes_per_sec}, {eta})").unwrap() + .progress_chars("#>-")); + pb.set_message(format!("Fetching player data for {video_title}")); -impl From for Format { - fn from(value: SubscriptionFormat) -> Self { - match value { - SubscriptionFormat::Json => Self::Json, - SubscriptionFormat::Yaml => Self::Yaml, - _ => Self::default(), + let res = async { + let player_data = rp + .query() + .player(video_id) + .await + .context(format!("Failed to fetch player data for video {video_id}"))?; + + let mut filter = StreamFilter::new(); + if let Some(res) = resolution { + if res == 0 { + filter = filter.no_video(); + } else { + filter = filter.video_max_res(res); + } } + + rustypipe_downloader::download_video( + &player_data, + output_dir, + output_fname, + None, + &filter, + ffmpeg, + http, + pb, + ) + .await + .context(format!( + "Failed to download video '{}' [{}]", + player_data.details.name, video_id + )) } + .await; + + if let Some(main) = main { + main.inc(1); + } + res } fn print_data(data: &T, format: Format, pretty: bool) { @@ -520,287 +292,58 @@ fn print_data(data: &T, format: Format, pretty: bool) { }; } -fn print_entities(items: &[impl YtEntity], with_type: bool) { - for e in items { - print_entity(e, with_type); - } -} - -fn print_entity(e: &impl YtEntity, with_type: bool) { - if with_type { - if let Some(t) = e.music_item_type() { - anstream::print!("{: >8} ", format!("{t:?}").dimmed()); - } - } - anstream::print!("[{}] {}", e.id(), e.name().bold()); - if let Some(n) = e.channel_name() { - anstream::print!(" - {}", n.cyan()); - } - println!(); -} - -fn fmt_print_entities( - items: &[T], - format: Option, - pretty: bool, - title: &str, -) { - match format { - Some(format) => print_data(&items, format, pretty), - None => { - print_h1(title); - print_entities(items, false); - } - } -} - -fn fmt_print_tracks(tracks: &[TrackItem], format: Option, pretty: bool, title: &str) { - match format { - Some(format) => print_data(&tracks, format, pretty), - None => { - print_h1(title); - print_tracks(tracks); - } - } -} - -fn fmt_print_subscriptions( - items: &[T], - format: Option, - pretty: bool, - title: &str, -) { - match format { - Some(SubscriptionFormat::Newpipe) => { - let subscriptions = items - .iter() - .map(|itm| NewpipeSubscription { - service_id: 0, - url: format!("https://www.youtube.com/channel/{}", itm.id()), - name: itm.name().to_owned(), - }) - .collect(); - let data = NewpipeSubscriptions { - app_version: "0.24.1", - app_version_int: 991, - subscriptions, - }; - print_data(&data, Format::Json, pretty); - } - Some(SubscriptionFormat::Opml) => { - let mut writer = if pretty { - quick_xml::Writer::new_with_indent(std::io::stdout(), b' ', 2) - } else { - quick_xml::Writer::new(std::io::stdout()) - }; - writer - .create_element("opml") - .with_attribute(("version", "1.1")) - .write_inner_content(|writer| { - writer - .create_element("body") - .write_inner_content(|writer| { - writer - .create_element("outline") - .with_attributes([ - ("text", title), - ("title", title), - ]) - .write_inner_content(|writer| { - for itm in items { - writer - .create_element("outline") - .with_attributes([ - ("text", itm.name()), - ("title", itm.name()), - ("type", "rss"), - ("xmlUrl", &format!("https://www.youtube.com/feeds/videos.xml?channel_id={}", itm.id())), - ]) - .write_empty()?; - } - Ok(()) - })?; - Ok(()) - })?; - Ok(()) - }) - .unwrap(); - println!(); - } - Some(format) => print_data(&items, format.into(), pretty), - None => { - print_h1(title); - print_entities(items, false); - } - } -} - -fn print_tracks(tracks: &[TrackItem]) { - for t in tracks { - if let Some(n) = t.track_nr { - anstream::print!("{} ", format!("{n:02}").yellow().bold()); - } - anstream::print!("[{}] {} - ", t.id, t.name.bold()); - print_artists(&t.artists); - print_duration(t.duration); - println!(); - } -} - -fn print_artists(artists: &[ArtistId]) { - for (i, a) in artists.iter().enumerate() { - if i > 0 { - print!(", "); - } - anstream::print!("{}", a.name.cyan()); - if let Some(id) = &a.id { - print!(" [{id}]"); - } - } -} - -fn print_duration(duration: Option) { - if let Some(d) = duration { - print!(" "); - let hours = d / 3600; - let minutes = (d / 60) % 60; - let seconds = d % 60; - if hours > 0 { - anstream::print!("{}", format!("{hours:02}:").yellow()); - } - anstream::print!("{}", format!("{minutes:02}:{seconds:02}").yellow()); - } -} - -fn print_music_search( - data: &MusicSearchResult, - format: Option, - pretty: bool, - with_type: bool, -) { - match format { - Some(format) => print_data(data, format, pretty), - None => { - print_h1("Music search"); - if let Some(corr) = &data.corrected_query { - anstream::println!("Did you mean `{}`?", corr.magenta()); - } - print_entities(&data.items.items, with_type); - } - } -} - -fn print_description(desc: Option) { - if let Some(desc) = desc { - if !desc.is_empty() { - print_h2("Description"); - println!("{}", desc.trim()); - } - } -} - -fn print_h1(title: &str) { - anstream::println!("{}", format!("{title}:").on_green().black()); -} - -fn print_h2(title: &str) { - anstream::println!("\n{}", format!("{title}:").green().underline()); -} - -fn print_verification(verification: Verification) { - match verification { - Verification::None => {} - Verification::Verified => print!(" ✓"), - Verification::Artist => print!(" ♪"), - } -} - -fn print_comments(comments: &[Comment]) { - print_h2("Comments"); - for c in comments { - if let Some(author) = &c.author { - anstream::print!("{} [{}]", author.name.cyan(), author.id); - print_verification(author.verification); - } else { - anstream::print!("{}", "Unknown author".magenta()); - } - if c.by_owner { - print!(" (Owner)"); - } - println!(); - print_richtext(&c.text); - anstream::print!("{} {}", "Likes:".blue(), c.like_count.unwrap_or_default()); - if c.hearted { - anstream::print!(" {}", "♥".red()); - } - println!(); - if let Some(ctoken) = &c.replies.ctoken { - println!("replies:{ctoken}"); - } - println!(); - } -} - -fn print_richtext(text: &RichText) { - for c in &text.0 { - match c { - rustypipe::model::richtext::TextComponent::Text { text, style } => { - if !text.is_empty() { - let mut tstyle = owo_colors::Style::new(); - - if style.bold { - tstyle = tstyle.bold(); - } - if style.italic { - tstyle = tstyle.italic(); - } - if style.strikethrough { - tstyle = tstyle.strikethrough(); - } - anstream::print!("{}", text.style(tstyle)); - } - } - rustypipe::model::richtext::TextComponent::Web { url, .. } => { - anstream::print!("{}", url.underline()); - } - rustypipe::model::richtext::TextComponent::YouTube { text, target } => { - if matches!(target, UrlTarget::Channel { .. }) { - anstream::print!("{}", text.cyan()); - } else { - anstream::print!("{}", target.to_url().underline()); - } - } - _ => {} - } - } - println!(); -} - async fn download_video( - dl: &Downloader, + rp: &RustyPipe, id: &str, - target: &DownloadTarget, - client_types: Option<&[ClientType]>, + output_dir: &str, + output_fname: Option, + resolution: Option, ) { - let mut q = target.apply(dl.id(id)); - if let Some(client_types) = client_types { - q = q.client_types(client_types); - } - let res = q.download().await; - if let Err(e) = res { - tracing::error!("[{id}]: {e}") - } + let http = ClientBuilder::new() + .user_agent("Mozilla/5.0 (Windows NT 10.0; Win64; rv:107.0) Gecko/20100101 Firefox/107.0") + .gzip(true) + .brotli(true) + .timeout(Duration::from_secs(10)) + .build() + .expect("unable to build the HTTP client"); + + // Indicatif setup + let multi = MultiProgress::new(); + + download_single_video( + id, + id, + output_dir, + output_fname, + resolution, + "ffmpeg", + rp, + http, + multi, + None, + ) + .await + .unwrap_or_else(|e| println!("ERROR: {e:?}")); } async fn download_videos( - dl: &Downloader, - videos: Vec, - target: &DownloadTarget, + rp: &RustyPipe, + videos: &[VideoId], + output_dir: &str, + output_fname: Option, + resolution: Option, parallel: usize, - client_types: Option<&[ClientType]>, - multi: MultiProgress, -) -> anyhow::Result<()> { +) { + let http = ClientBuilder::new() + .user_agent("Mozilla/5.0 (Windows NT 10.0; Win64; rv:107.0) Gecko/20100101 Firefox/107.0") + .gzip(true) + .brotli(true) + .timeout(Duration::from_secs(10)) + .build() + .expect("unable to build the HTTP client"); + // Indicatif setup + let multi = MultiProgress::new(); let main = multi.add(ProgressBar::new( videos.len().try_into().unwrap_or_default(), )); @@ -813,234 +356,154 @@ async fn download_videos( ); main.tick(); - let n_failed = Arc::new(AtomicUsize::default()); - stream::iter(videos) - .for_each_concurrent(parallel, |video| { - let dl = dl.clone(); - let main = main.clone(); - let id = video.id().to_owned(); - let n_failed = n_failed.clone(); - - let mut q = target.apply(dl.video(video)); - if let Some(client_types) = client_types { - q = q.client_types(client_types); - } - - async move { - if let Err(e) = q.download().await { - if !matches!(e, DownloadError::Exists(_)) { - tracing::error!("[{id}]: {e}"); - n_failed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - } - } else { - main.inc(1); - } - } + .map(|video| { + download_single_video( + &video.id, + &video.name, + output_dir, + output_fname.clone(), + resolution, + "ffmpeg", + rp, + http.clone(), + multi.clone(), + Some(main.clone()), + ) }) - .await; - - let n_failed = n_failed.load(std::sync::atomic::Ordering::Relaxed); - if n_failed > 0 { - anyhow::bail!("{n_failed} downloads failed"); - } - Ok(()) -} - -/// Stderr writer that suspends the progress bars before printing logs -#[derive(Clone)] -struct ProgWriter(MultiProgress); - -impl<'a> MakeWriter<'a> for ProgWriter { - type Writer = ProgWriter; - - fn make_writer(&'a self) -> Self::Writer { - self.clone() - } -} - -impl std::io::Write for ProgWriter { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - self.0.suspend(|| std::io::stderr().write(buf)) - } - - fn flush(&mut self) -> std::io::Result<()> { - std::io::stderr().flush() - } + .buffer_unordered(parallel) + .collect::>() + .await + .into_iter() + .for_each(|res| match res { + Ok(_) => {} + Err(e) => { + println!("ERROR: {e:?}"); + } + }); } #[tokio::main] async fn main() { - if let Err(e) = run().await { - println!("{} {}", "Error:".red().bold(), e); - std::process::exit(1); - } -} + env_logger::init(); -async fn run() -> anyhow::Result<()> { let cli = Cli::parse(); - let multi = MultiProgress::new(); - - let mut env_filter = EnvFilter::builder() - .with_default_directive(LevelFilter::INFO.into()) - .from_env_lossy(); - if cli.verbose { - env_filter = env_filter.add_directive("rustypipe=debug".parse().unwrap()); - } - - tracing_subscriber::fmt::SubscriberBuilder::default() - .with_env_filter(env_filter) - .with_writer(ProgWriter(multi.clone())) - .init(); let mut storage_dir = dirs::data_dir().expect("no data dir"); storage_dir.push("rustypipe"); + _ = std::fs::create_dir(&storage_dir); - if cli.cache_file.is_none() || cli.report_dir.is_none() { - std::fs::create_dir_all(&storage_dir).expect("could not create data dir"); - } - - let mut rp = RustyPipe::builder() - .storage_dir(storage_dir) - .visitor_data_opt(cli.vdata) - .timeout(Duration::from_secs(15)); - - if let Some(cache_file) = cli.cache_file { - rp = rp.storage(Box::new(FileStorage::new(cache_file))); - } - if let Some(report_dir) = cli.report_dir { - rp = rp.reporter(Box::new(FileReporter::new(report_dir))); - } - - if cli.report { - rp = rp - .report() - .reporter(Box::new(FileReporter::new("rustypipe_reports"))); - } - if let Some(lang) = cli.lang { - rp = rp.lang(Language::from_str(&lang).expect("invalid language")); - } - if let Some(country) = cli.country { - rp = rp.country(Country::from_str(&country.to_ascii_uppercase()).expect("invalid country")); - } - if let Some(botguard_bin) = cli.botguard_bin { - rp = rp.botguard_bin(botguard_bin); - } - if cli.tz_local { - rp = rp.timezone_local(); - } - - #[cfg(feature = "timezone")] - if let Some(timezone) = cli.tz { - use time::OffsetDateTime; - use time_tz::{Offset, TimeZone}; - - let tz = time_tz::timezones::get_by_name(&timezone).expect("invalid timezone"); - let offset = tz - .get_offset_utc(&OffsetDateTime::now_utc()) - .to_utc() - .whole_minutes(); - rp = rp.timezone(tz.name(), offset); - } - - if cli.no_botguard { - rp = rp.no_botguard(); - } - if cli.pot_cache { - rp = rp.po_token_cache(); - } - if cli.auth { - rp = rp.authenticated(); - } - let rp = rp.build()?; + let rp = RustyPipe::builder().storage_dir(storage_dir).build(); match cli.command { Commands::Download { id, - target, + output, resolution, - audio, parallel, - music, limit, - client_type, } => { - let url_target = rp.query().resolve_string(&id, false).await?; - - let mut filter = StreamFilter::new(); - if let Some(res) = resolution { - if res == 0 { - filter = filter - .no_video() - .audio_codecs([AudioCodec::Mp4a, AudioCodec::Opus]); - } else { - filter = filter.video_max_res(res); + // Cases: Existing folder, non-existing file with existing parent folder, + // Error cases: non-existing parent folder, existing file + let output_path = std::fs::canonicalize(output).unwrap(); + if output_path.is_file() { + println!("Output file already exists"); + return; + } + let (output_dir, output_fname) = if output_path.is_dir() { + (output_path.to_string_lossy().to_string(), None) + } else { + let output_dir_parent = output_path.parent().unwrap(); + if !output_dir_parent.is_dir() { + println!( + "Parent folder {} does not exist", + output_dir_parent.to_string_lossy() + ); + return; } - } - let mut dl = DownloaderBuilder::new() - .rustypipe(&rp) - .multi_progress(multi.clone()) - .path_precheck(); - if audio { - dl = dl.audio_tag().crop_cover(); - filter = filter.no_video(); - } - let dl = dl.stream_filter(filter).build(); - let cts = client_type.map(|c| c.into_iter().map(ClientType::from).collect::>()); + ( + output_dir_parent.to_string_lossy().to_string(), + Some( + output_path + .file_name() + .unwrap() + .to_string_lossy() + .to_string(), + ), + ) + }; - match url_target { + let target = rp.query().resolve_string(&id, false).await.unwrap(); + match target { UrlTarget::Video { id, .. } => { - download_video(&dl, &id, &target, cts.as_deref()).await; + download_video(&rp, &id, &output_dir, output_fname, resolution).await; } UrlTarget::Channel { id } => { - target.assert_dir(); - let mut channel = rp.query().channel_videos(id).await?; - channel.content.extend_limit(&rp.query(), limit).await?; - let videos = channel + let mut channel = rp.query().channel_videos(id).await.unwrap(); + channel + .content + .extend_limit(&rp.query(), limit) + .await + .unwrap(); + let videos: Vec = channel .content .items .into_iter() .take(limit) - .map(|v| DownloadVideo::from_entity(&v)) + .map(VideoId::from) .collect(); - download_videos(&dl, videos, &target, parallel, cts.as_deref(), multi).await?; + download_videos( + &rp, + &videos, + &output_dir, + output_fname, + resolution, + parallel, + ) + .await; } UrlTarget::Playlist { id } => { - target.assert_dir(); - let videos = if music { - let mut playlist = rp.query().music_playlist(id).await?; - playlist.tracks.extend_limit(&rp.query(), limit).await?; - playlist - .tracks - .items - .into_iter() - .take(limit) - .map(|v| DownloadVideo::from_track(&v)) - .collect() - } else { - let mut playlist = rp.query().playlist(id).await?; - playlist.videos.extend_limit(&rp.query(), limit).await?; - playlist - .videos - .items - .into_iter() - .take(limit) - .map(|v| DownloadVideo::from_entity(&v)) - .collect() - }; - download_videos(&dl, videos, &target, parallel, cts.as_deref(), multi).await?; + let mut playlist = rp.query().playlist(id).await.unwrap(); + playlist + .videos + .extend_limit(&rp.query(), limit) + .await + .unwrap(); + let videos: Vec = playlist + .videos + .items + .into_iter() + .take(limit) + .map(VideoId::from) + .collect(); + download_videos( + &rp, + &videos, + &output_dir, + output_fname, + resolution, + parallel, + ) + .await; } UrlTarget::Album { id } => { - target.assert_dir(); - let album = rp.query().music_album(id).await?; - let videos = album + let album = rp.query().music_album(id).await.unwrap(); + let videos: Vec = album .tracks .into_iter() .take(limit) - .map(|v| DownloadVideo::from_track(&v)) + .map(VideoId::from) .collect(); - download_videos(&dl, videos, &target, parallel, cts.as_deref(), multi).await?; + download_videos( + &rp, + &videos, + &output_dir, + output_fname, + resolution, + parallel, + ) + .await; } } } @@ -1051,231 +514,57 @@ async fn run() -> anyhow::Result<()> { limit, tab, music, - rss, comments, lyrics, player, - client_type, } => { - if let Some(ctoken) = id.strip_prefix("replies:") { - let mut replies = rp.query().video_comments(ctoken, None).await?; - replies.extend_limit(&rp.query(), limit).await?; - match format { - Some(format) => print_data(&replies.items, format, pretty), - None => print_comments(&replies.items), - } - return Ok(()); - } - - let target = rp.query().resolve_string(&id, false).await?; + let target = rp.query().resolve_string(&id, false).await.unwrap(); match target { UrlTarget::Video { id, .. } => { if lyrics { - let details = rp.query().music_details(&id).await?; + let details = rp.query().music_details(&id).await.unwrap(); match details.lyrics_id { Some(lyrics_id) => { - let lyrics = rp.query().music_lyrics(lyrics_id).await?; - match format { - Some(format) => print_data(&lyrics, format, pretty), - None => println!("{}\n\n{}", lyrics.body, lyrics.footer.blue()), - } + let lyrics = rp.query().music_lyrics(lyrics_id).await.unwrap(); + print_data(&lyrics, format, pretty); } None => eprintln!("no lyrics found"), } } else if music { - let details = rp.query().music_details(&id).await?; - match format { - Some(format) => print_data(&details, format, pretty), - None => { - let typ_str = match details.track.track_type { - TrackType::Track => "[Track]", - TrackType::Video => "[MV]", - TrackType::Episode => "[Episode]", - }; - anstream::println!("{}", typ_str.on_green().black()); - anstream::print!( - "{} [{}]", - details.track.name.green().bold(), - details.track.id - ); - print_duration(details.track.duration); - println!(); - print_artists(&details.track.artists); - println!(); - if details.track.track_type.is_track() { - anstream::println!( - "{} {}", - "Album:".blue(), - details - .track - .album - .as_ref() - .map(|b| b.id.as_str()) - .unwrap_or("None") - ) - } - if let Some(view_count) = details.track.view_count { - anstream::println!("{} {}", "Views:".blue(), view_count); - } - } - } + let details = rp.query().music_details(&id).await.unwrap(); + print_data(&details, format, pretty); } else if player { - let player = if let Some(client_types) = client_type { - let cts = client_types - .into_iter() - .map(ClientType::from) - .collect::>(); - rp.query().player_from_clients(&id, &cts).await - } else { - rp.query().player(&id).await - }?; - print_data(&player, format.unwrap_or_default(), pretty); + let player = rp.query().player(&id).await.unwrap(); + print_data(&player, format, pretty); } else { - let mut details = rp.query().video_details(&id).await?; + let mut details = rp.query().video_details(&id).await.unwrap(); match comments { Some(CommentsOrder::Top) => { - details.top_comments.extend_limit(rp.query(), limit).await?; + details + .top_comments + .extend_limit(rp.query(), limit) + .await + .unwrap(); } Some(CommentsOrder::Latest) => { details .latest_comments .extend_limit(rp.query(), limit) - .await?; + .await + .unwrap(); } None => {} } - match format { - Some(format) => print_data(&details, format, pretty), - None => { - anstream::println!( - "{}\n{} [{}]", - "[Video]".on_green().black(), - details.name.green().bold(), - details.id - ); - anstream::println!( - "{} {} [{}]", - "Channel:".blue(), - details.channel.name, - details.channel.id - ); - if let Some(subs) = details.channel.subscriber_count { - anstream::println!("{} {}", "Subscribers:".blue(), subs); - } - if let Some(date) = details.publish_date { - anstream::println!("{} {}", "Date:".blue(), date); - } - anstream::println!("{} {}", "Views:".blue(), details.view_count); - if let Some(likes) = details.like_count { - anstream::println!("{} {}", "Likes:".blue(), likes); - } - if let Some(comments) = details.top_comments.count { - anstream::println!("{} {}", "Comments:".blue(), comments); - } - if details.is_ccommons { - anstream::println!("{}", "Creative Commons".green()); - } - if details.is_live { - anstream::println!("{}", "Livestream".red()); - } - print_richtext(&details.description); - if !details.recommended.is_empty() { - print_h2("Recommended"); - print_entities(&details.recommended.items, false); - } - let comment_list = comments.map(|c| match c { - CommentsOrder::Top => &details.top_comments.items, - CommentsOrder::Latest => &details.latest_comments.items, - }); - if let Some(comment_list) = comment_list { - print_comments(comment_list) - } - } - } + print_data(&details, format, pretty); } } UrlTarget::Channel { id } => { if music { - let artist = rp.query().music_artist(&id, true).await?; - match format { - Some(format) => print_data(&artist, format, pretty), - None => { - anstream::println!( - "{}\n{} [{}]", - "[Artist]".on_green().black(), - artist.name.green().bold(), - artist.id - ); - if let Some(subs) = artist.subscriber_count { - anstream::println!("{} {}", "Subscribers:".blue(), subs); - } - if let Some(url) = artist.wikipedia_url { - anstream::println!("{} {}", "Wikipedia:".blue(), url); - } - if let Some(id) = artist.tracks_playlist_id { - anstream::println!("{} {}", "All tracks:".blue(), id); - } - if let Some(id) = artist.videos_playlist_id { - anstream::println!("{} {}", "All videos:".blue(), id); - } - if let Some(id) = artist.radio_id { - anstream::println!("{} {}", "Radio:".blue(), id); - } - print_description(artist.description); - if !artist.albums.is_empty() { - print_h2("Albums"); - for b in artist.albums { - anstream::print!( - "[{}] {} ({:?}", - b.id, - b.name.bold(), - b.album_type - ); - if let Some(y) = b.year { - print!(", {y}"); - } - println!(")"); - } - } - if !artist.playlists.is_empty() { - print_h2("Playlists"); - print_entities(&artist.playlists, false); - } - if !artist.similar_artists.is_empty() { - print_h2("Similar artists"); - print_entities(&artist.similar_artists, false); - } - } - } - } else if rss { - let rss = rp.query().channel_rss(&id).await?; - - match format { - Some(format) => print_data(&rss, format, pretty), - None => { - anstream::println!( - "{}\n{} [{}]\n{} {}", - "[Channel RSS]".on_green().black(), - rss.name.green().bold(), - rss.id, - "Created on:".blue(), - rss.create_date, - ); - if let Some(v) = rss.videos.first() { - anstream::println!( - "{} {} [{}]", - "Latest video:".blue(), - v.publish_date, - v.id - ); - } - println!(); - print_entities(&rss.videos, false); - } - } + let artist = rp.query().music_artist(&id, true).await.unwrap(); + print_data(&artist, format, pretty); } else { match tab { ChannelTab::Videos | ChannelTab::Shorts | ChannelTab::Live => { @@ -1286,199 +575,48 @@ async fn run() -> anyhow::Result<()> { _ => unreachable!(), }; let mut channel = - rp.query().channel_videos_tab(&id, video_tab).await?; + rp.query().channel_videos_tab(&id, video_tab).await.unwrap(); - channel.content.extend_limit(rp.query(), limit).await?; - - match format { - Some(format) => print_data(&channel, format, pretty), - None => { - anstream::print!( - "{}\n{} {} [{}]", - format!("[Channel {tab:?}]").on_green().black(), - channel.name.green().bold(), - channel.handle.unwrap_or_default(), - channel.id - ); - print_verification(channel.verification); - println!(); - if let Some(subs) = channel.subscriber_count { - anstream::println!( - "{} {}", - "Subscribers:".blue(), - subs - ); - } - if let Some(vids) = channel.video_count { - anstream::println!("{} {}", "Videos:".blue(), vids); - } - print_description(Some(channel.description)); - println!(); - print_entities(&channel.content.items, false); - } - } + channel + .content + .extend_limit(rp.query(), limit) + .await + .unwrap(); + print_data(&channel, format, pretty); } ChannelTab::Playlists => { - let channel = rp.query().channel_playlists(&id).await?; - - match format { - Some(format) => print_data(&channel, format, pretty), - None => { - anstream::println!( - "{}\n{} {} [{}]", - format!("[Channel {tab:?}]").on_green().black(), - channel.name.green().bold(), - channel.handle.unwrap_or_default(), - channel.id - ); - print_description(Some(channel.description)); - if let Some(subs) = channel.subscriber_count { - anstream::println!( - "{} {}", - "Subscribers:".blue(), - subs - ); - } - if let Some(vids) = channel.video_count { - anstream::println!("{} {}", "Videos:".blue(), vids); - } - println!(); - print_entities(&channel.content.items, false); - } - } + let channel = rp.query().channel_playlists(&id).await.unwrap(); + print_data(&channel, format, pretty); } ChannelTab::Info => { - let info = rp.query().channel_info(&id).await?; - - match format { - Some(format) => print_data(&info, format, pretty), - None => { - anstream::println!( - "{}\nID:{}", - "[Channel info]".on_green().black(), - info.id - ); - print_description(Some(info.description)); - if let Some(subs) = info.subscriber_count { - anstream::println!( - "{} {}", - "Subscribers:".blue(), - subs - ); - } - if let Some(vids) = info.video_count { - anstream::println!("{} {}", "Videos:".blue(), vids); - } - if let Some(views) = info.view_count { - anstream::println!("{} {}", "Views:".blue(), views); - } - if let Some(created) = info.create_date { - anstream::println!( - "{} {}", - "Created on:".blue(), - created - ); - } - if let Some(country) = info.country { - anstream::println!("{} {}", "Country:".blue(), country); - } - if !info.links.is_empty() { - print_h2("Links"); - for (name, url) in &info.links { - anstream::println!("{} {}", name.blue(), url); - } - } - } - } + let channel = rp.query().channel_info(&id).await.unwrap(); + print_data(&channel, format, pretty); } } } } UrlTarget::Playlist { id } => { if music { - let mut playlist = rp.query().music_playlist(&id).await?; - playlist.tracks.extend_limit(rp.query(), limit).await?; - match format { - Some(format) => print_data(&playlist, format, pretty), - None => { - anstream::println!( - "{}\n{} [{}]\n{} {}", - "[MusicPlaylist]".on_green().black(), - playlist.name.green().bold(), - playlist.id, - "Tracks:".blue(), - playlist.track_count.unwrap_or_default(), - ); - if let Some(n) = playlist.channel_name() { - anstream::print!("{} {}", "Author:".blue(), n.bold()); - if let Some(id) = playlist.channel_id() { - print!(" [{id}]"); - } - println!(); - } - print_description(playlist.description.map(|d| d.to_plaintext())); - println!(); - print_tracks(&playlist.tracks.items); - } - } + let mut playlist = rp.query().music_playlist(&id).await.unwrap(); + playlist + .tracks + .extend_limit(rp.query(), limit) + .await + .unwrap(); + print_data(&playlist, format, pretty); } else { - let mut playlist = rp.query().playlist(&id).await?; - playlist.videos.extend_limit(rp.query(), limit).await?; - match format { - Some(format) => print_data(&playlist, format, pretty), - None => { - anstream::println!( - "{}\n{} [{}]\n{} {}", - "[Playlist]".on_green().black(), - playlist.name.green().bold(), - playlist.id, - "Videos:".blue(), - playlist.video_count, - ); - if let Some(n) = playlist.channel_name() { - anstream::print!("{} {}", "Author:".blue(), n.bold()); - if let Some(id) = playlist.channel_id() { - print!(" [{id}]"); - } - println!(); - } - if let Some(last_update) = playlist.last_update { - anstream::println!("{} {}", "Last update:".blue(), last_update); - } - print_description(playlist.description.map(|d| d.to_plaintext())); - println!(); - print_entities(&playlist.videos.items, false); - } - } + let mut playlist = rp.query().playlist(&id).await.unwrap(); + playlist + .videos + .extend_limit(rp.query(), limit) + .await + .unwrap(); + print_data(&playlist, format, pretty); } } UrlTarget::Album { id } => { - let album = rp.query().music_album(&id).await?; - match format { - Some(format) => print_data(&album, format, pretty), - None => { - anstream::print!( - "{}\n{} [{}] ({:?}", - "[Album]".on_green().black(), - album.name.green().bold(), - album.id, - album.album_type - ); - if let Some(year) = album.year { - print!(", {year}"); - } - println!(")"); - if let Some(n) = album.channel_name() { - anstream::print!("{} {}", "Artist:".blue(), n); - if let Some(id) = album.channel_id() { - print!(" [{id}]"); - } - } - print_description(album.description.map(|d| d.to_plaintext())); - println!(); - print_tracks(&album.tracks); - } - } + let album = rp.query().music_album(&id).await.unwrap(); + print_data(&album, format, pretty); } } } @@ -1495,29 +633,10 @@ async fn run() -> anyhow::Result<()> { music, } => match music { None => match channel { - Some(channel_id) => { - rustypipe::validate::channel_id(&channel_id)?; - let channel = rp.query().channel_search(&channel_id, &query).await?; - - match format { - Some(format) => print_data(&channel, format, pretty), - None => { - anstream::print!( - "{}\n{} [{}]", - "[Channel search]".on_green().black(), - channel.name.green().bold(), - channel.id - ); - print_verification(channel.verification); - println!(); - if let Some(subs) = channel.subscriber_count { - anstream::println!("{} {}", "Subscribers:".blue(), subs); - } - print_description(Some(channel.description)); - println!(); - print_entities(&channel.content.items, false); - } - } + Some(channel) => { + rustypipe::validate::channel_id(&channel).unwrap(); + let res = rp.query().channel_search(&channel, &query).await.unwrap(); + print_data(&res, format, pretty); } None => { let filter = search_filter::SearchFilter::new() @@ -1525,311 +644,58 @@ async fn run() -> anyhow::Result<()> { .length_opt(length.map(search_filter::Length::from)) .date_opt(date.map(search_filter::UploadDate::from)) .sort_opt(order.map(search_filter::Order::from)); - let mut res = rp - .query() - .search_filter::(&query, &filter) - .await?; - res.items.extend_limit(rp.query(), limit).await?; - - match format { - Some(format) => print_data(&res, format, pretty), - None => { - print_h1("Search"); - if let Some(corr) = res.corrected_query { - anstream::println!("Did you mean `{}`?", corr.magenta()); - } - print_entities(&res.items.items, false); - } - } + let mut res = rp.query().search_filter(&query, &filter).await.unwrap(); + res.items.extend_limit(rp.query(), limit).await.unwrap(); + print_data(&res, format, pretty); } }, Some(MusicSearchCategory::All) => { - let res = rp.query().music_search_main(&query).await?; - print_music_search(&res, format, pretty, true); + let res = rp.query().music_search(&query).await.unwrap(); + print_data(&res, format, pretty); } Some(MusicSearchCategory::Tracks) => { - let mut res = rp.query().music_search_tracks(&query).await?; - res.items.extend_limit(rp.query(), limit).await?; - print_music_search(&res, format, pretty, false); + let mut res = rp.query().music_search_tracks(&query).await.unwrap(); + res.items.extend_limit(rp.query(), limit).await.unwrap(); + print_data(&res, format, pretty); } Some(MusicSearchCategory::Videos) => { - let mut res = rp.query().music_search_videos(&query).await?; - res.items.extend_limit(rp.query(), limit).await?; - print_music_search(&res, format, pretty, false); + let mut res = rp.query().music_search_videos(&query).await.unwrap(); + res.items.extend_limit(rp.query(), limit).await.unwrap(); + print_data(&res, format, pretty); } Some(MusicSearchCategory::Artists) => { - let mut res = rp.query().music_search_artists(&query).await?; - res.items.extend_limit(rp.query(), limit).await?; - print_music_search(&res, format, pretty, false); + let mut res = rp.query().music_search_artists(&query).await.unwrap(); + res.items.extend_limit(rp.query(), limit).await.unwrap(); + print_data(&res, format, pretty); } Some(MusicSearchCategory::Albums) => { - let mut res = rp.query().music_search_albums(&query).await?; - res.items.extend_limit(rp.query(), limit).await?; - print_music_search(&res, format, pretty, false); + let mut res = rp.query().music_search_albums(&query).await.unwrap(); + res.items.extend_limit(rp.query(), limit).await.unwrap(); + print_data(&res, format, pretty); } - Some(MusicSearchCategory::PlaylistsYtm | MusicSearchCategory::PlaylistsCommunity) => { + Some(MusicSearchCategory::Playlists) => { + let mut res = rp.query().music_search_playlists(&query).await.unwrap(); + res.items.extend_limit(rp.query(), limit).await.unwrap(); + print_data(&res, format, pretty); + } + Some(MusicSearchCategory::PlaylistsYtm) => { let mut res = rp .query() - .music_search_playlists( - &query, - music == Some(MusicSearchCategory::PlaylistsCommunity), - ) - .await?; - res.items.extend_limit(rp.query(), limit).await?; - print_music_search(&res, format, pretty, false); + .music_search_playlists_filter(&query, false) + .await + .unwrap(); + res.items.extend_limit(rp.query(), limit).await.unwrap(); + print_data(&res, format, pretty); } - Some(MusicSearchCategory::Users) => { - let mut res = rp.query().music_search_users(&query).await?; - res.items.extend_limit(rp.query(), limit).await?; - print_music_search(&res, format, pretty, false); + Some(MusicSearchCategory::PlaylistsCommunity) => { + let mut res = rp + .query() + .music_search_playlists_filter(&query, true) + .await + .unwrap(); + res.items.extend_limit(rp.query(), limit).await.unwrap(); + print_data(&res, format, pretty); } }, - Commands::History { - format, - pretty, - limit, - music, - search, - } => { - if music { - let mut history = rp.query().music_history().await?; - history.extend_limit(rp.query(), limit).await?; - - match format { - Some(format) => print_data(&history, format, pretty), - None => { - anstream::println!("{}", "[Music history]".on_green().black()); - - let mut last_date = None; - for item in history.items { - if last_date != item.playback_date { - println!(); - if let Some(dt) = item.playback_date { - anstream::println!("{}", dt.green().underline()); - } - last_date = item.playback_date; - } - - let t = item.item; - anstream::print!("[{}] {} - ", t.id, t.name.bold()); - print_artists(&t.artists); - print_duration(t.duration); - println!(); - } - } - } - } else { - let mut history = match search { - Some(query) => rp.query().history_search(query).await?, - None => rp.query().history().await?, - }; - history.extend_limit(rp.query(), limit).await?; - - match format { - Some(format) => print_data(&history, format, pretty), - None => { - anstream::println!("{}", "[History]".on_green().black()); - - let mut last_date = None; - for item in history.items { - if last_date != item.playback_date { - println!(); - if let Some(dt) = item.playback_date { - anstream::println!("{}", dt.green().underline()); - } - last_date = item.playback_date; - } - print_entity(&item.item, false); - } - } - } - } - } - Commands::Subscriptions { - format, - pretty, - limit, - music, - feed, - } => { - if music { - let mut subscriptions = rp.query().music_saved_artists().await?; - subscriptions.extend_limit(rp.query(), limit).await?; - fmt_print_subscriptions( - &subscriptions.items, - format, - pretty, - "YouTube Music artists", - ); - } else if feed { - let mut feed = rp.query().subscription_feed().await?; - feed.extend_limit(rp.query(), limit).await?; - fmt_print_entities(&feed.items, format.map(Format::from), pretty, "Feed"); - } else { - let mut subscriptions = rp.query().subscriptions().await?; - subscriptions.extend_limit(rp.query(), limit).await?; - fmt_print_subscriptions( - &subscriptions.items, - format, - pretty, - "YouTube subscriptions", - ); - } - } - Commands::Playlists { - format, - pretty, - limit, - music, - } => { - if music { - let mut playlists = rp.query().music_saved_playlists().await?; - playlists.extend_limit(rp.query(), limit).await?; - fmt_print_entities(&playlists.items, format, pretty, "Music playlists"); - } else { - let mut playlists = rp.query().saved_playlists().await?; - playlists.extend_limit(rp.query(), limit).await?; - fmt_print_entities(&playlists.items, format, pretty, "Saved playlists"); - } - } - Commands::Albums { - format, - pretty, - limit, - } => { - let mut albums = rp.query().music_saved_albums().await?; - albums.extend_limit(rp.query(), limit).await?; - fmt_print_entities(&albums.items, format, pretty, "Saved albums"); - } - Commands::Tracks { - format, - pretty, - limit, - } => { - let mut tracks = rp.query().music_saved_tracks().await?; - tracks.extend_limit(rp.query(), limit).await?; - fmt_print_tracks(&tracks.items, format, pretty, "Saved tracks"); - } - Commands::Releases { - videos, - format, - pretty, - } => { - if videos { - let releases = rp.query().music_new_videos().await?; - fmt_print_tracks(&releases, format, pretty, "New music videos"); - } else { - let releases = rp.query().music_new_albums().await?; - fmt_print_entities(&releases, format, pretty, "New albums"); - } - } - Commands::Charts { - country, - format, - pretty, - } => { - let country = match country { - Some(c) => Some(Country::from_str(&c)?), - None => None, - }; - let charts = rp.query().music_charts(country).await?; - match format { - Some(format) => print_data(&charts, format, pretty), - None => { - print_h1("Music charts"); - if let Some(plid) = &charts.top_playlist_id { - print_h2(&format!("Top tracks [{plid}]")); - } else { - print_h2("Top tracks"); - } - print_tracks(&charts.top_tracks); - if let Some(plid) = &charts.trending_playlist_id { - print_h2(&format!("Trending [{plid}]")); - } else { - print_h2("Trending"); - } - print_tracks(&charts.trending_tracks); - print_h2("Artists"); - print_entities(&charts.artists, false); - - if !charts.playlists.is_empty() { - print_h2("Playlists"); - print_entities(&charts.playlists, false); - } - } - } - } - Commands::Vdata => { - let vd = rp.query().get_visitor_data(true).await?; - println!("{vd}"); - } - Commands::Login { - cookie, - cookies_txt, - } => { - if cookie || cookies_txt.is_some() { - match rp.user_auth_check_cookie().await { - Ok(_) => { - println!("Already logged in."); - } - Err(rustypipe::error::Error::Auth(_)) => { - let cookie_raw = if let Some(cookie_txt) = cookies_txt { - std::fs::read_to_string(cookie_txt)? - } else { - println!("Enter cookie header or cookies.txt:"); - - // Read until 2 consecutive newlines - let mut line = String::new(); - let mut last_len = 0; - let mut stop = 0; - while stop < 2 { - std::io::stdin().read_line(&mut line)?; - if line.len() <= last_len + 1 { - stop += 1; - } else { - stop = 0; - } - last_len = line.len(); - } - line - }; - - if cookie_raw.contains('\t') { - rp.user_auth_set_cookie_txt(&cookie_raw).await?; - } else { - rp.user_auth_set_cookie(cookie_raw.trim()).await?; - } - anstream::println!("{}", "Logged in.".green()); - } - Err(e) => return Err(e.into()), - } - } else { - match rp.user_auth_check_login().await { - Ok(_) => { - println!("Already logged in."); - } - Err(rustypipe::error::Error::Auth(_)) => { - let device_code = rp.user_auth_get_code().await?; - println!( - "Open {} and enter the following code:", - device_code.verification_url - ); - anstream::println!("{}", device_code.user_code.blue()); - rp.user_auth_wait_for_login(&device_code).await?; - anstream::println!("{}", "Logged in.".green()); - } - Err(e) => return Err(e.into()), - } - } - } - Commands::Logout { cookie } => { - if cookie { - rp.user_auth_remove_cookie().await?; - } else { - rp.user_auth_logout().await?; - } - anstream::println!("{}", "Logged out.".red()); - } }; - Ok(()) } diff --git a/cliff.toml b/cliff.toml deleted file mode 100644 index b84a719..0000000 --- a/cliff.toml +++ /dev/null @@ -1,100 +0,0 @@ -# git-cliff ~ default configuration file -# https://git-cliff.org/docs/configuration -# -# Lines starting with "#" are comments. -# Configuration options are organized into tables and keys. -# See documentation for more information on available options. - -[changelog] -# changelog header -header = """ -# Changelog\n -All notable changes to this project will be documented in this file.\n -""" -# template for the changelog body -# https://keats.github.io/tera/docs/#introduction -body = """ -{% set repo_url = "https://codeberg.org/ThetaDev/rustypipe" %}\ -{% if version %}\ - {%set vname = version | split(pat="/") | last %} - {%if previous.version %}\ - ## [{{ vname }}]({{ repo_url }}/compare/{{ previous.version }}..{{ version }})\ - {% else %}\ - ## [{{ vname }}]({{ repo_url }}/commits/tag/{{ version }})\ - {% endif %} - {{ timestamp | date(format="%Y-%m-%d") }} -{% else %}\ - ## [unreleased] -{% endif %}\ -{% if previous.version %}\ -{% for group, commits in commits | group_by(attribute="group") %} - ### {{ group | striptags | trim | upper_first }} - {% for commit in commits %} - - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\ - {% if commit.breaking %}[**breaking**] {% endif %}\ - {{ commit.message | upper_first }} - \ - ([{{ commit.id | truncate(length=7, end="") }}]({{ repo_url }}/commit/{{ commit.id }}))\ - {% endfor %} -{% endfor %}\ -{% else %} -Initial release -{% endif %}\n -""" -# template for the changelog footer -footer = """ - -""" -# remove the leading and trailing s -trim = true -# postprocessors -postprocessors = [ - # { pattern = '', replace = "https://github.com/orhun/git-cliff" }, # replace repository URL -] - -[git] -# parse the commits based on https://www.conventionalcommits.org -conventional_commits = true -# filter out the commits that are not conventional -filter_unconventional = true -# process each line of a commit as an individual commit -split_commits = false -# regex for preprocessing the commit messages -commit_preprocessors = [ - # Replace issue numbers - #{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](/issues/${2}))"}, - # Check spelling of the commit with https://github.com/crate-ci/typos - # If the spelling is incorrect, it will be automatically fixed. - #{ pattern = '.*', replace_command = 'typos --write-changes -' }, -] -# regex for parsing and grouping commits -commit_parsers = [ - { message = "^feat", group = "🚀 Features" }, - { message = "^fix", group = "🐛 Bug Fixes" }, - { message = "^doc", group = "📚 Documentation" }, - { message = "^perf", group = "⚡ Performance" }, - { message = "^refactor", group = "🚜 Refactor" }, - { message = "^style", group = "🎨 Styling" }, - { message = "^test", skip = true }, - { message = "^chore\\(release\\)", skip = true }, - { message = "^chore\\(pr\\)", skip = true }, - { message = "^chore\\(pull\\)", skip = true }, - { message = "^chore", group = "⚙️ Miscellaneous Tasks" }, - { message = "^ci", skip = true }, - { body = ".*security", group = "🛡️ Security" }, - { message = "^revert", group = "◀️ Revert" }, -] -# protect breaking changes from being skipped due to matching a skipping commit_parser -protect_breaking_commits = false -# filter out the commits that are not matched by commit parsers -filter_commits = false -# regex for matching git tags -# tag_pattern = "v[0-9].*" -# regex for skipping tags -# skip_tags = "" -# regex for ignoring tags -# ignore_tags = "" -# sort the tags topologically -topo_order = false -# sort the commits inside sections by oldest/newest order -sort_commits = "oldest" -# limit the number of commits included in the changelog. -# limit_commits = 42 diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 4b602d8..6f48494 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -1,33 +1,26 @@ [package] name = "rustypipe-codegen" version = "0.1.0" -rust-version = "1.74.0" -edition.workspace = true -authors.workspace = true -license.workspace = true -repository.workspace = true +edition = "2021" publish = false [dependencies] -rustypipe = { path = "../", features = ["userdata"] } -reqwest.workspace = true -tokio = { workspace = true, features = ["rt-multi-thread"] } -futures-util.workspace = true -serde.workspace = true -serde_json.workspace = true -serde_plain.workspace = true -serde_with.workspace = true -once_cell.workspace = true -regex.workspace = true -path_macro.workspace = true -anyhow.workspace = true -tracing.workspace = true -tracing-subscriber.workspace = true -clap.workspace = true -phf_codegen.workspace = true -indicatif.workspace = true - -num_enum = "0.7.2" +rustypipe = { path = "../" } +reqwest = "0.11.11" +tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] } +futures = "0.3.21" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0.82" +serde_with = { version = "3.0.0", default-features = false, features = ["macros"] } +anyhow = "1.0" +log = "0.4.17" +env_logger = "0.10.0" +clap = { version = "4.0.29", features = ["derive"] } +phf_codegen = "0.11.1" +once_cell = "1.12.0" +regex = "1.7.1" +indicatif = "0.17.0" +num_enum = "0.6.1" +path_macro = "1.0.0" intl_pluralrules = "7.0.2" unic-langid = "0.9.1" -ordered_hash_map = { version = "0.4.0", features = ["serde"] } diff --git a/codegen/src/abtest.rs b/codegen/src/abtest.rs index de8001a..ac13a8c 100644 --- a/codegen/src/abtest.rs +++ b/codegen/src/abtest.rs @@ -1,20 +1,15 @@ use std::collections::BTreeMap; use anyhow::{bail, Result}; -use futures_util::{stream, StreamExt}; +use futures::{stream, StreamExt}; use indicatif::{ProgressBar, ProgressStyle}; use num_enum::TryFromPrimitive; -use once_cell::sync::Lazy; -use regex::Regex; -use rustypipe::client::{ClientType, RustyPipe, RustyPipeQuery}; -use rustypipe::model::{MusicItem, YouTubeItem}; +use rustypipe::client::{ClientType, RustyPipe, RustyPipeQuery, YTContext}; +use rustypipe::model::YouTubeItem; use rustypipe::param::search_filter::{ItemType, SearchFilter}; -use rustypipe::param::ChannelVideoTab; use serde::de::IgnoredAny; use serde::{Deserialize, Serialize}; -use crate::model::QCont; - #[derive( Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, TryFromPrimitive, Serialize, Deserialize, )] @@ -26,29 +21,12 @@ pub enum ABTest { TrendsVideoTab = 4, TrendsPageHeaderRenderer = 5, DiscographyPage = 6, - ShortDateFormat = 7, - TrackViewcount = 8, - PlaylistsForShorts = 9, - ChannelAboutModal = 10, - LikeButtonViewmodel = 11, - ChannelPageHeader = 12, - MusicPlaylistTwoColumn = 13, - CommentsFrameworkUpdate = 14, - ChannelShortsLockup = 15, - PlaylistPageHeader = 16, - ChannelPlaylistsLockup = 17, - MusicPlaylistFacepile = 18, - MusicAlbumGroupsReordered = 19, - MusicContinuationItemRenderer = 20, - AlbumRecommends = 21, - CommandExecutorCommand = 22, } -/// List of active A/B tests that are run when none is manually specified -const TESTS_TO_RUN: &[ABTest] = &[ - ABTest::MusicAlbumGroupsReordered, - ABTest::AlbumRecommends, - ABTest::CommandExecutorCommand, +const TESTS_TO_RUN: [ABTest; 3] = [ + ABTest::TrendsVideoTab, + ABTest::TrendsPageHeaderRenderer, + ABTest::DiscographyPage, ]; #[derive(Debug, Serialize, Deserialize)] @@ -63,6 +41,7 @@ pub struct ABTestRes { #[derive(Debug, Serialize)] struct QVideo<'a> { + context: YTContext<'a>, video_id: &'a str, content_check_ok: bool, racy_check_ok: bool, @@ -71,6 +50,7 @@ struct QVideo<'a> { #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QBrowse<'a> { + context: YTContext<'a>, browse_id: &'a str, #[serde(skip_serializing_if = "Option::is_none")] params: Option<&'a str>, @@ -85,6 +65,7 @@ pub async fn run_test( let rp = RustyPipe::new(); let pb = ProgressBar::new(n as u64); + let http = reqwest::Client::default(); pb.set_style( ProgressStyle::with_template( "{msg} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {pos}/{len}", @@ -96,8 +77,9 @@ pub async fn run_test( .map(|_| { let rp = rp.clone(); let pb = pb.clone(); + let http = http.clone(); async move { - let visitor_data = rp.query().get_visitor_data(true).await.unwrap(); + let visitor_data = get_visitor_data(&http).await; let query = rp.query().visitor_data(&visitor_data); let is_present = match ab { ABTest::AttributedTextDescription => attributed_text_description(&query).await, @@ -108,24 +90,6 @@ pub async fn run_test( ABTest::TrendsVideoTab => trends_video_tab(&query).await, ABTest::TrendsPageHeaderRenderer => trends_page_header_renderer(&query).await, ABTest::DiscographyPage => discography_page(&query).await, - ABTest::ShortDateFormat => short_date_format(&query).await, - ABTest::PlaylistsForShorts => playlists_for_shorts(&query).await, - ABTest::TrackViewcount => track_viewcount(&query).await, - ABTest::ChannelAboutModal => channel_about_modal(&query).await, - ABTest::LikeButtonViewmodel => like_button_viewmodel(&query).await, - ABTest::ChannelPageHeader => channel_page_header(&query).await, - ABTest::MusicPlaylistTwoColumn => music_playlist_two_column(&query).await, - ABTest::CommentsFrameworkUpdate => comments_framework_update(&query).await, - ABTest::ChannelShortsLockup => channel_shorts_lockup(&query).await, - ABTest::PlaylistPageHeader => playlist_page_header_renderer(&query).await, - ABTest::ChannelPlaylistsLockup => channel_playlists_lockup(&query).await, - ABTest::MusicPlaylistFacepile => music_playlist_facepile(&query).await, - ABTest::MusicAlbumGroupsReordered => music_album_groups_reordered(&query).await, - ABTest::MusicContinuationItemRenderer => { - music_continuation_item_renderer(&query).await - } - ABTest::AlbumRecommends => album_recommends(&query).await, - ABTest::CommandExecutorCommand => command_executor_command(&query).await, } .unwrap(); pb.inc(1); @@ -147,14 +111,30 @@ pub async fn run_test( (count, vd_present, vd_absent) } +async fn get_visitor_data(http: &reqwest::Client) -> String { + let resp = http.get("https://www.youtube.com").send().await.unwrap(); + resp.headers() + .get_all(reqwest::header::SET_COOKIE) + .iter() + .find_map(|c| { + if let Ok(cookie) = c.to_str() { + if let Some(after) = cookie.strip_prefix("__Secure-YEC=") { + return after.split_once(';').map(|s| s.0.to_owned()); + } + } + None + }) + .unwrap() +} + pub async fn run_all_tests(n: usize, concurrency: usize) -> Vec { let mut results = Vec::new(); for ab in TESTS_TO_RUN { - let (occurrences, vd_present, vd_absent) = run_test(*ab, n, concurrency).await; + let (occurrences, vd_present, vd_absent) = run_test(ab, n, concurrency).await; results.push(ABTestRes { - id: *ab as u16, - name: *ab, + id: ab as u16, + name: ab, tests: n, occurrences, vd_present, @@ -165,12 +145,14 @@ pub async fn run_all_tests(n: usize, concurrency: usize) -> Vec { } pub async fn attributed_text_description(rp: &RustyPipeQuery) -> Result { + let context = rp.get_context(ClientType::Desktop, true, None).await; let q = QVideo { + context, video_id: "ZeerrnuLi5E", content_check_ok: false, racy_check_ok: false, }; - let response_txt = rp.raw(ClientType::Desktop, "next", &q).await?; + let response_txt = rp.raw(ClientType::Desktop, "next", &q).await.unwrap(); if !response_txt.contains("\"Black Mamba\"") { bail!("invalid response data"); @@ -180,7 +162,7 @@ pub async fn attributed_text_description(rp: &RustyPipeQuery) -> Result { } pub async fn three_tab_channel_layout(rp: &RustyPipeQuery) -> Result { - let channel = rp.channel_videos("UCR-DXc1voovS8nhAvccRZhg").await?; + let channel = rp.channel_videos("UCR-DXc1voovS8nhAvccRZhg").await.unwrap(); Ok(channel.has_live || channel.has_shorts) } @@ -193,18 +175,20 @@ pub async fn channel_handles_in_search_results(rp: &RustyPipeQuery) -> Result channel .subscriber_count - .map(|sc| sc > 100 && channel.handle.is_some()) + .map(|sc| sc > 100 && channel.video_count.is_none()) .unwrap_or_default(), _ => false, })) } pub async fn trends_video_tab(rp: &RustyPipeQuery) -> Result { + let context = rp.get_context(ClientType::Desktop, true, None).await; let res = rp .raw( ClientType::Desktop, "browse", &QBrowse { + context, browse_id: "FEtrending", params: None, }, @@ -215,11 +199,13 @@ pub async fn trends_video_tab(rp: &RustyPipeQuery) -> Result { } pub async fn trends_page_header_renderer(rp: &RustyPipeQuery) -> Result { + let context = rp.get_context(ClientType::Desktop, true, None).await; let res = rp .raw( ClientType::Desktop, "browse", &QBrowse { + context, browse_id: "FEtrending", params: None, }, @@ -237,244 +223,10 @@ pub async fn trends_page_header_renderer(rp: &RustyPipeQuery) -> Result { } pub async fn discography_page(rp: &RustyPipeQuery) -> Result { - let id = "UC7cl4MmM6ZZ2TcFyMk_b4pg"; - let res = rp - .raw( - ClientType::DesktopMusic, - "browse", - &QBrowse { - browse_id: id, - params: None, - }, - ) - .await?; - Ok(res.contains(&format!("\"MPAD{id}\""))) -} - -pub async fn short_date_format(rp: &RustyPipeQuery) -> Result { - static SHORT_DATE: Lazy = Lazy::new(|| Regex::new("\\d(?:y|mo|w|d|h|min) ").unwrap()); - let channel = rp.channel_videos("UC2DjFE7Xf11URZqWBigcVOQ").await?; - - Ok(channel.content.items.iter().any(|itm| { - itm.publish_date_txt - .as_deref() - .map(|d| SHORT_DATE.is_match(d)) - .unwrap_or_default() - })) -} - -pub async fn playlists_for_shorts(rp: &RustyPipeQuery) -> Result { - let playlist = rp.playlist("UUSHh8gHdtzO2tXd593_bjErWg").await?; - let v1 = playlist - .videos - .items - .first() - .ok_or_else(|| anyhow::anyhow!("no videos"))?; - Ok(v1.publish_date_txt.is_none()) -} - -pub async fn track_viewcount(rp: &RustyPipeQuery) -> Result { - let res = rp.music_search_main("lieblingsmensch namika").await?; - - let track = &res - .items - .items - .iter() - .find_map(|itm| { - if let MusicItem::Track(track) = itm { - if track.id == "6485PhOtHzY" { - Some(track) - } else { - None - } - } else { - None - } - }) - .unwrap_or_else(|| { - panic!("could not find track, got {:#?}", &res.items.items); - }); - - Ok(track.view_count.is_some()) -} - -pub async fn channel_about_modal(rp: &RustyPipeQuery) -> Result { - let id = "UC2DjFE7Xf11URZqWBigcVOQ"; - let res = rp - .raw( - ClientType::Desktop, - "browse", - &QBrowse { - browse_id: id, - params: None, - }, - ) - .await?; - Ok(!res.contains("\"EgVhYm91dPIGBAoCEgA%3D\"")) -} - -pub async fn like_button_viewmodel(rp: &RustyPipeQuery) -> Result { - let res = rp - .raw( - ClientType::Desktop, - "next", - &QVideo { - video_id: "ZeerrnuLi5E", - content_check_ok: true, - racy_check_ok: true, - }, - ) - .await?; - Ok(res.contains("\"segmentedLikeDislikeButtonViewModel\"")) -} - -pub async fn channel_page_header(rp: &RustyPipeQuery) -> Result { - let channel = rp - .channel_videos_tab("UCh8gHdtzO2tXd593_bjErWg", ChannelVideoTab::Shorts) - .await?; - Ok(channel.video_count.is_some()) -} - -pub async fn music_playlist_two_column(rp: &RustyPipeQuery) -> Result { - let id = "VLRDCLAK5uy_kb7EBi6y3GrtJri4_ZH56Ms786DFEimbM"; - let res = rp - .raw( - ClientType::DesktopMusic, - "browse", - &QBrowse { - browse_id: id, - params: None, - }, - ) - .await?; - Ok(res.contains("\"musicResponsiveHeaderRenderer\"")) -} - -pub async fn comments_framework_update(rp: &RustyPipeQuery) -> Result { - let continuation = - "Eg0SC3dMZHBSN2d1S3k4GAYyJSIRIgt3TGRwUjdndUt5ODAAeAJCEGNvbW1lbnRzLXNlY3Rpb24%3D"; - let res = rp - .raw(ClientType::Desktop, "next", &QCont { continuation }) - .await?; - Ok(res.contains("\"frameworkUpdates\"")) -} - -pub async fn channel_shorts_lockup(rp: &RustyPipeQuery) -> Result { - let id = "UCh8gHdtzO2tXd593_bjErWg"; - let res = rp - .raw( - ClientType::Desktop, - "browse", - &QBrowse { - browse_id: id, - params: Some("EgZzaG9ydHPyBgUKA5oBAA%3D%3D"), - }, - ) - .await?; - Ok(res.contains("\"shortsLockupViewModel\"")) -} - -pub async fn playlist_page_header_renderer(rp: &RustyPipeQuery) -> Result { - let id = "VLPLZN_exA7d4RVmCQrG5VlWIjMOkMFZVVOc"; - let res = rp - .raw( - ClientType::Desktop, - "browse", - &QBrowse { - browse_id: id, - params: None, - }, - ) - .await?; - Ok(res.contains("\"pageHeaderRenderer\"")) -} - -pub async fn channel_playlists_lockup(rp: &RustyPipeQuery) -> Result { - let id = "UC2DjFE7Xf11URZqWBigcVOQ"; - let res = rp - .raw( - ClientType::Desktop, - "browse", - &QBrowse { - browse_id: id, - params: Some("EglwbGF5bGlzdHMgAQ%3D%3D"), - }, - ) - .await?; - Ok(res.contains("\"lockupViewModel\"")) -} - -pub async fn music_playlist_facepile(rp: &RustyPipeQuery) -> Result { - let id = "VLPL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe"; - let res = rp - .raw( - ClientType::DesktopMusic, - "browse", - &QBrowse { - browse_id: id, - params: None, - }, - ) - .await?; - Ok(res.contains("\"facepile\"")) -} - -pub async fn music_album_groups_reordered(rp: &RustyPipeQuery) -> Result { - let id = "UCOR4_bSVIXPsGa4BbCSt60Q"; - let res = rp - .raw( - ClientType::DesktopMusic, - "browse", - &QBrowse { - browse_id: id, - params: None, - }, - ) - .await?; - Ok(res.contains("\"Singles & EPs\"")) -} - -pub async fn music_continuation_item_renderer(rp: &RustyPipeQuery) -> Result { - let id = "VLPLbZIPy20-1pN7mqjckepWF78ndb6ci_qi"; - let res = rp - .raw( - ClientType::DesktopMusic, - "browse", - &QBrowse { - browse_id: id, - params: None, - }, - ) - .await?; - Ok(res.contains("\"continuationItemRenderer\"")) -} - -pub async fn album_recommends(rp: &RustyPipeQuery) -> Result { - let id = "MPREb_u1I69lSAe5v"; - let res = rp - .raw( - ClientType::DesktopMusic, - "browse", - &QBrowse { - browse_id: id, - params: None, - }, - ) - .await?; - Ok(res.contains("\"musicCarouselShelfRenderer\"")) -} - -pub async fn command_executor_command(rp: &RustyPipeQuery) -> Result { - let id = "VLPLbZIPy20-1pN7mqjckepWF78ndb6ci_qi"; - let res = rp - .raw( - ClientType::Desktop, - "browse", - &QBrowse { - browse_id: id, - params: None, - }, - ) - .await?; - Ok(res.contains("\"commandExecutorCommand\"")) + let artist = rp + .music_artist("UC7cl4MmM6ZZ2TcFyMk_b4pg", false) + .await + .unwrap(); + + Ok(artist.albums.len() <= 10) } diff --git a/codegen/src/collect_album_types.rs b/codegen/src/collect_album_types.rs index b5ac912..49fb56a 100644 --- a/codegen/src/collect_album_types.rs +++ b/codegen/src/collect_album_types.rs @@ -1,41 +1,28 @@ use std::{collections::BTreeMap, fs::File, io::BufReader}; -use futures_util::stream::{self, StreamExt}; +use futures::stream::{self, StreamExt}; use path_macro::path; use rustypipe::{ client::{ClientType, RustyPipe, RustyPipeQuery}, model::AlbumType, param::{Language, LANGUAGES}, }; -use serde::{Deserialize, Serialize}; -use serde_with::rust::deserialize_ignore_any; +use serde::Deserialize; use crate::{ - model::{ContentsRenderer, QBrowse, SectionList, Tab, TextRuns}, + model::{QBrowse, TextRuns}, util::{self, DICT_DIR}, }; -#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -#[serde(rename_all = "snake_case")] -enum AlbumTypeX { - Album, - Ep, - Single, - Audiobook, - Show, - AlbumRow, - SingleRow, -} - pub async fn collect_album_types(concurrency: usize) { let json_path = path!(*DICT_DIR / "album_type_samples.json"); let album_types = [ - (AlbumTypeX::Album, "MPREb_nlBWQROfvjo"), - (AlbumTypeX::Single, "MPREb_bHfHGoy7vuv"), - (AlbumTypeX::Ep, "MPREb_u1I69lSAe5v"), - (AlbumTypeX::Audiobook, "MPREb_gaoNzsQHedo"), - (AlbumTypeX::Show, "MPREb_cwzk8EUwypZ"), + (AlbumType::Album, "MPREb_nlBWQROfvjo"), + (AlbumType::Single, "MPREb_bHfHGoy7vuv"), + (AlbumType::Ep, "MPREb_u1I69lSAe5v"), + (AlbumType::Audiobook, "MPREb_gaoNzsQHedo"), + (AlbumType::Show, "MPREb_cwzk8EUwypZ"), ]; let rp = RustyPipe::new(); @@ -45,7 +32,7 @@ pub async fn collect_album_types(concurrency: usize) { let rp = rp.clone(); async move { let query = rp.query().lang(lang); - let mut data: BTreeMap = BTreeMap::new(); + let mut data: BTreeMap = BTreeMap::new(); for (album_type, id) in album_types { let atype_txt = get_album_type(&query, id).await; @@ -53,22 +40,6 @@ pub async fn collect_album_types(concurrency: usize) { data.insert(album_type, atype_txt); } - let (albums_txt, singles_txt) = get_album_groups(&query).await; - println!( - "collected {}-{:?} ({})", - lang, - AlbumTypeX::AlbumRow, - &albums_txt - ); - println!( - "collected {}-{:?} ({})", - lang, - AlbumTypeX::SingleRow, - &singles_txt - ); - data.insert(AlbumTypeX::AlbumRow, albums_txt); - data.insert(AlbumTypeX::SingleRow, singles_txt); - (lang, data) } }) @@ -84,7 +55,7 @@ pub fn write_samples_to_dict() { let json_path = path!(*DICT_DIR / "album_type_samples.json"); let json_file = File::open(json_path).unwrap(); - let collected: BTreeMap> = + let collected: BTreeMap> = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let mut dict = util::read_dict(); let langs = dict.keys().copied().collect::>(); @@ -96,12 +67,10 @@ pub fn write_samples_to_dict() { e_langs.push(lang); for lang in &e_langs { - collected.get(lang).unwrap().iter().for_each(|(t_str, v)| { - let t = - serde_plain::from_str::(t_str.split('_').next().unwrap()).unwrap(); + collected.get(lang).unwrap().iter().for_each(|(t, v)| { dict_entry .album_types - .insert(v.to_lowercase().trim().to_owned(), t); + .insert(v.to_lowercase().trim().to_owned(), *t); }); } } @@ -111,19 +80,13 @@ pub fn write_samples_to_dict() { #[derive(Debug, Deserialize)] struct AlbumData { - contents: AlbumContents, + header: Header, } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] -struct AlbumContents { - two_column_browse_results_renderer: ContentsRenderer>>, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AlbumHeader { - music_responsive_header_renderer: HeaderRenderer, +struct Header { + music_detail_header_renderer: HeaderRenderer, } #[derive(Debug, Deserialize)] @@ -132,7 +95,11 @@ struct HeaderRenderer { } async fn get_album_type(query: &RustyPipeQuery, id: &str) -> String { + let context = query + .get_context(ClientType::DesktopMusic, true, None) + .await; let body = QBrowse { + context, browse_id: id, params: None, }; @@ -143,20 +110,8 @@ async fn get_album_type(query: &RustyPipeQuery, id: &str) -> String { let album = serde_json::from_str::(&response_txt).unwrap(); album - .contents - .two_column_browse_results_renderer - .contents - .into_iter() - .next() - .unwrap() - .tab_renderer - .content - .section_list_renderer - .contents - .into_iter() - .next() - .unwrap() - .music_responsive_header_renderer + .header + .music_detail_header_renderer .subtitle .runs .into_iter() @@ -164,84 +119,3 @@ async fn get_album_type(query: &RustyPipeQuery, id: &str) -> String { .unwrap() .text } - -async fn get_album_groups(query: &RustyPipeQuery) -> (String, String) { - let body = QBrowse { - browse_id: "UCOR4_bSVIXPsGa4BbCSt60Q", - params: None, - }; - let response_txt = query - .clone() - .visitor_data("CgtwbzJZcS1XZWc1QSjM2JG8BjIKCgJERRIEEgAgCw%3D%3D") - .raw(ClientType::DesktopMusic, "browse", &body) - .await - .unwrap(); - let artist = serde_json::from_str::(&response_txt).unwrap(); - - let sections = artist - .contents - .single_column_browse_results_renderer - .contents - .into_iter() - .next() - .map(|c| c.tab_renderer.content.section_list_renderer.contents) - .unwrap(); - let titles = sections - .into_iter() - .filter_map(|s| { - if let ItemSection::MusicCarouselShelfRenderer(r) = s { - r.header - } else { - None - } - }) - .map(|h| { - h.music_carousel_shelf_basic_header_renderer - .title - .runs - .into_iter() - .next() - .unwrap() - .text - }) - .collect::>(); - assert!(titles.len() >= 2, "too few sections"); - - let mut titles_it = titles.into_iter(); - (titles_it.next().unwrap(), titles_it.next().unwrap()) -} - -#[derive(Debug, Deserialize)] -struct ArtistData { - contents: ArtistDataContents, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct ArtistDataContents { - single_column_browse_results_renderer: ContentsRenderer>>, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -enum ItemSection { - MusicCarouselShelfRenderer(MusicCarouselShelf), - #[serde(other, deserialize_with = "deserialize_ignore_any")] - None, -} - -#[derive(Debug, Deserialize)] -struct MusicCarouselShelf { - header: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct MusicCarouselShelfHeader { - music_carousel_shelf_basic_header_renderer: MusicCarouselShelfHeaderRenderer, -} - -#[derive(Debug, Deserialize)] -struct MusicCarouselShelfHeaderRenderer { - title: TextRuns, -} diff --git a/codegen/src/collect_album_versions_titles.rs b/codegen/src/collect_album_versions_titles.rs deleted file mode 100644 index 0cb513c..0000000 --- a/codegen/src/collect_album_versions_titles.rs +++ /dev/null @@ -1,130 +0,0 @@ -use std::{collections::BTreeMap, fs::File, io::BufReader}; - -use path_macro::path; -use rustypipe::{ - client::{ClientType, RustyPipe}, - param::{Language, LANGUAGES}, -}; -use serde::Deserialize; -use serde_with::rust::deserialize_ignore_any; - -use crate::{ - model::{QBrowse, SectionList, TextRuns}, - util::{self, DICT_DIR}, -}; - -pub async fn collect_album_versions_titles() { - let json_path = path!(*DICT_DIR / "other_versions_titles.json"); - let mut res = BTreeMap::new(); - - let rp = RustyPipe::new(); - - for lang in LANGUAGES { - let query = QBrowse { - browse_id: "MPREb_nlBWQROfvjo", - params: None, - }; - let raw_resp = rp - .query() - .lang(lang) - .raw(ClientType::DesktopMusic, "browse", &query) - .await - .unwrap(); - let data = serde_json::from_str::(&raw_resp).unwrap(); - let title = data - .contents - .two_column_browse_results_renderer - .secondary_contents - .section_list_renderer - .contents - .into_iter() - .find_map(|x| match x { - ItemSection::MusicCarouselShelfRenderer(music_carousel_shelf) => { - Some(music_carousel_shelf) - } - ItemSection::None => None, - }) - .expect("other versions") - .header - .expect("header") - .music_carousel_shelf_basic_header_renderer - .title - .runs - .into_iter() - .next() - .unwrap() - .text; - println!("{lang}: {title}"); - res.insert(lang, title); - } - - let file = File::create(json_path).unwrap(); - serde_json::to_writer_pretty(file, &res).unwrap(); -} - -pub fn write_samples_to_dict() { - let json_path = path!(*DICT_DIR / "other_versions_titles.json"); - let json_file = File::open(json_path).unwrap(); - let collected: BTreeMap = - serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let mut dict = util::read_dict(); - let langs = dict.keys().copied().collect::>(); - - for lang in langs { - let dict_entry = dict.entry(lang).or_default(); - - let e = collected.get(&lang).unwrap(); - assert_eq!(e, e.trim()); - dict_entry.album_versions_title = e.to_owned(); - - for lang in &dict_entry.equivalent { - let ee = collected.get(lang).unwrap(); - if ee != e { - panic!("equivalent lang conflict, lang: {lang}"); - } - } - } - - util::write_dict(dict); -} - -#[derive(Debug, Deserialize)] -struct AlbumData { - contents: AlbumDataContents, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct AlbumDataContents { - two_column_browse_results_renderer: X1, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct X1 { - secondary_contents: SectionList, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -enum ItemSection { - MusicCarouselShelfRenderer(MusicCarouselShelf), - #[serde(other, deserialize_with = "deserialize_ignore_any")] - None, -} - -#[derive(Debug, Deserialize)] -struct MusicCarouselShelf { - header: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct MusicCarouselShelfHeader { - music_carousel_shelf_basic_header_renderer: MusicCarouselShelfHeaderRenderer, -} - -#[derive(Debug, Deserialize)] -struct MusicCarouselShelfHeaderRenderer { - title: TextRuns, -} diff --git a/codegen/src/collect_chan_prefixes.rs b/codegen/src/collect_chan_prefixes.rs deleted file mode 100644 index 0d15bba..0000000 --- a/codegen/src/collect_chan_prefixes.rs +++ /dev/null @@ -1,75 +0,0 @@ -use std::{collections::BTreeMap, fs::File, io::BufReader}; - -use path_macro::path; -use rustypipe::{ - client::RustyPipe, - param::{Language, LANGUAGES}, -}; -use serde::{Deserialize, Serialize}; - -use crate::util::{self, DICT_DIR}; - -#[derive(Debug, Serialize, Deserialize)] -struct Entry { - prefix: String, - suffix: String, -} - -pub async fn collect_chan_prefixes() { - let cname = "kiernanchrisman"; - let json_path = path!(*DICT_DIR / "chan_prefixes.json"); - let mut res = BTreeMap::new(); - - let rp = RustyPipe::new(); - - for lang in LANGUAGES { - let playlist = rp - .query() - .lang(lang) - .playlist("PLZN_exA7d4RVmCQrG5VlWIjMOkMFZVVOc") - .await - .unwrap(); - let n = playlist.channel.unwrap().name; - let offset = n.find(cname).unwrap(); - let prefix = &n[..offset]; - let suffix = &n[(offset + cname.len())..]; - - res.insert( - lang, - Entry { - prefix: prefix.to_owned(), - suffix: suffix.to_owned(), - }, - ); - } - - let file = File::create(json_path).unwrap(); - serde_json::to_writer_pretty(file, &res).unwrap(); -} - -pub fn write_samples_to_dict() { - let json_path = path!(*DICT_DIR / "chan_prefixes.json"); - - let json_file = File::open(json_path).unwrap(); - let collected: BTreeMap = - serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let mut dict = util::read_dict(); - let langs = dict.keys().copied().collect::>(); - - for lang in langs { - let dict_entry = dict.entry(lang).or_default(); - - let e = collected.get(&lang).unwrap(); - dict_entry.chan_prefix = e.prefix.trim().to_owned(); - dict_entry.chan_suffix = e.suffix.trim().to_owned(); - - for lang in &dict_entry.equivalent { - let ee = collected.get(lang).unwrap(); - if ee.prefix != e.prefix || ee.suffix != e.suffix { - panic!("equivalent lang conflict, lang: {lang}"); - } - } - } - - util::write_dict(dict); -} diff --git a/codegen/src/collect_history_dates.rs b/codegen/src/collect_history_dates.rs deleted file mode 100644 index 838311b..0000000 --- a/codegen/src/collect_history_dates.rs +++ /dev/null @@ -1,110 +0,0 @@ -use std::{collections::BTreeMap, fs::File, io::BufReader}; - -use path_macro::path; -use rustypipe::{ - client::RustyPipe, - param::{Language, LANGUAGES}, -}; - -use crate::util::{self, DICT_DIR}; - -type CollectedDates = BTreeMap>; - -const THIS_WEEK: &str = "this_week"; -const LAST_WEEK: &str = "last_week"; - -pub async fn collect_dates_music() { - let json_path = path!(*DICT_DIR / "history_date_samples.json"); - let rp = RustyPipe::builder() - .storage_dir(path!(env!("CARGO_MANIFEST_DIR") / "..")) - .build() - .unwrap(); - - let mut res: CollectedDates = { - let json_file = File::open(&json_path).unwrap(); - serde_json::from_reader(BufReader::new(json_file)).unwrap() - }; - - for lang in LANGUAGES { - println!("{lang}"); - let history = rp.query().lang(lang).music_history().await.unwrap(); - if history.items.len() < 3 { - panic!("{lang} empty history") - } - - // The indexes have to be adapted before running - let entry = res.entry(lang).or_default(); - entry.insert( - THIS_WEEK.to_owned(), - history.items[0].playback_date_txt.clone().unwrap(), - ); - entry.insert( - LAST_WEEK.to_owned(), - history.items[18].playback_date_txt.clone().unwrap(), - ); - } - - let file = File::create(&json_path).unwrap(); - serde_json::to_writer_pretty(file, &res).unwrap(); -} - -pub async fn collect_dates() { - let json_path = path!(*DICT_DIR / "history_date_samples.json"); - let rp = RustyPipe::builder() - .storage_dir(path!(env!("CARGO_MANIFEST_DIR") / "..")) - .build() - .unwrap(); - - let mut res: CollectedDates = { - let json_file = File::open(&json_path).unwrap(); - serde_json::from_reader(BufReader::new(json_file)).unwrap() - }; - - for lang in LANGUAGES { - println!("{lang}"); - let history = rp.query().lang(lang).history().await.unwrap(); - if history.items.len() < 3 { - panic!("{lang} empty history") - } - - let entry = res.entry(lang).or_default(); - entry.insert( - "tuesday".to_owned(), - history.items[0].playback_date_txt.clone().unwrap(), - ); - entry.insert( - "0000-01-06".to_owned(), - history.items[1].playback_date_txt.clone().unwrap(), - ); - entry.insert( - "2024-12-28".to_owned(), - history.items[15].playback_date_txt.clone().unwrap(), - ); - } - - let file = File::create(&json_path).unwrap(); - serde_json::to_writer_pretty(file, &res).unwrap(); -} - -pub fn write_samples_to_dict() { - let json_path = path!(*DICT_DIR / "history_date_samples.json"); - - let json_file = File::open(json_path).unwrap(); - let collected_dates: CollectedDates = - serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let mut dict = util::read_dict(); - let langs = dict.keys().copied().collect::>(); - - for lang in langs { - let dict_entry = dict.entry(lang).or_default(); - let cd = &collected_dates[&lang]; - dict_entry - .timeago_nd_tokens - .insert(util::filter_datestr(&cd[THIS_WEEK]), "0Wl".to_owned()); - dict_entry - .timeago_nd_tokens - .insert(util::filter_datestr(&cd[LAST_WEEK]), "1Wl".to_owned()); - } - - util::write_dict(dict); -} diff --git a/codegen/src/collect_large_numbers.rs b/codegen/src/collect_large_numbers.rs index b8bb08e..59979ef 100644 --- a/codegen/src/collect_large_numbers.rs +++ b/codegen/src/collect_large_numbers.rs @@ -6,7 +6,7 @@ use std::{ }; use anyhow::{Context, Result}; -use futures_util::{stream, StreamExt}; +use futures::{stream, StreamExt}; use once_cell::sync::Lazy; use path_macro::path; use regex::Regex; @@ -350,6 +350,7 @@ async fn get_channel(query: &RustyPipeQuery, channel_id: &str) -> Result Result ClientType::DesktopMusic, "browse", &QBrowse { + context: query + .get_context(ClientType::DesktopMusic, true, None) + .await, browse_id: channel_id, params: None, }, diff --git a/codegen/src/collect_playlist_dates.rs b/codegen/src/collect_playlist_dates.rs index 4dbc782..4a87d36 100644 --- a/codegen/src/collect_playlist_dates.rs +++ b/codegen/src/collect_playlist_dates.rs @@ -5,8 +5,7 @@ use std::{ io::BufReader, }; -use futures_util::{stream, StreamExt}; -use ordered_hash_map::OrderedHashMap; +use futures::{stream, StreamExt}; use path_macro::path; use rustypipe::{ client::RustyPipe, @@ -66,9 +65,9 @@ pub async fn collect_dates(concurrency: usize) { // These are the sample playlists let cases = [ - (DateCase::Today, "PL3oW2tjiIxvQ98ZTLhBh5soCbE1mC3uAT"), - (DateCase::Yesterday, "PLGBuKfnErZlCkRRgt06em8nbXvcV5Sae7"), - (DateCase::Ago, "PLAQ7nLSEnhWTEihjeM1I-ToPDJEKfZHZu"), + (DateCase::Today, "PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj"), + (DateCase::Yesterday, "PL3oW2tjiIxvQ98ZTLhBh5soCbE1mC3uAT"), + (DateCase::Ago, "PLeDakahyfrO9Amk2GFrzpI4UWOkgqzoIE"), (DateCase::Jan, "PL1J-6JOckZtFjcni6Xj1pLYglJp6JCpKD"), (DateCase::Feb, "PL1J-6JOckZtETrbzwZE7mRIIK6BzWNLAs"), (DateCase::Mar, "PL1J-6JOckZtG3AVdvBXhMO64mB2k3BtKi"), @@ -171,7 +170,7 @@ pub fn write_samples_to_dict() { dict_entry.months = BTreeMap::new(); if collect_nd_tokens { - dict_entry.timeago_nd_tokens = OrderedHashMap::new(); + dict_entry.timeago_nd_tokens = BTreeMap::new(); } for datestr_table in &datestr_tables { diff --git a/codegen/src/collect_video_dates.rs b/codegen/src/collect_video_dates.rs deleted file mode 100644 index 38a157a..0000000 --- a/codegen/src/collect_video_dates.rs +++ /dev/null @@ -1,84 +0,0 @@ -use std::{ - collections::{BTreeMap, HashSet}, - fs::File, -}; - -use futures_util::{stream, StreamExt}; -use path_macro::path; -use rustypipe::{ - client::{RustyPipe, RustyPipeQuery}, - param::{Language, LANGUAGES}, -}; - -use crate::util::DICT_DIR; - -pub async fn collect_video_dates(concurrency: usize) { - let json_path = path!(*DICT_DIR / "timeago_samples_short.json"); - let rp = RustyPipe::builder() - .visitor_data("Cgtwel9tMkh2eHh0USiyzc6jBg%3D%3D") - .build() - .unwrap(); - - let channels = [ - "UCeY0bbntWzzVIaj2z3QigXg", - "UCcmpeVbSSQlZRvHfdC-CRwg", - "UC65afEgL62PGFWXY7n6CUbA", - "UCEOXxzW2vU0P-0THehuIIeg", - ]; - - let mut lang_strings: BTreeMap> = BTreeMap::new(); - for lang in LANGUAGES { - println!("{lang}"); - let query = rp.query().lang(lang); - let strings = stream::iter(channels) - .map(|id| get_channel_datestrings(&query, id)) - .buffered(concurrency) - .collect::>() - .await - .into_iter() - .flatten() - .collect::>(); - lang_strings.insert(lang, strings); - } - - let mut en_strings_uniq: HashSet<&str> = HashSet::new(); - let mut uniq_ids: HashSet = HashSet::new(); - - lang_strings[&Language::En] - .iter() - .enumerate() - .for_each(|(n, s)| { - if en_strings_uniq.insert(s) { - uniq_ids.insert(n); - } - }); - - let strings_map = lang_strings - .iter() - .map(|(lang, strings)| { - ( - lang, - strings - .iter() - .enumerate() - .filter(|(n, _)| uniq_ids.contains(n)) - .map(|(_, s)| s) - .collect::>(), - ) - }) - .collect::>(); - - let file = File::create(json_path).unwrap(); - serde_json::to_writer_pretty(file, &strings_map).unwrap(); -} - -async fn get_channel_datestrings(rp: &RustyPipeQuery, id: &str) -> Vec { - let channel = rp.channel_videos(id).await.unwrap(); - - channel - .content - .items - .into_iter() - .filter_map(|itm| itm.publish_date_txt) - .collect() -} diff --git a/codegen/src/collect_video_durations.rs b/codegen/src/collect_video_durations.rs index cfb5a64..8d5024f 100644 --- a/codegen/src/collect_video_durations.rs +++ b/codegen/src/collect_video_durations.rs @@ -5,7 +5,7 @@ use std::{ }; use anyhow::Result; -use futures_util::{stream, StreamExt}; +use futures::{stream, StreamExt}; use path_macro::path; use rustypipe::{ client::{ClientType, RustyPipe, RustyPipeQuery}, @@ -204,6 +204,8 @@ pub fn parse_video_durations() { parse(&mut words, lang, dict_entry.by_char, txt, *d); } + // dbg!(&words); + for (k, v) in words { if let Some(v) = v { dict_entry.timeago_tokens.insert(k, v.to_string()); @@ -268,6 +270,7 @@ async fn get_channel_vlengths( ClientType::Desktop, "browse", &QBrowse { + context: query.get_context(ClientType::Desktop, true, None).await, browse_id: channel_id, params: Some("EgZ2aWRlb3MYASAAMAE"), }, diff --git a/codegen/src/download_testfiles.rs b/codegen/src/download_testfiles.rs index cd85654..41daacc 100644 --- a/codegen/src/download_testfiles.rs +++ b/codegen/src/download_testfiles.rs @@ -8,7 +8,6 @@ use std::{ use path_macro::path; use rustypipe::{ client::{ClientType, RustyPipe}, - model::YouTubeItem, param::{ search_filter::{self, ItemType, SearchFilter}, ChannelVideoTab, Country, @@ -38,6 +37,8 @@ pub async fn download_testfiles() { search_cont().await; search_playlists().await; search_empty().await; + startpage().await; + startpage_cont().await; trending().await; music_playlist().await; @@ -62,23 +63,12 @@ pub async fn download_testfiles() { music_charts().await; music_genres().await; music_genre().await; - - // User data - history().await; - subscriptions().await; - subscription_feed().await; - - music_history().await; - music_saved_artists().await; - music_saved_albums().await; - music_saved_tracks().await; - music_saved_playlists().await; } const CLIENT_TYPES: [ClientType; 5] = [ ClientType::Desktop, ClientType::DesktopMusic, - ClientType::Tv, + ClientType::TvHtml5Embed, ClientType::Android, ClientType::Ios, ]; @@ -144,7 +134,6 @@ fn rp_testfile(json_path: &Path) -> RustyPipe { .report() .strict() .build() - .unwrap() } async fn player() { @@ -166,7 +155,7 @@ async fn player() { } async fn player_model() { - let rp = RustyPipe::builder().strict().build().unwrap(); + let rp = RustyPipe::builder().strict().build(); for (name, id) in [("multilanguage", "tVWWp1PqDus"), ("hdr", "LXb3EKWsInQ")] { let json_path = @@ -402,10 +391,7 @@ async fn search() { } let rp = rp_testfile(&json_path); - rp.query() - .search::("doobydoobap") - .await - .unwrap(); + rp.query().search("doobydoobap").await.unwrap(); } async fn search_cont() { @@ -415,11 +401,7 @@ async fn search_cont() { } let rp = RustyPipe::new(); - let search = rp - .query() - .search::("doobydoobap") - .await - .unwrap(); + let search = rp.query().search("doobydoobap").await.unwrap(); let rp = rp_testfile(&json_path); search.items.next(rp.query()).await.unwrap().unwrap(); @@ -433,7 +415,7 @@ async fn search_playlists() { let rp = rp_testfile(&json_path); rp.query() - .search_filter::("pop", &SearchFilter::new().item_type(ItemType::Playlist)) + .search_filter("pop", &SearchFilter::new().item_type(ItemType::Playlist)) .await .unwrap(); } @@ -446,7 +428,7 @@ async fn search_empty() { let rp = rp_testfile(&json_path); rp.query() - .search_filter::( + .search_filter( "test", &SearchFilter::new() .feature(search_filter::Feature::IsLive) @@ -456,6 +438,29 @@ async fn search_empty() { .unwrap(); } +async fn startpage() { + let json_path = path!(*TESTFILES_DIR / "trends" / "startpage.json"); + if json_path.exists() { + return; + } + + let rp = rp_testfile(&json_path); + rp.query().startpage().await.unwrap(); +} + +async fn startpage_cont() { + let json_path = path!(*TESTFILES_DIR / "trends" / "startpage_cont.json"); + if json_path.exists() { + return; + } + + let rp = RustyPipe::new(); + let startpage = rp.query().startpage().await.unwrap(); + + let rp = rp_testfile(&json_path); + startpage.next(rp.query()).await.unwrap(); +} + async fn trending() { let json_path = path!(*TESTFILES_DIR / "trends" / "trending_videos.json"); if json_path.exists() { @@ -466,36 +471,6 @@ async fn trending() { rp.query().trending().await.unwrap(); } -async fn history() { - let json_path = path!(*TESTFILES_DIR / "userdata" / "history.json"); - if json_path.exists() { - return; - } - - let rp = rp_testfile(&json_path); - rp.query().history().await.unwrap(); -} - -async fn subscriptions() { - let json_path = path!(*TESTFILES_DIR / "userdata" / "subscriptions.json"); - if json_path.exists() { - return; - } - - let rp = rp_testfile(&json_path); - rp.query().subscriptions().await.unwrap(); -} - -async fn subscription_feed() { - let json_path = path!(*TESTFILES_DIR / "userdata" / "subscription_feed.json"); - if json_path.exists() { - return; - } - - let rp = rp_testfile(&json_path); - rp.query().subscription_feed().await.unwrap(); -} - async fn music_playlist() { for (name, id) in [ ("short", "RDCLAK5uy_kFQXdnqMaQCVx2wpUM4ZfbsGCDibZtkJk"), @@ -582,7 +557,7 @@ async fn music_search() { } let rp = rp_testfile(&json_path); - rp.query().music_search_main(query).await.unwrap(); + rp.query().music_search(query).await.unwrap(); } } @@ -643,7 +618,7 @@ async fn music_search_playlists() { let rp = rp_testfile(&json_path); rp.query() - .music_search_playlists("pop", community) + .music_search_playlists_filter("pop", community) .await .unwrap(); } @@ -817,53 +792,3 @@ async fn music_genre() { rp.query().music_genre(id).await.unwrap(); } } - -async fn music_history() { - let json_path = path!(*TESTFILES_DIR / "music_userdata" / "music_history.json"); - if json_path.exists() { - return; - } - - let rp = rp_testfile(&json_path); - rp.query().music_history().await.unwrap(); -} - -async fn music_saved_artists() { - let json_path = path!(*TESTFILES_DIR / "music_userdata" / "saved_artists.json"); - if json_path.exists() { - return; - } - - let rp = rp_testfile(&json_path); - rp.query().music_saved_artists().await.unwrap(); -} - -async fn music_saved_albums() { - let json_path = path!(*TESTFILES_DIR / "music_userdata" / "saved_albums.json"); - if json_path.exists() { - return; - } - - let rp = rp_testfile(&json_path); - rp.query().music_saved_albums().await.unwrap(); -} - -async fn music_saved_tracks() { - let json_path = path!(*TESTFILES_DIR / "music_userdata" / "saved_tracks.json"); - if json_path.exists() { - return; - } - - let rp = rp_testfile(&json_path); - rp.query().music_saved_tracks().await.unwrap(); -} - -async fn music_saved_playlists() { - let json_path = path!(*TESTFILES_DIR / "music_userdata" / "saved_playlists.json"); - if json_path.exists() { - return; - } - - let rp = rp_testfile(&json_path); - rp.query().music_saved_playlists().await.unwrap(); -} diff --git a/codegen/src/gen_dictionary.rs b/codegen/src/gen_dictionary.rs index 549fb83..452a47b 100644 --- a/codegen/src/gen_dictionary.rs +++ b/codegen/src/gen_dictionary.rs @@ -10,7 +10,7 @@ use crate::{ }; fn parse_tu(tu: &str) -> (u8, Option) { - static TU_PATTERN: Lazy = Lazy::new(|| Regex::new(r"^(\d*)(\w*)$").unwrap()); + static TU_PATTERN: Lazy = Lazy::new(|| Regex::new(r"^(\d*)(\w?)$").unwrap()); match TU_PATTERN.captures(tu) { Some(cap) => ( cap.get(1).unwrap().as_str().parse().unwrap_or(1), @@ -22,8 +22,6 @@ fn parse_tu(tu: &str) -> (u8, Option) { "W" => Some(TimeUnit::Week), "M" => Some(TimeUnit::Month), "Y" => Some(TimeUnit::Year), - "Wl" => Some(TimeUnit::LastWeek), - "Wd" => Some(TimeUnit::LastWeekday), "" => None, _ => panic!("invalid time unit: {tu}"), }, @@ -45,7 +43,7 @@ pub fn generate_dictionary() { use crate::{ model::AlbumType, param::Language, - util::timeago::{TaToken, TimeUnit}, + util::timeago::{DateCmp, TaToken, TimeUnit}, }; /// Dictionary entry containing language-specific parsing information @@ -57,13 +55,14 @@ pub(crate) struct Entry { /// Identifiers: `Y`(ear), `M`(month), `W`(eek), `D`(ay), /// `h`(our), `m`(inute), `s`(econd) pub timeago_tokens: phf::Map<&'static str, TaToken>, - /// True if the month has to be parsed before the day + /// Order in which to parse numeric date components. Formatted as + /// a string of date identifiers (Y, M, D). /// /// Examples: /// - /// - 03.01.2020 => DMY => false - /// - 01/03/2020 => MDY => true - pub month_before_day: bool, + /// - 03.01.2020 => `"DMY"` + /// - Jan 3, 2020 => `"DY"` + pub date_order: &'static [DateCmp], /// Tokens for parsing month names. /// /// Format: Parsed token -> Month number (starting from 1) @@ -86,12 +85,6 @@ pub(crate) struct Entry { /// /// Format: Parsed text -> Album type pub album_types: phf::Map<&'static str, AlbumType>, - /// Channel name prefix on playlist pages (e.g. `by`) - pub chan_prefix: &'static str, - /// Channel name suffix on playlist pages - pub chan_suffix: &'static str, - /// "Other versions" title on album pages - pub album_versions_title: &'static str, } "#; @@ -140,6 +133,13 @@ pub(crate) fn entry(lang: Language) -> Entry { }; }); + // Date order + let mut date_order = "&[".to_owned(); + entry.date_order.chars().for_each(|c| { + write!(date_order, "DateCmp::{c}, ").unwrap(); + }); + date_order = date_order.trim_end_matches([' ', ',']).to_owned() + "]"; + // Number tokens let mut number_tokens = phf_codegen::Map::<&str>::new(); entry.number_tokens.iter().for_each(|(txt, mag)| { @@ -180,8 +180,8 @@ pub(crate) fn entry(lang: Language) -> Entry { .to_string() .replace('\n', "\n "); - write!(code_timeago_tokens, "{} => Entry {{\n timeago_tokens: {},\n month_before_day: {:?},\n months: {},\n timeago_nd_tokens: {},\n comma_decimal: {:?},\n number_tokens: {},\n number_nd_tokens: {},\n album_types: {},\n chan_prefix: {:?},\n chan_suffix: {:?},\n album_versions_title: {:?},\n }},\n ", - selector, code_ta_tokens, entry.month_before_day, code_months, code_ta_nd_tokens, entry.comma_decimal, code_number_tokens, code_number_nd_tokens, code_album_types, entry.chan_prefix, entry.chan_suffix, entry.album_versions_title).unwrap(); + write!(code_timeago_tokens, "{} => Entry {{\n timeago_tokens: {},\n date_order: {},\n months: {},\n timeago_nd_tokens: {},\n comma_decimal: {:?},\n number_tokens: {},\n number_nd_tokens: {},\n album_types: {},\n }},\n ", + selector, code_ta_tokens, date_order, code_months, code_ta_nd_tokens, entry.comma_decimal, code_number_tokens, code_number_nd_tokens, code_album_types).unwrap(); } code_timeago_tokens = code_timeago_tokens.trim_end().to_owned() + "\n }\n}\n"; diff --git a/codegen/src/gen_locales.rs b/codegen/src/gen_locales.rs index fdd0410..66e1da9 100644 --- a/codegen/src/gen_locales.rs +++ b/codegen/src/gen_locales.rs @@ -202,20 +202,11 @@ pub enum Country { .to_owned(); let mut code_lang_array = format!( - r#"/// Array of all available languages -/// The languages are sorted by their native names. This array can be used to display -/// a language selection or to get the language code from a language name using binary search. -pub const LANGUAGES: [Language; {}] = [ -"#, + "/// Array of all available languages\npub const LANGUAGES: [Language; {}] = [\n", languages.len() ); let mut code_country_array = format!( - r#"/// Array of all available countries -/// -/// The countries are sorted by their english names. This array can be used to display -/// a country selection or to get the country code from a country name using binary search. -pub const COUNTRIES: [Country; {}] = [ -"#, + "/// Array of all available countries\npub const COUNTRIES: [Country; {}] = [\n", countries.len() ); @@ -237,15 +228,16 @@ pub const COUNTRIES: [Country; {}] = [ .to_owned(); for (code, native_name) in &languages { - let enum_name = code.split('-').fold(String::new(), |mut output, c| { - let _ = write!( - output, - "{}{}", - c[0..1].to_owned().to_uppercase(), - c[1..].to_owned().to_lowercase() - ); - output - }); + let enum_name = code + .split('-') + .map(|c| { + format!( + "{}{}", + c[0..1].to_owned().to_uppercase(), + c[1..].to_owned().to_lowercase() + ) + }) + .collect::(); let en_name = lang_names.get(code).expect(code); @@ -261,6 +253,9 @@ pub const COUNTRIES: [Country; {}] = [ code_langs += &enum_name; code_langs += ",\n"; + // Language array + writeln!(code_lang_array, " Language::{enum_name},").unwrap(); + // Language names writeln!( code_lang_names, @@ -270,24 +265,6 @@ pub const COUNTRIES: [Country; {}] = [ } code_langs += "}\n"; - // Language array - let languages_by_name = languages - .iter() - .map(|(k, v)| (v, k)) - .collect::>(); - for code in languages_by_name.values() { - let enum_name = code.split('-').fold(String::new(), |mut output, c| { - let _ = write!( - output, - "{}{}", - c[0..1].to_owned().to_uppercase(), - c[1..].to_owned().to_lowercase() - ); - output - }); - writeln!(code_lang_array, " Language::{enum_name},").unwrap(); - } - for (c, n) in &countries { let enum_name = c[0..1].to_owned().to_uppercase() + &c[1..].to_owned().to_lowercase(); @@ -295,6 +272,9 @@ pub const COUNTRIES: [Country; {}] = [ writeln!(code_countries, " /// {n}").unwrap(); writeln!(code_countries, " {enum_name},").unwrap(); + // Country array + writeln!(code_country_array, " Country::{enum_name},").unwrap(); + // Country names writeln!( code_country_names, @@ -303,16 +283,6 @@ pub const COUNTRIES: [Country; {}] = [ .unwrap(); } - // Country array - let countries_by_name = countries - .iter() - .map(|(k, v)| (v, k)) - .collect::>(); - for c in countries_by_name.values() { - let enum_name = c[0..1].to_owned().to_uppercase() + &c[1..].to_owned().to_lowercase(); - writeln!(code_country_array, " Country::{enum_name},").unwrap(); - } - // Add Country::Zz / Global code_countries += " /// Global (can only be used for music charts)\n"; code_countries += " Zz,\n"; @@ -339,7 +309,7 @@ async fn get_locales() -> (BTreeMap, BTreeMap) { .post("https://www.youtube.com/youtubei/v1/account/account_menu?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false") .header(header::CONTENT_TYPE, "application/json") .body( - r#"{"context":{"client":{"clientName":"WEB","clientVersion":"2.20220914.06.00","platform":"DESKTOP","originalUrl":"https://www.youtube.com/","hl":"en","gl":"US"},"request":{"internalExperimentFlags":[],"useSsl":true},"user":{"lockedSafetyMode":false}}}"# + r##"{"context":{"client":{"clientName":"WEB","clientVersion":"2.20220914.06.00","platform":"DESKTOP","originalUrl":"https://www.youtube.com/","hl":"en","gl":"US"},"request":{"internalExperimentFlags":[],"useSsl":true},"user":{"lockedSafetyMode":false}}}"## ) .send().await .unwrap() diff --git a/codegen/src/main.rs b/codegen/src/main.rs index 74b5d85..e61a78f 100644 --- a/codegen/src/main.rs +++ b/codegen/src/main.rs @@ -2,12 +2,8 @@ mod abtest; mod collect_album_types; -mod collect_album_versions_titles; -mod collect_chan_prefixes; -mod collect_history_dates; mod collect_large_numbers; mod collect_playlist_dates; -mod collect_video_dates; mod collect_video_durations; mod download_testfiles; mod gen_dictionary; @@ -31,18 +27,10 @@ enum Commands { CollectLargeNumbers, CollectAlbumTypes, CollectVideoDurations, - CollectVideoDates, - CollectHistoryDates, - CollectMusicHistoryDates, - CollectChanPrefixes, - CollectAlbumVersionsTitles, ParsePlaylistDates, - ParseHistoryDates, ParseLargeNumbers, ParseAlbumTypes, ParseVideoDurations, - ParseChanPrefixes, - ParseAlbumVersionsTitles, GenLocales, GenDict, DownloadTestfiles, @@ -56,41 +44,29 @@ enum Commands { #[tokio::main] async fn main() { - tracing_subscriber::fmt::init(); + env_logger::init(); let cli = Cli::parse(); match cli.command { Commands::CollectPlaylistDates => { - collect_playlist_dates::collect_dates(cli.concurrency).await + collect_playlist_dates::collect_dates(cli.concurrency).await; } Commands::CollectLargeNumbers => { - collect_large_numbers::collect_large_numbers(cli.concurrency).await + collect_large_numbers::collect_large_numbers(cli.concurrency).await; } Commands::CollectAlbumTypes => { - collect_album_types::collect_album_types(cli.concurrency).await + collect_album_types::collect_album_types(cli.concurrency).await; } Commands::CollectVideoDurations => { - collect_video_durations::collect_video_durations(cli.concurrency).await - } - Commands::CollectVideoDates => { - collect_video_dates::collect_video_dates(cli.concurrency).await - } - Commands::CollectHistoryDates => collect_history_dates::collect_dates().await, - Commands::CollectMusicHistoryDates => collect_history_dates::collect_dates_music().await, - Commands::CollectChanPrefixes => collect_chan_prefixes::collect_chan_prefixes().await, - Commands::CollectAlbumVersionsTitles => { - collect_album_versions_titles::collect_album_versions_titles().await + collect_video_durations::collect_video_durations(cli.concurrency).await; } Commands::ParsePlaylistDates => collect_playlist_dates::write_samples_to_dict(), - Commands::ParseHistoryDates => collect_history_dates::write_samples_to_dict(), Commands::ParseLargeNumbers => collect_large_numbers::write_samples_to_dict(), Commands::ParseAlbumTypes => collect_album_types::write_samples_to_dict(), Commands::ParseVideoDurations => collect_video_durations::parse_video_durations(), - Commands::ParseChanPrefixes => collect_chan_prefixes::write_samples_to_dict(), - Commands::ParseAlbumVersionsTitles => { - collect_album_versions_titles::write_samples_to_dict() + Commands::GenLocales => { + gen_locales::generate_locales().await; } - Commands::GenLocales => gen_locales::generate_locales().await, Commands::GenDict => gen_dictionary::generate_dictionary(), Commands::DownloadTestfiles => download_testfiles::download_testfiles().await, Commands::AbTest { id, n } => { diff --git a/codegen/src/model.rs b/codegen/src/model.rs index 2d9929f..a5f9e11 100644 --- a/codegen/src/model.rs +++ b/codegen/src/model.rs @@ -1,7 +1,6 @@ use std::collections::BTreeMap; -use ordered_hash_map::OrderedHashMap; -use rustypipe::{model::AlbumType, param::Language}; +use rustypipe::{client::YTContext, model::AlbumType, param::Language}; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DefaultOnError, VecSkipError}; @@ -13,20 +12,13 @@ pub struct DictEntry { /// Should the language be parsed by character instead of by word? /// (e.g. Chinese/Japanese) pub by_char: bool, - /// True if the month has to be parsed before the day - /// - /// Examples: - /// - /// - 03.01.2020 => DMY => false - /// - 01/03/2020 => MDY => true - pub month_before_day: bool, /// Tokens for parsing timeago strings. /// /// Format: Parsed token -> \[Quantity\] Identifier /// /// Identifiers: `Y`(ear), `M`(month), `W`(eek), `D`(ay), /// `h`(our), `m`(inute), `s`(econd) - pub timeago_tokens: OrderedHashMap, + pub timeago_tokens: BTreeMap, /// Order in which to parse numeric date components. Formatted as /// a string of date identifiers (Y, M, D). /// @@ -42,7 +34,7 @@ pub struct DictEntry { /// Tokens for parsing date strings with no digits (e.g. Today, Tomorrow) /// /// Format: Parsed token -> \[Quantity\] Identifier - pub timeago_nd_tokens: OrderedHashMap, + pub timeago_nd_tokens: BTreeMap, /// Are commas (instead of points) used as decimal separators? pub comma_decimal: bool, /// Tokens for parsing decimal prefixes (K, M, B, ...) @@ -57,12 +49,6 @@ pub struct DictEntry { /// /// Format: Parsed text -> Album type pub album_types: BTreeMap, - /// Channel name prefix on playlist pages (e.g. `by`) - pub chan_prefix: String, - /// Channel name suffix on playlist pages - pub chan_suffix: String, - /// "Other versions" title on album pages - pub album_versions_title: String, } /// Parsed TimeAgo string, contains amount and time unit. @@ -76,12 +62,12 @@ pub struct TimeAgo { pub unit: TimeUnit, } -impl std::fmt::Display for TimeAgo { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl ToString for TimeAgo { + fn to_string(&self) -> String { if self.n > 1 { - write!(f, "{}{}", self.n, self.unit.as_str()) + format!("{}{}", self.n, self.unit.as_str()) } else { - f.write_str(self.unit.as_str()) + self.unit.as_str().to_owned() } } } @@ -97,8 +83,6 @@ pub enum TimeUnit { Week, Month, Year, - LastWeek, - LastWeekday, } impl TimeUnit { @@ -111,24 +95,14 @@ impl TimeUnit { TimeUnit::Week => "W", TimeUnit::Month => "M", TimeUnit::Year => "Y", - TimeUnit::LastWeek => "Wl", - TimeUnit::LastWeekday => "Wd", } } } -#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum ExtItemType { - Track, - Video, - Episode, - Playlist, - Artist, -} - #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct QBrowse<'a> { + pub context: YTContext<'a>, pub browse_id: &'a str, #[serde(skip_serializing_if = "Option::is_none")] pub params: Option<&'a str>, @@ -137,6 +111,7 @@ pub struct QBrowse<'a> { #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct QCont<'a> { + pub context: YTContext<'a>, pub continuation: &'a str, } @@ -154,7 +129,7 @@ pub struct Text { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Channel { - pub contents: TwoColumnBrowseResults, + pub contents: Contents, pub header: ChannelHeader, } @@ -172,7 +147,7 @@ pub struct HeaderRenderer { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct TwoColumnBrowseResults { +pub struct Contents { pub two_column_browse_results_renderer: TabsRenderer, } @@ -181,37 +156,24 @@ pub struct TwoColumnBrowseResults { #[serde(rename_all = "camelCase")] pub struct TabsRenderer { #[serde_as(as = "VecSkipError<_>")] - pub tabs: Vec>, + pub tabs: Vec, } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct ContentsRenderer { - #[serde(alias = "tabs")] - pub contents: Vec, +pub struct TabRendererWrap { + pub tab_renderer: TabRenderer, } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct Tab { - pub tab_renderer: TabRenderer, +pub struct TabRenderer { + pub content: RichGridRendererWrap, } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct TabRenderer { - pub content: T, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct SectionList { - pub section_list_renderer: ContentsRenderer, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct RichGrid { +pub struct RichGridRendererWrap { pub rich_grid_renderer: RichGridRenderer, } diff --git a/codegen/src/util.rs b/codegen/src/util.rs index bd62196..6d7d830 100644 --- a/codegen/src/util.rs +++ b/codegen/src/util.rs @@ -77,7 +77,7 @@ pub fn filter_datestr(string: &str) -> String { .to_lowercase() .chars() .filter_map(|c| { - if matches!(c, '\u{200b}' | '.' | ',') || c.is_ascii_digit() { + if c == '\u{200b}' || c.is_ascii_digit() { None } else if c == '-' { Some(' ') @@ -134,16 +134,12 @@ where if c.is_ascii_digit() { buf.push(c); } else if !buf.is_empty() { - if let Ok(n) = buf.parse::() { - numbers.push(n); - } + buf.parse::().map_or((), |n| numbers.push(n)); buf.clear(); } } if !buf.is_empty() { - if let Ok(n) = buf.parse::() { - numbers.push(n); - } + buf.parse::().map_or((), |n| numbers.push(n)); } numbers @@ -190,7 +186,7 @@ pub fn parse_largenum_en(string: &str) -> Option { /// and return the duration in seconds. pub fn parse_video_length(text: &str) -> Option { static VIDEO_LENGTH_REGEX: Lazy = - Lazy::new(|| Regex::new(r"(?:(\d+)[:.])?(\d{1,2})[:.](\d{2})").unwrap()); + Lazy::new(|| Regex::new(r#"(?:(\d+)[:.])?(\d{1,2})[:.](\d{2})"#).unwrap()); VIDEO_LENGTH_REGEX.captures(text).map(|cap| { let hrs = cap .get(1) diff --git a/downloader/CHANGELOG.md b/downloader/CHANGELOG.md deleted file mode 100644 index f0779d4..0000000 --- a/downloader/CHANGELOG.md +++ /dev/null @@ -1,175 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - - -## [v0.3.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.3.0..rustypipe-downloader/v0.3.1) - 2024-12-20 - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rustypipe to 0.11.0 - - -## [v0.3.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.7..rustypipe-downloader/v0.3.0) - 2025-02-09 - -### 🚀 Features - -- [**breaking**] Remove manual PO token options from downloader in favor of rustypipe-botguard - ([cddb32f](https://codeberg.org/ThetaDev/rustypipe/commit/cddb32f190276265258c6ab45b3d43a8891c4b39)) - -### 🐛 Bug Fixes - -- Ensure downloader futures are send - ([812ff4c](https://codeberg.org/ThetaDev/rustypipe/commit/812ff4c5bafffc5708a6d5066f1ebadb6d9fc958)) -- Download audio with dolby codec - ([9234005](https://codeberg.org/ThetaDev/rustypipe/commit/92340056f868007beccb64e9e26eb39abc40f7aa)) - -### 🚜 Refactor - -- [**breaking**] Add client_type field to DownloadError, rename cli option po-token-cache to pot-cache - ([594e675](https://codeberg.org/ThetaDev/rustypipe/commit/594e675b39efc5fbcdbd5e920a4d2cdee64f718e)) - -### 📚 Documentation - -- Add Botguard info to README - ([9957add](https://codeberg.org/ThetaDev/rustypipe/commit/9957add2b5d6391b2c1869d2019fd7dd91b8cd41)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rustypipe to 0.10.0 - - -## [v0.2.7](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.6..rustypipe-downloader/v0.2.7) - 2025-01-16 - -### 🚀 Features - -- Extract player DRM data - ([2af4001](https://codeberg.org/ThetaDev/rustypipe/commit/2af4001c75f2ff4f7c891aa59ac22c2c6b7902a2)) -- Prefer maxresdefault.jpg thumbnail if available - ([a8e97f4](https://codeberg.org/ThetaDev/rustypipe/commit/a8e97f411a1e769e52d8cbde11f0a4ca1535f7ef)) -- Add DRM and audio channel number filtering to StreamFilter - ([d5abee2](https://codeberg.org/ThetaDev/rustypipe/commit/d5abee275300ab1bc10fc8d6c35a4e3813fd2bd4)) - -### 🐛 Bug Fixes - -- Remove Unix file metadata usage (Windows compatibility) - ([5c6d992](https://codeberg.org/ThetaDev/rustypipe/commit/5c6d992939f55a203ac1784f1e9175ac1d498ce8)) - -### 📚 Documentation - -- Update README - ([0432477](https://codeberg.org/ThetaDev/rustypipe/commit/0432477451ecd5f64145d65239c721f4e44826c0)) -- Fix README - ([11442df](https://codeberg.org/ThetaDev/rustypipe/commit/11442dfd369599396357f5b7a7a4268a7b537f57)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rustypipe to 0.9.0 -- *(deps)* Update rust crate rstest to 0.24.0 (#20) - ([ab19034](https://codeberg.org/ThetaDev/rustypipe/commit/ab19034ab19baf090e83eada056559676ffdadce)) -- *(deps)* Update rust crate lofty to 0.22.0 - ([addeb82](https://codeberg.org/ThetaDev/rustypipe/commit/addeb821101aa968b95455604bc13bd24f50328f)) -- *(deps)* Update rust crate dirs to v6 (#24) - ([6a60425](https://codeberg.org/ThetaDev/rustypipe/commit/6a604252b1af7a9388db5dc170f737069cc31051)) - - -## [v0.2.6](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.5..rustypipe-downloader/v0.2.6) - 2024-12-20 - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rustypipe to 0.8.0 - - -## [v0.2.5](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.4..rustypipe-downloader/v0.2.5) - 2024-12-13 - -### 🐛 Bug Fixes - -- Replace futures dependency with futures-util - ([5c39bf4](https://codeberg.org/ThetaDev/rustypipe/commit/5c39bf4842b13d37a4277ea5506e15c179892ce5)) -- Remove empty tempfile after unsuccessful download - ([5262bec](https://codeberg.org/ThetaDev/rustypipe/commit/5262becca1e9e3e8262833764ef18c23bc401172)) - -### ⚙️ Miscellaneous Tasks - -- Add docs badge to README - ([706e881](https://codeberg.org/ThetaDev/rustypipe/commit/706e88134c0e94ce7d880735e9d31b3ff531a4f9)) - - -## [v0.2.4](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.3..rustypipe-downloader/v0.2.4) - 2024-11-10 - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rust crate thiserror to v2 (#16) - ([e1e1687](https://codeberg.org/ThetaDev/rustypipe/commit/e1e1687605603686ac5fd5deeb6aa8fecaf92494)) -- *(deps)* Update rustypipe to 0.7.0 - - -## [v0.2.3](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.2..rustypipe-downloader/v0.2.3) - 2024-10-28 - -### 🐛 Bug Fixes - -- Remove unnecessary image.rs dependencies - ([1b08166](https://codeberg.org/ThetaDev/rustypipe/commit/1b08166399cccb8394d2fdd82d54162c1a9e01be)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rustypipe to 0.6.0 - - -## [v0.2.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.1..rustypipe-downloader/v0.2.2) - 2024-10-13 - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rust crate rstest to 0.23.0 (#12) - ([96776e9](https://codeberg.org/ThetaDev/rustypipe/commit/96776e98d76fa1d31d5f84dbceafbe8f9dfd9085)) -- *(deps)* Update rustypipe to 0.5.0 - - -## [v0.2.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.0..rustypipe-downloader/v0.2.1) - 2024-09-10 - -### 📚 Documentation - -- Fix license badge URL - ([4a253e1](https://codeberg.org/ThetaDev/rustypipe/commit/4a253e1a47317e9999e6ad31ac5c411956a0986a)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rust crate tokio to 1.20.4 [security] (#10) - ([a445e51](https://codeberg.org/ThetaDev/rustypipe/commit/a445e51b54a9afc44cd9657260a0b3d2abddbfa6)) - - -## [v0.2.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.1.1..rustypipe-downloader/v0.2.0) - 2024-08-18 - -### 🚀 Features - -- Overhauled downloader - ([11a0038](https://codeberg.org/ThetaDev/rustypipe/commit/11a00383502917cd98245c3da349107289ba3aa9)) -- [**breaking**] Add TV client - ([e608811](https://codeberg.org/ThetaDev/rustypipe/commit/e608811e5f5615416241e67561671330097092cb)) -- Downloader: add audio tagging - ([1e1315a](https://codeberg.org/ThetaDev/rustypipe/commit/1e1315a8378bd0ad25b5f1614e83dabc4a0b40d5)) -- Downloader: add download_track fn, improve path templates - ([e1e4fb2](https://codeberg.org/ThetaDev/rustypipe/commit/e1e4fb29c190fec07f17c59ec88bef4f1c2a76a1)) -- Add audiotag+indicatif features to downloader - ([97fb057](https://codeberg.org/ThetaDev/rustypipe/commit/97fb0578b5c4954a596d8dee0c4b6e1d773a9300)) -- Add plaintext output to CLI - ([91b020e](https://codeberg.org/ThetaDev/rustypipe/commit/91b020efd498eff6e0f354a1de39439e252a79dd)) -- Add potoken option to downloader - ([904f821](https://codeberg.org/ThetaDev/rustypipe/commit/904f8215d84c810b04e4d2134718e786a4803ad2)) -- Add list of clients to downloader - ([5e646af](https://codeberg.org/ThetaDev/rustypipe/commit/5e646afd1edc6c0101501311527ea56d3bad5fd2)) -- Retry with different client after 403 error - ([d875b54](https://codeberg.org/ThetaDev/rustypipe/commit/d875b5442de9822ba7ddc6f05789f56a8962808c)) -- [**breaking**] Update channel model, addd handle + video_count, remove tv/mobile banner - ([e671570](https://codeberg.org/ThetaDev/rustypipe/commit/e6715700d950912031d5fbc1263f8770b6ffc49c)) - -### 🐛 Bug Fixes - -- *(deps)* Update quick-xml to v0.35.0 - ([298e4de](https://codeberg.org/ThetaDev/rustypipe/commit/298e4def93d1595fba91be103f014aa645a08937)) -- Improve deobfuscator (support multiple nsig name matches, error if mapping all streams fails) - ([8152ce6](https://codeberg.org/ThetaDev/rustypipe/commit/8152ce6b088b57be9b8419b754aca93805e5f34d)) -- Set tracing instrumentation level to Error - ([9da3b25](https://codeberg.org/ThetaDev/rustypipe/commit/9da3b25be2b2577f7bd0282c09d10d368ac8b73f)) -- Add docs.rs feature attributes - ([ec13cbb](https://codeberg.org/ThetaDev/rustypipe/commit/ec13cbb1f35081118dda0f7f35e3ef90f7ca79a8)) -- Show docs.rs feature flags - ([67a231d](https://codeberg.org/ThetaDev/rustypipe/commit/67a231d6d1b6427f500667729a59032f2b28cc65)) - -### ⚙️ Miscellaneous Tasks - -- *(deps)* Update rust crate quick-xml to 0.36.0 (#8) - ([b6bc05c](https://codeberg.org/ThetaDev/rustypipe/commit/b6bc05c1f39da9a846b2e3d1d24bcbccb031203b)) -- *(deps)* Update rust crate rstest to 0.22.0 (#9) - ([abb7832](https://codeberg.org/ThetaDev/rustypipe/commit/abb783219aba4b492c1dff03c2148acf1f51a55d)) -- Change repo URL to Codeberg - ([1793331](https://codeberg.org/ThetaDev/rustypipe/commit/17933315d947f76d5fe1aa52abf7ea24c3ce6381)) -- Adjust dependency versions - ([70c6f8c](https://codeberg.org/ThetaDev/rustypipe/commit/70c6f8c3b97baefd316fff90cc727524516657af)) - -### Todo - -- Update metadata - ([8692ca8](https://codeberg.org/ThetaDev/rustypipe/commit/8692ca81d972d0d2acf6fb4da79b9e0f5ebf4daf)) - - -## [v0.1.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.1.0..rustypipe-downloader/v0.1.1) - 2024-06-27 - -### 📚 Documentation - -- Add logo - ([6646078](https://codeberg.org/ThetaDev/rustypipe/commit/66460789449be0d5984cbdb6ec372e69323b7a88)) - -### ⚙️ Miscellaneous Tasks - -- Changelog: fix incorrect version URLs - ([97b6f07](https://codeberg.org/ThetaDev/rustypipe/commit/97b6f07399e80e00a6c015d013e744568be125dd)) -- Update rstest to v0.19.0 - ([50fd1f0](https://codeberg.org/ThetaDev/rustypipe/commit/50fd1f08caf39c1298654e06059cc393543e925b)) -- Introduce MSRV - ([5dbb288](https://codeberg.org/ThetaDev/rustypipe/commit/5dbb288a496d53a299effa2026f5258af7b1f176)) -- Fix clippy lints - ([45b9f2a](https://codeberg.org/ThetaDev/rustypipe/commit/45b9f2a627b4e7075ba0b1c5f16efcc19aef7922)) -- *(deps)* Update rust crate tokio to 1.20.4 [security] (#4) - ([ce3ec34](https://codeberg.org/ThetaDev/rustypipe/commit/ce3ec34337b8acac41410ea39264aab7423d5801)) -- *(deps)* Update rust crate quick-xml to 0.34.0 (#5) - ([1e8a1af](https://codeberg.org/ThetaDev/rustypipe/commit/1e8a1af08c873cee7feadf63c2eff62753a78f64)) -- *(deps)* Update rust crate rstest to 0.21.0 (#7) - ([c3af918](https://codeberg.org/ThetaDev/rustypipe/commit/c3af918ba53c6230c0e4aef822a0cb2cf120bf3f)) -- Update rustypipe to 0.2.0 - -## [v0.1.0](https://codeberg.org/ThetaDev/rustypipe/commits/tag/rustypipe-downloader/v0.1.0) - 2024-03-22 - -Initial release - - diff --git a/downloader/Cargo.toml b/downloader/Cargo.toml index dba74ce..fdf2087 100644 --- a/downloader/Cargo.toml +++ b/downloader/Cargo.toml @@ -1,14 +1,12 @@ [package] name = "rustypipe-downloader" -version = "0.3.1" -rust-version = "1.67.1" -edition.workspace = true -authors.workspace = true -license.workspace = true -repository.workspace = true -keywords.workspace = true -categories.workspace = true +version = "0.1.0" +edition = "2021" +authors = ["ThetaDev "] +license = "GPL-3.0" description = "Downloader extension for RustyPipe" +keywords = ["youtube", "video", "music"] +categories = ["multimedia"] [features] default = ["default-tls"] @@ -30,37 +28,18 @@ rustls-tls-native-roots = [ "rustypipe/rustls-tls-native-roots", ] -audiotag = ["dep:lofty", "dep:image", "dep:smartcrop2"] - [dependencies] -rustypipe.workspace = true -once_cell.workspace = true -regex.workspace = true -thiserror.workspace = true -futures-util.workspace = true -reqwest = { workspace = true, features = ["stream"] } -rand.workspace = true -tokio = { workspace = true, features = ["macros", "fs", "process"] } -indicatif = { workspace = true, optional = true } -filenamify.workspace = true -tracing.workspace = true -time.workspace = true -lofty = { version = "0.22.0", optional = true } -image = { version = "0.25.0", optional = true, default-features = false, features = [ - "rayon", - "jpeg", - "webp", +rustypipe = { path = "..", default-features = false } +rustypipe-postprocessor = { path = "../postprocessor" } +once_cell = "1.12.0" +regex = "1.6.0" +thiserror = "1.0.36" +futures = "0.3.21" +indicatif = "0.17.0" +filenamify = "0.1.0" +log = "0.4.17" +reqwest = { version = "0.11.11", default-features = false, features = [ + "stream", ] } -smartcrop2 = { version = "0.4.0", optional = true } - -[dev-dependencies] -path_macro.workspace = true -rstest.workspace = true -serde_json.workspace = true -temp_testdir = "0.2.3" - -[package.metadata.docs.rs] -# To build locally: -# RUSTDOCFLAGS="--cfg docsrs" cargo +nightly doc --features indicatif,audiotag --no-deps --open -features = ["indicatif", "audiotag"] -rustdoc-args = ["--cfg", "docsrs"] +rand = "0.8.5" +tokio = { version = "1.20.0", features = ["macros", "fs", "process"] } diff --git a/downloader/README.md b/downloader/README.md deleted file mode 100644 index 6a7d4ad..0000000 --- a/downloader/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# ![RustyPipe](https://codeberg.org/ThetaDev/rustypipe/raw/branch/main/notes/logo.svg) Downloader - -[![Current crates.io version](https://img.shields.io/crates/v/rustypipe-downloader.svg)](https://crates.io/crates/rustypipe-downloader) -[![License](https://img.shields.io/badge/License-GPL--3-blue.svg?style=flat)](https://opensource.org/licenses/GPL-3.0) -[![Docs](https://img.shields.io/docsrs/rustypipe-downloader/latest?style=flat)](https://docs.rs/rustypipe-downloader) -[![CI status](https://codeberg.org/ThetaDev/rustypipe/actions/workflows/ci.yaml/badge.svg?style=flat&label=CI)](https://codeberg.org/ThetaDev/rustypipe/actions/?workflow=ci.yaml) - -The downloader is a companion crate for RustyPipe that allows for easy and fast -downloading of video and audio files. - -## Features - -- Fast download of streams, bypassing YouTube's throttling -- Join video and audio streams using ffmpeg -- [Indicatif](https://crates.io/crates/indicatif) support to show download progress bars - (enable `indicatif` feature to use) -- Tag audio files with title, album, artist, date, description and album cover (enable - `audiotag` feature to use) -- Album covers are automatically cropped using smartcrop to ensure they are square - -## How to use - -For the downloader to work, you need to have ffmpeg installed on your system. If your -ffmpeg binary is located at a non-standard path, you can configure the location using -[`DownloaderBuilder::ffmpeg`]. - -At first you have to instantiate and configure the downloader using either -[`Downloader::new`] or the [`DownloaderBuilder`]. - -Then you can build a new download query with a video ID, stream filter and destination -path and finally download the video. - -```rust ignore -use rustypipe::param::StreamFilter; -use rustypipe_downloader::DownloaderBuilder; - -let dl = DownloaderBuilder::new() - .audio_tag() - .crop_cover() - .build(); - -let filter_audio = StreamFilter::new().no_video(); -dl.id("eRsGyueVLvQ").stream_filter(filter_audio).to_file("audio.opus").download().await; - -let filter_video = StreamFilter::new().video_max_res(720); -dl.id("eRsGyueVLvQ").stream_filter(filter_video).to_file("video.mp4").download().await; -``` diff --git a/downloader/src/error.rs b/downloader/src/error.rs deleted file mode 100644 index d45405d..0000000 --- a/downloader/src/error.rs +++ /dev/null @@ -1,59 +0,0 @@ -use std::{borrow::Cow, path::PathBuf}; - -use rustypipe::client::ClientType; - -/// Error from the video downloader -#[derive(thiserror::Error, Debug)] -#[non_exhaustive] -pub enum DownloadError { - /// RustyPipe error - #[error("{0}")] - RustyPipe(#[from] rustypipe::error::Error), - /// Error from the HTTP client - #[error("http error: {0}")] - Http(#[from] reqwest::Error), - /// 403 error trying to download video - #[error("YouTube returned 403 error; visitor_data={}", .visitor_data.as_deref().unwrap_or_default())] - Forbidden { - /// Client type used to fetch the failed stream - client_type: ClientType, - /// Visitor data used to fetch the failed stream - visitor_data: Option, - }, - /// File IO error - #[error(transparent)] - Io(#[from] std::io::Error), - /// FFmpeg returned an error - #[error("FFmpeg error: {0}")] - Ffmpeg(Cow<'static, str>), - /// Error parsing ranges for progressive download - #[error("Progressive download error: {0}")] - Progressive(Cow<'static, str>), - /// Video could not be downloaded because of invalid player data - #[error("source error: {0}")] - Source(Cow<'static, str>), - /// Download target already exists - #[error("file {0} already exists")] - Exists(PathBuf), - #[cfg(feature = "audiotag")] - /// Audio tagging error - #[error("Audio tag error: {0}")] - AudioTag(Cow<'static, str>), - /// Other error - #[error("error: {0}")] - Other(Cow<'static, str>), -} - -#[cfg(feature = "audiotag")] -impl From for DownloadError { - fn from(value: lofty::error::LoftyError) -> Self { - Self::AudioTag(value.to_string().into()) - } -} - -#[cfg(feature = "audiotag")] -impl From for DownloadError { - fn from(value: image::ImageError) -> Self { - Self::AudioTag(value.to_string().into()) - } -} diff --git a/downloader/src/lib.rs b/downloader/src/lib.rs index c63d5c5..7cc24ee 100644 --- a/downloader/src/lib.rs +++ b/downloader/src/lib.rs @@ -1,31 +1,20 @@ -#![doc = include_str!("../README.md")] -#![cfg_attr(docsrs, feature(doc_cfg))] -#![warn(missing_docs, clippy::todo, clippy::dbg_macro)] +#![warn(clippy::todo, clippy::dbg_macro)] + +//! # YouTube audio/video downloader -mod error; mod util; -use std::{ - borrow::Cow, - cmp::Ordering, - ffi::OsString, - ops::Range, - path::{Path, PathBuf}, - sync::Arc, - time::Duration, -}; +use std::{borrow::Cow, cmp::Ordering, ffi::OsString, ops::Range, path::PathBuf, time::Duration}; -use futures_util::stream::{self, StreamExt, TryStreamExt}; +use futures::stream::{self, StreamExt}; +use indicatif::{ProgressBar, ProgressStyle}; +use log::{debug, info}; use once_cell::sync::Lazy; use rand::Rng; use regex::Regex; -use reqwest::{header, Client, StatusCode, Url}; +use reqwest::{header, Client}; use rustypipe::{ - client::{ClientType, RustyPipe}, - model::{ - traits::{FileFormat, YtEntity}, - AudioCodec, TrackItem, VideoCodec, VideoPlayer, - }, + model::{traits::FileFormat, AudioCodec, AudioFormat, VideoCodec, VideoPlayer}, param::StreamFilter, }; use tokio::{ @@ -34,1041 +23,16 @@ use tokio::{ process::Command, }; -#[cfg(feature = "indicatif")] -use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; - -#[cfg(feature = "audiotag")] -use lofty::{config::WriteOptions, picture::Picture, prelude::*, tag::Tag}; -#[cfg(feature = "audiotag")] -use rustypipe::model::{richtext::ToPlaintext, VideoDetails, VideoPlayerDetails}; -#[cfg(feature = "audiotag")] -use time::{Date, OffsetDateTime}; - -pub use error::DownloadError; +use util::DownloadError; type Result = core::result::Result; const CHUNK_SIZE_MIN: u64 = 9_000_000; const CHUNK_SIZE_MAX: u64 = 10_000_000; -/// RustyPipe audio/video downloader -/// -/// The downloader uses an [`Arc`] internally, so if you are using the client -/// at multiple locations, you can just clone it. -#[derive(Clone)] -pub struct Downloader { - i: Arc, -} - -/// Builder to construct a new downloader -pub struct DownloaderBuilder { - rp: Option, - ffmpeg: String, - #[cfg(feature = "indicatif")] - multi: Option, - #[cfg(feature = "indicatif")] - progress_style: Option, - filter: StreamFilter, - video_format: DownloadVideoFormat, - n_retries: u32, - path_precheck: bool, - #[cfg(feature = "audiotag")] - audio_tag: bool, - #[cfg(feature = "audiotag")] - crop_cover: bool, - client_types: Option>, -} - -struct DownloaderInner { - /// YT client - rp: RustyPipe, - /// HTTP client - http: Client, - /// Path to the ffmpeg binary - ffmpeg: String, - /// Global progress - #[cfg(feature = "indicatif")] - multi: Option, - /// Progress style - #[cfg(feature = "indicatif")] - progress_style: ProgressStyle, - /// Default stream filter - filter: StreamFilter, - /// Default video format - video_format: DownloadVideoFormat, - /// Number of retries in case of 403 error - n_retries: u32, - /// Check if destination path exists before player is fetched - path_precheck: bool, - /// Apply metadata to audio files - #[cfg(feature = "audiotag")] - audio_tag: bool, - /// Crop YT thumbnails to ensure square album covers - #[cfg(feature = "audiotag")] - crop_cover: bool, - /// Client types for fetching videos - client_types: Option>, -} - -/// Download query -pub struct DownloadQuery { - /// RustyPipe Downloader - dl: Downloader, - /// Video to download - video: DownloadVideo, - /// Destination - dest: DownloadDest, - /// Progress bar - #[cfg(feature = "indicatif")] - progress: Option, - /// Stream filter - filter: Option, - /// Target video format - video_format: Option, - /// Client types for fetching videos - client_types: Option>, -} - -/// Video to be downloaded -#[derive(Default)] -pub struct DownloadVideo { - id: String, - name: Option, - channel_id: Option, - channel_name: Option, - album_id: Option, - album_name: Option, - track_nr: Option, -} - -impl DownloadVideo { - /// Get the YouTube video id - pub fn id(&self) -> &str { - &self.id - } - - /// Create a new DownloadVideo from a YouTube entity - pub fn from_entity(video: &impl YtEntity) -> Self { - DownloadVideo { - id: video.id().to_owned(), - name: Some(video.name().to_owned()), - channel_id: video.channel_id().map(str::to_owned), - channel_name: video - .channel_name() - .map(|n| n.strip_suffix("- Topic").unwrap_or(n).trim().to_owned()), - album_id: None, - album_name: None, - track_nr: None, - } - } - - /// Create a new DownloadVideo from a YTM track - pub fn from_track(track: &TrackItem) -> Self { - DownloadVideo { - id: track.id.to_owned(), - name: Some(track.name.to_owned()), - channel_id: track.channel_id().map(str::to_owned), - channel_name: track.channel_name().map(str::to_owned), - album_id: track.album.as_ref().map(|b| b.id.to_owned()), - album_name: track.album.as_ref().map(|b| b.name.to_owned()), - track_nr: track.track_nr, - } - } -} - -#[derive(Clone)] -enum DownloadDest { - Default, - File(PathBuf), - Dir(PathBuf), - Template(PathBuf), -} - -fn video_filename(v: &DownloadVideo) -> String { - let mut n = format!("{} [{}]", v.name.as_deref().unwrap_or_default(), v.id); - if let Some(track_nr) = v.track_nr { - n = format!("{track_nr:02} {n}"); - } - filenamify_lim(&n) -} - -/// Video container format for downloading -#[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] -pub enum DownloadVideoFormat { - /// .mp4 - #[default] - Mp4, - /// .mkv - Mkv, - /// .webm - Webm, -} - -impl DownloadVideoFormat { - /// Get the video format file extension - pub fn extension(&self) -> &'static str { - match self { - DownloadVideoFormat::Mp4 => "mp4", - DownloadVideoFormat::Mkv => "mkv", - DownloadVideoFormat::Webm => "webm", - } - } - - /// Get the video format from the given file extension - pub fn from_extension(ext: &str) -> Option { - match ext { - "mp4" => Some(Self::Mp4), - "mkv" => Some(Self::Mkv), - "webm" => Some(Self::Webm), - _ => None, - } - } -} - -impl DownloadDest { - fn get_dest_path(&self, v: &DownloadVideo) -> PathBuf { - static RE_TEMPLATE: Lazy = Lazy::new(|| Regex::new(r#"\{\w+\} *"#).unwrap()); - - match self { - DownloadDest::Default => PathBuf::from(video_filename(v)), - DownloadDest::File(p) => p.clone(), - DownloadDest::Dir(p) => p.join(video_filename(v)), - DownloadDest::Template(t) => t - .iter() - .map(|part| { - let s = part.to_string_lossy(); - - let (mut replaced, last_end) = RE_TEMPLATE.find_iter(&s).fold( - (String::new(), 0), - |(mut acc, last_end), m| { - acc += &s[last_end..m.start()]; - let ms = m.as_str(); - let trimmed = ms.trim_end_matches(' '); - let repl: Option> = match trimmed.trim_matches(['{', '}']) { - "id" => Some(v.id.as_str().into()), - "title" => v.name.as_deref().map(Cow::from), - "channel" => v.channel_name.as_deref().map(Cow::from), - "channelId" => v.channel_id.as_deref().map(Cow::from), - "album" => v.album_name.as_deref().map(Cow::from), - "albumId" => v.album_id.as_deref().map(Cow::from), - "track" => v.track_nr.map(|n| format!("{n:02}").into()), - _ => None, - }; - if let Some(repl) = repl { - acc += &repl; - acc += &ms[trimmed.len()..]; // preceeding whitespace - } - (acc, m.end()) - }, - ); - replaced += &s[last_end..]; - replaced = replaced.trim().to_owned(); - - if replaced.is_empty() { - "-".to_owned() - } else { - filenamify_lim(&replaced) - } - }) - .collect(), - } - } -} - -impl Default for DownloaderBuilder { - fn default() -> Self { - Self { - rp: None, - ffmpeg: "ffmpeg".to_owned(), - #[cfg(feature = "indicatif")] - multi: None, - #[cfg(feature = "indicatif")] - progress_style: None, - filter: StreamFilter::new(), - video_format: DownloadVideoFormat::Mp4, - n_retries: 3, - path_precheck: false, - #[cfg(feature = "audiotag")] - audio_tag: false, - #[cfg(feature = "audiotag")] - crop_cover: false, - client_types: None, - } - } -} - -impl DownloaderBuilder { - /// Create a new [`DownloaderBuilder`] - /// - /// This is the same as [`Downloader::builder`] - pub fn new() -> Self { - Self::default() - } - - /// Use a custom [`RustyPipe`] client - #[must_use] - pub fn rustypipe(mut self, rp: &RustyPipe) -> Self { - self.rp = Some(rp.clone()); - self - } - - /// Set the path to ffmpeg, used to join video and audio files - /// - /// The default system-wide `ffmpeg` binary is used by default. - #[must_use] - pub fn ffmpeg>(mut self, ffmpeg: S) -> Self { - self.ffmpeg = ffmpeg.into(); - self - } - - /// Set the indicatif [`MultiProgress`] used to show download progress - /// for all downloads - #[cfg(feature = "indicatif")] - #[cfg_attr(docsrs, doc(cfg(feature = "indicatif")))] - #[must_use] - pub fn multi_progress(mut self, progress: MultiProgress) -> Self { - self.multi = Some(progress); - self - } - - /// Set the indicatif [`ProgressStyle`] for the progress bars displayed under `multi_progress` - #[cfg(feature = "indicatif")] - #[cfg_attr(docsrs, doc(cfg(feature = "indicatif")))] - #[must_use] - pub fn progress_style(mut self, style: ProgressStyle) -> Self { - self.progress_style = Some(style); - self - } - - /// Set the default [`StreamFilter`] for all downloads. - /// - /// The filter can be overridden for individual download queries. - #[must_use] - pub fn stream_filter(mut self, filter: StreamFilter) -> Self { - self.filter = filter; - self - } - - /// Set the [`DownloadVideoFormat`] of downloaded videos - #[must_use] - pub fn video_format(mut self, video_format: DownloadVideoFormat) -> Self { - self.video_format = video_format; - self - } - - /// Set the number of retries in case a download fails with a 403 error - #[must_use] - pub fn n_retries(mut self, n_retries: u32) -> Self { - self.n_retries = n_retries; - self - } - - /// Enable path precheck - /// - /// The downloader will check if the destination path - /// (predicted from the entity to download and the StreamFilter) exists and - /// skips the download with [`DownloadError::Exists`] without fetching any player data. - /// - /// This allows fast resumption of playlist downloads. - #[must_use] - pub fn path_precheck(mut self) -> Self { - self.path_precheck = true; - self - } - - /// Enable audio tagging - #[cfg(feature = "audiotag")] - #[cfg_attr(docsrs, doc(cfg(feature = "audiotag")))] - #[must_use] - pub fn audio_tag(mut self) -> Self { - self.audio_tag = true; - self - } - - /// Crop YouTube thumbnails to get square album covers - #[cfg(feature = "audiotag")] - #[cfg_attr(docsrs, doc(cfg(feature = "audiotag")))] - #[must_use] - pub fn crop_cover(mut self) -> Self { - self.crop_cover = true; - self - } - - /// Set the [`ClientType`] used to fetch the YT player - #[must_use] - pub fn client_type(mut self, client_type: ClientType) -> Self { - self.client_types = Some(vec![client_type]); - self - } - - /// Set a list of client types used to fetch the YT player - /// - /// The clients are used in the given order. If a client cannot fetch the requested video, - /// an attempt is made with the next one. - #[must_use] - pub fn client_types>>(mut self, client_types: T) -> Self { - self.client_types = Some(client_types.into()); - self - } - - /// Create a new, configured [`Downloader`] instance - pub fn build(self) -> Downloader { - self.build_with_client( - Client::builder() - .timeout(Duration::from_secs(20)) - .build() - .expect("http client"), - ) - } - - /// Create a new, configured [`Downloader`] instance using a custom Reqwest [`Client`] - pub fn build_with_client(self, http_client: Client) -> Downloader { - Downloader { - i: Arc::new(DownloaderInner { - rp: self.rp.unwrap_or_default(), - http: http_client, - ffmpeg: self.ffmpeg, - #[cfg(feature = "indicatif")] - multi: self.multi, - #[cfg(feature = "indicatif")] - progress_style: self.progress_style.unwrap_or_else(|| { - ProgressStyle::with_template("{msg}\n{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({bytes_per_sec}, {eta})") - .unwrap() - .progress_chars("#>-") - }), - filter: self.filter, - video_format: self.video_format, - n_retries: self.n_retries, - path_precheck: self.path_precheck, - #[cfg(feature = "audiotag")] - audio_tag: self.audio_tag, - #[cfg(feature = "audiotag")] - crop_cover: self.crop_cover, - client_types: self.client_types, - }), - } - } -} - -impl Default for Downloader { - fn default() -> Self { - DownloaderBuilder::new().build() - } -} - -impl Downloader { - /// Create a new [`Downloader`] using the given [`RustyPipe`] instance - pub fn new(rp: &RustyPipe) -> Self { - DownloaderBuilder::new().rustypipe(rp).build() - } - - /// Create a new [`DownloaderBuilder`] - /// - /// This is the same as [`DownloaderBuilder::new`] - pub fn builder() -> DownloaderBuilder { - DownloaderBuilder::default() - } - - fn query(&self, video: DownloadVideo) -> DownloadQuery { - DownloadQuery { - dl: self.clone(), - video, - dest: DownloadDest::Default, - #[cfg(feature = "indicatif")] - progress: None, - filter: None, - video_format: None, - client_types: None, - } - } - - /// Download a video with the given ID - #[must_use] - pub fn id>(&self, video_id: S) -> DownloadQuery { - self.query(DownloadVideo { - id: video_id.into(), - ..Default::default() - }) - } - - /// Download a video from a DownloadVideo object - #[must_use] - pub fn video(&self, video: DownloadVideo) -> DownloadQuery { - self.query(video) - } - - /// Download a video from a [`YtEntity`] object (e.g. playlist/channel video) - /// - /// Providing an entity has the advantage that the download path can be determined before the video - /// is fetched, so already downloaded videos get skipped right away. - #[must_use] - pub fn entity(&self, video: &impl YtEntity) -> DownloadQuery { - self.query(DownloadVideo::from_entity(video)) - } - - /// Download a video from a [`TrackItem`] (YouTube Music album/playlist item) - /// - /// Providing an entity has the advantage that the download path can be determined before the video - /// is fetched, so already downloaded videos get skipped right away. - /// - /// If an album track is downloaded, this method will also add the track number to the downloaded file - #[must_use] - pub fn track(&self, track: &TrackItem) -> DownloadQuery { - self.query(DownloadVideo::from_track(track)) - } -} - -/// Output data from downloading a video -pub struct DownloadResult { - /// Download destination path - pub dest: PathBuf, - /// Fetched vvideo player data - pub player_data: VideoPlayer, -} - -impl DownloadQuery { - /// Update the video format from the given path extension - /// - /// The video format is not updated if it was already manually set - fn update_video_format(&mut self, path: &Path) { - if self.video_format.is_none() { - self.video_format = path - .extension() - .and_then(|ext| ext.to_str()) - .and_then(DownloadVideoFormat::from_extension); - } - } - - /// Download to the given file - /// - /// Note that the file extension may be changed to fit the reuested video/audio format. - /// Refer to the [`DownloadResult`] to get the actual path after downloading. - #[must_use] - pub fn to_file>(mut self, file: P) -> Self { - let file = file.into(); - self.update_video_format(&file); - self.dest = DownloadDest::File(file); - self - } - - /// Download to the given directory - /// - /// The filename is created by this template: `{track} {title} [{id}]`. - /// - /// You can use a custom filename template using [`DownloadQuery::to_template`] - #[must_use] - pub fn to_dir>(mut self, dir: P) -> Self { - self.dest = DownloadDest::Dir(dir.into()); - self - } - - /// Download to a path determined by a template - /// - /// Templates are paths that may contain variables for video metadata. - /// - /// ## Variables - /// - `{id}` Video ID - /// - `{title}` Video title - /// - `{channel}` Channel name - /// - `{channel_id}` Channel ID - /// - `{album}` Album - /// - `{album_id}` Album ID - /// - `{track}` Track number - /// - /// Whitespace between template variables is automatically removed if a variable - /// contains no data (e.g. `{track} {name}` is equal to `{name}` if a video without - /// track number is downloaded). - /// - /// Note that the file extension may be changed to fit the reuested video/audio format. - /// Refer to the [`DownloadResult`] to get the actual path after downloading. - #[must_use] - pub fn to_template>(mut self, tmpl: P) -> Self { - let tmpl = tmpl.into(); - self.update_video_format(&tmpl); - self.dest = DownloadDest::Template(tmpl); - self - } - - /// Show the progress of this download using a Indicatif [`ProgressBar`] - #[cfg(feature = "indicatif")] - #[cfg_attr(docsrs, doc(cfg(feature = "indicatif")))] - #[must_use] - pub fn progress_bar(mut self, progress: ProgressBar) -> Self { - self.progress = Some(progress); - self - } - - /// Set a [`StreamFilter`] for choosing a stream to be downloaded - #[must_use] - pub fn stream_filter(mut self, filter: StreamFilter) -> Self { - self.filter = Some(filter); - self - } - - /// Set the [`DownloadVideoFormat`] of downloaded videos - #[must_use] - pub fn video_format(mut self, video_format: DownloadVideoFormat) -> Self { - self.video_format = Some(video_format); - self - } - - /// Set the [`ClientType`] used to fetch the YT player - #[must_use] - pub fn client_type(mut self, client_type: ClientType) -> Self { - self.client_types = Some(vec![client_type]); - self - } - - /// Set a list of client types used to fetch the YT player - /// - /// The clients are used in the given order. If a client cannot fetch the requested video, - /// an attempt is made with the next one. - #[must_use] - pub fn client_types>>(mut self, client_types: T) -> Self { - self.client_types = Some(client_types.into()); - self - } - - /// Download the video - /// - /// If no download path is set, the video is downloaded to the current directory - /// with a filename created by this template: `{track} {title} [{id}]`. - #[tracing::instrument(skip(self), level="error", fields(id = self.video.id))] - pub async fn download(&self) -> Result { - let mut last_err = None; - let mut failed_client = None; - - // Progress bar - #[cfg(feature = "indicatif")] - let pb = match &self.progress { - Some(progress) => Some(progress.clone()), - None => self.dl.i.multi.clone().map(|m| { - let pb = ProgressBar::new(1); - pb.set_style(self.dl.i.progress_style.clone()); - m.add(pb) - }), - }; - - for n in 0..=self.dl.i.n_retries { - let err = match self - .download_attempt( - n, - failed_client, - #[cfg(feature = "indicatif")] - &pb, - ) - .await - { - Ok(res) => return Ok(res), - Err(DownloadError::Forbidden { - client_type, - visitor_data, - }) => { - failed_client = Some(client_type); - DownloadError::Forbidden { - client_type, - visitor_data, - } - } - Err(DownloadError::Http(e)) => { - if !e.is_timeout() { - return Err(DownloadError::Http(e)); - } - DownloadError::Http(e) - } - Err(e) => return Err(e), - }; - - if n != self.dl.i.n_retries { - tracing::warn!("Retry attempt #{}. Error: {}", n + 1, err); - tokio::time::sleep(Duration::from_secs(1)).await; - } - last_err = Some(err); - } - Err(last_err.unwrap()) - } - - async fn download_attempt( - &self, - #[allow(unused_variables)] n: u32, - failed_client: Option, - #[cfg(feature = "indicatif")] pb: &Option, - ) -> Result { - let filter = self.filter.as_ref().unwrap_or(&self.dl.i.filter); - let video_format = self.video_format.unwrap_or(self.dl.i.video_format); - - // Check if already downloaded - if self.video.name.is_some() && self.dl.i.path_precheck { - let op = self.dest.get_dest_path(&self.video); - - if filter.is_video_none() { - for ext in ["m4a", "opus"] { - let p = op.with_extension(ext); - if p.is_file() { - return Err(DownloadError::Exists(p)); - } - } - } else { - let p = op.with_extension(video_format.extension()); - if p.is_file() { - return Err(DownloadError::Exists(p)); - } - } - } - - #[cfg(feature = "indicatif")] - let attempt_suffix = if n > 0 { - format!(" (retry #{n})") - } else { - String::new() - }; - #[cfg(feature = "indicatif")] - if let Some(pb) = pb { - if let Some(n) = &self.video.name { - pb.set_message(format!("Fetching player data for {n}{attempt_suffix}")); - } else { - pb.set_message(format!("Fetching player data{attempt_suffix}")); - } - } - - let q = self.dl.i.rp.query(); - - let mut client_types = Cow::Borrowed( - self.client_types - .as_ref() - .or(self.dl.i.client_types.as_ref()) - .map(Vec::as_slice) - .unwrap_or(q.player_client_order()), - ); - - // If the last download failed, try another client if possible - if let Some(failed_client) = failed_client { - if let Some(pos) = client_types.iter().position(|c| c == &failed_client) { - let p2 = pos + 1; - if p2 < client_types.len() { - let mut v = client_types[p2..].to_vec(); - v.extend(&client_types[..p2]); - client_types = v.into(); - } - } - } - - let player_data = q.player_from_clients(&self.video.id, &client_types).await?; - let user_agent = q.user_agent(player_data.client_type); - - // Select streams to download - let (video, audio) = player_data.select_video_audio_stream(filter); - - if video.is_none() && audio.is_none() { - if player_data.drm.is_some() { - return Err(DownloadError::Source("video is DRM-protected".into())); - } - return Err(DownloadError::Source("no stream found".into())); - } - - let extension = match video { - Some(_) => video_format.extension(), - None => match audio { - Some(audio) => match audio.codec { - AudioCodec::Mp4a => "m4a", - AudioCodec::Opus => "opus", - AudioCodec::Ac3 => "ac3", - AudioCodec::Ec3 => "eac3", - _ => return Err(DownloadError::Source("unknown audio codec".into())), - }, - None => unreachable!(), - }, - }; - - let (name, details) = match &player_data.details.name { - Some(n) => (n.to_owned(), None), - None => { - let details = self.dl.i.rp.query().video_details(&self.video.id).await?; - (details.name.to_owned(), Some(details)) - } - }; - - let pv = DownloadVideo { - id: player_data.details.id.to_owned(), - name: Some(name.to_owned()), - channel_id: Some(player_data.details.channel_id.to_owned()), - channel_name: player_data - .details - .channel_name - .clone() - .or(details.as_ref().map(|d| d.channel.name.to_owned())), - album_id: self.video.album_id.to_owned(), - album_name: self.video.album_name.to_owned(), - track_nr: self.video.track_nr, - }; - let output_path = self.dest.get_dest_path(&pv).with_extension(extension); - - if output_path.exists() { - return Err(DownloadError::Exists(output_path)); - } - if let Some(parent) = output_path.parent() { - std::fs::create_dir_all(parent)?; - } - - let mut downloads: Vec = Vec::new(); - - if let Some(v) = video { - downloads.push(StreamDownload { - file: output_path.with_extension(format!("video{}", v.format.extension())), - url: v.url.clone(), - video_codec: Some(v.codec), - audio_codec: None, - }); - } - if let Some(a) = audio { - downloads.push(StreamDownload { - file: output_path.with_extension(format!("audio{}", a.format.extension())), - url: a.url.clone(), - video_codec: None, - audio_codec: Some(a.codec), - }); - } - - #[cfg(feature = "indicatif")] - if let Some(pb) = pb { - pb.set_message(format!("Downloading {name}{attempt_suffix}")) - } - let downloads = download_streams( - downloads, - &self.dl.i.http, - &user_agent, - #[cfg(feature = "indicatif")] - pb.clone(), - ) - .await - .map_err(|e| { - if let DownloadError::Http(e) = &e { - if e.status() == Some(StatusCode::FORBIDDEN) { - // 403 errors may occur due to bad visitor data IDs - if let Some(vd) = &player_data.visitor_data { - q.remove_visitor_data(vd); - } - return DownloadError::Forbidden { - client_type: player_data.client_type, - visitor_data: player_data.visitor_data.clone(), - }; - } - } - e - })?; - - #[cfg(feature = "indicatif")] - if let Some(pb) = &pb { - pb.set_message(format!("Converting {name}")); - pb.set_style( - ProgressStyle::with_template("{msg}\n{spinner:.green} [{elapsed_precise}]") - .unwrap(), - ); - pb.enable_steady_tick(Duration::from_millis(500)); - } - - convert_streams(&downloads, &output_path, &self.dl.i.ffmpeg, &name).await?; - - // Tag audio file - #[cfg(feature = "audiotag")] - if self.dl.i.audio_tag && video.is_none() && matches!(extension, "m4a" | "opus") { - let (details, track) = match details { - Some(d) => (d, self.dl.i.rp.query().music_details(&self.video.id).await?), - None => { - let q = self.dl.i.rp.query(); - tokio::try_join!( - q.video_details(&self.video.id), - q.music_details(&self.video.id) - )? - } - }; - self.apply_audio_tags( - &output_path, - details, - &player_data.details, - track.track, - pv.track_nr, - ) - .await?; - } - - #[cfg(feature = "indicatif")] - if let Some(pb) = pb { - pb.disable_steady_tick(); - } - - // Delete original files - for d in &downloads { - fs::remove_file(&d.file).await?; - } - - #[cfg(feature = "indicatif")] - if let Some(pb) = pb { - pb.finish_and_clear(); - } - Ok(DownloadResult { - dest: output_path, - player_data, - }) - } - - #[cfg(feature = "audiotag")] - async fn apply_audio_tags( - &self, - file: &Path, - details: VideoDetails, - player_details: &VideoPlayerDetails, - track: TrackItem, - track_nr: Option, - ) -> Result<()> { - use std::{io::Cursor, num::NonZeroU32}; - - let mut tagged_file = lofty::read_from_path(file)?; - let tag = match tagged_file.primary_tag_mut() { - Some(primary_tag) => primary_tag, - None => { - if let Some(first_tag) = tagged_file.first_tag_mut() { - first_tag - } else { - let tag_type = tagged_file.primary_tag_type(); - tagged_file.insert_tag(Tag::new(tag_type)); - - tagged_file.primary_tag_mut().unwrap() - } - } - }; - - let description = details.description.to_plaintext(); - - tag.set_album( - track - .album - .map(|b| b.name) - .unwrap_or_else(|| track.name.clone()), - ); - tag.set_artist( - track - .artists - .into_iter() - .next() - .map(|a| a.name) - .unwrap_or(details.channel.name), - ); - tag.set_title(track.name); - if let Some(release_date) = extract_yt_release_date(&description, details.publish_date) { - if let Ok(date_str) = release_date.format(&YMD_FORMAT) { - tag.insert_text(ItemKey::RecordingDate, date_str); - } - } - tag.set_comment(description); - if let Some(track_nr) = track_nr { - tag.set_track(track_nr.into()); - } - - // For YTM tracks the music details contain a high quality, square cover image, but for music videos - // the cover images are cropped and of worse resolution. - // Therefore we switch to the thumbnails from the player data if the music details contain no square - // thumbnails. - let thumbnail_music = track.cover.into_iter().max_by_key(|c| c.height); - let thumbnail = if thumbnail_music - .as_ref() - .map(|tn| tn.height == tn.width) - .unwrap_or_default() - { - thumbnail_music - } else { - let thumbnail_player = player_details - .thumbnail - .iter() - .max_by_key(|c| c.height) - .cloned(); - thumbnail_player.or(thumbnail_music) - }; - - if let Some(thumbnail) = thumbnail { - // Attempt to get the higher resolution, uncropped maxresdefault.jpg thumbnail if available - let mut resp = None; - if thumbnail.height != thumbnail.width { - if let Ok(x) = self - .dl - .i - .http - .get(format!( - "https://i.ytimg.com/vi/{}/maxresdefault.jpg", - track.id - )) - .send() - .await? - .error_for_status() - { - resp = Some(x); - } - } - - let resp = match resp { - Some(resp) => resp, - None => self - .dl - .i - .http - .get(thumbnail.url) - .send() - .await? - .error_for_status()?, - }; - - let img_type = resp - .headers() - .get(header::CONTENT_TYPE) - .and_then(|fmt| fmt.to_str().ok()) - .and_then(image::ImageFormat::from_mime_type); - let img_bts = resp.bytes().await?; - - let mut lofty_img = if self.dl.i.crop_cover { - // Crop cover image if it is not square - if thumbnail.height != thumbnail.width { - let mut img = if let Some(fmt) = img_type { - image::load_from_memory_with_format(&img_bts, fmt)? - } else { - image::load_from_memory(&img_bts)? - }; - - let crop = smartcrop::find_best_crop_no_borders( - &img, - NonZeroU32::MIN, - NonZeroU32::MIN, - ) - .map_err(|e| DownloadError::AudioTag(format!("image crop: {e}").into()))? - .crop; - img = img.crop_imm(crop.x, crop.y, crop.width, crop.height); - let mut enc_bts = Vec::new(); - img.write_with_encoder(image::codecs::jpeg::JpegEncoder::new_with_quality( - &mut enc_bts, - 90, - ))?; - let mut rd = Cursor::new(enc_bts); - Picture::from_reader(&mut rd)? - } else { - let mut rd = Cursor::new(img_bts); - Picture::from_reader(&mut rd)? - } - } else { - let mut rd = Cursor::new(img_bts); - Picture::from_reader(&mut rd)? - }; - - lofty_img.set_pic_type(lofty::picture::PictureType::CoverFront); - tag.set_picture(0, lofty_img); - } - - tag.save_to_path(file, WriteOptions::default())?; - Ok(()) - } -} - fn get_download_range(offset: u64, size: Option) -> Range { - let mut rng = rand::rng(); - let chunk_size = rng.random_range(CHUNK_SIZE_MIN..CHUNK_SIZE_MAX); + let mut rng = rand::thread_rng(); + let chunk_size = rng.gen_range(CHUNK_SIZE_MIN..CHUNK_SIZE_MAX); let mut chunk_end = offset + chunk_size; if let Some(size) = size { @@ -1082,7 +46,7 @@ fn get_download_range(offset: u64, size: Option) -> Range { } fn parse_cr_header(cr_header: &str) -> Result<(u64, u64)> { - static PATTERN: Lazy = Lazy::new(|| Regex::new(r"bytes (\d+)-(\d+)/(\d+)").unwrap()); + static PATTERN: Lazy = Lazy::new(|| Regex::new(r#"bytes (\d+)-(\d+)/(\d+)"#).unwrap()); let captures = PATTERN.captures(cr_header).ok_or_else(|| { DownloadError::Progressive( @@ -1100,26 +64,11 @@ fn parse_cr_header(cr_header: &str) -> Result<(u64, u64)> { )) } -fn filenamify_lim(name: &str) -> String { - let lim = 200; - let n = filenamify::filenamify(name); - - if n.len() > lim { - n.char_indices() - .take_while(|(i, _)| i < &lim) - .map(|(_, c)| c) - .collect::() - } else { - n - } -} - -async fn download_single_file( +async fn download_single_file>( url: &str, - output: &Path, - http: &Client, - user_agent: &str, - #[cfg(feature = "indicatif")] pb: Option, + output: P, + http: Client, + pb: ProgressBar, ) -> Result<()> { // Check if file is already downloaded let output_path: PathBuf = output.into(); @@ -1150,7 +99,6 @@ async fn download_single_file( let res = http .head(url.to_owned()) - .header(header::USER_AGENT, user_agent) .header(header::RANGE, "bytes=0-0") .send() .await? @@ -1177,11 +125,8 @@ async fn download_single_file( size = Some(original_size); offset = file_size; - #[cfg(feature = "indicatif")] - if let Some(pb) = &pb { - pb.inc_length(original_size); - pb.inc(offset); - } + pb.inc_length(original_size); + pb.inc(offset); } Ordering::Equal => { // Already downloaded @@ -1201,50 +146,16 @@ async fn download_single_file( } } - tracing::debug!("downloading {} to {}", url, output.to_string_lossy()); - let mut file = fs::OpenOptions::new() .append(true) .create(true) .open(&output_path_tmp) .await?; - let res = if is_gvideo && size.is_some() { - download_chunks_by_param( - http, - &mut file, - url, - size.unwrap(), - offset, - user_agent, - #[cfg(feature = "indicatif")] - pb, - ) - .await + if is_gvideo && size.is_some() { + download_chunks_by_param(http, &mut file, url, size.unwrap(), offset, pb).await?; } else { - download_chunks_by_header( - http, - &mut file, - url, - size, - offset, - user_agent, - #[cfg(feature = "indicatif")] - pb, - ) - .await - }; - - drop(file); - if let Err(e) = res { - // Remove temporary file if nothing was downloaded (e.g. 403 error) - if std::fs::metadata(&output_path_tmp) - .map(|md| md.len() == 0) - .unwrap_or_default() - { - _ = std::fs::remove_file(&output_path_tmp); - } - return Err(e); + download_chunks_by_header(http, &mut file, url, size, offset, pb).await?; } fs::rename(&output_path_tmp, &output_path).await?; @@ -1255,24 +166,22 @@ async fn download_single_file( // This is the standardized method that works on all web servers, // but I have observed throttling using this method. async fn download_chunks_by_header( - http: &Client, + http: Client, file: &mut File, url: &str, size: Option, offset: u64, - user_agent: &str, - #[cfg(feature = "indicatif")] pb: Option, + pb: ProgressBar, ) -> Result<()> { let mut offset = offset; let mut size = size; loop { let range = get_download_range(offset, size); - tracing::debug!("Fetching range {}-{}", range.start, range.end); + debug!("Fetching range {}-{}", range.start, range.end); let res = http .get(url.to_owned()) - .header(header::USER_AGENT, user_agent) .header(header::ORIGIN, "https://www.youtube.com") .header(header::REFERER, "https://www.youtube.com/") .header( @@ -1283,12 +192,6 @@ async fn download_chunks_by_header( .await? .error_for_status()?; - if res.content_length().unwrap_or_default() == 0 { - return Err(DownloadError::Progressive( - format!("empty chunk {}-{}", range.start, range.end).into(), - )); - } - // Content-Range: bytes 0-100/451368980 let cr_header = res .headers() @@ -1308,21 +211,15 @@ async fn download_chunks_by_header( offset = parsed_offset + 1; if size.is_none() { size = Some(parsed_size); - #[cfg(feature = "indicatif")] - if let Some(pb) = &pb { - pb.inc_length(parsed_size); - } + pb.inc_length(parsed_size); } - tracing::debug!("Retrieving chunks..."); + debug!("Retrieving chunks..."); let mut stream = res.bytes_stream(); while let Some(item) = stream.next().await { // Retrieve chunk. let mut chunk = item?; - #[cfg(feature = "indicatif")] - if let Some(pb) = &pb { - pb.inc(chunk.len() as u64); - } + pb.inc(chunk.len() as u64); file.write_all_buf(&mut chunk).await?; } @@ -1336,59 +233,42 @@ async fn download_chunks_by_header( // Use the `range` url parameter to download a stream in chunks. // This ist used by YouTube's web player. The file size // must be known beforehand (it is included in the stream url). -#[allow(clippy::too_many_arguments)] async fn download_chunks_by_param( - http: &Client, + http: Client, file: &mut File, url: &str, size: u64, offset: u64, - user_agent: &str, - #[cfg(feature = "indicatif")] pb: Option, + pb: ProgressBar, ) -> Result<()> { let mut offset = offset; - #[cfg(feature = "indicatif")] - if let Some(pb) = &pb { - pb.inc_length(size); - } + pb.inc_length(size); loop { let range = get_download_range(offset, Some(size)); - tracing::debug!("Fetching range {}-{}", range.start, range.end); - - let urlp = - Url::parse_with_params(url, [("range", &format!("{}-{}", range.start, range.end))]) - .map_err(|e| DownloadError::Progressive(format!("url parsing: {e}").into()))?; + debug!("Fetching range {}-{}", range.start, range.end); let res = http - .get(urlp) - .header(header::USER_AGENT, user_agent) + .get(format!("{}&range={}-{}", url, range.start, range.end)) .header(header::ORIGIN, "https://www.youtube.com") .header(header::REFERER, "https://www.youtube.com/") .send() .await? .error_for_status()?; - let clen = res.content_length().unwrap_or_default(); - if clen == 0 { - return Err(DownloadError::Progressive( - format!("empty chunk {}-{}", range.start, range.end).into(), - )); - } + let clen = res.content_length().unwrap(); + debug!("Retrieving chunks..."); let mut stream = res.bytes_stream(); while let Some(item) = stream.next().await { // Retrieve chunk. let mut chunk = item?; - #[cfg(feature = "indicatif")] - if let Some(pb) = &pb { - pb.inc(chunk.len() as u64); - } + pb.inc(chunk.len() as u64); file.write_all_buf(&mut chunk).await?; } offset += clen; - tracing::debug!("offset inc by {}, new: {}", clen, offset); + debug!("offset inc by {}, new: {}", clen, offset); if offset >= size { break; } @@ -1399,43 +279,180 @@ async fn download_chunks_by_param( #[allow(dead_code)] struct StreamDownload { file: PathBuf, + // track_name: String TODO: add for multiple audio languages, url: String, audio_codec: Option, video_codec: Option, } -async fn download_streams( - downloads: Vec, - http: &Client, - user_agent: &str, - #[cfg(feature = "indicatif")] pb: Option, -) -> Result> { - stream::iter(downloads.iter().map(Ok)) - .try_for_each_concurrent(2, |d| { - #[cfg(feature = "indicatif")] - let pb = pb.clone(); - async move { - download_single_file( - &d.url, - &d.file, - http, - user_agent, - #[cfg(feature = "indicatif")] - pb, - ) - .await - } - }) - .await?; +#[allow(clippy::too_many_arguments)] +pub async fn download_video( + player_data: &VideoPlayer, + output_dir: &str, + output_fname: Option, + output_format: Option, + filter: &StreamFilter<'_>, + ffmpeg: &str, + http: Client, + pb: ProgressBar, +) -> Result<()> { + // Download filepath + let download_dir = PathBuf::from(output_dir); + let title = player_data.details.name.clone(); + let output_fname_set = output_fname.is_some(); + let output_fname = output_fname.unwrap_or_else(|| { + filenamify::filenamify(format!("{} [{}]", title, player_data.details.id)) + }); - Ok(downloads) + // Select streams to download + let (video, audio) = player_data.select_video_audio_stream(filter); + + if video.is_none() && audio.is_none() { + return Err(DownloadError::Input("no stream found".into())); + } + + let format = output_format.unwrap_or( + match video { + Some(_) => "mp4", + None => match audio { + Some(audio) => match audio.codec { + AudioCodec::Mp4a => "m4a", + AudioCodec::Opus => "opus", + _ => return Err(DownloadError::Input("unknown audio codec".into())), + }, + None => unreachable!(), + }, + } + .to_owned(), + ); + + let output_path = download_dir.join(&output_fname).with_extension(&format); + if output_path.exists() { + // If the downloaded video already exists, only error if the download path was + // chosen explicitly. + if output_fname_set { + return Err(DownloadError::Input( + format!("File {} already exists", output_path.to_string_lossy()).into(), + ))?; + } + info!( + "Downloaded video {} already exists", + output_path.to_string_lossy() + ); + return Ok(()); + } + + match (video, audio) { + // Downloading combined video/audio stream (no conversion) + (Some(video), None) => { + pb.set_message(format!("Downloading {title}")); + download_single_file(&video.url, output_path, http, pb.clone()).await?; + } + // Downloading audio only + (None, Some(audio)) => { + pb.set_message(format!("Downloading {title}")); + + let (dl_path, postprocessor) = match (audio.format, audio.codec) { + (AudioFormat::M4a, AudioCodec::Mp4a) => (output_path.clone(), None), + (AudioFormat::Webm, AudioCodec::Opus) => ( + download_dir.join(format!( + "{}.audio{}", + output_fname, + audio.format.extension() + )), + Some(AudioFormat::Webm), + ), + _ => (output_path.clone(), None), + }; + + download_single_file(&audio.url, &dl_path, http, pb.clone()).await?; + + if let Some(postprocessor) = postprocessor { + pb.set_message(format!("Converting {title}")); + if postprocessor == AudioFormat::Webm { + rustypipe_postprocessor::ogg_from_webm::process(&dl_path, &output_path)?; + } + fs::remove_file(&dl_path).await?; + } + } + // Downloading split video/audio streams (requires conversion with ffmpeg) + _ => { + let mut downloads: Vec = Vec::new(); + + if let Some(v) = video { + downloads.push(StreamDownload { + file: download_dir.join(format!( + "{}.video{}", + output_fname, + v.format.extension() + )), + url: v.url.clone(), + video_codec: Some(v.codec), + audio_codec: None, + }); + } + if let Some(a) = audio { + downloads.push(StreamDownload { + file: download_dir.join(format!( + "{}.audio{}", + output_fname, + a.format.extension() + )), + url: a.url.clone(), + video_codec: None, + audio_codec: Some(a.codec), + }); + } + + pb.set_message(format!("Downloading {title}")); + download_streams(&downloads, http, pb.clone()).await?; + + pb.set_message(format!("Converting {title}")); + pb.set_style( + ProgressStyle::with_template("{msg}\n{spinner:.green} [{elapsed_precise}]") + .unwrap(), + ); + pb.enable_steady_tick(Duration::from_millis(100)); + convert_streams(&downloads, output_path, ffmpeg).await?; + pb.disable_steady_tick(); + + // Delete original files + stream::iter(&downloads) + .map(|d| fs::remove_file(d.file.clone())) + .buffer_unordered(downloads.len()) + .collect::>() + .await + .into_iter() + .collect::>()?; + } + } + + pb.finish_and_clear(); + Ok(()) } -async fn convert_streams( - downloads: &[StreamDownload], - output: &Path, +async fn download_streams( + downloads: &Vec, + http: Client, + pb: ProgressBar, +) -> Result<()> { + let n = downloads.len(); + + stream::iter(downloads) + .map(|d| download_single_file(&d.url, d.file.clone(), http.clone(), pb.clone())) + .buffer_unordered(n) + .collect::>() + .await + .into_iter() + .collect::>>()?; + + Ok(()) +} + +async fn convert_streams>( + downloads: &Vec, + output: P, ffmpeg: &str, - title: &str, ) -> Result<()> { let output_path: PathBuf = output.into(); @@ -1452,11 +469,11 @@ async fn convert_streams( args.append(&mut mapping_args); - args.push("-c".into()); - args.push("copy".into()); - - args.push("-metadata".into()); - args.push(format!("title={title}").into()); + // Combining multiple streams, keep codecs + if downloads.len() > 1 { + args.push("-c".into()); + args.push("copy".into()); + } args.push(output_path.into()); @@ -1473,84 +490,3 @@ async fn convert_streams( } Ok(()) } - -#[cfg(feature = "audiotag")] -const YMD_FORMAT: &[time::format_description::FormatItem] = - time::macros::format_description!("[year]-[month]-[day]"); - -#[cfg(feature = "audiotag")] -fn extract_yt_release_date( - description: &str, - publish_date: Option, -) -> Option { - static RELEASE_DATE_REGEX: Lazy = - Lazy::new(|| Regex::new(r"Released on: (\d{4}-\d{2}-\d{2})").unwrap()); - - RELEASE_DATE_REGEX - .captures(description) - .and_then(|cap| { - let raw_date = &cap[1]; - Date::parse(raw_date, YMD_FORMAT).ok() - }) - .map(|release_date| { - if let Some(upload_date) = publish_date { - // Prefer the video upload date if it lies within 4 days of the release date - let upload_date = upload_date.date(); - let diff = (upload_date - release_date).abs(); - if diff < time::Duration::days(4) { - return upload_date; - } - } - release_date - }) - .or_else(|| publish_date.map(|d| d.date())) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn template() { - let dest = - DownloadDest::Template(PathBuf::from("{channel}/{album}/{track} {title} [{id}]")); - let track_path = dest.get_dest_path(&DownloadVideo { - id: "a3Fo1vYyiDw".to_owned(), - name: Some("Volle Kraft voraus".to_owned()), - channel_id: Some("UCE7_p3lcXA-YXRZp2PjrgYw".to_owned()), - channel_name: Some("Helene Fischer".to_owned()), - album_id: Some("MPREb_O2gXCdCVGsZ".to_owned()), - album_name: Some("Rausch (Deluxe)".to_owned()), - track_nr: Some(1), - }); - assert_eq!( - track_path.to_str().unwrap(), - "Helene Fischer/Rausch (Deluxe)/01 Volle Kraft voraus [a3Fo1vYyiDw]" - ); - - let video_path = dest.get_dest_path(&DownloadVideo { - id: "5en96GIijXk".to_owned(), - name: Some("a pretty cloud, and a happy duck".to_owned()), - channel_id: Some("UCl2mFZoRqjw_ELax4Yisf6w".to_owned()), - channel_name: Some("Louis Rossmann".to_owned()), - album_id: None, - album_name: None, - track_nr: None, - }); - assert_eq!( - video_path.to_str().unwrap(), - "Louis Rossmann/-/a pretty cloud, and a happy duck [5en96GIijXk]" - ); - - let ido_path = dest.get_dest_path(&DownloadVideo { - id: "5en96GIijXk".to_owned(), - name: None, - channel_id: None, - channel_name: None, - album_id: None, - album_name: None, - track_nr: None, - }); - assert_eq!(ido_path.to_str().unwrap(), "-/-/[5en96GIijXk]"); - } -} diff --git a/downloader/src/util.rs b/downloader/src/util.rs index 5f87339..220470e 100644 --- a/downloader/src/util.rs +++ b/downloader/src/util.rs @@ -1,8 +1,28 @@ -use std::collections::BTreeMap; +use std::{borrow::Cow, collections::BTreeMap}; use reqwest::Url; -use crate::DownloadError; +/// Error from the video downloader +#[derive(thiserror::Error, Debug)] +#[non_exhaustive] +pub enum DownloadError { + /// Error from the HTTP client + #[error("http error: {0}")] + Http(#[from] reqwest::Error), + /// File IO error + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("FFmpeg error: {0}")] + Ffmpeg(Cow<'static, str>), + #[error("Progressive download error: {0}")] + Progressive(Cow<'static, str>), + #[error("input error: {0}")] + Input(Cow<'static, str>), + #[error("error: {0}")] + Other(Cow<'static, str>), + #[error("Postprocessing error: {0}")] + Postprocessing(#[from] rustypipe_postprocessor::PostprocessingError), +} /// Split an URL into its base string and parameter map /// diff --git a/downloader/tests/tests.rs b/downloader/tests/tests.rs deleted file mode 100644 index b8a3987..0000000 --- a/downloader/tests/tests.rs +++ /dev/null @@ -1,127 +0,0 @@ -use std::{fs, os::unix::fs::MetadataExt, path::Path, process::Command}; - -use path_macro::path; -use rstest::{fixture, rstest}; -use rustypipe::{client::RustyPipe, model::AudioCodec, param::StreamFilter}; -use rustypipe_downloader::Downloader; -use temp_testdir::TempDir; - -/// Get a new RusttyPipe instance -#[fixture] -fn rp() -> RustyPipe { - let vdata = std::env::var("YT_VDATA").ok(); - RustyPipe::builder() - .strict() - .storage_dir(path!(env!("CARGO_MANIFEST_DIR") / "..")) - .visitor_data_opt(vdata) - .build() - .unwrap() -} - -#[rstest] -#[tokio::test] -async fn download_video(rp: RustyPipe) { - let td = TempDir::default(); - let td_path = td.to_path_buf(); - - let dl = Downloader::builder().rustypipe(&rp).build(); - - let res = dl - .id("UXqq0ZvbOnk") - .to_dir(&td_path) - .stream_filter(StreamFilter::new().video_max_res(480)) - .download() - .await - .unwrap(); - - assert_eq!( - res.dest, - path!(td_path / "CHARGE - Blender Open Movie [UXqq0ZvbOnk].mp4") - ); - assert_eq!(res.player_data.details.id, "UXqq0ZvbOnk"); -} - -#[rstest] -#[tokio::test] -async fn download_music(rp: RustyPipe) { - let td = TempDir::default(); - let td_path = td.to_path_buf(); - - #[allow(unused_mut)] - let mut dl = Downloader::builder().rustypipe(&rp); - #[cfg(feature = "audiotag")] - { - dl = dl.audio_tag().crop_cover(); - } - let dl = dl.build(); - - let res = dl - .id("bVtv3st8bgc") - .to_dir(&td_path) - .stream_filter( - StreamFilter::new() - .no_video() - .audio_codecs([AudioCodec::Opus]), - ) - .download() - .await - .unwrap(); - - assert_eq!( - res.dest, - path!(td_path / "Lord of the Riffs [bVtv3st8bgc].opus") - ); - assert_eq!(res.player_data.details.id, "bVtv3st8bgc"); - let fm = fs::metadata(&res.dest).unwrap(); - assert_gte(fm.size(), 6_000_000, "file size"); - assert_audio_meta( - &res.dest, - "Lord of the Riffs", - "Alexander Nakarada", - "Lord of the Riffs", - "2022-02-05", - ); -} - -/// Assert that number A is greater than or equal to number B -#[track_caller] -fn assert_gte(a: T, b: T, msg: &str) { - assert!(a >= b, "expected >= {b} {msg}, got {a}"); -} - -#[track_caller] -fn assert_audio_meta(p: &Path, title: &str, artist: &str, album: &str, date: &str) { - let res = Command::new("ffprobe") - .args([ - "-loglevel", - "error", - "-show_entries", - "stream_tags", - "-of", - "json", - ]) - .arg(p) - .output() - .unwrap(); - if !res.status.success() { - panic!("ffprobe error\n{}", String::from_utf8_lossy(&res.stderr)) - } - let res_json = serde_json::from_slice::(&res.stdout).unwrap(); - let tags = &res_json["streams"][0]["tags"]; - assert_eq!(tags["TITLE"].as_str(), Some(title)); - assert_eq!(tags["ARTIST"].as_str(), Some(artist)); - assert_eq!(tags["ALBUM"].as_str(), Some(album)); - assert_eq!(tags["DATE"].as_str(), Some(date)); -} - -/// This is just a static check to make sure all RustyPipe futures can be sent -/// between threads safely. -/// Otherwise this may cause issues when integrating RustyPipe into async projects. -#[allow(unused)] -async fn all_send_and_sync() { - fn send_and_sync(t: T) {} - - let dl = Downloader::default(); - let dlq = dl.id(""); - send_and_sync(dlq.download()); -} diff --git a/notes/AB_Tests.md b/notes/AB_Tests.md index 8742ef6..9a2fcf8 100644 --- a/notes/AB_Tests.md +++ b/notes/AB_Tests.md @@ -3,13 +3,13 @@ When YouTube introduces a new feature, it does so gradually. When a user creates a new session, YouTube decided randomly which new features should be enabled. -YouTube sessions are identified by the visitor data ID. This cookie is sent with every -API request using the `context.client.visitor_data` JSON parameter. It is also returned -in the `responseContext.visitorData` response parameter and stored as the `__SECURE-YEC` -cookie. +YouTube sessions are identified by the visitor data cookie. This cookie is sent with +every API request using the `context.client.visitor_data` JSON parameter. It is also +returned in the `responseContext.visitorData` response parameter and stored as the +`__SECURE-YEC` cookie. -By sending the same visitor data ID, A/B tests can be reproduced, which is important for -testing alternative YouTube clients. +By sending the same visitor data cookie, A/B tests can be reproduced, which is important +for testing alternative YouTube clients. This page lists all A/B tests that were encountered while maintaining the RustyPipe client. @@ -24,14 +24,6 @@ to the new feature. - 🔴 **High** Changes to the functionality of YouTube that will require API changes for alternative clients -**Status:** - -- Discontinued (0%) -- Experimental (<3%) -- Common (>3%) -- Frequent (>40%) -- Stabilized (100%) - If you want to check how often these A/B tests occur, you can use the `codegen` tool with the following command: `rustypipe-codegen ab-test `. @@ -40,7 +32,6 @@ with the following command: `rustypipe-codegen ab-test `. - **Encountered on:** 24.09.2022 - **Impact:** 🟡 Medium - **Endpoint:** next (video details) -- **Status:** Stabilized ![A/B test 1 screenshot](./_img/ab_1.png) @@ -130,7 +121,6 @@ UTF-16 index seperately. - **Encountered on:** 11.10.2022 - **Impact:** 🔴 High - **Endpoint:** browse (channel videos) -- **Status:** Stabilized ![A/B test 2 screenshot](./_img/ab_2.webp) @@ -227,7 +217,6 @@ Additionally the channel tab response model was slightly changed, now using a - **Encountered on:** 20.11.2022 - **Impact:** 🟡 Medium - **Endpoint:** search -- **Status:** Stabilized ![A/B test 3 screenshot](./_img/ab_3.png) @@ -287,7 +276,6 @@ Note that channels without handles still use the old data model, even on the sam - **Encountered on:** 1.04.2023 - **Impact:** 🟢 Low - **Endpoint:** browse (trending videos) -- **Status:** Discontinued YouTube moved the list of trending videos from the main _trending_ page to a separate tab (Videos). @@ -305,14 +293,13 @@ The data model for the video shelves did not change. **NEW** -![A/B test 4 new screenshot](./_img/ab_4_new.png) +![A/B test 4 old screenshot](./_img/ab_4_new.png) ## [5] Page header renderer on the Trending page - **Encountered on:** 1.05.2023 - **Impact:** 🟢 Low - **Endpoint:** browse (trending videos) -- **Status:** Stabilized YouTube changed the header renderer type on the trending page to a `pageHeaderRenderer`. @@ -372,732 +359,20 @@ YouTube changed the header renderer type on the trending page to a `pageHeaderRe - **Encountered on:** 13.05.2023 - **Impact:** 🟡 Medium - **Endpoint:** browse (music artist) -- **Status:** Stabilized -YouTube merged the 2 sections for singles and albums on artist pages together. Now there -is only a _Top Releases_ section. +YouTube merged the 2 sections for singles and albums on artist pages together. Now +there is only a *Top Releases* section. YouTube also changed the way the full discography page is fetched, surprisingly making it easier for alternative clients. The discography page now has its own content ID in the format of `MPAD` (Music Page Artist Discography). This page can be fetched with a regular browse request without requiring parameters to be parsed or a -visitor data ID to be set, as it was the case with the old system. +visitor data cookie to be set, as it was the case with the old system. **OLD** -![A/B test 6 old screenshot](./_img/ab_6_old.png) +![A/B test 4 old screenshot](./_img/ab_6_old.png) **NEW** -![A/B test 6 screenshot](./_img/ab_6_new.png) - -## [7] Short timeago format - -- **Encountered on:** 28.05.2023 -- **Impact:** 🟢 Low -- **Status:** Discontinued - -YouTube changed their date format from the long format (_21 hours ago_, _3 days ago_) to -a short format (_21h ago_, _3d ago_). - -## [8] Track playback count in search results and artist views - -- **Encountered on:** 29.06.2023 -- **Impact:** 🟡 Medium -- **Status:** Stabilized - -YouTube added the track playback count to search results and top artist tracks. In -exchange, they removed the "Song" type identifier from search results. - -![A/B test 8 old screenshot](./_img/ab_8_old.png) - -![A/B test 8 screenshot](./_img/ab_8.png) - -## [9] Playlists for Shorts - -- **Encountered on:** 26.06.2023 -- **Impact:** 🟡 Medium -- **Endpoint:** browse (playlist) -- **Status:** Stabilized - -![A/B test 9 screenshot](./_img/ab_9.png) - -Original issue: https://github.com/TeamNewPipe/NewPipeExtractor/issues/10774 - -YouTube added a filter system for playlists, allowing users to only see shorts/full -videos. - -When shorts filter is enabled or when there are only shorts in a playlist, YouTube -return shorts UI elements instead of standard video ones, the ones that are also used -for shorts shelves in searches and suggestions and shorts in the corresponding channel -tab. - -Since the reel items dont include upload date information you can circumvent this new UI -by using the mobile client. But that may change in the future. - -## [10] Channel About modal - -- **Encountered on:** 03.11.2023 -- **Impact:** 🟡 Medium -- **Endpoint:** browse (channel info) -- **Status:** Stabilized - -![A/B test 10 screenshot](./_img/ab_10.png) - -YouTube replaced the _About_ channel tab with a modal. This changes the way additional -channel metadata has to be fetched. - -The new modal uses a continuation request with a token which can be easily generated. -Attempts to fetch the old about tab with the A/B test enabled will lead to a redirect to -the main tab. - -## [11] Like-Button viewmodel - -- **Encountered on:** 03.11.2023 -- **Impact:** 🟢 Low -- **Endpoint:** next -- **Status:** Stabilized - -YouTube introduced an updated data model for the like/dislike buttons. The new model -looks needlessly complex but contains the same parsing-relevant data as the old model -(accessibility text to get like count). - -```json -{ - "segmentedLikeDislikeButtonViewModel": { - "likeButtonViewModel": { - "likeButtonViewModel": { - "toggleButtonViewModel": { - "toggleButtonViewModel": { - "defaultButtonViewModel": { - "buttonViewModel": { - "iconName": "LIKE", - "title": "4.2M", - "accessibilityText": "like this video along with 4,209,059 other people" - } - } - } - } - } - } - } -} -``` - -## [12] New channel page header - -- **Encountered on:** 29.01.2024 -- **Impact:** 🟢 Low -- **Endpoint:** browse -- **Status:** Stabilized - -YouTube introduced a new data model for channel headers, based on a -`"pageHeaderRenderer"`. The new model comes with more needless complexity that needs to -be accomodated. There are also no mobile/TV header images available any more. - -```json -{ - "pageHeaderViewModel": { - "title": { - "dynamicTextViewModel": { - "text": { - "content": "Doobydobap", - "attachmentRuns": [ - { - "startIndex": 10, - "length": 0, - "element": { - "type": { - "imageType": { - "image": { - "sources": [ - { - "clientResource": { - "imageName": "CHECK_CIRCLE_FILLED" - }, - "width": 14, - "height": 14 - } - ] - } - } - } - } - } - ] - } - } - }, - "image": { - "decoratedAvatarViewModel": { - "avatar": { - "avatarViewModel": { - "image": { - "sources": [ - { - "url": "https://yt3.googleusercontent.com/dm5Aq93xvVJz0NoVO88ieBkDXmuShCujGPlZ7qETMEPTrXvPUCFI3-BB6Xs_P-r6Uk3mnBy9zA=s72-c-k-c0x00ffffff-no-rj", - "width": 72, - "height": 72 - } - ] - } - } - } - } - }, - "metadata": { - "contentMetadataViewModel": { - "metadataRows": [ - { - "metadataParts": [ - { - "text": { - "content": "@Doobydobap" - } - }, - { - "text": { - "content": "3.74M subscribers" - } - }, - { - "text": { - "content": "345 videos", - "styleRuns": [ - { - "startIndex": 0, - "length": 10 - } - ] - } - } - ] - } - ] - } - }, - "banner": { - "imageBannerViewModel": { - "image": { - "sources": [ - { - "url": "https://yt3.googleusercontent.com/BvnAqgiursrXpmS9AgDLtkOSTQfOG_Dqn0KzY5hcwO9XrHTEQTVgaflI913f9KRp7d0U2qBp=w1060-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj", - "width": 1060, - "height": 175 - } - ] - } - } - } - } -} -``` - -## [13] Music album/playlist 2-column layout - -- **Encountered on:** 29.02.2024 -- **Impact:** 🟢 Low -- **Endpoint:** browse -- **Status:** Stabilized - -![A/B test 13 screenshot](./_img/ab_13.png) - -YouTube Music updated the layout of album and playlist pages. The new layout shows the -cover on the left side of the playlist content. - -## [14] Comments Framework update - -- **Encountered on:** 31.01.2024 -- **Impact:** 🟢 Low -- **Endpoint:** next -- **Status:** Stabilized - -YouTube changed the data model for YouTube comments, now putting the content into a -seperate framework update object - -```json -{ - "frameworkUpdates": { - "onResponseReceivedEndpoints": [ - { - "clickTrackingParams": "CAAQg2ciEwi64q3dmKGFAxWvy0IFHc14BKM=", - "reloadContinuationItemsCommand": { - "targetId": "comments-section", - "continuationItems": [ - { - "commentThreadRenderer": { - "replies": { - "commentRepliesRenderer": { - "contents": [ - { - "continuationItemRenderer": { - "trigger": "CONTINUATION_TRIGGER_ON_ITEM_SHOWN", - "continuationEndpoint": { - "clickTrackingParams": "CHgQvnUiEwi64q3dmKGFAxWvy0IFHc14BKM=", - "commandMetadata": { - "webCommandMetadata": { - "sendPost": true, - "apiUrl": "/youtubei/v1/next" - } - }, - "continuationCommand": { - "token": "Eg0SC1FpcDFWa1R1TTcwGAYygwEaUBIaVWd5TlRUOHV4REVqZ1lxeWJJRjRBYUFCQWciAggAKhhVQ3lhZmx6ek9IMEdDNjgzRGxRLWZ6d2cyC1FpcDFWa1R1TTcwQAFICoIBAggBQi9jb21tZW50LXJlcGxpZXMtaXRlbS1VZ3lOVFQ4dXhERWpnWXF5YklGNEFhQUJBZw%3D%3D", - "request": "CONTINUATION_REQUEST_TYPE_WATCH_NEXT" - } - } - } - } - ], - "trackingParams": "CHgQvnUiEwi64q3dmKGFAxWvy0IFHc14BKM=", - "viewReplies": { - "buttonRenderer": { - "text": { "runs": [{ "text": "220 replies" }] }, - "icon": { "iconType": "ARROW_DROP_DOWN" }, - "trackingParams": "CHoQosAEIhMIuuKt3ZihhQMVr8tCBR3NeASj", - "iconPosition": "BUTTON_ICON_POSITION_TYPE_LEFT_OF_TEXT" - } - }, - "hideReplies": { - "buttonRenderer": { - "text": { "runs": [{ "text": "220 replies" }] }, - "icon": { "iconType": "ARROW_DROP_UP" }, - "trackingParams": "CHkQ280EIhMIuuKt3ZihhQMVr8tCBR3NeASj", - "iconPosition": "BUTTON_ICON_POSITION_TYPE_LEFT_OF_TEXT" - } - }, - "targetId": "comment-replies-item-UgyNTT8uxDEjgYqybIF4AaABAg" - } - }, - "trackingParams": "CHYQwnUYywEiEwi64q3dmKGFAxWvy0IFHc14BKM=", - "renderingPriority": "RENDERING_PRIORITY_PINNED_COMMENT", - "isModeratedElqComment": false, - "commentViewModel": { - "commentViewModel": { - "commentId": "UgyNTT8uxDEjgYqybIF4AaABAg" - } - } - } - } - ] - } - } - ], - "entityBatchUpdate": { - "mutations": [ - { - "entityKey": "EhpVZ3lOVFQ4dXhERWpnWXF5YklGNEFhQUJBZyAoKAE%3D", - "type": "ENTITY_MUTATION_TYPE_REPLACE", - "payload": { - "commentEntityPayload": { - "key": "EhpVZ3lOVFQ4dXhERWpnWXF5YklGNEFhQUJBZyAoKAE%3D", - "properties": { - "commentId": "UgyNTT8uxDEjgYqybIF4AaABAg", - "content": { - "content": "⚠️ Important notice: if you put any symbol immediately after markup, it will not work: *here is the comma*, without space.\n\nYou should leave space before and after , to make it work.\n\nSame for _underscore_, and -hyphen-.\n\nLeave space before opening and after closing underscore and hyphen. Put all dots and commas inside markup.", - "styleRuns": [ - { - "startIndex": 135, - "length": 28, - "weightLabel": "FONT_WEIGHT_MEDIUM" - }, - { - "startIndex": 267, - "length": 10, - "weightLabel": "FONT_WEIGHT_NORMAL", - "italic": true - }, - { - "startIndex": 282, - "length": 7, - "weightLabel": "FONT_WEIGHT_NORMAL", - "strikethrough": "LINE_STYLE_SINGLE" - } - ] - }, - "publishedTime": "2 years ago (edited)", - "replyLevel": 0, - "authorButtonA11y": "@kibizoid", - "toolbarStateKey": "EhpVZ3lOVFQ4dXhERWpnWXF5YklGNEFhQUJBZyAsKAE%3D", - "translateButtonEntityKey": "EhpVZ3lOVFQ4dXhERWpnWXF5YklGNEFhQUJBZyD_ASgB" - }, - "author": { - "channelId": "UCUJfyiofeHQTmxKwZ6cCwIg", - "displayName": "@kibizoid", - "avatarThumbnailUrl": "https://yt3.ggpht.com/ytc/AIdro_nY2PkIyojDqs9Bk5RY6J90-U7wePswTYl799DNJQ=s88-c-k-c0x00ffffff-no-rj", - "isVerified": false, - "isCurrentUser": false, - "isCreator": false, - "isArtist": false - }, - "avatar": { - "image": { - "sources": [ - { - "url": "https://yt3.ggpht.com/ytc/AIdro_nY2PkIyojDqs9Bk5RY6J90-U7wePswTYl799DNJQ=s88-c-k-c0x00ffffff-no-rj", - "width": 88, - "height": 88 - } - ] - } - } - } - } - } - ] - } - } -} -``` - -## [15] Channel shorts: shortsLockupViewModel - -- **Encountered on:** 10.09.2024 -- **Impact:** 🟢 Low -- **Endpoint:** browse -- **Status:** Stabilized - -YouTube changed the data model for the channel shorts tab - -```json -{ - "richItemRenderer": { - "content": { - "shortsLockupViewModel": { - "entityId": "shorts-shelf-item-ovaHmfy3O6U", - "accessibilityText": "hangover food, 17 million views - play Short", - "thumbnail": { - "sources": [ - { - "url": "https://i.ytimg.com/vi/ovaHmfy3O6U/oar2.jpg?sqp=-oaymwEdCJUDENAFSFWQAgHyq4qpAwwIARUAAIhCcAHAAQY=&rs=AOn4CLBg-kG4rAi-BQ8Xkp2hOtOu-oXDLQ", - "width": 405, - "height": 720 - } - ] - }, - "overlayMetadata": { - "primaryText": { - "content": "hangover food" - }, - "secondaryText": { - "content": "17M views" - } - } - } - } - } -} -``` - -## [16] New playlist header renderer - -- **Encountered on:** 11.10.2024 -- **Impact:** 🟢 Low -- **Endpoint:** browse -- **Status:** Stabilized - -```json -{ - "pageHeaderRenderer": { - "pageTitle": "LilyPichu", - "content": { - "pageHeaderViewModel": { - "title": { - "dynamicTextViewModel": { - "text": { - "content": "LilyPichu" - } - } - }, - "metadata": { - "contentMetadataViewModel": { - "metadataRows": [ - { - "metadataParts": [ - { - "avatarStack": { - "avatarStackViewModel": { - "avatars": [ - { - "avatarViewModel": { - "image": { - "sources": [ - { - "url": "https://yt3.ggpht.com/ytc/AIdro_kcjhSY2e8WlYjQABOB65Za8n3QYycNHP9zXwxjKpBfOg=s48-c-k-c0x00ffffff-no-rj", - "width": 48, - "height": 48 - } - ] - } - } - } - ], - "text": { - "content": "by Kevin Ramirez", - "commandRuns": [ - { - "startIndex": 0, - "length": 16, - "onTap": { - "innertubeCommand": { - "browseEndpoint": { - "browseId": "UCai7BcI5lrXC2vdc3ySku8A", - "canonicalBaseUrl": "/@XxthekevinramirezxX" - } - } - } - } - ] - } - } - } - } - ] - }, - { - "metadataParts": [ - { - "text": { - "content": "Playlist" - } - }, - { - "text": { - "content": "10 videos" - } - }, - { - "text": { - "content": "856 views" - } - } - ] - } - ] - } - }, - "actions": {}, - "description": { - "descriptionPreviewViewModel": { - "description": { "content": "Hello World" } - } - }, - "heroImage": { - "contentPreviewImageViewModel": { - "image": { - "sources": [ - { - "url": "https://i.ytimg.com/vi/DXuNJ267Vss/hqdefault.jpg?sqp=-oaymwEWCKgBEF5IWvKriqkDCQgBFQAAiEIYAQ==&rs=AOn4CLAHp6V96b70x4SWm9Pe6WEHnQhP6A", - "width": 168, - "height": 94 - } - ] - } - } - } - } - } - } -} -``` - -## [17] Channel playlists: lockupViewModel - -- **Encountered on:** 09.11.2024 -- **Impact:** 🟢 Low -- **Endpoint:** browse -- **Status:** Stabilized - -YouTube changed the data model for the channel playlists / podcasts / albums tab - -```json -{ - "lockupViewModel": { - "contentImage": { - "collectionThumbnailViewModel": { - "primaryThumbnail": { - "thumbnailViewModel": { - "image": { - "sources": [ - { - "url": "https://i.ytimg.com/vi/XYdmX8w8xwI/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLCqmf6TGfDinNXhgU29ZxOkv2u9sQ", - "width": 480, - "height": 270 - } - ] - }, - "overlays": [ - { - "thumbnailOverlayBadgeViewModel": { - "thumbnailBadges": [ - { - "thumbnailBadgeViewModel": { - "icon": { - "sources": [ - { - "clientResource": { - "imageName": "PLAYLISTS" - } - } - ] - }, - "text": "5 videos", - "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT", - "backgroundColor": { - "lightTheme": 2370867, - "darkTheme": 2370867 - } - } - } - ], - "position": "THUMBNAIL_OVERLAY_BADGE_POSITION_BOTTOM_END" - } - } - ] - } - } - } - }, - "metadata": { - "lockupMetadataViewModel": { - "title": { - "content": "Jellybean Components Series" - } - } - }, - "contentId": "PLvOlSehNtuHv268f0mW5m1t_hq_RVGRSA", - "contentType": "LOCKUP_CONTENT_TYPE_PLAYLIST" - } -} -``` - -## [18] Music playlists facepile avatar - -- **Encountered on:** 25.11.2024 -- **Impact:** 🟢 Low -- **Endpoint:** browse (YTM) -- **Status:** Stabilized - -YouTube changed the data model for the channel playlist owner avatar into a `facepile` -object. It now also contains the channel avatar. - -The model is also used for playlists owned by YouTube Music (with the avatar and -commandContext missing). - -```json -{ - "facepile": { - "avatarStackViewModel": { - "avatars": [ - { - "avatarViewModel": { - "image": { - "sources": [ - { - "url": "https://yt3.ggpht.com/ytc/AIdro_n9ALaLETwQH6_2WlXitIaIKV-IqBDWWquvyI2jucNAZaQ=s48-c-k-c0x00000000-no-cc-rj-rp" - } - ] - }, - "avatarImageSize": "AVATAR_SIZE_XS" - } - } - ], - "text": { - "content": "Chaosflo44" - }, - "rendererContext": { - "commandContext": { - "onTap": { - "innertubeCommand": { - "browseEndpoint": { - "browseId": "UCQM0bS4_04-Y4JuYrgmnpZQ", - "browseEndpointContextSupportedConfigs": { - "browseEndpointContextMusicConfig": { - "pageType": "MUSIC_PAGE_TYPE_USER_CHANNEL" - } - } - } - } - } - } - } - } - } -} -``` - -## [19] Music artist album groups reordered - -- **Encountered on:** 13.01.2025 -- **Impact:** 🟢 Low -- **Endpoint:** browse (YTM) -- **Status:** Frequent (59%) - -YouTube Music used to group artist albums into 2 rows: "Albums" and "Singles". - -These groups were changed into "Albums" and "Singles & EPs". Now the "Album" label is -omitted for albums in their group, while singles and EPs have a label with their type. - -## [20] Music continuation item renderer - -- **Encountered on:** 25.01.2025 -- **Impact:** 🟢 Low -- **Endpoint:** browse (YTM) -- **Status:** Stabilized - -YouTube Music now uses a `continuationItemRenderer` for music playlists instead of -putting the continuations in a separate attribute of the MusicShelf. - -The continuation response now uses a `onResponseReceivedActions` field for its music -items. - -YouTube Music now also sends a random 16-character string as a `clientScreenNonce` in -the request context. This is not mandatory though. - -## [21] Music album recommendations - -- **Encountered on:** 26.02.2025 -- **Impact:** 🟢 Low -- **Endpoint:** browse (YTM) -- **Status:** Common (15%) - -![A/B test 21 screenshot](./_img/ab_21.png) - -YouTube Music has added "Recommended" and "More from \" carousels to album -pages. The difficulty is distinguishing them reliably for parsing the album variants. - -The current solution is adding the "Other versions" title in all languages to the -dictionary and comparing it. - -## [22] commandExecutorCommand for continuations - -- **Encountered on:** 16.03.2025 -- **Impact:** 🟢 Low -- **Endpoint:** browse (YTM) -- **Status:** Experimental (1%) - -YouTube playlists may use a commandExecutorCommand which holds a list of commands: the -`continuationCommand` that needs to be extracted as well as a `playlistVotingRefreshPopupCommand`. - -```json -{ - "continuationItemRenderer": { - "continuationEndpoint": { - "commandExecutorCommand": { - "commands": [ - { - "playlistVotingRefreshPopupCommand": { - "command": {} - } - }, - { - "continuationCommand": { - "request": "CONTINUATION_REQUEST_TYPE_BROWSE", - "token": "4qmFsgKBARIkVkxQTGJaSVB5MjAtMXBON21xamNrZXBXRjc4bmRiNmNpX3FpGjRDQUY2SGxCVU9rTklTV2xGUkVreVVtdEZOVTVFU1hsU2FrWkRVa1JKZWs1NldRJTNEJTNEmgIiUExiWklQeTIwLTFwTjdtcWpja2VwV0Y3OG5kYjZjaV9xaQ%3D%3D" - } - } - ] - } - } - } -} -``` +![A/B test 4 old screenshot](./_img/ab_6_new.png) diff --git a/notes/_img/ab_10.png b/notes/_img/ab_10.png deleted file mode 100644 index 21101dd..0000000 Binary files a/notes/_img/ab_10.png and /dev/null differ diff --git a/notes/_img/ab_13.png b/notes/_img/ab_13.png deleted file mode 100644 index a372da6..0000000 Binary files a/notes/_img/ab_13.png and /dev/null differ diff --git a/notes/_img/ab_21.png b/notes/_img/ab_21.png deleted file mode 100644 index 929f5d3..0000000 Binary files a/notes/_img/ab_21.png and /dev/null differ diff --git a/notes/_img/ab_8.png b/notes/_img/ab_8.png deleted file mode 100644 index 42f426b..0000000 Binary files a/notes/_img/ab_8.png and /dev/null differ diff --git a/notes/_img/ab_8_old.png b/notes/_img/ab_8_old.png deleted file mode 100644 index 036e111..0000000 Binary files a/notes/_img/ab_8_old.png and /dev/null differ diff --git a/notes/_img/ab_9.png b/notes/_img/ab_9.png deleted file mode 100644 index adca439..0000000 Binary files a/notes/_img/ab_9.png and /dev/null differ diff --git a/notes/logo.png b/notes/logo.png deleted file mode 100644 index 0cf2448..0000000 Binary files a/notes/logo.png and /dev/null differ diff --git a/notes/logo.svg b/notes/logo.svg deleted file mode 100644 index b26ae18..0000000 --- a/notes/logo.svg +++ /dev/null @@ -1,110 +0,0 @@ - - - - diff --git a/notes/po_token.md b/notes/po_token.md deleted file mode 100644 index f85ec16..0000000 --- a/notes/po_token.md +++ /dev/null @@ -1,30 +0,0 @@ -# About the new `pot` token - -YouTube has implemented a new method to prevent downloaders and alternative clients from accessing -their videos. Now requests to YouTube's video servers require a `pot` URL parameter. - -It is currently only required in the web player. The YTM and embedded player sends the token, too, but does not require it (this may change in the future). - -The TV player does not use the token at all and is currently the best workaround. The only downside -is that the TV player does not return any video metadata like title and description text. - -The first part of a video file (range: 0-1007959 bytes) can be downloaded without the token. -Requesting more of the file requires the pot token to be set, otherwise YouTube responds with a 403 -error. - -The pot token is base64-formatted and usually starts with a M - -`MnToZ2brHmyo0ehfKtK_EWUq60dPYDXksNX_UsaniM_Uj6zbtiIZujCHY02hr7opxB_n3XHetJQCBV9cnNHovuhvDqrjfxsKR-sjn-eIxqv3qOZKphvyDpQzlYBnT2AXK41R-ti6iPonrvlvKIASNmYX2lhsEg==` - -The token is generated from YouTubes Botguard script. The token is bound to the visitor data ID -used to fetch the player data. - -This feature has been A/B-tested for a few weeks. During that time, refetching the player in case -of a 403 download error often made things work again. As of 08.08.2024 this new feature seems to be -stabilized and retrying requests does not work any more. - -## Getting a `pot` token - -You need a real browser environment to run YouTube's botguard and obtain a pot token. The Invidious project has created a script to -. -The script opens YouTube's embedded video player, starts playback and extracts the visitor data diff --git a/postprocessor/Cargo.toml b/postprocessor/Cargo.toml new file mode 100644 index 0000000..b55226d --- /dev/null +++ b/postprocessor/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "rustypipe-postprocessor" +version = "0.1.0" +edition = "2021" + +[dependencies] +thiserror = "1.0.40" + +[dev-dependencies] +path_macro = "1.0.0" +once_cell = "1.12.0" +temp_testdir = "0.2.3" diff --git a/postprocessor/src/crc.rs b/postprocessor/src/crc.rs new file mode 100644 index 0000000..b144721 --- /dev/null +++ b/postprocessor/src/crc.rs @@ -0,0 +1,75 @@ +// Ogg decoder and encoder written in Rust +// +// Original source: https://github.com/RustAudio/ogg +// +// Copyright (c) 2016-2017 est31 +// and contributors. All rights reserved. +// Redistribution or use only under the terms +// specified in the LICENSE file attached to this +// source distribution. + +/*! +Implementation of the CRC algorithm with the +vorbis specific parameters and setup +*/ + +// Lookup table to enable bytewise CRC32 calculation +static CRC_LOOKUP_ARRAY: &[u32] = &lookup_array(); + +const fn get_tbl_elem(idx: u32) -> u32 { + let mut r: u32 = idx << 24; + let mut i = 0; + while i < 8 { + r = (r << 1) ^ (-(((r >> 31) & 1) as i32) as u32 & 0x04c11db7); + i += 1; + } + r +} + +const fn lookup_array() -> [u32; 0x100] { + let mut lup_arr: [u32; 0x100] = [0; 0x100]; + let mut i = 0; + while i < 0x100 { + lup_arr[i] = get_tbl_elem(i as u32); + i += 1; + } + lup_arr +} + +pub fn crc32(array: &[u8]) -> u32 { + crc32_update(0, array) +} + +pub fn crc32_update(cur: u32, array: &[u8]) -> u32 { + let mut ret: u32 = cur; + for av in array { + ret = (ret << 8) ^ CRC_LOOKUP_ARRAY[(*av as u32 ^ (ret >> 24)) as usize]; + } + ret +} + +#[test] +fn test_crc32() { + // Test page taken from real Ogg file + let test_arr = &[ + 0x4f, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x74, + 0xa3, 0x90, 0x5b, 0x00, 0x00, 0x00, 0x00, + // The spec requires us to zero out the CRC field + /*0x6d, 0x94, 0x4e, 0x3d,*/ + 0x00, 0x00, 0x00, 0x00, 0x01, 0x1e, 0x01, 0x76, 0x6f, 0x72, 0x62, 0x69, 0x73, 0x00, 0x00, + 0x00, 0x00, 0x02, 0x44, 0xac, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0xb5, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb8, 0x01, + ]; + println!(); + println!( + "CRC of \"==!\" calculated as 0x{:08x} (expected 0x9f858776)", + crc32(&[61, 61, 33]) + ); + println!( + "Test page CRC calculated as 0x{:08x} (expected 0x3d4e946d)", + crc32(test_arr) + ); + assert_eq!(crc32(&[61, 61, 33]), 0x9f858776); + assert_eq!(crc32(test_arr), 0x3d4e946d); + assert_eq!(crc32(&test_arr[0..27]), 0x7b374db8); +} diff --git a/postprocessor/src/lib.rs b/postprocessor/src/lib.rs new file mode 100644 index 0000000..c89f696 --- /dev/null +++ b/postprocessor/src/lib.rs @@ -0,0 +1,73 @@ +#![allow(dead_code)] + +pub mod ogg_from_webm; + +mod crc; +mod ogg; +mod webm; + +/// Error from the postprocessor +#[derive(thiserror::Error, Debug)] +#[non_exhaustive] +pub enum PostprocessingError { + /// File IO error + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("file format not recognized: {0}")] + InvalidFormat(&'static str), + #[error("no such element: expected {0:#x}, got {1:#x}")] + NoSuchElement(u32, u32), + #[error("bad element: {0}")] + BadElement(String), + #[error("invalid encoded length")] + InvalidLength, + #[error("invalid string")] + InvalidString, + #[error("unexpected element size")] + UnexpectedSize, + #[error("unsupported op: {0}")] + Unsupported(&'static str), + #[error("int conversion: {0}")] + Conversion(#[from] std::num::TryFromIntError), +} + +pub(crate) type Result = std::result::Result; + +#[cfg(test)] +pub(crate) mod tests { + use std::{ + fs::File, + io::{BufReader, Read}, + os::unix::prelude::MetadataExt, + path::{Path, PathBuf}, + }; + + use once_cell::sync::Lazy; + use path_macro::path; + + pub static TESTFILES: Lazy = Lazy::new(|| { + path!(env!("CARGO_MANIFEST_DIR") / ".." / "testfiles") + .canonicalize() + .unwrap() + }); + + pub fn assert_files_eq, P2: AsRef>(p1: P, p2: P2) { + let f1 = File::open(p1).unwrap(); + let f2 = File::open(p2).unwrap(); + + let size = f1.metadata().unwrap().size(); + assert_eq!(size, f2.metadata().unwrap().size(), "File sizes dont match"); + + let mut r1 = BufReader::new(f1); + let mut r2 = BufReader::new(f2); + + for i in 0..size { + let mut b1 = [0; 1]; + let mut b2 = [0; 1]; + r1.read_exact(&mut b1).unwrap(); + r2.read_exact(&mut b2).unwrap(); + + assert_eq!(b1[0], b2[0], "Byte {i} does not match"); + } + } +} diff --git a/postprocessor/src/ogg.rs b/postprocessor/src/ogg.rs new file mode 100644 index 0000000..52d823a --- /dev/null +++ b/postprocessor/src/ogg.rs @@ -0,0 +1,131 @@ +use crate::{crc, PostprocessingError, Result}; + +pub struct OggWriter { + sequence_count: u32, + stream_id: u32, + packet_flag: u8, + segment_table: [u8; 255], + segment_table_size: u8, + pub segment_table_next_timestamp: u64, +} + +pub const FLAG_UNSET: u8 = 0x00; +pub const FLAG_FIRST: u8 = 0x02; +pub const FLAG_LAST: u8 = 0x04; +const HEADER_CHECKSUM_OFFSET: usize = 22; +const HEADER_SIZE: usize = 27; +pub const TIME_SCALE_NS: u64 = 1000000000; + +pub const METADATA_VORBIS: [u8; 15] = [ + 0x03, 0x76, 0x6f, 0x72, 0x62, 0x69, 0x73, // "vorbis" binary string + 0x00, 0x00, 0x00, 0x00, // writing application string size (not present) + 0x00, 0x00, 0x00, 0x00, // additional tags count (zero means no tags) +]; + +pub const METADATA_OPUS: [u8; 16] = [ + 0x4F, 0x70, 0x75, 0x73, 0x54, 0x61, 0x67, 0x73, // "OpusTags" binary string + 0x00, 0x00, 0x00, 0x00, // writing application string size (not present) + 0x00, 0x00, 0x00, 0x00, // additional tags count (zero means no tags) +]; + +impl OggWriter { + pub fn new() -> Self { + Self { + sequence_count: 0, + stream_id: 1, + packet_flag: FLAG_FIRST, + segment_table: [0; 255], + segment_table_size: 0, + segment_table_next_timestamp: TIME_SCALE_NS, + } + } + + pub fn make_packet_header(&mut self, gran_pos: u64, page: &[u8], immediate: bool) -> Vec { + let mut header = Vec::new(); + let mut length = HEADER_SIZE; + + header.extend_from_slice(b"OggS"); + header.push(0); // version + header.push(self.packet_flag); // type + + header.extend_from_slice(&gran_pos.to_le_bytes()); // granulate position + + header.extend_from_slice(&self.stream_id.to_le_bytes()); // bitstream serial number + header.extend_from_slice(&self.sequence_count.to_le_bytes()); // page sequence number + self.sequence_count += 1; + + header.extend_from_slice(&[0, 0, 0, 0]); // page checksum + + header.push(self.segment_table_size); + header.extend_from_slice(&self.segment_table[0..self.segment_table_size as usize]); + + length += self.segment_table_size as usize; + self.clear_segment_table(); + + let checksum = crc::crc32(&header[0..length]); + let checksum = crc::crc32_update(checksum, page); + Self::header_put_checksum(&mut header, checksum); + + if immediate { + self.segment_table_next_timestamp -= TIME_SCALE_NS; + } + header + } + + fn header_put_checksum(header: &mut Vec, checksum: u32) { + let cs_bts = checksum.to_le_bytes(); + let to_fill = (HEADER_CHECKSUM_OFFSET + cs_bts.len()).saturating_sub(header.len()); + for _ in 0..to_fill { + header.push(0); + } + header[HEADER_CHECKSUM_OFFSET..(HEADER_CHECKSUM_OFFSET + cs_bts.len())] + .copy_from_slice(&cs_bts[..]); + } + + pub fn clear_segment_table(&mut self) { + self.segment_table_next_timestamp += TIME_SCALE_NS; + self.packet_flag = FLAG_UNSET; + self.segment_table_size = 0; + } + + pub fn add_packet_segment(&mut self, size: u32) -> Result { + if size > 65025 { + return Err(PostprocessingError::Unsupported( + "page size cannot be larger than 65025", + )); + } + + let mut available = + (self.segment_table.len() as u32 - self.segment_table_size as u32) * 255; + let extra = (size % 255) == 0; + + if extra { + // add a zero byte entry in the table + // required to indicate the sample size is multiple of 255 + available -= 255; + } + + // check if possible add the segment, without overflow the table + if available < size { + return Ok(false); // not enough space on the page + } + + let mut seg = size; + while seg > 0 { + self.segment_table[self.segment_table_size as usize] = seg.min(255) as u8; + self.segment_table_size += 1; + seg = seg.saturating_sub(255); + } + + if extra { + self.segment_table[self.segment_table_size as usize] = 0; + self.segment_table_size += 1; + } + + Ok(true) + } + + pub fn set_packet_flag(&mut self, flag: u8) { + self.packet_flag = flag; + } +} diff --git a/postprocessor/src/ogg_from_webm.rs b/postprocessor/src/ogg_from_webm.rs new file mode 100644 index 0000000..b524fd4 --- /dev/null +++ b/postprocessor/src/ogg_from_webm.rs @@ -0,0 +1,179 @@ +use std::{ + fs::File, + io::{BufWriter, Read, Write}, + path::Path, +}; + +use crate::{ + ogg::{OggWriter, FLAG_LAST, METADATA_OPUS, METADATA_VORBIS, TIME_SCALE_NS}, + webm::{TrackKind, WebmReader}, + PostprocessingError, +}; + +pub fn test(mut source: impl Read) -> Result { + let mut buf = [0; 4]; + source.read_exact(&mut buf)?; + + match u32::from_be_bytes(buf) { + 0x1a45dfa3 => Ok(true), // webm/mkv + 0x4f676753 => Ok(false), // ogg + _ => Err(PostprocessingError::InvalidFormat("unknown magic number")), + } +} + +pub fn process, P2: AsRef>( + source: P, + dest: P2, +) -> Result<(), PostprocessingError> { + let mut webm = WebmReader::new(File::open(source)?)?; + webm.parse()?; + if !webm.get_next_segment()? { + return Err(PostprocessingError::InvalidFormat("no segment")); + } + + let mut ogg = OggWriter::new(); + let mut output = BufWriter::new(File::create(dest)?); + + // Select track + let track = match &webm.tracks { + Some(tracks) => { + let track_id = tracks + .iter() + .enumerate() + .find(|(_, track)| { + track.kind == TrackKind::Audio + && (track.codec_id == "A_OPUS" || track.codec_id == "A_VORBIS") + }) + .ok_or(PostprocessingError::InvalidFormat("no audio tracks"))? + .0; + webm.select_track(track_id).unwrap().clone() + } + None => return Err(PostprocessingError::InvalidFormat("no tracks")), + }; + + // Get sample rate + let res = get_sample_freq_from_track(&track.b_metadata); + + // Create packet with codec init data + if !track.codec_private.is_empty() { + ogg.add_packet_segment(track.codec_private.len().try_into()?)?; + let header = ogg.make_packet_header(0, &track.codec_private, true); + output.write_all(&header)?; + output.write_all(&track.codec_private)?; + } + + // Create packet with metadata + let metadata = match track.codec_id.as_str() { + "A_OPUS" => METADATA_OPUS.as_slice(), + "A_VORBIS" => METADATA_VORBIS.as_slice(), + _ => unreachable!(), + }; + ogg.add_packet_segment(metadata.len().try_into()?)?; + let header = ogg.make_packet_header(0, metadata, true); + output.write_all(&header)?; + output.write_all(metadata)?; + + // Calculate amount of packets + let mut page = Vec::new(); + let mut webm_block = None; + while !webm.is_done() { + let block = match webm_block { + Some(block) => { + webm_block = None; + Some(block) + } + None => webm.get_next_block()?, + }; + + if let Some(block) = block { + let timestamp = block.absolute_time_code_ns + track.codec_delay; + let added = if timestamp >= ogg.segment_table_next_timestamp { + false + } else { + ogg.add_packet_segment(block.data_size.try_into()?)? + }; + + if added { + webm.get_block_data(&block, &mut page)?; + continue; + } + } + + let mut elapsed_ns = track.codec_delay; + match block { + Some(block) => { + elapsed_ns += block.absolute_time_code_ns; + } + None => { + // TODO: move to ogg + ogg.set_packet_flag(FLAG_LAST); + elapsed_ns += webm.webm_block_last_timecode(); + + if track.default_duration > 0 { + elapsed_ns += track.default_duration; + } else { + elapsed_ns += webm.webm_block_near_duration(); + } + } + } + + // get the sample count in the page + let sample_count: f64 = (elapsed_ns as f64 / TIME_SCALE_NS as f64) * res as f64; + let sample_count = sample_count.ceil() as u64; + + let header = ogg.make_packet_header(sample_count, &page, false); + + // dump data + output.write_all(&header)?; + output.write_all(&page)?; + + page = Vec::new(); + webm_block = block; + } + + output.flush()?; + Ok(()) +} + +fn get_sample_freq_from_track(b_metadata: &[u8]) -> f32 { + let mut i = 0; + while i < b_metadata.len().saturating_sub(5) { + let id_bts: [u8; 2] = b_metadata[i..i + 2].try_into().unwrap(); + let id = u16::from_be_bytes(id_bts); + + if id == 0xB584 { + let freq_bts: [u8; 4] = b_metadata[i + 2..i + 6].try_into().unwrap(); + return f32::from_be_bytes(freq_bts); + } + i += 2; + } + 0.0 +} + +#[cfg(test)] +mod tests { + use path_macro::path; + + use crate::tests::{assert_files_eq, TESTFILES}; + + use super::*; + + #[test] + fn t_test() { + let path = path!(*TESTFILES / "postprocessor" / "audio1.webm"); + let mut file = File::open(&path).unwrap(); + assert!(test(&mut file).unwrap()); + } + + #[test] + fn t_process() { + let temp = temp_testdir::TempDir::default(); + + let source = path!(*TESTFILES / "postprocessor" / "audio1.webm"); + let dest = path!(temp / "audio1.ogg"); + let expect = path!(*TESTFILES / "postprocessor" / "conv" / "audio1.ogg"); + + process(&source, &dest).unwrap(); + assert_files_eq(&dest, &expect); + } +} diff --git a/postprocessor/src/webm.rs b/postprocessor/src/webm.rs new file mode 100644 index 0000000..117c324 --- /dev/null +++ b/postprocessor/src/webm.rs @@ -0,0 +1,736 @@ +use std::{ + fs::File, + io::{BufReader, Read, Seek, SeekFrom}, + os::unix::prelude::MetadataExt, +}; + +use crate::{PostprocessingError, Result}; + +#[derive(Debug)] +pub struct WebmReader { + stream: BufReader, + len: u64, + segment: Option, + cluster: Option, + pub tracks: Option>, + selected_track: Option, + done: bool, + first_segment: bool, + webm_block_near_duration: u64, + webm_block_last_timecode: u64, +} + +#[derive(Debug, Copy, Clone)] +struct Element { + typ: u32, + offset: u64, + content_size: u64, + size: u64, +} + +#[derive(Debug, Copy, Clone)] +pub struct Info { + pub timecode_scale: u64, + pub duration: u64, +} + +#[derive(Debug, Clone)] +pub struct WebmTrack { + pub track_number: u64, + track_type: u32, + pub codec_id: String, + pub codec_private: Vec, + pub b_metadata: Vec, + pub kind: TrackKind, + pub default_duration: u64, + pub codec_delay: u64, + pub seek_pre_roll: u64, +} + +#[derive(Debug, Clone)] +pub struct Segment { + pub info: Option, + tracks: Option>, + current_cluster: Option, + rf: Element, + first_cluster_in_segment: bool, +} + +impl Segment { + fn new(rf: Element) -> Segment { + Segment { + info: None, + tracks: None, + current_cluster: None, + rf, + first_cluster_in_segment: true, + } + } +} + +#[derive(Debug, Copy, Clone)] +pub struct SimpleBlock { + pub created_from_block: bool, + pub track_number: u64, + pub relative_time_code: u16, + pub absolute_time_code_ns: u64, + pub flags: u8, + pub offset: u64, + pub data_size: u64, + rf: Element, +} + +impl SimpleBlock { + fn new(rf: Element) -> Self { + Self { + created_from_block: false, + track_number: 0, + relative_time_code: 0, + absolute_time_code_ns: 0, + flags: 0, + offset: 0, + data_size: 0, + rf, + } + } + + pub fn is_keyframe(&self) -> bool { + (self.flags & 0x80) == 0x80 + } +} + +#[derive(Debug, Copy, Clone)] +pub struct Cluster { + rf: Element, + current_simple_block: Option, + current_block_group: Option, + timecode: u64, +} + +impl Cluster { + fn new(rf: Element) -> Self { + Self { + rf, + current_simple_block: None, + current_block_group: None, + timecode: 0, + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum TrackKind { + Audio, + Video, + Other, +} + +const ID_EMBL: u32 = 0x0A45DFA3; +const ID_EMBL_READ_VERSION: u32 = 0x02F7; +const ID_EMBL_DOC_TYPE: u32 = 0x0282; +const ID_EMBL_DOC_TYPE_READ_VERSION: u32 = 0x0285; +const ID_SEGMENT: u32 = 0x08538067; +const ID_INFO: u32 = 0x0549A966; +const ID_TIMECODE_SCALE: u32 = 0x0AD7B1; +const ID_DURATION: u32 = 0x489; +const ID_TRACKS: u32 = 0x0654AE6B; +const ID_TRACK_ENTRY: u32 = 0x2E; +const ID_TRACK_NUMBER: u32 = 0x57; +const ID_TRACK_TYPE: u32 = 0x03; +const ID_CODEC_ID: u32 = 0x06; +const ID_CODEC_PRIVATE: u32 = 0x23A2; +const ID_VIDEO: u32 = 0x60; +const ID_AUDIO: u32 = 0x61; +const ID_DEFAULT_DURATION: u32 = 0x3E383; +const ID_FLAG_LACING: u32 = 0x1C; +const ID_CODEC_DELAY: u32 = 0x16AA; +const ID_SEEK_PRE_ROLL: u32 = 0x16BB; +const ID_CLUSTER: u32 = 0x0F43B675; +const ID_TIMECODE: u32 = 0x67; +const ID_SIMPLE_BLOCK: u32 = 0x23; +const ID_BLOCK: u32 = 0x21; +const ID_GROUP_BLOCK: u32 = 0x20; + +impl WebmReader { + pub fn new(file: File) -> Result { + let md = file.metadata()?; + Ok(Self { + stream: BufReader::new(file), + len: md.size(), + segment: None, + cluster: None, + tracks: None, + selected_track: None, + done: false, + first_segment: false, + webm_block_near_duration: 0, + webm_block_last_timecode: 0, + }) + } + + /// Make sure the parser did not go beyond the current element and + /// skip to the start if the next element. + fn ensure(&mut self, rf: &Element) -> Result<()> { + let pos = self.stream.stream_position()?; + let elem_end = rf.offset + rf.size; + + if pos > elem_end { + return Err(PostprocessingError::Io(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + format!( + "parser go beyond limits of the Element. type={} offset={} size={} position={}", + rf.typ, rf.offset, rf.size, pos, + ), + ))); + } + + self.stream.seek(SeekFrom::Start(elem_end))?; + Ok(()) + } + + fn read_byte(&mut self) -> Result { + let mut bt = [0; 1]; + self.stream.read_exact(&mut bt)?; + Ok(bt[0]) + } + + fn read_u16(&mut self) -> Result { + let mut bt = [0; 2]; + self.stream.read_exact(&mut bt)?; + Ok(u16::from_be_bytes(bt)) + } + + fn read_number(&mut self, parent: &Element) -> Result { + let mut value: u64 = 0; + for _ in 0..parent.content_size { + let rd = self.read_byte()?; + value = (value << 8) | u64::from(rd); + } + Ok(value) + } + + fn read_string(&mut self, parent: &Element) -> Result { + String::from_utf8(self.read_blob(parent)?).map_err(|_| PostprocessingError::InvalidString) + } + + fn read_blob(&mut self, parent: &Element) -> Result> { + let length = parent.content_size as usize; + let mut buf = vec![0u8; length]; + self.stream.read_exact(&mut buf)?; + Ok(buf) + } + + fn read_encoded_number(&mut self) -> Result { + let mut value = u64::from(self.read_byte()?); + + if value > 0 { + let mut mask: u64 = 0x80; + + for size in 1..9 { + if (value & mask) == mask { + mask = 0xff; + mask >>= size; + + let mut number = value & mask; + + for _ in 1..size { + value = u64::from(self.read_byte()?); + number = (number << 8) | value; + } + return Ok(number); + } + + mask >>= 1; + } + } + Err(PostprocessingError::InvalidLength) + } + + fn read_element(&mut self) -> Result { + let offset = self.stream.stream_position()?; + let typ = self.read_encoded_number()? as u32; + let content_size = self.read_encoded_number()?; + let size = content_size + self.stream.stream_position()? - offset; + + Ok(Element { + typ, + offset, + content_size, + size, + }) + } + + fn read_element_expected(&mut self, expected: u32) -> Result { + let elem = self.read_element()?; + if expected != 0 && elem.typ != expected { + return Err(PostprocessingError::NoSuchElement(expected, elem.typ)); + } + Ok(elem) + } + + fn until_element(&mut self, rf: Option<&Element>, expected: &[u32]) -> Result> { + loop { + let brk = match rf { + Some(rf) => self.stream.stream_position()? >= (rf.offset + rf.size), + None => self.stream.stream_position()? >= self.len, + }; + if brk { + break; + } + + let elem = self.read_element()?; + if expected.is_empty() { + return Ok(Some(elem)); + } + for t in expected { + if &elem.typ == t { + return Ok(Some(elem)); + } + } + self.ensure(&elem)?; + } + Ok(None) + } + + fn read_ebml( + &mut self, + rf: &Element, + min_read_version: u64, + min_doc_type_version: u64, + ) -> Result { + let elem_v = self.until_element(Some(rf), &[ID_EMBL_READ_VERSION])?; + match elem_v { + Some(elem_v) => { + if self.read_number(&elem_v)? > min_read_version { + return Ok(false); + } + + let elem_t = self.until_element(Some(rf), &[ID_EMBL_DOC_TYPE])?; + match elem_t { + Some(elem_t) => { + if self.read_string(&elem_t)? != "webm" { + return Ok(false); + } + + let elem_tv = + self.until_element(Some(rf), &[ID_EMBL_DOC_TYPE_READ_VERSION])?; + match elem_tv { + Some(elem_tv) => { + Ok(self.read_number(&elem_tv)? <= min_doc_type_version) + } + None => Ok(false), + } + } + None => Ok(false), + } + } + None => Ok(false), + } + } + + fn read_info(&mut self, rf: &Element) -> Result { + let mut info = Info { + timecode_scale: 0, + duration: 0, + }; + + while let Some(elem) = self.until_element(Some(rf), &[ID_TIMECODE_SCALE, ID_DURATION])? { + match elem.typ { + ID_TIMECODE_SCALE => { + info.timecode_scale = self.read_number(&elem)?; + } + ID_DURATION => { + info.duration = self.read_number(&elem)?; + } + _ => {} + } + self.ensure(&elem)?; + } + + if info.timecode_scale == 0 { + return Err(PostprocessingError::BadElement( + "Element Timecode not found".to_owned(), + )); + } + Ok(info) + } + + fn read_segment( + &mut self, + rf: Element, + track_lacing_expected: u64, + metadata_expected: bool, + ) -> Result { + let mut seg = Segment::new(rf); + while let Some(elem) = self.until_element(Some(&rf), &[ID_INFO, ID_TRACKS, ID_CLUSTER])? { + match elem.typ { + ID_CLUSTER => { + seg.current_cluster = Some(elem); + break; + } + ID_INFO => { + seg.info = Some(self.read_info(&elem)?); + self.ensure(&elem)?; + } + ID_TRACKS => { + seg.tracks = Some(self.read_tracks(&elem, track_lacing_expected)?); + self.ensure(&elem)?; + } + _ => {} + } + } + + if metadata_expected && (seg.info.is_none() || seg.tracks.is_none()) { + return Err(PostprocessingError::BadElement(format!( + "Cluster element found without Info and/or Tracks element at position {}", + rf.offset + ))); + } + Ok(seg) + } + + fn read_tracks(&mut self, rf: &Element, lacing_expected: u64) -> Result> { + let mut tracks = Vec::new(); + + while let Some(elem_te) = self.until_element(Some(rf), &[ID_TRACK_ENTRY])? { + let mut entry = WebmTrack { + track_number: 0, + track_type: 0, + codec_id: String::new(), + codec_private: Vec::new(), + b_metadata: Vec::new(), + kind: TrackKind::Other, + default_duration: 0, + codec_delay: 0, + seek_pre_roll: 0, + }; + let mut drop = false; + + while let Some(elem) = self.until_element(Some(&elem_te), &[])? { + match elem.typ { + ID_TRACK_NUMBER => { + entry.track_number = self.read_number(&elem)?; + } + ID_TRACK_TYPE => { + entry.track_type = self.read_number(&elem)? as u32; + } + ID_CODEC_ID => { + entry.codec_id = self.read_string(&elem)?; + } + ID_CODEC_PRIVATE => { + entry.codec_private = self.read_blob(&elem)?; + } + ID_AUDIO | ID_VIDEO => { + entry.b_metadata = self.read_blob(&elem)?; + } + ID_DEFAULT_DURATION => { + entry.default_duration = self.read_number(&elem)?; + } + ID_FLAG_LACING => { + drop = self.read_number(&elem)? != lacing_expected; + } + ID_CODEC_DELAY => { + entry.codec_delay = self.read_number(&elem)?; + } + ID_SEEK_PRE_ROLL => { + entry.seek_pre_roll = self.read_number(&elem)?; + } + _ => {} + } + self.ensure(&elem)?; + } + entry.kind = match entry.track_type { + 1 => TrackKind::Video, + 2 => TrackKind::Audio, + _ => TrackKind::Other, + }; + if !drop { + tracks.push(entry); + } + self.ensure(&elem_te)?; + } + Ok(tracks) + } + + fn read_simple_block(&mut self, rf: Element) -> Result { + let mut sb = SimpleBlock::new(rf); + sb.track_number = self.read_encoded_number()?; + sb.relative_time_code = self.read_u16()?; + sb.flags = self.read_byte()?; + let pos = self.stream.stream_position()?; + sb.data_size = (rf.offset + rf.size) + .checked_sub(pos) + .ok_or(PostprocessingError::UnexpectedSize)?; + sb.offset = pos; + sb.created_from_block = rf.typ == ID_BLOCK; + Ok(sb) + } + + fn read_cluster(&mut self, rf: Element) -> Result { + let elem = self.until_element(Some(&rf), &[ID_TIMECODE])?; + let mut cl = Cluster::new(rf); + + match elem { + Some(elem) => { + cl.timecode = self.read_number(&elem)?; + } + None => { + return Err(PostprocessingError::BadElement(format!( + "Cluster at {} without Timecode element", + rf.offset + ))) + } + } + Ok(cl) + } + + fn inside_cluster_bounds(&mut self, cluster: &Cluster) -> Result { + Ok(self.stream.stream_position()? >= (cluster.rf.offset + cluster.rf.size)) + } + + pub fn get_next_cluster(&mut self) -> Result> { + if self.done { + return Ok(None); + } + + let (rf, cc) = if let Some(segment) = &mut self.segment { + if let Some(current_cluster) = segment.current_cluster { + if segment.first_cluster_in_segment { + segment.first_cluster_in_segment = false; + return Ok(Some(self.read_cluster(current_cluster)?)); + } + (segment.rf, current_cluster) + } else { + return Ok(None); + } + } else { + return Ok(None); + }; + self.ensure(&cc)?; + + let elem = self.until_element(Some(&rf), &[ID_CLUSTER])?; + match elem { + Some(elem) => { + self.segment.as_mut().unwrap().current_cluster = Some(elem); + Ok(Some(self.read_cluster(elem)?)) + } + None => Ok(None), + } + } + + pub fn get_next_simple_block(&mut self, cluster: &mut Cluster) -> Result> { + if self.inside_cluster_bounds(cluster)? { + return Ok(None); + } + + if let Some(current_block_group) = &cluster.current_block_group { + self.ensure(current_block_group)?; + cluster.current_block_group = None; + cluster.current_simple_block = None; + } else if let Some(current_simple_block) = &cluster.current_simple_block { + self.ensure(¤t_simple_block.rf)?; + } + + while !self.inside_cluster_bounds(cluster)? { + let elem = self.until_element(Some(&cluster.rf), &[ID_SIMPLE_BLOCK, ID_GROUP_BLOCK])?; + let nelem = match elem { + Some(elem) => { + if elem.typ == ID_GROUP_BLOCK { + cluster.current_block_group = Some(elem); + let block_elem = self.until_element(Some(&elem), &[ID_BLOCK])?; + + match block_elem { + Some(block_elem) => block_elem, + None => { + self.ensure(&elem)?; + cluster.current_block_group = None; + continue; + } + } + } else { + elem + } + } + None => return Ok(None), + }; + + let mut csb = self.read_simple_block(nelem)?; + + if self + .selected_track() + .map(|st| st.track_number == csb.track_number) + .unwrap_or_default() + { + csb.absolute_time_code_ns = (u64::from(csb.relative_time_code) + cluster.timecode) + * self + .segment + .as_ref() + .and_then(|seg| seg.info) + .map(|inf| inf.timecode_scale) + .unwrap_or(1); + + cluster.current_simple_block = Some(csb); + return Ok(Some(csb)); + } + + cluster.current_simple_block = Some(csb); + self.ensure(&nelem)?; + } + Ok(None) + } + + pub fn get_next_block(&mut self) -> Result> { + if self.segment.is_none() && !self.get_next_segment()? { + return Ok(None); + } + + if self.cluster.is_none() { + self.cluster = self.get_next_cluster()?; + if self.cluster.is_none() { + self.segment = None; + return self.get_next_block(); + } + } + + let mut c = self.cluster.unwrap(); + let res = self.get_next_simple_block(&mut c)?; + self.cluster = Some(c); + + match res { + Some(res) => { + self.webm_block_near_duration = + res.absolute_time_code_ns - self.webm_block_last_timecode; + self.webm_block_last_timecode = res.absolute_time_code_ns; + Ok(Some(res)) + } + None => { + self.cluster = None; + self.get_next_block() + } + } + } + + pub fn selected_track(&self) -> Option<&WebmTrack> { + if let (Some(tracks), Some(st)) = (&self.tracks, self.selected_track) { + return tracks.get(st); + } + None + } + + pub fn select_track(&mut self, index: usize) -> Option<&WebmTrack> { + if let Some(tracks) = &self.tracks { + match tracks.get(index) { + Some(track) => { + self.selected_track = Some(index); + Some(track) + } + None => None, + } + } else { + None + } + } + + pub fn parse(&mut self) -> Result<()> { + let elem_ebml = self.read_element_expected(ID_EMBL)?; + if !self.read_ebml(&elem_ebml, 1, 2)? { + return Err(PostprocessingError::InvalidFormat( + "Unsupported EBML data (WebM)", + )); + } + self.ensure(&elem_ebml)?; + + let elem_seg = self.until_element(None, &[ID_SEGMENT])?; + match elem_seg { + Some(elem_seg) => { + let seg = self.read_segment(elem_seg, 0, true)?; + // TODO: avoid this clone + self.tracks = seg.tracks.clone(); + self.segment = Some(seg); + self.selected_track = None; + self.done = false; + self.first_segment = true; + + Ok(()) + } + None => Err(PostprocessingError::InvalidFormat( + "Fragment element not found", + )), + } + } + + pub fn get_next_segment(&mut self) -> Result { + if self.done { + return Ok(false); + } + + if let Some(segment) = &self.segment { + if self.first_segment { + self.first_segment = false; + return Ok(true); + } + let srf = segment.rf; + self.ensure(&srf)?; + } + + let elem = self.until_element(None, &[ID_SEGMENT])?; + match elem { + Some(elem) => { + self.segment = Some(self.read_segment(elem, 0, false)?); + Ok(true) + } + None => { + self.done = true; + Ok(false) + } + } + } + + pub fn is_done(&self) -> bool { + self.done + } + + pub fn webm_block_near_duration(&self) -> u64 { + self.webm_block_near_duration + } + + pub fn webm_block_last_timecode(&self) -> u64 { + self.webm_block_last_timecode + } + + pub fn get_block_data(&mut self, block: &SimpleBlock, bytes: &mut Vec) -> Result<()> { + let old_pos = self.stream.stream_position()?; + self.stream.seek(SeekFrom::Start(block.offset))?; + + for _ in 0..block.data_size { + let mut bt = [0; 1]; + self.stream.read_exact(&mut bt)?; + bytes.push(bt[0]); + } + + self.stream.seek(SeekFrom::Start(old_pos))?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use path_macro::path; + + use crate::tests::TESTFILES; + + use super::*; + + #[test] + fn read() { + let path = path!(*TESTFILES / "postprocessor" / "audio1.webm"); + let file = File::open(&path).unwrap(); + let mut reader = WebmReader::new(file).unwrap(); + reader.parse().unwrap(); + reader.get_next_segment().unwrap(); + // TODO: check types + reader.select_track(0).unwrap(); + + dbg!(&reader); + } +} diff --git a/renovate.json b/renovate.json deleted file mode 100644 index 1ec2687..0000000 --- a/renovate.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": ["config:best-practices", ":preserveSemverRanges"], - "semanticCommits": "enabled", - "automerge": true, - "automergeStrategy": "squash", - "osvVulnerabilityAlerts": true, - "labels": ["dependency-upgrade"], - "enabledManagers": ["cargo"], - "prHourlyLimit": 5 -} diff --git a/src/cache.rs b/src/cache.rs index a1c90c4..970ca1a 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -16,12 +16,11 @@ //! the cache as a JSON file. use std::{ - fs::File, - io::Write, + fs, path::{Path, PathBuf}, }; -use tracing::error; +use log::error; pub(crate) const DEFAULT_CACHE_FILE: &str = "rustypipe_cache.json"; @@ -69,21 +68,7 @@ impl Default for FileStorage { impl CacheStorage for FileStorage { fn write(&self, data: &str) { - fn _write(path: &Path, data: &str) -> Result<(), std::io::Error> { - let mut f = File::create(path)?; - // Set cache file permissions to 0600 on Unix-based systems - #[cfg(target_family = "unix")] - { - use std::os::unix::fs::PermissionsExt; - let metadata = f.metadata()?; - let mut permissions = metadata.permissions(); - permissions.set_mode(0o600); - std::fs::set_permissions(path, permissions)?; - } - f.write_all(data.as_bytes()) - } - - _write(&self.path, data).unwrap_or_else(|e| { + fs::write(&self.path, data).unwrap_or_else(|e| { error!( "Could not write cache to file `{}`. Error: {}", self.path.to_string_lossy(), @@ -97,7 +82,7 @@ impl CacheStorage for FileStorage { return None; } - match std::fs::read_to_string(&self.path) { + match fs::read_to_string(&self.path) { Ok(data) => Some(data), Err(e) => { error!( diff --git a/src/client/channel.rs b/src/client/channel.rs index dc67579..867cc55 100644 --- a/src/client/channel.rs +++ b/src/client/channel.rs @@ -1,28 +1,23 @@ -use std::fmt::Debug; - use serde::Serialize; -use time::OffsetDateTime; use url::Url; use crate::{ - client::response::YouTubeListItem, error::{Error, ExtractionError}, model::{ paginator::{ContinuationEndpoint, Paginator}, - Channel, ChannelInfo, PlaylistItem, Verification, VideoItem, + Channel, ChannelInfo, PlaylistItem, VideoItem, YouTubeItem, }, param::{ChannelOrder, ChannelVideoTab, Language}, - serializer::{text::TextComponent, MapResult}, - util::{self, timeago, ProtoBuilder}, + serializer::MapResult, + util::{self, ProtoBuilder}, }; -use super::{ - response, ClientType, MapRespCtx, MapRespOptions, MapResponse, QContinuation, RustyPipeQuery, -}; +use super::{response, ClientType, MapResponse, RustyPipeQuery, YTContext}; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QChannel<'a> { + context: YTContext<'a>, browse_id: &'a str, params: ChannelTab, #[serde(skip_serializing_if = "Option::is_none")] @@ -39,6 +34,8 @@ enum ChannelTab { Live, #[serde(rename = "EglwbGF5bGlzdHMgAQ%3D%3D")] Playlists, + #[serde(rename = "EgVhYm91dPIGBAoCEgA%3D")] + Info, #[serde(rename = "EgZzZWFyY2jyBgQKAloA")] Search, } @@ -62,7 +59,9 @@ impl RustyPipeQuery { operation: &str, ) -> Result>, Error> { let channel_id = channel_id.as_ref(); + let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QChannel { + context, browse_id: channel_id, params, query, @@ -79,8 +78,7 @@ impl RustyPipeQuery { } /// Get the videos from a YouTube channel - #[tracing::instrument(skip(self), level = "error")] - pub async fn channel_videos + Debug>( + pub async fn channel_videos>( &self, channel_id: S, ) -> Result>, Error> { @@ -91,8 +89,7 @@ impl RustyPipeQuery { /// Get a ordered list of videos from a YouTube channel /// /// This function does not return channel metadata. - #[tracing::instrument(skip(self), level = "error")] - pub async fn channel_videos_order + Debug>( + pub async fn channel_videos_order>( &self, channel_id: S, order: ChannelOrder, @@ -102,8 +99,7 @@ impl RustyPipeQuery { } /// Get the videos of the given tab (Shorts, Livestreams) from a YouTube channel - #[tracing::instrument(skip(self), level = "error")] - pub async fn channel_videos_tab + Debug>( + pub async fn channel_videos_tab>( &self, channel_id: S, tab: ChannelVideoTab, @@ -115,24 +111,27 @@ impl RustyPipeQuery { /// Get a ordered list of videos from the given tab (Shorts, Livestreams) of a YouTube channel /// /// This function does not return channel metadata. - #[tracing::instrument(skip(self), level = "error")] - pub async fn channel_videos_tab_order + Debug>( + pub async fn channel_videos_tab_order>( &self, channel_id: S, tab: ChannelVideoTab, order: ChannelOrder, ) -> Result, Error> { + let visitor_data = match tab { + ChannelVideoTab::Shorts => Some(self.get_visitor_data().await?), + _ => None, + }; + self.continuation( - order_ctoken(channel_id.as_ref(), tab, order, &random_target()), + order_ctoken(channel_id.as_ref(), tab, order), ContinuationEndpoint::Browse, - None, + visitor_data.as_deref(), ) .await } /// Search the videos of a channel - #[tracing::instrument(skip(self), level = "error")] - pub async fn channel_search + Debug, S2: AsRef + Debug>( + pub async fn channel_search, S2: AsRef>( &self, channel_id: S, query: S2, @@ -147,13 +146,14 @@ impl RustyPipeQuery { } /// Get the playlists of a channel - #[tracing::instrument(skip(self), level = "error")] - pub async fn channel_playlists + Debug>( + pub async fn channel_playlists>( &self, channel_id: S, ) -> Result>, Error> { let channel_id = channel_id.as_ref(); + let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QChannel { + context, browse_id: channel_id, params: ChannelTab::Playlists, query: None, @@ -170,26 +170,25 @@ impl RustyPipeQuery { } /// Get additional metadata from the *About* tab of a channel - #[tracing::instrument(skip(self), level = "error")] - pub async fn channel_info + Debug>( + pub async fn channel_info>( &self, channel_id: S, - ) -> Result { + ) -> Result, Error> { let channel_id = channel_id.as_ref(); - let request_body = QContinuation { - continuation: &channel_info_ctoken(channel_id, &random_target()), + let context = self.get_context(ClientType::Desktop, true, None).await; + let request_body = QChannel { + context, + browse_id: channel_id, + params: ChannelTab::Info, + query: None, }; - self.execute_request_ctx::( + self.execute_request::( ClientType::Desktop, "channel_info", channel_id, "browse", &request_body, - MapRespOptions { - unlocalized: true, - ..Default::default() - }, ) .await } @@ -198,28 +197,27 @@ impl RustyPipeQuery { impl MapResponse>> for response::Channel { fn map_response( self, - ctx: &MapRespCtx<'_>, + id: &str, + lang: Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, ) -> Result>>, ExtractionError> { - let content = map_channel_content(ctx.id, self.contents, self.alerts)?; - let visitor_data = self - .response_context - .visitor_data - .or_else(|| ctx.visitor_data.map(str::to_owned)); + let content = map_channel_content(id, self.contents, self.alerts)?; let channel_data = map_channel( MapChannelData { header: self.header, metadata: self.metadata, microformat: self.microformat, - visitor_data: visitor_data.clone(), + visitor_data: self.response_context.visitor_data.clone(), has_shorts: content.has_shorts, has_live: content.has_live, }, - ctx, + id, + lang, )?; let mut mapper = response::YouTubeListMapper::::with_channel( - ctx.lang, + lang, &channel_data.c, channel_data.warnings, ); @@ -228,9 +226,8 @@ impl MapResponse>> for response::Channel { None, mapper.items, mapper.ctoken, - visitor_data, - ContinuationEndpoint::Browse, - false, + self.response_context.visitor_data, + crate::model::paginator::ContinuationEndpoint::Browse, ); Ok(MapResult { @@ -243,28 +240,27 @@ impl MapResponse>> for response::Channel { impl MapResponse>> for response::Channel { fn map_response( self, - ctx: &MapRespCtx<'_>, + id: &str, + lang: Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, ) -> Result>>, ExtractionError> { - let content = map_channel_content(ctx.id, self.contents, self.alerts)?; - let visitor_data = self - .response_context - .visitor_data - .or_else(|| ctx.visitor_data.map(str::to_owned)); + let content = map_channel_content(id, self.contents, self.alerts)?; let channel_data = map_channel( MapChannelData { header: self.header, metadata: self.metadata, microformat: self.microformat, - visitor_data, + visitor_data: self.response_context.visitor_data, has_shorts: content.has_shorts, has_live: content.has_live, }, - ctx, + id, + lang, )?; let mut mapper = response::YouTubeListMapper::::with_channel( - ctx.lang, + lang, &channel_data.c, channel_data.warnings, ); @@ -278,77 +274,59 @@ impl MapResponse>> for response::Channel { } } -impl MapResponse for response::ChannelAbout { - fn map_response(self, ctx: &MapRespCtx<'_>) -> Result, ExtractionError> { - // Channel info is always fetched in English. There is no localized data - // and it allows parsing the country name. - let lang = Language::En; +impl MapResponse> for response::Channel { + fn map_response( + self, + id: &str, + lang: Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, + ) -> Result>, ExtractionError> { + let content = map_channel_content(id, self.contents, self.alerts)?; + let channel_data = map_channel( + MapChannelData { + header: self.header, + metadata: self.metadata, + microformat: self.microformat, + visitor_data: self.response_context.visitor_data, + has_shorts: content.has_shorts, + has_live: content.has_live, + }, + id, + lang, + )?; - let ep = match self { - response::ChannelAbout::ReceivedEndpoints { - on_response_received_endpoints, - } => on_response_received_endpoints - .into_iter() - .next() - .ok_or(ExtractionError::InvalidData("no received endpoint".into()))?, - response::ChannelAbout::Content { contents } => { - // Handle errors (e.g. age restriction) when regular channel content was returned - map_channel_content(ctx.id, contents, None)?; - return Err(ExtractionError::InvalidData( - "could not extract aboutData".into(), - )); + let mut mapper = response::YouTubeListMapper::::new(lang); + mapper.map_response(content.content); + let mut warnings = mapper.warnings; + + let cinfo = mapper.channel_info.unwrap_or_else(|| { + warnings.push("no aboutFullMetadata".to_owned()); + ChannelInfo { + create_date: None, + view_count: None, + links: Vec::new(), } - }; - let continuations = ep.append_continuation_items_action.continuation_items; - let about = continuations - .c - .into_iter() - .next() - .ok_or(ExtractionError::InvalidData("no aboutChannel data".into()))? - .about_channel_renderer - .metadata - .about_channel_view_model; - let mut warnings = continuations.warnings; - - let links = about - .links - .into_iter() - .filter_map(|l| { - let lv = l.channel_external_link_view_model; - if let TextComponent::Web { url, .. } = lv.link { - Some((String::from(lv.title), util::sanitize_yt_url(&url))) - } else { - None - } - }) - .collect::>(); + }); Ok(MapResult { - c: ChannelInfo { - id: about.channel_id, - url: about.canonical_channel_url, - description: about.description, - subscriber_count: about - .subscriber_count_text - .and_then(|txt| util::parse_large_numstr_or_warn(&txt, lang, &mut warnings)), - video_count: about - .video_count_text - .and_then(|txt| util::parse_numeric_or_warn(&txt, &mut warnings)), - create_date: about.joined_date_text.and_then(|txt| { - timeago::parse_textual_date_or_warn(lang, ctx.utc_offset, &txt, &mut warnings) - .map(OffsetDateTime::date) - }), - view_count: about - .view_count_text - .and_then(|txt| util::parse_numeric_or_warn(&txt, &mut warnings)), - country: about.country.and_then(|c| util::country_from_name(&c)), - links, - }, + c: combine_channel_data(channel_data.c, cinfo), warnings, }) } } +fn map_vanity_url(url: &str, id: &str) -> Option { + if url.contains(id) { + return None; + } + + Url::parse(url).ok().map(|mut parsed_url| { + // The vanity URL from YouTube is http for some reason + _ = parsed_url.set_scheme("https"); + parsed_url.to_string() + }) +} + struct MapChannelData { header: Option, metadata: Option, @@ -360,41 +338,36 @@ struct MapChannelData { fn map_channel( d: MapChannelData, - ctx: &MapRespCtx<'_>, + id: &str, + lang: Language, ) -> Result>, ExtractionError> { let header = d.header.ok_or_else(|| ExtractionError::NotFound { - id: ctx.id.to_owned(), + id: id.to_owned(), msg: "no header".into(), })?; let metadata = d .metadata .ok_or_else(|| ExtractionError::NotFound { - id: ctx.id.to_owned(), + id: id.to_owned(), msg: "no metadata".into(), })? .channel_metadata_renderer; let microformat = d.microformat.ok_or_else(|| ExtractionError::NotFound { - id: ctx.id.to_owned(), + id: id.to_owned(), msg: "no microformat".into(), })?; - if metadata.external_id != ctx.id { + if metadata.external_id != id { return Err(ExtractionError::WrongResult(format!( "got wrong channel id {}, expected {}", - metadata.external_id, ctx.id + metadata.external_id, id ))); } - let handle = metadata + let vanity_url = metadata .vanity_channel_url .as_ref() - .and_then(|url| Url::parse(url).ok()) - .and_then(|url| { - url.path() - .strip_prefix('/') - .filter(|handle| util::CHANNEL_HANDLE_REGEX.is_match(handle)) - .map(str::to_owned) - }); + .and_then(|url| map_vanity_url(url, id)); let mut warnings = Vec::new(); Ok(MapResult { @@ -402,16 +375,17 @@ fn map_channel( response::channel::Header::C4TabbedHeaderRenderer(header) => Channel { id: metadata.external_id, name: metadata.title, - handle, - subscriber_count: header.subscriber_count_text.and_then(|txt| { - util::parse_large_numstr_or_warn(&txt, ctx.lang, &mut warnings) - }), - video_count: None, + subscriber_count: header + .subscriber_count_text + .and_then(|txt| util::parse_large_numstr_or_warn(&txt, lang, &mut warnings)), avatar: header.avatar.into(), verification: header.badges.into(), description: metadata.description, tags: microformat.microformat_data_renderer.tags, + vanity_url, banner: header.banner.into(), + mobile_banner: header.mobile_banner.into(), + tv_banner: header.tv_banner.into(), has_shorts: d.has_shorts, has_live: d.has_live, visitor_data: d.visitor_data, @@ -432,69 +406,19 @@ fn map_channel( Channel { id: metadata.external_id, name: metadata.title, - handle, subscriber_count: hdata.as_ref().and_then(|hdata| { hdata.0.as_ref().and_then(|txt| { - util::parse_large_numstr_or_warn(txt, ctx.lang, &mut warnings) + util::parse_large_numstr_or_warn(txt, lang, &mut warnings) }) }), - video_count: None, avatar: hdata.map(|hdata| hdata.1.into()).unwrap_or_default(), - // Since the carousel header is only used for YT-internal channels or special events - // (World Cup, Coachella, etc.) we can assume the channel to be verified verification: crate::model::Verification::Verified, description: metadata.description, tags: microformat.microformat_data_renderer.tags, + vanity_url, banner: Vec::new(), - has_shorts: d.has_shorts, - has_live: d.has_live, - visitor_data: d.visitor_data, - content: (), - } - } - response::channel::Header::PageHeaderRenderer(header) => { - let hdata = header.content.page_header_view_model; - // channel handle - subscriber count - video count - let md_rows = hdata.metadata.content_metadata_view_model.metadata_rows; - let (sub_part, vc_part) = if md_rows.len() > 1 { - let mp = &md_rows[1].metadata_parts; - (mp.first(), mp.get(1)) - } else { - ( - md_rows.first().and_then(|md| md.metadata_parts.get(1)), - None, - ) - }; - let subscriber_count = sub_part.and_then(|t| { - util::parse_large_numstr_or_warn::(t.as_str(), ctx.lang, &mut warnings) - }); - let video_count = vc_part.and_then(|t| { - util::parse_large_numstr_or_warn(t.as_str(), ctx.lang, &mut warnings) - }); - - Channel { - id: metadata.external_id, - name: metadata.title, - handle: handle.or_else(|| { - md_rows - .first() - .and_then(|md| md.metadata_parts.get(1)) - .map(|txt| txt.as_str().to_owned()) - .filter(|txt| util::CHANNEL_HANDLE_REGEX.is_match(txt)) - }), - subscriber_count, - video_count, - avatar: hdata - .image - .decorated_avatar_view_model - .avatar - .avatar_view_model - .image - .into(), - verification: hdata.title.map(Verification::from).unwrap_or_default(), - description: metadata.description, - tags: microformat.microformat_data_renderer.tags, - banner: hdata.banner.image_banner_view_model.image.into(), + mobile_banner: Vec::new(), + tv_banner: Vec::new(), has_shorts: d.has_shorts, has_live: d.has_live, visitor_data: d.visitor_data, @@ -520,6 +444,13 @@ fn map_channel_content( match contents { Some(contents) => { let tabs = contents.two_column_browse_results_renderer.contents; + if tabs.is_empty() { + return Err(ExtractionError::NotFound { + id: id.to_owned(), + msg: "no tabs".into(), + }); + } + let cmp_url_suffix = |endpoint: &response::channel::ChannelTabEndpoint, expect: &str| { endpoint @@ -534,46 +465,24 @@ fn map_channel_content( let mut featured_tab = false; for tab in &tabs { - if let Some(endpoint) = &tab.tab_renderer.endpoint { - if cmp_url_suffix(endpoint, "/featured") - && (tab.tab_renderer.content.section_list_renderer.is_some() - || tab.tab_renderer.content.rich_grid_renderer.is_some()) - { - featured_tab = true; - } else if cmp_url_suffix(endpoint, "/shorts") { - has_shorts = true; - } else if cmp_url_suffix(endpoint, "/streams") { - has_live = true; - } - } else { - // Check for age gate - if let Some(YouTubeListItem::ChannelAgeGateRenderer { - channel_title, - main_text, - }) = &tab - .tab_renderer - .content - .section_list_renderer - .as_ref() - .and_then(|c| c.contents.c.first()) - { - return Err(ExtractionError::Unavailable { - reason: crate::error::UnavailabilityReason::AgeRestricted, - msg: format!("{channel_title}: {main_text}"), - }); - } + if cmp_url_suffix(&tab.tab_renderer.endpoint, "/featured") + && (tab.tab_renderer.content.section_list_renderer.is_some() + || tab.tab_renderer.content.rich_grid_renderer.is_some()) + { + featured_tab = true; + } else if cmp_url_suffix(&tab.tab_renderer.endpoint, "/shorts") { + has_shorts = true; + } else if cmp_url_suffix(&tab.tab_renderer.endpoint, "/streams") { + has_live = true; } } - let channel_content = tabs - .into_iter() - .filter(|t| t.tab_renderer.endpoint.is_some()) - .find_map(|tab| { - tab.tab_renderer - .content - .rich_grid_renderer - .or(tab.tab_renderer.content.section_list_renderer) - }); + let channel_content = tabs.into_iter().find_map(|tab| { + tab.tab_renderer + .content + .rich_grid_renderer + .or(tab.tab_renderer.content.section_list_renderer) + }); // YouTube may show the "Featured" tab if the requested tab is empty/does not exist let content = if featured_tab { @@ -582,10 +491,9 @@ fn map_channel_content( match channel_content { Some(list) => list.contents, None => { - return Err(ExtractionError::NotFound { - id: id.to_owned(), - msg: "no tabs".into(), - }); + return Err(ExtractionError::InvalidData( + "could not extract content".into(), + )) } } }; @@ -604,14 +512,15 @@ fn combine_channel_data(channel_data: Channel<()>, content: T) -> Channel Channel { id: channel_data.id, name: channel_data.name, - handle: channel_data.handle, subscriber_count: channel_data.subscriber_count, - video_count: channel_data.video_count, avatar: channel_data.avatar, verification: channel_data.verification, description: channel_data.description, tags: channel_data.tags, + vanity_url: channel_data.vanity_url, banner: channel_data.banner, + mobile_banner: channel_data.mobile_banner, + tv_banner: channel_data.tv_banner, has_shorts: channel_data.has_shorts, has_live: channel_data.has_live, visitor_data: channel_data.visitor_data, @@ -620,7 +529,18 @@ fn combine_channel_data(channel_data: Channel<()>, content: T) -> Channel } /// Get the continuation token to fetch channel videos in the given order -fn order_ctoken( +fn order_ctoken(channel_id: &str, tab: ChannelVideoTab, order: ChannelOrder) -> String { + _order_ctoken( + channel_id, + tab, + order, + &format!("\n${}", util::random_uuid()), + ) +} + +/// Get the continuation token to fetch channel videos in the given order +/// (fixed targetId for testing) +fn _order_ctoken( channel_id: &str, tab: ChannelVideoTab, order: ChannelOrder, @@ -628,33 +548,7 @@ fn order_ctoken( ) -> String { let mut pb_tab = ProtoBuilder::new(); pb_tab.string(2, target_id); - - match tab { - ChannelVideoTab::Videos => match order { - ChannelOrder::Latest => { - pb_tab.varint(3, 1); - pb_tab.varint(4, 4); - } - ChannelOrder::Popular => { - pb_tab.varint(3, 2); - pb_tab.varint(4, 2); - } - ChannelOrder::Oldest => { - pb_tab.varint(3, 4); - pb_tab.varint(4, 5); - } - }, - ChannelVideoTab::Shorts => match order { - ChannelOrder::Latest => pb_tab.varint(4, 4), - ChannelOrder::Popular => pb_tab.varint(4, 2), - ChannelOrder::Oldest => pb_tab.varint(4, 5), - }, - ChannelVideoTab::Live => match order { - ChannelOrder::Latest => pb_tab.varint(5, 12), - ChannelOrder::Popular => pb_tab.varint(5, 14), - ChannelOrder::Oldest => pb_tab.varint(5, 13), - }, - } + pb_tab.varint(3, order as u64); let mut pb_3 = ProtoBuilder::new(); pb_3.embedded(tab.order_ctoken_id(), pb_tab); @@ -675,32 +569,6 @@ fn order_ctoken( pb.to_base64() } -/// Get the continuation token to fetch channel -fn channel_info_ctoken(channel_id: &str, target_id: &str) -> String { - let mut pb_3 = ProtoBuilder::new(); - pb_3.string(19, target_id); - - let mut pb_110 = ProtoBuilder::new(); - pb_110.embedded(3, pb_3); - - let mut pbi = ProtoBuilder::new(); - pbi.embedded(110, pb_110); - - let mut pb_80226972 = ProtoBuilder::new(); - pb_80226972.string(2, channel_id); - pb_80226972.string(3, &pbi.to_base64()); - - let mut pb = ProtoBuilder::new(); - pb.embedded(80_226_972, pb_80226972); - - pb.to_base64() -} - -/// Create a random UUId to build continuation tokens -fn random_target() -> String { - format!("\n${}", util::random_uuid()) -} - #[cfg(test)] mod tests { use std::{fs::File, io::BufReader}; @@ -709,15 +577,14 @@ mod tests { use rstest::rstest; use crate::{ - client::{response, MapRespCtx, MapResponse}, - error::{ExtractionError, UnavailabilityReason}, + client::{response, MapResponse}, model::{paginator::Paginator, Channel, ChannelInfo, PlaylistItem, VideoItem}, - param::{ChannelOrder, ChannelVideoTab}, + param::{ChannelOrder, ChannelVideoTab, Language}, serializer::MapResult, util::tests::TESTFILES, }; - use super::{channel_info_ctoken, order_ctoken}; + use super::_order_ctoken; #[rstest] #[case::base("videos_base", "UC2DjFE7Xf11URZqWBigcVOQ")] @@ -728,12 +595,9 @@ mod tests { #[case::upcoming("videos_upcoming", "UCcvfHa-GHSOHFAjU0-Ie57A")] #[case::richgrid("videos_20221011_richgrid", "UCh8gHdtzO2tXd593_bjErWg")] #[case::richgrid2("videos_20221011_richgrid2", "UC2DjFE7Xf11URZqWBigcVOQ")] - #[case::coachella("videos_20230415_coachella", "UCHF66aWLOxBW4l6VkSrS3cQ")] + #[case::richgrid2("videos_20230415_coachella", "UCHF66aWLOxBW4l6VkSrS3cQ")] #[case::shorts("shorts", "UCh8gHdtzO2tXd593_bjErWg")] #[case::livestreams("livestreams", "UC2DjFE7Xf11URZqWBigcVOQ")] - #[case::pageheader("shorts_20240129_pageheader", "UCh8gHdtzO2tXd593_bjErWg")] - #[case::pageheader2("videos_20240324_pageheader2", "UC2DjFE7Xf11URZqWBigcVOQ")] - #[case::lockup("shorts_20240910_lockup", "UCh8gHdtzO2tXd593_bjErWg")] fn map_channel_videos(#[case] name: &str, #[case] id: &str) { let json_path = path!(*TESTFILES / "channel" / format!("channel_{name}.json")); let json_file = File::open(json_path).unwrap(); @@ -741,7 +605,7 @@ mod tests { let channel: response::Channel = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult>> = - channel.map_response(&MapRespCtx::test(id)).unwrap(); + channel.map_response(id, Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -760,34 +624,15 @@ mod tests { } } - #[test] - fn channel_agegate() { - let json_path = path!(*TESTFILES / "channel" / format!("channel_agegate.json")); - let json_file = File::open(json_path).unwrap(); - - let channel: response::Channel = - serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let res: Result>>, ExtractionError> = - channel.map_response(&MapRespCtx::test("UCbfnHqxXs_K3kvaH-WlNlig")); - if let Err(ExtractionError::Unavailable { reason, msg }) = res { - assert_eq!(reason, UnavailabilityReason::AgeRestricted); - assert!(msg.starts_with("Laphroaig Whisky: ")); - } else { - panic!("invalid res: {res:?}") - } - } - #[rstest] - #[case::base("base")] - #[case::lockup("20241109_lockup")] - fn map_channel_playlists(#[case] name: &str) { - let json_path = path!(*TESTFILES / "channel" / format!("channel_playlists_{name}.json")); + fn map_channel_playlists() { + let json_path = path!(*TESTFILES / "channel" / "channel_playlists.json"); let json_file = File::open(json_path).unwrap(); let channel: response::Channel = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult>> = channel - .map_response(&MapRespCtx::test("UC2DjFE7Xf11URZqWBigcVOQ")) + .map_response("UC2DjFE7Xf11URZqWBigcVOQ", Language::En, None) .unwrap(); assert!( @@ -795,7 +640,7 @@ mod tests { "deserialization/mapping warnings: {:?}", map_res.warnings ); - insta::assert_ron_snapshot!(format!("map_channel_playlists_{name}"), map_res.c); + insta::assert_ron_snapshot!("map_channel_playlists", map_res.c); } #[rstest] @@ -803,10 +648,10 @@ mod tests { let json_path = path!(*TESTFILES / "channel" / "channel_info.json"); let json_file = File::open(json_path).unwrap(); - let channel: response::ChannelAbout = + let channel: response::Channel = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult = channel - .map_response(&MapRespCtx::test("UC2DjFE7Xf11U-RZqWBigcVOQ")) + let map_res: MapResult> = channel + .map_response("UC2DjFE7Xf11URZqWBigcVOQ", Language::En, None) .unwrap(); assert!( @@ -818,39 +663,31 @@ mod tests { } #[test] - fn t_order_ctoken() { + fn order_ctoken() { let channel_id = "UCXuqSBlHAE6Xw-yeJA0Tunw"; - let videos_popular_token = order_ctoken( + let videos_popular_token = _order_ctoken( channel_id, ChannelVideoTab::Videos, ChannelOrder::Popular, "\n$6461d7c8-0000-2040-87aa-089e0827e420", ); - assert_eq!(videos_popular_token, "4qmFsgJgEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaRDhnWXdHaTU2TEJJbUNpUTJORFl4WkRkak9DMHdNREF3TFRJd05EQXRPRGRoWVMwd09EbGxNRGd5TjJVME1qQVlBaUFD"); + assert_eq!(videos_popular_token, "4qmFsgJkEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaSDhnWXVHaXg2S2hJbUNpUTJORFl4WkRkak9DMHdNREF3TFRJd05EQXRPRGRoWVMwd09EbGxNRGd5TjJVME1qQVlBZyUzRCUzRA%3D%3D"); - let shorts_popular_token = order_ctoken( + let shorts_popular_token = _order_ctoken( channel_id, ChannelVideoTab::Shorts, ChannelOrder::Popular, "\n$64679ffb-0000-26b3-a1bd-582429d2c794", ); - assert_eq!(shorts_popular_token, "4qmFsgJkEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaSDhnWXVHaXhTS2hJbUNpUTJORFkzT1dabVlpMHdNREF3TFRJMllqTXRZVEZpWkMwMU9ESTBNamxrTW1NM09UUWdBZyUzRCUzRA%3D%3D"); + assert_eq!(shorts_popular_token, "4qmFsgJkEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaSDhnWXVHaXhTS2hJbUNpUTJORFkzT1dabVlpMHdNREF3TFRJMllqTXRZVEZpWkMwMU9ESTBNamxrTW1NM09UUVlBZyUzRCUzRA%3D%3D"); - let live_popular_token = order_ctoken( + let live_popular_token = _order_ctoken( channel_id, ChannelVideoTab::Live, ChannelOrder::Popular, "\n$64693069-0000-2a1e-8c7d-582429bd5ba8", ); - assert_eq!(live_popular_token, "4qmFsgJkEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaSDhnWXVHaXh5S2hJbUNpUTJORFk1TXpBMk9TMHdNREF3TFRKaE1XVXRPR00zWkMwMU9ESTBNamxpWkRWaVlUZ29EZyUzRCUzRA%3D%3D"); - } - - #[test] - fn t_channel_info_ctoken() { - let channel_id = "UCh8gHdtzO2tXd593_bjErWg"; - - let token = channel_info_ctoken(channel_id, "\n$655b339a-0000-20b9-92dc-582429d254b4"); - assert_eq!(token, "4qmFsgJgEhhVQ2g4Z0hkdHpPMnRYZDU5M19iakVyV2caRDhnWXJHaW1hQVNZS0pEWTFOV0l6TXpsaExUQXdNREF0TWpCaU9TMDVNbVJqTFRVNE1qUXlPV1F5TlRSaU5BJTNEJTNE"); + assert_eq!(live_popular_token, "4qmFsgJkEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaSDhnWXVHaXh5S2hJbUNpUTJORFk1TXpBMk9TMHdNREF3TFRKaE1XVXRPR00zWkMwMU9ESTBNamxpWkRWaVlUZ1lBZyUzRCUzRA%3D%3D"); } } diff --git a/src/client/channel_rss.rs b/src/client/channel_rss.rs index f3f7319..68f700c 100644 --- a/src/client/channel_rss.rs +++ b/src/client/channel_rss.rs @@ -1,10 +1,9 @@ -use std::fmt::Debug; +use std::collections::BTreeMap; use crate::{ error::{Error, ExtractionError}, model::ChannelRss, - report::Report, - util, + report::{Report, RustyPipeInfo}, }; use super::{response, RustyPipeQuery}; @@ -18,11 +17,7 @@ impl RustyPipeQuery { /// for checking a lot of channels or implementing a subscription feed. /// /// The downside of using the RSS feed is that it does not provide video durations. - #[tracing::instrument(skip(self), level = "error")] - pub async fn channel_rss + Debug>( - &self, - channel_id: S, - ) -> Result { + pub async fn channel_rss>(&self, channel_id: S) -> Result { let channel_id = channel_id.as_ref(); let url = format!("https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"); let xml = self @@ -37,15 +32,12 @@ impl RustyPipeQuery { _ => e, })?; - match quick_xml::de::from_str::(&xml) - .map_err(|e| ExtractionError::InvalidData(e.to_string().into())) - .and_then(|feed| feed.map_response(channel_id)) - { - Ok(res) => Ok(res), + match quick_xml::de::from_str::(&xml) { + Ok(feed) => Ok(feed.into()), Err(e) => { if let Some(reporter) = &self.client.inner.reporter { let report = Report { - info: self.rp_info(), + info: RustyPipeInfo::default(), level: crate::report::Level::ERR, operation: "channel_rss", error: Some(e.to_string()), @@ -54,103 +46,47 @@ impl RustyPipeQuery { http_request: crate::report::HTTPRequest { url: &url, method: "GET", + req_header: BTreeMap::new(), + req_body: String::new(), status: 200, - req_header: None, - req_body: None, resp_body: xml, }, }; reporter.report(&report); } - Err(Error::Extraction(e)) + + Err( + ExtractionError::InvalidData(format!("could not deserialize xml: {e}").into()) + .into(), + ) } } } } -impl response::ChannelRss { - fn map_response(self, id: &str) -> Result { - let channel_id = if self.channel_id.is_empty() { - self.entry - .iter() - .find_map(|entry| { - Some(entry.channel_id.as_str()) - .filter(|id| id.is_empty()) - .map(str::to_owned) - }) - .or_else(|| { - self.author - .uri - .strip_prefix("https://www.youtube.com/channel/") - .and_then(|id| { - if util::CHANNEL_ID_REGEX.is_match(id) { - Some(id.to_owned()) - } else { - None - } - }) - }) - .ok_or(ExtractionError::InvalidData( - "could not get channel id".into(), - ))? - } else if self.channel_id.len() == 22 { - // As of November 2023, YouTube seems to output channel IDs without the UC prefix - format!("UC{}", self.channel_id) - } else { - self.channel_id - }; - - if channel_id != id { - return Err(ExtractionError::WrongResult(format!( - "got wrong channel id {channel_id}, expected {id}", - ))); - } - - Ok(ChannelRss { - id: channel_id, - name: self.title, - videos: self - .entry - .into_iter() - .map(|item| crate::model::ChannelRssVideo { - id: item.video_id, - name: item.title, - description: item.media_group.description, - thumbnail: item.media_group.thumbnail.into(), - publish_date: item.published, - update_date: item.updated, - view_count: item.media_group.community.statistics.views, - like_count: item.media_group.community.rating.count, - }) - .collect(), - create_date: self.create_date, - }) - } -} - #[cfg(test)] mod tests { use std::{fs::File, io::BufReader}; - use crate::{client::response, util::tests::TESTFILES}; + use crate::{client::response, model::ChannelRss, util::tests::TESTFILES}; use path_macro::path; use rstest::rstest; #[rstest] - #[case::base("base", "UCHnyfMqiRRG1u-2MsSQLbXA")] - #[case::no_likes("no_likes", "UCdfxp4cUWsWryZOy-o427dw")] - #[case::no_channel_id("no_channel_id", "UCHnyfMqiRRG1u-2MsSQLbXA")] - #[case::trimmed_channel_id("trimmed_channel_id", "UCHnyfMqiRRG1u-2MsSQLbXA")] - fn map_channel_rss(#[case] name: &str, #[case] id: &str) { + #[case::base("base")] + #[case::no_likes("no_likes")] + #[case::no_channel_id("no_channel_id")] + fn map_channel_rss(#[case] name: &str) { let xml_path = path!(*TESTFILES / "channel_rss" / format!("{}.xml", name)); let xml_file = File::open(xml_path).unwrap(); let feed: response::ChannelRss = quick_xml::de::from_reader(BufReader::new(xml_file)).unwrap(); - let map_res = feed.map_response(id).unwrap(); + let map_res: ChannelRss = feed.into(); + insta::assert_ron_snapshot!(format!("map_channel_rss_{}", name), map_res); } } diff --git a/src/client/mod.rs b/src/client/mod.rs index 06386bc..ae7e180 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -18,38 +18,26 @@ mod trends; mod url_resolver; mod video_details; -#[cfg(feature = "userdata")] -#[cfg_attr(docsrs, doc(cfg(feature = "userdata")))] -mod music_userdata; -#[cfg(feature = "userdata")] -#[cfg_attr(docsrs, doc(cfg(feature = "userdata")))] -mod userdata; - #[cfg(feature = "rss")] #[cfg_attr(docsrs, doc(cfg(feature = "rss")))] mod channel_rss; -use std::collections::HashMap; -use std::ffi::OsString; use std::path::PathBuf; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use std::{borrow::Cow, fmt::Debug, time::Duration}; use once_cell::sync::Lazy; +use rand::Rng; use regex::Regex; use reqwest::{header, Client, ClientBuilder, Request, RequestBuilder, Response, StatusCode}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use sha1::{Digest, Sha1}; -use time::{OffsetDateTime, UtcOffset}; -use tokio::sync::RwLock as AsyncRwLock; +use time::OffsetDateTime; +use tokio::sync::RwLock; -use crate::error::AuthError; -use crate::util::VisitorDataCache; use crate::{ cache::{CacheStorage, FileStorage, DEFAULT_CACHE_FILE}, deobfuscate::DeobfData, error::{Error, ExtractionError}, - model::ArtistId, param::{Country, Language}, report::{FileReporter, Level, Report, Reporter, RustyPipeInfo, DEFAULT_REPORT_DIR}, serializer::MapResult, @@ -68,53 +56,35 @@ pub enum ClientType { Desktop, /// Client used by music.youtube.com /// - /// - can access YTM-specific data - /// - cannot access non-music content + /// can access YTM-specific data, cannot access non-music content DesktopMusic, - /// Client used by m.youtube.com + /// Client used by the embedded player for Smart TVs /// - /// - includes lower resolution audio streams - /// - does not return audio tracks in different languages - Mobile, - /// Client used by youtube.com/tv - /// - /// - Does not return video metadata when fetching the player - Tv, + /// can access age-restricted videos, cannot access non-embeddable videos + TvHtml5Embed, /// Client used by the Android app /// - /// - no obfuscated stream URLs - /// - includes lower resolution audio streams + /// no obfuscated stream URLs, includes lower resolution audio streams Android, /// Client used by the iOS app /// - /// - no obfuscated stream URLs + /// no obfuscated stream URLs Ios, } impl ClientType { fn is_web(self) -> bool { - matches!( - self, - ClientType::Desktop | ClientType::DesktopMusic | ClientType::Mobile - ) - } - - fn needs_deobf(self) -> bool { - !matches!(self, ClientType::Ios) - } - - fn needs_po_token(self) -> bool { - matches!( - self, - ClientType::Desktop | ClientType::DesktopMusic | ClientType::Mobile - ) + match self { + ClientType::Desktop | ClientType::DesktopMusic | ClientType::TvHtml5Embed => true, + ClientType::Android | ClientType::Ios => false, + } } } /// YouTube context request parameter #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] -struct YTContext<'a> { +pub struct YTContext<'a> { client: ClientInfo<'a>, /// only used on desktop #[serde(skip_serializing_if = "Option::is_none")] @@ -130,20 +100,15 @@ struct YTContext<'a> { struct ClientInfo<'a> { client_name: &'a str, client_version: Cow<'a, str>, - #[serde(skip_serializing_if = "str::is_empty")] - client_screen: &'a str, - #[serde(skip_serializing_if = "str::is_empty")] - device_model: &'a str, - #[serde(skip_serializing_if = "str::is_empty")] - os_name: &'a str, - #[serde(skip_serializing_if = "str::is_empty")] - os_version: &'a str, #[serde(skip_serializing_if = "Option::is_none")] - android_sdk_version: Option, + client_screen: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + device_model: Option<&'a str>, platform: &'a str, - #[serde(skip_serializing_if = "str::is_empty")] - original_url: &'a str, - visitor_data: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + original_url: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + visitor_data: Option<&'a str>, hl: Language, gl: Country, time_zone: &'a str, @@ -155,14 +120,11 @@ impl Default for ClientInfo<'_> { Self { client_name: "", client_version: Cow::default(), - client_screen: "", - device_model: "", - os_name: "", - os_version: "", - android_sdk_version: None, + client_screen: None, + device_model: None, platform: "", - original_url: "", - visitor_data: "", + original_url: None, + visitor_data: None, hl: Language::En, gl: Country::Us, time_zone: "UTC", @@ -199,22 +161,17 @@ struct ThirdParty<'a> { embed_url: &'a str, } -#[derive(Debug, Serialize)] -struct QBody<'a, T> { - context: YTContext<'a>, - #[serde(flatten)] - body: T, -} - #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QBrowse<'a> { + context: YTContext<'a>, browse_id: &'a str, } #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QBrowseParams<'a> { + context: YTContext<'a>, browse_id: &'a str, params: &'a str, } @@ -222,153 +179,38 @@ struct QBrowseParams<'a> { #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QContinuation<'a> { + context: YTContext<'a>, continuation: &'a str, } -#[derive(Debug, Serialize)] -struct OauthCodeRequest { - client_id: &'static str, - device_id: String, - device_model: &'static str, - scope: &'static str, -} +const DEFAULT_UA: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0"; -/// Device code used for logging a user into YouTube -/// -/// The login process works as follows: -/// 1. Obtain a user code and show it to the user -/// 2. The user opens the login page under , enters the code and logs in with his account -/// 3. The application has to check periodically if the login has succeeded using [`RustyPipe::user_auth_login`] or [`RustyPipe::user_auth_wait_for_login`] -/// 4. If the login is successful, the application receives a valid access/refresh token pair which can be used to access YouTube -#[derive(Debug, Deserialize)] -pub struct OauthDeviceCode { - device_code: String, - /// Code to be shown to the user to log himself in - pub user_code: String, - /// Time in seconds until the code expires - pub expires_in: u32, - /// Interval in seconds for checking if the login was completed - pub interval: u32, - /// URL to the login page () - pub verification_url: String, -} - -#[derive(Debug, Serialize)] -struct OauthTokenRequest<'a> { - client_id: &'static str, - client_secret: &'static str, - #[serde(skip_serializing_if = "Option::is_none")] - code: Option<&'a str>, - #[serde(skip_serializing_if = "Option::is_none")] - refresh_token: Option<&'a str>, - grant_type: &'static str, -} - -#[derive(Debug, Deserialize)] -#[serde(untagged)] -enum OauthTokenResponse { - Ok(OauthTokenResponseInner), - Error { - error: String, - #[serde(default)] - error_description: String, - }, -} - -#[derive(Debug, Deserialize)] -struct OauthTokenResponseInner { - access_token: String, - refresh_token: Option, - expires_in: u32, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct OauthToken { - access_token: String, - refresh_token: String, - #[serde(with = "time::serde::rfc3339")] - expires_at: OffsetDateTime, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct AuthCookie { - cookie: String, - #[serde(alias = "account_syncid", skip_serializing_if = "Option::is_none")] - channel_syncid: Option, - #[serde(skip_serializing_if = "Option::is_none")] - user_syncid: Option, - #[serde(skip_serializing_if = "Option::is_none")] - session_index: Option, -} - -impl OauthToken { - fn from_response( - value: OauthTokenResponseInner, - refresh_token: Option, - ) -> Result { - Ok(Self { - access_token: value.access_token, - refresh_token: value - .refresh_token - .or(refresh_token) - .ok_or(Error::Other("missing refresh token".into()))?, - expires_at: util::now_sec() + Duration::from_secs(value.expires_in.into()), - }) - } -} - -impl AuthCookie { - fn new(cookie: String) -> Self { - Self { - cookie, - channel_syncid: None, - session_index: None, - user_syncid: None, - } - } -} - -pub(crate) const DEFAULT_UA: &str = - "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0"; -pub(crate) const MOBILE_UA: &str = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.6778.135 Mobile Safari/537.36"; -pub(crate) const TV_UA: &str = "Mozilla/5.0 (SMART-TV; Linux; Tizen 5.0) AppleWebKit/538.1 (KHTML, like Gecko) Version/5.0 NativeTVAds Safari/538.1"; - -pub(crate) const CONSENT_COOKIE: &str = "SOCS=CAISAiAD"; +const CONSENT_COOKIE: &str = "CONSENT"; +const CONSENT_COOKIE_YES: &str = "YES+yt.462272069.de+FX+"; const YOUTUBEI_V1_URL: &str = "https://www.youtube.com/youtubei/v1/"; const YOUTUBEI_V1_GAPIS_URL: &str = "https://youtubei.googleapis.com/youtubei/v1/"; const YOUTUBE_MUSIC_V1_URL: &str = "https://music.youtube.com/youtubei/v1/"; -const YOUTUBEI_MOBILE_V1_URL: &str = "https://m.youtube.com/youtubei/v1/"; -const YOUTUBE_HOME_URL: &str = "https://www.youtube.com"; -pub(crate) const YOUTUBE_MUSIC_HOME_URL: &str = "https://music.youtube.com"; -const YOUTUBE_MOBILE_HOME_URL: &str = "https://m.youtube.com"; -const YOUTUBE_TV_URL: &str = "https://www.youtube.com/tv"; +const YOUTUBE_HOME_URL: &str = "https://www.youtube.com/"; +const YOUTUBE_MUSIC_HOME_URL: &str = "https://music.youtube.com/"; -const DISABLE_PRETTY_PRINT_PARAMETER: &str = "prettyPrint=false"; +const DISABLE_PRETTY_PRINT_PARAMETER: &str = "&prettyPrint=false"; -// Web client -const DESKTOP_CLIENT_VERSION: &str = "2.20241216.05.00"; -const DESKTOP_MUSIC_CLIENT_VERSION: &str = "1.20241216.01.00"; -const MOBILE_CLIENT_VERSION: &str = "2.20241217.07.00"; -const TV_CLIENT_VERSION: &str = "7.20241211.14.00"; +// Desktop client +const DESKTOP_CLIENT_VERSION: &str = "2.20230126.00.00"; +const DESKTOP_API_KEY: &str = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; +const TVHTML5_CLIENT_VERSION: &str = "2.0"; +const DESKTOP_MUSIC_API_KEY: &str = "AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30"; +const DESKTOP_MUSIC_CLIENT_VERSION: &str = "1.20230123.01.01"; -// Mobile app client -const IOS_CLIENT_VERSION: &str = "20.03.02"; -const IOS_VERSION: &str = "18_2_1"; -const IOS_VERSION_BUILD: &str = "18.2.1.22C161"; -const IOS_DEVICE_MODEL: &str = "iPhone16,2"; -const ANDROID_CLIENT_VERSION: &str = "19.44.38"; -const ANDROID_VERSION: &str = "11"; +// Mobile client +const MOBILE_CLIENT_VERSION: &str = "18.03.33"; +const ANDROID_API_KEY: &str = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w"; +const IOS_API_KEY: &str = "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc"; +const IOS_DEVICE_MODEL: &str = "iPhone14,5"; -const OAUTH_CLIENT_ID: &str = - "861556708454-d6dlm3lh05idd8npek18k6be8ba3oc68.apps.googleusercontent.com"; -const OAUTH_CLIENT_SECRET: &str = "SboVhoG9s0rNafixCSGGKXAT"; -const OAUTH_SCOPES: &str = "http://gdata.youtube.com https://www.googleapis.com/auth/youtube"; - -const BOTGUARD_API_VERSION: &str = "1"; - -static CLIENT_VERSION_REGEX: Lazy = - Lazy::new(|| Regex::new(r#""INNERTUBE_CONTEXT_CLIENT_VERSION":"([\w\d\._-]+?)""#).unwrap()); +static CLIENT_VERSION_REGEXES: Lazy<[Regex; 1]> = + Lazy::new(|| [Regex::new(r#"INNERTUBE_CONTEXT_CLIENT_VERSION":"([\w\d\._-]+?)""#).unwrap()]); /// The RustyPipe client used to access YouTube's API /// @@ -385,22 +227,17 @@ struct RustyPipeRef { storage: Option>, reporter: Option>, n_http_retries: u32, + consent_cookie: String, cache: CacheHolder, default_opts: RustyPipeOpts, - user_agent: Cow<'static, str>, - visitor_data_cache: VisitorDataCache, - botguard: Option, } #[derive(Clone)] struct RustyPipeOpts { lang: Language, country: Country, - timezone: Option, - utc_offset_minutes: i16, report: bool, strict: bool, - auth: Option, visitor_data: Option, } @@ -413,25 +250,6 @@ pub struct RustyPipeBuilder { user_agent: Option, default_opts: RustyPipeOpts, storage_dir: Option, - botguard_bin: DefaultOpt, - snapshot_file: Option, - po_token_cache: bool, -} - -struct BotguardCfg { - program: OsString, - version: String, - snapshot_file: PathBuf, - po_token_cache: bool, -} - -/// Proof-of-origin token -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PoToken { - /// PO token value - pub po_token: String, - /// Date until which the token is valid - pub valid_until: OffsetDateTime, } enum DefaultOpt { @@ -491,6 +309,7 @@ impl DefaultOpt { /// - [`music_search_albums`](RustyPipeQuery::music_search_albums) /// - [`music_search_artists`](RustyPipeQuery::music_search_artists) /// - [`music_search_playlists`](RustyPipeQuery::music_search_playlists) +/// - [`music_search_playlists_filter`](RustyPipeQuery::music_search_playlists_filter) /// - [`music_search_suggestion`](RustyPipeQuery::music_search_suggestion) /// - **Radio** /// - [`music_radio`](RustyPipeQuery::music_radio) @@ -508,29 +327,9 @@ impl DefaultOpt { /// - [`music_new_albums`](RustyPipeQuery::music_new_albums) /// - [`music_new_videos`](RustyPipeQuery::music_new_videos) /// -/// ### User data (🔒 Feature `userdata`) -/// -/// - **Playback history** -/// - [`history`](RustyPipeQuery::history) -/// - [`history_search`](RustyPipeQuery::history_search) -/// - [`music_history`](RustyPipeQuery::music_history) -/// - **YouTube library** -/// - [`liked_videos`](RustyPipeQuery::liked_videos) -/// - [`watch_later`](RustyPipeQuery::watch_later) -/// - [`saved_playlists`](RustyPipeQuery::saved_playlists) -/// - **Music library** -/// - [`music_saved_artists`](RustyPipeQuery::music_saved_artists) -/// - [`music_saved_albums`](RustyPipeQuery::music_saved_albums) -/// - [`music_saved_tracks`](RustyPipeQuery::music_saved_tracks) -/// - [`music_saved_playlists`](RustyPipeQuery::music_saved_playlists) -/// - [`music_liked_tracks`](RustyPipeQuery::music_liked_tracks) -/// - **Subscriptions** -/// - [`subscriptions`](RustyPipeQuery::subscriptions) -/// - [`subscription_feed`](RustyPipeQuery::subscription_feed) -/// /// ## Options /// -/// You can set the language, country and visitor data ID for individual requests. +/// You can set the language, country and visitor data cookie for individual requests. /// /// ``` /// # use rustypipe::client::RustyPipe; @@ -552,53 +351,38 @@ impl Default for RustyPipeOpts { Self { lang: Language::En, country: Country::Us, - timezone: None, - utc_offset_minutes: 0, report: false, strict: false, - auth: None, visitor_data: None, } } } -#[derive(Debug)] +#[derive(Default, Debug)] struct CacheHolder { - clients: HashMap>>, - deobf: AsyncRwLock>, - oauth_token: RwLock>, - auth_cookie: RwLock>, + desktop_client: RwLock>, + music_client: RwLock>, + deobf: RwLock>, } #[derive(Default, Debug, Clone, Serialize, Deserialize)] #[serde(default)] struct CacheData { - clients: HashMap>, + desktop_client: CacheEntry, + music_client: CacheEntry, deobf: CacheEntry, - #[serde(skip_serializing_if = "Option::is_none")] - oauth_token: Option, - #[serde(skip_serializing_if = "Option::is_none")] - auth_cookie: Option, } #[derive(Default, Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -struct CacheEntry { - #[serde( - with = "time::serde::rfc3339::option", - skip_serializing_if = "Option::is_none" - )] - last_update: Option, - /// If the entry failed to update, wait until this time before retrying - #[serde( - with = "time::serde::rfc3339::option", - skip_serializing_if = "Option::is_none" - )] - retry_at: Option, - /// RustyPipe version that failed to updated the entry - #[serde(skip_serializing_if = "Option::is_none")] - failed_version: Option, - data: Option, +#[serde(untagged)] +enum CacheEntry { + #[default] + None, + Some { + #[serde(with = "time::serde::rfc3339")] + last_update: OffsetDateTime, + data: T, + }, } #[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -606,59 +390,34 @@ struct ClientData { pub version: String, } -/// Result of a YouTube HTTP request +/// Result of a successful HTTP request struct RequestResult { /// Result of the deserialiation/mapping res: Result, Error>, status: StatusCode, body: String, - visitor_data: String, - request: Request, } impl CacheEntry { - /// Get the content of the cache if it is still fresh fn get(&self) -> Option<&T> { - self.data.as_ref().filter(|_| { - self.last_update.unwrap_or(OffsetDateTime::UNIX_EPOCH) - > (OffsetDateTime::now_utc() - time::Duration::days(1)) - }) - } - - /// Get the content of the cache, even if it is expired - fn get_expired(&self) -> Option<&T> { - self.data.as_ref() - } - - fn is_none(&self) -> bool { - self.data.is_none() - } - - /// Retry updating a cache entry only after a delay or a RustyPipe update - fn should_retry(&self) -> bool { - self.retry_at - .map(|d| OffsetDateTime::now_utc() > d) - .unwrap_or(true) - || self - .failed_version - .as_deref() - .map(|v| crate::VERSION != v) - .unwrap_or(true) - } - - fn retry_later(&mut self, delay_h: i64) { - self.retry_at = Some(util::now_sec() + time::Duration::hours(delay_h)); - self.failed_version = Some(crate::VERSION.to_owned()); + match self { + CacheEntry::Some { last_update, data } => { + if last_update < &(OffsetDateTime::now_utc() - time::Duration::hours(24)) { + None + } else { + Some(data) + } + } + CacheEntry::None => None, + } } } impl From for CacheEntry { fn from(f: T) -> Self { - Self { - last_update: Some(util::now_sec()), - retry_at: None, - failed_version: None, - data: Some(f), + Self::Some { + last_update: util::now_sec(), + data: f, } } } @@ -670,7 +429,7 @@ impl Default for RustyPipeBuilder { } impl RustyPipeBuilder { - /// Create a new [`RustyPipeBuilder`]. + /// Return a new `RustyPipeBuilder`. /// /// This is the same as [`RustyPipe::builder`] #[must_use] @@ -683,35 +442,23 @@ impl RustyPipeBuilder { n_http_retries: 2, user_agent: None, storage_dir: None, - botguard_bin: DefaultOpt::Default, - snapshot_file: None, - po_token_cache: false, } } - /// Create a new, configured [`RustyPipe`] instance. - pub fn build(self) -> Result { - self.build_with_client(ClientBuilder::new()) - } - - /// Create a new, configured RustyPipe instance using a Reqwest [`ClientBuilder`]. - pub fn build_with_client(self, mut client_builder: ClientBuilder) -> Result { - let user_agent = self - .user_agent - .map(Cow::Owned) - .unwrap_or(Cow::Borrowed(DEFAULT_UA)); - - client_builder = client_builder - .user_agent(user_agent.as_ref()) + /// Return a new, configured RustyPipe instance. + #[must_use] + pub fn build(self) -> RustyPipe { + let mut client_builder = ClientBuilder::new() + .user_agent(self.user_agent.unwrap_or_else(|| DEFAULT_UA.to_owned())) .gzip(true) .brotli(true) .redirect(reqwest::redirect::Policy::none()); - if let Some(timeout) = self.timeout.or_default(|| Duration::from_secs(20)) { + if let Some(timeout) = self.timeout.or_default(|| Duration::from_secs(10)) { client_builder = client_builder.timeout(timeout); } - let http = client_builder.build()?; + let http = client_builder.build().unwrap(); let storage_dir = self.storage_dir.unwrap_or_default(); @@ -721,63 +468,19 @@ impl RustyPipeBuilder { Box::new(FileStorage::new(cache_file)) }); - let mut cdata = if let Some(data) = storage.as_ref().and_then(|storage| storage.read()) { - match serde_json::from_str::(&data) { - Ok(data) => data, + let cdata = storage + .as_ref() + .and_then(|storage| storage.read()) + .and_then(|data| match serde_json::from_str::(&data) { + Ok(data) => Some(data), Err(e) => { - tracing::error!("Could not deserialize cache. Error: {}", e); - CacheData::default() + log::error!("Could not deserialize cache. Error: {}", e); + None } - } - } else { - CacheData::default() - }; + }) + .unwrap_or_default(); - let cache_clients = [ - ClientType::Desktop, - ClientType::DesktopMusic, - ClientType::Mobile, - ClientType::Tv, - ] - .into_iter() - .map(|c| { - ( - c, - AsyncRwLock::new(cdata.clients.remove(&c).unwrap_or_default()), - ) - }) - .collect::>(); - - let visitor_data_cache = VisitorDataCache::new(http.clone(), 50, 20); - - let botguard = match self.botguard_bin { - DefaultOpt::Some(botguard_bin) => Some(detect_botguard_bin(botguard_bin)?), - DefaultOpt::None => None, - DefaultOpt::Default => detect_botguard_bin("./rustypipe-botguard".into()) - .or_else(|_| detect_botguard_bin("rustypipe-botguard".into())) - .map_err(|e| tracing::debug!("could not detect rustypipe-botguard: {e}")) - .ok(), - } - .map(|(program, version)| { - tracing::debug!( - "rustypipe-botguard: using {} at {}", - version, - program.to_string_lossy() - ); - - BotguardCfg { - program: program.to_owned(), - version, - snapshot_file: self.snapshot_file.unwrap_or_else(|| { - let mut snapshot_file = storage_dir.clone(); - snapshot_file.push("bg_snapshot.bin"); - snapshot_file - }), - po_token_cache: self.po_token_cache, - } - }); - - Ok(RustyPipe { + RustyPipe { inner: Arc::new(RustyPipeRef { http, storage, @@ -787,18 +490,20 @@ impl RustyPipeBuilder { Box::new(FileReporter::new(report_dir)) }), n_http_retries: self.n_http_retries, + consent_cookie: format!( + "{}={}{}", + CONSENT_COOKIE, + CONSENT_COOKIE_YES, + rand::thread_rng().gen_range(100..1000) + ), cache: CacheHolder { - clients: cache_clients, - deobf: AsyncRwLock::new(cdata.deobf), - oauth_token: RwLock::new(cdata.oauth_token), - auth_cookie: RwLock::new(cdata.auth_cookie), + desktop_client: RwLock::new(cdata.desktop_client), + music_client: RwLock::new(cdata.music_client), + deobf: RwLock::new(cdata.deobf), }, default_opts: self.default_opts, - user_agent, - visitor_data_cache, - botguard, }), - }) + } } /// Set the default directory to store the cachefile and reports. @@ -851,7 +556,7 @@ impl RustyPipeBuilder { /// The timeout is applied from when the request starts connecting until the /// response body has finished. /// - /// **Default value**: 20s + /// **Default value**: 10s #[must_use] pub fn timeout(mut self, timeout: Duration) -> Self { self.timeout = DefaultOpt::Some(timeout); @@ -865,18 +570,18 @@ impl RustyPipeBuilder { self } - /// Set the maximum number of retries for YouTube requests. + /// Set the number of retries for HTTP requests. /// - /// If a request fails because of a serverside error and retries are enabled, + /// If a HTTP requests fails because of a serverside error and retries are enabled, /// RustyPipe waits 1 second before the next attempt. /// - /// The wait time is doubled for subsequent attempts (including a bit of + /// The waiting time is doubled for subsequent attempts (including a bit of /// random jitter to be less predictable). /// /// **Default value**: 2 #[must_use] pub fn n_http_retries(mut self, n_retries: u32) -> Self { - self.n_http_retries = n_retries.max(1); + self.n_http_retries = n_retries; self } @@ -916,29 +621,6 @@ impl RustyPipeBuilder { self } - /// Set the timezone and its associated UTC offset in minutes used - /// when accessing the YouTube API. - /// - /// **Default value**: `0` (UTC) - /// - /// **Info**: you can set this option for individual queries, too - #[must_use] - pub fn timezone>(mut self, timezone: S, utc_offset_minutes: i16) -> Self { - self.default_opts.timezone = Some(timezone.into()); - self.default_opts.utc_offset_minutes = utc_offset_minutes; - self - } - - /// Access the YouTube API using the local system timezone - /// - /// If the local timezone could not be determined, an error is logged and RustyPipe falls - /// back to UTC. - #[must_use] - pub fn timezone_local(self) -> Self { - let (timezone, utc_offset_minutes) = local_tz_offset(); - self.timezone(timezone, utc_offset_minutes) - } - /// Generate a report on every operation. /// /// This should only be used for debugging. @@ -962,31 +644,14 @@ impl RustyPipeBuilder { self } - /// Enable authentication for all requests - /// - /// Depending on the client type RustyPipe uses either the authentication cookie or the - /// OAuth token to authenticate requests. - #[must_use] - pub fn authenticated(mut self) -> Self { - self.default_opts.auth = Some(true); - self - } - - /// Disable authentication for all requests - #[must_use] - pub fn unauthenticated(mut self) -> Self { - self.default_opts.auth = Some(false); - self - } - - /// Set the YouTube visitor data ID + /// Set the YouTube visitor data cookie /// /// YouTube assigns a session cookie to each user which is used for personalized /// recommendations. By default, RustyPipe does not send this cookie to preserve /// user privacy. For requests that mandatate the cookie, a new one is requested /// for every query. /// - /// This option allows you to manually set the visitor data ID of your client, + /// This option allows you to manually set the visitor data cookie of your client, /// allowing you to get personalized recommendations or reproduce A/B tests. /// /// Note that YouTube has a rate limit on the number of requests from a single @@ -999,7 +664,7 @@ impl RustyPipeBuilder { self } - /// Set the YouTube visitor data ID to an optional value + /// Set the YouTube visitor data cookie to an optional value /// /// see also [`RustyPipeBuilder::visitor_data`] /// @@ -1009,53 +674,6 @@ impl RustyPipeBuilder { self.default_opts.visitor_data = visitor_data.map(S::into); self } - - /// Disable RustyPipe Botguard - /// - /// By default, RustyPipe uses the `rustypipe-botguard` binary if it is available. If you want to - /// use RustyPipe without Botguard, you can disable it. - #[must_use] - pub fn no_botguard(mut self) -> Self { - self.botguard_bin = DefaultOpt::None; - self - } - - /// Enable RustyPipe Botguard using the given binary - /// - /// Botguard is required to generate PO tokens for accessing streams on browser-based clients. - /// By default, RustyPipe uses the `rustypipe-botguard` binary if it is available. - /// - /// More information: - #[must_use] - pub fn botguard_bin>(mut self, botguard_bin: S) -> Self { - self.botguard_bin = DefaultOpt::Some(botguard_bin.into()); - self - } - - /// Set the path where the rustypipe-botguard snapshot file is stored - /// - /// After solving a Botguard challenge, rustypipe-botguard stores its - /// JavaScript environment in a snapshot file, so it can quickly generate additional tokens. - /// - /// By default the snapshot is stored in the storage_dir (Filename: bg_snapshot.bin). - #[must_use] - pub fn botguard_snapshot_file>(mut self, snapshot_file: P) -> Self { - self.snapshot_file = Some(snapshot_file.into()); - self - } - - /// Enable caching for session-bound PO tokens - /// - /// By default, RustyPipe calls Botguard for every player request to fetch both a - /// content-bound and a session-bound PO token. - /// - /// With caching enabled, the session-bound PO tokens are stored and reused. - /// Content-bound PO tokens are not used (they are not mandatory at the moment). - #[must_use] - pub fn po_token_cache(mut self) -> Self { - self.po_token_cache = true; - self - } } impl Default for RustyPipe { @@ -1069,9 +687,8 @@ impl RustyPipe { /// /// To create an instance with custom options, use [`RustyPipeBuilder`] instead. #[must_use] - #[allow(clippy::missing_panics_doc)] pub fn new() -> Self { - RustyPipeBuilder::new().build().unwrap() + RustyPipeBuilder::new().build() } /// Create a new [`RustyPipeBuilder`] @@ -1095,43 +712,35 @@ impl RustyPipe { async fn http_request(&self, request: &Request) -> Result { let mut last_resp = None; for n in 0..=self.inner.n_http_retries { - let resp = self.inner.http.execute(request.try_clone().unwrap()).await; + let resp = self + .inner + .http + .execute(request.try_clone().unwrap()) + .await?; - let err = match resp { - Ok(resp) => { - let status = resp.status(); - // Immediately return in case of success or unrecoverable status code - if status.is_success() - || (!status.is_server_error() && status != StatusCode::TOO_MANY_REQUESTS) - { - return Ok(resp); - } - last_resp = Some(Ok(resp)); - status.to_string() - } - Err(e) => { - // Retry in case of a timeout error - if !e.is_timeout() { - return Err(e); - } - last_resp = Some(Err(e)); - "timeout".to_string() - } - }; + let status = resp.status(); + // Immediately return in case of success or unrecoverable status code + if status.is_success() + || (!status.is_server_error() && status != StatusCode::TOO_MANY_REQUESTS) + { + return Ok(resp); + } // Retry in case of a recoverable status code (server err, too many requests) if n != self.inner.n_http_retries { let ms = util::retry_delay(n, 1000, 60000, 3); - tracing::warn!( + log::warn!( "Retry attempt #{}. Error: {}. Waiting {} ms", n + 1, - err, + status, ms ); tokio::time::sleep(Duration::from_millis(ms.into())).await; } + + last_resp = Some(resp); } - last_resp.unwrap() + Ok(last_resp.unwrap()) } /// Execute the given http request, returning an error in case of a @@ -1141,15 +750,7 @@ impl RustyPipe { let status = res.status(); if status.is_client_error() || status.is_server_error() { - let error_msg = if let Ok(body) = res.text().await { - serde_json::from_str::(&body) - .map(|r| Cow::from(r.error.message)) - .ok() - } else { - None - } - .unwrap_or_default(); - Err(Error::HttpStatus(status.into(), error_msg)) + Err(Error::HttpStatus(status.into(), "none".into())) } else { Ok(res) } @@ -1160,30 +761,35 @@ impl RustyPipe { Ok(self.http_request_estatus(request).await?.text().await?) } - async fn extract_client_version(&self, client_type: ClientType) -> Result { - let (sw_url, html_url, origin, ua) = match client_type { - ClientType::Desktop => ( - Some("https://www.youtube.com/sw.js"), - "https://www.youtube.com/results?search_query=", - YOUTUBE_HOME_URL, - None, - ), - ClientType::DesktopMusic => ( - Some("https://music.youtube.com/sw.js"), - YOUTUBE_MUSIC_HOME_URL, - YOUTUBE_MUSIC_HOME_URL, - None, - ), - ClientType::Mobile => ( - Some("https://m.youtube.com/sw.js"), - "https://m.youtube.com/results?search_query=", - YOUTUBE_MUSIC_HOME_URL, - Some(MOBILE_UA), - ), - ClientType::Tv => (None, YOUTUBE_TV_URL, YOUTUBE_TV_URL, Some(TV_UA)), - _ => panic!("cannot extract client version for {client_type:?}"), - }; + /// Extract the current version of the YouTube desktop client from the website. + async fn extract_desktop_client_version(&self) -> Result { + self.extract_client_version( + Some("https://www.youtube.com/sw.js"), + "https://www.youtube.com/results?search_query=", + YOUTUBE_HOME_URL, + None, + ) + .await + } + /// Extract the current version of the YouTube Music desktop client from the website. + async fn extract_music_client_version(&self) -> Result { + self.extract_client_version( + Some("https://music.youtube.com/sw.js"), + YOUTUBE_MUSIC_HOME_URL, + YOUTUBE_MUSIC_HOME_URL, + None, + ) + .await + } + + async fn extract_client_version( + &self, + sw_url: Option<&str>, + html_url: &str, + origin: &str, + ua: Option<&str>, + ) -> Result { let from_swjs = sw_url.map(|sw_url| async move { let swjs = self .http_request_txt( @@ -1193,15 +799,17 @@ impl RustyPipe { .get(sw_url) .header(header::ORIGIN, origin) .header(header::REFERER, origin) - .header(header::COOKIE, CONSENT_COOKIE) + .header(header::COOKIE, self.inner.consent_cookie.clone()) .build() .unwrap(), ) .await?; - util::get_cg_from_regex(&CLIENT_VERSION_REGEX, &swjs, 1).ok_or(Error::Extraction( - ExtractionError::InvalidData("Could not find client version in sw.js".into()), - )) + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1).ok_or( + Error::Extraction(ExtractionError::InvalidData(Cow::Borrowed( + "Could not find client version in sw.js", + ))), + ) }); let from_html = async { @@ -1212,9 +820,11 @@ impl RustyPipe { let html = self.http_request_txt(&builder.build().unwrap()).await?; - util::get_cg_from_regex(&CLIENT_VERSION_REGEX, &html, 1).ok_or(Error::Extraction( - ExtractionError::InvalidData("Could not find client version on html page".into()), - )) + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or( + Error::Extraction(ExtractionError::InvalidData(Cow::Borrowed( + "Could not find client version on html page", + ))), + ) }; if let Some(from_swjs) = from_swjs { @@ -1227,50 +837,71 @@ impl RustyPipe { } } - async fn get_client_version(&self, client_type: ClientType) -> Cow<'static, str> { + /// Get the current version of the YouTube web client from the following sources + /// + /// 1. from cache + /// 2. from YouTube's service worker script (`sw.js`) + /// 3. from the YouTube website + /// 4. fall back to the hardcoded version + async fn get_desktop_client_version(&self) -> String { // Write lock here to prevent concurrent tasks from fetching the same data - let mut client = self.inner.cache.clients[&client_type].write().await; + let mut desktop_client = self.inner.cache.desktop_client.write().await; - match client.get() { - Some(cdata) => cdata.version.clone().into(), + match desktop_client.get() { + Some(cdata) => cdata.version.clone(), None => { - if client.should_retry() { - tracing::debug!("getting {client_type:?} client version"); - match self.extract_client_version(client_type).await { - Ok(version) => { - *client = CacheEntry::from(ClientData { - version: version.clone(), - }); - drop(client); - self.store_cache().await; - return version.into(); - } - Err(e) => { - client.retry_later(1); - drop(client); - self.store_cache().await; - tracing::warn!( - "{e}, falling back to hardcoded {client_type:?} client version" - ); - } + log::debug!("getting desktop client version"); + match self.extract_desktop_client_version().await { + Ok(version) => { + *desktop_client = CacheEntry::from(ClientData { + version: version.clone(), + }); + drop(desktop_client); + self.store_cache().await; + version + } + Err(e) => { + log::warn!("{}, falling back to hardcoded desktop client version", e); + DESKTOP_CLIENT_VERSION.to_owned() } - } else { - tracing::warn!("falling back to hardcoded {client_type:?} client version") } - - match client_type { - ClientType::Desktop => DESKTOP_CLIENT_VERSION, - ClientType::DesktopMusic => DESKTOP_MUSIC_CLIENT_VERSION, - ClientType::Mobile => MOBILE_CLIENT_VERSION, - ClientType::Tv => TV_CLIENT_VERSION, - _ => unreachable!(), - } - .into() } } } - /// Get deobfuscation data (either from cache or extracted from YouTube's JavaScript code) + /// Get the current version of the YouTube Music web client from the following sources + /// + /// 1. from cache + /// 2. from YouTube Music's service worker script (`sw.js`) + /// 3. from the YouTube Music website + /// 4. fall back to the hardcoded version + async fn get_music_client_version(&self) -> String { + // Write lock here to prevent concurrent tasks from fetching the same data + let mut music_client = self.inner.cache.music_client.write().await; + + match music_client.get() { + Some(cdata) => cdata.version.clone(), + None => { + log::debug!("getting music client version"); + match self.extract_music_client_version().await { + Ok(version) => { + *music_client = CacheEntry::from(ClientData { + version: version.clone(), + }); + drop(music_client); + self.store_cache().await; + version + } + Err(e) => { + log::warn!("{}, falling back to hardcoded music client version", e); + DESKTOP_MUSIC_CLIENT_VERSION.to_owned() + } + } + } + } + } + + /// Instantiate a new deobfuscator from either cached or extracted YouTube JavaScript code. async fn get_deobf_data(&self) -> Result { // Write lock here to prevent concurrent tasks from fetching the same data let mut deobf_data = self.inner.cache.deobf.write().await; @@ -1278,451 +909,54 @@ impl RustyPipe { match deobf_data.get() { Some(deobf_data) => Ok(deobf_data.clone()), None => { - // Only attempt to fetch deobf data every 24 hours to avoid a flood of error reports - // if the client JS cannot be parsed - if deobf_data.should_retry() { - tracing::debug!("getting deobf data"); - - match DeobfData::extract(&self.inner.http, self.inner.reporter.as_deref()).await - { - Ok(new_data) => { - // Write new data to the cache - *deobf_data = CacheEntry::from(new_data.clone()); - drop(deobf_data); - self.store_cache().await; - Ok(new_data) - } - Err(e) => { - // Try to fall back to expired cache data if available, otherwise return error - deobf_data.retry_later(24); - let res = match deobf_data.get_expired() { - Some(d) => { - tracing::warn!("could not get new deobf data ({e}), falling back to expired cache"); - Ok(d.clone()) - } - None => Err(e), - }; - drop(deobf_data); - self.store_cache().await; - res - } - } - } else { - match deobf_data.get_expired() { - Some(d) => { - tracing::warn!( - "could not get new deobf data, falling back to expired cache" - ); - Ok(d.clone()) - } - None => Err(Error::Extraction(ExtractionError::Deobfuscation( - "could not get deobf data".into(), - ))), - } - } + log::debug!("getting deobfuscator"); + let new_data = DeobfData::download(self.inner.http.clone()).await?; + *deobf_data = CacheEntry::from(new_data.clone()); + drop(deobf_data); + self.store_cache().await; + Ok(new_data) } } } /// Write the current cache data to the storage backend. async fn store_cache(&self) { - let mut cache_clients = HashMap::new(); - for (c, lk) in &self.inner.cache.clients { - let v = lk.read().await.clone(); - if !v.is_none() { - cache_clients.insert(*c, v); - } - } - if let Some(storage) = &self.inner.storage { let cdata = CacheData { - clients: cache_clients, + desktop_client: self.inner.cache.desktop_client.read().await.clone(), + music_client: self.inner.cache.music_client.read().await.clone(), deobf: self.inner.cache.deobf.read().await.clone(), - oauth_token: self.inner.cache.oauth_token.read().unwrap().clone(), - auth_cookie: self.inner.cache.auth_cookie.read().unwrap().clone(), }; match serde_json::to_string(&cdata) { Ok(data) => storage.write(&data), - Err(e) => tracing::error!("Could not serialize cache. Error: {}", e), + Err(e) => log::error!("Could not serialize cache. Error: {}", e), } } } - /// Get a new device code for logging into YouTube - pub async fn user_auth_get_code(&self) -> Result { - tracing::debug!("getting OAuth user code"); - - let code_request = OauthCodeRequest { - client_id: OAUTH_CLIENT_ID, - device_id: util::random_uuid(), - device_model: "ytlr:samsung:smarttv", - scope: OAUTH_SCOPES, - }; - - self.inner - .http - .post("https://www.youtube.com/o/oauth2/device/code") - .header(header::USER_AGENT, TV_UA) - .header(header::ORIGIN, YOUTUBE_HOME_URL) - .header(header::REFERER, YOUTUBE_TV_URL) - .json(&code_request) - .send() - .await? - .error_for_status()? - .json::() - .await - .map_err(Error::from) - } - - /// Attempt to log in the user using the given device code + /// Request a new visitor data cookie from YouTube /// - /// Returns `true` if the user has successfully logged in using the code. - /// - /// Returns `false` if the user has not logged in yet, in this case repeat - /// the login attempt after a few seconds. - /// The function [`RustyPipe::user_auth_wait_for_login`] does this automatically. - pub async fn user_auth_login(&self, code: &OauthDeviceCode) -> Result { - tracing::debug!("OAuth login attempt (user_code: {})", code.user_code); + /// Since the cookie is shared between YT and YTM and the YTM page loads faster, + /// we request that. + async fn get_visitor_data(&self) -> Result { + log::debug!("getting YT visitor data"); + let resp = self.inner.http.get(YOUTUBE_MUSIC_HOME_URL).send().await?; - let token_request = OauthTokenRequest { - client_id: OAUTH_CLIENT_ID, - client_secret: OAUTH_CLIENT_SECRET, - code: Some(&code.device_code), - refresh_token: None, - grant_type: "http://oauth.net/grant_type/device/1.0", - }; - - let token_response = self - .inner - .http - .post("https://www.youtube.com/o/oauth2/token") - .header(header::USER_AGENT, TV_UA) - .header(header::ORIGIN, YOUTUBE_HOME_URL) - .header(header::REFERER, YOUTUBE_TV_URL) - .json(&token_request) - .send() - .await? - .error_for_status()? - .json::() - .await?; - - match token_response { - OauthTokenResponse::Ok(token) => { - let token = OauthToken::from_response(token, None)?; - { - let mut cache_token = self.inner.cache.oauth_token.write().unwrap(); - *cache_token = Some(token); + resp.headers() + .get_all(header::SET_COOKIE) + .iter() + .find_map(|c| { + if let Ok(cookie) = c.to_str() { + if let Some(after) = cookie.strip_prefix("__Secure-YEC=") { + return after.split_once(';').map(|s| s.0.to_owned()); + } } - self.store_cache().await; - Ok(true) - } - OauthTokenResponse::Error { - error, - error_description, - } => match error.as_str() { - "authorization_pending" => Ok(false), - "expired_token" => Err(Error::Auth(AuthError::DeviceCodeExpired)), - _ => Err(Error::Auth(AuthError::Other(format!( - "{error}: {error_description}" - )))), - }, - } - } - - /// Attempt to refresh the OAuth access token to check if the user is successfully logged in - /// and the session is still valid. - pub async fn user_auth_check_login(&self) -> Result<(), Error> { - let cache_token = self.inner.cache.oauth_token.read().unwrap().clone(); - if let Some(token) = cache_token { - let token = self.user_auth_refresh_token(&token.refresh_token).await?; - { - let mut cache_token = self.inner.cache.oauth_token.write().unwrap(); - *cache_token = Some(token.clone()); - } - self.store_cache().await; - Ok(()) - } else { - Err(Error::Auth(AuthError::NoLogin)) - } - } - - /// Attempt to log in the user using the given device code. - /// - /// This function waits until the login was successful or an error occurred. - pub async fn user_auth_wait_for_login(&self, code: &OauthDeviceCode) -> Result<(), Error> { - while !self.user_auth_login(code).await? { - tokio::time::sleep(Duration::from_secs(code.interval.into())).await; - } - Ok(()) - } - - /// Log out the user and remove the OAuth token from the cache - pub async fn user_auth_logout(&self) -> Result<(), Error> { - #[derive(Serialize)] - struct RevokeRequest<'a> { - token: &'a str, - } - - let cache_token = self - .inner - .cache - .oauth_token - .read() - .unwrap() - .clone() - .ok_or(Error::Auth(AuthError::NoLogin))?; - let revoke_request = RevokeRequest { - token: &cache_token.refresh_token, - }; - - let resp = self - .inner - .http - .post("https://www.youtube.com/o/oauth2/revoke") - .header(header::USER_AGENT, TV_UA) - .header(header::ORIGIN, YOUTUBE_HOME_URL) - .header(header::REFERER, YOUTUBE_TV_URL) - .json(&revoke_request) - .send() - .await?; - - if let Err(estatus) = resp.error_for_status_ref().map(|_| ()) { - if let Ok(OauthTokenResponse::Error { - error, - error_description, - }) = resp.json::().await - { - // User is already logged out - if error == "invalid_token" { - tracing::info!("user already logged out ({error}: {error_description})"); - } else { - return Err(Error::Other(format!("{error}: {error_description}").into())); - } - } else { - return Err(estatus.into()); - } - } - self.user_auth_remove_token().await; - Ok(()) - } - - /// Remove the stored OAuth token from the cache - async fn user_auth_remove_token(&self) { - { - let mut cache_token = self.inner.cache.oauth_token.write().unwrap(); - *cache_token = None; - } - self.store_cache().await; - } - - /// Obtain a new OAuth token using the given refresh token - async fn user_auth_refresh_token(&self, refresh_token: &str) -> Result { - tracing::debug!("refreshing OAuth token"); - - let token_request = OauthTokenRequest { - client_id: OAUTH_CLIENT_ID, - client_secret: OAUTH_CLIENT_SECRET, - code: None, - refresh_token: Some(refresh_token), - grant_type: "refresh_token", - }; - - let token_response = self - .inner - .http - .post("https://www.youtube.com/o/oauth2/token") - .header(header::USER_AGENT, TV_UA) - .header(header::ORIGIN, YOUTUBE_HOME_URL) - .header(header::REFERER, YOUTUBE_TV_URL) - .json(&token_request) - .send() - .await? - .json::() - .await?; - - match token_response { - OauthTokenResponse::Ok(token) => { - OauthToken::from_response(token, Some(refresh_token.to_owned())) - } - OauthTokenResponse::Error { - error, - error_description, - } => { - // If the token is expired or revoked, remove it from the client - if error == "invalid_grant" { - self.user_auth_remove_token().await; - } - Err(Error::Auth(AuthError::Refresh(format!( - "{error}: {error_description}" - )))) - } - } - } - - /// Get the OAuth access token for accessing YouTube as an authenticated user - pub async fn user_auth_access_token(&self) -> Result { - let cache_token = self.inner.cache.oauth_token.read().unwrap().clone(); - if let Some(token) = cache_token { - if token.expires_at < (OffsetDateTime::now_utc() + Duration::from_secs(60)) { - let token = self.user_auth_refresh_token(&token.refresh_token).await?; - let access_token = token.access_token.to_owned(); - - { - let mut cache_token = self.inner.cache.oauth_token.write().unwrap(); - *cache_token = Some(token.clone()); - } - self.store_cache().await; - - Ok(access_token) - } else { - Ok(token.access_token.to_owned()) - } - } else { - Err(Error::Auth(AuthError::NoLogin)) - } - } - - /// Get a copy of the authentication cookie from the cache - fn user_auth_cookie(&self) -> Result { - self.inner - .cache - .auth_cookie - .read() - .unwrap() - .clone() - .ok_or(Error::Auth(AuthError::NoLogin)) - } - - fn user_auth_datasync_id(&self) -> Result { - self.inner - .cache - .auth_cookie - .read() - .unwrap() - .as_ref() - .and_then(|c| c.user_syncid.as_ref().map(|id| id.to_owned())) - .ok_or(Error::Auth(AuthError::NoLogin)) - } - - /// Set the user authentication cookie - /// - /// The cookie is used for authenticated requests with browser-based clients - /// (Desktop, DesktopMusic, Mobile). - /// - /// **Note:** YouTube rotates cookies every few minutes when using the web application. - /// Do not use the session you obtained cookies from afterwards or it will - /// become invalid. - /// - /// I recommend to log in using Incognito mode, get the cookies from the devtools - /// and then close the page. - pub async fn user_auth_set_cookie>(&self, cookie: S) -> Result<(), Error> { - let cookie = cookie.into(); - if cookie.is_empty() { - return Err(Error::Auth(AuthError::NoLogin)); - } - let mut auth_cookie = AuthCookie::new(cookie); - self.extract_session_headers(&mut auth_cookie).await?; - { - let mut c = self.inner.cache.auth_cookie.write().unwrap(); - *c = Some(auth_cookie); - } - self.store_cache().await; - Ok(()) - } - - /// Parse the user authentication cookie from a Netscape HTTP Cookie File - /// - /// The cookie is used for authenticated requests with browser-based clients - /// (Desktop, DesktopMusic, Mobile). - /// - /// cookie.txt files can be extracted using browser plugins like - /// "Get cookies.txt LOCALLY" ([Firefox](https://addons.mozilla.org/de/firefox/addon/get-cookies-txt-locally/)) - /// ([Chromium](https://chromewebstore.google.com/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc)). - /// - /// **Note:** YouTube rotates cookies every few minutes when using the web application. - /// Do not use the session you obtained cookies from afterwards or it will - /// become invalid. - /// - /// I recommend to log in using Incognito mode, obtain the cookies and then close the page. - pub async fn user_auth_set_cookie_txt(&self, cookies: &str) -> Result<(), Error> { - let cookie = util::parse_netscape_cookies(cookies, ".youtube.com")?; - self.user_auth_set_cookie(cookie).await - } - - /// Remove the user authentication cookie from cache storage - pub async fn user_auth_remove_cookie(&self) -> Result<(), Error> { - { - let mut cookie = self.inner.cache.auth_cookie.write().unwrap(); - if cookie.is_none() { - return Err(Error::Auth(AuthError::NoLogin)); - } - *cookie = None; - } - self.store_cache().await; - Ok(()) - } - - /// Attempt to fetch the YouTube website with login cookies to check if the user is successfully logged in - /// and the session is still valid. - pub async fn user_auth_check_cookie(&self) -> Result<(), Error> { - let mut cookie = self.user_auth_cookie()?; - self.extract_session_headers(&mut cookie).await?; - Ok(()) - } - - /// Since YouTube allows multiple channels/profiles per account, cookie-authenticated requests must include - /// the X-Goog-AuthUser and X-Goog-PageId headers to specify which account should be used. - /// - /// The header values are included in the ytcfg object which is embedded in the html code. - async fn extract_session_headers(&self, auth_cookie: &mut AuthCookie) -> Result<(), Error> { - let re_session_id = Regex::new(r#""USER_SESSION_ID":"[\d]+?""#).unwrap(); - let re_sync_id = Regex::new(r#""datasyncId":"([\w|]+?)""#).unwrap(); - let re_session_index = Regex::new(r#""SESSION_INDEX":"([\d]+?)""#).unwrap(); - - let req = self - .inner - .http - .get("https://www.youtube.com/results?search_query=") - .header(header::COOKIE, &auth_cookie.cookie) - .build()?; - let html = self.http_request_txt(&req).await?; - - if !re_session_id.is_match(&html) { - tracing::debug!("session check failed: USER_SESSION_ID not found in reponse"); - return Err(Error::Auth(AuthError::NoLogin)); - } - - let datasync_id = - util::get_cg_from_regex(&re_sync_id, &html, 1).ok_or(Error::Extraction( - ExtractionError::InvalidData("could not find datasyncId on html page".into()), - ))?; - - // datasyncid is of the form "channel_syncid||user_syncid" for secondary channel - // and just "user_syncid||" for primary channel. - let (p1, p2) = - datasync_id - .split_once("||") - .ok_or(Error::Extraction(ExtractionError::InvalidData( - "datasyncId does not contain || seperator".into(), - )))?; - (auth_cookie.channel_syncid, auth_cookie.user_syncid) = if p2.is_empty() { - (None, Some(p1.to_owned())) - } else { - (Some(p1.to_owned()), Some(p2.to_owned())) - }; - - auth_cookie.session_index = Some( - util::get_cg_from_regex(&re_session_index, &html, 1).ok_or(Error::Extraction( - ExtractionError::InvalidData("could not find SESSION_INDEX on html page".into()), - ))?, - ); - Ok(()) - } - - /// Get the version string (e.g. `rustypipe-botguard 0.1.1`) of the used botguard binary - pub async fn version_botguard(&self) -> Option { - self.inner.botguard.as_ref().map(|bg| bg.version.to_owned()) + None + }) + .ok_or(Error::Extraction(ExtractionError::InvalidData( + Cow::Borrowed("could not get YTM cookies"), + ))) } } @@ -1745,22 +979,6 @@ impl RustyPipeQuery { self } - /// Set the timezone and its associated UTC offset in minutes used - /// when accessing the YouTube API. - #[must_use] - pub fn timezone>(mut self, timezone: S, utc_offset_minutes: i16) -> Self { - self.opts.timezone = Some(timezone.into()); - self.opts.utc_offset_minutes = utc_offset_minutes; - self - } - - /// Access the YouTube API using the local system timezone - #[must_use] - pub fn timezone_local(self) -> Self { - let (timezone, utc_offset_minutes) = local_tz_offset(); - self.timezone(timezone, utc_offset_minutes) - } - /// Generate a report on every operation. /// /// This should only be used for debugging. @@ -1780,31 +998,14 @@ impl RustyPipeQuery { self } - /// Enable authentication for this request - /// - /// Depending on the client type RustyPipe uses either the authentication cookie or the - /// OAuth token to authenticate requests. - #[must_use] - pub fn authenticated(mut self) -> Self { - self.opts.auth = Some(true); - self - } - - /// Disable authentication for this request - #[must_use] - pub fn unauthenticated(mut self) -> Self { - self.opts.auth = Some(false); - self - } - - /// Set the YouTube visitor data ID + /// Set the YouTube visitor data cookie /// /// YouTube assigns a session cookie to each user which is used for personalized /// recommendations. By default, RustyPipe does not send this cookie to preserve /// user privacy. For requests that mandatate the cookie, a new one is requested /// for every query. /// - /// This option allows you to manually set the visitor data ID of your query, + /// This option allows you to manually set the visitor data cookie of your query, /// allowing you to get personalized recommendations or reproduce A/B tests. /// /// Note that YouTube has a rate limit on the number of requests from a single @@ -1815,7 +1016,7 @@ impl RustyPipeQuery { self } - /// Set the YouTube visitor data ID to an optional value + /// Set the YouTube visitor data cookie to an optional value /// /// see also [`RustyPipeQuery::visitor_data`] #[must_use] @@ -1824,112 +1025,35 @@ impl RustyPipeQuery { self } - /// Get the user agent for the given client type - /// - /// This can be used for additional HTTP requests (e.g. downloading/streaming) - pub fn user_agent(&self, ctype: ClientType) -> Cow<'_, str> { - match ctype { - ClientType::Desktop | ClientType::DesktopMusic => { - Cow::Borrowed(&self.client.inner.user_agent) - } - ClientType::Mobile => MOBILE_UA.into(), - ClientType::Tv => TV_UA.into(), - ClientType::Android => format!( - "com.google.android.youtube/{} (Linux; U; Android {}) gzip", - ANDROID_CLIENT_VERSION, ANDROID_VERSION - ) - .into(), - ClientType::Ios => format!( - "com.google.ios.youtube/{} ({}; U; CPU iOS {} like Mac OS X)", - IOS_CLIENT_VERSION, IOS_DEVICE_MODEL, IOS_VERSION - ) - .into(), - } - } - - /// Return `true` if the client has stored login credentials for the given client type - /// and authentication has not been disabled - pub fn auth_enabled(&self, ctype: ClientType) -> bool { - if self.opts.auth == Some(false) { - return false; - } - if ctype.is_web() { - let auth_cookie = self.client.inner.cache.auth_cookie.read().unwrap(); - auth_cookie.is_some() - } else if ctype == ClientType::Tv { - let cache_token = self.client.inner.cache.oauth_token.read().unwrap(); - cache_token.is_some() - } else { - false - } - } - - /// Filter the given list of client types and iterate over those which have login credentials available. - pub fn auth_enabled_clients<'a>( - &self, - clients: &'a [ClientType], - ) -> impl Iterator + 'a { - let (has_cookie, has_token) = if self.opts.auth == Some(false) { - (false, false) - } else { - let auth_cookie = self.client.inner.cache.auth_cookie.read().unwrap(); - let oauth_token = self.client.inner.cache.oauth_token.read().unwrap(); - (auth_cookie.is_some(), oauth_token.is_some()) - }; - - clients - .iter() - .filter(move |c| { - if c.is_web() { - has_cookie - } else if **c == ClientType::Tv { - has_token - } else { - false - } - }) - .copied() - } - - /// Return the first client type from the given list which has login credentials available. - /// - /// Returns [`None`] if authentication has been disabled or there are no available client types. - pub fn auth_enabled_client(&self, clients: &[ClientType]) -> Option { - self.auth_enabled_clients(clients).next() - } - /// Create a new context object, which is included in every request to /// the YouTube API and contains language, country and device parameters. /// /// # Parameters /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...) /// - `localized`: Whether to include the configured language and country - async fn get_context<'a>( + pub async fn get_context<'a>( &'a self, ctype: ClientType, localized: bool, - visitor_data: &'a str, - ) -> YTContext<'a> { + visitor_data: Option<&'a str>, + ) -> YTContext { let (hl, gl) = if localized { (self.opts.lang, self.opts.country) } else { (Language::En, Country::Us) }; - let utc_offset_minutes = self.opts.utc_offset_minutes; - let time_zone = self.opts.timezone.as_deref().unwrap_or("UTC"); + let visitor_data = self.opts.visitor_data.as_deref().or(visitor_data); match ctype { ClientType::Desktop => YTContext { client: ClientInfo { client_name: "WEB", - client_version: self.client.get_client_version(ctype).await, + client_version: Cow::Owned(self.client.get_desktop_client_version().await), platform: "DESKTOP", - original_url: YOUTUBE_HOME_URL, + original_url: Some(YOUTUBE_HOME_URL), visitor_data, hl, gl, - time_zone, - utc_offset_minutes, ..Default::default() }, request: Some(RequestYT::default()), @@ -1939,70 +1063,43 @@ impl RustyPipeQuery { ClientType::DesktopMusic => YTContext { client: ClientInfo { client_name: "WEB_REMIX", - client_version: self.client.get_client_version(ctype).await, + client_version: Cow::Owned(self.client.get_music_client_version().await), platform: "DESKTOP", - original_url: YOUTUBE_MUSIC_HOME_URL, + original_url: Some(YOUTUBE_MUSIC_HOME_URL), visitor_data, hl, gl, - time_zone, - utc_offset_minutes, ..Default::default() }, request: Some(RequestYT::default()), user: User::default(), third_party: None, }, - ClientType::Mobile => YTContext { + ClientType::TvHtml5Embed => YTContext { client: ClientInfo { - client_name: "MWEB", - client_version: self.client.get_client_version(ctype).await, - platform: "MOBILE", - original_url: YOUTUBE_MOBILE_HOME_URL, - visitor_data, - hl, - gl, - time_zone, - utc_offset_minutes, - ..Default::default() - }, - request: Some(RequestYT::default()), - user: User::default(), - third_party: None, - }, - ClientType::Tv => YTContext { - client: ClientInfo { - client_name: "TVHTML5", - client_version: self.client.get_client_version(ctype).await, - client_screen: "WATCH", + client_name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", + client_version: Cow::Borrowed(TVHTML5_CLIENT_VERSION), + client_screen: Some("EMBED"), platform: "TV", - device_model: "SmartTV", visitor_data, hl, gl, - time_zone, - utc_offset_minutes, ..Default::default() }, request: Some(RequestYT::default()), user: User::default(), third_party: Some(ThirdParty { - embed_url: YOUTUBE_TV_URL, + embed_url: YOUTUBE_HOME_URL, }), }, ClientType::Android => YTContext { client: ClientInfo { client_name: "ANDROID", - client_version: ANDROID_CLIENT_VERSION.into(), - os_name: "Android", - os_version: ANDROID_VERSION, - android_sdk_version: Some(30), + client_version: Cow::Borrowed(MOBILE_CLIENT_VERSION), platform: "MOBILE", visitor_data, hl, gl, - time_zone, - utc_offset_minutes, ..Default::default() }, request: None, @@ -2012,16 +1109,12 @@ impl RustyPipeQuery { ClientType::Ios => YTContext { client: ClientInfo { client_name: "IOS", - client_version: IOS_CLIENT_VERSION.into(), - device_model: IOS_DEVICE_MODEL, - os_name: "iPhone", - os_version: IOS_VERSION_BUILD, + client_version: Cow::Borrowed(MOBILE_CLIENT_VERSION), + device_model: Some(IOS_DEVICE_MODEL), platform: "MOBILE", visitor_data, hl, gl, - time_zone, - utc_offset_minutes, ..Default::default() }, request: None, @@ -2038,332 +1131,96 @@ impl RustyPipeQuery { /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...) /// - `method`: HTTP method /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/?key=...`) - /// - `visitor_data`: YouTube visitor data ID - async fn request_builder( - &self, - ctype: ClientType, - endpoint: &str, - visitor_data: Option<&str>, - ) -> Result { - let mut r = match ctype { + async fn request_builder(&self, ctype: ClientType, endpoint: &str) -> RequestBuilder { + match ctype { ClientType::Desktop => self .client .inner .http .post(format!( - "{YOUTUBEI_V1_URL}{endpoint}?{DISABLE_PRETTY_PRINT_PARAMETER}" + "{YOUTUBEI_V1_URL}{endpoint}?key={DESKTOP_API_KEY}{DISABLE_PRETTY_PRINT_PARAMETER}" )) .header(header::ORIGIN, YOUTUBE_HOME_URL) .header(header::REFERER, YOUTUBE_HOME_URL) - .header(header::COOKIE, CONSENT_COOKIE) + .header(header::COOKIE, self.client.inner.consent_cookie.clone()) .header("X-YouTube-Client-Name", "1") .header( "X-YouTube-Client-Version", - self.client.get_client_version(ctype).await.into_owned(), + self.client.get_desktop_client_version().await, ), ClientType::DesktopMusic => self .client .inner .http .post(format!( - "{YOUTUBE_MUSIC_V1_URL}{endpoint}?{DISABLE_PRETTY_PRINT_PARAMETER}" + "{YOUTUBE_MUSIC_V1_URL}{endpoint}?key={DESKTOP_MUSIC_API_KEY}{DISABLE_PRETTY_PRINT_PARAMETER}" )) .header(header::ORIGIN, YOUTUBE_MUSIC_HOME_URL) .header(header::REFERER, YOUTUBE_MUSIC_HOME_URL) - .header(header::COOKIE, CONSENT_COOKIE) + .header(header::COOKIE, self.client.inner.consent_cookie.clone()) .header("X-YouTube-Client-Name", "67") .header( "X-YouTube-Client-Version", - self.client.get_client_version(ctype).await.into_owned(), + self.client.get_music_client_version().await, ), - ClientType::Mobile => self + ClientType::TvHtml5Embed => self .client .inner .http .post(format!( - "{YOUTUBEI_MOBILE_V1_URL}{endpoint}?{DISABLE_PRETTY_PRINT_PARAMETER}" - )) - .header(header::ORIGIN, YOUTUBE_MUSIC_HOME_URL) - .header(header::REFERER, YOUTUBE_MUSIC_HOME_URL) - .header(header::COOKIE, CONSENT_COOKIE) - .header("X-YouTube-Client-Name", "2") - .header( - "X-YouTube-Client-Version", - self.client.get_client_version(ctype).await.into_owned(), - ), - ClientType::Tv => self - .client - .inner - .http - .post(format!( - "{YOUTUBEI_V1_URL}{endpoint}?{DISABLE_PRETTY_PRINT_PARAMETER}" + "{YOUTUBEI_V1_URL}{endpoint}?key={DESKTOP_API_KEY}{DISABLE_PRETTY_PRINT_PARAMETER}" )) .header(header::ORIGIN, YOUTUBE_HOME_URL) - .header(header::REFERER, YOUTUBE_TV_URL) - .header("X-YouTube-Client-Name", "7") - .header( - "X-YouTube-Client-Version", - self.client.get_client_version(ctype).await.into_owned(), - ), + .header(header::REFERER, YOUTUBE_HOME_URL) + .header("X-YouTube-Client-Name", "1") + .header("X-YouTube-Client-Version", TVHTML5_CLIENT_VERSION), ClientType::Android => self .client .inner .http .post(format!( - "{YOUTUBEI_V1_GAPIS_URL}{endpoint}?{DISABLE_PRETTY_PRINT_PARAMETER}" + "{YOUTUBEI_V1_GAPIS_URL}{endpoint}?key={ANDROID_API_KEY}{DISABLE_PRETTY_PRINT_PARAMETER}" )) - .header("X-YouTube-Client-Name", "3") + .header( + header::USER_AGENT, + format!( + "com.google.android.youtube/{} (Linux; U; Android 12; {}) gzip", + MOBILE_CLIENT_VERSION, self.opts.country + ), + ) .header("X-Goog-Api-Format-Version", "2"), ClientType::Ios => self .client .inner .http .post(format!( - "{YOUTUBEI_V1_GAPIS_URL}{endpoint}?{DISABLE_PRETTY_PRINT_PARAMETER}" + "{YOUTUBEI_V1_GAPIS_URL}{endpoint}?key={IOS_API_KEY}{DISABLE_PRETTY_PRINT_PARAMETER}" )) - .header("X-YouTube-Client-Name", "5") + .header( + header::USER_AGENT, + format!( + "com.google.ios.youtube/{} ({}; U; CPU iOS 15_4 like Mac OS X; {})", + MOBILE_CLIENT_VERSION, IOS_DEVICE_MODEL, self.opts.country + ), + ) .header("X-Goog-Api-Format-Version", "2"), - }; - r = r - .header(header::CONTENT_TYPE, "application/json") - .header(header::USER_AGENT, self.user_agent(ctype).as_ref()); - if let Some(vdata) = self.opts.visitor_data.as_deref().or(visitor_data) { - r = r.header("X-Goog-EOM-Visitor-Id", vdata); } - - let mut cookie = None; - - if self.opts.auth == Some(true) { - if ctype.is_web() { - let auth_cookie = self.client.user_auth_cookie()?; - - if let Some(auth_header) = Self::sapisidhash_header(&auth_cookie.cookie, ctype) { - r = r.header(header::AUTHORIZATION, auth_header); - } - if let Some(session_index) = auth_cookie.session_index { - r = r.header("X-Goog-AuthUser", session_index); - } - if let Some(account_syncid) = auth_cookie.channel_syncid { - r = r.header("X-Goog-PageId", account_syncid); - } - cookie = Some(auth_cookie.cookie); - } else if ctype == ClientType::Tv { - let access_token = self.client.user_auth_access_token().await?; - r = r.header(header::AUTHORIZATION, format!("Bearer {}", access_token)); - } - } - - if ctype.is_web() { - r = r.header(header::COOKIE, cookie.as_deref().unwrap_or(CONSENT_COOKIE)); - } - - Ok(r) } - fn sapisidhash_header(cookie: &str, ctype: ClientType) -> Option { - let sapisid = cookie - .split(';') - .find_map(|c| c.trim().strip_prefix("SAPISID="))?; - let time_now = OffsetDateTime::now_utc().unix_timestamp(); - let mut sapisidhash = Sha1::new(); - sapisidhash.update(time_now.to_string()); - sapisidhash.update(" "); - sapisidhash.update(sapisid); - sapisidhash.update(" "); - sapisidhash.update(match ctype { - ClientType::DesktopMusic => YOUTUBE_MUSIC_HOME_URL, - ClientType::Mobile => YOUTUBE_MOBILE_HOME_URL, - _ => YOUTUBE_HOME_URL, - }); - - let sapisidhash_hex = data_encoding::HEXLOWER.encode(&sapisidhash.finalize()); - Some(format!("SAPISIDHASH {time_now}_{sapisidhash_hex}")) - } - - /// Get a YouTube visitor data ID, which is necessary for certain requests - pub async fn get_visitor_data(&self, force_new: bool) -> Result { - if force_new { - return self - .client - .inner - .visitor_data_cache - .new_visitor_data() - .await; - } - + /// Get a YouTube visitor data cookie, which is necessary for certain requests + async fn get_visitor_data(&self) -> Result { match &self.opts.visitor_data { Some(vd) => Ok(vd.clone()), - None => self.client.inner.visitor_data_cache.get().await, + None => self.client.get_visitor_data().await, } } - /// Remove a YouTube visitor data ID from the cache so it is not used again - pub fn remove_visitor_data(&self, visitor_data: &str) { - self.client.inner.visitor_data_cache.remove(visitor_data); - } - - /// Generate PO tokens - async fn get_po_tokens(&self, idents: &[&str]) -> Result<(Vec, OffsetDateTime), Error> { - let bg = self - .client - .inner - .botguard - .as_ref() - .ok_or(ExtractionError::Botguard("not enabled".into()))?; - - let start = std::time::Instant::now(); - let cmd = tokio::process::Command::new(&bg.program) - .arg("--snapshot-file") - .arg(&bg.snapshot_file) - .arg("--") - .args(idents) - .output() - .await - .map_err(|e| Error::Extraction(ExtractionError::Botguard(e.to_string().into())))?; - if !cmd.status.success() { - return Err(Error::Extraction(ExtractionError::Botguard( - String::from_utf8_lossy(&cmd.stderr).into_owned().into(), - ))); - } - - let output = String::from_utf8(cmd.stdout) - .map_err(|e| Error::Extraction(ExtractionError::Botguard(e.to_string().into())))?; - - let mut words = output.split_whitespace(); - let mut tokens = Vec::with_capacity(idents.len()); - for _ in 0..idents.len() { - tokens.push( - words - .next() - .ok_or(ExtractionError::Botguard("too few tokens returned".into()))? - .to_owned(), - ); - } - - let mut valid_until = None; - let mut from_snapshot = false; - for word in words { - if let Some((k, v)) = word.split_once('=') { - match k { - "valid_until" => { - valid_until = Some( - v.parse::() - .ok() - .and_then(|x| OffsetDateTime::from_unix_timestamp(x).ok()) - .ok_or(ExtractionError::Botguard( - format!("invalid validity date: {v}").into(), - ))?, - ); - } - "from_snapshot" => { - from_snapshot = v.eq_ignore_ascii_case("true") || v == "1"; - } - _ => {} - } - } - } - - let valid_until = - valid_until.unwrap_or_else(|| OffsetDateTime::now_utc() + time::Duration::hours(12)); - - tracing::debug!( - "generated PO token (valid_until {}, from_snapshot={}, took {}ms)", - valid_until, - from_snapshot, - start.elapsed().as_millis() - ); - Ok((tokens, valid_until)) - } - - /// Get a session-bound PO token (either from cache or newly generated) - async fn get_session_po_token(&self, visitor_data: &str) -> Result { - if let Some(po_token) = self.client.inner.visitor_data_cache.get_pot(visitor_data) { - return Ok(po_token); - } - - let po_token = self.get_po_token(visitor_data).await?; - self.client - .inner - .visitor_data_cache - .store_pot(visitor_data, po_token.clone()); - Ok(po_token) - } - - /// Get a PO token (Proof-of-origin token) - /// - /// PO tokens are used by the web-based YouTube clients for requesting player data and video streams. - /// - /// See for more information - pub async fn get_po_token>(&self, ident: S) -> Result { - let (tokens, valid_until) = self.get_po_tokens(&[ident.as_ref()]).await?; - - Ok(PoToken { - po_token: tokens.into_iter().next().unwrap(), - valid_until, - }) - } - - /// Get a new RustyPipeInfo object for reports - fn rp_info(&self) -> RustyPipeInfo<'_> { - RustyPipeInfo::new( - Some(self.opts.lang), - self.client - .inner - .botguard - .as_ref() - .map(|bg| bg.version.as_str()), - ) - } - - /// Execute a request to the YouTube API, then deobfuscate and map the response. - /// - /// Runs a single attempt, returns Ok with a erroneous RequestResult in case of a - /// HTTP or mapping error so it can be retried/reported. - async fn execute_request_attempt< - R: DeserializeOwned + MapResponse + Debug, - M, - B: Serialize + ?Sized, - >( + async fn yt_request_attempt + Debug, M>( &self, - ctype: ClientType, + request: &Request, id: &str, - endpoint: &str, - body: &B, - ctx_src: &MapRespOptions<'_>, + deobf: Option<&DeobfData>, ) -> Result, Error> { - let visitor_data = match ctx_src - .visitor_data - .or(self.opts.visitor_data.as_deref()) - .map(Cow::Borrowed) - { - Some(vd) => vd, - None => self.client.inner.visitor_data_cache.get().await?.into(), - }; - - let context = self - .get_context(ctype, !ctx_src.unlocalized, &visitor_data) - .await; - let req_body = QBody { context, body }; - - let ctx = MapRespCtx { - id, - lang: self.opts.lang, - utc_offset: UtcOffset::from_whole_seconds(i32::from(self.opts.utc_offset_minutes) * 60) - .map_err(|_| Error::Other("utc_offset overflow".into()))?, - deobf: ctx_src.deobf, - visitor_data: Some(&visitor_data), - client_type: ctype, - artist: ctx_src.artist.clone(), - authenticated: self.opts.auth.unwrap_or_default(), - session_po_token: ctx_src.session_po_token.clone(), - }; - - let request = self - .request_builder(ctype, endpoint, ctx.visitor_data) - .await? - .json(&req_body) - .build()?; - let response = self .client .inner @@ -2373,7 +1230,6 @@ impl RustyPipeQuery { let status = response.status(); let body = response.text().await?; - tracing::debug!("fetched {} bytes from YT", body.len()); let res = if status.is_client_error() || status.is_server_error() { let error_msg = serde_json::from_str::(&body) @@ -2381,18 +1237,17 @@ impl RustyPipeQuery { Err(match status { StatusCode::NOT_FOUND => Error::Extraction(ExtractionError::NotFound { - id: ctx.id.to_owned(), + id: id.to_owned(), msg: error_msg.unwrap_or("404".into()), }), StatusCode::BAD_REQUEST => { Error::Extraction(ExtractionError::BadRequest(error_msg.unwrap_or_default())) } - StatusCode::UNAUTHORIZED => Error::Auth(AuthError::NoLogin), _ => Error::HttpStatus(status.as_u16(), error_msg.unwrap_or_default()), }) } else { match serde_json::from_str::(&body) { - Ok(deserialized) => match deserialized.map_response(&ctx) { + Ok(deserialized) => match deserialized.map_response(id, self.opts.lang, deobf) { Ok(mapres) => Ok(mapres), Err(e) => Err(e.into()), }, @@ -2400,37 +1255,18 @@ impl RustyPipeQuery { } }; - tracing::trace!("mapped response"); - Ok(RequestResult { - res, - status, - body, - request, - visitor_data: visitor_data.into_owned(), - }) + Ok(RequestResult { res, status, body }) } - /// Execute a request to the YouTube API, then deobfuscate and map the response. - /// - /// Runs up to n_request_attempts, returns Ok with a erroneous RequestResult in case of a - /// HTTP or mapping error so it can be reported. - async fn execute_request_inner< - R: DeserializeOwned + MapResponse + Debug, - M, - B: Serialize + ?Sized, - >( + async fn yt_request + Debug, M>( &self, - ctype: ClientType, + request: &Request, id: &str, - endpoint: &str, - body: &B, - ctx_src: &MapRespOptions<'_>, + deobf: Option<&DeobfData>, ) -> Result, Error> { let mut last_resp = None; for n in 0..=self.client.inner.n_http_retries { - let resp = self - .execute_request_attempt::(ctype, id, endpoint, body, ctx_src) - .await?; + let resp = self.yt_request_attempt::(request, id, deobf).await?; let err = match &resp.res { Ok(_) => return Ok(resp), @@ -2442,12 +1278,9 @@ impl RustyPipeQuery { } }; - // Remove the used visitor data from cache if the request resulted in a recoverable error - self.remove_visitor_data(&resp.visitor_data); - if n != self.client.inner.n_http_retries { let ms = util::retry_delay(n, 1000, 60000, 3); - tracing::warn!( + log::warn!( "Retry attempt #{}. Error: {}. Waiting {} ms", n + 1, err, @@ -2474,8 +1307,8 @@ impl RustyPipeQuery { /// - `method`: HTTP method /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/?key=...`) /// - `body`: Serializable request body to be sent in json format - /// - `ctx_src`: Context source (additional parameters for fetching and mapping, used to build the MapRespCtx) - async fn execute_request_ctx< + /// - `deobf`: Deobfuscator (is passed to the mapper to deobfuscate stream URLs). + async fn execute_request_deobf< R: DeserializeOwned + MapResponse + Debug, M, B: Serialize + ?Sized, @@ -2486,14 +1319,17 @@ impl RustyPipeQuery { id: &str, endpoint: &str, body: &B, - ctx_src: MapRespOptions<'_>, + deobf: Option<&DeobfData>, ) -> Result { - tracing::debug!("getting {}({})", operation, id); + log::debug!("getting {}({})", operation, id); - let req_res = self - .execute_request_inner::(ctype, id, endpoint, body, &ctx_src) - .await?; - let request = req_res.request; + let request = self + .request_builder(ctype, endpoint) + .await + .json(body) + .build()?; + + let req_res = self.yt_request::(&request, id, deobf).await?; // Uncomment to debug response text // println!("{}", &req_res.body); @@ -2520,35 +1356,21 @@ impl RustyPipeQuery { if level > Level::DBG || self.opts.report { if let Some(reporter) = &self.client.inner.reporter { let report = Report { - info: self.rp_info(), + info: RustyPipeInfo::default(), level, operation: &format!("{operation}({id})"), error, msgs, - deobf_data: ctx_src.deobf.cloned(), + deobf_data: deobf.cloned(), http_request: crate::report::HTTPRequest { url: request.url().as_str(), method: request.method().as_str(), - req_header: Some( - request - .headers() - .iter() - .filter(|(k, _)| k != &header::COOKIE) - .map(|(k, v)| { - let vstr = if k == header::AUTHORIZATION { - "[redacted]" - } else { - v.to_str().unwrap_or_default() - }; - (k.as_str(), vstr.to_owned()) - }) - .collect(), - ), - req_body: request - .body() - .as_ref() - .and_then(|b| b.as_bytes()) - .map(|b| String::from_utf8_lossy(b).into_owned()), + req_header: request + .headers() + .iter() + .map(|(k, v)| (k.as_str(), v.to_str().unwrap_or_default().to_owned())) + .collect(), + req_body: serde_json::to_string(body).unwrap_or_default(), status: req_res.status.into(), resp_body: req_res.body, }, @@ -2589,41 +1411,21 @@ impl RustyPipeQuery { endpoint: &str, body: &B, ) -> Result { - self.execute_request_ctx::( - ctype, - operation, - id, - endpoint, - body, - MapRespOptions::default(), - ) - .await + self.execute_request_deobf::(ctype, operation, id, endpoint, body, None) + .await } /// Execute a request to the YouTube API and return the response string - /// - /// # Parameters - /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...) - /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/?key=...`) - /// - `body`: Serializable request body to be sent in json format pub async fn raw( &self, ctype: ClientType, endpoint: &str, body: &B, ) -> Result { - let visitor_data = match self.opts.visitor_data.as_deref().map(Cow::Borrowed) { - Some(vd) => vd, - None => self.client.inner.visitor_data_cache.get().await?.into(), - }; - - let context = self.get_context(ctype, true, &visitor_data).await; - let req_body = QBody { context, body }; - let request = self - .request_builder(ctype, endpoint, None) - .await? - .json(&req_body) + .request_builder(ctype, endpoint) + .await + .json(body) .build()?; self.client.http_request_txt(&request).await @@ -2636,49 +1438,6 @@ impl AsRef for RustyPipeQuery { } } -/// Additional data needed for mapping YouTube responses -struct MapRespCtx<'a> { - id: &'a str, - lang: Language, - utc_offset: UtcOffset, - deobf: Option<&'a DeobfData>, - visitor_data: Option<&'a str>, - client_type: ClientType, - artist: Option, - authenticated: bool, - session_po_token: Option, -} - -/// Options to give to the mapper when making requests; -/// used to construct the [`MapRespCtx`] -#[derive(Default)] -struct MapRespOptions<'a> { - visitor_data: Option<&'a str>, - deobf: Option<&'a DeobfData>, - artist: Option, - unlocalized: bool, - session_po_token: Option, -} - -#[allow(clippy::needless_lifetimes)] -impl<'a> MapRespCtx<'a> { - /// Create a [`MapRespCtx`] for testing - #[cfg(test)] - fn test(id: &'a str) -> Self { - Self { - id, - lang: Language::En, - utc_offset: UtcOffset::UTC, - deobf: None, - visitor_data: None, - client_type: ClientType::Desktop, - artist: None, - authenticated: false, - session_po_token: None, - } - } -} - /// Implement this for YouTube API response structs that need to be mapped to /// RustyPipe models. trait MapResponse { @@ -2694,123 +1453,52 @@ trait MapResponse { /// that the returned entity matches this ID and return an error instead. /// - `lang`: Language of the request. Used for mapping localized information like dates. /// - `deobf`: Deobfuscator (if passed to the `execute_request_deobf` method) - /// - `visitor_data`: Visitor data option of the client - fn map_response(self, ctx: &MapRespCtx<'_>) -> Result, ExtractionError>; + fn map_response( + self, + id: &str, + lang: Language, + deobf: Option<&DeobfData>, + ) -> Result, ExtractionError>; } fn validate_country(country: Country) -> Country { if country == Country::Zz { - tracing::warn!("Country:Zz (Global) can only be used for fetching music charts, falling back to Country:Us"); + log::warn!("Country:Zz (Global) can only be used for fetching music charts, falling back to Country:Us"); Country::Us } else { country } } -fn local_tz_offset() -> (String, i16) { - match ( - localzone::get_local_zone().ok_or(Error::Other("could not get local timezone".into())), - UtcOffset::current_local_offset().map_err(|_| Error::Other("indeterminate offset".into())), - ) { - (Ok(timezone), Ok(offset)) => (timezone, offset.whole_minutes()), - (Err(e), _) | (_, Err(e)) => { - tracing::error!("{e}"); - ("UTC".to_owned(), 0) - } - } -} - -/// Check if a valid Botguard binary is available at the given location -fn detect_botguard_bin(program: OsString) -> Result<(OsString, String), Error> { - let out = std::process::Command::new(&program) - .arg("--version") - .output() - .map_err(|e| { - if e.kind() == std::io::ErrorKind::NotFound { - Error::Other("rustypipe-botguard binary not found".into()) - } else { - Error::Other(format!("error calling rustypipe-botguard {e}").into()) - } - })?; - if !out.status.success() { - return Err(Error::Extraction(ExtractionError::Botguard( - format!("version check failed with status {}", out.status).into(), - ))); - } - let output = String::from_utf8_lossy(&out.stdout); - let pat = "rustypipe-botguard-api "; - let pos = output.find(pat).ok_or(Error::Other( - "no rustypipe-botguard-api version returned".into(), - ))? + pat.len(); - let pos_end = output[pos..] - .char_indices() - .find(|(_, c)| !c.is_ascii_digit()) - .map(|(p, _)| p + pos) - .unwrap_or(output.len()); - let api_version = &output[pos..pos_end]; - if api_version != BOTGUARD_API_VERSION { - return Err(Error::Other( - format!( - "incompatible rustypipe-botguard-api version {api_version}, expected {BOTGUARD_API_VERSION}" - ) - .into(), - )); - } - let version = output[..pos].lines().next().unwrap_or_default().to_owned(); - Ok((program, version)) -} - #[cfg(test)] mod tests { use super::*; - use rstest::rstest; - - // 1.20240506.01.00-canary_control_1.20240508.01.01 - // 1.20240508.01.01-canary_experiment_1.20240506.01.00 fn get_major_version(version: &str) -> u32 { let parts = version.split('.').collect::>(); - assert!(parts.len() >= 4, "version: {version}"); + assert_eq!(parts.len(), 4); parts[0].parse().unwrap() } - #[rstest] - #[case(ClientType::Desktop, 2)] - #[case(ClientType::DesktopMusic, 1)] - #[case(ClientType::Mobile, 2)] - #[case(ClientType::Tv, 1)] - #[tokio::test] - async fn extract_desktop_client_version(#[case] client_type: ClientType, #[case] major: u32) { + #[test] + fn t_extract_desktop_client_version() { let rp = RustyPipe::new(); - let version = rp.extract_client_version(client_type).await.unwrap(); - assert!(get_major_version(&version) >= major); + let version = tokio_test::block_on(rp.extract_desktop_client_version()).unwrap(); + assert!(get_major_version(&version) >= 2); } - #[tokio::test] - async fn get_visitor_data() { + #[test] + fn t_extract_music_client_version() { let rp = RustyPipe::new(); - let visitor_data = rp.query().get_visitor_data(true).await.unwrap(); - - assert!( - visitor_data.starts_with("Cg") && visitor_data.len() > 23, - "invalid visitor data: {visitor_data}" - ); + let version = tokio_test::block_on(rp.extract_music_client_version()).unwrap(); + assert!(get_major_version(&version) >= 1); } - #[tokio::test] - async fn get_po_token() { - let rp = RustyPipe::builder().build().unwrap(); - let ident = "Cgt4eDYyVVJveGQtbyiLyvu8BjIKCgJERRIEEgAgKw=="; - let po_token = rp.query().get_po_token(ident).await.unwrap(); - - let token_bts = data_encoding::BASE64URL - .decode(po_token.po_token.as_bytes()) - .unwrap(); - assert_eq!(token_bts.len(), ident.len() + 74); - assert!( - po_token.valid_until > OffsetDateTime::now_utc() + time::Duration::minutes(30), - "valid until {}", - po_token.valid_until - ) + #[test] + fn t_get_visitor_data() { + let rp = RustyPipe::new(); + let visitor_data = tokio_test::block_on(rp.get_visitor_data()).unwrap(); + assert!(visitor_data.ends_with("%3D")); + assert_eq!(visitor_data.len(), 32) } } diff --git a/src/client/music_artist.rs b/src/client/music_artist.rs index 13cbeda..58b926d 100644 --- a/src/client/music_artist.rs +++ b/src/client/music_artist.rs @@ -2,26 +2,17 @@ use std::borrow::Cow; use once_cell::sync::Lazy; use regex::Regex; -use tracing::debug; use crate::{ - client::{ - response::{music_item::map_album_type, url_endpoint::NavigationEndpoint}, - MapRespOptions, QContinuation, - }, error::{Error, ExtractionError}, - model::{ - paginator::Paginator, traits::FromYtItem, AlbumItem, AlbumType, ArtistId, MusicArtist, - MusicItem, - }, - param::{AlbumFilter, AlbumOrder}, + model::{AlbumItem, ArtistId, MusicArtist}, serializer::MapResult, - util::{self, ProtoBuilder}, + util, }; use super::{ response::{self, music_item::MusicListMapper, url_endpoint::PageType}, - ClientType, MapRespCtx, MapResponse, QBrowse, QBrowseParams, RustyPipeQuery, + ClientType, MapResponse, QBrowse, RustyPipeQuery, }; impl RustyPipeQuery { @@ -37,7 +28,7 @@ impl RustyPipeQuery { let res = self._music_artist(artist_id, all_albums).await; if let Err(Error::Extraction(ExtractionError::Redirect(id))) = res { - debug!("music artist {} redirects to {}", artist_id, &id); + log::debug!("music artist {} redirects to {}", artist_id, &id); self._music_artist(&id, all_albums).await } else { res @@ -45,7 +36,9 @@ impl RustyPipeQuery { } async fn _music_artist(&self, artist_id: &str, all_albums: bool) -> Result { + let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { + context, browse_id: artist_id, }; @@ -61,9 +54,7 @@ impl RustyPipeQuery { .await?; if can_fetch_more { - artist.albums = self - .music_artist_albums(artist_id, None, Some(AlbumOrder::Recency)) - .await?; + artist.albums = self.music_artist_albums(artist_id).await?; } Ok(artist) @@ -80,59 +71,32 @@ impl RustyPipeQuery { } /// Get a list of all albums of a YouTube Music artist - pub async fn music_artist_albums( - &self, - artist_id: &str, - filter: Option, - order: Option, - ) -> Result, Error> { - let request_body = QBrowseParams { + pub async fn music_artist_albums(&self, artist_id: &str) -> Result, Error> { + let context = self.get_context(ClientType::DesktopMusic, true, None).await; + let request_body = QBrowse { + context, browse_id: &format!("{}{}", util::ARTIST_DISCOGRAPHY_PREFIX, artist_id), - params: &albums_param(filter, order), }; - let first_page = self - .execute_request::( - ClientType::DesktopMusic, - "music_artist_albums", - artist_id, - "browse", - &request_body, - ) - .await?; - - let mut albums = first_page.albums; - let mut ctoken = first_page.ctoken; - - while let Some(tkn) = &ctoken { - let request_body = QContinuation { continuation: tkn }; - let resp: Paginator = self - .execute_request_ctx::, _>( - ClientType::DesktopMusic, - "music_artist_albums_cont", - artist_id, - "browse", - &request_body, - MapRespOptions { - artist: Some(first_page.artist.clone()), - visitor_data: first_page.visitor_data.as_deref(), - ..Default::default() - }, - ) - .await?; - if resp.items.is_empty() { - tracing::warn!("artist albums [{artist_id}] empty continuation"); - } - ctoken = resp.ctoken; - albums.extend(resp.items.into_iter().filter_map(AlbumItem::from_ytm_item)); - } - Ok(albums) + self.execute_request::( + ClientType::DesktopMusic, + "music_artist_albums", + artist_id, + "browse", + &request_body, + ) + .await } } impl MapResponse for response::MusicArtist { - fn map_response(self, ctx: &MapRespCtx<'_>) -> Result, ExtractionError> { - let mapped = map_artist_page(self, ctx, false)?; + fn map_response( + self, + id: &str, + lang: crate::param::Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, + ) -> Result, ExtractionError> { + let mapped = map_artist_page(self, id, lang, false)?; Ok(MapResult { c: mapped.c.0, warnings: mapped.warnings, @@ -143,35 +107,23 @@ impl MapResponse for response::MusicArtist { impl MapResponse<(MusicArtist, bool)> for response::MusicArtist { fn map_response( self, - ctx: &MapRespCtx<'_>, + id: &str, + lang: crate::param::Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, ) -> Result, ExtractionError> { - map_artist_page(self, ctx, true) + map_artist_page(self, id, lang, true) } } fn map_artist_page( res: response::MusicArtist, - ctx: &MapRespCtx<'_>, + id: &str, + lang: crate::param::Language, skip_extendables: bool, ) -> Result, ExtractionError> { - let contents = match res.contents { - Some(c) => c, - None => { - if res.microformat.microformat_data_renderer.noindex { - return Err(ExtractionError::NotFound { - id: ctx.id.to_owned(), - msg: "no contents".into(), - }); - } else { - return Err(ExtractionError::InvalidData("no contents".into())); - } - } - }; + // dbg!(&res); - let header = res - .header - .ok_or(ExtractionError::InvalidData("no header".into()))? - .music_immersive_header_renderer; + let header = res.header.music_immersive_header_renderer; if let Some(share) = header.share_endpoint { let pb = share.share_entity_endpoint.serialized_share_entity; @@ -182,24 +134,26 @@ fn map_artist_page( .and_then(|pb| util::string_from_pb(pb, 3)); if let Some(share_channel_id) = share_channel_id { - if share_channel_id != ctx.id { + if share_channel_id != id { return Err(ExtractionError::Redirect(share_channel_id)); } } } - let sections = contents + let sections = res + .contents .single_column_browse_results_renderer .contents .into_iter() .next() - .map(|c| c.tab_renderer.content.section_list_renderer.contents) + .and_then(|tab| tab.tab_renderer.content) + .map(|c| c.section_list_renderer.contents) .unwrap_or_default(); let mut mapper = MusicListMapper::with_artist( - ctx.lang, + lang, ArtistId { - id: Some(ctx.id.to_owned()), + id: Some(id.to_owned()), name: header.title.clone(), }, ); @@ -224,56 +178,50 @@ fn map_artist_page( } } } - mapper.album_type = AlbumType::Single; + mapper.map_response(shelf.contents); } response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => { let mut extendable_albums = false; - mapper.album_type = AlbumType::Single; if let Some(h) = shelf.header { if let Some(button) = h .music_carousel_shelf_basic_header_renderer .more_content_button { - if let NavigationEndpoint::Browse { - browse_endpoint, .. - } = button.button_renderer.navigation_endpoint + if let Some(bep) = + button.button_renderer.navigation_endpoint.browse_endpoint { - // Music videos - if browse_endpoint - .browse_endpoint_context_supported_configs - .map(|cfg| { - cfg.browse_endpoint_context_music_config.page_type - == PageType::Playlist - }) - .unwrap_or_default() - { - if videos_playlist_id.is_none() { - videos_playlist_id = Some(browse_endpoint.browse_id); - } - } else if browse_endpoint - .browse_id - .starts_with(util::ARTIST_DISCOGRAPHY_PREFIX) - { - can_fetch_more = true; - extendable_albums = true; - } else { - // Peek at the first item to determine type - if let Some(response::music_item::MusicResponseItem::MusicTwoRowItemRenderer(item)) = shelf.contents.c.first() { - if let Some(PageType::Album) = item.navigation_endpoint.page_type() { + if let Some(cfg) = bep.browse_endpoint_context_supported_configs { + match cfg.browse_endpoint_context_music_config.page_type { + // Music videos + PageType::Playlist => { + if videos_playlist_id.is_none() { + videos_playlist_id = Some(bep.browse_id); + } + } + // Albums + PageType::ArtistDiscography => { can_fetch_more = true; extendable_albums = true; } + // Albums or playlists + PageType::Artist => { + // Peek at the first item to determine type + if let Some(response::music_item::MusicResponseItem::MusicTwoRowItemRenderer(item)) = shelf.contents.c.first() { + if let Some(PageType::Album) = item.navigation_endpoint.browse_endpoint.as_ref().and_then(|be| { + be.browse_endpoint_context_supported_configs.as_ref().map(|config| { + config.browse_endpoint_context_music_config.page_type + })}) { + can_fetch_more = true; + extendable_albums = true; + } + } + } + _ => {} } } } } - mapper.album_type = map_album_type( - h.music_carousel_shelf_basic_header_renderer - .title - .first_str(), - ctx.lang, - ); } if !skip_extendables || !extendable_albums { @@ -284,6 +232,7 @@ fn map_artist_page( } } + mapper.check_unknown()?; let mut mapped = mapper.group_items(); static WIKIPEDIA_REGEX: Lazy = @@ -302,18 +251,16 @@ fn map_artist_page( }); let radio_id = header.start_radio_button.and_then(|b| { - if let NavigationEndpoint::Watch { watch_endpoint } = b.button_renderer.navigation_endpoint - { - watch_endpoint.playlist_id - } else { - None - } + b.button_renderer + .navigation_endpoint + .watch_endpoint + .and_then(|w| w.playlist_id) }); Ok(MapResult { c: ( MusicArtist { - id: ctx.id.to_owned(), + id: id.to_owned(), name: header.title, header_image: header.thumbnail.into(), description: header.description, @@ -321,7 +268,7 @@ fn map_artist_page( subscriber_count: header.subscription_button.and_then(|btn| { util::parse_large_numstr_or_warn( &btn.subscribe_button_renderer.subscriber_count_text, - ctx.lang, + lang, &mut mapped.warnings, ) }), @@ -339,24 +286,17 @@ fn map_artist_page( }) } -#[derive(Debug)] -struct FirstAlbumPage { - albums: Vec, - ctoken: Option, - artist: ArtistId, - visitor_data: Option, -} - -impl MapResponse for response::MusicArtistAlbums { +impl MapResponse> for response::MusicArtistAlbums { fn map_response( self, - ctx: &MapRespCtx<'_>, - ) -> Result, ExtractionError> { + id: &str, + lang: crate::param::Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, + ) -> Result>, ExtractionError> { + // dbg!(&self); + let Some(header) = self.header else { - return Err(ExtractionError::NotFound { - id: ctx.id.into(), - msg: "no header".into(), - }); + return Err(ExtractionError::NotFound { id: id.into(), msg: "no header".into() }); }; let grids = self @@ -371,56 +311,28 @@ impl MapResponse for response::MusicArtistAlbums { .section_list_renderer .contents; - let artist_id = ArtistId { - id: Some(ctx.id.to_owned()), - name: header.music_header_renderer.title, - }; - let mut mapper = MusicListMapper::with_artist(ctx.lang, artist_id.clone()); - let mut ctoken = None; + let mut mapper = MusicListMapper::with_artist( + lang, + ArtistId { + id: Some(id.to_owned()), + name: header.music_header_renderer.title, + }, + ); + for grid in grids { mapper.map_response(grid.grid_renderer.items); - if ctoken.is_none() { - ctoken = grid - .grid_renderer - .continuations - .into_iter() - .next() - .map(|g| g.next_continuation_data.continuation); - } } + mapper.check_unknown()?; let mapped = mapper.group_items(); Ok(MapResult { - c: FirstAlbumPage { - albums: mapped.c.albums, - ctoken, - artist: artist_id, - visitor_data: ctx.visitor_data.map(str::to_owned), - }, + c: mapped.c.albums, warnings: mapped.warnings, }) } } -fn albums_param(filter: Option, order: Option) -> String { - let mut pb_filter = ProtoBuilder::new(); - if let Some(filter) = filter { - pb_filter.varint(1, filter as u64); - } - if let Some(order) = order { - pb_filter.varint(2, order as u64); - } - pb_filter.bytes(3, &[1, 2]); - - let mut pb_48 = ProtoBuilder::new(); - pb_48.embedded(15, pb_filter); - - let mut pb_3 = ProtoBuilder::new(); - pb_3.embedded(48, pb_48); - pb_3.to_base64() -} - #[cfg(test)] mod tests { use std::{fs::File, io::BufReader}; @@ -428,7 +340,7 @@ mod tests { use path_macro::path; use rstest::rstest; - use crate::util::tests::TESTFILES; + use crate::{param::Language, util::tests::TESTFILES}; use super::*; @@ -437,7 +349,6 @@ mod tests { #[case::only_singles("only_singles", "UCfwCE5VhPMGxNPFxtVv7lRw")] #[case::no_artist("no_artist", "UCh8gHdtzO2tXd593_bjErWg")] #[case::only_more_singles("only_more_singles", "UC0aXrjVxG5pZr99v77wZdPQ")] - #[case::grouped_albums("20250113_grouped_albums", "UCOR4_bSVIXPsGa4BbCSt60Q")] fn map_music_artist(#[case] name: &str, #[case] id: &str) { let json_path = path!(*TESTFILES / "music_artist" / format!("artist_{name}.json")); let json_file = File::open(json_path).unwrap(); @@ -451,7 +362,7 @@ mod tests { let resp: response::MusicArtist = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult<(MusicArtist, bool)> = - resp.map_response(&MapRespCtx::test(id)).unwrap(); + resp.map_response(id, Language::En, None).unwrap(); let (mut artist, can_fetch_more) = map_res.c; assert!( @@ -461,42 +372,19 @@ mod tests { ); assert_eq!(can_fetch_more, album_page_path.is_some()); - // Album overview if let Some(album_page_path) = album_page_path { let json_file = File::open(album_page_path).unwrap(); let resp: response::MusicArtistAlbums = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult = - resp.map_response(&MapRespCtx::test(id)).unwrap(); + let mut map_res: MapResult> = + resp.map_response(id, Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), "deserialization/mapping warnings: {:?}", map_res.warnings ); - artist.albums = map_res.c.albums; - - // Album overview continuation - for i in 2..10 { - let cont_path = - path!(*TESTFILES / "music_artist" / format!("artist_{name}_{i}.json")); - if !cont_path.is_file() { - break; - } - let json_file = File::open(cont_path).unwrap(); - let resp: response::MusicContinuation = - serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult> = - resp.map_response(&MapRespCtx::test(id)).unwrap(); - assert!(!map_res.c.items.is_empty()); - artist.albums.extend( - map_res - .c - .items - .into_iter() - .filter_map(AlbumItem::from_ytm_item), - ); - } + artist.albums.append(&mut map_res.c); } insta::assert_ron_snapshot!(format!("map_music_artist_{name}"), artist); @@ -510,7 +398,7 @@ mod tests { let artist: response::MusicArtist = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult = artist - .map_response(&MapRespCtx::test("UClmXPfaYhXOYsNn_QUyheWQ")) + .map_response("UClmXPfaYhXOYsNn_QUyheWQ", Language::En, None) .unwrap(); assert!( @@ -529,12 +417,12 @@ mod tests { let artist: response::MusicArtist = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let res: Result, ExtractionError> = - artist.map_response(&MapRespCtx::test("UCLkAepWjdylmXSltofFvsYQ")); + artist.map_response("UCLkAepWjdylmXSltofFvsYQ", Language::En, None); let e = res.unwrap_err(); match e { ExtractionError::Redirect(id) => { - assert_eq!(id, "UCOR4_bSVIXPsGa4BbCSt60Q"); + assert_eq!(id, "UCOR4_bSVIXPsGa4BbCSt60Q") } _ => panic!("error: {e}"), } diff --git a/src/client/music_charts.rs b/src/client/music_charts.rs index 6df5b17..1176dfc 100644 --- a/src/client/music_charts.rs +++ b/src/client/music_charts.rs @@ -11,12 +11,13 @@ use crate::{ use super::{ response::{self, music_item::MusicListMapper, url_endpoint::MusicPageType}, - ClientType, MapRespCtx, MapResponse, RustyPipeQuery, + ClientType, MapResponse, RustyPipeQuery, YTContext, }; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QCharts<'a> { + context: YTContext<'a>, browse_id: &'a str, params: &'a str, #[serde(skip_serializing_if = "Option::is_none")] @@ -31,9 +32,10 @@ struct FormData { impl RustyPipeQuery { /// Get the YouTube Music charts for a given country - #[tracing::instrument(skip(self), level = "error")] pub async fn music_charts(&self, country: Option) -> Result { + let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QCharts { + context, browse_id: "FEmusic_charts", params: "sgYPRkVtdXNpY19leHBsb3Jl", form_data: country.map(|c| FormData { @@ -53,7 +55,12 @@ impl RustyPipeQuery { } impl MapResponse for response::MusicCharts { - fn map_response(self, ctx: &MapRespCtx<'_>) -> Result, ExtractionError> { + fn map_response( + self, + _id: &str, + lang: crate::param::Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, + ) -> Result, crate::error::ExtractionError> { let countries = self .framework_updates .map(|fwu| { @@ -68,9 +75,9 @@ impl MapResponse for response::MusicCharts { let mut top_playlist_id = None; let mut trending_playlist_id = None; - let mut mapper_top = MusicListMapper::new(ctx.lang); - let mut mapper_trending = MusicListMapper::new(ctx.lang); - let mut mapper_other = MusicListMapper::new(ctx.lang); + let mut mapper_top = MusicListMapper::new(lang); + let mut mapper_trending = MusicListMapper::new(lang); + let mut mapper_other = MusicListMapper::new(lang); self.contents .single_column_browse_results_renderer @@ -89,9 +96,8 @@ impl MapResponse for response::MusicCharts { h.music_carousel_shelf_basic_header_renderer .more_content_button .and_then(|btn| btn.button_renderer.navigation_endpoint.music_page()) - .map(|mp| (mp.typ, mp.id)) }) { - Some((MusicPageType::Playlist { .. }, id)) => { + Some((MusicPageType::Playlist, id)) => { // Top music videos (first shelf with associated playlist) if top_playlist_id.is_none() { mapper_top.map_response(shelf.contents); @@ -112,13 +118,17 @@ impl MapResponse for response::MusicCharts { response::music_charts::ItemSection::None => {} }); + mapper_top.check_unknown()?; + mapper_trending.check_unknown()?; + mapper_other.check_unknown()?; + let mapped_top = mapper_top.conv_items::(); - let mapped_trending = mapper_trending.conv_items::(); - let mapped_other = mapper_other.group_items(); + let mut mapped_trending = mapper_trending.conv_items::(); + let mut mapped_other = mapper_other.group_items(); let mut warnings = mapped_top.warnings; - warnings.extend(mapped_trending.warnings); - warnings.extend(mapped_other.warnings); + warnings.append(&mut mapped_trending.warnings); + warnings.append(&mut mapped_other.warnings); Ok(MapResult { c: MusicCharts { @@ -142,6 +152,7 @@ mod tests { use rstest::rstest; use super::*; + use crate::param::Language; #[rstest] #[case::default("global")] @@ -153,7 +164,7 @@ mod tests { let charts: response::MusicCharts = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult = charts.map_response(&MapRespCtx::test("")).unwrap(); + let map_res: MapResult = charts.map_response("", Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), diff --git a/src/client/music_details.rs b/src/client/music_details.rs index 1eb4e51..06578f9 100644 --- a/src/client/music_details.rs +++ b/src/client/music_details.rs @@ -1,13 +1,11 @@ -use std::{borrow::Cow, fmt::Debug}; +use std::borrow::Cow; use serde::Serialize; use crate::{ error::{Error, ExtractionError}, - model::{ - paginator::{ContinuationEndpoint, Paginator}, - ArtistId, Lyrics, MusicRelated, TrackDetails, TrackItem, - }, + model::{paginator::Paginator, ArtistId, Lyrics, MusicRelated, TrackDetails, TrackItem}, + param::Language, serializer::MapResult, }; @@ -16,11 +14,12 @@ use super::{ self, music_item::{map_queue_item, MusicListMapper}, }, - ClientType, MapRespCtx, MapResponse, QBrowse, RustyPipeQuery, + ClientType, MapResponse, QBrowse, RustyPipeQuery, YTContext, }; #[derive(Debug, Serialize)] struct QMusicDetails<'a> { + context: YTContext<'a>, video_id: &'a str, enable_persistent_playlist_panel: bool, is_audio_only: bool, @@ -29,6 +28,7 @@ struct QMusicDetails<'a> { #[derive(Debug, Serialize)] struct QRadio<'a> { + context: YTContext<'a>, playlist_id: &'a str, params: &'a str, enable_persistent_playlist_panel: bool, @@ -37,14 +37,12 @@ struct QRadio<'a> { } impl RustyPipeQuery { - /// Get the metadata of a YouTube Music track - #[tracing::instrument(skip(self), level = "error")] - pub async fn music_details + Debug>( - &self, - video_id: S, - ) -> Result { + /// Get the metadata of a YouTube music track + pub async fn music_details>(&self, video_id: S) -> Result { let video_id = video_id.as_ref(); + let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QMusicDetails { + context, video_id, enable_persistent_playlist_panel: true, is_audio_only: true, @@ -61,13 +59,14 @@ impl RustyPipeQuery { .await } - /// Get the lyrics of a YouTube Music track + /// Get the lyrics of a YouTube music track /// /// The `lyrics_id` has to be obtained using [`RustyPipeQuery::music_details`]. - #[tracing::instrument(skip(self), level = "error")] - pub async fn music_lyrics + Debug>(&self, lyrics_id: S) -> Result { + pub async fn music_lyrics>(&self, lyrics_id: S) -> Result { let lyrics_id = lyrics_id.as_ref(); + let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { + context, browse_id: lyrics_id, }; @@ -84,13 +83,11 @@ impl RustyPipeQuery { /// Get related items (tracks, playlists, artists) to a YouTube Music track /// /// The `related_id` has to be obtained using [`RustyPipeQuery::music_details`]. - #[tracing::instrument(skip(self), level = "error")] - pub async fn music_related + Debug>( - &self, - related_id: S, - ) -> Result { + pub async fn music_related>(&self, related_id: S) -> Result { let related_id = related_id.as_ref(); + let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { + context, browse_id: related_id, }; @@ -107,13 +104,17 @@ impl RustyPipeQuery { /// Get a YouTube Music radio (a dynamically generated playlist) /// /// The `radio_id` can be obtained using [`RustyPipeQuery::music_artist`] to get an artist's radio. - #[tracing::instrument(skip(self), level = "error")] - pub async fn music_radio + Debug>( + pub async fn music_radio>( &self, radio_id: S, ) -> Result, Error> { let radio_id = radio_id.as_ref(); + let visitor_data = self.get_visitor_data().await?; + let context = self + .get_context(ClientType::DesktopMusic, true, Some(&visitor_data)) + .await; let request_body = QRadio { + context, playlist_id: radio_id, params: "wAEB8gECeAE%3D", enable_persistent_playlist_panel: true, @@ -132,8 +133,7 @@ impl RustyPipeQuery { } /// Get a YouTube Music radio (a dynamically generated playlist) for a track - #[tracing::instrument(skip(self), level = "error")] - pub async fn music_radio_track + Debug>( + pub async fn music_radio_track>( &self, video_id: S, ) -> Result, Error> { @@ -142,8 +142,7 @@ impl RustyPipeQuery { } /// Get a YouTube Music radio (a dynamically generated playlist) for a playlist - #[tracing::instrument(skip(self), level = "error")] - pub async fn music_radio_playlist + Debug>( + pub async fn music_radio_playlist>( &self, playlist_id: S, ) -> Result, Error> { @@ -155,7 +154,9 @@ impl RustyPipeQuery { impl MapResponse for response::MusicDetails { fn map_response( self, - ctx: &MapRespCtx<'_>, + id: &str, + lang: Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, ) -> Result, ExtractionError> { let tabs = self .contents @@ -193,7 +194,7 @@ impl MapResponse for response::MusicDetails { } let content = content.ok_or_else(|| ExtractionError::NotFound { - id: ctx.id.to_owned(), + id: id.to_owned(), msg: "no content".into(), })?; let track_item = content @@ -207,7 +208,14 @@ impl MapResponse for response::MusicDetails { response::music_item::PlaylistPanelVideo::None => None, }) .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no video item")))?; - let mut track = map_queue_item(track_item, ctx.lang); + let mut track = map_queue_item(track_item, lang); + + if track.c.id != id { + return Err(ExtractionError::WrongResult(format!( + "got wrong video id {}, expected {}", + track.c.id, id + ))); + } let mut warnings = content.contents.warnings; warnings.append(&mut track.warnings); @@ -226,7 +234,9 @@ impl MapResponse for response::MusicDetails { impl MapResponse> for response::MusicDetails { fn map_response( self, - ctx: &MapRespCtx<'_>, + id: &str, + lang: Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, ) -> Result>, ExtractionError> { let tabs = self .contents @@ -239,7 +249,7 @@ impl MapResponse> for response::MusicDetails { .into_iter() .find_map(|t| t.tab_renderer.content) .ok_or_else(|| ExtractionError::NotFound { - id: ctx.id.to_owned(), + id: id.to_owned(), msg: "no content".into(), })? .music_queue_renderer @@ -254,7 +264,7 @@ impl MapResponse> for response::MusicDetails { .into_iter() .filter_map(|item| match item { response::music_item::PlaylistPanelVideo::PlaylistPanelVideoRenderer(item) => { - let mut track = map_queue_item(item, ctx.lang); + let mut track = map_queue_item(item, lang); warnings.append(&mut track.warnings); Some(track.c) } @@ -274,8 +284,7 @@ impl MapResponse> for response::MusicDetails { tracks, ctoken, None, - ContinuationEndpoint::MusicNext, - false, + crate::model::paginator::ContinuationEndpoint::MusicNext, ), warnings, }) @@ -283,17 +292,27 @@ impl MapResponse> for response::MusicDetails { } impl MapResponse for response::MusicLyrics { - fn map_response(self, ctx: &MapRespCtx<'_>) -> Result, ExtractionError> { + fn map_response( + self, + id: &str, + _lang: Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, + ) -> Result, ExtractionError> { let lyrics = self .contents - .into_res() - .map_err(|msg| ExtractionError::NotFound { - id: ctx.id.to_owned(), - msg: msg.into(), - })? - .into_iter() - .find_map(|item| item.music_description_shelf_renderer) - .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))?; + .section_list_renderer + .and_then(|sl| { + sl.contents + .into_iter() + .find_map(|item| item.music_description_shelf_renderer) + }) + .ok_or(match self.contents.message_renderer { + Some(msg) => ExtractionError::NotFound { + id: id.to_owned(), + msg: msg.text.into(), + }, + None => ExtractionError::InvalidData(Cow::Borrowed("no content")), + })?; Ok(MapResult { c: Lyrics { @@ -308,44 +327,43 @@ impl MapResponse for response::MusicLyrics { impl MapResponse for response::MusicRelated { fn map_response( self, - ctx: &MapRespCtx<'_>, + _id: &str, + lang: Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, ) -> Result, ExtractionError> { - let contents = self - .contents - .into_res() - .map_err(|msg| ExtractionError::NotFound { - id: ctx.id.to_owned(), - msg: msg.into(), - })?; - // Find artist - let artist_id = contents.iter().find_map(|section| match section { - response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => { - shelf.header.as_ref().and_then(|h| { - h.music_carousel_shelf_basic_header_renderer - .title - .0 - .iter() - .find_map(|c| { - let artist = ArtistId::from(c.clone()); - if artist.id.is_some() { - Some(artist) - } else { - None - } - }) - }) - } - _ => None, - }); + let artist_id = self + .contents + .section_list_renderer + .contents + .iter() + .find_map(|section| match section { + response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => { + shelf.header.as_ref().and_then(|h| { + h.music_carousel_shelf_basic_header_renderer + .title + .0 + .iter() + .find_map(|c| { + let artist = ArtistId::from(c.clone()); + if artist.id.is_some() { + Some(artist) + } else { + None + } + }) + }) + } + _ => None, + }); - let mut mapper_tracks = MusicListMapper::new(ctx.lang); + let mut mapper_tracks = MusicListMapper::new(lang); let mut mapper = match artist_id { - Some(artist_id) => MusicListMapper::with_artist(ctx.lang, artist_id), - None => MusicListMapper::new(ctx.lang), + Some(artist_id) => MusicListMapper::with_artist(lang, artist_id), + None => MusicListMapper::new(lang), }; - let mut sections = contents.into_iter(); + let mut sections = self.contents.section_list_renderer.contents.into_iter(); if let Some(response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf)) = sections.next() { @@ -362,6 +380,9 @@ impl MapResponse for response::MusicRelated { _ => {} }); + mapper.check_unknown()?; + mapper_tracks.check_unknown()?; + let mapped_tracks = mapper_tracks.conv_items(); let mut mapped = mapper.group_items(); @@ -389,7 +410,7 @@ mod tests { use rstest::rstest; use super::*; - use crate::{model, util::tests::TESTFILES}; + use crate::{model, param::Language, util::tests::TESTFILES}; #[rstest] #[case::mv("mv", "ZeerrnuLi5E")] @@ -401,7 +422,7 @@ mod tests { let details: response::MusicDetails = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult = - details.map_response(&MapRespCtx::test(id)).unwrap(); + details.map_response(id, Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -421,7 +442,7 @@ mod tests { let radio: response::MusicDetails = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - radio.map_response(&MapRespCtx::test(id)).unwrap(); + radio.map_response(id, Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -438,7 +459,7 @@ mod tests { let lyrics: response::MusicLyrics = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult = lyrics.map_response(&MapRespCtx::test("")).unwrap(); + let map_res: MapResult = lyrics.map_response("", Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -455,7 +476,7 @@ mod tests { let lyrics: response::MusicRelated = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult = lyrics.map_response(&MapRespCtx::test("")).unwrap(); + let map_res: MapResult = lyrics.map_response("", Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), diff --git a/src/client/music_genres.rs b/src/client/music_genres.rs index c931fa4..f471f87 100644 --- a/src/client/music_genres.rs +++ b/src/client/music_genres.rs @@ -1,4 +1,4 @@ -use std::{borrow::Cow, fmt::Debug}; +use std::borrow::Cow; use crate::{ error::{Error, ExtractionError}, @@ -7,15 +7,16 @@ use crate::{ }; use super::{ - response::{self, music_item::MusicListMapper, url_endpoint::NavigationEndpoint}, - ClientType, MapRespCtx, MapResponse, QBrowse, QBrowseParams, RustyPipeQuery, + response::{self, music_item::MusicListMapper}, + ClientType, MapResponse, QBrowse, QBrowseParams, RustyPipeQuery, }; impl RustyPipeQuery { /// Get a list of moods and genres from YouTube Music - #[tracing::instrument(skip(self), level = "error")] pub async fn music_genres(&self) -> Result, Error> { + let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { + context, browse_id: "FEmusic_moods_and_genres", }; @@ -30,13 +31,11 @@ impl RustyPipeQuery { } /// Get the playlists from a YouTube Music genre - #[tracing::instrument(skip(self), level = "error")] - pub async fn music_genre + Debug>( - &self, - genre_id: S, - ) -> Result { + pub async fn music_genre>(&self, genre_id: S) -> Result { let genre_id = genre_id.as_ref(); + let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowseParams { + context, browse_id: "FEmusic_moods_and_genres_category", params: genre_id, }; @@ -55,8 +54,10 @@ impl RustyPipeQuery { impl MapResponse> for response::MusicGenres { fn map_response( self, - _ctx: &MapRespCtx<'_>, - ) -> Result>, ExtractionError> { + _id: &str, + _lang: crate::param::Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, + ) -> Result>, ExtractionError> { let content = self .contents .single_column_browse_results_renderer @@ -104,7 +105,14 @@ impl MapResponse> for response::MusicGenres { } impl MapResponse for response::MusicGenre { - fn map_response(self, ctx: &MapRespCtx<'_>) -> Result, ExtractionError> { + fn map_response( + self, + id: &str, + lang: crate::param::Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, + ) -> Result, ExtractionError> { + // dbg!(&self); + let content = self .contents .single_column_browse_results_renderer @@ -136,20 +144,18 @@ impl MapResponse for response::MusicGenre { h.music_carousel_shelf_basic_header_renderer .more_content_button .and_then(|btn| { - if let NavigationEndpoint::Browse { - browse_endpoint, .. - } = btn.button_renderer.navigation_endpoint - { - if browse_endpoint.browse_id - == "FEmusic_moods_and_genres_category" - { - Some(browse_endpoint.params) - } else { - None - } - } else { - None - } + btn.button_renderer + .navigation_endpoint + .browse_endpoint + .and_then(|browse| { + if browse.browse_id + == "FEmusic_moods_and_genres_category" + { + Some(browse.params) + } else { + None + } + }) }) }), shelf.contents, @@ -164,7 +170,7 @@ impl MapResponse for response::MusicGenre { _ => return None, }; - let mut mapper = MusicListMapper::new(ctx.lang); + let mut mapper = MusicListMapper::new(lang); mapper.map_response(items); let mut mapped = mapper.conv_items(); warnings.append(&mut mapped.warnings); @@ -179,7 +185,7 @@ impl MapResponse for response::MusicGenre { Ok(MapResult { c: MusicGenre { - id: ctx.id.to_owned(), + id: id.to_owned(), name: self.header.music_header_renderer.title, sections, }, @@ -196,7 +202,7 @@ mod tests { use rstest::rstest; use super::*; - use crate::{model, util::tests::TESTFILES}; + use crate::{model, param::Language, util::tests::TESTFILES}; #[test] fn map_music_genres() { @@ -206,7 +212,7 @@ mod tests { let playlist: response::MusicGenres = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - playlist.map_response(&MapRespCtx::test("")).unwrap(); + playlist.map_response("", Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -226,7 +232,7 @@ mod tests { let playlist: response::MusicGenre = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult = - playlist.map_response(&MapRespCtx::test(id)).unwrap(); + playlist.map_response(id, Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), diff --git a/src/client/music_new.rs b/src/client/music_new.rs index fb6cf17..e80bff1 100644 --- a/src/client/music_new.rs +++ b/src/client/music_new.rs @@ -4,16 +4,16 @@ use crate::{ client::response::music_item::MusicListMapper, error::{Error, ExtractionError}, model::{traits::FromYtItem, AlbumItem, TrackItem}, - serializer::MapResult, }; -use super::{response, ClientType, MapRespCtx, MapResponse, QBrowse, RustyPipeQuery}; +use super::{response, ClientType, MapResponse, QBrowse, RustyPipeQuery}; impl RustyPipeQuery { /// Get the new albums that were released on YouTube Music - #[tracing::instrument(skip(self), level = "error")] pub async fn music_new_albums(&self) -> Result, Error> { + let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { + context, browse_id: "FEmusic_new_releases_albums", }; @@ -28,9 +28,10 @@ impl RustyPipeQuery { } /// Get the new music videos that were released on YouTube Music - #[tracing::instrument(skip(self), level = "error")] pub async fn music_new_videos(&self) -> Result, Error> { + let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { + context, browse_id: "FEmusic_new_releases_videos", }; @@ -46,7 +47,12 @@ impl RustyPipeQuery { } impl MapResponse> for response::MusicNew { - fn map_response(self, ctx: &MapRespCtx<'_>) -> Result>, ExtractionError> { + fn map_response( + self, + _id: &str, + lang: crate::param::Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, + ) -> Result>, ExtractionError> { let items = self .contents .single_column_browse_results_renderer @@ -64,8 +70,9 @@ impl MapResponse> for response::MusicNew { .grid_renderer .items; - let mut mapper = MusicListMapper::new(ctx.lang); + let mut mapper = MusicListMapper::new(lang); mapper.map_response(items); + mapper.check_unknown()?; Ok(mapper.conv_items()) } @@ -79,7 +86,7 @@ mod tests { use rstest::rstest; use super::*; - use crate::{serializer::MapResult, util::tests::TESTFILES}; + use crate::{param::Language, serializer::MapResult, util::tests::TESTFILES}; #[rstest] #[case::default("default")] @@ -90,7 +97,7 @@ mod tests { let new_albums: response::MusicNew = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - new_albums.map_response(&MapRespCtx::test("")).unwrap(); + new_albums.map_response("", Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -102,15 +109,14 @@ mod tests { #[rstest] #[case::default("default")] - #[case::default("w_podcasts")] fn map_music_new_videos(#[case] name: &str) { let json_path = path!(*TESTFILES / "music_new" / format!("videos_{name}.json")); let json_file = File::open(json_path).unwrap(); - let new_videos: response::MusicNew = + let new_albums: response::MusicNew = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - new_videos.map_response(&MapRespCtx::test("")).unwrap(); + new_albums.map_response("", Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), diff --git a/src/client/music_playlist.rs b/src/client/music_playlist.rs index 3ef6e95..0bea1b3 100644 --- a/src/client/music_playlist.rs +++ b/src/client/music_playlist.rs @@ -1,36 +1,30 @@ -use std::{borrow::Cow, fmt::Debug}; +use std::borrow::Cow; use crate::{ - client::response::url_endpoint::NavigationEndpoint, error::{Error, ExtractionError}, - model::{ - paginator::{ContinuationEndpoint, Paginator}, - richtext::RichText, - AlbumId, ChannelId, MusicAlbum, MusicPlaylist, TrackItem, TrackType, - }, - serializer::{text::TextComponents, MapResult}, - util::{self, dictionary, TryRemove, DOT_SEPARATOR}, + model::{paginator::Paginator, AlbumId, ChannelId, MusicAlbum, MusicPlaylist, TrackItem}, + serializer::MapResult, + util::{self, TryRemove, DOT_SEPARATOR}, }; -use self::response::url_endpoint::MusicPageType; - use super::{ response::{ self, music_item::{map_album_type, map_artist_id, map_artists, MusicListMapper}, }, - ClientType, MapRespCtx, MapResponse, QBrowse, RustyPipeQuery, + ClientType, MapResponse, QBrowse, RustyPipeQuery, }; impl RustyPipeQuery { /// Get a playlist from YouTube Music - #[tracing::instrument(skip(self), level = "error")] - pub async fn music_playlist + Debug>( + pub async fn music_playlist>( &self, playlist_id: S, ) -> Result { let playlist_id = playlist_id.as_ref(); + let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { + context, browse_id: &format!("VL{playlist_id}"), }; @@ -45,13 +39,11 @@ impl RustyPipeQuery { } /// Get an album from YouTube Music - #[tracing::instrument(skip(self), level = "error")] - pub async fn music_album + Debug>( - &self, - album_id: S, - ) -> Result { + pub async fn music_album>(&self, album_id: S) -> Result { let album_id = album_id.as_ref(); + let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { + context, browse_id: album_id, }; @@ -87,7 +79,7 @@ impl RustyPipeQuery { .iter() .enumerate() .filter_map(|(i, track)| { - if track.track_type.is_video() && !track.unavailable { + if track.is_video { Some((i, track.name.clone())) } else { None @@ -95,55 +87,27 @@ impl RustyPipeQuery { }) .collect::>(); - let last_tn = album - .tracks - .last() - .and_then(|t| t.track_nr) - .unwrap_or_default(); - if !to_replace.is_empty() || last_tn < album.track_count { - tracing::debug!( - "fetching album playlist ({} tracks, {} to replace)", - album.track_count, - to_replace.len() - ); + if !to_replace.is_empty() { let mut playlist = self.music_playlist(playlist_id).await?; playlist .tracks - .extend_limit(&self, album.track_count.into()) + .extend_limit(&self, album.tracks.len()) .await?; for (i, title) in to_replace { let found_track = playlist.tracks.items.iter().find_map(|track| { - if track.name == title && track.track_type.is_track() { - Some((track.id.clone(), track.duration, track.unavailable)) + if track.name == title && !track.is_video { + Some((track.id.clone(), track.duration)) } else { None } }); - if let Some((track_id, duration, unavailable)) = found_track { + if let Some((track_id, duration)) = found_track { album.tracks[i].id = track_id; if let Some(duration) = duration { album.tracks[i].duration = Some(duration); } - album.tracks[i].track_type = TrackType::Track; - album.tracks[i].unavailable = unavailable; - } - } - - // Extend the list of album tracks with the ones from the playlist if the playlist returned more tracks - // This is the case for albums with more than 200 tracks (e.g. audiobooks) - // Note: in some cases the playlist may contain a loop of repeating tracks. If a track was found in the playlist - // that already exists in the album, stop. - if album.tracks.len() < playlist.tracks.items.len() { - let mut tn = last_tn; - for mut t in playlist.tracks.items.into_iter().skip(album.tracks.len()) { - if album.tracks.iter().any(|at| at.id == t.id) { - break; - } - tn += 1; - t.album = album.tracks.first().and_then(|t| t.album.clone()); - t.track_nr = Some(tn); - album.tracks.push(t); + album.tracks[i].is_video = false; } } } @@ -155,51 +119,22 @@ impl RustyPipeQuery { impl MapResponse for response::MusicPlaylist { fn map_response( self, - ctx: &MapRespCtx<'_>, + id: &str, + lang: crate::param::Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, ) -> Result, ExtractionError> { - let contents = match self.contents { - Some(c) => c, - None => { - if self.microformat.microformat_data_renderer.noindex { - return Err(ExtractionError::NotFound { - id: ctx.id.to_owned(), - msg: "no contents".into(), - }); - } else { - return Err(ExtractionError::InvalidData("no contents".into())); - } - } - }; + // dbg!(&self); - let (header, music_contents) = match contents { - response::music_playlist::Contents::SingleColumnBrowseResultsRenderer(c) => ( - self.header, - c.contents - .into_iter() - .next() - .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))? - .tab_renderer - .content - .section_list_renderer, - ), - response::music_playlist::Contents::TwoColumnBrowseResultsRenderer { - secondary_contents, - tabs, - } => ( - tabs.into_iter() - .next() - .and_then(|t| { - t.tab_renderer - .content - .section_list_renderer - .contents - .into_iter() - .next() - }) - .or(self.header), - secondary_contents.section_list_renderer, - ), - }; + let music_contents = self + .contents + .single_column_browse_results_renderer + .contents + .into_iter() + .next() + .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))? + .tab_renderer + .content + .section_list_renderer; let shelf = music_contents .contents .into_iter() @@ -212,28 +147,26 @@ impl MapResponse for response::MusicPlaylist { )))?; if let Some(playlist_id) = shelf.playlist_id { - if playlist_id != ctx.id { + if playlist_id != id { return Err(ExtractionError::WrongResult(format!( - "got wrong playlist id {}, expected {}", - playlist_id, ctx.id + "got wrong playlist id {playlist_id}, expected {id}" ))); } } - let mut mapper = MusicListMapper::new(ctx.lang); + let mut mapper = MusicListMapper::new(lang); mapper.map_response(shelf.contents); - - let ctoken = mapper.ctoken.clone().or_else(|| { - shelf - .continuations - .into_iter() - .next() - .map(|cont| cont.next_continuation_data.continuation) - }); + mapper.check_unknown()?; let map_res = mapper.conv_items(); + let ctoken = shelf + .continuations + .into_iter() + .next() + .map(|cont| cont.next_continuation_data.continuation); + let track_count = if ctoken.is_some() { - header.as_ref().and_then(|h| { + self.header.as_ref().and_then(|h| { let parts = h .music_detail_header_renderer .second_subtitle @@ -253,57 +186,31 @@ impl MapResponse for response::MusicPlaylist { .next() .map(|c| c.next_continuation_data.continuation); - let (from_ytm, channel, name, thumbnail, description) = match header { + let (from_ytm, channel, name, thumbnail, description) = match self.header { Some(header) => { let h = header.music_detail_header_renderer; - let (from_ytm, channel) = match h.facepile { - Some(facepile) => { - let from_ytm = facepile.avatar_stack_view_model.text.starts_with("YouTube"); - let channel = facepile - .avatar_stack_view_model - .renderer_context - .command_context - .and_then(|c| { - c.on_tap - .innertube_command - .music_page() - .filter(|p| p.typ == MusicPageType::User) - .map(|p| p.id) - }) - .map(|id| ChannelId { - id, - name: facepile.avatar_stack_view_model.text, - }); - - (from_ytm && channel.is_none(), channel) - } - None => { - let st = match h.strapline_text_one { - Some(s) => s, - None => h.subtitle, - }; - - let from_ytm = st.0.iter().any(util::is_ytm); - let channel = st.0.into_iter().find_map(|c| ChannelId::try_from(c).ok()); - (from_ytm, channel) - } - }; + let from_ytm = h.subtitle.0.iter().any(util::is_ytm); + let channel = h + .subtitle + .0 + .into_iter() + .find_map(|c| ChannelId::try_from(c).ok()); ( from_ytm, channel, h.title, h.thumbnail.into(), - h.description.map(TextComponents::from), + h.description, ) } None => { // Album playlists fetched via the playlist method dont include a header let (album, cover) = map_res .c - .iter() - .find_map(|t: &TrackItem| { + .first() + .and_then(|t: &TrackItem| { t.album.as_ref().map(|a| (a.clone(), t.cover.clone())) }) .ok_or(ExtractionError::InvalidData(Cow::Borrowed( @@ -311,11 +218,10 @@ impl MapResponse for response::MusicPlaylist { )))?; if !map_res.c.iter().all(|t| { - t.unavailable - || t.album - .as_ref() - .map(|a| a.id == album.id) - .unwrap_or_default() + t.album + .as_ref() + .map(|a| a.id == album.id) + .unwrap_or_default() }) { return Err(ExtractionError::InvalidData(Cow::Borrowed( "album playlist containing items from different albums", @@ -328,28 +234,26 @@ impl MapResponse for response::MusicPlaylist { Ok(MapResult { c: MusicPlaylist { - id: ctx.id.to_owned(), + id: id.to_owned(), name, thumbnail, channel, - description: description.map(RichText::from), + description, track_count, from_ytm, tracks: Paginator::new_ext( track_count, map_res.c, ctoken, - ctx.visitor_data.map(str::to_owned), - ContinuationEndpoint::MusicBrowse, - ctx.authenticated, + None, + crate::model::paginator::ContinuationEndpoint::MusicBrowse, ), related_playlists: Paginator::new_ext( None, Vec::new(), related_ctoken, - ctx.visitor_data.map(str::to_owned), - ContinuationEndpoint::MusicBrowse, - ctx.authenticated, + None, + crate::model::paginator::ContinuationEndpoint::MusicBrowse, ), }, warnings: map_res.warnings, @@ -358,73 +262,38 @@ impl MapResponse for response::MusicPlaylist { } impl MapResponse for response::MusicPlaylist { - fn map_response(self, ctx: &MapRespCtx<'_>) -> Result, ExtractionError> { - let contents = match self.contents { - Some(c) => c, - None => { - if self.microformat.microformat_data_renderer.noindex { - return Err(ExtractionError::NotFound { - id: ctx.id.to_owned(), - msg: "no contents".into(), - }); - } else { - return Err(ExtractionError::InvalidData("no contents".into())); - } - } - }; + fn map_response( + self, + id: &str, + lang: crate::param::Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, + ) -> Result, ExtractionError> { + // dbg!(&self); - let (header, sections) = match contents { - response::music_playlist::Contents::SingleColumnBrowseResultsRenderer(c) => ( - self.header, - c.contents - .into_iter() - .next() - .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))? - .tab_renderer - .content - .section_list_renderer - .contents, - ), - response::music_playlist::Contents::TwoColumnBrowseResultsRenderer { - secondary_contents, - tabs, - } => ( - tabs.into_iter() - .next() - .and_then(|t| { - t.tab_renderer - .content - .section_list_renderer - .contents - .into_iter() - .next() - }) - .or(self.header), - secondary_contents.section_list_renderer.contents, - ), - }; - let header = header + let header = self + .header .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no header")))? .music_detail_header_renderer; + let sections = self + .contents + .single_column_browse_results_renderer + .contents + .into_iter() + .next() + .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))? + .tab_renderer + .content + .section_list_renderer + .contents; + let mut shelf = None; let mut album_variants = None; for section in sections { match section { response::music_item::ItemSection::MusicShelfRenderer(sh) => shelf = Some(sh), response::music_item::ItemSection::MusicCarouselShelfRenderer(sh) => { - if sh - .header - .map(|h| { - h.music_carousel_shelf_basic_header_renderer - .title - .first_str() - == dictionary::entry(ctx.lang).album_versions_title - }) - .unwrap_or_default() - { - album_variants = Some(sh.contents); - } + album_variants = Some(sh.contents); } _ => (), } @@ -435,37 +304,27 @@ impl MapResponse for response::MusicPlaylist { let mut subtitle_split = header.subtitle.split(util::DOT_SEPARATOR); - let (year_txt, artists_p) = match header.strapline_text_one { - // New (2column) album layout - Some(sl) => { + let (year_txt, artists_p) = match subtitle_split.len() { + 3.. => { let year_txt = subtitle_split - .try_swap_remove(1) - .and_then(|t| t.0.first().map(|c| c.as_str().to_owned())); - (year_txt, Some(sl)) + .swap_remove(2) + .0 + .get(0) + .map(|c| c.as_str().to_owned()); + (year_txt, subtitle_split.try_swap_remove(1)) } - // Old album layout - None => match subtitle_split.len() { - 3.. => { - let year_txt = subtitle_split - .swap_remove(2) - .0 - .first() - .map(|c| c.as_str().to_owned()); - (year_txt, subtitle_split.try_swap_remove(1)) + 2 => { + // The second part may either be the year or the artist + let p2 = subtitle_split.swap_remove(1); + let is_year = + p2.0.len() == 1 && p2.0[0].as_str().chars().all(|c| c.is_ascii_digit()); + if is_year { + (Some(p2.0[0].as_str().to_owned()), None) + } else { + (None, Some(p2)) } - 2 => { - // The second part may either be the year or the artist - let p2 = subtitle_split.swap_remove(1); - let is_year = - p2.0.len() == 1 && p2.0[0].as_str().chars().all(|c| c.is_ascii_digit()); - if is_year { - (Some(p2.0[0].as_str().to_owned()), None) - } else { - (None, Some(p2)) - } - } - _ => (None, None), - }, + } + _ => (None, None), }; let (artists, by_va) = map_artists(artists_p); @@ -475,68 +334,35 @@ impl MapResponse for response::MusicPlaylist { .map(|part| part.to_string()) .unwrap_or_default(); - let album_type = map_album_type(album_type_txt.as_str(), ctx.lang); + let album_type = map_album_type(album_type_txt.as_str(), lang); let year = year_txt.and_then(|txt| util::parse_numeric(&txt).ok()); - fn map_playlist_id(ep: &NavigationEndpoint) -> Option { - if let NavigationEndpoint::WatchPlaylist { - watch_playlist_endpoint, - } = ep - { - Some(watch_playlist_endpoint.playlist_id.to_owned()) - } else { - None - } - } - - let playlist_id = self - .microformat - .microformat_data_renderer - .url_canonical - .and_then(|x| { - x.strip_prefix("https://music.youtube.com/playlist?list=") - .map(str::to_owned) - }); - let (playlist_id, artist_id) = header + let (artist_id, playlist_id) = header .menu - .or_else(|| header.buttons.into_iter().next()) .map(|menu| { ( - playlist_id.or_else(|| { - menu.menu_renderer - .top_level_buttons - .iter() - .find_map(|btn| { - map_playlist_id(&btn.button_renderer.navigation_endpoint) - }) - .or_else(|| { - menu.menu_renderer.items.iter().find_map(|itm| { - map_playlist_id( - &itm.menu_navigation_item_renderer.navigation_endpoint, - ) - }) - }) - }), map_artist_id(menu.menu_renderer.items), + menu.menu_renderer + .top_level_buttons + .into_iter() + .next() + .map(|btn| { + btn.button_renderer + .navigation_endpoint + .watch_playlist_endpoint + .playlist_id + }), ) }) .unwrap_or_default(); let artist_id = artist_id.or_else(|| artists.first().and_then(|a| a.id.clone())); - let second_subtitle_parts = header - .second_subtitle - .split(|p| p == DOT_SEPARATOR) - .collect::>(); - let track_count = second_subtitle_parts - .get(usize::from(second_subtitle_parts.len() > 2)) - .and_then(|txt| util::parse_numeric::(&txt[0]).ok()); - let mut mapper = MusicListMapper::with_album( - ctx.lang, + lang, artists.clone(), by_va, AlbumId { - id: ctx.id.to_owned(), + id: id.to_owned(), name: header.title.clone(), }, ); @@ -544,7 +370,7 @@ impl MapResponse for response::MusicPlaylist { let tracks_res = mapper.conv_items(); let mut warnings = tracks_res.warnings; - let mut variants_mapper = MusicListMapper::new(ctx.lang); + let mut variants_mapper = MusicListMapper::new(lang); if let Some(res) = album_variants { variants_mapper.map_response(res); } @@ -553,19 +379,16 @@ impl MapResponse for response::MusicPlaylist { Ok(MapResult { c: MusicAlbum { - id: ctx.id.to_owned(), + id: id.to_owned(), playlist_id, name: header.title, cover: header.thumbnail.into(), artists, artist_id, - description: header - .description - .map(|t| RichText::from(TextComponents::from(t))), + description: header.description, album_type, year, by_va, - track_count: track_count.unwrap_or(tracks_res.c.len() as u16), tracks: tracks_res.c, variants: variants_res.c, }, @@ -582,15 +405,12 @@ mod tests { use rstest::rstest; use super::*; - use crate::{model, util::tests::TESTFILES}; + use crate::{model, param::Language, util::tests::TESTFILES}; #[rstest] #[case::short("short", "RDCLAK5uy_kFQXdnqMaQCVx2wpUM4ZfbsGCDibZtkJk")] #[case::long("long", "PL5dDx681T4bR7ZF1IuWzOv1omlRbE7PiJ")] #[case::nomusic("nomusic", "PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe")] - #[case::two_columns("20240228_twoColumns", "RDCLAK5uy_kb7EBi6y3GrtJri4_ZH56Ms786DFEimbM")] - #[case::n_album("20240228_album", "OLAK5uy_kdSWBZ-9AZDkYkuy0QCc3p0KO9DEHVNH0")] - #[case::facepile("20241125_facepile", "PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe")] fn map_music_playlist(#[case] name: &str, #[case] id: &str) { let json_path = path!(*TESTFILES / "music_playlist" / format!("playlist_{name}.json")); let json_file = File::open(json_path).unwrap(); @@ -598,7 +418,7 @@ mod tests { let playlist: response::MusicPlaylist = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult = - playlist.map_response(&MapRespCtx::test(id)).unwrap(); + playlist.map_response(id, Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -616,8 +436,6 @@ mod tests { #[case::single("single", "MPREb_bHfHGoy7vuv")] #[case::description("description", "MPREb_PiyfuVl6aYd")] #[case::unavailable("unavailable", "MPREb_AzuWg8qAVVl")] - #[case::two_columns("20240228_twoColumns", "MPREb_bHfHGoy7vuv")] - #[case::recommends("20250225_recommends", "MPREb_u1I69lSAe5v")] fn map_music_album(#[case] name: &str, #[case] id: &str) { let json_path = path!(*TESTFILES / "music_playlist" / format!("album_{name}.json")); let json_file = File::open(json_path).unwrap(); @@ -625,7 +443,7 @@ mod tests { let playlist: response::MusicPlaylist = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult = - playlist.map_response(&MapRespCtx::test(id)).unwrap(); + playlist.map_response(id, Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), diff --git a/src/client/music_search.rs b/src/client/music_search.rs index a083dcc..eafb524 100644 --- a/src/client/music_search.rs +++ b/src/client/music_search.rs @@ -1,4 +1,4 @@ -use std::{borrow::Cow, fmt::Debug}; +use std::borrow::Cow; use serde::Serialize; @@ -6,45 +6,96 @@ use crate::{ client::response::music_item::MusicListMapper, error::{Error, ExtractionError}, model::{ - paginator::{ContinuationEndpoint, Paginator}, - traits::FromYtItem, - AlbumItem, ArtistItem, MusicItem, MusicPlaylistItem, MusicSearchResult, - MusicSearchSuggestion, TrackItem, UserItem, + paginator::Paginator, traits::FromYtItem, AlbumItem, ArtistItem, MusicPlaylistItem, + MusicSearchFiltered, MusicSearchResult, MusicSearchSuggestion, TrackItem, }, - param::search_filter::MusicSearchFilter, serializer::MapResult, }; -use super::{response, ClientType, MapRespCtx, MapResponse, RustyPipeQuery}; +use super::{response, ClientType, MapResponse, RustyPipeQuery, YTContext}; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QSearch<'a> { + context: YTContext<'a>, query: &'a str, #[serde(skip_serializing_if = "Option::is_none")] - params: Option<&'a str>, + params: Option, } #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QSearchSuggestion<'a> { + context: YTContext<'a>, input: &'a str, } +#[derive(Debug, Serialize)] +enum Params { + #[serde(rename = "EgWKAQIIAWoMEAMQBBAJEA4QChAF")] + Tracks, + #[serde(rename = "EgWKAQIQAWoMEAMQBBAJEA4QChAF")] + Videos, + #[serde(rename = "EgWKAQIYAWoMEAMQBBAJEA4QChAF")] + Albums, + #[serde(rename = "EgWKAQIgAWoMEAMQBBAJEA4QChAF")] + Artists, + #[serde(rename = "EgWKAQIoAWoMEAMQBBAJEA4QChAF")] + Playlists, + #[serde(rename = "EgeKAQQoADgBagwQAxAEEAkQDhAKEAU%3D")] + YtmPlaylists, + #[serde(rename = "EgeKAQQoAEABagwQAxAEEAkQDhAKEAU%3D")] + CommunityPlaylists, +} + impl RustyPipeQuery { - /// Search YouTube Music. - /// - /// This is a generic implementation which casts items to the given type or filters - /// them out. - pub async fn music_search>( + /// Search YouTube Music. Returns items from any type. + pub async fn music_search>(&self, query: S) -> Result { + let query = query.as_ref(); + let context = self.get_context(ClientType::DesktopMusic, true, None).await; + let request_body = QSearch { + context, + query, + params: None, + }; + + self.execute_request::( + ClientType::DesktopMusic, + "music_search", + query, + "search", + &request_body, + ) + .await + } + + /// Search YouTube Music tracks + pub async fn music_search_tracks>( &self, query: S, - filter: Option, - ) -> Result, Error> { + ) -> Result, Error> { + self._music_search_tracks(query, Params::Tracks).await + } + + /// Search YouTube Music videos + pub async fn music_search_videos>( + &self, + query: S, + ) -> Result, Error> { + self._music_search_tracks(query, Params::Videos).await + } + + async fn _music_search_tracks>( + &self, + query: S, + params: Params, + ) -> Result, Error> { let query = query.as_ref(); + let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QSearch { + context, query, - params: filter.map(MusicSearchFilter::params), + params: Some(params), }; self.execute_request::( @@ -57,87 +108,111 @@ impl RustyPipeQuery { .await } - /// Search YouTube Music and return items of all types - pub async fn music_search_main>( - &self, - query: S, - ) -> Result, Error> { - self.music_search(query, None).await - } - - /// Search YouTube Music artists - pub async fn music_search_artists>( - &self, - query: S, - ) -> Result, Error> { - self.music_search(query, Some(MusicSearchFilter::Artists)) - .await - } - /// Search YouTube Music albums pub async fn music_search_albums>( &self, query: S, - ) -> Result, Error> { - self.music_search(query, Some(MusicSearchFilter::Albums)) - .await - } - - /// Search YouTube Music tracks - pub async fn music_search_tracks>( - &self, - query: S, - ) -> Result, Error> { - self.music_search(query, Some(MusicSearchFilter::Tracks)) - .await - } - - /// Search YouTube Music videos - pub async fn music_search_videos>( - &self, - query: S, - ) -> Result, Error> { - self.music_search(query, Some(MusicSearchFilter::Videos)) - .await - } - - /// Search YouTube Music playlists - /// - /// Playlists are filtered whether they are created by users - /// (`community=true`) or by YouTube Music (`community=false`) - pub async fn music_search_playlists + Debug>( - &self, - query: S, - community: bool, - ) -> Result, Error> { - self.music_search( + ) -> Result, Error> { + let query = query.as_ref(); + let context = self.get_context(ClientType::DesktopMusic, true, None).await; + let request_body = QSearch { + context, query, - Some(if community { - MusicSearchFilter::CommunityPlaylists - } else { - MusicSearchFilter::YtmPlaylists - }), + params: Some(Params::Albums), + }; + + self.execute_request::( + ClientType::DesktopMusic, + "music_search_albums", + query, + "search", + &request_body, ) .await } - /// Search YouTube Music users - pub async fn music_search_users>( + /// Search YouTube Music artists + pub async fn music_search_artists( + &self, + query: &str, + ) -> Result, Error> { + let context = self.get_context(ClientType::DesktopMusic, true, None).await; + let request_body = QSearch { + context, + query, + params: Some(Params::Artists), + }; + + self.execute_request::( + ClientType::DesktopMusic, + "music_search_albums", + query, + "search", + &request_body, + ) + .await + } + + /// Search YouTube Music playlists + pub async fn music_search_playlists>( &self, query: S, - ) -> Result, Error> { - self.music_search(query, Some(MusicSearchFilter::Users)) - .await + ) -> Result, Error> { + self._music_search_playlists(query, Params::Playlists).await + } + + /// Search YouTube Music playlists that were created by users + /// (`community=true`) or by YouTube Music (`community=false`) + pub async fn music_search_playlists_filter>( + &self, + query: S, + community: bool, + ) -> Result, Error> { + self._music_search_playlists( + query, + if community { + Params::CommunityPlaylists + } else { + Params::YtmPlaylists + }, + ) + .await + } + + async fn _music_search_playlists>( + &self, + query: S, + params: Params, + ) -> Result, Error> { + let query = query.as_ref(); + let context = self.get_context(ClientType::DesktopMusic, true, None).await; + let request_body = QSearch { + context, + query, + params: Some(params), + }; + + self.execute_request::( + ClientType::DesktopMusic, + "music_search_playlists", + query, + "search", + &request_body, + ) + .await } /// Get YouTube Music search suggestions - #[tracing::instrument(skip(self), level = "error")] - pub async fn music_search_suggestion + Debug>( + pub async fn music_search_suggestion>( &self, query: S, ) -> Result { let query = query.as_ref(); - let request_body = QSearchSuggestion { input: query }; + let context = self.get_context(ClientType::DesktopMusic, true, None).await; + let request_body = QSearchSuggestion { + context, + input: query, + }; self.execute_request::( ClientType::DesktopMusic, @@ -150,11 +225,80 @@ impl RustyPipeQuery { } } -impl MapResponse> for response::MusicSearch { +impl MapResponse for response::MusicSearch { fn map_response( self, - ctx: &MapRespCtx<'_>, - ) -> Result>, ExtractionError> { + _id: &str, + lang: crate::param::Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, + ) -> Result, crate::error::ExtractionError> { + // dbg!(&self); + + let sections = self + .contents + .tabbed_search_results_renderer + .contents + .into_iter() + .next() + .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no tab")))? + .tab_renderer + .content + .section_list_renderer + .contents; + + let mut corrected_query = None; + let mut order = Vec::new(); + let mut mapper = MusicListMapper::new(lang); + + sections.into_iter().for_each(|section| match section { + response::music_search::ItemSection::MusicShelfRenderer(shelf) => { + if let Some(etype) = mapper.map_response(shelf.contents) { + if !order.contains(&etype) { + order.push(etype); + } + } + } + response::music_search::ItemSection::MusicCardShelfRenderer(card) => { + if let Some(etype) = mapper.map_card(card) { + if !order.contains(&etype) { + order.push(etype); + } + } + } + response::music_search::ItemSection::ItemSectionRenderer { contents } => { + if let Some(corrected) = contents.into_iter().next() { + corrected_query = Some(corrected.showing_results_for_renderer.corrected_query); + } + } + response::music_search::ItemSection::None => {} + }); + + mapper.check_unknown()?; + let map_res = mapper.group_items(); + + Ok(MapResult { + c: MusicSearchResult { + tracks: map_res.c.tracks, + albums: map_res.c.albums, + artists: map_res.c.artists, + playlists: map_res.c.playlists, + corrected_query, + order, + }, + warnings: map_res.warnings, + }) + } +} + +impl MapResponse> for response::MusicSearch { + fn map_response( + self, + _id: &str, + lang: crate::param::Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, + ) -> Result>, ExtractionError> { + // dbg!(&self); + let tabs = self.contents.tabbed_search_results_renderer.contents; let sections = tabs .into_iter() @@ -167,7 +311,7 @@ impl MapResponse> for response::MusicSearch let mut corrected_query = None; let mut ctoken = None; - let mut mapper = MusicListMapper::new(ctx.lang); + let mut mapper = MusicListMapper::new(lang); sections.into_iter().for_each(|section| match section { response::music_search::ItemSection::MusicShelfRenderer(shelf) => { @@ -187,18 +331,17 @@ impl MapResponse> for response::MusicSearch response::music_search::ItemSection::None => {} }); - let ctoken = ctoken.or(mapper.ctoken.clone()); + mapper.check_unknown()?; let map_res = mapper.conv_items(); Ok(MapResult { - c: MusicSearchResult { + c: MusicSearchFiltered { items: Paginator::new_ext( None, map_res.c, ctoken, - ctx.visitor_data.map(str::to_owned), - ContinuationEndpoint::MusicSearch, - false, + None, + crate::model::paginator::ContinuationEndpoint::MusicSearch, ), corrected_query, }, @@ -210,9 +353,11 @@ impl MapResponse> for response::MusicSearch impl MapResponse for response::MusicSearchSuggestion { fn map_response( self, - ctx: &MapRespCtx<'_>, + _id: &str, + lang: crate::param::Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, ) -> Result, ExtractionError> { - let mut mapper = MusicListMapper::new_search_suggest(ctx.lang); + let mut mapper = MusicListMapper::new(lang); let mut terms = Vec::new(); for section in self.contents { @@ -231,6 +376,7 @@ impl MapResponse for response::MusicSearchSuggestion { } } + mapper.check_unknown()?; let map_res = mapper.conv_items(); Ok(MapResult { @@ -251,11 +397,12 @@ mod tests { use rstest::rstest; use crate::{ - client::{response, MapRespCtx, MapResponse}, + client::{response, MapResponse}, model::{ - AlbumItem, ArtistItem, MusicItem, MusicPlaylistItem, MusicSearchResult, + AlbumItem, ArtistItem, MusicPlaylistItem, MusicSearchFiltered, MusicSearchResult, MusicSearchSuggestion, TrackItem, }, + param::Language, serializer::MapResult, util::tests::TESTFILES, }; @@ -265,15 +412,14 @@ mod tests { #[case::typo("typo")] #[case::radio("radio")] #[case::artist("artist")] - #[case::live("live")] fn map_music_search_main(#[case] name: &str) { let json_path = path!(*TESTFILES / "music_search" / format!("main_{name}.json")); let json_file = File::open(json_path).unwrap(); let search: response::MusicSearch = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult> = - search.map_response(&MapRespCtx::test("")).unwrap(); + let map_res: MapResult = + search.map_response("", Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -295,8 +441,8 @@ mod tests { let search: response::MusicSearch = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult> = - search.map_response(&MapRespCtx::test("")).unwrap(); + let map_res: MapResult> = + search.map_response("", Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -314,8 +460,8 @@ mod tests { let search: response::MusicSearch = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult> = - search.map_response(&MapRespCtx::test("")).unwrap(); + let map_res: MapResult> = + search.map_response("", Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -333,8 +479,8 @@ mod tests { let search: response::MusicSearch = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult> = - search.map_response(&MapRespCtx::test("")).unwrap(); + let map_res: MapResult> = + search.map_response("", Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -354,8 +500,8 @@ mod tests { let search: response::MusicSearch = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult> = - search.map_response(&MapRespCtx::test("")).unwrap(); + let map_res: MapResult> = + search.map_response("", Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -376,7 +522,7 @@ mod tests { let suggestion: response::MusicSearchSuggestion = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult = - suggestion.map_response(&MapRespCtx::test("")).unwrap(); + suggestion.map_response("", Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), diff --git a/src/client/music_userdata.rs b/src/client/music_userdata.rs deleted file mode 100644 index 8c256cb..0000000 --- a/src/client/music_userdata.rs +++ /dev/null @@ -1,228 +0,0 @@ -use std::fmt::Debug; - -use crate::{ - client::{ - response::{self, music_item::MusicListMapper}, - ClientType, MapResponse, QBrowseParams, RustyPipeQuery, - }, - error::{Error, ExtractionError}, - model::{ - paginator::{ContinuationEndpoint, Paginator}, - AlbumItem, ArtistItem, HistoryItem, MusicPlaylist, MusicPlaylistItem, TrackItem, - }, - serializer::MapResult, -}; - -use super::{MapRespCtx, MapRespOptions, QContinuation}; - -impl RustyPipeQuery { - /// Get a list of tracks from YouTube Music which the current user recently played - /// - /// Requires authentication cookies. - #[tracing::instrument(skip(self), level = "error")] - pub async fn music_history(&self) -> Result>, Error> { - let request_body = QBrowseParams { - browse_id: "FEmusic_history", - params: "oggECgIIAQ%3D%3D", - }; - - self.clone() - .authenticated() - .execute_request::( - ClientType::DesktopMusic, - "music_history", - "", - "browse", - &request_body, - ) - .await - } - - /// Get more YouTube Music history items from the given continuation token - #[tracing::instrument(skip(self), level = "error")] - pub async fn music_history_continuation + Debug>( - &self, - ctoken: S, - visitor_data: Option<&str>, - ) -> Result>, Error> { - let ctoken = ctoken.as_ref(); - let request_body = QContinuation { - continuation: ctoken, - }; - - self.clone() - .authenticated() - .execute_request_ctx::( - ClientType::Desktop, - "history_continuation", - ctoken, - "browse", - &request_body, - MapRespOptions { - visitor_data, - ..Default::default() - }, - ) - .await - } - - /// Get a list of YouTube Music artists which the current user subscribed to - /// - /// Requires authentication cookies. - #[tracing::instrument(skip(self), level = "error")] - pub async fn music_saved_artists(&self) -> Result, Error> { - self.clone() - .authenticated() - .continuation( - "4qmFsgIyEh5GRW11c2ljX2xpYnJhcnlfY29ycHVzX2FydGlzdHMaEGdnTUdLZ1FJQUJBQm9BWUI%3D", - ContinuationEndpoint::MusicBrowse, - None, - ) - .await - } - - /// Get a list of YouTube Music albums which the current user has added to their collection - /// - /// Requires authentication cookies. - #[tracing::instrument(skip(self), level = "error")] - pub async fn music_saved_albums(&self) -> Result, Error> { - self.clone() - .authenticated() - .continuation( - "4qmFsgIoEhRGRW11c2ljX2xpa2VkX2FsYnVtcxoQZ2dNR0tnUUlBQkFCb0FZQg%3D%3D", - ContinuationEndpoint::MusicBrowse, - None, - ) - .await - } - - /// Get a list of YouTube Music tracks which the current user has added to their collection - /// - /// Contains both liked tracks and tracks from saved albums. - /// - /// Requires authentication cookies. - #[tracing::instrument(skip(self), level = "error")] - pub async fn music_saved_tracks(&self) -> Result, Error> { - self.clone() - .authenticated() - .continuation( - "4qmFsgIoEhRGRW11c2ljX2xpa2VkX3ZpZGVvcxoQZ2dNR0tnUUlBQkFCb0FZQg%3D%3D", - ContinuationEndpoint::MusicBrowse, - None, - ) - .await - } - - /// Get a list of YouTube Music playlists which the current user has added to their collection - /// - /// Requires authentication cookies. - #[tracing::instrument(skip(self), level = "error")] - pub async fn music_saved_playlists(&self) -> Result, Error> { - self.clone() - .authenticated() - .continuation( - "4qmFsgIrEhdGRW11c2ljX2xpa2VkX3BsYXlsaXN0cxoQZ2dNR0tnUUlBQkFCb0FZQg%3D%3D", - ContinuationEndpoint::MusicBrowse, - None, - ) - .await - } - - /// Get all liked YouTube Music tracks of the logged-in user - /// - /// The difference to [`RustyPipeQuery::music_saved_tracks`] is that this function only returns - /// tracks that were explicitly liked by the user. - /// - /// Requires authentication cookies. - pub async fn music_liked_tracks(&self) -> Result { - self.clone() - .authenticated() - .music_playlist("LM") - .await - .map_err(crate::util::map_internal_playlist_err) - } -} - -impl MapResponse>> for response::MusicHistory { - fn map_response( - self, - ctx: &MapRespCtx<'_>, - ) -> Result>>, ExtractionError> { - let contents = match self.contents { - response::music_playlist::Contents::SingleColumnBrowseResultsRenderer(c) => { - c.contents - .into_iter() - .next() - .ok_or(ExtractionError::InvalidData("no content".into()))? - .tab_renderer - .content - .section_list_renderer - } - response::music_playlist::Contents::TwoColumnBrowseResultsRenderer { - secondary_contents, - .. - } => secondary_contents.section_list_renderer, - }; - - let mut map_res = MapResult::default(); - - for shelf in contents.contents { - let shelf = if let response::music_item::ItemSection::MusicShelfRenderer(s) = shelf { - s - } else { - continue; - }; - let mut mapper = MusicListMapper::new(ctx.lang); - mapper.map_response(shelf.contents); - mapper.conv_history_items(shelf.title, ctx.utc_offset, &mut map_res); - } - - let ctoken = contents - .continuations - .into_iter() - .next() - .map(|c| c.next_continuation_data.continuation); - - Ok(MapResult { - c: Paginator::new_ext( - None, - map_res.c, - ctoken, - ctx.visitor_data.map(str::to_owned), - ContinuationEndpoint::MusicBrowse, - true, - ), - warnings: map_res.warnings, - }) - } -} - -#[cfg(test)] -mod tests { - use std::{fs::File, io::BufReader}; - - use path_macro::path; - - use crate::util::tests::TESTFILES; - - use super::*; - - #[test] - fn map_history() { - let json_path = path!(*TESTFILES / "music_userdata" / "music_history.json"); - let json_file = File::open(json_path).unwrap(); - - let history: response::MusicHistory = - serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res = history.map_response(&MapRespCtx::test("")).unwrap(); - - assert!( - map_res.warnings.is_empty(), - "deserialization/mapping warnings: {:?}", - map_res.warnings - ); - insta::assert_ron_snapshot!(map_res.c, { - ".items[].playback_date" => "[date]", - }); - } -} diff --git a/src/client/pagination.rs b/src/client/pagination.rs index f0f052e..62cf220 100644 --- a/src/client/pagination.rs +++ b/src/client/pagination.rs @@ -1,5 +1,3 @@ -use std::fmt::Debug; - use crate::error::{Error, ExtractionError}; use crate::model::{ paginator::{ContinuationEndpoint, Paginator}, @@ -8,21 +6,12 @@ use crate::model::{ }; use crate::serializer::MapResult; -#[cfg(feature = "userdata")] -use crate::model::{HistoryItem, TrackItem, VideoItem}; - -use super::response::{ - music_item::{map_queue_item, MusicListMapper, PlaylistPanelVideo}, - YouTubeListItem, -}; -use super::{ - response, ClientType, MapRespCtx, MapRespOptions, MapResponse, QContinuation, RustyPipeQuery, -}; +use super::response::music_item::{map_queue_item, MusicListMapper, PlaylistPanelVideo}; +use super::{response, ClientType, MapResponse, QContinuation, RustyPipeQuery}; impl RustyPipeQuery { /// Get more YouTube items from the given continuation token and endpoint - #[tracing::instrument(skip(self), level = "error")] - pub async fn continuation + Debug>( + pub async fn continuation>( &self, ctoken: S, endpoint: ContinuationEndpoint, @@ -30,118 +19,107 @@ impl RustyPipeQuery { ) -> Result, Error> { let ctoken = ctoken.as_ref(); if endpoint.is_music() { + let context = self + .get_context(ClientType::DesktopMusic, true, visitor_data) + .await; let request_body = QContinuation { + context, continuation: ctoken, }; let p = self - .execute_request_ctx::, _>( + .execute_request::, _>( ClientType::DesktopMusic, "music_continuation", ctoken, endpoint.as_str(), &request_body, - MapRespOptions { - visitor_data, - ..Default::default() - }, ) .await?; - Ok(map_ytm_paginator(p, endpoint)) + Ok(map_ytm_paginator(p, visitor_data, endpoint)) } else { + let context = self + .get_context(ClientType::Desktop, true, visitor_data) + .await; let request_body = QContinuation { + context, continuation: ctoken, }; let p = self - .execute_request_ctx::, _>( + .execute_request::, _>( ClientType::Desktop, "continuation", ctoken, endpoint.as_str(), &request_body, - MapRespOptions { - visitor_data, - ..Default::default() - }, ) .await?; - Ok(map_yt_paginator(p, endpoint)) + Ok(map_yt_paginator(p, visitor_data, endpoint)) } } } fn map_yt_paginator( p: Paginator, + visitor_data: Option<&str>, endpoint: ContinuationEndpoint, ) -> Paginator { Paginator { count: p.count, items: p.items.into_iter().filter_map(T::from_yt_item).collect(), ctoken: p.ctoken, - visitor_data: p.visitor_data, + visitor_data: visitor_data.map(str::to_owned), endpoint, - authenticated: p.authenticated, } } fn map_ytm_paginator( p: Paginator, + visitor_data: Option<&str>, endpoint: ContinuationEndpoint, ) -> Paginator { Paginator { count: p.count, items: p.items.into_iter().filter_map(T::from_ytm_item).collect(), ctoken: p.ctoken, - visitor_data: p.visitor_data, + visitor_data: visitor_data.map(str::to_owned), endpoint, - authenticated: p.authenticated, } } -fn continuation_items(response: response::Continuation) -> MapResult> { - response - .on_response_received_actions - .and_then(|actions| { - actions - .into_iter() - .map(|action| action.append_continuation_items_action.continuation_items) - .reduce(|mut acc, mut items| { - acc.c.append(&mut items.c); - acc.warnings.append(&mut items.warnings); - acc - }) - }) - .or_else(|| { - response - .continuation_contents - .map(|contents| contents.rich_grid_continuation.contents) - }) - .unwrap_or_default() -} - impl MapResponse> for response::Continuation { fn map_response( self, - ctx: &MapRespCtx<'_>, + _id: &str, + lang: crate::param::Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, ) -> Result>, ExtractionError> { - let estimated_results = self.estimated_results; - let items = continuation_items(self); + let items = self + .on_response_received_actions + .and_then(|actions| { + actions + .into_iter() + .map(|action| action.append_continuation_items_action.continuation_items) + .reduce(|mut acc, mut items| { + acc.c.append(&mut items.c); + acc.warnings.append(&mut items.warnings); + acc + }) + }) + .or_else(|| { + self.continuation_contents + .map(|contents| contents.rich_grid_continuation.contents) + }) + .unwrap_or_default(); - let mut mapper = response::YouTubeListMapper::::new(ctx.lang); + let mut mapper = response::YouTubeListMapper::::new(lang); mapper.map_response(items); Ok(MapResult { - c: Paginator::new_ext( - estimated_results, - mapper.items, - mapper.ctoken, - ctx.visitor_data.map(str::to_owned), - ContinuationEndpoint::Browse, - ctx.authenticated, - ), + c: Paginator::new(self.estimated_results, mapper.items, mapper.ctoken), warnings: mapper.warnings, }) } @@ -150,13 +128,11 @@ impl MapResponse> for response::Continuation { impl MapResponse> for response::MusicContinuation { fn map_response( self, - ctx: &MapRespCtx<'_>, + _id: &str, + lang: crate::param::Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, ) -> Result>, ExtractionError> { - let mut mapper = if let Some(artist) = &ctx.artist { - MusicListMapper::with_artist(ctx.lang, artist.clone()) - } else { - MusicListMapper::new(ctx.lang) - }; + let mut mapper = MusicListMapper::new(lang); let mut continuations = Vec::new(); match self.continuation_contents { @@ -174,11 +150,7 @@ impl MapResponse> for response::MusicContinuation { response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => { mapper.map_response(shelf.contents); } - response::music_item::ItemSection::GridRenderer(mut grid) => { - mapper.map_response(grid.items); - continuations.append(&mut grid.continuations); - } - response::music_item::ItemSection::None => {} + _ => {} } } } @@ -189,133 +161,23 @@ impl MapResponse> for response::MusicContinuation { mapper.add_warnings(&mut panel.contents.warnings); panel.contents.c.into_iter().for_each(|item| { if let PlaylistPanelVideo::PlaylistPanelVideoRenderer(item) = item { - let mut track = map_queue_item(item, ctx.lang); + let mut track = map_queue_item(item, lang); mapper.add_item(MusicItem::Track(track.c)); mapper.add_warnings(&mut track.warnings); } }); } - Some(response::music_item::ContinuationContents::GridContinuation(mut grid)) => { - mapper.map_response(grid.items); - continuations.append(&mut grid.continuations); - } None => {} } - for a in self.on_response_received_actions { - mapper.map_response(a.append_continuation_items_action.continuation_items); - } - - let ctoken = mapper.ctoken.clone().or_else(|| { - continuations - .into_iter() - .next() - .map(|cont| cont.next_continuation_data.continuation) - }); let map_res = mapper.items(); - - Ok(MapResult { - c: Paginator::new_ext( - None, - map_res.c, - ctoken, - ctx.visitor_data.map(str::to_owned), - ContinuationEndpoint::MusicBrowse, - ctx.authenticated, - ), - warnings: map_res.warnings, - }) - } -} - -#[cfg(feature = "userdata")] -impl MapResponse>> for response::Continuation { - fn map_response( - self, - ctx: &MapRespCtx<'_>, - ) -> Result>>, ExtractionError> { - let mut map_res = MapResult::default(); - let mut ctoken = None; - - let items = continuation_items(self); - for item in items.c { - match item { - response::YouTubeListItem::ItemSectionRenderer { header, contents } => { - let mut mapper = response::YouTubeListMapper::::new(ctx.lang); - mapper.map_response(contents); - mapper.conv_history_items( - header.map(|h| h.item_section_header_renderer.title), - ctx.utc_offset, - &mut map_res, - ); - } - response::YouTubeListItem::ContinuationItemRenderer(ep) => { - if ctoken.is_none() { - ctoken = ep.continuation_endpoint.into_token(); - } - } - _ => {} - } - } - - Ok(MapResult { - c: Paginator::new_ext( - None, - map_res.c, - ctoken, - ctx.visitor_data.map(str::to_owned), - ContinuationEndpoint::Browse, - ctx.authenticated, - ), - warnings: map_res.warnings, - }) - } -} - -#[cfg(feature = "userdata")] -impl MapResponse>> for response::MusicContinuation { - fn map_response( - self, - ctx: &MapRespCtx<'_>, - ) -> Result>>, ExtractionError> { - let mut map_res = MapResult::default(); - let mut continuations = Vec::new(); - - let mut map_shelf = |shelf: response::music_item::MusicShelf| { - let mut mapper = MusicListMapper::new(ctx.lang); - mapper.map_response(shelf.contents); - mapper.conv_history_items(shelf.title, ctx.utc_offset, &mut map_res); - continuations.extend(shelf.continuations); - }; - - match self.continuation_contents { - Some(response::music_item::ContinuationContents::MusicShelfContinuation(shelf)) => { - map_shelf(shelf); - } - Some(response::music_item::ContinuationContents::SectionListContinuation(contents)) => { - for c in contents.contents { - if let response::music_item::ItemSection::MusicShelfRenderer(shelf) = c { - map_shelf(shelf); - } - } - } - _ => {} - } - let ctoken = continuations .into_iter() .next() .map(|cont| cont.next_continuation_data.continuation); Ok(MapResult { - c: Paginator::new_ext( - None, - map_res.c, - ctoken, - ctx.visitor_data.map(str::to_owned), - ContinuationEndpoint::MusicBrowse, - ctx.authenticated, - ), + c: Paginator::new(None, map_res.c, ctoken), warnings: map_res.warnings, }) } @@ -325,18 +187,12 @@ impl Paginator { /// Get the next page from the paginator (or `None` if the paginator is exhausted) pub async fn next>(&self, query: Q) -> Result, Error> { Ok(match &self.ctoken { - Some(ctoken) => { - let q = if self.authenticated { - &query.as_ref().clone().authenticated() - } else { - query.as_ref() - }; - - Some( - q.continuation(ctoken, self.endpoint, self.visitor_data.as_deref()) - .await?, - ) - } + Some(ctoken) => Some( + query + .as_ref() + .continuation(ctoken, self.endpoint, self.visitor_data.as_deref()) + .await?, + ), _ => None, }) } @@ -350,9 +206,6 @@ impl Paginator { let mut items = paginator.items; self.items.append(&mut items); self.ctoken = paginator.ctoken; - if paginator.visitor_data.is_some() { - self.visitor_data = paginator.visitor_data; - } Ok(true) } Ok(None) => Ok(false), @@ -395,19 +248,6 @@ impl Paginator { } Ok(()) } - - /// Extend the items of the paginator until the paginator is exhausted. - pub async fn extend_all>(&mut self, query: Q) -> Result<(), Error> { - let query = query.as_ref(); - loop { - match self.extend(query).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } } impl Paginator { @@ -425,40 +265,6 @@ impl Paginator { } } -#[cfg(feature = "userdata")] -#[cfg_attr(docsrs, doc(cfg(feature = "userdata")))] -impl Paginator> { - /// Get the next page from the paginator (or `None` if the paginator is exhausted) - pub async fn next>(&self, query: Q) -> Result, Error> { - Ok(match &self.ctoken { - Some(ctoken) => Some( - query - .as_ref() - .history_continuation(ctoken, self.visitor_data.as_deref()) - .await?, - ), - _ => None, - }) - } -} - -#[cfg(feature = "userdata")] -#[cfg_attr(docsrs, doc(cfg(feature = "userdata")))] -impl Paginator> { - /// Get the next page from the paginator (or `None` if the paginator is exhausted) - pub async fn next>(&self, query: Q) -> Result, Error> { - Ok(match &self.ctoken { - Some(ctoken) => Some( - query - .as_ref() - .music_history_continuation(ctoken, self.visitor_data.as_deref()) - .await?, - ), - _ => None, - }) - } -} - macro_rules! paginator { ($entity_type:ty) => { impl Paginator<$entity_type> { @@ -474,9 +280,6 @@ macro_rules! paginator { let mut items = paginator.items; self.items.append(&mut items); self.ctoken = paginator.ctoken; - if paginator.visitor_data.is_some() { - self.visitor_data = paginator.visitor_data; - } Ok(true) } Ok(None) => Ok(false), @@ -519,33 +322,11 @@ macro_rules! paginator { } Ok(()) } - - /// Extend the items of the paginator until the paginator is exhausted. - pub async fn extend_all>( - &mut self, - query: Q, - ) -> Result<(), Error> { - let query = query.as_ref(); - loop { - match self.extend(query).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } } }; } paginator!(Comment); -#[cfg(feature = "userdata")] -#[cfg_attr(docsrs, doc(cfg(feature = "userdata")))] -paginator!(HistoryItem); -#[cfg(feature = "userdata")] -#[cfg_attr(docsrs, doc(cfg(feature = "userdata")))] -paginator!(HistoryItem); #[cfg(test)] mod tests { @@ -556,15 +337,14 @@ mod tests { use super::*; use crate::{ - model::{ - AlbumItem, ArtistItem, ChannelItem, MusicPlaylistItem, PlaylistItem, TrackItem, - VideoItem, - }, + model::{MusicPlaylistItem, PlaylistItem, TrackItem, VideoItem}, + param::Language, util::tests::TESTFILES, }; #[rstest] #[case::search("search", path!("search" / "cont.json"))] + #[case::startpage("startpage", path!("trends" / "startpage_cont.json"))] #[case::recommendations("recommendations", path!("video_details" / "recommendations.json"))] fn map_continuation_items(#[case] name: &str, #[case] path: PathBuf) { let json_path = path!(*TESTFILES / path); @@ -573,7 +353,7 @@ mod tests { let items: response::Continuation = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - items.map_response(&MapRespCtx::test("")).unwrap(); + items.map_response("", Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -595,9 +375,9 @@ mod tests { let items: response::Continuation = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - items.map_response(&MapRespCtx::test("")).unwrap(); + items.map_response("", Language::En, None).unwrap(); let paginator: Paginator = - map_yt_paginator(map_res.c, ContinuationEndpoint::Browse); + map_yt_paginator(map_res.c, None, ContinuationEndpoint::Browse); assert!( map_res.warnings.is_empty(), @@ -618,30 +398,9 @@ mod tests { let items: response::Continuation = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - items.map_response(&MapRespCtx::test("")).unwrap(); + items.map_response("", Language::En, None).unwrap(); let paginator: Paginator = - map_yt_paginator(map_res.c, ContinuationEndpoint::Browse); - - assert!( - map_res.warnings.is_empty(), - "deserialization/mapping warnings: {:?}", - map_res.warnings - ); - insta::assert_ron_snapshot!(format!("map_{name}"), paginator); - } - - #[rstest] - #[case::subscriptions("subscriptions", path!("userdata" / "subscriptions.json"))] - fn map_continuation_channels(#[case] name: &str, #[case] path: PathBuf) { - let json_path = path!(*TESTFILES / path); - let json_file = File::open(json_path).unwrap(); - - let items: response::Continuation = - serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult> = - items.map_response(&MapRespCtx::test("")).unwrap(); - let paginator: Paginator = - map_yt_paginator(map_res.c, ContinuationEndpoint::Browse); + map_yt_paginator(map_res.c, None, ContinuationEndpoint::Browse); assert!( map_res.warnings.is_empty(), @@ -655,7 +414,6 @@ mod tests { #[case::playlist_tracks("playlist_tracks", path!("music_playlist" / "playlist_cont.json"))] #[case::search_tracks("search_tracks", path!("music_search" / "tracks_cont.json"))] #[case::radio_tracks("radio_tracks", path!("music_details" / "radio_cont.json"))] - #[case::saved_tracks("saved_tracks", path!("music_userdata" / "saved_tracks.json"))] fn map_continuation_tracks(#[case] name: &str, #[case] path: PathBuf) { let json_path = path!(*TESTFILES / path); let json_file = File::open(json_path).unwrap(); @@ -663,51 +421,9 @@ mod tests { let items: response::MusicContinuation = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - items.map_response(&MapRespCtx::test("")).unwrap(); + items.map_response("", Language::En, None).unwrap(); let paginator: Paginator = - map_ytm_paginator(map_res.c, ContinuationEndpoint::MusicBrowse); - - assert!( - map_res.warnings.is_empty(), - "deserialization/mapping warnings: {:?}", - map_res.warnings - ); - insta::assert_ron_snapshot!(format!("map_{name}"), paginator); - } - - #[rstest] - #[case::saved_artists("saved_artists", path!("music_userdata" / "saved_artists.json"))] - fn map_continuation_artists(#[case] name: &str, #[case] path: PathBuf) { - let json_path = path!(*TESTFILES / path); - let json_file = File::open(json_path).unwrap(); - - let items: response::MusicContinuation = - serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult> = - items.map_response(&MapRespCtx::test("")).unwrap(); - let paginator: Paginator = - map_ytm_paginator(map_res.c, ContinuationEndpoint::MusicBrowse); - - assert!( - map_res.warnings.is_empty(), - "deserialization/mapping warnings: {:?}", - map_res.warnings - ); - insta::assert_ron_snapshot!(format!("map_{name}"), paginator); - } - - #[rstest] - #[case::saved_albums("saved_albums", path!("music_userdata" / "saved_albums.json"))] - fn map_continuation_albums(#[case] name: &str, #[case] path: PathBuf) { - let json_path = path!(*TESTFILES / path); - let json_file = File::open(json_path).unwrap(); - - let items: response::MusicContinuation = - serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult> = - items.map_response(&MapRespCtx::test("")).unwrap(); - let paginator: Paginator = - map_ytm_paginator(map_res.c, ContinuationEndpoint::MusicBrowse); + map_ytm_paginator(map_res.c, None, ContinuationEndpoint::MusicBrowse); assert!( map_res.warnings.is_empty(), @@ -719,7 +435,6 @@ mod tests { #[rstest] #[case::playlist_related("playlist_related", path!("music_playlist" / "playlist_related.json"))] - #[case::saved_playlists("saved_playlists", path!("music_userdata" / "saved_playlists.json"))] fn map_continuation_music_playlists(#[case] name: &str, #[case] path: PathBuf) { let json_path = path!(*TESTFILES / path); let json_file = File::open(json_path).unwrap(); @@ -727,9 +442,9 @@ mod tests { let items: response::MusicContinuation = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - items.map_response(&MapRespCtx::test("")).unwrap(); + items.map_response("", Language::En, None).unwrap(); let paginator: Paginator = - map_ytm_paginator(map_res.c, ContinuationEndpoint::MusicBrowse); + map_ytm_paginator(map_res.c, None, ContinuationEndpoint::MusicBrowse); assert!( map_res.warnings.is_empty(), diff --git a/src/client/player.rs b/src/client/player.rs index 9bae601..cda8f70 100644 --- a/src/client/player.rs +++ b/src/client/player.rs @@ -1,49 +1,45 @@ use std::{ borrow::Cow, - collections::{BTreeMap, HashMap, HashSet}, - fmt::Debug, + collections::{BTreeMap, HashMap}, }; use once_cell::sync::Lazy; use regex::Regex; use serde::Serialize; -use time::OffsetDateTime; use url::Url; use crate::{ - deobfuscate::{DeobfData, Deobfuscator}, - error::{internal::DeobfError, AuthError, Error, ExtractionError, UnavailabilityReason}, + deobfuscate::Deobfuscator, + error::{internal::DeobfError, Error, ExtractionError, UnavailabilityReason}, model::{ - traits::QualityOrd, AudioCodec, AudioFormat, AudioStream, AudioTrack, DrmLicense, - DrmSystem, Frameset, Subtitle, VideoCodec, VideoFormat, VideoPlayer, VideoPlayerDetails, - VideoPlayerDrm, VideoStream, + traits::QualityOrd, AudioCodec, AudioFormat, AudioStream, AudioTrack, ChannelId, Frameset, + Subtitle, VideoCodec, VideoFormat, VideoPlayer, VideoPlayerDetails, VideoStream, }, + param::Language, util, }; use super::{ - response::{ - self, - player::{self, Format}, - }, - ClientType, MapRespCtx, MapRespOptions, MapResponse, MapResult, PoToken, RustyPipeQuery, + response::{self, player}, + ClientType, MapResponse, MapResult, RustyPipeQuery, YTContext, }; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QPlayer<'a> { + context: YTContext<'a>, /// Website playback context #[serde(skip_serializing_if = "Option::is_none")] playback_context: Option>, + /// Content playback nonce (mobile only, 16 random chars) + #[serde(skip_serializing_if = "Option::is_none")] + cpn: Option, /// YouTube video ID video_id: &'a str, /// Set to true to allow extraction of streams with sensitive content content_check_ok: bool, /// Probably refers to allowing sensitive content, too racy_check_ok: bool, - /// Botguard data - #[serde(skip_serializing_if = "Option::is_none")] - service_integrity_dimensions: Option, } #[derive(Debug, Serialize)] @@ -61,240 +57,94 @@ struct QContentPlaybackContext<'a> { referer: String, } -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct QDrmLicense<'a> { - drm_system: &'a str, - video_id: &'a str, - cpn: &'a str, - session_id: &'a str, - license_request: &'a str, - drm_params: &'a str, - drm_video_feature: &'a str, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct ServiceIntegrity { - po_token: String, -} - -#[derive(Default)] -struct PlayerPoToken { - visitor_data: Option, - session_po_token: Option, - content_po_token: Option, -} - impl RustyPipeQuery { /// Get YouTube player data (video/audio streams + basic metadata) - pub async fn player + Debug>(&self, video_id: S) -> Result { - self.player_from_clients(video_id, self.player_client_order()) - .await - } - - /// Get YouTube player data (video/audio streams + basic metadata) using a list of clients. - /// - /// The clients are used in the given order. If a client cannot fetch the requested video, - /// an attempt is made with the next one. - pub async fn player_from_clients + Debug>( - &self, - video_id: S, - clients: &[ClientType], - ) -> Result { + pub async fn player>(&self, video_id: S) -> Result { let video_id = video_id.as_ref(); - let mut last_e = None; - let mut query = Cow::Borrowed(self); - let mut clients_iter = clients.iter().peekable(); - let mut failed_clients = HashSet::new(); + let desktop_res = self.player_from_client(video_id, ClientType::Desktop).await; - while let Some(client) = clients_iter.next() { - if query.opts.auth == Some(true) && !self.auth_enabled(*client) { - // If no client has auth enabled, return NoLogin error instead of "no clients" - if last_e.is_none() { - last_e = Some(Error::Auth(AuthError::NoLogin)); - } - continue; - } - if failed_clients.contains(client) { - continue; - } + match desktop_res { + Ok(res) => Ok(res), + Err(Error::Extraction(e)) => { + if e.switch_client() { + let tv_res = self + .player_from_client(video_id, ClientType::TvHtml5Embed) + .await; - let res = query.player_from_client(video_id, *client).await; - match res { - Ok(res) => return Ok(res), - Err(Error::Extraction(e)) => { - if e.use_login() && query.opts.auth.is_none() { - clients_iter = clients.iter().peekable(); - query = Cow::Owned(self.clone().authenticated()); - } else if !e.switch_client() { - return Err(Error::Extraction(e)); + match tv_res { + // Output desktop client error if the tv client is unsupported + Err(Error::Extraction(ExtractionError::VideoUnavailable { + reason: UnavailabilityReason::UnsupportedClient, + .. + })) => Err(Error::Extraction(e)), + _ => tv_res, } - if let Some(next_client) = clients_iter.peek() { - tracing::warn!("error fetching player with {client:?} client: {e}; retrying with {next_client:?} client"); - } - last_e = Some(Error::Extraction(e)); - failed_clients.insert(*client); + } else { + Err(Error::Extraction(e)) } - Err(e) => return Err(e), } - } - Err(last_e.unwrap_or(Error::Other("no clients".into()))) - } - - async fn get_player_po_token(&self, video_id: &str) -> Result { - if let Some(bg) = &self.client.inner.botguard { - let (ident, visitor_data) = if self.opts.auth == Some(true) { - (self.client.user_auth_datasync_id()?, None) - } else { - let visitor_data = self.get_visitor_data(false).await?; - (visitor_data.to_owned(), Some(visitor_data)) - }; - - if bg.po_token_cache { - let session_token = self.get_session_po_token(&ident).await?; - Ok(PlayerPoToken { - visitor_data, - session_po_token: Some(session_token), - content_po_token: None, - }) - } else { - let (po_tokens, valid_until) = self.get_po_tokens(&[video_id, &ident]).await?; - let mut po_tokens = po_tokens.into_iter(); - let po_token = po_tokens.next().unwrap(); - let session_po_token = po_tokens.next().unwrap(); - Ok(PlayerPoToken { - visitor_data, - session_po_token: Some(PoToken { - po_token: session_po_token, - valid_until, - }), - content_po_token: Some(ServiceIntegrity { po_token }), - }) - } - } else { - Ok(PlayerPoToken::default()) + Err(e) => Err(e), } } /// Get YouTube player data (video/audio streams + basic metadata) using the specified client - #[tracing::instrument(skip(self), level = "error")] - pub async fn player_from_client + Debug>( + pub async fn player_from_client>( &self, video_id: S, client_type: ClientType, ) -> Result { - if self.opts.auth == Some(true) { - tracing::info!("fetching {client_type:?} player with login"); - } else { - tracing::debug!("fetching {client_type:?} player"); - } let video_id = video_id.as_ref(); + let (context, deobf) = tokio::join!( + self.get_context(client_type, false, None), + self.client.get_deobf_data() + ); + let deobf = deobf?; - let (deobf, player_po) = tokio::try_join!( - async { - if client_type.needs_deobf() { - Ok::<_, Error>(Some(self.client.get_deobf_data().await?)) - } else { - Ok(None) - } - }, - async { - if client_type.needs_po_token() { - self.get_player_po_token(video_id).await - } else { - Ok(PlayerPoToken::default()) - } + let request_body = if client_type.is_web() { + QPlayer { + context, + playback_context: Some(QPlaybackContext { + content_playback_context: QContentPlaybackContext { + signature_timestamp: &deobf.sts, + referer: format!("https://www.youtube.com/watch?v={video_id}"), + }, + }), + cpn: None, + video_id, + content_check_ok: true, + racy_check_ok: true, + } + } else { + QPlayer { + context, + playback_context: None, + cpn: Some(util::generate_content_playback_nonce()), + video_id, + content_check_ok: true, + racy_check_ok: true, } - )?; - - let playback_context = deobf.as_ref().map(|deobf| QPlaybackContext { - content_playback_context: QContentPlaybackContext { - signature_timestamp: &deobf.sts, - referer: format!("https://www.youtube.com/watch?v={video_id}"), - }, - }); - - let request_body = QPlayer { - playback_context, - video_id, - content_check_ok: true, - racy_check_ok: true, - service_integrity_dimensions: player_po.content_po_token, }; - self.execute_request_ctx::( + self.execute_request_deobf::( client_type, "player", video_id, "player", &request_body, - MapRespOptions { - visitor_data: player_po.visitor_data.as_deref(), - deobf: deobf.as_ref(), - unlocalized: true, - session_po_token: player_po.session_po_token, - ..Default::default() - }, + Some(&deobf), ) .await } - - /// Get the default order of client types when fetching player data - /// - /// The order may change in the future in case YouTube applies changes to their - /// platform that disable a client or make it less reliable. - pub fn player_client_order(&self) -> &'static [ClientType] { - if self.client.inner.botguard.is_some() { - &[ClientType::Desktop, ClientType::Ios, ClientType::Tv] - } else { - &[ClientType::Ios, ClientType::Tv] - } - } - - /// Get a license to play back DRM protected videos - /// - /// Requires authentication (either via OAuth or cookies). - #[tracing::instrument(skip(self), level = "error")] - pub async fn drm_license( - &self, - video_id: &str, - drm_system: DrmSystem, - session_id: &str, - drm_params: &str, - license_request: &[u8], - ) -> Result { - let client_type = self - .auth_enabled_client(&[ClientType::Desktop, ClientType::Tv]) - .ok_or(Error::Auth(AuthError::NoLogin))?; - let request_body = QDrmLicense { - drm_system: drm_system.req_param(), - video_id, - cpn: &util::generate_content_playback_nonce(), - session_id, - license_request: &data_encoding::BASE64.encode(license_request), - drm_params, - drm_video_feature: "DRM_VIDEO_FEATURE_SDR", - }; - - self.clone() - .authenticated() - .execute_request::( - client_type, - "drm_license", - video_id, - "player/get_drm_license", - &request_body, - ) - .await - } } impl MapResponse for response::Player { fn map_response( self, - ctx: &MapRespCtx<'_>, + id: &str, + _lang: Language, + deobf: Option<&crate::deobfuscate::DeobfData>, ) -> Result, ExtractionError> { + let deobf = Deobfuscator::new(deobf.unwrap())?; let mut warnings = vec![]; // Check playability status @@ -307,29 +157,23 @@ impl MapResponse for response::Player { error_screen, } => { let mut msg = reason; - if let Some(error_screen) = error_screen.player_error_message_renderer { + if let Some(error_screen) = error_screen { msg.push_str(" - "); - msg.push_str(&error_screen.subreason); + msg.push_str(&error_screen.player_error_message_renderer.subreason); } - let reason = if error_screen.player_captcha_view_model.is_some() { - UnavailabilityReason::Captcha - } else { - msg.split_whitespace() - .find_map(|word| match word { - "payment" => Some(UnavailabilityReason::Paid), - "Premium" => Some(UnavailabilityReason::Premium), - "members-only" => Some(UnavailabilityReason::MembersOnly), - "country" => Some(UnavailabilityReason::Geoblocked), - "version" | "websites" => Some(UnavailabilityReason::UnsupportedClient), - "bot" => Some(UnavailabilityReason::IpBan), - "VPN/Proxy" => Some(UnavailabilityReason::VpnBan), - "later." => Some(UnavailabilityReason::TryAgain), - _ => None, - }) - .unwrap_or_default() - }; - return Err(ExtractionError::Unavailable { reason, msg }); + let reason = msg + .split_whitespace() + .find_map(|word| match word { + "payment" => Some(UnavailabilityReason::Paid), + "Premium" => Some(UnavailabilityReason::Premium), + "members-only" => Some(UnavailabilityReason::MembersOnly), + "country" => Some(UnavailabilityReason::Geoblocked), + "Android" | "websites" => Some(UnavailabilityReason::UnsupportedClient), + _ => None, + }) + .unwrap_or_default(); + return Err(ExtractionError::VideoUnavailable { reason, msg }); } response::player::PlayabilityStatus::LoginRequired { reason, messages } => { let mut msg = reason; @@ -348,14 +192,13 @@ impl MapResponse for response::Player { .find_map(|word| match word { "age" | "inappropriate" => Some(UnavailabilityReason::AgeRestricted), "private" => Some(UnavailabilityReason::Private), - "bot" => Some(UnavailabilityReason::IpBan), _ => None, }) .unwrap_or_default(); - return Err(ExtractionError::Unavailable { reason, msg }); + return Err(ExtractionError::VideoUnavailable { reason, msg }); } response::player::PlayabilityStatus::LiveStreamOffline { reason } => { - return Err(ExtractionError::Unavailable { + return Err(ExtractionError::VideoUnavailable { reason: UnavailabilityReason::OfflineLivestream, msg: reason, }); @@ -363,14 +206,14 @@ impl MapResponse for response::Player { response::player::PlayabilityStatus::Error { reason } => { // reason (censored): "This video has been removed for violating YouTube's policy on hate speech. Learn more about combating hate speech in your country." // reason: "This video is unavailable" - return Err(ExtractionError::Unavailable { + return Err(ExtractionError::VideoUnavailable { reason: UnavailabilityReason::Deleted, msg: reason, }); } }; - let streaming_data = + let mut streaming_data = self.streaming_data .ok_or(ExtractionError::InvalidData(Cow::Borrowed( "no streaming data", @@ -381,55 +224,77 @@ impl MapResponse for response::Player { "no video details", )))?; - if video_details.video_id != ctx.id { + if video_details.video_id != id { return Err(ExtractionError::WrongResult(format!( "video id {}, expected {}", - video_details.video_id, ctx.id + video_details.video_id, id ))); } - // Sometimes YouTube Desktop does not output any URLs for adaptive streams. - // Since this is currently rare, it is best to retry the request in this case. - if !is_live - && !streaming_data.adaptive_formats.c.is_empty() - && streaming_data - .adaptive_formats - .c - .iter() - .all(|f| f.url.is_none() && f.signature_cipher.is_none()) - { - return Err(ExtractionError::Unavailable { - reason: UnavailabilityReason::TryAgain, - msg: "no adaptive stream URLs".to_owned(), - }); - } let video_info = VideoPlayerDetails { id: video_details.video_id, name: video_details.title, description: video_details.short_description, - duration: video_details.length_seconds, + length: video_details.length_seconds, thumbnail: video_details.thumbnail.into(), - channel_id: video_details.channel_id, - channel_name: video_details.author, + channel: ChannelId { + id: video_details.channel_id, + name: video_details.author, + }, view_count: video_details.view_count, keywords: video_details.keywords, is_live, is_live_content: video_details.is_live_content, }; - let streams = if !is_live { - let mut mapper = StreamsMapper::new( - ctx.deobf, - ctx.session_po_token.as_ref().map(|t| t.po_token.as_str()), - )?; - mapper.map_streams(streaming_data.formats); - mapper.map_streams(streaming_data.adaptive_formats); - let mut res = mapper.output()?; - warnings.append(&mut res.warnings); - res.c - } else { - Streams::default() - }; + let mut formats = streaming_data.formats.c; + formats.append(&mut streaming_data.adaptive_formats.c); + + let mut video_streams: Vec = Vec::new(); + let mut video_only_streams: Vec = Vec::new(); + let mut audio_streams: Vec = Vec::new(); + + if !is_live { + let mut last_nsig: [String; 2] = [String::new(), String::new()]; + + warnings.append(&mut streaming_data.formats.warnings); + warnings.append(&mut streaming_data.adaptive_formats.warnings); + + for f in formats { + if f.format_type == player::FormatType::FormatStreamTypeOtf { + continue; + } + + match (f.is_video(), f.is_audio()) { + (true, true) => { + let mut map_res = map_video_stream(f, &deobf, &mut last_nsig); + warnings.append(&mut map_res.warnings); + if let Some(c) = map_res.c { + video_streams.push(c); + }; + } + (true, false) => { + let mut map_res = map_video_stream(f, &deobf, &mut last_nsig); + warnings.append(&mut map_res.warnings); + if let Some(c) = map_res.c { + video_only_streams.push(c); + }; + } + (false, true) => { + let mut map_res = map_audio_stream(f, &deobf, &mut last_nsig); + warnings.append(&mut map_res.warnings); + if let Some(c) = map_res.c { + audio_streams.push(c); + }; + } + (false, false) => warnings.push(format!("invalid stream: itag {}", f.itag)), + } + } + } + + video_streams.sort_by(QualityOrd::quality_cmp); + video_only_streams.sort_by(QualityOrd::quality_cmp); + audio_streams.sort_by(QualityOrd::quality_cmp); let subtitles = self.captions.map_or(Vec::new(), |captions| { captions @@ -476,8 +341,9 @@ impl MapResponse for response::Player { + "&sigh=" + sigh; - let sprite_count = - util::div_ceil(total_count, frames_per_page_x * frames_per_page_y); + let sprite_count = (f64::from(total_count) + / f64::from(frames_per_page_x * frames_per_page_y)) + .ceil() as u32; Some(Frameset { url_template: url, @@ -495,366 +361,235 @@ impl MapResponse for response::Player { }) .unwrap_or_default(); - let drm = streaming_data - .drm_params - .zip(self.heartbeat_params.drm_session_id) - .map(|(drm_params, drm_session_id)| VideoPlayerDrm { - widevine_service_cert: self - .player_config - .web_drm_config - .and_then(|c| c.widevine_service_cert) - .and_then(|c| data_encoding::BASE64URL.decode(c.as_bytes()).ok()), - drm_params, - authorized_track_types: streaming_data - .initial_authorized_drm_track_types - .into_iter() - .map(|t| t.into()) - .collect(), - drm_session_id, - }); - - let mut valid_until = OffsetDateTime::now_utc() - + time::Duration::seconds(streaming_data.expires_in_seconds.into()); - if let Some(pot) = &ctx.session_po_token { - valid_until = valid_until.min(pot.valid_until); - } - Ok(MapResult { c: VideoPlayer { details: video_info, - video_streams: streams.video_streams, - video_only_streams: streams.video_only_streams, - audio_streams: streams.audio_streams, + video_streams, + video_only_streams, + audio_streams, subtitles, expires_in_seconds: streaming_data.expires_in_seconds, - valid_until, hls_manifest_url: streaming_data.hls_manifest_url, dash_manifest_url: streaming_data.dash_manifest_url, preview_frames, - drm, - client_type: ctx.client_type, - visitor_data: self - .response_context - .visitor_data - .or_else(|| ctx.visitor_data.map(str::to_owned)), + visitor_data: self.response_context.visitor_data, }, warnings, }) } } -struct StreamsMapper<'a> { - deobf: Option, - session_po_token: Option<&'a str>, - streams: Streams, - warnings: Vec, - /// First stream mapping error - first_err: Option, - /// Last obfuscated nsig parameter (cache) - last_nsig: String, - /// Last deobfuscated nsig parameter - last_nsig_deobf: String, +fn cipher_to_url_params( + signature_cipher: &str, + deobf: &Deobfuscator, +) -> Result<(Url, BTreeMap), DeobfError> { + let params: HashMap, Cow> = + url::form_urlencoded::parse(signature_cipher.as_bytes()).collect(); + + // Parameters: + // `s`: Obfuscated signature + // `sp`: Signature parameter + // `url`: URL that is missing the signature parameter + + let sig = params.get("s").ok_or(DeobfError::Extraction("s param"))?; + let sp = params.get("sp").ok_or(DeobfError::Extraction("sp param"))?; + let raw_url = params + .get("url") + .ok_or(DeobfError::Extraction("no url param"))?; + let (url_base, mut url_params) = + util::url_to_params(raw_url).or(Err(DeobfError::Extraction("url params")))?; + + let deobf_sig = deobf.deobfuscate_sig(sig)?; + url_params.insert(sp.to_string(), deobf_sig); + + Ok((url_base, url_params)) } -#[derive(Default)] -struct Streams { - video_streams: Vec, - video_only_streams: Vec, - audio_streams: Vec, -} - -impl<'a> StreamsMapper<'a> { - fn new( - deobf_data: Option<&DeobfData>, - session_po_token: Option<&'a str>, - ) -> Result { - let deobf = match deobf_data { - Some(deobf_data) => Some(Deobfuscator::new(deobf_data)?), - None => None, +fn deobf_nsig( + url_params: &mut BTreeMap, + deobf: &Deobfuscator, + last_nsig: &mut [String; 2], +) -> Result<(), DeobfError> { + let nsig: String; + if let Some(n) = url_params.get("n") { + nsig = if n == &last_nsig[0] { + last_nsig[1].clone() + } else { + let nsig = deobf.deobfuscate_nsig(n)?; + last_nsig[0] = n.to_string(); + last_nsig[1] = nsig.clone(); + nsig }; - Ok(Self { - deobf, - session_po_token, - streams: Streams::default(), - warnings: Vec::new(), - first_err: None, - last_nsig: String::new(), - last_nsig_deobf: String::new(), - }) - } - - fn map_streams(&mut self, mut streams: MapResult>) { - self.warnings.append(&mut streams.warnings); - - let map_e = |m: &mut Self, e: ExtractionError| { - m.warnings.push(e.to_string()); - if m.first_err.is_none() { - m.first_err = Some(e); - } - }; - - for f in streams.c { - if f.format_type == player::FormatType::FormatStreamTypeOtf { - continue; - } - - match (f.is_video(), f.is_audio()) { - (true, true) => match self.map_video_stream(f) { - Ok(c) => self.streams.video_streams.push(c), - Err(e) => map_e(self, e), - }, - (true, false) => match self.map_video_stream(f) { - Ok(c) => self.streams.video_only_streams.push(c), - Err(e) => map_e(self, e), - }, - (false, true) => match self.map_audio_stream(f) { - Ok(c) => self.streams.audio_streams.push(c), - Err(e) => map_e(self, e), - }, - (false, false) => self - .warnings - .push(format!("invalid stream: itag {}", f.itag)), - } - } - } - - fn output(mut self) -> Result, ExtractionError> { - // If we did not extract any streams and there were mapping errors, fail with the first error - if self.streams.video_streams.is_empty() - && (self.streams.video_only_streams.is_empty() || self.streams.audio_streams.is_empty()) - { - if let Some(e) = self.first_err { - return Err(e); - } - } - - self.streams.video_streams.sort_by(QualityOrd::quality_cmp); - self.streams - .video_only_streams - .sort_by(QualityOrd::quality_cmp); - self.streams.audio_streams.sort_by(QualityOrd::quality_cmp); - - Ok(MapResult { - c: self.streams, - warnings: self.warnings, - }) - } - - fn deobf(&self) -> Result<&Deobfuscator, DeobfError> { - self.deobf - .as_ref() - .ok_or(DeobfError::Other("no deobfuscator".into())) - } - - fn cipher_to_url_params( - &self, - signature_cipher: &str, - ) -> Result<(Url, BTreeMap), DeobfError> { - let params: HashMap, Cow> = - url::form_urlencoded::parse(signature_cipher.as_bytes()).collect(); - - // Parameters: - // `s`: Obfuscated signature - // `sp`: Signature parameter - // `url`: URL that is missing the signature parameter - - let sig = params.get("s").ok_or(DeobfError::Extraction("s param"))?; - let sp = params.get("sp").ok_or(DeobfError::Extraction("sp param"))?; - let raw_url = params - .get("url") - .ok_or(DeobfError::Extraction("no url param"))?; - let (url_base, mut url_params) = - util::url_to_params(raw_url).or(Err(DeobfError::Extraction("url params")))?; - - let deobf_sig = self.deobf()?.deobfuscate_sig(sig)?; - url_params.insert(sp.to_string(), deobf_sig); - - Ok((url_base, url_params)) - } - - fn deobf_nsig(&mut self, url_params: &mut BTreeMap) -> Result<(), DeobfError> { - if let Some(n) = url_params.get("n") { - let nsig = if n == &self.last_nsig { - self.last_nsig_deobf.to_owned() - } else { - let nsig = self.deobf()?.deobfuscate_nsig(n)?; - self.last_nsig.clone_from(n); - self.last_nsig_deobf.clone_from(&nsig); - nsig - }; - - url_params.insert("n".to_owned(), nsig); - }; - Ok(()) - } - - fn map_url( - &mut self, - url: &Option, - signature_cipher: &Option, - ) -> Result { - let (url_base, mut url_params) = - match url { - Some(url) => util::url_to_params(url).map_err(|_| { - ExtractionError::InvalidData(format!("Could not parse url `{url}`").into()) - }), - None => match signature_cipher { - Some(signature_cipher) => { - self.cipher_to_url_params(signature_cipher).map_err(|e| { - ExtractionError::InvalidData( - format!("Could not deobfuscate signatureCipher `{signature_cipher}`: {e}") - .into(), - ) - }) - } - None => Err(ExtractionError::InvalidData( - "stream contained neither url or cipher".into(), - )), - }, - }?; - - self.deobf_nsig(&mut url_params)?; - if let Some(pot) = self.session_po_token { - url_params.insert("pot".to_owned(), pot.to_owned()); - } - - let url = Url::parse_with_params(url_base.as_str(), url_params.iter()) - .map_err(|_| ExtractionError::InvalidData("could not combine URL".into()))?; - - Ok(UrlMapRes { - url: url.to_string(), - xtags: url_params.get("xtags").cloned(), - }) - } - - fn map_video_stream(&mut self, f: player::Format) -> Result { - let Some((mtype, codecs)) = parse_mime(&f.mime_type) else { - return Err(ExtractionError::InvalidData( - format!( - "Invalid mime type `{}` in video format {:?}", - &f.mime_type, &f - ) - .into(), - )); - }; - let Some(format) = get_video_format(mtype) else { - return Err(ExtractionError::InvalidData( - format!("invalid video format. itag: {}", f.itag).into(), - )); - }; - let map_res = self.map_url(&f.url, &f.signature_cipher)?; - - Ok(VideoStream { - url: map_res.url, - itag: f.itag, - bitrate: f.bitrate, - average_bitrate: f.average_bitrate.unwrap_or(f.bitrate), - size: f.content_length, - index_range: f.index_range, - init_range: f.init_range, - duration_ms: f.approx_duration_ms, - // Note that the format has already been verified using - // is_video(), so these unwraps are safe - width: f.width.unwrap(), - height: f.height.unwrap(), - fps: f.fps.unwrap(), - quality: f.quality_label.unwrap(), - hdr: f.color_info.unwrap_or_default().primaries - == player::Primaries::ColorPrimariesBt2020, - format, - codec: get_video_codec(codecs), - mime: f.mime_type, - drm_track_type: f.drm_track_type.map(|t| t.into()), - drm_systems: f.drm_families.into_iter().map(|t| t.into()).collect(), - }) - } - - fn map_audio_stream(&mut self, f: player::Format) -> Result { - let Some((mtype, codecs)) = parse_mime(&f.mime_type) else { - return Err(ExtractionError::InvalidData( - format!( - "Invalid mime type `{}` in video format {:?}", - &f.mime_type, &f - ) - .into(), - )); - }; - let format = get_audio_format(mtype).ok_or_else(|| { - ExtractionError::InvalidData(format!("invalid audio format. itag: {}", f.itag).into()) - })?; - let map_res = self.map_url(&f.url, &f.signature_cipher)?; - - Ok(AudioStream { - url: map_res.url, - itag: f.itag, - bitrate: f.bitrate, - average_bitrate: f.average_bitrate.unwrap_or(f.bitrate), - size: f.content_length.ok_or_else(|| { - ExtractionError::InvalidData( - format!("no audio content length. itag: {}", f.itag).into(), - ) - })?, - index_range: f.index_range, - init_range: f.init_range, - duration_ms: f.approx_duration_ms, - format, - codec: get_audio_codec(codecs), - mime: f.mime_type, - channels: f.audio_channels, - loudness_db: f.loudness_db, - track: f - .audio_track - .map(|t| self.map_audio_track(t, map_res.xtags)), - drm_track_type: f.drm_track_type.map(|t| t.into()), - drm_systems: f.drm_families.into_iter().map(|t| t.into()).collect(), - }) - } - - fn map_audio_track( - &mut self, - track: response::player::AudioTrack, - xtags: Option, - ) -> AudioTrack { - let mut lang = None; - let mut track_type = None; - - if let Some(xtags) = xtags { - xtags - .split(':') - .filter_map(|param| param.split_once('=')) - .for_each(|(k, v)| match k { - "lang" => { - lang = Some(v.to_owned()); - } - "acont" => match serde_plain::from_str(v) { - Ok(v) => { - track_type = Some(v); - } - Err(_) => { - self.warnings - .push(format!("could not parse audio track type `{v}`")); - } - }, - _ => {} - }); - } - - AudioTrack { - id: track.id, - lang, - lang_name: track.display_name, - is_default: track.audio_is_default, - track_type, - } - } + url_params.insert("n".to_owned(), nsig); + }; + Ok(()) } struct UrlMapRes { url: String, + throttled: bool, xtags: Option, } +fn map_url( + url: &Option, + signature_cipher: &Option, + deobf: &Deobfuscator, + last_nsig: &mut [String; 2], +) -> MapResult> { + let x = match url { + Some(url) => util::url_to_params(url).map_err(|_| format!("Could not parse url `{url}`")), + None => match signature_cipher { + Some(signature_cipher) => cipher_to_url_params(signature_cipher, deobf).map_err(|e| { + format!("Could not deobfuscate signatureCipher `{signature_cipher}`: {e}") + }), + None => Err("stream contained neither url or cipher".to_owned()), + }, + }; + + let (url_base, mut url_params) = match x { + Ok(x) => x, + Err(e) => { + return MapResult { + c: None, + warnings: vec![e], + } + } + }; + + let mut warnings = vec![]; + let mut throttled = false; + deobf_nsig(&mut url_params, deobf, last_nsig).unwrap_or_else(|e| { + warnings.push(format!( + "Could not deobfuscate nsig (params: {url_params:?}): {e}" + )); + throttled = true; + }); + + match Url::parse_with_params(url_base.as_str(), url_params.iter()) { + Ok(url) => MapResult { + c: Some(UrlMapRes { + url: url.to_string(), + throttled, + xtags: url_params.get("xtags").cloned(), + }), + warnings, + }, + Err(_) => MapResult { + c: None, + warnings: vec![format!( + "url could not be joined. url: `{url_base}` params: {url_params:?}" + )], + }, + } +} + +fn map_video_stream( + f: player::Format, + deobf: &Deobfuscator, + last_nsig: &mut [String; 2], +) -> MapResult> { + let Some((mtype, codecs)) = parse_mime(&f.mime_type) else { + return MapResult { + c: None, + warnings: vec![format!( + "Invalid mime type `{}` in video format {:?}", + &f.mime_type, &f + )], + } + }; + let Some(format) = get_video_format(mtype) else { + return MapResult { + c: None, + warnings: vec![format!("invalid video format. itag: {}", f.itag)], + } + }; + let map_res = map_url(&f.url, &f.signature_cipher, deobf, last_nsig); + + match map_res.c { + Some(url) => MapResult { + c: Some(VideoStream { + url: url.url, + itag: f.itag, + bitrate: f.bitrate, + average_bitrate: f.average_bitrate.unwrap_or(f.bitrate), + size: f.content_length, + index_range: f.index_range, + init_range: f.init_range, + duration_ms: f.approx_duration_ms, + // Note that the format has already been verified using + // is_video(), so these unwraps are safe + width: f.width.unwrap(), + height: f.height.unwrap(), + fps: f.fps.unwrap(), + quality: f.quality_label.unwrap(), + hdr: f.color_info.unwrap_or_default().primaries + == player::Primaries::ColorPrimariesBt2020, + format, + codec: get_video_codec(codecs), + mime: f.mime_type, + throttled: url.throttled, + }), + warnings: map_res.warnings, + }, + None => MapResult { + c: None, + warnings: map_res.warnings, + }, + } +} + +fn map_audio_stream( + f: player::Format, + deobf: &Deobfuscator, + last_nsig: &mut [String; 2], +) -> MapResult> { + let Some((mtype, codecs)) = parse_mime(&f.mime_type) else { + return MapResult { + c: None, + warnings: vec![format!( + "Invalid mime type `{}` in video format {:?}", + &f.mime_type, &f + )], + } + }; + let Some(format) = get_audio_format(mtype) else { + return MapResult { + c: None, + warnings: vec![format!("invalid audio format. itag: {}", f.itag)], + } + }; + let map_res = map_url(&f.url, &f.signature_cipher, deobf, last_nsig); + let mut warnings = map_res.warnings; + + match map_res.c { + Some(url) => MapResult { + c: Some(AudioStream { + url: url.url, + itag: f.itag, + bitrate: f.bitrate, + average_bitrate: f.average_bitrate.unwrap_or(f.bitrate), + size: f.content_length.unwrap(), + index_range: f.index_range, + init_range: f.init_range, + duration_ms: f.approx_duration_ms, + format, + codec: get_audio_codec(codecs), + mime: f.mime_type, + channels: f.audio_channels, + loudness_db: f.loudness_db, + throttled: url.throttled, + track: f + .audio_track + .map(|t| map_audio_track(t, url.xtags, &mut warnings)), + }), + warnings, + }, + None => MapResult { c: None, warnings }, + } +} + fn parse_mime(mime: &str) -> Option<(&str, Vec<&str>)> { static PATTERN: Lazy = Lazy::new(|| Regex::new(r#"(\w+/\w+);\scodecs="([a-zA-Z-0-9.,\s]*)""#).unwrap()); @@ -909,42 +644,45 @@ fn get_audio_codec(codecs: Vec<&str>) -> AudioCodec { return AudioCodec::Mp4a; } else if codec.starts_with("opus") { return AudioCodec::Opus; - } else if codec.starts_with("ac-3") { - return AudioCodec::Ac3; - } else if codec.starts_with("ec-3") { - return AudioCodec::Ec3; } } AudioCodec::Unknown } -impl MapResponse for response::DrmLicense { - fn map_response(self, _ctx: &MapRespCtx<'_>) -> Result, ExtractionError> { - if self.status != "LICENSE_STATUS_OK" { - return Err(ExtractionError::InvalidData(self.status.into())); - } +fn map_audio_track( + track: response::player::AudioTrack, + xtags: Option, + warnings: &mut Vec, +) -> AudioTrack { + let mut lang = None; + let mut track_type = None; - let license = DrmLicense { - license: data_encoding::BASE64URL - .decode(self.license.as_bytes()) - .map_err(|_| ExtractionError::InvalidData("license: invalid b64".into()))?, - authorized_formats: self - .authorized_formats - .into_iter() - .filter_map(|f| { - let key: Option<[u8; 16]> = data_encoding::BASE64URL - .decode(f.key_id.as_bytes()) - .ok() - .and_then(|k| k.try_into().ok()); - key.map(|k| (f.track_type.into(), k)) - }) - .collect(), - }; + if let Some(xtags) = xtags { + xtags + .split(':') + .filter_map(|param| param.split_once('=')) + .for_each(|(k, v)| match k { + "lang" => { + lang = Some(v.to_owned()); + } + "acont" => match serde_plain::from_str(v) { + Ok(v) => { + track_type = Some(v); + } + Err(_) => { + warnings.push(format!("could not parse audio track type `{v}`")); + } + }, + _ => {} + }); + } - Ok(MapResult { - c: license, - warnings: Vec::new(), - }) + AudioTrack { + id: track.id, + lang, + lang_name: track.display_name, + is_default: track.audio_is_default, + track_type, } } @@ -954,10 +692,9 @@ mod tests { use path_macro::path; use rstest::rstest; - use time::UtcOffset; use super::*; - use crate::{deobfuscate::DeobfData, param::Language, util::tests::TESTFILES}; + use crate::{deobfuscate::DeobfData, util::tests::TESTFILES}; static DEOBF_DATA: Lazy = Lazy::new(|| { DeobfData { @@ -969,31 +706,18 @@ mod tests { }); #[rstest] - #[case::desktop(ClientType::Desktop)] - #[case::desktop_music(ClientType::DesktopMusic)] - #[case::tv(ClientType::Tv)] - #[case::android(ClientType::Android)] - #[case::ios(ClientType::Ios)] - fn map_player_data(#[case] client_type: ClientType) { - let name = serde_plain::to_string(&client_type) - .unwrap() - .replace('_', ""); + #[case::desktop("desktop")] + #[case::desktop_music("desktopmusic")] + #[case::tv_html5_embed("tvhtml5embed")] + #[case::android("android")] + #[case::ios("ios")] + fn map_player_data(#[case] name: &str) { let json_path = path!(*TESTFILES / "player" / format!("{name}_video.json")); let json_file = File::open(json_path).unwrap(); let resp: response::Player = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res = resp - .map_response(&MapRespCtx { - id: "pPvd8UxmSbQ", - lang: Language::En, - utc_offset: UtcOffset::UTC, - deobf: Some(&DEOBF_DATA), - visitor_data: None, - client_type, - artist: None, - authenticated: false, - session_po_token: None, - }) + .map_response("pPvd8UxmSbQ", Language::En, Some(&DEOBF_DATA)) .unwrap(); assert!( @@ -1001,20 +725,39 @@ mod tests { "deserialization/mapping warnings: {:?}", map_res.warnings ); + let is_desktop = name == "desktop" || name == "desktopmusic"; insta::assert_ron_snapshot!(format!("map_player_data_{name}"), map_res.c, { - ".valid_until" => "[date]" + ".details.publish_date" => insta::dynamic_redaction(move |value, _path| { + if is_desktop { + assert!(value.as_str().unwrap().starts_with("2019-05-30T00:00:00")); + "2019-05-30T00:00:00" + } else { + assert_eq!(value, insta::internals::Content::None); + "~" + } + }), }); } #[test] fn cipher_to_url() { let signature_cipher = "s=w%3DAe%3DA6aDNQLkViKS7LOm9QtxZJHKwb53riq9qEFw-ecBWJCAiA%3DcEg0tn3dty9jEHszfzh4Ud__bg9CEHVx4ix-7dKsIPAhIQRw8JQ0qOA&sp=sig&url=https://rr5---sn-h0jelnez.googlevideo.com/videoplayback%3Fexpire%3D1659376413%26ei%3Dvb7nYvH5BMK8gAfBj7ToBQ%26ip%3D2003%253Ade%253Aaf06%253A6300%253Ac750%253A1b77%253Ac74a%253A80e3%26id%3Do-AB_BABwrXZJN428ZwDxq5ScPn2AbcGODnRlTVhCQ3mj2%26itag%3D251%26source%3Dyoutube%26requiressl%3Dyes%26mh%3DhH%26mm%3D31%252C26%26mn%3Dsn-h0jelnez%252Csn-4g5ednsl%26ms%3Dau%252Conr%26mv%3Dm%26mvi%3D5%26pl%3D37%26initcwndbps%3D1588750%26spc%3DlT-Khi831z8dTejFIRCvCEwx_6romtM%26vprv%3D1%26mime%3Daudio%252Fwebm%26ns%3Db_Mq_qlTFcSGlG9RpwpM9xQH%26gir%3Dyes%26clen%3D3781277%26dur%3D229.301%26lmt%3D1655510291473933%26mt%3D1659354538%26fvip%3D5%26keepalive%3Dyes%26fexp%3D24001373%252C24007246%26c%3DWEB%26rbqsm%3Dfr%26txp%3D4532434%26n%3Dd2g6G2hVqWIXxedQ%26sparams%3Dexpire%252Cei%252Cip%252Cid%252Citag%252Csource%252Crequiressl%252Cspc%252Cvprv%252Cmime%252Cns%252Cgir%252Cclen%252Cdur%252Clmt%26lsparams%3Dmh%252Cmm%252Cmn%252Cms%252Cmv%252Cmvi%252Cpl%252Cinitcwndbps%26lsig%3DAG3C_xAwRQIgCKCGJ1iu4wlaGXy3jcJyU3inh9dr1FIfqYOZEG_MdmACIQCbungkQYFk7EhD6K2YvLaHFMjKOFWjw001_tLb0lPDtg%253D%253D"; - let mut mapper = StreamsMapper::new(Some(&DEOBF_DATA), None).unwrap(); - let url = mapper - .map_url(&None, &Some(signature_cipher.to_owned())) - .unwrap() - .url; + let mut last_nsig: [String; 2] = ["".to_owned(), "".to_owned()]; + let deobf = Deobfuscator::new(&DEOBF_DATA).unwrap(); + let map_res = map_url( + &None, + &Some(signature_cipher.to_owned()), + &deobf, + &mut last_nsig, + ); + let url = map_res.c.unwrap(); - assert_eq!(url, "https://rr5---sn-h0jelnez.googlevideo.com/videoplayback?c=WEB&clen=3781277&dur=229.301&ei=vb7nYvH5BMK8gAfBj7ToBQ&expire=1659376413&fexp=24001373%2C24007246&fvip=5&gir=yes&id=o-AB_BABwrXZJN428ZwDxq5ScPn2AbcGODnRlTVhCQ3mj2&initcwndbps=1588750&ip=2003%3Ade%3Aaf06%3A6300%3Ac750%3A1b77%3Ac74a%3A80e3&itag=251&keepalive=yes&lmt=1655510291473933&lsig=AG3C_xAwRQIgCKCGJ1iu4wlaGXy3jcJyU3inh9dr1FIfqYOZEG_MdmACIQCbungkQYFk7EhD6K2YvLaHFMjKOFWjw001_tLb0lPDtg%3D%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=hH&mime=audio%2Fwebm&mm=31%2C26&mn=sn-h0jelnez%2Csn-4g5ednsl&ms=au%2Conr&mt=1659354538&mv=m&mvi=5&n=XzXGSfGusw6OCQ&ns=b_Mq_qlTFcSGlG9RpwpM9xQH&pl=37&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRQIhAPIsKd7-xi4xVHEC9gb__dU4hzfzsHEj9ytd3nt0gEceAiACJWBcw-wFEq9qir35bwKHJZxtQ9mOL7SKiVkLQNDa6A%3D%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-Khi831z8dTejFIRCvCEwx_6romtM&txp=4532434&vprv=1"); + assert_eq!(url.url, "https://rr5---sn-h0jelnez.googlevideo.com/videoplayback?c=WEB&clen=3781277&dur=229.301&ei=vb7nYvH5BMK8gAfBj7ToBQ&expire=1659376413&fexp=24001373%2C24007246&fvip=5&gir=yes&id=o-AB_BABwrXZJN428ZwDxq5ScPn2AbcGODnRlTVhCQ3mj2&initcwndbps=1588750&ip=2003%3Ade%3Aaf06%3A6300%3Ac750%3A1b77%3Ac74a%3A80e3&itag=251&keepalive=yes&lmt=1655510291473933&lsig=AG3C_xAwRQIgCKCGJ1iu4wlaGXy3jcJyU3inh9dr1FIfqYOZEG_MdmACIQCbungkQYFk7EhD6K2YvLaHFMjKOFWjw001_tLb0lPDtg%3D%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=hH&mime=audio%2Fwebm&mm=31%2C26&mn=sn-h0jelnez%2Csn-4g5ednsl&ms=au%2Conr&mt=1659354538&mv=m&mvi=5&n=XzXGSfGusw6OCQ&ns=b_Mq_qlTFcSGlG9RpwpM9xQH&pl=37&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRQIhAPIsKd7-xi4xVHEC9gb__dU4hzfzsHEj9ytd3nt0gEceAiACJWBcw-wFEq9qir35bwKHJZxtQ9mOL7SKiVkLQNDa6A%3D%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-Khi831z8dTejFIRCvCEwx_6romtM&txp=4532434&vprv=1"); + assert!(!url.throttled); + assert!( + map_res.warnings.is_empty(), + "deserialization/mapping warnings: {:?}", + map_res.warnings + ); } } diff --git a/src/client/playlist.rs b/src/client/playlist.rs index c080dd6..18340df 100644 --- a/src/client/playlist.rs +++ b/src/client/playlist.rs @@ -1,26 +1,22 @@ -use std::{borrow::Cow, convert::TryFrom, fmt::Debug}; +use std::{borrow::Cow, convert::TryFrom}; use time::OffsetDateTime; use crate::{ error::{Error, ExtractionError}, - model::{ - paginator::{ContinuationEndpoint, Paginator}, - richtext::RichText, - ChannelId, Playlist, VideoItem, - }, - serializer::text::{TextComponent, TextComponents}, - util::{self, dictionary, timeago, TryRemove}, + model::{paginator::Paginator, ChannelId, Playlist, VideoItem}, + util::{self, timeago, TryRemove}, }; -use super::{response, ClientType, MapRespCtx, MapResponse, MapResult, QBrowse, RustyPipeQuery}; +use super::{response, ClientType, MapResponse, MapResult, QBrowse, RustyPipeQuery}; impl RustyPipeQuery { /// Get a YouTube playlist - #[tracing::instrument(skip(self), level = "error")] - pub async fn playlist + Debug>(&self, playlist_id: S) -> Result { + pub async fn playlist>(&self, playlist_id: S) -> Result { let playlist_id = playlist_id.as_ref(); + let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QBrowse { + context, browse_id: &format!("VL{playlist_id}"), }; @@ -36,9 +32,14 @@ impl RustyPipeQuery { } impl MapResponse for response::Playlist { - fn map_response(self, ctx: &MapRespCtx<'_>) -> Result, ExtractionError> { + fn map_response( + self, + id: &str, + lang: crate::param::Language, + _deobf: Option<&crate::deobfuscate::DeobfData>, + ) -> Result, ExtractionError> { let (Some(contents), Some(header)) = (self.contents, self.header) else { - return Err(response::alerts_to_err(ctx.id, self.alerts)); + return Err(response::alerts_to_err(id, self.alerts)); }; let video_items = contents @@ -68,10 +69,10 @@ impl MapResponse for response::Playlist { .playlist_video_list_renderer .contents; - let mut mapper = response::YouTubeListMapper::::new(ctx.lang); + let mut mapper = response::YouTubeListMapper::::new(lang); mapper.map_response(video_items); - let (description, thumbnails, last_update_txt) = match self.sidebar { + let (thumbnails, last_update_txt) = match self.sidebar { Some(sidebar) => { let sidebar_items = sidebar.playlist_sidebar_renderer.contents; let mut primary = @@ -85,155 +86,73 @@ impl MapResponse for response::Playlist { ( primary .playlist_sidebar_primary_info_renderer - .description - .filter(|d| !d.0.is_empty()), - Some( - primary - .playlist_sidebar_primary_info_renderer - .thumbnail_renderer - .playlist_video_thumbnail_renderer - .thumbnail, - ), + .thumbnail_renderer + .playlist_video_thumbnail_renderer + .thumbnail, primary .playlist_sidebar_primary_info_renderer .stats .try_swap_remove(2), ) } - None => (None, None, None), + None => { + let header_banner = header + .playlist_header_renderer + .playlist_header_banner + .ok_or(ExtractionError::InvalidData(Cow::Borrowed( + "no thumbnail found", + )))?; + + let mut byline = header.playlist_header_renderer.byline; + let last_update_txt = byline + .try_swap_remove(1) + .map(|b| b.playlist_byline_renderer.text); + + ( + header_banner.hero_playlist_thumbnail_renderer.thumbnail, + last_update_txt, + ) + } }; - let (name, playlist_id, channel, n_videos_txt, description2, thumbnails2, last_update_txt2) = - match header { - response::playlist::Header::PlaylistHeaderRenderer(header_renderer) => { - let mut byline = header_renderer.byline; - let last_update_txt = byline - .try_swap_remove(1) - .map(|b| b.playlist_byline_renderer.text); - - ( - header_renderer.title, - header_renderer.playlist_id, - header_renderer - .owner_text - .and_then(|link| ChannelId::try_from(link).ok()), - header_renderer.num_videos_text, - header_renderer - .description_text - .map(|text| TextComponents(vec![TextComponent::new(text)])), - header_renderer - .playlist_header_banner - .map(|b| b.hero_playlist_thumbnail_renderer.thumbnail), - last_update_txt, - ) - } - response::playlist::Header::PageHeaderRenderer(content_renderer) => { - let h = content_renderer.content.page_header_view_model; - let rows = h.metadata.content_metadata_view_model.metadata_rows; - let n_videos_txt = rows - .get(1) - .and_then(|r| r.metadata_parts.get(1)) - .map(|p| p.as_str().to_owned()) - .ok_or(ExtractionError::InvalidData("no video count".into()))?; - let mut channel = rows - .into_iter() - .next() - .and_then(|r| r.metadata_parts.into_iter().next()) - .and_then(|p| match p { - response::MetadataPart::Text { .. } => None, - response::MetadataPart::AvatarStack { avatar_stack } => { - ChannelId::try_from(avatar_stack.avatar_stack_view_model.text).ok() - } - }); - // remove "by" prefix - if let Some(c) = channel.as_mut() { - let entry = dictionary::entry(ctx.lang); - let n = c.name.strip_prefix(entry.chan_prefix).unwrap_or(&c.name); - let n = n.strip_suffix(entry.chan_suffix).unwrap_or(n); - c.name = n.trim().to_owned(); - } - - let playlist_id = h - .actions - .flexible_actions_view_model - .actions_rows - .into_iter() - .next() - .and_then(|r| r.actions.into_iter().next()) - .and_then(|a| { - a.button_view_model - .on_tap - .innertube_command - .into_playlist_id() - }) - .ok_or(ExtractionError::InvalidData("no playlist id".into()))?; - ( - h.title.dynamic_text_view_model.text, - playlist_id, - channel, - n_videos_txt, - h.description.description_preview_view_model.description, - h.hero_image.content_preview_image_view_model.image.into(), - None, - ) - } - }; - let n_videos = if mapper.ctoken.is_some() { - util::parse_numeric(&n_videos_txt) - .map_err(|_| ExtractionError::InvalidData("no video count".into()))? + util::parse_numeric(&header.playlist_header_renderer.num_videos_text) + .map_err(|_| ExtractionError::InvalidData(Cow::Borrowed("no video count")))? } else { mapper.items.len() as u64 }; - if playlist_id != ctx.id { + let playlist_id = header.playlist_header_renderer.playlist_id; + if playlist_id != id { return Err(ExtractionError::WrongResult(format!( - "got wrong playlist id {}, expected {}", - playlist_id, ctx.id + "got wrong playlist id {playlist_id}, expected {id}" ))); } - let description = description.or(description2).map(RichText::from); - let thumbnails = thumbnails - .or(thumbnails2) - .ok_or(ExtractionError::InvalidData(Cow::Borrowed( - "no thumbnail found", - )))?; - let last_update = last_update_txt - .as_deref() - .or(last_update_txt2.as_deref()) - .and_then(|txt| { - timeago::parse_textual_date_or_warn( - ctx.lang, - ctx.utc_offset, - txt, - &mut mapper.warnings, - ) + let name = header.playlist_header_renderer.title; + let description = header.playlist_header_renderer.description_text; + let channel = header + .playlist_header_renderer + .owner_text + .and_then(|link| ChannelId::try_from(link).ok()); + + let last_update = last_update_txt.as_ref().and_then(|txt| { + timeago::parse_textual_date_or_warn(lang, txt, &mut mapper.warnings) .map(OffsetDateTime::date) - }); + }); Ok(MapResult { c: Playlist { id: playlist_id, name, - videos: Paginator::new_ext( - Some(n_videos), - mapper.items, - mapper.ctoken, - ctx.visitor_data.map(str::to_owned), - ContinuationEndpoint::Browse, - ctx.authenticated, - ), + videos: Paginator::new(Some(n_videos), mapper.items, mapper.ctoken), video_count: n_videos, thumbnail: thumbnails.into(), description, channel, last_update, last_update_txt, - visitor_data: self - .response_context - .visitor_data - .or_else(|| ctx.visitor_data.map(str::to_owned)), + visitor_data: self.response_context.visitor_data, }, warnings: mapper.warnings, }) @@ -247,7 +166,7 @@ mod tests { use path_macro::path; use rstest::rstest; - use crate::util::tests::TESTFILES; + use crate::{param::Language, util::tests::TESTFILES}; use super::*; @@ -256,15 +175,13 @@ mod tests { #[case::long("long", "PL5dDx681T4bR7ZF1IuWzOv1omlRbE7PiJ")] #[case::nomusic("nomusic", "PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe")] #[case::live("live", "UULVvqRdlKsE5Q8mf8YXbdIJLw")] - #[case::pageheader("20241011_pageheader", "PLT2w2oBf1TZKyvY_M6JsASs73m-wjLzH5")] - #[case::cmdexecutor("20250316_cmdexecutor", "PLbZIPy20-1pN7mqjckepWF78ndb6ci_qi")] fn map_playlist_data(#[case] name: &str, #[case] id: &str) { let json_path = path!(*TESTFILES / "playlist" / format!("playlist_{name}.json")); let json_file = File::open(json_path).unwrap(); let playlist: response::Playlist = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res = playlist.map_response(&MapRespCtx::test(id)).unwrap(); + let map_res = playlist.map_response(id, Language::En, None).unwrap(); assert!( map_res.warnings.is_empty(), diff --git a/src/client/response/channel.rs b/src/client/response/channel.rs index 3ed5de5..826a663 100644 --- a/src/client/response/channel.rs +++ b/src/client/response/channel.rs @@ -2,14 +2,10 @@ use serde::Deserialize; use serde_with::{rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError}; use super::{ - video_item::YouTubeListRenderer, Alert, AttachmentRun, AvatarViewModel, ChannelBadge, - ContentRenderer, ContentsRenderer, ContinuationActionWrap, ImageView, - PageHeaderRendererContent, PhMetadataView, ResponseContext, Thumbnails, TwoColumnBrowseResults, -}; -use crate::{ - model::Verification, - serializer::text::{AttributedText, Text, TextComponent}, + video_item::YouTubeListRenderer, Alert, ChannelBadge, ContentsRenderer, ResponseContext, + Thumbnails, TwoColumnBrowseResults, }; +use crate::serializer::text::Text; #[serde_as] #[derive(Debug, Deserialize)] @@ -40,7 +36,7 @@ pub(crate) struct TabRendererWrap { pub(crate) struct TabRenderer { #[serde(default)] pub content: TabContent, - pub endpoint: Option, + pub endpoint: ChannelTabEndpoint, } #[serde_as] @@ -75,12 +71,10 @@ pub(crate) struct ChannelTabWebCommandMetadata { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] -#[allow(clippy::enum_variant_names)] pub(crate) enum Header { C4TabbedHeaderRenderer(HeaderRenderer), /// Used for special channels like YouTube Music CarouselHeaderRenderer(ContentsRenderer), - PageHeaderRenderer(ContentRenderer>), } #[serde_as] @@ -99,6 +93,11 @@ pub(crate) struct HeaderRenderer { pub badges: Vec, #[serde(default)] pub banner: Thumbnails, + #[serde(default)] + pub mobile_banner: Thumbnails, + /// Fullscreen (16:9) channel banner + #[serde(default)] + pub tv_banner: Thumbnails, } #[serde_as] @@ -118,59 +117,6 @@ pub(crate) enum CarouselHeaderRendererItem { None, } -#[serde_as] -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct PageHeaderRendererInner { - /// Channel title (only used to extract verification badges) - #[serde_as(as = "DefaultOnError")] - pub title: Option, - /// Channel avatar - pub image: PhAvatarView, - /// Channel metadata (subscribers, video count) - pub metadata: PhMetadataView, - #[serde(default)] - pub banner: PhBannerView, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct PhTitleView { - pub dynamic_text_view_model: PhTitleView2, -} - -#[derive(Default, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct PhTitleView2 { - pub text: PhTitleView3, -} - -#[serde_as] -#[derive(Default, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct PhTitleView3 { - #[serde_as(as = "VecSkipError<_>")] - pub attachment_runs: Vec, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct PhAvatarView { - pub decorated_avatar_view_model: PhAvatarView2, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct PhAvatarView2 { - pub avatar: AvatarViewModel, -} - -#[derive(Default, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct PhBannerView { - pub image_banner_view_model: ImageView, -} - #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct Metadata { @@ -199,85 +145,3 @@ pub(crate) struct MicroformatDataRenderer { #[serde(default)] pub tags: Vec, } - -#[serde_as] -#[derive(Debug, Deserialize)] -#[serde(untagged)] -pub(crate) enum ChannelAbout { - #[serde(rename_all = "camelCase")] - ReceivedEndpoints { - #[serde_as(as = "VecSkipError<_>")] - on_response_received_endpoints: Vec>, - }, - Content { - contents: Option, - }, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct AboutChannelRendererWrap { - pub about_channel_renderer: AboutChannelRenderer, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct AboutChannelRenderer { - pub metadata: ChannelMetadata, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct ChannelMetadata { - pub about_channel_view_model: ChannelMetadataView, -} - -#[serde_as] -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct ChannelMetadataView { - pub channel_id: String, - pub canonical_channel_url: String, - pub country: Option, - #[serde(default)] - pub description: String, - #[serde_as(as = "Option")] - pub joined_date_text: Option, - #[serde_as(as = "Option")] - pub subscriber_count_text: Option, - #[serde_as(as = "Option")] - pub video_count_text: Option, - #[serde_as(as = "Option")] - pub view_count_text: Option, - #[serde(default)] - pub links: Vec, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct ExternalLink { - pub channel_external_link_view_model: ExternalLinkInner, -} - -#[serde_as] -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct ExternalLinkInner { - #[serde_as(as = "AttributedText")] - pub title: TextComponent, - #[serde_as(as = "AttributedText")] - pub link: TextComponent, -} - -impl From for crate::model::Verification { - fn from(value: PhTitleView) -> Self { - value - .dynamic_text_view_model - .text - .attachment_runs - .into_iter() - .next() - .map(Verification::from) - .unwrap_or_default() - } -} diff --git a/src/client/response/channel_rss.rs b/src/client/response/channel_rss.rs index 92f28c9..d3e1e6f 100644 --- a/src/client/response/channel_rss.rs +++ b/src/client/response/channel_rss.rs @@ -1,6 +1,8 @@ use serde::Deserialize; use time::OffsetDateTime; +use crate::util; + #[derive(Debug, Deserialize)] pub(crate) struct ChannelRss { #[serde(rename = "channelId")] @@ -78,3 +80,52 @@ impl From for crate::model::Thumbnail { } } } + +impl From for crate::model::ChannelRss { + fn from(feed: ChannelRss) -> Self { + let id = if feed.channel_id.is_empty() { + feed.entry + .iter() + .find_map(|entry| { + Some(entry.channel_id.as_str()) + .filter(|id| id.is_empty()) + .map(str::to_owned) + }) + .or_else(|| { + feed.author + .uri + .strip_prefix("https://www.youtube.com/channel/") + .and_then(|id| { + if util::CHANNEL_ID_REGEX.is_match(id) { + Some(id.to_owned()) + } else { + None + } + }) + }) + .unwrap_or_default() + } else { + feed.channel_id + }; + + Self { + id, + name: feed.title, + videos: feed + .entry + .into_iter() + .map(|item| crate::model::ChannelRssVideo { + id: item.video_id, + name: item.title, + description: item.media_group.description, + thumbnail: item.media_group.thumbnail.into(), + publish_date: item.published, + update_date: item.updated, + view_count: item.media_group.community.statistics.views, + like_count: item.media_group.community.rating.count, + }) + .collect(), + create_date: feed.create_date, + } + } +} diff --git a/src/client/response/history.rs b/src/client/response/history.rs deleted file mode 100644 index 11d88af..0000000 --- a/src/client/response/history.rs +++ /dev/null @@ -1,8 +0,0 @@ -use serde::Deserialize; - -use super::{video_item::YouTubeListRendererWrap, Tab, TwoColumnBrowseResults}; - -#[derive(Debug, Deserialize)] -pub(crate) struct History { - pub contents: TwoColumnBrowseResults>, -} diff --git a/src/client/response/mod.rs b/src/client/response/mod.rs index e160826..2538963 100644 --- a/src/client/response/mod.rs +++ b/src/client/response/mod.rs @@ -16,7 +16,6 @@ pub(crate) mod video_details; pub(crate) mod video_item; pub(crate) use channel::Channel; -pub(crate) use channel::ChannelAbout; pub(crate) use music_artist::MusicArtist; pub(crate) use music_artist::MusicArtistAlbums; pub(crate) use music_charts::MusicCharts; @@ -30,11 +29,11 @@ pub(crate) use music_new::MusicNew; pub(crate) use music_playlist::MusicPlaylist; pub(crate) use music_search::MusicSearch; pub(crate) use music_search::MusicSearchSuggestion; -pub(crate) use player::DrmLicense; pub(crate) use player::Player; pub(crate) use playlist::Playlist; pub(crate) use search::Search; pub(crate) use search::SearchSuggestion; +pub(crate) use trends::Startpage; pub(crate) use trends::Trending; pub(crate) use url_endpoint::ResolvedUrl; pub(crate) use video_details::VideoComments; @@ -47,28 +46,17 @@ pub(crate) mod channel_rss; #[cfg(feature = "rss")] pub(crate) use channel_rss::ChannelRss; -#[cfg(feature = "userdata")] -pub(crate) mod history; -#[cfg(feature = "userdata")] -pub(crate) use history::History; -#[cfg(feature = "userdata")] -pub(crate) mod music_history; -#[cfg(feature = "userdata")] -pub(crate) use music_history::MusicHistory; - use std::borrow::Cow; -use std::collections::HashMap; use std::marker::PhantomData; use serde::{ de::{IgnoredAny, Visitor}, Deserialize, }; -use serde_with::{serde_as, DisplayFromStr, VecSkipError}; +use serde_with::{json::JsonString, serde_as, VecSkipError}; use crate::error::ExtractionError; -use crate::serializer::text::{AttributedText, Text, TextComponent}; -use crate::serializer::{MapResult, VecSkipErrorWrap}; +use crate::serializer::{text::Text, MapResult, VecSkipErrorWrap}; use self::video_item::YouTubeListRenderer; @@ -78,9 +66,6 @@ pub(crate) struct ContentRenderer { pub content: T, } -/// Deserializes any object with an array field named `contents`, `tabs` or `items`. -/// -/// Invalid items are skipped #[derive(Debug)] pub(crate) struct ContentsRenderer { pub contents: Vec, @@ -117,24 +102,12 @@ pub(crate) struct ThumbnailsWrap { pub thumbnail: Thumbnails, } -#[derive(Default, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct ImageView { - pub image: Thumbnails, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct AvatarViewModel { - pub avatar_view_model: ImageView, -} - /// List of images in different resolutions. /// Not only used for thumbnails, but also for avatars and banners. #[derive(Default, Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct Thumbnails { - #[serde(default, alias = "sources")] + #[serde(default)] pub thumbnails: Vec, } @@ -152,16 +125,9 @@ pub(crate) struct ContinuationItemRenderer { pub continuation_endpoint: ContinuationEndpoint, } -#[derive(Debug, Deserialize)] -#[serde(untagged)] -pub(crate) enum ContinuationEndpoint { - ContinuationCommand(ContinuationCommandWrap), - CommandExecutorCommand(CommandExecutorCommandWrap), -} - #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] -pub(crate) struct ContinuationCommandWrap { +pub(crate) struct ContinuationEndpoint { pub continuation_command: ContinuationCommand, } @@ -171,34 +137,7 @@ pub(crate) struct ContinuationCommand { pub token: String, } -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct CommandExecutorCommandWrap { - pub command_executor_command: CommandExecutorCommand, -} - #[serde_as] -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct CommandExecutorCommand { - #[serde_as(as = "VecSkipError<_>")] - commands: Vec, -} - -impl ContinuationEndpoint { - pub fn into_token(self) -> Option { - match self { - Self::ContinuationCommand(cmd) => Some(cmd.continuation_command.token), - Self::CommandExecutorCommand(cmd) => cmd - .command_executor_command - .commands - .into_iter() - .next() - .map(|c| c.continuation_command.token), - } - } -} - #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct Icon { @@ -238,92 +177,23 @@ pub(crate) enum ChannelBadgeStyle { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct Alert { - pub alert_renderer: TextBox, + pub alert_renderer: AlertRenderer, } #[serde_as] #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] -pub(crate) struct TextBox { +pub(crate) struct AlertRenderer { #[serde_as(as = "Text")] pub text: String, } -#[serde_as] -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct SimpleHeaderRenderer { - #[serde_as(as = "Text")] - pub title: String, -} - -#[serde_as] -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct TextComponentBox { - #[serde_as(as = "AttributedText")] - pub text: TextComponent, -} - #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct ResponseContext { pub visitor_data: Option, } -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct AttachmentRun { - pub element: AttachmentRunElement, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct AttachmentRunElement { - #[serde(rename = "type")] - pub typ: AttachmentRunElementType, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct AttachmentRunElementType { - pub image_type: AttachmentRunElementImageType, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct AttachmentRunElementImageType { - pub image: AttachmentRunElementImage, -} - -#[serde_as] -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct AttachmentRunElementImage { - #[serde_as(as = "VecSkipError<_>")] - pub sources: Vec, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct AttachmentRunElementImageSource { - pub client_resource: ClientResource, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct ClientResource { - pub image_name: IconName, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub enum IconName { - CheckCircleFilled, - #[serde(alias = "AUDIO_BADGE")] - MusicFilled, -} - // CONTINUATION #[serde_as] @@ -331,14 +201,14 @@ pub enum IconName { #[serde(rename_all = "camelCase")] pub(crate) struct Continuation { /// Number of search results - #[serde_as(as = "Option")] + #[serde_as(as = "Option")] pub estimated_results: Option, #[serde( alias = "onResponseReceivedCommands", alias = "onResponseReceivedEndpoints" )] #[serde_as(as = "Option>")] - pub on_response_received_actions: Option>>, + pub on_response_received_actions: Option>, /// Used for channel video rich grid renderer /// /// A/B test seen on 19.10.2022 @@ -347,15 +217,15 @@ pub(crate) struct Continuation { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] -pub(crate) struct ContinuationActionWrap { +pub(crate) struct ContinuationActionWrap { #[serde(alias = "reloadContinuationItemsCommand")] - pub append_continuation_items_action: ContinuationAction, + pub append_continuation_items_action: ContinuationAction, } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] -pub(crate) struct ContinuationAction { - pub continuation_items: MapResult>, +pub(crate) struct ContinuationAction { + pub continuation_items: MapResult>, } #[derive(Debug, Deserialize)] @@ -462,27 +332,14 @@ impl From for Vec { } } -impl ContentImage { - pub(crate) fn into_image(self) -> ImageViewOl { - match self { - ContentImage::ThumbnailViewModel(image) => image, - ContentImage::CollectionThumbnailViewModel { primary_thumbnail } => { - primary_thumbnail.thumbnail_view_model - } - } - } -} - impl From> for crate::model::Verification { fn from(badges: Vec) -> Self { - badges - .first() - .map_or(crate::model::Verification::None, |b| { - match b.metadata_badge_renderer.style { - ChannelBadgeStyle::BadgeStyleTypeVerified => Self::Verified, - ChannelBadgeStyle::BadgeStyleTypeVerifiedArtist => Self::Artist, - } - }) + badges.get(0).map_or(crate::model::Verification::None, |b| { + match b.metadata_badge_renderer.style { + ChannelBadgeStyle::BadgeStyleTypeVerified => Self::Verified, + ChannelBadgeStyle::BadgeStyleTypeVerifiedArtist => Self::Artist, + } + }) } } @@ -496,25 +353,6 @@ impl From for crate::model::Verification { } } -impl From for crate::model::Verification { - fn from(value: AttachmentRun) -> Self { - match value - .element - .typ - .image_type - .image - .sources - .into_iter() - .next() - .map(|s| s.client_resource.image_name) - { - Some(IconName::CheckCircleFilled) => Self::Verified, - Some(IconName::MusicFilled) => Self::Artist, - None => Self::None, - } - } -} - pub(crate) fn alerts_to_err(id: &str, alerts: Option>) -> ExtractionError { ExtractionError::NotFound { id: id.to_owned(), @@ -530,201 +368,3 @@ pub(crate) fn alerts_to_err(id: &str, alerts: Option>) -> ExtractionE .unwrap_or_default(), } } - -// FRAMEWORK UPDATES - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct FrameworkUpdates { - pub entity_batch_update: EntityBatchUpdate, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct EntityBatchUpdate { - pub mutations: FrameworkUpdateMutations, -} - -/// List of update mutations that deserializes into a HashMap (entity_key => payload) -#[derive(Debug)] -pub(crate) struct FrameworkUpdateMutations { - pub items: HashMap, - pub warnings: Vec, -} - -impl<'de, T> Deserialize<'de> for FrameworkUpdateMutations -where - T: Deserialize<'de>, -{ - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - struct SeqVisitor(PhantomData); - - #[derive(serde::Deserialize)] - #[serde(untagged)] - enum MutationOrError { - #[serde(rename_all = "camelCase")] - Good { - entity_key: String, - payload: T, - }, - Error(serde_json::Value), - } - - impl<'de, T> Visitor<'de> for SeqVisitor - where - T: Deserialize<'de>, - { - type Value = FrameworkUpdateMutations; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - formatter.write_str("sequence of entity mutations") - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let mut items = HashMap::with_capacity(seq.size_hint().unwrap_or_default()); - let mut warnings = Vec::new(); - - while let Some(value) = seq.next_element::>()? { - match value { - MutationOrError::Good { - entity_key, - payload, - } => { - items.insert(entity_key, payload); - } - MutationOrError::Error(value) => { - warnings.push(format!( - "error deserializing item: {}", - serde_json::to_string(&value).unwrap_or_default() - )); - } - } - } - - Ok(FrameworkUpdateMutations { items, warnings }) - } - } - - deserializer.deserialize_seq(SeqVisitor(PhantomData::)) - } -} - -// PAGE HEADER - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct PageHeaderRendererContent { - pub page_header_view_model: T, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct PhMetadataView { - pub content_metadata_view_model: PhMetadataView2, -} - -#[serde_as] -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct PhMetadataView2 { - #[serde_as(as = "VecSkipError<_>")] - pub metadata_rows: Vec, -} - -#[serde_as] -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct PhMetadataRow { - #[serde_as(as = "VecSkipError<_>")] - pub metadata_parts: Vec, -} - -#[serde_as] -#[derive(Debug, Deserialize)] -#[serde(untagged)] -pub(crate) enum MetadataPart { - Text { - #[serde_as(as = "AttributedText")] - text: TextComponent, - }, - #[serde(rename_all = "camelCase")] - AvatarStack { avatar_stack: AvatarStackInner }, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct AvatarStackInner { - pub avatar_stack_view_model: TextComponentBox, -} - -impl MetadataPart { - pub fn into_text_component(self) -> TextComponent { - match self { - MetadataPart::Text { text } => text, - MetadataPart::AvatarStack { avatar_stack } => avatar_stack.avatar_stack_view_model.text, - } - } - - pub fn as_str(&self) -> &str { - match self { - MetadataPart::Text { text } => text.as_str(), - MetadataPart::AvatarStack { avatar_stack } => { - avatar_stack.avatar_stack_view_model.text.as_str() - } - } - } -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) enum ContentImage { - ThumbnailViewModel(ImageViewOl), - #[serde(rename_all = "camelCase")] - CollectionThumbnailViewModel { - primary_thumbnail: ThumbnailViewModelWrap, - }, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct ThumbnailViewModelWrap { - pub thumbnail_view_model: ImageViewOl, -} - -#[serde_as] -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct ImageViewOl { - pub image: Thumbnails, - #[serde_as(as = "VecSkipError<_>")] - pub overlays: Vec, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct ImageViewOverlay { - pub thumbnail_overlay_badge_view_model: ThumbnailOverlayBadgeViewModel, -} - -#[serde_as] -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct ThumbnailOverlayBadgeViewModel { - #[serde_as(as = "VecSkipError<_>")] - pub thumbnail_badges: Vec, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct ThumbnailBadges { - pub thumbnail_badge_view_model: TextBox, -} - -#[derive(Debug, Deserialize)] -pub(crate) struct Empty {} diff --git a/src/client/response/music_artist.rs b/src/client/response/music_artist.rs index d510cf9..af4efaf 100644 --- a/src/client/response/music_artist.rs +++ b/src/client/response/music_artist.rs @@ -5,8 +5,7 @@ use crate::serializer::text::Text; use super::{ music_item::{ - Button, Grid, ItemSection, MusicMicroformat, MusicThumbnailRenderer, SimpleHeader, - SingleColumnBrowseResult, + Button, Grid, ItemSection, MusicThumbnailRenderer, SimpleHeader, SingleColumnBrowseResult, }, SectionList, Tab, }; @@ -15,10 +14,8 @@ use super::{ #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct MusicArtist { - pub contents: Option>>>, - pub header: Option
, - #[serde(default)] - pub microformat: MusicMicroformat, + pub contents: SingleColumnBrowseResult>>>, + pub header: Header, } #[derive(Debug, Deserialize)] diff --git a/src/client/response/music_details.rs b/src/client/response/music_details.rs index a27c8c3..0d7e6d2 100644 --- a/src/client/response/music_details.rs +++ b/src/client/response/music_details.rs @@ -1,13 +1,14 @@ use serde::Deserialize; -use serde_with::{serde_as, DefaultOnError, VecSkipError}; +use serde_with::serde_as; +use serde_with::DefaultOnError; use crate::serializer::text::Text; +use super::AlertRenderer; use super::ContentsRenderer; -use super::TextBox; use super::{ music_item::{ItemSection, PlaylistPanelRenderer}, - ContentRenderer, + ContentRenderer, SectionList, }; /// Response model for YouTube Music track details @@ -35,11 +36,9 @@ pub(crate) struct TabbedRenderer { pub watch_next_tabbed_results_renderer: TabbedRendererInner, } -#[serde_as] #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct TabbedRendererInner { - #[serde_as(as = "VecSkipError<_>")] pub tabs: Vec, } @@ -108,14 +107,14 @@ pub(crate) struct PlaylistPanel { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct MusicLyrics { - pub contents: ListOrMessage, + pub contents: LyricsContents, } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] -pub(crate) enum ListOrMessage { - SectionListRenderer(ContentsRenderer), - MessageRenderer(TextBox), +pub(crate) struct LyricsContents { + pub message_renderer: Option, + pub section_list_renderer: Option>, } #[derive(Debug, Deserialize)] @@ -137,14 +136,5 @@ pub(crate) struct LyricsRenderer { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct MusicRelated { - pub contents: ListOrMessage, -} - -impl ListOrMessage { - pub fn into_res(self) -> Result, String> { - match self { - ListOrMessage::SectionListRenderer(c) => Ok(c.contents), - ListOrMessage::MessageRenderer(msg) => Err(msg.text), - } - } + pub contents: SectionList, } diff --git a/src/client/response/music_history.rs b/src/client/response/music_history.rs deleted file mode 100644 index 888113c..0000000 --- a/src/client/response/music_history.rs +++ /dev/null @@ -1,8 +0,0 @@ -use serde::Deserialize; - -use super::music_playlist::Contents; - -#[derive(Debug, Deserialize)] -pub(crate) struct MusicHistory { - pub contents: Contents, -} diff --git a/src/client/response/music_item.rs b/src/client/response/music_item.rs index 5d9907d..3edd74e 100644 --- a/src/client/response/music_item.rs +++ b/src/client/response/music_item.rs @@ -2,31 +2,25 @@ use serde::Deserialize; use serde_with::{rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError}; use crate::{ + error::ExtractionError, model::{ self, traits::FromYtItem, AlbumId, AlbumItem, AlbumType, ArtistId, ArtistItem, ChannelId, - MusicItem, MusicItemType, MusicPlaylistItem, TrackItem, UserItem, + MusicItem, MusicItemType, MusicPlaylistItem, TrackItem, }, param::Language, serializer::{ - text::{Text, TextComponent, TextComponents}, + text::{Text, TextComponents}, MapResult, }, - util::{self, dictionary, timeago}, + util::{self, dictionary}, }; use super::{ - url_endpoint::{ - BrowseEndpointWrap, MusicPage, MusicPageType, MusicVideoType, NavigationEndpoint, PageType, - }, - ContentsRenderer, ContinuationActionWrap, ContinuationEndpoint, MusicContinuationData, - SimpleHeaderRenderer, Thumbnails, ThumbnailsWrap, + url_endpoint::{BrowseEndpointWrap, MusicPageType, NavigationEndpoint, PageType}, + ContentsRenderer, MusicContinuationData, Thumbnails, ThumbnailsWrap, }; -#[cfg(feature = "userdata")] -use crate::model::HistoryItem; -#[cfg(feature = "userdata")] -use time::UtcOffset; - +#[serde_as] #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) enum ItemSection { @@ -44,9 +38,6 @@ pub(crate) enum ItemSection { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct MusicShelf { - #[cfg(feature = "userdata")] - #[serde_as(as = "Option")] - pub title: Option, /// Playlist ID (only for playlists) pub playlist_id: Option, pub contents: MapResult>, @@ -88,15 +79,9 @@ pub(crate) struct MusicCardShelf { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] -#[allow(clippy::enum_variant_names)] pub(crate) enum MusicResponseItem { MusicResponsiveListItemRenderer(ListMusicItem), MusicTwoRowItemRenderer(CoverMusicItem), - MessageRenderer(serde::de::IgnoredAny), - #[serde(rename_all = "camelCase")] - ContinuationItemRenderer { - continuation_endpoint: ContinuationEndpoint, - }, } #[serde_as] @@ -181,9 +166,6 @@ pub(crate) struct ListMusicItem { #[serde_as(as = "Option")] pub index: Option, pub menu: Option, - #[serde(default)] - #[serde_as(deserialize_as = "VecSkipError<_>")] - pub badges: Vec, } #[derive(Default, Debug, Copy, Clone, Deserialize)] @@ -284,7 +266,7 @@ pub(crate) struct QueueMusicItem { #[derive(Default, Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct MusicThumbnailRenderer { - #[serde(default, alias = "croppedSquareThumbnailRenderer")] + #[serde(alias = "croppedSquareThumbnailRenderer")] pub music_thumbnail_renderer: ThumbnailsWrap, } @@ -333,14 +315,10 @@ impl From for Vec { } /// Music list continuation response model -#[serde_as] #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct MusicContinuation { pub continuation_contents: Option, - #[serde(default)] - #[serde_as(as = "VecSkipError<_>")] - pub on_response_received_actions: Vec>, } #[derive(Debug, Deserialize)] @@ -351,7 +329,6 @@ pub(crate) enum ContinuationContents { MusicShelfContinuation(MusicShelf), SectionListContinuation(ContentsRenderer), PlaylistPanelContinuation(PlaylistPanelRenderer), - GridContinuation(GridRenderer), } #[derive(Debug, Deserialize)] @@ -398,21 +375,25 @@ pub(crate) struct Grid { pub grid_renderer: GridRenderer, } -#[serde_as] #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct GridRenderer { pub items: MapResult>, pub header: Option, - #[serde(default)] - #[serde_as(as = "VecSkipError<_>")] - pub continuations: Vec, } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct GridHeader { - pub grid_header_renderer: SimpleHeaderRenderer, + pub grid_header_renderer: GridHeaderRenderer, +} + +#[serde_as] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct GridHeaderRenderer { + #[serde_as(as = "Text")] + pub title: String, } #[derive(Debug, Deserialize)] @@ -427,26 +408,12 @@ pub(crate) struct SimpleHeader { pub music_header_renderer: SimpleHeaderRenderer, } +#[serde_as] #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] -pub(crate) enum TrackBadge { - LiveBadgeRenderer {}, -} - -#[serde_as] -#[derive(Default, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct MusicMicroformat { - #[serde_as(as = "DefaultOnError")] - pub microformat_data_renderer: MicroformatData, -} - -#[derive(Default, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct MicroformatData { - pub url_canonical: Option, - #[serde(default)] - pub noindex: bool, +pub(crate) struct SimpleHeaderRenderer { + #[serde_as(as = "Text")] + pub title: String, } /* @@ -459,13 +426,11 @@ pub(crate) struct MusicListMapper { /// Artists list + various artists flag artists: Option<(Vec, bool)>, album: Option, - /// Default album type in case an album is unlabeled - pub album_type: AlbumType, artist_page: bool, - search_suggestion: bool, items: Vec, warnings: Vec, - pub ctoken: Option, + /// True if unknown items were mapped + has_unknown: bool, } #[derive(Debug)] @@ -482,26 +447,10 @@ impl MusicListMapper { lang, artists: None, album: None, - album_type: AlbumType::Single, artist_page: false, - search_suggestion: false, items: Vec::new(), warnings: Vec::new(), - ctoken: None, - } - } - - pub fn new_search_suggest(lang: Language) -> Self { - Self { - lang, - artists: None, - album: None, - album_type: AlbumType::Single, - artist_page: false, - search_suggestion: true, - items: Vec::new(), - warnings: Vec::new(), - ctoken: None, + has_unknown: false, } } @@ -511,12 +460,10 @@ impl MusicListMapper { lang, artists: Some((vec![artist], false)), album: None, - album_type: AlbumType::Single, artist_page: true, - search_suggestion: false, items: Vec::new(), warnings: Vec::new(), - ctoken: None, + has_unknown: false, } } @@ -526,30 +473,442 @@ impl MusicListMapper { lang, artists: Some((artists, by_va)), album: Some(album), - album_type: AlbumType::Single, artist_page: false, - search_suggestion: false, items: Vec::new(), warnings: Vec::new(), - ctoken: None, + has_unknown: false, } } - /// Map a MusicResponseItem (list item or tile) fn map_item(&mut self, item: MusicResponseItem) -> Result, String> { match item { // List item - MusicResponseItem::MusicResponsiveListItemRenderer(item) => self.map_list_item(item), - // Tile - MusicResponseItem::MusicTwoRowItemRenderer(item) => self.map_tile(item), - MusicResponseItem::MessageRenderer(_) => Ok(None), - MusicResponseItem::ContinuationItemRenderer { - continuation_endpoint, - } => { - if self.ctoken.is_none() { - self.ctoken = continuation_endpoint.into_token(); + MusicResponseItem::MusicResponsiveListItemRenderer(item) => { + let mut columns = item.flex_columns.into_iter(); + let c1 = columns.next(); + let c2 = columns.next(); + let c3 = columns.next(); + + let title = c1.as_ref().map(|col| col.renderer.text.to_string()); + + let first_tn = item + .thumbnail + .music_thumbnail_renderer + .thumbnail + .thumbnails + .first(); + + let pt_id = item + .navigation_endpoint + .and_then(NavigationEndpoint::music_page) + .or_else(|| { + c1.and_then(|c1| { + c1.renderer.text.0.into_iter().next().and_then(|t| match t { + crate::serializer::text::TextComponent::Video { + video_id, + is_video, + .. + } => Some((MusicPageType::Track { is_video }, video_id)), + crate::serializer::text::TextComponent::Browse { + page_type, + browse_id, + .. + } => Some((page_type.into(), browse_id)), + _ => None, + }) + }) + }) + .or_else(|| { + item.playlist_item_data.map(|d| { + ( + MusicPageType::Track { + is_video: self.album.is_none() + && !first_tn + .map(|tn| tn.height == tn.width) + .unwrap_or_default(), + }, + d.video_id, + ) + }) + }) + .or_else(|| { + first_tn.and_then(|tn| { + util::video_id_from_thumbnail_url(&tn.url).map(|id| { + ( + MusicPageType::Track { + is_video: self.album.is_none() && tn.width != tn.height, + }, + id, + ) + }) + }) + }); + + match pt_id { + // Track + Some((MusicPageType::Track { is_video }, id)) => { + let title = + title.ok_or_else(|| format!("track {id}: could not get title"))?; + + let (artists_p, album_p, duration_p) = match item.flex_column_display_style + { + // Search result + FlexColumnDisplayStyle::TwoLines => { + // Is this a related track? + if !is_video && item.item_height == ItemHeight::Compact { + ( + c2.map(TextComponents::from), + c3.map(TextComponents::from), + None, + ) + } else { + let mut subtitle_parts = c2 + .ok_or_else(|| { + format!("track {id}: could not get subtitle") + })? + .renderer + .text + .split(util::DOT_SEPARATOR) + .into_iter(); + + // Is this a related video? + if item.item_height == ItemHeight::Compact { + (subtitle_parts.next(), subtitle_parts.next(), None) + } + // Is it a podcast episode? + else if subtitle_parts.len() <= 3 && c3.is_some() { + (subtitle_parts.rev().next(), None, None) + } else { + // Skip first part (track type) + if subtitle_parts.len() > 3 + || (is_video && subtitle_parts.len() == 2) + { + subtitle_parts.next(); + } + + ( + subtitle_parts.next(), + subtitle_parts.next(), + subtitle_parts.next(), + ) + } + } + } + // Playlist item + FlexColumnDisplayStyle::Default => ( + c2.map(TextComponents::from), + c3.map(TextComponents::from), + item.fixed_columns + .into_iter() + .next() + .map(TextComponents::from), + ), + }; + + let duration = + duration_p.and_then(|p| util::parse_video_length(p.first_str())); + + let (album, view_count) = match (item.flex_column_display_style, is_video) { + // The album field contains the view count for search videos + (FlexColumnDisplayStyle::TwoLines, true) => ( + None, + album_p.and_then(|p| { + util::parse_large_numstr_or_warn( + p.first_str(), + self.lang, + &mut self.warnings, + ) + }), + ), + (_, false) => ( + album_p.and_then(|p| { + p.0.into_iter().find_map(|c| AlbumId::try_from(c).ok()) + }), + None, + ), + (FlexColumnDisplayStyle::Default, true) => (None, None), + }; + let album = album.or_else(|| self.album.clone()); + + let (mut artists, by_va) = map_artists(artists_p); + + // Extract artist id from dropdown menu + let artist_id = map_artist_id_fallback(item.menu, artists.first()); + + // Fall back to the artist given when constructing the mapper. + // This is used for extracting artist pages. + // On some albums, the artist name of the tracks is not given but different + // from the album artist. In this case dont copy the album artist. + if let Some((fb_artists, _)) = &self.artists { + if artists.is_empty() + && (self.artist_page + || artist_id.is_none() + || fb_artists.iter().any(|fb_id| { + fb_id + .id + .as_deref() + .map(|aid| artist_id.as_deref() == Some(aid)) + .unwrap_or_default() + })) + { + artists = fb_artists.clone(); + } + } + + let track_nr = item.index.and_then(|txt| util::parse_numeric(&txt).ok()); + + self.items.push(MusicItem::Track(TrackItem { + id, + name: title, + duration, + cover: item.thumbnail.into(), + artists, + artist_id, + album, + view_count, + is_video, + track_nr, + by_va, + })); + Ok(Some(MusicItemType::Track)) + } + // Artist / Album / Playlist + Some((page_type, id)) => { + let mut subtitle_parts = c2 + .ok_or_else(|| "could not get subtitle".to_owned())? + .renderer + .text + .split(util::DOT_SEPARATOR) + .into_iter(); + + let title = + title.ok_or_else(|| format!("track {id}: could not get title"))?; + + let subtitle_p1 = subtitle_parts.next(); + let subtitle_p2 = subtitle_parts.next(); + let subtitle_p3 = subtitle_parts.next(); + + match page_type { + MusicPageType::Artist => { + let subscriber_count = subtitle_p2.and_then(|p| { + util::parse_large_numstr_or_warn( + p.first_str(), + self.lang, + &mut self.warnings, + ) + }); + + self.items.push(MusicItem::Artist(ArtistItem { + id, + name: title, + avatar: item.thumbnail.into(), + subscriber_count, + })); + Ok(Some(MusicItemType::Artist)) + } + MusicPageType::Album => { + let album_type = subtitle_p1 + .map(|st| map_album_type(st.first_str(), self.lang)) + .unwrap_or_default(); + + let (artists, by_va) = map_artists(subtitle_p2); + + let artist_id = map_artist_id_fallback(item.menu, artists.first()); + + let year = subtitle_p3 + .and_then(|st| util::parse_numeric(st.first_str()).ok()); + + self.items.push(MusicItem::Album(AlbumItem { + id, + name: title, + cover: item.thumbnail.into(), + artists, + artist_id, + album_type, + year, + by_va, + })); + Ok(Some(MusicItemType::Album)) + } + MusicPageType::Playlist => { + // Part 1 may be the "Playlist" label + let (channel_p, tcount_p) = match subtitle_p3 { + Some(_) => (subtitle_p2, subtitle_p3), + None => (subtitle_p1, subtitle_p2), + }; + + let from_ytm = channel_p + .as_ref() + .and_then(|p| p.0.first()) + .map(util::is_ytm) + .unwrap_or_default(); + let channel = channel_p.and_then(|p| { + p.0.into_iter().find_map(|c| ChannelId::try_from(c).ok()) + }); + let track_count = tcount_p + .filter(|_| from_ytm) + .and_then(|p| util::parse_numeric(p.first_str()).ok()); + + self.items.push(MusicItem::Playlist(MusicPlaylistItem { + id, + name: title, + thumbnail: item.thumbnail.into(), + channel, + track_count, + from_ytm, + })); + Ok(Some(MusicItemType::Playlist)) + } + MusicPageType::None => { + // There may be broken YT channels from the artist search. They can be skipped. + Ok(None) + } + // Tracks were already handled above + MusicPageType::Track { .. } => unreachable!(), + MusicPageType::Unknown => { + self.has_unknown = true; + Ok(None) + } + } + } + None => { + if item.music_item_renderer_display_policy == DisplayPolicy::GreyOut { + Ok(None) + } else { + Err("could not determine item type".to_owned()) + } + } + } + } + // Tile + MusicResponseItem::MusicTwoRowItemRenderer(item) => { + let mut subtitle_parts = item.subtitle.split(util::DOT_SEPARATOR).into_iter(); + let subtitle_p1 = subtitle_parts.next(); + let subtitle_p2 = subtitle_parts.next(); + + match item.navigation_endpoint.music_page() { + Some((page_type, id)) => match page_type { + MusicPageType::Track { is_video } => { + let (artists, by_va) = map_artists(subtitle_p1); + + self.items.push(MusicItem::Track(TrackItem { + id, + name: item.title, + duration: None, + cover: item.thumbnail_renderer.into(), + artist_id: artists.first().and_then(|a| a.id.clone()), + artists, + album: None, + view_count: subtitle_p2.and_then(|c| { + util::parse_large_numstr_or_warn( + c.first_str(), + self.lang, + &mut self.warnings, + ) + }), + is_video, + track_nr: None, + by_va, + })); + Ok(Some(MusicItemType::Track)) + } + MusicPageType::Artist => { + let subscriber_count = subtitle_p1.and_then(|p| { + util::parse_large_numstr_or_warn( + p.first_str(), + self.lang, + &mut self.warnings, + ) + }); + + self.items.push(MusicItem::Artist(ArtistItem { + id, + name: item.title, + avatar: item.thumbnail_renderer.into(), + subscriber_count, + })); + Ok(Some(MusicItemType::Artist)) + } + MusicPageType::Album => { + let mut year = None; + let mut album_type = AlbumType::Single; + + let (artists, by_va) = + match (subtitle_p1, subtitle_p2, &self.artists, self.artist_page) { + // "2022" (Artist singles) + (Some(year_txt), None, Some(artists), true) => { + year = util::parse_numeric(year_txt.first_str()).ok(); + artists.clone() + } + // "Album", "2022" (Artist albums) + (Some(atype_txt), Some(year_txt), Some(artists), true) => { + year = util::parse_numeric(year_txt.first_str()).ok(); + album_type = + map_album_type(atype_txt.first_str(), self.lang); + artists.clone() + } + // Album on artist page with unknown year + (None, None, Some(artists), true) => artists.clone(), + // "Album", <"Oonagh"> (Album variants, new releases) + (Some(atype_txt), Some(p2), _, false) => { + album_type = + map_album_type(atype_txt.first_str(), self.lang); + map_artists(Some(p2)) + } + // "Album" (Album variants, no artist) + (Some(atype_txt), None, _, false) => { + album_type = + map_album_type(atype_txt.first_str(), self.lang); + (Vec::new(), true) + } + _ => { + return Err(format!( + "could not parse subtitle of album {id}" + )); + } + }; + + self.items.push(MusicItem::Album(AlbumItem { + id, + name: item.title, + cover: item.thumbnail_renderer.into(), + artist_id: artists.first().and_then(|a| a.id.clone()), + artists, + album_type, + year, + by_va, + })); + Ok(Some(MusicItemType::Album)) + } + MusicPageType::Playlist => { + // When the playlist subtitle has only 1 part, it is a playlist from YT Music + // (featured on the startpage or in genres) + let from_ytm = subtitle_p2 + .as_ref() + .and_then(|p| p.0.first()) + .map_or(true, util::is_ytm); + let channel = subtitle_p2.and_then(|p| { + p.0.into_iter().find_map(|c| ChannelId::try_from(c).ok()) + }); + + self.items.push(MusicItem::Playlist(MusicPlaylistItem { + id, + name: item.title, + thumbnail: item.thumbnail_renderer.into(), + channel, + track_count: None, + from_ytm, + })); + Ok(Some(MusicItemType::Playlist)) + } + MusicPageType::None => Ok(None), + MusicPageType::Unknown => { + self.has_unknown = true; + Ok(None) + } + }, + None => Err("could not determine item type".to_owned()), } - Ok(None) } } } @@ -570,512 +929,6 @@ impl MusicListMapper { etype } - /// Map a ListMusicItem (album/playlist item, search result) - fn map_list_item(&mut self, item: ListMusicItem) -> Result, String> { - let mut columns = item.flex_columns.into_iter(); - let c1 = columns.next(); - let c2 = columns.next(); - let c3 = columns.next(); - let c4 = columns.next(); - - let title = c1.as_ref().map(|col| col.renderer.text.to_string()); - - let first_tn = item - .thumbnail - .music_thumbnail_renderer - .thumbnail - .thumbnails - .first(); - - let music_page = item - .navigation_endpoint - .and_then(NavigationEndpoint::music_page) - .or_else(|| { - c1.and_then(|c1| { - c1.renderer - .text - .0 - .into_iter() - .next() - .and_then(TextComponent::music_page) - }) - }) - .or_else(|| { - item.playlist_item_data.map(|d| MusicPage { - id: d.video_id, - typ: MusicPageType::Track { - vtype: MusicVideoType::from_is_video( - self.album.is_none() - && !first_tn.map(|tn| tn.height == tn.width).unwrap_or_default(), - ), - }, - }) - }) - .or_else(|| { - first_tn.and_then(|tn| { - util::video_id_from_thumbnail_url(&tn.url).map(|id| MusicPage { - id, - typ: MusicPageType::Track { - vtype: MusicVideoType::from_is_video( - self.album.is_none() && tn.width != tn.height, - ), - }, - }) - }) - }); - - match music_page.map(|mp| (mp.typ, mp.id)) { - // Track - Some((MusicPageType::Track { vtype }, id)) => { - let title = title.ok_or_else(|| format!("track {id}: could not get title"))?; - - #[derive(Default)] - struct Parsed { - artists: Option, - album: Option, - duration: Option, - view_count: Option, - } - - // Dont map music livestreams - if item - .badges - .iter() - .any(|b| matches!(b, TrackBadge::LiveBadgeRenderer {})) - { - return Ok(None); - } - - let p = match item.flex_column_display_style { - // Search result - FlexColumnDisplayStyle::TwoLines => { - // Is this a related track (from the "similar titles" tab in the player)? - if vtype != MusicVideoType::Video && item.item_height == ItemHeight::Compact - { - Parsed { - artists: c2.map(TextComponents::from), - album: c3.map(TextComponents::from), - ..Default::default() - } - } else { - let mut subtitle_parts = c2 - .ok_or_else(|| format!("track {id}: could not get subtitle"))? - .renderer - .text - .split(util::DOT_SEPARATOR) - .into_iter(); - - // Is this a related video? - if item.item_height == ItemHeight::Compact { - Parsed { - artists: subtitle_parts.next(), - view_count: subtitle_parts.next(), - ..Default::default() - } - } - // Is this an item from search suggestion? - else if self.search_suggestion { - // Skip first part (track type) - subtitle_parts.next(); - Parsed { - artists: subtitle_parts.next(), - album: c3.map(TextComponents::from), - view_count: subtitle_parts.next(), - ..Default::default() - } - } - // Is it a podcast episode? - else if vtype == MusicVideoType::Episode { - Parsed { - artists: subtitle_parts.next_back(), - ..Default::default() - } - } else { - // Skip first part (track type) - if subtitle_parts.len() > 3 - || (vtype == MusicVideoType::Video && subtitle_parts.len() == 2) - { - subtitle_parts.next(); - } - - match vtype { - MusicVideoType::Video => Parsed { - artists: subtitle_parts.next(), - view_count: subtitle_parts.next(), - duration: subtitle_parts.next(), - ..Default::default() - }, - _ => Parsed { - artists: subtitle_parts.next(), - album: subtitle_parts.next(), - duration: subtitle_parts.next(), - view_count: c3.map(TextComponents::from), - }, - } - } - } - } - // Playlist item - FlexColumnDisplayStyle::Default => { - let artists = c2.map(TextComponents::from); - let duration = item - .fixed_columns - .into_iter() - .next() - .map(TextComponents::from); - if self.album.is_some() { - Parsed { - artists, - view_count: c3.map(TextComponents::from), - duration, - ..Default::default() - } - } else if self.artist_page && c4.is_some() { - Parsed { - artists, - view_count: c3.map(TextComponents::from), - album: c4.map(TextComponents::from), - duration, - } - } else { - Parsed { - artists, - album: c3.map(TextComponents::from), - duration, - ..Default::default() - } - } - } - }; - - let duration = p - .duration - .and_then(|p| util::parse_video_length(p.first_str())); - let album = p - .album - .and_then(|p| p.0.into_iter().find_map(|c| AlbumId::try_from(c).ok())) - .or_else(|| self.album.clone()); - let view_count = p.view_count.and_then(|p| { - util::parse_large_numstr_or_warn(p.first_str(), self.lang, &mut self.warnings) - }); - let (mut artists, by_va) = map_artists(p.artists); - - // Extract artist id from dropdown menu - let artist_id = map_artist_id_fallback(item.menu, artists.first()); - - // Fall back to the artist given when constructing the mapper. - // This is used for extracting artist pages. - // On some albums, the artist name of the tracks is not given but different - // from the album artist. In this case dont copy the album artist. - if let Some((fb_artists, _)) = &self.artists { - if artists.is_empty() - && (self.artist_page - || artist_id.is_none() - || fb_artists.iter().any(|fb_id| { - fb_id - .id - .as_deref() - .map(|aid| artist_id.as_deref() == Some(aid)) - .unwrap_or_default() - })) - { - artists.clone_from(fb_artists); - } - } - - let track_nr = item.index.and_then(|txt| util::parse_numeric(&txt).ok()); - - self.items.push(MusicItem::Track(TrackItem { - id, - name: title, - duration, - cover: item.thumbnail.into(), - artists, - artist_id, - album, - view_count, - track_type: vtype.into(), - track_nr, - by_va, - unavailable: item.music_item_renderer_display_policy == DisplayPolicy::GreyOut, - })); - Ok(Some(MusicItemType::Track)) - } - // Artist / Album / Playlist - Some((page_type, id)) => { - // Ignore "Shuffle all" button and builtin "Liked music" and "Saved episodes" playlists - if page_type == MusicPageType::None - || (page_type == (MusicPageType::Playlist { is_podcast: false }) - && matches!(id.as_str(), "MLCT" | "LM" | "SE")) - { - return Ok(None); - } - - let mut subtitle_parts = c2 - .ok_or_else(|| format!("{id}: could not get subtitle"))? - .renderer - .text - .split(util::DOT_SEPARATOR) - .into_iter(); - - let title = title.ok_or_else(|| format!("track {id}: could not get title"))?; - - let subtitle_p1 = subtitle_parts.next(); - let subtitle_p2 = subtitle_parts.next(); - let subtitle_p3 = subtitle_parts.next(); - - match page_type { - MusicPageType::Artist => { - let subscriber_count = subtitle_p2.and_then(|p| { - util::parse_large_numstr_or_warn( - p.first_str(), - self.lang, - &mut self.warnings, - ) - }); - - self.items.push(MusicItem::Artist(ArtistItem { - id, - name: title, - avatar: item.thumbnail.into(), - subscriber_count, - })); - Ok(Some(MusicItemType::Artist)) - } - MusicPageType::Album => { - let album_type = subtitle_p1 - .map(|st| map_album_type(st.first_str(), self.lang)) - .unwrap_or_default(); - - let (mut artists, by_va) = map_artists(subtitle_p2); - let artist_id = map_artist_id_fallback(item.menu, artists.first()); - - // Album artist links may be invisible on the search page, so - // fall back to menu data - if let Some(a1) = artists.first_mut() { - if a1.id.is_none() { - a1.id.clone_from(&artist_id); - } - } - - let year = - subtitle_p3.and_then(|st| util::parse_numeric(st.first_str()).ok()); - - self.items.push(MusicItem::Album(AlbumItem { - id, - name: title, - cover: item.thumbnail.into(), - artists, - artist_id, - album_type, - year, - by_va, - })); - Ok(Some(MusicItemType::Album)) - } - MusicPageType::Playlist { is_podcast } => { - // Part 1 may be the "Playlist" label - let (channel_p, tcount_p) = match subtitle_p3 { - Some(_) => (subtitle_p2, subtitle_p3), - None => (subtitle_p1, subtitle_p2), - }; - - let from_ytm = channel_p - .as_ref() - .and_then(|p| p.0.first()) - .map(util::is_ytm) - .unwrap_or_default(); - let channel = channel_p.and_then(|p| { - p.0.into_iter().find_map(|c| ChannelId::try_from(c).ok()) - }); - let track_count = tcount_p - .filter(|_| from_ytm) - .and_then(|p| util::parse_numeric(p.first_str()).ok()); - - self.items.push(MusicItem::Playlist(MusicPlaylistItem { - id, - name: title, - thumbnail: item.thumbnail.into(), - channel, - track_count, - from_ytm, - is_podcast, - })); - Ok(Some(MusicItemType::Playlist)) - } - MusicPageType::User => { - // Part 1 may be the "Profile" label - let handle = map_channel_handle(subtitle_p2.as_ref()) - .or_else(|| map_channel_handle(subtitle_p1.as_ref())); - - self.items.push(MusicItem::User(UserItem { - id, - name: title, - handle, - avatar: item.thumbnail.into(), - })); - Ok(Some(MusicItemType::User)) - } - MusicPageType::None => { - // There may be broken YT channels from the artist search. They can be skipped. - Ok(None) - } - // Tracks were already handled above - MusicPageType::Track { .. } => unreachable!(), - } - } - None => { - if item.music_item_renderer_display_policy == DisplayPolicy::GreyOut { - Ok(None) - } else { - Err("could not determine item type".to_owned()) - } - } - } - } - - /// Map a CoverMusicItem (album/playlist tile) - fn map_tile(&mut self, item: CoverMusicItem) -> Result, String> { - let mut subtitle_parts = item.subtitle.split(util::DOT_SEPARATOR).into_iter(); - let subtitle_p1 = subtitle_parts.next(); - let subtitle_p2 = subtitle_parts.next(); - - match item.navigation_endpoint.music_page() { - Some(music_page) => match music_page.typ { - MusicPageType::Track { vtype } => { - let (artists, by_va, view_count, duration) = if vtype == MusicVideoType::Episode - { - let (artists, by_va) = map_artists(subtitle_p2); - let duration = subtitle_p1.and_then(|s| { - timeago::parse_video_duration_or_warn( - self.lang, - s.first_str(), - &mut self.warnings, - ) - }); - (artists, by_va, None, duration) - } else { - let (artists, by_va) = map_artists(subtitle_p1); - let view_count = subtitle_p2.and_then(|c| { - util::parse_large_numstr_or_warn( - c.first_str(), - self.lang, - &mut self.warnings, - ) - }); - (artists, by_va, view_count, None) - }; - - self.items.push(MusicItem::Track(TrackItem { - id: music_page.id, - name: item.title, - duration, - cover: item.thumbnail_renderer.into(), - artist_id: artists.first().and_then(|a| a.id.clone()), - artists, - album: None, - view_count, - track_type: vtype.into(), - track_nr: None, - by_va, - unavailable: false, - })); - Ok(Some(MusicItemType::Track)) - } - MusicPageType::Artist => { - let subscriber_count = subtitle_p1.and_then(|p| { - util::parse_large_numstr_or_warn( - p.first_str(), - self.lang, - &mut self.warnings, - ) - }); - - self.items.push(MusicItem::Artist(ArtistItem { - id: music_page.id, - name: item.title, - avatar: item.thumbnail_renderer.into(), - subscriber_count, - })); - Ok(Some(MusicItemType::Artist)) - } - MusicPageType::Album => { - let mut year = None; - let mut album_type = self.album_type; - - let (artists, by_va) = - match (subtitle_p1, subtitle_p2, &self.artists, self.artist_page) { - // "2022" (Artist singles) - (Some(year_txt), None, Some(artists), true) => { - year = util::parse_numeric(year_txt.first_str()).ok(); - artists.clone() - } - // "Album", "2022" (Artist albums) - (Some(atype_txt), Some(year_txt), Some(artists), true) => { - year = util::parse_numeric(year_txt.first_str()).ok(); - album_type = map_album_type(atype_txt.first_str(), self.lang); - artists.clone() - } - // Album on artist page with unknown year - (None, None, Some(artists), true) => artists.clone(), - // "Album", <"Oonagh"> (Album variants, new releases) - (Some(atype_txt), Some(p2), _, false) => { - album_type = map_album_type(atype_txt.first_str(), self.lang); - map_artists(Some(p2)) - } - // "Album" (Album variants, no artist) - (Some(atype_txt), None, _, false) => { - album_type = map_album_type(atype_txt.first_str(), self.lang); - (Vec::new(), true) - } - _ => { - return Err(format!( - "could not parse subtitle of album {}", - music_page.id - )); - } - }; - - self.items.push(MusicItem::Album(AlbumItem { - id: music_page.id, - name: item.title, - cover: item.thumbnail_renderer.into(), - artist_id: artists.first().and_then(|a| a.id.clone()), - artists, - album_type, - year, - by_va, - })); - Ok(Some(MusicItemType::Album)) - } - MusicPageType::Playlist { is_podcast } => { - // When the playlist subtitle has only 1 part, it is a playlist from YT Music - // (featured on the startpage or in genres) - let from_ytm = subtitle_p2 - .as_ref() - .and_then(|p| p.0.first()) - .map_or(true, util::is_ytm); - let channel = subtitle_p2 - .and_then(|p| p.0.into_iter().find_map(|c| ChannelId::try_from(c).ok())); - - self.items.push(MusicItem::Playlist(MusicPlaylistItem { - id: music_page.id, - name: item.title, - thumbnail: item.thumbnail_renderer.into(), - channel, - track_count: None, - from_ytm, - is_podcast, - })); - Ok(Some(MusicItemType::Playlist)) - } - MusicPageType::None | MusicPageType::User => Ok(None), - }, - None => Err("could not determine item type".to_owned()), - } - } - - /// Map a MusicCardShelf (used for the top search result) pub fn map_card(&mut self, card: MusicCardShelf) -> Option { /* "Artist" " • " "" @@ -1091,7 +944,7 @@ impl MusicListMapper { let subtitle_p4 = subtitle_parts.next(); let item_type = match card.on_tap.music_page() { - Some(music_page) => match music_page.typ { + Some((page_type, id)) => match page_type { MusicPageType::Artist => { let subscriber_count = subtitle_p2.and_then(|p| { util::parse_large_numstr_or_warn( @@ -1102,7 +955,7 @@ impl MusicListMapper { }); self.items.push(MusicItem::Artist(ArtistItem { - id: music_page.id, + id, name: card.title, avatar: card.thumbnail.into(), subscriber_count, @@ -1116,7 +969,7 @@ impl MusicListMapper { .unwrap_or_default(); self.items.push(MusicItem::Album(AlbumItem { - id: music_page.id, + id, name: card.title, cover: card.thumbnail.into(), artist_id: artists.first().and_then(|a| a.id.clone()), @@ -1127,66 +980,46 @@ impl MusicListMapper { })); Some(MusicItemType::Album) } - MusicPageType::Track { vtype } => { - if vtype == MusicVideoType::Episode { - let (artists, by_va) = map_artists(subtitle_p3); - - self.items.push(MusicItem::Track(TrackItem { - id: music_page.id, - name: card.title, - duration: None, - cover: card.thumbnail.into(), - artist_id: artists.first().and_then(|a| a.id.clone()), - artists, - album: None, - view_count: None, - track_type: vtype.into(), - track_nr: None, - by_va, - unavailable: false, - })); + MusicPageType::Track { is_video } => { + let (artists, by_va) = map_artists(subtitle_p2); + let duration = + subtitle_p4.and_then(|p| util::parse_video_length(p.first_str())); + let (album, view_count) = if is_video { + ( + None, + subtitle_p3.and_then(|p| { + util::parse_large_numstr_or_warn( + p.first_str(), + self.lang, + &mut self.warnings, + ) + }), + ) } else { - let (artists, by_va) = map_artists(subtitle_p2); - let duration = - subtitle_p4.and_then(|p| util::parse_video_length(p.first_str())); - let (album, view_count) = if vtype.is_video() { - ( - None, - subtitle_p3.and_then(|p| { - util::parse_large_numstr_or_warn( - p.first_str(), - self.lang, - &mut self.warnings, - ) - }), - ) - } else { - ( - subtitle_p3.and_then(|p| { - p.0.into_iter().find_map(|c| AlbumId::try_from(c).ok()) - }), - None, - ) - }; + ( + subtitle_p3.and_then(|p| { + p.0.into_iter().find_map(|c| AlbumId::try_from(c).ok()) + }), + None, + ) + }; - self.items.push(MusicItem::Track(TrackItem { - id: music_page.id, - name: card.title, - duration, - cover: card.thumbnail.into(), - artist_id: artists.first().and_then(|a| a.id.clone()), - artists, - album, - view_count, - track_type: vtype.into(), - track_nr: None, - by_va, - unavailable: false, - })); - } + self.items.push(MusicItem::Track(TrackItem { + id, + name: card.title, + duration, + cover: card.thumbnail.into(), + artist_id: artists.first().and_then(|a| a.id.clone()), + artists, + album, + view_count, + is_video, + track_nr: None, + by_va, + })); Some(MusicItemType::Track) } - MusicPageType::Playlist { is_podcast } => { + MusicPageType::Playlist => { let from_ytm = subtitle_p2 .as_ref() .and_then(|p| p.0.first()) @@ -1197,30 +1030,20 @@ impl MusicListMapper { subtitle_p3.and_then(|p| util::parse_numeric(p.first_str()).ok()); self.items.push(MusicItem::Playlist(MusicPlaylistItem { - id: music_page.id, + id, name: card.title, thumbnail: card.thumbnail.into(), channel, track_count, from_ytm, - is_podcast, })); Some(MusicItemType::Playlist) } - MusicPageType::User => { - // Part 1 may be the "Profile" label - let handle = map_channel_handle(subtitle_p2.as_ref()) - .or_else(|| map_channel_handle(subtitle_p1.as_ref())); - - self.items.push(MusicItem::User(UserItem { - id: music_page.id, - name: card.title, - handle, - avatar: card.thumbnail.into(), - })); - Some(MusicItemType::User) - } MusicPageType::None => None, + MusicPageType::Unknown => { + self.has_unknown = true; + None + } }, None => { self.warnings @@ -1282,7 +1105,6 @@ impl MusicListMapper { MusicItem::Album(album) => albums.push(album), MusicItem::Artist(artist) => artists.push(artist), MusicItem::Playlist(playlist) => playlists.push(playlist), - MusicItem::User(_) => {} } } @@ -1297,31 +1119,18 @@ impl MusicListMapper { } } - #[cfg(feature = "userdata")] - pub fn conv_history_items( - self, - date_txt: Option, - utc_offset: UtcOffset, - res: &mut MapResult>>, - ) { - res.warnings.extend(self.warnings); - res.c.extend( - self.items - .into_iter() - .filter_map(TrackItem::from_ytm_item) - .map(|item| HistoryItem { - item, - playback_date: date_txt.as_deref().and_then(|s| { - timeago::parse_textual_date_to_d( - self.lang, - utc_offset, - s, - &mut res.warnings, - ) - }), - playback_date_txt: date_txt.clone(), - }), - ); + /// Sometimes the YT Music API returns responses containing unknown items. + /// + /// In this case, the response data is likely missing some fields, which leads to + /// parsing errors and wrong data being extracted. + /// + /// Therefore it is safest to discard such responses and retry the request. + pub fn check_unknown(&self) -> Result<(), ExtractionError> { + if self.has_unknown { + Err(ExtractionError::InvalidData("unknown YTM items".into())) + } else { + Ok(()) + } } } @@ -1360,30 +1169,22 @@ fn map_artist_id_fallback( .or_else(|| fallback_artist.and_then(|a| a.id.clone())) } -fn map_channel_handle(st: Option<&TextComponents>) -> Option { - st.map(|t| t.first_str()) - .filter(|t| t.starts_with('@')) - .map(str::to_owned) -} - pub(crate) fn map_artist_id(entries: Vec) -> Option { entries.into_iter().find_map(|i| { - if let NavigationEndpoint::Browse { - browse_endpoint, .. - } = i.menu_navigation_item_renderer.navigation_endpoint - { - browse_endpoint - .browse_endpoint_context_supported_configs + let ep = i + .menu_navigation_item_renderer + .navigation_endpoint + .browse_endpoint; + ep.and_then(|ep| { + ep.browse_endpoint_context_supported_configs .and_then(|cfg| { if cfg.browse_endpoint_context_music_config.page_type == PageType::Artist { - Some(browse_endpoint.browse_id) + Some(ep.browse_id) } else { None } }) - } else { - None - } + }) }) } @@ -1436,10 +1237,9 @@ pub(crate) fn map_queue_item(item: QueueMusicItem, lang: Language) -> MapResult< artist_id, album, view_count, - track_type: MusicVideoType::from_is_video(is_video).into(), + is_video, track_nr: None, by_va, - unavailable: false, }, warnings, } @@ -1458,19 +1258,14 @@ mod tests { fn map_album_type_samples() { let json_path = path!(*TESTFILES / "dict" / "album_type_samples.json"); let json_file = File::open(json_path).unwrap(); - let atype_samples: BTreeMap> = + let atype_samples: BTreeMap> = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - for (lang, entry) in &atype_samples { - for (album_type_str, txt) in entry { - let album_type_n = album_type_str.split('_').next().unwrap(); - let album_type = serde_plain::from_str::(album_type_n).unwrap(); + atype_samples.iter().for_each(|(lang, entry)| { + entry.iter().for_each(|(album_type, txt)| { let res = map_album_type(txt, *lang); - assert_eq!( - res, album_type, - "{album_type_str}: lang: {lang}, txt: {txt}" - ); - } - } + assert_eq!(res, *album_type, "lang: {lang}, txt: {txt}"); + }); + }); } } diff --git a/src/client/response/music_playlist.rs b/src/client/response/music_playlist.rs index 84202f0..8de1c91 100644 --- a/src/client/response/music_playlist.rs +++ b/src/client/response/music_playlist.rs @@ -1,45 +1,27 @@ use serde::Deserialize; use serde_with::{serde_as, DefaultOnError, VecSkipError}; -use crate::serializer::text::{AttributedText, Text, TextComponents}; +use crate::serializer::text::{Text, TextComponents}; use super::{ music_item::{ - Button, ItemSection, MusicContentsRenderer, MusicItemMenuEntry, MusicMicroformat, - MusicThumbnailRenderer, + ItemSection, MusicContentsRenderer, MusicItemMenuEntry, MusicThumbnailRenderer, + SingleColumnBrowseResult, }, - url_endpoint::OnTapWrap, - ContentsRenderer, SectionList, Tab, + Tab, }; /// Response model for YouTube Music playlists and albums #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct MusicPlaylist { - pub contents: Option, + pub contents: SingleColumnBrowseResult>, pub header: Option
, - #[serde(default)] - pub microformat: MusicMicroformat, -} - -#[serde_as] -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) enum Contents { - SingleColumnBrowseResultsRenderer(ContentsRenderer>), - #[serde(rename_all = "camelCase")] - TwoColumnBrowseResultsRenderer { - /// List content - secondary_contents: PlSectionList, - /// Header - #[serde_as(as = "VecSkipError<_>")] - tabs: Vec>>, - }, } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] -pub(crate) struct PlSectionList { +pub(crate) struct SectionList { /// Includes a continuation token for fetching recommendations pub section_list_renderer: MusicContentsRenderer, } @@ -47,7 +29,6 @@ pub(crate) struct PlSectionList { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct Header { - #[serde(alias = "musicResponsiveHeaderRenderer")] pub music_detail_header_renderer: HeaderRenderer, } @@ -67,13 +48,12 @@ pub(crate) struct HeaderRenderer { pub subtitle: TextComponents, /// Playlist/album description. May contain hashtags which are /// displayed as search links on the YouTube website. - pub description: Option, + #[serde_as(as = "Option")] + pub description: Option, /// Playlist thumbnail / album cover. /// Missing on artist_tracks view. #[serde(default)] pub thumbnail: MusicThumbnailRenderer, - /// Channel (only on TwoColumnBrowseResultsRenderer) - pub strapline_text_one: Option, /// Number of tracks + playtime. /// Missing on artist_tracks view. /// @@ -83,32 +63,9 @@ pub(crate) struct HeaderRenderer { #[serde(default)] #[serde_as(as = "Text")] pub second_subtitle: Vec, - /// Channel (newer data model) - #[serde(default)] - #[serde_as(as = "DefaultOnError")] - pub facepile: Option, #[serde(default)] #[serde_as(as = "DefaultOnError")] pub menu: Option, - #[serde(default)] - #[serde_as(as = "VecSkipError<_>")] - pub buttons: Vec, -} - -#[derive(Debug, Deserialize)] -#[serde(untagged)] -pub(crate) enum Description { - #[serde(rename_all = "camelCase")] - Shelf { - music_description_shelf_renderer: DescriptionShelf, - }, - Text(TextComponents), -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct DescriptionShelf { - pub description: TextComponents, } #[derive(Debug, Deserialize)] @@ -123,41 +80,31 @@ pub(crate) struct HeaderMenu { pub(crate) struct HeaderMenuRenderer { #[serde(default)] #[serde_as(as = "VecSkipError<_>")] - pub top_level_buttons: Vec