fix: parse channel subscriber/video count correctly

fix: handling albums with unavailable tracks

.forgejo/workflows/ci.yaml

@@ -0,0 +1,68 @@
+name: CI
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+
+jobs:
+  Test:
+    runs-on: cimaster-latest
+    services:
+      warpproxy:
+        image: thetadev256/warpproxy
+        env:
+          WARP_DEVICE_ID: ${{ secrets.WARP_DEVICE_ID }}
+          WARP_ACCESS_TOKEN: ${{ secrets.WARP_ACCESS_TOKEN }}
+          WARP_LICENSE_KEY: ${{ secrets.WARP_LICENSE_KEY }}
+          WARP_PRIVATE_KEY: ${{ secrets.WARP_PRIVATE_KEY }}
+    steps:
+      - name: 📦 Checkout repository
+        uses: actions/checkout@v4
+
+      - name: 🦀 Setup Rust cache
+        uses: https://github.com/Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: "true"
+
+      - name: Download rustypipe-botguard
+        run: |
+          TARGET=$(rustc --version --verbose | grep "host:" | sed -e 's/^host: //')
+          cd ~
+          curl -SsL -o rustypipe-botguard.tar.xz "https://codeberg.org/ThetaDev/rustypipe-botguard/releases/download/v0.1.1/rustypipe-botguard-v0.1.1-${TARGET}.tar.xz"
+          cd /usr/local/bin
+          sudo tar -xJf ~/rustypipe-botguard.tar.xz
+          rm ~/rustypipe-botguard.tar.xz
+          rustypipe-botguard --version
+
+      - name: 📎 Clippy
+        run: |
+          cargo clippy --all --tests --features=rss,userdata,indicatif,audiotag -- -D warnings
+          cargo clippy --package=rustypipe --tests -- -D warnings
+          cargo clippy --package=rustypipe-downloader -- -D warnings
+          cargo clippy --package=rustypipe-cli -- -D warnings
+          cargo clippy --package=rustypipe-cli --features=timezone -- -D warnings
+
+      - name: 🧪 Test
+        run: cargo nextest run --config-file ~/.config/nextest.toml --profile ci --retries 2 --features rss,userdata --workspace -- --skip 'user_data::'
+        env:
+          ALL_PROXY: "http://warpproxy:8124"
+
+      - name: Move test report
+        if: always()
+        run: mv target/nextest/ci/junit.xml junit.xml || true
+
+      - name: 💌 Upload test report
+        if: always()
+        uses: https://code.forgejo.org/forgejo/upload-artifact@v4
+        with:
+          name: test
+          path: |
+            junit.xml
+            rustypipe_reports
+
+      - name: 🔗 Artifactview PR comment
+        if: ${{ always() && github.event_name == 'pull_request' }}
+        run: |
+          if [[ "$GITEA_ACTIONS" == "true" ]]; then RUN_NUMBER="$GITHUB_RUN_NUMBER"; else RUN_NUMBER="$GITHUB_RUN_ID"; fi
+          curl -SsL --fail-with-body -w "\n" -X POST https://av.thetadev.de/.well-known/api/prComment -H "Content-Type: application/json" \
+            --data '{"url": "'"$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$RUN_NUMBER"'", "pr": ${{ github.event.number }}, "artifact_titles": {"test":"🧪 Test report"}, "artifact_paths": {"test":"/junit.xml?viewer=1"}}'

.forgejo/workflows/release-cli.yaml

@@ -0,0 +1,69 @@
+name: Release CLI
+on:
+  push:
+    tags:
+      - "rustypipe-cli/v*.*.*"
+
+jobs:
+  Release:
+    runs-on: cimaster-latest
+    steps:
+      - name: 📦 Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup cross compilation
+        run: |
+          rustup target add x86_64-pc-windows-msvc x86_64-apple-darwin aarch64-apple-darwin
+          cargo install cargo-xwin
+
+          # https://wapl.es/rust/2019/02/17/rust-cross-compile-linux-to-macos.html/
+          sudo apt-get install -y llvm clang cmake
+          cd ~
+          git clone https://github.com/tpoechtrager/osxcross
+          cd osxcross
+          wget -nc "https://github.com/joseluisq/macosx-sdks/releases/download/12.3/MacOSX12.3.sdk.tar.xz"
+          mv MacOSX12.3.sdk.tar.xz tarballs/
+          UNATTENDED=yes OSX_VERSION_MIN=12.3 ./build.sh
+          OSXCROSS_BIN="$(pwd)/target/bin"
+
+          echo "CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=$(find "$OSXCROSS_BIN" -name "x86_64-apple-darwin*-clang")" >> $GITHUB_ENV
+          echo "CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS=-Car=$(find "$OSXCROSS_BIN" -name "x86_64-apple-darwin*-ar"),-Clink-arg=-undefined,-Clink-arg=dynamic_lookup" >> $GITHUB_ENV
+          echo "CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=$(find "$OSXCROSS_BIN" -name "aarch64-apple-darwin*-clang")" >> $GITHUB_ENV
+          echo "CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS=-Car=$(find "$OSXCROSS_BIN" -name "aarch64-apple-darwin*-ar"),-Clink-arg=-undefined,-Clink-arg=dynamic_lookup" >> $GITHUB_ENV
+
+      - name: ⚒️ Build application
+        run: |
+          export PATH="$PATH:$HOME/osxcross/target/bin"
+          CRATE="rustypipe-cli"
+          PKG_CONFIG_SYSROOT_DIR=/usr/x86_64-linux-gnu cargo build --release --package=$CRATE --target x86_64-unknown-linux-gnu
+          PKG_CONFIG_SYSROOT_DIR=/usr/aarch64-linux-gnu cargo build --release --package=$CRATE --target aarch64-unknown-linux-gnu
+          CC="$CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER" CXX="$CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER++" cargo build --release --package=$CRATE --target x86_64-apple-darwin
+          CC="$CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER" CXX="$CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER++" cargo build --release --package=$CRATE --target aarch64-apple-darwin
+          cargo xwin build --release --package=$CRATE --target x86_64-pc-windows-msvc
+
+      - name: Prepare release
+        run: |
+          CRATE="rustypipe-cli"
+          BIN="rustypipe"
+          echo "CRATE=$CRATE" >> "$GITHUB_ENV"
+          echo "CRATE_VERSION=$(echo '${{ github.ref_name }}' | awk 'BEGIN{RS="/"} NR==2{print}')" >> "$GITHUB_ENV"
+          CL_PATH="cli/CHANGELOG.md"
+          {
+            echo 'CHANGELOG<<END_OF_FILE'
+            awk 'BEGIN{RS="(^|\n)## [^\n]+\n*"} NR==2 { print }' "$CL_PATH"
+            echo END_OF_FILE
+          } >> "$GITHUB_ENV"
+
+          mkdir dist
+
+          for arch in x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu x86_64-apple-darwin aarch64-apple-darwin; do
+            tar -cJf "dist/${BIN}-${CRATE_VERSION}-${arch}.tar.xz" -C target/${arch}/release "${BIN}"
+          done
+          (cd target/x86_64-pc-windows-msvc/release && zip -9 "../../../dist/${BIN}-${CRATE_VERSION}-x86_64-pc-windows-msvc.zip" "${BIN}.exe")
+
+      - name: 🎉 Publish release
+        uses: https://gitea.com/actions/release-action@main
+        with:
+          title: "${{ env.CRATE }} ${{ env.CRATE_VERSION }}"
+          body: "${{ env.CHANGELOG }}"
+          files: dist/*

.forgejo/workflows/release.yaml

@@ -0,0 +1,34 @@
+name: Release
+on:
+  push:
+    tags:
+      - "*/v*.*.*"
+
+jobs:
+  Release:
+    runs-on: cimaster-latest
+    steps:
+      - name: 📦 Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Get variables
+        run: |
+          CRATE=$(echo '${{ github.ref_name }}' | awk 'BEGIN{RS="/"} NR==1{print}')
+          echo "CRATE=$CRATE" >> "$GITHUB_ENV"
+          echo "CRATE_VERSION=$(echo '${{ github.ref_name }}' | awk 'BEGIN{RS="/"} NR==2{print}')" >> "$GITHUB_ENV"
+          CL_PATH="CHANGELOG.md"
+          if [[ "$CRATE" != "rustypipe" ]]; then pfx="rustypipe-"; CL_PATH="${CRATE#"$pfx"}/$CL_PATH"; fi
+          {
+            echo 'CHANGELOG<<END_OF_FILE'
+            awk 'BEGIN{RS="(^|\n)## [^\n]+\n*"} NR==2 { print }' "$CL_PATH"
+            echo END_OF_FILE
+          } >> "$GITHUB_ENV"
+
+      - name: 📤 Publish crate on crates.io
+        run: cargo publish --token ${{ secrets.CARGO_TOKEN }} --package "${{ env.CRATE }}"
+
+      - name: 🎉 Publish release
+        uses: https://gitea.com/actions/release-action@main
+        with:
+          title: "${{ env.CRATE }} ${{ env.CRATE_VERSION }}"
+          body: "${{ env.CHANGELOG }}"

.forgejo/workflows/renovate.yaml.bak

@@ -0,0 +1,63 @@
+name: renovate
+
+on:
+  push:
+    branches: ["main"]
+    paths:
+      - ".forgejo/workflows/renovate.yaml"
+      - "renovate.json"
+  schedule:
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+
+env:
+  RENOVATE_REPOSITORIES: ${{ github.repository }}
+
+jobs:
+  renovate:
+    runs-on: docker
+    container:
+      image: renovate/renovate:39
+
+    steps:
+      - name: Load renovate repo cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            .tmp/cache/renovate/repository
+            .tmp/cache/renovate/renovate-cache-sqlite
+            .tmp/osv
+          key: repo-cache-${{ github.run_id }}
+          restore-keys: |
+            repo-cache-
+
+      - name: Run renovate
+        run: renovate
+        env:
+          LOG_LEVEL: debug
+          RENOVATE_BASE_DIR: ${{ github.workspace }}/.tmp
+          RENOVATE_ENDPOINT: ${{ github.server_url }}
+          RENOVATE_PLATFORM: gitea
+          RENOVATE_REPOSITORY_CACHE: 'enabled'
+          RENOVATE_TOKEN: ${{ secrets.FORGEJO_CI_BOT_TOKEN }}
+          GITHUB_COM_TOKEN: ${{ secrets.GH_PUBLIC_TOKEN }}
+          RENOVATE_GIT_AUTHOR: 'Renovate Bot <forgejo-renovate-action@forgejo.org>'
+
+          RENOVATE_X_SQLITE_PACKAGE_CACHE: true
+
+          GIT_AUTHOR_NAME: 'Renovate Bot'
+          GIT_AUTHOR_EMAIL: 'forgejo-renovate-action@forgejo.org'
+          GIT_COMMITTER_NAME: 'Renovate Bot'
+          GIT_COMMITTER_EMAIL: 'forgejo-renovate-action@forgejo.org'
+
+          OSV_OFFLINE_ROOT_DIR: ${{ github.workspace }}/.tmp/osv
+
+      - name: Save renovate repo cache
+        if: always() && env.RENOVATE_DRY_RUN != 'full'
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            .tmp/cache/renovate/repository
+            .tmp/cache/renovate/renovate-cache-sqlite
+            .tmp/osv
+          key: repo-cache-${{ github.run_id }}

.gitignore

@@ -4,4 +4,5 @@
 *.snap.new
 
 rustypipe_reports
-rustypipe_cache.json
+rustypipe_cache*.json
+bg_snapshot.bin

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v5.0.0
     hooks:
       - id: end-of-file-fixer
       - id: check-json
@@ -10,4 +10,8 @@ repos:
     hooks:
       - id: cargo-fmt
       - id: cargo-clippy
-        args: ["--all", "--all-features", "--", "-D", "warnings"]
+        name: cargo-clippy rustypipe
+        args: ["--package=rustypipe", "--tests", "--", "-D", "warnings"]
+      - id: cargo-clippy
+        name: cargo-clippy workspace
+        args: ["--all", "--tests", "--features=rss,userdata,indicatif,audiotag", "--", "-D", "warnings"]

.woodpecker.yml

@@ -1,10 +0,0 @@
-pipeline:
-  test:
-    image: rust:latest
-    environment:
-      - CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
-    commands:
-      - rustup component add rustfmt clippy
-      - cargo fmt --all --check
-      - cargo clippy --all --all-features -- -D warnings
-      - cargo test --workspace

CHANGELOG.md

@@ -0,0 +1,396 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+
+## [v0.11.4](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.11.3..rustypipe/v0.11.4) - 2025-04-23
+
+### 🚀 Features
+
+- Player: handle VPN ban and captcha required error messages - ([be6da5e](https://codeberg.org/ThetaDev/rustypipe/commit/be6da5e7e3558ef39773bf45bcb8afbf006bacec))
+
+### 🐛 Bug Fixes
+
+- Deobfuscator: handle 1-char long global variables, find nsig fn (player 6450230e) - ([d675987](https://codeberg.org/ThetaDev/rustypipe/commit/d675987654972c6aa4cc2b291d25bc49fa60173e))
+
+
+## [v0.11.3](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.11.2..rustypipe/v0.11.3) - 2025-04-03
+
+### 🐛 Bug Fixes
+
+- Deobfuscator: global variable extraction fixed - ([ac44e95](https://codeberg.org/ThetaDev/rustypipe/commit/ac44e95a88d95f9d2d1ec672f86ca9d31d6991b9))
+- Deobfuscator: small simplification - ([189ba81](https://codeberg.org/ThetaDev/rustypipe/commit/189ba81a42e6c09f6af4d2768c449c22b864101e))
+- Deobfuscator: handle global functions as well - ([939a7ae](https://codeberg.org/ThetaDev/rustypipe/commit/939a7aea61a3eee4c1e67bfbfc835f0ce3934171))
+- Handle music playlist/album not found - ([ea80717](https://codeberg.org/ThetaDev/rustypipe/commit/ea80717f692b2c45b5063c362c9fa8ebca5a3471))
+- Switch client if no adaptive stream URLs were returned - ([187bf1c](https://codeberg.org/ThetaDev/rustypipe/commit/187bf1c9a0e846bff205e0d71a19c5a1ce7b1943))
+- Handle music artist not found - ([daf3d03](https://codeberg.org/ThetaDev/rustypipe/commit/daf3d035be38b59aef1ae205ac91c2bbdda2fe66))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rust crate rand to 0.9.0 - ([af415dd](https://codeberg.org/ThetaDev/rustypipe/commit/af415ddf8f94f00edb918f271d8e6336503e9faf))
+
+
+## [v0.11.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.11.1..rustypipe/v0.11.2) - 2025-03-24
+
+### 🐛 Bug Fixes
+
+- A/B test 22: commandExecutorCommand for playlist continuations - ([e8acbfb](https://codeberg.org/ThetaDev/rustypipe/commit/e8acbfbbcf5d31b5ac34410ddf334e5534e3762f))
+- Extract deobf data with global strings variable - ([4ce6746](https://codeberg.org/ThetaDev/rustypipe/commit/4ce6746be538564e79f7e3c67d7a91aaa53f48ea))
+- Handle player returning no adaptive stream URLs - ([07db7b1](https://codeberg.org/ThetaDev/rustypipe/commit/07db7b1166e912e1554f98f2ae20c2c356fed38f))
+
+
+## [v0.11.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.11.0..rustypipe/v0.11.1) - 2025-03-16
+
+### 🐛 Bug Fixes
+
+- Simplify get_player_from_clients logic - ([c04b606](https://codeberg.org/ThetaDev/rustypipe/commit/c04b60604d2628bf8f0e3de453c243adbb966e57))
+- Desktop client: generate PO token from user_syncid when authenticated - ([8342cae](https://codeberg.org/ThetaDev/rustypipe/commit/8342caeb0f566a38060a6ec69f3ca65b9a2afcd6))
+- Always skip failed clients - ([63a6f50](https://codeberg.org/ThetaDev/rustypipe/commit/63a6f50a8b5ad6bb984282335c1481ae3cd2fe83))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rust crate rstest to 0.25.0 - ([9ed1306](https://codeberg.org/ThetaDev/rustypipe/commit/9ed1306f3aaeb993c409997ddfbc47499e4f4d22))
+
+
+## [v0.11.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.10.0..rustypipe/v0.11.0) - 2025-02-26
+
+### 🚀 Features
+
+- Add original album track count, fix fetching albums with more than 200 tracks - ([544782f](https://codeberg.org/ThetaDev/rustypipe/commit/544782f8de728cda0aca9a1cb95837cdfbd001f1))
+
+### 🐛 Bug Fixes
+
+- A/B test 21: music album recommendations - ([6737512](https://codeberg.org/ThetaDev/rustypipe/commit/6737512f5f67c8cd05d4552dd0e0f24381035b35))
+
+
+## [v0.10.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.9.0..rustypipe/v0.10.0) - 2025-02-09
+
+### 🚀 Features
+
+- Add visitor data cache, remove random visitor data - ([b12f4c5](https://codeberg.org/ThetaDev/rustypipe/commit/b12f4c5d821a9189d7ed8410ad860824b6d052ef))
+- Add support for rustypipe-botguard to get PO tokens - ([b90a252](https://codeberg.org/ThetaDev/rustypipe/commit/b90a252a5e1bf05a5294168b0ec16a73cbb88f42))
+- Add session po token cache - ([b72b501](https://codeberg.org/ThetaDev/rustypipe/commit/b72b501b6dbcf4333b24cd80e7c8c61b0c21ec91))
+- Check rustypipe-botguard-api version - ([8385b87](https://codeberg.org/ThetaDev/rustypipe/commit/8385b87c63677f32a240679a78702f53072e517a))
+- Rewrite request attempt system, retry with different visitor data - ([dfd03ed](https://codeberg.org/ThetaDev/rustypipe/commit/dfd03edfadff2657e9cfbf04e5d313ba409520ac))
+- Log failed player fetch attempts with player_from_clients - ([8e35358](https://codeberg.org/ThetaDev/rustypipe/commit/8e35358c8941301f6ebf7646a11ab22711082569))
+- Add timezone query option - ([3a2370b](https://codeberg.org/ThetaDev/rustypipe/commit/3a2370b97ca3d0f40d72d66a23295557317d29fb))
+- [**breaking**] Add userdata feature for all personal data queries (playback history, subscriptions) - ([65cb424](https://codeberg.org/ThetaDev/rustypipe/commit/65cb4244c6ab547f53d0cb12af802c4189188c86))
+- Add RustyPipe::version_botguard fn, detect rustypipe-botguard in current dir, add botguard version to report - ([1d755b7](https://codeberg.org/ThetaDev/rustypipe/commit/1d755b76bf4569f7d0bb90a65494ac8e7aae499a))
+
+### 🐛 Bug Fixes
+
+- Parsing history dates - ([af7dc10](https://codeberg.org/ThetaDev/rustypipe/commit/af7dc1016322a87dd8fec0b739939c2b12b6f400))
+- A/V streams incorrectly recognized as video-only - ([2b891ca](https://codeberg.org/ThetaDev/rustypipe/commit/2b891ca0788f91f16dbb9203191cb3d2092ecc74))
+- Update iOS client - ([e915416](https://codeberg.org/ThetaDev/rustypipe/commit/e91541629d6c944c1001f5883e3c1264aeeb3969))
+- A/B test 20: music continuation item renderer - ([9c67f8f](https://codeberg.org/ThetaDev/rustypipe/commit/9c67f8f85bef8214848dc9d17bff6cff252e015e))
+- Include whole request body in report - ([15245c1](https://codeberg.org/ThetaDev/rustypipe/commit/15245c18b584e42523762b94fcc7284d483660a0))
+- Extracting nsig fn when outside variable starts with $ - ([eda16e3](https://codeberg.org/ThetaDev/rustypipe/commit/eda16e378730a3b57c4982a626df1622a93c574a))
+- Retry updating deobf data after a RustyPipe update - ([50ab1f7](https://codeberg.org/ThetaDev/rustypipe/commit/50ab1f7a5d8aeaa3720264b4a4b27805bb0e8121))
+- Allow player data to be fetched without botguard - ([29c854b](https://codeberg.org/ThetaDev/rustypipe/commit/29c854b20d7a6677415b1744e7ba7ecd4f594ea5))
+- Output full request body in reports, clean up `get_player_po_token` - ([a0d850f](https://codeberg.org/ThetaDev/rustypipe/commit/a0d850f8e01428a73bbd66397d0dbf797b45958f))
+- Correct timezone offset for parsed dates, add timezone_local option - ([a5a7be5](https://codeberg.org/ThetaDev/rustypipe/commit/a5a7be5b4e0a0b73d7e1dc802ebd7bd48dafc76d))
+- Use localzone crate to get local tz - ([5acbf0e](https://codeberg.org/ThetaDev/rustypipe/commit/5acbf0e456b1f10707e0a56125d993a8129eee3a))
+- Only use cached potokens with min. 10min lifetime - ([0c94267](https://codeberg.org/ThetaDev/rustypipe/commit/0c94267d0371b2b26c7b5c9abfa156d5cde2153e))
+
+### 📚 Documentation
+
+- Add Botguard info to README - ([9957add](https://codeberg.org/ThetaDev/rustypipe/commit/9957add2b5d6391b2c1869d2019fd7dd91b8cd41))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rust crate rquickjs to 0.9.0 (#33) - ([2c8ac41](https://codeberg.org/ThetaDev/rustypipe/commit/2c8ac410aa535d83f8bcc7181f81914b13bceb77))
+
+
+## [v0.9.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.8.0..rustypipe/v0.9.0) - 2025-01-16
+
+### 🚀 Features
+
+- Add functions to fetch a user's history and subscriptions - ([14e3995](https://codeberg.org/ThetaDev/rustypipe/commit/14e399594f97a1228a8c2991a14dd8745af1beb7))
+- Add history item dates, extend timeago parser - ([320a8c2](https://codeberg.org/ThetaDev/rustypipe/commit/320a8c2c24217ad5697f0424c4f994bbbe31f3aa))
+- Add session headers when using cookie auth - ([3c95b52](https://codeberg.org/ThetaDev/rustypipe/commit/3c95b52ceaf0df2d67ee0d2f2ac658f666f29836))
+- Add cookies.txt parser, add cookie auth + history cmds to CLI - ([cf498e4](https://codeberg.org/ThetaDev/rustypipe/commit/cf498e4a8f9318b0197bc3f0cbaf7043c53adb9d))
+- Add method to get saved_playlists - ([27f64fc](https://codeberg.org/ThetaDev/rustypipe/commit/27f64fc412e833d5bd19ad72913aae19358e98b9))
+- Extract player DRM data - ([2af4001](https://codeberg.org/ThetaDev/rustypipe/commit/2af4001c75f2ff4f7c891aa59ac22c2c6b7902a2))
+- Add Dolby audio codecs (ac-3, ec-3) - ([a7f8c78](https://codeberg.org/ThetaDev/rustypipe/commit/a7f8c789b1a34710274c4630e027ef868397aea2))
+- Add DRM and audio channel number filtering to StreamFilter - ([d5abee2](https://codeberg.org/ThetaDev/rustypipe/commit/d5abee275300ab1bc10fc8d6c35a4e3813fd2bd4))
+- Set cache file permissions to 600 - ([dee8a99](https://codeberg.org/ThetaDev/rustypipe/commit/dee8a99e7a8d071c987709a01f02ee8fecf2d776))
+
+### 🐛 Bug Fixes
+
+- Dont leak authorization and cookie header in reports - ([75fce91](https://codeberg.org/ThetaDev/rustypipe/commit/75fce91353c02cd498f27d21b08261c23ea03d70))
+- Require new time crate version which added Month::length - ([ec7a195](https://codeberg.org/ThetaDev/rustypipe/commit/ec7a195c98f39346c4c8db875212c3843580450e))
+- Parsing numbers (it), dates (kn) - ([63f86b6](https://codeberg.org/ThetaDev/rustypipe/commit/63f86b6e186aa1d2dcaf7e9169ccebb2265e5905))
+- Accept user-specific playlist ids (LL, WL) - ([97c3f30](https://codeberg.org/ThetaDev/rustypipe/commit/97c3f30d180d3e62b7e19f22d191d7fd7614daca))
+- Only use auth-enabled clients for fetching player with auth option enabled - ([2b2b4af](https://codeberg.org/ThetaDev/rustypipe/commit/2b2b4af0b26cdd0d4bf2218d3f527abd88658abf))
+- A/B test 19: Music artist album groups reordered - ([5daad1b](https://codeberg.org/ThetaDev/rustypipe/commit/5daad1b700e8dcf1f3e803db1685f08f27794898))
+- Switch to rquickjs crate for deobfuscator - ([75c3746](https://codeberg.org/ThetaDev/rustypipe/commit/75c3746890f3428f3314b7b10c9ec816ad275836))
+- Player_from_clients method not send/sync - ([9c512c3](https://codeberg.org/ThetaDev/rustypipe/commit/9c512c3c4dbec0fc3b973536733d61ba61125a92))
+
+### 📚 Documentation
+
+- Update README - ([0432477](https://codeberg.org/ThetaDev/rustypipe/commit/0432477451ecd5f64145d65239c721f4e44826c0))
+- Fix README - ([11442df](https://codeberg.org/ThetaDev/rustypipe/commit/11442dfd369599396357f5b7a7a4268a7b537f57))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rust crate rstest to 0.24.0 (#20) - ([ab19034](https://codeberg.org/ThetaDev/rustypipe/commit/ab19034ab19baf090e83eada056559676ffdadce))
+- *(deps)* Update rust crate dirs to v6 (#24) - ([6a60425](https://codeberg.org/ThetaDev/rustypipe/commit/6a604252b1af7a9388db5dc170f737069cc31051))
+- Update pre-commit hooks - ([7cd9246](https://codeberg.org/ThetaDev/rustypipe/commit/7cd9246260493d7839018cb39a2dfb4dded8b343))
+
+
+## [v0.8.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.7.2..rustypipe/v0.8.0) - 2024-12-20
+
+### 🚀 Features
+
+- Log warning when generating report - ([258f18a](https://codeberg.org/ThetaDev/rustypipe/commit/258f18a99d848ae7e6808beddad054037a3b3799))
+- Add auto-dubbed audio tracks, improved StreamFilter - ([1d1ae17](https://codeberg.org/ThetaDev/rustypipe/commit/1d1ae17ffc16724667d43142aa57abda2e6468e4))
+
+### 🐛 Bug Fixes
+
+- Replace deprecated call to `time::util::days_in_year_month` - ([69ef6ae](https://codeberg.org/ThetaDev/rustypipe/commit/69ef6ae51e9b09a9b9c06057e717bf6f054c9803))
+- Nsig fn extra variable extraction - ([8014741](https://codeberg.org/ThetaDev/rustypipe/commit/80147413ee3190bb530f8f6b02738bcc787a6444))
+- Deobf function extraction, allow $ in variable names - ([8cadbc1](https://codeberg.org/ThetaDev/rustypipe/commit/8cadbc1a4c865d085e30249dba0f353472456a32))
+- Remove leading zero-width-space from comments, ensure space after links - ([162959c](https://codeberg.org/ThetaDev/rustypipe/commit/162959ca4513a03496776fae905b4bf20c79899c))
+- Update client versions, enable Opus audio with iOS client - ([1b60c97](https://codeberg.org/ThetaDev/rustypipe/commit/1b60c97a183b9d74b92df14b5b113c61aba1be7f))
+- Extract transcript from comment voice replies - ([30f60c3](https://codeberg.org/ThetaDev/rustypipe/commit/30f60c30f9d87d39585db93c1c9e274f48d688ba))
+- Error 400 when fetching player with login - ([5ce84c4](https://codeberg.org/ThetaDev/rustypipe/commit/5ce84c44a6844f692258066c83e04df875e0aa91))
+
+### ⚙️ Miscellaneous Tasks
+
+- Update user agent - ([53e5846](https://codeberg.org/ThetaDev/rustypipe/commit/53e5846286e8db920622152c2a0a57ddc7c41d25))
+
+
+## [v0.7.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.7.1..rustypipe/v0.7.2) - 2024-12-13
+
+### 🐛 Bug Fixes
+
+- Replace futures dependency with futures-util - ([5c39bf4](https://codeberg.org/ThetaDev/rustypipe/commit/5c39bf4842b13d37a4277ea5506e15c179892ce5))
+- Lifetime-related lints - ([c4feff3](https://codeberg.org/ThetaDev/rustypipe/commit/c4feff37a5989097b575c43d89c26427d92d77b9))
+- Limit retry attempts to fetch client versions and deobf data - ([44ae456](https://codeberg.org/ThetaDev/rustypipe/commit/44ae456d2c654679837da8ec44932c44b1b01195))
+- Deobfuscation function extraction - ([f5437aa](https://codeberg.org/ThetaDev/rustypipe/commit/f5437aa127b2b7c5a08839643e30ea1ec989d30b))
+
+
+## [v0.7.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.7.0..rustypipe/v0.7.1) - 2024-11-25
+
+### 🐛 Bug Fixes
+
+- Disable Android client - ([a846b72](https://codeberg.org/ThetaDev/rustypipe/commit/a846b729e3519e3d5e62bdf028d9b48a7f8ea2ce))
+- A/B test 18: music playlist facepile avatar model - ([6c8108c](https://codeberg.org/ThetaDev/rustypipe/commit/6c8108c94acf9ca2336381bdca7c97b24a809521))
+
+### ⚙️ Miscellaneous Tasks
+
+- Add docs badge to README - ([706e881](https://codeberg.org/ThetaDev/rustypipe/commit/706e88134c0e94ce7d880735e9d31b3ff531a4f9))
+
+
+## [v0.7.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.6.0..rustypipe/v0.7.0) - 2024-11-10
+
+### 🚀 Features
+
+- Allow searching for YTM users - ([50010b7](https://codeberg.org/ThetaDev/rustypipe/commit/50010b7b0856d3ce05fe7a9d5989e526089bc2ef))
+- [**breaking**] Replace `TrackItem::is_video` attr with TrackType enum; serde lowercase AlbumType enum for consistency - ([044094a](https://codeberg.org/ThetaDev/rustypipe/commit/044094a4b70f05c46a459fa1597e23f4224b7b0b))
+
+### 🐛 Bug Fixes
+
+- Fetch unlocalized player data to interpret errors correctly; regression introduced with v0.6.0 - ([0919cbd](https://codeberg.org/ThetaDev/rustypipe/commit/0919cbd0dfe28ea00610c67a694e5f319e80635f))
+- A/B test 17: channel playlists lockupViewModel - ([342119d](https://codeberg.org/ThetaDev/rustypipe/commit/342119dba6f3dc2152eef1fc9841264a9e56b9f0))
+- [**breaking**] Serde: lowercase Verification enum - ([badb3ae](https://codeberg.org/ThetaDev/rustypipe/commit/badb3aef8249315909160b8ff73df3019f07cf97))
+- Parsing videos using LockupViewModel (Music video recommendations) - ([870ff79](https://codeberg.org/ThetaDev/rustypipe/commit/870ff79ee07dfab1f4f2be3a401cd5320ed587da))
+- Parsing lockup playlists with "MIX" instead of view count - ([ac8fbc3](https://codeberg.org/ThetaDev/rustypipe/commit/ac8fbc3e679819189e2791c323975acaf1b43035))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rust crate thiserror to v2 (#16) - ([e1e1687](https://codeberg.org/ThetaDev/rustypipe/commit/e1e1687605603686ac5fd5deeb6aa8fecaf92494))
+
+
+## [v0.6.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.5.0..rustypipe/v0.6.0) - 2024-10-28
+
+### 🚀 Features
+
+- [**breaking**] Remove TvHtml5Embed client as it got disabled - ([9e835c8](https://codeberg.org/ThetaDev/rustypipe/commit/9e835c8f38a3dd28c65561b2f9bb7a0f530c24f1))
+- [**breaking**] Generate random visitorData, remove `RustyPipeQuery::get_context` and `YTContext<'a>` from public API - ([7c4f44d](https://codeberg.org/ThetaDev/rustypipe/commit/7c4f44d09c4d813efff9e7d1059ddacd226b9e9d))
+- Add OAuth user login to access age-restricted videos - ([1cc3f9a](https://codeberg.org/ThetaDev/rustypipe/commit/1cc3f9ad74908d33e247ba6243103bfc22540164))
+- Add user_auth_logout method - ([9e2fe61](https://codeberg.org/ThetaDev/rustypipe/commit/9e2fe61267846ce216e0c498d8fa9ee672e03cbf))
+- Revoke OAuth token when logging out - ([62f8a92](https://codeberg.org/ThetaDev/rustypipe/commit/62f8a9210c23e1f02c711a2294af8766ca6b70e2))
+
+### 🐛 Bug Fixes
+
+- Skip serializing empty cache entries - ([be18d89](https://codeberg.org/ThetaDev/rustypipe/commit/be18d89ea65e35ddcf0f31bea3360e5db209fb9f))
+- Fetch artist albums continuation - ([b589061](https://codeberg.org/ThetaDev/rustypipe/commit/b589061a40245637b4fe619a26892291d87d25e6))
+- Update channel order tokens - ([79a6281](https://codeberg.org/ThetaDev/rustypipe/commit/79a62816ff62d94e5c706f45b1ce5971e5e58a81))
+- Handle auth errors - ([512223f](https://codeberg.org/ThetaDev/rustypipe/commit/512223fd83fb1ba2ba7ad96ed050a70bb7ec294d))
+- Use same visitor data for fetching artist album continuations - ([7b0499f](https://codeberg.org/ThetaDev/rustypipe/commit/7b0499f6b7cbf6ac4b83695adadfebb3f30349c7))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rust crate fancy-regex to 0.14.0 (#14) - ([94194e0](https://codeberg.org/ThetaDev/rustypipe/commit/94194e019c46ca49c343086e80e8eb75c52f4bc6))
+- *(deps)* Update rust crate quick-xml to 0.37.0 (#15) - ([0662b5c](https://codeberg.org/ThetaDev/rustypipe/commit/0662b5ccfccc922b28629f11ea52c3eb35f9efd2))
+
+
+## [v0.5.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.4.0..rustypipe/v0.5.0) - 2024-10-13
+
+### 🚀 Features
+
+- Add mobile client - ([71d3ec6](https://codeberg.org/ThetaDev/rustypipe/commit/71d3ec65ddafa966ef6b41cf4eb71687ba4b594c))
+
+### 🐛 Bug Fixes
+
+- Prioritize visitor_data argument before opts - ([ace0fae](https://codeberg.org/ThetaDev/rustypipe/commit/ace0fae1005217cd396000176e7c01682eae026f))
+- Ignore live tracks in YTM searches - ([f3f2e1d](https://codeberg.org/ThetaDev/rustypipe/commit/f3f2e1d3ca1e9c838c682356bb5a7ded6951c8e5))
+- A/B test 16 (pageHeaderRenderer on playlist pages) - ([e65f145](https://codeberg.org/ThetaDev/rustypipe/commit/e65f14556f3003fa59fee3f9f1410fb5ddf63219))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rust crate rstest to 0.23.0 (#12) - ([96776e9](https://codeberg.org/ThetaDev/rustypipe/commit/96776e98d76fa1d31d5f84dbceafbe8f9dfd9085))
+
+
+## [v0.4.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.3.0..rustypipe/v0.4.0) - 2024-09-10
+
+### 🚀 Features
+
+- Add RustyPipe version constant - ([7a019f5](https://codeberg.org/ThetaDev/rustypipe/commit/7a019f5706e19f7fe9f2e16e3b94d7b98cc8aca9))
+
+### 🐛 Bug Fixes
+
+- Show docs.rs feature flags - ([67a231d](https://codeberg.org/ThetaDev/rustypipe/commit/67a231d6d1b6427f500667729a59032f2b28cc65))
+- A/B test 15 (parsing channel shortsLockupViewModel) - ([7972df0](https://codeberg.org/ThetaDev/rustypipe/commit/7972df0df498edd7801e25037b9b2456367f9204))
+
+### 📚 Documentation
+
+- Fix license badge URL - ([4a253e1](https://codeberg.org/ThetaDev/rustypipe/commit/4a253e1a47317e9999e6ad31ac5c411956a0986a))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rust crate tokio to 1.20.4 [security] (#10) - ([a445e51](https://codeberg.org/ThetaDev/rustypipe/commit/a445e51b54a9afc44cd9657260a0b3d2abddbfa6))
+
+
+## [v0.3.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.2.1..rustypipe/v0.3.0) - 2024-08-18
+
+### 🚀 Features
+
+- Add client_type to VideoPlayer, simplify MapResponse trait - ([90540c6](https://codeberg.org/ThetaDev/rustypipe/commit/90540c6aaad658d4ce24ed41450d8509bac711bd))
+- Add http_client method to RustyPipe and user_agent method to RustyPipeQuery - ([3d6de53](https://codeberg.org/ThetaDev/rustypipe/commit/3d6de5354599ea691351e0ca161154e53f2e0b41))
+- Add channel_id and channel_name getters to YtEntity trait - ([bbbe9b4](https://codeberg.org/ThetaDev/rustypipe/commit/bbbe9b4b322c6b5b30764772e282c6823aeea524))
+- [**breaking**] Make StreamFilter use Vec internally, remove lifetime - ([821984b](https://codeberg.org/ThetaDev/rustypipe/commit/821984bbd51d65cf96b1d14087417ef968eaf9b2))
+- Overhauled downloader - ([11a0038](https://codeberg.org/ThetaDev/rustypipe/commit/11a00383502917cd98245c3da349107289ba3aa9))
+- Add player_from_clients function to specify client order - ([72b5dfe](https://codeberg.org/ThetaDev/rustypipe/commit/72b5dfec69ec25445b94cb0976662416a5df56ef))
+- [**breaking**] Add TV client - ([e608811](https://codeberg.org/ThetaDev/rustypipe/commit/e608811e5f5615416241e67561671330097092cb))
+- Downloader: add audio tagging - ([1e1315a](https://codeberg.org/ThetaDev/rustypipe/commit/1e1315a8378bd0ad25b5f1614e83dabc4a0b40d5))
+- Add audiotag+indicatif features to downloader - ([97fb057](https://codeberg.org/ThetaDev/rustypipe/commit/97fb0578b5c4954a596d8dee0c4b6e1d773a9300))
+- Add YtEntity trait to YouTubeItem and MusicItem - ([114a86a](https://codeberg.org/ThetaDev/rustypipe/commit/114a86a3823a175875aa2aeb31a61a6799ef13bc))
+- Change default player client order - ([97904d7](https://codeberg.org/ThetaDev/rustypipe/commit/97904d77374c2c937a49dc7905759c2d8e8ef9ae))
+- [**breaking**] Update channel model, addd handle + video_count, remove tv/mobile banner - ([e671570](https://codeberg.org/ThetaDev/rustypipe/commit/e6715700d950912031d5fbc1263f8770b6ffc49c))
+- [**breaking**] Add handle to ChannelItem, remove video_count - ([1cffb27](https://codeberg.org/ThetaDev/rustypipe/commit/1cffb27cc0b64929f9627f5839df2d73b81988a4))
+- [**breaking**] Remove startpage - ([3599aca](https://codeberg.org/ThetaDev/rustypipe/commit/3599acafef1a21fa6f8dea97902eb4a3fb048c14))
+
+### 🐛 Bug Fixes
+
+- [**breaking**] Extracting nsig function, remove field `throttled` from Video/Audio stream model - ([dd0565b](https://codeberg.org/ThetaDev/rustypipe/commit/dd0565ba98acb3289ed220fd2a3aaf86bb8b0788))
+- Make nsig_fn regex more generic - ([fb7af3b](https://codeberg.org/ThetaDev/rustypipe/commit/fb7af3b96698b452b6b24d1e094ba13a245cb83c))
+- Improve deobfuscator (support multiple nsig name matches, error if mapping all streams fails) - ([8152ce6](https://codeberg.org/ThetaDev/rustypipe/commit/8152ce6b088b57be9b8419b754aca93805e5f34d))
+- Nsig fn extraction - ([3c83e11](https://codeberg.org/ThetaDev/rustypipe/commit/3c83e11e753f8eb6efea5d453a7c819c487b3464))
+- Add var to deobf fn assignment - ([c6bd03f](https://codeberg.org/ThetaDev/rustypipe/commit/c6bd03fb70871ae1b764be18f88e86e71818fc56))
+- Make Verification enum exhaustive - ([d053ac3](https://codeberg.org/ThetaDev/rustypipe/commit/d053ac3eba810a7241df91f2f50bcbe1fd968c86))
+- Extraction error message - ([d36ba59](https://codeberg.org/ThetaDev/rustypipe/commit/d36ba595dab0bbaef1012ebfa8930fc0e6bf8167))
+- Set tracing instrumentation level to Error - ([9da3b25](https://codeberg.org/ThetaDev/rustypipe/commit/9da3b25be2b2577f7bd0282c09d10d368ac8b73f))
+- Detect ip-ban error message - ([da39c64](https://codeberg.org/ThetaDev/rustypipe/commit/da39c64f302bc2edc4214bbe25a0a9eb54063b09))
+- Player_from_clients: fall back to TvHtml5Embed client - ([d0ae796](https://codeberg.org/ThetaDev/rustypipe/commit/d0ae7961ba91d56c8b9a8d1c545875e869b818f5))
+- Parsing channels without banner - ([5a6b2c3](https://codeberg.org/ThetaDev/rustypipe/commit/5a6b2c3a621f6b20c1324ea8b9c03426e3d8018b))
+- Get TV client version - ([ee3ae40](https://codeberg.org/ThetaDev/rustypipe/commit/ee3ae40395263c5989784c7e00038ff13bc1151a))
+
+### ⚙️ Miscellaneous Tasks
+
+- Renovate: disable approveMajorUpdates - ([4743f9d](https://codeberg.org/ThetaDev/rustypipe/commit/4743f9d8e101b58ad6a43548495da9f4f381b9f4))
+- Renovate: disable scheduleDaily - ([015bd6f](https://codeberg.org/ThetaDev/rustypipe/commit/015bd6fcbf04163565fcb190b163ecfdb5664e11))
+- Renovate: enable automerge - ([882abc5](https://codeberg.org/ThetaDev/rustypipe/commit/882abc53ca894229ee78ec0edaa723d9ea61bbcb))
+- *(deps)* Update rust crate quick-xml to 0.36.0 (#8) - ([b6bc05c](https://codeberg.org/ThetaDev/rustypipe/commit/b6bc05c1f39da9a846b2e3d1d24bcbccb031203b))
+- *(deps)* Update rust crate rstest to 0.22.0 (#9) - ([abb7832](https://codeberg.org/ThetaDev/rustypipe/commit/abb783219aba4b492c1dff03c2148acf1f51a55d))
+- Change repo URL to Codeberg - ([1793331](https://codeberg.org/ThetaDev/rustypipe/commit/17933315d947f76d5fe1aa52abf7ea24c3ce6381))
+- Adjust dependency versions - ([70c6f8c](https://codeberg.org/ThetaDev/rustypipe/commit/70c6f8c3b97baefd316fff90cc727524516657af))
+
+### Todo
+
+- Update metadata - ([8692ca8](https://codeberg.org/ThetaDev/rustypipe/commit/8692ca81d972d0d2acf6fb4da79b9e0f5ebf4daf))
+
+
+## [v0.2.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.2.0..rustypipe/v0.2.1) - 2024-07-01
+
+### 🐛 Bug Fixes
+
+- *(deps)* Update quick-xml to v0.35.0 - ([298e4de](https://codeberg.org/ThetaDev/rustypipe/commit/298e4def93d1595fba91be103f014aa645a08937))
+
+
+## [v0.2.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.1.3..rustypipe/v0.2.0) - 2024-06-27
+
+### 🚀 Features
+
+- Add text formatting (bold/italic/strikethrough) - ([b8825f9](https://codeberg.org/ThetaDev/rustypipe/commit/b8825f9199365c873a4f0edd98a435e986b8daa2))
+- Prefix chip-style web links (social media) with the service name - ([6c41ef2](https://codeberg.org/ThetaDev/rustypipe/commit/6c41ef2fb2531e10a12c271e2d48504510a3b0bf))
+- Make get_visitor_data() public - ([da1d1bd](https://codeberg.org/ThetaDev/rustypipe/commit/da1d1bd2a0b214da10436ae221c90a0f88697b9a))
+- Add UnavailabilityReason: IpBan - ([401d4e8](https://codeberg.org/ThetaDev/rustypipe/commit/401d4e8255b1e86444319fed6d114dfbd0f80bbd))
+- Add YtEntity trait - ([792e3b3](https://codeberg.org/ThetaDev/rustypipe/commit/792e3b31e0101087a167935baad39a2e3b4296d0))
+
+### 🐛 Bug Fixes
+
+- Remove Innertube API keys, update android player params - ([a8fb337](https://codeberg.org/ThetaDev/rustypipe/commit/a8fb337fae9cb0112e0152f9a0a19ebae49c2a4d))
+- Parsing error when no `music_related` content available - ([8fbd6b9](https://codeberg.org/ThetaDev/rustypipe/commit/8fbd6b95b6f01108b46f53fe60a56b0c561e40c1))
+- Parsing audiobook type in European Portuguese - ([041ce2d](https://codeberg.org/ThetaDev/rustypipe/commit/041ce2d08f6021c88e8890034f551f7e01b2f012))
+- Renovate ci token - ([e0759eb](https://codeberg.org/ThetaDev/rustypipe/commit/e0759ebce32a5520245bb2c0cb920734b04ee7dc))
+
+### 🚜 Refactor
+
+- [**breaking**] Rename VideoItem/VideoPlayerDetails.length to duration for consistency - ([94e8d24](https://codeberg.org/ThetaDev/rustypipe/commit/94e8d24c6848b8bfca70dd03a7d89547ba9d6051))
+
+### 📚 Documentation
+
+- Add logo - ([6646078](https://codeberg.org/ThetaDev/rustypipe/commit/66460789449be0d5984cbdb6ec372e69323b7a88))
+
+### ⚙️ Miscellaneous Tasks
+
+- Changelog: fix incorrect version URLs - ([97b6f07](https://codeberg.org/ThetaDev/rustypipe/commit/97b6f07399e80e00a6c015d013e744568be125dd))
+- Update rstest to v0.19.0 - ([50fd1f0](https://codeberg.org/ThetaDev/rustypipe/commit/50fd1f08caf39c1298654e06059cc393543e925b))
+- Introduce MSRV - ([5dbb288](https://codeberg.org/ThetaDev/rustypipe/commit/5dbb288a496d53a299effa2026f5258af7b1f176))
+- Fix clippy lints - ([45b9f2a](https://codeberg.org/ThetaDev/rustypipe/commit/45b9f2a627b4e7075ba0b1c5f16efcc19aef7922))
+- Vscode: enable rss feature by default - ([e75ffbb](https://codeberg.org/ThetaDev/rustypipe/commit/e75ffbb5da6198086385ea96383ab9d0791592a5))
+- Configure Renovate (#3) - ([44c2deb](https://codeberg.org/ThetaDev/rustypipe/commit/44c2debea61f70c24ad6d827987e85e2132ed3d1))
+- *(deps)* Update rust crate tokio to 1.20.4 [security] (#4) - ([ce3ec34](https://codeberg.org/ThetaDev/rustypipe/commit/ce3ec34337b8acac41410ea39264aab7423d5801))
+- *(deps)* Update rust crate quick-xml to 0.34.0 (#5) - ([1e8a1af](https://codeberg.org/ThetaDev/rustypipe/commit/1e8a1af08c873cee7feadf63c2eff62753a78f64))
+- *(deps)* Update rust crate rstest to 0.21.0 (#7) - ([c3af918](https://codeberg.org/ThetaDev/rustypipe/commit/c3af918ba53c6230c0e4aef822a0cb2cf120bf3f))
+
+## [v0.1.3](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.1.2..rustypipe/v0.1.3) - 2024-04-01
+
+### 🐛 Bug Fixes
+
+- Parse new comment model (A/B#14 frameworkUpdates) - ([b0331f7](https://codeberg.org/ThetaDev/rustypipe/commit/b0331f7250f5d7d61a45209150739d2cb08b4280))
+
+### ◀️ Revert
+
+- "fix: improve VecLogErr messages" (leads to infinite loop) - ([348c852](https://codeberg.org/ThetaDev/rustypipe/commit/348c8523fe847f2f6ce98317375a7ab65e778ed2))
+
+
+## [v0.1.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.1.1..rustypipe/v0.1.2) - 2024-03-26
+
+### 🐛 Bug Fixes
+
+- Correctly parse subscriber count with new channel header - ([180dd98](https://codeberg.org/ThetaDev/rustypipe/commit/180dd9891a14b4da9f130a73d73aecc3822fce2f))
+
+
+## [v0.1.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.1.0..rustypipe/v0.1.1) - 2024-03-26
+
+### 🐛 Bug Fixes
+
+- Specify internal dependency versions - ([6598a23](https://codeberg.org/ThetaDev/rustypipe/commit/6598a23d0699e6fe298275a67e0146a19c422c88))
+- Move package attributes to workspace - ([e4b204e](https://codeberg.org/ThetaDev/rustypipe/commit/e4b204eae65f450471be0890b0198d2f30714b3b))
+- Parsing music details with video description tab - ([a81c3e8](https://codeberg.org/ThetaDev/rustypipe/commit/a81c3e83366fdf72d01dd3ee00fb2e831f7aaa26))
+
+### ⚙️ Miscellaneous Tasks
+
+- Changes to release command - ([0bcced1](https://codeberg.org/ThetaDev/rustypipe/commit/0bcced1db377198a54c9c7d03b8d038125a2bfe4))
+- Update user agent (FF 115.0) - ([be314d5](https://codeberg.org/ThetaDev/rustypipe/commit/be314d57ea1d99bfdc80649351ee3e7845541238))
+- Fix release script (unquoted include paths) - ([78ba9cb](https://codeberg.org/ThetaDev/rustypipe/commit/78ba9cb34c6bba3aba177583b242d3f76ea9847d))
+
+
+## [v0.1.0](https://codeberg.org/ThetaDev/rustypipe/commits/tag/rustypipe/v0.1.0) - 2024-03-22
+
+Initial release
+
+<!-- generated by git-cliff -->

Cargo.toml

@@ -1,64 +1,132 @@
 [package]
 name = "rustypipe"
-version = "0.1.0"
-edition = "2021"
-authors = ["ThetaDev <t.testboy@gmail.com>"]
-license = "GPL-3.0"
+version = "0.11.4"
+rust-version = "1.67.1"
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+keywords.workspace = true
+categories.workspace = true
 description = "Client for the public YouTube / YouTube Music API (Innertube), inspired by NewPipe"
-keywords = ["youtube", "video", "music"]
 
-include = ["/src", "README.md", "LICENSE", "!snapshots"]
+include = ["/src", "README.md", "CHANGELOG.md", "LICENSE", "!snapshots"]
 
 [workspace]
 members = [".", "codegen", "downloader", "cli"]
 
+[workspace.package]
+edition = "2021"
+authors = ["ThetaDev <thetadev@magenta.de>"]
+license = "GPL-3.0"
+repository = "https://codeberg.org/ThetaDev/rustypipe"
+keywords = ["youtube", "video", "music"]
+categories = ["api-bindings", "multimedia"]
+
+[workspace.dependencies]
+rquickjs = "0.9.0"
+once_cell = "1.12.0"
+regex = "1.6.0"
+fancy-regex = "0.14.0"
+thiserror = "2.0.0"
+url = "2.2.0"
+reqwest = { version = "0.12.0", default-features = false }
+tokio = "1.20.4"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0.82"
+serde_with = { version = "3.0.0", default-features = false, features = [
+    "alloc",
+    "macros",
+] }
+serde_plain = "1.0.0"
+sha1 = "0.10.0"
+rand = "0.9.0"
+time = { version = "0.3.37", features = [
+    "macros",
+    "serde-human-readable",
+    "serde-well-known",
+    "local-offset",
+] }
+futures-util = "0.3.31"
+ress = "0.11.0"
+phf = "0.11.0"
+phf_codegen = "0.11.0"
+data-encoding = "2.0.0"
+urlencoding = "2.1.0"
+quick-xml = { version = "0.37.0", features = ["serialize"] }
+tracing = { version = "0.1.0", features = ["log"] }
+localzone = "0.3.1"
+
+# CLI
+indicatif = "0.17.0"
+anyhow = "1.0"
+clap = { version = "4.0.0", features = ["derive"] }
+tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
+serde_yaml = "0.9.0"
+dirs = "6.0.0"
+filenamify = "0.1.0"
+
+# Testing
+rstest = "0.25.0"
+tokio-test = "0.4.2"
+insta = { version = "1.17.1", features = ["ron", "redactions"] }
+path_macro = "1.0.0"
+tracing-test = "0.2.5"
+
+# Included crates
+rustypipe = { path = ".", version = "0.11.4", default-features = false }
+rustypipe-downloader = { path = "./downloader", version = "0.3.1", default-features = false, features = [
+    "indicatif",
+    "audiotag",
+] }
+
 [features]
 default = ["default-tls"]
 
-rss = ["quick-xml"]
+rss = ["dep:quick-xml"]
+userdata = []
 
-# Reqwest TLS
+# Reqwest TLS options
 default-tls = ["reqwest/default-tls"]
+native-tls = ["reqwest/native-tls"]
+native-tls-alpn = ["reqwest/native-tls-alpn"]
+native-tls-vendored = ["reqwest/native-tls-vendored"]
 rustls-tls-webpki-roots = ["reqwest/rustls-tls-webpki-roots"]
 rustls-tls-native-roots = ["reqwest/rustls-tls-native-roots"]
 
 [dependencies]
-quick-js-dtp = { version = "0.4.1", default-features = false, features = [
-    "patch-dateparser",
-] }
-once_cell = "1.12.0"
-regex = "1.6.0"
-fancy-regex = "0.11.0"
-thiserror = "1.0.36"
-url = "2.2.2"
-log = "0.4.17"
-reqwest = { version = "0.11.11", default-features = false, features = [
-    "json",
-    "gzip",
-    "brotli",
-] }
-tokio = { version = "1.20.0", features = ["macros", "time"] }
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0.82"
-serde_with = { version = "2.0.0", features = ["json"] }
-rand = "0.8.5"
-time = { version = "0.3.15", features = [
-    "macros",
-    "serde",
-    "serde-well-known",
-] }
-futures = "0.3.21"
-ress = "0.11.4"
-phf = "0.11.1"
-base64 = "0.21.0"
-urlencoding = "2.1.2"
-quick-xml = { version = "0.28.1", features = ["serialize"], optional = true }
+rquickjs.workspace = true
+once_cell.workspace = true
+regex.workspace = true
+fancy-regex.workspace = true
+thiserror.workspace = true
+url.workspace = true
+reqwest = { workspace = true, features = ["json", "gzip", "brotli"] }
+tokio = { workspace = true, features = ["macros", "time", "process"] }
+serde.workspace = true
+serde_json.workspace = true
+serde_with.workspace = true
+serde_plain.workspace = true
+sha1.workspace = true
+rand.workspace = true
+time.workspace = true
+ress.workspace = true
+phf.workspace = true
+data-encoding.workspace = true
+urlencoding.workspace = true
+tracing.workspace = true
+localzone.workspace = true
+quick-xml = { workspace = true, optional = true }
 
 [dev-dependencies]
-env_logger = "0.10.0"
-test-log = "0.2.11"
-rstest = "0.17.0"
-temp_testdir = "0.2.3"
-tokio-test = "0.4.2"
-insta = { version = "1.17.1", features = ["ron", "redactions"] }
-path_macro = "1.0.0"
+rstest.workspace = true
+tokio-test.workspace = true
+insta.workspace = true
+path_macro.workspace = true
+tracing-test.workspace = true
+
+[package.metadata.docs.rs]
+# To build locally:
+# RUSTDOCFLAGS="--cfg docsrs" cargo +nightly doc --features rss,userdata --no-deps --open
+features = ["rss", "userdata"]
+rustdoc-args = ["--cfg", "docsrs"]

DEVELOPMENT.md

@@ -0,0 +1,26 @@
+## Development
+
+**Requirements:**
+
+- Current version of stable Rust
+- [`just`](https://github.com/casey/just) task runner
+- [`nextest`](https://nexte.st) test runner
+- [`pre-commit`](https://pre-commit.com/)
+- yq (YAML processor)
+
+### Tasks
+
+**Testing**
+
+- `just test` Run unit+integration tests
+- `just unittest` Run unit tests
+- `just testyt` Run YouTube integration tests
+- `just testintl` Run YouTube integration tests for all supported languages (this takes
+  a long time and is therefore not run in CI)
+- `YT_LANG=de just testyt` Run YouTube integration tests for a specific language
+
+**Tools**
+
+- `just testfiles` Download missing testfiles for unit tests
+- `just report2yaml` Convert RustyPipe reports into a more readable yaml format
+  (requires `yq`)

Justfile

@@ -1,23 +1,92 @@
 test:
-    cargo test --all-features
+    # cargo test --features=rss,userdata
+    cargo nextest run --workspace --features=rss,userdata --no-fail-fast --retries 1 -- --skip 'user_data::'
 
 unittest:
-    cargo test --all-features --lib
+    cargo nextest run --features=rss,userdata --no-fail-fast --lib
 
 testyt:
-    cargo test --all-features --test youtube
+    cargo nextest run --features=rss,userdata --no-fail-fast --retries 1 --test youtube -- --skip 'user_data::'
 
-testyt10:
+testyt-cookie:
+    cargo nextest run --features=rss,userdata --no-fail-fast --retries 1 --test youtube
+
+testyt-localized:
+    YT_LANG=th cargo nextest run --features=rss,userdata --no-fail-fast --retries 1 --test youtube -- \
+        --skip 'user_data::' --skip 'search_suggestion' --skip 'isrc_search_languages'
+
+testintl:
     #!/usr/bin/env bash
-    set -e
-    for i in {1..10}; do \
-        echo "---TEST RUN $i---"; \
-        cargo test --all-features --test youtube; \
+    LANGUAGES=(
+        "af" "am" "ar" "as" "az" "be" "bg" "bn" "bs" "ca" "cs" "da" "de" "el"
+        "en" "en-GB" "en-IN"
+        "es" "es-419" "es-US" "et" "eu" "fa" "fi" "fil" "fr" "fr-CA" "gl" "gu"
+        "hi" "hr" "hu" "hy" "id" "is" "it" "iw" "ja" "ka" "kk" "km" "kn" "ko" "ky"
+        "lo" "lt" "lv" "mk" "ml" "mn" "mr" "ms" "my" "ne" "nl" "no" "or" "pa" "pl"
+        "pt" "pt-PT" "ro" "ru" "si" "sk" "sl" "sq" "sr" "sr-Latn" "sv" "sw" "ta"
+        "te" "th" "tr" "uk" "ur" "uz" "vi" "zh-CN" "zh-HK" "zh-TW" "zu"
+    )
+
+    N_FAILED=0
+
+    for YT_LANG in "${LANGUAGES[@]}"; do
+        echo "---TESTS FOR $YT_LANG ---"
+
+        if YT_LANG="$YT_LANG" cargo nextest run --no-fail-fast --retries 1 --test-threads 4 --test youtube -- \
+            --skip 'user_data::' --skip 'search_suggestion' --skip 'isrc_search_languages' --skip 'resolve_'; then
+            echo "--- $YT_LANG COMPLETED ---"
+        else
+            echo "--- $YT_LANG FAILED ---"
+            ((N_FAILED++))
+        fi
     done
 
+    exit "$N_FAILED"
+
 testfiles:
-    cargo run -p rustypipe-codegen -- -d . download-testfiles
+    cargo run -p rustypipe-codegen download-testfiles
 
 report2yaml:
     mkdir -p rustypipe_reports/conv
-    for f in rustypipe_reports/*.json; do yq '.http_request.resp_body' $f | yq -o json -P > rustypipe_reports/conv/`basename $f .json`_body.json; yq e -Pi $f; mv $f rustypipe_reports/conv/`basename $f .json`.yaml; done;
+    for f in rustypipe_reports/*.json; do yq '.http_request.resp_body' $f | yq -o json -P > rustypipe_reports/conv/`basename $f .json`_body.json; yq e -Pi "del(.http_request.resp_body)" $f; mv $f rustypipe_reports/conv/`basename $f .json`.yaml; done;
+
+release crate="rustypipe":
+    #!/usr/bin/env bash
+    set -e
+
+    CRATE="{{crate}}"
+    CHANGELOG="CHANGELOG.md"
+
+    if [ "$CRATE" = "rustypipe" ]; then
+        INCLUDES="--exclude-path 'notes/**' --exclude-path 'cli/**' --exclude-path 'downloader/**'"
+    else
+        if [ ! -d "$CRATE" ]; then
+            echo "$CRATE does not exist."; exit 1
+        fi
+        INCLUDES="--include-path README.md --include-path LICENSE --include-path Cargo.toml --include-path '$CRATE/**'"
+        CHANGELOG="$CRATE/$CHANGELOG"
+        CRATE="rustypipe-$CRATE" # Add crate name prefix
+    fi
+
+    VERSION=$(cargo pkgid --package "$CRATE" | tr '#@' '\n' | tail -n 1)
+    TAG="${CRATE}/v${VERSION}"
+    echo "Releasing $TAG:"
+
+    if git rev-parse "$TAG" >/dev/null 2>&1; then echo "version tag $TAG already exists"; exit 1; fi
+
+    CLIFF_ARGS="--tag '${TAG}' --tag-pattern '${CRATE}/v*' --unreleased $INCLUDES"
+    echo "git-cliff $CLIFF_ARGS"
+    if [ -f "$CHANGELOG" ]; then
+        eval "git-cliff $CLIFF_ARGS --prepend '$CHANGELOG'"
+    else
+        eval "git-cliff $CLIFF_ARGS --output '$CHANGELOG'"
+    fi
+
+    editor "$CHANGELOG"
+
+    git add .
+    git commit -m "chore(release): release $CRATE v$VERSION"
+
+    awk 'BEGIN{RS="(^|\n)## [^\n]+\n*"} NR==2 { print }' "$CHANGELOG" | git tag -as -F - --cleanup whitespace "$TAG"
+
+    echo "🚀 Run 'git push origin $TAG' to publish"

README.md

@@ -1,31 +1,285 @@
-# RustyPipe
+# ![RustyPipe](https://codeberg.org/ThetaDev/rustypipe/raw/branch/main/notes/logo.svg)
 
-Client for the public YouTube / YouTube Music API (Innertube),
-inspired by [NewPipe](https://github.com/TeamNewPipe/NewPipeExtractor).
+[![Current crates.io version](https://img.shields.io/crates/v/rustypipe.svg)](https://crates.io/crates/rustypipe)
+[![License](https://img.shields.io/badge/License-GPL--3-blue.svg?style=flat)](https://opensource.org/licenses/GPL-3.0)
+[![Docs](https://img.shields.io/docsrs/rustypipe/latest?style=flat)](https://docs.rs/rustypipe)
+[![CI status](https://codeberg.org/ThetaDev/rustypipe/actions/workflows/ci.yaml/badge.svg?style=flat&label=CI)](https://codeberg.org/ThetaDev/rustypipe/actions/?workflow=ci.yaml)
+
+RustyPipe is a fully featured Rust client for the public YouTube / YouTube Music API
+(Innertube), inspired by [NewPipe](https://github.com/TeamNewPipe/NewPipeExtractor).
 
 ## Features
 
 ### YouTube
 
-- [X] **Player** (video/audio streams, subtitles)
-- [X] **Playlist**
-- [X] **VideoDetails** (metadata, comments, recommended videos)
-- [X] **Channel** (videos, shorts, livestreams, playlists, info, search)
-- [X] **ChannelRSS**
-- [X] **Search** (with filters)
-- [X] **Search suggestions**
-- [X] **Trending**
-- [X] **URL resolver**
+- **Player** (video/audio streams, subtitles)
+- **VideoDetails** (metadata, comments, recommended videos)
+- **Playlist**
+- **Channel** (videos, shorts, livestreams, playlists, info, search)
+- **ChannelRSS**
+- **Search** (with filters)
+- **Search suggestions**
+- **Trending**
+- **URL resolver**
+- **Subscriptions**
+- **Playback history**
 
 ### YouTube Music
 
-- [X] **Playlist**
-- [X] **Album**
-- [X] **Artist**
-- [X] **Search**
-- [X] **Search suggestions**
-- [X] **Radio**
-- [X] **Track details** (lyrics, recommendations)
-- [X] **Moods/Genres**
-- [X] **Charts**
-- [X] **New**
+- **Playlist**
+- **Album**
+- **Artist**
+- **Search**
+- **Search suggestions**
+- **Radio**
+- **Track details** (lyrics, recommendations)
+- **Moods/Genres**
+- **Charts**
+- **New** (albums, music videos)
+- **Saved items**
+- **Playback history**
+
+## Getting started
+
+The RustyPipe library works as follows: at first you have to instantiate a RustyPipe
+client. You can either create it with default options or use the `RustyPipe::builder()`
+to customize it.
+
+For fetching data you have to start with a new RustyPipe query object (`rp.query()`).
+The query object holds options for an individual query (e.g. content language or
+country). You can adjust these options with setter methods. Finally call your query
+method to fetch the data you need.
+
+All query methods are async, you need the tokio runtime to execute them.
+
+```rust ignore
+let rp = RustyPipe::new();
+let rp = RustyPipe::builder().storage_dir("/app/data").build().unwrap();
+let channel = rp.query().lang(Language::De).channel_videos("UCl2mFZoRqjw_ELax4Yisf6w").await.unwrap();
+```
+
+Here are a few examples to get you started:
+
+### Cargo.toml
+
+```toml
+[dependencies]
+rustypipe = "0.1.3"
+tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] }
+```
+
+### Watch a video
+
+```rust ignore
+use std::process::Command;
+
+use rustypipe::{client::RustyPipe, param::StreamFilter};
+
+#[tokio::main]
+async fn main() {
+    // Create a client
+    let rp = RustyPipe::new();
+    // Fetch the player
+    let player = rp.query().player("pPvd8UxmSbQ").await.unwrap();
+    // Select the best streams
+    let (video, audio) = player.select_video_audio_stream(&StreamFilter::default());
+
+    // Open mpv player
+    let mut args = vec![video.expect("no video stream").url.to_owned()];
+    if let Some(audio) = audio {
+        args.push(format!("--audio-file={}", audio.url));
+    }
+    Command::new("mpv").args(args).output().unwrap();
+}
+```
+
+### Get a playlist
+
+```rust ignore
+use rustypipe::client::RustyPipe
+
+#[tokio::main]
+async fn main() {
+    // Create a client
+    let rp = RustyPipe::new();
+    // Get the playlist
+    let playlist = rp
+        .query()
+        .playlist("PL2_OBreMn7FrsiSW0VDZjdq0xqUKkZYHT")
+        .await
+        .unwrap();
+    // Get all items (maximum: 1000)
+    playlist.videos.extend_limit(rp.query(), 1000).await.unwrap();
+
+    println!("Name: {}", playlist.name);
+    println!("Author: {}", playlist.channel.unwrap().name);
+    println!("Last update: {}", playlist.last_update.unwrap());
+
+    playlist
+        .videos
+        .items
+        .iter()
+        .for_each(|v| println!("[{}] {} ({}s)", v.id, v.name, v.length));
+}
+```
+
+**Output:**
+
+```txt
+Name: Homelab
+Author: Jeff Geerling
+Last update: 2023-05-04
+[cVWF3u-y-Zg] I put a computer in my computer (720s)
+[ecdm3oA-QdQ] 6-in-1: Build a 6-node Ceph cluster on this Mini ITX Motherboard (783s)
+[xvE4HNJZeIg] Scrapyard Server: Fastest all-SSD NAS! (733s)
+[RvnG-ywF6_s] Nanosecond clock sync with a Raspberry Pi (836s)
+[R2S2RMNv7OU] I made the Petabyte Raspberry Pi even faster! (572s)
+[FG--PtrDmw4] Hiding Macs in my Rack! (515s)
+...
+```
+
+### Get a channel
+
+```rust ignore
+use rustypipe::client::RustyPipe
+
+#[tokio::main]
+async fn main() {
+    // Create a client
+    let rp = RustyPipe::new();
+    // Get the channel
+    let channel = rp
+        .query()
+        .channel_videos("UCl2mFZoRqjw_ELax4Yisf6w")
+        .await
+        .unwrap();
+
+    println!("Name: {}", channel.name);
+    println!("Description: {}", channel.description);
+    println!("Subscribers: {}", channel.subscriber_count.unwrap());
+
+    channel
+        .content
+        .items
+        .iter()
+        .for_each(|v| println!("[{}] {} ({}s)", v.id, v.name, v.length.unwrap()));
+}
+```
+
+**Output:**
+
+```txt
+Name: Louis Rossmann
+Description: I discuss random things of interest to me. (...)
+Subscribers: 1780000
+[qBHgJx_rb8E] Introducing Rossmann senior, a genuine fossil 😃 (122s)
+[TmV8eAtXc3s] Am I wrong about CompTIA? (592s)
+[CjOJJc1qzdY] How FUTO projects loosen Google's grip on your life! (588s)
+[0A10JtkkL9A] a private moment between a man and his kitten (522s)
+[zbHq5_1Cd5U] Is Texas mandating auto repair shops use OEM parts? SB1083 analysis & breakdown; tldr, no. (645s)
+[6Fv8bd9ICb4] Who owns this? (199s)
+...
+```
+
+## Crate features
+
+Some features of RustyPipe are gated behind features to avoid compiling unneeded
+dependencies.
+
+- `rss` Fetch a channel's RSS feed, which is faster than fetching the channel page
+- `userdata` Add functions to fetch YouTube user data (watch history, subscriptions,
+  music library)
+
+You can also choose the TLS library used for making web requests using the same features
+as the reqwest crate (`default-tls`, `native-tls`, `native-tls-alpn`,
+`native-tls-vendored`, `rustls-tls-webpki-roots`, `rustls-tls-native-roots`).
+
+## Cache storage
+
+The RustyPipe cache holds the current version numbers for all clients, the JavaScript
+code used to deobfuscate video URLs and the authentication token/cookies. Never share
+the contents of the cache if you are using authentication.
+
+By default the cache is written to a JSON file named `rustypipe_cache.json` in the
+current working directory. This path can be changed with the `storage_dir` option of the
+RustyPipeBuilder. The RustyPipe CLI stores its cache in the userdata folder. The full
+path on Linux is `~/.local/share/rustypipe/rustypipe_cache.json`.
+
+You can integrate your own cache storage backend (e.g. database storage) by implementing
+the `CacheStorage` trait.
+
+## Reports
+
+RustyPipe has a builtin error reporting system. If a YouTube response cannot be
+deserialized or parsed, the original response data along with some request metadata is
+written to a JSON file in the folder `rustypipe_reports`, located in RustyPipe's storage
+directory (current folder by default, `~/.local/share/rustypipe` for the CLI).
+
+When submitting a bug report to the RustyPipe project, you can share this report to help
+resolve the issue.
+
+RustyPipe reports come in 3 severity levels:
+
+- DBG (no error occurred, report creation was enabled by the `RustyPipeQuery::report`
+  query option)
+- WRN (parts of the response could not be deserialized/parsed, response data may be
+  incomplete)
+- ERR (entire response could not be deserialized/parsed, RustyPipe returned an error)
+
+## PO tokens
+
+Since August 2024 YouTube requires PO tokens to access streams from web-based clients
+(Desktop, Mobile). Otherwise streams will return a 403 error.
+
+Generating PO tokens requires a simulated browser environment, which would be too large
+to include in RustyPipe directly.
+
+Therefore, the PO token generation is handled by a seperate CLI application
+([rustypipe-botguard](https://codeberg.org/ThetaDev/rustypipe-botguard)) which is called
+by the RustyPipe crate. RustyPipe automatically detects the rustypipe-botguard binary if
+it is located in PATH or the current working directory. If your rustypipe-botguard
+binary is located at a different path, you can specify it with the `.botguard_bin(path)`
+option.
+
+## Authentication
+
+RustyPipe supports authenticating with your YouTube account to access
+age-restricted/private videos and user information. There are 2 supported authentication
+methods: OAuth and cookies.
+
+To execute a query with authentication, use the `.authenticated()` query option. This
+option is enabled by default for queries that always require authentication like
+fetching user data. RustyPipe may automatically use authentication in case a video is
+age-restricted or your IP address is banned by YouTube. If you never want to use
+authentication, set the `.unauthenticated()` query option.
+
+### OAuth
+
+OAuth is the authentication method used by the YouTube TV client. It is more
+user-friendly than extracting cookies, however it only works with the TV client. This
+means that you can only fetch videos and not access any user data.
+
+To login using OAuth, you first have to get a new device code using the
+`rp.user_auth_get_code()` function. You can then enter the code on
+<https://google.com/device> and log in with your Google account. After generating the
+code, you can call the `rp.user_auth_wait_for_login()` function which waits until the
+user has logged in and stores the authentication token in the cache.
+
+### Cookies
+
+Authenticating with cookies allows you to use the functionality of the YouTube/YouTube
+Music Desktop client. You can fetch your subscribed channels, playlists and your music
+collection. You can also fetch videos using the Desktop client, including private
+videos, as long as you have access to them.
+
+To authenticate with cookies you have to log into YouTube in a fresh browser session
+(open Incognito/Private mode). Then extract the cookies from the developer tools or by
+using browser plugins like "Get cookies.txt LOCALLY"
+([Firefox](https://addons.mozilla.org/de/firefox/addon/get-cookies-txt-locally/))
+([Chromium](https://chromewebstore.google.com/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc)).
+Close the browser window after extracting the cookies to prevent YouTube from rotating
+the cookies.
+
+You can then add the cookies to your RustyPipe client using the `user_auth_set_cookie`
+or `user_auth_set_cookie_txt` function. The cookies are stored in the cache file. To log
+out, use the function `user_auth_remove_cookie`.

cli/CHANGELOG.md

@@ -0,0 +1,207 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+
+## [v0.7.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.7.1..rustypipe-cli/v0.7.2) - 2025-03-16
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rustypipe to 0.11.1
+- *(deps)* Update rustypipe-downloader to 0.3.1
+- *(deps)* Update rust crate rstest to 0.25.0 - ([9ed1306](https://codeberg.org/ThetaDev/rustypipe/commit/9ed1306f3aaeb993c409997ddfbc47499e4f4d22))
+
+
+## [v0.7.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.7.0..rustypipe-cli/v0.7.1) - 2025-02-26
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rustypipe to 0.11.0 - ([035c07f](https://codeberg.org/ThetaDev/rustypipe/commit/035c07f170aa293bcc626f27998c2b2b28660881))
+
+
+## [v0.7.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.6.0..rustypipe-cli/v0.7.0) - 2025-02-09
+
+### 🚀 Features
+
+- Add support for rustypipe-botguard to get PO tokens - ([b90a252](https://codeberg.org/ThetaDev/rustypipe/commit/b90a252a5e1bf05a5294168b0ec16a73cbb88f42))
+- [**breaking**] Remove manual PO token options from downloader/cli, add new rustypipe-botguard options - ([cddb32f](https://codeberg.org/ThetaDev/rustypipe/commit/cddb32f190276265258c6ab45b3d43a8891c4b39))
+- Add session po token cache - ([b72b501](https://codeberg.org/ThetaDev/rustypipe/commit/b72b501b6dbcf4333b24cd80e7c8c61b0c21ec91))
+- Add timezone query option - ([3a2370b](https://codeberg.org/ThetaDev/rustypipe/commit/3a2370b97ca3d0f40d72d66a23295557317d29fb))
+- Add --timezone-local CLI option - ([4f2bb47](https://codeberg.org/ThetaDev/rustypipe/commit/4f2bb47ab42ae0c68a64f3b3c2831fa7850b6f56))
+- Add verbose flag - ([629b590](https://codeberg.org/ThetaDev/rustypipe/commit/629b5905da653c6fe0f3c6b5814dd2f49030e7ed))
+
+### 🐛 Bug Fixes
+
+- Parsing mixed-case language codes like zh-CN - ([9c73ed4](https://codeberg.org/ThetaDev/rustypipe/commit/9c73ed4b3008cb093c0fa7fd94fd9f1ba8cd3627))
+
+### 🚜 Refactor
+
+- [**breaking**] Add client_type field to DownloadError, rename cli option po-token-cache to pot-cache - ([594e675](https://codeberg.org/ThetaDev/rustypipe/commit/594e675b39efc5fbcdbd5e920a4d2cdee64f718e))
+- Rename rustypipe-cli binary to rustypipe - ([c1a872e](https://codeberg.org/ThetaDev/rustypipe/commit/c1a872e1c14ea0956053bd7c65f6875b1cb3bc55))
+
+### 📚 Documentation
+
+- Add Botguard info to README - ([9957add](https://codeberg.org/ThetaDev/rustypipe/commit/9957add2b5d6391b2c1869d2019fd7dd91b8cd41))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rustypipe to 0.10.0
+- *(deps)* Update rust crate rquickjs to 0.9.0 (#33) - ([2c8ac41](https://codeberg.org/ThetaDev/rustypipe/commit/2c8ac410aa535d83f8bcc7181f81914b13bceb77))
+
+
+## [v0.6.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.5.0..rustypipe-cli/v0.6.0) - 2025-01-16
+
+### 🚀 Features
+
+- Add functions to fetch a user's history and subscriptions - ([14e3995](https://codeberg.org/ThetaDev/rustypipe/commit/14e399594f97a1228a8c2991a14dd8745af1beb7))
+- Add history item dates, extend timeago parser - ([320a8c2](https://codeberg.org/ThetaDev/rustypipe/commit/320a8c2c24217ad5697f0424c4f994bbbe31f3aa))
+- Add cookies.txt parser, add cookie auth + history cmds to CLI - ([cf498e4](https://codeberg.org/ThetaDev/rustypipe/commit/cf498e4a8f9318b0197bc3f0cbaf7043c53adb9d))
+- Add CLI commands to fetch user library and YTM releases/charts - ([a1b43ad](https://codeberg.org/ThetaDev/rustypipe/commit/a1b43ad70a66cfcbaba8ef302ac8699f243e56e7))
+- Export subscriptions as OPML / NewPipe JSON - ([c90d966](https://codeberg.org/ThetaDev/rustypipe/commit/c90d966b17eab24e957d980695888a459707055c))
+
+### 📚 Documentation
+
+- Update README - ([0432477](https://codeberg.org/ThetaDev/rustypipe/commit/0432477451ecd5f64145d65239c721f4e44826c0))
+- Fix README - ([11442df](https://codeberg.org/ThetaDev/rustypipe/commit/11442dfd369599396357f5b7a7a4268a7b537f57))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rust crate rstest to 0.24.0 (#20) - ([ab19034](https://codeberg.org/ThetaDev/rustypipe/commit/ab19034ab19baf090e83eada056559676ffdadce))
+- *(deps)* Update rust crate dirs to v6 (#24) - ([6a60425](https://codeberg.org/ThetaDev/rustypipe/commit/6a604252b1af7a9388db5dc170f737069cc31051))
+
+
+## [v0.5.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.4.0..rustypipe-cli/v0.5.0) - 2024-12-20
+
+### 🚀 Features
+
+- Get comment replies, rich text formatting - ([dceba44](https://codeberg.org/ThetaDev/rustypipe/commit/dceba442fe1a1d5d8d2a6d9422ff699593131f6d))
+
+### 🐛 Bug Fixes
+
+- Replace futures dependency with futures-util - ([5c39bf4](https://codeberg.org/ThetaDev/rustypipe/commit/5c39bf4842b13d37a4277ea5506e15c179892ce5))
+- Error 400 when fetching player with login - ([5ce84c4](https://codeberg.org/ThetaDev/rustypipe/commit/5ce84c44a6844f692258066c83e04df875e0aa91))
+
+### ⚙️ Miscellaneous Tasks
+
+- Add docs badge to README - ([706e881](https://codeberg.org/ThetaDev/rustypipe/commit/706e88134c0e94ce7d880735e9d31b3ff531a4f9))
+- *(deps)* Update rustypipe to 0.8.0
+
+
+## [v0.4.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.3.0..rustypipe-cli/v0.4.0) - 2024-11-10
+
+### 🚀 Features
+
+- Allow searching for YTM users - ([50010b7](https://codeberg.org/ThetaDev/rustypipe/commit/50010b7b0856d3ce05fe7a9d5989e526089bc2ef))
+- [**breaking**] Replace `TrackItem::is_video` attr with TrackType enum; serde lowercase AlbumType enum for consistency - ([044094a](https://codeberg.org/ThetaDev/rustypipe/commit/044094a4b70f05c46a459fa1597e23f4224b7b0b))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rust crate thiserror to v2 (#16) - ([e1e1687](https://codeberg.org/ThetaDev/rustypipe/commit/e1e1687605603686ac5fd5deeb6aa8fecaf92494))
+
+
+## [v0.3.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.2.2..rustypipe-cli/v0.3.0) - 2024-10-28
+
+### 🚀 Features
+
+- [**breaking**] Remove TvHtml5Embed client as it got disabled - ([9e835c8](https://codeberg.org/ThetaDev/rustypipe/commit/9e835c8f38a3dd28c65561b2f9bb7a0f530c24f1))
+- Add OAuth user login to access age-restricted videos - ([1cc3f9a](https://codeberg.org/ThetaDev/rustypipe/commit/1cc3f9ad74908d33e247ba6243103bfc22540164))
+- Revoke OAuth token when logging out - ([62f8a92](https://codeberg.org/ThetaDev/rustypipe/commit/62f8a9210c23e1f02c711a2294af8766ca6b70e2))
+
+
+## [v0.2.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.2.1..rustypipe-cli/v0.2.2) - 2024-10-13
+
+### 🚀 Features
+
+- Add mobile client - ([71d3ec6](https://codeberg.org/ThetaDev/rustypipe/commit/71d3ec65ddafa966ef6b41cf4eb71687ba4b594c))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rust crate rstest to 0.23.0 (#12) - ([96776e9](https://codeberg.org/ThetaDev/rustypipe/commit/96776e98d76fa1d31d5f84dbceafbe8f9dfd9085))
+- *(deps)* Update rustypipe to 0.5.0
+
+
+## [v0.2.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.2.0..rustypipe-cli/v0.2.1) - 2024-09-10
+
+### 🚀 Features
+
+- Add RustyPipe version constant - ([7a019f5](https://codeberg.org/ThetaDev/rustypipe/commit/7a019f5706e19f7fe9f2e16e3b94d7b98cc8aca9))
+
+### 📚 Documentation
+
+- Fix license badge URL - ([4a253e1](https://codeberg.org/ThetaDev/rustypipe/commit/4a253e1a47317e9999e6ad31ac5c411956a0986a))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rust crate tokio to 1.20.4 [security] (#10) - ([a445e51](https://codeberg.org/ThetaDev/rustypipe/commit/a445e51b54a9afc44cd9657260a0b3d2abddbfa6))
+
+
+## [v0.2.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.1.1..rustypipe-cli/v0.2.0) - 2024-08-18
+
+### 🚀 Features
+
+- Overhauled downloader - ([11a0038](https://codeberg.org/ThetaDev/rustypipe/commit/11a00383502917cd98245c3da349107289ba3aa9))
+- [**breaking**] Add TV client - ([e608811](https://codeberg.org/ThetaDev/rustypipe/commit/e608811e5f5615416241e67561671330097092cb))
+- Downloader: add audio tagging - ([1e1315a](https://codeberg.org/ThetaDev/rustypipe/commit/1e1315a8378bd0ad25b5f1614e83dabc4a0b40d5))
+- Downloader: add download_track fn, improve path templates - ([e1e4fb2](https://codeberg.org/ThetaDev/rustypipe/commit/e1e4fb29c190fec07f17c59ec88bef4f1c2a76a1))
+- Add audiotag+indicatif features to downloader - ([97fb057](https://codeberg.org/ThetaDev/rustypipe/commit/97fb0578b5c4954a596d8dee0c4b6e1d773a9300))
+- Add plaintext output to CLI - ([91b020e](https://codeberg.org/ThetaDev/rustypipe/commit/91b020efd498eff6e0f354a1de39439e252a79dd))
+- Add potoken option to downloader - ([904f821](https://codeberg.org/ThetaDev/rustypipe/commit/904f8215d84c810b04e4d2134718e786a4803ad2))
+- Print error message - ([8f16e5b](https://codeberg.org/ThetaDev/rustypipe/commit/8f16e5ba6eec3fd6aba1bb6a19571c65fb69ce0e))
+- Add list of clients to downloader - ([5e646af](https://codeberg.org/ThetaDev/rustypipe/commit/5e646afd1edc6c0101501311527ea56d3bad5fd2))
+- Retry with different client after 403 error - ([d875b54](https://codeberg.org/ThetaDev/rustypipe/commit/d875b5442de9822ba7ddc6f05789f56a8962808c))
+- Add option to fetch RSS feed - ([03c4d3c](https://codeberg.org/ThetaDev/rustypipe/commit/03c4d3c392386e06f2673f0e0783e22d10087989))
+- [**breaking**] Update channel model, addd handle + video_count, remove tv/mobile banner - ([e671570](https://codeberg.org/ThetaDev/rustypipe/commit/e6715700d950912031d5fbc1263f8770b6ffc49c))
+
+### 🐛 Bug Fixes
+
+- *(deps)* Update quick-xml to v0.35.0 - ([298e4de](https://codeberg.org/ThetaDev/rustypipe/commit/298e4def93d1595fba91be103f014aa645a08937))
+- Improve deobfuscator (support multiple nsig name matches, error if mapping all streams fails) - ([8152ce6](https://codeberg.org/ThetaDev/rustypipe/commit/8152ce6b088b57be9b8419b754aca93805e5f34d))
+- Cli: print video ID when logging errors - ([2c7a3fb](https://codeberg.org/ThetaDev/rustypipe/commit/2c7a3fb5cc153ff0b8b5e79234ae497d916e471c))
+- Use anstream + owo-color for colorful CLI output - ([e8324cf](https://codeberg.org/ThetaDev/rustypipe/commit/e8324cf3b065cb977adbc9529b1ef5ee18c3dd47))
+- Use native tls by default for CLI - ([f37432a](https://codeberg.org/ThetaDev/rustypipe/commit/f37432a48c1f93cab5f7942f791daf7b27cb1565))
+- Detect ip-ban error message - ([da39c64](https://codeberg.org/ThetaDev/rustypipe/commit/da39c64f302bc2edc4214bbe25a0a9eb54063b09))
+- Dont store cache in current dir with --report option - ([6009de7](https://codeberg.org/ThetaDev/rustypipe/commit/6009de7bddc6031f2af17005c473c17934327c02))
+- Show docs.rs feature flags - ([67a231d](https://codeberg.org/ThetaDev/rustypipe/commit/67a231d6d1b6427f500667729a59032f2b28cc65))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rust crate quick-xml to 0.36.0 (#8) - ([b6bc05c](https://codeberg.org/ThetaDev/rustypipe/commit/b6bc05c1f39da9a846b2e3d1d24bcbccb031203b))
+- *(deps)* Update rust crate rstest to 0.22.0 (#9) - ([abb7832](https://codeberg.org/ThetaDev/rustypipe/commit/abb783219aba4b492c1dff03c2148acf1f51a55d))
+- Change repo URL to Codeberg - ([1793331](https://codeberg.org/ThetaDev/rustypipe/commit/17933315d947f76d5fe1aa52abf7ea24c3ce6381))
+- Adjust dependency versions - ([70c6f8c](https://codeberg.org/ThetaDev/rustypipe/commit/70c6f8c3b97baefd316fff90cc727524516657af))
+
+### Todo
+
+- Update metadata - ([8692ca8](https://codeberg.org/ThetaDev/rustypipe/commit/8692ca81d972d0d2acf6fb4da79b9e0f5ebf4daf))
+
+
+## [v0.1.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.1.0..rustypipe-cli/v0.1.1) - 2024-06-27
+
+### 🚀 Features
+
+- CLI: setting player type - ([16e0e28](https://codeberg.org/ThetaDev/rustypipe/commit/16e0e28c4866bb69d8e4c06eef94176f329a1c27))
+
+### 🐛 Bug Fixes
+
+- Clippy warning - ([8420c2f](https://codeberg.org/ThetaDev/rustypipe/commit/8420c2f8dbd2791b524ceca2e19fb68e5b918bfa))
+
+### 📚 Documentation
+
+- Add logo - ([6646078](https://codeberg.org/ThetaDev/rustypipe/commit/66460789449be0d5984cbdb6ec372e69323b7a88))
+
+### ⚙️ Miscellaneous Tasks
+
+- Changelog: fix incorrect version URLs - ([97b6f07](https://codeberg.org/ThetaDev/rustypipe/commit/97b6f07399e80e00a6c015d013e744568be125dd))
+- Update rstest to v0.19.0 - ([50fd1f0](https://codeberg.org/ThetaDev/rustypipe/commit/50fd1f08caf39c1298654e06059cc393543e925b))
+- Introduce MSRV - ([5dbb288](https://codeberg.org/ThetaDev/rustypipe/commit/5dbb288a496d53a299effa2026f5258af7b1f176))
+- Fix clippy lints - ([45b9f2a](https://codeberg.org/ThetaDev/rustypipe/commit/45b9f2a627b4e7075ba0b1c5f16efcc19aef7922))
+- *(deps)* Update rust crate tokio to 1.20.4 [security] (#4) - ([ce3ec34](https://codeberg.org/ThetaDev/rustypipe/commit/ce3ec34337b8acac41410ea39264aab7423d5801))
+- *(deps)* Update rust crate quick-xml to 0.34.0 (#5) - ([1e8a1af](https://codeberg.org/ThetaDev/rustypipe/commit/1e8a1af08c873cee7feadf63c2eff62753a78f64))
+- *(deps)* Update rust crate rstest to 0.21.0 (#7) - ([c3af918](https://codeberg.org/ThetaDev/rustypipe/commit/c3af918ba53c6230c0e4aef822a0cb2cf120bf3f))
+- Update rustypipe to 0.2.0
+
+## [v0.1.0](https://codeberg.org/ThetaDev/rustypipe/commits/tag/rustypipe-cli/v0.1.0) - 2024-03-22
+
+Initial release
+
+<!-- generated by git-cliff -->

cli/Cargo.toml

@@ -1,18 +1,70 @@
 [package]
 name = "rustypipe-cli"
-version = "0.1.0"
-edition = "2021"
+version = "0.7.2"
+rust-version = "1.70.0"
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+keywords.workspace = true
+categories.workspace = true
+description = "CLI for RustyPipe - download videos and extract data from YouTube / YouTube Music"
+
+[features]
+default = ["native-tls"]
+timezone = ["dep:time", "dep:time-tz"]
+
+# Reqwest TLS options
+native-tls = [
+    "reqwest/native-tls",
+    "rustypipe/native-tls",
+    "rustypipe-downloader/native-tls",
+]
+native-tls-alpn = [
+    "reqwest/native-tls-alpn",
+    "rustypipe/native-tls-alpn",
+    "rustypipe-downloader/native-tls-alpn",
+]
+native-tls-vendored = [
+    "reqwest/native-tls-vendored",
+    "rustypipe/native-tls-vendored",
+    "rustypipe-downloader/native-tls-vendored",
+]
+rustls-tls-webpki-roots = [
+    "reqwest/rustls-tls-webpki-roots",
+    "rustypipe/rustls-tls-webpki-roots",
+    "rustypipe-downloader/rustls-tls-webpki-roots",
+]
+rustls-tls-native-roots = [
+    "reqwest/rustls-tls-native-roots",
+    "rustypipe/rustls-tls-native-roots",
+    "rustypipe-downloader/rustls-tls-native-roots",
+]
 
 [dependencies]
-rustypipe = { path = "../" }
-rustypipe-downloader = { path = "../downloader" }
-reqwest = { version = "0.11.11", default_features = false }
-tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] }
-indicatif = "0.17.0"
-futures = "0.3.21"
-anyhow = "1.0"
-clap = { version = "4.0.29", features = ["derive"] }
-env_logger = "0.10.0"
-serde = "1.0"
-serde_json = "1.0.82"
-serde_yaml = "0.9.19"
+rustypipe = { workspace = true, features = ["rss", "userdata"] }
+rustypipe-downloader.workspace = true
+reqwest.workspace = true
+tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
+futures-util.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+quick-xml.workspace = true
+time = { workspace = true, optional = true }
+time-tz = { version = "2.0.0", optional = true }
+
+indicatif.workspace = true
+anyhow.workspace = true
+clap.workspace = true
+tracing.workspace = true
+tracing-subscriber.workspace = true
+serde_yaml.workspace = true
+dirs.workspace = true
+
+anstream = "0.6.15"
+owo-colors = "4.0.0"
+const_format = "0.2.33"
+
+[[bin]]
+name = "rustypipe"
+path = "src/main.rs"

cli/README.md

@@ -0,0 +1,174 @@
+# ![RustyPipe](https://codeberg.org/ThetaDev/rustypipe/raw/branch/main/notes/logo.svg) CLI
+
+[![Current crates.io version](https://img.shields.io/crates/v/rustypipe-cli.svg)](https://crates.io/crates/rustypipe-cli)
+[![License](https://img.shields.io/badge/License-GPL--3-blue.svg?style=flat)](https://opensource.org/licenses/GPL-3.0)
+[![CI status](https://codeberg.org/ThetaDev/rustypipe/actions/workflows/ci.yaml/badge.svg?style=flat&label=CI)](https://codeberg.org/ThetaDev/rustypipe/actions/?workflow=ci.yaml)
+
+The RustyPipe CLI is a powerful YouTube client for the command line. It allows you to
+access most of the features of the RustyPipe crate: getting data from YouTube and
+downloading videos.
+
+## Installation
+
+You can download a compiled version of RustyPipe here:
+<https://codeberg.org/ThetaDev/rustypipe/releases>
+
+Alternatively, you can compile it yourself by installing [Rust](https://rustup.rs/) and
+running `cargo install rustypipe-cli`.
+
+To be able to access streams from web-based clients (Desktop, Mobile) you need to
+download [rustypipe-botguard](https://codeberg.org/ThetaDev/rustypipe-botguard/releases)
+and place the binary either in the PATH or the current working directory.
+
+For downloading videos you also need to have ffmpeg installed.
+
+## `get`: Fetch information
+
+You can call the get command with any YouTube entity ID or URL and RustyPipe will fetch
+the associated metadata. It can fetch channels, playlists, albums and videos.
+
+**Usage:** `rustypipe get UC2TXq_t06Hjdr2g_KdKpHQg`
+
+- `-l`, `--limit` Limit the number of list items to fetch
+- `-t`, `--tab` Channel tab (options: **videos**, shorts, live, playlists, info)
+- `-m, --music` Use the YouTube Music API
+- `--rss`Fetch the RSS feed of a channel
+- `--comments` Get comments (options: top, latest)
+- `--lyrics` Get the lyrics for YTM tracks
+- `--player` Get the player data instead of the video details when fetching videos
+- `-c`, `--client-type` YT clients used to fetch player data (options: desktop, tv,
+  tv-embed, android, ios; if multiple clients are specified, they are attempted in
+  order)
+
+## `search`: Search YouTube
+
+With the search command you can search the entire YouTube platform or individual
+channels. YouTube Music search is also supported.
+
+Note that search filters are only supported when searching YouTube. They have no effect
+when searching YTM or individual channels.
+
+**Usage:** `rustypipe search "query"`
+
+### Options
+
+- `-l`, `--limit` Limit the number of list items to fetch
+
+- `--item-type` Filter results by item type
+- `--length` Filter results by video length
+- `--date` Filter results by upload date (options: hour, day, week, month, year)
+- `--order` Sort search results (options: rating, date, views)
+- `--channel` Channel ID for searching channel videos
+- `-m`, `--music` Search YouTube Music in the given category (options: all, tracks,
+  videos, artists, albums, playlists-ytm, playlists-community)
+
+## `dl`: Download videos
+
+The downloader can download individual videos, playlists, albums and channels. Multiple
+videos can be downloaded in parallel for improved performance.
+
+**Usage:** `rustypipe dl eRsGyueVLvQ`
+
+### Options
+
+- `-o`, `--output` Download to the given directory
+- `--output-file` Download to the given file
+- `--template` Download to a path determined by a template
+
+- `-r`, `--resolution` Video resolution (e.g. 720, 1080). Set to 0 for audio-only
+- `-a`, `--audio` Download only the audio track and write track metadata + album cover
+- `-p`, `--parallel` Number of videos downloaded in parallel (default: 8)
+- `-m`, `--music` Use YouTube Music for downloading playlists
+- `-l`, `--limit` Limit the number of videos to download (default: 1000)
+- `-c`, `--client-type` YT clients used to fetch player data (options: desktop, tv,
+  tv-embed, android, ios; if multiple clients are specified, they are attempted in
+  order)
+
+## `vdata`: Get visitor data
+
+You can use the vdata command to get a new visitor data ID. This feature may come in
+handy for testing and reproducing A/B tests.
+
+## `releases` Get YouTube Music new releases
+
+Get a list of new albums or music videos on YouTube Music
+
+**Usage:** `rustypipe releases` or `rustypipe releases --videos`
+
+## `charts`: Get YouTube Music charts
+
+Get a list of the most popular tracks and artists for a given country
+
+**Usage:** `rustypipe charts DE`
+
+## `history`: Get YouTube playback history
+
+Get a list of recently played videos or tracks
+
+### Options
+
+- `-l`, `--limit` Limit the number of list items to fetch
+- `--search` Search the playback history (unavailable on YouTube Music)
+- `-m`, `--music` Get the YouTube Music playback history
+
+## `subscriptions`: Get subscribed channels
+
+You can use the RustyPipe CLI to get a list of the channels you subscribed to. With the
+`--format` flag you can export then in different formats, including OPML and NewPipe
+JSON.
+
+With the `--feed` option you can output a list of the latest videos from your
+subscription feed instead.
+
+### Options
+
+- `-l`, `--limit` Limit the number of list items to fetch
+- `-m`, `--music` Get a list of subscribed YouTube Music artists
+- `--feed` Output YouTube Music subscription feed
+
+## `playlists`, `albums`, `tracks`: Get your YouTube library
+
+Fetch a list of all the items saved in your YouTube/YouTube Music profile.
+
+### Options
+
+- `-l`, `--limit` Limit the number of list items to fetch
+- `-m`, `--music` (only for playlists): Get your YouTube Music playlists
+
+## Global options
+
+- **Proxy:** RustyPipe respects the environment variables `HTTP_PROXY`, `HTTPS_PROXY`
+  and `ALL_PROXY`
+- **Logging:** Enable debug logging with the `-v` (verbose) flag. If you want more
+  fine-grained control, use the `RUST_LOG` environment variable.
+- **Visitor data:** A custom visitor data ID can be used with the `--vdata` flag
+- **Authentication:** Use the commands `rustypipe login` and `rustypipe login --cookie`
+  to log into your Google account using either OAuth or YouTube cookies. With the
+  `--auth` flag you can use authentication for any request.
+- `--lang` Change the YouTube content language
+- `--country` Change the YouTube content country
+- `--tz` Use a specific
+  [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) (e.g.
+  Europe/Berlin, Australia/Sydney)
+
+  **Note:** this requires building rustypipe-cli with the `timezone` feature
+
+- `--local-tz` Use the local timezone instead of UTC
+- `--report` Generate a report on every request and store it in a `rustypipe_reports`
+  folder in the current directory
+- `--cache-file` Change the RustyPipe cache file location (Default:
+  `~/.local/share/rustypipe/rustypipe_cache.json`)
+- `--report-dir` Change the RustyPipe report directory location (Default:
+  `~/.local/share/rustypipe/rustypipe_reports`)
+- `--botguard-bin` Use a
+  [rustypipe-botguard](https://codeberg.org/ThetaDev/rustypipe-botguard) binary from the
+  given path for generating PO tokens
+- `--no-botguard` Disable Botguard, only download videos using clients that dont require
+  it
+- `--pot-cache` Enable caching for session-bound PO tokens
+
+### Output format
+
+By default, the CLI outputs YouTube data in a human-readable text format. If you want to
+store the data or process it with a script, you should choose a machine readable output
+format. You can choose both JSON and YAML with the `-f, --format` flag.

cliff.toml

@@ -0,0 +1,100 @@
+# git-cliff ~ default configuration file
+# https://git-cliff.org/docs/configuration
+#
+# Lines starting with "#" are comments.
+# Configuration options are organized into tables and keys.
+# See documentation for more information on available options.
+
+[changelog]
+# changelog header
+header = """
+# Changelog\n
+All notable changes to this project will be documented in this file.\n
+"""
+# template for the changelog body
+# https://keats.github.io/tera/docs/#introduction
+body = """
+{% set repo_url = "https://codeberg.org/ThetaDev/rustypipe" %}\
+{% if version %}\
+    {%set vname = version | split(pat="/") | last %}
+    {%if previous.version %}\
+        ## [{{ vname }}]({{ repo_url }}/compare/{{ previous.version }}..{{ version }})\
+    {% else %}\
+        ## [{{ vname }}]({{ repo_url }}/commits/tag/{{ version }})\
+    {% endif %} - {{ timestamp | date(format="%Y-%m-%d") }}
+{% else %}\
+    ## [unreleased]
+{% endif %}\
+{% if previous.version %}\
+{% for group, commits in commits | group_by(attribute="group") %}
+    ### {{ group | striptags | trim | upper_first }}
+    {% for commit in commits %}
+        - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\
+            {% if commit.breaking %}[**breaking**] {% endif %}\
+            {{ commit.message | upper_first }} - \
+            ([{{ commit.id | truncate(length=7, end="") }}]({{ repo_url }}/commit/{{ commit.id }}))\
+    {% endfor %}
+{% endfor %}\
+{% else %}
+Initial release
+{% endif %}\n
+"""
+# template for the changelog footer
+footer = """
+<!-- generated by git-cliff -->
+"""
+# remove the leading and trailing s
+trim = true
+# postprocessors
+postprocessors = [
+  # { pattern = '<REPO>', replace = "https://github.com/orhun/git-cliff" }, # replace repository URL
+]
+
+[git]
+# parse the commits based on https://www.conventionalcommits.org
+conventional_commits = true
+# filter out the commits that are not conventional
+filter_unconventional = true
+# process each line of a commit as an individual commit
+split_commits = false
+# regex for preprocessing the commit messages
+commit_preprocessors = [
+  # Replace issue numbers
+  #{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](<REPO>/issues/${2}))"},
+  # Check spelling of the commit with https://github.com/crate-ci/typos
+  # If the spelling is incorrect, it will be automatically fixed.
+  #{ pattern = '.*', replace_command = 'typos --write-changes -' },
+]
+# regex for parsing and grouping commits
+commit_parsers = [
+  { message = "^feat", group = "<!-- 0 -->🚀 Features" },
+  { message = "^fix", group = "<!-- 1 -->🐛 Bug Fixes" },
+  { message = "^doc", group = "<!-- 3 -->📚 Documentation" },
+  { message = "^perf", group = "<!-- 4 -->⚡ Performance" },
+  { message = "^refactor", group = "<!-- 2 -->🚜 Refactor" },
+  { message = "^style", group = "<!-- 5 -->🎨 Styling" },
+  { message = "^test", skip = true },
+  { message = "^chore\\(release\\)", skip = true },
+  { message = "^chore\\(pr\\)", skip = true },
+  { message = "^chore\\(pull\\)", skip = true },
+  { message = "^chore", group = "<!-- 7 -->⚙️ Miscellaneous Tasks" },
+  { message = "^ci", skip = true },
+  { body = ".*security", group = "<!-- 8 -->🛡️ Security" },
+  { message = "^revert", group = "<!-- 9 -->◀️ Revert" },
+]
+# protect breaking changes from being skipped due to matching a skipping commit_parser
+protect_breaking_commits = false
+# filter out the commits that are not matched by commit parsers
+filter_commits = false
+# regex for matching git tags
+# tag_pattern = "v[0-9].*"
+# regex for skipping tags
+# skip_tags = ""
+# regex for ignoring tags
+# ignore_tags = ""
+# sort the tags topologically
+topo_order = false
+# sort the commits inside sections by oldest/newest order
+sort_commits = "oldest"
+# limit the number of commits included in the changelog.
+# limit_commits = 42

codegen/Cargo.toml

@@ -1,23 +1,33 @@
 [package]
 name = "rustypipe-codegen"
 version = "0.1.0"
-edition = "2021"
+rust-version = "1.74.0"
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+publish = false
 
 [dependencies]
-rustypipe = { path = "../" }
-reqwest = "0.11.11"
-tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] }
-futures = "0.3.21"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0.82"
-serde_with = "2.0.0"
-anyhow = "1.0"
-log = "0.4.17"
-env_logger = "0.10.0"
-clap = { version = "4.0.29", features = ["derive"] }
-phf_codegen = "0.11.1"
-once_cell = "1.12.0"
-regex = "1.7.1"
-indicatif = "0.17.0"
-num_enum = "0.5.7"
-path_macro = "1.0.0"
+rustypipe = { path = "../", features = ["userdata"] }
+reqwest.workspace = true
+tokio = { workspace = true, features = ["rt-multi-thread"] }
+futures-util.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+serde_plain.workspace = true
+serde_with.workspace = true
+once_cell.workspace = true
+regex.workspace = true
+path_macro.workspace = true
+anyhow.workspace = true
+tracing.workspace = true
+tracing-subscriber.workspace = true
+clap.workspace = true
+phf_codegen.workspace = true
+indicatif.workspace = true
+
+num_enum = "0.7.2"
+intl_pluralrules = "7.0.2"
+unic-langid = "0.9.1"
+ordered_hash_map = { version = "0.4.0", features = ["serde"] }

codegen/src/abtest.rs

@@ -1,12 +1,20 @@
+use std::collections::BTreeMap;
+
 use anyhow::{bail, Result};
-use futures::{stream, StreamExt};
+use futures_util::{stream, StreamExt};
 use indicatif::{ProgressBar, ProgressStyle};
 use num_enum::TryFromPrimitive;
-use rustypipe::client::{ClientType, RustyPipe, YTContext};
-use rustypipe::model::YouTubeItem;
+use once_cell::sync::Lazy;
+use regex::Regex;
+use rustypipe::client::{ClientType, RustyPipe, RustyPipeQuery};
+use rustypipe::model::{MusicItem, YouTubeItem};
 use rustypipe::param::search_filter::{ItemType, SearchFilter};
+use rustypipe::param::ChannelVideoTab;
+use serde::de::IgnoredAny;
 use serde::{Deserialize, Serialize};
 
+use crate::model::QCont;
+
 #[derive(
     Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, TryFromPrimitive, Serialize, Deserialize,
 )]
@@ -16,9 +24,32 @@ pub enum ABTest {
     ThreeTabChannelLayout = 2,
     ChannelHandlesInSearchResults = 3,
     TrendsVideoTab = 4,
+    TrendsPageHeaderRenderer = 5,
+    DiscographyPage = 6,
+    ShortDateFormat = 7,
+    TrackViewcount = 8,
+    PlaylistsForShorts = 9,
+    ChannelAboutModal = 10,
+    LikeButtonViewmodel = 11,
+    ChannelPageHeader = 12,
+    MusicPlaylistTwoColumn = 13,
+    CommentsFrameworkUpdate = 14,
+    ChannelShortsLockup = 15,
+    PlaylistPageHeader = 16,
+    ChannelPlaylistsLockup = 17,
+    MusicPlaylistFacepile = 18,
+    MusicAlbumGroupsReordered = 19,
+    MusicContinuationItemRenderer = 20,
+    AlbumRecommends = 21,
+    CommandExecutorCommand = 22,
 }
 
-const TESTS_TO_RUN: [ABTest; 1] = [ABTest::TrendsVideoTab];
+/// List of active A/B tests that are run when none is manually specified
+const TESTS_TO_RUN: &[ABTest] = &[
+    ABTest::MusicAlbumGroupsReordered,
+    ABTest::AlbumRecommends,
+    ABTest::CommandExecutorCommand,
+];
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct ABTestRes {
@@ -32,7 +63,6 @@ pub struct ABTestRes {
 
 #[derive(Debug, Serialize)]
 struct QVideo<'a> {
-    context: YTContext<'a>,
     video_id: &'a str,
     content_check_ok: bool,
     racy_check_ok: bool,
@@ -41,7 +71,6 @@ struct QVideo<'a> {
 #[derive(Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
 struct QBrowse<'a> {
-    context: YTContext<'a>,
     browse_id: &'a str,
     #[serde(skip_serializing_if = "Option::is_none")]
     params: Option<&'a str>,
@@ -56,7 +85,6 @@ pub async fn run_test(
 
     let rp = RustyPipe::new();
     let pb = ProgressBar::new(n as u64);
-    let http = reqwest::Client::default();
     pb.set_style(
         ProgressStyle::with_template(
             "{msg} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {pos}/{len}",
@@ -68,20 +96,36 @@ pub async fn run_test(
         .map(|_| {
             let rp = rp.clone();
             let pb = pb.clone();
-            let http = http.clone();
             async move {
-                let visitor_data = get_visitor_data(&http).await;
+                let visitor_data = rp.query().get_visitor_data(true).await.unwrap();
+                let query = rp.query().visitor_data(&visitor_data);
                 let is_present = match ab {
-                    ABTest::AttributedTextDescription => {
-                        attributed_text_description(&rp, &visitor_data).await
-                    }
-                    ABTest::ThreeTabChannelLayout => {
-                        three_tab_channel_layout(&rp, &visitor_data).await
-                    }
+                    ABTest::AttributedTextDescription => attributed_text_description(&query).await,
+                    ABTest::ThreeTabChannelLayout => three_tab_channel_layout(&query).await,
                     ABTest::ChannelHandlesInSearchResults => {
-                        channel_handles_in_search_results(&rp, &visitor_data).await
+                        channel_handles_in_search_results(&query).await
                     }
-                    ABTest::TrendsVideoTab => trends_video_tab(&rp, &visitor_data).await,
+                    ABTest::TrendsVideoTab => trends_video_tab(&query).await,
+                    ABTest::TrendsPageHeaderRenderer => trends_page_header_renderer(&query).await,
+                    ABTest::DiscographyPage => discography_page(&query).await,
+                    ABTest::ShortDateFormat => short_date_format(&query).await,
+                    ABTest::PlaylistsForShorts => playlists_for_shorts(&query).await,
+                    ABTest::TrackViewcount => track_viewcount(&query).await,
+                    ABTest::ChannelAboutModal => channel_about_modal(&query).await,
+                    ABTest::LikeButtonViewmodel => like_button_viewmodel(&query).await,
+                    ABTest::ChannelPageHeader => channel_page_header(&query).await,
+                    ABTest::MusicPlaylistTwoColumn => music_playlist_two_column(&query).await,
+                    ABTest::CommentsFrameworkUpdate => comments_framework_update(&query).await,
+                    ABTest::ChannelShortsLockup => channel_shorts_lockup(&query).await,
+                    ABTest::PlaylistPageHeader => playlist_page_header_renderer(&query).await,
+                    ABTest::ChannelPlaylistsLockup => channel_playlists_lockup(&query).await,
+                    ABTest::MusicPlaylistFacepile => music_playlist_facepile(&query).await,
+                    ABTest::MusicAlbumGroupsReordered => music_album_groups_reordered(&query).await,
+                    ABTest::MusicContinuationItemRenderer => {
+                        music_continuation_item_renderer(&query).await
+                    }
+                    ABTest::AlbumRecommends => album_recommends(&query).await,
+                    ABTest::CommandExecutorCommand => command_executor_command(&query).await,
                 }
                 .unwrap();
                 pb.inc(1);
@@ -95,38 +139,22 @@ pub async fn run_test(
     let count = results.iter().filter(|(p, _)| *p).count();
     let vd_present = results
         .iter()
-        .find_map(|(p, vd)| if *p { Some(vd.to_owned()) } else { None });
+        .find_map(|(p, vd)| if *p { Some(vd.clone()) } else { None });
     let vd_absent = results
         .iter()
-        .find_map(|(p, vd)| if !*p { Some(vd.to_owned()) } else { None });
+        .find_map(|(p, vd)| if *p { None } else { Some(vd.clone()) });
 
     (count, vd_present, vd_absent)
 }
 
-async fn get_visitor_data(http: &reqwest::Client) -> String {
-    let resp = http.get("https://www.youtube.com").send().await.unwrap();
-    resp.headers()
-        .get_all(reqwest::header::SET_COOKIE)
-        .iter()
-        .find_map(|c| {
-            if let Ok(cookie) = c.to_str() {
-                if let Some(after) = cookie.strip_prefix("__Secure-YEC=") {
-                    return after.split_once(';').map(|s| s.0.to_owned());
-                }
-            }
-            None
-        })
-        .unwrap()
-}
-
 pub async fn run_all_tests(n: usize, concurrency: usize) -> Vec<ABTestRes> {
     let mut results = Vec::new();
 
     for ab in TESTS_TO_RUN {
-        let (occurrences, vd_present, vd_absent) = run_test(ab, n, concurrency).await;
+        let (occurrences, vd_present, vd_absent) = run_test(*ab, n, concurrency).await;
         results.push(ABTestRes {
-            id: ab as u16,
-            name: ab,
+            id: *ab as u16,
+            name: *ab,
             tests: n,
             occurrences,
             vd_present,
@@ -136,18 +164,13 @@ pub async fn run_all_tests(n: usize, concurrency: usize) -> Vec<ABTestRes> {
     results
 }
 
-pub async fn attributed_text_description(rp: &RustyPipe, visitor_data: &str) -> Result<bool> {
-    let query = rp.query();
-    let context = query
-        .get_context(ClientType::Desktop, true, Some(visitor_data))
-        .await;
+pub async fn attributed_text_description(rp: &RustyPipeQuery) -> Result<bool> {
     let q = QVideo {
-        context,
         video_id: "ZeerrnuLi5E",
         content_check_ok: false,
         racy_check_ok: false,
     };
-    let response_txt = query.raw(ClientType::Desktop, "next", &q).await.unwrap();
+    let response_txt = rp.raw(ClientType::Desktop, "next", &q).await?;
 
     if !response_txt.contains("\"Black Mamba\"") {
         bail!("invalid response data");
@@ -156,20 +179,13 @@ pub async fn attributed_text_description(rp: &RustyPipe, visitor_data: &str) ->
     Ok(response_txt.contains("\"attributedDescription\""))
 }
 
-pub async fn three_tab_channel_layout(rp: &RustyPipe, visitor_data: &str) -> Result<bool> {
-    let channel = rp
-        .query()
-        .visitor_data(visitor_data)
-        .channel_videos("UCR-DXc1voovS8nhAvccRZhg")
-        .await
-        .unwrap();
+pub async fn three_tab_channel_layout(rp: &RustyPipeQuery) -> Result<bool> {
+    let channel = rp.channel_videos("UCR-DXc1voovS8nhAvccRZhg").await?;
     Ok(channel.has_live || channel.has_shorts)
 }
 
-pub async fn channel_handles_in_search_results(rp: &RustyPipe, visitor_data: &str) -> Result<bool> {
+pub async fn channel_handles_in_search_results(rp: &RustyPipeQuery) -> Result<bool> {
     let search = rp
-        .query()
-        .visitor_data(visitor_data)
         .search_filter("rust", &SearchFilter::new().item_type(ItemType::Channel))
         .await
         .unwrap();
@@ -177,21 +193,18 @@ pub async fn channel_handles_in_search_results(rp: &RustyPipe, visitor_data: &st
     Ok(search.items.items.iter().any(|itm| match itm {
         YouTubeItem::Channel(channel) => channel
             .subscriber_count
-            .map(|sc| sc > 100 && channel.video_count.is_none())
+            .map(|sc| sc > 100 && channel.handle.is_some())
             .unwrap_or_default(),
         _ => false,
     }))
 }
 
-pub async fn trends_video_tab(rp: &RustyPipe, visitor_data: &str) -> Result<bool> {
-    let query = rp.query().visitor_data(visitor_data);
-    let context = query.get_context(ClientType::Desktop, true, None).await;
-    let res = query
+pub async fn trends_video_tab(rp: &RustyPipeQuery) -> Result<bool> {
+    let res = rp
         .raw(
             ClientType::Desktop,
             "browse",
             &QBrowse {
-                context,
                 browse_id: "FEtrending",
                 params: None,
             },
@@ -200,3 +213,268 @@ pub async fn trends_video_tab(rp: &RustyPipe, visitor_data: &str) -> Result<bool
 
     Ok(res.contains("\"4gIOGgxtb3N0X3BvcHVsYXI%3D\""))
 }
+
+pub async fn trends_page_header_renderer(rp: &RustyPipeQuery) -> Result<bool> {
+    let res = rp
+        .raw(
+            ClientType::Desktop,
+            "browse",
+            &QBrowse {
+                browse_id: "FEtrending",
+                params: None,
+            },
+        )
+        .await?;
+
+    #[derive(Debug, Deserialize)]
+    struct D {
+        header: BTreeMap<String, IgnoredAny>,
+    }
+
+    let data = serde_json::from_str::<D>(&res)?;
+
+    Ok(data.header.contains_key("pageHeaderRenderer"))
+}
+
+pub async fn discography_page(rp: &RustyPipeQuery) -> Result<bool> {
+    let id = "UC7cl4MmM6ZZ2TcFyMk_b4pg";
+    let res = rp
+        .raw(
+            ClientType::DesktopMusic,
+            "browse",
+            &QBrowse {
+                browse_id: id,
+                params: None,
+            },
+        )
+        .await?;
+    Ok(res.contains(&format!("\"MPAD{id}\"")))
+}
+
+pub async fn short_date_format(rp: &RustyPipeQuery) -> Result<bool> {
+    static SHORT_DATE: Lazy<Regex> = Lazy::new(|| Regex::new("\\d(?:y|mo|w|d|h|min) ").unwrap());
+    let channel = rp.channel_videos("UC2DjFE7Xf11URZqWBigcVOQ").await?;
+
+    Ok(channel.content.items.iter().any(|itm| {
+        itm.publish_date_txt
+            .as_deref()
+            .map(|d| SHORT_DATE.is_match(d))
+            .unwrap_or_default()
+    }))
+}
+
+pub async fn playlists_for_shorts(rp: &RustyPipeQuery) -> Result<bool> {
+    let playlist = rp.playlist("UUSHh8gHdtzO2tXd593_bjErWg").await?;
+    let v1 = playlist
+        .videos
+        .items
+        .first()
+        .ok_or_else(|| anyhow::anyhow!("no videos"))?;
+    Ok(v1.publish_date_txt.is_none())
+}
+
+pub async fn track_viewcount(rp: &RustyPipeQuery) -> Result<bool> {
+    let res = rp.music_search_main("lieblingsmensch namika").await?;
+
+    let track = &res
+        .items
+        .items
+        .iter()
+        .find_map(|itm| {
+            if let MusicItem::Track(track) = itm {
+                if track.id == "6485PhOtHzY" {
+                    Some(track)
+                } else {
+                    None
+                }
+            } else {
+                None
+            }
+        })
+        .unwrap_or_else(|| {
+            panic!("could not find track, got {:#?}", &res.items.items);
+        });
+
+    Ok(track.view_count.is_some())
+}
+
+pub async fn channel_about_modal(rp: &RustyPipeQuery) -> Result<bool> {
+    let id = "UC2DjFE7Xf11URZqWBigcVOQ";
+    let res = rp
+        .raw(
+            ClientType::Desktop,
+            "browse",
+            &QBrowse {
+                browse_id: id,
+                params: None,
+            },
+        )
+        .await?;
+    Ok(!res.contains("\"EgVhYm91dPIGBAoCEgA%3D\""))
+}
+
+pub async fn like_button_viewmodel(rp: &RustyPipeQuery) -> Result<bool> {
+    let res = rp
+        .raw(
+            ClientType::Desktop,
+            "next",
+            &QVideo {
+                video_id: "ZeerrnuLi5E",
+                content_check_ok: true,
+                racy_check_ok: true,
+            },
+        )
+        .await?;
+    Ok(res.contains("\"segmentedLikeDislikeButtonViewModel\""))
+}
+
+pub async fn channel_page_header(rp: &RustyPipeQuery) -> Result<bool> {
+    let channel = rp
+        .channel_videos_tab("UCh8gHdtzO2tXd593_bjErWg", ChannelVideoTab::Shorts)
+        .await?;
+    Ok(channel.video_count.is_some())
+}
+
+pub async fn music_playlist_two_column(rp: &RustyPipeQuery) -> Result<bool> {
+    let id = "VLRDCLAK5uy_kb7EBi6y3GrtJri4_ZH56Ms786DFEimbM";
+    let res = rp
+        .raw(
+            ClientType::DesktopMusic,
+            "browse",
+            &QBrowse {
+                browse_id: id,
+                params: None,
+            },
+        )
+        .await?;
+    Ok(res.contains("\"musicResponsiveHeaderRenderer\""))
+}
+
+pub async fn comments_framework_update(rp: &RustyPipeQuery) -> Result<bool> {
+    let continuation =
+        "Eg0SC3dMZHBSN2d1S3k4GAYyJSIRIgt3TGRwUjdndUt5ODAAeAJCEGNvbW1lbnRzLXNlY3Rpb24%3D";
+    let res = rp
+        .raw(ClientType::Desktop, "next", &QCont { continuation })
+        .await?;
+    Ok(res.contains("\"frameworkUpdates\""))
+}
+
+pub async fn channel_shorts_lockup(rp: &RustyPipeQuery) -> Result<bool> {
+    let id = "UCh8gHdtzO2tXd593_bjErWg";
+    let res = rp
+        .raw(
+            ClientType::Desktop,
+            "browse",
+            &QBrowse {
+                browse_id: id,
+                params: Some("EgZzaG9ydHPyBgUKA5oBAA%3D%3D"),
+            },
+        )
+        .await?;
+    Ok(res.contains("\"shortsLockupViewModel\""))
+}
+
+pub async fn playlist_page_header_renderer(rp: &RustyPipeQuery) -> Result<bool> {
+    let id = "VLPLZN_exA7d4RVmCQrG5VlWIjMOkMFZVVOc";
+    let res = rp
+        .raw(
+            ClientType::Desktop,
+            "browse",
+            &QBrowse {
+                browse_id: id,
+                params: None,
+            },
+        )
+        .await?;
+    Ok(res.contains("\"pageHeaderRenderer\""))
+}
+
+pub async fn channel_playlists_lockup(rp: &RustyPipeQuery) -> Result<bool> {
+    let id = "UC2DjFE7Xf11URZqWBigcVOQ";
+    let res = rp
+        .raw(
+            ClientType::Desktop,
+            "browse",
+            &QBrowse {
+                browse_id: id,
+                params: Some("EglwbGF5bGlzdHMgAQ%3D%3D"),
+            },
+        )
+        .await?;
+    Ok(res.contains("\"lockupViewModel\""))
+}
+
+pub async fn music_playlist_facepile(rp: &RustyPipeQuery) -> Result<bool> {
+    let id = "VLPL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe";
+    let res = rp
+        .raw(
+            ClientType::DesktopMusic,
+            "browse",
+            &QBrowse {
+                browse_id: id,
+                params: None,
+            },
+        )
+        .await?;
+    Ok(res.contains("\"facepile\""))
+}
+
+pub async fn music_album_groups_reordered(rp: &RustyPipeQuery) -> Result<bool> {
+    let id = "UCOR4_bSVIXPsGa4BbCSt60Q";
+    let res = rp
+        .raw(
+            ClientType::DesktopMusic,
+            "browse",
+            &QBrowse {
+                browse_id: id,
+                params: None,
+            },
+        )
+        .await?;
+    Ok(res.contains("\"Singles & EPs\""))
+}
+
+pub async fn music_continuation_item_renderer(rp: &RustyPipeQuery) -> Result<bool> {
+    let id = "VLPLbZIPy20-1pN7mqjckepWF78ndb6ci_qi";
+    let res = rp
+        .raw(
+            ClientType::DesktopMusic,
+            "browse",
+            &QBrowse {
+                browse_id: id,
+                params: None,
+            },
+        )
+        .await?;
+    Ok(res.contains("\"continuationItemRenderer\""))
+}
+
+pub async fn album_recommends(rp: &RustyPipeQuery) -> Result<bool> {
+    let id = "MPREb_u1I69lSAe5v";
+    let res = rp
+        .raw(
+            ClientType::DesktopMusic,
+            "browse",
+            &QBrowse {
+                browse_id: id,
+                params: None,
+            },
+        )
+        .await?;
+    Ok(res.contains("\"musicCarouselShelfRenderer\""))
+}
+
+pub async fn command_executor_command(rp: &RustyPipeQuery) -> Result<bool> {
+    let id = "VLPLbZIPy20-1pN7mqjckepWF78ndb6ci_qi";
+    let res = rp
+        .raw(
+            ClientType::Desktop,
+            "browse",
+            &QBrowse {
+                browse_id: id,
+                params: None,
+            },
+        )
+        .await?;
+    Ok(res.contains("\"commandExecutorCommand\""))
+}

codegen/src/collect_album_types.rs

@@ -1,25 +1,41 @@
-use std::{collections::BTreeMap, fs::File, io::BufReader, path::Path};
+use std::{collections::BTreeMap, fs::File, io::BufReader};
 
-use futures::stream::{self, StreamExt};
+use futures_util::stream::{self, StreamExt};
 use path_macro::path;
 use rustypipe::{
-    client::{ClientType, RustyPipe, RustyPipeQuery, YTContext},
+    client::{ClientType, RustyPipe, RustyPipeQuery},
     model::AlbumType,
-    param::{locale::LANGUAGES, Language},
+    param::{Language, LANGUAGES},
 };
 use serde::{Deserialize, Serialize};
+use serde_with::rust::deserialize_ignore_any;
 
-use crate::util::{self, TextRuns};
+use crate::{
+    model::{ContentsRenderer, QBrowse, SectionList, Tab, TextRuns},
+    util::{self, DICT_DIR},
+};
 
-pub async fn collect_album_types(project_root: &Path, concurrency: usize) {
-    let json_path = path!(project_root / "testfiles" / "dict" / "album_type_samples.json");
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+#[serde(rename_all = "snake_case")]
+enum AlbumTypeX {
+    Album,
+    Ep,
+    Single,
+    Audiobook,
+    Show,
+    AlbumRow,
+    SingleRow,
+}
+
+pub async fn collect_album_types(concurrency: usize) {
+    let json_path = path!(*DICT_DIR / "album_type_samples.json");
 
     let album_types = [
-        (AlbumType::Album, "MPREb_nlBWQROfvjo"),
-        (AlbumType::Single, "MPREb_bHfHGoy7vuv"),
-        (AlbumType::Ep, "MPREb_u1I69lSAe5v"),
-        (AlbumType::Audiobook, "MPREb_gaoNzsQHedo"),
-        (AlbumType::Show, "MPREb_cwzk8EUwypZ"),
+        (AlbumTypeX::Album, "MPREb_nlBWQROfvjo"),
+        (AlbumTypeX::Single, "MPREb_bHfHGoy7vuv"),
+        (AlbumTypeX::Ep, "MPREb_u1I69lSAe5v"),
+        (AlbumTypeX::Audiobook, "MPREb_gaoNzsQHedo"),
+        (AlbumTypeX::Show, "MPREb_cwzk8EUwypZ"),
     ];
 
     let rp = RustyPipe::new();
@@ -29,7 +45,7 @@ pub async fn collect_album_types(project_root: &Path, concurrency: usize) {
             let rp = rp.clone();
             async move {
                 let query = rp.query().lang(lang);
-                let mut data: BTreeMap<AlbumType, String> = BTreeMap::new();
+                let mut data: BTreeMap<AlbumTypeX, String> = BTreeMap::new();
 
                 for (album_type, id) in album_types {
                     let atype_txt = get_album_type(&query, id).await;
@@ -37,6 +53,22 @@ pub async fn collect_album_types(project_root: &Path, concurrency: usize) {
                     data.insert(album_type, atype_txt);
                 }
 
+                let (albums_txt, singles_txt) = get_album_groups(&query).await;
+                println!(
+                    "collected {}-{:?} ({})",
+                    lang,
+                    AlbumTypeX::AlbumRow,
+                    &albums_txt
+                );
+                println!(
+                    "collected {}-{:?} ({})",
+                    lang,
+                    AlbumTypeX::SingleRow,
+                    &singles_txt
+                );
+                data.insert(AlbumTypeX::AlbumRow, albums_txt);
+                data.insert(AlbumTypeX::SingleRow, singles_txt);
+
                 (lang, data)
             }
         })
@@ -48,14 +80,14 @@ pub async fn collect_album_types(project_root: &Path, concurrency: usize) {
     serde_json::to_writer_pretty(file, &collected_album_types).unwrap();
 }
 
-pub fn write_samples_to_dict(project_root: &Path) {
-    let json_path = path!(project_root / "testfiles" / "dict" / "album_type_samples.json");
+pub fn write_samples_to_dict() {
+    let json_path = path!(*DICT_DIR / "album_type_samples.json");
 
     let json_file = File::open(json_path).unwrap();
-    let collected: BTreeMap<Language, BTreeMap<AlbumType, String>> =
+    let collected: BTreeMap<Language, BTreeMap<String, String>> =
         serde_json::from_reader(BufReader::new(json_file)).unwrap();
-    let mut dict = util::read_dict(project_root);
-    let langs = dict.keys().map(|k| k.to_owned()).collect::<Vec<_>>();
+    let mut dict = util::read_dict();
+    let langs = dict.keys().copied().collect::<Vec<_>>();
 
     for lang in langs {
         let dict_entry = dict.entry(lang).or_default();
@@ -63,27 +95,35 @@ pub fn write_samples_to_dict(project_root: &Path) {
         let mut e_langs = dict_entry.equivalent.clone();
         e_langs.push(lang);
 
-        e_langs.iter().for_each(|lang| {
-            collected.get(lang).unwrap().iter().for_each(|(t, v)| {
+        for lang in &e_langs {
+            collected.get(lang).unwrap().iter().for_each(|(t_str, v)| {
+                let t =
+                    serde_plain::from_str::<AlbumType>(t_str.split('_').next().unwrap()).unwrap();
                 dict_entry
                     .album_types
-                    .insert(v.to_lowercase().trim().to_owned(), *t);
+                    .insert(v.to_lowercase().trim().to_owned(), t);
             });
-        });
+        }
     }
 
-    util::write_dict(project_root, &dict);
+    util::write_dict(dict);
 }
 
 #[derive(Debug, Deserialize)]
 struct AlbumData {
-    header: Header,
+    contents: AlbumContents,
 }
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-struct Header {
-    music_detail_header_renderer: HeaderRenderer,
+struct AlbumContents {
+    two_column_browse_results_renderer: ContentsRenderer<Tab<SectionList<AlbumHeader>>>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct AlbumHeader {
+    music_responsive_header_renderer: HeaderRenderer,
 }
 
 #[derive(Debug, Deserialize)]
@@ -91,20 +131,10 @@ struct HeaderRenderer {
     subtitle: TextRuns,
 }
 
-#[derive(Debug, Serialize)]
-#[serde(rename_all = "camelCase")]
-struct QBrowse<'a> {
-    context: YTContext<'a>,
-    browse_id: &'a str,
-}
-
 async fn get_album_type(query: &RustyPipeQuery, id: &str) -> String {
-    let context = query
-        .get_context(ClientType::DesktopMusic, true, None)
-        .await;
     let body = QBrowse {
-        context,
         browse_id: id,
+        params: None,
     };
     let response_txt = query
         .raw(ClientType::DesktopMusic, "browse", &body)
@@ -113,8 +143,20 @@ async fn get_album_type(query: &RustyPipeQuery, id: &str) -> String {
     let album = serde_json::from_str::<AlbumData>(&response_txt).unwrap();
 
     album
-        .header
-        .music_detail_header_renderer
+        .contents
+        .two_column_browse_results_renderer
+        .contents
+        .into_iter()
+        .next()
+        .unwrap()
+        .tab_renderer
+        .content
+        .section_list_renderer
+        .contents
+        .into_iter()
+        .next()
+        .unwrap()
+        .music_responsive_header_renderer
         .subtitle
         .runs
         .into_iter()
@@ -122,3 +164,84 @@ async fn get_album_type(query: &RustyPipeQuery, id: &str) -> String {
         .unwrap()
         .text
 }
+
+async fn get_album_groups(query: &RustyPipeQuery) -> (String, String) {
+    let body = QBrowse {
+        browse_id: "UCOR4_bSVIXPsGa4BbCSt60Q",
+        params: None,
+    };
+    let response_txt = query
+        .clone()
+        .visitor_data("CgtwbzJZcS1XZWc1QSjM2JG8BjIKCgJERRIEEgAgCw%3D%3D")
+        .raw(ClientType::DesktopMusic, "browse", &body)
+        .await
+        .unwrap();
+    let artist = serde_json::from_str::<ArtistData>(&response_txt).unwrap();
+
+    let sections = artist
+        .contents
+        .single_column_browse_results_renderer
+        .contents
+        .into_iter()
+        .next()
+        .map(|c| c.tab_renderer.content.section_list_renderer.contents)
+        .unwrap();
+    let titles = sections
+        .into_iter()
+        .filter_map(|s| {
+            if let ItemSection::MusicCarouselShelfRenderer(r) = s {
+                r.header
+            } else {
+                None
+            }
+        })
+        .map(|h| {
+            h.music_carousel_shelf_basic_header_renderer
+                .title
+                .runs
+                .into_iter()
+                .next()
+                .unwrap()
+                .text
+        })
+        .collect::<Vec<_>>();
+    assert!(titles.len() >= 2, "too few sections");
+
+    let mut titles_it = titles.into_iter();
+    (titles_it.next().unwrap(), titles_it.next().unwrap())
+}
+
+#[derive(Debug, Deserialize)]
+struct ArtistData {
+    contents: ArtistDataContents,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ArtistDataContents {
+    single_column_browse_results_renderer: ContentsRenderer<Tab<SectionList<ItemSection>>>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+enum ItemSection {
+    MusicCarouselShelfRenderer(MusicCarouselShelf),
+    #[serde(other, deserialize_with = "deserialize_ignore_any")]
+    None,
+}
+
+#[derive(Debug, Deserialize)]
+struct MusicCarouselShelf {
+    header: Option<MusicCarouselShelfHeader>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct MusicCarouselShelfHeader {
+    music_carousel_shelf_basic_header_renderer: MusicCarouselShelfHeaderRenderer,
+}
+
+#[derive(Debug, Deserialize)]
+struct MusicCarouselShelfHeaderRenderer {
+    title: TextRuns,
+}

codegen/src/collect_album_versions_titles.rs

@@ -0,0 +1,130 @@
+use std::{collections::BTreeMap, fs::File, io::BufReader};
+
+use path_macro::path;
+use rustypipe::{
+    client::{ClientType, RustyPipe},
+    param::{Language, LANGUAGES},
+};
+use serde::Deserialize;
+use serde_with::rust::deserialize_ignore_any;
+
+use crate::{
+    model::{QBrowse, SectionList, TextRuns},
+    util::{self, DICT_DIR},
+};
+
+pub async fn collect_album_versions_titles() {
+    let json_path = path!(*DICT_DIR / "other_versions_titles.json");
+    let mut res = BTreeMap::new();
+
+    let rp = RustyPipe::new();
+
+    for lang in LANGUAGES {
+        let query = QBrowse {
+            browse_id: "MPREb_nlBWQROfvjo",
+            params: None,
+        };
+        let raw_resp = rp
+            .query()
+            .lang(lang)
+            .raw(ClientType::DesktopMusic, "browse", &query)
+            .await
+            .unwrap();
+        let data = serde_json::from_str::<AlbumData>(&raw_resp).unwrap();
+        let title = data
+            .contents
+            .two_column_browse_results_renderer
+            .secondary_contents
+            .section_list_renderer
+            .contents
+            .into_iter()
+            .find_map(|x| match x {
+                ItemSection::MusicCarouselShelfRenderer(music_carousel_shelf) => {
+                    Some(music_carousel_shelf)
+                }
+                ItemSection::None => None,
+            })
+            .expect("other versions")
+            .header
+            .expect("header")
+            .music_carousel_shelf_basic_header_renderer
+            .title
+            .runs
+            .into_iter()
+            .next()
+            .unwrap()
+            .text;
+        println!("{lang}: {title}");
+        res.insert(lang, title);
+    }
+
+    let file = File::create(json_path).unwrap();
+    serde_json::to_writer_pretty(file, &res).unwrap();
+}
+
+pub fn write_samples_to_dict() {
+    let json_path = path!(*DICT_DIR / "other_versions_titles.json");
+    let json_file = File::open(json_path).unwrap();
+    let collected: BTreeMap<Language, String> =
+        serde_json::from_reader(BufReader::new(json_file)).unwrap();
+    let mut dict = util::read_dict();
+    let langs = dict.keys().copied().collect::<Vec<_>>();
+
+    for lang in langs {
+        let dict_entry = dict.entry(lang).or_default();
+
+        let e = collected.get(&lang).unwrap();
+        assert_eq!(e, e.trim());
+        dict_entry.album_versions_title = e.to_owned();
+
+        for lang in &dict_entry.equivalent {
+            let ee = collected.get(lang).unwrap();
+            if ee != e {
+                panic!("equivalent lang conflict, lang: {lang}");
+            }
+        }
+    }
+
+    util::write_dict(dict);
+}
+
+#[derive(Debug, Deserialize)]
+struct AlbumData {
+    contents: AlbumDataContents,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct AlbumDataContents {
+    two_column_browse_results_renderer: X1,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct X1 {
+    secondary_contents: SectionList<ItemSection>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+enum ItemSection {
+    MusicCarouselShelfRenderer(MusicCarouselShelf),
+    #[serde(other, deserialize_with = "deserialize_ignore_any")]
+    None,
+}
+
+#[derive(Debug, Deserialize)]
+struct MusicCarouselShelf {
+    header: Option<MusicCarouselShelfHeader>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct MusicCarouselShelfHeader {
+    music_carousel_shelf_basic_header_renderer: MusicCarouselShelfHeaderRenderer,
+}
+
+#[derive(Debug, Deserialize)]
+struct MusicCarouselShelfHeaderRenderer {
+    title: TextRuns,
+}

codegen/src/collect_chan_prefixes.rs

@@ -0,0 +1,75 @@
+use std::{collections::BTreeMap, fs::File, io::BufReader};
+
+use path_macro::path;
+use rustypipe::{
+    client::RustyPipe,
+    param::{Language, LANGUAGES},
+};
+use serde::{Deserialize, Serialize};
+
+use crate::util::{self, DICT_DIR};
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Entry {
+    prefix: String,
+    suffix: String,
+}
+
+pub async fn collect_chan_prefixes() {
+    let cname = "kiernanchrisman";
+    let json_path = path!(*DICT_DIR / "chan_prefixes.json");
+    let mut res = BTreeMap::new();
+
+    let rp = RustyPipe::new();
+
+    for lang in LANGUAGES {
+        let playlist = rp
+            .query()
+            .lang(lang)
+            .playlist("PLZN_exA7d4RVmCQrG5VlWIjMOkMFZVVOc")
+            .await
+            .unwrap();
+        let n = playlist.channel.unwrap().name;
+        let offset = n.find(cname).unwrap();
+        let prefix = &n[..offset];
+        let suffix = &n[(offset + cname.len())..];
+
+        res.insert(
+            lang,
+            Entry {
+                prefix: prefix.to_owned(),
+                suffix: suffix.to_owned(),
+            },
+        );
+    }
+
+    let file = File::create(json_path).unwrap();
+    serde_json::to_writer_pretty(file, &res).unwrap();
+}
+
+pub fn write_samples_to_dict() {
+    let json_path = path!(*DICT_DIR / "chan_prefixes.json");
+
+    let json_file = File::open(json_path).unwrap();
+    let collected: BTreeMap<Language, Entry> =
+        serde_json::from_reader(BufReader::new(json_file)).unwrap();
+    let mut dict = util::read_dict();
+    let langs = dict.keys().copied().collect::<Vec<_>>();
+
+    for lang in langs {
+        let dict_entry = dict.entry(lang).or_default();
+
+        let e = collected.get(&lang).unwrap();
+        dict_entry.chan_prefix = e.prefix.trim().to_owned();
+        dict_entry.chan_suffix = e.suffix.trim().to_owned();
+
+        for lang in &dict_entry.equivalent {
+            let ee = collected.get(lang).unwrap();
+            if ee.prefix != e.prefix || ee.suffix != e.suffix {
+                panic!("equivalent lang conflict, lang: {lang}");
+            }
+        }
+    }
+
+    util::write_dict(dict);
+}

codegen/src/collect_datetimes.rs

@@ -1,93 +0,0 @@
-use std::{collections::BTreeMap, fs::File, io::BufReader, path::Path};
-
-use futures::{stream, StreamExt};
-use path_macro::path;
-use rustypipe::{
-    client::RustyPipe,
-    param::{locale::LANGUAGES, Language},
-};
-
-use crate::util;
-
-type CollectedDates = BTreeMap<Language, String>;
-
-const FILENAME: &str = "datetime_samples.json";
-
-// A channel with an upcoming video or livestream
-const CHANNEL_ID: &str = "UCWxlUwW9BgGISaakjGM37aw";
-const VIDEO_ID: &str = "p9FfS9l2NVA";
-
-const YEAR: u64 = 2023;
-const YEAR_SHORT: u64 = 23;
-const MONTH: u64 = 4;
-const DAY: u64 = 14;
-const HOUR: u64 = 15;
-const HOUR_12: u64 = 3;
-const MINUTE: u64 = 0;
-
-/// Collect upcoming video dates from the TV client in every supported language
-/// and write them to `testfiles/dict/datetime_samples.json`
-pub async fn collect_datetimes(project_root: &Path, concurrency: usize) {
-    let json_path = path!(project_root / "testfiles" / "dict" / FILENAME);
-
-    let rp = RustyPipe::new();
-    let collected_dates: CollectedDates = stream::iter(LANGUAGES)
-        .map(|lang| {
-            let rp = rp.clone();
-            println!("collecting {lang}");
-            async move {
-                let channel = rp.query().lang(lang).channel_tv(CHANNEL_ID).await.unwrap();
-                let video = channel
-                    .videos
-                    .into_iter()
-                    .find(|v| v.id == VIDEO_ID)
-                    .unwrap();
-                (
-                    lang,
-                    video
-                        .publish_date_txt
-                        .unwrap_or_else(|| panic!("no publish_date_txt in {}", lang)),
-                )
-            }
-        })
-        .buffer_unordered(concurrency)
-        .collect()
-        .await;
-
-    let file = File::create(json_path).unwrap();
-    serde_json::to_writer_pretty(file, &collected_dates).unwrap();
-}
-
-/// Attempt to parse the numbers collected by `collect-datetimes`
-/// and write the results to `dictionary.json`.
-pub fn write_samples_to_dict(project_root: &Path) {
-    let json_path = path!(project_root / "testfiles" / "dict" / FILENAME);
-
-    let json_file = File::open(json_path).unwrap();
-    let collected_dates: CollectedDates =
-        serde_json::from_reader(BufReader::new(json_file)).unwrap();
-    let mut dict = util::read_dict(project_root);
-    let langs = dict.keys().map(|k| k.to_owned()).collect::<Vec<_>>();
-
-    for lang in langs {
-        let datestr = &collected_dates[&lang];
-        let numbers = util::parse_numeric_vec::<u64>(datestr);
-        let order = numbers
-            .iter()
-            .map(|n| match *n {
-                YEAR => 'Y',
-                YEAR_SHORT => 'y',
-                MONTH => 'M',
-                DAY => 'D',
-                HOUR => 'H',
-                HOUR_12 => 'h',
-                MINUTE => 'm',
-                _ => panic!("unknown number {n} in {datestr} ({lang})"),
-            })
-            .collect::<String>();
-        assert_eq!(order.len(), 5);
-        dict.get_mut(&lang).unwrap().datetime_order = order;
-    }
-
-    util::write_dict(project_root, &dict);
-}

codegen/src/collect_history_dates.rs

@@ -0,0 +1,110 @@
+use std::{collections::BTreeMap, fs::File, io::BufReader};
+
+use path_macro::path;
+use rustypipe::{
+    client::RustyPipe,
+    param::{Language, LANGUAGES},
+};
+
+use crate::util::{self, DICT_DIR};
+
+type CollectedDates = BTreeMap<Language, BTreeMap<String, String>>;
+
+const THIS_WEEK: &str = "this_week";
+const LAST_WEEK: &str = "last_week";
+
+pub async fn collect_dates_music() {
+    let json_path = path!(*DICT_DIR / "history_date_samples.json");
+    let rp = RustyPipe::builder()
+        .storage_dir(path!(env!("CARGO_MANIFEST_DIR") / ".."))
+        .build()
+        .unwrap();
+
+    let mut res: CollectedDates = {
+        let json_file = File::open(&json_path).unwrap();
+        serde_json::from_reader(BufReader::new(json_file)).unwrap()
+    };
+
+    for lang in LANGUAGES {
+        println!("{lang}");
+        let history = rp.query().lang(lang).music_history().await.unwrap();
+        if history.items.len() < 3 {
+            panic!("{lang} empty history")
+        }
+
+        // The indexes have to be adapted before running
+        let entry = res.entry(lang).or_default();
+        entry.insert(
+            THIS_WEEK.to_owned(),
+            history.items[0].playback_date_txt.clone().unwrap(),
+        );
+        entry.insert(
+            LAST_WEEK.to_owned(),
+            history.items[18].playback_date_txt.clone().unwrap(),
+        );
+    }
+
+    let file = File::create(&json_path).unwrap();
+    serde_json::to_writer_pretty(file, &res).unwrap();
+}
+
+pub async fn collect_dates() {
+    let json_path = path!(*DICT_DIR / "history_date_samples.json");
+    let rp = RustyPipe::builder()
+        .storage_dir(path!(env!("CARGO_MANIFEST_DIR") / ".."))
+        .build()
+        .unwrap();
+
+    let mut res: CollectedDates = {
+        let json_file = File::open(&json_path).unwrap();
+        serde_json::from_reader(BufReader::new(json_file)).unwrap()
+    };
+
+    for lang in LANGUAGES {
+        println!("{lang}");
+        let history = rp.query().lang(lang).history().await.unwrap();
+        if history.items.len() < 3 {
+            panic!("{lang} empty history")
+        }
+
+        let entry = res.entry(lang).or_default();
+        entry.insert(
+            "tuesday".to_owned(),
+            history.items[0].playback_date_txt.clone().unwrap(),
+        );
+        entry.insert(
+            "0000-01-06".to_owned(),
+            history.items[1].playback_date_txt.clone().unwrap(),
+        );
+        entry.insert(
+            "2024-12-28".to_owned(),
+            history.items[15].playback_date_txt.clone().unwrap(),
+        );
+    }
+
+    let file = File::create(&json_path).unwrap();
+    serde_json::to_writer_pretty(file, &res).unwrap();
+}
+
+pub fn write_samples_to_dict() {
+    let json_path = path!(*DICT_DIR / "history_date_samples.json");
+
+    let json_file = File::open(json_path).unwrap();
+    let collected_dates: CollectedDates =
+        serde_json::from_reader(BufReader::new(json_file)).unwrap();
+    let mut dict = util::read_dict();
+    let langs = dict.keys().copied().collect::<Vec<_>>();
+
+    for lang in langs {
+        let dict_entry = dict.entry(lang).or_default();
+        let cd = &collected_dates[&lang];
+        dict_entry
+            .timeago_nd_tokens
+            .insert(util::filter_datestr(&cd[THIS_WEEK]), "0Wl".to_owned());
+        dict_entry
+            .timeago_nd_tokens
+            .insert(util::filter_datestr(&cd[LAST_WEEK]), "1Wl".to_owned());
+    }
+
+    util::write_dict(dict);
+}

codegen/src/collect_large_numbers.rs

@@ -1,25 +1,32 @@
-use std::collections::{HashMap, HashSet};
-use std::{collections::BTreeMap, fs::File, io::BufReader, path::Path};
+use std::sync::Arc;
+use std::{
+    collections::{BTreeMap, HashMap, HashSet},
+    fs::File,
+    io::BufReader,
+};
 
 use anyhow::{Context, Result};
-use futures::{stream, StreamExt};
+use futures_util::{stream, StreamExt};
 use once_cell::sync::Lazy;
 use path_macro::path;
 use regex::Regex;
-use reqwest::{header, Client};
-use rustypipe::param::{locale::LANGUAGES, Language};
+use rustypipe::client::{ClientType, RustyPipe, RustyPipeQuery};
+use rustypipe::param::{Language, LANGUAGES};
 use serde::Deserialize;
-use serde_with::serde_as;
-use serde_with::VecSkipError;
 
-use crate::util::{self, Text};
+use crate::model::{Channel, ContinuationResponse};
+use crate::util::DICT_DIR;
+use crate::{
+    model::{QBrowse, QCont, TextRuns},
+    util,
+};
 
-type CollectedNumbers = BTreeMap<Language, BTreeMap<u8, (String, u64)>>;
+type CollectedNumbers = BTreeMap<Language, BTreeMap<String, u64>>;
 
 /// Collect video view count texts in every supported language
 /// and write them to `testfiles/dict/large_number_samples.json`.
 ///
-/// YouTube's API outputs the subscriber count of a channel only in a
+/// YouTube's API outputs subscriber and view counts only in a
 /// approximated format (e.g *880K subscribers*), which varies
 /// by language.
 ///
@@ -30,99 +37,117 @@ type CollectedNumbers = BTreeMap<Language, BTreeMap<u8, (String, u64)>>;
 /// We extract these instead of subscriber counts because the YouTube API
 /// outputs view counts both in approximated and exact format, so we can use
 /// the exact counts to figure out the tokens.
-pub async fn collect_large_numbers(project_root: &Path, concurrency: usize) {
-    let json_path = path!(project_root / "testfiles" / "dict" / "large_number_samples.json");
-    let json_path_all =
-        path!(project_root / "testfiles" / "dict" / "large_number_samples_all.json");
+pub async fn collect_large_numbers(concurrency: usize) {
+    let json_path = path!(*DICT_DIR / "large_number_samples_all.json");
+    let rp = RustyPipe::new();
 
     let channels = [
-        "UCq-Fj5jknLsUf-MWSy4_brA", // 10e8 (225M)
-        "UCcdwLMPsaU2ezNSJU1nFoBQ", // 10e7 (60M)
-        "UC6mIxFTvXkWQVEHPsEdflzQ", // 10e6 (1.7M)
-        "UCD0y51PJfvkZNe3y3FR5riw", // 10e5 (125K)
-        "UCNcN0dW43zE0Om3278fjY8A", // 10e4 (27K)
+        "UCq-Fj5jknLsUf-MWSy4_brA", // 10e8 (241M)
+        "UCcdwLMPsaU2ezNSJU1nFoBQ", // 10e7 (67M)
+        "UC6mIxFTvXkWQVEHPsEdflzQ", // 10e6 (1.8M)
+        "UCD0y51PJfvkZNe3y3FR5riw", // 10e5 (126K)
+        "UCNcN0dW43zE0Om3278fjY8A", // 10e4 (33K)
         "UC0QEucPrn0-Ddi3JBTcs5Kw", // 10e3 (5K)
-        "UCXvtcj9xUQhaqPaitFf2DqA", // (170)
-        "UCq-XMc01T641v-4P3hQYJWg", // (636)
+        "UCXvtcj9xUQhaqPaitFf2DqA", // (275)
+        "UCq-XMc01T641v-4P3hQYJWg", // (695)
+        "UCaZL4eLD7a30Fa8QI-sRi_g", // (31K)
+        "UCO-dylEoJozPTxGYd8fTQxA", // (5)
+        "UCQXYK94vDqOEkPbTCyL0OjA", // (1)
     ];
 
-    let collected_numbers_all: BTreeMap<Language, BTreeMap<String, u64>> = stream::iter(LANGUAGES)
-        .map(|lang| async move {
-            let mut entry = BTreeMap::new();
+    // YTM outputs the subscriber count in a shortened format in some languages
+    let music_channels = [
+        "UC_1N84buVNgR_-3gDZ9Jtxg", // 10e8 (158M)
+        "UCRw0x9_EfawqmgDI2IgQLLg", // 10e7 (29M)
+        "UChWu2clmvJ5wN_0Ic5dnqmw", // 10e6 (1.9M)
+        "UCOYiPDuimprrGHgFy4_Fw8Q", // 10e5 (149K)
+        "UC8nZf9WyVIxNMly_hy2PTyQ", // 10e4 (17K)
+        "UCaltNL5XvZ7dKvBsBPi-gqg", // 10e3 (8K)
+    ];
 
-            for (n, ch_id) in channels.iter().enumerate() {
-                let channel = get_channel(ch_id, lang)
-                    .await
-                    .context(format!("{lang}-{n}"))
-                    .unwrap();
+    // Build a lookup table for the channel's subscriber counts
+    let subscriber_counts: Arc<BTreeMap<String, u64>> = stream::iter(channels)
+        .map(|c| {
+            let rp = rp.query();
+            async move {
+                let channel = get_channel(&rp, c).await.unwrap();
 
-                channel.view_counts.iter().for_each(|(num, txt)| {
-                    entry.insert(txt.to_owned(), *num);
-                });
-
-                println!("collected {lang}-{n}");
+                let n = util::parse_largenum_en(&channel.subscriber_count).unwrap();
+                (c.to_owned(), n)
             }
+        })
+        .buffer_unordered(concurrency)
+        .collect::<BTreeMap<_, _>>()
+        .await
+        .into();
 
-            (lang, entry)
+    let music_subscriber_counts: Arc<BTreeMap<String, u64>> = stream::iter(music_channels)
+        .map(|c| {
+            let rp = rp.query();
+            async move {
+                let subscriber_count = music_channel_subscribers(&rp, c).await.unwrap();
+
+                let n = util::parse_largenum_en(&subscriber_count).unwrap();
+                (c.to_owned(), n)
+            }
+        })
+        .buffer_unordered(concurrency)
+        .collect::<BTreeMap<_, _>>()
+        .await
+        .into();
+
+    let collected_numbers: CollectedNumbers = stream::iter(LANGUAGES)
+        .map(|lang| {
+            let rp = rp.query().lang(lang);
+            let subscriber_counts = subscriber_counts.clone();
+            let music_subscriber_counts = music_subscriber_counts.clone();
+            async move {
+                let mut entry = BTreeMap::new();
+
+                for (n, ch_id) in channels.iter().enumerate() {
+                    let channel = get_channel(&rp, ch_id)
+                        .await
+                        .context(format!("{lang}-{n}"))
+                        .unwrap();
+
+                    channel.view_counts.iter().for_each(|(num, txt)| {
+                        entry.insert(txt.clone(), *num);
+                    });
+                    entry.insert(channel.subscriber_count, subscriber_counts[*ch_id]);
+
+                    println!("collected {lang}-{n}");
+                }
+
+                for (n, ch_id) in music_channels.iter().enumerate() {
+                    let subscriber_count = music_channel_subscribers(&rp, ch_id)
+                        .await
+                        .context(format!("{lang}-music-{n}"))
+                        .unwrap();
+                    entry.insert(subscriber_count, music_subscriber_counts[*ch_id]);
+                    println!("collected {lang}-music-{n}");
+                }
+
+                (lang, entry)
+            }
         })
         .buffer_unordered(concurrency)
         .collect()
         .await;
 
-    let collected_numbers: CollectedNumbers = collected_numbers_all
-        .iter()
-        .map(|(lang, entry)| {
-            let mut e2 = BTreeMap::new();
-            entry.iter().for_each(|(txt, num)| {
-                e2.insert(get_mag(*num), (txt.to_owned(), *num));
-            });
-            (*lang, e2)
-        })
-        .collect();
-
     let file = File::create(json_path).unwrap();
     serde_json::to_writer_pretty(file, &collected_numbers).unwrap();
-
-    let file = File::create(json_path_all).unwrap();
-    serde_json::to_writer_pretty(file, &collected_numbers_all).unwrap();
 }
 
 /// Attempt to parse the numbers collected by `collect-large-numbers`
 /// and write the results to `dictionary.json`.
-pub fn write_samples_to_dict(project_root: &Path) {
-    /*
-    Manual corrections:
-    as
-    "কোঃটা": 9,
-    "নিঃটা": 6,
-    "নিযুতটা": 6,
-    "লাখটা": 5,
-    "হাজাৰটা": 3
-
-    ar
-    "ألف": 3,
-    "آلاف": 3,
-    "مليار": 9,
-    "مليون": 6
-
-    bn
-    "লাটি": 5,
-    "শত": 2,
-    "হাটি": 3,
-    "কোটি": 7
-
-    es/es-US
-    "mil": 3,
-    "M": 6
-    */
-
-    let json_path = path!(project_root / "testfiles" / "dict" / "large_number_samples.json");
+pub fn write_samples_to_dict() {
+    let json_path = path!(*DICT_DIR / "large_number_samples.json");
 
     let json_file = File::open(json_path).unwrap();
     let collected_nums: CollectedNumbers =
         serde_json::from_reader(BufReader::new(json_file)).unwrap();
-    let mut dict = util::read_dict(project_root);
-    let langs = dict.keys().map(|k| k.to_owned()).collect::<Vec<_>>();
+    let mut dict = util::read_dict();
+    let langs = dict.keys().copied().collect::<Vec<_>>();
 
     static POINT_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"\d(\.|,)\d{1,3}(?:\D|$)").unwrap());
 
@@ -132,11 +157,9 @@ pub fn write_samples_to_dict(project_root: &Path) {
         let mut e_langs = dict_entry.equivalent.clone();
         e_langs.push(lang);
 
-        let comma_decimal = collected_nums
-            .get(&lang)
-            .unwrap()
+        let comma_decimal = collected_nums[&lang]
             .iter()
-            .find_map(|(mag, (txt, _))| {
+            .find_map(|(txt, val)| {
                 let point = POINT_REGEX
                     .captures(txt)
                     .map(|c| c.get(1).unwrap().as_str());
@@ -146,16 +169,14 @@ pub fn write_samples_to_dict(project_root: &Path) {
                     // If the number parsed from all digits has the same order of
                     // magnitude as the actual number, it must be a separator.
                     // Otherwise it is a decimal point
-                    return Some((get_mag(num_all) == *mag) ^ (point == ","));
+                    return Some((get_mag(num_all) == get_mag(*val)) ^ (point == ","));
                 }
+
                 None
             })
             .unwrap();
 
-        let decimal_point = match comma_decimal {
-            true => ",",
-            false => ".",
-        };
+        let decimal_point = if comma_decimal { "," } else { "." };
 
         // Search for tokens
 
@@ -165,6 +186,7 @@ pub fn write_samples_to_dict(project_root: &Path) {
         // If the token is found again with a different derived order of magnitude,
         // its value in the map is set to None.
         let mut found_tokens: HashMap<String, Option<u8>> = HashMap::new();
+        let mut found_nd_tokens: HashMap<String, Option<u8>> = HashMap::new();
 
         let mut insert_token = |token: String, mag: u8| {
             let found_token = found_tokens.entry(token).or_insert(match mag {
@@ -179,46 +201,77 @@ pub fn write_samples_to_dict(project_root: &Path) {
             }
         };
 
+        let mut insert_nd_token = |token: String, n: Option<u8>| {
+            let found_token = found_nd_tokens.entry(token).or_insert(n);
+
+            if let Some(f) = found_token {
+                if Some(*f) != n {
+                    *found_token = None;
+                }
+            }
+        };
+
         for lang in e_langs {
             let entry = collected_nums.get(&lang).unwrap();
 
-            entry.iter().for_each(|(mag, (txt, _))| {
+            for (txt, val) in entry.iter() {
                 let filtered = util::filter_largenumstr(txt);
+                let mag = get_mag(*val);
 
-                let tokens: Vec<String> = match dict_entry.by_char {
-                    true => filtered.chars().map(|c| c.to_string()).collect(),
-                    false => filtered.split_whitespace().map(|c| c.to_string()).collect(),
+                let tokens: Vec<String> = if dict_entry.by_char || lang == Language::Ko {
+                    filtered.chars().map(|c| c.to_string()).collect()
+                } else {
+                    filtered
+                        .split_whitespace()
+                        .map(std::string::ToString::to_string)
+                        .collect()
                 };
 
-                let num_before_point =
-                    util::parse_numeric::<u64>(txt.split(decimal_point).next().unwrap()).unwrap();
-                let mag_before_point = get_mag(num_before_point);
-                let mut mag_remaining = mag - mag_before_point;
+                match util::parse_numeric::<u64>(txt.split(decimal_point).next().unwrap()) {
+                    Ok(num_before_point) => {
+                        let mag_before_point = get_mag(num_before_point);
+                        let mut mag_remaining = mag - mag_before_point;
 
-                tokens.iter().for_each(|t| {
-                    // These tokens are correct in all languages
-                    // and are used to parse combined prefixes like `1.1K crore` (en-IN)
-                    let known_tmag: u8 = if t.len() == 1 {
-                        match t.as_str() {
-                            "K" | "k" => 3,
-                            // 'm' means 10^3 in Catalan, 'B' means 10^3 in Turkish
-                            // 'M' means 10^9 in Indonesian
-                            _ => 0,
+                        for t in &tokens {
+                            // These tokens are correct in all languages
+                            // and are used to parse combined prefixes like `1.1K crore` (en-IN)
+                            let known_tmag: u8 = if t.len() == 1 {
+                                match t.as_str() {
+                                    "K" | "k" => 3,
+                                    // 'm' means 10^3 in Catalan, 'B' means 10^3 in Turkish
+                                    // 'M' means 10^9 in Indonesian
+                                    _ => 0,
+                                }
+                            } else {
+                                0
+                            };
+
+                            // K/M/B
+                            if known_tmag > 0 {
+                                mag_remaining = mag_remaining
+                                    .checked_sub(known_tmag)
+                                    .expect("known magnitude incorrect");
+                            } else {
+                                insert_token(t.clone(), mag_remaining);
+                            }
+                            insert_nd_token(t.clone(), None);
                         }
-                    } else {
-                        0
-                    };
-
-                    // K/M/B
-                    if known_tmag > 0 {
-                        mag_remaining = mag_remaining
-                            .checked_sub(known_tmag)
-                            .expect("known magnitude incorrect");
-                    } else {
-                        insert_token(t.to_owned(), mag_remaining);
                     }
-                });
-            });
+                    Err(e) => {
+                        if matches!(e.kind(), std::num::IntErrorKind::Empty) {
+                            // Text does not contain any digits, search for nd_tokens
+                            for t in &tokens {
+                                insert_nd_token(
+                                    t.clone(),
+                                    Some((*val).try_into().expect("nd_token value too large")),
+                                );
+                            }
+                        } else {
+                            panic!("{e}, txt: {txt}")
+                        }
+                    }
+                }
+            }
         }
 
         // Insert collected data into dictionary
@@ -226,6 +279,10 @@ pub fn write_samples_to_dict(project_root: &Path) {
             .into_iter()
             .filter_map(|(k, v)| v.map(|v| (k, v)))
             .collect();
+        dict_entry.number_nd_tokens = found_nd_tokens
+            .into_iter()
+            .filter_map(|(k, v)| v.map(|v| (k, v)))
+            .collect();
         dict_entry.comma_decimal = comma_decimal;
 
         // Check for duplicates
@@ -233,9 +290,13 @@ pub fn write_samples_to_dict(project_root: &Path) {
         if !dict_entry.number_tokens.values().all(|x| uniq.insert(x)) {
             println!("Warning: collected duplicate tokens for {lang}");
         }
+        let mut uniq = HashSet::new();
+        if !dict_entry.number_nd_tokens.values().all(|x| uniq.insert(x)) {
+            println!("Warning: collected duplicate nd_tokens for {lang}");
+        }
     }
 
-    util::write_dict(project_root, &dict);
+    util::write_dict(dict);
 }
 
 fn get_mag(n: u64) -> u8 {
@@ -243,145 +304,147 @@ fn get_mag(n: u64) -> u8 {
 }
 
 /*
-YouTube channel videos response
+YouTube Music channel data
 */
 
-#[derive(Clone, Debug, Deserialize)]
+#[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-struct Channel {
-    contents: Contents,
+struct MusicChannel {
+    header: MusicHeader,
 }
 
-#[derive(Clone, Debug, Deserialize)]
+#[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-struct Contents {
-    two_column_browse_results_renderer: TabsRenderer,
+struct MusicHeader {
+    #[serde(alias = "musicVisualHeaderRenderer")]
+    music_immersive_header_renderer: MusicHeaderRenderer,
 }
 
-#[serde_as]
-#[derive(Clone, Debug, Deserialize)]
+#[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-struct TabsRenderer {
-    #[serde_as(as = "VecSkipError<_>")]
-    tabs: Vec<TabRendererWrap>,
+struct MusicHeaderRenderer {
+    subscription_button: SubscriptionButton,
 }
 
-#[derive(Clone, Debug, Deserialize)]
+#[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-struct TabRendererWrap {
-    tab_renderer: TabRenderer,
+struct SubscriptionButton {
+    subscribe_button_renderer: SubscriptionButtonRenderer,
 }
 
-#[derive(Clone, Debug, Deserialize)]
+#[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-struct TabRenderer {
-    content: SectionListRendererWrap,
+struct SubscriptionButtonRenderer {
+    subscriber_count_text: TextRuns,
 }
 
-#[derive(Clone, Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct SectionListRendererWrap {
-    section_list_renderer: SectionListRenderer,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct SectionListRenderer {
-    contents: Vec<ItemSectionRendererWrap>,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct ItemSectionRendererWrap {
-    item_section_renderer: ItemSectionRenderer,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct ItemSectionRenderer {
-    contents: Vec<GridRendererWrap>,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct GridRendererWrap {
-    grid_renderer: GridRenderer,
-}
-
-#[serde_as]
-#[derive(Clone, Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct GridRenderer {
-    #[serde_as(as = "VecSkipError<_>")]
-    items: Vec<VideoListItem>,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct VideoListItem {
-    grid_video_renderer: GridVideoRenderer,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct GridVideoRenderer {
-    /// `24,194 views`
-    view_count_text: Text,
-    /// `19K views`
-    short_view_count_text: Text,
-}
-
-#[derive(Clone, Debug)]
+#[derive(Debug)]
 struct ChannelData {
-    view_counts: Vec<(u64, String)>,
+    view_counts: BTreeMap<u64, String>,
+    subscriber_count: String,
 }
 
-async fn get_channel(channel_id: &str, lang: Language) -> Result<ChannelData> {
-    let client = Client::new();
+async fn get_channel(query: &RustyPipeQuery, channel_id: &str) -> Result<ChannelData> {
+    let resp = query
+        .raw(
+            ClientType::Desktop,
+            "browse",
+            &QBrowse {
+                browse_id: channel_id,
+                params: Some("EgZ2aWRlb3MYASAAMAE"),
+            },
+        )
+        .await?;
 
-    let body = format!(
-        "{}{}{}{}{}",
-        r##"{"context":{"client":{"clientName":"WEB","clientVersion":"2.20220914.06.00","platform":"DESKTOP","originalUrl":"https://www.youtube.com/","hl":""##,
-        lang,
-        r##"","gl":"US"},"request":{"internalExperimentFlags":[],"useSsl":true},"user":{"lockedSafetyMode":false}},"params":"EgZ2aWRlb3MYASAAMAE%3D","browseId":""##,
-        channel_id,
-        "\"}"
-    );
+    let channel = serde_json::from_str::<Channel>(&resp)?;
 
-    let resp = client
-        .post("https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false")
-        .header(header::CONTENT_TYPE, "application/json")
-        .body(body)
-        .send().await?
-        .error_for_status()?;
+    let tab = &channel.contents.two_column_browse_results_renderer.tabs[0]
+        .tab_renderer
+        .content
+        .rich_grid_renderer;
 
-    let channel = resp.json::<Channel>().await?;
+    let popular_token = tab.header.as_ref().and_then(|h| {
+        h.feed_filter_chip_bar_renderer.contents.get(1).map(|c| {
+            c.chip_cloud_chip_renderer
+                .navigation_endpoint
+                .continuation_command
+                .token
+                .clone()
+        })
+    });
+
+    let mut view_counts: BTreeMap<u64, String> = tab
+        .contents
+        .iter()
+        .map(|itm| {
+            let v = &itm.rich_item_renderer.content.video_renderer;
+            (
+                util::parse_numeric(&v.view_count_text.text).unwrap_or_default(),
+                v.short_view_count_text.text.clone(),
+            )
+        })
+        .collect();
+
+    if let Some(popular_token) = popular_token {
+        let resp = query
+            .raw(
+                ClientType::Desktop,
+                "browse",
+                &QCont {
+                    continuation: &popular_token,
+                },
+            )
+            .await?;
+
+        let continuation = serde_json::from_str::<ContinuationResponse>(&resp)?;
+
+        for action in &continuation.on_response_received_actions {
+            action
+                .reload_continuation_items_command
+                .continuation_items
+                .iter()
+                .for_each(|itm| {
+                    let v = &itm.rich_item_renderer.content.video_renderer;
+                    view_counts.insert(
+                        util::parse_numeric(&v.view_count_text.text).unwrap(),
+                        v.short_view_count_text.text.clone(),
+                    );
+                });
+        }
+    }
 
     Ok(ChannelData {
-        view_counts: channel
-            .contents
-            .two_column_browse_results_renderer
-            .tabs
-            .get(0)
-            .map(|tab| {
-                tab.tab_renderer.content.section_list_renderer.contents[0]
-                    .item_section_renderer
-                    .contents[0]
-                    .grid_renderer
-                    .items
-                    .iter()
-                    .map(|itm| {
-                        (
-                            util::parse_numeric(&itm.grid_video_renderer.view_count_text.text)
-                                .unwrap(),
-                            itm.grid_video_renderer
-                                .short_view_count_text
-                                .text
-                                .to_owned(),
-                        )
-                    })
-                    .collect()
-            })
-            .unwrap_or_default(),
+        view_counts,
+        subscriber_count: channel
+            .header
+            .c4_tabbed_header_renderer
+            .subscriber_count_text
+            .text,
     })
 }
+
+async fn music_channel_subscribers(query: &RustyPipeQuery, channel_id: &str) -> Result<String> {
+    let resp = query
+        .raw(
+            ClientType::DesktopMusic,
+            "browse",
+            &QBrowse {
+                browse_id: channel_id,
+                params: None,
+            },
+        )
+        .await?;
+
+    let channel = serde_json::from_str::<MusicChannel>(&resp)?;
+    channel
+        .header
+        .music_immersive_header_renderer
+        .subscription_button
+        .subscribe_button_renderer
+        .subscriber_count_text
+        .runs
+        .into_iter()
+        .next()
+        .map(|t| t.text)
+        .ok_or_else(|| anyhow::anyhow!("no text"))
+}

codegen/src/collect_playlist_dates.rs

@@ -3,19 +3,18 @@ use std::{
     fs::File,
     hash::Hash,
     io::BufReader,
-    path::Path,
 };
 
-use futures::{stream, StreamExt};
+use futures_util::{stream, StreamExt};
+use ordered_hash_map::OrderedHashMap;
 use path_macro::path;
 use rustypipe::{
     client::RustyPipe,
-    param::{locale::LANGUAGES, Language},
-    timeago::{self, TimeAgo},
+    param::{Language, LANGUAGES},
 };
 use serde::{Deserialize, Serialize};
 
-use crate::util;
+use crate::util::{self, DICT_DIR};
 
 type CollectedDates = BTreeMap<Language, BTreeMap<DateCase, String>>;
 
@@ -62,17 +61,14 @@ enum DateCase {
 ///
 /// Because the relative dates change with time, the first three playlists
 /// have to checked and eventually changed before running the program.
-pub async fn collect_dates(project_root: &Path, concurrency: usize) {
-    let json_path = path!(project_root / "testfiles" / "dict" / "playlist_samples.json");
+pub async fn collect_dates(concurrency: usize) {
+    let json_path = path!(*DICT_DIR / "playlist_samples.json");
 
     // These are the sample playlists
     let cases = [
-        (
-            DateCase::Today,
-            "RDCLAK5uy_kj3rhiar1LINmyDcuFnXihEO0K1NQa2jI",
-        ),
-        (DateCase::Yesterday, "PL7zsB-C3aNu2yRY2869T0zj1FhtRIu5am"),
-        (DateCase::Ago, "PLmB6td997u3kUOrfFwkULZ910ho44oQSy"),
+        (DateCase::Today, "PL3oW2tjiIxvQ98ZTLhBh5soCbE1mC3uAT"),
+        (DateCase::Yesterday, "PLGBuKfnErZlCkRRgt06em8nbXvcV5Sae7"),
+        (DateCase::Ago, "PLAQ7nLSEnhWTEihjeM1I-ToPDJEKfZHZu"),
         (DateCase::Jan, "PL1J-6JOckZtFjcni6Xj1pLYglJp6JCpKD"),
         (DateCase::Feb, "PL1J-6JOckZtETrbzwZE7mRIIK6BzWNLAs"),
         (DateCase::Mar, "PL1J-6JOckZtG3AVdvBXhMO64mB2k3BtKi"),
@@ -90,6 +86,7 @@ pub async fn collect_dates(project_root: &Path, concurrency: usize) {
     let rp = RustyPipe::new();
     let collected_dates = stream::iter(LANGUAGES)
         .map(|lang| {
+            println!("{lang}");
             let rp = rp.clone();
             async move {
                 let mut map: BTreeMap<DateCase, String> = BTreeMap::new();
@@ -115,14 +112,14 @@ pub async fn collect_dates(project_root: &Path, concurrency: usize) {
 ///
 /// The ND (no digit) tokens (today, tomorrow) of some languages cannot be
 /// parsed automatically and require manual work.
-pub fn write_samples_to_dict(project_root: &Path) {
-    let json_path = path!(project_root / "testfiles" / "dict" / "playlist_samples.json");
+pub fn write_samples_to_dict() {
+    let json_path = path!(*DICT_DIR / "playlist_samples.json");
 
     let json_file = File::open(json_path).unwrap();
     let collected_dates: CollectedDates =
         serde_json::from_reader(BufReader::new(json_file)).unwrap();
-    let mut dict = util::read_dict(project_root);
-    let langs = dict.keys().map(|k| k.to_owned()).collect::<Vec<_>>();
+    let mut dict = util::read_dict();
+    let langs = dict.keys().copied().collect::<Vec<_>>();
 
     let months = [
         DateCase::Jan,
@@ -163,30 +160,18 @@ pub fn write_samples_to_dict(project_root: &Path) {
             .for_each(|l| datestr_tables.push(collected_dates.get(l).unwrap()));
 
         let dict_entry = dict.entry(lang).or_default();
-        let mut num_order = "".to_owned();
+        let mut num_order = String::new();
 
         let collect_nd_tokens = !matches!(
             lang,
             // ND tokens of these languages must be edited manually
-            Language::Ja
-            | Language::ZhCn
-            | Language::ZhHk
-            | Language::ZhTw
-            | Language::Ko
-            | Language::Gu
-            | Language::Pa
-            | Language::Ur
-            | Language::Uz
-            | Language::Te
-            | Language::PtPt
-            // Singhalese YT translation has an error (today == tomorrow)
-            | Language::Si
+            Language::Ja | Language::ZhCn | Language::ZhHk | Language::ZhTw
         );
 
         dict_entry.months = BTreeMap::new();
 
         if collect_nd_tokens {
-            dict_entry.timeago_nd_tokens = BTreeMap::new();
+            dict_entry.timeago_nd_tokens = OrderedHashMap::new();
         }
 
         for datestr_table in &datestr_tables {
@@ -212,20 +197,6 @@ pub fn write_samples_to_dict(project_root: &Path) {
                 parse(datestr_table.get(&DateCase::Jan).unwrap(), 0);
             }
 
-            // n days ago
-            {
-                let datestr = datestr_table.get(&DateCase::Ago).unwrap();
-                let tago = timeago::parse_timeago(lang, datestr);
-                assert_eq!(
-                    tago,
-                    Some(TimeAgo {
-                        n: 3,
-                        unit: timeago::TimeUnit::Day
-                    }),
-                    "lang: {lang}, txt: {datestr}"
-                );
-            }
-
             // Absolute dates (Jan 3, 2020)
             months.iter().enumerate().for_each(|(n, m)| {
                 let datestr = datestr_table.get(m).unwrap();
@@ -266,38 +237,36 @@ pub fn write_samples_to_dict(project_root: &Path) {
                     });
             });
 
-            month_words.iter().for_each(|(word, m)| {
+            for (word, m) in &month_words {
                 if *m != 0 {
-                    dict_entry.months.insert(word.to_owned(), *m as u8);
+                    dict_entry.months.insert(word.clone(), *m as u8);
                 };
-            });
+            }
 
             if collect_nd_tokens {
-                td_words.iter().for_each(|(word, n)| {
+                for (word, n) in &td_words {
                     match n {
                         // Today
                         1 => {
                             dict_entry
                                 .timeago_nd_tokens
-                                .insert(word.to_owned(), "0D".to_owned());
+                                .insert(word.clone(), "0D".to_owned());
                         }
                         // Yesterday
                         2 => {
                             dict_entry
                                 .timeago_nd_tokens
-                                .insert(word.to_owned(), "1D".to_owned());
+                                .insert(word.clone(), "1D".to_owned());
                         }
                         _ => {}
                     };
-                });
+                }
 
-                if datestr_tables.len() == 1 {
-                    assert_eq!(
-                        dict_entry.timeago_nd_tokens.len(),
-                        2,
-                        "lang: {}, nd_tokens: {:?}",
+                if datestr_tables.len() == 1 && dict_entry.timeago_nd_tokens.len() > 2 {
+                    println!(
+                        "INFO: {} has {} nd_tokens. Check manually.",
                         lang,
-                        &dict_entry.timeago_nd_tokens
+                        dict_entry.timeago_nd_tokens.len()
                     );
                 }
             }
@@ -305,5 +274,5 @@ pub fn write_samples_to_dict(project_root: &Path) {
         dict_entry.date_order = num_order;
     }
 
-    util::write_dict(project_root, &dict);
+    util::write_dict(dict);
 }

codegen/src/collect_video_dates.rs

@@ -0,0 +1,84 @@
+use std::{
+    collections::{BTreeMap, HashSet},
+    fs::File,
+};
+
+use futures_util::{stream, StreamExt};
+use path_macro::path;
+use rustypipe::{
+    client::{RustyPipe, RustyPipeQuery},
+    param::{Language, LANGUAGES},
+};
+
+use crate::util::DICT_DIR;
+
+pub async fn collect_video_dates(concurrency: usize) {
+    let json_path = path!(*DICT_DIR / "timeago_samples_short.json");
+    let rp = RustyPipe::builder()
+        .visitor_data("Cgtwel9tMkh2eHh0USiyzc6jBg%3D%3D")
+        .build()
+        .unwrap();
+
+    let channels = [
+        "UCeY0bbntWzzVIaj2z3QigXg",
+        "UCcmpeVbSSQlZRvHfdC-CRwg",
+        "UC65afEgL62PGFWXY7n6CUbA",
+        "UCEOXxzW2vU0P-0THehuIIeg",
+    ];
+
+    let mut lang_strings: BTreeMap<Language, Vec<String>> = BTreeMap::new();
+    for lang in LANGUAGES {
+        println!("{lang}");
+        let query = rp.query().lang(lang);
+        let strings = stream::iter(channels)
+            .map(|id| get_channel_datestrings(&query, id))
+            .buffered(concurrency)
+            .collect::<Vec<_>>()
+            .await
+            .into_iter()
+            .flatten()
+            .collect::<Vec<_>>();
+        lang_strings.insert(lang, strings);
+    }
+
+    let mut en_strings_uniq: HashSet<&str> = HashSet::new();
+    let mut uniq_ids: HashSet<usize> = HashSet::new();
+
+    lang_strings[&Language::En]
+        .iter()
+        .enumerate()
+        .for_each(|(n, s)| {
+            if en_strings_uniq.insert(s) {
+                uniq_ids.insert(n);
+            }
+        });
+
+    let strings_map = lang_strings
+        .iter()
+        .map(|(lang, strings)| {
+            (
+                lang,
+                strings
+                    .iter()
+                    .enumerate()
+                    .filter(|(n, _)| uniq_ids.contains(n))
+                    .map(|(_, s)| s)
+                    .collect::<Vec<_>>(),
+            )
+        })
+        .collect::<BTreeMap<_, _>>();
+
+    let file = File::create(json_path).unwrap();
+    serde_json::to_writer_pretty(file, &strings_map).unwrap();
+}
+
+async fn get_channel_datestrings(rp: &RustyPipeQuery, id: &str) -> Vec<String> {
+    let channel = rp.channel_videos(id).await.unwrap();
+
+    channel
+        .content
+        .items
+        .into_iter()
+        .filter_map(|itm| itm.publish_date_txt)
+        .collect()
+}

codegen/src/collect_video_durations.rs

@@ -0,0 +1,373 @@
+use std::{
+    collections::{BTreeMap, HashMap},
+    fs::File,
+    io::BufReader,
+};
+
+use anyhow::Result;
+use futures_util::{stream, StreamExt};
+use path_macro::path;
+use rustypipe::{
+    client::{ClientType, RustyPipe, RustyPipeQuery},
+    param::{Language, LANGUAGES},
+};
+
+use crate::{
+    model::{Channel, QBrowse, TimeAgo, TimeUnit},
+    util::{self, DICT_DIR},
+};
+
+type CollectedDurations = BTreeMap<Language, BTreeMap<String, u32>>;
+
+/// Collect the video duration texts in every supported language
+/// and write them to `testfiles/dict/video_duration_samples.json`.
+///
+/// The length of YouTube short videos is only available in textual form.
+/// To parse it correctly, we need to collect samples of this text in every
+/// language. We collect these samples from regular channel videos because these
+/// include a textual duration in addition to the easy to parse "mm:ss"
+/// duration format.
+pub async fn collect_video_durations(concurrency: usize) {
+    let json_path = path!(*DICT_DIR / "video_duration_samples.json");
+    let rp = RustyPipe::new();
+
+    let channels = [
+        "UCq-Fj5jknLsUf-MWSy4_brA",
+        "UCMcS5ITpSohfr8Ppzlo4vKw",
+        "UCXuqSBlHAE6Xw-yeJA0Tunw",
+    ];
+
+    let durations: CollectedDurations = stream::iter(LANGUAGES)
+        .map(|lang| {
+            let rp = rp.query().lang(lang);
+            async move {
+                let mut map = BTreeMap::new();
+
+                for (n, ch_id) in channels.iter().enumerate() {
+                    get_channel_vlengths(&rp, ch_id, &mut map).await.unwrap();
+                    println!("collected {lang}-{n}");
+                }
+
+                // Since we are only parsing shorts durations, we do not need durations >= 1h
+                let map = map.into_iter().filter(|(_, v)| v < &3600).collect();
+                (lang, map)
+            }
+        })
+        .buffer_unordered(concurrency)
+        .collect()
+        .await;
+
+    let file = File::create(json_path).unwrap();
+    serde_json::to_writer_pretty(file, &durations).unwrap();
+}
+
+pub fn parse_video_durations() {
+    let json_path = path!(*DICT_DIR / "video_duration_samples.json");
+    let json_file = File::open(json_path).unwrap();
+    let durations: CollectedDurations = serde_json::from_reader(BufReader::new(json_file)).unwrap();
+
+    let mut dict = util::read_dict();
+    let langs = dict.keys().copied().collect::<Vec<_>>();
+
+    for lang in langs {
+        let dict_entry = dict.entry(lang).or_default();
+
+        let mut e_langs = dict_entry.equivalent.clone();
+        e_langs.push(lang);
+
+        for lang in e_langs {
+            let mut words = HashMap::new();
+
+            fn check_add_word(
+                words: &mut HashMap<String, Option<TimeAgo>>,
+                by_char: bool,
+                val: u32,
+                expect: u32,
+                w: &str,
+                unit: TimeUnit,
+            ) -> bool {
+                let ok = val == expect || val * 2 == expect;
+                if ok {
+                    let mut ins = |w: &str, val: &mut TimeAgo| {
+                        // Filter stop words
+                        if matches!(
+                            w,
+                            "na" | "y"
+                                | "و"
+                                | "ja"
+                                | "et"
+                                | "e"
+                                | "i"
+                                | "և"
+                                | "og"
+                                | "en"
+                                | "и"
+                                | "a"
+                                | "és"
+                                | "ir"
+                                | "un"
+                                | "și"
+                                | "in"
+                                | "และ"
+                                | "\u{0456}"
+                                | "鐘"
+                                | "eta"
+                                | "અને"
+                                | "और"
+                                | "കൂടാതെ"
+                                | "සහ"
+                        ) {
+                            return;
+                        }
+
+                        let entry = words.entry(w.to_owned()).or_insert(Some(*val));
+                        if let Some(e) = entry {
+                            if e != val {
+                                *entry = None;
+                            }
+                        }
+                    };
+
+                    let mut val = TimeAgo {
+                        n: (expect / val).try_into().unwrap(),
+                        unit,
+                    };
+
+                    if by_char {
+                        w.chars().for_each(|c| {
+                            if !c.is_whitespace() {
+                                ins(&c.to_string(), &mut val);
+                            }
+                        });
+                    } else {
+                        w.split_whitespace().for_each(|w| ins(w, &mut val));
+                    }
+                }
+                ok
+            }
+
+            fn parse(
+                words: &mut HashMap<String, Option<TimeAgo>>,
+                lang: Language,
+                by_char: bool,
+                txt: &str,
+                d: u32,
+            ) {
+                let (m, s) = split_duration(d);
+
+                let mut parts =
+                    split_duration_txt(txt, matches!(lang, Language::Si | Language::Sw))
+                        .into_iter();
+
+                let p1 = parts.next().unwrap();
+                let p1_n = p1.digits.parse::<u32>().unwrap_or(1);
+                let p2: Option<DurationTxtSegment> = parts.next();
+
+                match p2 {
+                    Some(p2) => {
+                        let p2_n = p2.digits.parse::<u32>().unwrap_or(1);
+
+                        assert!(
+                            check_add_word(words, by_char, p1_n, m, &p1.word, TimeUnit::Minute),
+                            "{txt}: min parse error"
+                        );
+                        assert!(
+                            check_add_word(words, by_char, p2_n, s, &p2.word, TimeUnit::Second),
+                            "{txt}: sec parse error"
+                        );
+                    }
+                    None => {
+                        if s == 0 {
+                            assert!(
+                                check_add_word(words, by_char, p1_n, m, &p1.word, TimeUnit::Minute),
+                                "{txt}: min parse error"
+                            );
+                        } else if m == 0 {
+                            assert!(
+                                check_add_word(words, by_char, p1_n, s, &p1.word, TimeUnit::Second),
+                                "{txt}: sec parse error"
+                            );
+                        } else {
+                            let p = txt
+                                .find([',', 'و'])
+                                .unwrap_or_else(|| panic!("`{txt}`: only 1 part"));
+                            parse(words, lang, by_char, &txt[0..p], m);
+                            parse(words, lang, by_char, &txt[p..], s);
+                        }
+                    }
+                }
+
+                assert!(parts.next().is_none(), "`{txt}`: more than 2 parts");
+            }
+
+            for (txt, d) in &durations[&lang] {
+                parse(&mut words, lang, dict_entry.by_char, txt, *d);
+            }
+
+            for (k, v) in words {
+                if let Some(v) = v {
+                    dict_entry.timeago_tokens.insert(k, v.to_string());
+                }
+            }
+        }
+    }
+
+    util::write_dict(dict);
+}
+
+fn split_duration(d: u32) -> (u32, u32) {
+    (d / 60, d % 60)
+}
+
+#[derive(Debug, Default)]
+struct DurationTxtSegment {
+    digits: String,
+    word: String,
+}
+
+fn split_duration_txt(txt: &str, start_c: bool) -> Vec<DurationTxtSegment> {
+    let mut segments = Vec::new();
+
+    // 1: parse digits, 2: parse word
+    let mut state: u8 = 0;
+    let mut seg = DurationTxtSegment::default();
+
+    for c in txt.chars() {
+        if c.is_ascii_digit() {
+            if state == 2 && (!seg.digits.is_empty() || (!start_c && segments.is_empty())) {
+                segments.push(seg);
+                seg = DurationTxtSegment::default();
+            }
+            seg.digits.push(c);
+            state = 1;
+        } else {
+            if (state == 1) && (!seg.word.is_empty() || (start_c && segments.is_empty())) {
+                segments.push(seg);
+                seg = DurationTxtSegment::default();
+            }
+            if c != ',' {
+                c.to_lowercase().for_each(|c| seg.word.push(c));
+            }
+            state = 2;
+        }
+    }
+    if !seg.word.is_empty() || !seg.digits.is_empty() {
+        segments.push(seg);
+    }
+
+    segments
+}
+
+async fn get_channel_vlengths(
+    query: &RustyPipeQuery,
+    channel_id: &str,
+    map: &mut BTreeMap<String, u32>,
+) -> Result<()> {
+    let resp = query
+        .raw(
+            ClientType::Desktop,
+            "browse",
+            &QBrowse {
+                browse_id: channel_id,
+                params: Some("EgZ2aWRlb3MYASAAMAE"),
+            },
+        )
+        .await?;
+
+    let channel = serde_json::from_str::<Channel>(&resp)?;
+
+    let tab = channel
+        .contents
+        .two_column_browse_results_renderer
+        .tabs
+        .into_iter()
+        .next()
+        .unwrap()
+        .tab_renderer
+        .content
+        .rich_grid_renderer;
+
+    tab.contents.into_iter().for_each(|c| {
+        let lt = c.rich_item_renderer.content.video_renderer.length_text;
+        let duration = util::parse_video_length(&lt.simple_text).unwrap();
+        map.insert(lt.accessibility.accessibility_data.label, duration);
+    });
+
+    Ok(())
+}
+
+#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord)]
+enum PluralCategory {
+    Zero,
+    One,
+    Two,
+    Few,
+    Many,
+    Other,
+}
+
+impl From<intl_pluralrules::PluralCategory> for PluralCategory {
+    fn from(value: intl_pluralrules::PluralCategory) -> Self {
+        match value {
+            intl_pluralrules::PluralCategory::ZERO => Self::Zero,
+            intl_pluralrules::PluralCategory::ONE => Self::One,
+            intl_pluralrules::PluralCategory::TWO => Self::Two,
+            intl_pluralrules::PluralCategory::FEW => Self::Few,
+            intl_pluralrules::PluralCategory::MANY => Self::Many,
+            intl_pluralrules::PluralCategory::OTHER => Self::Other,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use std::collections::HashSet;
+    use std::io::BufReader;
+
+    use intl_pluralrules::{PluralRuleType, PluralRules};
+    use unic_langid::LanguageIdentifier;
+
+    /// Verify that the duration sample set covers all pluralization variants of the languages
+    #[test]
+    fn check_video_duration_samples() {
+        let json_path = path!(*DICT_DIR / "video_duration_samples.json");
+        let json_file = File::open(json_path).unwrap();
+        let durations: CollectedDurations =
+            serde_json::from_reader(BufReader::new(json_file)).unwrap();
+        let mut failed = false;
+
+        for (lang, durations) in durations {
+            let ul: LanguageIdentifier =
+                lang.to_string().split('-').next().unwrap().parse().unwrap();
+
+            let pr = PluralRules::create(ul, PluralRuleType::CARDINAL)
+                .unwrap_or_else(|_| panic!("{}", lang.to_string()));
+
+            let mut plurals_m: HashSet<PluralCategory> = HashSet::new();
+            for n in 1..60 {
+                plurals_m.insert(pr.select(n).unwrap().into());
+            }
+            let mut plurals_s = plurals_m.clone();
+
+            for v in durations.values() {
+                let (m, s) = split_duration(*v);
+                plurals_m.remove(&pr.select(m).unwrap().into());
+                plurals_s.remove(&pr.select(s).unwrap().into());
+            }
+
+            if !plurals_m.is_empty() {
+                println!("{lang}: missing minutes {plurals_m:?}");
+                failed = true;
+            }
+
+            if !plurals_s.is_empty() {
+                println!("{lang}: missing seconds {plurals_m:?}");
+                failed = true;
+            }
+        }
+
+        assert!(!failed);
+    }
+}

codegen/src/download_testfiles.rs

@@ -5,71 +5,80 @@ use std::{
     sync::Mutex,
 };
 
+use path_macro::path;
 use rustypipe::{
     client::{ClientType, RustyPipe},
+    model::YouTubeItem,
     param::{
         search_filter::{self, ItemType, SearchFilter},
-        Country,
+        ChannelVideoTab, Country,
     },
     report::{Report, Reporter},
 };
 
-pub async fn download_testfiles(project_root: &Path) {
-    let mut testfiles = project_root.to_path_buf();
-    testfiles.push("testfiles");
+use crate::util::TESTFILES_DIR;
 
-    player(&testfiles).await;
-    player_model(&testfiles).await;
-    playlist(&testfiles).await;
-    playlist_cont(&testfiles).await;
-    video_details(&testfiles).await;
-    comments_top(&testfiles).await;
-    comments_latest(&testfiles).await;
-    recommendations(&testfiles).await;
-    channel_videos(&testfiles).await;
-    channel_shorts(&testfiles).await;
-    channel_livestreams(&testfiles).await;
-    channel_playlists(&testfiles).await;
-    channel_info(&testfiles).await;
-    channel_videos_cont(&testfiles).await;
-    channel_playlists_cont(&testfiles).await;
-    channel_tv(&testfiles).await;
-    search(&testfiles).await;
-    search_cont(&testfiles).await;
-    search_playlists(&testfiles).await;
-    search_empty(&testfiles).await;
-    startpage(&testfiles).await;
-    startpage_cont(&testfiles).await;
-    trending(&testfiles).await;
+pub async fn download_testfiles() {
+    player().await;
+    player_model().await;
+    playlist().await;
+    playlist_cont().await;
+    video_details().await;
+    comments_top().await;
+    comments_latest().await;
+    recommendations().await;
+    channel_videos().await;
+    channel_shorts().await;
+    channel_livestreams().await;
+    channel_playlists().await;
+    channel_info().await;
+    channel_videos_cont().await;
+    channel_playlists_cont().await;
+    search().await;
+    search_cont().await;
+    search_playlists().await;
+    search_empty().await;
+    trending().await;
 
-    music_playlist(&testfiles).await;
-    music_playlist_cont(&testfiles).await;
-    music_playlist_related(&testfiles).await;
-    music_album(&testfiles).await;
-    music_search(&testfiles).await;
-    music_search_tracks(&testfiles).await;
-    music_search_albums(&testfiles).await;
-    music_search_artists(&testfiles).await;
-    music_search_playlists(&testfiles).await;
-    music_search_cont(&testfiles).await;
-    music_search_suggestion(&testfiles).await;
-    music_artist(&testfiles).await;
-    music_details(&testfiles).await;
-    music_lyrics(&testfiles).await;
-    music_related(&testfiles).await;
-    music_radio(&testfiles).await;
-    music_radio_cont(&testfiles).await;
-    music_new_albums(&testfiles).await;
-    music_new_videos(&testfiles).await;
-    music_charts(&testfiles).await;
-    music_genres(&testfiles).await;
-    music_genre(&testfiles).await;
+    music_playlist().await;
+    music_playlist_cont().await;
+    music_playlist_related().await;
+    music_album().await;
+    music_search().await;
+    music_search_tracks().await;
+    music_search_albums().await;
+    music_search_artists().await;
+    music_search_playlists().await;
+    music_search_cont().await;
+    music_search_suggestion().await;
+    music_artist().await;
+    music_details().await;
+    music_lyrics().await;
+    music_related().await;
+    music_radio().await;
+    music_radio_cont().await;
+    music_new_albums().await;
+    music_new_videos().await;
+    music_charts().await;
+    music_genres().await;
+    music_genre().await;
+
+    // User data
+    history().await;
+    subscriptions().await;
+    subscription_feed().await;
+
+    music_history().await;
+    music_saved_artists().await;
+    music_saved_albums().await;
+    music_saved_tracks().await;
+    music_saved_playlists().await;
 }
 
 const CLIENT_TYPES: [ClientType; 5] = [
     ClientType::Desktop,
     ClientType::DesktopMusic,
-    ClientType::TvHtml5Embed,
+    ClientType::Tv,
     ClientType::Android,
     ClientType::Ios,
 ];
@@ -135,16 +144,15 @@ fn rp_testfile(json_path: &Path) -> RustyPipe {
         .report()
         .strict()
         .build()
+        .unwrap()
 }
 
-async fn player(testfiles: &Path) {
+async fn player() {
     let video_id = "pPvd8UxmSbQ";
 
     for client_type in CLIENT_TYPES {
-        let mut json_path = testfiles.to_path_buf();
-        json_path.push("player");
-        json_path.push(format!("{client_type:?}_video.json").to_lowercase());
-
+        let json_path =
+            path!(*TESTFILES_DIR / "player" / format!("{client_type:?}_video.json").to_lowercase());
         if json_path.exists() {
             continue;
         }
@@ -157,14 +165,12 @@ async fn player(testfiles: &Path) {
     }
 }
 
-async fn player_model(testfiles: &Path) {
-    let rp = RustyPipe::builder().strict().build();
+async fn player_model() {
+    let rp = RustyPipe::builder().strict().build().unwrap();
 
     for (name, id) in [("multilanguage", "tVWWp1PqDus"), ("hdr", "LXb3EKWsInQ")] {
-        let mut json_path = testfiles.to_path_buf();
-        json_path.push("player_model");
-        json_path.push(format!("{name}.json").to_lowercase());
-
+        let json_path =
+            path!(*TESTFILES_DIR / "player_model" / format!("{name}.json").to_lowercase());
         if json_path.exists() {
             continue;
         }
@@ -181,15 +187,14 @@ async fn player_model(testfiles: &Path) {
     }
 }
 
-async fn playlist(testfiles: &Path) {
+async fn playlist() {
     for (name, id) in [
         ("short", "RDCLAK5uy_kFQXdnqMaQCVx2wpUM4ZfbsGCDibZtkJk"),
         ("long", "PL5dDx681T4bR7ZF1IuWzOv1omlRbE7PiJ"),
         ("nomusic", "PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe"),
+        ("live", "UULVvqRdlKsE5Q8mf8YXbdIJLw"),
     ] {
-        let mut json_path = testfiles.to_path_buf();
-        json_path.push("playlist");
-        json_path.push(format!("playlist_{name}.json"));
+        let json_path = path!(*TESTFILES_DIR / "playlist" / format!("playlist_{name}.json"));
         if json_path.exists() {
             continue;
         }
@@ -199,10 +204,8 @@ async fn playlist(testfiles: &Path) {
     }
 }
 
-async fn playlist_cont(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("playlist");
-    json_path.push("playlist_cont.json");
+async fn playlist_cont() {
+    let json_path = path!(*TESTFILES_DIR / "playlist" / "playlist_cont.json");
     if json_path.exists() {
         return;
     }
@@ -218,7 +221,7 @@ async fn playlist_cont(testfiles: &Path) {
     playlist.videos.next(rp.query()).await.unwrap().unwrap();
 }
 
-async fn video_details(testfiles: &Path) {
+async fn video_details() {
     for (name, id) in [
         ("music", "XuM2onMGvTI"),
         ("mv", "ZeerrnuLi5E"),
@@ -227,9 +230,8 @@ async fn video_details(testfiles: &Path) {
         ("live", "86YLFOog4GM"),
         ("agegate", "HRKu0cvrr_o"),
     ] {
-        let mut json_path = testfiles.to_path_buf();
-        json_path.push("video_details");
-        json_path.push(format!("video_details_{name}.json"));
+        let json_path =
+            path!(*TESTFILES_DIR / "video_details" / format!("video_details_{name}.json"));
         if json_path.exists() {
             continue;
         }
@@ -239,10 +241,8 @@ async fn video_details(testfiles: &Path) {
     }
 }
 
-async fn comments_top(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("video_details");
-    json_path.push("comments_top.json");
+async fn comments_top() {
+    let json_path = path!(*TESTFILES_DIR / "video_details" / "comments_top.json");
     if json_path.exists() {
         return;
     }
@@ -259,10 +259,8 @@ async fn comments_top(testfiles: &Path) {
         .unwrap();
 }
 
-async fn comments_latest(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("video_details");
-    json_path.push("comments_latest.json");
+async fn comments_latest() {
+    let json_path = path!(*TESTFILES_DIR / "video_details" / "comments_latest.json");
     if json_path.exists() {
         return;
     }
@@ -279,10 +277,8 @@ async fn comments_latest(testfiles: &Path) {
         .unwrap();
 }
 
-async fn recommendations(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("video_details");
-    json_path.push("recommendations.json");
+async fn recommendations() {
+    let json_path = path!(*TESTFILES_DIR / "video_details" / "recommendations.json");
     if json_path.exists() {
         return;
     }
@@ -294,7 +290,7 @@ async fn recommendations(testfiles: &Path) {
     details.recommended.next(rp.query()).await.unwrap();
 }
 
-async fn channel_videos(testfiles: &Path) {
+async fn channel_videos() {
     for (name, id) in [
         ("base", "UC2DjFE7Xf11URZqWBigcVOQ"),
         ("music", "UC_vmjW5e1xEHhYjY2a0kK1A"), // YouTube Music channels have no videos
@@ -303,9 +299,7 @@ async fn channel_videos(testfiles: &Path) {
         ("empty", "UCxBa895m48H5idw5li7h-0g"),
         ("upcoming", "UCcvfHa-GHSOHFAjU0-Ie57A"),
     ] {
-        let mut json_path = testfiles.to_path_buf();
-        json_path.push("channel");
-        json_path.push(format!("channel_videos_{name}.json"));
+        let json_path = path!(*TESTFILES_DIR / "channel" / format!("channel_videos_{name}.json"));
         if json_path.exists() {
             continue;
         }
@@ -315,40 +309,34 @@ async fn channel_videos(testfiles: &Path) {
     }
 }
 
-async fn channel_shorts(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("channel");
-    json_path.push("channel_shorts.json");
+async fn channel_shorts() {
+    let json_path = path!(*TESTFILES_DIR / "channel" / "channel_shorts.json");
     if json_path.exists() {
         return;
     }
 
     let rp = rp_testfile(&json_path);
     rp.query()
-        .channel_shorts("UCh8gHdtzO2tXd593_bjErWg")
+        .channel_videos_tab("UCh8gHdtzO2tXd593_bjErWg", ChannelVideoTab::Shorts)
         .await
         .unwrap();
 }
 
-async fn channel_livestreams(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("channel");
-    json_path.push("channel_livestreams.json");
+async fn channel_livestreams() {
+    let json_path = path!(*TESTFILES_DIR / "channel" / "channel_livestreams.json");
     if json_path.exists() {
         return;
     }
 
     let rp = rp_testfile(&json_path);
     rp.query()
-        .channel_livestreams("UC2DjFE7Xf11URZqWBigcVOQ")
+        .channel_videos_tab("UC2DjFE7Xf11URZqWBigcVOQ", ChannelVideoTab::Live)
         .await
         .unwrap();
 }
 
-async fn channel_playlists(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("channel");
-    json_path.push("channel_playlists.json");
+async fn channel_playlists() {
+    let json_path = path!(*TESTFILES_DIR / "channel" / "channel_playlists.json");
     if json_path.exists() {
         return;
     }
@@ -360,10 +348,8 @@ async fn channel_playlists(testfiles: &Path) {
         .unwrap();
 }
 
-async fn channel_info(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("channel");
-    json_path.push("channel_info.json");
+async fn channel_info() {
+    let json_path = path!(*TESTFILES_DIR / "channel" / "channel_info.json");
     if json_path.exists() {
         return;
     }
@@ -375,10 +361,8 @@ async fn channel_info(testfiles: &Path) {
         .unwrap();
 }
 
-async fn channel_videos_cont(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("channel");
-    json_path.push("channel_videos_cont.json");
+async fn channel_videos_cont() {
+    let json_path = path!(*TESTFILES_DIR / "channel" / "channel_videos_cont.json");
     if json_path.exists() {
         return;
     }
@@ -394,10 +378,8 @@ async fn channel_videos_cont(testfiles: &Path) {
     videos.content.next(rp.query()).await.unwrap().unwrap();
 }
 
-async fn channel_playlists_cont(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("channel");
-    json_path.push("channel_playlists_cont.json");
+async fn channel_playlists_cont() {
+    let json_path = path!(*TESTFILES_DIR / "channel" / "channel_playlists_cont.json");
     if json_path.exists() {
         return;
     }
@@ -413,79 +395,58 @@ async fn channel_playlists_cont(testfiles: &Path) {
     playlists.content.next(rp.query()).await.unwrap().unwrap();
 }
 
-async fn channel_tv(testfiles: &Path) {
-    for (name, id) in [
-        ("base", "UCXuqSBlHAE6Xw-yeJA0Tunw"),
-        ("music", "UC_vmjW5e1xEHhYjY2a0kK1A"),
-        ("live", "UCSJ4gkVC6NrvII8umztf0Ow"),
-        ("live_upcoming", "UCWxlUwW9BgGISaakjGM37aw"),
-        ("onevideo", "UCAkeE1thnToEXZTao-CZkHw"),
-    ] {
-        let mut json_path = testfiles.to_path_buf();
-        json_path.push("channel_tv");
-        json_path.push(format!("{name}.json"));
-        if json_path.exists() {
-            continue;
-        }
-
-        let rp = rp_testfile(&json_path);
-        rp.query().channel_tv(id).await.unwrap();
-    }
-}
-
-async fn search(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("search");
-    json_path.push("default.json");
+async fn search() {
+    let json_path = path!(*TESTFILES_DIR / "search" / "default.json");
     if json_path.exists() {
         return;
     }
 
     let rp = rp_testfile(&json_path);
-    rp.query().search("doobydoobap").await.unwrap();
+    rp.query()
+        .search::<YouTubeItem, _>("doobydoobap")
+        .await
+        .unwrap();
 }
 
-async fn search_cont(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("search");
-    json_path.push("cont.json");
+async fn search_cont() {
+    let json_path = path!(*TESTFILES_DIR / "search" / "cont.json");
     if json_path.exists() {
         return;
     }
 
     let rp = RustyPipe::new();
-    let search = rp.query().search("doobydoobap").await.unwrap();
+    let search = rp
+        .query()
+        .search::<YouTubeItem, _>("doobydoobap")
+        .await
+        .unwrap();
 
     let rp = rp_testfile(&json_path);
     search.items.next(rp.query()).await.unwrap().unwrap();
 }
 
-async fn search_playlists(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("search");
-    json_path.push("playlists.json");
+async fn search_playlists() {
+    let json_path = path!(*TESTFILES_DIR / "search" / "playlists.json");
     if json_path.exists() {
         return;
     }
 
     let rp = rp_testfile(&json_path);
     rp.query()
-        .search_filter("pop", &SearchFilter::new().item_type(ItemType::Playlist))
+        .search_filter::<YouTubeItem, _>("pop", &SearchFilter::new().item_type(ItemType::Playlist))
         .await
         .unwrap();
 }
 
-async fn search_empty(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("search");
-    json_path.push("empty.json");
+async fn search_empty() {
+    let json_path = path!(*TESTFILES_DIR / "search" / "empty.json");
     if json_path.exists() {
         return;
     }
 
     let rp = rp_testfile(&json_path);
     rp.query()
-        .search_filter(
+        .search_filter::<YouTubeItem, _>(
             "test",
             &SearchFilter::new()
                 .feature(search_filter::Feature::IsLive)
@@ -495,37 +456,8 @@ async fn search_empty(testfiles: &Path) {
         .unwrap();
 }
 
-async fn startpage(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("trends");
-    json_path.push("startpage.json");
-    if json_path.exists() {
-        return;
-    }
-
-    let rp = rp_testfile(&json_path);
-    rp.query().startpage().await.unwrap();
-}
-
-async fn startpage_cont(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("trends");
-    json_path.push("startpage_cont.json");
-    if json_path.exists() {
-        return;
-    }
-
-    let rp = RustyPipe::new();
-    let startpage = rp.query().startpage().await.unwrap();
-
-    let rp = rp_testfile(&json_path);
-    startpage.next(rp.query()).await.unwrap();
-}
-
-async fn trending(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("trends");
-    json_path.push("trending.json");
+async fn trending() {
+    let json_path = path!(*TESTFILES_DIR / "trends" / "trending_videos.json");
     if json_path.exists() {
         return;
     }
@@ -534,15 +466,43 @@ async fn trending(testfiles: &Path) {
     rp.query().trending().await.unwrap();
 }
 
-async fn music_playlist(testfiles: &Path) {
+async fn history() {
+    let json_path = path!(*TESTFILES_DIR / "userdata" / "history.json");
+    if json_path.exists() {
+        return;
+    }
+
+    let rp = rp_testfile(&json_path);
+    rp.query().history().await.unwrap();
+}
+
+async fn subscriptions() {
+    let json_path = path!(*TESTFILES_DIR / "userdata" / "subscriptions.json");
+    if json_path.exists() {
+        return;
+    }
+
+    let rp = rp_testfile(&json_path);
+    rp.query().subscriptions().await.unwrap();
+}
+
+async fn subscription_feed() {
+    let json_path = path!(*TESTFILES_DIR / "userdata" / "subscription_feed.json");
+    if json_path.exists() {
+        return;
+    }
+
+    let rp = rp_testfile(&json_path);
+    rp.query().subscription_feed().await.unwrap();
+}
+
+async fn music_playlist() {
     for (name, id) in [
         ("short", "RDCLAK5uy_kFQXdnqMaQCVx2wpUM4ZfbsGCDibZtkJk"),
         ("long", "PL5dDx681T4bR7ZF1IuWzOv1omlRbE7PiJ"),
         ("nomusic", "PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe"),
     ] {
-        let mut json_path = testfiles.to_path_buf();
-        json_path.push("music_playlist");
-        json_path.push(format!("playlist_{name}.json"));
+        let json_path = path!(*TESTFILES_DIR / "music_playlist" / format!("playlist_{name}.json"));
         if json_path.exists() {
             continue;
         }
@@ -552,10 +512,8 @@ async fn music_playlist(testfiles: &Path) {
     }
 }
 
-async fn music_playlist_cont(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("music_playlist");
-    json_path.push("playlist_cont.json");
+async fn music_playlist_cont() {
+    let json_path = path!(*TESTFILES_DIR / "music_playlist" / "playlist_cont.json");
     if json_path.exists() {
         return;
     }
@@ -571,10 +529,8 @@ async fn music_playlist_cont(testfiles: &Path) {
     playlist.tracks.next(rp.query()).await.unwrap().unwrap();
 }
 
-async fn music_playlist_related(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("music_playlist");
-    json_path.push("playlist_related.json");
+async fn music_playlist_related() {
+    let json_path = path!(*TESTFILES_DIR / "music_playlist" / "playlist_related.json");
     if json_path.exists() {
         return;
     }
@@ -595,7 +551,7 @@ async fn music_playlist_related(testfiles: &Path) {
         .unwrap();
 }
 
-async fn music_album(testfiles: &Path) {
+async fn music_album() {
     for (name, id) in [
         ("one_artist", "MPREb_nlBWQROfvjo"),
         ("various_artists", "MPREb_8QkDeEIawvX"),
@@ -603,9 +559,7 @@ async fn music_album(testfiles: &Path) {
         ("description", "MPREb_PiyfuVl6aYd"),
         ("unavailable", "MPREb_AzuWg8qAVVl"),
     ] {
-        let mut json_path = testfiles.to_path_buf();
-        json_path.push("music_playlist");
-        json_path.push(format!("album_{name}.json"));
+        let json_path = path!(*TESTFILES_DIR / "music_playlist" / format!("album_{name}.json"));
         if json_path.exists() {
             continue;
         }
@@ -615,26 +569,24 @@ async fn music_album(testfiles: &Path) {
     }
 }
 
-async fn music_search(testfiles: &Path) {
+async fn music_search() {
     for (name, query) in [
         ("default", "black mamba"),
         ("typo", "liblingsmensch"),
         ("radio", "pop radio"),
         ("artist", "taylor swift"),
     ] {
-        let mut json_path = testfiles.to_path_buf();
-        json_path.push("music_search");
-        json_path.push(format!("main_{name}.json"));
+        let json_path = path!(*TESTFILES_DIR / "music_search" / format!("main_{name}.json"));
         if json_path.exists() {
             continue;
         }
 
         let rp = rp_testfile(&json_path);
-        rp.query().music_search(query).await.unwrap();
+        rp.query().music_search_main(query).await.unwrap();
     }
 }
 
-async fn music_search_tracks(testfiles: &Path) {
+async fn music_search_tracks() {
     for (name, query, videos) in [
         ("default", "black mamba", false),
         ("videos", "black mamba", true),
@@ -645,9 +597,7 @@ async fn music_search_tracks(testfiles: &Path) {
             false,
         ),
     ] {
-        let mut json_path = testfiles.to_path_buf();
-        json_path.push("music_search");
-        json_path.push(format!("tracks_{name}.json"));
+        let json_path = path!(*TESTFILES_DIR / "music_search" / format!("tracks_{name}.json"));
         if json_path.exists() {
             continue;
         }
@@ -661,10 +611,8 @@ async fn music_search_tracks(testfiles: &Path) {
     }
 }
 
-async fn music_search_albums(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("music_search");
-    json_path.push("albums.json");
+async fn music_search_albums() {
+    let json_path = path!(*TESTFILES_DIR / "music_search" / "albums.json");
     if json_path.exists() {
         return;
     }
@@ -673,10 +621,8 @@ async fn music_search_albums(testfiles: &Path) {
     rp.query().music_search_albums("black mamba").await.unwrap();
 }
 
-async fn music_search_artists(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("music_search");
-    json_path.push("artists.json");
+async fn music_search_artists() {
+    let json_path = path!(*TESTFILES_DIR / "music_search" / "artists.json");
     if json_path.exists() {
         return;
     }
@@ -688,27 +634,23 @@ async fn music_search_artists(testfiles: &Path) {
         .unwrap();
 }
 
-async fn music_search_playlists(testfiles: &Path) {
+async fn music_search_playlists() {
     for (name, community) in [("ytm", false), ("community", true)] {
-        let mut json_path = testfiles.to_path_buf();
-        json_path.push("music_search");
-        json_path.push(format!("playlists_{name}.json"));
+        let json_path = path!(*TESTFILES_DIR / "music_search" / format!("playlists_{name}.json"));
         if json_path.exists() {
             continue;
         }
 
         let rp = rp_testfile(&json_path);
         rp.query()
-            .music_search_playlists_filter("pop", community)
+            .music_search_playlists("pop", community)
             .await
             .unwrap();
     }
 }
 
-async fn music_search_cont(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("music_search");
-    json_path.push("tracks_cont.json");
+async fn music_search_cont() {
+    let json_path = path!(*TESTFILES_DIR / "music_search" / "tracks_cont.json");
     if json_path.exists() {
         return;
     }
@@ -720,11 +662,9 @@ async fn music_search_cont(testfiles: &Path) {
     res.items.next(rp.query()).await.unwrap().unwrap();
 }
 
-async fn music_search_suggestion(testfiles: &Path) {
+async fn music_search_suggestion() {
     for (name, query) in [("default", "t"), ("empty", "reujbhevmfndxnjrze")] {
-        let mut json_path = testfiles.to_path_buf();
-        json_path.push("music_search");
-        json_path.push(format!("suggestion_{name}.json"));
+        let json_path = path!(*TESTFILES_DIR / "music_search" / format!("suggestion_{name}.json"));
         if json_path.exists() {
             continue;
         }
@@ -734,18 +674,15 @@ async fn music_search_suggestion(testfiles: &Path) {
     }
 }
 
-async fn music_artist(testfiles: &Path) {
+async fn music_artist() {
     for (name, id, all_albums) in [
         ("default", "UClmXPfaYhXOYsNn_QUyheWQ", true),
-        ("no_more_albums", "UC_vmjW5e1xEHhYjY2a0kK1A", true),
         ("only_singles", "UCfwCE5VhPMGxNPFxtVv7lRw", true),
         ("no_artist", "UCh8gHdtzO2tXd593_bjErWg", true),
         ("only_more_singles", "UC0aXrjVxG5pZr99v77wZdPQ", true),
         ("secondary_channel", "UCC9192yGQD25eBZgFZ84MPw", false),
     ] {
-        let mut json_path = testfiles.to_path_buf();
-        json_path.push("music_artist");
-        json_path.push(format!("artist_{name}.json"));
+        let json_path = path!(*TESTFILES_DIR / "music_artist" / format!("artist_{name}.json"));
         if json_path.exists() {
             continue;
         }
@@ -755,11 +692,9 @@ async fn music_artist(testfiles: &Path) {
     }
 }
 
-async fn music_details(testfiles: &Path) {
+async fn music_details() {
     for (name, id) in [("mv", "ZeerrnuLi5E"), ("track", "7nigXQS1Xb0")] {
-        let mut json_path = testfiles.to_path_buf();
-        json_path.push("music_details");
-        json_path.push(format!("details_{name}.json"));
+        let json_path = path!(*TESTFILES_DIR / "music_details" / format!("details_{name}.json"));
         if json_path.exists() {
             continue;
         }
@@ -769,10 +704,8 @@ async fn music_details(testfiles: &Path) {
     }
 }
 
-async fn music_lyrics(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("music_details");
-    json_path.push("lyrics.json");
+async fn music_lyrics() {
+    let json_path = path!(*TESTFILES_DIR / "music_details" / "lyrics.json");
     if json_path.exists() {
         return;
     }
@@ -787,10 +720,8 @@ async fn music_lyrics(testfiles: &Path) {
         .unwrap();
 }
 
-async fn music_related(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("music_details");
-    json_path.push("related.json");
+async fn music_related() {
+    let json_path = path!(*TESTFILES_DIR / "music_details" / "related.json");
     if json_path.exists() {
         return;
     }
@@ -805,11 +736,9 @@ async fn music_related(testfiles: &Path) {
         .unwrap();
 }
 
-async fn music_radio(testfiles: &Path) {
+async fn music_radio() {
     for (name, id) in [("mv", "RDAMVMZeerrnuLi5E"), ("track", "RDAMVM7nigXQS1Xb0")] {
-        let mut json_path = testfiles.to_path_buf();
-        json_path.push("music_details");
-        json_path.push(format!("radio_{name}.json"));
+        let json_path = path!(*TESTFILES_DIR / "music_details" / format!("radio_{name}.json"));
         if json_path.exists() {
             continue;
         }
@@ -819,10 +748,8 @@ async fn music_radio(testfiles: &Path) {
     }
 }
 
-async fn music_radio_cont(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("music_details");
-    json_path.push("radio_cont.json");
+async fn music_radio_cont() {
+    let json_path = path!(*TESTFILES_DIR / "music_details" / "radio_cont.json");
     if json_path.exists() {
         return;
     }
@@ -834,10 +761,8 @@ async fn music_radio_cont(testfiles: &Path) {
     res.next(rp.query()).await.unwrap().unwrap();
 }
 
-async fn music_new_albums(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("music_new");
-    json_path.push("albums_default.json");
+async fn music_new_albums() {
+    let json_path = path!(*TESTFILES_DIR / "music_new" / "albums_default.json");
     if json_path.exists() {
         return;
     }
@@ -846,10 +771,8 @@ async fn music_new_albums(testfiles: &Path) {
     rp.query().music_new_albums().await.unwrap();
 }
 
-async fn music_new_videos(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("music_new");
-    json_path.push("videos_default.json");
+async fn music_new_videos() {
+    let json_path = path!(*TESTFILES_DIR / "music_new" / "videos_default.json");
     if json_path.exists() {
         return;
     }
@@ -858,11 +781,9 @@ async fn music_new_videos(testfiles: &Path) {
     rp.query().music_new_videos().await.unwrap();
 }
 
-async fn music_charts(testfiles: &Path) {
+async fn music_charts() {
     for (name, country) in [("global", Some(Country::Zz)), ("US", Some(Country::Us))] {
-        let mut json_path = testfiles.to_path_buf();
-        json_path.push("music_charts");
-        json_path.push(&format!("charts_{name}.json"));
+        let json_path = path!(*TESTFILES_DIR / "music_charts" / format!("charts_{name}.json"));
         if json_path.exists() {
             continue;
         }
@@ -872,10 +793,8 @@ async fn music_charts(testfiles: &Path) {
     }
 }
 
-async fn music_genres(testfiles: &Path) {
-    let mut json_path = testfiles.to_path_buf();
-    json_path.push("music_genres");
-    json_path.push("genres.json");
+async fn music_genres() {
+    let json_path = path!(*TESTFILES_DIR / "music_genres" / "genres.json");
     if json_path.exists() {
         return;
     }
@@ -884,14 +803,12 @@ async fn music_genres(testfiles: &Path) {
     rp.query().music_genres().await.unwrap();
 }
 
-async fn music_genre(testfiles: &Path) {
+async fn music_genre() {
     for (name, id) in [
         ("default", "ggMPOg1uX1lMbVZmbzl6NlJ3"),
         ("mood", "ggMPOg1uX1JOQWZFeDByc2Jm"),
     ] {
-        let mut json_path = testfiles.to_path_buf();
-        json_path.push("music_genres");
-        json_path.push(&format!("genre_{name}.json"));
+        let json_path = path!(*TESTFILES_DIR / "music_genres" / format!("genre_{name}.json"));
         if json_path.exists() {
             continue;
         }
@@ -900,3 +817,53 @@ async fn music_genre(testfiles: &Path) {
         rp.query().music_genre(id).await.unwrap();
     }
 }
+
+async fn music_history() {
+    let json_path = path!(*TESTFILES_DIR / "music_userdata" / "music_history.json");
+    if json_path.exists() {
+        return;
+    }
+
+    let rp = rp_testfile(&json_path);
+    rp.query().music_history().await.unwrap();
+}
+
+async fn music_saved_artists() {
+    let json_path = path!(*TESTFILES_DIR / "music_userdata" / "saved_artists.json");
+    if json_path.exists() {
+        return;
+    }
+
+    let rp = rp_testfile(&json_path);
+    rp.query().music_saved_artists().await.unwrap();
+}
+
+async fn music_saved_albums() {
+    let json_path = path!(*TESTFILES_DIR / "music_userdata" / "saved_albums.json");
+    if json_path.exists() {
+        return;
+    }
+
+    let rp = rp_testfile(&json_path);
+    rp.query().music_saved_albums().await.unwrap();
+}
+
+async fn music_saved_tracks() {
+    let json_path = path!(*TESTFILES_DIR / "music_userdata" / "saved_tracks.json");
+    if json_path.exists() {
+        return;
+    }
+
+    let rp = rp_testfile(&json_path);
+    rp.query().music_saved_tracks().await.unwrap();
+}
+
+async fn music_saved_playlists() {
+    let json_path = path!(*TESTFILES_DIR / "music_userdata" / "saved_playlists.json");
+    if json_path.exists() {
+        return;
+    }
+
+    let rp = rp_testfile(&json_path);
+    rp.query().music_saved_playlists().await.unwrap();
+}

codegen/src/gen_dictionary.rs

@@ -1,16 +1,16 @@
 use std::fmt::Write;
-use std::path::Path;
 
 use once_cell::sync::Lazy;
+use path_macro::path;
 use regex::Regex;
-use rustypipe::timeago::TimeUnit;
 
-use crate::util;
-
-const TARGET_PATH: &str = "src/util/dictionary.rs";
+use crate::{
+    model::TimeUnit,
+    util::{self, SRC_DIR},
+};
 
 fn parse_tu(tu: &str) -> (u8, Option<TimeUnit>) {
-    static TU_PATTERN: Lazy<Regex> = Lazy::new(|| Regex::new(r"^(\d*)(\w?)$").unwrap());
+    static TU_PATTERN: Lazy<Regex> = Lazy::new(|| Regex::new(r"^(\d*)(\w*)$").unwrap());
     match TU_PATTERN.captures(tu) {
         Some(cap) => (
             cap.get(1).unwrap().as_str().parse().unwrap_or(1),
@@ -22,6 +22,8 @@ fn parse_tu(tu: &str) -> (u8, Option<TimeUnit>) {
                 "W" => Some(TimeUnit::Week),
                 "M" => Some(TimeUnit::Month),
                 "Y" => Some(TimeUnit::Year),
+                "Wl" => Some(TimeUnit::LastWeek),
+                "Wd" => Some(TimeUnit::LastWeekday),
                 "" => None,
                 _ => panic!("invalid time unit: {tu}"),
             },
@@ -30,36 +32,24 @@ fn parse_tu(tu: &str) -> (u8, Option<TimeUnit>) {
     }
 }
 
-fn parse_date_cmp(c: char) -> &'static str {
-    match c {
-        'Y' => "Y",
-        'y' => "YShort",
-        'M' => "M",
-        'D' => "D",
-        'H' => "Hr",
-        'h' => "Hr12",
-        'm' => "Mi",
-        _ => panic!("invalid date cmp: {c}"),
-    }
-}
-
-pub fn generate_dictionary(project_root: &Path) {
-    let dict = util::read_dict(project_root);
+pub fn generate_dictionary() {
+    let dict = util::read_dict();
 
     let code_head = r#"// This file is automatically generated. DO NOT EDIT.
 // See codegen/gen_dictionary.rs for the generation code.
+#![allow(clippy::unreadable_literal)]
+
+//! The dictionary contains the information required to parse dates and numbers
+//! in all supported languages.
+
 use crate::{
     model::AlbumType,
     param::Language,
-    timeago::{DateCmp, TaToken, TimeUnit},
+    util::timeago::{TaToken, TimeUnit},
 };
 
-/// The dictionary contains the information required to parse dates and numbers
-/// in all supported languages.
+/// Dictionary entry containing language-specific parsing information
 pub(crate) struct Entry {
-    /// Should the language be parsed by character instead of by word?
-    /// (e.g. Chinese/Japanese)
-    pub by_char: bool,
     /// Tokens for parsing timeago strings.
     ///
     /// Format: Parsed token -> \[Quantity\] Identifier
@@ -67,20 +57,13 @@ pub(crate) struct Entry {
     /// Identifiers: `Y`(ear), `M`(month), `W`(eek), `D`(ay),
     /// `h`(our), `m`(inute), `s`(econd)
     pub timeago_tokens: phf::Map<&'static str, TaToken>,
-    /// Order in which to parse numeric date components.
+    /// True if the month has to be parsed before the day
     ///
     /// Examples:
     ///
-    /// - 03.01.2020 => `"DMY"`
-    /// - Jan 3, 2020 => `"DY"`
-    pub date_order: &'static [DateCmp],
-    /// Order in which to parse datetimes.
-    ///
-    /// Examples:
-    ///
-    /// - 2023-04-14 15:00 => `[Y,M,D,Hr,Mi]`
-    /// - 4/14/23, 3:00 PM => `[M,D,YShort,Hr12,Mi]`
-    pub datetime_order: &'static [DateCmp],
+    /// - 03.01.2020 => DMY => false
+    /// - 01/03/2020 => MDY => true
+    pub month_before_day: bool,
     /// Tokens for parsing month names.
     ///
     /// Format: Parsed token -> Month number (starting from 1)
@@ -95,10 +78,20 @@ pub(crate) struct Entry {
     ///
     /// Format: Parsed token -> decimal power
     pub number_tokens: phf::Map<&'static str, u8>,
+    /// Tokens for parsing number strings with no digits (e.g. "No videos")
+    ///
+    /// Format: Parsed token -> value
+    pub number_nd_tokens: phf::Map<&'static str, u8>,
     /// Names of album types (Album, Single, ...)
     ///
     /// Format: Parsed text -> Album type
     pub album_types: phf::Map<&'static str, AlbumType>,
+    /// Channel name prefix on playlist pages (e.g. `by`)
+    pub chan_prefix: &'static str,
+    /// Channel name suffix on playlist pages
+    pub chan_suffix: &'static str,
+    /// "Other versions" title on album pages
+    pub album_versions_title: &'static str,
 }
 "#;
 
@@ -108,11 +101,11 @@ pub(crate) fn entry(lang: Language) -> Entry {
         "#
     .to_owned();
 
-    dict.iter().for_each(|(lang, entry)| {
+    for (lang, entry) in &dict {
         // Match selector
         let mut selector = format!("Language::{lang:?}");
         entry.equivalent.iter().for_each(|eq| {
-            let _ = write!(selector, " | Language::{eq:?}");
+            write!(selector, " | Language::{eq:?}").unwrap();
         });
 
         // Timeago tokens
@@ -147,47 +140,54 @@ pub(crate) fn entry(lang: Language) -> Entry {
             };
         });
 
-        // Date order
-        let mut date_order = "&[".to_owned();
-        entry.date_order.chars().for_each(|c| {
-            let _ = write!(date_order, "DateCmp::{}, ", parse_date_cmp(c));
-        });
-        date_order = date_order.trim_end_matches([' ', ',']).to_owned() + "]";
-
-        // Datetime order
-        let mut datetime_order = "&[".to_owned();
-        entry.datetime_order.chars().for_each(|c| {
-            let _ = write!(datetime_order, "DateCmp::{}, ", parse_date_cmp(c));
-        });
-        datetime_order = datetime_order.trim_end_matches([' ', ',']).to_owned() + "]";
-
         // Number tokens
         let mut number_tokens = phf_codegen::Map::<&str>::new();
         entry.number_tokens.iter().for_each(|(txt, mag)| {
             number_tokens.entry(txt, &mag.to_string());
         });
 
+        // Number nd tokens
+        let mut number_nd_tokens = phf_codegen::Map::<&str>::new();
+        entry.number_nd_tokens.iter().for_each(|(txt, mag)| {
+            number_nd_tokens.entry(txt, &mag.to_string());
+        });
+
         // Album types
         let mut album_types = phf_codegen::Map::<&str>::new();
         entry.album_types.iter().for_each(|(txt, album_type)| {
             album_types.entry(txt, &format!("AlbumType::{album_type:?}"));
         });
 
-        let code_ta_tokens = &ta_tokens.build().to_string().replace('\n', "\n            ");
-        let code_ta_nd_tokens = &ta_nd_tokens.build().to_string().replace('\n', "\n            ");
+        let code_ta_tokens = &ta_tokens
+            .build()
+            .to_string()
+            .replace('\n', "\n            ");
+        let code_ta_nd_tokens = &ta_nd_tokens
+            .build()
+            .to_string()
+            .replace('\n', "\n            ");
         let code_months = &months.build().to_string().replace('\n', "\n            ");
-        let code_number_tokens = &number_tokens.build().to_string().replace('\n', "\n            ");
-        let code_album_types = &album_types.build().to_string().replace('\n', "\n            ");
+        let code_number_tokens = &number_tokens
+            .build()
+            .to_string()
+            .replace('\n', "\n            ");
+        let code_number_nd_tokens = &number_nd_tokens
+            .build()
+            .to_string()
+            .replace('\n', "\n            ");
+        let code_album_types = &album_types
+            .build()
+            .to_string()
+            .replace('\n', "\n            ");
 
-        let _ = write!(code_timeago_tokens, "{} => Entry {{\n            by_char: {:?},\n            timeago_tokens: {},\n            date_order: {},\n            datetime_order: {},\n            months: {},\n            timeago_nd_tokens: {},\n            comma_decimal: {:?},\n            number_tokens: {},\n            album_types: {},\n        }},\n        ",
-        selector, entry.by_char, code_ta_tokens, date_order, datetime_order, code_months, code_ta_nd_tokens, entry.comma_decimal, code_number_tokens, code_album_types);
-    });
+        write!(code_timeago_tokens, "{} => Entry {{\n            timeago_tokens: {},\n            month_before_day: {:?},\n            months: {},\n            timeago_nd_tokens: {},\n            comma_decimal: {:?},\n            number_tokens: {},\n            number_nd_tokens: {},\n            album_types: {},\n            chan_prefix: {:?},\n            chan_suffix: {:?},\n            album_versions_title: {:?},\n        }},\n        ",
+        selector, code_ta_tokens, entry.month_before_day, code_months, code_ta_nd_tokens, entry.comma_decimal, code_number_tokens, code_number_nd_tokens, code_album_types, entry.chan_prefix, entry.chan_suffix, entry.album_versions_title).unwrap();
+    }
 
     code_timeago_tokens = code_timeago_tokens.trim_end().to_owned() + "\n    }\n}\n";
 
     let code = format!("{code_head}\n{code_timeago_tokens}");
 
-    let mut target_path = project_root.to_path_buf();
-    target_path.push(TARGET_PATH);
+    let target_path = path!(*SRC_DIR / "util" / "dictionary.rs");
     std::fs::write(target_path, code).unwrap();
 }

codegen/src/gen_locales.rs

@@ -1,14 +1,18 @@
 use std::collections::BTreeMap;
 use std::fmt::Write;
-use std::path::Path;
+use std::fs::File;
+use std::io::BufReader;
 
+use path_macro::path;
 use reqwest::header;
 use reqwest::Client;
 use serde::Deserialize;
 use serde_with::serde_as;
 use serde_with::VecSkipError;
 
-use crate::util::Text;
+use crate::model::Text;
+use crate::util::DICT_DIR;
+use crate::util::SRC_DIR;
 
 #[serde_as]
 #[derive(Clone, Debug, Deserialize)]
@@ -137,47 +141,48 @@ struct LanguageCountryCommand {
     hl: String,
 }
 
-pub async fn generate_locales(project_root: &Path) {
+pub async fn generate_locales() {
     let (languages, countries) = get_locales().await;
 
+    let json_path = path!(*DICT_DIR / "lang_names.json");
+    let json_file = File::open(json_path).unwrap();
+    let lang_names: BTreeMap<String, String> =
+        serde_json::from_reader(BufReader::new(json_file)).unwrap();
+
     let code_head = r#"// This file is automatically generated. DO NOT EDIT.
 
 //! Languages and countries
 
-use std::{fmt::Display, str::FromStr};
+use std::str::FromStr;
 
 use serde::{Deserialize, Serialize};
+
+use crate::error::Error;
 "#;
 
-    let code_foot = r#"impl Display for Language {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.write_str(
-            &serde_json::to_string(self).map_or("".to_owned(), |s| s[1..s.len() - 1].to_owned()),
-        )
-    }
-}
+    let code_foot = r#"impl FromStr for Language {
+    type Err = Error;
 
-impl Display for Country {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.write_str(
-            &serde_json::to_string(self).map_or("".to_owned(), |s| s[1..s.len() - 1].to_owned()),
-        )
-    }
-}
-
-impl FromStr for Language {
-    type Err = serde_json::Error;
     fn from_str(s: &str) -> Result<Self, Self::Err> {
-        serde_json::from_str(&format!("\"{}\"", s))
+        let mut sub = s;
+        loop {
+            if let Ok(v) = serde_plain::from_str(sub) {
+                return Ok(v);
+            }
+            match sub.rfind('-') {
+                Some(pos) => {
+                    sub = &sub[..pos];
+                }
+                None => return Err(Error::Other("could not parse language `{s}`".into())),
+            }
+        }
     }
 }
 
-impl FromStr for Country {
-    type Err = serde_json::Error;
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        serde_json::from_str(&format!("\"{}\"", s))
-    }
-}
+serde_plain::derive_display_from_serialize!(Language);
+
+serde_plain::derive_fromstr_from_deserialize!(Country, Error);
+serde_plain::derive_display_from_serialize!(Country);
 "#;
 
     let mut code_langs = r#"/// Available languages
@@ -197,11 +202,20 @@ pub enum Country {
     .to_owned();
 
     let mut code_lang_array = format!(
-        "/// Array of all available languages\npub const LANGUAGES: [Language; {}] = [\n",
+        r#"/// Array of all available languages
+/// The languages are sorted by their native names. This array can be used to display
+/// a language selection or to get the language code from a language name using binary search.
+pub const LANGUAGES: [Language; {}] = [
+"#,
         languages.len()
     );
     let mut code_country_array = format!(
-        "/// Array of all available countries\npub const COUNTRIES: [Country; {}] = [\n",
+        r#"/// Array of all available countries
+///
+/// The countries are sorted by their english names. This array can be used to display
+/// a country selection or to get the country code from a country name using binary search.
+pub const COUNTRIES: [Country; {}] = [
+"#,
         countries.len()
     );
 
@@ -222,55 +236,82 @@ pub enum Country {
 "#
     .to_owned();
 
-    languages.iter().for_each(|(c, n)| {
-        let enum_name = c
-            .split('-')
-            .map(|c| {
-                format!(
-                    "{}{}",
-                    c[0..1].to_owned().to_uppercase(),
-                    c[1..].to_owned().to_lowercase()
-                )
-            })
-            .collect::<String>();
+    for (code, native_name) in &languages {
+        let enum_name = code.split('-').fold(String::new(), |mut output, c| {
+            let _ = write!(
+                output,
+                "{}{}",
+                c[0..1].to_owned().to_uppercase(),
+                c[1..].to_owned().to_lowercase()
+            );
+            output
+        });
+
+        let en_name = lang_names.get(code).expect(code);
 
         // Language enum
-        write!(code_langs, "    /// {n}\n    ").unwrap();
-        if c.contains('-') {
-            write!(code_langs, "#[serde(rename = \"{c}\")]\n    ").unwrap();
+        if en_name == native_name || code.starts_with("en") {
+            write!(code_langs, "    /// {native_name}\n    ").unwrap();
+        } else {
+            write!(code_langs, "    /// {en_name} / {native_name}\n    ").unwrap();
+        }
+        if code.contains('-') {
+            write!(code_langs, "#[serde(rename = \"{code}\")]\n    ").unwrap();
         }
         code_langs += &enum_name;
         code_langs += ",\n";
 
-        // Language array
-        writeln!(code_lang_array, "    Language::{enum_name},").unwrap();
-
         // Language names
         writeln!(
             code_lang_names,
-            "            Language::{enum_name} => \"{n}\","
+            "            Language::{enum_name} => \"{native_name}\","
         )
         .unwrap();
-    });
+    }
     code_langs += "}\n";
 
-    countries.iter().for_each(|(c, n)| {
+    // Language array
+    let languages_by_name = languages
+        .iter()
+        .map(|(k, v)| (v, k))
+        .collect::<BTreeMap<_, _>>();
+    for code in languages_by_name.values() {
+        let enum_name = code.split('-').fold(String::new(), |mut output, c| {
+            let _ = write!(
+                output,
+                "{}{}",
+                c[0..1].to_owned().to_uppercase(),
+                c[1..].to_owned().to_lowercase()
+            );
+            output
+        });
+        writeln!(code_lang_array, "    Language::{enum_name},").unwrap();
+    }
+
+    for (c, n) in &countries {
         let enum_name = c[0..1].to_owned().to_uppercase() + &c[1..].to_owned().to_lowercase();
 
         // Country enum
         writeln!(code_countries, "    /// {n}").unwrap();
         writeln!(code_countries, "    {enum_name},").unwrap();
 
-        // Country array
-        writeln!(code_country_array, "    Country::{enum_name},").unwrap();
-
         // Country names
         writeln!(
             code_country_names,
             "            Country::{enum_name} => \"{n}\","
         )
         .unwrap();
-    });
+    }
+
+    // Country array
+    let countries_by_name = countries
+        .iter()
+        .map(|(k, v)| (v, k))
+        .collect::<BTreeMap<_, _>>();
+    for c in countries_by_name.values() {
+        let enum_name = c[0..1].to_owned().to_uppercase() + &c[1..].to_owned().to_lowercase();
+        writeln!(code_country_array, "    Country::{enum_name},").unwrap();
+    }
 
     // Add Country::Zz / Global
     code_countries += "    /// Global (can only be used for music charts)\n";
@@ -288,8 +329,7 @@ pub enum Country {
         "{code_head}\n{code_langs}\n{code_countries}\n{code_lang_array}\n{code_country_array}\n{code_lang_names}\n{code_country_names}\n{code_foot}"
     );
 
-    let mut target_path = project_root.to_path_buf();
-    target_path.push("src/param/locale.rs");
+    let target_path = path!(*SRC_DIR / "param" / "locale.rs");
     std::fs::write(target_path, code).unwrap();
 }
 
@@ -299,7 +339,7 @@ async fn get_locales() -> (BTreeMap<String, String>, BTreeMap<String, String>) {
         .post("https://www.youtube.com/youtubei/v1/account/account_menu?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false")
         .header(header::CONTENT_TYPE, "application/json")
         .body(
-            r##"{"context":{"client":{"clientName":"WEB","clientVersion":"2.20220914.06.00","platform":"DESKTOP","originalUrl":"https://www.youtube.com/","hl":"en","gl":"US"},"request":{"internalExperimentFlags":[],"useSsl":true},"user":{"lockedSafetyMode":false}}}"##
+            r#"{"context":{"client":{"clientName":"WEB","clientVersion":"2.20220914.06.00","platform":"DESKTOP","originalUrl":"https://www.youtube.com/","hl":"en","gl":"US"},"request":{"internalExperimentFlags":[],"useSsl":true},"user":{"lockedSafetyMode":false}}}"#
         )
         .send().await
         .unwrap()
@@ -358,8 +398,8 @@ fn map_language_section(section: &CompactLinkRendererWrap) -> BTreeMap<String, S
                     .actions[0]
                     .select_language_command
                     .hl
-                    .to_owned(),
-                i.compact_link_renderer.title.text.to_owned(),
+                    .clone(),
+                i.compact_link_renderer.title.text.clone(),
             )
         })
         .collect()

codegen/src/main.rs

@@ -1,23 +1,26 @@
+#![warn(clippy::todo)]
+
 mod abtest;
 mod collect_album_types;
-mod collect_datetimes;
+mod collect_album_versions_titles;
+mod collect_chan_prefixes;
+mod collect_history_dates;
 mod collect_large_numbers;
 mod collect_playlist_dates;
+mod collect_video_dates;
+mod collect_video_durations;
 mod download_testfiles;
 mod gen_dictionary;
 mod gen_locales;
+mod model;
 mod util;
 
-use std::path::PathBuf;
-
 use clap::{Parser, Subcommand};
 
 #[derive(Parser)]
 struct Cli {
     #[clap(subcommand)]
     command: Commands,
-    #[clap(short = 'd', default_value = "..")]
-    project_root: PathBuf,
     #[clap(short, default_value = "8")]
     concurrency: usize,
 }
@@ -27,11 +30,19 @@ enum Commands {
     CollectPlaylistDates,
     CollectLargeNumbers,
     CollectAlbumTypes,
-    CollectDatetimes,
+    CollectVideoDurations,
+    CollectVideoDates,
+    CollectHistoryDates,
+    CollectMusicHistoryDates,
+    CollectChanPrefixes,
+    CollectAlbumVersionsTitles,
     ParsePlaylistDates,
+    ParseHistoryDates,
     ParseLargeNumbers,
     ParseAlbumTypes,
-    ParseDatetimes,
+    ParseVideoDurations,
+    ParseChanPrefixes,
+    ParseAlbumVersionsTitles,
     GenLocales,
     GenDict,
     DownloadTestfiles,
@@ -45,37 +56,43 @@ enum Commands {
 
 #[tokio::main]
 async fn main() {
-    env_logger::init();
+    tracing_subscriber::fmt::init();
     let cli = Cli::parse();
 
     match cli.command {
         Commands::CollectPlaylistDates => {
-            collect_playlist_dates::collect_dates(&cli.project_root, cli.concurrency).await;
+            collect_playlist_dates::collect_dates(cli.concurrency).await
         }
         Commands::CollectLargeNumbers => {
-            collect_large_numbers::collect_large_numbers(&cli.project_root, cli.concurrency).await;
+            collect_large_numbers::collect_large_numbers(cli.concurrency).await
         }
         Commands::CollectAlbumTypes => {
-            collect_album_types::collect_album_types(&cli.project_root, cli.concurrency).await;
+            collect_album_types::collect_album_types(cli.concurrency).await
         }
-        Commands::CollectDatetimes => {
-            collect_datetimes::collect_datetimes(&cli.project_root, cli.concurrency).await;
+        Commands::CollectVideoDurations => {
+            collect_video_durations::collect_video_durations(cli.concurrency).await
         }
-        Commands::ParsePlaylistDates => {
-            collect_playlist_dates::write_samples_to_dict(&cli.project_root)
+        Commands::CollectVideoDates => {
+            collect_video_dates::collect_video_dates(cli.concurrency).await
         }
-        Commands::ParseLargeNumbers => {
-            collect_large_numbers::write_samples_to_dict(&cli.project_root)
+        Commands::CollectHistoryDates => collect_history_dates::collect_dates().await,
+        Commands::CollectMusicHistoryDates => collect_history_dates::collect_dates_music().await,
+        Commands::CollectChanPrefixes => collect_chan_prefixes::collect_chan_prefixes().await,
+        Commands::CollectAlbumVersionsTitles => {
+            collect_album_versions_titles::collect_album_versions_titles().await
         }
-        Commands::ParseAlbumTypes => collect_album_types::write_samples_to_dict(&cli.project_root),
-        Commands::ParseDatetimes => collect_datetimes::write_samples_to_dict(&cli.project_root),
-        Commands::GenLocales => {
-            gen_locales::generate_locales(&cli.project_root).await;
-        }
-        Commands::GenDict => gen_dictionary::generate_dictionary(&cli.project_root),
-        Commands::DownloadTestfiles => {
-            download_testfiles::download_testfiles(&cli.project_root).await
+        Commands::ParsePlaylistDates => collect_playlist_dates::write_samples_to_dict(),
+        Commands::ParseHistoryDates => collect_history_dates::write_samples_to_dict(),
+        Commands::ParseLargeNumbers => collect_large_numbers::write_samples_to_dict(),
+        Commands::ParseAlbumTypes => collect_album_types::write_samples_to_dict(),
+        Commands::ParseVideoDurations => collect_video_durations::parse_video_durations(),
+        Commands::ParseChanPrefixes => collect_chan_prefixes::write_samples_to_dict(),
+        Commands::ParseAlbumVersionsTitles => {
+            collect_album_versions_titles::write_samples_to_dict()
         }
+        Commands::GenLocales => gen_locales::generate_locales().await,
+        Commands::GenDict => gen_dictionary::generate_dictionary(),
+        Commands::DownloadTestfiles => download_testfiles::download_testfiles().await,
         Commands::AbTest { id, n } => {
             match id {
                 Some(id) => {
@@ -99,7 +116,7 @@ async fn main() {
                 }
                 None => {
                     let res = abtest::run_all_tests(n, cli.concurrency).await;
-                    println!("{}", serde_json::to_string_pretty(&res).unwrap())
+                    println!("{}", serde_json::to_string_pretty(&res).unwrap());
                 }
             };
         }

codegen/src/model.rs

@@ -0,0 +1,333 @@
+use std::collections::BTreeMap;
+
+use ordered_hash_map::OrderedHashMap;
+use rustypipe::{model::AlbumType, param::Language};
+use serde::{Deserialize, Serialize};
+use serde_with::{serde_as, DefaultOnError, VecSkipError};
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(default)]
+pub struct DictEntry {
+    /// List of languages that should be treated equally (e.g. EnUs/EnGb/EnIn)
+    pub equivalent: Vec<Language>,
+    /// Should the language be parsed by character instead of by word?
+    /// (e.g. Chinese/Japanese)
+    pub by_char: bool,
+    /// True if the month has to be parsed before the day
+    ///
+    /// Examples:
+    ///
+    /// - 03.01.2020 => DMY => false
+    /// - 01/03/2020 => MDY => true
+    pub month_before_day: bool,
+    /// Tokens for parsing timeago strings.
+    ///
+    /// Format: Parsed token -> \[Quantity\] Identifier
+    ///
+    /// Identifiers: `Y`(ear), `M`(month), `W`(eek), `D`(ay),
+    /// `h`(our), `m`(inute), `s`(econd)
+    pub timeago_tokens: OrderedHashMap<String, String>,
+    /// Order in which to parse numeric date components. Formatted as
+    /// a string of date identifiers (Y, M, D).
+    ///
+    /// Examples:
+    ///
+    /// - 03.01.2020 => `"DMY"`
+    /// - Jan 3, 2020 => `"DY"`
+    pub date_order: String,
+    /// Tokens for parsing month names.
+    ///
+    /// Format: Parsed token -> Month number (starting from 1)
+    pub months: BTreeMap<String, u8>,
+    /// Tokens for parsing date strings with no digits (e.g. Today, Tomorrow)
+    ///
+    /// Format: Parsed token -> \[Quantity\] Identifier
+    pub timeago_nd_tokens: OrderedHashMap<String, String>,
+    /// Are commas (instead of points) used as decimal separators?
+    pub comma_decimal: bool,
+    /// Tokens for parsing decimal prefixes (K, M, B, ...)
+    ///
+    /// Format: Parsed token -> decimal power
+    pub number_tokens: BTreeMap<String, u8>,
+    /// Tokens for parsing number strings with no digits (e.g. "No videos")
+    ///
+    /// Format: Parsed token -> value
+    pub number_nd_tokens: BTreeMap<String, u8>,
+    /// Names of album types (Album, Single, ...)
+    ///
+    /// Format: Parsed text -> Album type
+    pub album_types: BTreeMap<String, AlbumType>,
+    /// Channel name prefix on playlist pages (e.g. `by`)
+    pub chan_prefix: String,
+    /// Channel name suffix on playlist pages
+    pub chan_suffix: String,
+    /// "Other versions" title on album pages
+    pub album_versions_title: String,
+}
+
+/// Parsed TimeAgo string, contains amount and time unit.
+///
+/// Example: "14 hours ago" => `TimeAgo {n: 14, unit: TimeUnit::Hour}`
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct TimeAgo {
+    /// Number of time units
+    pub n: u8,
+    /// Time unit
+    pub unit: TimeUnit,
+}
+
+impl std::fmt::Display for TimeAgo {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        if self.n > 1 {
+            write!(f, "{}{}", self.n, self.unit.as_str())
+        } else {
+            f.write_str(self.unit.as_str())
+        }
+    }
+}
+
+/// Parsed time unit
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[serde(rename_all = "lowercase")]
+pub enum TimeUnit {
+    Second,
+    Minute,
+    Hour,
+    Day,
+    Week,
+    Month,
+    Year,
+    LastWeek,
+    LastWeekday,
+}
+
+impl TimeUnit {
+    pub fn as_str(&self) -> &str {
+        match self {
+            TimeUnit::Second => "s",
+            TimeUnit::Minute => "m",
+            TimeUnit::Hour => "h",
+            TimeUnit::Day => "D",
+            TimeUnit::Week => "W",
+            TimeUnit::Month => "M",
+            TimeUnit::Year => "Y",
+            TimeUnit::LastWeek => "Wl",
+            TimeUnit::LastWeekday => "Wd",
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum ExtItemType {
+    Track,
+    Video,
+    Episode,
+    Playlist,
+    Artist,
+}
+
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct QBrowse<'a> {
+    pub browse_id: &'a str,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub params: Option<&'a str>,
+}
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct QCont<'a> {
+    pub continuation: &'a str,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+pub struct TextRuns {
+    pub runs: Vec<Text>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+pub struct Text {
+    #[serde(alias = "simpleText")]
+    pub text: String,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Channel {
+    pub contents: TwoColumnBrowseResults,
+    pub header: ChannelHeader,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ChannelHeader {
+    pub c4_tabbed_header_renderer: HeaderRenderer,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct HeaderRenderer {
+    pub subscriber_count_text: Text,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct TwoColumnBrowseResults {
+    pub two_column_browse_results_renderer: TabsRenderer,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct TabsRenderer {
+    #[serde_as(as = "VecSkipError<_>")]
+    pub tabs: Vec<Tab<RichGrid>>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ContentsRenderer<T> {
+    #[serde(alias = "tabs")]
+    pub contents: Vec<T>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Tab<T> {
+    pub tab_renderer: TabRenderer<T>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct TabRenderer<T> {
+    pub content: T,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct SectionList<T> {
+    pub section_list_renderer: ContentsRenderer<T>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RichGrid {
+    pub rich_grid_renderer: RichGridRenderer,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RichGridRenderer {
+    #[serde_as(as = "VecSkipError<_>")]
+    pub contents: Vec<RichItemRendererWrap>,
+    #[serde(default)]
+    #[serde_as(as = "DefaultOnError")]
+    pub header: Option<RichGridHeader>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RichItemRendererWrap {
+    pub rich_item_renderer: RichItemRenderer,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RichItemRenderer {
+    pub content: VideoRendererWrap,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct VideoRendererWrap {
+    pub video_renderer: VideoRenderer,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct VideoRenderer {
+    /// `24,194 views`
+    pub view_count_text: Text,
+    /// `19K views`
+    pub short_view_count_text: Text,
+    pub length_text: LengthText,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct LengthText {
+    /// `18 minutes, 26 seconds`
+    pub accessibility: Accessibility,
+    /// `18:26`
+    pub simple_text: String,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Accessibility {
+    pub accessibility_data: AccessibilityData,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct AccessibilityData {
+    pub label: String,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RichGridHeader {
+    pub feed_filter_chip_bar_renderer: ChipBar,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ChipBar {
+    pub contents: Vec<Chip>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Chip {
+    pub chip_cloud_chip_renderer: ChipRenderer,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ChipRenderer {
+    pub navigation_endpoint: NavigationEndpoint,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct NavigationEndpoint {
+    pub continuation_command: ContinuationCommand,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ContinuationCommand {
+    pub token: String,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ContinuationResponse {
+    pub on_response_received_actions: Vec<ContinuationAction>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ContinuationAction {
+    pub reload_continuation_items_command: ContinuationItemsWrap,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ContinuationItemsWrap {
+    #[serde_as(as = "VecSkipError<_>")]
+    pub continuation_items: Vec<RichItemRendererWrap>,
+}

codegen/src/util.rs

@@ -1,92 +1,75 @@
-use std::{
-    collections::BTreeMap,
-    fs::File,
-    io::BufReader,
-    path::{Path, PathBuf},
-    str::FromStr,
-};
+use std::{collections::BTreeMap, fs::File, io::BufReader, path::PathBuf, str::FromStr};
 
 use once_cell::sync::Lazy;
 use path_macro::path;
-use rustypipe::{model::AlbumType, param::Language};
+use regex::Regex;
+use rustypipe::param::Language;
 use serde::{Deserialize, Serialize};
 
-static DICT_PATH: Lazy<PathBuf> = Lazy::new(|| path!("testfiles" / "dict" / "dictionary.json"));
+use crate::model::DictEntry;
+
+/// Get the path of the `testfiles` directory
+pub static TESTFILES_DIR: Lazy<PathBuf> = Lazy::new(|| {
+    path!(env!("CARGO_MANIFEST_DIR") / ".." / "testfiles")
+        .canonicalize()
+        .unwrap()
+});
+/// Get the path of the `dict` directory
+pub static DICT_DIR: Lazy<PathBuf> = Lazy::new(|| path!(*TESTFILES_DIR / "dict"));
+/// Get the path of the `src` directory
+pub static SRC_DIR: Lazy<PathBuf> = Lazy::new(|| path!(env!("CARGO_MANIFEST_DIR") / ".." / "src"));
 
 type Dictionary = BTreeMap<Language, DictEntry>;
+type DictionaryOverride = BTreeMap<Language, DictOverrideEntry>;
 
 #[derive(Debug, Default, Serialize, Deserialize)]
 #[serde(default)]
-pub struct DictEntry {
-    /// List of languages that should be treated equally (e.g. EnUs/EnGb/EnIn)
-    pub equivalent: Vec<Language>,
-    /// Should the language be parsed by character instead of by word?
-    /// (e.g. Chinese/Japanese)
-    pub by_char: bool,
-    /// Tokens for parsing timeago strings.
-    ///
-    /// Format: Parsed token -> \[Quantity\] Identifier
-    ///
-    /// Identifiers: `Y`(ear), `M`(month), `W`(eek), `D`(ay),
-    /// `h`(our), `m`(inute), `s`(econd)
-    pub timeago_tokens: BTreeMap<String, String>,
-    /// Order in which to parse numeric date components. Formatted as
-    /// a string of date identifiers (Y, M, D).
-    ///
-    /// Examples:
-    ///
-    /// - 03.01.2020 => `"DMY"`
-    /// - Jan 3, 2020 => `"DY"`
-    pub date_order: String,
-    /// Order in which to parse datetimes. Formatted as a string of
-    /// date/time identifiers (Y, y, M, D, H, h, m).
-    ///
-    /// Examples:
-    ///
-    /// - 2023-04-14 15:00 => `"YMDHm"`
-    /// - 4/14/23, 3:00 PM => `"MDyhm"`
-    pub datetime_order: String,
-    /// Tokens for parsing month names.
-    ///
-    /// Format: Parsed token -> Month number (starting from 1)
-    pub months: BTreeMap<String, u8>,
-    /// Tokens for parsing date strings with no digits (e.g. Today, Tomorrow)
-    ///
-    /// Format: Parsed token -> \[Quantity\] Identifier
-    pub timeago_nd_tokens: BTreeMap<String, String>,
-    /// Are commas (instead of points) used as decimal separators?
-    pub comma_decimal: bool,
-    /// Tokens for parsing decimal prefixes (K, M, B, ...)
-    ///
-    /// Format: Parsed token -> decimal power
-    pub number_tokens: BTreeMap<String, u8>,
-    /// Names of album types (Album, Single, ...)
-    ///
-    /// Format: Parsed text -> Album type
-    pub album_types: BTreeMap<String, AlbumType>,
+struct DictOverrideEntry {
+    number_tokens: BTreeMap<String, Option<u8>>,
+    number_nd_tokens: BTreeMap<String, Option<u8>>,
 }
 
-#[derive(Clone, Debug, Deserialize)]
-pub struct TextRuns {
-    pub runs: Vec<Text>,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-pub struct Text {
-    #[serde(alias = "simpleText")]
-    pub text: String,
-}
-
-pub fn read_dict(project_root: &Path) -> Dictionary {
-    let json_path = path!(project_root / *DICT_PATH);
+pub fn read_dict() -> Dictionary {
+    let json_path = path!(*DICT_DIR / "dictionary.json");
     let json_file = File::open(json_path).unwrap();
     serde_json::from_reader(BufReader::new(json_file)).unwrap()
 }
 
-pub fn write_dict(project_root: &Path, dict: &Dictionary) {
-    let json_path = path!(project_root / *DICT_PATH);
+fn read_dict_override() -> DictionaryOverride {
+    let json_path = path!(*DICT_DIR / "dictionary_override.json");
+    let json_file = File::open(json_path).unwrap();
+    serde_json::from_reader(BufReader::new(json_file)).unwrap()
+}
+
+pub fn write_dict(dict: Dictionary) {
+    let dict_override = read_dict_override();
+
+    let json_path = path!(*DICT_DIR / "dictionary.json");
     let json_file = File::create(json_path).unwrap();
-    serde_json::to_writer_pretty(json_file, dict).unwrap();
+
+    fn apply_map<K: Clone + Ord, V: Clone>(map: &mut BTreeMap<K, V>, or: &BTreeMap<K, Option<V>>) {
+        or.iter().for_each(|(key, val)| match val {
+            Some(val) => {
+                map.insert(key.clone(), val.clone());
+            }
+            None => {
+                map.remove(key);
+            }
+        });
+    }
+
+    let dict: Dictionary = dict
+        .into_iter()
+        .map(|(lang, mut entry)| {
+            if let Some(or) = dict_override.get(&lang) {
+                apply_map(&mut entry.number_tokens, &or.number_tokens);
+                apply_map(&mut entry.number_nd_tokens, &or.number_nd_tokens);
+            }
+            (lang, entry)
+        })
+        .collect();
+
+    serde_json::to_writer_pretty(json_file, &dict).unwrap();
 }
 
 pub fn filter_datestr(string: &str) -> String {
@@ -94,7 +77,7 @@ pub fn filter_datestr(string: &str) -> String {
         .to_lowercase()
         .chars()
         .filter_map(|c| {
-            if c == '\u{200b}' || c.is_ascii_digit() {
+            if matches!(c, '\u{200b}' | '.' | ',') || c.is_ascii_digit() {
                 None
             } else if c == '-' {
                 Some(' ')
@@ -108,7 +91,20 @@ pub fn filter_datestr(string: &str) -> String {
 pub fn filter_largenumstr(string: &str) -> String {
     string
         .chars()
-        .filter(|c| !matches!(c, '\u{200b}' | '.' | ',') && !c.is_ascii_digit())
+        .filter(|c| {
+            !matches!(
+                c,
+                '\u{200b}'
+                    | '\u{202b}'
+                    | '\u{202c}'
+                    | '\u{202e}'
+                    | '\u{200e}'
+                    | '\u{200f}'
+                    | '.'
+                    | ','
+            ) && !c.is_ascii_digit()
+        })
+        .flat_map(char::to_lowercase)
         .collect()
 }
 
@@ -138,13 +134,77 @@ where
         if c.is_ascii_digit() {
             buf.push(c);
         } else if !buf.is_empty() {
-            buf.parse::<F>().map_or((), |n| numbers.push(n));
+            if let Ok(n) = buf.parse::<F>() {
+                numbers.push(n);
+            }
             buf.clear();
         }
     }
     if !buf.is_empty() {
-        buf.parse::<F>().map_or((), |n| numbers.push(n));
+        if let Ok(n) = buf.parse::<F>() {
+            numbers.push(n);
+        }
     }
 
     numbers
 }
+
+pub fn parse_largenum_en(string: &str) -> Option<u64> {
+    let (num, mut exp, filtered) = {
+        let mut buf = String::new();
+        let mut filtered = String::new();
+        let mut exp = 0;
+        let mut after_point = false;
+        for c in string.chars() {
+            if c.is_ascii_digit() {
+                buf.push(c);
+
+                if after_point {
+                    exp -= 1;
+                }
+            } else if c == '.' {
+                after_point = true;
+            } else if !matches!(c, '\u{200b}' | '.' | ',') {
+                filtered.push(c);
+            }
+        }
+        (buf.parse::<u64>().ok()?, exp, filtered)
+    };
+
+    let lookup_token = |token: &str| match token {
+        "K" => Some(3),
+        "M" => Some(6),
+        "B" => Some(9),
+        _ => None,
+    };
+
+    exp += filtered
+        .split_whitespace()
+        .filter_map(lookup_token)
+        .sum::<i32>();
+
+    num.checked_mul((10_u64).checked_pow(exp.try_into().ok()?)?)
+}
+
+/// Parse textual video length (e.g. `0:49`, `2:02` or `1:48:18`)
+/// and return the duration in seconds.
+pub fn parse_video_length(text: &str) -> Option<u32> {
+    static VIDEO_LENGTH_REGEX: Lazy<Regex> =
+        Lazy::new(|| Regex::new(r"(?:(\d+)[:.])?(\d{1,2})[:.](\d{2})").unwrap());
+    VIDEO_LENGTH_REGEX.captures(text).map(|cap| {
+        let hrs = cap
+            .get(1)
+            .and_then(|x| x.as_str().parse::<u32>().ok())
+            .unwrap_or_default();
+        let min = cap
+            .get(2)
+            .and_then(|x| x.as_str().parse::<u32>().ok())
+            .unwrap_or_default();
+        let sec = cap
+            .get(3)
+            .and_then(|x| x.as_str().parse::<u32>().ok())
+            .unwrap_or_default();
+
+        hrs * 3600 + min * 60 + sec
+    })
+}

downloader/CHANGELOG.md

@@ -0,0 +1,175 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+
+## [v0.3.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.3.0..rustypipe-downloader/v0.3.1) - 2024-12-20
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rustypipe to 0.11.0
+
+
+## [v0.3.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.7..rustypipe-downloader/v0.3.0) - 2025-02-09
+
+### 🚀 Features
+
+- [**breaking**] Remove manual PO token options from downloader in favor of rustypipe-botguard - ([cddb32f](https://codeberg.org/ThetaDev/rustypipe/commit/cddb32f190276265258c6ab45b3d43a8891c4b39))
+
+### 🐛 Bug Fixes
+
+- Ensure downloader futures are send - ([812ff4c](https://codeberg.org/ThetaDev/rustypipe/commit/812ff4c5bafffc5708a6d5066f1ebadb6d9fc958))
+- Download audio with dolby codec - ([9234005](https://codeberg.org/ThetaDev/rustypipe/commit/92340056f868007beccb64e9e26eb39abc40f7aa))
+
+### 🚜 Refactor
+
+- [**breaking**] Add client_type field to DownloadError, rename cli option po-token-cache to pot-cache - ([594e675](https://codeberg.org/ThetaDev/rustypipe/commit/594e675b39efc5fbcdbd5e920a4d2cdee64f718e))
+
+### 📚 Documentation
+
+- Add Botguard info to README - ([9957add](https://codeberg.org/ThetaDev/rustypipe/commit/9957add2b5d6391b2c1869d2019fd7dd91b8cd41))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rustypipe to 0.10.0
+
+
+## [v0.2.7](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.6..rustypipe-downloader/v0.2.7) - 2025-01-16
+
+### 🚀 Features
+
+- Extract player DRM data - ([2af4001](https://codeberg.org/ThetaDev/rustypipe/commit/2af4001c75f2ff4f7c891aa59ac22c2c6b7902a2))
+- Prefer maxresdefault.jpg thumbnail if available - ([a8e97f4](https://codeberg.org/ThetaDev/rustypipe/commit/a8e97f411a1e769e52d8cbde11f0a4ca1535f7ef))
+- Add DRM and audio channel number filtering to StreamFilter - ([d5abee2](https://codeberg.org/ThetaDev/rustypipe/commit/d5abee275300ab1bc10fc8d6c35a4e3813fd2bd4))
+
+### 🐛 Bug Fixes
+
+- Remove Unix file metadata usage (Windows compatibility) - ([5c6d992](https://codeberg.org/ThetaDev/rustypipe/commit/5c6d992939f55a203ac1784f1e9175ac1d498ce8))
+
+### 📚 Documentation
+
+- Update README - ([0432477](https://codeberg.org/ThetaDev/rustypipe/commit/0432477451ecd5f64145d65239c721f4e44826c0))
+- Fix README - ([11442df](https://codeberg.org/ThetaDev/rustypipe/commit/11442dfd369599396357f5b7a7a4268a7b537f57))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rustypipe to 0.9.0
+- *(deps)* Update rust crate rstest to 0.24.0 (#20) - ([ab19034](https://codeberg.org/ThetaDev/rustypipe/commit/ab19034ab19baf090e83eada056559676ffdadce))
+- *(deps)* Update rust crate lofty to 0.22.0 - ([addeb82](https://codeberg.org/ThetaDev/rustypipe/commit/addeb821101aa968b95455604bc13bd24f50328f))
+- *(deps)* Update rust crate dirs to v6 (#24) - ([6a60425](https://codeberg.org/ThetaDev/rustypipe/commit/6a604252b1af7a9388db5dc170f737069cc31051))
+
+
+## [v0.2.6](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.5..rustypipe-downloader/v0.2.6) - 2024-12-20
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rustypipe to 0.8.0
+
+
+## [v0.2.5](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.4..rustypipe-downloader/v0.2.5) - 2024-12-13
+
+### 🐛 Bug Fixes
+
+- Replace futures dependency with futures-util - ([5c39bf4](https://codeberg.org/ThetaDev/rustypipe/commit/5c39bf4842b13d37a4277ea5506e15c179892ce5))
+- Remove empty tempfile after unsuccessful download - ([5262bec](https://codeberg.org/ThetaDev/rustypipe/commit/5262becca1e9e3e8262833764ef18c23bc401172))
+
+### ⚙️ Miscellaneous Tasks
+
+- Add docs badge to README - ([706e881](https://codeberg.org/ThetaDev/rustypipe/commit/706e88134c0e94ce7d880735e9d31b3ff531a4f9))
+
+
+## [v0.2.4](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.3..rustypipe-downloader/v0.2.4) - 2024-11-10
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rust crate thiserror to v2 (#16) - ([e1e1687](https://codeberg.org/ThetaDev/rustypipe/commit/e1e1687605603686ac5fd5deeb6aa8fecaf92494))
+- *(deps)* Update rustypipe to 0.7.0
+
+
+## [v0.2.3](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.2..rustypipe-downloader/v0.2.3) - 2024-10-28
+
+### 🐛 Bug Fixes
+
+- Remove unnecessary image.rs dependencies - ([1b08166](https://codeberg.org/ThetaDev/rustypipe/commit/1b08166399cccb8394d2fdd82d54162c1a9e01be))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rustypipe to 0.6.0
+
+
+## [v0.2.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.1..rustypipe-downloader/v0.2.2) - 2024-10-13
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rust crate rstest to 0.23.0 (#12) - ([96776e9](https://codeberg.org/ThetaDev/rustypipe/commit/96776e98d76fa1d31d5f84dbceafbe8f9dfd9085))
+- *(deps)* Update rustypipe to 0.5.0
+
+
+## [v0.2.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.2.0..rustypipe-downloader/v0.2.1) - 2024-09-10
+
+### 📚 Documentation
+
+- Fix license badge URL - ([4a253e1](https://codeberg.org/ThetaDev/rustypipe/commit/4a253e1a47317e9999e6ad31ac5c411956a0986a))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rust crate tokio to 1.20.4 [security] (#10) - ([a445e51](https://codeberg.org/ThetaDev/rustypipe/commit/a445e51b54a9afc44cd9657260a0b3d2abddbfa6))
+
+
+## [v0.2.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.1.1..rustypipe-downloader/v0.2.0) - 2024-08-18
+
+### 🚀 Features
+
+- Overhauled downloader - ([11a0038](https://codeberg.org/ThetaDev/rustypipe/commit/11a00383502917cd98245c3da349107289ba3aa9))
+- [**breaking**] Add TV client - ([e608811](https://codeberg.org/ThetaDev/rustypipe/commit/e608811e5f5615416241e67561671330097092cb))
+- Downloader: add audio tagging - ([1e1315a](https://codeberg.org/ThetaDev/rustypipe/commit/1e1315a8378bd0ad25b5f1614e83dabc4a0b40d5))
+- Downloader: add download_track fn, improve path templates - ([e1e4fb2](https://codeberg.org/ThetaDev/rustypipe/commit/e1e4fb29c190fec07f17c59ec88bef4f1c2a76a1))
+- Add audiotag+indicatif features to downloader - ([97fb057](https://codeberg.org/ThetaDev/rustypipe/commit/97fb0578b5c4954a596d8dee0c4b6e1d773a9300))
+- Add plaintext output to CLI - ([91b020e](https://codeberg.org/ThetaDev/rustypipe/commit/91b020efd498eff6e0f354a1de39439e252a79dd))
+- Add potoken option to downloader - ([904f821](https://codeberg.org/ThetaDev/rustypipe/commit/904f8215d84c810b04e4d2134718e786a4803ad2))
+- Add list of clients to downloader - ([5e646af](https://codeberg.org/ThetaDev/rustypipe/commit/5e646afd1edc6c0101501311527ea56d3bad5fd2))
+- Retry with different client after 403 error - ([d875b54](https://codeberg.org/ThetaDev/rustypipe/commit/d875b5442de9822ba7ddc6f05789f56a8962808c))
+- [**breaking**] Update channel model, addd handle + video_count, remove tv/mobile banner - ([e671570](https://codeberg.org/ThetaDev/rustypipe/commit/e6715700d950912031d5fbc1263f8770b6ffc49c))
+
+### 🐛 Bug Fixes
+
+- *(deps)* Update quick-xml to v0.35.0 - ([298e4de](https://codeberg.org/ThetaDev/rustypipe/commit/298e4def93d1595fba91be103f014aa645a08937))
+- Improve deobfuscator (support multiple nsig name matches, error if mapping all streams fails) - ([8152ce6](https://codeberg.org/ThetaDev/rustypipe/commit/8152ce6b088b57be9b8419b754aca93805e5f34d))
+- Set tracing instrumentation level to Error - ([9da3b25](https://codeberg.org/ThetaDev/rustypipe/commit/9da3b25be2b2577f7bd0282c09d10d368ac8b73f))
+- Add docs.rs feature attributes - ([ec13cbb](https://codeberg.org/ThetaDev/rustypipe/commit/ec13cbb1f35081118dda0f7f35e3ef90f7ca79a8))
+- Show docs.rs feature flags - ([67a231d](https://codeberg.org/ThetaDev/rustypipe/commit/67a231d6d1b6427f500667729a59032f2b28cc65))
+
+### ⚙️ Miscellaneous Tasks
+
+- *(deps)* Update rust crate quick-xml to 0.36.0 (#8) - ([b6bc05c](https://codeberg.org/ThetaDev/rustypipe/commit/b6bc05c1f39da9a846b2e3d1d24bcbccb031203b))
+- *(deps)* Update rust crate rstest to 0.22.0 (#9) - ([abb7832](https://codeberg.org/ThetaDev/rustypipe/commit/abb783219aba4b492c1dff03c2148acf1f51a55d))
+- Change repo URL to Codeberg - ([1793331](https://codeberg.org/ThetaDev/rustypipe/commit/17933315d947f76d5fe1aa52abf7ea24c3ce6381))
+- Adjust dependency versions - ([70c6f8c](https://codeberg.org/ThetaDev/rustypipe/commit/70c6f8c3b97baefd316fff90cc727524516657af))
+
+### Todo
+
+- Update metadata - ([8692ca8](https://codeberg.org/ThetaDev/rustypipe/commit/8692ca81d972d0d2acf6fb4da79b9e0f5ebf4daf))
+
+
+## [v0.1.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-downloader/v0.1.0..rustypipe-downloader/v0.1.1) - 2024-06-27
+
+### 📚 Documentation
+
+- Add logo - ([6646078](https://codeberg.org/ThetaDev/rustypipe/commit/66460789449be0d5984cbdb6ec372e69323b7a88))
+
+### ⚙️ Miscellaneous Tasks
+
+- Changelog: fix incorrect version URLs - ([97b6f07](https://codeberg.org/ThetaDev/rustypipe/commit/97b6f07399e80e00a6c015d013e744568be125dd))
+- Update rstest to v0.19.0 - ([50fd1f0](https://codeberg.org/ThetaDev/rustypipe/commit/50fd1f08caf39c1298654e06059cc393543e925b))
+- Introduce MSRV - ([5dbb288](https://codeberg.org/ThetaDev/rustypipe/commit/5dbb288a496d53a299effa2026f5258af7b1f176))
+- Fix clippy lints - ([45b9f2a](https://codeberg.org/ThetaDev/rustypipe/commit/45b9f2a627b4e7075ba0b1c5f16efcc19aef7922))
+- *(deps)* Update rust crate tokio to 1.20.4 [security] (#4) - ([ce3ec34](https://codeberg.org/ThetaDev/rustypipe/commit/ce3ec34337b8acac41410ea39264aab7423d5801))
+- *(deps)* Update rust crate quick-xml to 0.34.0 (#5) - ([1e8a1af](https://codeberg.org/ThetaDev/rustypipe/commit/1e8a1af08c873cee7feadf63c2eff62753a78f64))
+- *(deps)* Update rust crate rstest to 0.21.0 (#7) - ([c3af918](https://codeberg.org/ThetaDev/rustypipe/commit/c3af918ba53c6230c0e4aef822a0cb2cf120bf3f))
+- Update rustypipe to 0.2.0
+
+## [v0.1.0](https://codeberg.org/ThetaDev/rustypipe/commits/tag/rustypipe-downloader/v0.1.0) - 2024-03-22
+
+Initial release
+
+<!-- generated by git-cliff -->

downloader/Cargo.toml

@@ -1,11 +1,26 @@
 [package]
 name = "rustypipe-downloader"
-version = "0.1.0"
-edition = "2021"
+version = "0.3.1"
+rust-version = "1.67.1"
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+keywords.workspace = true
+categories.workspace = true
+description = "Downloader extension for RustyPipe"
 
 [features]
-# Reqwest TLS
+default = ["default-tls"]
+
+# Reqwest TLS options
 default-tls = ["reqwest/default-tls", "rustypipe/default-tls"]
+native-tls = ["reqwest/native-tls", "rustypipe/native-tls"]
+native-tls-alpn = ["reqwest/native-tls-alpn", "rustypipe/native-tls-alpn"]
+native-tls-vendored = [
+    "reqwest/native-tls-vendored",
+    "rustypipe/native-tls-vendored",
+]
 rustls-tls-webpki-roots = [
     "reqwest/rustls-tls-webpki-roots",
     "rustypipe/rustls-tls-webpki-roots",
@@ -15,17 +30,37 @@ rustls-tls-native-roots = [
     "rustypipe/rustls-tls-native-roots",
 ]
 
+audiotag = ["dep:lofty", "dep:image", "dep:smartcrop2"]
+
 [dependencies]
-rustypipe = { path = "..", default-features = false }
-once_cell = "1.12.0"
-regex = "1.6.0"
-thiserror = "1.0.36"
-futures = "0.3.21"
-indicatif = "0.17.0"
-filenamify = "0.1.0"
-log = "0.4.17"
-reqwest = { version = "0.11.11", default-features = false, features = [
-    "stream",
+rustypipe.workspace = true
+once_cell.workspace = true
+regex.workspace = true
+thiserror.workspace = true
+futures-util.workspace = true
+reqwest = { workspace = true, features = ["stream"] }
+rand.workspace = true
+tokio = { workspace = true, features = ["macros", "fs", "process"] }
+indicatif = { workspace = true, optional = true }
+filenamify.workspace = true
+tracing.workspace = true
+time.workspace = true
+lofty = { version = "0.22.0", optional = true }
+image = { version = "0.25.0", optional = true, default-features = false, features = [
+    "rayon",
+    "jpeg",
+    "webp",
 ] }
-rand = "0.8.5"
-tokio = { version = "1.20.0", features = ["macros", "fs", "process"] }
+smartcrop2 = { version = "0.4.0", optional = true }
+
+[dev-dependencies]
+path_macro.workspace = true
+rstest.workspace = true
+serde_json.workspace = true
+temp_testdir = "0.2.3"
+
+[package.metadata.docs.rs]
+# To build locally:
+# RUSTDOCFLAGS="--cfg docsrs" cargo +nightly doc --features indicatif,audiotag --no-deps --open
+features = ["indicatif", "audiotag"]
+rustdoc-args = ["--cfg", "docsrs"]

downloader/README.md

@@ -0,0 +1,47 @@
+# ![RustyPipe](https://codeberg.org/ThetaDev/rustypipe/raw/branch/main/notes/logo.svg) Downloader
+
+[![Current crates.io version](https://img.shields.io/crates/v/rustypipe-downloader.svg)](https://crates.io/crates/rustypipe-downloader)
+[![License](https://img.shields.io/badge/License-GPL--3-blue.svg?style=flat)](https://opensource.org/licenses/GPL-3.0)
+[![Docs](https://img.shields.io/docsrs/rustypipe-downloader/latest?style=flat)](https://docs.rs/rustypipe-downloader)
+[![CI status](https://codeberg.org/ThetaDev/rustypipe/actions/workflows/ci.yaml/badge.svg?style=flat&label=CI)](https://codeberg.org/ThetaDev/rustypipe/actions/?workflow=ci.yaml)
+
+The downloader is a companion crate for RustyPipe that allows for easy and fast
+downloading of video and audio files.
+
+## Features
+
+- Fast download of streams, bypassing YouTube's throttling
+- Join video and audio streams using ffmpeg
+- [Indicatif](https://crates.io/crates/indicatif) support to show download progress bars
+  (enable `indicatif` feature to use)
+- Tag audio files with title, album, artist, date, description and album cover (enable
+  `audiotag` feature to use)
+- Album covers are automatically cropped using smartcrop to ensure they are square
+
+## How to use
+
+For the downloader to work, you need to have ffmpeg installed on your system. If your
+ffmpeg binary is located at a non-standard path, you can configure the location using
+[`DownloaderBuilder::ffmpeg`].
+
+At first you have to instantiate and configure the downloader using either
+[`Downloader::new`] or the [`DownloaderBuilder`].
+
+Then you can build a new download query with a video ID, stream filter and destination
+path and finally download the video.
+
+```rust ignore
+use rustypipe::param::StreamFilter;
+use rustypipe_downloader::DownloaderBuilder;
+
+let dl = DownloaderBuilder::new()
+    .audio_tag()
+    .crop_cover()
+    .build();
+
+let filter_audio = StreamFilter::new().no_video();
+dl.id("eRsGyueVLvQ").stream_filter(filter_audio).to_file("audio.opus").download().await;
+
+let filter_video = StreamFilter::new().video_max_res(720);
+dl.id("eRsGyueVLvQ").stream_filter(filter_video).to_file("video.mp4").download().await;
+```

downloader/src/error.rs

@@ -0,0 +1,59 @@
+use std::{borrow::Cow, path::PathBuf};
+
+use rustypipe::client::ClientType;
+
+/// Error from the video downloader
+#[derive(thiserror::Error, Debug)]
+#[non_exhaustive]
+pub enum DownloadError {
+    /// RustyPipe error
+    #[error("{0}")]
+    RustyPipe(#[from] rustypipe::error::Error),
+    /// Error from the HTTP client
+    #[error("http error: {0}")]
+    Http(#[from] reqwest::Error),
+    /// 403 error trying to download video
+    #[error("YouTube returned 403 error; visitor_data={}", .visitor_data.as_deref().unwrap_or_default())]
+    Forbidden {
+        /// Client type used to fetch the failed stream
+        client_type: ClientType,
+        /// Visitor data used to fetch the failed stream
+        visitor_data: Option<String>,
+    },
+    /// File IO error
+    #[error(transparent)]
+    Io(#[from] std::io::Error),
+    /// FFmpeg returned an error
+    #[error("FFmpeg error: {0}")]
+    Ffmpeg(Cow<'static, str>),
+    /// Error parsing ranges for progressive download
+    #[error("Progressive download error: {0}")]
+    Progressive(Cow<'static, str>),
+    /// Video could not be downloaded because of invalid player data
+    #[error("source error: {0}")]
+    Source(Cow<'static, str>),
+    /// Download target already exists
+    #[error("file {0} already exists")]
+    Exists(PathBuf),
+    #[cfg(feature = "audiotag")]
+    /// Audio tagging error
+    #[error("Audio tag error: {0}")]
+    AudioTag(Cow<'static, str>),
+    /// Other error
+    #[error("error: {0}")]
+    Other(Cow<'static, str>),
+}
+
+#[cfg(feature = "audiotag")]
+impl From<lofty::error::LoftyError> for DownloadError {
+    fn from(value: lofty::error::LoftyError) -> Self {
+        Self::AudioTag(value.to_string().into())
+    }
+}
+
+#[cfg(feature = "audiotag")]
+impl From<image::ImageError> for DownloadError {
+    fn from(value: image::ImageError) -> Self {
+        Self::AudioTag(value.to_string().into())
+    }
+}

downloader/src/util.rs

@@ -1,26 +1,8 @@
-use std::{borrow::Cow, collections::BTreeMap};
+use std::collections::BTreeMap;
 
 use reqwest::Url;
 
-/// Error from the video downloader
-#[derive(thiserror::Error, Debug)]
-#[non_exhaustive]
-pub enum DownloadError {
-    /// Error from the HTTP client
-    #[error("http error: {0}")]
-    Http(#[from] reqwest::Error),
-    /// File IO error
-    #[error(transparent)]
-    Io(#[from] std::io::Error),
-    #[error("FFmpeg error: {0}")]
-    Ffmpeg(Cow<'static, str>),
-    #[error("Progressive download error: {0}")]
-    Progressive(Cow<'static, str>),
-    #[error("input error: {0}")]
-    Input(Cow<'static, str>),
-    #[error("error: {0}")]
-    Other(Cow<'static, str>),
-}
+use crate::DownloadError;
 
 /// Split an URL into its base string and parameter map
 ///

downloader/tests/tests.rs

@@ -0,0 +1,127 @@
+use std::{fs, os::unix::fs::MetadataExt, path::Path, process::Command};
+
+use path_macro::path;
+use rstest::{fixture, rstest};
+use rustypipe::{client::RustyPipe, model::AudioCodec, param::StreamFilter};
+use rustypipe_downloader::Downloader;
+use temp_testdir::TempDir;
+
+/// Get a new RusttyPipe instance
+#[fixture]
+fn rp() -> RustyPipe {
+    let vdata = std::env::var("YT_VDATA").ok();
+    RustyPipe::builder()
+        .strict()
+        .storage_dir(path!(env!("CARGO_MANIFEST_DIR") / ".."))
+        .visitor_data_opt(vdata)
+        .build()
+        .unwrap()
+}
+
+#[rstest]
+#[tokio::test]
+async fn download_video(rp: RustyPipe) {
+    let td = TempDir::default();
+    let td_path = td.to_path_buf();
+
+    let dl = Downloader::builder().rustypipe(&rp).build();
+
+    let res = dl
+        .id("UXqq0ZvbOnk")
+        .to_dir(&td_path)
+        .stream_filter(StreamFilter::new().video_max_res(480))
+        .download()
+        .await
+        .unwrap();
+
+    assert_eq!(
+        res.dest,
+        path!(td_path / "CHARGE - Blender Open Movie [UXqq0ZvbOnk].mp4")
+    );
+    assert_eq!(res.player_data.details.id, "UXqq0ZvbOnk");
+}
+
+#[rstest]
+#[tokio::test]
+async fn download_music(rp: RustyPipe) {
+    let td = TempDir::default();
+    let td_path = td.to_path_buf();
+
+    #[allow(unused_mut)]
+    let mut dl = Downloader::builder().rustypipe(&rp);
+    #[cfg(feature = "audiotag")]
+    {
+        dl = dl.audio_tag().crop_cover();
+    }
+    let dl = dl.build();
+
+    let res = dl
+        .id("bVtv3st8bgc")
+        .to_dir(&td_path)
+        .stream_filter(
+            StreamFilter::new()
+                .no_video()
+                .audio_codecs([AudioCodec::Opus]),
+        )
+        .download()
+        .await
+        .unwrap();
+
+    assert_eq!(
+        res.dest,
+        path!(td_path / "Lord of the Riffs [bVtv3st8bgc].opus")
+    );
+    assert_eq!(res.player_data.details.id, "bVtv3st8bgc");
+    let fm = fs::metadata(&res.dest).unwrap();
+    assert_gte(fm.size(), 6_000_000, "file size");
+    assert_audio_meta(
+        &res.dest,
+        "Lord of the Riffs",
+        "Alexander Nakarada",
+        "Lord of the Riffs",
+        "2022-02-05",
+    );
+}
+
+/// Assert that number A is greater than or equal to number B
+#[track_caller]
+fn assert_gte<T: PartialOrd + std::fmt::Display>(a: T, b: T, msg: &str) {
+    assert!(a >= b, "expected >= {b} {msg}, got {a}");
+}
+
+#[track_caller]
+fn assert_audio_meta(p: &Path, title: &str, artist: &str, album: &str, date: &str) {
+    let res = Command::new("ffprobe")
+        .args([
+            "-loglevel",
+            "error",
+            "-show_entries",
+            "stream_tags",
+            "-of",
+            "json",
+        ])
+        .arg(p)
+        .output()
+        .unwrap();
+    if !res.status.success() {
+        panic!("ffprobe error\n{}", String::from_utf8_lossy(&res.stderr))
+    }
+    let res_json = serde_json::from_slice::<serde_json::Value>(&res.stdout).unwrap();
+    let tags = &res_json["streams"][0]["tags"];
+    assert_eq!(tags["TITLE"].as_str(), Some(title));
+    assert_eq!(tags["ARTIST"].as_str(), Some(artist));
+    assert_eq!(tags["ALBUM"].as_str(), Some(album));
+    assert_eq!(tags["DATE"].as_str(), Some(date));
+}
+
+/// This is just a static check to make sure all RustyPipe futures can be sent
+/// between threads safely.
+/// Otherwise this may cause issues when integrating RustyPipe into async projects.
+#[allow(unused)]
+async fn all_send_and_sync() {
+    fn send_and_sync<T: Send + Sync>(t: T) {}
+
+    let dl = Downloader::default();
+    let dlq = dl.id("");
+    send_and_sync(dlq.download());
+}

notes/AB_Tests.md

@@ -3,47 +3,59 @@
 When YouTube introduces a new feature, it does so gradually. When a user creates a new
 session, YouTube decided randomly which new features should be enabled.
 
-YouTube sessions are identified by the visitor data cookie. This cookie is sent with every
-API request using the `context.client.visitor_data` JSON parameter. It is also returned in the
-`responseContext.visitorData` response parameter and stored as the `__SECURE-YEC` cookie.
+YouTube sessions are identified by the visitor data ID. This cookie is sent with every
+API request using the `context.client.visitor_data` JSON parameter. It is also returned
+in the `responseContext.visitorData` response parameter and stored as the `__SECURE-YEC`
+cookie.
 
-By sending the same visitor data cookie, A/B tests can be reproduced, which is important for testing
-alternative YouTube clients.
+By sending the same visitor data ID, A/B tests can be reproduced, which is important for
+testing alternative YouTube clients.
 
-This page lists all A/B tests that were encountered while maintaining the RustyPipe client.
+This page lists all A/B tests that were encountered while maintaining the RustyPipe
+client.
 
 **Impact rating:**
 
-The impact ratings shows how much effort it takes to adapt alternative YouTube clients to the
-new feature.
+The impact ratings shows how much effort it takes to adapt alternative YouTube clients
+to the new feature.
 
 - 🟢 **Low** Minor incompatibility (e.g. parameter name change)
 - 🟡 **Medium** Extensive changes to the response data model OR removal of parameters
-- 🔴 **High** Changes to the functionality of YouTube that will require API changes
-  for alternative clients
+- 🔴 **High** Changes to the functionality of YouTube that will require API changes for
+  alternative clients
 
-If you want to check how often these A/B tests occur, you can use the `codegen` tool with the
-following command: `rustypipe-codegen ab-test <id>`.
+**Status:**
+
+- Discontinued (0%)
+- Experimental (<3%)
+- Common (>3%)
+- Frequent (>40%)
+- Stabilized (100%)
+
+If you want to check how often these A/B tests occur, you can use the `codegen` tool
+with the following command: `rustypipe-codegen ab-test <id>`.
 
 ## [1] Attributed text description
 
 - **Encountered on:** 24.09.2022
 - **Impact:** 🟡 Medium
 - **Endpoint:** next (video details)
+- **Status:** Stabilized
 
 ![A/B test 1 screenshot](./_img/ab_1.png)
 
-YouTube shows internal links (channels, videos, playlists) in the video description
-as buttons with the YouTube icon. To accomplish this, they completely changed the underlying
-data model.
+YouTube shows internal links (channels, videos, playlists) in the video description as
+buttons with the YouTube icon. To accomplish this, they completely changed the
+underlying data model.
 
-The new format uses a string with the entire plaintext content along with a list of `"commandRuns"`
-which include the link data and the position of the links within the text.
+The new format uses a string with the entire plaintext content along with a list of
+`"commandRuns"` which include the link data and the position of the links within the
+text.
 
 Note that the position and length parameter refer to the number of UTF-16 characters. If
 you are implementing this in a language which does not use UTF-16 as its internal string
-representation, you have to iterate over the unicode codepoints and keep track of the UTF-16
-index seperately.
+representation, you have to iterate over the unicode codepoints and keep track of the
+UTF-16 index seperately.
 
 **OLD**
 
@@ -118,20 +130,22 @@ index seperately.
 - **Encountered on:** 11.10.2022
 - **Impact:** 🔴 High
 - **Endpoint:** browse (channel videos)
+- **Status:** Stabilized
 
 ![A/B test 2 screenshot](./_img/ab_2.webp)
 
 YouTube changed their channel page layout, putting livestreams and short videos into
 separate tabs.
 
-Fetching the videos page now only returns a subset of a channel's videos. To get all videos
-from a channel, you would have to run up to 3 queries.
+Fetching the videos page now only returns a subset of a channel's videos. To get all
+videos from a channel, you would have to run up to 3 queries.
 
-Even though it has its disadvantages, the RSS feed is now probably the best way for keeping
-track of a channel's new uploads.
+Even though it has its disadvantages, the RSS feed is now probably the best way for
+keeping track of a channel's new uploads.
 
-Additionally the channel tab response model was slightly changed, now using a `"RichGridRenderer"`.
-Short videos also have their own data models (`"reelItemRenderer"`).
+Additionally the channel tab response model was slightly changed, now using a
+`"RichGridRenderer"`. Short videos also have their own data models
+(`"reelItemRenderer"`).
 
 **RichGrid**
 
@@ -213,6 +227,7 @@ Short videos also have their own data models (`"reelItemRenderer"`).
 - **Encountered on:** 20.11.2022
 - **Impact:** 🟡 Medium
 - **Endpoint:** search
+- **Status:** Stabilized
 
 ![A/B test 3 screenshot](./_img/ab_3.png)
 
@@ -272,9 +287,10 @@ Note that channels without handles still use the old data model, even on the sam
 - **Encountered on:** 1.04.2023
 - **Impact:** 🟢 Low
 - **Endpoint:** browse (trending videos)
+- **Status:** Discontinued
 
-YouTube moved the list of trending videos from the main *trending* page to a
-separate tab (Videos).
+YouTube moved the list of trending videos from the main _trending_ page to a separate
+tab (Videos).
 
 The video tab is fetched with the params `4gIOGgxtb3N0X3BvcHVsYXI%3D`.
 
@@ -289,4 +305,799 @@ The data model for the video shelves did not change.
 
 **NEW**
 
-![A/B test 4 old screenshot](./_img/ab_4_new.png)
+![A/B test 4 new screenshot](./_img/ab_4_new.png)
+
+## [5] Page header renderer on the Trending page
+
+- **Encountered on:** 1.05.2023
+- **Impact:** 🟢 Low
+- **Endpoint:** browse (trending videos)
+- **Status:** Stabilized
+
+YouTube changed the header renderer type on the trending page to a `pageHeaderRenderer`.
+
+**OLD**
+
+```json
+{
+  "c4TabbedHeaderRenderer": {
+    "avatar": {
+      "thumbnails": [
+        {
+          "height": 100,
+          "url": "https://www.youtube.com/img/trending/avatar/trending_avatar.png",
+          "width": 100
+        }
+      ]
+    },
+    "title": "Trending",
+    "trackingParams": "CBAQ8DsiEwiXi_iUht76AhVM6hEIHfgTB2g="
+  }
+}
+```
+
+**NEW**
+
+```json
+{
+  "pageHeaderRenderer": {
+    "pageTitle": "Trending",
+    "content": {
+      "pageHeaderViewModel": {
+        "title": {
+          "dynamicTextViewModel": { "text": { "content": "Trending" } }
+        },
+        "image": {
+          "contentPreviewImageViewModel": {
+            "image": {
+              "sources": [
+                {
+                  "url": "https://www.youtube.com/img/trending/avatar/trending.png",
+                  "width": 100,
+                  "height": 100
+                }
+              ]
+            },
+            "style": "CONTENT_PREVIEW_IMAGE_STYLE_CIRCLE"
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+## [6] New Music Discography page
+
+- **Encountered on:** 13.05.2023
+- **Impact:** 🟡 Medium
+- **Endpoint:** browse (music artist)
+- **Status:** Stabilized
+
+YouTube merged the 2 sections for singles and albums on artist pages together. Now there
+is only a _Top Releases_ section.
+
+YouTube also changed the way the full discography page is fetched, surprisingly making
+it easier for alternative clients. The discography page now has its own content ID in
+the format of `MPAD<channel id>` (Music Page Artist Discography). This page can be
+fetched with a regular browse request without requiring parameters to be parsed or a
+visitor data ID to be set, as it was the case with the old system.
+
+**OLD**
+
+![A/B test 6 old screenshot](./_img/ab_6_old.png)
+
+**NEW**
+
+![A/B test 6 screenshot](./_img/ab_6_new.png)
+
+## [7] Short timeago format
+
+- **Encountered on:** 28.05.2023
+- **Impact:** 🟢 Low
+- **Status:** Discontinued
+
+YouTube changed their date format from the long format (_21 hours ago_, _3 days ago_) to
+a short format (_21h ago_, _3d ago_).
+
+## [8] Track playback count in search results and artist views
+
+- **Encountered on:** 29.06.2023
+- **Impact:** 🟡 Medium
+- **Status:** Stabilized
+
+YouTube added the track playback count to search results and top artist tracks. In
+exchange, they removed the "Song" type identifier from search results.
+
+![A/B test 8 old screenshot](./_img/ab_8_old.png)
+
+![A/B test 8 screenshot](./_img/ab_8.png)
+
+## [9] Playlists for Shorts
+
+- **Encountered on:** 26.06.2023
+- **Impact:** 🟡 Medium
+- **Endpoint:** browse (playlist)
+- **Status:** Stabilized
+
+![A/B test 9 screenshot](./_img/ab_9.png)
+
+Original issue: https://github.com/TeamNewPipe/NewPipeExtractor/issues/10774
+
+YouTube added a filter system for playlists, allowing users to only see shorts/full
+videos.
+
+When shorts filter is enabled or when there are only shorts in a playlist, YouTube
+return shorts UI elements instead of standard video ones, the ones that are also used
+for shorts shelves in searches and suggestions and shorts in the corresponding channel
+tab.
+
+Since the reel items dont include upload date information you can circumvent this new UI
+by using the mobile client. But that may change in the future.
+
+## [10] Channel About modal
+
+- **Encountered on:** 03.11.2023
+- **Impact:** 🟡 Medium
+- **Endpoint:** browse (channel info)
+- **Status:** Stabilized
+
+![A/B test 10 screenshot](./_img/ab_10.png)
+
+YouTube replaced the _About_ channel tab with a modal. This changes the way additional
+channel metadata has to be fetched.
+
+The new modal uses a continuation request with a token which can be easily generated.
+Attempts to fetch the old about tab with the A/B test enabled will lead to a redirect to
+the main tab.
+
+## [11] Like-Button viewmodel
+
+- **Encountered on:** 03.11.2023
+- **Impact:** 🟢 Low
+- **Endpoint:** next
+- **Status:** Stabilized
+
+YouTube introduced an updated data model for the like/dislike buttons. The new model
+looks needlessly complex but contains the same parsing-relevant data as the old model
+(accessibility text to get like count).
+
+```json
+{
+  "segmentedLikeDislikeButtonViewModel": {
+    "likeButtonViewModel": {
+      "likeButtonViewModel": {
+        "toggleButtonViewModel": {
+          "toggleButtonViewModel": {
+            "defaultButtonViewModel": {
+              "buttonViewModel": {
+                "iconName": "LIKE",
+                "title": "4.2M",
+                "accessibilityText": "like this video along with 4,209,059 other people"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+## [12] New channel page header
+
+- **Encountered on:** 29.01.2024
+- **Impact:** 🟢 Low
+- **Endpoint:** browse
+- **Status:** Stabilized
+
+YouTube introduced a new data model for channel headers, based on a
+`"pageHeaderRenderer"`. The new model comes with more needless complexity that needs to
+be accomodated. There are also no mobile/TV header images available any more.
+
+```json
+{
+  "pageHeaderViewModel": {
+    "title": {
+      "dynamicTextViewModel": {
+        "text": {
+          "content": "Doobydobap",
+          "attachmentRuns": [
+            {
+              "startIndex": 10,
+              "length": 0,
+              "element": {
+                "type": {
+                  "imageType": {
+                    "image": {
+                      "sources": [
+                        {
+                          "clientResource": {
+                            "imageName": "CHECK_CIRCLE_FILLED"
+                          },
+                          "width": 14,
+                          "height": 14
+                        }
+                      ]
+                    }
+                  }
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    "image": {
+      "decoratedAvatarViewModel": {
+        "avatar": {
+          "avatarViewModel": {
+            "image": {
+              "sources": [
+                {
+                  "url": "https://yt3.googleusercontent.com/dm5Aq93xvVJz0NoVO88ieBkDXmuShCujGPlZ7qETMEPTrXvPUCFI3-BB6Xs_P-r6Uk3mnBy9zA=s72-c-k-c0x00ffffff-no-rj",
+                  "width": 72,
+                  "height": 72
+                }
+              ]
+            }
+          }
+        }
+      }
+    },
+    "metadata": {
+      "contentMetadataViewModel": {
+        "metadataRows": [
+          {
+            "metadataParts": [
+              {
+                "text": {
+                  "content": "@Doobydobap"
+                }
+              },
+              {
+                "text": {
+                  "content": "3.74M subscribers"
+                }
+              },
+              {
+                "text": {
+                  "content": "345 videos",
+                  "styleRuns": [
+                    {
+                      "startIndex": 0,
+                      "length": 10
+                    }
+                  ]
+                }
+              }
+            ]
+          }
+        ]
+      }
+    },
+    "banner": {
+      "imageBannerViewModel": {
+        "image": {
+          "sources": [
+            {
+              "url": "https://yt3.googleusercontent.com/BvnAqgiursrXpmS9AgDLtkOSTQfOG_Dqn0KzY5hcwO9XrHTEQTVgaflI913f9KRp7d0U2qBp=w1060-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
+              "width": 1060,
+              "height": 175
+            }
+          ]
+        }
+      }
+    }
+  }
+}
+```
+
+## [13] Music album/playlist 2-column layout
+
+- **Encountered on:** 29.02.2024
+- **Impact:** 🟢 Low
+- **Endpoint:** browse
+- **Status:** Stabilized
+
+![A/B test 13 screenshot](./_img/ab_13.png)
+
+YouTube Music updated the layout of album and playlist pages. The new layout shows the
+cover on the left side of the playlist content.
+
+## [14] Comments Framework update
+
+- **Encountered on:** 31.01.2024
+- **Impact:** 🟢 Low
+- **Endpoint:** next
+- **Status:** Stabilized
+
+YouTube changed the data model for YouTube comments, now putting the content into a
+seperate framework update object
+
+```json
+{
+  "frameworkUpdates": {
+    "onResponseReceivedEndpoints": [
+      {
+        "clickTrackingParams": "CAAQg2ciEwi64q3dmKGFAxWvy0IFHc14BKM=",
+        "reloadContinuationItemsCommand": {
+          "targetId": "comments-section",
+          "continuationItems": [
+            {
+              "commentThreadRenderer": {
+                "replies": {
+                  "commentRepliesRenderer": {
+                    "contents": [
+                      {
+                        "continuationItemRenderer": {
+                          "trigger": "CONTINUATION_TRIGGER_ON_ITEM_SHOWN",
+                          "continuationEndpoint": {
+                            "clickTrackingParams": "CHgQvnUiEwi64q3dmKGFAxWvy0IFHc14BKM=",
+                            "commandMetadata": {
+                              "webCommandMetadata": {
+                                "sendPost": true,
+                                "apiUrl": "/youtubei/v1/next"
+                              }
+                            },
+                            "continuationCommand": {
+                              "token": "Eg0SC1FpcDFWa1R1TTcwGAYygwEaUBIaVWd5TlRUOHV4REVqZ1lxeWJJRjRBYUFCQWciAggAKhhVQ3lhZmx6ek9IMEdDNjgzRGxRLWZ6d2cyC1FpcDFWa1R1TTcwQAFICoIBAggBQi9jb21tZW50LXJlcGxpZXMtaXRlbS1VZ3lOVFQ4dXhERWpnWXF5YklGNEFhQUJBZw%3D%3D",
+                              "request": "CONTINUATION_REQUEST_TYPE_WATCH_NEXT"
+                            }
+                          }
+                        }
+                      }
+                    ],
+                    "trackingParams": "CHgQvnUiEwi64q3dmKGFAxWvy0IFHc14BKM=",
+                    "viewReplies": {
+                      "buttonRenderer": {
+                        "text": { "runs": [{ "text": "220 replies" }] },
+                        "icon": { "iconType": "ARROW_DROP_DOWN" },
+                        "trackingParams": "CHoQosAEIhMIuuKt3ZihhQMVr8tCBR3NeASj",
+                        "iconPosition": "BUTTON_ICON_POSITION_TYPE_LEFT_OF_TEXT"
+                      }
+                    },
+                    "hideReplies": {
+                      "buttonRenderer": {
+                        "text": { "runs": [{ "text": "220 replies" }] },
+                        "icon": { "iconType": "ARROW_DROP_UP" },
+                        "trackingParams": "CHkQ280EIhMIuuKt3ZihhQMVr8tCBR3NeASj",
+                        "iconPosition": "BUTTON_ICON_POSITION_TYPE_LEFT_OF_TEXT"
+                      }
+                    },
+                    "targetId": "comment-replies-item-UgyNTT8uxDEjgYqybIF4AaABAg"
+                  }
+                },
+                "trackingParams": "CHYQwnUYywEiEwi64q3dmKGFAxWvy0IFHc14BKM=",
+                "renderingPriority": "RENDERING_PRIORITY_PINNED_COMMENT",
+                "isModeratedElqComment": false,
+                "commentViewModel": {
+                  "commentViewModel": {
+                    "commentId": "UgyNTT8uxDEjgYqybIF4AaABAg"
+                  }
+                }
+              }
+            }
+          ]
+        }
+      }
+    ],
+    "entityBatchUpdate": {
+      "mutations": [
+        {
+          "entityKey": "EhpVZ3lOVFQ4dXhERWpnWXF5YklGNEFhQUJBZyAoKAE%3D",
+          "type": "ENTITY_MUTATION_TYPE_REPLACE",
+          "payload": {
+            "commentEntityPayload": {
+              "key": "EhpVZ3lOVFQ4dXhERWpnWXF5YklGNEFhQUJBZyAoKAE%3D",
+              "properties": {
+                "commentId": "UgyNTT8uxDEjgYqybIF4AaABAg",
+                "content": {
+                  "content": "⚠️ Important notice: if you put any symbol immediately after markup, it will not work: *here is the comma*, without space.\n\nYou should leave space before and after , to make it work.\n\nSame for _underscore_, and -hyphen-.\n\nLeave space before opening and after closing underscore and hyphen. Put all dots and commas inside markup.",
+                  "styleRuns": [
+                    {
+                      "startIndex": 135,
+                      "length": 28,
+                      "weightLabel": "FONT_WEIGHT_MEDIUM"
+                    },
+                    {
+                      "startIndex": 267,
+                      "length": 10,
+                      "weightLabel": "FONT_WEIGHT_NORMAL",
+                      "italic": true
+                    },
+                    {
+                      "startIndex": 282,
+                      "length": 7,
+                      "weightLabel": "FONT_WEIGHT_NORMAL",
+                      "strikethrough": "LINE_STYLE_SINGLE"
+                    }
+                  ]
+                },
+                "publishedTime": "2 years ago (edited)",
+                "replyLevel": 0,
+                "authorButtonA11y": "@kibizoid",
+                "toolbarStateKey": "EhpVZ3lOVFQ4dXhERWpnWXF5YklGNEFhQUJBZyAsKAE%3D",
+                "translateButtonEntityKey": "EhpVZ3lOVFQ4dXhERWpnWXF5YklGNEFhQUJBZyD_ASgB"
+              },
+              "author": {
+                "channelId": "UCUJfyiofeHQTmxKwZ6cCwIg",
+                "displayName": "@kibizoid",
+                "avatarThumbnailUrl": "https://yt3.ggpht.com/ytc/AIdro_nY2PkIyojDqs9Bk5RY6J90-U7wePswTYl799DNJQ=s88-c-k-c0x00ffffff-no-rj",
+                "isVerified": false,
+                "isCurrentUser": false,
+                "isCreator": false,
+                "isArtist": false
+              },
+              "avatar": {
+                "image": {
+                  "sources": [
+                    {
+                      "url": "https://yt3.ggpht.com/ytc/AIdro_nY2PkIyojDqs9Bk5RY6J90-U7wePswTYl799DNJQ=s88-c-k-c0x00ffffff-no-rj",
+                      "width": 88,
+                      "height": 88
+                    }
+                  ]
+                }
+              }
+            }
+          }
+        }
+      ]
+    }
+  }
+}
+```
+
+## [15] Channel shorts: shortsLockupViewModel
+
+- **Encountered on:** 10.09.2024
+- **Impact:** 🟢 Low
+- **Endpoint:** browse
+- **Status:** Stabilized
+
+YouTube changed the data model for the channel shorts tab
+
+```json
+{
+  "richItemRenderer": {
+    "content": {
+      "shortsLockupViewModel": {
+        "entityId": "shorts-shelf-item-ovaHmfy3O6U",
+        "accessibilityText": "hangover food, 17 million views - play Short",
+        "thumbnail": {
+          "sources": [
+            {
+              "url": "https://i.ytimg.com/vi/ovaHmfy3O6U/oar2.jpg?sqp=-oaymwEdCJUDENAFSFWQAgHyq4qpAwwIARUAAIhCcAHAAQY=&rs=AOn4CLBg-kG4rAi-BQ8Xkp2hOtOu-oXDLQ",
+              "width": 405,
+              "height": 720
+            }
+          ]
+        },
+        "overlayMetadata": {
+          "primaryText": {
+            "content": "hangover food"
+          },
+          "secondaryText": {
+            "content": "17M views"
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+## [16] New playlist header renderer
+
+- **Encountered on:** 11.10.2024
+- **Impact:** 🟢 Low
+- **Endpoint:** browse
+- **Status:** Stabilized
+
+```json
+{
+  "pageHeaderRenderer": {
+    "pageTitle": "LilyPichu",
+    "content": {
+      "pageHeaderViewModel": {
+        "title": {
+          "dynamicTextViewModel": {
+            "text": {
+              "content": "LilyPichu"
+            }
+          }
+        },
+        "metadata": {
+          "contentMetadataViewModel": {
+            "metadataRows": [
+              {
+                "metadataParts": [
+                  {
+                    "avatarStack": {
+                      "avatarStackViewModel": {
+                        "avatars": [
+                          {
+                            "avatarViewModel": {
+                              "image": {
+                                "sources": [
+                                  {
+                                    "url": "https://yt3.ggpht.com/ytc/AIdro_kcjhSY2e8WlYjQABOB65Za8n3QYycNHP9zXwxjKpBfOg=s48-c-k-c0x00ffffff-no-rj",
+                                    "width": 48,
+                                    "height": 48
+                                  }
+                                ]
+                              }
+                            }
+                          }
+                        ],
+                        "text": {
+                          "content": "by Kevin Ramirez",
+                          "commandRuns": [
+                            {
+                              "startIndex": 0,
+                              "length": 16,
+                              "onTap": {
+                                "innertubeCommand": {
+                                  "browseEndpoint": {
+                                    "browseId": "UCai7BcI5lrXC2vdc3ySku8A",
+                                    "canonicalBaseUrl": "/@XxthekevinramirezxX"
+                                  }
+                                }
+                              }
+                            }
+                          ]
+                        }
+                      }
+                    }
+                  }
+                ]
+              },
+              {
+                "metadataParts": [
+                  {
+                    "text": {
+                      "content": "Playlist"
+                    }
+                  },
+                  {
+                    "text": {
+                      "content": "10 videos"
+                    }
+                  },
+                  {
+                    "text": {
+                      "content": "856 views"
+                    }
+                  }
+                ]
+              }
+            ]
+          }
+        },
+        "actions": {},
+        "description": {
+          "descriptionPreviewViewModel": {
+            "description": { "content": "Hello World" }
+          }
+        },
+        "heroImage": {
+          "contentPreviewImageViewModel": {
+            "image": {
+              "sources": [
+                {
+                  "url": "https://i.ytimg.com/vi/DXuNJ267Vss/hqdefault.jpg?sqp=-oaymwEWCKgBEF5IWvKriqkDCQgBFQAAiEIYAQ==&rs=AOn4CLAHp6V96b70x4SWm9Pe6WEHnQhP6A",
+                  "width": 168,
+                  "height": 94
+                }
+              ]
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+## [17] Channel playlists: lockupViewModel
+
+- **Encountered on:** 09.11.2024
+- **Impact:** 🟢 Low
+- **Endpoint:** browse
+- **Status:** Stabilized
+
+YouTube changed the data model for the channel playlists / podcasts / albums tab
+
+```json
+{
+  "lockupViewModel": {
+    "contentImage": {
+      "collectionThumbnailViewModel": {
+        "primaryThumbnail": {
+          "thumbnailViewModel": {
+            "image": {
+              "sources": [
+                {
+                  "url": "https://i.ytimg.com/vi/XYdmX8w8xwI/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLCqmf6TGfDinNXhgU29ZxOkv2u9sQ",
+                  "width": 480,
+                  "height": 270
+                }
+              ]
+            },
+            "overlays": [
+              {
+                "thumbnailOverlayBadgeViewModel": {
+                  "thumbnailBadges": [
+                    {
+                      "thumbnailBadgeViewModel": {
+                        "icon": {
+                          "sources": [
+                            {
+                              "clientResource": {
+                                "imageName": "PLAYLISTS"
+                              }
+                            }
+                          ]
+                        },
+                        "text": "5 videos",
+                        "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT",
+                        "backgroundColor": {
+                          "lightTheme": 2370867,
+                          "darkTheme": 2370867
+                        }
+                      }
+                    }
+                  ],
+                  "position": "THUMBNAIL_OVERLAY_BADGE_POSITION_BOTTOM_END"
+                }
+              }
+            ]
+          }
+        }
+      }
+    },
+    "metadata": {
+      "lockupMetadataViewModel": {
+        "title": {
+          "content": "Jellybean Components Series"
+        }
+      }
+    },
+    "contentId": "PLvOlSehNtuHv268f0mW5m1t_hq_RVGRSA",
+    "contentType": "LOCKUP_CONTENT_TYPE_PLAYLIST"
+  }
+}
+```
+
+## [18] Music playlists facepile avatar
+
+- **Encountered on:** 25.11.2024
+- **Impact:** 🟢 Low
+- **Endpoint:** browse (YTM)
+- **Status:** Stabilized
+
+YouTube changed the data model for the channel playlist owner avatar into a `facepile`
+object. It now also contains the channel avatar.
+
+The model is also used for playlists owned by YouTube Music (with the avatar and
+commandContext missing).
+
+```json
+{
+  "facepile": {
+    "avatarStackViewModel": {
+      "avatars": [
+        {
+          "avatarViewModel": {
+            "image": {
+              "sources": [
+                {
+                  "url": "https://yt3.ggpht.com/ytc/AIdro_n9ALaLETwQH6_2WlXitIaIKV-IqBDWWquvyI2jucNAZaQ=s48-c-k-c0x00000000-no-cc-rj-rp"
+                }
+              ]
+            },
+            "avatarImageSize": "AVATAR_SIZE_XS"
+          }
+        }
+      ],
+      "text": {
+        "content": "Chaosflo44"
+      },
+      "rendererContext": {
+        "commandContext": {
+          "onTap": {
+            "innertubeCommand": {
+              "browseEndpoint": {
+                "browseId": "UCQM0bS4_04-Y4JuYrgmnpZQ",
+                "browseEndpointContextSupportedConfigs": {
+                  "browseEndpointContextMusicConfig": {
+                    "pageType": "MUSIC_PAGE_TYPE_USER_CHANNEL"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+## [19] Music artist album groups reordered
+
+- **Encountered on:** 13.01.2025
+- **Impact:** 🟢 Low
+- **Endpoint:** browse (YTM)
+- **Status:** Frequent (59%)
+
+YouTube Music used to group artist albums into 2 rows: "Albums" and "Singles".
+
+These groups were changed into "Albums" and "Singles & EPs". Now the "Album" label is
+omitted for albums in their group, while singles and EPs have a label with their type.
+
+## [20] Music continuation item renderer
+
+- **Encountered on:** 25.01.2025
+- **Impact:** 🟢 Low
+- **Endpoint:** browse (YTM)
+- **Status:** Stabilized
+
+YouTube Music now uses a `continuationItemRenderer` for music playlists instead of
+putting the continuations in a separate attribute of the MusicShelf.
+
+The continuation response now uses a `onResponseReceivedActions` field for its music
+items.
+
+YouTube Music now also sends a random 16-character string as a `clientScreenNonce` in
+the request context. This is not mandatory though.
+
+## [21] Music album recommendations
+
+- **Encountered on:** 26.02.2025
+- **Impact:** 🟢 Low
+- **Endpoint:** browse (YTM)
+- **Status:** Common (15%)
+
+![A/B test 21 screenshot](./_img/ab_21.png)
+
+YouTube Music has added "Recommended" and "More from \<Artist\>" carousels to album
+pages. The difficulty is distinguishing them reliably for parsing the album variants.
+
+The current solution is adding the "Other versions" title in all languages to the
+dictionary and comparing it.
+
+## [22] commandExecutorCommand for continuations
+
+- **Encountered on:** 16.03.2025
+- **Impact:** 🟢 Low
+- **Endpoint:** browse (YTM)
+- **Status:** Experimental (1%)
+
+YouTube playlists may use a commandExecutorCommand which holds a list of commands: the
+`continuationCommand` that needs to be extracted as well as a `playlistVotingRefreshPopupCommand`.
+
+```json
+{
+  "continuationItemRenderer": {
+    "continuationEndpoint": {
+      "commandExecutorCommand": {
+        "commands": [
+          {
+            "playlistVotingRefreshPopupCommand": {
+              "command": {}
+            }
+          },
+          {
+            "continuationCommand": {
+              "request": "CONTINUATION_REQUEST_TYPE_BROWSE",
+              "token": "4qmFsgKBARIkVkxQTGJaSVB5MjAtMXBON21xamNrZXBXRjc4bmRiNmNpX3FpGjRDQUY2SGxCVU9rTklTV2xGUkVreVVtdEZOVTVFU1hsU2FrWkRVa1JKZWs1NldRJTNEJTNEmgIiUExiWklQeTIwLTFwTjdtcWpja2VwV0Y3OG5kYjZjaV9xaQ%3D%3D"
+            }
+          }
+        ]
+      }
+    }
+  }
+}
+```

notes/channel_order.md

@@ -0,0 +1,69 @@
+# Channel order
+
+Fields:
+
+- `2:0:string` Channel ID
+- `15:0:embedded` Videos tab
+- `10:0:embedded` Shorts tab
+- `14:0:embedded` Livestreams tab
+- `2:0:string`: targetId for YouTube's web framework (`"\n$"` + any UUID)
+- `3:1:varint` Sort order (1: Latest, 2: Popular)
+
+Popular videos
+
+```json
+{
+  "80226972:0:embedded": {
+    "2:0:string": "UCXuqSBlHAE6Xw-yeJA0Tunw",
+    "3:1:base64": {
+      "110:0:embedded": {
+        "3:0:embedded": {
+          "15:0:embedded": {
+            "2:0:string": "\n$6461d7c8-0000-2040-87aa-089e0827e420",
+            "3:1:varint": 2
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+Popular shorts
+```json
+{
+  "80226972:0:embedded": {
+    "2:0:string": "UCXuqSBlHAE6Xw-yeJA0Tunw",
+    "3:1:base64": {
+      "110:0:embedded": {
+        "3:0:embedded": {
+          "10:0:embedded": {
+            "2:0:string": "\n$64679ffb-0000-26b3-a1bd-582429d2c794",
+            "3:1:varint": 2
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+Popular streams
+
+```json
+{
+  "80226972:0:embedded": {
+    "2:0:string": "UCXuqSBlHAE6Xw-yeJA0Tunw",
+    "3:1:base64": {
+      "110:0:embedded": {
+        "3:0:embedded": {
+          "14:0:embedded": {
+            "2:0:string": "\n$64693069-0000-2a1e-8c7d-582429bd5ba8",
+            "3:1:varint": 2
+          }
+        }
+      }
+    }
+  }
+}
+```

notes/channel_playlist.txt

@@ -0,0 +1,18 @@
+Source: https://github.com/TeamNewPipe/NewPipe/pull/9182#issuecomment-1508938841
+
+Note: we recently discovered that YouTube system playlists exist for regular videos of channels, for livestreams, and shorts as chronological ones (the shorts one was already known) and popular ones.
+They correspond basically to the results of the sort filters available on the channels streams tab on YouTube's interface
+
+So, basically shortcuts for the lazy/incurious?
+
+Same procedure as the one described in the 0.24.1 changelog, except that you need to change the prefix UU (all user uploads) to:
+
+UULF for regular videos only,
+UULV for livestreams only,
+UUSH for shorts only,
+UULP for popular regular videos,
+UUPS for popular shorts,
+UUPV for popular livestreams
+UUMF: members only regular videos
+UUMV: members only livestreams
+UUMS is probably for members-only shorts, we need to found a channel making shorts restricted to channel members

notes/dictionary.md

@@ -0,0 +1,34 @@
+# Parsing localized data from YouTube
+
+Since YouTube's API is outputting the website as it should be rendered by the client,
+the data received from the API is already localized. This affects dates, times and
+number formats.
+
+To be able to successfully parse them, we need to collect samples in every language and
+build a dictionary.
+
+### Timeago
+
+- Relative date format used for video upload dates and comments.
+- Examples: "1 hour ago", "3 months ago"
+
+### Playlist dates
+
+- Playlist update dates are always day-accurate, either as textual dates or in the form
+  of "n days ago"
+- Examples: "Last updated on Jan 3, 2020", "Updated today", "Updated yesterday",
+  "Updated 3 days ago"
+
+### Video duration
+
+- In Danisch ("da") video durations are formatted using dots instead of colons. Example:
+  "12.31", "3.03.52"
+
+### Numbers
+
+- Large numbers (subscriber/view counts) are rounded and shown using a decimal prefix
+- Examples: "1.4M views"
+- There is an exception for the value 0 ("no views") and in some languages for the value
+  1 (pt: "Um vídeo")
+- Special case: Language "gu", "જોવાયાની સંખ્યા" = "no views", contains no unique tokens
+  to parse

notes/logo.svg

@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   width="530"
+   height="80"
+   viewBox="0 0 140.22916 21.166667"
+   version="1.1"
+   id="svg5"
+   inkscape:version="1.2.2 (b0a8486541, 2022-12-01)"
+   sodipodi:docname="logo.svg"
+   xml:space="preserve"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg"><sodipodi:namedview
+     id="namedview7"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:showpageshadow="2"
+     inkscape:pageopacity="0"
+     inkscape:pagecheckerboard="false"
+     inkscape:deskcolor="#d1d1d1"
+     inkscape:document-units="px"
+     showgrid="false"
+     inkscape:zoom="1.329974"
+     inkscape:cx="206.77097"
+     inkscape:cy="117.29553"
+     inkscape:window-width="2516"
+     inkscape:window-height="1051"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg5" /><defs
+     id="defs2" /><g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"><g
+       aria-label="RUSTYPIPE"
+       id="text236"
+       style="font-size:21.1667px;line-height:1.25;display:inline;stroke-width:0.264583"
+       transform="translate(-22.622596,-15.875)"><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#ff2000;fill-opacity:1"
+         d="m 51.720162,28.78667 h -0.846668 v -3.238506 h 0.719667 q 0.656168,0 0.994835,-0.04233 0.338668,-0.0635 0.529168,-0.211667 0.169333,-0.148167 0.232834,-0.444501 0.0635,-0.296334 0.0635,-0.867834 0,-0.571501 -0.0635,-0.867835 -0.0635,-0.317501 -0.232834,-0.465668 -0.169334,-0.148166 -0.508001,-0.1905 -0.3175,-0.04233 -1.016002,-0.04233 h -0.719667 v -3.238505 h 2.18017 q 1.502835,0 2.43417,0.296333 0.931335,0.296334 1.439336,0.910169 0.465667,0.5715 0.613834,1.418168 0.169334,0.846668 0.169334,2.180171 0,1.714502 -0.317501,2.645837 -0.4445,1.185335 -1.566335,1.672169 l 2.201336,5.439842 h -4.445007 z"
+         id="path2732" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#8c441a;fill-opacity:1"
+         d="m 45.751152,19.176988 h 4.23334 v 14.562689 h -4.23334 z"
+         id="path2711" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#ff2000;fill-opacity:1"
+         d="m 67.701016,19.176988 h 4.23334 v 7.916346 q 0,2.074336 -0.08467,3.132671 -0.08467,1.058335 -0.465667,1.778003 -0.423334,0.825501 -1.291169,1.270002 -0.867834,0.444501 -2.391837,0.571501 z"
+         id="path2736" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#8c441a;fill-opacity:1"
+         d="m 65.94418,33.909011 q -2.222504,0 -3.429006,-0.359834 -1.206501,-0.359834 -1.778002,-1.164168 -0.529168,-0.740835 -0.656168,-1.883837 -0.127,-1.143002 -0.127,-3.407838 v -7.916346 h 4.23334 v 8.763014 q 0,0.783167 0.04233,1.502835 0.04233,0.571501 0.190501,0.825502 0.148166,0.254 0.508,0.3175 0.317501,0.08467 1.016002,0.08467 h 0.486834 q 0.169334,0 0.381001,-0.04233 v 3.259672 q -0.148167,0.02117 -0.423334,0.02117 z"
+         id="path2713" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#ff2000;fill-opacity:1"
+         d="m 80.041215,33.909011 q -2.11667,0 -3.937006,-0.1905 -1.079502,-0.105834 -1.629836,-0.211667 v -3.090339 q 1.248835,0.105834 2.857504,0.211667 1.016002,0.04233 1.439336,0.04233 1.143002,0 1.502836,-0.0635 v 3.302005 z"
+         id="path2742" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#8c441a;fill-opacity:1"
+         d="m 81.16305,29.442837 q 0,-0.4445 -0.0635,-0.635001 -0.0635,-0.211667 -0.211667,-0.275167 -0.148167,-0.08467 -0.508001,-0.127 l -2.603504,-0.317501 q -2.30717,-0.254 -3.111505,-1.502835 -0.359834,-0.529168 -0.486834,-1.291169 -0.127,-0.762001 -0.127,-1.86267 0,-2.349503 1.206502,-3.386672 0.973668,-0.846668 3.048004,-0.994835 v 4.254507 q 0,0.275167 0.02117,0.465668 0.02117,0.1905 0.08467,0.296333 0.0635,0.127001 0.211667,0.190501 0.148167,0.04233 0.444501,0.0635 l 2.921004,0.359834 q 0.910168,0.127 1.481669,0.3175 0.571501,0.1905 0.973669,0.592668 0.952501,0.994835 0.952501,3.534839 0,2.688171 -1.185335,3.767672 -0.529168,0.486835 -1.291169,0.719668 -0.740834,0.211667 -1.756836,0.275167 z"
+         id="path2740" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#ff2000;fill-opacity:1"
+         d="m 84.655556,22.521326 q -0.698502,-0.08467 -2.455338,-0.232833 -0.973668,-0.04233 -1.566335,-0.04233 -0.762002,0 -1.439336,0.04233 v -3.280839 h 0.529167 q 1.73567,0 3.598339,0.232834 0.592668,0.08467 1.333503,0.232833 z"
+         id="path2715" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#8c441a;fill-opacity:1"
+         d="m 90.222387,23.410328 h 4.23334 v 10.329349 h -4.23334 z"
+         id="path2746" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#ff2000;fill-opacity:1"
+         d="M 86.391214,19.176988 H 98.308067 V 22.56366 H 86.391214 Z"
+         id="path2717" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#ff2000;fill-opacity:1"
+         d="m 106.33024,23.685495 1.9685,-4.508507 h 4.23334 l -1.651,3.429005 -0.6985,1.439336 -1.33351,2.772837 -0.52916,1.100669 z"
+         id="path2750" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#8c441a;fill-opacity:1"
+         d="m 103.59973,28.934836 -0.1905,-0.423334 -0.55033,-1.079501 -0.52917,-1.121835 q -0.0847,-0.148167 -0.21167,-0.444501 l -0.86783,-1.79917 -2.328342,-4.889507 h 4.318012 l 4.59317,9.779015 v 4.783674 h -4.23334 z"
+         id="path2719" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#ff2000;fill-opacity:1"
+         d="m 118.92441,25.971498 h 0.508 q 0.67733,0 0.99483,-0.04233 0.33867,-0.0635 0.52917,-0.254 0.1905,-0.169334 0.23283,-0.486835 0.0635,-0.338667 0.0635,-0.994834 0,-0.656168 -0.0635,-0.973669 -0.0423,-0.338667 -0.23283,-0.529167 -0.1905,-0.169334 -0.52917,-0.211667 -0.33866,-0.0635 -0.99483,-0.0635 h -0.508 v -3.238505 h 1.75683 q 1.56634,0 2.54001,0.3175 0.99483,0.296334 1.50283,0.931335 0.48684,0.592668 0.65617,1.502836 0.16933,0.889001 0.16933,2.264837 0,1.312335 -0.16933,2.18017 -0.14817,0.867834 -0.61383,1.460502 -0.52917,0.677334 -1.52401,1.037168 -0.97366,0.338668 -2.56117,0.338668 h -1.75683 z"
+         id="path2754" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#8c441a;fill-opacity:1"
+         d="m 113.80207,19.176988 h 4.23334 v 14.562689 h -4.23334 z"
+         id="path2721" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#8c441a;fill-opacity:1"
+         d="m 127.4546,19.176988 h 4.23334 v 14.562689 h -4.23334 z"
+         id="path2723" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#ff2000;fill-opacity:1"
+         d="m 139.56195,25.971498 h 0.508 q 0.67734,0 0.99484,-0.04233 0.33866,-0.0635 0.52916,-0.254 0.1905,-0.169334 0.23284,-0.486835 0.0635,-0.338667 0.0635,-0.994834 0,-0.656168 -0.0635,-0.973669 -0.0423,-0.338667 -0.23284,-0.529167 -0.1905,-0.169334 -0.52916,-0.211667 -0.33867,-0.0635 -0.99484,-0.0635 h -0.508 v -3.238505 h 1.75684 q 1.56633,0 2.54,0.3175 0.99484,0.296334 1.50284,0.931335 0.48683,0.592668 0.65616,1.502836 0.16934,0.889001 0.16934,2.264837 0,1.312335 -0.16934,2.18017 -0.14816,0.867834 -0.61383,1.460502 -0.52917,0.677334 -1.524,1.037168 -0.97367,0.338668 -2.56117,0.338668 h -1.75684 z"
+         id="path2760" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#8c441a;fill-opacity:1"
+         d="m 134.43961,19.176988 h 4.23334 v 14.562689 h -4.23334 z"
+         id="path2725" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#ff2000;fill-opacity:1"
+         d="m 153.21448,30.501172 h 5.37635 v 3.238505 h -5.37635 z"
+         id="path2768" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#8c441a;fill-opacity:1"
+         d="m 153.21448,24.764996 h 4.38151 v 3.238506 h -4.38151 z"
+         id="path2766" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#ff2000;fill-opacity:1"
+         d="m 153.21448,19.176988 h 5.37635 v 3.238505 h -5.37635 z"
+         id="path2764" /><path
+         style="font-family:'Saira Stencil One';-inkscape-font-specification:'Saira Stencil One';fill:#8c441a;fill-opacity:1"
+         d="m 148.09214,19.176988 h 4.23334 v 14.562689 h -4.23334 z"
+         id="path2727" /></g><path
+       d="m 17.157261,11.267722 c 0.02821,-0.225786 0.04939,-0.451553 0.04939,-0.684389 0,-0.232826 -0.02107,-0.465666 -0.04939,-0.7055542 l 1.488721,-1.150055 c 0.134053,-0.105841 0.169334,-0.296333 0.08466,-0.451555 l -1.411108,-2.441223 c -0.08466,-0.155226 -0.275166,-0.218719 -0.43039,-0.155226 l -1.75683,0.705555 c -0.366888,-0.275166 -0.747887,-0.515056 -1.192389,-0.691443 l -0.261066,-1.869722 c -0.02822,-0.169333 -0.1764,-0.296332 -0.352775,-0.296332 h -2.822222 c -0.176401,0 -0.324554,0.127013 -0.352776,0.296332 l -0.2610673,1.869722 c -0.444501,0.176373 -0.825501,0.416277 -1.192388,0.691443 l -1.756835,-0.705555 c -0.155226,-0.06349 -0.345719,0 -0.430385,0.155226 l -1.411112,2.441223 c -0.09173,0.155226 -0.04939,0.34572 0.08467,0.451555 l 1.488722,1.150055 c -0.02822,0.2398932 -0.04938,0.4727232 -0.04938,0.7055542 0,0.232826 0.02107,0.458611 0.04938,0.684389 l -1.488722,1.171221 c -0.134053,0.10584 -0.1764,0.296333 -0.08467,0.451556 l 1.411112,2.44122 c 0.08466,0.155227 0.275165,0.211654 0.430385,0.155227 l 1.756835,-0.712611 c 0.366887,0.282222 0.747887,0.522112 1.192388,0.698501 l 0.2610673,1.86972 c 0.02821,0.169333 0.1764,0.296333 0.352776,0.296333 h 2.822222 c 0.176399,0 0.324554,-0.126987 0.352775,-0.296333 l 0.261066,-1.86972 c 0.444502,-0.183439 0.825501,-0.416279 1.192389,-0.698501 l 1.75683,0.712611 c 0.155227,0.05646 0.345723,0 0.43039,-0.155227 l 1.41111,-2.44122 c 0.08466,-0.155227 0.04939,-0.345721 -0.08466,-0.451556 z"
+       id="path458"
+       style="fill:none;fill-opacity:1;stroke:#8c441a;stroke-width:1.5875;stroke-dasharray:none;stroke-opacity:1"
+       sodipodi:nodetypes="csccccccccssccccccccsccccccccsscccccccc" /><path
+       style="fill:#ff2000;fill-opacity:1;stroke-width:0.829285"
+       d="M 10.29091,13.0712 14.594918,10.583335 10.29091,8.0954668 V 13.0712"
+       id="path1225" /></g></svg>

notes/po_token.md

@@ -0,0 +1,30 @@
+# About the new `pot` token
+
+YouTube has implemented a new method to prevent downloaders and alternative clients from accessing
+their videos. Now requests to YouTube's video servers require a `pot` URL parameter.
+
+It is currently only required in the web player. The YTM and embedded player sends the token, too, but does not require it (this may change in the future).
+
+The TV player does not use the token at all and is currently the best workaround. The only downside
+is that the TV player does not return any video metadata like title and description text.
+
+The first part of a video file (range: 0-1007959 bytes) can be downloaded without the token.
+Requesting more of the file requires the pot token to be set, otherwise YouTube responds with a 403
+error.
+
+The pot token is base64-formatted and usually starts with a M
+
+`MnToZ2brHmyo0ehfKtK_EWUq60dPYDXksNX_UsaniM_Uj6zbtiIZujCHY02hr7opxB_n3XHetJQCBV9cnNHovuhvDqrjfxsKR-sjn-eIxqv3qOZKphvyDpQzlYBnT2AXK41R-ti6iPonrvlvKIASNmYX2lhsEg==`
+
+The token is generated from YouTubes Botguard script. The token is bound to the visitor data ID
+used to fetch the player data.
+
+This feature has been A/B-tested for a few weeks. During that time, refetching the player in case
+of a 403 download error often made things work again. As of 08.08.2024 this new feature seems to be
+stabilized and retrying requests does not work any more.
+
+## Getting a `pot` token
+
+You need a real browser environment to run YouTube's botguard and obtain a pot token. The Invidious project has created a script to
+<https://github.com/iv-org/youtube-trusted-session-generator/tree/master>.
+The script opens YouTube's embedded video player, starts playback and extracts the visitor data

renovate.json

@@ -0,0 +1,11 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": ["config:best-practices", ":preserveSemverRanges"],
+  "semanticCommits": "enabled",
+  "automerge": true,
+  "automergeStrategy": "squash",
+  "osvVulnerabilityAlerts": true,
+  "labels": ["dependency-upgrade"],
+  "enabledManagers": ["cargo"],
+  "prHourlyLimit": 5
+}

src/cache.rs

@@ -1,20 +1,40 @@
-//! Persistent cache storage
+//! # Persistent cache storage
+//!
+//! RustyPipe caches some information fetched from YouTube: specifically
+//! the client versions and the JavaScript code used to deobfuscate the stream URLs.
+//!
+//! Without a persistent cache storage, this information would have to be re-fetched
+//! with every new instantiation of the client. This would make operation a lot slower,
+//! especially with CLI applications. For this reason, persisting the cache between
+//! program executions is recommended.
+//!
+//! Since there are many diferent ways to store this data (Text file, SQL, Redis, etc),
+//! RustyPipe allows you to plug in your own cache storage by implementing the
+//! [`CacheStorage`] trait.
+//!
+//! RustyPipe already comes with the [`FileStorage`] implementation which stores
+//! the cache as a JSON file.
 
 use std::{
-    fs,
+    fs::File,
+    io::Write,
     path::{Path, PathBuf},
 };
 
-use log::error;
+use tracing::error;
 
+pub(crate) const DEFAULT_CACHE_FILE: &str = "rustypipe_cache.json";
+
+/// Cache storage trait
+///
 /// RustyPipe has to cache some information fetched from YouTube: specifically
 /// the client versions and the JavaScript code used to deobfuscate the stream URLs.
 ///
 /// This trait is used to abstract the cache storage behavior so you can store
 /// cache data in your preferred way (File, SQL, Redis, etc).
 ///
-/// The cache is read when building the [`crate::client::RustyPipe`] client and updated
-/// whenever additional data is fetched.
+/// The cache is read when building the [`RustyPipe`](crate::client::RustyPipe)
+/// client and updated whenever additional data is fetched.
 pub trait CacheStorage: Sync + Send {
     /// Write the given string to the cache
     fn write(&self, data: &str);
@@ -42,14 +62,28 @@ impl FileStorage {
 impl Default for FileStorage {
     fn default() -> Self {
         Self {
-            path: Path::new("rustypipe_cache.json").into(),
+            path: Path::new(DEFAULT_CACHE_FILE).into(),
         }
     }
 }
 
 impl CacheStorage for FileStorage {
     fn write(&self, data: &str) {
-        fs::write(&self.path, data).unwrap_or_else(|e| {
+        fn _write(path: &Path, data: &str) -> Result<(), std::io::Error> {
+            let mut f = File::create(path)?;
+            // Set cache file permissions to 0600 on Unix-based systems
+            #[cfg(target_family = "unix")]
+            {
+                use std::os::unix::fs::PermissionsExt;
+                let metadata = f.metadata()?;
+                let mut permissions = metadata.permissions();
+                permissions.set_mode(0o600);
+                std::fs::set_permissions(path, permissions)?;
+            }
+            f.write_all(data.as_bytes())
+        }
+
+        _write(&self.path, data).unwrap_or_else(|e| {
             error!(
                 "Could not write cache to file `{}`. Error: {}",
                 self.path.to_string_lossy(),
@@ -63,7 +97,7 @@ impl CacheStorage for FileStorage {
             return None;
         }
 
-        match fs::read_to_string(&self.path) {
+        match std::fs::read_to_string(&self.path) {
             Ok(data) => Some(data),
             Err(e) => {
                 error!(

src/client/channel_rss.rs

@@ -1,9 +1,10 @@
-use std::collections::BTreeMap;
+use std::fmt::Debug;
 
 use crate::{
     error::{Error, ExtractionError},
     model::ChannelRss,
     report::Report,
+    util,
 };
 
 use super::{response, RustyPipeQuery};
@@ -15,77 +16,141 @@ impl RustyPipeQuery {
     ///
     /// Fetching RSS feeds is a lot faster than querying the InnerTube API, so this method is great
     /// for checking a lot of channels or implementing a subscription feed.
-    pub async fn channel_rss<S: AsRef<str>>(&self, channel_id: S) -> Result<ChannelRss, Error> {
-        let url = format!(
-            "https://www.youtube.com/feeds/videos.xml?channel_id={}",
-            channel_id.as_ref()
-        );
+    ///
+    /// The downside of using the RSS feed is that it does not provide video durations.
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn channel_rss<S: AsRef<str> + Debug>(
+        &self,
+        channel_id: S,
+    ) -> Result<ChannelRss, Error> {
+        let channel_id = channel_id.as_ref();
+        let url = format!("https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}");
         let xml = self
             .client
-            .http_request_txt(self.client.inner.http.get(&url).build()?)
+            .http_request_txt(&self.client.inner.http.get(&url).build()?)
             .await
             .map_err(|e| match e {
-                Error::HttpStatus(404, _) => Error::Extraction(
-                    ExtractionError::ContentUnavailable("Channel not found".into()),
-                ),
+                Error::HttpStatus(404, _) => Error::Extraction(ExtractionError::NotFound {
+                    id: channel_id.to_owned(),
+                    msg: "404".into(),
+                }),
                 _ => e,
             })?;
 
-        match quick_xml::de::from_str::<response::ChannelRss>(&xml) {
-            Ok(feed) => Ok(feed.into()),
+        match quick_xml::de::from_str::<response::ChannelRss>(&xml)
+            .map_err(|e| ExtractionError::InvalidData(e.to_string().into()))
+            .and_then(|feed| feed.map_response(channel_id))
+        {
+            Ok(res) => Ok(res),
             Err(e) => {
                 if let Some(reporter) = &self.client.inner.reporter {
                     let report = Report {
-                        info: Default::default(),
+                        info: self.rp_info(),
                         level: crate::report::Level::ERR,
-                        operation: "channel_rss".to_owned(),
+                        operation: "channel_rss",
                         error: Some(e.to_string()),
                         msgs: Vec::new(),
                         deobf_data: None,
                         http_request: crate::report::HTTPRequest {
-                            url,
-                            method: "GET".to_owned(),
-                            req_header: BTreeMap::new(),
-                            req_body: String::new(),
+                            url: &url,
+                            method: "GET",
                             status: 200,
+                            req_header: None,
+                            req_body: None,
                             resp_body: xml,
                         },
                     };
 
                     reporter.report(&report);
                 }
-
-                Err(
-                    ExtractionError::InvalidData(format!("could not deserialize xml: {e}").into())
-                        .into(),
-                )
+                Err(Error::Extraction(e))
             }
         }
     }
 }
 
+impl response::ChannelRss {
+    fn map_response(self, id: &str) -> Result<ChannelRss, ExtractionError> {
+        let channel_id = if self.channel_id.is_empty() {
+            self.entry
+                .iter()
+                .find_map(|entry| {
+                    Some(entry.channel_id.as_str())
+                        .filter(|id| id.is_empty())
+                        .map(str::to_owned)
+                })
+                .or_else(|| {
+                    self.author
+                        .uri
+                        .strip_prefix("https://www.youtube.com/channel/")
+                        .and_then(|id| {
+                            if util::CHANNEL_ID_REGEX.is_match(id) {
+                                Some(id.to_owned())
+                            } else {
+                                None
+                            }
+                        })
+                })
+                .ok_or(ExtractionError::InvalidData(
+                    "could not get channel id".into(),
+                ))?
+        } else if self.channel_id.len() == 22 {
+            // As of November 2023, YouTube seems to output channel IDs without the UC prefix
+            format!("UC{}", self.channel_id)
+        } else {
+            self.channel_id
+        };
+
+        if channel_id != id {
+            return Err(ExtractionError::WrongResult(format!(
+                "got wrong channel id {channel_id}, expected {id}",
+            )));
+        }
+
+        Ok(ChannelRss {
+            id: channel_id,
+            name: self.title,
+            videos: self
+                .entry
+                .into_iter()
+                .map(|item| crate::model::ChannelRssVideo {
+                    id: item.video_id,
+                    name: item.title,
+                    description: item.media_group.description,
+                    thumbnail: item.media_group.thumbnail.into(),
+                    publish_date: item.published,
+                    update_date: item.updated,
+                    view_count: item.media_group.community.statistics.views,
+                    like_count: item.media_group.community.rating.count,
+                })
+                .collect(),
+            create_date: self.create_date,
+        })
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::{fs::File, io::BufReader};
 
-    use crate::{client::response, model::ChannelRss, util::tests::TESTFILES};
+    use crate::{client::response, util::tests::TESTFILES};
 
     use path_macro::path;
     use rstest::rstest;
 
     #[rstest]
-    #[case::base("base")]
-    #[case::no_likes("no_likes")]
-    #[case::no_channel_id("no_channel_id")]
-    fn map_channel_rss(#[case] name: &str) {
+    #[case::base("base", "UCHnyfMqiRRG1u-2MsSQLbXA")]
+    #[case::no_likes("no_likes", "UCdfxp4cUWsWryZOy-o427dw")]
+    #[case::no_channel_id("no_channel_id", "UCHnyfMqiRRG1u-2MsSQLbXA")]
+    #[case::trimmed_channel_id("trimmed_channel_id", "UCHnyfMqiRRG1u-2MsSQLbXA")]
+    fn map_channel_rss(#[case] name: &str, #[case] id: &str) {
         let xml_path = path!(*TESTFILES / "channel_rss" / format!("{}.xml", name));
         let xml_file = File::open(xml_path).unwrap();
 
         let feed: response::ChannelRss =
             quick_xml::de::from_reader(BufReader::new(xml_file)).unwrap();
 
-        let map_res: ChannelRss = feed.into();
-
+        let map_res = feed.map_response(id).unwrap();
         insta::assert_ron_snapshot!(format!("map_channel_rss_{}", name), map_res);
     }
 }

src/client/channel_tv.rs

@@ -1,231 +0,0 @@
-use super::{
-    response,
-    response::video_item::{IsLive, IsShort, IsUpcoming},
-    ClientType, MapResponse, QBrowse, RustyPipeQuery,
-};
-
-use crate::{
-    error::{Error, ExtractionError},
-    model::{ChannelTag, ChannelTv, Verification, VideoItem},
-    param::Language,
-    serializer::MapResult,
-    timeago,
-    util::{self, TryRemove},
-};
-
-impl RustyPipeQuery {
-    /// Get the latest videos of a YouTube channel using the SmartTV client
-    pub async fn channel_tv<S: AsRef<str>>(&self, channel_id: S) -> Result<ChannelTv, Error> {
-        let channel_id = channel_id.as_ref();
-        let context = self.get_context(ClientType::TvHtml5, true, None).await;
-        let request_body = QBrowse {
-            browse_id: channel_id,
-            context,
-        };
-
-        self.execute_request::<response::ChannelTv, _, _>(
-            ClientType::TvHtml5,
-            "channel_tv",
-            channel_id.as_ref(),
-            "browse",
-            &request_body,
-        )
-        .await
-    }
-}
-
-impl MapResponse<ChannelTv> for response::ChannelTv {
-    fn map_response(
-        self,
-        id: &str,
-        lang: Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
-    ) -> Result<MapResult<ChannelTv>, ExtractionError> {
-        // dbg!(&self);
-
-        let cr = self
-            .contents
-            .tv_browse_renderer
-            .content
-            .tv_surface_content_renderer;
-
-        let header = cr.header.tv_surface_header_renderer;
-        let content = cr.content.section_list_renderer.contents;
-
-        let subscribe_btn = header.buttons.into_iter().next();
-        let subscriber_count = subscribe_btn
-            .as_ref()
-            .and_then(|b| b.subscribe_button_renderer.subscriber_count_text.as_deref())
-            .and_then(|txt| util::parse_large_numstr(txt, lang));
-        let channel_id = subscribe_btn
-            .map(|b| b.subscribe_button_renderer.channel_id)
-            .unwrap_or_else(|| id.to_owned());
-
-        let uploads = content.into_iter().find(|shelf| {
-            shelf
-                .shelf_renderer
-                .endpoint
-                .browse_endpoint
-                .as_ref()
-                .map(|ep| ep.params == "EgZ2aWRlb3MYAyAAcADyBgsKCToCCAGiAQIIAQ%3D%3D")
-                .unwrap_or_default()
-        });
-
-        let mut warnings = Vec::new();
-        let videos = uploads
-            .map(|uploads| {
-                let mut items = uploads
-                    .shelf_renderer
-                    .content
-                    .horizontal_list_renderer
-                    .items;
-                warnings.append(&mut items.warnings);
-
-                items
-                    .c
-                    .into_iter()
-                    .filter_map(|v| {
-                        let v = v.tile_renderer;
-
-                        match v.content_type {
-                            response::channel_tv::ContentType::Video => {
-                                let h = v.header.tile_header_renderer;
-                                let mut m = v.metadata.tile_metadata_renderer;
-
-                                let length = h.thumbnail_overlays.first().and_then(|overlay| {
-                                    util::parse_video_length(
-                                        &overlay.thumbnail_overlay_time_status_renderer.text,
-                                    )
-                                });
-
-                                let is_upcoming = h.thumbnail_overlays.is_upcoming();
-
-                                // Normal video:
-                                // Line1: "Channel name", Line2: "View count" "•" "Upload date"
-                                // Current stream:
-                                // Line1: "Channel name", Line2: "10k watching"
-                                // Upcoming stream:
-                                // Line1: "Channel name", Line2: "Scheduled for 4/15/23, 12:00 AM"
-                                let (view_count, publish_date_txt) = m
-                                    .lines
-                                    .try_swap_remove(1)
-                                    .map(|mut line| {
-                                        let date_i = if is_upcoming { 0 } else { 2 };
-
-                                        let view_count =
-                                            line.line_renderer.items.get(0).and_then(|vc| {
-                                                util::parse_large_numstr(
-                                                    &vc.line_item_renderer.text,
-                                                    lang,
-                                                )
-                                            });
-                                        let publish_date_txt = line
-                                            .line_renderer
-                                            .items
-                                            .try_swap_remove(date_i)
-                                            .map(|dt| dt.line_item_renderer.text);
-                                        (view_count, publish_date_txt)
-                                    })
-                                    .unwrap_or_default();
-
-                                let publish_date = publish_date_txt.as_deref().and_then(|txt| {
-                                    if is_upcoming {
-                                        timeago::parse_datetime_or_warn(lang, txt, &mut warnings)
-                                    } else {
-                                        timeago::parse_textual_date_or_warn(
-                                            lang,
-                                            txt,
-                                            &mut warnings,
-                                        )
-                                    }
-                                });
-
-                                Some(VideoItem {
-                                    id: v.content_id,
-                                    name: m.title,
-                                    length,
-                                    thumbnail: h.thumbnail.into(),
-                                    channel: Some(ChannelTag {
-                                        id: channel_id.to_owned(),
-                                        name: header.title.to_owned(),
-                                        avatar: Vec::new(),
-                                        verification: Verification::None,
-                                        subscriber_count,
-                                    }),
-                                    publish_date,
-                                    publish_date_txt,
-                                    view_count,
-                                    is_live: h.thumbnail_overlays.is_live(),
-                                    is_short: h.thumbnail_overlays.is_short(),
-                                    is_upcoming,
-                                    short_description: None,
-                                })
-                            }
-                            _ => None,
-                        }
-                    })
-                    .collect()
-            })
-            .unwrap_or_default();
-
-        Ok(MapResult {
-            c: ChannelTv {
-                id: channel_id,
-                name: header.title,
-                subscriber_count,
-                avatar: header.thumbnail.into(),
-                tv_banner: header.banner.into(),
-                videos,
-                visitor_data: self.response_context.visitor_data,
-            },
-            warnings,
-        })
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::{fs::File, io::BufReader};
-
-    use path_macro::path;
-    use rstest::rstest;
-
-    use crate::{
-        client::{response, MapResponse},
-        model::ChannelTv,
-        param::Language,
-        serializer::MapResult,
-        util::tests::TESTFILES,
-    };
-
-    #[rstest]
-    #[case::base("base", "UCXuqSBlHAE6Xw-yeJA0Tunw")]
-    #[case::music("music", "UC_vmjW5e1xEHhYjY2a0kK1A")]
-    #[case::live("live", "UCSJ4gkVC6NrvII8umztf0Ow")]
-    #[case::live_upcoming("live_upcoming", "UC9CoZyztR-8Xok8Pptzpq1Q")]
-    #[case::onevideo("onevideo", "UCAkeE1thnToEXZTao-CZkHw")]
-    fn map_channel_videos(#[case] name: &str, #[case] id: &str) {
-        let json_path = path!(*TESTFILES / "channel_tv" / format!("{name}.json"));
-        let json_file = File::open(json_path).unwrap();
-
-        let channel: response::ChannelTv =
-            serde_json::from_reader(BufReader::new(json_file)).unwrap();
-        let map_res: MapResult<ChannelTv> = channel.map_response(id, Language::En, None).unwrap();
-
-        assert!(
-            map_res.warnings.is_empty(),
-            "deserialization/mapping warnings: {:?}",
-            map_res.warnings
-        );
-
-        if name == "live_upcoming" {
-            insta::assert_ron_snapshot!(format!("map_channel_{name}"), map_res.c, {
-                ".videos[1:].publish_date" => "[date]",
-            });
-        } else {
-            insta::assert_ron_snapshot!(format!("map_channel_{name}"), map_res.c, {
-                ".videos[].publish_date" => "[date]",
-            });
-        }
-    }
-}

src/client/music_artist.rs

@@ -1,19 +1,27 @@
-use std::{borrow::Cow, rc::Rc};
+use std::borrow::Cow;
 
-use futures::{stream, StreamExt};
 use once_cell::sync::Lazy;
 use regex::Regex;
+use tracing::debug;
 
 use crate::{
+    client::{
+        response::{music_item::map_album_type, url_endpoint::NavigationEndpoint},
+        MapRespOptions, QContinuation,
+    },
     error::{Error, ExtractionError},
-    model::{AlbumItem, ArtistId, MusicArtist},
+    model::{
+        paginator::Paginator, traits::FromYtItem, AlbumItem, AlbumType, ArtistId, MusicArtist,
+        MusicItem,
+    },
+    param::{AlbumFilter, AlbumOrder},
     serializer::MapResult,
-    util::{self, TryRemove},
+    util::{self, ProtoBuilder},
 };
 
 use super::{
     response::{self, music_item::MusicListMapper, url_endpoint::PageType},
-    ClientType, MapResponse, QBrowse, QBrowseParams, RustyPipeQuery,
+    ClientType, MapRespCtx, MapResponse, QBrowse, QBrowseParams, RustyPipeQuery,
 };
 
 impl RustyPipeQuery {
@@ -26,119 +34,105 @@ impl RustyPipeQuery {
         all_albums: bool,
     ) -> Result<MusicArtist, Error> {
         let artist_id = artist_id.as_ref();
-        let visitor_data = match all_albums {
-            true => Some(self.get_ytm_visitor_data().await?),
-            false => None,
-        };
-
-        let res = self._music_artist(artist_id, visitor_data.as_deref()).await;
+        let res = self._music_artist(artist_id, all_albums).await;
 
         if let Err(Error::Extraction(ExtractionError::Redirect(id))) = res {
-            log::debug!("music artist {} redirects to {}", artist_id, &id);
-            self._music_artist(&id, visitor_data.as_deref()).await
+            debug!("music artist {} redirects to {}", artist_id, &id);
+            self._music_artist(&id, all_albums).await
         } else {
             res
         }
     }
 
-    async fn _music_artist(
-        &self,
-        artist_id: &str,
-        all_albums_vdata: Option<&str>,
-    ) -> Result<MusicArtist, Error> {
-        match all_albums_vdata {
-            Some(visitor_data) => {
-                let context = self
-                    .get_context(ClientType::DesktopMusic, true, Some(visitor_data))
-                    .await;
-                let request_body = QBrowse {
-                    context,
-                    browse_id: artist_id,
-                };
+    async fn _music_artist(&self, artist_id: &str, all_albums: bool) -> Result<MusicArtist, Error> {
+        let request_body = QBrowse {
+            browse_id: artist_id,
+        };
 
-                let (mut artist, album_page_params) = self
-                    .execute_request::<response::MusicArtist, _, _>(
-                        ClientType::DesktopMusic,
-                        "music_artist",
-                        artist_id,
-                        "browse",
-                        &request_body,
-                    )
-                    .await?;
-
-                let visitor_data = Rc::new(visitor_data);
-                let album_page_results = stream::iter(album_page_params)
-                    .map(|params| {
-                        let visitor_data = visitor_data.clone();
-                        async move {
-                            self.music_artist_album_page(artist_id, &params, &visitor_data)
-                                .await
-                        }
-                    })
-                    .buffer_unordered(2)
-                    .collect::<Vec<_>>()
-                    .await;
-
-                for res in album_page_results {
-                    let mut res = res?;
-                    artist.albums.append(&mut res);
-                }
-
-                Ok(artist)
-            }
-            None => {
-                let context = self.get_context(ClientType::DesktopMusic, true, None).await;
-                let request_body = QBrowse {
-                    context,
-                    browse_id: artist_id,
-                };
-
-                self.execute_request::<response::MusicArtist, _, _>(
+        if all_albums {
+            let (mut artist, can_fetch_more) = self
+                .execute_request::<response::MusicArtist, _, _>(
                     ClientType::DesktopMusic,
                     "music_artist",
                     artist_id,
                     "browse",
                     &request_body,
                 )
-                .await
+                .await?;
+
+            if can_fetch_more {
+                artist.albums = self
+                    .music_artist_albums(artist_id, None, Some(AlbumOrder::Recency))
+                    .await?;
             }
+
+            Ok(artist)
+        } else {
+            self.execute_request::<response::MusicArtist, _, _>(
+                ClientType::DesktopMusic,
+                "music_artist",
+                artist_id,
+                "browse",
+                &request_body,
+            )
+            .await
         }
     }
 
-    async fn music_artist_album_page(
+    /// Get a list of all albums of a YouTube Music artist
+    pub async fn music_artist_albums(
         &self,
         artist_id: &str,
-        params: &str,
-        visitor_data: &str,
+        filter: Option<AlbumFilter>,
+        order: Option<AlbumOrder>,
     ) -> Result<Vec<AlbumItem>, Error> {
-        let context = self
-            .get_context(ClientType::DesktopMusic, true, Some(visitor_data))
-            .await;
         let request_body = QBrowseParams {
-            context,
-            browse_id: artist_id,
-            params,
+            browse_id: &format!("{}{}", util::ARTIST_DISCOGRAPHY_PREFIX, artist_id),
+            params: &albums_param(filter, order),
         };
 
-        self.execute_request::<response::MusicArtistAlbums, _, _>(
-            ClientType::DesktopMusic,
-            "music_artist_albums",
-            artist_id,
-            "browse",
-            &request_body,
-        )
-        .await
+        let first_page = self
+            .execute_request::<response::MusicArtistAlbums, _, _>(
+                ClientType::DesktopMusic,
+                "music_artist_albums",
+                artist_id,
+                "browse",
+                &request_body,
+            )
+            .await?;
+
+        let mut albums = first_page.albums;
+        let mut ctoken = first_page.ctoken;
+
+        while let Some(tkn) = &ctoken {
+            let request_body = QContinuation { continuation: tkn };
+            let resp: Paginator<MusicItem> = self
+                .execute_request_ctx::<response::MusicContinuation, Paginator<MusicItem>, _>(
+                    ClientType::DesktopMusic,
+                    "music_artist_albums_cont",
+                    artist_id,
+                    "browse",
+                    &request_body,
+                    MapRespOptions {
+                        artist: Some(first_page.artist.clone()),
+                        visitor_data: first_page.visitor_data.as_deref(),
+                        ..Default::default()
+                    },
+                )
+                .await?;
+            if resp.items.is_empty() {
+                tracing::warn!("artist albums [{artist_id}] empty continuation");
+            }
+            ctoken = resp.ctoken;
+            albums.extend(resp.items.into_iter().filter_map(AlbumItem::from_ytm_item));
+        }
+        Ok(albums)
     }
 }
 
 impl MapResponse<MusicArtist> for response::MusicArtist {
-    fn map_response(
-        self,
-        id: &str,
-        lang: crate::param::Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
-    ) -> Result<MapResult<MusicArtist>, ExtractionError> {
-        let mapped = map_artist_page(self, id, lang, false)?;
+    fn map_response(self, ctx: &MapRespCtx<'_>) -> Result<MapResult<MusicArtist>, ExtractionError> {
+        let mapped = map_artist_page(self, ctx, false)?;
         Ok(MapResult {
             c: mapped.c.0,
             warnings: mapped.warnings,
@@ -146,26 +140,38 @@ impl MapResponse<MusicArtist> for response::MusicArtist {
     }
 }
 
-impl MapResponse<(MusicArtist, Vec<String>)> for response::MusicArtist {
+impl MapResponse<(MusicArtist, bool)> for response::MusicArtist {
     fn map_response(
         self,
-        id: &str,
-        lang: crate::param::Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
-    ) -> Result<MapResult<(MusicArtist, Vec<String>)>, ExtractionError> {
-        map_artist_page(self, id, lang, true)
+        ctx: &MapRespCtx<'_>,
+    ) -> Result<MapResult<(MusicArtist, bool)>, ExtractionError> {
+        map_artist_page(self, ctx, true)
     }
 }
 
 fn map_artist_page(
     res: response::MusicArtist,
-    id: &str,
-    lang: crate::param::Language,
+    ctx: &MapRespCtx<'_>,
     skip_extendables: bool,
-) -> Result<MapResult<(MusicArtist, Vec<String>)>, ExtractionError> {
-    // dbg!(&res);
+) -> Result<MapResult<(MusicArtist, bool)>, ExtractionError> {
+    let contents = match res.contents {
+        Some(c) => c,
+        None => {
+            if res.microformat.microformat_data_renderer.noindex {
+                return Err(ExtractionError::NotFound {
+                    id: ctx.id.to_owned(),
+                    msg: "no contents".into(),
+                });
+            } else {
+                return Err(ExtractionError::InvalidData("no contents".into()));
+            }
+        }
+    };
 
-    let header = res.header.music_immersive_header_renderer;
+    let header = res
+        .header
+        .ok_or(ExtractionError::InvalidData("no header".into()))?
+        .music_immersive_header_renderer;
 
     if let Some(share) = header.share_endpoint {
         let pb = share.share_entity_endpoint.serialized_share_entity;
@@ -176,33 +182,31 @@ fn map_artist_page(
             .and_then(|pb| util::string_from_pb(pb, 3));
 
         if let Some(share_channel_id) = share_channel_id {
-            if share_channel_id != id {
+            if share_channel_id != ctx.id {
                 return Err(ExtractionError::Redirect(share_channel_id));
             }
         }
     }
 
-    let sections = res
-        .contents
+    let sections = contents
         .single_column_browse_results_renderer
         .contents
         .into_iter()
         .next()
-        .and_then(|tab| tab.tab_renderer.content)
-        .map(|c| c.section_list_renderer.contents)
+        .map(|c| c.tab_renderer.content.section_list_renderer.contents)
         .unwrap_or_default();
 
     let mut mapper = MusicListMapper::with_artist(
-        lang,
+        ctx.lang,
         ArtistId {
-            id: Some(id.to_owned()),
-            name: header.title.to_owned(),
+            id: Some(ctx.id.to_owned()),
+            name: header.title.clone(),
         },
     );
 
     let mut tracks_playlist_id = None;
     let mut videos_playlist_id = None;
-    let mut album_page_params = Vec::new();
+    let mut can_fetch_more = false;
 
     for section in sections {
         match section {
@@ -220,45 +224,56 @@ fn map_artist_page(
                         }
                     }
                 }
-
+                mapper.album_type = AlbumType::Single;
                 mapper.map_response(shelf.contents);
             }
             response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => {
                 let mut extendable_albums = false;
+                mapper.album_type = AlbumType::Single;
                 if let Some(h) = shelf.header {
                     if let Some(button) = h
                         .music_carousel_shelf_basic_header_renderer
                         .more_content_button
                     {
-                        if let Some(bep) =
-                            button.button_renderer.navigation_endpoint.browse_endpoint
+                        if let NavigationEndpoint::Browse {
+                            browse_endpoint, ..
+                        } = button.button_renderer.navigation_endpoint
                         {
-                            if let Some(cfg) = bep.browse_endpoint_context_supported_configs {
-                                match cfg.browse_endpoint_context_music_config.page_type {
-                                    // Music videos
-                                    PageType::Playlist => {
-                                        if videos_playlist_id.is_none() {
-                                            videos_playlist_id = Some(bep.browse_id);
-                                        }
+                            // Music videos
+                            if browse_endpoint
+                                .browse_endpoint_context_supported_configs
+                                .map(|cfg| {
+                                    cfg.browse_endpoint_context_music_config.page_type
+                                        == PageType::Playlist
+                                })
+                                .unwrap_or_default()
+                            {
+                                if videos_playlist_id.is_none() {
+                                    videos_playlist_id = Some(browse_endpoint.browse_id);
+                                }
+                            } else if browse_endpoint
+                                .browse_id
+                                .starts_with(util::ARTIST_DISCOGRAPHY_PREFIX)
+                            {
+                                can_fetch_more = true;
+                                extendable_albums = true;
+                            } else {
+                                // Peek at the first item to determine type
+                                if let Some(response::music_item::MusicResponseItem::MusicTwoRowItemRenderer(item)) = shelf.contents.c.first() {
+                                    if let Some(PageType::Album) = item.navigation_endpoint.page_type() {
+                                        can_fetch_more = true;
+                                        extendable_albums = true;
                                     }
-                                    // Albums or playlists
-                                    PageType::Artist => {
-                                        // Peek at the first item to determine type
-                                        if let Some(response::music_item::MusicResponseItem::MusicTwoRowItemRenderer(item)) = shelf.contents.c.first() {
-                                            if let Some(PageType::Album) = item.navigation_endpoint.browse_endpoint.as_ref().and_then(|be| {
-                                                be.browse_endpoint_context_supported_configs.as_ref().map(|config| {
-                                                        config.browse_endpoint_context_music_config.page_type
-                                                })}) {
-                                                    album_page_params.push(bep.params);
-                                                    extendable_albums = true;
-                                                }
-                                        }
-                                    }
-                                    _ => {}
                                 }
                             }
                         }
                     }
+                    mapper.album_type = map_album_type(
+                        h.music_carousel_shelf_basic_header_renderer
+                            .title
+                            .first_str(),
+                        ctx.lang,
+                    );
                 }
 
                 if !skip_extendables || !extendable_albums {
@@ -269,7 +284,7 @@ fn map_artist_page(
         }
     }
 
-    let mapped = mapper.group_items();
+    let mut mapped = mapper.group_items();
 
     static WIKIPEDIA_REGEX: Lazy<Regex> =
         Lazy::new(|| Regex::new(r"\(?https://[a-z\d-]+\.wikipedia.org/wiki/[^\s]+").unwrap());
@@ -287,24 +302,27 @@ fn map_artist_page(
     });
 
     let radio_id = header.start_radio_button.and_then(|b| {
-        b.button_renderer
-            .navigation_endpoint
-            .watch_endpoint
-            .and_then(|w| w.playlist_id)
+        if let NavigationEndpoint::Watch { watch_endpoint } = b.button_renderer.navigation_endpoint
+        {
+            watch_endpoint.playlist_id
+        } else {
+            None
+        }
     });
 
     Ok(MapResult {
         c: (
             MusicArtist {
-                id: id.to_owned(),
+                id: ctx.id.to_owned(),
                 name: header.title,
                 header_image: header.thumbnail.into(),
                 description: header.description,
                 wikipedia_url,
                 subscriber_count: header.subscription_button.and_then(|btn| {
-                    util::parse_large_numstr(
+                    util::parse_large_numstr_or_warn(
                         &btn.subscribe_button_renderer.subscriber_count_text,
-                        lang,
+                        ctx.lang,
+                        &mut mapped.warnings,
                     )
                 }),
                 tracks: mapped.c.tracks,
@@ -315,51 +333,94 @@ fn map_artist_page(
                 videos_playlist_id,
                 radio_id,
             },
-            album_page_params,
+            can_fetch_more,
         ),
         warnings: mapped.warnings,
     })
 }
 
-impl MapResponse<Vec<AlbumItem>> for response::MusicArtistAlbums {
+#[derive(Debug)]
+struct FirstAlbumPage {
+    albums: Vec<AlbumItem>,
+    ctoken: Option<String>,
+    artist: ArtistId,
+    visitor_data: Option<String>,
+}
+
+impl MapResponse<FirstAlbumPage> for response::MusicArtistAlbums {
     fn map_response(
         self,
-        id: &str,
-        lang: crate::param::Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
-    ) -> Result<MapResult<Vec<AlbumItem>>, ExtractionError> {
-        // dbg!(&self);
+        ctx: &MapRespCtx<'_>,
+    ) -> Result<MapResult<FirstAlbumPage>, ExtractionError> {
+        let Some(header) = self.header else {
+            return Err(ExtractionError::NotFound {
+                id: ctx.id.into(),
+                msg: "no header".into(),
+            });
+        };
 
-        let mut content = self.contents.single_column_browse_results_renderer.contents;
-        let grids = content
-            .try_swap_remove(0)
+        let grids = self
+            .contents
+            .single_column_browse_results_renderer
+            .contents
+            .into_iter()
+            .next()
             .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))?
             .tab_renderer
             .content
             .section_list_renderer
             .contents;
 
-        let mut mapper = MusicListMapper::with_artist(
-            lang,
-            ArtistId {
-                id: Some(id.to_owned()),
-                name: self.header.music_header_renderer.title,
-            },
-        );
-
+        let artist_id = ArtistId {
+            id: Some(ctx.id.to_owned()),
+            name: header.music_header_renderer.title,
+        };
+        let mut mapper = MusicListMapper::with_artist(ctx.lang, artist_id.clone());
+        let mut ctoken = None;
         for grid in grids {
             mapper.map_response(grid.grid_renderer.items);
+            if ctoken.is_none() {
+                ctoken = grid
+                    .grid_renderer
+                    .continuations
+                    .into_iter()
+                    .next()
+                    .map(|g| g.next_continuation_data.continuation);
+            }
         }
 
         let mapped = mapper.group_items();
 
         Ok(MapResult {
-            c: mapped.c.albums,
+            c: FirstAlbumPage {
+                albums: mapped.c.albums,
+                ctoken,
+                artist: artist_id,
+                visitor_data: ctx.visitor_data.map(str::to_owned),
+            },
             warnings: mapped.warnings,
         })
     }
 }
 
+fn albums_param(filter: Option<AlbumFilter>, order: Option<AlbumOrder>) -> String {
+    let mut pb_filter = ProtoBuilder::new();
+    if let Some(filter) = filter {
+        pb_filter.varint(1, filter as u64);
+    }
+    if let Some(order) = order {
+        pb_filter.varint(2, order as u64);
+    }
+    pb_filter.bytes(3, &[1, 2]);
+
+    let mut pb_48 = ProtoBuilder::new();
+    pb_48.embedded(15, pb_filter);
+
+    let mut pb_3 = ProtoBuilder::new();
+    pb_3.embedded(48, pb_48);
+    pb_3.to_base64()
+}
+
 #[cfg(test)]
 mod tests {
     use std::{fs::File, io::BufReader};
@@ -367,55 +428,75 @@ mod tests {
     use path_macro::path;
     use rstest::rstest;
 
-    use crate::{param::Language, util::tests::TESTFILES};
+    use crate::util::tests::TESTFILES;
 
     use super::*;
 
     #[rstest]
     #[case::default("default", "UClmXPfaYhXOYsNn_QUyheWQ")]
-    #[case::no_more_albums("no_more_albums", "UC_vmjW5e1xEHhYjY2a0kK1A")]
     #[case::only_singles("only_singles", "UCfwCE5VhPMGxNPFxtVv7lRw")]
     #[case::no_artist("no_artist", "UCh8gHdtzO2tXd593_bjErWg")]
     #[case::only_more_singles("only_more_singles", "UC0aXrjVxG5pZr99v77wZdPQ")]
+    #[case::grouped_albums("20250113_grouped_albums", "UCOR4_bSVIXPsGa4BbCSt60Q")]
     fn map_music_artist(#[case] name: &str, #[case] id: &str) {
         let json_path = path!(*TESTFILES / "music_artist" / format!("artist_{name}.json"));
         let json_file = File::open(json_path).unwrap();
 
-        let mut album_page_paths = Vec::new();
-        for i in 1..=2 {
-            let json_path = path!(*TESTFILES / "music_artist" / format!("artist_{name}_{i}.json"));
-            if !json_path.exists() {
-                break;
-            }
-            album_page_paths.push(json_path);
+        let mut album_page_path = None;
+        let json_path = path!(*TESTFILES / "music_artist" / format!("artist_{name}_1.json"));
+        if json_path.exists() {
+            album_page_path = Some(json_path);
         }
 
         let resp: response::MusicArtist =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
-        let map_res: MapResult<(MusicArtist, Vec<String>)> =
-            resp.map_response(id, Language::En, None).unwrap();
-        let (mut artist, album_page_params) = map_res.c;
+        let map_res: MapResult<(MusicArtist, bool)> =
+            resp.map_response(&MapRespCtx::test(id)).unwrap();
+        let (mut artist, can_fetch_more) = map_res.c;
 
         assert!(
             map_res.warnings.is_empty(),
             "deserialization/mapping warnings: {:?}",
             map_res.warnings
         );
-        assert_eq!(album_page_params.len(), album_page_paths.len());
+        assert_eq!(can_fetch_more, album_page_path.is_some());
 
-        for json_path in album_page_paths {
-            let json_file = File::open(json_path).unwrap();
+        // Album overview
+        if let Some(album_page_path) = album_page_path {
+            let json_file = File::open(album_page_path).unwrap();
             let resp: response::MusicArtistAlbums =
                 serde_json::from_reader(BufReader::new(json_file)).unwrap();
-            let mut map_res: MapResult<Vec<AlbumItem>> =
-                resp.map_response(id, Language::En, None).unwrap();
+            let map_res: MapResult<FirstAlbumPage> =
+                resp.map_response(&MapRespCtx::test(id)).unwrap();
 
             assert!(
                 map_res.warnings.is_empty(),
                 "deserialization/mapping warnings: {:?}",
                 map_res.warnings
             );
-            artist.albums.append(&mut map_res.c);
+            artist.albums = map_res.c.albums;
+
+            // Album overview continuation
+            for i in 2..10 {
+                let cont_path =
+                    path!(*TESTFILES / "music_artist" / format!("artist_{name}_{i}.json"));
+                if !cont_path.is_file() {
+                    break;
+                }
+                let json_file = File::open(cont_path).unwrap();
+                let resp: response::MusicContinuation =
+                    serde_json::from_reader(BufReader::new(json_file)).unwrap();
+                let map_res: MapResult<Paginator<MusicItem>> =
+                    resp.map_response(&MapRespCtx::test(id)).unwrap();
+                assert!(!map_res.c.items.is_empty());
+                artist.albums.extend(
+                    map_res
+                        .c
+                        .items
+                        .into_iter()
+                        .filter_map(AlbumItem::from_ytm_item),
+                );
+            }
         }
 
         insta::assert_ron_snapshot!(format!("map_music_artist_{name}"), artist);
@@ -429,7 +510,7 @@ mod tests {
         let artist: response::MusicArtist =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
         let map_res: MapResult<MusicArtist> = artist
-            .map_response("UClmXPfaYhXOYsNn_QUyheWQ", Language::En, None)
+            .map_response(&MapRespCtx::test("UClmXPfaYhXOYsNn_QUyheWQ"))
             .unwrap();
 
         assert!(
@@ -448,12 +529,12 @@ mod tests {
         let artist: response::MusicArtist =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
         let res: Result<MapResult<MusicArtist>, ExtractionError> =
-            artist.map_response("UCLkAepWjdylmXSltofFvsYQ", Language::En, None);
+            artist.map_response(&MapRespCtx::test("UCLkAepWjdylmXSltofFvsYQ"));
         let e = res.unwrap_err();
 
         match e {
             ExtractionError::Redirect(id) => {
-                assert_eq!(id, "UCOR4_bSVIXPsGa4BbCSt60Q")
+                assert_eq!(id, "UCOR4_bSVIXPsGa4BbCSt60Q");
             }
             _ => panic!("error: {e}"),
         }

src/client/music_charts.rs

@@ -11,13 +11,12 @@ use crate::{
 
 use super::{
     response::{self, music_item::MusicListMapper, url_endpoint::MusicPageType},
-    ClientType, MapResponse, RustyPipeQuery, YTContext,
+    ClientType, MapRespCtx, MapResponse, RustyPipeQuery,
 };
 
 #[derive(Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
 struct QCharts<'a> {
-    context: YTContext<'a>,
     browse_id: &'a str,
     params: &'a str,
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -32,10 +31,9 @@ struct FormData {
 
 impl RustyPipeQuery {
     /// Get the YouTube Music charts for a given country
+    #[tracing::instrument(skip(self), level = "error")]
     pub async fn music_charts(&self, country: Option<Country>) -> Result<MusicCharts, Error> {
-        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
         let request_body = QCharts {
-            context,
             browse_id: "FEmusic_charts",
             params: "sgYPRkVtdXNpY19leHBsb3Jl",
             form_data: country.map(|c| FormData {
@@ -55,12 +53,7 @@ impl RustyPipeQuery {
 }
 
 impl MapResponse<MusicCharts> for response::MusicCharts {
-    fn map_response(
-        self,
-        _id: &str,
-        lang: crate::param::Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
-    ) -> Result<crate::serializer::MapResult<MusicCharts>, crate::error::ExtractionError> {
+    fn map_response(self, ctx: &MapRespCtx<'_>) -> Result<MapResult<MusicCharts>, ExtractionError> {
         let countries = self
             .framework_updates
             .map(|fwu| {
@@ -75,9 +68,9 @@ impl MapResponse<MusicCharts> for response::MusicCharts {
         let mut top_playlist_id = None;
         let mut trending_playlist_id = None;
 
-        let mut mapper_top = MusicListMapper::new(lang);
-        let mut mapper_trending = MusicListMapper::new(lang);
-        let mut mapper_other = MusicListMapper::new(lang);
+        let mut mapper_top = MusicListMapper::new(ctx.lang);
+        let mut mapper_trending = MusicListMapper::new(ctx.lang);
+        let mut mapper_other = MusicListMapper::new(ctx.lang);
 
         self.contents
             .single_column_browse_results_renderer
@@ -96,8 +89,9 @@ impl MapResponse<MusicCharts> for response::MusicCharts {
                         h.music_carousel_shelf_basic_header_renderer
                             .more_content_button
                             .and_then(|btn| btn.button_renderer.navigation_endpoint.music_page())
+                            .map(|mp| (mp.typ, mp.id))
                     }) {
-                        Some((MusicPageType::Playlist, id)) => {
+                        Some((MusicPageType::Playlist { .. }, id)) => {
                             // Top music videos (first shelf with associated playlist)
                             if top_playlist_id.is_none() {
                                 mapper_top.map_response(shelf.contents);
@@ -119,12 +113,12 @@ impl MapResponse<MusicCharts> for response::MusicCharts {
             });
 
         let mapped_top = mapper_top.conv_items::<TrackItem>();
-        let mut mapped_trending = mapper_trending.conv_items::<TrackItem>();
-        let mut mapped_other = mapper_other.group_items();
+        let mapped_trending = mapper_trending.conv_items::<TrackItem>();
+        let mapped_other = mapper_other.group_items();
 
         let mut warnings = mapped_top.warnings;
-        warnings.append(&mut mapped_trending.warnings);
-        warnings.append(&mut mapped_other.warnings);
+        warnings.extend(mapped_trending.warnings);
+        warnings.extend(mapped_other.warnings);
 
         Ok(MapResult {
             c: MusicCharts {
@@ -148,7 +142,6 @@ mod tests {
     use rstest::rstest;
 
     use super::*;
-    use crate::param::Language;
 
     #[rstest]
     #[case::default("global")]
@@ -160,7 +153,7 @@ mod tests {
 
         let charts: response::MusicCharts =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
-        let map_res: MapResult<MusicCharts> = charts.map_response("", Language::En, None).unwrap();
+        let map_res: MapResult<MusicCharts> = charts.map_response(&MapRespCtx::test("")).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),

src/client/music_details.rs

@@ -1,11 +1,13 @@
-use std::borrow::Cow;
+use std::{borrow::Cow, fmt::Debug};
 
 use serde::Serialize;
 
 use crate::{
     error::{Error, ExtractionError},
-    model::{paginator::Paginator, ArtistId, Lyrics, MusicRelated, TrackDetails, TrackItem},
-    param::Language,
+    model::{
+        paginator::{ContinuationEndpoint, Paginator},
+        ArtistId, Lyrics, MusicRelated, TrackDetails, TrackItem,
+    },
     serializer::MapResult,
 };
 
@@ -14,12 +16,11 @@ use super::{
         self,
         music_item::{map_queue_item, MusicListMapper},
     },
-    ClientType, MapResponse, QBrowse, RustyPipeQuery, YTContext,
+    ClientType, MapRespCtx, MapResponse, QBrowse, RustyPipeQuery,
 };
 
 #[derive(Debug, Serialize)]
 struct QMusicDetails<'a> {
-    context: YTContext<'a>,
     video_id: &'a str,
     enable_persistent_playlist_panel: bool,
     is_audio_only: bool,
@@ -28,7 +29,6 @@ struct QMusicDetails<'a> {
 
 #[derive(Debug, Serialize)]
 struct QRadio<'a> {
-    context: YTContext<'a>,
     playlist_id: &'a str,
     params: &'a str,
     enable_persistent_playlist_panel: bool,
@@ -37,12 +37,14 @@ struct QRadio<'a> {
 }
 
 impl RustyPipeQuery {
-    /// Get the metadata of a YouTube music track
-    pub async fn music_details<S: AsRef<str>>(&self, video_id: S) -> Result<TrackDetails, Error> {
+    /// Get the metadata of a YouTube Music track
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn music_details<S: AsRef<str> + Debug>(
+        &self,
+        video_id: S,
+    ) -> Result<TrackDetails, Error> {
         let video_id = video_id.as_ref();
-        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
         let request_body = QMusicDetails {
-            context,
             video_id,
             enable_persistent_playlist_panel: true,
             is_audio_only: true,
@@ -59,14 +61,13 @@ impl RustyPipeQuery {
         .await
     }
 
-    /// Get the lyrics of a YouTube music track
+    /// Get the lyrics of a YouTube Music track
     ///
     /// The `lyrics_id` has to be obtained using [`RustyPipeQuery::music_details`].
-    pub async fn music_lyrics<S: AsRef<str>>(&self, lyrics_id: S) -> Result<Lyrics, Error> {
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn music_lyrics<S: AsRef<str> + Debug>(&self, lyrics_id: S) -> Result<Lyrics, Error> {
         let lyrics_id = lyrics_id.as_ref();
-        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
         let request_body = QBrowse {
-            context,
             browse_id: lyrics_id,
         };
 
@@ -83,11 +84,13 @@ impl RustyPipeQuery {
     /// Get related items (tracks, playlists, artists) to a YouTube Music track
     ///
     /// The `related_id` has to be obtained using [`RustyPipeQuery::music_details`].
-    pub async fn music_related<S: AsRef<str>>(&self, related_id: S) -> Result<MusicRelated, Error> {
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn music_related<S: AsRef<str> + Debug>(
+        &self,
+        related_id: S,
+    ) -> Result<MusicRelated, Error> {
         let related_id = related_id.as_ref();
-        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
         let request_body = QBrowse {
-            context,
             browse_id: related_id,
         };
 
@@ -104,17 +107,13 @@ impl RustyPipeQuery {
     /// Get a YouTube Music radio (a dynamically generated playlist)
     ///
     /// The `radio_id` can be obtained using [`RustyPipeQuery::music_artist`] to get an artist's radio.
-    pub async fn music_radio<S: AsRef<str>>(
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn music_radio<S: AsRef<str> + Debug>(
         &self,
         radio_id: S,
     ) -> Result<Paginator<TrackItem>, Error> {
         let radio_id = radio_id.as_ref();
-        let visitor_data = self.get_ytm_visitor_data().await?;
-        let context = self
-            .get_context(ClientType::DesktopMusic, true, Some(&visitor_data))
-            .await;
         let request_body = QRadio {
-            context,
             playlist_id: radio_id,
             params: "wAEB8gECeAE%3D",
             enable_persistent_playlist_panel: true,
@@ -133,7 +132,8 @@ impl RustyPipeQuery {
     }
 
     /// Get a YouTube Music radio (a dynamically generated playlist) for a track
-    pub async fn music_radio_track<S: AsRef<str>>(
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn music_radio_track<S: AsRef<str> + Debug>(
         &self,
         video_id: S,
     ) -> Result<Paginator<TrackItem>, Error> {
@@ -142,7 +142,8 @@ impl RustyPipeQuery {
     }
 
     /// Get a YouTube Music radio (a dynamically generated playlist) for a playlist
-    pub async fn music_radio_playlist<S: AsRef<str>>(
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn music_radio_playlist<S: AsRef<str> + Debug>(
         &self,
         playlist_id: S,
     ) -> Result<Paginator<TrackItem>, Error> {
@@ -154,9 +155,7 @@ impl RustyPipeQuery {
 impl MapResponse<TrackDetails> for response::MusicDetails {
     fn map_response(
         self,
-        id: &str,
-        lang: Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
+        ctx: &MapRespCtx<'_>,
     ) -> Result<MapResult<TrackDetails>, ExtractionError> {
         let tabs = self
             .contents
@@ -193,9 +192,10 @@ impl MapResponse<TrackDetails> for response::MusicDetails {
             }
         }
 
-        let content = content.ok_or(ExtractionError::ContentUnavailable(Cow::Borrowed(
-            "track not found",
-        )))?;
+        let content = content.ok_or_else(|| ExtractionError::NotFound {
+            id: ctx.id.to_owned(),
+            msg: "no content".into(),
+        })?;
         let track_item = content
             .contents
             .c
@@ -207,22 +207,18 @@ impl MapResponse<TrackDetails> for response::MusicDetails {
                 response::music_item::PlaylistPanelVideo::None => None,
             })
             .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no video item")))?;
-        let track = map_queue_item(track_item, lang);
+        let mut track = map_queue_item(track_item, ctx.lang);
 
-        if track.id != id {
-            return Err(ExtractionError::WrongResult(format!(
-                "got wrong video id {}, expected {}",
-                track.id, id
-            )));
-        }
+        let mut warnings = content.contents.warnings;
+        warnings.append(&mut track.warnings);
 
         Ok(MapResult {
             c: TrackDetails {
-                track,
+                track: track.c,
                 lyrics_id,
                 related_id,
             },
-            warnings: content.contents.warnings,
+            warnings,
         })
     }
 }
@@ -230,9 +226,7 @@ impl MapResponse<TrackDetails> for response::MusicDetails {
 impl MapResponse<Paginator<TrackItem>> for response::MusicDetails {
     fn map_response(
         self,
-        _id: &str,
-        lang: Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
+        ctx: &MapRespCtx<'_>,
     ) -> Result<MapResult<Paginator<TrackItem>>, ExtractionError> {
         let tabs = self
             .contents
@@ -244,20 +238,25 @@ impl MapResponse<Paginator<TrackItem>> for response::MusicDetails {
         let content = tabs
             .into_iter()
             .find_map(|t| t.tab_renderer.content)
-            .ok_or(ExtractionError::ContentUnavailable(Cow::Borrowed(
-                "radio unavailable",
-            )))?
+            .ok_or_else(|| ExtractionError::NotFound {
+                id: ctx.id.to_owned(),
+                msg: "no content".into(),
+            })?
             .music_queue_renderer
             .content
             .playlist_panel_renderer;
 
+        let mut warnings = content.contents.warnings;
+
         let tracks = content
             .contents
             .c
             .into_iter()
             .filter_map(|item| match item {
                 response::music_item::PlaylistPanelVideo::PlaylistPanelVideoRenderer(item) => {
-                    Some(map_queue_item(item, lang))
+                    let mut track = map_queue_item(item, ctx.lang);
+                    warnings.append(&mut track.warnings);
+                    Some(track.c)
                 }
                 response::music_item::PlaylistPanelVideo::None => None,
             })
@@ -275,32 +274,26 @@ impl MapResponse<Paginator<TrackItem>> for response::MusicDetails {
                 tracks,
                 ctoken,
                 None,
-                crate::model::paginator::ContinuationEndpoint::MusicNext,
+                ContinuationEndpoint::MusicNext,
+                false,
             ),
-            warnings: content.contents.warnings,
+            warnings,
         })
     }
 }
 
 impl MapResponse<Lyrics> for response::MusicLyrics {
-    fn map_response(
-        self,
-        _id: &str,
-        _lang: Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
-    ) -> Result<MapResult<Lyrics>, ExtractionError> {
+    fn map_response(self, ctx: &MapRespCtx<'_>) -> Result<MapResult<Lyrics>, ExtractionError> {
         let lyrics = self
             .contents
-            .section_list_renderer
-            .and_then(|sl| {
-                sl.contents
-                    .into_iter()
-                    .find_map(|item| item.music_description_shelf_renderer)
-            })
-            .ok_or(match self.contents.message_renderer {
-                Some(msg) => ExtractionError::ContentUnavailable(Cow::Owned(msg.text)),
-                None => ExtractionError::InvalidData(Cow::Borrowed("no content")),
-            })?;
+            .into_res()
+            .map_err(|msg| ExtractionError::NotFound {
+                id: ctx.id.to_owned(),
+                msg: msg.into(),
+            })?
+            .into_iter()
+            .find_map(|item| item.music_description_shelf_renderer)
+            .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))?;
 
         Ok(MapResult {
             c: Lyrics {
@@ -315,43 +308,44 @@ impl MapResponse<Lyrics> for response::MusicLyrics {
 impl MapResponse<MusicRelated> for response::MusicRelated {
     fn map_response(
         self,
-        _id: &str,
-        lang: Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
+        ctx: &MapRespCtx<'_>,
     ) -> Result<MapResult<MusicRelated>, ExtractionError> {
-        // Find artist
-        let artist_id = self
+        let contents = self
             .contents
-            .section_list_renderer
-            .contents
-            .iter()
-            .find_map(|section| match section {
-                response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => {
-                    shelf.header.as_ref().and_then(|h| {
-                        h.music_carousel_shelf_basic_header_renderer
-                            .title
-                            .0
-                            .iter()
-                            .find_map(|c| {
-                                let artist = ArtistId::from(c.clone());
-                                if artist.id.is_some() {
-                                    Some(artist)
-                                } else {
-                                    None
-                                }
-                            })
-                    })
-                }
-                _ => None,
-            });
+            .into_res()
+            .map_err(|msg| ExtractionError::NotFound {
+                id: ctx.id.to_owned(),
+                msg: msg.into(),
+            })?;
 
-        let mut mapper_tracks = MusicListMapper::new(lang);
+        // Find artist
+        let artist_id = contents.iter().find_map(|section| match section {
+            response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => {
+                shelf.header.as_ref().and_then(|h| {
+                    h.music_carousel_shelf_basic_header_renderer
+                        .title
+                        .0
+                        .iter()
+                        .find_map(|c| {
+                            let artist = ArtistId::from(c.clone());
+                            if artist.id.is_some() {
+                                Some(artist)
+                            } else {
+                                None
+                            }
+                        })
+                })
+            }
+            _ => None,
+        });
+
+        let mut mapper_tracks = MusicListMapper::new(ctx.lang);
         let mut mapper = match artist_id {
-            Some(artist_id) => MusicListMapper::with_artist(lang, artist_id),
-            None => MusicListMapper::new(lang),
+            Some(artist_id) => MusicListMapper::with_artist(ctx.lang, artist_id),
+            None => MusicListMapper::new(ctx.lang),
         };
 
-        let mut sections = self.contents.section_list_renderer.contents.into_iter();
+        let mut sections = contents.into_iter();
         if let Some(response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf)) =
             sections.next()
         {
@@ -395,7 +389,7 @@ mod tests {
     use rstest::rstest;
 
     use super::*;
-    use crate::{model, param::Language, util::tests::TESTFILES};
+    use crate::{model, util::tests::TESTFILES};
 
     #[rstest]
     #[case::mv("mv", "ZeerrnuLi5E")]
@@ -407,7 +401,7 @@ mod tests {
         let details: response::MusicDetails =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
         let map_res: MapResult<model::TrackDetails> =
-            details.map_response(id, Language::En, None).unwrap();
+            details.map_response(&MapRespCtx::test(id)).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),
@@ -427,7 +421,7 @@ mod tests {
         let radio: response::MusicDetails =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
         let map_res: MapResult<Paginator<TrackItem>> =
-            radio.map_response(id, Language::En, None).unwrap();
+            radio.map_response(&MapRespCtx::test(id)).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),
@@ -444,7 +438,7 @@ mod tests {
 
         let lyrics: response::MusicLyrics =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
-        let map_res: MapResult<Lyrics> = lyrics.map_response("", Language::En, None).unwrap();
+        let map_res: MapResult<Lyrics> = lyrics.map_response(&MapRespCtx::test("")).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),
@@ -461,7 +455,7 @@ mod tests {
 
         let lyrics: response::MusicRelated =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
-        let map_res: MapResult<MusicRelated> = lyrics.map_response("", Language::En, None).unwrap();
+        let map_res: MapResult<MusicRelated> = lyrics.map_response(&MapRespCtx::test("")).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),

src/client/music_genres.rs

@@ -1,4 +1,4 @@
-use std::borrow::Cow;
+use std::{borrow::Cow, fmt::Debug};
 
 use crate::{
     error::{Error, ExtractionError},
@@ -7,16 +7,15 @@ use crate::{
 };
 
 use super::{
-    response::{self, music_item::MusicListMapper},
-    ClientType, MapResponse, QBrowse, QBrowseParams, RustyPipeQuery,
+    response::{self, music_item::MusicListMapper, url_endpoint::NavigationEndpoint},
+    ClientType, MapRespCtx, MapResponse, QBrowse, QBrowseParams, RustyPipeQuery,
 };
 
 impl RustyPipeQuery {
     /// Get a list of moods and genres from YouTube Music
+    #[tracing::instrument(skip(self), level = "error")]
     pub async fn music_genres(&self) -> Result<Vec<MusicGenreItem>, Error> {
-        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
         let request_body = QBrowse {
-            context,
             browse_id: "FEmusic_moods_and_genres",
         };
 
@@ -31,11 +30,13 @@ impl RustyPipeQuery {
     }
 
     /// Get the playlists from a YouTube Music genre
-    pub async fn music_genre<S: AsRef<str>>(&self, genre_id: S) -> Result<MusicGenre, Error> {
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn music_genre<S: AsRef<str> + Debug>(
+        &self,
+        genre_id: S,
+    ) -> Result<MusicGenre, Error> {
         let genre_id = genre_id.as_ref();
-        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
         let request_body = QBrowseParams {
-            context,
             browse_id: "FEmusic_moods_and_genres_category",
             params: genre_id,
         };
@@ -54,10 +55,8 @@ impl RustyPipeQuery {
 impl MapResponse<Vec<MusicGenreItem>> for response::MusicGenres {
     fn map_response(
         self,
-        _id: &str,
-        _lang: crate::param::Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
-    ) -> Result<crate::serializer::MapResult<Vec<MusicGenreItem>>, ExtractionError> {
+        _ctx: &MapRespCtx<'_>,
+    ) -> Result<MapResult<Vec<MusicGenreItem>>, ExtractionError> {
         let content = self
             .contents
             .single_column_browse_results_renderer
@@ -81,7 +80,7 @@ impl MapResponse<Vec<MusicGenreItem>> for response::MusicGenres {
         let genres = content_iter
             .enumerate()
             .flat_map(|(i, grid)| {
-                let mut grid = grid.grid_renderer.items;
+                let mut grid = grid.grid_renderer.contents;
                 warnings.append(&mut grid.warnings);
                 grid.c.into_iter().filter_map(move |section| match section {
                     response::music_genres::NavigationButton::MusicNavigationButtonRenderer(
@@ -105,14 +104,7 @@ impl MapResponse<Vec<MusicGenreItem>> for response::MusicGenres {
 }
 
 impl MapResponse<MusicGenre> for response::MusicGenre {
-    fn map_response(
-        self,
-        id: &str,
-        lang: crate::param::Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
-    ) -> Result<crate::serializer::MapResult<MusicGenre>, ExtractionError> {
-        // dbg!(&self);
-
+    fn map_response(self, ctx: &MapRespCtx<'_>) -> Result<MapResult<MusicGenre>, ExtractionError> {
         let content = self
             .contents
             .single_column_browse_results_renderer
@@ -144,18 +136,20 @@ impl MapResponse<MusicGenre> for response::MusicGenre {
                             h.music_carousel_shelf_basic_header_renderer
                                 .more_content_button
                                 .and_then(|btn| {
-                                    btn.button_renderer
-                                        .navigation_endpoint
-                                        .browse_endpoint
-                                        .and_then(|browse| {
-                                            if browse.browse_id
-                                                == "FEmusic_moods_and_genres_category"
-                                            {
-                                                Some(browse.params)
-                                            } else {
-                                                None
-                                            }
-                                        })
+                                    if let NavigationEndpoint::Browse {
+                                        browse_endpoint, ..
+                                    } = btn.button_renderer.navigation_endpoint
+                                    {
+                                        if browse_endpoint.browse_id
+                                            == "FEmusic_moods_and_genres_category"
+                                        {
+                                            Some(browse_endpoint.params)
+                                        } else {
+                                            None
+                                        }
+                                    } else {
+                                        None
+                                    }
                                 })
                         }),
                         shelf.contents,
@@ -170,7 +164,7 @@ impl MapResponse<MusicGenre> for response::MusicGenre {
                     _ => return None,
                 };
 
-                let mut mapper = MusicListMapper::new(lang);
+                let mut mapper = MusicListMapper::new(ctx.lang);
                 mapper.map_response(items);
                 let mut mapped = mapper.conv_items();
                 warnings.append(&mut mapped.warnings);
@@ -185,7 +179,7 @@ impl MapResponse<MusicGenre> for response::MusicGenre {
 
         Ok(MapResult {
             c: MusicGenre {
-                id: id.to_owned(),
+                id: ctx.id.to_owned(),
                 name: self.header.music_header_renderer.title,
                 sections,
             },
@@ -202,7 +196,7 @@ mod tests {
     use rstest::rstest;
 
     use super::*;
-    use crate::{model, param::Language, util::tests::TESTFILES};
+    use crate::{model, util::tests::TESTFILES};
 
     #[test]
     fn map_music_genres() {
@@ -212,7 +206,7 @@ mod tests {
         let playlist: response::MusicGenres =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
         let map_res: MapResult<Vec<model::MusicGenreItem>> =
-            playlist.map_response("", Language::En, None).unwrap();
+            playlist.map_response(&MapRespCtx::test("")).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),
@@ -232,7 +226,7 @@ mod tests {
         let playlist: response::MusicGenre =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
         let map_res: MapResult<model::MusicGenre> =
-            playlist.map_response(id, Language::En, None).unwrap();
+            playlist.map_response(&MapRespCtx::test(id)).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),

src/client/music_new.rs

@@ -4,16 +4,16 @@ use crate::{
     client::response::music_item::MusicListMapper,
     error::{Error, ExtractionError},
     model::{traits::FromYtItem, AlbumItem, TrackItem},
+    serializer::MapResult,
 };
 
-use super::{response, ClientType, MapResponse, QBrowse, RustyPipeQuery};
+use super::{response, ClientType, MapRespCtx, MapResponse, QBrowse, RustyPipeQuery};
 
 impl RustyPipeQuery {
     /// Get the new albums that were released on YouTube Music
+    #[tracing::instrument(skip(self), level = "error")]
     pub async fn music_new_albums(&self) -> Result<Vec<AlbumItem>, Error> {
-        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
         let request_body = QBrowse {
-            context,
             browse_id: "FEmusic_new_releases_albums",
         };
 
@@ -28,10 +28,9 @@ impl RustyPipeQuery {
     }
 
     /// Get the new music videos that were released on YouTube Music
+    #[tracing::instrument(skip(self), level = "error")]
     pub async fn music_new_videos(&self) -> Result<Vec<TrackItem>, Error> {
-        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
         let request_body = QBrowse {
-            context,
             browse_id: "FEmusic_new_releases_videos",
         };
 
@@ -47,12 +46,7 @@ impl RustyPipeQuery {
 }
 
 impl<T: FromYtItem> MapResponse<Vec<T>> for response::MusicNew {
-    fn map_response(
-        self,
-        _id: &str,
-        lang: crate::param::Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
-    ) -> Result<crate::serializer::MapResult<Vec<T>>, ExtractionError> {
+    fn map_response(self, ctx: &MapRespCtx<'_>) -> Result<MapResult<Vec<T>>, ExtractionError> {
         let items = self
             .contents
             .single_column_browse_results_renderer
@@ -70,7 +64,7 @@ impl<T: FromYtItem> MapResponse<Vec<T>> for response::MusicNew {
             .grid_renderer
             .items;
 
-        let mut mapper = MusicListMapper::new(lang);
+        let mut mapper = MusicListMapper::new(ctx.lang);
         mapper.map_response(items);
 
         Ok(mapper.conv_items())
@@ -85,7 +79,7 @@ mod tests {
     use rstest::rstest;
 
     use super::*;
-    use crate::{param::Language, serializer::MapResult, util::tests::TESTFILES};
+    use crate::{serializer::MapResult, util::tests::TESTFILES};
 
     #[rstest]
     #[case::default("default")]
@@ -96,7 +90,7 @@ mod tests {
         let new_albums: response::MusicNew =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
         let map_res: MapResult<Vec<AlbumItem>> =
-            new_albums.map_response("", Language::En, None).unwrap();
+            new_albums.map_response(&MapRespCtx::test("")).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),
@@ -108,14 +102,15 @@ mod tests {
 
     #[rstest]
     #[case::default("default")]
+    #[case::default("w_podcasts")]
     fn map_music_new_videos(#[case] name: &str) {
         let json_path = path!(*TESTFILES / "music_new" / format!("videos_{name}.json"));
         let json_file = File::open(json_path).unwrap();
 
-        let new_albums: response::MusicNew =
+        let new_videos: response::MusicNew =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
         let map_res: MapResult<Vec<TrackItem>> =
-            new_albums.map_response("", Language::En, None).unwrap();
+            new_videos.map_response(&MapRespCtx::test("")).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),

src/client/music_playlist.rs

@@ -1,30 +1,36 @@
-use std::borrow::Cow;
+use std::{borrow::Cow, fmt::Debug};
 
 use crate::{
+    client::response::url_endpoint::NavigationEndpoint,
     error::{Error, ExtractionError},
-    model::{paginator::Paginator, AlbumId, ChannelId, MusicAlbum, MusicPlaylist, TrackItem},
-    serializer::MapResult,
-    util::{self, TryRemove},
+    model::{
+        paginator::{ContinuationEndpoint, Paginator},
+        richtext::RichText,
+        AlbumId, ChannelId, MusicAlbum, MusicPlaylist, TrackItem, TrackType,
+    },
+    serializer::{text::TextComponents, MapResult},
+    util::{self, dictionary, TryRemove, DOT_SEPARATOR},
 };
 
+use self::response::url_endpoint::MusicPageType;
+
 use super::{
     response::{
         self,
         music_item::{map_album_type, map_artist_id, map_artists, MusicListMapper},
     },
-    ClientType, MapResponse, QBrowse, RustyPipeQuery,
+    ClientType, MapRespCtx, MapResponse, QBrowse, RustyPipeQuery,
 };
 
 impl RustyPipeQuery {
     /// Get a playlist from YouTube Music
-    pub async fn music_playlist<S: AsRef<str>>(
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn music_playlist<S: AsRef<str> + Debug>(
         &self,
         playlist_id: S,
     ) -> Result<MusicPlaylist, Error> {
         let playlist_id = playlist_id.as_ref();
-        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
         let request_body = QBrowse {
-            context,
             browse_id: &format!("VL{playlist_id}"),
         };
 
@@ -39,11 +45,13 @@ impl RustyPipeQuery {
     }
 
     /// Get an album from YouTube Music
-    pub async fn music_album<S: AsRef<str>>(&self, album_id: S) -> Result<MusicAlbum, Error> {
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn music_album<S: AsRef<str> + Debug>(
+        &self,
+        album_id: S,
+    ) -> Result<MusicAlbum, Error> {
         let album_id = album_id.as_ref();
-        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
         let request_body = QBrowse {
-            context,
             browse_id: album_id,
         };
 
@@ -60,7 +68,7 @@ impl RustyPipeQuery {
         // In rare cases, albums may have track numbers =0 (example: MPREb_RM0QfZ0eSKL)
         // They should be replaced with the track number derived from the previous track.
         let mut n_prev = 0;
-        for track in album.tracks.iter_mut() {
+        for track in &mut album.tracks {
             let tn = track.track_nr.unwrap_or_default();
             if tn == 0 {
                 n_prev += 1;
@@ -79,35 +87,63 @@ impl RustyPipeQuery {
                 .iter()
                 .enumerate()
                 .filter_map(|(i, track)| {
-                    if track.is_video {
-                        Some((i, track.name.to_owned()))
+                    if track.track_type.is_video() && !track.unavailable {
+                        Some((i, track.name.clone()))
                     } else {
                         None
                     }
                 })
                 .collect::<Vec<_>>();
 
-            if !to_replace.is_empty() {
+            let last_tn = album
+                .tracks
+                .last()
+                .and_then(|t| t.track_nr)
+                .unwrap_or_default();
+            if !to_replace.is_empty() || last_tn < album.track_count {
+                tracing::debug!(
+                    "fetching album playlist ({} tracks, {} to replace)",
+                    album.track_count,
+                    to_replace.len()
+                );
                 let mut playlist = self.music_playlist(playlist_id).await?;
                 playlist
                     .tracks
-                    .extend_limit(&self, album.tracks.len())
+                    .extend_limit(&self, album.track_count.into())
                     .await?;
 
                 for (i, title) in to_replace {
                     let found_track = playlist.tracks.items.iter().find_map(|track| {
-                        if track.name == title && !track.is_video {
-                            Some((track.id.to_owned(), track.duration))
+                        if track.name == title && track.track_type.is_track() {
+                            Some((track.id.clone(), track.duration, track.unavailable))
                         } else {
                             None
                         }
                     });
-                    if let Some((track_id, duration)) = found_track {
+                    if let Some((track_id, duration, unavailable)) = found_track {
                         album.tracks[i].id = track_id;
                         if let Some(duration) = duration {
                             album.tracks[i].duration = Some(duration);
                         }
-                        album.tracks[i].is_video = false;
+                        album.tracks[i].track_type = TrackType::Track;
+                        album.tracks[i].unavailable = unavailable;
+                    }
+                }
+
+                // Extend the list of album tracks with the ones from the playlist if the playlist returned more tracks
+                // This is the case for albums with more than 200 tracks (e.g. audiobooks)
+                // Note: in some cases the playlist may contain a loop of repeating tracks. If a track was found in the playlist
+                // that already exists in the album, stop.
+                if album.tracks.len() < playlist.tracks.items.len() {
+                    let mut tn = last_tn;
+                    for mut t in playlist.tracks.items.into_iter().skip(album.tracks.len()) {
+                        if album.tracks.iter().any(|at| at.id == t.id) {
+                            break;
+                        }
+                        tn += 1;
+                        t.album = album.tracks.first().and_then(|t| t.album.clone());
+                        t.track_nr = Some(tn);
+                        album.tracks.push(t);
                     }
                 }
             }
@@ -119,20 +155,52 @@ impl RustyPipeQuery {
 impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
     fn map_response(
         self,
-        id: &str,
-        lang: crate::param::Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
+        ctx: &MapRespCtx<'_>,
     ) -> Result<MapResult<MusicPlaylist>, ExtractionError> {
-        // dbg!(&self);
+        let contents = match self.contents {
+            Some(c) => c,
+            None => {
+                if self.microformat.microformat_data_renderer.noindex {
+                    return Err(ExtractionError::NotFound {
+                        id: ctx.id.to_owned(),
+                        msg: "no contents".into(),
+                    });
+                } else {
+                    return Err(ExtractionError::InvalidData("no contents".into()));
+                }
+            }
+        };
 
-        let mut content = self.contents.single_column_browse_results_renderer.contents;
-        let mut music_contents = content
-            .try_swap_remove(0)
-            .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))?
-            .tab_renderer
-            .content
-            .section_list_renderer;
-        let mut shelf = music_contents
+        let (header, music_contents) = match contents {
+            response::music_playlist::Contents::SingleColumnBrowseResultsRenderer(c) => (
+                self.header,
+                c.contents
+                    .into_iter()
+                    .next()
+                    .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))?
+                    .tab_renderer
+                    .content
+                    .section_list_renderer,
+            ),
+            response::music_playlist::Contents::TwoColumnBrowseResultsRenderer {
+                secondary_contents,
+                tabs,
+            } => (
+                tabs.into_iter()
+                    .next()
+                    .and_then(|t| {
+                        t.tab_renderer
+                            .content
+                            .section_list_renderer
+                            .contents
+                            .into_iter()
+                            .next()
+                    })
+                    .or(self.header),
+                secondary_contents.section_list_renderer,
+            ),
+        };
+        let shelf = music_contents
             .contents
             .into_iter()
             .find_map(|section| match section {
@@ -144,66 +212,98 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
             )))?;
 
         if let Some(playlist_id) = shelf.playlist_id {
-            if playlist_id != id {
+            if playlist_id != ctx.id {
                 return Err(ExtractionError::WrongResult(format!(
-                    "got wrong playlist id {playlist_id}, expected {id}"
+                    "got wrong playlist id {}, expected {}",
+                    playlist_id, ctx.id
                 )));
             }
         }
 
-        let mut mapper = MusicListMapper::new(lang);
+        let mut mapper = MusicListMapper::new(ctx.lang);
         mapper.map_response(shelf.contents);
+
+        let ctoken = mapper.ctoken.clone().or_else(|| {
+            shelf
+                .continuations
+                .into_iter()
+                .next()
+                .map(|cont| cont.next_continuation_data.continuation)
+        });
         let map_res = mapper.conv_items();
 
-        let ctoken = shelf
-            .continuations
-            .try_swap_remove(0)
-            .map(|cont| cont.next_continuation_data.continuation);
-
-        let track_count = match ctoken {
-            Some(_) => self.header.as_ref().and_then(|h| {
-                h.music_detail_header_renderer
+        let track_count = if ctoken.is_some() {
+            header.as_ref().and_then(|h| {
+                let parts = h
+                    .music_detail_header_renderer
                     .second_subtitle
-                    .first()
-                    .and_then(|txt| util::parse_numeric::<u64>(txt).ok())
-            }),
-            None => Some(map_res.c.len() as u64),
+                    .split(|p| p == DOT_SEPARATOR)
+                    .collect::<Vec<_>>();
+                parts
+                    .get(usize::from(parts.len() > 2))
+                    .and_then(|txt| util::parse_numeric::<u64>(&txt[0]).ok())
+            })
+        } else {
+            Some(map_res.c.len() as u64)
         };
 
         let related_ctoken = music_contents
             .continuations
-            .try_swap_remove(0)
+            .into_iter()
+            .next()
             .map(|c| c.next_continuation_data.continuation);
 
-        let (from_ytm, channel, name, thumbnail, description) = match self.header {
+        let (from_ytm, channel, name, thumbnail, description) = match header {
             Some(header) => {
                 let h = header.music_detail_header_renderer;
 
-                let from_ytm = h
-                    .subtitle
-                    .0
-                    .iter()
-                    .any(|c| c.as_str() == util::YT_MUSIC_NAME);
-                let channel = h
-                    .subtitle
-                    .0
-                    .into_iter()
-                    .find_map(|c| ChannelId::try_from(c).ok());
+                let (from_ytm, channel) = match h.facepile {
+                    Some(facepile) => {
+                        let from_ytm = facepile.avatar_stack_view_model.text.starts_with("YouTube");
+                        let channel = facepile
+                            .avatar_stack_view_model
+                            .renderer_context
+                            .command_context
+                            .and_then(|c| {
+                                c.on_tap
+                                    .innertube_command
+                                    .music_page()
+                                    .filter(|p| p.typ == MusicPageType::User)
+                                    .map(|p| p.id)
+                            })
+                            .map(|id| ChannelId {
+                                id,
+                                name: facepile.avatar_stack_view_model.text,
+                            });
+
+                        (from_ytm && channel.is_none(), channel)
+                    }
+                    None => {
+                        let st = match h.strapline_text_one {
+                            Some(s) => s,
+                            None => h.subtitle,
+                        };
+
+                        let from_ytm = st.0.iter().any(util::is_ytm);
+                        let channel = st.0.into_iter().find_map(|c| ChannelId::try_from(c).ok());
+                        (from_ytm, channel)
+                    }
+                };
 
                 (
                     from_ytm,
                     channel,
                     h.title,
                     h.thumbnail.into(),
-                    h.description,
+                    h.description.map(TextComponents::from),
                 )
             }
             None => {
                 // Album playlists fetched via the playlist method dont include a header
                 let (album, cover) = map_res
                     .c
-                    .first()
-                    .and_then(|t: &TrackItem| {
+                    .iter()
+                    .find_map(|t: &TrackItem| {
                         t.album.as_ref().map(|a| (a.clone(), t.cover.clone()))
                     })
                     .ok_or(ExtractionError::InvalidData(Cow::Borrowed(
@@ -211,10 +311,11 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
                     )))?;
 
                 if !map_res.c.iter().all(|t| {
-                    t.album
-                        .as_ref()
-                        .map(|a| a.id == album.id)
-                        .unwrap_or_default()
+                    t.unavailable
+                        || t.album
+                            .as_ref()
+                            .map(|a| a.id == album.id)
+                            .unwrap_or_default()
                 }) {
                     return Err(ExtractionError::InvalidData(Cow::Borrowed(
                         "album playlist containing items from different albums",
@@ -227,26 +328,28 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
 
         Ok(MapResult {
             c: MusicPlaylist {
-                id: id.to_owned(),
+                id: ctx.id.to_owned(),
                 name,
                 thumbnail,
                 channel,
-                description,
+                description: description.map(RichText::from),
                 track_count,
                 from_ytm,
                 tracks: Paginator::new_ext(
                     track_count,
                     map_res.c,
                     ctoken,
-                    None,
-                    crate::model::paginator::ContinuationEndpoint::MusicBrowse,
+                    ctx.visitor_data.map(str::to_owned),
+                    ContinuationEndpoint::MusicBrowse,
+                    ctx.authenticated,
                 ),
                 related_playlists: Paginator::new_ext(
                     None,
                     Vec::new(),
                     related_ctoken,
-                    None,
-                    crate::model::paginator::ContinuationEndpoint::MusicBrowse,
+                    ctx.visitor_data.map(str::to_owned),
+                    ContinuationEndpoint::MusicBrowse,
+                    ctx.authenticated,
                 ),
             },
             warnings: map_res.warnings,
@@ -255,35 +358,73 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
 }
 
 impl MapResponse<MusicAlbum> for response::MusicPlaylist {
-    fn map_response(
-        self,
-        id: &str,
-        lang: crate::param::Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
-    ) -> Result<MapResult<MusicAlbum>, ExtractionError> {
-        // dbg!(&self);
+    fn map_response(self, ctx: &MapRespCtx<'_>) -> Result<MapResult<MusicAlbum>, ExtractionError> {
+        let contents = match self.contents {
+            Some(c) => c,
+            None => {
+                if self.microformat.microformat_data_renderer.noindex {
+                    return Err(ExtractionError::NotFound {
+                        id: ctx.id.to_owned(),
+                        msg: "no contents".into(),
+                    });
+                } else {
+                    return Err(ExtractionError::InvalidData("no contents".into()));
+                }
+            }
+        };
 
-        let header = self
-            .header
+        let (header, sections) = match contents {
+            response::music_playlist::Contents::SingleColumnBrowseResultsRenderer(c) => (
+                self.header,
+                c.contents
+                    .into_iter()
+                    .next()
+                    .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))?
+                    .tab_renderer
+                    .content
+                    .section_list_renderer
+                    .contents,
+            ),
+            response::music_playlist::Contents::TwoColumnBrowseResultsRenderer {
+                secondary_contents,
+                tabs,
+            } => (
+                tabs.into_iter()
+                    .next()
+                    .and_then(|t| {
+                        t.tab_renderer
+                            .content
+                            .section_list_renderer
+                            .contents
+                            .into_iter()
+                            .next()
+                    })
+                    .or(self.header),
+                secondary_contents.section_list_renderer.contents,
+            ),
+        };
+        let header = header
             .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no header")))?
             .music_detail_header_renderer;
 
-        let mut content = self.contents.single_column_browse_results_renderer.contents;
-        let sections = content
-            .try_swap_remove(0)
-            .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))?
-            .tab_renderer
-            .content
-            .section_list_renderer
-            .contents;
-
         let mut shelf = None;
         let mut album_variants = None;
         for section in sections {
             match section {
                 response::music_item::ItemSection::MusicShelfRenderer(sh) => shelf = Some(sh),
                 response::music_item::ItemSection::MusicCarouselShelfRenderer(sh) => {
-                    album_variants = Some(sh.contents)
+                    if sh
+                        .header
+                        .map(|h| {
+                            h.music_carousel_shelf_basic_header_renderer
+                                .title
+                                .first_str()
+                                == dictionary::entry(ctx.lang).album_versions_title
+                        })
+                        .unwrap_or_default()
+                    {
+                        album_variants = Some(sh.contents);
+                    }
                 }
                 _ => (),
             }
@@ -294,71 +435,116 @@ impl MapResponse<MusicAlbum> for response::MusicPlaylist {
 
         let mut subtitle_split = header.subtitle.split(util::DOT_SEPARATOR);
 
-        let (year_txt, artists_p) = match subtitle_split.len() {
-            3.. => {
+        let (year_txt, artists_p) = match header.strapline_text_one {
+            // New (2column) album layout
+            Some(sl) => {
                 let year_txt = subtitle_split
-                    .swap_remove(2)
-                    .0
-                    .get(0)
-                    .map(|c| c.as_str().to_owned());
-                (year_txt, subtitle_split.try_swap_remove(1))
+                    .try_swap_remove(1)
+                    .and_then(|t| t.0.first().map(|c| c.as_str().to_owned()));
+                (year_txt, Some(sl))
             }
-            2 => {
-                // The second part may either be the year or the artist
-                let p2 = subtitle_split.swap_remove(1);
-                let is_year =
-                    p2.0.len() == 1 && p2.0[0].as_str().chars().all(|c| c.is_ascii_digit());
-                if is_year {
-                    (Some(p2.0[0].as_str().to_owned()), None)
-                } else {
-                    (None, Some(p2))
+            // Old album layout
+            None => match subtitle_split.len() {
+                3.. => {
+                    let year_txt = subtitle_split
+                        .swap_remove(2)
+                        .0
+                        .first()
+                        .map(|c| c.as_str().to_owned());
+                    (year_txt, subtitle_split.try_swap_remove(1))
                 }
-            }
-            _ => (None, None),
+                2 => {
+                    // The second part may either be the year or the artist
+                    let p2 = subtitle_split.swap_remove(1);
+                    let is_year =
+                        p2.0.len() == 1 && p2.0[0].as_str().chars().all(|c| c.is_ascii_digit());
+                    if is_year {
+                        (Some(p2.0[0].as_str().to_owned()), None)
+                    } else {
+                        (None, Some(p2))
+                    }
+                }
+                _ => (None, None),
+            },
         };
 
         let (artists, by_va) = map_artists(artists_p);
         let album_type_txt = subtitle_split
-            .try_swap_remove(0)
+            .into_iter()
+            .next()
             .map(|part| part.to_string())
             .unwrap_or_default();
 
-        let album_type = map_album_type(album_type_txt.as_str(), lang);
+        let album_type = map_album_type(album_type_txt.as_str(), ctx.lang);
         let year = year_txt.and_then(|txt| util::parse_numeric(&txt).ok());
 
-        let (artist_id, playlist_id) = header
+        fn map_playlist_id(ep: &NavigationEndpoint) -> Option<String> {
+            if let NavigationEndpoint::WatchPlaylist {
+                watch_playlist_endpoint,
+            } = ep
+            {
+                Some(watch_playlist_endpoint.playlist_id.to_owned())
+            } else {
+                None
+            }
+        }
+
+        let playlist_id = self
+            .microformat
+            .microformat_data_renderer
+            .url_canonical
+            .and_then(|x| {
+                x.strip_prefix("https://music.youtube.com/playlist?list=")
+                    .map(str::to_owned)
+            });
+        let (playlist_id, artist_id) = header
             .menu
-            .map(|mut menu| {
+            .or_else(|| header.buttons.into_iter().next())
+            .map(|menu| {
                 (
+                    playlist_id.or_else(|| {
+                        menu.menu_renderer
+                            .top_level_buttons
+                            .iter()
+                            .find_map(|btn| {
+                                map_playlist_id(&btn.button_renderer.navigation_endpoint)
+                            })
+                            .or_else(|| {
+                                menu.menu_renderer.items.iter().find_map(|itm| {
+                                    map_playlist_id(
+                                        &itm.menu_navigation_item_renderer.navigation_endpoint,
+                                    )
+                                })
+                            })
+                    }),
                     map_artist_id(menu.menu_renderer.items),
-                    menu.menu_renderer
-                        .top_level_buttons
-                        .try_swap_remove(0)
-                        .map(|btn| {
-                            btn.button_renderer
-                                .navigation_endpoint
-                                .watch_playlist_endpoint
-                                .playlist_id
-                        }),
                 )
             })
             .unwrap_or_default();
-        let artist_id = artist_id.or_else(|| artists.first().and_then(|a| a.id.to_owned()));
+        let artist_id = artist_id.or_else(|| artists.first().and_then(|a| a.id.clone()));
+
+        let second_subtitle_parts = header
+            .second_subtitle
+            .split(|p| p == DOT_SEPARATOR)
+            .collect::<Vec<_>>();
+        let track_count = second_subtitle_parts
+            .get(usize::from(second_subtitle_parts.len() > 2))
+            .and_then(|txt| util::parse_numeric::<u16>(&txt[0]).ok());
 
         let mut mapper = MusicListMapper::with_album(
-            lang,
+            ctx.lang,
             artists.clone(),
             by_va,
             AlbumId {
-                id: id.to_owned(),
-                name: header.title.to_owned(),
+                id: ctx.id.to_owned(),
+                name: header.title.clone(),
             },
         );
         mapper.map_response(shelf.contents);
         let tracks_res = mapper.conv_items();
         let mut warnings = tracks_res.warnings;
 
-        let mut variants_mapper = MusicListMapper::new(lang);
+        let mut variants_mapper = MusicListMapper::new(ctx.lang);
         if let Some(res) = album_variants {
             variants_mapper.map_response(res);
         }
@@ -367,16 +553,19 @@ impl MapResponse<MusicAlbum> for response::MusicPlaylist {
 
         Ok(MapResult {
             c: MusicAlbum {
-                id: id.to_owned(),
+                id: ctx.id.to_owned(),
                 playlist_id,
                 name: header.title,
                 cover: header.thumbnail.into(),
                 artists,
                 artist_id,
-                description: header.description,
+                description: header
+                    .description
+                    .map(|t| RichText::from(TextComponents::from(t))),
                 album_type,
                 year,
                 by_va,
+                track_count: track_count.unwrap_or(tracks_res.c.len() as u16),
                 tracks: tracks_res.c,
                 variants: variants_res.c,
             },
@@ -393,12 +582,15 @@ mod tests {
     use rstest::rstest;
 
     use super::*;
-    use crate::{model, param::Language, util::tests::TESTFILES};
+    use crate::{model, util::tests::TESTFILES};
 
     #[rstest]
     #[case::short("short", "RDCLAK5uy_kFQXdnqMaQCVx2wpUM4ZfbsGCDibZtkJk")]
     #[case::long("long", "PL5dDx681T4bR7ZF1IuWzOv1omlRbE7PiJ")]
     #[case::nomusic("nomusic", "PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe")]
+    #[case::two_columns("20240228_twoColumns", "RDCLAK5uy_kb7EBi6y3GrtJri4_ZH56Ms786DFEimbM")]
+    #[case::n_album("20240228_album", "OLAK5uy_kdSWBZ-9AZDkYkuy0QCc3p0KO9DEHVNH0")]
+    #[case::facepile("20241125_facepile", "PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe")]
     fn map_music_playlist(#[case] name: &str, #[case] id: &str) {
         let json_path = path!(*TESTFILES / "music_playlist" / format!("playlist_{name}.json"));
         let json_file = File::open(json_path).unwrap();
@@ -406,7 +598,7 @@ mod tests {
         let playlist: response::MusicPlaylist =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
         let map_res: MapResult<model::MusicPlaylist> =
-            playlist.map_response(id, Language::En, None).unwrap();
+            playlist.map_response(&MapRespCtx::test(id)).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),
@@ -424,6 +616,8 @@ mod tests {
     #[case::single("single", "MPREb_bHfHGoy7vuv")]
     #[case::description("description", "MPREb_PiyfuVl6aYd")]
     #[case::unavailable("unavailable", "MPREb_AzuWg8qAVVl")]
+    #[case::two_columns("20240228_twoColumns", "MPREb_bHfHGoy7vuv")]
+    #[case::recommends("20250225_recommends", "MPREb_u1I69lSAe5v")]
     fn map_music_album(#[case] name: &str, #[case] id: &str) {
         let json_path = path!(*TESTFILES / "music_playlist" / format!("album_{name}.json"));
         let json_file = File::open(json_path).unwrap();
@@ -431,7 +625,7 @@ mod tests {
         let playlist: response::MusicPlaylist =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
         let map_res: MapResult<model::MusicAlbum> =
-            playlist.map_response(id, Language::En, None).unwrap();
+            playlist.map_response(&MapRespCtx::test(id)).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),

src/client/music_search.rs

@@ -1,4 +1,4 @@
-use std::borrow::Cow;
+use std::{borrow::Cow, fmt::Debug};
 
 use serde::Serialize;
 
@@ -6,97 +6,45 @@ use crate::{
     client::response::music_item::MusicListMapper,
     error::{Error, ExtractionError},
     model::{
-        paginator::Paginator, traits::FromYtItem, AlbumItem, ArtistItem, MusicPlaylistItem,
-        MusicSearchFiltered, MusicSearchResult, MusicSearchSuggestion, TrackItem,
+        paginator::{ContinuationEndpoint, Paginator},
+        traits::FromYtItem,
+        AlbumItem, ArtistItem, MusicItem, MusicPlaylistItem, MusicSearchResult,
+        MusicSearchSuggestion, TrackItem, UserItem,
     },
+    param::search_filter::MusicSearchFilter,
     serializer::MapResult,
-    util::TryRemove,
 };
 
-use super::{response, ClientType, MapResponse, RustyPipeQuery, YTContext};
+use super::{response, ClientType, MapRespCtx, MapResponse, RustyPipeQuery};
 
 #[derive(Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
 struct QSearch<'a> {
-    context: YTContext<'a>,
     query: &'a str,
     #[serde(skip_serializing_if = "Option::is_none")]
-    params: Option<Params>,
+    params: Option<&'a str>,
 }
 
 #[derive(Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
 struct QSearchSuggestion<'a> {
-    context: YTContext<'a>,
     input: &'a str,
 }
 
-#[derive(Debug, Serialize)]
-enum Params {
-    #[serde(rename = "EgWKAQIIAWoMEAMQBBAJEA4QChAF")]
-    Tracks,
-    #[serde(rename = "EgWKAQIQAWoMEAMQBBAJEA4QChAF")]
-    Videos,
-    #[serde(rename = "EgWKAQIYAWoMEAMQBBAJEA4QChAF")]
-    Albums,
-    #[serde(rename = "EgWKAQIgAWoMEAMQBBAJEA4QChAF")]
-    Artists,
-    #[serde(rename = "EgWKAQIoAWoMEAMQBBAJEA4QChAF")]
-    Playlists,
-    #[serde(rename = "EgeKAQQoADgBagwQAxAEEAkQDhAKEAU%3D")]
-    YtmPlaylists,
-    #[serde(rename = "EgeKAQQoAEABagwQAxAEEAkQDhAKEAU%3D")]
-    CommunityPlaylists,
-}
-
 impl RustyPipeQuery {
-    /// Search YouTube Music. Returns items from any type.
-    pub async fn music_search<S: AsRef<str>>(&self, query: S) -> Result<MusicSearchResult, Error> {
+    /// Search YouTube Music.
+    ///
+    /// This is a generic implementation which casts items to the given type or filters
+    /// them out.
+    pub async fn music_search<T: FromYtItem, S: AsRef<str>>(
+        &self,
+        query: S,
+        filter: Option<MusicSearchFilter>,
+    ) -> Result<MusicSearchResult<T>, Error> {
         let query = query.as_ref();
-        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
         let request_body = QSearch {
-            context,
             query,
-            params: None,
-        };
-
-        self.execute_request::<response::MusicSearch, _, _>(
-            ClientType::DesktopMusic,
-            "music_search",
-            query,
-            "search",
-            &request_body,
-        )
-        .await
-    }
-
-    /// Search YouTube Music tracks
-    pub async fn music_search_tracks<S: AsRef<str>>(
-        &self,
-        query: S,
-    ) -> Result<MusicSearchFiltered<TrackItem>, Error> {
-        self._music_search_tracks(query, Params::Tracks).await
-    }
-
-    /// Search YouTube Music videos
-    pub async fn music_search_videos<S: AsRef<str>>(
-        &self,
-        query: S,
-    ) -> Result<MusicSearchFiltered<TrackItem>, Error> {
-        self._music_search_tracks(query, Params::Videos).await
-    }
-
-    async fn _music_search_tracks<S: AsRef<str>>(
-        &self,
-        query: S,
-        params: Params,
-    ) -> Result<MusicSearchFiltered<TrackItem>, Error> {
-        let query = query.as_ref();
-        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
-        let request_body = QSearch {
-            context,
-            query,
-            params: Some(params),
+            params: filter.map(MusicSearchFilter::params),
         };
 
         self.execute_request::<response::MusicSearch, _, _>(
@@ -109,110 +57,87 @@ impl RustyPipeQuery {
         .await
     }
 
+    /// Search YouTube Music and return items of all types
+    pub async fn music_search_main<S: AsRef<str>>(
+        &self,
+        query: S,
+    ) -> Result<MusicSearchResult<MusicItem>, Error> {
+        self.music_search(query, None).await
+    }
+
+    /// Search YouTube Music artists
+    pub async fn music_search_artists<S: AsRef<str>>(
+        &self,
+        query: S,
+    ) -> Result<MusicSearchResult<ArtistItem>, Error> {
+        self.music_search(query, Some(MusicSearchFilter::Artists))
+            .await
+    }
+
     /// Search YouTube Music albums
     pub async fn music_search_albums<S: AsRef<str>>(
         &self,
         query: S,
-    ) -> Result<MusicSearchFiltered<AlbumItem>, Error> {
-        let query = query.as_ref();
-        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
-        let request_body = QSearch {
-            context,
-            query,
-            params: Some(Params::Albums),
-        };
-
-        self.execute_request::<response::MusicSearch, _, _>(
-            ClientType::DesktopMusic,
-            "music_search_albums",
-            query,
-            "search",
-            &request_body,
-        )
-        .await
+    ) -> Result<MusicSearchResult<AlbumItem>, Error> {
+        self.music_search(query, Some(MusicSearchFilter::Albums))
+            .await
     }
 
-    /// Search YouTube Music artists
-    pub async fn music_search_artists(
+    /// Search YouTube Music tracks
+    pub async fn music_search_tracks<S: AsRef<str>>(
         &self,
-        query: &str,
-    ) -> Result<MusicSearchFiltered<ArtistItem>, Error> {
-        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
-        let request_body = QSearch {
-            context,
-            query,
-            params: Some(Params::Artists),
-        };
+        query: S,
+    ) -> Result<MusicSearchResult<TrackItem>, Error> {
+        self.music_search(query, Some(MusicSearchFilter::Tracks))
+            .await
+    }
 
-        self.execute_request::<response::MusicSearch, _, _>(
-            ClientType::DesktopMusic,
-            "music_search_albums",
-            query,
-            "search",
-            &request_body,
-        )
-        .await
+    /// Search YouTube Music videos
+    pub async fn music_search_videos<S: AsRef<str>>(
+        &self,
+        query: S,
+    ) -> Result<MusicSearchResult<TrackItem>, Error> {
+        self.music_search(query, Some(MusicSearchFilter::Videos))
+            .await
     }
 
     /// Search YouTube Music playlists
-    pub async fn music_search_playlists<S: AsRef<str>>(
-        &self,
-        query: S,
-    ) -> Result<MusicSearchFiltered<MusicPlaylistItem>, Error> {
-        self._music_search_playlists(query, Params::Playlists).await
-    }
-
-    /// Search YouTube Music playlists that were created by users
+    ///
+    /// Playlists are filtered whether they are created by users
     /// (`community=true`) or by YouTube Music (`community=false`)
-    pub async fn music_search_playlists_filter<S: AsRef<str>>(
+    pub async fn music_search_playlists<S: AsRef<str> + Debug>(
         &self,
         query: S,
         community: bool,
-    ) -> Result<MusicSearchFiltered<MusicPlaylistItem>, Error> {
-        self._music_search_playlists(
+    ) -> Result<MusicSearchResult<MusicPlaylistItem>, Error> {
+        self.music_search(
             query,
-            match community {
-                true => Params::CommunityPlaylists,
-                false => Params::YtmPlaylists,
-            },
+            Some(if community {
+                MusicSearchFilter::CommunityPlaylists
+            } else {
+                MusicSearchFilter::YtmPlaylists
+            }),
         )
         .await
     }
 
-    async fn _music_search_playlists<S: AsRef<str>>(
+    /// Search YouTube Music users
+    pub async fn music_search_users<S: AsRef<str>>(
         &self,
         query: S,
-        params: Params,
-    ) -> Result<MusicSearchFiltered<MusicPlaylistItem>, Error> {
-        let query = query.as_ref();
-        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
-        let request_body = QSearch {
-            context,
-            query,
-            params: Some(params),
-        };
-
-        self.execute_request::<response::MusicSearch, _, _>(
-            ClientType::DesktopMusic,
-            "music_search_playlists",
-            query,
-            "search",
-            &request_body,
-        )
-        .await
+    ) -> Result<MusicSearchResult<UserItem>, Error> {
+        self.music_search(query, Some(MusicSearchFilter::Users))
+            .await
     }
 
     /// Get YouTube Music search suggestions
-    pub async fn music_search_suggestion<S: AsRef<str>>(
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn music_search_suggestion<S: AsRef<str> + Debug>(
         &self,
         query: S,
     ) -> Result<MusicSearchSuggestion, Error> {
         let query = query.as_ref();
-        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
-        let request_body = QSearchSuggestion {
-            context,
-            input: query,
-        };
+        let request_body = QSearchSuggestion { input: query };
 
         self.execute_request::<response::MusicSearchSuggestion, _, _>(
             ClientType::DesktopMusic,
@@ -225,79 +150,15 @@ impl RustyPipeQuery {
     }
 }
 
-impl MapResponse<MusicSearchResult> for response::MusicSearch {
+impl<T: FromYtItem> MapResponse<MusicSearchResult<T>> for response::MusicSearch {
     fn map_response(
         self,
-        _id: &str,
-        lang: crate::param::Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
-    ) -> Result<MapResult<MusicSearchResult>, crate::error::ExtractionError> {
-        // dbg!(&self);
-
-        let mut tabs = self.contents.tabbed_search_results_renderer.contents;
+        ctx: &MapRespCtx<'_>,
+    ) -> Result<MapResult<MusicSearchResult<T>>, ExtractionError> {
+        let tabs = self.contents.tabbed_search_results_renderer.contents;
         let sections = tabs
-            .try_swap_remove(0)
-            .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no tab")))?
-            .tab_renderer
-            .content
-            .section_list_renderer
-            .contents;
-
-        let mut corrected_query = None;
-        let mut order = Vec::new();
-        let mut mapper = MusicListMapper::new(lang);
-
-        sections.into_iter().for_each(|section| match section {
-            response::music_search::ItemSection::MusicShelfRenderer(shelf) => {
-                if let Some(etype) = mapper.map_response(shelf.contents) {
-                    if !order.contains(&etype) {
-                        order.push(etype);
-                    }
-                }
-            }
-            response::music_search::ItemSection::MusicCardShelfRenderer(card) => {
-                if let Some(etype) = mapper.map_card(card) {
-                    if !order.contains(&etype) {
-                        order.push(etype);
-                    }
-                }
-            }
-            response::music_search::ItemSection::ItemSectionRenderer { mut contents } => {
-                if let Some(corrected) = contents.try_swap_remove(0) {
-                    corrected_query = Some(corrected.showing_results_for_renderer.corrected_query)
-                }
-            }
-            response::music_search::ItemSection::None => {}
-        });
-
-        let map_res = mapper.group_items();
-
-        Ok(MapResult {
-            c: MusicSearchResult {
-                tracks: map_res.c.tracks,
-                albums: map_res.c.albums,
-                artists: map_res.c.artists,
-                playlists: map_res.c.playlists,
-                corrected_query,
-                order,
-            },
-            warnings: map_res.warnings,
-        })
-    }
-}
-
-impl<T: FromYtItem> MapResponse<MusicSearchFiltered<T>> for response::MusicSearch {
-    fn map_response(
-        self,
-        _id: &str,
-        lang: crate::param::Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
-    ) -> Result<MapResult<MusicSearchFiltered<T>>, ExtractionError> {
-        // dbg!(&self);
-
-        let mut tabs = self.contents.tabbed_search_results_renderer.contents;
-        let sections = tabs
-            .try_swap_remove(0)
+            .into_iter()
+            .next()
             .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no tab")))?
             .tab_renderer
             .content
@@ -306,36 +167,38 @@ impl<T: FromYtItem> MapResponse<MusicSearchFiltered<T>> for response::MusicSearc
 
         let mut corrected_query = None;
         let mut ctoken = None;
-        let mut mapper = MusicListMapper::new(lang);
+        let mut mapper = MusicListMapper::new(ctx.lang);
 
         sections.into_iter().for_each(|section| match section {
-            response::music_search::ItemSection::MusicShelfRenderer(mut shelf) => {
+            response::music_search::ItemSection::MusicShelfRenderer(shelf) => {
                 mapper.map_response(shelf.contents);
-                if let Some(cont) = shelf.continuations.try_swap_remove(0) {
+                if let Some(cont) = shelf.continuations.into_iter().next() {
                     ctoken = Some(cont.next_continuation_data.continuation);
                 }
             }
             response::music_search::ItemSection::MusicCardShelfRenderer(card) => {
                 mapper.map_card(card);
             }
-            response::music_search::ItemSection::ItemSectionRenderer { mut contents } => {
-                if let Some(corrected) = contents.try_swap_remove(0) {
-                    corrected_query = Some(corrected.showing_results_for_renderer.corrected_query)
+            response::music_search::ItemSection::ItemSectionRenderer { contents } => {
+                if let Some(corrected) = contents.into_iter().next() {
+                    corrected_query = Some(corrected.showing_results_for_renderer.corrected_query);
                 }
             }
             response::music_search::ItemSection::None => {}
         });
 
+        let ctoken = ctoken.or(mapper.ctoken.clone());
         let map_res = mapper.conv_items();
 
         Ok(MapResult {
-            c: MusicSearchFiltered {
+            c: MusicSearchResult {
                 items: Paginator::new_ext(
                     None,
                     map_res.c,
                     ctoken,
-                    None,
-                    crate::model::paginator::ContinuationEndpoint::MusicSearch,
+                    ctx.visitor_data.map(str::to_owned),
+                    ContinuationEndpoint::MusicSearch,
+                    false,
                 ),
                 corrected_query,
             },
@@ -347,11 +210,9 @@ impl<T: FromYtItem> MapResponse<MusicSearchFiltered<T>> for response::MusicSearc
 impl MapResponse<MusicSearchSuggestion> for response::MusicSearchSuggestion {
     fn map_response(
         self,
-        _id: &str,
-        lang: crate::param::Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
+        ctx: &MapRespCtx<'_>,
     ) -> Result<MapResult<MusicSearchSuggestion>, ExtractionError> {
-        let mut mapper = MusicListMapper::new(lang);
+        let mut mapper = MusicListMapper::new_search_suggest(ctx.lang);
         let mut terms = Vec::new();
 
         for section in self.contents {
@@ -390,12 +251,11 @@ mod tests {
     use rstest::rstest;
 
     use crate::{
-        client::{response, MapResponse},
+        client::{response, MapRespCtx, MapResponse},
         model::{
-            AlbumItem, ArtistItem, MusicPlaylistItem, MusicSearchFiltered, MusicSearchResult,
+            AlbumItem, ArtistItem, MusicItem, MusicPlaylistItem, MusicSearchResult,
             MusicSearchSuggestion, TrackItem,
         },
-        param::Language,
         serializer::MapResult,
         util::tests::TESTFILES,
     };
@@ -404,15 +264,16 @@ mod tests {
     #[case::default("default")]
     #[case::typo("typo")]
     #[case::radio("radio")]
-    #[case::radio("artist")]
+    #[case::artist("artist")]
+    #[case::live("live")]
     fn map_music_search_main(#[case] name: &str) {
         let json_path = path!(*TESTFILES / "music_search" / format!("main_{name}.json"));
         let json_file = File::open(json_path).unwrap();
 
         let search: response::MusicSearch =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
-        let map_res: MapResult<MusicSearchResult> =
-            search.map_response("", Language::En, None).unwrap();
+        let map_res: MapResult<MusicSearchResult<MusicItem>> =
+            search.map_response(&MapRespCtx::test("")).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),
@@ -434,8 +295,8 @@ mod tests {
 
         let search: response::MusicSearch =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
-        let map_res: MapResult<MusicSearchFiltered<TrackItem>> =
-            search.map_response("", Language::En, None).unwrap();
+        let map_res: MapResult<MusicSearchResult<TrackItem>> =
+            search.map_response(&MapRespCtx::test("")).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),
@@ -453,8 +314,8 @@ mod tests {
 
         let search: response::MusicSearch =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
-        let map_res: MapResult<MusicSearchFiltered<AlbumItem>> =
-            search.map_response("", Language::En, None).unwrap();
+        let map_res: MapResult<MusicSearchResult<AlbumItem>> =
+            search.map_response(&MapRespCtx::test("")).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),
@@ -472,8 +333,8 @@ mod tests {
 
         let search: response::MusicSearch =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
-        let map_res: MapResult<MusicSearchFiltered<ArtistItem>> =
-            search.map_response("", Language::En, None).unwrap();
+        let map_res: MapResult<MusicSearchResult<ArtistItem>> =
+            search.map_response(&MapRespCtx::test("")).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),
@@ -493,8 +354,8 @@ mod tests {
 
         let search: response::MusicSearch =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
-        let map_res: MapResult<MusicSearchFiltered<MusicPlaylistItem>> =
-            search.map_response("", Language::En, None).unwrap();
+        let map_res: MapResult<MusicSearchResult<MusicPlaylistItem>> =
+            search.map_response(&MapRespCtx::test("")).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),
@@ -515,7 +376,7 @@ mod tests {
         let suggestion: response::MusicSearchSuggestion =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
         let map_res: MapResult<MusicSearchSuggestion> =
-            suggestion.map_response("", Language::En, None).unwrap();
+            suggestion.map_response(&MapRespCtx::test("")).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),

src/client/music_userdata.rs

@@ -0,0 +1,228 @@
+use std::fmt::Debug;
+
+use crate::{
+    client::{
+        response::{self, music_item::MusicListMapper},
+        ClientType, MapResponse, QBrowseParams, RustyPipeQuery,
+    },
+    error::{Error, ExtractionError},
+    model::{
+        paginator::{ContinuationEndpoint, Paginator},
+        AlbumItem, ArtistItem, HistoryItem, MusicPlaylist, MusicPlaylistItem, TrackItem,
+    },
+    serializer::MapResult,
+};
+
+use super::{MapRespCtx, MapRespOptions, QContinuation};
+
+impl RustyPipeQuery {
+    /// Get a list of tracks from YouTube Music which the current user recently played
+    ///
+    /// Requires authentication cookies.
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn music_history(&self) -> Result<Paginator<HistoryItem<TrackItem>>, Error> {
+        let request_body = QBrowseParams {
+            browse_id: "FEmusic_history",
+            params: "oggECgIIAQ%3D%3D",
+        };
+
+        self.clone()
+            .authenticated()
+            .execute_request::<response::MusicHistory, _, _>(
+                ClientType::DesktopMusic,
+                "music_history",
+                "",
+                "browse",
+                &request_body,
+            )
+            .await
+    }
+
+    /// Get more YouTube Music history items from the given continuation token
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn music_history_continuation<S: AsRef<str> + Debug>(
+        &self,
+        ctoken: S,
+        visitor_data: Option<&str>,
+    ) -> Result<Paginator<HistoryItem<TrackItem>>, Error> {
+        let ctoken = ctoken.as_ref();
+        let request_body = QContinuation {
+            continuation: ctoken,
+        };
+
+        self.clone()
+            .authenticated()
+            .execute_request_ctx::<response::MusicContinuation, _, _>(
+                ClientType::Desktop,
+                "history_continuation",
+                ctoken,
+                "browse",
+                &request_body,
+                MapRespOptions {
+                    visitor_data,
+                    ..Default::default()
+                },
+            )
+            .await
+    }
+
+    /// Get a list of YouTube Music artists which the current user subscribed to
+    ///
+    /// Requires authentication cookies.
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn music_saved_artists(&self) -> Result<Paginator<ArtistItem>, Error> {
+        self.clone()
+            .authenticated()
+            .continuation(
+                "4qmFsgIyEh5GRW11c2ljX2xpYnJhcnlfY29ycHVzX2FydGlzdHMaEGdnTUdLZ1FJQUJBQm9BWUI%3D",
+                ContinuationEndpoint::MusicBrowse,
+                None,
+            )
+            .await
+    }
+
+    /// Get a list of YouTube Music albums which the current user has added to their collection
+    ///
+    /// Requires authentication cookies.
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn music_saved_albums(&self) -> Result<Paginator<AlbumItem>, Error> {
+        self.clone()
+            .authenticated()
+            .continuation(
+                "4qmFsgIoEhRGRW11c2ljX2xpa2VkX2FsYnVtcxoQZ2dNR0tnUUlBQkFCb0FZQg%3D%3D",
+                ContinuationEndpoint::MusicBrowse,
+                None,
+            )
+            .await
+    }
+
+    /// Get a list of YouTube Music tracks which the current user has added to their collection
+    ///
+    /// Contains both liked tracks and tracks from saved albums.
+    ///
+    /// Requires authentication cookies.
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn music_saved_tracks(&self) -> Result<Paginator<TrackItem>, Error> {
+        self.clone()
+            .authenticated()
+            .continuation(
+                "4qmFsgIoEhRGRW11c2ljX2xpa2VkX3ZpZGVvcxoQZ2dNR0tnUUlBQkFCb0FZQg%3D%3D",
+                ContinuationEndpoint::MusicBrowse,
+                None,
+            )
+            .await
+    }
+
+    /// Get a list of YouTube Music playlists which the current user has added to their collection
+    ///
+    /// Requires authentication cookies.
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn music_saved_playlists(&self) -> Result<Paginator<MusicPlaylistItem>, Error> {
+        self.clone()
+            .authenticated()
+            .continuation(
+                "4qmFsgIrEhdGRW11c2ljX2xpa2VkX3BsYXlsaXN0cxoQZ2dNR0tnUUlBQkFCb0FZQg%3D%3D",
+                ContinuationEndpoint::MusicBrowse,
+                None,
+            )
+            .await
+    }
+
+    /// Get all liked YouTube Music tracks of the logged-in user
+    ///
+    /// The difference to [`RustyPipeQuery::music_saved_tracks`] is that this function only returns
+    /// tracks that were explicitly liked by the user.
+    ///
+    /// Requires authentication cookies.
+    pub async fn music_liked_tracks(&self) -> Result<MusicPlaylist, Error> {
+        self.clone()
+            .authenticated()
+            .music_playlist("LM")
+            .await
+            .map_err(crate::util::map_internal_playlist_err)
+    }
+}
+
+impl MapResponse<Paginator<HistoryItem<TrackItem>>> for response::MusicHistory {
+    fn map_response(
+        self,
+        ctx: &MapRespCtx<'_>,
+    ) -> Result<MapResult<Paginator<HistoryItem<TrackItem>>>, ExtractionError> {
+        let contents = match self.contents {
+            response::music_playlist::Contents::SingleColumnBrowseResultsRenderer(c) => {
+                c.contents
+                    .into_iter()
+                    .next()
+                    .ok_or(ExtractionError::InvalidData("no content".into()))?
+                    .tab_renderer
+                    .content
+                    .section_list_renderer
+            }
+            response::music_playlist::Contents::TwoColumnBrowseResultsRenderer {
+                secondary_contents,
+                ..
+            } => secondary_contents.section_list_renderer,
+        };
+
+        let mut map_res = MapResult::default();
+
+        for shelf in contents.contents {
+            let shelf = if let response::music_item::ItemSection::MusicShelfRenderer(s) = shelf {
+                s
+            } else {
+                continue;
+            };
+            let mut mapper = MusicListMapper::new(ctx.lang);
+            mapper.map_response(shelf.contents);
+            mapper.conv_history_items(shelf.title, ctx.utc_offset, &mut map_res);
+        }
+
+        let ctoken = contents
+            .continuations
+            .into_iter()
+            .next()
+            .map(|c| c.next_continuation_data.continuation);
+
+        Ok(MapResult {
+            c: Paginator::new_ext(
+                None,
+                map_res.c,
+                ctoken,
+                ctx.visitor_data.map(str::to_owned),
+                ContinuationEndpoint::MusicBrowse,
+                true,
+            ),
+            warnings: map_res.warnings,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{fs::File, io::BufReader};
+
+    use path_macro::path;
+
+    use crate::util::tests::TESTFILES;
+
+    use super::*;
+
+    #[test]
+    fn map_history() {
+        let json_path = path!(*TESTFILES / "music_userdata" / "music_history.json");
+        let json_file = File::open(json_path).unwrap();
+
+        let history: response::MusicHistory =
+            serde_json::from_reader(BufReader::new(json_file)).unwrap();
+        let map_res = history.map_response(&MapRespCtx::test("")).unwrap();
+
+        assert!(
+            map_res.warnings.is_empty(),
+            "deserialization/mapping warnings: {:?}",
+            map_res.warnings
+        );
+        insta::assert_ron_snapshot!(map_res.c, {
+            ".items[].playback_date" => "[date]",
+        });
+    }
+}

src/client/pagination.rs

@@ -1,18 +1,28 @@
+use std::fmt::Debug;
+
 use crate::error::{Error, ExtractionError};
 use crate::model::{
     paginator::{ContinuationEndpoint, Paginator},
     traits::FromYtItem,
-    Comment, MusicItem, PlaylistVideo, YouTubeItem,
+    Comment, MusicItem, YouTubeItem,
 };
 use crate::serializer::MapResult;
-use crate::util::TryRemove;
 
-use super::response::music_item::{map_queue_item, MusicListMapper, PlaylistPanelVideo};
-use super::{response, ClientType, MapResponse, QContinuation, RustyPipeQuery};
+#[cfg(feature = "userdata")]
+use crate::model::{HistoryItem, TrackItem, VideoItem};
+
+use super::response::{
+    music_item::{map_queue_item, MusicListMapper, PlaylistPanelVideo},
+    YouTubeListItem,
+};
+use super::{
+    response, ClientType, MapRespCtx, MapRespOptions, MapResponse, QContinuation, RustyPipeQuery,
+};
 
 impl RustyPipeQuery {
     /// Get more YouTube items from the given continuation token and endpoint
-    pub async fn continuation<T: FromYtItem, S: AsRef<str>>(
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn continuation<T: FromYtItem, S: AsRef<str> + Debug>(
         &self,
         ctoken: S,
         endpoint: ContinuationEndpoint,
@@ -20,102 +30,118 @@ impl RustyPipeQuery {
     ) -> Result<Paginator<T>, Error> {
         let ctoken = ctoken.as_ref();
         if endpoint.is_music() {
-            let context = self
-                .get_context(ClientType::DesktopMusic, true, visitor_data)
-                .await;
             let request_body = QContinuation {
-                context,
                 continuation: ctoken,
             };
 
             let p = self
-                .execute_request::<response::MusicContinuation, Paginator<MusicItem>, _>(
+                .execute_request_ctx::<response::MusicContinuation, Paginator<MusicItem>, _>(
                     ClientType::DesktopMusic,
                     "music_continuation",
                     ctoken,
                     endpoint.as_str(),
                     &request_body,
+                    MapRespOptions {
+                        visitor_data,
+                        ..Default::default()
+                    },
                 )
                 .await?;
 
-            Ok(map_ytm_paginator(p, visitor_data, endpoint))
+            Ok(map_ytm_paginator(p, endpoint))
         } else {
-            let context = self
-                .get_context(ClientType::Desktop, true, visitor_data)
-                .await;
             let request_body = QContinuation {
-                context,
                 continuation: ctoken,
             };
 
             let p = self
-                .execute_request::<response::Continuation, Paginator<YouTubeItem>, _>(
+                .execute_request_ctx::<response::Continuation, Paginator<YouTubeItem>, _>(
                     ClientType::Desktop,
                     "continuation",
                     ctoken,
                     endpoint.as_str(),
                     &request_body,
+                    MapRespOptions {
+                        visitor_data,
+                        ..Default::default()
+                    },
                 )
                 .await?;
 
-            Ok(map_yt_paginator(p, visitor_data, endpoint))
+            Ok(map_yt_paginator(p, endpoint))
         }
     }
 }
 
 fn map_yt_paginator<T: FromYtItem>(
     p: Paginator<YouTubeItem>,
-    visitor_data: Option<&str>,
     endpoint: ContinuationEndpoint,
 ) -> Paginator<T> {
     Paginator {
         count: p.count,
         items: p.items.into_iter().filter_map(T::from_yt_item).collect(),
         ctoken: p.ctoken,
-        visitor_data: visitor_data.map(str::to_owned),
+        visitor_data: p.visitor_data,
         endpoint,
+        authenticated: p.authenticated,
     }
 }
 
 fn map_ytm_paginator<T: FromYtItem>(
     p: Paginator<MusicItem>,
-    visitor_data: Option<&str>,
     endpoint: ContinuationEndpoint,
 ) -> Paginator<T> {
     Paginator {
         count: p.count,
         items: p.items.into_iter().filter_map(T::from_ytm_item).collect(),
         ctoken: p.ctoken,
-        visitor_data: visitor_data.map(str::to_owned),
+        visitor_data: p.visitor_data,
         endpoint,
+        authenticated: p.authenticated,
     }
 }
 
+fn continuation_items(response: response::Continuation) -> MapResult<Vec<YouTubeListItem>> {
+    response
+        .on_response_received_actions
+        .and_then(|actions| {
+            actions
+                .into_iter()
+                .map(|action| action.append_continuation_items_action.continuation_items)
+                .reduce(|mut acc, mut items| {
+                    acc.c.append(&mut items.c);
+                    acc.warnings.append(&mut items.warnings);
+                    acc
+                })
+        })
+        .or_else(|| {
+            response
+                .continuation_contents
+                .map(|contents| contents.rich_grid_continuation.contents)
+        })
+        .unwrap_or_default()
+}
+
 impl MapResponse<Paginator<YouTubeItem>> for response::Continuation {
     fn map_response(
         self,
-        _id: &str,
-        lang: crate::param::Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
+        ctx: &MapRespCtx<'_>,
     ) -> Result<MapResult<Paginator<YouTubeItem>>, ExtractionError> {
-        let items = self
-            .on_response_received_actions
-            .and_then(|mut actions| {
-                actions
-                    .try_swap_remove(0)
-                    .map(|action| action.append_continuation_items_action.continuation_items)
-            })
-            .or_else(|| {
-                self.continuation_contents
-                    .map(|contents| contents.rich_grid_continuation.contents)
-            })
-            .unwrap_or_default();
+        let estimated_results = self.estimated_results;
+        let items = continuation_items(self);
 
-        let mut mapper = response::YouTubeListMapper::<YouTubeItem>::new(lang);
+        let mut mapper = response::YouTubeListMapper::<YouTubeItem>::new(ctx.lang);
         mapper.map_response(items);
 
         Ok(MapResult {
-            c: Paginator::new(self.estimated_results, mapper.items, mapper.ctoken),
+            c: Paginator::new_ext(
+                estimated_results,
+                mapper.items,
+                mapper.ctoken,
+                ctx.visitor_data.map(str::to_owned),
+                ContinuationEndpoint::Browse,
+                ctx.authenticated,
+            ),
             warnings: mapper.warnings,
         })
     }
@@ -124,11 +150,13 @@ impl MapResponse<Paginator<YouTubeItem>> for response::Continuation {
 impl MapResponse<Paginator<MusicItem>> for response::MusicContinuation {
     fn map_response(
         self,
-        _id: &str,
-        lang: crate::param::Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
+        ctx: &MapRespCtx<'_>,
     ) -> Result<MapResult<Paginator<MusicItem>>, ExtractionError> {
-        let mut mapper = MusicListMapper::new(lang);
+        let mut mapper = if let Some(artist) = &ctx.artist {
+            MusicListMapper::with_artist(ctx.lang, artist.clone())
+        } else {
+            MusicListMapper::new(ctx.lang)
+        };
         let mut continuations = Vec::new();
 
         match self.continuation_contents {
@@ -146,7 +174,11 @@ impl MapResponse<Paginator<MusicItem>> for response::MusicContinuation {
                         response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => {
                             mapper.map_response(shelf.contents);
                         }
-                        _ => {}
+                        response::music_item::ItemSection::GridRenderer(mut grid) => {
+                            mapper.map_response(grid.items);
+                            continuations.append(&mut grid.continuations);
+                        }
+                        response::music_item::ItemSection::None => {}
                     }
                 }
             }
@@ -157,20 +189,133 @@ impl MapResponse<Paginator<MusicItem>> for response::MusicContinuation {
                 mapper.add_warnings(&mut panel.contents.warnings);
                 panel.contents.c.into_iter().for_each(|item| {
                     if let PlaylistPanelVideo::PlaylistPanelVideoRenderer(item) = item {
-                        mapper.add_item(MusicItem::Track(map_queue_item(item, lang)))
+                        let mut track = map_queue_item(item, ctx.lang);
+                        mapper.add_item(MusicItem::Track(track.c));
+                        mapper.add_warnings(&mut track.warnings);
                     }
                 });
             }
+            Some(response::music_item::ContinuationContents::GridContinuation(mut grid)) => {
+                mapper.map_response(grid.items);
+                continuations.append(&mut grid.continuations);
+            }
             None => {}
         }
 
+        for a in self.on_response_received_actions {
+            mapper.map_response(a.append_continuation_items_action.continuation_items);
+        }
+
+        let ctoken = mapper.ctoken.clone().or_else(|| {
+            continuations
+                .into_iter()
+                .next()
+                .map(|cont| cont.next_continuation_data.continuation)
+        });
         let map_res = mapper.items();
+
+        Ok(MapResult {
+            c: Paginator::new_ext(
+                None,
+                map_res.c,
+                ctoken,
+                ctx.visitor_data.map(str::to_owned),
+                ContinuationEndpoint::MusicBrowse,
+                ctx.authenticated,
+            ),
+            warnings: map_res.warnings,
+        })
+    }
+}
+
+#[cfg(feature = "userdata")]
+impl MapResponse<Paginator<HistoryItem<VideoItem>>> for response::Continuation {
+    fn map_response(
+        self,
+        ctx: &MapRespCtx<'_>,
+    ) -> Result<MapResult<Paginator<HistoryItem<VideoItem>>>, ExtractionError> {
+        let mut map_res = MapResult::default();
+        let mut ctoken = None;
+
+        let items = continuation_items(self);
+        for item in items.c {
+            match item {
+                response::YouTubeListItem::ItemSectionRenderer { header, contents } => {
+                    let mut mapper = response::YouTubeListMapper::<VideoItem>::new(ctx.lang);
+                    mapper.map_response(contents);
+                    mapper.conv_history_items(
+                        header.map(|h| h.item_section_header_renderer.title),
+                        ctx.utc_offset,
+                        &mut map_res,
+                    );
+                }
+                response::YouTubeListItem::ContinuationItemRenderer(ep) => {
+                    if ctoken.is_none() {
+                        ctoken = ep.continuation_endpoint.into_token();
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        Ok(MapResult {
+            c: Paginator::new_ext(
+                None,
+                map_res.c,
+                ctoken,
+                ctx.visitor_data.map(str::to_owned),
+                ContinuationEndpoint::Browse,
+                ctx.authenticated,
+            ),
+            warnings: map_res.warnings,
+        })
+    }
+}
+
+#[cfg(feature = "userdata")]
+impl MapResponse<Paginator<HistoryItem<TrackItem>>> for response::MusicContinuation {
+    fn map_response(
+        self,
+        ctx: &MapRespCtx<'_>,
+    ) -> Result<MapResult<Paginator<HistoryItem<TrackItem>>>, ExtractionError> {
+        let mut map_res = MapResult::default();
+        let mut continuations = Vec::new();
+
+        let mut map_shelf = |shelf: response::music_item::MusicShelf| {
+            let mut mapper = MusicListMapper::new(ctx.lang);
+            mapper.map_response(shelf.contents);
+            mapper.conv_history_items(shelf.title, ctx.utc_offset, &mut map_res);
+            continuations.extend(shelf.continuations);
+        };
+
+        match self.continuation_contents {
+            Some(response::music_item::ContinuationContents::MusicShelfContinuation(shelf)) => {
+                map_shelf(shelf);
+            }
+            Some(response::music_item::ContinuationContents::SectionListContinuation(contents)) => {
+                for c in contents.contents {
+                    if let response::music_item::ItemSection::MusicShelfRenderer(shelf) = c {
+                        map_shelf(shelf);
+                    }
+                }
+            }
+            _ => {}
+        }
+
         let ctoken = continuations
-            .try_swap_remove(0)
+            .into_iter()
+            .next()
             .map(|cont| cont.next_continuation_data.continuation);
 
         Ok(MapResult {
-            c: Paginator::new(None, map_res.c, ctoken),
+            c: Paginator::new_ext(
+                None,
+                map_res.c,
+                ctoken,
+                ctx.visitor_data.map(str::to_owned),
+                ContinuationEndpoint::MusicBrowse,
+                ctx.authenticated,
+            ),
             warnings: map_res.warnings,
         })
     }
@@ -180,12 +325,18 @@ impl<T: FromYtItem> Paginator<T> {
     /// Get the next page from the paginator (or `None` if the paginator is exhausted)
     pub async fn next<Q: AsRef<RustyPipeQuery>>(&self, query: Q) -> Result<Option<Self>, Error> {
         Ok(match &self.ctoken {
-            Some(ctoken) => Some(
-                query
-                    .as_ref()
-                    .continuation(ctoken, self.endpoint, self.visitor_data.as_deref())
-                    .await?,
-            ),
+            Some(ctoken) => {
+                let q = if self.authenticated {
+                    &query.as_ref().clone().authenticated()
+                } else {
+                    query.as_ref()
+                };
+
+                Some(
+                    q.continuation(ctoken, self.endpoint, self.visitor_data.as_deref())
+                        .await?,
+                )
+            }
             _ => None,
         })
     }
@@ -199,6 +350,9 @@ impl<T: FromYtItem> Paginator<T> {
                 let mut items = paginator.items;
                 self.items.append(&mut items);
                 self.ctoken = paginator.ctoken;
+                if paginator.visitor_data.is_some() {
+                    self.visitor_data = paginator.visitor_data;
+                }
                 Ok(true)
             }
             Ok(None) => Ok(false),
@@ -241,6 +395,19 @@ impl<T: FromYtItem> Paginator<T> {
         }
         Ok(())
     }
+
+    /// Extend the items of the paginator until the paginator is exhausted.
+    pub async fn extend_all<Q: AsRef<RustyPipeQuery>>(&mut self, query: Q) -> Result<(), Error> {
+        let query = query.as_ref();
+        loop {
+            match self.extend(query).await {
+                Ok(false) => break,
+                Err(e) => return Err(e),
+                _ => {}
+            }
+        }
+        Ok(())
+    }
 }
 
 impl Paginator<Comment> {
@@ -258,12 +425,36 @@ impl Paginator<Comment> {
     }
 }
 
-impl Paginator<PlaylistVideo> {
+#[cfg(feature = "userdata")]
+#[cfg_attr(docsrs, doc(cfg(feature = "userdata")))]
+impl Paginator<HistoryItem<VideoItem>> {
     /// Get the next page from the paginator (or `None` if the paginator is exhausted)
     pub async fn next<Q: AsRef<RustyPipeQuery>>(&self, query: Q) -> Result<Option<Self>, Error> {
         Ok(match &self.ctoken {
-            Some(ctoken) => Some(query.as_ref().playlist_continuation(ctoken).await?),
-            None => None,
+            Some(ctoken) => Some(
+                query
+                    .as_ref()
+                    .history_continuation(ctoken, self.visitor_data.as_deref())
+                    .await?,
+            ),
+            _ => None,
+        })
+    }
+}
+
+#[cfg(feature = "userdata")]
+#[cfg_attr(docsrs, doc(cfg(feature = "userdata")))]
+impl Paginator<HistoryItem<TrackItem>> {
+    /// Get the next page from the paginator (or `None` if the paginator is exhausted)
+    pub async fn next<Q: AsRef<RustyPipeQuery>>(&self, query: Q) -> Result<Option<Self>, Error> {
+        Ok(match &self.ctoken {
+            Some(ctoken) => Some(
+                query
+                    .as_ref()
+                    .music_history_continuation(ctoken, self.visitor_data.as_deref())
+                    .await?,
+            ),
+            _ => None,
         })
     }
 }
@@ -283,6 +474,9 @@ macro_rules! paginator {
                         let mut items = paginator.items;
                         self.items.append(&mut items);
                         self.ctoken = paginator.ctoken;
+                        if paginator.visitor_data.is_some() {
+                            self.visitor_data = paginator.visitor_data;
+                        }
                         Ok(true)
                     }
                     Ok(None) => Ok(false),
@@ -325,12 +519,33 @@ macro_rules! paginator {
                 }
                 Ok(())
             }
+
+            /// Extend the items of the paginator until the paginator is exhausted.
+            pub async fn extend_all<Q: AsRef<RustyPipeQuery>>(
+                &mut self,
+                query: Q,
+            ) -> Result<(), Error> {
+                let query = query.as_ref();
+                loop {
+                    match self.extend(query).await {
+                        Ok(false) => break,
+                        Err(e) => return Err(e),
+                        _ => {}
+                    }
+                }
+                Ok(())
+            }
         }
     };
 }
 
 paginator!(Comment);
-paginator!(PlaylistVideo);
+#[cfg(feature = "userdata")]
+#[cfg_attr(docsrs, doc(cfg(feature = "userdata")))]
+paginator!(HistoryItem<VideoItem>);
+#[cfg(feature = "userdata")]
+#[cfg_attr(docsrs, doc(cfg(feature = "userdata")))]
+paginator!(HistoryItem<TrackItem>);
 
 #[cfg(test)]
 mod tests {
@@ -341,15 +556,16 @@ mod tests {
 
     use super::*;
     use crate::{
-        model::{MusicPlaylistItem, PlaylistItem, TrackItem},
-        param::Language,
+        model::{
+            AlbumItem, ArtistItem, ChannelItem, MusicPlaylistItem, PlaylistItem, TrackItem,
+            VideoItem,
+        },
         util::tests::TESTFILES,
     };
 
     #[rstest]
-    #[case("search", path!("search" / "cont.json"))]
-    #[case("startpage", path!("trends" / "startpage_cont.json"))]
-    #[case("recommendations", path!("video_details" / "recommendations.json"))]
+    #[case::search("search", path!("search" / "cont.json"))]
+    #[case::recommendations("recommendations", path!("video_details" / "recommendations.json"))]
     fn map_continuation_items(#[case] name: &str, #[case] path: PathBuf) {
         let json_path = path!(*TESTFILES / path);
         let json_file = File::open(json_path).unwrap();
@@ -357,7 +573,7 @@ mod tests {
         let items: response::Continuation =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
         let map_res: MapResult<Paginator<YouTubeItem>> =
-            items.map_response("", Language::En, None).unwrap();
+            items.map_response(&MapRespCtx::test("")).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),
@@ -370,7 +586,31 @@ mod tests {
     }
 
     #[rstest]
-    #[case("channel_playlists", path!("channel" / "channel_playlists_cont.json"))]
+    #[case::channel_videos("channel_videos", path!("channel" / "channel_videos_cont.json"))]
+    #[case::playlist("playlist", path!("playlist" / "playlist_cont.json"))]
+    fn map_continuation_videos(#[case] name: &str, #[case] path: PathBuf) {
+        let json_path = path!(*TESTFILES / path);
+        let json_file = File::open(json_path).unwrap();
+
+        let items: response::Continuation =
+            serde_json::from_reader(BufReader::new(json_file)).unwrap();
+        let map_res: MapResult<Paginator<YouTubeItem>> =
+            items.map_response(&MapRespCtx::test("")).unwrap();
+        let paginator: Paginator<VideoItem> =
+            map_yt_paginator(map_res.c, ContinuationEndpoint::Browse);
+
+        assert!(
+            map_res.warnings.is_empty(),
+            "deserialization/mapping warnings: {:?}",
+            map_res.warnings
+        );
+        insta::assert_ron_snapshot!(format!("map_{name}"), paginator, {
+            ".items[].publish_date" => "[date]",
+        });
+    }
+
+    #[rstest]
+    #[case::channel_playlists("channel_playlists", path!("channel" / "channel_playlists_cont.json"))]
     fn map_continuation_playlists(#[case] name: &str, #[case] path: PathBuf) {
         let json_path = path!(*TESTFILES / path);
         let json_file = File::open(json_path).unwrap();
@@ -378,9 +618,9 @@ mod tests {
         let items: response::Continuation =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
         let map_res: MapResult<Paginator<YouTubeItem>> =
-            items.map_response("", Language::En, None).unwrap();
+            items.map_response(&MapRespCtx::test("")).unwrap();
         let paginator: Paginator<PlaylistItem> =
-            map_yt_paginator(map_res.c, None, ContinuationEndpoint::Browse);
+            map_yt_paginator(map_res.c, ContinuationEndpoint::Browse);
 
         assert!(
             map_res.warnings.is_empty(),
@@ -391,9 +631,31 @@ mod tests {
     }
 
     #[rstest]
-    #[case("playlist_tracks", path!("music_playlist" / "playlist_cont.json"))]
-    #[case("search_tracks", path!("music_search" / "tracks_cont.json"))]
-    #[case("radio_tracks", path!("music_details" / "radio_cont.json"))]
+    #[case::subscriptions("subscriptions", path!("userdata" / "subscriptions.json"))]
+    fn map_continuation_channels(#[case] name: &str, #[case] path: PathBuf) {
+        let json_path = path!(*TESTFILES / path);
+        let json_file = File::open(json_path).unwrap();
+
+        let items: response::Continuation =
+            serde_json::from_reader(BufReader::new(json_file)).unwrap();
+        let map_res: MapResult<Paginator<YouTubeItem>> =
+            items.map_response(&MapRespCtx::test("")).unwrap();
+        let paginator: Paginator<ChannelItem> =
+            map_yt_paginator(map_res.c, ContinuationEndpoint::Browse);
+
+        assert!(
+            map_res.warnings.is_empty(),
+            "deserialization/mapping warnings: {:?}",
+            map_res.warnings
+        );
+        insta::assert_ron_snapshot!(format!("map_{name}"), paginator);
+    }
+
+    #[rstest]
+    #[case::playlist_tracks("playlist_tracks", path!("music_playlist" / "playlist_cont.json"))]
+    #[case::search_tracks("search_tracks", path!("music_search" / "tracks_cont.json"))]
+    #[case::radio_tracks("radio_tracks", path!("music_details" / "radio_cont.json"))]
+    #[case::saved_tracks("saved_tracks", path!("music_userdata" / "saved_tracks.json"))]
     fn map_continuation_tracks(#[case] name: &str, #[case] path: PathBuf) {
         let json_path = path!(*TESTFILES / path);
         let json_file = File::open(json_path).unwrap();
@@ -401,9 +663,9 @@ mod tests {
         let items: response::MusicContinuation =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
         let map_res: MapResult<Paginator<MusicItem>> =
-            items.map_response("", Language::En, None).unwrap();
+            items.map_response(&MapRespCtx::test("")).unwrap();
         let paginator: Paginator<TrackItem> =
-            map_ytm_paginator(map_res.c, None, ContinuationEndpoint::MusicBrowse);
+            map_ytm_paginator(map_res.c, ContinuationEndpoint::MusicBrowse);
 
         assert!(
             map_res.warnings.is_empty(),
@@ -414,7 +676,50 @@ mod tests {
     }
 
     #[rstest]
-    #[case("playlist_related", path!("music_playlist" / "playlist_related.json"))]
+    #[case::saved_artists("saved_artists", path!("music_userdata" / "saved_artists.json"))]
+    fn map_continuation_artists(#[case] name: &str, #[case] path: PathBuf) {
+        let json_path = path!(*TESTFILES / path);
+        let json_file = File::open(json_path).unwrap();
+
+        let items: response::MusicContinuation =
+            serde_json::from_reader(BufReader::new(json_file)).unwrap();
+        let map_res: MapResult<Paginator<MusicItem>> =
+            items.map_response(&MapRespCtx::test("")).unwrap();
+        let paginator: Paginator<ArtistItem> =
+            map_ytm_paginator(map_res.c, ContinuationEndpoint::MusicBrowse);
+
+        assert!(
+            map_res.warnings.is_empty(),
+            "deserialization/mapping warnings: {:?}",
+            map_res.warnings
+        );
+        insta::assert_ron_snapshot!(format!("map_{name}"), paginator);
+    }
+
+    #[rstest]
+    #[case::saved_albums("saved_albums", path!("music_userdata" / "saved_albums.json"))]
+    fn map_continuation_albums(#[case] name: &str, #[case] path: PathBuf) {
+        let json_path = path!(*TESTFILES / path);
+        let json_file = File::open(json_path).unwrap();
+
+        let items: response::MusicContinuation =
+            serde_json::from_reader(BufReader::new(json_file)).unwrap();
+        let map_res: MapResult<Paginator<MusicItem>> =
+            items.map_response(&MapRespCtx::test("")).unwrap();
+        let paginator: Paginator<AlbumItem> =
+            map_ytm_paginator(map_res.c, ContinuationEndpoint::MusicBrowse);
+
+        assert!(
+            map_res.warnings.is_empty(),
+            "deserialization/mapping warnings: {:?}",
+            map_res.warnings
+        );
+        insta::assert_ron_snapshot!(format!("map_{name}"), paginator);
+    }
+
+    #[rstest]
+    #[case::playlist_related("playlist_related", path!("music_playlist" / "playlist_related.json"))]
+    #[case::saved_playlists("saved_playlists", path!("music_userdata" / "saved_playlists.json"))]
     fn map_continuation_music_playlists(#[case] name: &str, #[case] path: PathBuf) {
         let json_path = path!(*TESTFILES / path);
         let json_file = File::open(json_path).unwrap();
@@ -422,9 +727,9 @@ mod tests {
         let items: response::MusicContinuation =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
         let map_res: MapResult<Paginator<MusicItem>> =
-            items.map_response("", Language::En, None).unwrap();
+            items.map_response(&MapRespCtx::test("")).unwrap();
         let paginator: Paginator<MusicPlaylistItem> =
-            map_ytm_paginator(map_res.c, None, ContinuationEndpoint::MusicBrowse);
+            map_ytm_paginator(map_res.c, ContinuationEndpoint::MusicBrowse);
 
         assert!(
             map_res.warnings.is_empty(),

src/client/playlist.rs

@@ -1,23 +1,26 @@
-use std::{borrow::Cow, convert::TryFrom};
+use std::{borrow::Cow, convert::TryFrom, fmt::Debug};
 
 use time::OffsetDateTime;
 
 use crate::{
     error::{Error, ExtractionError},
-    model::{paginator::Paginator, ChannelId, Playlist, PlaylistVideo},
-    timeago,
-    util::{self, TryRemove},
+    model::{
+        paginator::{ContinuationEndpoint, Paginator},
+        richtext::RichText,
+        ChannelId, Playlist, VideoItem,
+    },
+    serializer::text::{TextComponent, TextComponents},
+    util::{self, dictionary, timeago, TryRemove},
 };
 
-use super::{response, ClientType, MapResponse, MapResult, QBrowse, QContinuation, RustyPipeQuery};
+use super::{response, ClientType, MapRespCtx, MapResponse, MapResult, QBrowse, RustyPipeQuery};
 
 impl RustyPipeQuery {
     /// Get a YouTube playlist
-    pub async fn playlist<S: AsRef<str>>(&self, playlist_id: S) -> Result<Playlist, Error> {
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn playlist<S: AsRef<str> + Debug>(&self, playlist_id: S) -> Result<Playlist, Error> {
         let playlist_id = playlist_id.as_ref();
-        let context = self.get_context(ClientType::Desktop, true, None).await;
         let request_body = QBrowse {
-            context,
             browse_id: &format!("VL{playlist_id}"),
         };
 
@@ -30,46 +33,19 @@ impl RustyPipeQuery {
         )
         .await
     }
-
-    /// Get more playlist items using the given continuation token
-    pub async fn playlist_continuation<S: AsRef<str>>(
-        &self,
-        ctoken: S,
-    ) -> Result<Paginator<PlaylistVideo>, Error> {
-        let ctoken = ctoken.as_ref();
-        let context = self.get_context(ClientType::Desktop, true, None).await;
-        let request_body = QContinuation {
-            context,
-            continuation: ctoken,
-        };
-
-        self.execute_request::<response::PlaylistCont, _, _>(
-            ClientType::Desktop,
-            "playlist_continuation",
-            ctoken,
-            "browse",
-            &request_body,
-        )
-        .await
-    }
 }
 
 impl MapResponse<Playlist> for response::Playlist {
-    fn map_response(
-        self,
-        id: &str,
-        lang: crate::param::Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
-    ) -> Result<MapResult<Playlist>, ExtractionError> {
-        let (contents, header) = match (self.contents, self.header) {
-            (Some(contents), Some(header)) => (contents, header),
-            _ => return Err(response::alerts_to_err(self.alerts)),
+    fn map_response(self, ctx: &MapRespCtx<'_>) -> Result<MapResult<Playlist>, ExtractionError> {
+        let (Some(contents), Some(header)) = (self.contents, self.header) else {
+            return Err(response::alerts_to_err(ctx.id, self.alerts));
         };
 
-        let mut tcbr_contents = contents.two_column_browse_results_renderer.contents;
-
-        let video_items = tcbr_contents
-            .try_swap_remove(0)
+        let video_items = contents
+            .two_column_browse_results_renderer
+            .contents
+            .into_iter()
+            .next()
             .ok_or(ExtractionError::InvalidData(Cow::Borrowed(
                 "twoColumnBrowseResultsRenderer empty",
             )))?
@@ -77,27 +53,31 @@ impl MapResponse<Playlist> for response::Playlist {
             .content
             .section_list_renderer
             .contents
-            .try_swap_remove(0)
+            .into_iter()
+            .next()
             .ok_or(ExtractionError::InvalidData(Cow::Borrowed(
                 "sectionListRenderer empty",
             )))?
             .item_section_renderer
             .contents
-            .try_swap_remove(0)
+            .into_iter()
+            .next()
             .ok_or(ExtractionError::InvalidData(Cow::Borrowed(
                 "itemSectionRenderer empty",
             )))?
             .playlist_video_list_renderer
             .contents;
 
-        let (videos, ctoken) = map_playlist_items(video_items.c);
+        let mut mapper = response::YouTubeListMapper::<VideoItem>::new(ctx.lang);
+        mapper.map_response(video_items);
 
-        let (thumbnails, last_update_txt) = match self.sidebar {
+        let (description, thumbnails, last_update_txt) = match self.sidebar {
             Some(sidebar) => {
-                let mut sidebar_items = sidebar.playlist_sidebar_renderer.items;
+                let sidebar_items = sidebar.playlist_sidebar_renderer.contents;
                 let mut primary =
                     sidebar_items
-                        .try_swap_remove(0)
+                        .into_iter()
+                        .next()
                         .ok_or(ExtractionError::InvalidData(Cow::Borrowed(
                             "no primary sidebar",
                         )))?;
@@ -105,130 +85,161 @@ impl MapResponse<Playlist> for response::Playlist {
                 (
                     primary
                         .playlist_sidebar_primary_info_renderer
-                        .thumbnail_renderer
-                        .playlist_video_thumbnail_renderer
-                        .thumbnail,
+                        .description
+                        .filter(|d| !d.0.is_empty()),
+                    Some(
+                        primary
+                            .playlist_sidebar_primary_info_renderer
+                            .thumbnail_renderer
+                            .playlist_video_thumbnail_renderer
+                            .thumbnail,
+                    ),
                     primary
                         .playlist_sidebar_primary_info_renderer
                         .stats
                         .try_swap_remove(2),
                 )
             }
-            None => {
-                let header_banner = header
-                    .playlist_header_renderer
-                    .playlist_header_banner
-                    .ok_or(ExtractionError::InvalidData(Cow::Borrowed(
-                        "no thumbnail found",
-                    )))?;
-
-                let mut byline = header.playlist_header_renderer.byline;
-                let last_update_txt = byline
-                    .try_swap_remove(1)
-                    .map(|b| b.playlist_byline_renderer.text);
-
-                (
-                    header_banner.hero_playlist_thumbnail_renderer.thumbnail,
-                    last_update_txt,
-                )
-            }
+            None => (None, None, None),
         };
 
-        let n_videos = match ctoken {
-            Some(_) => util::parse_numeric(&header.playlist_header_renderer.num_videos_text)
-                .map_err(|_| ExtractionError::InvalidData(Cow::Borrowed("no video count")))?,
-            None => videos.len() as u64,
+        let (name, playlist_id, channel, n_videos_txt, description2, thumbnails2, last_update_txt2) =
+            match header {
+                response::playlist::Header::PlaylistHeaderRenderer(header_renderer) => {
+                    let mut byline = header_renderer.byline;
+                    let last_update_txt = byline
+                        .try_swap_remove(1)
+                        .map(|b| b.playlist_byline_renderer.text);
+
+                    (
+                        header_renderer.title,
+                        header_renderer.playlist_id,
+                        header_renderer
+                            .owner_text
+                            .and_then(|link| ChannelId::try_from(link).ok()),
+                        header_renderer.num_videos_text,
+                        header_renderer
+                            .description_text
+                            .map(|text| TextComponents(vec![TextComponent::new(text)])),
+                        header_renderer
+                            .playlist_header_banner
+                            .map(|b| b.hero_playlist_thumbnail_renderer.thumbnail),
+                        last_update_txt,
+                    )
+                }
+                response::playlist::Header::PageHeaderRenderer(content_renderer) => {
+                    let h = content_renderer.content.page_header_view_model;
+                    let rows = h.metadata.content_metadata_view_model.metadata_rows;
+                    let n_videos_txt = rows
+                        .get(1)
+                        .and_then(|r| r.metadata_parts.get(1))
+                        .map(|p| p.as_str().to_owned())
+                        .ok_or(ExtractionError::InvalidData("no video count".into()))?;
+                    let mut channel = rows
+                        .into_iter()
+                        .next()
+                        .and_then(|r| r.metadata_parts.into_iter().next())
+                        .and_then(|p| match p {
+                            response::MetadataPart::Text { .. } => None,
+                            response::MetadataPart::AvatarStack { avatar_stack } => {
+                                ChannelId::try_from(avatar_stack.avatar_stack_view_model.text).ok()
+                            }
+                        });
+                    // remove "by" prefix
+                    if let Some(c) = channel.as_mut() {
+                        let entry = dictionary::entry(ctx.lang);
+                        let n = c.name.strip_prefix(entry.chan_prefix).unwrap_or(&c.name);
+                        let n = n.strip_suffix(entry.chan_suffix).unwrap_or(n);
+                        c.name = n.trim().to_owned();
+                    }
+
+                    let playlist_id = h
+                        .actions
+                        .flexible_actions_view_model
+                        .actions_rows
+                        .into_iter()
+                        .next()
+                        .and_then(|r| r.actions.into_iter().next())
+                        .and_then(|a| {
+                            a.button_view_model
+                                .on_tap
+                                .innertube_command
+                                .into_playlist_id()
+                        })
+                        .ok_or(ExtractionError::InvalidData("no playlist id".into()))?;
+                    (
+                        h.title.dynamic_text_view_model.text,
+                        playlist_id,
+                        channel,
+                        n_videos_txt,
+                        h.description.description_preview_view_model.description,
+                        h.hero_image.content_preview_image_view_model.image.into(),
+                        None,
+                    )
+                }
+            };
+
+        let n_videos = if mapper.ctoken.is_some() {
+            util::parse_numeric(&n_videos_txt)
+                .map_err(|_| ExtractionError::InvalidData("no video count".into()))?
+        } else {
+            mapper.items.len() as u64
         };
 
-        let playlist_id = header.playlist_header_renderer.playlist_id;
-        if playlist_id != id {
+        if playlist_id != ctx.id {
             return Err(ExtractionError::WrongResult(format!(
-                "got wrong playlist id {playlist_id}, expected {id}"
+                "got wrong playlist id {}, expected {}",
+                playlist_id, ctx.id
             )));
         }
 
-        let name = header.playlist_header_renderer.title;
-        let description = header.playlist_header_renderer.description_text;
-        let channel = header
-            .playlist_header_renderer
-            .owner_text
-            .and_then(|link| ChannelId::try_from(link).ok());
-
-        let mut warnings = video_items.warnings;
-        let last_update = last_update_txt.as_ref().and_then(|txt| {
-            timeago::parse_textual_date_or_warn(lang, txt, &mut warnings).map(OffsetDateTime::date)
-        });
+        let description = description.or(description2).map(RichText::from);
+        let thumbnails = thumbnails
+            .or(thumbnails2)
+            .ok_or(ExtractionError::InvalidData(Cow::Borrowed(
+                "no thumbnail found",
+            )))?;
+        let last_update = last_update_txt
+            .as_deref()
+            .or(last_update_txt2.as_deref())
+            .and_then(|txt| {
+                timeago::parse_textual_date_or_warn(
+                    ctx.lang,
+                    ctx.utc_offset,
+                    txt,
+                    &mut mapper.warnings,
+                )
+                .map(OffsetDateTime::date)
+            });
 
         Ok(MapResult {
             c: Playlist {
                 id: playlist_id,
                 name,
-                videos: Paginator::new(Some(n_videos), videos, ctoken),
+                videos: Paginator::new_ext(
+                    Some(n_videos),
+                    mapper.items,
+                    mapper.ctoken,
+                    ctx.visitor_data.map(str::to_owned),
+                    ContinuationEndpoint::Browse,
+                    ctx.authenticated,
+                ),
                 video_count: n_videos,
                 thumbnail: thumbnails.into(),
                 description,
                 channel,
                 last_update,
                 last_update_txt,
-                visitor_data: self.response_context.visitor_data,
+                visitor_data: self
+                    .response_context
+                    .visitor_data
+                    .or_else(|| ctx.visitor_data.map(str::to_owned)),
             },
-            warnings,
+            warnings: mapper.warnings,
         })
     }
 }
 
-impl MapResponse<Paginator<PlaylistVideo>> for response::PlaylistCont {
-    fn map_response(
-        self,
-        _id: &str,
-        _lang: crate::param::Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
-    ) -> Result<MapResult<Paginator<PlaylistVideo>>, ExtractionError> {
-        let action = self.on_response_received_actions.into_iter().next();
-
-        let ((items, ctoken), warnings) = action
-            .map(|action| {
-                (
-                    map_playlist_items(
-                        action.append_continuation_items_action.continuation_items.c,
-                    ),
-                    action
-                        .append_continuation_items_action
-                        .continuation_items
-                        .warnings,
-                )
-            })
-            .unwrap_or_default();
-
-        Ok(MapResult {
-            c: Paginator::new(None, items, ctoken),
-            warnings,
-        })
-    }
-}
-
-fn map_playlist_items(
-    items: Vec<response::playlist::PlaylistItem>,
-) -> (Vec<PlaylistVideo>, Option<String>) {
-    let mut ctoken: Option<String> = None;
-    let videos = items
-        .into_iter()
-        .filter_map(|it| match it {
-            response::playlist::PlaylistItem::PlaylistVideoRenderer(video) => {
-                PlaylistVideo::try_from(video).ok()
-            }
-            response::playlist::PlaylistItem::ContinuationItemRenderer {
-                continuation_endpoint,
-            } => {
-                ctoken = Some(continuation_endpoint.continuation_command.token);
-                None
-            }
-            response::playlist::PlaylistItem::None => None,
-        })
-        .collect::<Vec<_>>();
-    (videos, ctoken)
-}
-
 #[cfg(test)]
 mod tests {
     use std::{fs::File, io::BufReader};
@@ -236,7 +247,7 @@ mod tests {
     use path_macro::path;
     use rstest::rstest;
 
-    use crate::{param::Language, util::tests::TESTFILES};
+    use crate::util::tests::TESTFILES;
 
     use super::*;
 
@@ -244,13 +255,16 @@ mod tests {
     #[case::short("short", "RDCLAK5uy_kFQXdnqMaQCVx2wpUM4ZfbsGCDibZtkJk")]
     #[case::long("long", "PL5dDx681T4bR7ZF1IuWzOv1omlRbE7PiJ")]
     #[case::nomusic("nomusic", "PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe")]
+    #[case::live("live", "UULVvqRdlKsE5Q8mf8YXbdIJLw")]
+    #[case::pageheader("20241011_pageheader", "PLT2w2oBf1TZKyvY_M6JsASs73m-wjLzH5")]
+    #[case::cmdexecutor("20250316_cmdexecutor", "PLbZIPy20-1pN7mqjckepWF78ndb6ci_qi")]
     fn map_playlist_data(#[case] name: &str, #[case] id: &str) {
         let json_path = path!(*TESTFILES / "playlist" / format!("playlist_{name}.json"));
         let json_file = File::open(json_path).unwrap();
 
         let playlist: response::Playlist =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
-        let map_res = playlist.map_response(id, Language::En, None).unwrap();
+        let map_res = playlist.map_response(&MapRespCtx::test(id)).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),
@@ -258,24 +272,8 @@ mod tests {
             map_res.warnings
         );
         insta::assert_ron_snapshot!(format!("map_playlist_data_{name}"), map_res.c, {
-            ".last_update" => "[date]"
+            ".last_update" => "[date]",
+            ".videos.items[].publish_date" => "[date]",
         });
     }
-
-    #[test]
-    fn map_playlist_cont() {
-        let json_path = path!(*TESTFILES / "playlist" / "playlist_cont.json");
-        let json_file = File::open(json_path).unwrap();
-
-        let playlist: response::PlaylistCont =
-            serde_json::from_reader(BufReader::new(json_file)).unwrap();
-        let map_res = playlist.map_response("", Language::En, None).unwrap();
-
-        assert!(
-            map_res.warnings.is_empty(),
-            "deserialization/mapping warnings: {:?}",
-            map_res.warnings
-        );
-        insta::assert_ron_snapshot!("map_playlist_cont", map_res.c);
-    }
 }

src/client/response/channel.rs

@@ -2,10 +2,14 @@ use serde::Deserialize;
 use serde_with::{rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError};
 
 use super::{
-    video_item::YouTubeListRenderer, Alert, ChannelBadge, ContentsRenderer, ResponseContext,
-    Thumbnails,
+    video_item::YouTubeListRenderer, Alert, AttachmentRun, AvatarViewModel, ChannelBadge,
+    ContentRenderer, ContentsRenderer, ContinuationActionWrap, ImageView,
+    PageHeaderRendererContent, PhMetadataView, ResponseContext, Thumbnails, TwoColumnBrowseResults,
+};
+use crate::{
+    model::Verification,
+    serializer::text::{AttributedText, Text, TextComponent},
 };
-use crate::serializer::text::Text;
 
 #[serde_as]
 #[derive(Debug, Deserialize)]
@@ -22,21 +26,7 @@ pub(crate) struct Channel {
     pub response_context: ResponseContext,
 }
 
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct Contents {
-    pub two_column_browse_results_renderer: TabsRenderer,
-}
-
-/// YouTube channel tab view. Contains multiple tabs
-/// (Home, Videos, Playlists, About...). We can ignore unknown tabs.
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct TabsRenderer {
-    #[serde_as(as = "VecSkipError<_>")]
-    pub tabs: Vec<TabRendererWrap>,
-}
+pub(crate) type Contents = TwoColumnBrowseResults<TabRendererWrap>;
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -50,7 +40,7 @@ pub(crate) struct TabRendererWrap {
 pub(crate) struct TabRenderer {
     #[serde(default)]
     pub content: TabContent,
-    pub endpoint: ChannelTabEndpoint,
+    pub endpoint: Option<ChannelTabEndpoint>,
 }
 
 #[serde_as]
@@ -85,10 +75,12 @@ pub(crate) struct ChannelTabWebCommandMetadata {
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
+#[allow(clippy::enum_variant_names)]
 pub(crate) enum Header {
     C4TabbedHeaderRenderer(HeaderRenderer),
     /// Used for special channels like YouTube Music
     CarouselHeaderRenderer(ContentsRenderer<CarouselHeaderRendererItem>),
+    PageHeaderRenderer(ContentRenderer<PageHeaderRendererContent<PageHeaderRendererInner>>),
 }
 
 #[serde_as]
@@ -107,11 +99,6 @@ pub(crate) struct HeaderRenderer {
     pub badges: Vec<ChannelBadge>,
     #[serde(default)]
     pub banner: Thumbnails,
-    #[serde(default)]
-    pub mobile_banner: Thumbnails,
-    /// Fullscreen (16:9) channel banner
-    #[serde(default)]
-    pub tv_banner: Thumbnails,
 }
 
 #[serde_as]
@@ -122,6 +109,8 @@ pub(crate) enum CarouselHeaderRendererItem {
     TopicChannelDetailsRenderer {
         #[serde_as(as = "Option<Text>")]
         subscriber_count_text: Option<String>,
+        #[serde_as(as = "Option<Text>")]
+        subtitle: Option<String>,
         #[serde(default)]
         avatar: Thumbnails,
     },
@@ -129,6 +118,59 @@ pub(crate) enum CarouselHeaderRendererItem {
     None,
 }
 
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PageHeaderRendererInner {
+    /// Channel title (only used to extract verification badges)
+    #[serde_as(as = "DefaultOnError")]
+    pub title: Option<PhTitleView>,
+    /// Channel avatar
+    pub image: PhAvatarView,
+    /// Channel metadata (subscribers, video count)
+    pub metadata: PhMetadataView,
+    #[serde(default)]
+    pub banner: PhBannerView,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PhTitleView {
+    pub dynamic_text_view_model: PhTitleView2,
+}
+
+#[derive(Default, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PhTitleView2 {
+    pub text: PhTitleView3,
+}
+
+#[serde_as]
+#[derive(Default, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PhTitleView3 {
+    #[serde_as(as = "VecSkipError<_>")]
+    pub attachment_runs: Vec<AttachmentRun>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PhAvatarView {
+    pub decorated_avatar_view_model: PhAvatarView2,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PhAvatarView2 {
+    pub avatar: AvatarViewModel,
+}
+
+#[derive(Default, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PhBannerView {
+    pub image_banner_view_model: ImageView,
+}
+
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct Metadata {
@@ -157,3 +199,85 @@ pub(crate) struct MicroformatDataRenderer {
     #[serde(default)]
     pub tags: Vec<String>,
 }
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(untagged)]
+pub(crate) enum ChannelAbout {
+    #[serde(rename_all = "camelCase")]
+    ReceivedEndpoints {
+        #[serde_as(as = "VecSkipError<_>")]
+        on_response_received_endpoints: Vec<ContinuationActionWrap<AboutChannelRendererWrap>>,
+    },
+    Content {
+        contents: Option<Contents>,
+    },
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct AboutChannelRendererWrap {
+    pub about_channel_renderer: AboutChannelRenderer,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct AboutChannelRenderer {
+    pub metadata: ChannelMetadata,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ChannelMetadata {
+    pub about_channel_view_model: ChannelMetadataView,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ChannelMetadataView {
+    pub channel_id: String,
+    pub canonical_channel_url: String,
+    pub country: Option<String>,
+    #[serde(default)]
+    pub description: String,
+    #[serde_as(as = "Option<Text>")]
+    pub joined_date_text: Option<String>,
+    #[serde_as(as = "Option<Text>")]
+    pub subscriber_count_text: Option<String>,
+    #[serde_as(as = "Option<Text>")]
+    pub video_count_text: Option<String>,
+    #[serde_as(as = "Option<Text>")]
+    pub view_count_text: Option<String>,
+    #[serde(default)]
+    pub links: Vec<ExternalLink>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ExternalLink {
+    pub channel_external_link_view_model: ExternalLinkInner,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ExternalLinkInner {
+    #[serde_as(as = "AttributedText")]
+    pub title: TextComponent,
+    #[serde_as(as = "AttributedText")]
+    pub link: TextComponent,
+}
+
+impl From<PhTitleView> for crate::model::Verification {
+    fn from(value: PhTitleView) -> Self {
+        value
+            .dynamic_text_view_model
+            .text
+            .attachment_runs
+            .into_iter()
+            .next()
+            .map(Verification::from)
+            .unwrap_or_default()
+    }
+}

src/client/response/channel_rss.rs

@@ -1,8 +1,6 @@
 use serde::Deserialize;
 use time::OffsetDateTime;
 
-use crate::util;
-
 #[derive(Debug, Deserialize)]
 pub(crate) struct ChannelRss {
     #[serde(rename = "channelId")]
@@ -80,54 +78,3 @@ impl From<Thumbnail> for crate::model::Thumbnail {
         }
     }
 }
-
-impl From<ChannelRss> for crate::model::ChannelRss {
-    fn from(feed: ChannelRss) -> Self {
-        let id = if feed.channel_id.is_empty() {
-            feed.entry
-                .iter()
-                .find_map(|entry| {
-                    if !entry.channel_id.is_empty() {
-                        Some(entry.channel_id.to_owned())
-                    } else {
-                        None
-                    }
-                })
-                .or_else(|| {
-                    feed.author
-                        .uri
-                        .strip_prefix("https://www.youtube.com/channel/")
-                        .and_then(|id| {
-                            if util::CHANNEL_ID_REGEX.is_match(id) {
-                                Some(id.to_owned())
-                            } else {
-                                None
-                            }
-                        })
-                })
-                .unwrap_or_default()
-        } else {
-            feed.channel_id
-        };
-
-        Self {
-            id,
-            name: feed.title,
-            videos: feed
-                .entry
-                .into_iter()
-                .map(|item| crate::model::ChannelRssVideo {
-                    id: item.video_id,
-                    name: item.title,
-                    description: item.media_group.description,
-                    thumbnail: item.media_group.thumbnail.into(),
-                    publish_date: item.published,
-                    update_date: item.updated,
-                    view_count: item.media_group.community.statistics.views,
-                    like_count: item.media_group.community.rating.count,
-                })
-                .collect(),
-            create_date: feed.create_date,
-        }
-    }
-}

src/client/response/channel_tv.rs

@@ -1,202 +0,0 @@
-use serde::Deserialize;
-use serde_with::{serde_as, VecSkipError};
-
-use super::{
-    url_endpoint::NavigationEndpoint, video_item::TimeOverlay, ContentRenderer, ResponseContext,
-    Thumbnails,
-};
-use crate::serializer::{text::Text, MapResult, VecLogError};
-
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct ChannelTv {
-    pub contents: Contents,
-    pub response_context: ResponseContext,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct Contents {
-    pub tv_browse_renderer: ContentRenderer<TvSurface>,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct TvSurface {
-    pub tv_surface_content_renderer: SurfaceContentRenderer,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct SurfaceContentRenderer {
-    #[serde(default)]
-    pub content: SurfaceContent,
-    pub header: SurfaceHeader,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct SurfaceHeader {
-    pub tv_surface_header_renderer: SurfaceHeaderRenderer,
-}
-
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct SurfaceHeaderRenderer {
-    // TODO: really?
-    // #[serde(default)]
-    #[serde_as(as = "Text")]
-    pub title: String,
-    /// Channel avatar
-    #[serde(default)]
-    pub thumbnail: Thumbnails,
-    #[serde(default)]
-    pub banner: Thumbnails,
-    #[serde_as(as = "VecSkipError<_>")]
-    pub buttons: Vec<SubscribeButton>,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct SubscribeButton {
-    pub subscribe_button_renderer: SubscribeButtonRenderer,
-}
-
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct SubscribeButtonRenderer {
-    pub channel_id: String,
-    #[serde_as(as = "Option<Text>")]
-    pub subscriber_count_text: Option<String>,
-}
-
-#[derive(Default, Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct SurfaceContent {
-    pub section_list_renderer: SectionList,
-}
-
-#[serde_as]
-#[derive(Default, Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct SectionList {
-    #[serde_as(as = "VecSkipError<_>")]
-    pub contents: Vec<Shelf>,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct Shelf {
-    pub shelf_renderer: ShelfRenderer,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct ShelfRenderer {
-    pub content: ShelfContent,
-    pub endpoint: NavigationEndpoint,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct ShelfContent {
-    pub horizontal_list_renderer: HorizontalListRenderer,
-}
-
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct HorizontalListRenderer {
-    #[serde_as(as = "VecLogError<_>")]
-    pub items: MapResult<Vec<Tile>>,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct Tile {
-    pub tile_renderer: TileRenderer,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct TileRenderer {
-    pub content_id: String,
-    pub content_type: ContentType,
-    pub header: TileHeader,
-    pub metadata: Metadata,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct TileHeader {
-    pub tile_header_renderer: TileHeaderRenderer,
-}
-
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct TileHeaderRenderer {
-    pub thumbnail: Thumbnails,
-    /// Contains Live tag
-    #[serde(default)]
-    #[serde_as(as = "VecSkipError<_>")]
-    pub thumbnail_overlays: Vec<TimeOverlay>,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct Metadata {
-    pub tile_metadata_renderer: MetadataRenderer,
-}
-
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct MetadataRenderer {
-    #[serde_as(as = "Text")]
-    pub title: String,
-    #[serde(default)]
-    #[serde_as(as = "VecSkipError<_>")]
-    pub lines: Vec<Line>,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct Line {
-    pub line_renderer: LineRenderer,
-}
-
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct LineRenderer {
-    #[serde_as(as = "VecSkipError<_>")]
-    pub items: Vec<LineItem>,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct LineItem {
-    pub line_item_renderer: LineItemRenderer,
-}
-
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct LineItemRenderer {
-    #[serde_as(as = "Text")]
-    pub text: String,
-}
-
-#[derive(Debug, Deserialize)]
-pub(crate) enum ContentType {
-    #[serde(rename = "TILE_CONTENT_TYPE_VIDEO")]
-    Video,
-    #[serde(rename = "TILE_CONTENT_TYPE_CHANNEL")]
-    Channel,
-    #[serde(rename = "TILE_CONTENT_TYPE_PLAYLIST")]
-    Playlist,
-}

src/client/response/history.rs

@@ -0,0 +1,8 @@
+use serde::Deserialize;
+
+use super::{video_item::YouTubeListRendererWrap, Tab, TwoColumnBrowseResults};
+
+#[derive(Debug, Deserialize)]
+pub(crate) struct History {
+    pub contents: TwoColumnBrowseResults<Tab<YouTubeListRendererWrap>>,
+}

src/client/response/mod.rs

@@ -1,5 +1,4 @@
 pub(crate) mod channel;
-pub(crate) mod channel_tv;
 pub(crate) mod music_artist;
 pub(crate) mod music_charts;
 pub(crate) mod music_details;
@@ -17,7 +16,7 @@ pub(crate) mod video_details;
 pub(crate) mod video_item;
 
 pub(crate) use channel::Channel;
-pub(crate) use channel_tv::ChannelTv;
+pub(crate) use channel::ChannelAbout;
 pub(crate) use music_artist::MusicArtist;
 pub(crate) use music_artist::MusicArtistAlbums;
 pub(crate) use music_charts::MusicCharts;
@@ -31,11 +30,11 @@ pub(crate) use music_new::MusicNew;
 pub(crate) use music_playlist::MusicPlaylist;
 pub(crate) use music_search::MusicSearch;
 pub(crate) use music_search::MusicSearchSuggestion;
+pub(crate) use player::DrmLicense;
 pub(crate) use player::Player;
 pub(crate) use playlist::Playlist;
-pub(crate) use playlist::PlaylistCont;
 pub(crate) use search::Search;
-pub(crate) use trends::Startpage;
+pub(crate) use search::SearchSuggestion;
 pub(crate) use trends::Trending;
 pub(crate) use url_endpoint::ResolvedUrl;
 pub(crate) use video_details::VideoComments;
@@ -48,12 +47,28 @@ pub(crate) mod channel_rss;
 #[cfg(feature = "rss")]
 pub(crate) use channel_rss::ChannelRss;
 
-use serde::Deserialize;
-use serde_with::{json::JsonString, serde_as, VecSkipError};
+#[cfg(feature = "userdata")]
+pub(crate) mod history;
+#[cfg(feature = "userdata")]
+pub(crate) use history::History;
+#[cfg(feature = "userdata")]
+pub(crate) mod music_history;
+#[cfg(feature = "userdata")]
+pub(crate) use music_history::MusicHistory;
+
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::marker::PhantomData;
+
+use serde::{
+    de::{IgnoredAny, Visitor},
+    Deserialize,
+};
+use serde_with::{serde_as, DisplayFromStr, VecSkipError};
 
 use crate::error::ExtractionError;
-use crate::serializer::MapResult;
-use crate::serializer::{text::Text, VecLogError};
+use crate::serializer::text::{AttributedText, Text, TextComponent};
+use crate::serializer::{MapResult, VecSkipErrorWrap};
 
 use self::video_item::YouTubeListRenderer;
 
@@ -63,13 +78,20 @@ pub(crate) struct ContentRenderer<T> {
     pub content: T,
 }
 
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
+/// Deserializes any object with an array field named `contents`, `tabs` or `items`.
+///
+/// Invalid items are skipped
+#[derive(Debug)]
 pub(crate) struct ContentsRenderer<T> {
-    #[serde(alias = "tabs")]
     pub contents: Vec<T>,
 }
 
+#[derive(Debug, Deserialize)]
+pub(crate) struct ContentsRendererLogged<T> {
+    #[serde(alias = "items")]
+    pub contents: MapResult<Vec<T>>,
+}
+
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct Tab<T> {
@@ -82,6 +104,12 @@ pub(crate) struct SectionList<T> {
     pub section_list_renderer: ContentsRenderer<T>,
 }
 
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct TwoColumnBrowseResults<T> {
+    pub two_column_browse_results_renderer: ContentsRenderer<T>,
+}
+
 #[derive(Default, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct ThumbnailsWrap {
@@ -89,12 +117,24 @@ pub(crate) struct ThumbnailsWrap {
     pub thumbnail: Thumbnails,
 }
 
+#[derive(Default, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ImageView {
+    pub image: Thumbnails,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct AvatarViewModel {
+    pub avatar_view_model: ImageView,
+}
+
 /// List of images in different resolutions.
 /// Not only used for thumbnails, but also for avatars and banners.
 #[derive(Default, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct Thumbnails {
-    #[serde(default)]
+    #[serde(default, alias = "sources")]
     pub thumbnails: Vec<Thumbnail>,
 }
 
@@ -112,9 +152,16 @@ pub(crate) struct ContinuationItemRenderer {
     pub continuation_endpoint: ContinuationEndpoint,
 }
 
+#[derive(Debug, Deserialize)]
+#[serde(untagged)]
+pub(crate) enum ContinuationEndpoint {
+    ContinuationCommand(ContinuationCommandWrap),
+    CommandExecutorCommand(CommandExecutorCommandWrap),
+}
+
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub(crate) struct ContinuationEndpoint {
+pub(crate) struct ContinuationCommandWrap {
     pub continuation_command: ContinuationCommand,
 }
 
@@ -124,7 +171,34 @@ pub(crate) struct ContinuationCommand {
     pub token: String,
 }
 
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct CommandExecutorCommandWrap {
+    pub command_executor_command: CommandExecutorCommand,
+}
+
 #[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct CommandExecutorCommand {
+    #[serde_as(as = "VecSkipError<_>")]
+    commands: Vec<ContinuationCommandWrap>,
+}
+
+impl ContinuationEndpoint {
+    pub fn into_token(self) -> Option<String> {
+        match self {
+            Self::ContinuationCommand(cmd) => Some(cmd.continuation_command.token),
+            Self::CommandExecutorCommand(cmd) => cmd
+                .command_executor_command
+                .commands
+                .into_iter()
+                .next()
+                .map(|c| c.continuation_command.token),
+        }
+    }
+}
+
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct Icon {
@@ -164,23 +238,92 @@ pub(crate) enum ChannelBadgeStyle {
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct Alert {
-    pub alert_renderer: AlertRenderer,
+    pub alert_renderer: TextBox,
 }
 
 #[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub(crate) struct AlertRenderer {
+pub(crate) struct TextBox {
     #[serde_as(as = "Text")]
     pub text: String,
 }
 
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct SimpleHeaderRenderer {
+    #[serde_as(as = "Text")]
+    pub title: String,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct TextComponentBox {
+    #[serde_as(as = "AttributedText")]
+    pub text: TextComponent,
+}
+
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct ResponseContext {
     pub visitor_data: Option<String>,
 }
 
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct AttachmentRun {
+    pub element: AttachmentRunElement,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct AttachmentRunElement {
+    #[serde(rename = "type")]
+    pub typ: AttachmentRunElementType,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct AttachmentRunElementType {
+    pub image_type: AttachmentRunElementImageType,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct AttachmentRunElementImageType {
+    pub image: AttachmentRunElementImage,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct AttachmentRunElementImage {
+    #[serde_as(as = "VecSkipError<_>")]
+    pub sources: Vec<AttachmentRunElementImageSource>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct AttachmentRunElementImageSource {
+    pub client_resource: ClientResource,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ClientResource {
+    pub image_name: IconName,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
+pub enum IconName {
+    CheckCircleFilled,
+    #[serde(alias = "AUDIO_BADGE")]
+    MusicFilled,
+}
+
 // CONTINUATION
 
 #[serde_as]
@@ -188,14 +331,14 @@ pub(crate) struct ResponseContext {
 #[serde(rename_all = "camelCase")]
 pub(crate) struct Continuation {
     /// Number of search results
-    #[serde_as(as = "Option<JsonString>")]
+    #[serde_as(as = "Option<DisplayFromStr>")]
     pub estimated_results: Option<u64>,
     #[serde(
         alias = "onResponseReceivedCommands",
         alias = "onResponseReceivedEndpoints"
     )]
     #[serde_as(as = "Option<VecSkipError<_>>")]
-    pub on_response_received_actions: Option<Vec<ContinuationActionWrap>>,
+    pub on_response_received_actions: Option<Vec<ContinuationActionWrap<YouTubeListItem>>>,
     /// Used for channel video rich grid renderer
     ///
     /// A/B test seen on 19.10.2022
@@ -204,16 +347,15 @@ pub(crate) struct Continuation {
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub(crate) struct ContinuationActionWrap {
-    pub append_continuation_items_action: ContinuationAction,
+pub(crate) struct ContinuationActionWrap<T> {
+    #[serde(alias = "reloadContinuationItemsCommand")]
+    pub append_continuation_items_action: ContinuationAction<T>,
 }
 
-#[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub(crate) struct ContinuationAction {
-    #[serde_as(as = "VecLogError<_>")]
-    pub continuation_items: MapResult<Vec<YouTubeListItem>>,
+pub(crate) struct ContinuationAction<T> {
+    pub continuation_items: MapResult<Vec<T>>,
 }
 
 #[derive(Debug, Deserialize)]
@@ -249,9 +391,53 @@ pub(crate) struct ErrorResponseContent {
     pub message: String,
 }
 
-/*
-#MAPPING
-*/
+// DESERIALIZER
+
+impl<'de, T> Deserialize<'de> for ContentsRenderer<T>
+where
+    T: Deserialize<'de>,
+{
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        struct ItemVisitor<T>(PhantomData<T>);
+
+        impl<'de, T> Visitor<'de> for ItemVisitor<T>
+        where
+            T: Deserialize<'de>,
+        {
+            type Value = ContentsRenderer<T>;
+
+            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+                formatter.write_str("map")
+            }
+
+            fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
+            where
+                A: serde::de::MapAccess<'de>,
+            {
+                let mut contents = None;
+
+                while let Some(k) = map.next_key::<Cow<'de, str>>()? {
+                    if k == "contents" || k == "tabs" || k == "items" {
+                        contents = Some(ContentsRenderer {
+                            contents: map.next_value::<VecSkipErrorWrap<T>>()?.0,
+                        });
+                    } else {
+                        map.next_value::<IgnoredAny>()?;
+                    }
+                }
+
+                contents.ok_or(serde::de::Error::missing_field("contents"))
+            }
+        }
+
+        deserializer.deserialize_map(ItemVisitor(PhantomData::<T>))
+    }
+}
+
+// MAPPING
 
 impl From<Thumbnail> for crate::model::Thumbnail {
     fn from(tn: Thumbnail) -> Self {
@@ -276,14 +462,27 @@ impl From<Thumbnails> for Vec<crate::model::Thumbnail> {
     }
 }
 
+impl ContentImage {
+    pub(crate) fn into_image(self) -> ImageViewOl {
+        match self {
+            ContentImage::ThumbnailViewModel(image) => image,
+            ContentImage::CollectionThumbnailViewModel { primary_thumbnail } => {
+                primary_thumbnail.thumbnail_view_model
+            }
+        }
+    }
+}
+
 impl From<Vec<ChannelBadge>> for crate::model::Verification {
     fn from(badges: Vec<ChannelBadge>) -> Self {
-        badges.get(0).map_or(crate::model::Verification::None, |b| {
-            match b.metadata_badge_renderer.style {
-                ChannelBadgeStyle::BadgeStyleTypeVerified => Self::Verified,
-                ChannelBadgeStyle::BadgeStyleTypeVerifiedArtist => Self::Artist,
-            }
-        })
+        badges
+            .first()
+            .map_or(crate::model::Verification::None, |b| {
+                match b.metadata_badge_renderer.style {
+                    ChannelBadgeStyle::BadgeStyleTypeVerified => Self::Verified,
+                    ChannelBadgeStyle::BadgeStyleTypeVerifiedArtist => Self::Artist,
+                }
+            })
     }
 }
 
@@ -292,21 +491,240 @@ impl From<Icon> for crate::model::Verification {
         match icon.icon_type {
             IconType::Check => Self::Verified,
             IconType::OfficialArtistBadge => Self::Artist,
-            _ => Self::None,
+            IconType::Like => Self::None,
         }
     }
 }
 
-pub(crate) fn alerts_to_err(alerts: Option<Vec<Alert>>) -> ExtractionError {
-    match alerts {
-        Some(alerts) => ExtractionError::ContentUnavailable(
-            alerts
-                .into_iter()
-                .map(|a| a.alert_renderer.text)
-                .collect::<Vec<_>>()
-                .join(" ")
-                .into(),
-        ),
-        None => ExtractionError::ContentUnavailable("content not found".into()),
+impl From<AttachmentRun> for crate::model::Verification {
+    fn from(value: AttachmentRun) -> Self {
+        match value
+            .element
+            .typ
+            .image_type
+            .image
+            .sources
+            .into_iter()
+            .next()
+            .map(|s| s.client_resource.image_name)
+        {
+            Some(IconName::CheckCircleFilled) => Self::Verified,
+            Some(IconName::MusicFilled) => Self::Artist,
+            None => Self::None,
+        }
     }
 }
+
+pub(crate) fn alerts_to_err(id: &str, alerts: Option<Vec<Alert>>) -> ExtractionError {
+    ExtractionError::NotFound {
+        id: id.to_owned(),
+        msg: alerts
+            .map(|alerts| {
+                alerts
+                    .into_iter()
+                    .map(|a| a.alert_renderer.text)
+                    .collect::<Vec<_>>()
+                    .join(" ")
+                    .into()
+            })
+            .unwrap_or_default(),
+    }
+}
+
+// FRAMEWORK UPDATES
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct FrameworkUpdates<T> {
+    pub entity_batch_update: EntityBatchUpdate<T>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct EntityBatchUpdate<T> {
+    pub mutations: FrameworkUpdateMutations<T>,
+}
+
+/// List of update mutations that deserializes into a HashMap (entity_key => payload)
+#[derive(Debug)]
+pub(crate) struct FrameworkUpdateMutations<T> {
+    pub items: HashMap<String, T>,
+    pub warnings: Vec<String>,
+}
+
+impl<'de, T> Deserialize<'de> for FrameworkUpdateMutations<T>
+where
+    T: Deserialize<'de>,
+{
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        struct SeqVisitor<T>(PhantomData<T>);
+
+        #[derive(serde::Deserialize)]
+        #[serde(untagged)]
+        enum MutationOrError<T> {
+            #[serde(rename_all = "camelCase")]
+            Good {
+                entity_key: String,
+                payload: T,
+            },
+            Error(serde_json::Value),
+        }
+
+        impl<'de, T> Visitor<'de> for SeqVisitor<T>
+        where
+            T: Deserialize<'de>,
+        {
+            type Value = FrameworkUpdateMutations<T>;
+
+            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+                formatter.write_str("sequence of entity mutations")
+            }
+
+            fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+            where
+                A: serde::de::SeqAccess<'de>,
+            {
+                let mut items = HashMap::with_capacity(seq.size_hint().unwrap_or_default());
+                let mut warnings = Vec::new();
+
+                while let Some(value) = seq.next_element::<MutationOrError<T>>()? {
+                    match value {
+                        MutationOrError::Good {
+                            entity_key,
+                            payload,
+                        } => {
+                            items.insert(entity_key, payload);
+                        }
+                        MutationOrError::Error(value) => {
+                            warnings.push(format!(
+                                "error deserializing item: {}",
+                                serde_json::to_string(&value).unwrap_or_default()
+                            ));
+                        }
+                    }
+                }
+
+                Ok(FrameworkUpdateMutations { items, warnings })
+            }
+        }
+
+        deserializer.deserialize_seq(SeqVisitor(PhantomData::<T>))
+    }
+}
+
+// PAGE HEADER
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PageHeaderRendererContent<T> {
+    pub page_header_view_model: T,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PhMetadataView {
+    pub content_metadata_view_model: PhMetadataView2,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PhMetadataView2 {
+    #[serde_as(as = "VecSkipError<_>")]
+    pub metadata_rows: Vec<PhMetadataRow>,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PhMetadataRow {
+    #[serde_as(as = "VecSkipError<_>")]
+    pub metadata_parts: Vec<MetadataPart>,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(untagged)]
+pub(crate) enum MetadataPart {
+    Text {
+        #[serde_as(as = "AttributedText")]
+        text: TextComponent,
+    },
+    #[serde(rename_all = "camelCase")]
+    AvatarStack { avatar_stack: AvatarStackInner },
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct AvatarStackInner {
+    pub avatar_stack_view_model: TextComponentBox,
+}
+
+impl MetadataPart {
+    pub fn into_text_component(self) -> TextComponent {
+        match self {
+            MetadataPart::Text { text } => text,
+            MetadataPart::AvatarStack { avatar_stack } => avatar_stack.avatar_stack_view_model.text,
+        }
+    }
+
+    pub fn as_str(&self) -> &str {
+        match self {
+            MetadataPart::Text { text } => text.as_str(),
+            MetadataPart::AvatarStack { avatar_stack } => {
+                avatar_stack.avatar_stack_view_model.text.as_str()
+            }
+        }
+    }
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) enum ContentImage {
+    ThumbnailViewModel(ImageViewOl),
+    #[serde(rename_all = "camelCase")]
+    CollectionThumbnailViewModel {
+        primary_thumbnail: ThumbnailViewModelWrap,
+    },
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ThumbnailViewModelWrap {
+    pub thumbnail_view_model: ImageViewOl,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ImageViewOl {
+    pub image: Thumbnails,
+    #[serde_as(as = "VecSkipError<_>")]
+    pub overlays: Vec<ImageViewOverlay>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ImageViewOverlay {
+    pub thumbnail_overlay_badge_view_model: ThumbnailOverlayBadgeViewModel,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ThumbnailOverlayBadgeViewModel {
+    #[serde_as(as = "VecSkipError<_>")]
+    pub thumbnail_badges: Vec<ThumbnailBadges>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ThumbnailBadges {
+    pub thumbnail_badge_view_model: TextBox,
+}
+
+#[derive(Debug, Deserialize)]
+pub(crate) struct Empty {}

src/client/response/music_artist.rs

@@ -5,7 +5,8 @@ use crate::serializer::text::Text;
 
 use super::{
     music_item::{
-        Button, Grid, ItemSection, MusicThumbnailRenderer, SimpleHeader, SingleColumnBrowseResult,
+        Button, Grid, ItemSection, MusicMicroformat, MusicThumbnailRenderer, SimpleHeader,
+        SingleColumnBrowseResult,
     },
     SectionList, Tab,
 };
@@ -14,8 +15,10 @@ use super::{
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct MusicArtist {
-    pub contents: SingleColumnBrowseResult<Tab<Option<SectionList<ItemSection>>>>,
-    pub header: Header,
+    pub contents: Option<SingleColumnBrowseResult<Tab<SectionList<ItemSection>>>>,
+    pub header: Option<Header>,
+    #[serde(default)]
+    pub microformat: MusicMicroformat,
 }
 
 #[derive(Debug, Deserialize)]
@@ -73,9 +76,12 @@ pub(crate) struct ShareEntityEndpoint {
 }
 
 /// Response model for YouTube Music artist album page
+#[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct MusicArtistAlbums {
-    pub header: SimpleHeader,
+    #[serde(default)]
+    #[serde_as(as = "DefaultOnError")]
+    pub header: Option<SimpleHeader>,
     pub contents: SingleColumnBrowseResult<Tab<SectionList<Grid>>>,
 }

src/client/response/music_details.rs

@@ -1,14 +1,13 @@
 use serde::Deserialize;
-use serde_with::serde_as;
-use serde_with::DefaultOnError;
+use serde_with::{serde_as, DefaultOnError, VecSkipError};
 
 use crate::serializer::text::Text;
 
-use super::AlertRenderer;
 use super::ContentsRenderer;
+use super::TextBox;
 use super::{
     music_item::{ItemSection, PlaylistPanelRenderer},
-    ContentRenderer, SectionList,
+    ContentRenderer,
 };
 
 /// Response model for YouTube Music track details
@@ -36,9 +35,11 @@ pub(crate) struct TabbedRenderer {
     pub watch_next_tabbed_results_renderer: TabbedRendererInner,
 }
 
+#[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct TabbedRendererInner {
+    #[serde_as(as = "VecSkipError<_>")]
     pub tabs: Vec<Tab>,
 }
 
@@ -107,14 +108,14 @@ pub(crate) struct PlaylistPanel {
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct MusicLyrics {
-    pub contents: LyricsContents,
+    pub contents: ListOrMessage<LyricsSection>,
 }
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub(crate) struct LyricsContents {
-    pub message_renderer: Option<AlertRenderer>,
-    pub section_list_renderer: Option<ContentsRenderer<LyricsSection>>,
+pub(crate) enum ListOrMessage<T> {
+    SectionListRenderer(ContentsRenderer<T>),
+    MessageRenderer(TextBox),
 }
 
 #[derive(Debug, Deserialize)]
@@ -136,5 +137,14 @@ pub(crate) struct LyricsRenderer {
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct MusicRelated {
-    pub contents: SectionList<ItemSection>,
+    pub contents: ListOrMessage<ItemSection>,
+}
+
+impl<T> ListOrMessage<T> {
+    pub fn into_res(self) -> Result<Vec<T>, String> {
+        match self {
+            ListOrMessage::SectionListRenderer(c) => Ok(c.contents),
+            ListOrMessage::MessageRenderer(msg) => Err(msg.text),
+        }
+    }
 }

src/client/response/music_genres.rs

@@ -1,12 +1,12 @@
 use serde::Deserialize;
 use serde_with::{rust::deserialize_ignore_any, serde_as};
 
-use crate::serializer::{text::Text, MapResult, VecLogError};
+use crate::serializer::text::Text;
 
 use super::{
     music_item::{ItemSection, SimpleHeader, SingleColumnBrowseResult},
     url_endpoint::BrowseEndpointWrap,
-    SectionList, Tab,
+    ContentsRendererLogged, SectionList, Tab,
 };
 
 #[derive(Debug, Deserialize)]
@@ -18,15 +18,7 @@ pub(crate) struct MusicGenres {
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct Grid {
-    pub grid_renderer: GridRenderer,
-}
-
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct GridRenderer {
-    #[serde_as(as = "VecLogError<_>")]
-    pub items: MapResult<Vec<NavigationButton>>,
+    pub grid_renderer: ContentsRendererLogged<NavigationButton>,
 }
 
 #[derive(Debug, Deserialize)]

src/client/response/music_history.rs

@@ -0,0 +1,8 @@
+use serde::Deserialize;
+
+use super::music_playlist::Contents;
+
+#[derive(Debug, Deserialize)]
+pub(crate) struct MusicHistory {
+    pub contents: Contents,
+}

src/client/response/music_playlist.rs

@@ -1,27 +1,45 @@
 use serde::Deserialize;
 use serde_with::{serde_as, DefaultOnError, VecSkipError};
 
-use crate::serializer::text::{Text, TextComponents};
+use crate::serializer::text::{AttributedText, Text, TextComponents};
 
 use super::{
     music_item::{
-        ItemSection, MusicContentsRenderer, MusicItemMenuEntry, MusicThumbnailRenderer,
-        SingleColumnBrowseResult,
+        Button, ItemSection, MusicContentsRenderer, MusicItemMenuEntry, MusicMicroformat,
+        MusicThumbnailRenderer,
     },
-    Tab,
+    url_endpoint::OnTapWrap,
+    ContentsRenderer, SectionList, Tab,
 };
 
 /// Response model for YouTube Music playlists and albums
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct MusicPlaylist {
-    pub contents: SingleColumnBrowseResult<Tab<SectionList>>,
+    pub contents: Option<Contents>,
     pub header: Option<Header>,
+    #[serde(default)]
+    pub microformat: MusicMicroformat,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) enum Contents {
+    SingleColumnBrowseResultsRenderer(ContentsRenderer<Tab<PlSectionList>>),
+    #[serde(rename_all = "camelCase")]
+    TwoColumnBrowseResultsRenderer {
+        /// List content
+        secondary_contents: PlSectionList,
+        /// Header
+        #[serde_as(as = "VecSkipError<_>")]
+        tabs: Vec<Tab<SectionList<Header>>>,
+    },
 }
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub(crate) struct SectionList {
+pub(crate) struct PlSectionList {
     /// Includes a continuation token for fetching recommendations
     pub section_list_renderer: MusicContentsRenderer<ItemSection>,
 }
@@ -29,6 +47,7 @@ pub(crate) struct SectionList {
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct Header {
+    #[serde(alias = "musicResponsiveHeaderRenderer")]
     pub music_detail_header_renderer: HeaderRenderer,
 }
 
@@ -48,22 +67,48 @@ pub(crate) struct HeaderRenderer {
     pub subtitle: TextComponents,
     /// Playlist/album description. May contain hashtags which are
     /// displayed as search links on the YouTube website.
-    #[serde_as(as = "Option<Text>")]
-    pub description: Option<String>,
+    pub description: Option<Description>,
     /// Playlist thumbnail / album cover.
     /// Missing on artist_tracks view.
     #[serde(default)]
     pub thumbnail: MusicThumbnailRenderer,
+    /// Channel (only on TwoColumnBrowseResultsRenderer)
+    pub strapline_text_one: Option<TextComponents>,
     /// Number of tracks + playtime.
     /// Missing on artist_tracks view.
     ///
     /// `"64 songs", " • ", "3 hours, 40 minutes"`
+    ///
+    /// `"1B views", " • ", "200 songs", " • ", "6+ hours"`
     #[serde(default)]
     #[serde_as(as = "Text")]
     pub second_subtitle: Vec<String>,
+    /// Channel (newer data model)
+    #[serde(default)]
+    #[serde_as(as = "DefaultOnError")]
+    pub facepile: Option<AvatarStackViewModelWrap>,
     #[serde(default)]
     #[serde_as(as = "DefaultOnError")]
     pub menu: Option<HeaderMenu>,
+    #[serde(default)]
+    #[serde_as(as = "VecSkipError<_>")]
+    pub buttons: Vec<HeaderMenu>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(untagged)]
+pub(crate) enum Description {
+    #[serde(rename_all = "camelCase")]
+    Shelf {
+        music_description_shelf_renderer: DescriptionShelf,
+    },
+    Text(TextComponents),
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct DescriptionShelf {
+    pub description: TextComponents,
 }
 
 #[derive(Debug, Deserialize)]
@@ -78,31 +123,41 @@ pub(crate) struct HeaderMenu {
 pub(crate) struct HeaderMenuRenderer {
     #[serde(default)]
     #[serde_as(as = "VecSkipError<_>")]
-    pub top_level_buttons: Vec<TopLevelButton>,
+    pub top_level_buttons: Vec<Button>,
     #[serde_as(as = "VecSkipError<_>")]
     pub items: Vec<MusicItemMenuEntry>,
 }
 
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct TopLevelButton {
-    pub button_renderer: ButtonRenderer,
+impl From<Description> for TextComponents {
+    fn from(value: Description) -> Self {
+        match value {
+            Description::Text(v) => v,
+            Description::Shelf {
+                music_description_shelf_renderer,
+            } => music_description_shelf_renderer.description,
+        }
+    }
 }
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub(crate) struct ButtonRenderer {
-    pub navigation_endpoint: PlaylistEndpoint,
+pub(crate) struct AvatarStackViewModelWrap {
+    pub avatar_stack_view_model: AvatarStackViewModel,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct AvatarStackViewModel {
+    // #[serde(default)]
+    // pub avatars: Vec<AvatarViewModel>,
+    #[serde_as(as = "AttributedText")]
+    pub text: String,
+    pub renderer_context: AvatarStackRendererContext,
 }
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub(crate) struct PlaylistEndpoint {
-    pub watch_playlist_endpoint: PlaylistWatchEndpoint,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct PlaylistWatchEndpoint {
-    pub playlist_id: String,
+pub(crate) struct AvatarStackRendererContext {
+    pub command_context: Option<OnTapWrap>,
 }

src/client/response/player.rs

@@ -2,11 +2,12 @@ use std::ops::Range;
 
 use serde::Deserialize;
 use serde_with::serde_as;
-use serde_with::{json::JsonString, DefaultOnError};
+use serde_with::{DefaultOnError, DisplayFromStr, VecSkipError};
 
-use super::{ResponseContext, Thumbnails};
-use crate::serializer::{text::Text, MapResult, VecLogError};
+use super::{Empty, ResponseContext, Thumbnails};
+use crate::serializer::{text::Text, MapResult};
 
+#[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct Player {
@@ -14,7 +15,14 @@ pub(crate) struct Player {
     pub streaming_data: Option<StreamingData>,
     pub captions: Option<Captions>,
     pub video_details: Option<VideoDetails>,
+    #[serde(default)]
+    #[serde_as(deserialize_as = "DefaultOnError")]
+    pub storyboards: Option<Storyboards>,
     pub response_context: ResponseContext,
+    #[serde(default)]
+    pub player_config: PlayerConfig,
+    #[serde(default)]
+    pub heartbeat_params: HeartbeatParams,
 }
 
 #[serde_as]
@@ -29,14 +37,15 @@ pub(crate) enum PlayabilityStatus {
         #[serde(default)]
         reason: String,
         #[serde(default)]
-        #[serde_as(deserialize_as = "DefaultOnError")]
-        error_screen: Option<ErrorScreen>,
+        error_screen: ErrorScreen,
     },
     /// Age limit / Private video
     #[serde(rename_all = "camelCase")]
     LoginRequired {
         #[serde(default)]
         reason: String,
+        #[serde(default)]
+        messages: Vec<String>,
     },
     #[serde(rename_all = "camelCase")]
     LiveStreamOffline {
@@ -51,17 +60,18 @@ pub(crate) enum PlayabilityStatus {
     },
 }
 
-#[derive(Debug, Deserialize)]
-pub(crate) struct Empty {}
-
-#[derive(Debug, Deserialize)]
+#[serde_as]
+#[derive(Default, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct ErrorScreen {
-    pub player_error_message_renderer: ErrorMessage,
+    #[serde(default)]
+    #[serde_as(deserialize_as = "DefaultOnError")]
+    pub player_error_message_renderer: Option<ErrorMessage>,
+    pub player_captcha_view_model: Option<Empty>,
 }
 
 #[serde_as]
-#[derive(Debug, Deserialize)]
+#[derive(Default, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct ErrorMessage {
     #[serde_as(as = "Text")]
@@ -72,18 +82,20 @@ pub(crate) struct ErrorMessage {
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct StreamingData {
-    #[serde_as(as = "JsonString")]
+    #[serde_as(as = "DisplayFromStr")]
     pub expires_in_seconds: u32,
     #[serde(default)]
-    #[serde_as(as = "VecLogError<_>")]
     pub formats: MapResult<Vec<Format>>,
     #[serde(default)]
-    #[serde_as(as = "VecLogError<_>")]
     pub adaptive_formats: MapResult<Vec<Format>>,
     /// Only on livestreams
     pub dash_manifest_url: Option<String>,
     /// Only on livestreams
     pub hls_manifest_url: Option<String>,
+    pub drm_params: Option<String>,
+    #[serde(default)]
+    #[serde_as(deserialize_as = "VecSkipError<_>")]
+    pub initial_authorized_drm_track_types: Vec<DrmTrackType>,
 }
 
 #[serde_as]
@@ -102,7 +114,7 @@ pub(crate) struct Format {
 
     pub width: Option<u32>,
     pub height: Option<u32>,
-    #[serde_as(as = "Option<JsonString>")]
+    #[serde_as(as = "Option<DisplayFromStr>")]
     pub approx_duration_ms: Option<u32>,
 
     #[serde_as(as = "Option<crate::serializer::Range>")]
@@ -110,7 +122,7 @@ pub(crate) struct Format {
     #[serde_as(as = "Option<crate::serializer::Range>")]
     pub init_range: Option<Range<u32>>,
 
-    #[serde_as(as = "Option<JsonString>")]
+    #[serde_as(as = "Option<DisplayFromStr>")]
     pub content_length: Option<u64>,
 
     #[serde(default)]
@@ -125,20 +137,23 @@ pub(crate) struct Format {
     #[serde(default)]
     #[serde_as(deserialize_as = "DefaultOnError")]
     pub audio_quality: Option<AudioQuality>,
-    #[serde_as(as = "Option<JsonString>")]
+    #[serde_as(as = "Option<DisplayFromStr>")]
     pub audio_sample_rate: Option<u32>,
     pub audio_channels: Option<u8>,
     pub loudness_db: Option<f32>,
     pub audio_track: Option<AudioTrack>,
 
     pub signature_cipher: Option<String>,
+
+    #[serde(default)]
+    #[serde_as(deserialize_as = "VecSkipError<_>")]
+    pub drm_families: Vec<DrmFamily>,
+    pub drm_track_type: Option<DrmTrackType>,
 }
 
 impl Format {
     pub fn is_audio(&self) -> bool {
-        self.content_length.is_some()
-            && self.audio_quality.is_some()
-            && self.audio_sample_rate.is_some()
+        self.audio_quality.is_some() && self.audio_sample_rate.is_some()
     }
 
     pub fn is_video(&self) -> bool {
@@ -150,7 +165,7 @@ impl Format {
     }
 }
 
-#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
 #[serde(rename_all = "lowercase")]
 pub(crate) enum Quality {
     Tiny,
@@ -164,17 +179,19 @@ pub(crate) enum Quality {
     Hd2160,
 }
 
-#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
 pub(crate) enum AudioQuality {
-    #[serde(rename = "AUDIO_QUALITY_LOW", alias = "low")]
+    #[serde(rename = "AUDIO_QUALITY_ULTRALOW")]
+    UltraLow,
+    #[serde(rename = "AUDIO_QUALITY_LOW")]
     Low,
-    #[serde(rename = "AUDIO_QUALITY_MEDIUM", alias = "medium")]
+    #[serde(rename = "AUDIO_QUALITY_MEDIUM")]
     Medium,
-    #[serde(rename = "AUDIO_QUALITY_HIGH", alias = "high")]
+    #[serde(rename = "AUDIO_QUALITY_HIGH")]
     High,
 }
 
-#[derive(Default, Clone, Copy, Debug, Deserialize, PartialEq, Eq, Hash)]
+#[derive(Default, Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
 #[serde(rename_all = "SCREAMING_SNAKE_CASE")]
 pub(crate) enum FormatType {
     #[default]
@@ -189,7 +206,7 @@ pub(crate) struct ColorInfo {
     pub primaries: Primaries,
 }
 
-#[derive(Default, Clone, Copy, Debug, Deserialize, PartialEq, Eq, Hash)]
+#[derive(Default, Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
 #[serde(rename_all = "SCREAMING_SNAKE_CASE")]
 pub(crate) enum Primaries {
     #[default]
@@ -197,6 +214,24 @@ pub(crate) enum Primaries {
     ColorPrimariesBt2020,
 }
 
+#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
+#[allow(clippy::enum_variant_names)]
+pub(crate) enum DrmTrackType {
+    DrmTrackTypeAudio,
+    DrmTrackTypeSd,
+    DrmTrackTypeHd,
+    DrmTrackTypeUhd1,
+}
+
+#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
+pub(crate) enum DrmFamily {
+    Widevine,
+    Playready,
+    Fairplay,
+}
+
 #[derive(Default, Debug, Deserialize)]
 #[serde(default, rename_all = "camelCase")]
 pub(crate) struct AudioTrack {
@@ -232,8 +267,8 @@ pub(crate) struct CaptionTrack {
 #[serde(rename_all = "camelCase")]
 pub(crate) struct VideoDetails {
     pub video_id: String,
-    pub title: String,
-    #[serde_as(as = "JsonString")]
+    pub title: Option<String>,
+    #[serde_as(as = "DisplayFromStr")]
     pub length_seconds: u32,
     #[serde(default)]
     pub keywords: Vec<String>,
@@ -241,8 +276,74 @@ pub(crate) struct VideoDetails {
     pub short_description: Option<String>,
     #[serde(default)]
     pub thumbnail: Thumbnails,
-    #[serde_as(as = "JsonString")]
-    pub view_count: u64,
-    pub author: String,
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    pub view_count: Option<u64>,
+    pub author: Option<String>,
     pub is_live_content: bool,
 }
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct Storyboards {
+    pub player_storyboard_spec_renderer: StoryboardRenderer,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct StoryboardRenderer {
+    pub spec: String,
+}
+
+#[derive(Default, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PlayerConfig {
+    pub web_drm_config: Option<WebDrmConfig>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct WebDrmConfig {
+    pub widevine_service_cert: Option<String>,
+}
+
+#[derive(Default, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct HeartbeatParams {
+    pub drm_session_id: Option<String>,
+}
+
+impl From<DrmTrackType> for crate::model::DrmTrackType {
+    fn from(value: DrmTrackType) -> Self {
+        match value {
+            DrmTrackType::DrmTrackTypeAudio => Self::Audio,
+            DrmTrackType::DrmTrackTypeSd => Self::Sd,
+            DrmTrackType::DrmTrackTypeHd => Self::Hd,
+            DrmTrackType::DrmTrackTypeUhd1 => Self::Uhd1,
+        }
+    }
+}
+
+impl From<DrmFamily> for crate::model::DrmSystem {
+    fn from(value: DrmFamily) -> Self {
+        match value {
+            DrmFamily::Widevine => Self::Widevine,
+            DrmFamily::Playready => Self::Playready,
+            DrmFamily::Fairplay => Self::Fairplay,
+        }
+    }
+}
+
+#[derive(Default, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct DrmLicense {
+    pub status: String,
+    pub license: String,
+    pub authorized_formats: Vec<AuthorizedFormat>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct AuthorizedFormat {
+    pub track_type: DrmTrackType,
+    pub key_id: String,
+}

src/client/response/playlist.rs

@@ -1,22 +1,19 @@
 use serde::Deserialize;
-use serde_with::{
-    json::JsonString, rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError,
-};
+use serde_with::{serde_as, DefaultOnError, VecSkipError};
 
-use crate::serializer::text::{Text, TextComponent};
-use crate::serializer::{MapResult, VecLogError};
-use crate::util::MappingError;
+use crate::serializer::text::{AttributedText, Text, TextComponent, TextComponents};
 
 use super::{
-    Alert, ContentsRenderer, ContinuationEndpoint, ResponseContext, SectionList, Tab, Thumbnails,
-    ThumbnailsWrap,
+    url_endpoint::OnTapWrap, video_item::YouTubeListRenderer, Alert, ContentRenderer,
+    ContentsRenderer, ImageView, PageHeaderRendererContent, PhMetadataView, ResponseContext,
+    SectionList, Tab, TextBox, ThumbnailsWrap, TwoColumnBrowseResults,
 };
 
 #[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct Playlist {
-    pub contents: Option<Contents>,
+    pub contents: Option<TwoColumnBrowseResults<Tab<SectionList<ItemSection>>>>,
     pub header: Option<Header>,
     pub sidebar: Option<Sidebar>,
     #[serde_as(as = "Option<DefaultOnError>")]
@@ -24,21 +21,6 @@ pub(crate) struct Playlist {
     pub response_context: ResponseContext,
 }
 
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct PlaylistCont {
-    #[serde(default)]
-    #[serde_as(as = "VecSkipError<_>")]
-    pub on_response_received_actions: Vec<OnResponseReceivedAction>,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct Contents {
-    pub two_column_browse_results_renderer: ContentsRenderer<Tab<SectionList<ItemSection>>>,
-}
-
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct ItemSection {
@@ -48,21 +30,15 @@ pub(crate) struct ItemSection {
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct PlaylistVideoListRenderer {
-    pub playlist_video_list_renderer: PlaylistVideoList,
-}
-
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct PlaylistVideoList {
-    #[serde_as(as = "VecLogError<_>")]
-    pub contents: MapResult<Vec<PlaylistItem>>,
+    #[serde(alias = "richGridRenderer")]
+    pub playlist_video_list_renderer: YouTubeListRenderer,
 }
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub(crate) struct Header {
-    pub playlist_header_renderer: HeaderRenderer,
+pub(crate) enum Header {
+    PlaylistHeaderRenderer(HeaderRenderer),
+    PageHeaderRenderer(ContentRenderer<PageHeaderRendererContent<PageHeaderRendererInner>>),
 }
 
 #[serde_as]
@@ -94,29 +70,13 @@ pub(crate) struct PlaylistHeaderBanner {
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct Byline {
-    pub playlist_byline_renderer: BylineRenderer,
-}
-
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct BylineRenderer {
-    #[serde_as(as = "Text")]
-    pub text: String,
+    pub playlist_byline_renderer: TextBox,
 }
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct Sidebar {
-    pub playlist_sidebar_renderer: SidebarRenderer,
-}
-
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct SidebarRenderer {
-    #[serde_as(as = "VecSkipError<_>")]
-    pub items: Vec<SidebarItemPrimary>,
+    pub playlist_sidebar_renderer: ContentsRenderer<SidebarItemPrimary>,
 }
 
 #[derive(Debug, Deserialize)]
@@ -129,6 +89,7 @@ pub(crate) struct SidebarItemPrimary {
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct SidebarPrimaryInfoRenderer {
+    pub description: Option<TextComponents>,
     pub thumbnail_renderer: PlaylistThumbnailRenderer,
     /// - `"495", " videos"`
     /// - `"3,310,996 views"`
@@ -145,64 +106,72 @@ pub(crate) struct PlaylistThumbnailRenderer {
     pub playlist_video_thumbnail_renderer: ThumbnailsWrap,
 }
 
-#[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub(crate) enum PlaylistItem {
-    /// Video in playlist
-    PlaylistVideoRenderer(PlaylistVideoRenderer),
-    /// Continauation items are located at the end of a list
-    /// and contain the continuation token for progressive loading
-    #[serde(rename_all = "camelCase")]
-    ContinuationItemRenderer {
-        continuation_endpoint: ContinuationEndpoint,
-    },
-    /// No video list item (e.g. ad) or unimplemented item
-    #[serde(other, deserialize_with = "deserialize_ignore_any")]
-    None,
+pub(crate) struct PageHeaderRendererInner {
+    pub title: PhTitleView,
+    pub metadata: PhMetadataView,
+    pub actions: PhActions,
+    pub description: PhDescription,
+    pub hero_image: PhHeroImage,
 }
 
-/// Video displayed in a playlist
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct PlaylistVideoRenderer {
-    pub video_id: String,
-    pub thumbnail: Thumbnails,
-    #[serde_as(as = "Text")]
-    pub title: String,
-    #[serde(rename = "shortBylineText")]
-    pub channel: TextComponent,
-    #[serde_as(as = "JsonString")]
-    pub length_seconds: u32,
-}
-
-impl TryFrom<PlaylistVideoRenderer> for crate::model::PlaylistVideo {
-    type Error = MappingError;
-
-    fn try_from(video: PlaylistVideoRenderer) -> Result<Self, Self::Error> {
-        Ok(Self {
-            id: video.video_id,
-            name: video.title,
-            length: video.length_seconds,
-            thumbnail: video.thumbnail.into(),
-            channel: crate::model::ChannelId::try_from(video.channel)?,
-        })
-    }
-}
-
-// Continuation
-
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub(crate) struct OnResponseReceivedAction {
-    pub append_continuation_items_action: AppendAction,
+pub(crate) struct PhDescription {
+    pub description_preview_view_model: PhDescription2,
 }
 
 #[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub(crate) struct AppendAction {
-    #[serde_as(as = "VecLogError<_>")]
-    pub continuation_items: MapResult<Vec<PlaylistItem>>,
+pub(crate) struct PhDescription2 {
+    #[serde_as(as = "Option<AttributedText>")]
+    pub description: Option<TextComponents>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PhHeroImage {
+    pub content_preview_image_view_model: ImageView,
+}
+
+#[derive(Default, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PhTitleView {
+    pub dynamic_text_view_model: PhTitleInner,
+}
+
+#[serde_as]
+#[derive(Default, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PhTitleInner {
+    #[serde_as(as = "AttributedText")]
+    pub text: String,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PhActions {
+    pub flexible_actions_view_model: PhActions2,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PhActions2 {
+    pub actions_rows: Vec<ActionsRow>,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ActionsRow {
+    #[serde_as(as = "VecSkipError<_>")]
+    pub actions: Vec<ButtonAction>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ButtonAction {
+    pub button_view_model: OnTapWrap,
 }

src/client/response/search.rs

@@ -1,5 +1,8 @@
-use serde::Deserialize;
-use serde_with::{json::JsonString, serde_as};
+use serde::{
+    de::{IgnoredAny, Visitor},
+    Deserialize,
+};
+use serde_with::{serde_as, DisplayFromStr};
 
 use super::{video_item::YouTubeListRendererWrap, ResponseContext};
 
@@ -7,7 +10,7 @@ use super::{video_item::YouTubeListRendererWrap, ResponseContext};
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct Search {
-    #[serde_as(as = "Option<JsonString>")]
+    #[serde_as(as = "Option<DisplayFromStr>")]
     pub estimated_results: Option<u64>,
     pub contents: Contents,
     pub response_context: ResponseContext,
@@ -24,3 +27,42 @@ pub(crate) struct Contents {
 pub(crate) struct TwoColumnSearchResultsRenderer {
     pub primary_contents: YouTubeListRendererWrap,
 }
+
+#[derive(Debug, Deserialize)]
+pub(crate) struct SearchSuggestion(IgnoredAny, pub Vec<SearchSuggestionItem>, IgnoredAny);
+
+#[derive(Debug)]
+pub(crate) struct SearchSuggestionItem(pub String);
+
+impl<'de> Deserialize<'de> for SearchSuggestionItem {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        struct ItemVisitor;
+
+        impl<'de> Visitor<'de> for ItemVisitor {
+            type Value = SearchSuggestionItem;
+
+            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+                formatter.write_str("search suggestion item")
+            }
+
+            fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+            where
+                A: serde::de::SeqAccess<'de>,
+            {
+                match seq.next_element::<String>()? {
+                    Some(s) => {
+                        // Ignore the rest of the list
+                        while seq.next_element::<IgnoredAny>()?.is_some() {}
+                        Ok(SearchSuggestionItem(s))
+                    }
+                    None => Err(serde::de::Error::invalid_length(0, &"1")),
+                }
+            }
+        }
+
+        deserializer.deserialize_seq(ItemVisitor)
+    }
+}

src/client/response/trends.rs

@@ -1,14 +1,6 @@
 use serde::Deserialize;
-use serde_with::{serde_as, VecSkipError};
 
-use super::{video_item::YouTubeListRendererWrap, ResponseContext, Tab};
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct Startpage {
-    pub contents: Contents,
-    pub response_context: ResponseContext,
-}
+use super::{video_item::YouTubeListRendererWrap, Tab, TwoColumnBrowseResults};
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -16,16 +8,4 @@ pub(crate) struct Trending {
     pub contents: Contents,
 }
 
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct Contents {
-    pub two_column_browse_results_renderer: BrowseResults,
-}
-
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct BrowseResults {
-    #[serde_as(as = "VecSkipError<_>")]
-    pub tabs: Vec<Tab<YouTubeListRendererWrap>>,
-}
+type Contents = TwoColumnBrowseResults<Tab<YouTubeListRendererWrap>>;

src/client/response/url_endpoint.rs

@@ -1,7 +1,12 @@
 use serde::Deserialize;
 use serde_with::{serde_as, DefaultOnError};
 
-use crate::model::UrlTarget;
+use crate::{
+    model::{TrackType, UrlTarget},
+    util,
+};
+
+use super::Empty;
 
 /// navigation/resolve_url response model
 #[derive(Debug, Deserialize)]
@@ -11,21 +16,30 @@ pub(crate) struct ResolvedUrl {
 }
 
 #[serde_as]
-#[derive(Debug, Deserialize, Default)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct NavigationEndpoint {
-    #[serde(default)]
-    #[serde_as(deserialize_as = "DefaultOnError")]
-    pub watch_endpoint: Option<WatchEndpoint>,
-    #[serde(default)]
-    #[serde_as(deserialize_as = "DefaultOnError")]
-    pub browse_endpoint: Option<BrowseEndpoint>,
-    #[serde(default)]
-    #[serde_as(deserialize_as = "DefaultOnError")]
-    pub url_endpoint: Option<UrlEndpoint>,
-    #[serde(default)]
-    #[serde_as(deserialize_as = "DefaultOnError")]
-    pub command_metadata: Option<CommandMetadata>,
+#[derive(Debug, Deserialize)]
+#[serde(untagged)]
+pub(crate) enum NavigationEndpoint {
+    #[serde(rename_all = "camelCase")]
+    Watch {
+        #[serde(alias = "reelWatchEndpoint")]
+        watch_endpoint: WatchEndpoint,
+    },
+    #[serde(rename_all = "camelCase")]
+    Browse {
+        browse_endpoint: BrowseEndpoint,
+        #[serde(default)]
+        #[serde_as(deserialize_as = "DefaultOnError")]
+        command_metadata: Option<CommandMetadata>,
+    },
+    #[serde(rename_all = "camelCase")]
+    Url { url_endpoint: UrlEndpoint },
+    #[serde(rename_all = "camelCase")]
+    WatchPlaylist {
+        watch_playlist_endpoint: WatchPlaylistEndpoint,
+    },
+    #[serde(rename_all = "camelCase")]
+    #[allow(unused)]
+    CreatePlaylist { create_playlist_endpoint: Empty },
 }
 
 #[derive(Debug, Deserialize)]
@@ -52,6 +66,12 @@ pub(crate) struct BrowseEndpointWrap {
     pub browse_endpoint: BrowseEndpoint,
 }
 
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct WatchPlaylistEndpoint {
+    pub playlist_id: String,
+}
+
 impl<'de> Deserialize<'de> for BrowseEndpoint {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
@@ -69,6 +89,7 @@ impl<'de> Deserialize<'de> for BrowseEndpoint {
         let bep = BEp::deserialize(deserializer)?;
 
         // Remove the VL prefix from the playlist id
+        #[allow(clippy::map_unwrap_or)]
         let browse_id = bep
             .browse_endpoint_context_supported_configs
             .as_ref()
@@ -102,9 +123,12 @@ pub(crate) struct BrowseEndpointConfig {
     pub browse_endpoint_context_music_config: BrowseEndpointMusicConfig,
 }
 
+#[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct BrowseEndpointMusicConfig {
+    #[serde(default)]
+    #[serde_as(as = "DefaultOnError")]
     pub page_type: PageType,
 }
 
@@ -114,9 +138,12 @@ pub(crate) struct CommandMetadata {
     pub web_command_metadata: WebCommandMetadata,
 }
 
+#[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct WebCommandMetadata {
+    #[serde(default)]
+    #[serde_as(as = "DefaultOnError")]
     pub web_page_type: PageType,
 }
 
@@ -135,16 +162,54 @@ pub(crate) struct WatchEndpointConfig {
     pub music_video_type: MusicVideoType,
 }
 
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct OnTap {
+    pub innertube_command: NavigationEndpoint,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct OnTapWrap {
+    pub on_tap: OnTap,
+}
+
 #[derive(Default, Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
 pub(crate) enum MusicVideoType {
     #[default]
-    #[serde(rename = "MUSIC_VIDEO_TYPE_OMV")]
+    #[serde(rename = "MUSIC_VIDEO_TYPE_OMV", alias = "MUSIC_VIDEO_TYPE_UGC")]
     Video,
     #[serde(rename = "MUSIC_VIDEO_TYPE_ATV")]
     Track,
+    #[serde(rename = "MUSIC_VIDEO_TYPE_PODCAST_EPISODE")]
+    Episode,
 }
 
-#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
+impl MusicVideoType {
+    pub fn is_video(self) -> bool {
+        self != Self::Track
+    }
+
+    pub fn from_is_video(is_video: bool) -> Self {
+        if is_video {
+            Self::Video
+        } else {
+            Self::Track
+        }
+    }
+}
+
+impl From<MusicVideoType> for TrackType {
+    fn from(value: MusicVideoType) -> Self {
+        match value {
+            MusicVideoType::Video => Self::Video,
+            MusicVideoType::Track => Self::Track,
+            MusicVideoType::Episode => Self::Episode,
+        }
+    }
+}
+
+#[derive(Default, Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
 pub(crate) enum PageType {
     #[serde(
         rename = "MUSIC_PAGE_TYPE_ARTIST",
@@ -160,15 +225,28 @@ pub(crate) enum PageType {
     Channel,
     #[serde(rename = "MUSIC_PAGE_TYPE_PLAYLIST", alias = "WEB_PAGE_TYPE_PLAYLIST")]
     Playlist,
+    #[serde(rename = "MUSIC_PAGE_TYPE_PODCAST_SHOW_DETAIL_PAGE")]
+    Podcast,
+    #[serde(rename = "MUSIC_PAGE_TYPE_NON_MUSIC_AUDIO_TRACK_PAGE")]
+    Episode,
+    #[default]
+    Unknown,
 }
 
 impl PageType {
-    pub(crate) fn to_url_target(self, id: String) -> UrlTarget {
+    pub(crate) fn to_url_target(self, id: String) -> Option<UrlTarget> {
         match self {
-            PageType::Artist => UrlTarget::Channel { id },
-            PageType::Album => UrlTarget::Album { id },
-            PageType::Channel => UrlTarget::Channel { id },
-            PageType::Playlist => UrlTarget::Playlist { id },
+            PageType::Artist | PageType::Channel => Some(UrlTarget::Channel { id }),
+            PageType::Album => Some(UrlTarget::Album { id }),
+            PageType::Playlist => Some(UrlTarget::Playlist { id }),
+            PageType::Podcast => Some(UrlTarget::Playlist {
+                id: util::strip_prefix(&id, util::PODCAST_PLAYLIST_PREFIX),
+            }),
+            PageType::Episode => Some(UrlTarget::Video {
+                id: util::strip_prefix(&id, util::PODCAST_EPISODE_PREFIX),
+                start_time: 0,
+            }),
+            PageType::Unknown => None,
         }
     }
 }
@@ -177,8 +255,9 @@ impl PageType {
 pub(crate) enum MusicPageType {
     Artist,
     Album,
-    Playlist,
-    Track { is_video: bool },
+    Playlist { is_podcast: bool },
+    Track { vtype: MusicVideoType },
+    User,
     None,
 }
 
@@ -187,45 +266,131 @@ impl From<PageType> for MusicPageType {
         match t {
             PageType::Artist => MusicPageType::Artist,
             PageType::Album => MusicPageType::Album,
-            PageType::Playlist => MusicPageType::Playlist,
-            PageType::Channel => MusicPageType::None,
+            PageType::Playlist => MusicPageType::Playlist { is_podcast: false },
+            PageType::Podcast => MusicPageType::Playlist { is_podcast: true },
+            PageType::Channel => MusicPageType::User,
+            PageType::Episode => MusicPageType::Track {
+                vtype: MusicVideoType::Episode,
+            },
+            PageType::Unknown => MusicPageType::None,
+        }
+    }
+}
+
+pub(crate) struct MusicPage {
+    pub id: String,
+    pub typ: MusicPageType,
+}
+
+impl MusicPage {
+    /// Create a new MusicPage object, applying the required ID fixes when
+    /// mapping a browse link
+    pub fn from_browse(mut id: String, typ: PageType) -> Self {
+        if typ == PageType::Podcast {
+            id = util::strip_prefix(&id, util::PODCAST_PLAYLIST_PREFIX);
+        } else if typ == PageType::Episode && id.len() == 15 {
+            id = util::strip_prefix(&id, util::PODCAST_EPISODE_PREFIX);
+        }
+
+        Self {
+            id,
+            typ: typ.into(),
         }
     }
 }
 
 impl NavigationEndpoint {
-    pub(crate) fn music_page(self) -> Option<(MusicPageType, String)> {
-        self.browse_endpoint
-            .and_then(|be| {
-                be.browse_endpoint_context_supported_configs.map(|config| {
-                    (
-                        config.browse_endpoint_context_music_config.page_type.into(),
-                        be.browse_id,
+    /// Get the YouTube Music page and id from a browse/watch endpoint
+    pub(crate) fn music_page(self) -> Option<MusicPage> {
+        match self {
+            NavigationEndpoint::Watch { watch_endpoint } => {
+                if watch_endpoint
+                    .playlist_id
+                    .map(|plid| plid.starts_with("RDQM"))
+                    .unwrap_or_default()
+                {
+                    // Genre radios (e.g. "pop radio") will be skipped
+                    Some(MusicPage {
+                        id: watch_endpoint.video_id,
+                        typ: MusicPageType::None,
+                    })
+                } else {
+                    Some(MusicPage {
+                        id: watch_endpoint.video_id,
+                        typ: MusicPageType::Track {
+                            vtype: watch_endpoint
+                                .watch_endpoint_music_supported_configs
+                                .watch_endpoint_music_config
+                                .music_video_type,
+                        },
+                    })
+                }
+            }
+            NavigationEndpoint::Browse {
+                browse_endpoint, ..
+            } => browse_endpoint
+                .browse_endpoint_context_supported_configs
+                .map(|config| {
+                    MusicPage::from_browse(
+                        browse_endpoint.browse_id,
+                        config.browse_endpoint_context_music_config.page_type,
                     )
+                }),
+            NavigationEndpoint::Url { .. } => None,
+            NavigationEndpoint::WatchPlaylist {
+                watch_playlist_endpoint,
+            } => Some(MusicPage {
+                id: watch_playlist_endpoint.playlist_id,
+                typ: MusicPageType::Playlist { is_podcast: false },
+            }),
+            NavigationEndpoint::CreatePlaylist { .. } => Some(MusicPage {
+                id: String::new(),
+                typ: MusicPageType::None,
+            }),
+        }
+    }
+
+    /// Get the page type of a browse endpoint
+    pub(crate) fn page_type(&self) -> Option<PageType> {
+        if let NavigationEndpoint::Browse {
+            browse_endpoint,
+            command_metadata,
+        } = self
+        {
+            browse_endpoint
+                .browse_endpoint_context_supported_configs
+                .as_ref()
+                .map(|c| c.browse_endpoint_context_music_config.page_type)
+                .or_else(|| {
+                    command_metadata
+                        .as_ref()
+                        .map(|c| c.web_command_metadata.web_page_type)
                 })
-            })
-            .or_else(|| {
-                self.watch_endpoint.map(|watch| {
-                    if watch
-                        .playlist_id
-                        .map(|plid| plid.starts_with("RDQM"))
+        } else {
+            None
+        }
+    }
+
+    pub(crate) fn into_playlist_id(self) -> Option<String> {
+        match self {
+            NavigationEndpoint::Watch { watch_endpoint } => watch_endpoint.playlist_id,
+            NavigationEndpoint::Browse {
+                browse_endpoint,
+                command_metadata,
+            } => Some(browse_endpoint.browse_id).filter(|_| {
+                browse_endpoint
+                    .browse_endpoint_context_supported_configs
+                    .map(|c| c.browse_endpoint_context_music_config.page_type == PageType::Playlist)
+                    .unwrap_or_default()
+                    || command_metadata
+                        .map(|c| c.web_command_metadata.web_page_type == PageType::Playlist)
                         .unwrap_or_default()
-                    {
-                        // Genre radios (e.g. "pop radio") will be skipped
-                        (MusicPageType::None, watch.video_id)
-                    } else {
-                        (
-                            MusicPageType::Track {
-                                is_video: watch
-                                    .watch_endpoint_music_supported_configs
-                                    .watch_endpoint_music_config
-                                    .music_video_type
-                                    == MusicVideoType::Video,
-                            },
-                            watch.video_id,
-                        )
-                    }
-                })
-            })
+            }),
+            NavigationEndpoint::Url { .. } => None,
+            NavigationEndpoint::WatchPlaylist {
+                watch_playlist_endpoint,
+            } => Some(watch_playlist_endpoint.playlist_id),
+            NavigationEndpoint::CreatePlaylist { .. } => None,
+        }
     }
 }

src/client/response/video_details.rs

@@ -3,24 +3,25 @@
 use serde::Deserialize;
 use serde_with::{rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError};
 
-use crate::serializer::text::TextComponent;
 use crate::serializer::{
-    text::{AccessibilityText, AttributedText, Text, TextComponents},
-    MapResult, VecLogError,
+    text::{AccessibilityText, AttributedText, Text, TextComponent, TextComponents},
+    MapResult,
 };
 
 use super::{
     url_endpoint::BrowseEndpointWrap, ContinuationEndpoint, ContinuationItemRenderer, Icon,
     MusicContinuationData, Thumbnails,
 };
-use super::{ChannelBadge, ResponseContext, YouTubeListItem};
+use super::{
+    ChannelBadge, ContentsRendererLogged, FrameworkUpdates, ImageView, ResponseContext,
+    YouTubeListItem,
+};
 
 /*
 #VIDEO DETAILS
 */
 
 /// Video details response
-#[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct VideoDetails {
@@ -29,7 +30,6 @@ pub(crate) struct VideoDetails {
     /// Video ID
     pub current_video_endpoint: Option<CurrentVideoEndpoint>,
     /// Video chapters + comment section
-    #[serde_as(as = "VecLogError<_>")]
     pub engagement_panels: MapResult<Vec<EngagementPanel>>,
     pub response_context: ResponseContext,
 }
@@ -60,11 +60,9 @@ pub(crate) struct VideoResultsWrap {
 }
 
 /// Video metadata items
-#[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct VideoResults {
-    #[serde_as(as = "Option<VecLogError<_>>")]
     pub contents: Option<MapResult<Vec<VideoResultsItem>>>,
 }
 
@@ -81,8 +79,8 @@ pub(crate) enum VideoResultsItem {
         /// Like/Dislike button
         video_actions: VideoActions,
         /// Absolute textual date (e.g. `Dec 29, 2019`)
-        #[serde_as(as = "Text")]
-        date_text: String,
+        #[serde_as(as = "Option<Text>")]
+        date_text: Option<String>,
     },
     #[serde(rename_all = "camelCase")]
     VideoSecondaryInfoRenderer {
@@ -151,6 +149,46 @@ pub(crate) enum TopLevelButton {
     SegmentedLikeDislikeButtonRenderer {
         like_button: ToggleButtonWrap,
     },
+    #[serde(rename_all = "camelCase")]
+    SegmentedLikeDislikeButtonViewModel {
+        like_button_view_model: LikeButtonViewModelWrap,
+    },
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct LikeButtonViewModelWrap {
+    pub like_button_view_model: LikeButtonViewModel,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct LikeButtonViewModel {
+    pub toggle_button_view_model: ToggleButtonViewModelWrap,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ToggleButtonViewModelWrap {
+    pub toggle_button_view_model: ToggleButtonViewModel,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ToggleButtonViewModel {
+    pub default_button_view_model: ButtonViewModelWrap,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ButtonViewModelWrap {
+    pub button_view_model: ButtonViewModel,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ButtonViewModel {
+    pub accessibility_text: String,
 }
 
 /// Like/Dislike button
@@ -303,7 +341,6 @@ pub(crate) struct RecommendationResultsWrap {
 #[serde(rename_all = "camelCase")]
 pub(crate) struct RecommendationResults {
     /// Can be `None` for age-restricted videos
-    #[serde_as(as = "Option<VecLogError<_>>")]
     pub results: Option<MapResult<Vec<YouTubeListItem>>>,
     #[serde_as(as = "Option<VecSkipError<_>>")]
     pub continuations: Option<Vec<MusicContinuationData>>,
@@ -341,16 +378,7 @@ pub(crate) enum EngagementPanelRenderer {
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct ChapterMarkersContent {
-    pub macro_markers_list_renderer: MacroMarkersListRenderer,
-}
-
-/// Chapter markers
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct MacroMarkersListRenderer {
-    #[serde_as(as = "VecLogError<_>")]
-    pub contents: MapResult<Vec<MacroMarkersListItem>>,
+    pub macro_markers_list_renderer: ContentsRendererLogged<MacroMarkersListItem>,
 }
 
 /// Chapter marker
@@ -436,7 +464,6 @@ pub(crate) struct CommentItemSectionHeaderMenuItem {
 */
 
 /// Video comments continuation response
-#[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct VideoComments {
@@ -450,8 +477,8 @@ pub(crate) struct VideoComments {
     /// - Comment replies: appendContinuationItemsAction
     ///   - n*commentRenderer, continuationItemRenderer:
     ///     replies + continuation
-    #[serde_as(as = "VecLogError<_>")]
     pub on_response_received_endpoints: MapResult<Vec<CommentsContItem>>,
+    pub framework_updates: Option<FrameworkUpdates<Payload>>,
 }
 
 /// Video comments continuation
@@ -463,11 +490,9 @@ pub(crate) struct CommentsContItem {
 }
 
 /// Video comments continuation action
-#[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct AppendComments {
-    #[serde_as(as = "VecLogError<_>")]
     pub continuation_items: MapResult<Vec<CommentListItem>>,
 }
 
@@ -476,23 +501,13 @@ pub(crate) struct AppendComments {
 #[serde(rename_all = "camelCase")]
 pub(crate) enum CommentListItem {
     /// Top-level comment
-    #[serde(rename_all = "camelCase")]
-    CommentThreadRenderer {
-        comment: Comment,
-        /// Continuation token to fetch replies
-        #[serde(default)]
-        replies: Replies,
-        #[serde(default)]
-        #[serde_as(deserialize_as = "DefaultOnError")]
-        rendering_priority: CommentPriority,
-    },
+    CommentThreadRenderer(CommentThreadRenderer),
     /// Reply comment
     CommentRenderer(CommentRenderer),
+    /// Reply comment (A/B #14)
+    CommentViewModel(CommentViewModel),
     /// Continuation token to fetch more comments
-    #[serde(rename_all = "camelCase")]
-    ContinuationItemRenderer {
-        continuation_endpoint: ContinuationEndpoint,
-    },
+    ContinuationItemRenderer(ContinuationItemVariants),
     /// Header of the comment section (contains number of comments)
     #[serde(rename_all = "camelCase")]
     CommentsHeaderRenderer {
@@ -502,6 +517,45 @@ pub(crate) enum CommentListItem {
     },
 }
 
+#[derive(Debug, Deserialize)]
+#[serde(untagged)]
+pub(crate) enum ContinuationItemVariants {
+    #[serde(rename_all = "camelCase")]
+    Ep {
+        continuation_endpoint: ContinuationEndpoint,
+    },
+    Btn {
+        button: ContinuationButton,
+    },
+}
+
+impl ContinuationItemVariants {
+    pub fn into_token(self) -> Option<String> {
+        match self {
+            ContinuationItemVariants::Ep {
+                continuation_endpoint,
+            } => continuation_endpoint,
+            ContinuationItemVariants::Btn { button } => button.button_renderer.command,
+        }
+        .into_token()
+    }
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct CommentThreadRenderer {
+    /// Missing on the FrameworkUpdate data model (A/B #14)
+    pub comment: Option<Comment>,
+    pub comment_view_model: Option<CommentViewModelWrap>,
+    /// Continuation token to fetch replies
+    #[serde(default)]
+    pub replies: Replies,
+    #[serde(default)]
+    #[serde_as(deserialize_as = "DefaultOnError")]
+    pub rendering_priority: CommentPriority,
+}
+
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct Comment {
@@ -536,11 +590,13 @@ pub(crate) struct CommentRenderer {
     pub author_comment_badge: Option<AuthorCommentBadge>,
     #[serde(default)]
     pub reply_count: u64,
+    #[serde_as(as = "Option<Text>")]
+    pub vote_count: Option<String>,
     /// Buttons for comment interaction (Like/Dislike/Reply)
     pub action_buttons: CommentActionButtons,
 }
 
-#[derive(Default, Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
+#[derive(Default, Clone, Copy, Debug, Deserialize)]
 #[serde(rename_all = "SCREAMING_SNAKE_CASE")]
 pub(crate) enum CommentPriority {
     /// Default rendering priority
@@ -550,6 +606,27 @@ pub(crate) enum CommentPriority {
     RenderingPriorityPinnedComment,
 }
 
+impl From<CommentPriority> for bool {
+    fn from(value: CommentPriority) -> Self {
+        matches!(value, CommentPriority::RenderingPriorityPinnedComment)
+    }
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct CommentViewModelWrap {
+    pub comment_view_model: CommentViewModel,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct CommentViewModel {
+    pub comment_id: String,
+    pub comment_key: String,
+    pub comment_surface_key: String,
+    pub toolbar_state_key: String,
+}
+
 /// Does not contain replies directly but a continuation token
 /// for fetching them.
 #[derive(Default, Debug, Deserialize)]
@@ -581,7 +658,6 @@ pub(crate) struct CommentActionButtons {
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct CommentActionButtonsRenderer {
-    pub like_button: ToggleButtonWrap,
     pub creator_heart: Option<CreatorHeart>,
 }
 
@@ -614,3 +690,107 @@ pub(crate) struct AuthorCommentBadgeRenderer {
     /// Artist: `OFFICIAL_ARTIST_BADGE`
     pub icon: Icon,
 }
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) enum Payload {
+    CommentEntityPayload(CommentEntityPayload),
+    CommentSurfaceEntityPayload(CommentSurfaceEntityPayload),
+    #[serde(rename_all = "camelCase")]
+    EngagementToolbarStateEntityPayload {
+        heart_state: HeartState,
+    },
+    #[serde(other, deserialize_with = "deserialize_ignore_any")]
+    None,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct CommentEntityPayload {
+    pub properties: CommentProperties,
+    #[serde(default)]
+    #[serde_as(as = "DefaultOnError")]
+    pub author: Option<CommentAuthor>,
+    pub toolbar: CommentToolbar,
+    #[serde(default)]
+    pub avatar: ImageView,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct CommentSurfaceEntityPayload {
+    pub voice_reply_container_view_model: Option<VoiceReplyContainer>,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct CommentProperties {
+    #[serde_as(as = "AttributedText")]
+    pub content: TextComponents,
+    pub published_time: String,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct CommentAuthor {
+    pub channel_id: String,
+    pub display_name: String,
+    #[serde(default)]
+    pub is_verified: bool,
+    #[serde(default)]
+    pub is_artist: bool,
+    #[serde(default)]
+    pub is_creator: bool,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct CommentToolbar {
+    pub like_count_notliked: String,
+    pub reply_count: String,
+}
+
+#[derive(Debug, Copy, Clone, Deserialize)]
+#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
+pub(crate) enum HeartState {
+    ToolbarHeartStateUnhearted,
+    ToolbarHeartStateHearted,
+}
+
+impl From<HeartState> for bool {
+    fn from(value: HeartState) -> Self {
+        match value {
+            HeartState::ToolbarHeartStateUnhearted => false,
+            HeartState::ToolbarHeartStateHearted => true,
+        }
+    }
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ContinuationButton {
+    pub button_renderer: ContinuationButtonRenderer,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ContinuationButtonRenderer {
+    pub command: ContinuationEndpoint,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct VoiceReplyContainer {
+    pub voice_reply_container_view_model: VoiceReplyContainer2,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct VoiceReplyContainer2 {
+    #[serde_as(as = "AttributedText")]
+    pub transcript_text: TextComponents,
+}

src/client/response/video_item.rs

@@ -1,26 +1,25 @@
-use once_cell::sync::Lazy;
-use regex::Regex;
 use serde::Deserialize;
 use serde_with::{
-    json::JsonString, rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError,
+    rust::deserialize_ignore_any, serde_as, DefaultOnError, DisplayFromStr, VecSkipError,
 };
-use time::{Duration, OffsetDateTime};
+use time::OffsetDateTime;
 
-use super::{url_endpoint::NavigationEndpoint, ChannelBadge, ContinuationEndpoint, Thumbnails};
+use super::{ChannelBadge, ContentImage, ContinuationItemRenderer, PhMetadataView, Thumbnails};
 use crate::{
-    model::{
-        Channel, ChannelId, ChannelInfo, ChannelItem, ChannelTag, PlaylistItem, VideoItem,
-        YouTubeItem,
-    },
+    model::{Channel, ChannelItem, ChannelTag, PlaylistItem, VideoItem, YouTubeItem},
     param::Language,
     serializer::{
-        text::{AccessibilityText, Text, TextComponent},
-        MapResult, VecLogError,
+        text::{AttributedText, Text, TextComponent},
+        MapResult,
     },
-    timeago,
-    util::{self, TryRemove},
+    util::{self, timeago, TryRemove},
 };
 
+#[cfg(feature = "userdata")]
+use crate::{client::response::SimpleHeaderRenderer, model::HistoryItem};
+#[cfg(feature = "userdata")]
+use time::UtcOffset;
+
 #[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -28,18 +27,19 @@ pub(crate) enum YouTubeListItem {
     #[serde(alias = "gridVideoRenderer", alias = "compactVideoRenderer")]
     VideoRenderer(VideoRenderer),
     ReelItemRenderer(ReelItemRenderer),
+    ShortsLockupViewModel(ShortsLockupViewModel),
+    PlaylistVideoRenderer(PlaylistVideoRenderer),
 
     #[serde(alias = "gridPlaylistRenderer")]
     PlaylistRenderer(PlaylistRenderer),
 
     ChannelRenderer(ChannelRenderer),
 
-    /// Continauation items are located at the end of a list
+    LockupViewModel(LockupViewModel),
+
+    /// Continuation items are located at the end of a list
     /// and contain the continuation token for progressive loading
-    #[serde(rename_all = "camelCase")]
-    ContinuationItemRenderer {
-        continuation_endpoint: ContinuationEndpoint,
-    },
+    ContinuationItemRenderer(ContinuationItemRenderer),
 
     /// Corrected search query
     #[serde(rename_all = "camelCase")]
@@ -48,9 +48,6 @@ pub(crate) enum YouTubeListItem {
         corrected_query: String,
     },
 
-    /// Channel metadata (about tab)
-    ChannelAboutFullMetadataRenderer(ChannelFullMetadata),
-
     /// Contains video on startpage
     ///
     /// Seems to be currently A/B tested on the channel page,
@@ -68,11 +65,20 @@ pub(crate) enum YouTubeListItem {
     /// GridRenderer: contains videos on channel page
     #[serde(alias = "expandedShelfContentsRenderer", alias = "gridRenderer")]
     ItemSectionRenderer {
+        #[cfg(feature = "userdata")]
+        header: Option<ItemSectionHeader>,
         #[serde(alias = "items")]
-        #[serde_as(as = "VecLogError<_>")]
         contents: MapResult<Vec<YouTubeListItem>>,
     },
 
+    /// Age-restricted channel
+    #[serde(rename_all = "camelCase")]
+    ChannelAgeGateRenderer {
+        channel_title: String,
+        #[serde_as(as = "Text")]
+        main_text: String,
+    },
+
     /// No video list item (e.g. ad) or unimplemented item
     ///
     /// Unimplemented:
@@ -135,18 +141,98 @@ pub(crate) struct ReelItemRenderer {
     /// Contains `No views` if the view count is zero
     #[serde_as(as = "Option<Text>")]
     pub view_count_text: Option<String>,
-    /// video duration
-    ///
-    /// Example: `the horror maze - 44 seconds - play video`
-    ///
-    /// Dashes may be `\u2013` (emdash)
-    #[serde_as(as = "Option<AccessibilityText>")]
-    pub accessibility: Option<String>,
     #[serde(default)]
     #[serde_as(as = "DefaultOnError")]
     pub navigation_endpoint: Option<ReelNavigationEndpoint>,
 }
 
+// New short video item
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ShortsLockupViewModel {
+    /// `shorts-shelf-item-[video_id]`
+    pub entity_id: String,
+    pub thumbnail: Thumbnails,
+    pub overlay_metadata: ShortsOverlayMetadata,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ShortsOverlayMetadata {
+    /// Title
+    #[serde_as(as = "AttributedText")]
+    pub primary_text: String,
+    /// View count
+    #[serde_as(as = "Option<AttributedText>")]
+    pub secondary_text: Option<String>,
+}
+
+/// Generalized list item, currently only used for channel playlists and YTM items
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct LockupViewModel {
+    pub content_id: String,
+    #[serde(default)]
+    #[serde_as(deserialize_as = "DefaultOnError")]
+    pub content_type: LockupContentType,
+    pub content_image: ContentImage,
+    pub metadata: LockupViewModelMetadata,
+}
+
+#[derive(Default, Debug, Deserialize)]
+#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
+#[allow(clippy::enum_variant_names)]
+pub(crate) enum LockupContentType {
+    LockupContentTypePlaylist,
+    LockupContentTypeVideo,
+    #[default]
+    Unknown,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct LockupViewModelMetadata {
+    pub lockup_metadata_view_model: LockupViewModelMetadataInner,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct LockupViewModelMetadataInner {
+    #[serde_as(as = "AttributedText")]
+    pub title: String,
+    pub metadata: PhMetadataView,
+}
+
+/// Video displayed in a playlist
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PlaylistVideoRenderer {
+    pub video_id: String,
+    pub thumbnail: Thumbnails,
+    #[serde_as(as = "Text")]
+    pub title: String,
+    #[serde(rename = "shortBylineText")]
+    pub channel: TextComponent,
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    pub length_seconds: Option<u32>,
+    /// Regular video: `["29K views", " • ", "13 years ago"]`
+    /// Livestream: `["66K", " watching"]`
+    /// Upcoming: `["8", " waiting"]`
+    #[serde(default)]
+    #[serde_as(as = "DefaultOnError<Text>")]
+    pub video_info: Vec<String>,
+    /// Contains Short/Live tag
+    #[serde(default)]
+    #[serde_as(as = "VecSkipError<_>")]
+    pub thumbnail_overlays: Vec<TimeOverlay>,
+    /// Release date for upcoming videos
+    pub upcoming_event_data: Option<UpcomingEventData>,
+}
+
 /// Playlist displayed in search results
 #[serde_as]
 #[derive(Debug, Deserialize)]
@@ -161,7 +247,7 @@ pub(crate) struct PlaylistRenderer {
     /// The first item of this list contains the playlist thumbnail,
     /// subsequent items contain very small thumbnails of the next playlist videos
     pub thumbnails: Option<Vec<Thumbnails>>,
-    #[serde_as(as = "Option<JsonString>")]
+    #[serde_as(as = "Option<DisplayFromStr>")]
     pub video_count: Option<u64>,
     #[serde_as(as = "Option<Text>")]
     pub video_count_short_text: Option<String>,
@@ -206,20 +292,25 @@ pub(crate) struct YouTubeListRendererWrap {
     pub section_list_renderer: YouTubeListRenderer,
 }
 
-#[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct YouTubeListRenderer {
-    #[serde_as(as = "VecLogError<_>")]
     pub contents: MapResult<Vec<YouTubeListItem>>,
 }
 
+#[cfg(feature = "userdata")]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ItemSectionHeader {
+    pub item_section_header_renderer: SimpleHeaderRenderer,
+}
+
 #[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct UpcomingEventData {
     /// Unixtime in seconds
-    #[serde_as(as = "JsonString")]
+    #[serde_as(as = "DisplayFromStr")]
     pub start_time: i64,
 }
 
@@ -273,7 +364,6 @@ pub(crate) enum TimeOverlayStyle {
     Default,
     Live,
     Shorts,
-    Upcoming,
 }
 
 #[serde_as]
@@ -335,40 +425,14 @@ pub(crate) struct ReelPlayerHeaderRenderer {
     pub timestamp_text: String,
 }
 
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct ChannelFullMetadata {
-    #[serde_as(as = "Text")]
-    pub joined_date_text: String,
-    #[serde_as(as = "Option<Text>")]
-    pub view_count_text: Option<String>,
-    #[serde(default)]
-    #[serde_as(as = "VecSkipError<_>")]
-    pub primary_links: Vec<PrimaryLink>,
-}
-
-#[serde_as]
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct PrimaryLink {
-    #[serde_as(as = "Text")]
-    pub title: String,
-    pub navigation_endpoint: NavigationEndpoint,
-}
-
-pub(crate) trait IsLive {
+trait IsLive {
     fn is_live(&self) -> bool;
 }
 
-pub(crate) trait IsShort {
+trait IsShort {
     fn is_short(&self) -> bool;
 }
 
-pub(crate) trait IsUpcoming {
-    fn is_upcoming(&self) -> bool;
-}
-
 impl IsLive for Vec<VideoBadge> {
     fn is_live(&self) -> bool {
         self.iter().any(|badge| {
@@ -393,14 +457,6 @@ impl IsShort for Vec<TimeOverlay> {
     }
 }
 
-impl IsUpcoming for Vec<TimeOverlay> {
-    fn is_upcoming(&self) -> bool {
-        self.iter().any(|overlay| {
-            overlay.thumbnail_overlay_time_status_renderer.style == TimeOverlayStyle::Upcoming
-        })
-    }
-}
-
 /// Result of mapping a list of different YouTube enities
 /// (videos, channels, playlists)
 #[derive(Debug)]
@@ -412,7 +468,6 @@ pub(crate) struct YouTubeListMapper<T> {
     pub warnings: Vec<String>,
     pub ctoken: Option<String>,
     pub corrected_query: Option<String>,
-    pub channel_info: Option<ChannelInfo>,
 }
 
 impl<T> YouTubeListMapper<T> {
@@ -424,56 +479,59 @@ impl<T> YouTubeListMapper<T> {
             warnings: Vec::new(),
             ctoken: None,
             corrected_query: None,
-            channel_info: None,
         }
     }
 
-    pub fn with_channel<C>(lang: Language, channel: &Channel<C>) -> Self {
+    pub fn with_channel<C>(lang: Language, channel: &Channel<C>, warnings: Vec<String>) -> Self {
         Self {
             lang,
             channel: Some(ChannelTag {
-                id: channel.id.to_owned(),
-                name: channel.name.to_owned(),
+                id: channel.id.clone(),
+                name: channel.name.clone(),
                 avatar: Vec::new(),
                 verification: channel.verification,
                 subscriber_count: channel.subscriber_count,
             }),
             items: Vec::new(),
-            warnings: Vec::new(),
+            warnings,
             ctoken: None,
             corrected_query: None,
-            channel_info: None,
         }
     }
 
     fn map_video(&mut self, video: VideoRenderer) -> VideoItem {
-        let mut tn_overlays = video.thumbnail_overlays;
+        let is_live = video.thumbnail_overlays.is_live() || video.badges.is_live();
+        let is_short = video.thumbnail_overlays.is_short();
+
         let length_text = video.length_text.or_else(|| {
-            tn_overlays
-                .try_swap_remove(0)
-                .map(|overlay| overlay.thumbnail_overlay_time_status_renderer.text)
+            video
+                .thumbnail_overlays
+                .into_iter()
+                .find(|ol| {
+                    ol.thumbnail_overlay_time_status_renderer.style == TimeOverlayStyle::Default
+                })
+                .map(|ol| ol.thumbnail_overlay_time_status_renderer.text)
         });
 
         VideoItem {
             id: video.video_id,
             name: video.title,
-            length: length_text.and_then(|txt| util::parse_video_length(&txt)),
+            duration: length_text.and_then(|txt| util::parse_video_length(&txt)),
             thumbnail: video.thumbnail.into(),
             channel: video
                 .channel
-                .and_then(|c| {
-                    ChannelId::try_from(c).ok().map(|c| ChannelTag {
-                        id: c.id,
-                        name: c.name,
-                        avatar: video
-                            .channel_thumbnail_supported_renderers
-                            .map(|tn| tn.channel_thumbnail_with_link_renderer.thumbnail)
-                            .or(video.channel_thumbnail)
-                            .unwrap_or_default()
-                            .into(),
-                        verification: video.owner_badges.into(),
-                        subscriber_count: None,
-                    })
+                .and_then(|c| ChannelTag::try_from(c).ok())
+                .map(|mut c| {
+                    c.avatar = video
+                        .channel_thumbnail_supported_renderers
+                        .map(|tn| tn.channel_thumbnail_with_link_renderer.thumbnail)
+                        .or(video.channel_thumbnail)
+                        .unwrap_or_default()
+                        .into();
+                    if !c.verification.verified() {
+                        c.verification = video.owner_badges.into();
+                    }
+                    c
                 })
                 .or_else(|| self.channel.clone()),
             publish_date: video
@@ -489,20 +547,17 @@ impl<T> YouTubeListMapper<T> {
             view_count: video
                 .view_count_text
                 .map(|txt| util::parse_numeric(&txt).unwrap_or_default()),
-            is_live: tn_overlays.is_live() || video.badges.is_live(),
-            is_short: tn_overlays.is_short(),
+            is_live,
+            is_short,
             is_upcoming: video.upcoming_event_data.is_some(),
             short_description: video
                 .detailed_metadata_snippets
-                .and_then(|mut snippets| snippets.try_swap_remove(0).map(|s| s.snippet_text))
+                .and_then(|snippets| snippets.into_iter().next().map(|s| s.snippet_text))
                 .or(video.description_snippet),
         }
     }
 
-    fn map_short_video(&mut self, video: ReelItemRenderer, lang: Language) -> VideoItem {
-        static ACCESSIBILITY_SEP_REGEX: Lazy<Regex> =
-            Lazy::new(|| Regex::new(" [-\u{2013}] (.+) [-\u{2013}] ").unwrap());
-
+    fn map_short_video(&mut self, video: ReelItemRenderer) -> VideoItem {
         let pub_date_txt = video.navigation_endpoint.map(|n| {
             n.reel_watch_endpoint
                 .overlay
@@ -515,23 +570,16 @@ impl<T> YouTubeListMapper<T> {
         VideoItem {
             id: video.video_id,
             name: video.headline,
-            length: video.accessibility.and_then(|acc| {
-                ACCESSIBILITY_SEP_REGEX.captures(&acc).and_then(|cap| {
-                    cap.get(1).and_then(|c| {
-                        timeago::parse_timeago_or_warn(self.lang, c.as_str(), &mut self.warnings)
-                            .map(|ta| Duration::from(ta).whole_seconds() as u32)
-                    })
-                })
-            }),
+            duration: None,
             thumbnail: video.thumbnail.into(),
             channel: self.channel.clone(),
             publish_date: pub_date_txt.as_ref().and_then(|txt| {
                 timeago::parse_timeago_dt_or_warn(self.lang, txt, &mut self.warnings)
             }),
             publish_date_txt: pub_date_txt,
-            view_count: video
-                .view_count_text
-                .map(|txt| util::parse_large_numstr(&txt, lang).unwrap_or_default()),
+            view_count: video.view_count_text.and_then(|txt| {
+                util::parse_large_numstr_or_warn(&txt, self.lang, &mut self.warnings)
+            }),
             is_live: false,
             is_short: true,
             is_upcoming: false,
@@ -539,6 +587,84 @@ impl<T> YouTubeListMapper<T> {
         }
     }
 
+    fn map_short_video2(&mut self, video: ShortsLockupViewModel) -> Option<VideoItem> {
+        if let Some(video_id) = video.entity_id.strip_prefix("shorts-shelf-item-") {
+            Some(VideoItem {
+                id: video_id.to_owned(),
+                name: video.overlay_metadata.primary_text,
+                duration: None,
+                thumbnail: video.thumbnail.into(),
+                channel: self.channel.clone(),
+                publish_date: None,
+                publish_date_txt: None,
+                view_count: video.overlay_metadata.secondary_text.and_then(|txt| {
+                    util::parse_large_numstr_or_warn(&txt, self.lang, &mut self.warnings)
+                }),
+                is_live: false,
+                is_short: true,
+                is_upcoming: false,
+                short_description: None,
+            })
+        } else {
+            self.warnings
+                .push(format!("invalid shorts entityId: {}", video.entity_id));
+            None
+        }
+    }
+
+    fn map_playlist_video(&mut self, video: PlaylistVideoRenderer) -> VideoItem {
+        let channel = ChannelTag::try_from(video.channel).ok();
+        let mut video_info = video.video_info.into_iter();
+        let video_info1 = video_info
+            .next()
+            .map(|s| match video_info.next().as_deref() {
+                None | Some(util::DOT_SEPARATOR) => s,
+                Some(s2) => s + s2,
+            });
+        let video_info2 = video_info.next();
+
+        // RU: "7 лет назад" " • " "210 млн просмотров" (order flipped)
+        let (view_count_txt, publish_date_txt) =
+            if self.lang == Language::Ru && video_info2.is_some() {
+                (video_info2, video_info1)
+            } else {
+                (video_info1, video_info2)
+            };
+
+        let is_live = video.thumbnail_overlays.is_live();
+
+        let publish_date = video
+            .upcoming_event_data
+            .as_ref()
+            .and_then(|upc| OffsetDateTime::from_unix_timestamp(upc.start_time).ok())
+            .or_else(|| {
+                if is_live {
+                    None
+                } else {
+                    publish_date_txt.as_ref().and_then(|txt| {
+                        timeago::parse_timeago_dt_or_warn(self.lang, txt, &mut self.warnings)
+                    })
+                }
+            });
+
+        VideoItem {
+            id: video.video_id,
+            name: video.title,
+            duration: video.length_seconds,
+            thumbnail: video.thumbnail.into(),
+            channel,
+            publish_date,
+            publish_date_txt,
+            view_count: view_count_txt.and_then(|txt| {
+                util::parse_large_numstr_or_warn(&txt, self.lang, &mut self.warnings)
+            }),
+            is_live,
+            is_short: video.thumbnail_overlays.is_short(),
+            is_upcoming: video.upcoming_event_data.is_some(),
+            short_description: None,
+        }
+    }
+
     fn map_playlist(&self, playlist: PlaylistRenderer) -> PlaylistItem {
         PlaylistItem {
             id: playlist.playlist_id,
@@ -550,14 +676,12 @@ impl<T> YouTubeListMapper<T> {
                 .into(),
             channel: playlist
                 .channel
-                .and_then(|c| {
-                    ChannelId::try_from(c).ok().map(|c| ChannelTag {
-                        id: c.id,
-                        name: c.name,
-                        avatar: Vec::new(),
-                        verification: playlist.owner_badges.into(),
-                        subscriber_count: None,
-                    })
+                .and_then(|c| ChannelTag::try_from(c).ok())
+                .map(|mut c| {
+                    if !c.verification.verified() {
+                        c.verification = playlist.owner_badges.into();
+                    }
+                    c
                 })
                 .or_else(|| self.channel.clone()),
             video_count: playlist.video_count.or_else(|| {
@@ -570,28 +694,112 @@ impl<T> YouTubeListMapper<T> {
 
     fn map_channel(&mut self, channel: ChannelRenderer) -> ChannelItem {
         // channel handle instead of subscriber count (A/B test 3)
-        let (sc_txt, vc_text) = match channel
+        let (handle, sc_txt) = if channel
             .subscriber_count_text
             .as_ref()
             .map(|txt| txt.starts_with('@'))
             .unwrap_or_default()
         {
-            true => (channel.video_count_text, None),
-            false => (channel.subscriber_count_text, channel.video_count_text),
+            (channel.subscriber_count_text, channel.video_count_text)
+        } else {
+            (None, channel.subscriber_count_text)
         };
 
         ChannelItem {
             id: channel.channel_id,
             name: channel.title,
+            handle,
             avatar: channel.thumbnail.into(),
             verification: channel.owner_badges.into(),
-            subscriber_count: sc_txt
-                .and_then(|txt| util::parse_numeric_or_warn(&txt, &mut self.warnings)),
-            video_count: vc_text
-                .and_then(|txt| util::parse_numeric_or_warn(&txt, &mut self.warnings)),
+            subscriber_count: sc_txt.and_then(|txt| {
+                util::parse_large_numstr_or_warn(&txt, self.lang, &mut self.warnings)
+            }),
             short_description: channel.description_snippet,
         }
     }
+
+    fn map_lockup(&mut self, lockup: LockupViewModel) -> Option<YouTubeItem> {
+        let md = lockup.metadata.lockup_metadata_view_model;
+        let tn = lockup.content_image.into_image();
+        match lockup.content_type {
+            LockupContentType::LockupContentTypePlaylist => {
+                Some(YouTubeItem::Playlist(PlaylistItem {
+                    id: lockup.content_id,
+                    name: md.title,
+                    thumbnail: tn.image.into(),
+                    channel: self.channel.clone(),
+                    video_count: tn
+                        .overlays
+                        .first()
+                        .and_then(|ol| {
+                            ol.thumbnail_overlay_badge_view_model
+                                .thumbnail_badges
+                                .first()
+                        })
+                        .and_then(|badge| {
+                            util::parse_numeric(&badge.thumbnail_badge_view_model.text).ok()
+                        }),
+                }))
+            }
+            LockupContentType::LockupContentTypeVideo => {
+                let mut mdr = md
+                    .metadata
+                    .content_metadata_view_model
+                    .metadata_rows
+                    .into_iter();
+                let channel = mdr
+                    .next()
+                    .and_then(|r| r.metadata_parts.into_iter().next())
+                    .and_then(|p| ChannelTag::try_from(p.into_text_component()).ok());
+                let (view_count, publish_date_txt) = mdr
+                    .next()
+                    .map(|metadata_row| {
+                        let mut parts = metadata_row.metadata_parts.into_iter();
+                        let p1 = parts.next();
+                        let p2 = parts.next();
+                        (
+                            p1.and_then(|p| {
+                                util::parse_large_numstr_or_warn(
+                                    p.as_str(),
+                                    self.lang,
+                                    &mut self.warnings,
+                                )
+                            }),
+                            p2.map(|p2| p2.into_text_component().into_string()),
+                        )
+                    })
+                    .unwrap_or_default();
+
+                Some(YouTubeItem::Video(VideoItem {
+                    id: lockup.content_id,
+                    name: md.title,
+                    duration: tn
+                        .overlays
+                        .first()
+                        .and_then(|ol| {
+                            ol.thumbnail_overlay_badge_view_model
+                                .thumbnail_badges
+                                .first()
+                        })
+                        .and_then(|badge| {
+                            util::parse_video_length(&badge.thumbnail_badge_view_model.text)
+                        }),
+                    thumbnail: tn.image.into(),
+                    channel,
+                    publish_date: publish_date_txt.as_deref().and_then(|t| {
+                        timeago::parse_timeago_dt_or_warn(self.lang, t, &mut self.warnings)
+                    }),
+                    publish_date_txt,
+                    view_count,
+                    is_live: false,
+                    is_short: false,
+                    is_upcoming: false,
+                    short_description: None,
+                }))
+            }
+            LockupContentType::Unknown => None,
+        }
+    }
 }
 
 impl YouTubeListMapper<YouTubeItem> {
@@ -601,8 +809,17 @@ impl YouTubeListMapper<YouTubeItem> {
                 let mapped = YouTubeItem::Video(self.map_video(video));
                 self.items.push(mapped);
             }
+            YouTubeListItem::ShortsLockupViewModel(video) => {
+                if let Some(mapped) = self.map_short_video2(video) {
+                    self.items.push(YouTubeItem::Video(mapped));
+                }
+            }
             YouTubeListItem::ReelItemRenderer(video) => {
-                let mapped = self.map_short_video(video, self.lang);
+                let mapped = self.map_short_video(video);
+                self.items.push(YouTubeItem::Video(mapped));
+            }
+            YouTubeListItem::PlaylistVideoRenderer(video) => {
+                let mapped = self.map_playlist_video(video);
                 self.items.push(YouTubeItem::Video(mapped));
             }
             YouTubeListItem::PlaylistRenderer(playlist) => {
@@ -613,42 +830,27 @@ impl YouTubeListMapper<YouTubeItem> {
                 let mapped = YouTubeItem::Channel(self.map_channel(channel));
                 self.items.push(mapped);
             }
-            YouTubeListItem::ContinuationItemRenderer {
-                continuation_endpoint,
-            } => self.ctoken = Some(continuation_endpoint.continuation_command.token),
+            YouTubeListItem::LockupViewModel(lockup) => {
+                if let Some(mapped) = self.map_lockup(lockup) {
+                    self.items.push(mapped);
+                }
+            }
+            YouTubeListItem::ContinuationItemRenderer(r) => {
+                if self.ctoken.is_none() {
+                    self.ctoken = r.continuation_endpoint.into_token();
+                }
+            }
             YouTubeListItem::ShowingResultsForRenderer { corrected_query } => {
                 self.corrected_query = Some(corrected_query);
             }
-            YouTubeListItem::ChannelAboutFullMetadataRenderer(meta) => {
-                self.channel_info = Some(ChannelInfo {
-                    create_date: timeago::parse_textual_date_or_warn(
-                        self.lang,
-                        &meta.joined_date_text,
-                        &mut self.warnings,
-                    )
-                    .map(OffsetDateTime::date),
-                    view_count: meta
-                        .view_count_text
-                        .and_then(|txt| util::parse_numeric_or_warn(&txt, &mut self.warnings)),
-                    links: meta
-                        .primary_links
-                        .into_iter()
-                        .filter_map(|l| {
-                            l.navigation_endpoint
-                                .url_endpoint
-                                .map(|url| (l.title, util::sanitize_yt_url(&url.url)))
-                        })
-                        .collect(),
-                })
-            }
             YouTubeListItem::RichItemRenderer { content } => {
                 self.map_item(*content);
             }
-            YouTubeListItem::ItemSectionRenderer { mut contents } => {
+            YouTubeListItem::ItemSectionRenderer { mut contents, .. } => {
                 self.warnings.append(&mut contents.warnings);
                 contents.c.into_iter().for_each(|it| self.map_item(it));
             }
-            YouTubeListItem::None => {}
+            YouTubeListItem::None | YouTubeListItem::ChannelAgeGateRenderer { .. } => {}
         }
     }
 
@@ -666,19 +868,35 @@ impl YouTubeListMapper<VideoItem> {
                 self.items.push(mapped);
             }
             YouTubeListItem::ReelItemRenderer(video) => {
-                let mapped = self.map_short_video(video, self.lang);
+                let mapped = self.map_short_video(video);
                 self.items.push(mapped);
             }
-            YouTubeListItem::ContinuationItemRenderer {
-                continuation_endpoint,
-            } => self.ctoken = Some(continuation_endpoint.continuation_command.token),
+            YouTubeListItem::ShortsLockupViewModel(video) => {
+                if let Some(mapped) = self.map_short_video2(video) {
+                    self.items.push(mapped);
+                }
+            }
+            YouTubeListItem::PlaylistVideoRenderer(video) => {
+                let mapped = self.map_playlist_video(video);
+                self.items.push(mapped);
+            }
+            YouTubeListItem::LockupViewModel(lockup) => {
+                if let Some(YouTubeItem::Video(mapped)) = self.map_lockup(lockup) {
+                    self.items.push(mapped);
+                }
+            }
+            YouTubeListItem::ContinuationItemRenderer(r) => {
+                if self.ctoken.is_none() {
+                    self.ctoken = r.continuation_endpoint.into_token();
+                }
+            }
             YouTubeListItem::ShowingResultsForRenderer { corrected_query } => {
                 self.corrected_query = Some(corrected_query);
             }
             YouTubeListItem::RichItemRenderer { content } => {
                 self.map_item(*content);
             }
-            YouTubeListItem::ItemSectionRenderer { mut contents } => {
+            YouTubeListItem::ItemSectionRenderer { mut contents, .. } => {
                 self.warnings.append(&mut contents.warnings);
                 contents.c.into_iter().for_each(|it| self.map_item(it));
             }
@@ -690,6 +908,23 @@ impl YouTubeListMapper<VideoItem> {
         self.warnings.append(&mut res.warnings);
         res.c.into_iter().for_each(|item| self.map_item(item));
     }
+
+    #[cfg(feature = "userdata")]
+    pub(crate) fn conv_history_items(
+        self,
+        date_txt: Option<String>,
+        utc_offset: UtcOffset,
+        res: &mut MapResult<Vec<HistoryItem<VideoItem>>>,
+    ) {
+        res.warnings.extend(self.warnings);
+        res.c.extend(self.items.into_iter().map(|item| HistoryItem {
+            item,
+            playback_date: date_txt.as_deref().and_then(|s| {
+                timeago::parse_textual_date_to_d(self.lang, utc_offset, s, &mut res.warnings)
+            }),
+            playback_date_txt: date_txt.clone(),
+        }));
+    }
 }
 
 impl YouTubeListMapper<PlaylistItem> {
@@ -697,18 +932,25 @@ impl YouTubeListMapper<PlaylistItem> {
         match item {
             YouTubeListItem::PlaylistRenderer(playlist) => {
                 let mapped = self.map_playlist(playlist);
-                self.items.push(mapped)
+                self.items.push(mapped);
+            }
+            YouTubeListItem::LockupViewModel(lockup) => {
+                if let Some(YouTubeItem::Playlist(mapped)) = self.map_lockup(lockup) {
+                    self.items.push(mapped);
+                }
+            }
+            YouTubeListItem::ContinuationItemRenderer(r) => {
+                if self.ctoken.is_none() {
+                    self.ctoken = r.continuation_endpoint.into_token();
+                }
             }
-            YouTubeListItem::ContinuationItemRenderer {
-                continuation_endpoint,
-            } => self.ctoken = Some(continuation_endpoint.continuation_command.token),
             YouTubeListItem::ShowingResultsForRenderer { corrected_query } => {
                 self.corrected_query = Some(corrected_query);
             }
             YouTubeListItem::RichItemRenderer { content } => {
                 self.map_item(*content);
             }
-            YouTubeListItem::ItemSectionRenderer { mut contents } => {
+            YouTubeListItem::ItemSectionRenderer { mut contents, .. } => {
                 self.warnings.append(&mut contents.warnings);
                 contents.c.into_iter().for_each(|it| self.map_item(it));
             }

src/client/search.rs

@@ -1,33 +1,37 @@
-use std::borrow::Cow;
+use std::fmt::Debug;
 
-use serde::{de::IgnoredAny, Serialize};
+use serde::Serialize;
 
 use crate::{
     error::{Error, ExtractionError},
-    model::{paginator::Paginator, SearchResult, YouTubeItem},
+    model::{
+        paginator::{ContinuationEndpoint, Paginator},
+        traits::FromYtItem,
+        SearchResult, YouTubeItem,
+    },
     param::search_filter::SearchFilter,
 };
 
-use super::{response, ClientType, MapResponse, MapResult, RustyPipeQuery, YTContext};
+use super::{response, ClientType, MapRespCtx, MapResponse, MapResult, RustyPipeQuery};
 
 #[derive(Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
 struct QSearch<'a> {
-    context: YTContext<'a>,
     query: &'a str,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    params: Option<String>,
+    params: &'a str,
 }
 
 impl RustyPipeQuery {
     /// Search YouTube
-    pub async fn search<S: AsRef<str>>(&self, query: S) -> Result<SearchResult, Error> {
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn search<T: FromYtItem, S: AsRef<str> + Debug>(
+        &self,
+        query: S,
+    ) -> Result<SearchResult<T>, Error> {
         let query = query.as_ref();
-        let context = self.get_context(ClientType::Desktop, true, None).await;
         let request_body = QSearch {
-            context,
             query,
-            params: None,
+            params: "8AEB",
         };
 
         self.execute_request::<response::Search, _, _>(
@@ -41,17 +45,16 @@ impl RustyPipeQuery {
     }
 
     /// Search YouTube using the given [`SearchFilter`]
-    pub async fn search_filter<S: AsRef<str>>(
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn search_filter<T: FromYtItem, S: AsRef<str> + Debug>(
         &self,
         query: S,
         filter: &SearchFilter,
-    ) -> Result<SearchResult, Error> {
+    ) -> Result<SearchResult<T>, Error> {
         let query = query.as_ref();
-        let context = self.get_context(ClientType::Desktop, true, None).await;
         let request_body = QSearch {
-            context,
             query,
-            params: Some(filter.encode()),
+            params: &filter.encode(),
         };
 
         self.execute_request::<response::Search, _, _>(
@@ -65,40 +68,38 @@ impl RustyPipeQuery {
     }
 
     /// Get YouTube search suggestions
-    pub async fn search_suggestion<S: AsRef<str>>(&self, query: S) -> Result<Vec<String>, Error> {
-        let url = url::Url::parse_with_params("https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&gs_rn=64&gs_ri=youtube&ds=yt&cp=1&gs_id=4&xhr=t&xssi=t",
-            &[("hl", self.opts.lang.to_string()), ("gl", self.opts.country.to_string()), ("q", query.as_ref().to_owned())]
-        ).map_err(|_| Error::Other("could not build url".into()))?;
+    #[tracing::instrument(skip(self), level = "error")]
+    pub async fn search_suggestion<S: AsRef<str> + Debug>(
+        &self,
+        query: S,
+    ) -> Result<Vec<String>, Error> {
+        let url = url::Url::parse_with_params(
+            "https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&xhr=t",
+            &[
+                ("hl", self.opts.lang.to_string()),
+                ("gl", self.opts.country.to_string()),
+                ("q", query.as_ref().to_owned()),
+            ],
+        )
+        .map_err(|_| Error::Other("could not build url".into()))?;
 
         let response = self
             .client
-            .http_request_txt(self.client.inner.http.get(url).build()?)
+            .http_request_txt(&self.client.inner.http.get(url).build()?)
             .await?;
 
-        let trimmed = response
-            .get(5..)
-            .ok_or(Error::Extraction(ExtractionError::InvalidData(
-                Cow::Borrowed("could not get string slice"),
-            )))?;
-
-        let parsed = serde_json::from_str::<(
-            IgnoredAny,
-            Vec<(String, IgnoredAny, IgnoredAny)>,
-            IgnoredAny,
-        )>(trimmed)
-        .map_err(|e| Error::Extraction(ExtractionError::InvalidData(e.to_string().into())))?;
+        let parsed = serde_json::from_str::<response::SearchSuggestion>(&response)
+            .map_err(|e| Error::Extraction(ExtractionError::InvalidData(e.to_string().into())))?;
 
         Ok(parsed.1.into_iter().map(|item| item.0).collect())
     }
 }
 
-impl MapResponse<SearchResult> for response::Search {
+impl<T: FromYtItem> MapResponse<SearchResult<T>> for response::Search {
     fn map_response(
         self,
-        _id: &str,
-        lang: crate::param::Language,
-        _deobf: Option<&crate::deobfuscate::DeobfData>,
-    ) -> Result<MapResult<SearchResult>, ExtractionError> {
+        ctx: &MapRespCtx<'_>,
+    ) -> Result<MapResult<SearchResult<T>>, ExtractionError> {
         let items = self
             .contents
             .two_column_search_results_renderer
@@ -106,20 +107,28 @@ impl MapResponse<SearchResult> for response::Search {
             .section_list_renderer
             .contents;
 
-        let mut mapper = response::YouTubeListMapper::<YouTubeItem>::new(lang);
+        let mut mapper = response::YouTubeListMapper::<YouTubeItem>::new(ctx.lang);
         mapper.map_response(items);
 
         Ok(MapResult {
             c: SearchResult {
                 items: Paginator::new_ext(
                     self.estimated_results,
-                    mapper.items,
+                    mapper
+                        .items
+                        .into_iter()
+                        .filter_map(T::from_yt_item)
+                        .collect(),
                     mapper.ctoken,
-                    None,
-                    crate::model::paginator::ContinuationEndpoint::Search,
+                    ctx.visitor_data.map(str::to_owned),
+                    ContinuationEndpoint::Search,
+                    false,
                 ),
                 corrected_query: mapper.corrected_query,
-                visitor_data: self.response_context.visitor_data,
+                visitor_data: self
+                    .response_context
+                    .visitor_data
+                    .or_else(|| ctx.visitor_data.map(str::to_owned)),
             },
             warnings: mapper.warnings,
         })
@@ -134,9 +143,8 @@ mod tests {
     use rstest::rstest;
 
     use crate::{
-        client::{response, MapResponse},
-        model::SearchResult,
-        param::Language,
+        client::{response, MapRespCtx, MapResponse},
+        model::{SearchResult, YouTubeItem},
         serializer::MapResult,
         util::tests::TESTFILES,
     };
@@ -151,7 +159,8 @@ mod tests {
         let json_file = File::open(json_path).unwrap();
 
         let search: response::Search = serde_json::from_reader(BufReader::new(json_file)).unwrap();
-        let map_res: MapResult<SearchResult> = search.map_response("", Language::En, None).unwrap();
+        let map_res: MapResult<SearchResult<YouTubeItem>> =
+            search.map_response(&MapRespCtx::test("")).unwrap();
 
         assert!(
             map_res.warnings.is_empty(),

src/client/snapshots/rustypipe__client__channel__tests__map_channel_info.snap

@@ -2,166 +2,28 @@
 source: src/client/channel.rs
 expression: map_res.c
 ---
-Channel(
+ChannelInfo(
   id: "UC2DjFE7Xf11URZqWBigcVOQ",
-  name: "EEVblog",
-  subscriber_count: Some(881000),
-  avatar: [
-    Thumbnail(
-      url: "https://yt3.ggpht.com/ytc/AMLnZu9eKk4Nd16fX4Rn1TF1G7ReluwOl6M5558FTYAM=s48-c-k-c0x00ffffff-no-rj",
-      width: 48,
-      height: 48,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/ytc/AMLnZu9eKk4Nd16fX4Rn1TF1G7ReluwOl6M5558FTYAM=s88-c-k-c0x00ffffff-no-rj",
-      width: 88,
-      height: 88,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/ytc/AMLnZu9eKk4Nd16fX4Rn1TF1G7ReluwOl6M5558FTYAM=s176-c-k-c0x00ffffff-no-rj",
-      width: 176,
-      height: 176,
-    ),
-  ],
-  verification: Verified,
+  url: "http://www.youtube.com/@EEVblog",
   description: "NO SCRIPT, NO FEAR, ALL OPINION\nAn off-the-cuff Video Blog about Electronics Engineering, for engineers, hobbyists, enthusiasts, hackers and Makers\nHosted by Dave Jones from Sydney Australia\n\nDONATIONS:\nBitcoin: 3KqyH1U3qrMPnkLufM2oHDU7YB4zVZeFyZ\nEthereum: 0x99ccc4d2654ba40744a1f678d9868ecb15e91206\nPayPal: david@alternatezone.com\n\nPatreon: https://www.patreon.com/eevblog\n\nEEVblog2: http://www.youtube.com/EEVblog2\nEEVdiscover: https://www.youtube.com/channel/UCkGvUEt8iQLmq3aJIMjT2qQ\n\nEMAIL:\nAdvertising/Commercial: eevblog+business@gmail.com\nFan mail: eevblog+fan@gmail.com\nHate Mail: eevblog+hate@gmail.com\n\nI DON\'T DO PAID VIDEO SPONSORSHIPS, DON\'T ASK!\n\nPLEASE:\nDo NOT ask for personal advice on something, post it in the EEVblog forum.\nI read ALL email, but please don\'t be offended if I don\'t have time to reply, I get a LOT of email.\n\nMailbag\nPO Box 7949\nBaulkham Hills NSW 2153\nAUSTRALIA",
-  tags: [
-    "electronics",
-    "engineering",
-    "maker",
-    "hacker",
-    "design",
-    "circuit",
-    "hardware",
-    "pic",
-    "atmel",
-    "oscilloscope",
-    "multimeter",
-    "diy",
-    "hobby",
-    "review",
-    "teardown",
-    "microcontroller",
-    "arduino",
-    "video",
-    "blog",
-    "tutorial",
-    "how-to",
-    "interview",
-    "rant",
-    "industry",
-    "news",
-    "mailbag",
-    "dumpster diving",
-    "debunking",
+  subscriber_count: Some(920000),
+  video_count: Some(1920),
+  create_date: Some("2009-04-04"),
+  view_count: Some(199087682),
+  country: Some(AU),
+  links: [
+    ("EEVblog Web Site", "http://www.eevblog.com/"),
+    ("Twitter", "http://www.twitter.com/eevblog"),
+    ("Facebook", "http://www.facebook.com/EEVblog"),
+    ("EEVdiscover", "https://www.youtube.com/channel/UCkGvUEt8iQLmq3aJIMjT2qQ"),
+    ("The EEVblog Forum", "http://www.eevblog.com/forum"),
+    ("EEVblog Merchandise (T-Shirts)", "http://www.eevblog.com/merch"),
+    ("EEVblog Donations", "http://www.eevblog.com/donations/"),
+    ("Patreon", "https://www.patreon.com/eevblog"),
+    ("SubscribeStar", "https://www.subscribestar.com/eevblog"),
+    ("The AmpHour Radio Show", "http://www.theamphour.com/"),
+    ("Flickr", "http://www.flickr.com/photos/eevblog"),
+    ("EEVblog AMAZON Store", "http://www.amazon.com/gp/redirect.html?ie=UTF8&location=http%3A%2F%2Fwww.amazon.com%2F&tag=ee04-20&linkCode=ur2&camp=1789&creative=390957"),
+    ("2nd EEVblog Channel", "http://www.youtube.com/EEVblog2"),
   ],
-  vanity_url: Some("https://www.youtube.com/c/EevblogDave"),
-  banner: [
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1060-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
-      width: 1060,
-      height: 175,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1138-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
-      width: 1138,
-      height: 188,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1707-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
-      width: 1707,
-      height: 283,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w2120-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
-      width: 2120,
-      height: 351,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w2276-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
-      width: 2276,
-      height: 377,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w2560-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
-      width: 2560,
-      height: 424,
-    ),
-  ],
-  mobile_banner: [
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w320-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
-      width: 320,
-      height: 88,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w640-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
-      width: 640,
-      height: 175,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w960-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
-      width: 960,
-      height: 263,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1280-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
-      width: 1280,
-      height: 351,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1440-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
-      width: 1440,
-      height: 395,
-    ),
-  ],
-  tv_banner: [
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w320-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
-      width: 320,
-      height: 180,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w854-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
-      width: 854,
-      height: 480,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1280-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
-      width: 1280,
-      height: 720,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1920-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
-      width: 1920,
-      height: 1080,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w2120-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
-      width: 2120,
-      height: 1192,
-    ),
-  ],
-  has_shorts: false,
-  has_live: false,
-  visitor_data: Some("CgszMUUzZDlGLWxiRSipqr2ZBg%3D%3D"),
-  content: ChannelInfo(
-    create_date: Some("2009-04-04"),
-    view_count: Some(186854342),
-    links: [
-      ("EEVblog Web Site", "http://www.eevblog.com/"),
-      ("Twitter", "http://www.twitter.com/eevblog"),
-      ("Facebook", "http://www.facebook.com/EEVblog"),
-      ("EEVdiscover", "https://www.youtube.com/channel/UCkGvUEt8iQLmq3aJIMjT2qQ"),
-      ("The EEVblog Forum", "http://www.eevblog.com/forum"),
-      ("EEVblog Merchandise (T-Shirts)", "http://www.eevblog.com/merch"),
-      ("EEVblog Donations", "http://www.eevblog.com/donations/"),
-      ("Patreon", "https://www.patreon.com/eevblog"),
-      ("SubscribeStar", "https://www.subscribestar.com/eevblog"),
-      ("The AmpHour Radio Show", "http://www.theamphour.com/"),
-      ("Flickr", "http://www.flickr.com/photos/eevblog"),
-      ("EEVblog AMAZON Store", "http://www.amazon.com/gp/redirect.html?ie=UTF8&location=http%3A%2F%2Fwww.amazon.com%2F&tag=ee04-20&linkCode=ur2&camp=1789&creative=390957"),
-      ("2nd EEVblog Channel", "http://www.youtube.com/EEVblog2"),
-    ],
-  ),
 )

src/client/snapshots/rustypipe__client__channel__tests__map_channel_livestreams.snap

@@ -5,7 +5,9 @@ expression: map_res.c
 Channel(
   id: "UC2DjFE7Xf11URZqWBigcVOQ",
   name: "EEVblog",
+  handle: None,
   subscriber_count: Some(884000),
+  video_count: None,
   avatar: [
     Thumbnail(
       url: "https://yt3.ggpht.com/ytc/AMLnZu9eKk4Nd16fX4Rn1TF1G7ReluwOl6M5558FTYAM=s48-c-k-c0x00ffffff-no-rj",
@@ -23,7 +25,7 @@ Channel(
       height: 176,
     ),
   ],
-  verification: Verified,
+  verification: verified,
   description: "NO SCRIPT, NO FEAR, ALL OPINION\nAn off-the-cuff Video Blog about Electronics Engineering, for engineers, hobbyists, enthusiasts, hackers and Makers\nHosted by Dave Jones from Sydney Australia\n\nDONATIONS:\nBitcoin: 3KqyH1U3qrMPnkLufM2oHDU7YB4zVZeFyZ\nEthereum: 0x99ccc4d2654ba40744a1f678d9868ecb15e91206\nPayPal: david@alternatezone.com\n\nPatreon: https://www.patreon.com/eevblog\n\nEEVblog2: http://www.youtube.com/EEVblog2\nEEVdiscover: https://www.youtube.com/channel/UCkGvUEt8iQLmq3aJIMjT2qQ\n\nEMAIL:\nAdvertising/Commercial: eevblog+business@gmail.com\nFan mail: eevblog+fan@gmail.com\nHate Mail: eevblog+hate@gmail.com\n\nI DON\'T DO PAID VIDEO SPONSORSHIPS, DON\'T ASK!\n\nPLEASE:\nDo NOT ask for personal advice on something, post it in the EEVblog forum.\nI read ALL email, but please don\'t be offended if I don\'t have time to reply, I get a LOT of email.\n\nMailbag\nPO Box 7949\nBaulkham Hills NSW 2153\nAUSTRALIA",
   tags: [
     "electronics",
@@ -55,7 +57,6 @@ Channel(
     "dumpster diving",
     "debunking",
   ],
-  vanity_url: Some("https://www.youtube.com/c/EevblogDave"),
   banner: [
     Thumbnail(
       url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1060-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
@@ -88,60 +89,6 @@ Channel(
       height: 424,
     ),
   ],
-  mobile_banner: [
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w320-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
-      width: 320,
-      height: 88,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w640-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
-      width: 640,
-      height: 175,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w960-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
-      width: 960,
-      height: 263,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1280-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
-      width: 1280,
-      height: 351,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1440-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
-      width: 1440,
-      height: 395,
-    ),
-  ],
-  tv_banner: [
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w320-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
-      width: 320,
-      height: 180,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w854-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
-      width: 854,
-      height: 480,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1280-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
-      width: 1280,
-      height: 720,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1920-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
-      width: 1920,
-      height: 1080,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w2120-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
-      width: 2120,
-      height: 1192,
-    ),
-  ],
   has_shorts: false,
   has_live: true,
   visitor_data: None,
@@ -151,7 +98,7 @@ Channel(
       VideoItem(
         id: "hhs95CI6Dsg",
         name: "MARS 2020 Landing LIVE",
-        length: Some(6321),
+        duration: Some(6321),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/hhs95CI6Dsg/hqdefault.jpg?sqp=-oaymwE1CKgBEF5IVfKriqkDKAgBFQAAiEIYAXABwAEG8AEB-AHUBoAC4AOKAgwIABABGGUgZShlMA8=&rs=AOn4CLAlPp2e1tF8gyf1cJisZGTMleissg",
@@ -178,7 +125,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -192,7 +139,7 @@ Channel(
       VideoItem(
         id: "cpQk2n-wmQ4",
         name: "LIVE Soldering",
-        length: Some(7046),
+        duration: Some(7046),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/cpQk2n-wmQ4/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLCoS3qwdY2rDbhkWJOWHisORlMKnA",
@@ -219,7 +166,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -233,7 +180,7 @@ Channel(
       VideoItem(
         id: "kIDV_XN9oA8",
         name: "LIVE Soldering",
-        length: Some(4353),
+        duration: Some(4353),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/kIDV_XN9oA8/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLBG3KVoFpBFIYCG2mrox_kEq6Arug",
@@ -260,7 +207,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -274,7 +221,7 @@ Channel(
       VideoItem(
         id: "DWS4Qp3Yn0A",
         name: "Apollo 11 Launch LIVE - 50 Years Later",
-        length: Some(4560),
+        duration: Some(4560),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/DWS4Qp3Yn0A/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLAFkIQ4er8qDNMlD9H8lPzfSnE99g",
@@ -301,7 +248,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -315,7 +262,7 @@ Channel(
       VideoItem(
         id: "LwjTe3SiVXg",
         name: "EEVblog LIVE Q&A",
-        length: Some(3943),
+        duration: Some(3943),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/LwjTe3SiVXg/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLAzTlnjBJLT3KJVN4teMlX_svuaNA",
@@ -342,7 +289,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -356,7 +303,7 @@ Channel(
       VideoItem(
         id: "skPiz3GrVNs",
         name: "LIVE Keysight Scope Draw #2",
-        length: Some(2445),
+        duration: Some(2445),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/skPiz3GrVNs/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLBFiIfUBfoL0Q9CLR9Pc8bXy-zclg",
@@ -383,7 +330,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -397,7 +344,7 @@ Channel(
       VideoItem(
         id: "HZc-Ctvgv5Y",
         name: "LIVE Keysight Scope Draw",
-        length: Some(6455),
+        duration: Some(6455),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/HZc-Ctvgv5Y/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLDQM1_QPh6u5_BFonLCdFPz-AcpkQ",
@@ -424,7 +371,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -438,7 +385,7 @@ Channel(
       VideoItem(
         id: "5ilODYy2zGE",
         name: "Ask Dave LIVE - March 8th 2019",
-        length: Some(10645),
+        duration: Some(10645),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/5ilODYy2zGE/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLCft4f7Lw3l3_u55bzUibWXr-UHTQ",
@@ -465,7 +412,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -479,7 +426,7 @@ Channel(
       VideoItem(
         id: "gQ7TTuiDH1M",
         name: "Ask Dave LIVE - Jan 28th 2019",
-        length: Some(17228),
+        duration: Some(17228),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/gQ7TTuiDH1M/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLAUPZz1xzckl5xzdBRonA_1WNWIyg",
@@ -506,7 +453,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -520,7 +467,7 @@ Channel(
       VideoItem(
         id: "qpw9dKxL2Ho",
         name: "LIVE KiCAD 5 PCB Design",
-        length: Some(8003),
+        duration: Some(8003),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/qpw9dKxL2Ho/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLAC-kI2770I7JgVCTYExG0vXoYoxA",
@@ -547,7 +494,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -561,7 +508,7 @@ Channel(
       VideoItem(
         id: "wECZoUNd2GY",
         name: "EEVblog LIVE DIY TTL Computer Build",
-        length: Some(14599),
+        duration: Some(14599),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/wECZoUNd2GY/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLDzZwAD6bQQEaYuZEzmQ0sgQKc1yA",
@@ -588,7 +535,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -602,7 +549,7 @@ Channel(
       VideoItem(
         id: "bV99dn-tWDk",
         name: "EEVblog LIVE Scope Draw",
-        length: Some(2694),
+        duration: Some(2694),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/bV99dn-tWDk/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLAR4ckJxAituVMFCyWpYhHXozqQRA",
@@ -629,7 +576,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -643,7 +590,7 @@ Channel(
       VideoItem(
         id: "-NGRIFiu_p0",
         name: "EEVblog LIVE SHOW - End of 2017",
-        length: Some(12238),
+        duration: Some(12238),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/-NGRIFiu_p0/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLBjMmIdgjiSMBQ2X73h6-NtVUIqSg",
@@ -670,7 +617,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -684,7 +631,7 @@ Channel(
       VideoItem(
         id: "zgE6_x4rM5k",
         name: "LIVE Show Giveaway",
-        length: Some(5533),
+        duration: Some(5533),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/zgE6_x4rM5k/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLBjb92wUNqOvTKs9TCLCThvdkdz3A",
@@ -711,7 +658,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -725,7 +672,7 @@ Channel(
       VideoItem(
         id: "9DjABCJN2M8",
         name: "LIVE Testing of the Batteriser",
-        length: Some(10747),
+        duration: Some(10747),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/9DjABCJN2M8/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLBXhnnHCuNfSzHZC64KFsfHPPJDNg",
@@ -752,7 +699,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -766,7 +713,7 @@ Channel(
       VideoItem(
         id: "cAsUI2YhqN4",
         name: "LIVE Unboxing of the Batteriser! (Batteroo)",
-        length: Some(3102),
+        duration: Some(3102),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/cAsUI2YhqN4/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLCOE1MyG1nFXs9D2qdK78bpN1mc_g",
@@ -793,7 +740,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -807,7 +754,7 @@ Channel(
       VideoItem(
         id: "CLYKwFMW9J0",
         name: "Juno Live Again",
-        length: Some(811),
+        duration: Some(811),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/CLYKwFMW9J0/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLC7WO4HX0e7M58ddoJD5dkVjdKHYQ",
@@ -834,7 +781,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -848,7 +795,7 @@ Channel(
       VideoItem(
         id: "nV43vM9VcUA",
         name: "Juno Live",
-        length: Some(190),
+        duration: Some(190),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/nV43vM9VcUA/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLCy-zEVPDvomCCi8YoP8Ig_Hrhzfw",
@@ -875,7 +822,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -889,7 +836,7 @@ Channel(
       VideoItem(
         id: "38uFiWzcDnc",
         name: "Juno Orbital Insertion Live",
-        length: Some(1731),
+        duration: Some(1731),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/38uFiWzcDnc/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLALhrDygxFH4T2c-4efZqVaJnYY7g",
@@ -916,7 +863,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -930,7 +877,7 @@ Channel(
       VideoItem(
         id: "ib80yjc9VlM",
         name: "Juno Jupiter Live",
-        length: Some(581),
+        duration: Some(581),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/ib80yjc9VlM/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLDbJJvzoEmwUc7nAm6GLJpoZJKmgQ",
@@ -957,7 +904,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -971,7 +918,7 @@ Channel(
       VideoItem(
         id: "rQRakYpb8-g",
         name: "eevSTREAM: Lab Rearrangement Part 2",
-        length: Some(8616),
+        duration: Some(8616),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/rQRakYpb8-g/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLAdGJH0yhCQ7kmI3d3JXVv_7xzJAQ",
@@ -998,7 +945,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -1012,7 +959,7 @@ Channel(
       VideoItem(
         id: "DwLEFKu2XWg",
         name: "eevSTREAM: Lab Rearrangement Part 1",
-        length: Some(768),
+        duration: Some(768),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/DwLEFKu2XWg/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLCXvSePgZ8NIKQTviqWvROVZFRPpA",
@@ -1039,7 +986,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -1053,7 +1000,7 @@ Channel(
       VideoItem(
         id: "VeUDXQR3F2o",
         name: "Live Show",
-        length: Some(10360),
+        duration: Some(10360),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/VeUDXQR3F2o/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLDmgrfQXMTaGMahuP8F_UHJAomFbg",
@@ -1080,7 +1027,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -1094,7 +1041,7 @@ Channel(
       VideoItem(
         id: "PgZx25vVwoI",
         name: "Live Giveaway",
-        length: Some(1808),
+        duration: Some(1808),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/PgZx25vVwoI/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLDTrMmoCfISxG0YSqC4oEyKGHdK_A",
@@ -1121,7 +1068,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -1135,7 +1082,7 @@ Channel(
       VideoItem(
         id: "jUtzoO-ur34",
         name: "Inventables X-Carve LIVE Build Part 4",
-        length: Some(10665),
+        duration: Some(10665),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/jUtzoO-ur34/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLCO35sFP8D_Q08HxMZkNHFO8MmpDg",
@@ -1162,7 +1109,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -1176,7 +1123,7 @@ Channel(
       VideoItem(
         id: "199gtbX1y4M",
         name: "Inventables X-Carve LIVE Build Part 3 + Batteriser Rant",
-        length: Some(6267),
+        duration: Some(6267),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/199gtbX1y4M/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLAg3bMS00xpSXmNn1f5hXu_jWWC1w",
@@ -1203,7 +1150,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -1217,7 +1164,7 @@ Channel(
       VideoItem(
         id: "nQH4I_p7-MI",
         name: "Inventables X-Carve LIVE Build Part 2",
-        length: Some(17643),
+        duration: Some(17643),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/nQH4I_p7-MI/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLBMIA1YzQefFwGj5UFikXuYS2Nkng",
@@ -1244,7 +1191,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -1258,7 +1205,7 @@ Channel(
       VideoItem(
         id: "XBMNFXGKpaw",
         name: "Inventables X-Carve LIVE Build",
-        length: Some(5479),
+        duration: Some(5479),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/XBMNFXGKpaw/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLCV980wWO8tdx0aFDXwPn9aBQ2xlA",
@@ -1285,7 +1232,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -1299,7 +1246,7 @@ Channel(
       VideoItem(
         id: "yl6DGgiE3J8",
         name: "Apollo Saturn LVDC Live testing",
-        length: Some(1076),
+        duration: Some(1076),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/yl6DGgiE3J8/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLCugABHuqqPZQjV9cEm0JFh7R5aiA",
@@ -1326,7 +1273,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",
@@ -1340,7 +1287,7 @@ Channel(
       VideoItem(
         id: "EEMcIZAcKjc",
         name: "LIVE EEVblog Mailbag",
-        length: Some(7344),
+        duration: Some(7344),
         thumbnail: [
           Thumbnail(
             url: "https://i.ytimg.com/vi/EEMcIZAcKjc/hqdefault.jpg?sqp=-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLCg16HpJqC9mNwkYOf8b0cfAuNLOA",
@@ -1367,7 +1314,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(884000),
         )),
         publish_date: "[date]",

src/client/snapshots/rustypipe__client__channel__tests__map_channel_playlists_20241109_lockup.snap

@@ -0,0 +1,672 @@
+---
+source: src/client/channel.rs
+expression: map_res.c
+---
+Channel(
+  id: "UC2DjFE7Xf11URZqWBigcVOQ",
+  name: "EEVblog",
+  handle: Some("@EEVblog"),
+  subscriber_count: Some(952000),
+  video_count: Some(2000),
+  avatar: [
+    Thumbnail(
+      url: "https://yt3.googleusercontent.com/ytc/AIdro_l17lYcTcRSydZeQK-RuiSfEeH5eX9m4irSNQj6109v5MQ=s72-c-k-c0x00ffffff-no-rj",
+      width: 72,
+      height: 72,
+    ),
+    Thumbnail(
+      url: "https://yt3.googleusercontent.com/ytc/AIdro_l17lYcTcRSydZeQK-RuiSfEeH5eX9m4irSNQj6109v5MQ=s120-c-k-c0x00ffffff-no-rj",
+      width: 120,
+      height: 120,
+    ),
+    Thumbnail(
+      url: "https://yt3.googleusercontent.com/ytc/AIdro_l17lYcTcRSydZeQK-RuiSfEeH5eX9m4irSNQj6109v5MQ=s160-c-k-c0x00ffffff-no-rj",
+      width: 160,
+      height: 160,
+    ),
+  ],
+  verification: verified,
+  description: "NO SCRIPT, NO FEAR, ALL OPINION\nAn off-the-cuff Video Blog about Electronics Engineering, for engineers, hobbyists, enthusiasts, hackers and Makers\nHosted by Dave Jones from Sydney Australia\n\nDONATIONS:\nBitcoin: 3KqyH1U3qrMPnkLufM2oHDU7YB4zVZeFyZ\nEthereum: 0x99ccc4d2654ba40744a1f678d9868ecb15e91206\nPayPal: david@alternatezone.com\n\nPatreon: https://www.patreon.com/eevblog\n\nEEVblog2: http://www.youtube.com/EEVblog2\nEEVdiscover: https://www.youtube.com/channel/UCkGvUEt8iQLmq3aJIMjT2qQ\n\nEMAIL:\nAdvertising/Commercial: eevblog+business@gmail.com\nFan mail: eevblog+fan@gmail.com\nHate Mail: eevblog+hate@gmail.com\n\nI DON\'T DO PAID VIDEO SPONSORSHIPS, DON\'T ASK!\n\nPLEASE:\nDo NOT ask for personal advice on something, post it in the EEVblog forum.\nI read ALL email, but please don\'t be offended if I don\'t have time to reply, I get a LOT of email.\n\nMailbag\nPO Box 7949\nBaulkham Hills NSW 2153\nAUSTRALIA",
+  tags: [
+    "electronics",
+    "engineering",
+    "maker",
+    "hacker",
+    "design",
+    "circuit",
+    "hardware",
+    "pic",
+    "atmel",
+    "oscilloscope",
+    "multimeter",
+    "diy",
+    "hobby",
+    "review",
+    "teardown",
+    "microcontroller",
+    "arduino",
+    "video",
+    "blog",
+    "tutorial",
+    "how-to",
+    "interview",
+    "rant",
+    "industry",
+    "news",
+    "mailbag",
+    "dumpster diving",
+    "debunking",
+  ],
+  banner: [
+    Thumbnail(
+      url: "https://yt3.googleusercontent.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1060-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
+      width: 1060,
+      height: 175,
+    ),
+    Thumbnail(
+      url: "https://yt3.googleusercontent.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1138-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
+      width: 1138,
+      height: 188,
+    ),
+    Thumbnail(
+      url: "https://yt3.googleusercontent.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1707-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
+      width: 1707,
+      height: 283,
+    ),
+    Thumbnail(
+      url: "https://yt3.googleusercontent.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w2120-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
+      width: 2120,
+      height: 351,
+    ),
+    Thumbnail(
+      url: "https://yt3.googleusercontent.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w2276-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
+      width: 2276,
+      height: 377,
+    ),
+    Thumbnail(
+      url: "https://yt3.googleusercontent.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w2560-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
+      width: 2560,
+      height: 424,
+    ),
+  ],
+  has_shorts: true,
+  has_live: true,
+  visitor_data: None,
+  content: Paginator(
+    count: None,
+    items: [
+      PlaylistItem(
+        id: "PLvOlSehNtuHv268f0mW5m1t_hq_RVGRSA",
+        name: "Jellybean Components Series",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/XYdmX8w8xwI/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLCqmf6TGfDinNXhgU29ZxOkv2u9sQ",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(5),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHu46I7nFuUg3LC3PpiWTR4f",
+        name: "Tandy Electronics / Radio Shack & Computers",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/uUXxY6gA-7g/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLAlIVvQ4Axx40Xa_i8F56qmppXEXg",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(11),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHuS01_RNCnvpzyk7bycYCmM",
+        name: "Open Source Hardware",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/m_8jh_MpWBE/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLBx6U5iikp5rSO78dIWdy1RQ_BLNQ",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(4),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHuwwQ1fpquOJuA5MSfD4iD6",
+        name: "Fluke Multimeters",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/ymJc5oxthlw/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLDAOiw39aJajjAdroLnuj_fh60Ryw",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(22),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHs2LwEdDwTp3n7mxb-MyBbo",
+        name: "EEVacademy Digital Design Tutorial Series",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/lJ3q9RHIatU/hqdefault.jpg?sqp=-oaymwExCOADEI4CSFryq4qpAyMIARUAAIhCGAHwAQH4Af4JgALQBYoCDAgAEAEYQyBXKGUwDw==&rs=AOn4CLBaaQaTJzi7H-zjwSsTlNJdBsyqvQ",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(5),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHu2v8THrRMt8E9ziHtRXPm7",
+        name: "AI / ChatGPT",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/g5_Ts9SWbYs/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLBmZPW6EiAvTCsI86BFg4BxXLj66A",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(3),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHvXuXRmoBUys09Dwi1heNii",
+        name: "Shorts",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/ndvJtQ8nxV4/hqdefault.jpg?sqp=-oaymwExCOADEI4CSFryq4qpAyMIARUAAIhCGAHwAQH4AbYIgAKAD4oCDAgAEAEYNyBTKH8wDw==&rs=AOn4CLDD0qOLs38KPJtqdG6zCeVLQMf62Q",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(1),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHv3gxNg5BGoZJJu9htoAGB6",
+        name: "Microcontrollers",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/L9Wrv7nW-S8/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLDiAT5izyig1ntMSUhvSOVuYSsG1Q",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(1),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHvllTQ-vwvY26E3Bvrov93Y",
+        name: "Bypass Capacitors",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/1xicZF9glH0/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLAFb2FcbpdtAG1xLjmdkdIm1hFvgA",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(4),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHtOV3AEwhuea4TnviddKfAj",
+        name: "MacGyver Project",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/4yosozyeIP4/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLAkwsCiJjFkWhYxtcg5NgfnQbkZrA",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(3),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHuvHE5GQrQJxWXHdmW2l5IF",
+        name: "Calculators",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/S3R4r2xvVYQ/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLB7HH5drG-33c1SyRe9kyZBrXvm3A",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(1),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHs6wRwVSaErU0BEnLiHfnKJ",
+        name: "BM235",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/WPyEFB4cHkA/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLAzBuQFV8T9hM8adlPvv58C9TeDug",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(9),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHu4k0ZkKFLsysSB5iava6Qu",
+        name: "Vibration Measurement",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/uus_cpZiqsU/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLCqdsjWVFaLOkEcXgbZD2Eca8MnuQ",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(3),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHtdQF-m5UFZ5GEjABadI3kI",
+        name: "Component Selection",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/uq1DMWtjL2U/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLAbgb1Jdb5P69JGdZQ-a8asLLyYdA",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(6),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHtlndPUSOPgsujUdq1c5Mr9",
+        name: "Solar Roadways",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/oIImmlfCyzo/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLBxApgyGu3dNXRGoqLctVUnESpEIA",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(23),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHvD6M_7WeN071OVsZFE0_q-",
+        name: "Electronics Tutorials - AC Circuit Theory Series",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/rrPtvYYJ2-g/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLBEVc71xxSjJ-xlA_dDQaYIjdHyUw",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(3),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHtVLq2MDPIz82BWMIZcuwhK",
+        name: "Electronics Tutorial - DC Fundamentals",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/xSRe_4TQbuo/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLDP4V24_MG6vzvUZsHep9WFSCCY6Q",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(8),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHvIDfW3x2p4BY6l4RYgfBJE",
+        name: "Oscilloscope Probing",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/OiAmER1OJh4/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLAXeGAvEc8y3pEsPUxWdsNIP9UmPw",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(14),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHu6Jjb8U82eKQfvKhJVl0Bu",
+        name: "Thermal Design",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/8ruFVmxf0zs/hqdefault.jpg?sqp=-oaymwExCOADEI4CSFryq4qpAyMIARUAAIhCGAHwAQH4Af4JgALQBYoCDAgAEAEYfyA1KDUwDw==&rs=AOn4CLD6PMawyYXKe8KT1-Y6vWjQc2xIDw",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(9),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHs-X2Awg33PCBNrP2BGFVhC",
+        name: "Electric Cars",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/CPcZm1Tu5VI/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLCsm8De0QaHPaeCZqxMp_F464fWzg",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(9),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHuLODLTeq3PM-OJRP2nzNUa",
+        name: "Designing a better uCurrent",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/0AEVilxXAAo/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLCjotFuRjPPBHd2LWzt3lviPj9HaA",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(3),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHtvTKP4RTNW1-08Kmzy1pvA",
+        name: "EMC Compliance & Measurement",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/lYmfVMWbIHQ/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLBtygEqMXx7Lwe5SuBWt2q0CSahYA",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(8),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHuUTpCrTVX7BdU68l2aVqMv",
+        name: "Power Counter Display Project",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/nTpE1Nw3Yy4/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLAbPl28_i7isizY6A1t2_c6gV8BAQ",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(2),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHvm120Tq40nKrM5SUBlolN3",
+        name: "Live - Ask Dave",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/gQ7TTuiDH1M/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLBMnucUil90WeDSIeFz8mZCOtEv9g",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(3),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHsiF93KOLoF1KAHArmIW9lC",
+        name: "Padauk Microcontroller",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/r45r4rV5JOI/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLCn4kGWcjBOhk3vN8QPMDa9L3mkKA",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(10),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHvxTzBLwUFw4My4rtrNFzED",
+        name: "Other Debunking Videos",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/WopuF9vD7KE/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLBv5buh3qMs4feQaPj6Fy6bxl_vuA",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(1),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHt2pJ7X5tumuM4Wa3r1OC7Q",
+        name: "Audio & Speakers",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/qHbkw0Gm7pk/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLCJBYXTDttGHTm51j3bfwqxOqVFig",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(9),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHtX7OearWdmqGzqiu4DHKWi",
+        name: "Cameras",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/g9umAQ1-an4/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLCB5jNm9U-rypnpthK_N321LpYWew",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(16),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHu-TaNRp27_PiXjBG5wY9Gv",
+        name: "Cryptocurrency",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/ibPgfzd9zd8/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLDe3IXT88HR3XxnxfqrpAxh6pfYMg",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(7),
+      ),
+      PlaylistItem(
+        id: "PLvOlSehNtuHvmK-VGcZ33ZuATmcNB8tvH",
+        name: "LCD Tutorial",
+        thumbnail: [
+          Thumbnail(
+            url: "https://i.ytimg.com/vi/ZYvxgl-9tNM/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLDv2WT4Chl1_H2G43AjfSFpPcKVoA",
+            width: 480,
+            height: 270,
+          ),
+        ],
+        channel: Some(ChannelTag(
+          id: "UC2DjFE7Xf11URZqWBigcVOQ",
+          name: "EEVblog",
+          avatar: [],
+          verification: verified,
+          subscriber_count: Some(952000),
+        )),
+        video_count: Some(6),
+      ),
+    ],
+    ctoken: Some("4qmFsgLCARIYVUMyRGpGRTdYZjExVVJacVdCaWdjVk9RGnRFZ2x3YkdGNWJHbHpkSE1ZQXlBQk1BRTRBZW9EUEVOblRrUlJhbEZUU2tKSmFWVkZlREpVTW5oVVdsZG9UMlJJVmtsa2JURk1URlphU0ZreGIzcE5NWEF4VVZaU2RGa3dOVU5QU0ZJeVUwTm5PQSUzRCUzRJoCL2Jyb3dzZS1mZWVkVUMyRGpGRTdYZjExVVJacVdCaWdjVk9RcGxheWxpc3RzMTA0"),
+    endpoint: browse,
+  ),
+)

src/client/snapshots/rustypipe__client__channel__tests__map_channel_playlists_base.snap

@@ -5,7 +5,9 @@ expression: map_res.c
 Channel(
   id: "UC2DjFE7Xf11URZqWBigcVOQ",
   name: "EEVblog",
+  handle: None,
   subscriber_count: Some(881000),
+  video_count: None,
   avatar: [
     Thumbnail(
       url: "https://yt3.ggpht.com/ytc/AMLnZu9eKk4Nd16fX4Rn1TF1G7ReluwOl6M5558FTYAM=s48-c-k-c0x00ffffff-no-rj",
@@ -23,7 +25,7 @@ Channel(
       height: 176,
     ),
   ],
-  verification: Verified,
+  verification: verified,
   description: "NO SCRIPT, NO FEAR, ALL OPINION\nAn off-the-cuff Video Blog about Electronics Engineering, for engineers, hobbyists, enthusiasts, hackers and Makers\nHosted by Dave Jones from Sydney Australia\n\nDONATIONS:\nBitcoin: 3KqyH1U3qrMPnkLufM2oHDU7YB4zVZeFyZ\nEthereum: 0x99ccc4d2654ba40744a1f678d9868ecb15e91206\nPayPal: david@alternatezone.com\n\nPatreon: https://www.patreon.com/eevblog\n\nEEVblog2: http://www.youtube.com/EEVblog2\nEEVdiscover: https://www.youtube.com/channel/UCkGvUEt8iQLmq3aJIMjT2qQ\n\nEMAIL:\nAdvertising/Commercial: eevblog+business@gmail.com\nFan mail: eevblog+fan@gmail.com\nHate Mail: eevblog+hate@gmail.com\n\nI DON\'T DO PAID VIDEO SPONSORSHIPS, DON\'T ASK!\n\nPLEASE:\nDo NOT ask for personal advice on something, post it in the EEVblog forum.\nI read ALL email, but please don\'t be offended if I don\'t have time to reply, I get a LOT of email.\n\nMailbag\nPO Box 7949\nBaulkham Hills NSW 2153\nAUSTRALIA",
   tags: [
     "electronics",
@@ -55,7 +57,6 @@ Channel(
     "dumpster diving",
     "debunking",
   ],
-  vanity_url: Some("https://www.youtube.com/c/EevblogDave"),
   banner: [
     Thumbnail(
       url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1060-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
@@ -88,60 +89,6 @@ Channel(
       height: 424,
     ),
   ],
-  mobile_banner: [
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w320-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
-      width: 320,
-      height: 88,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w640-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
-      width: 640,
-      height: 175,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w960-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
-      width: 960,
-      height: 263,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1280-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
-      width: 1280,
-      height: 351,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1440-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
-      width: 1440,
-      height: 395,
-    ),
-  ],
-  tv_banner: [
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w320-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
-      width: 320,
-      height: 180,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w854-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
-      width: 854,
-      height: 480,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1280-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
-      width: 1280,
-      height: 720,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1920-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
-      width: 1920,
-      height: 1080,
-    ),
-    Thumbnail(
-      url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w2120-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
-      width: 2120,
-      height: 1192,
-    ),
-  ],
   has_shorts: false,
   has_live: false,
   visitor_data: Some("CgttaWpyTVpUN1AyZyioqr2ZBg%3D%3D"),
@@ -162,7 +109,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(2),
@@ -181,7 +128,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(1),
@@ -200,7 +147,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(9),
@@ -219,7 +166,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(2),
@@ -238,7 +185,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(4),
@@ -257,7 +204,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(18),
@@ -276,7 +223,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(3),
@@ -295,7 +242,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(8),
@@ -314,7 +261,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(13),
@@ -333,7 +280,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(9),
@@ -352,7 +299,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(7),
@@ -371,7 +318,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(3),
@@ -390,7 +337,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(8),
@@ -409,7 +356,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(2),
@@ -428,7 +375,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(3),
@@ -447,7 +394,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(10),
@@ -466,7 +413,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(1),
@@ -485,7 +432,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(9),
@@ -504,7 +451,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(16),
@@ -523,7 +470,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(7),
@@ -542,7 +489,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(6),
@@ -561,7 +508,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(12),
@@ -580,7 +527,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(1),
@@ -599,7 +546,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(5),
@@ -618,7 +565,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(2),
@@ -637,7 +584,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(4),
@@ -656,7 +603,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(1),
@@ -675,7 +622,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(2),
@@ -694,7 +641,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(9),
@@ -713,7 +660,7 @@ Channel(
           id: "UC2DjFE7Xf11URZqWBigcVOQ",
           name: "EEVblog",
           avatar: [],
-          verification: Verified,
+          verification: verified,
           subscriber_count: Some(881000),
         )),
         video_count: Some(1),