Compare commits

..

No commits in common. "main" and "rustypipe-cli/v0.7.1" have entirely different histories.

30 changed files with 319 additions and 34286 deletions

View file

@ -3,44 +3,6 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
## [v0.11.3](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.11.2..rustypipe/v0.11.3) - 2025-04-03
### 🐛 Bug Fixes
- Deobfuscator: global variable extraction fixed - ([ac44e95](https://codeberg.org/ThetaDev/rustypipe/commit/ac44e95a88d95f9d2d1ec672f86ca9d31d6991b9))
- Deobfuscator: small simplification - ([189ba81](https://codeberg.org/ThetaDev/rustypipe/commit/189ba81a42e6c09f6af4d2768c449c22b864101e))
- Deobfuscator: handle global functions as well - ([939a7ae](https://codeberg.org/ThetaDev/rustypipe/commit/939a7aea61a3eee4c1e67bfbfc835f0ce3934171))
- Handle music playlist/album not found - ([ea80717](https://codeberg.org/ThetaDev/rustypipe/commit/ea80717f692b2c45b5063c362c9fa8ebca5a3471))
- Switch client if no adaptive stream URLs were returned - ([187bf1c](https://codeberg.org/ThetaDev/rustypipe/commit/187bf1c9a0e846bff205e0d71a19c5a1ce7b1943))
- Handle music artist not found - ([daf3d03](https://codeberg.org/ThetaDev/rustypipe/commit/daf3d035be38b59aef1ae205ac91c2bbdda2fe66))
### ⚙️ Miscellaneous Tasks
- *(deps)* Update rust crate rand to 0.9.0 - ([af415dd](https://codeberg.org/ThetaDev/rustypipe/commit/af415ddf8f94f00edb918f271d8e6336503e9faf))
## [v0.11.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.11.1..rustypipe/v0.11.2) - 2025-03-24
### 🐛 Bug Fixes
- A/B test 22: commandExecutorCommand for playlist continuations - ([e8acbfb](https://codeberg.org/ThetaDev/rustypipe/commit/e8acbfbbcf5d31b5ac34410ddf334e5534e3762f))
- Extract deobf data with global strings variable - ([4ce6746](https://codeberg.org/ThetaDev/rustypipe/commit/4ce6746be538564e79f7e3c67d7a91aaa53f48ea))
- Handle player returning no adaptive stream URLs - ([07db7b1](https://codeberg.org/ThetaDev/rustypipe/commit/07db7b1166e912e1554f98f2ae20c2c356fed38f))
## [v0.11.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.11.0..rustypipe/v0.11.1) - 2025-03-16
### 🐛 Bug Fixes
- Simplify get_player_from_clients logic - ([c04b606](https://codeberg.org/ThetaDev/rustypipe/commit/c04b60604d2628bf8f0e3de453c243adbb966e57))
- Desktop client: generate PO token from user_syncid when authenticated - ([8342cae](https://codeberg.org/ThetaDev/rustypipe/commit/8342caeb0f566a38060a6ec69f3ca65b9a2afcd6))
- Always skip failed clients - ([63a6f50](https://codeberg.org/ThetaDev/rustypipe/commit/63a6f50a8b5ad6bb984282335c1481ae3cd2fe83))
### ⚙️ Miscellaneous Tasks
- *(deps)* Update rust crate rstest to 0.25.0 - ([9ed1306](https://codeberg.org/ThetaDev/rustypipe/commit/9ed1306f3aaeb993c409997ddfbc47499e4f4d22))
## [v0.11.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.10.0..rustypipe/v0.11.0) - 2025-02-26 ## [v0.11.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.10.0..rustypipe/v0.11.0) - 2025-02-26
### 🚀 Features ### 🚀 Features

View file

@ -1,6 +1,6 @@
[package] [package]
name = "rustypipe" name = "rustypipe"
version = "0.11.3" version = "0.11.0"
rust-version = "1.67.1" rust-version = "1.67.1"
edition.workspace = true edition.workspace = true
authors.workspace = true authors.workspace = true
@ -40,7 +40,7 @@ serde_with = { version = "3.0.0", default-features = false, features = [
] } ] }
serde_plain = "1.0.0" serde_plain = "1.0.0"
sha1 = "0.10.0" sha1 = "0.10.0"
rand = "0.9.0" rand = "0.8.0"
time = { version = "0.3.37", features = [ time = { version = "0.3.37", features = [
"macros", "macros",
"serde-human-readable", "serde-human-readable",
@ -67,15 +67,15 @@ dirs = "6.0.0"
filenamify = "0.1.0" filenamify = "0.1.0"
# Testing # Testing
rstest = "0.25.0" rstest = "0.24.0"
tokio-test = "0.4.2" tokio-test = "0.4.2"
insta = { version = "1.17.1", features = ["ron", "redactions"] } insta = { version = "1.17.1", features = ["ron", "redactions"] }
path_macro = "1.0.0" path_macro = "1.0.0"
tracing-test = "0.2.5" tracing-test = "0.2.5"
# Included crates # Included crates
rustypipe = { path = ".", version = "0.11.3", default-features = false } rustypipe = { path = ".", version = "0.11.0", default-features = false }
rustypipe-downloader = { path = "./downloader", version = "0.3.1", default-features = false, features = [ rustypipe-downloader = { path = "./downloader", version = "0.3.0", default-features = false, features = [
"indicatif", "indicatif",
"audiotag", "audiotag",
] } ] }

View file

@ -3,15 +3,6 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
## [v0.7.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.7.1..rustypipe-cli/v0.7.2) - 2025-03-16
### ⚙️ Miscellaneous Tasks
- *(deps)* Update rustypipe to 0.11.1
- *(deps)* Update rustypipe-downloader to 0.3.1
- *(deps)* Update rust crate rstest to 0.25.0 - ([9ed1306](https://codeberg.org/ThetaDev/rustypipe/commit/9ed1306f3aaeb993c409997ddfbc47499e4f4d22))
## [v0.7.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.7.0..rustypipe-cli/v0.7.1) - 2025-02-26 ## [v0.7.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.7.0..rustypipe-cli/v0.7.1) - 2025-02-26
### ⚙️ Miscellaneous Tasks ### ⚙️ Miscellaneous Tasks

View file

@ -1,6 +1,6 @@
[package] [package]
name = "rustypipe-cli" name = "rustypipe-cli"
version = "0.7.2" version = "0.7.1"
rust-version = "1.70.0" rust-version = "1.70.0"
edition.workspace = true edition.workspace = true
authors.workspace = true authors.workspace = true

View file

@ -41,15 +41,10 @@ pub enum ABTest {
MusicAlbumGroupsReordered = 19, MusicAlbumGroupsReordered = 19,
MusicContinuationItemRenderer = 20, MusicContinuationItemRenderer = 20,
AlbumRecommends = 21, AlbumRecommends = 21,
CommandExecutorCommand = 22,
} }
/// List of active A/B tests that are run when none is manually specified /// List of active A/B tests that are run when none is manually specified
const TESTS_TO_RUN: &[ABTest] = &[ const TESTS_TO_RUN: &[ABTest] = &[ABTest::MusicAlbumGroupsReordered, ABTest::AlbumRecommends];
ABTest::MusicAlbumGroupsReordered,
ABTest::AlbumRecommends,
ABTest::CommandExecutorCommand,
];
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize, Deserialize)]
pub struct ABTestRes { pub struct ABTestRes {
@ -125,7 +120,6 @@ pub async fn run_test(
music_continuation_item_renderer(&query).await music_continuation_item_renderer(&query).await
} }
ABTest::AlbumRecommends => album_recommends(&query).await, ABTest::AlbumRecommends => album_recommends(&query).await,
ABTest::CommandExecutorCommand => command_executor_command(&query).await,
} }
.unwrap(); .unwrap();
pb.inc(1); pb.inc(1);
@ -463,18 +457,3 @@ pub async fn album_recommends(rp: &RustyPipeQuery) -> Result<bool> {
.await?; .await?;
Ok(res.contains("\"musicCarouselShelfRenderer\"")) Ok(res.contains("\"musicCarouselShelfRenderer\""))
} }
pub async fn command_executor_command(rp: &RustyPipeQuery) -> Result<bool> {
let id = "VLPLbZIPy20-1pN7mqjckepWF78ndb6ci_qi";
let res = rp
.raw(
ClientType::Desktop,
"browse",
&QBrowse {
browse_id: id,
params: None,
},
)
.await?;
Ok(res.contains("\"commandExecutorCommand\""))
}

View file

@ -51,7 +51,7 @@ image = { version = "0.25.0", optional = true, default-features = false, feature
"jpeg", "jpeg",
"webp", "webp",
] } ] }
smartcrop2 = { version = "0.4.0", optional = true } smartcrop2 = { version = "0.3.1", optional = true }
[dev-dependencies] [dev-dependencies]
path_macro.workspace = true path_macro.workspace = true

View file

@ -1033,11 +1033,7 @@ impl DownloadQuery {
image::load_from_memory(&img_bts)? image::load_from_memory(&img_bts)?
}; };
let crop = smartcrop::find_best_crop_no_borders( let crop = smartcrop::find_best_crop(&img, NonZeroU32::MIN, NonZeroU32::MIN)
&img,
NonZeroU32::MIN,
NonZeroU32::MIN,
)
.map_err(|e| DownloadError::AudioTag(format!("image crop: {e}").into()))? .map_err(|e| DownloadError::AudioTag(format!("image crop: {e}").into()))?
.crop; .crop;
img = img.crop_imm(crop.x, crop.y, crop.width, crop.height); img = img.crop_imm(crop.x, crop.y, crop.width, crop.height);
@ -1067,8 +1063,8 @@ impl DownloadQuery {
} }
fn get_download_range(offset: u64, size: Option<u64>) -> Range<u64> { fn get_download_range(offset: u64, size: Option<u64>) -> Range<u64> {
let mut rng = rand::rng(); let mut rng = rand::thread_rng();
let chunk_size = rng.random_range(CHUNK_SIZE_MIN..CHUNK_SIZE_MAX); let chunk_size = rng.gen_range(CHUNK_SIZE_MIN..CHUNK_SIZE_MAX);
let mut chunk_end = offset + chunk_size; let mut chunk_end = offset + chunk_size;
if let Some(size) = size { if let Some(size) = size {
@ -1201,8 +1197,6 @@ async fn download_single_file(
} }
} }
tracing::debug!("downloading {} to {}", url, output.to_string_lossy());
let mut file = fs::OpenOptions::new() let mut file = fs::OpenOptions::new()
.append(true) .append(true)
.create(true) .create(true)

View file

@ -1030,7 +1030,7 @@ commandContext missing).
- **Encountered on:** 13.01.2025 - **Encountered on:** 13.01.2025
- **Impact:** 🟢 Low - **Impact:** 🟢 Low
- **Endpoint:** browse (YTM) - **Endpoint:** browse (YTM)
- **Status:** Frequent (59%) - **Status:** Common (10%)
YouTube Music used to group artist albums into 2 rows: "Albums" and "Singles". YouTube Music used to group artist albums into 2 rows: "Albums" and "Singles".
@ -1067,37 +1067,3 @@ pages. The difficulty is distinguishing them reliably for parsing the album vari
The current solution is adding the "Other versions" title in all languages to the The current solution is adding the "Other versions" title in all languages to the
dictionary and comparing it. dictionary and comparing it.
## [22] commandExecutorCommand for continuations
- **Encountered on:** 16.03.2025
- **Impact:** 🟢 Low
- **Endpoint:** browse (YTM)
- **Status:** Experimental (1%)
YouTube playlists may use a commandExecutorCommand which holds a list of commands: the
`continuationCommand` that needs to be extracted as well as a `playlistVotingRefreshPopupCommand`.
```json
{
"continuationItemRenderer": {
"continuationEndpoint": {
"commandExecutorCommand": {
"commands": [
{
"playlistVotingRefreshPopupCommand": {
"command": {}
}
},
{
"continuationCommand": {
"request": "CONTINUATION_REQUEST_TYPE_BROWSE",
"token": "4qmFsgKBARIkVkxQTGJaSVB5MjAtMXBON21xamNrZXBXRjc4bmRiNmNpX3FpGjRDQUY2SGxCVU9rTklTV2xGUkVreVVtdEZOVTVFU1hsU2FrWkRVa1JKZWs1NldRJTNEJTNEmgIiUExiWklQeTIwLTFwTjdtcWpja2VwV0Y3OG5kYjZjaV9xaQ%3D%3D"
}
}
]
}
}
}
}
```

View file

@ -293,10 +293,8 @@ struct OauthToken {
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
struct AuthCookie { struct AuthCookie {
cookie: String, cookie: String,
#[serde(alias = "account_syncid", skip_serializing_if = "Option::is_none")]
channel_syncid: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
user_syncid: Option<String>, account_syncid: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
session_index: Option<String>, session_index: Option<String>,
} }
@ -321,9 +319,8 @@ impl AuthCookie {
fn new(cookie: String) -> Self { fn new(cookie: String) -> Self {
Self { Self {
cookie, cookie,
channel_syncid: None, account_syncid: None,
session_index: None, session_index: None,
user_syncid: None,
} }
} }
} }
@ -1594,17 +1591,6 @@ impl RustyPipe {
.ok_or(Error::Auth(AuthError::NoLogin)) .ok_or(Error::Auth(AuthError::NoLogin))
} }
fn user_auth_datasync_id(&self) -> Result<String, Error> {
self.inner
.cache
.auth_cookie
.read()
.unwrap()
.as_ref()
.and_then(|c| c.user_syncid.as_ref().map(|id| id.to_owned()))
.ok_or(Error::Auth(AuthError::NoLogin))
}
/// Set the user authentication cookie /// Set the user authentication cookie
/// ///
/// The cookie is used for authenticated requests with browser-based clients /// The cookie is used for authenticated requests with browser-based clients
@ -1699,17 +1685,17 @@ impl RustyPipe {
))?; ))?;
// datasyncid is of the form "channel_syncid||user_syncid" for secondary channel // datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
// and just "user_syncid||" for primary channel. // and just "user_syncid||" for primary channel. We only want the channel_syncid
let (p1, p2) = let (channel_syncid, user_syncid) =
datasync_id datasync_id
.split_once("||") .split_once("||")
.ok_or(Error::Extraction(ExtractionError::InvalidData( .ok_or(Error::Extraction(ExtractionError::InvalidData(
"datasyncId does not contain || seperator".into(), "datasyncId does not contain || seperator".into(),
)))?; )))?;
(auth_cookie.channel_syncid, auth_cookie.user_syncid) = if p2.is_empty() { auth_cookie.account_syncid = if user_syncid.is_empty() {
(None, Some(p1.to_owned())) None
} else { } else {
(Some(p1.to_owned()), Some(p2.to_owned())) Some(channel_syncid.to_owned())
}; };
auth_cookie.session_index = Some( auth_cookie.session_index = Some(
@ -2143,7 +2129,7 @@ impl RustyPipeQuery {
if let Some(session_index) = auth_cookie.session_index { if let Some(session_index) = auth_cookie.session_index {
r = r.header("X-Goog-AuthUser", session_index); r = r.header("X-Goog-AuthUser", session_index);
} }
if let Some(account_syncid) = auth_cookie.channel_syncid { if let Some(account_syncid) = auth_cookie.account_syncid {
r = r.header("X-Goog-PageId", account_syncid); r = r.header("X-Goog-PageId", account_syncid);
} }
cookie = Some(auth_cookie.cookie); cookie = Some(auth_cookie.cookie);

View file

@ -154,24 +154,7 @@ fn map_artist_page(
ctx: &MapRespCtx<'_>, ctx: &MapRespCtx<'_>,
skip_extendables: bool, skip_extendables: bool,
) -> Result<MapResult<(MusicArtist, bool)>, ExtractionError> { ) -> Result<MapResult<(MusicArtist, bool)>, ExtractionError> {
let contents = match res.contents { let header = res.header.music_immersive_header_renderer;
Some(c) => c,
None => {
if res.microformat.microformat_data_renderer.noindex {
return Err(ExtractionError::NotFound {
id: ctx.id.to_owned(),
msg: "no contents".into(),
});
} else {
return Err(ExtractionError::InvalidData("no contents".into()));
}
}
};
let header = res
.header
.ok_or(ExtractionError::InvalidData("no header".into()))?
.music_immersive_header_renderer;
if let Some(share) = header.share_endpoint { if let Some(share) = header.share_endpoint {
let pb = share.share_entity_endpoint.serialized_share_entity; let pb = share.share_entity_endpoint.serialized_share_entity;
@ -188,7 +171,8 @@ fn map_artist_page(
} }
} }
let sections = contents let sections = res
.contents
.single_column_browse_results_renderer .single_column_browse_results_renderer
.contents .contents
.into_iter() .into_iter()

View file

@ -151,21 +151,7 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
self, self,
ctx: &MapRespCtx<'_>, ctx: &MapRespCtx<'_>,
) -> Result<MapResult<MusicPlaylist>, ExtractionError> { ) -> Result<MapResult<MusicPlaylist>, ExtractionError> {
let contents = match self.contents { let (header, music_contents) = match self.contents {
Some(c) => c,
None => {
if self.microformat.microformat_data_renderer.noindex {
return Err(ExtractionError::NotFound {
id: ctx.id.to_owned(),
msg: "no contents".into(),
});
} else {
return Err(ExtractionError::InvalidData("no contents".into()));
}
}
};
let (header, music_contents) = match contents {
response::music_playlist::Contents::SingleColumnBrowseResultsRenderer(c) => ( response::music_playlist::Contents::SingleColumnBrowseResultsRenderer(c) => (
self.header, self.header,
c.contents c.contents
@ -352,21 +338,7 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
impl MapResponse<MusicAlbum> for response::MusicPlaylist { impl MapResponse<MusicAlbum> for response::MusicPlaylist {
fn map_response(self, ctx: &MapRespCtx<'_>) -> Result<MapResult<MusicAlbum>, ExtractionError> { fn map_response(self, ctx: &MapRespCtx<'_>) -> Result<MapResult<MusicAlbum>, ExtractionError> {
let contents = match self.contents { let (header, sections) = match self.contents {
Some(c) => c,
None => {
if self.microformat.microformat_data_renderer.noindex {
return Err(ExtractionError::NotFound {
id: ctx.id.to_owned(),
msg: "no contents".into(),
});
} else {
return Err(ExtractionError::InvalidData("no contents".into()));
}
}
};
let (header, sections) = match contents {
response::music_playlist::Contents::SingleColumnBrowseResultsRenderer(c) => ( response::music_playlist::Contents::SingleColumnBrowseResultsRenderer(c) => (
self.header, self.header,
c.contents c.contents
@ -482,12 +454,10 @@ impl MapResponse<MusicAlbum> for response::MusicPlaylist {
} }
} }
let playlist_id = self let playlist_id = self.microformat.and_then(|mf| {
.microformat mf.microformat_data_renderer
.microformat_data_renderer
.url_canonical .url_canonical
.and_then(|x| { .strip_prefix("https://music.youtube.com/playlist?list=")
x.strip_prefix("https://music.youtube.com/playlist?list=")
.map(str::to_owned) .map(str::to_owned)
}); });
let (playlist_id, artist_id) = header let (playlist_id, artist_id) = header

View file

@ -249,9 +249,11 @@ impl MapResponse<Paginator<HistoryItem<VideoItem>>> for response::Continuation {
&mut map_res, &mut map_res,
); );
} }
response::YouTubeListItem::ContinuationItemRenderer(ep) => { response::YouTubeListItem::ContinuationItemRenderer {
continuation_endpoint,
} => {
if ctoken.is_none() { if ctoken.is_none() {
ctoken = ep.continuation_endpoint.into_token(); ctoken = Some(continuation_endpoint.continuation_command.token);
} }
} }
_ => {} _ => {}

View file

@ -1,6 +1,6 @@
use std::{ use std::{
borrow::Cow, borrow::Cow,
collections::{BTreeMap, HashMap, HashSet}, collections::{BTreeMap, HashMap},
fmt::Debug, fmt::Debug,
}; };
@ -104,29 +104,42 @@ impl RustyPipeQuery {
) -> Result<VideoPlayer, Error> { ) -> Result<VideoPlayer, Error> {
let video_id = video_id.as_ref(); let video_id = video_id.as_ref();
let mut last_e = None; let mut last_e = None;
let mut query = Cow::Borrowed(self);
let mut clients_iter = clients.iter().peekable(); let mut clients_iter = clients.iter().peekable();
let mut failed_clients = HashSet::new();
while let Some(client) = clients_iter.next() { while let Some(client) = clients_iter.next() {
if query.opts.auth == Some(true) && !self.auth_enabled(*client) { if self.opts.auth == Some(true) && !self.auth_enabled(*client) {
// If no client has auth enabled, return NoLogin error instead of "no clients" // If no client has auth enabled, return NoLogin error instead of "no clients"
if last_e.is_none() { if last_e.is_none() {
last_e = Some(Error::Auth(AuthError::NoLogin)); last_e = Some(Error::Auth(AuthError::NoLogin));
} }
continue; continue;
} }
if failed_clients.contains(client) {
continue;
}
let res = query.player_from_client(video_id, *client).await; let res = self.player_from_client(video_id, *client).await;
match res { match res {
Ok(res) => return Ok(res), Ok(res) => return Ok(res),
Err(Error::Extraction(e)) => { Err(Error::Extraction(e)) => {
if e.use_login() && query.opts.auth.is_none() { if e.use_login() {
clients_iter = clients.iter().peekable(); if let Some(c) = self.auth_enabled_client(clients) {
query = Cow::Owned(self.clone().authenticated()); tracing::info!("{e}; fetching player with login");
match self
.clone()
.authenticated()
.player_from_client(video_id, c)
.await
{
Ok(res) => return Ok(res),
Err(Error::Extraction(e)) => {
if !e.switch_client() {
return Err(Error::Extraction(e));
}
}
Err(e) => return Err(e),
}
} else {
return Err(Error::Extraction(e));
}
} else if !e.switch_client() { } else if !e.switch_client() {
return Err(Error::Extraction(e)); return Err(Error::Extraction(e));
} }
@ -134,7 +147,6 @@ impl RustyPipeQuery {
tracing::warn!("error fetching player with {client:?} client: {e}; retrying with {next_client:?} client"); tracing::warn!("error fetching player with {client:?} client: {e}; retrying with {next_client:?} client");
} }
last_e = Some(Error::Extraction(e)); last_e = Some(Error::Extraction(e));
failed_clients.insert(*client);
} }
Err(e) => return Err(e), Err(e) => return Err(e),
} }
@ -144,27 +156,22 @@ impl RustyPipeQuery {
async fn get_player_po_token(&self, video_id: &str) -> Result<PlayerPoToken, Error> { async fn get_player_po_token(&self, video_id: &str) -> Result<PlayerPoToken, Error> {
if let Some(bg) = &self.client.inner.botguard { if let Some(bg) = &self.client.inner.botguard {
let (ident, visitor_data) = if self.opts.auth == Some(true) {
(self.client.user_auth_datasync_id()?, None)
} else {
let visitor_data = self.get_visitor_data(false).await?; let visitor_data = self.get_visitor_data(false).await?;
(visitor_data.to_owned(), Some(visitor_data))
};
if bg.po_token_cache { if bg.po_token_cache {
let session_token = self.get_session_po_token(&ident).await?; let session_token = self.get_session_po_token(&visitor_data).await?;
Ok(PlayerPoToken { Ok(PlayerPoToken {
visitor_data, visitor_data: Some(visitor_data),
session_po_token: Some(session_token), session_po_token: Some(session_token),
content_po_token: None, content_po_token: None,
}) })
} else { } else {
let (po_tokens, valid_until) = self.get_po_tokens(&[video_id, &ident]).await?; let (po_tokens, valid_until) =
self.get_po_tokens(&[video_id, &visitor_data]).await?;
let mut po_tokens = po_tokens.into_iter(); let mut po_tokens = po_tokens.into_iter();
let po_token = po_tokens.next().unwrap(); let po_token = po_tokens.next().unwrap();
let session_po_token = po_tokens.next().unwrap(); let session_po_token = po_tokens.next().unwrap();
Ok(PlayerPoToken { Ok(PlayerPoToken {
visitor_data, visitor_data: Some(visitor_data),
session_po_token: Some(PoToken { session_po_token: Some(PoToken {
po_token: session_po_token, po_token: session_po_token,
valid_until, valid_until,
@ -184,11 +191,6 @@ impl RustyPipeQuery {
video_id: S, video_id: S,
client_type: ClientType, client_type: ClientType,
) -> Result<VideoPlayer, Error> { ) -> Result<VideoPlayer, Error> {
if self.opts.auth == Some(true) {
tracing::info!("fetching {client_type:?} player with login");
} else {
tracing::debug!("fetching {client_type:?} player");
}
let video_id = video_id.as_ref(); let video_id = video_id.as_ref();
let (deobf, player_po) = tokio::try_join!( let (deobf, player_po) = tokio::try_join!(
@ -383,21 +385,6 @@ impl MapResponse<VideoPlayer> for response::Player {
video_details.video_id, ctx.id video_details.video_id, ctx.id
))); )));
} }
// Sometimes YouTube Desktop does not output any URLs for adaptive streams.
// Since this is currently rare, it is best to retry the request in this case.
if !is_live
&& !streaming_data.adaptive_formats.c.is_empty()
&& streaming_data
.adaptive_formats
.c
.iter()
.all(|f| f.url.is_none() && f.signature_cipher.is_none())
{
return Err(ExtractionError::Unavailable {
reason: UnavailabilityReason::TryAgain,
msg: "no adaptive stream URLs".to_owned(),
});
}
let video_info = VideoPlayerDetails { let video_info = VideoPlayerDetails {
id: video_details.video_id, id: video_details.video_id,
@ -640,7 +627,7 @@ impl<'a> StreamsMapper<'a> {
fn deobf(&self) -> Result<&Deobfuscator, DeobfError> { fn deobf(&self) -> Result<&Deobfuscator, DeobfError> {
self.deobf self.deobf
.as_ref() .as_ref()
.ok_or(DeobfError::Other("no deobfuscator".into())) .ok_or(DeobfError::Other("no deobfuscator"))
} }
fn cipher_to_url_params( fn cipher_to_url_params(

View file

@ -257,7 +257,6 @@ mod tests {
#[case::nomusic("nomusic", "PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe")] #[case::nomusic("nomusic", "PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe")]
#[case::live("live", "UULVvqRdlKsE5Q8mf8YXbdIJLw")] #[case::live("live", "UULVvqRdlKsE5Q8mf8YXbdIJLw")]
#[case::pageheader("20241011_pageheader", "PLT2w2oBf1TZKyvY_M6JsASs73m-wjLzH5")] #[case::pageheader("20241011_pageheader", "PLT2w2oBf1TZKyvY_M6JsASs73m-wjLzH5")]
#[case::cmdexecutor("20250316_cmdexecutor", "PLbZIPy20-1pN7mqjckepWF78ndb6ci_qi")]
fn map_playlist_data(#[case] name: &str, #[case] id: &str) { fn map_playlist_data(#[case] name: &str, #[case] id: &str) {
let json_path = path!(*TESTFILES / "playlist" / format!("playlist_{name}.json")); let json_path = path!(*TESTFILES / "playlist" / format!("playlist_{name}.json"));
let json_file = File::open(json_path).unwrap(); let json_file = File::open(json_path).unwrap();

View file

@ -152,16 +152,9 @@ pub(crate) struct ContinuationItemRenderer {
pub continuation_endpoint: ContinuationEndpoint, pub continuation_endpoint: ContinuationEndpoint,
} }
#[derive(Debug, Deserialize)]
#[serde(untagged)]
pub(crate) enum ContinuationEndpoint {
ContinuationCommand(ContinuationCommandWrap),
CommandExecutorCommand(CommandExecutorCommandWrap),
}
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub(crate) struct ContinuationCommandWrap { pub(crate) struct ContinuationEndpoint {
pub continuation_command: ContinuationCommand, pub continuation_command: ContinuationCommand,
} }
@ -171,34 +164,7 @@ pub(crate) struct ContinuationCommand {
pub token: String, pub token: String,
} }
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct CommandExecutorCommandWrap {
pub command_executor_command: CommandExecutorCommand,
}
#[serde_as] #[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct CommandExecutorCommand {
#[serde_as(as = "VecSkipError<_>")]
commands: Vec<ContinuationCommandWrap>,
}
impl ContinuationEndpoint {
pub fn into_token(self) -> Option<String> {
match self {
Self::ContinuationCommand(cmd) => Some(cmd.continuation_command.token),
Self::CommandExecutorCommand(cmd) => cmd
.command_executor_command
.commands
.into_iter()
.next()
.map(|c| c.continuation_command.token),
}
}
}
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub(crate) struct Icon { pub(crate) struct Icon {

View file

@ -5,8 +5,7 @@ use crate::serializer::text::Text;
use super::{ use super::{
music_item::{ music_item::{
Button, Grid, ItemSection, MusicMicroformat, MusicThumbnailRenderer, SimpleHeader, Button, Grid, ItemSection, MusicThumbnailRenderer, SimpleHeader, SingleColumnBrowseResult,
SingleColumnBrowseResult,
}, },
SectionList, Tab, SectionList, Tab,
}; };
@ -15,10 +14,8 @@ use super::{
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub(crate) struct MusicArtist { pub(crate) struct MusicArtist {
pub contents: Option<SingleColumnBrowseResult<Tab<SectionList<ItemSection>>>>, pub contents: SingleColumnBrowseResult<Tab<SectionList<ItemSection>>>,
pub header: Option<Header>, pub header: Header,
#[serde(default)]
pub microformat: MusicMicroformat,
} }
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]

View file

@ -433,22 +433,6 @@ pub(crate) enum TrackBadge {
LiveBadgeRenderer {}, LiveBadgeRenderer {},
} }
#[serde_as]
#[derive(Default, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct MusicMicroformat {
#[serde_as(as = "DefaultOnError")]
pub microformat_data_renderer: MicroformatData,
}
#[derive(Default, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct MicroformatData {
pub url_canonical: Option<String>,
#[serde(default)]
pub noindex: bool,
}
/* /*
#MAPPER #MAPPER
*/ */
@ -546,9 +530,7 @@ impl MusicListMapper {
MusicResponseItem::ContinuationItemRenderer { MusicResponseItem::ContinuationItemRenderer {
continuation_endpoint, continuation_endpoint,
} => { } => {
if self.ctoken.is_none() { self.ctoken = Some(continuation_endpoint.continuation_command.token);
self.ctoken = continuation_endpoint.into_token();
}
Ok(None) Ok(None)
} }
} }

View file

@ -5,21 +5,22 @@ use crate::serializer::text::{AttributedText, Text, TextComponents};
use super::{ use super::{
music_item::{ music_item::{
Button, ItemSection, MusicContentsRenderer, MusicItemMenuEntry, MusicMicroformat, Button, ItemSection, MusicContentsRenderer, MusicItemMenuEntry, MusicThumbnailRenderer,
MusicThumbnailRenderer,
}, },
url_endpoint::OnTapWrap, url_endpoint::OnTapWrap,
ContentsRenderer, SectionList, Tab, ContentsRenderer, SectionList, Tab,
}; };
/// Response model for YouTube Music playlists and albums /// Response model for YouTube Music playlists and albums
#[serde_as]
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub(crate) struct MusicPlaylist { pub(crate) struct MusicPlaylist {
pub contents: Option<Contents>, pub contents: Contents,
pub header: Option<Header>, pub header: Option<Header>,
#[serde(default)] #[serde(default)]
pub microformat: MusicMicroformat, #[serde_as(as = "DefaultOnError")]
pub microformat: Option<Microformat>,
} }
#[serde_as] #[serde_as]
@ -161,3 +162,15 @@ pub(crate) struct AvatarStackViewModel {
pub(crate) struct AvatarStackRendererContext { pub(crate) struct AvatarStackRendererContext {
pub command_context: Option<OnTapWrap>, pub command_context: Option<OnTapWrap>,
} }
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct Microformat {
pub microformat_data_renderer: MicroformatData,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct MicroformatData {
pub url_canonical: String,
}

View file

@ -530,14 +530,15 @@ pub(crate) enum ContinuationItemVariants {
} }
impl ContinuationItemVariants { impl ContinuationItemVariants {
pub fn into_token(self) -> Option<String> { pub fn token(self) -> String {
match self { match self {
ContinuationItemVariants::Ep { ContinuationItemVariants::Ep {
continuation_endpoint, continuation_endpoint,
} => continuation_endpoint, } => continuation_endpoint,
ContinuationItemVariants::Btn { button } => button.button_renderer.command, ContinuationItemVariants::Btn { button } => button.button_renderer.command,
} }
.into_token() .continuation_command
.token
} }
} }

View file

@ -4,7 +4,7 @@ use serde_with::{
}; };
use time::OffsetDateTime; use time::OffsetDateTime;
use super::{ChannelBadge, ContentImage, ContinuationItemRenderer, PhMetadataView, Thumbnails}; use super::{ChannelBadge, ContentImage, ContinuationEndpoint, PhMetadataView, Thumbnails};
use crate::{ use crate::{
model::{Channel, ChannelItem, ChannelTag, PlaylistItem, VideoItem, YouTubeItem}, model::{Channel, ChannelItem, ChannelTag, PlaylistItem, VideoItem, YouTubeItem},
param::Language, param::Language,
@ -37,9 +37,12 @@ pub(crate) enum YouTubeListItem {
LockupViewModel(LockupViewModel), LockupViewModel(LockupViewModel),
/// Continuation items are located at the end of a list /// Continauation items are located at the end of a list
/// and contain the continuation token for progressive loading /// and contain the continuation token for progressive loading
ContinuationItemRenderer(ContinuationItemRenderer), #[serde(rename_all = "camelCase")]
ContinuationItemRenderer {
continuation_endpoint: ContinuationEndpoint,
},
/// Corrected search query /// Corrected search query
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
@ -835,11 +838,9 @@ impl YouTubeListMapper<YouTubeItem> {
self.items.push(mapped); self.items.push(mapped);
} }
} }
YouTubeListItem::ContinuationItemRenderer(r) => { YouTubeListItem::ContinuationItemRenderer {
if self.ctoken.is_none() { continuation_endpoint,
self.ctoken = r.continuation_endpoint.into_token(); } => self.ctoken = Some(continuation_endpoint.continuation_command.token),
}
}
YouTubeListItem::ShowingResultsForRenderer { corrected_query } => { YouTubeListItem::ShowingResultsForRenderer { corrected_query } => {
self.corrected_query = Some(corrected_query); self.corrected_query = Some(corrected_query);
} }
@ -885,11 +886,9 @@ impl YouTubeListMapper<VideoItem> {
self.items.push(mapped); self.items.push(mapped);
} }
} }
YouTubeListItem::ContinuationItemRenderer(r) => { YouTubeListItem::ContinuationItemRenderer {
if self.ctoken.is_none() { continuation_endpoint,
self.ctoken = r.continuation_endpoint.into_token(); } => self.ctoken = Some(continuation_endpoint.continuation_command.token),
}
}
YouTubeListItem::ShowingResultsForRenderer { corrected_query } => { YouTubeListItem::ShowingResultsForRenderer { corrected_query } => {
self.corrected_query = Some(corrected_query); self.corrected_query = Some(corrected_query);
} }
@ -939,11 +938,9 @@ impl YouTubeListMapper<PlaylistItem> {
self.items.push(mapped); self.items.push(mapped);
} }
} }
YouTubeListItem::ContinuationItemRenderer(r) => { YouTubeListItem::ContinuationItemRenderer {
if self.ctoken.is_none() { continuation_endpoint,
self.ctoken = r.continuation_endpoint.into_token(); } => self.ctoken = Some(continuation_endpoint.continuation_command.token),
}
}
YouTubeListItem::ShowingResultsForRenderer { corrected_query } => { YouTubeListItem::ShowingResultsForRenderer { corrected_query } => {
self.corrected_query = Some(corrected_query); self.corrected_query = Some(corrected_query);
} }

View file

@ -207,9 +207,11 @@ impl MapResponse<Paginator<HistoryItem<VideoItem>>> for response::History {
&mut map_res, &mut map_res,
); );
} }
response::YouTubeListItem::ContinuationItemRenderer(ep) => { response::YouTubeListItem::ContinuationItemRenderer {
continuation_endpoint,
} => {
if ctoken.is_none() { if ctoken.is_none() {
ctoken = ep.continuation_endpoint.into_token(); ctoken = Some(continuation_endpoint.continuation_command.token);
} }
} }
_ => {} _ => {}

View file

@ -208,10 +208,11 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
) )
}); });
let comment_ctoken = comment_ctoken_section.and_then(|s| { let comment_ctoken = comment_ctoken_section.map(|s| {
s.continuation_item_renderer s.continuation_item_renderer
.continuation_endpoint .continuation_endpoint
.into_token() .continuation_command
.token
}); });
let (owner, description, is_ccommons) = match secondary_info { let (owner, description, is_ccommons) = match secondary_info {
@ -332,7 +333,7 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
.sub_menu_items; .sub_menu_items;
items items
.try_swap_remove(1) .try_swap_remove(1)
.and_then(|c| c.service_endpoint.into_token()) .map(|c| c.service_endpoint.continuation_command.token)
}); });
Ok(MapResult { Ok(MapResult {
@ -452,9 +453,7 @@ impl MapResponse<Paginator<Comment>> for response::VideoComments {
} }
} }
response::video_details::CommentListItem::ContinuationItemRenderer(cont) => { response::video_details::CommentListItem::ContinuationItemRenderer(cont) => {
if ctoken.is_none() { ctoken = Some(cont.token());
ctoken = cont.into_token();
}
} }
response::video_details::CommentListItem::CommentsHeaderRenderer { count_text } => { response::video_details::CommentListItem::CommentsHeaderRenderer { count_text } => {
comment_count = count_text comment_count = count_text
@ -521,9 +520,7 @@ fn map_replies(
)) ))
} }
response::video_details::CommentListItem::ContinuationItemRenderer(cont) => { response::video_details::CommentListItem::ContinuationItemRenderer(cont) => {
if reply_ctoken.is_none() { reply_ctoken = Some(cont.token());
reply_ctoken = cont.into_token();
}
None None
} }
_ => None, _ => None,

View file

@ -3,7 +3,7 @@ use std::collections::HashMap;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use regex::Regex; use regex::Regex;
use reqwest::Client; use reqwest::Client;
use ress::tokens::{Keyword, Punct, Token}; use ress::tokens::Token;
use rquickjs::{Context, Runtime}; use rquickjs::{Context, Runtime};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -106,7 +106,7 @@ impl Deobfuscator {
.with(|ctx| call_fn(&ctx, DEOBF_NSIG_FUNC_NAME, nsig))?; .with(|ctx| call_fn(&ctx, DEOBF_NSIG_FUNC_NAME, nsig))?;
tracing::trace!("deobf nsig: {nsig} -> {res}"); tracing::trace!("deobf nsig: {nsig} -> {res}");
if res.starts_with("enhanced_except_") || res.ends_with(nsig) { if res.starts_with("enhanced_except_") || res.ends_with(nsig) {
return Err(DeobfError::Other("nsig fn returned an exception".into())); return Err(DeobfError::Other("nsig fn returned an exception"));
} }
Ok(res) Ok(res)
} }
@ -134,21 +134,55 @@ fn caller_function(mapped_name: &str, fn_name: &str) -> String {
} }
fn get_sig_fn(player_js: &str) -> Result<String, DeobfError> { fn get_sig_fn(player_js: &str) -> Result<String, DeobfError> {
let name = get_sig_fn_name(player_js)?; let dfunc_name = get_sig_fn_name(player_js)?;
let code = extract_js_fn(player_js, &name)?;
let js_fn = format!("{}{}", code, caller_function(DEOBF_SIG_FUNC_NAME, &name));
let function_pattern_str = format!(
r#"({}=function\([\w]+\)\{{.+?\}})"#,
dfunc_name.replace('$', "\\$")
);
let function_pattern = Regex::new(&function_pattern_str)
.map_err(|_| DeobfError::Other("could not parse sig fn pattern regex"))?;
let deobfuscate_function = format!(
"var {};",
&function_pattern
.captures(player_js)
.ok_or(DeobfError::Extraction("sig fn"))?[1]
);
let helper_object_name_pattern = Regex::new(r";([\w\$]{2,3})\...\(").unwrap();
let helper_object_name = helper_object_name_pattern
.captures(&deobfuscate_function)
.ok_or(DeobfError::Extraction("sig fn helper object name"))?
.get(1)
.unwrap()
.as_str();
let helper_pattern_str = format!(
r#"(var {}=\{{.+?\}}\}};)"#,
helper_object_name.replace('$', "\\$")
);
let helper_pattern = Regex::new(&helper_pattern_str)
.map_err(|_| DeobfError::Other("could not parse helper pattern regex"))?;
let player_js_nonl = player_js.replace('\n', "");
let helper_object = &helper_pattern
.captures(&player_js_nonl)
.ok_or(DeobfError::Extraction("sig fn helper object"))?[1];
let js_fn = helper_object.to_owned()
+ &deobfuscate_function
+ &caller_function(DEOBF_SIG_FUNC_NAME, &dfunc_name);
tracing::trace!("sig_fn: {js_fn}"); tracing::trace!("sig_fn: {js_fn}");
verify_fn(&js_fn, DEOBF_SIG_FUNC_NAME)?; verify_fn(&js_fn, DEOBF_SIG_FUNC_NAME)?;
tracing::debug!("successfully extracted sig fn `{name}`"); tracing::debug!("successfully extracted sig fn `{dfunc_name}`");
Ok(js_fn) Ok(js_fn)
} }
fn get_nsig_fn_names(player_js: &str) -> impl Iterator<Item = String> + '_ { fn get_nsig_fn_names(player_js: &str) -> impl Iterator<Item = String> + '_ {
static FUNCTION_NAME_REGEX: Lazy<Regex> = Lazy::new(|| { static FUNCTION_NAME_REGEX: Lazy<Regex> = Lazy::new(|| {
// x.get( OR index.m3u8 OR delete x.y.file .. y=functionName[array_num](z) .. x.set( // x.get( .. y=functionName[array_num](z) .. x.set(
Regex::new(r#"(?:[\w$]\.get\(|index\.m3u8|delete [\w$]+\.[\w$]+\.file).+[a-zA-Z]=([\w$]{2,})(?:\[(\d+)\])?\([a-zA-Z0-9]\).+[a-zA-Z0-9]\.set\("#) Regex::new(r#"(?:[\w$]\.get\(|index\.m3u8).+[a-zA-Z]=([\w$]{2,})(?:\[(\d+)\])?\([a-zA-Z0-9]\).+[a-zA-Z0-9]\.set\("#)
.unwrap() .unwrap()
}); });
@ -172,71 +206,26 @@ fn get_nsig_fn_names(player_js: &str) -> impl Iterator<Item = String> + '_ {
}) })
} }
fn extract_js_fn(js: &str, name: &str) -> Result<String, DeobfError> { fn extract_js_fn(js: &str, offset: usize, name: &str) -> Result<String, DeobfError> {
let function_base_re = Regex::new(&format!(r#"{}\s*=\s*function\("#, regex::escape(name)))
.map_err(|e| DeobfError::Other(format!("parsing regex for {name}: {e}").into()))?;
let offset = function_base_re
.find(js)
.ok_or(DeobfError::Extraction("could not find function base"))?
.start();
let scan = ress::Scanner::new(&js[offset..]); let scan = ress::Scanner::new(&js[offset..]);
let mut state = 0; let mut state = 0;
let mut level = 0;
#[derive(Default, Clone, PartialEq, Eq)] let mut start = 0;
struct Level { let mut end = 0;
brace: isize,
paren: isize,
bracket: isize,
}
let mut level = Level::default();
let mut start = 0usize;
let mut end = 0usize;
let mut period_before = false; let mut period_before = false;
let mut function_before = false; let mut last_ident = None;
let mut idents: HashMap<String, bool> = HashMap::new(); let mut idents: HashMap<String, usize> = HashMap::new();
// Set if the current statement is a variable/function param definition
// First value is the brace level, second is true if we are on the right hand side of an assignment
let mut var_def_stmt: Option<(Level, bool)> = None;
let global_objects = [ let global_objects = [
"globalThis", "NaN", "Infinity", "Object", "Function", "Boolean", "Symbol", "Error", "Number", "BigInt",
"NaN", "Math", "Date", "String", "RegExp", "Array", "Map", "Set",
"undefined",
"Infinity",
"Object",
"Function",
"Boolean",
"Symbol",
"Error",
"Number",
"BigInt",
"Math",
"Date",
"String",
"RegExp",
"Array",
"Map",
"Set",
"eval",
"isFinite",
"isNaN",
"parseFloat",
"parseInt",
"decodeURI",
"decodeURIComponent",
"encodeURI",
"encodeURIComponent",
"escape",
"unescape",
]; ];
for item in scan { for item in scan {
let it = item?; let it = item?;
let token = it.token; let token = it.token;
match state { match state {
// Looking for fn name // Looking for fn name
0 => { 0 => {
@ -247,116 +236,47 @@ fn extract_js_fn(js: &str, name: &str) -> Result<String, DeobfError> {
} }
// Looking for equals // Looking for equals
1 => { 1 => {
if token.matches_punct(Punct::Equal) { if token.matches_punct(ress::tokens::Punct::Equal) {
state = 2; state = 2;
} else { } else {
state = 0; state = 0;
} }
} }
2 => { 2 => {
match &token { // Looking for begin/end braces
Token::Punct(punct) => { if token.matches_punct(ress::tokens::Punct::OpenBrace) {
let var_def_this_lvl = || { level += 1;
var_def_stmt } else if token.matches_punct(ress::tokens::Punct::CloseBrace) {
.as_ref() level -= 1;
.map(|(x, _)| x == &level)
.unwrap_or_default()
};
match punct { if level == 0 {
Punct::OpenBrace => {
level.brace += 1;
}
Punct::CloseBrace => {
if var_def_this_lvl() {
var_def_stmt = None;
}
level.brace -= 1;
if level.brace == 0 {
end = it.span.end; end = it.span.end;
state = 3; state = 3;
break; break;
} }
} }
Punct::OpenParen => {
level.paren += 1; // Looking for variable names
} if let Token::Ident(id) = &token {
Punct::CloseParen => { if !period_before {
if var_def_this_lvl() { let id_str = id.to_string();
var_def_stmt = None; if !global_objects.contains(&id_str.as_str()) {
} last_ident = Some(id.to_string());
level.paren -= 1;
}
Punct::OpenBracket => {
level.bracket += 1;
}
Punct::CloseBracket => {
if var_def_this_lvl() {
var_def_stmt = None;
}
level.bracket -= 1;
}
Punct::SemiColon => {
if var_def_this_lvl() {
var_def_stmt = None;
} }
} }
Punct::Comma => { } else if last_ident.is_some()
if let Some((lvl, rhs)) = &mut var_def_stmt { && !token.matches_punct(ress::tokens::Punct::OpenParen)
if lvl == &level {
*rhs = false;
}
}
}
Punct::Equal => {
if let Some((lvl, rhs)) = &mut var_def_stmt {
if lvl == &level {
*rhs = true;
}
}
}
_ => {}
}
}
Token::Keyword(kw) => match kw {
Keyword::Var(_) | Keyword::Let(_) | Keyword::Const(_) => {
var_def_stmt = Some((level.clone(), false));
}
Keyword::Function(_) => {
let mut l = level.clone();
l.paren += 1;
var_def_stmt = Some((l, false));
}
_ => {}
},
Token::Ident(id) => {
// Ignore object attributes and 1char long local vars
if !period_before
&& id.as_ref().len() > 1
&& !global_objects.contains(&id.as_ref())
{ {
// If we are on the left hand side of a variable definition statement let n = idents.entry(last_ident.unwrap()).or_default();
// or after "function", mark the variable name as defined *n += 1;
if var_def_stmt last_ident = None;
.as_ref()
.map(|(lvl, rhs)| lvl == &level && !rhs)
.unwrap_or_default()
|| function_before
{
idents.insert(id.to_string(), true);
} else { } else {
idents.entry(id.to_string()).or_default(); last_ident = None;
}
}
}
_ => {}
} }
} }
_ => break, _ => break,
}; };
period_before = token.matches_punct(Punct::Period); period_before = token.matches_punct(ress::tokens::Punct::Period);
function_before = matches!(&token, Token::Keyword(Keyword::Function(_)));
} }
if state != 3 { if state != 3 {
@ -367,10 +287,9 @@ fn extract_js_fn(js: &str, name: &str) -> Result<String, DeobfError> {
let mut code = format!("var {};", &js[fn_range.clone()]); let mut code = format!("var {};", &js[fn_range.clone()]);
let rt = rquickjs::Runtime::new()?; let rt = rquickjs::Runtime::new()?;
for (ident, _) in idents.into_iter().filter(|(_, v)| !v) { for (ident, _) in idents.into_iter().filter(|(_, v)| *v == 1) {
let var_pattern_str = format!(r#"(^|[^\w$\.]){}\s*=[^=]"#, regex::escape(&ident)); let var_pattern_str = format!(r#"(^|[^\w$]){}\s*=[^=]"#, regex::escape(&ident));
let re = Regex::new(&var_pattern_str) let re = Regex::new(&var_pattern_str).unwrap();
.map_err(|e| DeobfError::Other(format!("parsing regex for {ident}: {e}").into()))?;
let found_variable = re let found_variable = re
.captures_iter(js) .captures_iter(js)
.filter(|cap| { .filter(|cap| {
@ -428,13 +347,13 @@ fn extract_js_var(js: &str) -> Option<&str> {
if let Token::Punct(p) = &token { if let Token::Punct(p) = &token {
match p { match p {
Punct::OpenBrace => braces.push(b'{'), ress::tokens::Punct::OpenBrace => braces.push(b'{'),
Punct::OpenBracket => braces.push(b'['), ress::tokens::Punct::OpenBracket => braces.push(b'['),
Punct::OpenParen => braces.push(b'('), ress::tokens::Punct::OpenParen => braces.push(b'('),
Punct::CloseBrace => close_brace(&mut braces, b'{')?, ress::tokens::Punct::CloseBrace => close_brace(&mut braces, b'{')?,
Punct::CloseBracket => close_brace(&mut braces, b'[')?, ress::tokens::Punct::CloseBracket => close_brace(&mut braces, b'[')?,
Punct::CloseParen => close_brace(&mut braces, b'(')?, ress::tokens::Punct::CloseParen => close_brace(&mut braces, b'(')?,
Punct::Comma | Punct::SemiColon => { ress::tokens::Punct::Comma | ress::tokens::Punct::SemiColon => {
if braces.is_empty() { if braces.is_empty() {
end = it.span.start; end = it.span.start;
break; break;
@ -469,19 +388,23 @@ fn verify_fn(js_fn: &str, fn_name: &str) -> Result<(), DeobfError> {
})?; })?;
if res.is_empty() { if res.is_empty() {
return Err(DeobfError::Other( return Err(DeobfError::Other("deobfuscation fn returned empty string"));
"deobfuscation fn returned empty string".into(),
));
} }
if res.starts_with("enhanced_except_") || res.ends_with(&testinp) { if res.starts_with("enhanced_except_") || res.ends_with(&testinp) {
return Err(DeobfError::Other("nsig fn returned an exception".into())); return Err(DeobfError::Other("nsig fn returned an exception"));
} }
Ok(()) Ok(())
} }
fn get_nsig_fn(player_js: &str) -> Result<String, DeobfError> { fn get_nsig_fn(player_js: &str) -> Result<String, DeobfError> {
let extract_fn = |name: &str| -> Result<String, DeobfError> { let extract_fn = |name: &str| -> Result<String, DeobfError> {
let code = extract_js_fn(player_js, name)?; let function_base = format!("{name}=function");
let offset = player_js
.find(&function_base)
.ok_or(DeobfError::Extraction("could not find function base"))?;
let code = extract_js_fn(player_js, offset, name)?;
let js_fn = format!("{}{}", code, caller_function(DEOBF_NSIG_FUNC_NAME, name)); let js_fn = format!("{}{}", code, caller_function(DEOBF_NSIG_FUNC_NAME, name));
tracing::trace!("nsig_fn: {js_fn}"); tracing::trace!("nsig_fn: {js_fn}");
verify_fn(&js_fn, DEOBF_NSIG_FUNC_NAME)?; verify_fn(&js_fn, DEOBF_NSIG_FUNC_NAME)?;
@ -549,9 +472,7 @@ mod tests {
std::fs::read_to_string(js_path).unwrap() std::fs::read_to_string(js_path).unwrap()
}); });
const SIG_DEOBF_FUNC: &str = r#"var qB={w8:function(a){a.reverse()}, const SIG_DEOBF_FUNC: &str = r#"var qB={w8:function(a){a.reverse()},EC:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c},Np:function(a,b){a.splice(0,b)}};var Rva=function(a){a=a.split("");qB.Np(a,3);qB.w8(a,41);qB.EC(a,55);qB.Np(a,3);qB.w8(a,33);qB.Np(a,3);qB.EC(a,48);qB.EC(a,17);qB.EC(a,43);return a.join("")};var deobf_sig=Rva;"#;
EC:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c},
Np:function(a,b){a.splice(0,b)}}; var Rva=function(a){a=a.split("");qB.Np(a,3);qB.w8(a,41);qB.EC(a,55);qB.Np(a,3);qB.w8(a,33);qB.Np(a,3);qB.EC(a,48);qB.EC(a,17);qB.EC(a,43);return a.join("")};var deobf_sig=Rva;"#;
const NSIG_DEOBF_FUNC: &str = r#"var Vo=function(a){var b=a.split(""),c=[function(d,e,f){var h=f.length;d.forEach(function(l,m,n){this.push(n[m]=f[(f.indexOf(l)-f.indexOf(this[m])+m+h--)%f.length])},e.split(""))}, const NSIG_DEOBF_FUNC: &str = r#"var Vo=function(a){var b=a.split(""),c=[function(d,e,f){var h=f.length;d.forEach(function(l,m,n){this.push(n[m]=f[(f.indexOf(l)-f.indexOf(this[m])+m+h--)%f.length])},e.split(""))},
928409064,-595856984,1403221911,653089124,-168714481,-1883008765,158931990,1346921902,361518508,1403221911,-362174697,-233641452,function(){for(var d=64,e=[];++d-e.length-32;){switch(d){case 91:d=44;continue;case 123:d=65;break;case 65:d-=18;continue;case 58:d=96;continue;case 46:d=95}e.push(String.fromCharCode(d))}return e}, 928409064,-595856984,1403221911,653089124,-168714481,-1883008765,158931990,1346921902,361518508,1403221911,-362174697,-233641452,function(){for(var d=64,e=[];++d-e.length-32;){switch(d){case 91:d=44;continue;case 123:d=65;break;case 65:d-=18;continue;case 58:d=96;continue;case 46:d=95}e.push(String.fromCharCode(d))}return e},
b,158931990,791141857,-907319795,-1776185924,1595027902,-829736173,function(d,e){e=(e%d.length+d.length)%d.length;d.splice(0,1,d.splice(e,1,d[0])[0])}, b,158931990,791141857,-907319795,-1776185924,1595027902,-829736173,function(d,e){e=(e%d.length+d.length)%d.length;d.splice(0,1,d.splice(e,1,d[0])[0])},
@ -604,7 +525,7 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
#[test] #[test]
fn t_extract_js_fn() { fn t_extract_js_fn() {
let base_js = "Wka = function(d){let x=10/2;return /,,[/,913,/](,)}/}let a = 42;"; let base_js = "Wka = function(d){let x=10/2;return /,,[/,913,/](,)}/}let a = 42;";
let res = extract_js_fn(base_js, "Wka").unwrap(); let res = extract_js_fn(base_js, 0, "Wka").unwrap();
assert_eq!( assert_eq!(
res, res,
"var Wka = function(d){let x=10/2;return /,,[/,913,/](,)}/};" "var Wka = function(d){let x=10/2;return /,,[/,913,/](,)}/};"
@ -615,7 +536,7 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
fn t_extract_js_fn_eviljs() { fn t_extract_js_fn_eviljs() {
// Evil JavaScript code containing braces within strings and regular expressions // Evil JavaScript code containing braces within strings and regular expressions
let base_js = "Wka = function(d){var x = [/,,/,913,/(,)}/,\"abcdef}\\\"\",];var y = 10/2/1;return x[1][y];}//some={}random-padding+;"; let base_js = "Wka = function(d){var x = [/,,/,913,/(,)}/,\"abcdef}\\\"\",];var y = 10/2/1;return x[1][y];}//some={}random-padding+;";
let res = extract_js_fn(base_js, "Wka").unwrap(); let res = extract_js_fn(base_js, 0, "Wka").unwrap();
assert_eq!( assert_eq!(
res, res,
"var Wka = function(d){var x = [/,,/,913,/(,)}/,\"abcdef}\\\"\",];var y = 10/2/1;return x[1][y];};" "var Wka = function(d){var x = [/,,/,913,/(,)}/,\"abcdef}\\\"\",];var y = 10/2/1;return x[1][y];};"
@ -624,43 +545,33 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
#[test] #[test]
fn t_extract_js_fn_outside_vars() { fn t_extract_js_fn_outside_vars() {
let base_js = "let a1 = 42;foo();var b1=11;var da=77;bar();Wka = function(da){var xy=1+2+a1*b1;return xy;}"; let base_js = "let a = 42;foo();var b=11;bar();Wka = function(d){var x=1+2+a*b;return x;}";
let res = extract_js_fn(base_js, "Wka").unwrap(); let res = extract_js_fn(base_js, 0, "Wka").unwrap();
// order of variables is non-reproducible // order of variables is non-reproducible
assert!( assert!(
res == "var a1 = 42; var b1=11; var Wka = function(da){var xy=1+2+a1*b1;return xy;};" res == "var a = 42; var b=11; var Wka = function(d){var x=1+2+a*b;return x;};"
|| res == "var b1=11; var a1 = 42; var Wka = function(da){var xy=1+2+a1*b1;return xy;};", || res == "var b=11; var a = 42; var Wka = function(d){var x=1+2+a*b;return x;};",
"got {res}" "got {res}"
); );
} }
#[test] #[test]
fn t_extract_js_fn_outside_vars2() { fn t_extract_js_fn_outside_vars2() {
let base_js = "{let a1 = {v1:1,v2:2}}foo();Wka = function(d){var x=1+2+a1.v1;return x;}"; let base_js = "{let a = {v1:1,v2:2}}foo();Wka = function(d){var x=1+2+a.v1;return x;}";
let res = extract_js_fn(base_js, "Wka").unwrap(); let res = extract_js_fn(base_js, 0, "Wka").unwrap();
assert_eq!( assert_eq!(
res, res,
"var a1 = {v1:1,v2:2}; var Wka = function(d){var x=1+2+a1.v1;return x;};" "var a = {v1:1,v2:2}; var Wka = function(d){var x=1+2+a.v1;return x;};"
); );
} }
#[test] #[test]
fn t_extract_js_fn_outside_vars3() { fn t_extract_js_fn_outside_vars3() {
let base_js = "Wka = function(d){var x=1+2+a1[0];return x;};let a1=[1,2,3]"; let base_js = "Wka = function(d){var x=1+2+a[0];return x;};let a=[1,2,3]";
let res = extract_js_fn(base_js, "Wka").unwrap(); let res = extract_js_fn(base_js, 0, "Wka").unwrap();
assert_eq!( assert_eq!(
res, res,
"var a1=[1,2,3]; var Wka = function(d){var x=1+2+a1[0];return x;};" "var a=[1,2,3]; var Wka = function(d){var x=1+2+a[0];return x;};"
);
}
#[test]
fn t_extract_js_fn_outside_vars4() {
let base_js = "let a0=123456;let a1=function(a){return a};let Wka = function(d){var x=1+2+a1();return x;}";
let res = extract_js_fn(base_js, "Wka").unwrap();
assert_eq!(
res,
"var a1=function(a){return a}; var Wka = function(d){var x=1+2+a1();return x;};"
); );
} }
@ -714,86 +625,65 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
} }
// Test cases from https://github.com/yt-dlp/yt-dlp/blob/master/test/test_youtube_signature.py // Test cases from https://github.com/yt-dlp/yt-dlp/blob/master/test/test_youtube_signature.py
#[rstest]
#[case("6ed0d907", "AOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL2QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0")]
#[case("3bb1f723", "MyOSJXtKI3m-uME_jv7-pT12gOFC02RFkGoqWpzE0Cs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA")]
#[case("2f1832d2", "0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xxAj7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJ2OySqa0q")]
#[tokio::test] #[tokio::test]
#[traced_test] #[traced_test]
async fn sig_tests() { async fn sig_tests(#[case] js_hash: &str, #[case] exp_sig: &str) {
let cases = [
("6ed0d907", "AOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL2QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0"),
("3bb1f723", "MyOSJXtKI3m-uME_jv7-pT12gOFC02RFkGoqWpzE0Cs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA"),
("2f1832d2", "0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xxAj7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJ2OySqa0q"),
("643afba4", "AAOAOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7vgpDL0QwbdV06sCIEzpWqMGkFR20CFOS21Tp-7vj_EMu-m37KtXJoOy1"),
("363db69b", "0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpz2ICs6EVdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA"),
];
for (js_hash, exp_sig) in cases {
let span = tracing::span!(tracing::Level::ERROR, "sig_test", js_hash);
let _enter = span.enter();
let (js_url, js_path) = player_js_file(js_hash).await; let (js_url, js_path) = player_js_file(js_hash).await;
let player_js = std::fs::read_to_string(js_path).unwrap(); let player_js = std::fs::read_to_string(js_path).unwrap();
let deobf_data = DeobfData::extract_fns(&js_url, &player_js).unwrap(); let deobf_data = DeobfData::extract_fns(&js_url, &player_js).unwrap();
let deobf = Deobfuscator::new(&deobf_data).unwrap(); let deobf = Deobfuscator::new(&deobf_data).unwrap();
let deobf_sig = deobf.deobfuscate_sig("2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA").unwrap(); let deobf_sig = deobf.deobfuscate_sig("2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA").unwrap();
assert_eq!(deobf_sig, exp_sig, "[{js_hash}]"); assert_eq!(deobf_sig, exp_sig, "js: {js_hash}");
}
} }
#[rstest]
#[case("7862ca1f", "X_LCxVDjAavgE5t", "yxJ1dM6iz5ogUg")]
#[case("9216d1f7", "SLp9F5bwjAdhE9F-", "gWnb9IK2DJ8Q1w")]
#[case("f8cb7a3b", "oBo2h5euWy6osrUt", "ivXHpm7qJjJN")]
#[case("2dfe380c", "oBo2h5euWy6osrUt", "3DIBbn3qdQ")]
#[case("f1ca6900", "cu3wyu6LQn2hse", "jvxetvmlI9AN9Q")]
#[case("8040e515", "wvOFaY-yjgDuIEg5", "HkfBFDHmgw4rsw")]
#[case("e06dea74", "AiuodmaDDYw8d3y4bf", "ankd8eza2T6Qmw")]
#[case("5dd88d1d", "kSxKFLeqzv_ZyHSAt", "n8gS8oRlHOxPFA")]
#[case("324f67b9", "xdftNy7dh9QGnhW", "22qLGxrmX8F1rA")]
#[case("4c3f79c5", "TDCstCG66tEAO5pR9o", "dbxNtZ14c-yWyw")]
#[case("c81bbb4a", "gre3EcLurNY2vqp94", "Z9DfGxWP115WTg")]
#[case("1f7d5369", "batNX7sYqIJdkJ", "IhOkL_zxbkOZBw")]
#[case("009f1d77", "5dwFHw8aFWQUQtffRq", "audescmLUzI3jw")]
#[case("dc0c6770", "5EHDMgYLV6HPGk_Mu-kk", "n9lUJLHbxUI0GQ")]
#[case("113ca41c", "cgYl-tlYkhjT7A", "hI7BBr2zUgcmMg")]
#[case("c57c113c", "M92UUMHa8PdvPd3wyM", "3hPqLJsiNZx7yA")]
#[case("5a3b6271", "B2j7f_UPT4rfje85Lu_e", "m5DmNymaGQ5RdQ")]
#[case("7a062b77", "NRcE3y3mVtm_cV-W", "VbsCYUATvqlt5w")]
#[case("dac945fd", "o8BkRxXhuYsBCWi6RplPdP", "3Lx32v_hmzTm6A")]
#[case("6f20102c", "lE8DhoDmKqnmJJ", "pJTTX6XyJP2BYw")]
#[case("cfa9e7cb", "aCi3iElgd2kq0bxVbQ", "QX1y8jGb2IbZ0w")]
#[case("8c7583ff", "1wWCVpRR96eAmMI87L", "KSkWAVv1ZQxC3A")]
#[case("b7910ca8", "_hXMCwMt9qE310D", "LoZMgkkofRMCZQ")]
#[case("590f65a6", "1tm7-g_A9zsI8_Lay_", "xI4Vem4Put_rOg")]
#[case("b22ef6e7", "b6HcntHGkvBLk_FRf", "kNPW6A7FyP2l8A")]
#[case("3400486c", "lL46g3XifCKUZn1Xfw", "z767lhet6V2Skl")]
#[case("20dfca59", "-fLCxedkAk4LUTK2", "O8kfRq1y1eyHGw")]
#[case("b12cc44b", "keLa5R2U00sR9SQK", "N1OGyujjEwMnLw")]
#[case("3bb1f723", "gK15nzVyaXE9RsMP3z", "ZFFWFLPWx9DEgQ")]
#[case("2f1832d2", "YWt1qdbe8SAfkoPHW5d", "RrRjWQOJmBiP")]
#[case("19d2ae9d", "YWt1qdbe8SAfkoPHW5d", "CS6dVTYzpZrAZ5TD")]
#[tokio::test] #[tokio::test]
#[traced_test] #[traced_test]
async fn nsig_tests() { async fn nsig_tests(#[case] js_hash: &str, #[case] nsig_in: &str, #[case] expect: &str) {
let cases = [
("7862ca1f", "X_LCxVDjAavgE5t", "yxJ1dM6iz5ogUg"),
("9216d1f7", "SLp9F5bwjAdhE9F-", "gWnb9IK2DJ8Q1w"),
("f8cb7a3b", "oBo2h5euWy6osrUt", "ivXHpm7qJjJN"),
("2dfe380c", "oBo2h5euWy6osrUt", "3DIBbn3qdQ"),
("f1ca6900", "cu3wyu6LQn2hse", "jvxetvmlI9AN9Q"),
("8040e515", "wvOFaY-yjgDuIEg5", "HkfBFDHmgw4rsw"),
("e06dea74", "AiuodmaDDYw8d3y4bf", "ankd8eza2T6Qmw"),
("5dd88d1d", "kSxKFLeqzv_ZyHSAt", "n8gS8oRlHOxPFA"),
("324f67b9", "xdftNy7dh9QGnhW", "22qLGxrmX8F1rA"),
("4c3f79c5", "TDCstCG66tEAO5pR9o", "dbxNtZ14c-yWyw"),
("c81bbb4a", "gre3EcLurNY2vqp94", "Z9DfGxWP115WTg"),
("1f7d5369", "batNX7sYqIJdkJ", "IhOkL_zxbkOZBw"),
("009f1d77", "5dwFHw8aFWQUQtffRq", "audescmLUzI3jw"),
("dc0c6770", "5EHDMgYLV6HPGk_Mu-kk", "n9lUJLHbxUI0GQ"),
("113ca41c", "cgYl-tlYkhjT7A", "hI7BBr2zUgcmMg"),
("c57c113c", "M92UUMHa8PdvPd3wyM", "3hPqLJsiNZx7yA"),
("5a3b6271", "B2j7f_UPT4rfje85Lu_e", "m5DmNymaGQ5RdQ"),
("7a062b77", "NRcE3y3mVtm_cV-W", "VbsCYUATvqlt5w"),
("dac945fd", "o8BkRxXhuYsBCWi6RplPdP", "3Lx32v_hmzTm6A"),
("6f20102c", "lE8DhoDmKqnmJJ", "pJTTX6XyJP2BYw"),
("cfa9e7cb", "aCi3iElgd2kq0bxVbQ", "QX1y8jGb2IbZ0w"),
("8c7583ff", "1wWCVpRR96eAmMI87L", "KSkWAVv1ZQxC3A"),
("b7910ca8", "_hXMCwMt9qE310D", "LoZMgkkofRMCZQ"),
("590f65a6", "1tm7-g_A9zsI8_Lay_", "xI4Vem4Put_rOg"),
("b22ef6e7", "b6HcntHGkvBLk_FRf", "kNPW6A7FyP2l8A"),
("3400486c", "lL46g3XifCKUZn1Xfw", "z767lhet6V2Skl"),
("20dfca59", "-fLCxedkAk4LUTK2", "O8kfRq1y1eyHGw"),
("b12cc44b", "keLa5R2U00sR9SQK", "N1OGyujjEwMnLw"),
("3bb1f723", "gK15nzVyaXE9RsMP3z", "ZFFWFLPWx9DEgQ"),
("2f1832d2", "YWt1qdbe8SAfkoPHW5d", "RrRjWQOJmBiP"),
("19d2ae9d", "YWt1qdbe8SAfkoPHW5d", "CS6dVTYzpZrAZ5TD"),
("e7567ecf", "Sy4aDGc0VpYRR9ew_", "5UPOT1VhoZxNLQ"),
("d50f54ef", "Ha7507LzRmH3Utygtj", "XFTb2HoeOE5MHg"),
("074a8365", "Ha7507LzRmH3Utygtj", "ufTsrE0IVYrkl8v"),
("643afba4", "N5uAlLqm0eg1GyHO", "dCBQOejdq5s-ww"),
("69f581a5", "-qIP447rVlTTwaZjY", "KNcGOksBAvwqQg"),
("363db69b", "eWYu5d5YeY_4LyEDc", "XJQqf-N7Xra3gg"),
];
for (js_hash, nsig_in, exp_nsig) in cases {
let span = tracing::span!(tracing::Level::ERROR, "nsig_test", js_hash);
let _enter = span.enter();
let (js_url, js_path) = player_js_file(js_hash).await; let (js_url, js_path) = player_js_file(js_hash).await;
let player_js = std::fs::read_to_string(js_path).unwrap(); let player_js = std::fs::read_to_string(js_path).unwrap();
let deobf_data = DeobfData::extract_fns(&js_url, &player_js).expect(js_hash); let deobf_data = DeobfData::extract_fns(&js_url, &player_js).unwrap();
let deobf = Deobfuscator::new(&deobf_data).expect(js_hash); let deobf = Deobfuscator::new(&deobf_data).unwrap();
let deobf_nsig = deobf.deobfuscate_nsig(nsig_in).expect(js_hash); let deobf_nsig = deobf.deobfuscate_nsig(nsig_in).unwrap();
assert_eq!(deobf_nsig, exp_nsig, "[{js_hash}]"); assert_eq!(deobf_nsig, expect, "js: {js_hash}");
}
} }
#[tokio::test] #[tokio::test]

View file

@ -151,8 +151,6 @@ pub enum AuthError {
} }
pub(crate) mod internal { pub(crate) mod internal {
use std::borrow::Cow;
use super::{Error, ExtractionError}; use super::{Error, ExtractionError};
/// Error that occurred during the initialization /// Error that occurred during the initialization
@ -170,7 +168,7 @@ pub(crate) mod internal {
Extraction(&'static str), Extraction(&'static str),
/// Unspecified error /// Unspecified error
#[error("error: {0}")] #[error("error: {0}")]
Other(Cow<'static, str>), Other(&'static str),
} }
impl From<DeobfError> for Error { impl From<DeobfError> for Error {

View file

@ -75,10 +75,10 @@ pub fn get_cg_from_fancy_regexes(regexes: &[&str], text: &str, cg_name: &str) ->
/// Generate a random string with given length and byte charset. /// Generate a random string with given length and byte charset.
fn random_string(charset: &[u8], length: usize) -> String { fn random_string(charset: &[u8], length: usize) -> String {
let mut result = String::with_capacity(length); let mut result = String::with_capacity(length);
let mut rng = rand::rng(); let mut rng = rand::thread_rng();
for _ in 0..length { for _ in 0..length {
result.push(char::from(charset[rng.random_range(0..charset.len())])); result.push(char::from(charset[rng.gen_range(0..charset.len())]));
} }
result result
@ -90,14 +90,14 @@ pub fn generate_content_playback_nonce() -> String {
} }
pub fn random_uuid() -> String { pub fn random_uuid() -> String {
let mut rng = rand::rng(); let mut rng = rand::thread_rng();
format!( format!(
"{:08x}-{:04x}-{:04x}-{:04x}-{:012x}", "{:08x}-{:04x}-{:04x}-{:04x}-{:012x}",
rng.random::<u32>(), rng.gen::<u32>(),
rng.random::<u16>(), rng.gen::<u16>(),
rng.random::<u16>(), rng.gen::<u16>(),
rng.random::<u16>(), rng.gen::<u16>(),
rng.random::<u64>() & 0xffff_ffff_ffff, rng.gen::<u64>() & 0xffff_ffff_ffff,
) )
} }
@ -229,7 +229,7 @@ pub fn retry_delay(
backoff_base: u32, backoff_base: u32,
) -> u32 { ) -> u32 {
let unjittered_delay = backoff_base.checked_pow(n_past_retries).unwrap_or(u32::MAX); let unjittered_delay = backoff_base.checked_pow(n_past_retries).unwrap_or(u32::MAX);
let jitter_factor = rand::rng().random_range(800..1500); let jitter_factor = rand::thread_rng().gen_range(800..1500);
let jittered_delay = unjittered_delay let jittered_delay = unjittered_delay
.checked_mul(jitter_factor) .checked_mul(jitter_factor)
.unwrap_or(u32::MAX); .unwrap_or(u32::MAX);

View file

@ -148,8 +148,8 @@ impl VisitorDataCache {
{ {
let vds = self.inner.visitor_data.read().unwrap(); let vds = self.inner.visitor_data.read().unwrap();
if !vds.is_empty() { if !vds.is_empty() {
let mut rng = rand::rng(); let mut rng = rand::thread_rng();
let vd = vds[rng.random_range(0..vds.len())].to_owned(); let vd = vds[rng.gen_range(0..vds.len())].to_owned();
tracing::debug!("visitor data {vd} picked from cache"); tracing::debug!("visitor data {vd} picked from cache");
return Ok(vd); return Ok(vd);
} }

File diff suppressed because it is too large Load diff

View file

@ -42,8 +42,8 @@ MusicArtist(
by_va: false, by_va: false,
), ),
AlbumItem( AlbumItem(
id: "MPREb_HrCgErOdgCv", id: "MPREb_6PEkIQE7sWY",
name: "Freiheit", name: "An deiner Seite (Online Version)",
cover: "[cover]", cover: "[cover]",
artists: [ artists: [
ArtistId( ArtistId(
@ -52,8 +52,8 @@ MusicArtist(
), ),
], ],
artist_id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"), artist_id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
album_type: album, album_type: ep,
year: Some(2004), year: Some(2008),
by_va: false, by_va: false,
), ),
AlbumItem( AlbumItem(
@ -87,8 +87,8 @@ MusicArtist(
by_va: false, by_va: false,
), ),
AlbumItem( AlbumItem(
id: "MPREb_Oq0WKqNwSVY", id: "MPREb_QEClJsuO9xM",
name: "Das 2. Gebot", name: "So wie Du warst",
cover: "[cover]", cover: "[cover]",
artists: [ artists: [
ArtistId( ArtistId(
@ -97,8 +97,8 @@ MusicArtist(
), ),
], ],
artist_id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"), artist_id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
album_type: album, album_type: single,
year: Some(2003), year: Some(2012),
by_va: false, by_va: false,
), ),
AlbumItem( AlbumItem(
@ -251,21 +251,6 @@ MusicArtist(
year: Some(2015), year: Some(2015),
by_va: false, by_va: false,
), ),
AlbumItem(
id: "MPREb_ohcGTZrqKPZ",
name: "Zelluloid",
cover: "[cover]",
artists: [
ArtistId(
id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
name: "Unheilig",
),
],
artist_id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
album_type: album,
year: Some(2004),
by_va: false,
),
AlbumItem( AlbumItem(
id: "MPREb_pWpeXxATZYb", id: "MPREb_pWpeXxATZYb",
name: "Wir sind alle wie eins", name: "Wir sind alle wie eins",
@ -281,6 +266,21 @@ MusicArtist(
year: Some(2014), year: Some(2014),
by_va: false, by_va: false,
), ),
AlbumItem(
id: "MPREb_rHhaDLqalbT",
name: "Winter (EP)",
cover: "[cover]",
artists: [
ArtistId(
id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
name: "Unheilig",
),
],
artist_id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
album_type: ep,
year: Some(2010),
by_va: false,
),
AlbumItem( AlbumItem(
id: "MPREb_saXgTKNPaSu", id: "MPREb_saXgTKNPaSu",
name: "Zeit zu gehen", name: "Zeit zu gehen",

View file

@ -30,7 +30,7 @@ use rustypipe::validate;
#[case::desktop(ClientType::Desktop)] #[case::desktop(ClientType::Desktop)]
#[case::tv(ClientType::Tv)] #[case::tv(ClientType::Tv)]
#[case::mobile(ClientType::Mobile)] #[case::mobile(ClientType::Mobile)]
// #[case::android(ClientType::Android)] Removed since it requires Android device attestation #[case::android(ClientType::Android)]
#[case::ios(ClientType::Ios)] #[case::ios(ClientType::Ios)]
#[tokio::test] #[tokio::test]
async fn get_player_from_client(#[case] client_type: ClientType, rp: RustyPipe) { async fn get_player_from_client(#[case] client_type: ClientType, rp: RustyPipe) {
@ -2119,13 +2119,11 @@ async fn music_search_artists(rp: RustyPipe, unlocalized: bool) {
#[rstest] #[rstest]
#[tokio::test] #[tokio::test]
async fn music_search_artists_cont(rp: RustyPipe) { async fn music_search_artists_cont(rp: RustyPipe) {
let res = rp.query().music_search_artists("girls").await.unwrap(); let res = rp.query().music_search_artists("boys").await.unwrap();
assert_eq!(res.corrected_query, None); assert_eq!(res.corrected_query, None);
if !res.items.is_exhausted() {
assert_next(res.items, rp.query(), 15, 2, true).await; assert_next(res.items, rp.query(), 15, 2, true).await;
} }
}
#[rstest] #[rstest]
#[tokio::test] #[tokio::test]
@ -2596,7 +2594,7 @@ async fn music_genres(rp: RustyPipe, unlocalized: bool) {
} }
#[rstest] #[rstest]
#[case::party("ggMPOg1uX2w1aW1CRDFTSUNo", "Party")] #[case::chill("ggMPOg1uX1JOQWZFeDByc2Jm", "Chill")]
#[case::pop("ggMPOg1uX1lMbVZmbzl6NlJ3", "Pop")] #[case::pop("ggMPOg1uX1lMbVZmbzl6NlJ3", "Pop")]
#[tokio::test] #[tokio::test]
async fn music_genre(#[case] id: &str, #[case] name: &str, rp: RustyPipe, unlocalized: bool) { async fn music_genre(#[case] id: &str, #[case] name: &str, rp: RustyPipe, unlocalized: bool) {
@ -2640,7 +2638,7 @@ async fn music_genre(#[case] id: &str, #[case] name: &str, rp: RustyPipe, unloca
let subgenres = check_music_genre(genre, id, name, unlocalized); let subgenres = check_music_genre(genre, id, name, unlocalized);
if name == "Party" { if name == "Chill" {
assert_gte(subgenres.len(), 2, "subgenres"); assert_gte(subgenres.len(), 2, "subgenres");
} }
@ -2931,11 +2929,7 @@ async fn assert_next<T: FromYtItem, Q: AsRef<RustyPipeQuery>>(
} }
for i in 0..n_pages { for i in 0..n_pages {
match p.next(query).await.unwrap() { p = p.next(query).await.unwrap().expect("paginator exhausted");
Some(np) => p = np,
None => panic!("paginator exhausted after {i} pages"),
}
assert_gte( assert_gte(
p.items.len(), p.items.len(),
min_items, min_items,