Compare commits

...

25 commits

Author SHA1 Message Date
c6abd89087
test: fix tests 2025-04-18 16:38:44 +02:00
703f350b6b
chore(release): release rustypipe v0.11.3 2025-04-03 13:39:28 +02:00
af415ddf8f chore(deps): update rust crate rand to 0.9.0 2025-04-03 11:08:18 +00:00
daf3d035be
fix: handle music artist not found 2025-03-31 18:11:14 +02:00
187bf1c9a0
fix: switch client if no adaptive stream URLs were returned 2025-03-26 02:44:08 +01:00
ea80717f69
fix: handle music playlist/album not found 2025-03-26 02:35:03 +01:00
939a7aea61
fix: deobfuscator: handle global functions as well 2025-03-26 02:12:18 +01:00
47bea4eed2
test: update music_artist_basic snapshot 2025-03-26 01:38:35 +01:00
189ba81a42
fix: extractor: small simplification 2025-03-26 01:38:12 +01:00
ac44e95a88
fix: extractor: global variable extraction fixed 2025-03-26 01:20:35 +01:00
23c8775326
chore(release): release rustypipe v0.11.2 2025-03-24 01:50:53 +01:00
07db7b1166
fix: handle player returning no adaptive stream URLs 2025-03-24 01:28:07 +01:00
4ce6746be5
fix: extract deobf data with global strings variable 2025-03-24 01:12:01 +01:00
e8acbfbbcf
fix: A/B test 22: commandExecutorCommand for playlist continuations 2025-03-16 19:45:14 +01:00
fcf27aa3b2
chore(release): release rustypipe-cli v0.7.2 2025-03-16 18:20:32 +01:00
64ed3b14e3
chore(release): release rustypipe v0.11.1 2025-03-16 18:13:55 +01:00
63a6f50a8b
fix: always skip failed clients 2025-03-16 16:51:43 +01:00
8342caeb0f
fix: desktop client: generate PO token from user_syncid when authenticated 2025-03-16 01:56:29 +01:00
c04b60604d
fix: simplify get_player_from_clients logic 2025-03-16 01:24:54 +01:00
2f18efa1cf
fix: log download URL 2025-03-16 01:21:29 +01:00
b8f61c9bae
test: skip android client test 2025-03-04 22:50:33 +01:00
9ed1306f3a
chore(deps): update rust crate rstest to 0.25.0 2025-03-04 22:48:10 +01:00
6d481c16d0
update smartcrop2 to v0.4.0, remove black borders from album covers 2025-03-04 22:38:01 +01:00
144a670da1
chore(release): release rustypipe-cli v0.7.1 2025-02-26 19:48:12 +01:00
035c07f170
chore(deps): update rustypipe to 0.11.0 2025-02-26 19:47:42 +01:00
30 changed files with 34293 additions and 319 deletions

View file

@ -3,6 +3,44 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
## [v0.11.3](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.11.2..rustypipe/v0.11.3) - 2025-04-03
### 🐛 Bug Fixes
- Deobfuscator: global variable extraction fixed - ([ac44e95](https://codeberg.org/ThetaDev/rustypipe/commit/ac44e95a88d95f9d2d1ec672f86ca9d31d6991b9))
- Deobfuscator: small simplification - ([189ba81](https://codeberg.org/ThetaDev/rustypipe/commit/189ba81a42e6c09f6af4d2768c449c22b864101e))
- Deobfuscator: handle global functions as well - ([939a7ae](https://codeberg.org/ThetaDev/rustypipe/commit/939a7aea61a3eee4c1e67bfbfc835f0ce3934171))
- Handle music playlist/album not found - ([ea80717](https://codeberg.org/ThetaDev/rustypipe/commit/ea80717f692b2c45b5063c362c9fa8ebca5a3471))
- Switch client if no adaptive stream URLs were returned - ([187bf1c](https://codeberg.org/ThetaDev/rustypipe/commit/187bf1c9a0e846bff205e0d71a19c5a1ce7b1943))
- Handle music artist not found - ([daf3d03](https://codeberg.org/ThetaDev/rustypipe/commit/daf3d035be38b59aef1ae205ac91c2bbdda2fe66))
### ⚙️ Miscellaneous Tasks
- *(deps)* Update rust crate rand to 0.9.0 - ([af415dd](https://codeberg.org/ThetaDev/rustypipe/commit/af415ddf8f94f00edb918f271d8e6336503e9faf))
## [v0.11.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.11.1..rustypipe/v0.11.2) - 2025-03-24
### 🐛 Bug Fixes
- A/B test 22: commandExecutorCommand for playlist continuations - ([e8acbfb](https://codeberg.org/ThetaDev/rustypipe/commit/e8acbfbbcf5d31b5ac34410ddf334e5534e3762f))
- Extract deobf data with global strings variable - ([4ce6746](https://codeberg.org/ThetaDev/rustypipe/commit/4ce6746be538564e79f7e3c67d7a91aaa53f48ea))
- Handle player returning no adaptive stream URLs - ([07db7b1](https://codeberg.org/ThetaDev/rustypipe/commit/07db7b1166e912e1554f98f2ae20c2c356fed38f))
## [v0.11.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.11.0..rustypipe/v0.11.1) - 2025-03-16
### 🐛 Bug Fixes
- Simplify get_player_from_clients logic - ([c04b606](https://codeberg.org/ThetaDev/rustypipe/commit/c04b60604d2628bf8f0e3de453c243adbb966e57))
- Desktop client: generate PO token from user_syncid when authenticated - ([8342cae](https://codeberg.org/ThetaDev/rustypipe/commit/8342caeb0f566a38060a6ec69f3ca65b9a2afcd6))
- Always skip failed clients - ([63a6f50](https://codeberg.org/ThetaDev/rustypipe/commit/63a6f50a8b5ad6bb984282335c1481ae3cd2fe83))
### ⚙️ Miscellaneous Tasks
- *(deps)* Update rust crate rstest to 0.25.0 - ([9ed1306](https://codeberg.org/ThetaDev/rustypipe/commit/9ed1306f3aaeb993c409997ddfbc47499e4f4d22))
## [v0.11.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.10.0..rustypipe/v0.11.0) - 2025-02-26 ## [v0.11.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe/v0.10.0..rustypipe/v0.11.0) - 2025-02-26
### 🚀 Features ### 🚀 Features

View file

@ -1,6 +1,6 @@
[package] [package]
name = "rustypipe" name = "rustypipe"
version = "0.11.0" version = "0.11.3"
rust-version = "1.67.1" rust-version = "1.67.1"
edition.workspace = true edition.workspace = true
authors.workspace = true authors.workspace = true
@ -40,7 +40,7 @@ serde_with = { version = "3.0.0", default-features = false, features = [
] } ] }
serde_plain = "1.0.0" serde_plain = "1.0.0"
sha1 = "0.10.0" sha1 = "0.10.0"
rand = "0.8.0" rand = "0.9.0"
time = { version = "0.3.37", features = [ time = { version = "0.3.37", features = [
"macros", "macros",
"serde-human-readable", "serde-human-readable",
@ -67,15 +67,15 @@ dirs = "6.0.0"
filenamify = "0.1.0" filenamify = "0.1.0"
# Testing # Testing
rstest = "0.24.0" rstest = "0.25.0"
tokio-test = "0.4.2" tokio-test = "0.4.2"
insta = { version = "1.17.1", features = ["ron", "redactions"] } insta = { version = "1.17.1", features = ["ron", "redactions"] }
path_macro = "1.0.0" path_macro = "1.0.0"
tracing-test = "0.2.5" tracing-test = "0.2.5"
# Included crates # Included crates
rustypipe = { path = ".", version = "0.11.0", default-features = false } rustypipe = { path = ".", version = "0.11.3", default-features = false }
rustypipe-downloader = { path = "./downloader", version = "0.3.0", default-features = false, features = [ rustypipe-downloader = { path = "./downloader", version = "0.3.1", default-features = false, features = [
"indicatif", "indicatif",
"audiotag", "audiotag",
] } ] }

View file

@ -3,6 +3,22 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
## [v0.7.2](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.7.1..rustypipe-cli/v0.7.2) - 2025-03-16
### ⚙️ Miscellaneous Tasks
- *(deps)* Update rustypipe to 0.11.1
- *(deps)* Update rustypipe-downloader to 0.3.1
- *(deps)* Update rust crate rstest to 0.25.0 - ([9ed1306](https://codeberg.org/ThetaDev/rustypipe/commit/9ed1306f3aaeb993c409997ddfbc47499e4f4d22))
## [v0.7.1](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.7.0..rustypipe-cli/v0.7.1) - 2025-02-26
### ⚙️ Miscellaneous Tasks
- *(deps)* Update rustypipe to 0.11.0 - ([035c07f](https://codeberg.org/ThetaDev/rustypipe/commit/035c07f170aa293bcc626f27998c2b2b28660881))
## [v0.7.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.6.0..rustypipe-cli/v0.7.0) - 2025-02-09 ## [v0.7.0](https://codeberg.org/ThetaDev/rustypipe/compare/rustypipe-cli/v0.6.0..rustypipe-cli/v0.7.0) - 2025-02-09
### 🚀 Features ### 🚀 Features

View file

@ -1,6 +1,6 @@
[package] [package]
name = "rustypipe-cli" name = "rustypipe-cli"
version = "0.7.0" version = "0.7.2"
rust-version = "1.70.0" rust-version = "1.70.0"
edition.workspace = true edition.workspace = true
authors.workspace = true authors.workspace = true

View file

@ -41,10 +41,15 @@ pub enum ABTest {
MusicAlbumGroupsReordered = 19, MusicAlbumGroupsReordered = 19,
MusicContinuationItemRenderer = 20, MusicContinuationItemRenderer = 20,
AlbumRecommends = 21, AlbumRecommends = 21,
CommandExecutorCommand = 22,
} }
/// List of active A/B tests that are run when none is manually specified /// List of active A/B tests that are run when none is manually specified
const TESTS_TO_RUN: &[ABTest] = &[ABTest::MusicAlbumGroupsReordered, ABTest::AlbumRecommends]; const TESTS_TO_RUN: &[ABTest] = &[
ABTest::MusicAlbumGroupsReordered,
ABTest::AlbumRecommends,
ABTest::CommandExecutorCommand,
];
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize, Deserialize)]
pub struct ABTestRes { pub struct ABTestRes {
@ -120,6 +125,7 @@ pub async fn run_test(
music_continuation_item_renderer(&query).await music_continuation_item_renderer(&query).await
} }
ABTest::AlbumRecommends => album_recommends(&query).await, ABTest::AlbumRecommends => album_recommends(&query).await,
ABTest::CommandExecutorCommand => command_executor_command(&query).await,
} }
.unwrap(); .unwrap();
pb.inc(1); pb.inc(1);
@ -457,3 +463,18 @@ pub async fn album_recommends(rp: &RustyPipeQuery) -> Result<bool> {
.await?; .await?;
Ok(res.contains("\"musicCarouselShelfRenderer\"")) Ok(res.contains("\"musicCarouselShelfRenderer\""))
} }
pub async fn command_executor_command(rp: &RustyPipeQuery) -> Result<bool> {
let id = "VLPLbZIPy20-1pN7mqjckepWF78ndb6ci_qi";
let res = rp
.raw(
ClientType::Desktop,
"browse",
&QBrowse {
browse_id: id,
params: None,
},
)
.await?;
Ok(res.contains("\"commandExecutorCommand\""))
}

View file

@ -51,7 +51,7 @@ image = { version = "0.25.0", optional = true, default-features = false, feature
"jpeg", "jpeg",
"webp", "webp",
] } ] }
smartcrop2 = { version = "0.3.1", optional = true } smartcrop2 = { version = "0.4.0", optional = true }
[dev-dependencies] [dev-dependencies]
path_macro.workspace = true path_macro.workspace = true

View file

@ -1033,7 +1033,11 @@ impl DownloadQuery {
image::load_from_memory(&img_bts)? image::load_from_memory(&img_bts)?
}; };
let crop = smartcrop::find_best_crop(&img, NonZeroU32::MIN, NonZeroU32::MIN) let crop = smartcrop::find_best_crop_no_borders(
&img,
NonZeroU32::MIN,
NonZeroU32::MIN,
)
.map_err(|e| DownloadError::AudioTag(format!("image crop: {e}").into()))? .map_err(|e| DownloadError::AudioTag(format!("image crop: {e}").into()))?
.crop; .crop;
img = img.crop_imm(crop.x, crop.y, crop.width, crop.height); img = img.crop_imm(crop.x, crop.y, crop.width, crop.height);
@ -1063,8 +1067,8 @@ impl DownloadQuery {
} }
fn get_download_range(offset: u64, size: Option<u64>) -> Range<u64> { fn get_download_range(offset: u64, size: Option<u64>) -> Range<u64> {
let mut rng = rand::thread_rng(); let mut rng = rand::rng();
let chunk_size = rng.gen_range(CHUNK_SIZE_MIN..CHUNK_SIZE_MAX); let chunk_size = rng.random_range(CHUNK_SIZE_MIN..CHUNK_SIZE_MAX);
let mut chunk_end = offset + chunk_size; let mut chunk_end = offset + chunk_size;
if let Some(size) = size { if let Some(size) = size {
@ -1197,6 +1201,8 @@ async fn download_single_file(
} }
} }
tracing::debug!("downloading {} to {}", url, output.to_string_lossy());
let mut file = fs::OpenOptions::new() let mut file = fs::OpenOptions::new()
.append(true) .append(true)
.create(true) .create(true)

View file

@ -1030,7 +1030,7 @@ commandContext missing).
- **Encountered on:** 13.01.2025 - **Encountered on:** 13.01.2025
- **Impact:** 🟢 Low - **Impact:** 🟢 Low
- **Endpoint:** browse (YTM) - **Endpoint:** browse (YTM)
- **Status:** Common (10%) - **Status:** Frequent (59%)
YouTube Music used to group artist albums into 2 rows: "Albums" and "Singles". YouTube Music used to group artist albums into 2 rows: "Albums" and "Singles".
@ -1067,3 +1067,37 @@ pages. The difficulty is distinguishing them reliably for parsing the album vari
The current solution is adding the "Other versions" title in all languages to the The current solution is adding the "Other versions" title in all languages to the
dictionary and comparing it. dictionary and comparing it.
## [22] commandExecutorCommand for continuations
- **Encountered on:** 16.03.2025
- **Impact:** 🟢 Low
- **Endpoint:** browse (YTM)
- **Status:** Experimental (1%)
YouTube playlists may use a commandExecutorCommand which holds a list of commands: the
`continuationCommand` that needs to be extracted as well as a `playlistVotingRefreshPopupCommand`.
```json
{
"continuationItemRenderer": {
"continuationEndpoint": {
"commandExecutorCommand": {
"commands": [
{
"playlistVotingRefreshPopupCommand": {
"command": {}
}
},
{
"continuationCommand": {
"request": "CONTINUATION_REQUEST_TYPE_BROWSE",
"token": "4qmFsgKBARIkVkxQTGJaSVB5MjAtMXBON21xamNrZXBXRjc4bmRiNmNpX3FpGjRDQUY2SGxCVU9rTklTV2xGUkVreVVtdEZOVTVFU1hsU2FrWkRVa1JKZWs1NldRJTNEJTNEmgIiUExiWklQeTIwLTFwTjdtcWpja2VwV0Y3OG5kYjZjaV9xaQ%3D%3D"
}
}
]
}
}
}
}
```

View file

@ -293,8 +293,10 @@ struct OauthToken {
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
struct AuthCookie { struct AuthCookie {
cookie: String, cookie: String,
#[serde(alias = "account_syncid", skip_serializing_if = "Option::is_none")]
channel_syncid: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
account_syncid: Option<String>, user_syncid: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
session_index: Option<String>, session_index: Option<String>,
} }
@ -319,8 +321,9 @@ impl AuthCookie {
fn new(cookie: String) -> Self { fn new(cookie: String) -> Self {
Self { Self {
cookie, cookie,
account_syncid: None, channel_syncid: None,
session_index: None, session_index: None,
user_syncid: None,
} }
} }
} }
@ -1591,6 +1594,17 @@ impl RustyPipe {
.ok_or(Error::Auth(AuthError::NoLogin)) .ok_or(Error::Auth(AuthError::NoLogin))
} }
fn user_auth_datasync_id(&self) -> Result<String, Error> {
self.inner
.cache
.auth_cookie
.read()
.unwrap()
.as_ref()
.and_then(|c| c.user_syncid.as_ref().map(|id| id.to_owned()))
.ok_or(Error::Auth(AuthError::NoLogin))
}
/// Set the user authentication cookie /// Set the user authentication cookie
/// ///
/// The cookie is used for authenticated requests with browser-based clients /// The cookie is used for authenticated requests with browser-based clients
@ -1685,17 +1699,17 @@ impl RustyPipe {
))?; ))?;
// datasyncid is of the form "channel_syncid||user_syncid" for secondary channel // datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
// and just "user_syncid||" for primary channel. We only want the channel_syncid // and just "user_syncid||" for primary channel.
let (channel_syncid, user_syncid) = let (p1, p2) =
datasync_id datasync_id
.split_once("||") .split_once("||")
.ok_or(Error::Extraction(ExtractionError::InvalidData( .ok_or(Error::Extraction(ExtractionError::InvalidData(
"datasyncId does not contain || seperator".into(), "datasyncId does not contain || seperator".into(),
)))?; )))?;
auth_cookie.account_syncid = if user_syncid.is_empty() { (auth_cookie.channel_syncid, auth_cookie.user_syncid) = if p2.is_empty() {
None (None, Some(p1.to_owned()))
} else { } else {
Some(channel_syncid.to_owned()) (Some(p1.to_owned()), Some(p2.to_owned()))
}; };
auth_cookie.session_index = Some( auth_cookie.session_index = Some(
@ -2129,7 +2143,7 @@ impl RustyPipeQuery {
if let Some(session_index) = auth_cookie.session_index { if let Some(session_index) = auth_cookie.session_index {
r = r.header("X-Goog-AuthUser", session_index); r = r.header("X-Goog-AuthUser", session_index);
} }
if let Some(account_syncid) = auth_cookie.account_syncid { if let Some(account_syncid) = auth_cookie.channel_syncid {
r = r.header("X-Goog-PageId", account_syncid); r = r.header("X-Goog-PageId", account_syncid);
} }
cookie = Some(auth_cookie.cookie); cookie = Some(auth_cookie.cookie);

View file

@ -154,7 +154,24 @@ fn map_artist_page(
ctx: &MapRespCtx<'_>, ctx: &MapRespCtx<'_>,
skip_extendables: bool, skip_extendables: bool,
) -> Result<MapResult<(MusicArtist, bool)>, ExtractionError> { ) -> Result<MapResult<(MusicArtist, bool)>, ExtractionError> {
let header = res.header.music_immersive_header_renderer; let contents = match res.contents {
Some(c) => c,
None => {
if res.microformat.microformat_data_renderer.noindex {
return Err(ExtractionError::NotFound {
id: ctx.id.to_owned(),
msg: "no contents".into(),
});
} else {
return Err(ExtractionError::InvalidData("no contents".into()));
}
}
};
let header = res
.header
.ok_or(ExtractionError::InvalidData("no header".into()))?
.music_immersive_header_renderer;
if let Some(share) = header.share_endpoint { if let Some(share) = header.share_endpoint {
let pb = share.share_entity_endpoint.serialized_share_entity; let pb = share.share_entity_endpoint.serialized_share_entity;
@ -171,8 +188,7 @@ fn map_artist_page(
} }
} }
let sections = res let sections = contents
.contents
.single_column_browse_results_renderer .single_column_browse_results_renderer
.contents .contents
.into_iter() .into_iter()

View file

@ -151,7 +151,21 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
self, self,
ctx: &MapRespCtx<'_>, ctx: &MapRespCtx<'_>,
) -> Result<MapResult<MusicPlaylist>, ExtractionError> { ) -> Result<MapResult<MusicPlaylist>, ExtractionError> {
let (header, music_contents) = match self.contents { let contents = match self.contents {
Some(c) => c,
None => {
if self.microformat.microformat_data_renderer.noindex {
return Err(ExtractionError::NotFound {
id: ctx.id.to_owned(),
msg: "no contents".into(),
});
} else {
return Err(ExtractionError::InvalidData("no contents".into()));
}
}
};
let (header, music_contents) = match contents {
response::music_playlist::Contents::SingleColumnBrowseResultsRenderer(c) => ( response::music_playlist::Contents::SingleColumnBrowseResultsRenderer(c) => (
self.header, self.header,
c.contents c.contents
@ -338,7 +352,21 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
impl MapResponse<MusicAlbum> for response::MusicPlaylist { impl MapResponse<MusicAlbum> for response::MusicPlaylist {
fn map_response(self, ctx: &MapRespCtx<'_>) -> Result<MapResult<MusicAlbum>, ExtractionError> { fn map_response(self, ctx: &MapRespCtx<'_>) -> Result<MapResult<MusicAlbum>, ExtractionError> {
let (header, sections) = match self.contents { let contents = match self.contents {
Some(c) => c,
None => {
if self.microformat.microformat_data_renderer.noindex {
return Err(ExtractionError::NotFound {
id: ctx.id.to_owned(),
msg: "no contents".into(),
});
} else {
return Err(ExtractionError::InvalidData("no contents".into()));
}
}
};
let (header, sections) = match contents {
response::music_playlist::Contents::SingleColumnBrowseResultsRenderer(c) => ( response::music_playlist::Contents::SingleColumnBrowseResultsRenderer(c) => (
self.header, self.header,
c.contents c.contents
@ -454,10 +482,12 @@ impl MapResponse<MusicAlbum> for response::MusicPlaylist {
} }
} }
let playlist_id = self.microformat.and_then(|mf| { let playlist_id = self
mf.microformat_data_renderer .microformat
.microformat_data_renderer
.url_canonical .url_canonical
.strip_prefix("https://music.youtube.com/playlist?list=") .and_then(|x| {
x.strip_prefix("https://music.youtube.com/playlist?list=")
.map(str::to_owned) .map(str::to_owned)
}); });
let (playlist_id, artist_id) = header let (playlist_id, artist_id) = header

View file

@ -249,11 +249,9 @@ impl MapResponse<Paginator<HistoryItem<VideoItem>>> for response::Continuation {
&mut map_res, &mut map_res,
); );
} }
response::YouTubeListItem::ContinuationItemRenderer { response::YouTubeListItem::ContinuationItemRenderer(ep) => {
continuation_endpoint,
} => {
if ctoken.is_none() { if ctoken.is_none() {
ctoken = Some(continuation_endpoint.continuation_command.token); ctoken = ep.continuation_endpoint.into_token();
} }
} }
_ => {} _ => {}

View file

@ -1,6 +1,6 @@
use std::{ use std::{
borrow::Cow, borrow::Cow,
collections::{BTreeMap, HashMap}, collections::{BTreeMap, HashMap, HashSet},
fmt::Debug, fmt::Debug,
}; };
@ -104,42 +104,29 @@ impl RustyPipeQuery {
) -> Result<VideoPlayer, Error> { ) -> Result<VideoPlayer, Error> {
let video_id = video_id.as_ref(); let video_id = video_id.as_ref();
let mut last_e = None; let mut last_e = None;
let mut query = Cow::Borrowed(self);
let mut clients_iter = clients.iter().peekable(); let mut clients_iter = clients.iter().peekable();
let mut failed_clients = HashSet::new();
while let Some(client) = clients_iter.next() { while let Some(client) = clients_iter.next() {
if self.opts.auth == Some(true) && !self.auth_enabled(*client) { if query.opts.auth == Some(true) && !self.auth_enabled(*client) {
// If no client has auth enabled, return NoLogin error instead of "no clients" // If no client has auth enabled, return NoLogin error instead of "no clients"
if last_e.is_none() { if last_e.is_none() {
last_e = Some(Error::Auth(AuthError::NoLogin)); last_e = Some(Error::Auth(AuthError::NoLogin));
} }
continue; continue;
} }
if failed_clients.contains(client) {
continue;
}
let res = self.player_from_client(video_id, *client).await; let res = query.player_from_client(video_id, *client).await;
match res { match res {
Ok(res) => return Ok(res), Ok(res) => return Ok(res),
Err(Error::Extraction(e)) => { Err(Error::Extraction(e)) => {
if e.use_login() { if e.use_login() && query.opts.auth.is_none() {
if let Some(c) = self.auth_enabled_client(clients) { clients_iter = clients.iter().peekable();
tracing::info!("{e}; fetching player with login"); query = Cow::Owned(self.clone().authenticated());
match self
.clone()
.authenticated()
.player_from_client(video_id, c)
.await
{
Ok(res) => return Ok(res),
Err(Error::Extraction(e)) => {
if !e.switch_client() {
return Err(Error::Extraction(e));
}
}
Err(e) => return Err(e),
}
} else {
return Err(Error::Extraction(e));
}
} else if !e.switch_client() { } else if !e.switch_client() {
return Err(Error::Extraction(e)); return Err(Error::Extraction(e));
} }
@ -147,6 +134,7 @@ impl RustyPipeQuery {
tracing::warn!("error fetching player with {client:?} client: {e}; retrying with {next_client:?} client"); tracing::warn!("error fetching player with {client:?} client: {e}; retrying with {next_client:?} client");
} }
last_e = Some(Error::Extraction(e)); last_e = Some(Error::Extraction(e));
failed_clients.insert(*client);
} }
Err(e) => return Err(e), Err(e) => return Err(e),
} }
@ -156,22 +144,27 @@ impl RustyPipeQuery {
async fn get_player_po_token(&self, video_id: &str) -> Result<PlayerPoToken, Error> { async fn get_player_po_token(&self, video_id: &str) -> Result<PlayerPoToken, Error> {
if let Some(bg) = &self.client.inner.botguard { if let Some(bg) = &self.client.inner.botguard {
let (ident, visitor_data) = if self.opts.auth == Some(true) {
(self.client.user_auth_datasync_id()?, None)
} else {
let visitor_data = self.get_visitor_data(false).await?; let visitor_data = self.get_visitor_data(false).await?;
(visitor_data.to_owned(), Some(visitor_data))
};
if bg.po_token_cache { if bg.po_token_cache {
let session_token = self.get_session_po_token(&visitor_data).await?; let session_token = self.get_session_po_token(&ident).await?;
Ok(PlayerPoToken { Ok(PlayerPoToken {
visitor_data: Some(visitor_data), visitor_data,
session_po_token: Some(session_token), session_po_token: Some(session_token),
content_po_token: None, content_po_token: None,
}) })
} else { } else {
let (po_tokens, valid_until) = let (po_tokens, valid_until) = self.get_po_tokens(&[video_id, &ident]).await?;
self.get_po_tokens(&[video_id, &visitor_data]).await?;
let mut po_tokens = po_tokens.into_iter(); let mut po_tokens = po_tokens.into_iter();
let po_token = po_tokens.next().unwrap(); let po_token = po_tokens.next().unwrap();
let session_po_token = po_tokens.next().unwrap(); let session_po_token = po_tokens.next().unwrap();
Ok(PlayerPoToken { Ok(PlayerPoToken {
visitor_data: Some(visitor_data), visitor_data,
session_po_token: Some(PoToken { session_po_token: Some(PoToken {
po_token: session_po_token, po_token: session_po_token,
valid_until, valid_until,
@ -191,6 +184,11 @@ impl RustyPipeQuery {
video_id: S, video_id: S,
client_type: ClientType, client_type: ClientType,
) -> Result<VideoPlayer, Error> { ) -> Result<VideoPlayer, Error> {
if self.opts.auth == Some(true) {
tracing::info!("fetching {client_type:?} player with login");
} else {
tracing::debug!("fetching {client_type:?} player");
}
let video_id = video_id.as_ref(); let video_id = video_id.as_ref();
let (deobf, player_po) = tokio::try_join!( let (deobf, player_po) = tokio::try_join!(
@ -385,6 +383,21 @@ impl MapResponse<VideoPlayer> for response::Player {
video_details.video_id, ctx.id video_details.video_id, ctx.id
))); )));
} }
// Sometimes YouTube Desktop does not output any URLs for adaptive streams.
// Since this is currently rare, it is best to retry the request in this case.
if !is_live
&& !streaming_data.adaptive_formats.c.is_empty()
&& streaming_data
.adaptive_formats
.c
.iter()
.all(|f| f.url.is_none() && f.signature_cipher.is_none())
{
return Err(ExtractionError::Unavailable {
reason: UnavailabilityReason::TryAgain,
msg: "no adaptive stream URLs".to_owned(),
});
}
let video_info = VideoPlayerDetails { let video_info = VideoPlayerDetails {
id: video_details.video_id, id: video_details.video_id,
@ -627,7 +640,7 @@ impl<'a> StreamsMapper<'a> {
fn deobf(&self) -> Result<&Deobfuscator, DeobfError> { fn deobf(&self) -> Result<&Deobfuscator, DeobfError> {
self.deobf self.deobf
.as_ref() .as_ref()
.ok_or(DeobfError::Other("no deobfuscator")) .ok_or(DeobfError::Other("no deobfuscator".into()))
} }
fn cipher_to_url_params( fn cipher_to_url_params(

View file

@ -257,6 +257,7 @@ mod tests {
#[case::nomusic("nomusic", "PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe")] #[case::nomusic("nomusic", "PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe")]
#[case::live("live", "UULVvqRdlKsE5Q8mf8YXbdIJLw")] #[case::live("live", "UULVvqRdlKsE5Q8mf8YXbdIJLw")]
#[case::pageheader("20241011_pageheader", "PLT2w2oBf1TZKyvY_M6JsASs73m-wjLzH5")] #[case::pageheader("20241011_pageheader", "PLT2w2oBf1TZKyvY_M6JsASs73m-wjLzH5")]
#[case::cmdexecutor("20250316_cmdexecutor", "PLbZIPy20-1pN7mqjckepWF78ndb6ci_qi")]
fn map_playlist_data(#[case] name: &str, #[case] id: &str) { fn map_playlist_data(#[case] name: &str, #[case] id: &str) {
let json_path = path!(*TESTFILES / "playlist" / format!("playlist_{name}.json")); let json_path = path!(*TESTFILES / "playlist" / format!("playlist_{name}.json"));
let json_file = File::open(json_path).unwrap(); let json_file = File::open(json_path).unwrap();

View file

@ -152,9 +152,16 @@ pub(crate) struct ContinuationItemRenderer {
pub continuation_endpoint: ContinuationEndpoint, pub continuation_endpoint: ContinuationEndpoint,
} }
#[derive(Debug, Deserialize)]
#[serde(untagged)]
pub(crate) enum ContinuationEndpoint {
ContinuationCommand(ContinuationCommandWrap),
CommandExecutorCommand(CommandExecutorCommandWrap),
}
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub(crate) struct ContinuationEndpoint { pub(crate) struct ContinuationCommandWrap {
pub continuation_command: ContinuationCommand, pub continuation_command: ContinuationCommand,
} }
@ -164,7 +171,34 @@ pub(crate) struct ContinuationCommand {
pub token: String, pub token: String,
} }
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct CommandExecutorCommandWrap {
pub command_executor_command: CommandExecutorCommand,
}
#[serde_as] #[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct CommandExecutorCommand {
#[serde_as(as = "VecSkipError<_>")]
commands: Vec<ContinuationCommandWrap>,
}
impl ContinuationEndpoint {
pub fn into_token(self) -> Option<String> {
match self {
Self::ContinuationCommand(cmd) => Some(cmd.continuation_command.token),
Self::CommandExecutorCommand(cmd) => cmd
.command_executor_command
.commands
.into_iter()
.next()
.map(|c| c.continuation_command.token),
}
}
}
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub(crate) struct Icon { pub(crate) struct Icon {

View file

@ -5,7 +5,8 @@ use crate::serializer::text::Text;
use super::{ use super::{
music_item::{ music_item::{
Button, Grid, ItemSection, MusicThumbnailRenderer, SimpleHeader, SingleColumnBrowseResult, Button, Grid, ItemSection, MusicMicroformat, MusicThumbnailRenderer, SimpleHeader,
SingleColumnBrowseResult,
}, },
SectionList, Tab, SectionList, Tab,
}; };
@ -14,8 +15,10 @@ use super::{
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub(crate) struct MusicArtist { pub(crate) struct MusicArtist {
pub contents: SingleColumnBrowseResult<Tab<SectionList<ItemSection>>>, pub contents: Option<SingleColumnBrowseResult<Tab<SectionList<ItemSection>>>>,
pub header: Header, pub header: Option<Header>,
#[serde(default)]
pub microformat: MusicMicroformat,
} }
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]

View file

@ -433,6 +433,22 @@ pub(crate) enum TrackBadge {
LiveBadgeRenderer {}, LiveBadgeRenderer {},
} }
#[serde_as]
#[derive(Default, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct MusicMicroformat {
#[serde_as(as = "DefaultOnError")]
pub microformat_data_renderer: MicroformatData,
}
#[derive(Default, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct MicroformatData {
pub url_canonical: Option<String>,
#[serde(default)]
pub noindex: bool,
}
/* /*
#MAPPER #MAPPER
*/ */
@ -530,7 +546,9 @@ impl MusicListMapper {
MusicResponseItem::ContinuationItemRenderer { MusicResponseItem::ContinuationItemRenderer {
continuation_endpoint, continuation_endpoint,
} => { } => {
self.ctoken = Some(continuation_endpoint.continuation_command.token); if self.ctoken.is_none() {
self.ctoken = continuation_endpoint.into_token();
}
Ok(None) Ok(None)
} }
} }

View file

@ -5,22 +5,21 @@ use crate::serializer::text::{AttributedText, Text, TextComponents};
use super::{ use super::{
music_item::{ music_item::{
Button, ItemSection, MusicContentsRenderer, MusicItemMenuEntry, MusicThumbnailRenderer, Button, ItemSection, MusicContentsRenderer, MusicItemMenuEntry, MusicMicroformat,
MusicThumbnailRenderer,
}, },
url_endpoint::OnTapWrap, url_endpoint::OnTapWrap,
ContentsRenderer, SectionList, Tab, ContentsRenderer, SectionList, Tab,
}; };
/// Response model for YouTube Music playlists and albums /// Response model for YouTube Music playlists and albums
#[serde_as]
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub(crate) struct MusicPlaylist { pub(crate) struct MusicPlaylist {
pub contents: Contents, pub contents: Option<Contents>,
pub header: Option<Header>, pub header: Option<Header>,
#[serde(default)] #[serde(default)]
#[serde_as(as = "DefaultOnError")] pub microformat: MusicMicroformat,
pub microformat: Option<Microformat>,
} }
#[serde_as] #[serde_as]
@ -162,15 +161,3 @@ pub(crate) struct AvatarStackViewModel {
pub(crate) struct AvatarStackRendererContext { pub(crate) struct AvatarStackRendererContext {
pub command_context: Option<OnTapWrap>, pub command_context: Option<OnTapWrap>,
} }
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct Microformat {
pub microformat_data_renderer: MicroformatData,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct MicroformatData {
pub url_canonical: String,
}

View file

@ -530,15 +530,14 @@ pub(crate) enum ContinuationItemVariants {
} }
impl ContinuationItemVariants { impl ContinuationItemVariants {
pub fn token(self) -> String { pub fn into_token(self) -> Option<String> {
match self { match self {
ContinuationItemVariants::Ep { ContinuationItemVariants::Ep {
continuation_endpoint, continuation_endpoint,
} => continuation_endpoint, } => continuation_endpoint,
ContinuationItemVariants::Btn { button } => button.button_renderer.command, ContinuationItemVariants::Btn { button } => button.button_renderer.command,
} }
.continuation_command .into_token()
.token
} }
} }

View file

@ -4,7 +4,7 @@ use serde_with::{
}; };
use time::OffsetDateTime; use time::OffsetDateTime;
use super::{ChannelBadge, ContentImage, ContinuationEndpoint, PhMetadataView, Thumbnails}; use super::{ChannelBadge, ContentImage, ContinuationItemRenderer, PhMetadataView, Thumbnails};
use crate::{ use crate::{
model::{Channel, ChannelItem, ChannelTag, PlaylistItem, VideoItem, YouTubeItem}, model::{Channel, ChannelItem, ChannelTag, PlaylistItem, VideoItem, YouTubeItem},
param::Language, param::Language,
@ -37,12 +37,9 @@ pub(crate) enum YouTubeListItem {
LockupViewModel(LockupViewModel), LockupViewModel(LockupViewModel),
/// Continauation items are located at the end of a list /// Continuation items are located at the end of a list
/// and contain the continuation token for progressive loading /// and contain the continuation token for progressive loading
#[serde(rename_all = "camelCase")] ContinuationItemRenderer(ContinuationItemRenderer),
ContinuationItemRenderer {
continuation_endpoint: ContinuationEndpoint,
},
/// Corrected search query /// Corrected search query
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
@ -838,9 +835,11 @@ impl YouTubeListMapper<YouTubeItem> {
self.items.push(mapped); self.items.push(mapped);
} }
} }
YouTubeListItem::ContinuationItemRenderer { YouTubeListItem::ContinuationItemRenderer(r) => {
continuation_endpoint, if self.ctoken.is_none() {
} => self.ctoken = Some(continuation_endpoint.continuation_command.token), self.ctoken = r.continuation_endpoint.into_token();
}
}
YouTubeListItem::ShowingResultsForRenderer { corrected_query } => { YouTubeListItem::ShowingResultsForRenderer { corrected_query } => {
self.corrected_query = Some(corrected_query); self.corrected_query = Some(corrected_query);
} }
@ -886,9 +885,11 @@ impl YouTubeListMapper<VideoItem> {
self.items.push(mapped); self.items.push(mapped);
} }
} }
YouTubeListItem::ContinuationItemRenderer { YouTubeListItem::ContinuationItemRenderer(r) => {
continuation_endpoint, if self.ctoken.is_none() {
} => self.ctoken = Some(continuation_endpoint.continuation_command.token), self.ctoken = r.continuation_endpoint.into_token();
}
}
YouTubeListItem::ShowingResultsForRenderer { corrected_query } => { YouTubeListItem::ShowingResultsForRenderer { corrected_query } => {
self.corrected_query = Some(corrected_query); self.corrected_query = Some(corrected_query);
} }
@ -938,9 +939,11 @@ impl YouTubeListMapper<PlaylistItem> {
self.items.push(mapped); self.items.push(mapped);
} }
} }
YouTubeListItem::ContinuationItemRenderer { YouTubeListItem::ContinuationItemRenderer(r) => {
continuation_endpoint, if self.ctoken.is_none() {
} => self.ctoken = Some(continuation_endpoint.continuation_command.token), self.ctoken = r.continuation_endpoint.into_token();
}
}
YouTubeListItem::ShowingResultsForRenderer { corrected_query } => { YouTubeListItem::ShowingResultsForRenderer { corrected_query } => {
self.corrected_query = Some(corrected_query); self.corrected_query = Some(corrected_query);
} }

View file

@ -207,11 +207,9 @@ impl MapResponse<Paginator<HistoryItem<VideoItem>>> for response::History {
&mut map_res, &mut map_res,
); );
} }
response::YouTubeListItem::ContinuationItemRenderer { response::YouTubeListItem::ContinuationItemRenderer(ep) => {
continuation_endpoint,
} => {
if ctoken.is_none() { if ctoken.is_none() {
ctoken = Some(continuation_endpoint.continuation_command.token); ctoken = ep.continuation_endpoint.into_token();
} }
} }
_ => {} _ => {}

View file

@ -208,11 +208,10 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
) )
}); });
let comment_ctoken = comment_ctoken_section.map(|s| { let comment_ctoken = comment_ctoken_section.and_then(|s| {
s.continuation_item_renderer s.continuation_item_renderer
.continuation_endpoint .continuation_endpoint
.continuation_command .into_token()
.token
}); });
let (owner, description, is_ccommons) = match secondary_info { let (owner, description, is_ccommons) = match secondary_info {
@ -333,7 +332,7 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
.sub_menu_items; .sub_menu_items;
items items
.try_swap_remove(1) .try_swap_remove(1)
.map(|c| c.service_endpoint.continuation_command.token) .and_then(|c| c.service_endpoint.into_token())
}); });
Ok(MapResult { Ok(MapResult {
@ -453,7 +452,9 @@ impl MapResponse<Paginator<Comment>> for response::VideoComments {
} }
} }
response::video_details::CommentListItem::ContinuationItemRenderer(cont) => { response::video_details::CommentListItem::ContinuationItemRenderer(cont) => {
ctoken = Some(cont.token()); if ctoken.is_none() {
ctoken = cont.into_token();
}
} }
response::video_details::CommentListItem::CommentsHeaderRenderer { count_text } => { response::video_details::CommentListItem::CommentsHeaderRenderer { count_text } => {
comment_count = count_text comment_count = count_text
@ -520,7 +521,9 @@ fn map_replies(
)) ))
} }
response::video_details::CommentListItem::ContinuationItemRenderer(cont) => { response::video_details::CommentListItem::ContinuationItemRenderer(cont) => {
reply_ctoken = Some(cont.token()); if reply_ctoken.is_none() {
reply_ctoken = cont.into_token();
}
None None
} }
_ => None, _ => None,

View file

@ -3,7 +3,7 @@ use std::collections::HashMap;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use regex::Regex; use regex::Regex;
use reqwest::Client; use reqwest::Client;
use ress::tokens::Token; use ress::tokens::{Keyword, Punct, Token};
use rquickjs::{Context, Runtime}; use rquickjs::{Context, Runtime};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -106,7 +106,7 @@ impl Deobfuscator {
.with(|ctx| call_fn(&ctx, DEOBF_NSIG_FUNC_NAME, nsig))?; .with(|ctx| call_fn(&ctx, DEOBF_NSIG_FUNC_NAME, nsig))?;
tracing::trace!("deobf nsig: {nsig} -> {res}"); tracing::trace!("deobf nsig: {nsig} -> {res}");
if res.starts_with("enhanced_except_") || res.ends_with(nsig) { if res.starts_with("enhanced_except_") || res.ends_with(nsig) {
return Err(DeobfError::Other("nsig fn returned an exception")); return Err(DeobfError::Other("nsig fn returned an exception".into()));
} }
Ok(res) Ok(res)
} }
@ -134,55 +134,21 @@ fn caller_function(mapped_name: &str, fn_name: &str) -> String {
} }
fn get_sig_fn(player_js: &str) -> Result<String, DeobfError> { fn get_sig_fn(player_js: &str) -> Result<String, DeobfError> {
let dfunc_name = get_sig_fn_name(player_js)?; let name = get_sig_fn_name(player_js)?;
let code = extract_js_fn(player_js, &name)?;
let js_fn = format!("{}{}", code, caller_function(DEOBF_SIG_FUNC_NAME, &name));
let function_pattern_str = format!(
r#"({}=function\([\w]+\)\{{.+?\}})"#,
dfunc_name.replace('$', "\\$")
);
let function_pattern = Regex::new(&function_pattern_str)
.map_err(|_| DeobfError::Other("could not parse sig fn pattern regex"))?;
let deobfuscate_function = format!(
"var {};",
&function_pattern
.captures(player_js)
.ok_or(DeobfError::Extraction("sig fn"))?[1]
);
let helper_object_name_pattern = Regex::new(r";([\w\$]{2,3})\...\(").unwrap();
let helper_object_name = helper_object_name_pattern
.captures(&deobfuscate_function)
.ok_or(DeobfError::Extraction("sig fn helper object name"))?
.get(1)
.unwrap()
.as_str();
let helper_pattern_str = format!(
r#"(var {}=\{{.+?\}}\}};)"#,
helper_object_name.replace('$', "\\$")
);
let helper_pattern = Regex::new(&helper_pattern_str)
.map_err(|_| DeobfError::Other("could not parse helper pattern regex"))?;
let player_js_nonl = player_js.replace('\n', "");
let helper_object = &helper_pattern
.captures(&player_js_nonl)
.ok_or(DeobfError::Extraction("sig fn helper object"))?[1];
let js_fn = helper_object.to_owned()
+ &deobfuscate_function
+ &caller_function(DEOBF_SIG_FUNC_NAME, &dfunc_name);
tracing::trace!("sig_fn: {js_fn}"); tracing::trace!("sig_fn: {js_fn}");
verify_fn(&js_fn, DEOBF_SIG_FUNC_NAME)?; verify_fn(&js_fn, DEOBF_SIG_FUNC_NAME)?;
tracing::debug!("successfully extracted sig fn `{dfunc_name}`"); tracing::debug!("successfully extracted sig fn `{name}`");
Ok(js_fn) Ok(js_fn)
} }
fn get_nsig_fn_names(player_js: &str) -> impl Iterator<Item = String> + '_ { fn get_nsig_fn_names(player_js: &str) -> impl Iterator<Item = String> + '_ {
static FUNCTION_NAME_REGEX: Lazy<Regex> = Lazy::new(|| { static FUNCTION_NAME_REGEX: Lazy<Regex> = Lazy::new(|| {
// x.get( .. y=functionName[array_num](z) .. x.set( // x.get( OR index.m3u8 OR delete x.y.file .. y=functionName[array_num](z) .. x.set(
Regex::new(r#"(?:[\w$]\.get\(|index\.m3u8).+[a-zA-Z]=([\w$]{2,})(?:\[(\d+)\])?\([a-zA-Z0-9]\).+[a-zA-Z0-9]\.set\("#) Regex::new(r#"(?:[\w$]\.get\(|index\.m3u8|delete [\w$]+\.[\w$]+\.file).+[a-zA-Z]=([\w$]{2,})(?:\[(\d+)\])?\([a-zA-Z0-9]\).+[a-zA-Z0-9]\.set\("#)
.unwrap() .unwrap()
}); });
@ -206,26 +172,71 @@ fn get_nsig_fn_names(player_js: &str) -> impl Iterator<Item = String> + '_ {
}) })
} }
fn extract_js_fn(js: &str, offset: usize, name: &str) -> Result<String, DeobfError> { fn extract_js_fn(js: &str, name: &str) -> Result<String, DeobfError> {
let function_base_re = Regex::new(&format!(r#"{}\s*=\s*function\("#, regex::escape(name)))
.map_err(|e| DeobfError::Other(format!("parsing regex for {name}: {e}").into()))?;
let offset = function_base_re
.find(js)
.ok_or(DeobfError::Extraction("could not find function base"))?
.start();
let scan = ress::Scanner::new(&js[offset..]); let scan = ress::Scanner::new(&js[offset..]);
let mut state = 0; let mut state = 0;
let mut level = 0;
let mut start = 0; #[derive(Default, Clone, PartialEq, Eq)]
let mut end = 0; struct Level {
brace: isize,
paren: isize,
bracket: isize,
}
let mut level = Level::default();
let mut start = 0usize;
let mut end = 0usize;
let mut period_before = false; let mut period_before = false;
let mut last_ident = None; let mut function_before = false;
let mut idents: HashMap<String, usize> = HashMap::new(); let mut idents: HashMap<String, bool> = HashMap::new();
// Set if the current statement is a variable/function param definition
// First value is the brace level, second is true if we are on the right hand side of an assignment
let mut var_def_stmt: Option<(Level, bool)> = None;
let global_objects = [ let global_objects = [
"NaN", "Infinity", "Object", "Function", "Boolean", "Symbol", "Error", "Number", "BigInt", "globalThis",
"Math", "Date", "String", "RegExp", "Array", "Map", "Set", "NaN",
"undefined",
"Infinity",
"Object",
"Function",
"Boolean",
"Symbol",
"Error",
"Number",
"BigInt",
"Math",
"Date",
"String",
"RegExp",
"Array",
"Map",
"Set",
"eval",
"isFinite",
"isNaN",
"parseFloat",
"parseInt",
"decodeURI",
"decodeURIComponent",
"encodeURI",
"encodeURIComponent",
"escape",
"unescape",
]; ];
for item in scan { for item in scan {
let it = item?; let it = item?;
let token = it.token; let token = it.token;
match state { match state {
// Looking for fn name // Looking for fn name
0 => { 0 => {
@ -236,47 +247,116 @@ fn extract_js_fn(js: &str, offset: usize, name: &str) -> Result<String, DeobfErr
} }
// Looking for equals // Looking for equals
1 => { 1 => {
if token.matches_punct(ress::tokens::Punct::Equal) { if token.matches_punct(Punct::Equal) {
state = 2; state = 2;
} else { } else {
state = 0; state = 0;
} }
} }
2 => { 2 => {
// Looking for begin/end braces match &token {
if token.matches_punct(ress::tokens::Punct::OpenBrace) { Token::Punct(punct) => {
level += 1; let var_def_this_lvl = || {
} else if token.matches_punct(ress::tokens::Punct::CloseBrace) { var_def_stmt
level -= 1; .as_ref()
.map(|(x, _)| x == &level)
.unwrap_or_default()
};
if level == 0 { match punct {
Punct::OpenBrace => {
level.brace += 1;
}
Punct::CloseBrace => {
if var_def_this_lvl() {
var_def_stmt = None;
}
level.brace -= 1;
if level.brace == 0 {
end = it.span.end; end = it.span.end;
state = 3; state = 3;
break; break;
} }
} }
Punct::OpenParen => {
// Looking for variable names level.paren += 1;
if let Token::Ident(id) = &token { }
if !period_before { Punct::CloseParen => {
let id_str = id.to_string(); if var_def_this_lvl() {
if !global_objects.contains(&id_str.as_str()) { var_def_stmt = None;
last_ident = Some(id.to_string()); }
level.paren -= 1;
}
Punct::OpenBracket => {
level.bracket += 1;
}
Punct::CloseBracket => {
if var_def_this_lvl() {
var_def_stmt = None;
}
level.bracket -= 1;
}
Punct::SemiColon => {
if var_def_this_lvl() {
var_def_stmt = None;
} }
} }
} else if last_ident.is_some() Punct::Comma => {
&& !token.matches_punct(ress::tokens::Punct::OpenParen) if let Some((lvl, rhs)) = &mut var_def_stmt {
if lvl == &level {
*rhs = false;
}
}
}
Punct::Equal => {
if let Some((lvl, rhs)) = &mut var_def_stmt {
if lvl == &level {
*rhs = true;
}
}
}
_ => {}
}
}
Token::Keyword(kw) => match kw {
Keyword::Var(_) | Keyword::Let(_) | Keyword::Const(_) => {
var_def_stmt = Some((level.clone(), false));
}
Keyword::Function(_) => {
let mut l = level.clone();
l.paren += 1;
var_def_stmt = Some((l, false));
}
_ => {}
},
Token::Ident(id) => {
// Ignore object attributes and 1char long local vars
if !period_before
&& id.as_ref().len() > 1
&& !global_objects.contains(&id.as_ref())
{ {
let n = idents.entry(last_ident.unwrap()).or_default(); // If we are on the left hand side of a variable definition statement
*n += 1; // or after "function", mark the variable name as defined
last_ident = None; if var_def_stmt
.as_ref()
.map(|(lvl, rhs)| lvl == &level && !rhs)
.unwrap_or_default()
|| function_before
{
idents.insert(id.to_string(), true);
} else { } else {
last_ident = None; idents.entry(id.to_string()).or_default();
}
}
}
_ => {}
} }
} }
_ => break, _ => break,
}; };
period_before = token.matches_punct(ress::tokens::Punct::Period); period_before = token.matches_punct(Punct::Period);
function_before = matches!(&token, Token::Keyword(Keyword::Function(_)));
} }
if state != 3 { if state != 3 {
@ -287,9 +367,10 @@ fn extract_js_fn(js: &str, offset: usize, name: &str) -> Result<String, DeobfErr
let mut code = format!("var {};", &js[fn_range.clone()]); let mut code = format!("var {};", &js[fn_range.clone()]);
let rt = rquickjs::Runtime::new()?; let rt = rquickjs::Runtime::new()?;
for (ident, _) in idents.into_iter().filter(|(_, v)| *v == 1) { for (ident, _) in idents.into_iter().filter(|(_, v)| !v) {
let var_pattern_str = format!(r#"(^|[^\w$]){}\s*=[^=]"#, regex::escape(&ident)); let var_pattern_str = format!(r#"(^|[^\w$\.]){}\s*=[^=]"#, regex::escape(&ident));
let re = Regex::new(&var_pattern_str).unwrap(); let re = Regex::new(&var_pattern_str)
.map_err(|e| DeobfError::Other(format!("parsing regex for {ident}: {e}").into()))?;
let found_variable = re let found_variable = re
.captures_iter(js) .captures_iter(js)
.filter(|cap| { .filter(|cap| {
@ -347,13 +428,13 @@ fn extract_js_var(js: &str) -> Option<&str> {
if let Token::Punct(p) = &token { if let Token::Punct(p) = &token {
match p { match p {
ress::tokens::Punct::OpenBrace => braces.push(b'{'), Punct::OpenBrace => braces.push(b'{'),
ress::tokens::Punct::OpenBracket => braces.push(b'['), Punct::OpenBracket => braces.push(b'['),
ress::tokens::Punct::OpenParen => braces.push(b'('), Punct::OpenParen => braces.push(b'('),
ress::tokens::Punct::CloseBrace => close_brace(&mut braces, b'{')?, Punct::CloseBrace => close_brace(&mut braces, b'{')?,
ress::tokens::Punct::CloseBracket => close_brace(&mut braces, b'[')?, Punct::CloseBracket => close_brace(&mut braces, b'[')?,
ress::tokens::Punct::CloseParen => close_brace(&mut braces, b'(')?, Punct::CloseParen => close_brace(&mut braces, b'(')?,
ress::tokens::Punct::Comma | ress::tokens::Punct::SemiColon => { Punct::Comma | Punct::SemiColon => {
if braces.is_empty() { if braces.is_empty() {
end = it.span.start; end = it.span.start;
break; break;
@ -388,23 +469,19 @@ fn verify_fn(js_fn: &str, fn_name: &str) -> Result<(), DeobfError> {
})?; })?;
if res.is_empty() { if res.is_empty() {
return Err(DeobfError::Other("deobfuscation fn returned empty string")); return Err(DeobfError::Other(
"deobfuscation fn returned empty string".into(),
));
} }
if res.starts_with("enhanced_except_") || res.ends_with(&testinp) { if res.starts_with("enhanced_except_") || res.ends_with(&testinp) {
return Err(DeobfError::Other("nsig fn returned an exception")); return Err(DeobfError::Other("nsig fn returned an exception".into()));
} }
Ok(()) Ok(())
} }
fn get_nsig_fn(player_js: &str) -> Result<String, DeobfError> { fn get_nsig_fn(player_js: &str) -> Result<String, DeobfError> {
let extract_fn = |name: &str| -> Result<String, DeobfError> { let extract_fn = |name: &str| -> Result<String, DeobfError> {
let function_base = format!("{name}=function"); let code = extract_js_fn(player_js, name)?;
let offset = player_js
.find(&function_base)
.ok_or(DeobfError::Extraction("could not find function base"))?;
let code = extract_js_fn(player_js, offset, name)?;
let js_fn = format!("{}{}", code, caller_function(DEOBF_NSIG_FUNC_NAME, name)); let js_fn = format!("{}{}", code, caller_function(DEOBF_NSIG_FUNC_NAME, name));
tracing::trace!("nsig_fn: {js_fn}"); tracing::trace!("nsig_fn: {js_fn}");
verify_fn(&js_fn, DEOBF_NSIG_FUNC_NAME)?; verify_fn(&js_fn, DEOBF_NSIG_FUNC_NAME)?;
@ -472,7 +549,9 @@ mod tests {
std::fs::read_to_string(js_path).unwrap() std::fs::read_to_string(js_path).unwrap()
}); });
const SIG_DEOBF_FUNC: &str = r#"var qB={w8:function(a){a.reverse()},EC:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c},Np:function(a,b){a.splice(0,b)}};var Rva=function(a){a=a.split("");qB.Np(a,3);qB.w8(a,41);qB.EC(a,55);qB.Np(a,3);qB.w8(a,33);qB.Np(a,3);qB.EC(a,48);qB.EC(a,17);qB.EC(a,43);return a.join("")};var deobf_sig=Rva;"#; const SIG_DEOBF_FUNC: &str = r#"var qB={w8:function(a){a.reverse()},
EC:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c},
Np:function(a,b){a.splice(0,b)}}; var Rva=function(a){a=a.split("");qB.Np(a,3);qB.w8(a,41);qB.EC(a,55);qB.Np(a,3);qB.w8(a,33);qB.Np(a,3);qB.EC(a,48);qB.EC(a,17);qB.EC(a,43);return a.join("")};var deobf_sig=Rva;"#;
const NSIG_DEOBF_FUNC: &str = r#"var Vo=function(a){var b=a.split(""),c=[function(d,e,f){var h=f.length;d.forEach(function(l,m,n){this.push(n[m]=f[(f.indexOf(l)-f.indexOf(this[m])+m+h--)%f.length])},e.split(""))}, const NSIG_DEOBF_FUNC: &str = r#"var Vo=function(a){var b=a.split(""),c=[function(d,e,f){var h=f.length;d.forEach(function(l,m,n){this.push(n[m]=f[(f.indexOf(l)-f.indexOf(this[m])+m+h--)%f.length])},e.split(""))},
928409064,-595856984,1403221911,653089124,-168714481,-1883008765,158931990,1346921902,361518508,1403221911,-362174697,-233641452,function(){for(var d=64,e=[];++d-e.length-32;){switch(d){case 91:d=44;continue;case 123:d=65;break;case 65:d-=18;continue;case 58:d=96;continue;case 46:d=95}e.push(String.fromCharCode(d))}return e}, 928409064,-595856984,1403221911,653089124,-168714481,-1883008765,158931990,1346921902,361518508,1403221911,-362174697,-233641452,function(){for(var d=64,e=[];++d-e.length-32;){switch(d){case 91:d=44;continue;case 123:d=65;break;case 65:d-=18;continue;case 58:d=96;continue;case 46:d=95}e.push(String.fromCharCode(d))}return e},
b,158931990,791141857,-907319795,-1776185924,1595027902,-829736173,function(d,e){e=(e%d.length+d.length)%d.length;d.splice(0,1,d.splice(e,1,d[0])[0])}, b,158931990,791141857,-907319795,-1776185924,1595027902,-829736173,function(d,e){e=(e%d.length+d.length)%d.length;d.splice(0,1,d.splice(e,1,d[0])[0])},
@ -525,7 +604,7 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
#[test] #[test]
fn t_extract_js_fn() { fn t_extract_js_fn() {
let base_js = "Wka = function(d){let x=10/2;return /,,[/,913,/](,)}/}let a = 42;"; let base_js = "Wka = function(d){let x=10/2;return /,,[/,913,/](,)}/}let a = 42;";
let res = extract_js_fn(base_js, 0, "Wka").unwrap(); let res = extract_js_fn(base_js, "Wka").unwrap();
assert_eq!( assert_eq!(
res, res,
"var Wka = function(d){let x=10/2;return /,,[/,913,/](,)}/};" "var Wka = function(d){let x=10/2;return /,,[/,913,/](,)}/};"
@ -536,7 +615,7 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
fn t_extract_js_fn_eviljs() { fn t_extract_js_fn_eviljs() {
// Evil JavaScript code containing braces within strings and regular expressions // Evil JavaScript code containing braces within strings and regular expressions
let base_js = "Wka = function(d){var x = [/,,/,913,/(,)}/,\"abcdef}\\\"\",];var y = 10/2/1;return x[1][y];}//some={}random-padding+;"; let base_js = "Wka = function(d){var x = [/,,/,913,/(,)}/,\"abcdef}\\\"\",];var y = 10/2/1;return x[1][y];}//some={}random-padding+;";
let res = extract_js_fn(base_js, 0, "Wka").unwrap(); let res = extract_js_fn(base_js, "Wka").unwrap();
assert_eq!( assert_eq!(
res, res,
"var Wka = function(d){var x = [/,,/,913,/(,)}/,\"abcdef}\\\"\",];var y = 10/2/1;return x[1][y];};" "var Wka = function(d){var x = [/,,/,913,/(,)}/,\"abcdef}\\\"\",];var y = 10/2/1;return x[1][y];};"
@ -545,33 +624,43 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
#[test] #[test]
fn t_extract_js_fn_outside_vars() { fn t_extract_js_fn_outside_vars() {
let base_js = "let a = 42;foo();var b=11;bar();Wka = function(d){var x=1+2+a*b;return x;}"; let base_js = "let a1 = 42;foo();var b1=11;var da=77;bar();Wka = function(da){var xy=1+2+a1*b1;return xy;}";
let res = extract_js_fn(base_js, 0, "Wka").unwrap(); let res = extract_js_fn(base_js, "Wka").unwrap();
// order of variables is non-reproducible // order of variables is non-reproducible
assert!( assert!(
res == "var a = 42; var b=11; var Wka = function(d){var x=1+2+a*b;return x;};" res == "var a1 = 42; var b1=11; var Wka = function(da){var xy=1+2+a1*b1;return xy;};"
|| res == "var b=11; var a = 42; var Wka = function(d){var x=1+2+a*b;return x;};", || res == "var b1=11; var a1 = 42; var Wka = function(da){var xy=1+2+a1*b1;return xy;};",
"got {res}" "got {res}"
); );
} }
#[test] #[test]
fn t_extract_js_fn_outside_vars2() { fn t_extract_js_fn_outside_vars2() {
let base_js = "{let a = {v1:1,v2:2}}foo();Wka = function(d){var x=1+2+a.v1;return x;}"; let base_js = "{let a1 = {v1:1,v2:2}}foo();Wka = function(d){var x=1+2+a1.v1;return x;}";
let res = extract_js_fn(base_js, 0, "Wka").unwrap(); let res = extract_js_fn(base_js, "Wka").unwrap();
assert_eq!( assert_eq!(
res, res,
"var a = {v1:1,v2:2}; var Wka = function(d){var x=1+2+a.v1;return x;};" "var a1 = {v1:1,v2:2}; var Wka = function(d){var x=1+2+a1.v1;return x;};"
); );
} }
#[test] #[test]
fn t_extract_js_fn_outside_vars3() { fn t_extract_js_fn_outside_vars3() {
let base_js = "Wka = function(d){var x=1+2+a[0];return x;};let a=[1,2,3]"; let base_js = "Wka = function(d){var x=1+2+a1[0];return x;};let a1=[1,2,3]";
let res = extract_js_fn(base_js, 0, "Wka").unwrap(); let res = extract_js_fn(base_js, "Wka").unwrap();
assert_eq!( assert_eq!(
res, res,
"var a=[1,2,3]; var Wka = function(d){var x=1+2+a[0];return x;};" "var a1=[1,2,3]; var Wka = function(d){var x=1+2+a1[0];return x;};"
);
}
#[test]
fn t_extract_js_fn_outside_vars4() {
let base_js = "let a0=123456;let a1=function(a){return a};let Wka = function(d){var x=1+2+a1();return x;}";
let res = extract_js_fn(base_js, "Wka").unwrap();
assert_eq!(
res,
"var a1=function(a){return a}; var Wka = function(d){var x=1+2+a1();return x;};"
); );
} }
@ -625,65 +714,86 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
} }
// Test cases from https://github.com/yt-dlp/yt-dlp/blob/master/test/test_youtube_signature.py // Test cases from https://github.com/yt-dlp/yt-dlp/blob/master/test/test_youtube_signature.py
#[rstest]
#[case("6ed0d907", "AOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL2QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0")]
#[case("3bb1f723", "MyOSJXtKI3m-uME_jv7-pT12gOFC02RFkGoqWpzE0Cs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA")]
#[case("2f1832d2", "0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xxAj7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJ2OySqa0q")]
#[tokio::test] #[tokio::test]
#[traced_test] #[traced_test]
async fn sig_tests(#[case] js_hash: &str, #[case] exp_sig: &str) { async fn sig_tests() {
let cases = [
("6ed0d907", "AOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL2QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0"),
("3bb1f723", "MyOSJXtKI3m-uME_jv7-pT12gOFC02RFkGoqWpzE0Cs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA"),
("2f1832d2", "0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xxAj7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJ2OySqa0q"),
("643afba4", "AAOAOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7vgpDL0QwbdV06sCIEzpWqMGkFR20CFOS21Tp-7vj_EMu-m37KtXJoOy1"),
("363db69b", "0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpz2ICs6EVdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA"),
];
for (js_hash, exp_sig) in cases {
let span = tracing::span!(tracing::Level::ERROR, "sig_test", js_hash);
let _enter = span.enter();
let (js_url, js_path) = player_js_file(js_hash).await; let (js_url, js_path) = player_js_file(js_hash).await;
let player_js = std::fs::read_to_string(js_path).unwrap(); let player_js = std::fs::read_to_string(js_path).unwrap();
let deobf_data = DeobfData::extract_fns(&js_url, &player_js).unwrap(); let deobf_data = DeobfData::extract_fns(&js_url, &player_js).unwrap();
let deobf = Deobfuscator::new(&deobf_data).unwrap(); let deobf = Deobfuscator::new(&deobf_data).unwrap();
let deobf_sig = deobf.deobfuscate_sig("2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA").unwrap(); let deobf_sig = deobf.deobfuscate_sig("2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA").unwrap();
assert_eq!(deobf_sig, exp_sig, "js: {js_hash}"); assert_eq!(deobf_sig, exp_sig, "[{js_hash}]");
}
} }
#[rstest]
#[case("7862ca1f", "X_LCxVDjAavgE5t", "yxJ1dM6iz5ogUg")]
#[case("9216d1f7", "SLp9F5bwjAdhE9F-", "gWnb9IK2DJ8Q1w")]
#[case("f8cb7a3b", "oBo2h5euWy6osrUt", "ivXHpm7qJjJN")]
#[case("2dfe380c", "oBo2h5euWy6osrUt", "3DIBbn3qdQ")]
#[case("f1ca6900", "cu3wyu6LQn2hse", "jvxetvmlI9AN9Q")]
#[case("8040e515", "wvOFaY-yjgDuIEg5", "HkfBFDHmgw4rsw")]
#[case("e06dea74", "AiuodmaDDYw8d3y4bf", "ankd8eza2T6Qmw")]
#[case("5dd88d1d", "kSxKFLeqzv_ZyHSAt", "n8gS8oRlHOxPFA")]
#[case("324f67b9", "xdftNy7dh9QGnhW", "22qLGxrmX8F1rA")]
#[case("4c3f79c5", "TDCstCG66tEAO5pR9o", "dbxNtZ14c-yWyw")]
#[case("c81bbb4a", "gre3EcLurNY2vqp94", "Z9DfGxWP115WTg")]
#[case("1f7d5369", "batNX7sYqIJdkJ", "IhOkL_zxbkOZBw")]
#[case("009f1d77", "5dwFHw8aFWQUQtffRq", "audescmLUzI3jw")]
#[case("dc0c6770", "5EHDMgYLV6HPGk_Mu-kk", "n9lUJLHbxUI0GQ")]
#[case("113ca41c", "cgYl-tlYkhjT7A", "hI7BBr2zUgcmMg")]
#[case("c57c113c", "M92UUMHa8PdvPd3wyM", "3hPqLJsiNZx7yA")]
#[case("5a3b6271", "B2j7f_UPT4rfje85Lu_e", "m5DmNymaGQ5RdQ")]
#[case("7a062b77", "NRcE3y3mVtm_cV-W", "VbsCYUATvqlt5w")]
#[case("dac945fd", "o8BkRxXhuYsBCWi6RplPdP", "3Lx32v_hmzTm6A")]
#[case("6f20102c", "lE8DhoDmKqnmJJ", "pJTTX6XyJP2BYw")]
#[case("cfa9e7cb", "aCi3iElgd2kq0bxVbQ", "QX1y8jGb2IbZ0w")]
#[case("8c7583ff", "1wWCVpRR96eAmMI87L", "KSkWAVv1ZQxC3A")]
#[case("b7910ca8", "_hXMCwMt9qE310D", "LoZMgkkofRMCZQ")]
#[case("590f65a6", "1tm7-g_A9zsI8_Lay_", "xI4Vem4Put_rOg")]
#[case("b22ef6e7", "b6HcntHGkvBLk_FRf", "kNPW6A7FyP2l8A")]
#[case("3400486c", "lL46g3XifCKUZn1Xfw", "z767lhet6V2Skl")]
#[case("20dfca59", "-fLCxedkAk4LUTK2", "O8kfRq1y1eyHGw")]
#[case("b12cc44b", "keLa5R2U00sR9SQK", "N1OGyujjEwMnLw")]
#[case("3bb1f723", "gK15nzVyaXE9RsMP3z", "ZFFWFLPWx9DEgQ")]
#[case("2f1832d2", "YWt1qdbe8SAfkoPHW5d", "RrRjWQOJmBiP")]
#[case("19d2ae9d", "YWt1qdbe8SAfkoPHW5d", "CS6dVTYzpZrAZ5TD")]
#[tokio::test] #[tokio::test]
#[traced_test] #[traced_test]
async fn nsig_tests(#[case] js_hash: &str, #[case] nsig_in: &str, #[case] expect: &str) { async fn nsig_tests() {
let cases = [
("7862ca1f", "X_LCxVDjAavgE5t", "yxJ1dM6iz5ogUg"),
("9216d1f7", "SLp9F5bwjAdhE9F-", "gWnb9IK2DJ8Q1w"),
("f8cb7a3b", "oBo2h5euWy6osrUt", "ivXHpm7qJjJN"),
("2dfe380c", "oBo2h5euWy6osrUt", "3DIBbn3qdQ"),
("f1ca6900", "cu3wyu6LQn2hse", "jvxetvmlI9AN9Q"),
("8040e515", "wvOFaY-yjgDuIEg5", "HkfBFDHmgw4rsw"),
("e06dea74", "AiuodmaDDYw8d3y4bf", "ankd8eza2T6Qmw"),
("5dd88d1d", "kSxKFLeqzv_ZyHSAt", "n8gS8oRlHOxPFA"),
("324f67b9", "xdftNy7dh9QGnhW", "22qLGxrmX8F1rA"),
("4c3f79c5", "TDCstCG66tEAO5pR9o", "dbxNtZ14c-yWyw"),
("c81bbb4a", "gre3EcLurNY2vqp94", "Z9DfGxWP115WTg"),
("1f7d5369", "batNX7sYqIJdkJ", "IhOkL_zxbkOZBw"),
("009f1d77", "5dwFHw8aFWQUQtffRq", "audescmLUzI3jw"),
("dc0c6770", "5EHDMgYLV6HPGk_Mu-kk", "n9lUJLHbxUI0GQ"),
("113ca41c", "cgYl-tlYkhjT7A", "hI7BBr2zUgcmMg"),
("c57c113c", "M92UUMHa8PdvPd3wyM", "3hPqLJsiNZx7yA"),
("5a3b6271", "B2j7f_UPT4rfje85Lu_e", "m5DmNymaGQ5RdQ"),
("7a062b77", "NRcE3y3mVtm_cV-W", "VbsCYUATvqlt5w"),
("dac945fd", "o8BkRxXhuYsBCWi6RplPdP", "3Lx32v_hmzTm6A"),
("6f20102c", "lE8DhoDmKqnmJJ", "pJTTX6XyJP2BYw"),
("cfa9e7cb", "aCi3iElgd2kq0bxVbQ", "QX1y8jGb2IbZ0w"),
("8c7583ff", "1wWCVpRR96eAmMI87L", "KSkWAVv1ZQxC3A"),
("b7910ca8", "_hXMCwMt9qE310D", "LoZMgkkofRMCZQ"),
("590f65a6", "1tm7-g_A9zsI8_Lay_", "xI4Vem4Put_rOg"),
("b22ef6e7", "b6HcntHGkvBLk_FRf", "kNPW6A7FyP2l8A"),
("3400486c", "lL46g3XifCKUZn1Xfw", "z767lhet6V2Skl"),
("20dfca59", "-fLCxedkAk4LUTK2", "O8kfRq1y1eyHGw"),
("b12cc44b", "keLa5R2U00sR9SQK", "N1OGyujjEwMnLw"),
("3bb1f723", "gK15nzVyaXE9RsMP3z", "ZFFWFLPWx9DEgQ"),
("2f1832d2", "YWt1qdbe8SAfkoPHW5d", "RrRjWQOJmBiP"),
("19d2ae9d", "YWt1qdbe8SAfkoPHW5d", "CS6dVTYzpZrAZ5TD"),
("e7567ecf", "Sy4aDGc0VpYRR9ew_", "5UPOT1VhoZxNLQ"),
("d50f54ef", "Ha7507LzRmH3Utygtj", "XFTb2HoeOE5MHg"),
("074a8365", "Ha7507LzRmH3Utygtj", "ufTsrE0IVYrkl8v"),
("643afba4", "N5uAlLqm0eg1GyHO", "dCBQOejdq5s-ww"),
("69f581a5", "-qIP447rVlTTwaZjY", "KNcGOksBAvwqQg"),
("363db69b", "eWYu5d5YeY_4LyEDc", "XJQqf-N7Xra3gg"),
];
for (js_hash, nsig_in, exp_nsig) in cases {
let span = tracing::span!(tracing::Level::ERROR, "nsig_test", js_hash);
let _enter = span.enter();
let (js_url, js_path) = player_js_file(js_hash).await; let (js_url, js_path) = player_js_file(js_hash).await;
let player_js = std::fs::read_to_string(js_path).unwrap(); let player_js = std::fs::read_to_string(js_path).unwrap();
let deobf_data = DeobfData::extract_fns(&js_url, &player_js).unwrap(); let deobf_data = DeobfData::extract_fns(&js_url, &player_js).expect(js_hash);
let deobf = Deobfuscator::new(&deobf_data).unwrap(); let deobf = Deobfuscator::new(&deobf_data).expect(js_hash);
let deobf_nsig = deobf.deobfuscate_nsig(nsig_in).unwrap(); let deobf_nsig = deobf.deobfuscate_nsig(nsig_in).expect(js_hash);
assert_eq!(deobf_nsig, expect, "js: {js_hash}"); assert_eq!(deobf_nsig, exp_nsig, "[{js_hash}]");
}
} }
#[tokio::test] #[tokio::test]

View file

@ -151,6 +151,8 @@ pub enum AuthError {
} }
pub(crate) mod internal { pub(crate) mod internal {
use std::borrow::Cow;
use super::{Error, ExtractionError}; use super::{Error, ExtractionError};
/// Error that occurred during the initialization /// Error that occurred during the initialization
@ -168,7 +170,7 @@ pub(crate) mod internal {
Extraction(&'static str), Extraction(&'static str),
/// Unspecified error /// Unspecified error
#[error("error: {0}")] #[error("error: {0}")]
Other(&'static str), Other(Cow<'static, str>),
} }
impl From<DeobfError> for Error { impl From<DeobfError> for Error {

View file

@ -75,10 +75,10 @@ pub fn get_cg_from_fancy_regexes(regexes: &[&str], text: &str, cg_name: &str) ->
/// Generate a random string with given length and byte charset. /// Generate a random string with given length and byte charset.
fn random_string(charset: &[u8], length: usize) -> String { fn random_string(charset: &[u8], length: usize) -> String {
let mut result = String::with_capacity(length); let mut result = String::with_capacity(length);
let mut rng = rand::thread_rng(); let mut rng = rand::rng();
for _ in 0..length { for _ in 0..length {
result.push(char::from(charset[rng.gen_range(0..charset.len())])); result.push(char::from(charset[rng.random_range(0..charset.len())]));
} }
result result
@ -90,14 +90,14 @@ pub fn generate_content_playback_nonce() -> String {
} }
pub fn random_uuid() -> String { pub fn random_uuid() -> String {
let mut rng = rand::thread_rng(); let mut rng = rand::rng();
format!( format!(
"{:08x}-{:04x}-{:04x}-{:04x}-{:012x}", "{:08x}-{:04x}-{:04x}-{:04x}-{:012x}",
rng.gen::<u32>(), rng.random::<u32>(),
rng.gen::<u16>(), rng.random::<u16>(),
rng.gen::<u16>(), rng.random::<u16>(),
rng.gen::<u16>(), rng.random::<u16>(),
rng.gen::<u64>() & 0xffff_ffff_ffff, rng.random::<u64>() & 0xffff_ffff_ffff,
) )
} }
@ -229,7 +229,7 @@ pub fn retry_delay(
backoff_base: u32, backoff_base: u32,
) -> u32 { ) -> u32 {
let unjittered_delay = backoff_base.checked_pow(n_past_retries).unwrap_or(u32::MAX); let unjittered_delay = backoff_base.checked_pow(n_past_retries).unwrap_or(u32::MAX);
let jitter_factor = rand::thread_rng().gen_range(800..1500); let jitter_factor = rand::rng().random_range(800..1500);
let jittered_delay = unjittered_delay let jittered_delay = unjittered_delay
.checked_mul(jitter_factor) .checked_mul(jitter_factor)
.unwrap_or(u32::MAX); .unwrap_or(u32::MAX);

View file

@ -148,8 +148,8 @@ impl VisitorDataCache {
{ {
let vds = self.inner.visitor_data.read().unwrap(); let vds = self.inner.visitor_data.read().unwrap();
if !vds.is_empty() { if !vds.is_empty() {
let mut rng = rand::thread_rng(); let mut rng = rand::rng();
let vd = vds[rng.gen_range(0..vds.len())].to_owned(); let vd = vds[rng.random_range(0..vds.len())].to_owned();
tracing::debug!("visitor data {vd} picked from cache"); tracing::debug!("visitor data {vd} picked from cache");
return Ok(vd); return Ok(vd);
} }

File diff suppressed because it is too large Load diff

View file

@ -42,8 +42,8 @@ MusicArtist(
by_va: false, by_va: false,
), ),
AlbumItem( AlbumItem(
id: "MPREb_6PEkIQE7sWY", id: "MPREb_HrCgErOdgCv",
name: "An deiner Seite (Online Version)", name: "Freiheit",
cover: "[cover]", cover: "[cover]",
artists: [ artists: [
ArtistId( ArtistId(
@ -52,8 +52,8 @@ MusicArtist(
), ),
], ],
artist_id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"), artist_id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
album_type: ep, album_type: album,
year: Some(2008), year: Some(2004),
by_va: false, by_va: false,
), ),
AlbumItem( AlbumItem(
@ -87,8 +87,8 @@ MusicArtist(
by_va: false, by_va: false,
), ),
AlbumItem( AlbumItem(
id: "MPREb_QEClJsuO9xM", id: "MPREb_Oq0WKqNwSVY",
name: "So wie Du warst", name: "Das 2. Gebot",
cover: "[cover]", cover: "[cover]",
artists: [ artists: [
ArtistId( ArtistId(
@ -97,8 +97,8 @@ MusicArtist(
), ),
], ],
artist_id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"), artist_id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
album_type: single, album_type: album,
year: Some(2012), year: Some(2003),
by_va: false, by_va: false,
), ),
AlbumItem( AlbumItem(
@ -251,6 +251,21 @@ MusicArtist(
year: Some(2015), year: Some(2015),
by_va: false, by_va: false,
), ),
AlbumItem(
id: "MPREb_ohcGTZrqKPZ",
name: "Zelluloid",
cover: "[cover]",
artists: [
ArtistId(
id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
name: "Unheilig",
),
],
artist_id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
album_type: album,
year: Some(2004),
by_va: false,
),
AlbumItem( AlbumItem(
id: "MPREb_pWpeXxATZYb", id: "MPREb_pWpeXxATZYb",
name: "Wir sind alle wie eins", name: "Wir sind alle wie eins",
@ -266,21 +281,6 @@ MusicArtist(
year: Some(2014), year: Some(2014),
by_va: false, by_va: false,
), ),
AlbumItem(
id: "MPREb_rHhaDLqalbT",
name: "Winter (EP)",
cover: "[cover]",
artists: [
ArtistId(
id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
name: "Unheilig",
),
],
artist_id: Some("UC7cl4MmM6ZZ2TcFyMk_b4pg"),
album_type: ep,
year: Some(2010),
by_va: false,
),
AlbumItem( AlbumItem(
id: "MPREb_saXgTKNPaSu", id: "MPREb_saXgTKNPaSu",
name: "Zeit zu gehen", name: "Zeit zu gehen",

View file

@ -30,7 +30,7 @@ use rustypipe::validate;
#[case::desktop(ClientType::Desktop)] #[case::desktop(ClientType::Desktop)]
#[case::tv(ClientType::Tv)] #[case::tv(ClientType::Tv)]
#[case::mobile(ClientType::Mobile)] #[case::mobile(ClientType::Mobile)]
#[case::android(ClientType::Android)] // #[case::android(ClientType::Android)] Removed since it requires Android device attestation
#[case::ios(ClientType::Ios)] #[case::ios(ClientType::Ios)]
#[tokio::test] #[tokio::test]
async fn get_player_from_client(#[case] client_type: ClientType, rp: RustyPipe) { async fn get_player_from_client(#[case] client_type: ClientType, rp: RustyPipe) {
@ -2119,11 +2119,13 @@ async fn music_search_artists(rp: RustyPipe, unlocalized: bool) {
#[rstest] #[rstest]
#[tokio::test] #[tokio::test]
async fn music_search_artists_cont(rp: RustyPipe) { async fn music_search_artists_cont(rp: RustyPipe) {
let res = rp.query().music_search_artists("boys").await.unwrap(); let res = rp.query().music_search_artists("girls").await.unwrap();
assert_eq!(res.corrected_query, None); assert_eq!(res.corrected_query, None);
if !res.items.is_exhausted() {
assert_next(res.items, rp.query(), 15, 2, true).await; assert_next(res.items, rp.query(), 15, 2, true).await;
} }
}
#[rstest] #[rstest]
#[tokio::test] #[tokio::test]
@ -2594,7 +2596,7 @@ async fn music_genres(rp: RustyPipe, unlocalized: bool) {
} }
#[rstest] #[rstest]
#[case::chill("ggMPOg1uX1JOQWZFeDByc2Jm", "Chill")] #[case::party("ggMPOg1uX2w1aW1CRDFTSUNo", "Party")]
#[case::pop("ggMPOg1uX1lMbVZmbzl6NlJ3", "Pop")] #[case::pop("ggMPOg1uX1lMbVZmbzl6NlJ3", "Pop")]
#[tokio::test] #[tokio::test]
async fn music_genre(#[case] id: &str, #[case] name: &str, rp: RustyPipe, unlocalized: bool) { async fn music_genre(#[case] id: &str, #[case] name: &str, rp: RustyPipe, unlocalized: bool) {
@ -2638,7 +2640,7 @@ async fn music_genre(#[case] id: &str, #[case] name: &str, rp: RustyPipe, unloca
let subgenres = check_music_genre(genre, id, name, unlocalized); let subgenres = check_music_genre(genre, id, name, unlocalized);
if name == "Chill" { if name == "Party" {
assert_gte(subgenres.len(), 2, "subgenres"); assert_gte(subgenres.len(), 2, "subgenres");
} }
@ -2929,7 +2931,11 @@ async fn assert_next<T: FromYtItem, Q: AsRef<RustyPipeQuery>>(
} }
for i in 0..n_pages { for i in 0..n_pages {
p = p.next(query).await.unwrap().expect("paginator exhausted"); match p.next(query).await.unwrap() {
Some(np) => p = np,
None => panic!("paginator exhausted after {i} pages"),
}
assert_gte( assert_gte(
p.items.len(), p.items.len(),
min_items, min_items,