Compare commits

...

3 commits

Author SHA1 Message Date
452f765ffd fix: handling new podcast links
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
2023-11-04 01:14:54 +01:00
ba06e2c8c8 fix: a/b test 10: channel about modal 2023-11-03 21:46:55 +01:00
cced125390 fix: Handle trimmed channel ID from RSS feed 2023-11-03 20:45:19 +01:00
28 changed files with 3305 additions and 3111 deletions

View file

@ -26,6 +26,7 @@ pub enum ABTest {
ShortDateFormat = 7, ShortDateFormat = 7,
TrackViewcount = 8, TrackViewcount = 8,
PlaylistsForShorts = 9, PlaylistsForShorts = 9,
ChannelAboutModal = 10,
} }
const TESTS_TO_RUN: [ABTest; 3] = [ const TESTS_TO_RUN: [ABTest; 3] = [
@ -98,6 +99,7 @@ pub async fn run_test(
ABTest::ShortDateFormat => short_date_format(&query).await, ABTest::ShortDateFormat => short_date_format(&query).await,
ABTest::PlaylistsForShorts => playlists_for_shorts(&query).await, ABTest::PlaylistsForShorts => playlists_for_shorts(&query).await,
ABTest::TrackViewcount => track_viewcount(&query).await, ABTest::TrackViewcount => track_viewcount(&query).await,
ABTest::ChannelAboutModal => channel_about_modal(&query).await,
} }
.unwrap(); .unwrap();
pb.inc(1); pb.inc(1);
@ -259,6 +261,16 @@ pub async fn short_date_format(rp: &RustyPipeQuery) -> Result<bool> {
})) }))
} }
pub async fn playlists_for_shorts(rp: &RustyPipeQuery) -> Result<bool> {
let playlist = rp.playlist("UUSHh8gHdtzO2tXd593_bjErWg").await?;
let v1 = playlist
.videos
.items
.first()
.ok_or_else(|| anyhow::anyhow!("no videos"))?;
Ok(v1.publish_date_txt.is_none())
}
pub async fn track_viewcount(rp: &RustyPipeQuery) -> Result<bool> { pub async fn track_viewcount(rp: &RustyPipeQuery) -> Result<bool> {
let res = rp.music_search("lieblingsmensch namika").await?; let res = rp.music_search("lieblingsmensch namika").await?;
@ -273,12 +285,19 @@ pub async fn track_viewcount(rp: &RustyPipeQuery) -> Result<bool> {
Ok(track.view_count.is_some()) Ok(track.view_count.is_some())
} }
pub async fn playlists_for_shorts(rp: &RustyPipeQuery) -> Result<bool> { pub async fn channel_about_modal(rp: &RustyPipeQuery) -> Result<bool> {
let playlist = rp.playlist("UUSHh8gHdtzO2tXd593_bjErWg").await?; let id = "UC2DjFE7Xf11URZqWBigcVOQ";
let v1 = playlist let res = rp
.videos .raw(
.items ClientType::Desktop,
.first() "browse",
.ok_or_else(|| anyhow::anyhow!("no videos"))?; &QBrowse {
Ok(v1.publish_date_txt.is_none()) context: rp.get_context(ClientType::Desktop, true, None).await,
browse_id: id,
params: None,
},
)
.await
.unwrap();
Ok(!res.contains("\"EgVhYm91dPIGBAoCEgA%3D\""))
} }

View file

@ -339,7 +339,7 @@ async fn channel_playlists() {
} }
async fn channel_info() { async fn channel_info() {
let json_path = path!(*TESTFILES_DIR / "channel" / "channel_info.json"); let json_path = path!(*TESTFILES_DIR / "channel" / "channel_info2.json");
if json_path.exists() { if json_path.exists() {
return; return;
} }

View file

@ -202,11 +202,20 @@ pub enum Country {
.to_owned(); .to_owned();
let mut code_lang_array = format!( let mut code_lang_array = format!(
"/// Array of all available languages\npub const LANGUAGES: [Language; {}] = [\n", r#"/// Array of all available languages
/// The languages are sorted by their native names. This array can be used to display
/// a language selection or to get the language code from a language name using binary search.
pub const LANGUAGES: [Language; {}] = [
"#,
languages.len() languages.len()
); );
let mut code_country_array = format!( let mut code_country_array = format!(
"/// Array of all available countries\npub const COUNTRIES: [Country; {}] = [\n", r#"/// Array of all available countries
///
/// The countries are sorted by their english names. This array can be used to display
/// a country selection or to get the country code from a country name using binary search.
pub const COUNTRIES: [Country; {}] = [
"#,
countries.len() countries.len()
); );
@ -252,9 +261,6 @@ pub enum Country {
code_langs += &enum_name; code_langs += &enum_name;
code_langs += ",\n"; code_langs += ",\n";
// Language array
writeln!(code_lang_array, " Language::{enum_name},").unwrap();
// Language names // Language names
writeln!( writeln!(
code_lang_names, code_lang_names,
@ -264,6 +270,24 @@ pub enum Country {
} }
code_langs += "}\n"; code_langs += "}\n";
// Language array
let languages_by_name = languages
.iter()
.map(|(k, v)| (v, k))
.collect::<BTreeMap<_, _>>();
for code in languages_by_name.values() {
let enum_name = code.split('-').fold(String::new(), |mut output, c| {
let _ = write!(
output,
"{}{}",
c[0..1].to_owned().to_uppercase(),
c[1..].to_owned().to_lowercase()
);
output
});
writeln!(code_lang_array, " Language::{enum_name},").unwrap();
}
for (c, n) in &countries { for (c, n) in &countries {
let enum_name = c[0..1].to_owned().to_uppercase() + &c[1..].to_owned().to_lowercase(); let enum_name = c[0..1].to_owned().to_uppercase() + &c[1..].to_owned().to_lowercase();
@ -271,9 +295,6 @@ pub enum Country {
writeln!(code_countries, " /// {n}").unwrap(); writeln!(code_countries, " /// {n}").unwrap();
writeln!(code_countries, " {enum_name},").unwrap(); writeln!(code_countries, " {enum_name},").unwrap();
// Country array
writeln!(code_country_array, " Country::{enum_name},").unwrap();
// Country names // Country names
writeln!( writeln!(
code_country_names, code_country_names,
@ -282,6 +303,16 @@ pub enum Country {
.unwrap(); .unwrap();
} }
// Country array
let countries_by_name = countries
.iter()
.map(|(k, v)| (v, k))
.collect::<BTreeMap<_, _>>();
for c in countries_by_name.values() {
let enum_name = c[0..1].to_owned().to_uppercase() + &c[1..].to_owned().to_lowercase();
writeln!(code_country_array, " Country::{enum_name},").unwrap();
}
// Add Country::Zz / Global // Add Country::Zz / Global
code_countries += " /// Global (can only be used for music charts)\n"; code_countries += " /// Global (can only be used for music charts)\n";
code_countries += " Zz,\n"; code_countries += " Zz,\n";

View file

@ -417,3 +417,18 @@ tab.
Since the reel items dont include upload date information you can circumvent this new UI Since the reel items dont include upload date information you can circumvent this new UI
by using the mobile client. But that may change in the future. by using the mobile client. But that may change in the future.
## [10] Channel About modal
- **Encountered on:** 03.11.2023
- **Impact:** 🟡 Medium
- **Endpoint:** browse (channel info)
![A/B test 10 screenshot](./_img/ab_10.png)
YouTube replaced the *About* channel tab with a modal. This changes the way additional
channel metadata has to be fetched.
The new modal uses a continuation request with a token which can be easily generated.
Attempts to fetch the old about tab with the A/B test enabled will lead to a redirect to
the main tab.

BIN
notes/_img/ab_10.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 43 KiB

View file

@ -1,20 +1,21 @@
use std::fmt::Debug; use std::fmt::Debug;
use serde::Serialize; use serde::Serialize;
use time::OffsetDateTime;
use url::Url; use url::Url;
use crate::{ use crate::{
error::{Error, ExtractionError}, error::{Error, ExtractionError},
model::{ model::{
paginator::{ContinuationEndpoint, Paginator}, paginator::{ContinuationEndpoint, Paginator},
Channel, ChannelInfo, PlaylistItem, VideoItem, YouTubeItem, Channel, ChannelInfo, PlaylistItem, VideoItem,
}, },
param::{ChannelOrder, ChannelVideoTab, Language}, param::{ChannelOrder, ChannelVideoTab, Language},
serializer::MapResult, serializer::{text::TextComponent, MapResult},
util::{self, ProtoBuilder}, util::{self, timeago, ProtoBuilder},
}; };
use super::{response, ClientType, MapResponse, RustyPipeQuery, YTContext}; use super::{response, ClientType, MapResponse, QContinuation, RustyPipeQuery, YTContext};
#[derive(Debug, Serialize)] #[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
@ -36,8 +37,6 @@ enum ChannelTab {
Live, Live,
#[serde(rename = "EglwbGF5bGlzdHMgAQ%3D%3D")] #[serde(rename = "EglwbGF5bGlzdHMgAQ%3D%3D")]
Playlists, Playlists,
#[serde(rename = "EgVhYm91dPIGBAoCEgA%3D")]
Info,
#[serde(rename = "EgZzZWFyY2jyBgQKAloA")] #[serde(rename = "EgZzZWFyY2jyBgQKAloA")]
Search, Search,
} }
@ -126,7 +125,7 @@ impl RustyPipeQuery {
let visitor_data = Some(self.get_visitor_data().await?); let visitor_data = Some(self.get_visitor_data().await?);
self.continuation( self.continuation(
order_ctoken(channel_id.as_ref(), tab, order), order_ctoken(channel_id.as_ref(), tab, order, &random_target()),
ContinuationEndpoint::Browse, ContinuationEndpoint::Browse,
visitor_data.as_deref(), visitor_data.as_deref(),
) )
@ -179,19 +178,17 @@ impl RustyPipeQuery {
pub async fn channel_info<S: AsRef<str> + Debug>( pub async fn channel_info<S: AsRef<str> + Debug>(
&self, &self,
channel_id: S, channel_id: S,
) -> Result<Channel<ChannelInfo>, Error> { ) -> Result<ChannelInfo, Error> {
let channel_id = channel_id.as_ref(); let channel_id = channel_id.as_ref();
let context = self.get_context(ClientType::Desktop, true, None).await; let context = self.get_context(ClientType::Desktop, false, None).await;
let request_body = QChannel { let request_body = QContinuation {
context, context,
browse_id: channel_id, continuation: &channel_info_ctoken(channel_id, &random_target()),
params: ChannelTab::Info,
query: None,
}; };
self.execute_request::<response::Channel, _, _>( self.execute_request::<response::ChannelAbout, _, _>(
ClientType::Desktop, ClientType::Desktop,
"channel_info", "channel_info2",
channel_id, channel_id,
"browse", "browse",
&request_body, &request_body,
@ -290,46 +287,64 @@ impl MapResponse<Channel<Paginator<PlaylistItem>>> for response::Channel {
} }
} }
impl MapResponse<Channel<ChannelInfo>> for response::Channel { impl MapResponse<ChannelInfo> for response::ChannelAbout {
fn map_response( fn map_response(
self, self,
id: &str, _id: &str,
lang: Language, lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
vdata: Option<&str>, _visitor_data: Option<&str>,
) -> Result<MapResult<Channel<ChannelInfo>>, ExtractionError> { ) -> Result<MapResult<ChannelInfo>, ExtractionError> {
let content = map_channel_content(id, self.contents, self.alerts)?; let ep = self
let channel_data = map_channel( .on_response_received_endpoints
MapChannelData { .into_iter()
header: self.header, .next()
metadata: self.metadata, .ok_or(ExtractionError::InvalidData("no received endpoint".into()))?;
microformat: self.microformat, let continuations = ep.append_continuation_items_action.continuation_items;
visitor_data: self let about = continuations
.response_context .c
.visitor_data .into_iter()
.or_else(|| vdata.map(str::to_owned)), .next()
has_shorts: content.has_shorts, .ok_or(ExtractionError::InvalidData("no aboutChannel data".into()))?
has_live: content.has_live, .about_channel_renderer
}, .metadata
id, .about_channel_view_model;
lang, let mut warnings = continuations.warnings;
)?;
let mut mapper = response::YouTubeListMapper::<YouTubeItem>::new(lang); let links = about
mapper.map_response(content.content); .links
let mut warnings = mapper.warnings; .into_iter()
.filter_map(|l| {
let cinfo = mapper.channel_info.unwrap_or_else(|| { let lv = l.channel_external_link_view_model;
warnings.push("no aboutFullMetadata".to_owned()); if let TextComponent::Web { url, .. } = lv.link {
ChannelInfo { Some((String::from(lv.title), util::sanitize_yt_url(&url)))
create_date: None, } else {
view_count: None, None
links: Vec::new(), }
} })
}); .collect::<Vec<_>>();
Ok(MapResult { Ok(MapResult {
c: combine_channel_data(channel_data.c, cinfo), c: ChannelInfo {
id: about.channel_id,
url: about.canonical_channel_url,
description: about.description,
subscriber_count: about
.subscriber_count_text
.and_then(|txt| util::parse_large_numstr_or_warn(&txt, lang, &mut warnings)),
video_count: about
.video_count_text
.and_then(|txt| util::parse_numeric_or_warn(&txt, &mut warnings)),
create_date: about.joined_date_text.and_then(|txt| {
timeago::parse_textual_date_or_warn(lang, &txt, &mut warnings)
.map(OffsetDateTime::date)
}),
view_count: about
.view_count_text
.and_then(|txt| util::parse_numeric_or_warn(&txt, &mut warnings)),
country: about.country.and_then(|c| util::country_from_name(&c)),
links,
},
warnings, warnings,
}) })
} }
@ -549,18 +564,7 @@ fn combine_channel_data<T>(channel_data: Channel<()>, content: T) -> Channel<T>
} }
/// Get the continuation token to fetch channel videos in the given order /// Get the continuation token to fetch channel videos in the given order
fn order_ctoken(channel_id: &str, tab: ChannelVideoTab, order: ChannelOrder) -> String { fn order_ctoken(
_order_ctoken(
channel_id,
tab,
order,
&format!("\n${}", util::random_uuid()),
)
}
/// Get the continuation token to fetch channel videos in the given order
/// (fixed targetId for testing)
fn _order_ctoken(
channel_id: &str, channel_id: &str,
tab: ChannelVideoTab, tab: ChannelVideoTab,
order: ChannelOrder, order: ChannelOrder,
@ -589,6 +593,32 @@ fn _order_ctoken(
pb.to_base64() pb.to_base64()
} }
/// Get the continuation token to fetch channel
fn channel_info_ctoken(channel_id: &str, target_id: &str) -> String {
let mut pb_3 = ProtoBuilder::new();
pb_3.string(19, target_id);
let mut pb_110 = ProtoBuilder::new();
pb_110.embedded(3, pb_3);
let mut pbi = ProtoBuilder::new();
pbi.embedded(110, pb_110);
let mut pb_80226972 = ProtoBuilder::new();
pb_80226972.string(2, channel_id);
pb_80226972.string(3, &pbi.to_base64());
let mut pb = ProtoBuilder::new();
pb.embedded(80_226_972, pb_80226972);
pb.to_base64()
}
/// Create a random UUId to build continuation tokens
fn random_target() -> String {
format!("\n${}", util::random_uuid())
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use std::{fs::File, io::BufReader}; use std::{fs::File, io::BufReader};
@ -604,7 +634,7 @@ mod tests {
util::tests::TESTFILES, util::tests::TESTFILES,
}; };
use super::_order_ctoken; use super::{channel_info_ctoken, order_ctoken};
#[rstest] #[rstest]
#[case::base("videos_base", "UC2DjFE7Xf11URZqWBigcVOQ")] #[case::base("videos_base", "UC2DjFE7Xf11URZqWBigcVOQ")]
@ -668,10 +698,10 @@ mod tests {
let json_path = path!(*TESTFILES / "channel" / "channel_info.json"); let json_path = path!(*TESTFILES / "channel" / "channel_info.json");
let json_file = File::open(json_path).unwrap(); let json_file = File::open(json_path).unwrap();
let channel: response::Channel = let channel: response::ChannelAbout =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Channel<ChannelInfo>> = channel let map_res: MapResult<ChannelInfo> = channel
.map_response("UC2DjFE7Xf11URZqWBigcVOQ", Language::En, None, None) .map_response("UC2DjFE7Xf11U-RZqWBigcVOQ", Language::En, None, None)
.unwrap(); .unwrap();
assert!( assert!(
@ -683,10 +713,10 @@ mod tests {
} }
#[test] #[test]
fn order_ctoken() { fn t_order_ctoken() {
let channel_id = "UCXuqSBlHAE6Xw-yeJA0Tunw"; let channel_id = "UCXuqSBlHAE6Xw-yeJA0Tunw";
let videos_popular_token = _order_ctoken( let videos_popular_token = order_ctoken(
channel_id, channel_id,
ChannelVideoTab::Videos, ChannelVideoTab::Videos,
ChannelOrder::Popular, ChannelOrder::Popular,
@ -694,7 +724,7 @@ mod tests {
); );
assert_eq!(videos_popular_token, "4qmFsgJkEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaSDhnWXVHaXg2S2hJbUNpUTJORFl4WkRkak9DMHdNREF3TFRJd05EQXRPRGRoWVMwd09EbGxNRGd5TjJVME1qQVlBZyUzRCUzRA%3D%3D"); assert_eq!(videos_popular_token, "4qmFsgJkEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaSDhnWXVHaXg2S2hJbUNpUTJORFl4WkRkak9DMHdNREF3TFRJd05EQXRPRGRoWVMwd09EbGxNRGd5TjJVME1qQVlBZyUzRCUzRA%3D%3D");
let shorts_popular_token = _order_ctoken( let shorts_popular_token = order_ctoken(
channel_id, channel_id,
ChannelVideoTab::Shorts, ChannelVideoTab::Shorts,
ChannelOrder::Popular, ChannelOrder::Popular,
@ -702,7 +732,7 @@ mod tests {
); );
assert_eq!(shorts_popular_token, "4qmFsgJkEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaSDhnWXVHaXhTS2hJbUNpUTJORFkzT1dabVlpMHdNREF3TFRJMllqTXRZVEZpWkMwMU9ESTBNamxrTW1NM09UUVlBZyUzRCUzRA%3D%3D"); assert_eq!(shorts_popular_token, "4qmFsgJkEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaSDhnWXVHaXhTS2hJbUNpUTJORFkzT1dabVlpMHdNREF3TFRJMllqTXRZVEZpWkMwMU9ESTBNamxrTW1NM09UUVlBZyUzRCUzRA%3D%3D");
let live_popular_token = _order_ctoken( let live_popular_token = order_ctoken(
channel_id, channel_id,
ChannelVideoTab::Live, ChannelVideoTab::Live,
ChannelOrder::Popular, ChannelOrder::Popular,
@ -710,4 +740,12 @@ mod tests {
); );
assert_eq!(live_popular_token, "4qmFsgJkEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaSDhnWXVHaXh5S2hJbUNpUTJORFk1TXpBMk9TMHdNREF3TFRKaE1XVXRPR00zWkMwMU9ESTBNamxpWkRWaVlUZ1lBZyUzRCUzRA%3D%3D"); assert_eq!(live_popular_token, "4qmFsgJkEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaSDhnWXVHaXh5S2hJbUNpUTJORFk1TXpBMk9TMHdNREF3TFRKaE1XVXRPR00zWkMwMU9ESTBNamxpWkRWaVlUZ1lBZyUzRCUzRA%3D%3D");
} }
#[test]
fn t_channel_info_ctoken() {
let channel_id = "UCh8gHdtzO2tXd593_bjErWg";
let token = channel_info_ctoken(channel_id, "\n$655b339a-0000-20b9-92dc-582429d254b4");
assert_eq!(token, "4qmFsgJgEhhVQ2g4Z0hkdHpPMnRYZDU5M19iakVyV2caRDhnWXJHaW1hQVNZS0pEWTFOV0l6TXpsaExUQXdNREF0TWpCaU9TMDVNbVJqTFRVNE1qUXlPV1F5TlRSaU5BJTNEJTNE");
}
} }

View file

@ -4,6 +4,7 @@ use crate::{
error::{Error, ExtractionError}, error::{Error, ExtractionError},
model::ChannelRss, model::ChannelRss,
report::{Report, RustyPipeInfo}, report::{Report, RustyPipeInfo},
util,
}; };
use super::{response, RustyPipeQuery}; use super::{response, RustyPipeQuery};
@ -36,8 +37,11 @@ impl RustyPipeQuery {
_ => e, _ => e,
})?; })?;
match quick_xml::de::from_str::<response::ChannelRss>(&xml) { match quick_xml::de::from_str::<response::ChannelRss>(&xml)
Ok(feed) => Ok(feed.into()), .map_err(|e| ExtractionError::InvalidData(e.to_string().into()))
.and_then(|feed| feed.map_response(channel_id))
{
Ok(res) => Ok(res),
Err(e) => { Err(e) => {
if let Some(reporter) = &self.client.inner.reporter { if let Some(reporter) = &self.client.inner.reporter {
let report = Report { let report = Report {
@ -59,38 +63,94 @@ impl RustyPipeQuery {
reporter.report(&report); reporter.report(&report);
} }
Err(Error::Extraction(e))
Err(
ExtractionError::InvalidData(format!("could not deserialize xml: {e}").into())
.into(),
)
} }
} }
} }
} }
impl response::ChannelRss {
fn map_response(self, id: &str) -> Result<ChannelRss, ExtractionError> {
let channel_id = if self.channel_id.is_empty() {
self.entry
.iter()
.find_map(|entry| {
Some(entry.channel_id.as_str())
.filter(|id| id.is_empty())
.map(str::to_owned)
})
.or_else(|| {
self.author
.uri
.strip_prefix("https://www.youtube.com/channel/")
.and_then(|id| {
if util::CHANNEL_ID_REGEX.is_match(id) {
Some(id.to_owned())
} else {
None
}
})
})
.ok_or(ExtractionError::InvalidData(
"could not get channel id".into(),
))?
} else if self.channel_id.len() == 22 {
// As of November 2023, YouTube seems to output channel IDs without the UC prefix
format!("UC{}", self.channel_id)
} else {
self.channel_id
};
if channel_id != id {
return Err(ExtractionError::WrongResult(format!(
"got wrong channel id {channel_id}, expected {id}",
)));
}
Ok(ChannelRss {
id: channel_id,
name: self.title,
videos: self
.entry
.into_iter()
.map(|item| crate::model::ChannelRssVideo {
id: item.video_id,
name: item.title,
description: item.media_group.description,
thumbnail: item.media_group.thumbnail.into(),
publish_date: item.published,
update_date: item.updated,
view_count: item.media_group.community.statistics.views,
like_count: item.media_group.community.rating.count,
})
.collect(),
create_date: self.create_date,
})
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use std::{fs::File, io::BufReader}; use std::{fs::File, io::BufReader};
use crate::{client::response, model::ChannelRss, util::tests::TESTFILES}; use crate::{client::response, util::tests::TESTFILES};
use path_macro::path; use path_macro::path;
use rstest::rstest; use rstest::rstest;
#[rstest] #[rstest]
#[case::base("base")] #[case::base("base", "UCHnyfMqiRRG1u-2MsSQLbXA")]
#[case::no_likes("no_likes")] #[case::no_likes("no_likes", "UCdfxp4cUWsWryZOy-o427dw")]
#[case::no_channel_id("no_channel_id")] #[case::no_channel_id("no_channel_id", "UCHnyfMqiRRG1u-2MsSQLbXA")]
fn map_channel_rss(#[case] name: &str) { #[case::trimmed_channel_id("trimmed_channel_id", "UCHnyfMqiRRG1u-2MsSQLbXA")]
fn map_channel_rss(#[case] name: &str, #[case] id: &str) {
let xml_path = path!(*TESTFILES / "channel_rss" / format!("{}.xml", name)); let xml_path = path!(*TESTFILES / "channel_rss" / format!("{}.xml", name));
let xml_file = File::open(xml_path).unwrap(); let xml_file = File::open(xml_path).unwrap();
let feed: response::ChannelRss = let feed: response::ChannelRss =
quick_xml::de::from_reader(BufReader::new(xml_file)).unwrap(); quick_xml::de::from_reader(BufReader::new(xml_file)).unwrap();
let map_res: ChannelRss = feed.into(); let map_res = feed.map_response(id).unwrap();
insta::assert_ron_snapshot!(format!("map_channel_rss_{}", name), map_res); insta::assert_ron_snapshot!(format!("map_channel_rss_{}", name), map_res);
} }
} }

View file

@ -235,7 +235,6 @@ fn map_artist_page(
} }
} }
mapper.check_unknown()?;
let mut mapped = mapper.group_items(); let mut mapped = mapper.group_items();
static WIKIPEDIA_REGEX: Lazy<Regex> = static WIKIPEDIA_REGEX: Lazy<Regex> =
@ -332,7 +331,6 @@ impl MapResponse<Vec<AlbumItem>> for response::MusicArtistAlbums {
mapper.map_response(grid.grid_renderer.items); mapper.map_response(grid.grid_renderer.items);
} }
mapper.check_unknown()?;
let mapped = mapper.group_items(); let mapped = mapper.group_items();
Ok(MapResult { Ok(MapResult {

View file

@ -98,6 +98,7 @@ impl MapResponse<MusicCharts> for response::MusicCharts {
h.music_carousel_shelf_basic_header_renderer h.music_carousel_shelf_basic_header_renderer
.more_content_button .more_content_button
.and_then(|btn| btn.button_renderer.navigation_endpoint.music_page()) .and_then(|btn| btn.button_renderer.navigation_endpoint.music_page())
.map(|mp| (mp.typ, mp.id))
}) { }) {
Some((MusicPageType::Playlist, id)) => { Some((MusicPageType::Playlist, id)) => {
// Top music videos (first shelf with associated playlist) // Top music videos (first shelf with associated playlist)
@ -120,10 +121,6 @@ impl MapResponse<MusicCharts> for response::MusicCharts {
response::music_charts::ItemSection::None => {} response::music_charts::ItemSection::None => {}
}); });
mapper_top.check_unknown()?;
mapper_trending.check_unknown()?;
mapper_other.check_unknown()?;
let mapped_top = mapper_top.conv_items::<TrackItem>(); let mapped_top = mapper_top.conv_items::<TrackItem>();
let mut mapped_trending = mapper_trending.conv_items::<TrackItem>(); let mut mapped_trending = mapper_trending.conv_items::<TrackItem>();
let mut mapped_other = mapper_other.group_items(); let mut mapped_other = mapper_other.group_items();

View file

@ -387,9 +387,6 @@ impl MapResponse<MusicRelated> for response::MusicRelated {
_ => {} _ => {}
}); });
mapper.check_unknown()?;
mapper_tracks.check_unknown()?;
let mapped_tracks = mapper_tracks.conv_items(); let mapped_tracks = mapper_tracks.conv_items();
let mut mapped = mapper.group_items(); let mut mapped = mapper.group_items();

View file

@ -75,7 +75,6 @@ impl<T: FromYtItem> MapResponse<Vec<T>> for response::MusicNew {
let mut mapper = MusicListMapper::new(lang); let mut mapper = MusicListMapper::new(lang);
mapper.map_response(items); mapper.map_response(items);
mapper.check_unknown()?;
Ok(mapper.conv_items()) Ok(mapper.conv_items())
} }

View file

@ -174,7 +174,6 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
let mut mapper = MusicListMapper::new(lang); let mut mapper = MusicListMapper::new(lang);
mapper.map_response(shelf.contents); mapper.map_response(shelf.contents);
mapper.check_unknown()?;
let map_res = mapper.conv_items(); let map_res = mapper.conv_items();
let ctoken = shelf let ctoken = shelf

View file

@ -266,7 +266,6 @@ impl MapResponse<MusicSearchResult> for response::MusicSearch {
response::music_search::ItemSection::None => {} response::music_search::ItemSection::None => {}
}); });
mapper.check_unknown()?;
let map_res = mapper.group_items(); let map_res = mapper.group_items();
Ok(MapResult { Ok(MapResult {
@ -325,7 +324,6 @@ impl<T: FromYtItem> MapResponse<MusicSearchFiltered<T>> for response::MusicSearc
response::music_search::ItemSection::None => {} response::music_search::ItemSection::None => {}
}); });
mapper.check_unknown()?;
let map_res = mapper.conv_items(); let map_res = mapper.conv_items();
Ok(MapResult { Ok(MapResult {
@ -371,7 +369,6 @@ impl MapResponse<MusicSearchSuggestion> for response::MusicSearchSuggestion {
} }
} }
mapper.check_unknown()?;
let map_res = mapper.conv_items(); let map_res = mapper.conv_items();
Ok(MapResult { Ok(MapResult {

View file

@ -2,10 +2,10 @@ use serde::Deserialize;
use serde_with::{rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError}; use serde_with::{rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError};
use super::{ use super::{
video_item::YouTubeListRenderer, Alert, ChannelBadge, ContentsRenderer, ResponseContext, video_item::YouTubeListRenderer, Alert, ChannelBadge, ContentsRenderer, ContinuationActionWrap,
Thumbnails, TwoColumnBrowseResults, ResponseContext, Thumbnails, TwoColumnBrowseResults,
}; };
use crate::serializer::text::Text; use crate::serializer::text::{AttributedText, Text, TextComponent};
#[serde_as] #[serde_as]
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
@ -145,3 +145,66 @@ pub(crate) struct MicroformatDataRenderer {
#[serde(default)] #[serde(default)]
pub tags: Vec<String>, pub tags: Vec<String>,
} }
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ChannelAbout {
#[serde_as(as = "VecSkipError<_>")]
pub on_response_received_endpoints: Vec<ContinuationActionWrap<AboutChannelRendererWrap>>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct AboutChannelRendererWrap {
pub about_channel_renderer: AboutChannelRenderer,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct AboutChannelRenderer {
pub metadata: ChannelMetadata,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ChannelMetadata {
pub about_channel_view_model: ChannelMetadataView,
}
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ChannelMetadataView {
pub channel_id: String,
pub canonical_channel_url: String,
pub country: Option<String>,
#[serde(default)]
pub description: String,
#[serde_as(as = "Option<Text>")]
pub joined_date_text: Option<String>,
#[serde_as(as = "Option<Text>")]
pub subscriber_count_text: Option<String>,
#[serde_as(as = "Option<Text>")]
pub video_count_text: Option<String>,
#[serde_as(as = "Option<Text>")]
pub view_count_text: Option<String>,
#[serde(default)]
pub links: Vec<ExternalLink>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ExternalLink {
pub channel_external_link_view_model: ExternalLinkInner,
}
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ExternalLinkInner {
#[serde_as(as = "AttributedText")]
pub title: TextComponent,
#[serde_as(as = "AttributedText")]
pub link: TextComponent,
}

View file

@ -1,8 +1,6 @@
use serde::Deserialize; use serde::Deserialize;
use time::OffsetDateTime; use time::OffsetDateTime;
use crate::util;
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
pub(crate) struct ChannelRss { pub(crate) struct ChannelRss {
#[serde(rename = "channelId")] #[serde(rename = "channelId")]
@ -80,52 +78,3 @@ impl From<Thumbnail> for crate::model::Thumbnail {
} }
} }
} }
impl From<ChannelRss> for crate::model::ChannelRss {
fn from(feed: ChannelRss) -> Self {
let id = if feed.channel_id.is_empty() {
feed.entry
.iter()
.find_map(|entry| {
Some(entry.channel_id.as_str())
.filter(|id| id.is_empty())
.map(str::to_owned)
})
.or_else(|| {
feed.author
.uri
.strip_prefix("https://www.youtube.com/channel/")
.and_then(|id| {
if util::CHANNEL_ID_REGEX.is_match(id) {
Some(id.to_owned())
} else {
None
}
})
})
.unwrap_or_default()
} else {
feed.channel_id
};
Self {
id,
name: feed.title,
videos: feed
.entry
.into_iter()
.map(|item| crate::model::ChannelRssVideo {
id: item.video_id,
name: item.title,
description: item.media_group.description,
thumbnail: item.media_group.thumbnail.into(),
publish_date: item.published,
update_date: item.updated,
view_count: item.media_group.community.statistics.views,
like_count: item.media_group.community.rating.count,
})
.collect(),
create_date: feed.create_date,
}
}
}

View file

@ -16,6 +16,7 @@ pub(crate) mod video_details;
pub(crate) mod video_item; pub(crate) mod video_item;
pub(crate) use channel::Channel; pub(crate) use channel::Channel;
pub(crate) use channel::ChannelAbout;
pub(crate) use music_artist::MusicArtist; pub(crate) use music_artist::MusicArtist;
pub(crate) use music_artist::MusicArtistAlbums; pub(crate) use music_artist::MusicArtistAlbums;
pub(crate) use music_charts::MusicCharts; pub(crate) use music_charts::MusicCharts;
@ -208,7 +209,7 @@ pub(crate) struct Continuation {
alias = "onResponseReceivedEndpoints" alias = "onResponseReceivedEndpoints"
)] )]
#[serde_as(as = "Option<VecSkipError<_>>")] #[serde_as(as = "Option<VecSkipError<_>>")]
pub on_response_received_actions: Option<Vec<ContinuationActionWrap>>, pub on_response_received_actions: Option<Vec<ContinuationActionWrap<YouTubeListItem>>>,
/// Used for channel video rich grid renderer /// Used for channel video rich grid renderer
/// ///
/// A/B test seen on 19.10.2022 /// A/B test seen on 19.10.2022
@ -217,15 +218,15 @@ pub(crate) struct Continuation {
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub(crate) struct ContinuationActionWrap { pub(crate) struct ContinuationActionWrap<T> {
#[serde(alias = "reloadContinuationItemsCommand")] #[serde(alias = "reloadContinuationItemsCommand")]
pub append_continuation_items_action: ContinuationAction, pub append_continuation_items_action: ContinuationAction<T>,
} }
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub(crate) struct ContinuationAction { pub(crate) struct ContinuationAction<T> {
pub continuation_items: MapResult<Vec<YouTubeListItem>>, pub continuation_items: MapResult<Vec<T>>,
} }
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]

View file

@ -2,14 +2,13 @@ use serde::Deserialize;
use serde_with::{rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError}; use serde_with::{rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError};
use crate::{ use crate::{
error::ExtractionError,
model::{ model::{
self, traits::FromYtItem, AlbumId, AlbumItem, AlbumType, ArtistId, ArtistItem, ChannelId, self, traits::FromYtItem, AlbumId, AlbumItem, AlbumType, ArtistId, ArtistItem, ChannelId,
MusicItem, MusicItemType, MusicPlaylistItem, TrackItem, MusicItem, MusicItemType, MusicPlaylistItem, TrackItem,
}, },
param::Language, param::Language,
serializer::{ serializer::{
text::{Text, TextComponents}, text::{Text, TextComponent, TextComponents},
MapResult, MapResult,
}, },
util::{self, dictionary}, util::{self, dictionary},
@ -17,7 +16,7 @@ use crate::{
use super::{ use super::{
url_endpoint::{ url_endpoint::{
BrowseEndpointWrap, MusicPageType, MusicVideoType, NavigationEndpoint, PageType, BrowseEndpointWrap, MusicPage, MusicPageType, MusicVideoType, NavigationEndpoint, PageType,
}, },
ContentsRenderer, MusicContinuationData, Thumbnails, ThumbnailsWrap, ContentsRenderer, MusicContinuationData, Thumbnails, ThumbnailsWrap,
}; };
@ -434,8 +433,6 @@ pub(crate) struct MusicListMapper {
search_suggestion: bool, search_suggestion: bool,
items: Vec<MusicItem>, items: Vec<MusicItem>,
warnings: Vec<String>, warnings: Vec<String>,
/// True if unknown items were mapped
has_unknown: bool,
} }
#[derive(Debug)] #[derive(Debug)]
@ -456,7 +453,6 @@ impl MusicListMapper {
search_suggestion: false, search_suggestion: false,
items: Vec::new(), items: Vec::new(),
warnings: Vec::new(), warnings: Vec::new(),
has_unknown: false,
} }
} }
@ -469,7 +465,6 @@ impl MusicListMapper {
search_suggestion: true, search_suggestion: true,
items: Vec::new(), items: Vec::new(),
warnings: Vec::new(), warnings: Vec::new(),
has_unknown: false,
} }
} }
@ -483,7 +478,6 @@ impl MusicListMapper {
search_suggestion: false, search_suggestion: false,
items: Vec::new(), items: Vec::new(),
warnings: Vec::new(), warnings: Vec::new(),
has_unknown: false,
} }
} }
@ -497,7 +491,6 @@ impl MusicListMapper {
search_suggestion: false, search_suggestion: false,
items: Vec::new(), items: Vec::new(),
warnings: Vec::new(), warnings: Vec::new(),
has_unknown: false,
} }
} }
@ -545,55 +538,44 @@ impl MusicListMapper {
.thumbnails .thumbnails
.first(); .first();
let pt_id = item let music_page = item
.navigation_endpoint .navigation_endpoint
.and_then(NavigationEndpoint::music_page) .and_then(NavigationEndpoint::music_page)
.or_else(|| { .or_else(|| {
c1.and_then(|c1| { c1.and_then(|c1| {
c1.renderer.text.0.into_iter().next().and_then(|t| match t { c1.renderer
crate::serializer::text::TextComponent::Video { .text
video_id, vtype, .. .0
} => Some((MusicPageType::Track { vtype }, video_id)), .into_iter()
crate::serializer::text::TextComponent::Browse { .next()
page_type, .and_then(TextComponent::music_page)
browse_id,
..
} => Some((page_type.into(), browse_id)),
_ => None,
})
}) })
}) })
.or_else(|| { .or_else(|| {
item.playlist_item_data.map(|d| { item.playlist_item_data.map(|d| MusicPage {
( id: d.video_id,
MusicPageType::Track { typ: MusicPageType::Track {
vtype: MusicVideoType::from_is_video( vtype: MusicVideoType::from_is_video(
self.album.is_none() self.album.is_none()
&& !first_tn && !first_tn.map(|tn| tn.height == tn.width).unwrap_or_default(),
.map(|tn| tn.height == tn.width) ),
.unwrap_or_default(), },
),
},
d.video_id,
)
}) })
}) })
.or_else(|| { .or_else(|| {
first_tn.and_then(|tn| { first_tn.and_then(|tn| {
util::video_id_from_thumbnail_url(&tn.url).map(|id| { util::video_id_from_thumbnail_url(&tn.url).map(|id| MusicPage {
( id,
MusicPageType::Track { typ: MusicPageType::Track {
vtype: MusicVideoType::from_is_video( vtype: MusicVideoType::from_is_video(
self.album.is_none() && tn.width != tn.height, self.album.is_none() && tn.width != tn.height,
), ),
}, },
id,
)
}) })
}) })
}); });
match pt_id { match music_page.map(|mp| (mp.typ, mp.id)) {
// Track // Track
Some((MusicPageType::Track { vtype }, id)) => { Some((MusicPageType::Track { vtype }, id)) => {
let title = title.ok_or_else(|| format!("track {id}: could not get title"))?; let title = title.ok_or_else(|| format!("track {id}: could not get title"))?;
@ -852,10 +834,6 @@ impl MusicListMapper {
} }
// Tracks were already handled above // Tracks were already handled above
MusicPageType::Track { .. } => unreachable!(), MusicPageType::Track { .. } => unreachable!(),
MusicPageType::Unknown => {
self.has_unknown = true;
Ok(None)
}
} }
} }
None => { None => {
@ -875,12 +853,12 @@ impl MusicListMapper {
let subtitle_p2 = subtitle_parts.next(); let subtitle_p2 = subtitle_parts.next();
match item.navigation_endpoint.music_page() { match item.navigation_endpoint.music_page() {
Some((page_type, id)) => match page_type { Some(music_page) => match music_page.typ {
MusicPageType::Track { vtype } => { MusicPageType::Track { vtype } => {
let (artists, by_va) = map_artists(subtitle_p1); let (artists, by_va) = map_artists(subtitle_p1);
self.items.push(MusicItem::Track(TrackItem { self.items.push(MusicItem::Track(TrackItem {
id, id: music_page.id,
name: item.title, name: item.title,
duration: None, duration: None,
cover: item.thumbnail_renderer.into(), cover: item.thumbnail_renderer.into(),
@ -910,7 +888,7 @@ impl MusicListMapper {
}); });
self.items.push(MusicItem::Artist(ArtistItem { self.items.push(MusicItem::Artist(ArtistItem {
id, id: music_page.id,
name: item.title, name: item.title,
avatar: item.thumbnail_renderer.into(), avatar: item.thumbnail_renderer.into(),
subscriber_count, subscriber_count,
@ -947,12 +925,15 @@ impl MusicListMapper {
(Vec::new(), true) (Vec::new(), true)
} }
_ => { _ => {
return Err(format!("could not parse subtitle of album {id}")); return Err(format!(
"could not parse subtitle of album {}",
music_page.id
));
} }
}; };
self.items.push(MusicItem::Album(AlbumItem { self.items.push(MusicItem::Album(AlbumItem {
id, id: music_page.id,
name: item.title, name: item.title,
cover: item.thumbnail_renderer.into(), cover: item.thumbnail_renderer.into(),
artist_id: artists.first().and_then(|a| a.id.clone()), artist_id: artists.first().and_then(|a| a.id.clone()),
@ -974,7 +955,7 @@ impl MusicListMapper {
.and_then(|p| p.0.into_iter().find_map(|c| ChannelId::try_from(c).ok())); .and_then(|p| p.0.into_iter().find_map(|c| ChannelId::try_from(c).ok()));
self.items.push(MusicItem::Playlist(MusicPlaylistItem { self.items.push(MusicItem::Playlist(MusicPlaylistItem {
id, id: music_page.id,
name: item.title, name: item.title,
thumbnail: item.thumbnail_renderer.into(), thumbnail: item.thumbnail_renderer.into(),
channel, channel,
@ -984,10 +965,6 @@ impl MusicListMapper {
Ok(Some(MusicItemType::Playlist)) Ok(Some(MusicItemType::Playlist))
} }
MusicPageType::None => Ok(None), MusicPageType::None => Ok(None),
MusicPageType::Unknown => {
self.has_unknown = true;
Ok(None)
}
}, },
None => Err("could not determine item type".to_owned()), None => Err("could not determine item type".to_owned()),
} }
@ -1009,7 +986,7 @@ impl MusicListMapper {
let subtitle_p4 = subtitle_parts.next(); let subtitle_p4 = subtitle_parts.next();
let item_type = match card.on_tap.music_page() { let item_type = match card.on_tap.music_page() {
Some((page_type, id)) => match page_type { Some(music_page) => match music_page.typ {
MusicPageType::Artist => { MusicPageType::Artist => {
let subscriber_count = subtitle_p2.and_then(|p| { let subscriber_count = subtitle_p2.and_then(|p| {
util::parse_large_numstr_or_warn( util::parse_large_numstr_or_warn(
@ -1020,7 +997,7 @@ impl MusicListMapper {
}); });
self.items.push(MusicItem::Artist(ArtistItem { self.items.push(MusicItem::Artist(ArtistItem {
id, id: music_page.id,
name: card.title, name: card.title,
avatar: card.thumbnail.into(), avatar: card.thumbnail.into(),
subscriber_count, subscriber_count,
@ -1034,7 +1011,7 @@ impl MusicListMapper {
.unwrap_or_default(); .unwrap_or_default();
self.items.push(MusicItem::Album(AlbumItem { self.items.push(MusicItem::Album(AlbumItem {
id, id: music_page.id,
name: card.title, name: card.title,
cover: card.thumbnail.into(), cover: card.thumbnail.into(),
artist_id: artists.first().and_then(|a| a.id.clone()), artist_id: artists.first().and_then(|a| a.id.clone()),
@ -1050,7 +1027,7 @@ impl MusicListMapper {
let (artists, by_va) = map_artists(subtitle_p3); let (artists, by_va) = map_artists(subtitle_p3);
self.items.push(MusicItem::Track(TrackItem { self.items.push(MusicItem::Track(TrackItem {
id, id: music_page.id,
name: card.title, name: card.title,
duration: None, duration: None,
cover: card.thumbnail.into(), cover: card.thumbnail.into(),
@ -1087,7 +1064,7 @@ impl MusicListMapper {
}; };
self.items.push(MusicItem::Track(TrackItem { self.items.push(MusicItem::Track(TrackItem {
id, id: music_page.id,
name: card.title, name: card.title,
duration, duration,
cover: card.thumbnail.into(), cover: card.thumbnail.into(),
@ -1113,7 +1090,7 @@ impl MusicListMapper {
subtitle_p3.and_then(|p| util::parse_numeric(p.first_str()).ok()); subtitle_p3.and_then(|p| util::parse_numeric(p.first_str()).ok());
self.items.push(MusicItem::Playlist(MusicPlaylistItem { self.items.push(MusicItem::Playlist(MusicPlaylistItem {
id, id: music_page.id,
name: card.title, name: card.title,
thumbnail: card.thumbnail.into(), thumbnail: card.thumbnail.into(),
channel, channel,
@ -1123,10 +1100,6 @@ impl MusicListMapper {
Some(MusicItemType::Playlist) Some(MusicItemType::Playlist)
} }
MusicPageType::None => None, MusicPageType::None => None,
MusicPageType::Unknown => {
self.has_unknown = true;
None
}
}, },
None => { None => {
self.warnings self.warnings
@ -1201,20 +1174,6 @@ impl MusicListMapper {
warnings: self.warnings, warnings: self.warnings,
} }
} }
/// Sometimes the YT Music API returns responses containing unknown items.
///
/// In this case, the response data is likely missing some fields, which leads to
/// parsing errors and wrong data being extracted.
///
/// Therefore it is safest to discard such responses and retry the request.
pub fn check_unknown(&self) -> Result<(), ExtractionError> {
if self.has_unknown {
Err(ExtractionError::InvalidData("unknown YTM items".into()))
} else {
Ok(())
}
}
} }
/// Map TextComponents containing artist names to a list of artists and a 'Various Artists' flag /// Map TextComponents containing artist names to a list of artists and a 'Various Artists' flag

View file

@ -185,6 +185,10 @@ pub(crate) enum PageType {
Channel, Channel,
#[serde(rename = "MUSIC_PAGE_TYPE_PLAYLIST", alias = "WEB_PAGE_TYPE_PLAYLIST")] #[serde(rename = "MUSIC_PAGE_TYPE_PLAYLIST", alias = "WEB_PAGE_TYPE_PLAYLIST")]
Playlist, Playlist,
#[serde(rename = "MUSIC_PAGE_TYPE_PODCAST_SHOW_DETAIL_PAGE")]
Podcast,
#[serde(rename = "MUSIC_PAGE_TYPE_NON_MUSIC_AUDIO_TRACK_PAGE")]
Episode,
#[default] #[default]
Unknown, Unknown,
} }
@ -195,6 +199,13 @@ impl PageType {
PageType::Artist | PageType::Channel => Some(UrlTarget::Channel { id }), PageType::Artist | PageType::Channel => Some(UrlTarget::Channel { id }),
PageType::Album => Some(UrlTarget::Album { id }), PageType::Album => Some(UrlTarget::Album { id }),
PageType::Playlist => Some(UrlTarget::Playlist { id }), PageType::Playlist => Some(UrlTarget::Playlist { id }),
PageType::Podcast => Some(UrlTarget::Playlist {
id: util::strip_prefix(&id, util::PODCAST_PLAYLIST_PREFIX),
}),
PageType::Episode => Some(UrlTarget::Video {
id: util::strip_prefix(&id, util::PODCAST_EPISODE_PREFIX),
start_time: 0,
}),
PageType::Unknown => None, PageType::Unknown => None,
} }
} }
@ -206,7 +217,6 @@ pub(crate) enum MusicPageType {
Album, Album,
Playlist, Playlist,
Track { vtype: MusicVideoType }, Track { vtype: MusicVideoType },
Unknown,
None, None,
} }
@ -215,16 +225,40 @@ impl From<PageType> for MusicPageType {
match t { match t {
PageType::Artist => MusicPageType::Artist, PageType::Artist => MusicPageType::Artist,
PageType::Album => MusicPageType::Album, PageType::Album => MusicPageType::Album,
PageType::Playlist => MusicPageType::Playlist, PageType::Playlist | PageType::Podcast => MusicPageType::Playlist,
PageType::Channel => MusicPageType::None, PageType::Channel | PageType::Unknown => MusicPageType::None,
PageType::Unknown => MusicPageType::Unknown, PageType::Episode => MusicPageType::Track {
vtype: MusicVideoType::Episode,
},
}
}
}
pub(crate) struct MusicPage {
pub id: String,
pub typ: MusicPageType,
}
impl MusicPage {
/// Create a new MusicPage object, applying the required ID fixes when
/// mapping a browse link
pub fn from_browse(mut id: String, typ: PageType) -> Self {
if typ == PageType::Podcast {
id = util::strip_prefix(&id, util::PODCAST_PLAYLIST_PREFIX);
} else if typ == PageType::Episode && id.len() == 15 {
id = util::strip_prefix(&id, util::PODCAST_EPISODE_PREFIX);
}
Self {
id,
typ: typ.into(),
} }
} }
} }
impl NavigationEndpoint { impl NavigationEndpoint {
/// Get the YouTube Music page and id from a browse/watch endpoint /// Get the YouTube Music page and id from a browse/watch endpoint
pub(crate) fn music_page(self) -> Option<(MusicPageType, String)> { pub(crate) fn music_page(self) -> Option<MusicPage> {
match self { match self {
NavigationEndpoint::Watch { watch_endpoint } => { NavigationEndpoint::Watch { watch_endpoint } => {
if watch_endpoint if watch_endpoint
@ -233,17 +267,20 @@ impl NavigationEndpoint {
.unwrap_or_default() .unwrap_or_default()
{ {
// Genre radios (e.g. "pop radio") will be skipped // Genre radios (e.g. "pop radio") will be skipped
Some((MusicPageType::None, watch_endpoint.video_id)) Some(MusicPage {
id: watch_endpoint.video_id,
typ: MusicPageType::None,
})
} else { } else {
Some(( Some(MusicPage {
MusicPageType::Track { id: watch_endpoint.video_id,
typ: MusicPageType::Track {
vtype: watch_endpoint vtype: watch_endpoint
.watch_endpoint_music_supported_configs .watch_endpoint_music_supported_configs
.watch_endpoint_music_config .watch_endpoint_music_config
.music_video_type, .music_video_type,
}, },
watch_endpoint.video_id, })
))
} }
} }
NavigationEndpoint::Browse { NavigationEndpoint::Browse {
@ -251,9 +288,9 @@ impl NavigationEndpoint {
} => browse_endpoint } => browse_endpoint
.browse_endpoint_context_supported_configs .browse_endpoint_context_supported_configs
.map(|config| { .map(|config| {
( MusicPage::from_browse(
config.browse_endpoint_context_music_config.page_type.into(),
browse_endpoint.browse_id, browse_endpoint.browse_id,
config.browse_endpoint_context_music_config.page_type,
) )
}), }),
NavigationEndpoint::Url { .. } => None, NavigationEndpoint::Url { .. } => None,
@ -280,14 +317,4 @@ impl NavigationEndpoint {
None None
} }
} }
/// Get the sanitized URL from a url endpoint
pub(crate) fn url(&self) -> Option<String> {
match self {
NavigationEndpoint::Url { url_endpoint } => {
Some(util::sanitize_yt_url(&url_endpoint.url))
}
_ => None,
}
}
} }

View file

@ -6,15 +6,15 @@ use serde_with::{
}; };
use time::OffsetDateTime; use time::OffsetDateTime;
use super::{url_endpoint::NavigationEndpoint, ChannelBadge, ContinuationEndpoint, Thumbnails}; use super::{ChannelBadge, ContinuationEndpoint, Thumbnails};
use crate::{ use crate::{
model::{ model::{
Channel, ChannelId, ChannelInfo, ChannelItem, ChannelTag, PlaylistItem, Verification, Channel, ChannelId, ChannelItem, ChannelTag, PlaylistItem, Verification, VideoItem,
VideoItem, YouTubeItem, YouTubeItem,
}, },
param::Language, param::Language,
serializer::{ serializer::{
text::{AccessibilityText, AttributedText, Text, TextComponent}, text::{AccessibilityText, Text, TextComponent},
MapResult, MapResult,
}, },
util::{self, timeago, TryRemove}, util::{self, timeago, TryRemove},
@ -48,9 +48,6 @@ pub(crate) enum YouTubeListItem {
corrected_query: String, corrected_query: String,
}, },
/// Channel metadata (about tab)
ChannelAboutFullMetadataRenderer(ChannelFullMetadata),
/// Contains video on startpage /// Contains video on startpage
/// ///
/// Seems to be currently A/B tested on the channel page, /// Seems to be currently A/B tested on the channel page,
@ -358,47 +355,6 @@ pub(crate) struct ReelPlayerHeaderRenderer {
pub timestamp_text: String, pub timestamp_text: String,
} }
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ChannelFullMetadata {
#[serde_as(as = "Text")]
pub joined_date_text: String,
#[serde_as(as = "Option<Text>")]
pub view_count_text: Option<String>,
#[serde(default)]
#[serde_as(as = "VecSkipError<_>")]
pub primary_links: Vec<PrimaryLink>,
#[serde(default)]
// #[serde_as(as = "VecSkipError<_>")]
pub links: Vec<ExternalLink>,
}
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct PrimaryLink {
#[serde_as(as = "Text")]
pub title: String,
pub navigation_endpoint: NavigationEndpoint,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ExternalLink {
pub channel_external_link_view_model: ExternalLinkInner,
}
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ExternalLinkInner {
#[serde_as(as = "AttributedText")]
pub title: TextComponent,
#[serde_as(as = "AttributedText")]
pub link: TextComponent,
}
trait IsLive { trait IsLive {
fn is_live(&self) -> bool; fn is_live(&self) -> bool;
} }
@ -446,7 +402,6 @@ pub(crate) struct YouTubeListMapper<T> {
pub warnings: Vec<String>, pub warnings: Vec<String>,
pub ctoken: Option<String>, pub ctoken: Option<String>,
pub corrected_query: Option<String>, pub corrected_query: Option<String>,
pub channel_info: Option<ChannelInfo>,
} }
impl<T> YouTubeListMapper<T> { impl<T> YouTubeListMapper<T> {
@ -458,7 +413,6 @@ impl<T> YouTubeListMapper<T> {
warnings: Vec::new(), warnings: Vec::new(),
ctoken: None, ctoken: None,
corrected_query: None, corrected_query: None,
channel_info: None,
} }
} }
@ -476,7 +430,6 @@ impl<T> YouTubeListMapper<T> {
warnings, warnings,
ctoken: None, ctoken: None,
corrected_query: None, corrected_query: None,
channel_info: None,
} }
} }
@ -744,32 +697,6 @@ impl YouTubeListMapper<YouTubeItem> {
YouTubeListItem::ShowingResultsForRenderer { corrected_query } => { YouTubeListItem::ShowingResultsForRenderer { corrected_query } => {
self.corrected_query = Some(corrected_query); self.corrected_query = Some(corrected_query);
} }
YouTubeListItem::ChannelAboutFullMetadataRenderer(meta) => {
let mut links = meta
.primary_links
.into_iter()
.filter_map(|l| l.navigation_endpoint.url().map(|url| (l.title, url)))
.collect::<Vec<_>>();
for l in meta.links {
let l = l.channel_external_link_view_model;
if let TextComponent::Web { url, .. } = l.link {
links.push((l.title.into(), util::sanitize_yt_url(&url)));
}
}
self.channel_info = Some(ChannelInfo {
create_date: timeago::parse_textual_date_or_warn(
self.lang,
&meta.joined_date_text,
&mut self.warnings,
)
.map(OffsetDateTime::date),
view_count: meta
.view_count_text
.and_then(|txt| util::parse_numeric_or_warn(&txt, &mut self.warnings)),
links,
});
}
YouTubeListItem::RichItemRenderer { content } => { YouTubeListItem::RichItemRenderer { content } => {
self.map_item(*content); self.map_item(*content);
} }

View file

@ -2,166 +2,28 @@
source: src/client/channel.rs source: src/client/channel.rs
expression: map_res.c expression: map_res.c
--- ---
Channel( ChannelInfo(
id: "UC2DjFE7Xf11URZqWBigcVOQ", id: "UC2DjFE7Xf11URZqWBigcVOQ",
name: "EEVblog", url: "http://www.youtube.com/@EEVblog",
subscriber_count: Some(881000),
avatar: [
Thumbnail(
url: "https://yt3.ggpht.com/ytc/AMLnZu9eKk4Nd16fX4Rn1TF1G7ReluwOl6M5558FTYAM=s48-c-k-c0x00ffffff-no-rj",
width: 48,
height: 48,
),
Thumbnail(
url: "https://yt3.ggpht.com/ytc/AMLnZu9eKk4Nd16fX4Rn1TF1G7ReluwOl6M5558FTYAM=s88-c-k-c0x00ffffff-no-rj",
width: 88,
height: 88,
),
Thumbnail(
url: "https://yt3.ggpht.com/ytc/AMLnZu9eKk4Nd16fX4Rn1TF1G7ReluwOl6M5558FTYAM=s176-c-k-c0x00ffffff-no-rj",
width: 176,
height: 176,
),
],
verification: Verified,
description: "NO SCRIPT, NO FEAR, ALL OPINION\nAn off-the-cuff Video Blog about Electronics Engineering, for engineers, hobbyists, enthusiasts, hackers and Makers\nHosted by Dave Jones from Sydney Australia\n\nDONATIONS:\nBitcoin: 3KqyH1U3qrMPnkLufM2oHDU7YB4zVZeFyZ\nEthereum: 0x99ccc4d2654ba40744a1f678d9868ecb15e91206\nPayPal: david@alternatezone.com\n\nPatreon: https://www.patreon.com/eevblog\n\nEEVblog2: http://www.youtube.com/EEVblog2\nEEVdiscover: https://www.youtube.com/channel/UCkGvUEt8iQLmq3aJIMjT2qQ\n\nEMAIL:\nAdvertising/Commercial: eevblog+business@gmail.com\nFan mail: eevblog+fan@gmail.com\nHate Mail: eevblog+hate@gmail.com\n\nI DON\'T DO PAID VIDEO SPONSORSHIPS, DON\'T ASK!\n\nPLEASE:\nDo NOT ask for personal advice on something, post it in the EEVblog forum.\nI read ALL email, but please don\'t be offended if I don\'t have time to reply, I get a LOT of email.\n\nMailbag\nPO Box 7949\nBaulkham Hills NSW 2153\nAUSTRALIA", description: "NO SCRIPT, NO FEAR, ALL OPINION\nAn off-the-cuff Video Blog about Electronics Engineering, for engineers, hobbyists, enthusiasts, hackers and Makers\nHosted by Dave Jones from Sydney Australia\n\nDONATIONS:\nBitcoin: 3KqyH1U3qrMPnkLufM2oHDU7YB4zVZeFyZ\nEthereum: 0x99ccc4d2654ba40744a1f678d9868ecb15e91206\nPayPal: david@alternatezone.com\n\nPatreon: https://www.patreon.com/eevblog\n\nEEVblog2: http://www.youtube.com/EEVblog2\nEEVdiscover: https://www.youtube.com/channel/UCkGvUEt8iQLmq3aJIMjT2qQ\n\nEMAIL:\nAdvertising/Commercial: eevblog+business@gmail.com\nFan mail: eevblog+fan@gmail.com\nHate Mail: eevblog+hate@gmail.com\n\nI DON\'T DO PAID VIDEO SPONSORSHIPS, DON\'T ASK!\n\nPLEASE:\nDo NOT ask for personal advice on something, post it in the EEVblog forum.\nI read ALL email, but please don\'t be offended if I don\'t have time to reply, I get a LOT of email.\n\nMailbag\nPO Box 7949\nBaulkham Hills NSW 2153\nAUSTRALIA",
tags: [ subscriber_count: Some(920000),
"electronics", video_count: Some(1920),
"engineering", create_date: Some("2009-04-04"),
"maker", view_count: Some(199087682),
"hacker", country: Some(AU),
"design", links: [
"circuit", ("EEVblog Web Site", "http://www.eevblog.com/"),
"hardware", ("Twitter", "http://www.twitter.com/eevblog"),
"pic", ("Facebook", "http://www.facebook.com/EEVblog"),
"atmel", ("EEVdiscover", "https://www.youtube.com/channel/UCkGvUEt8iQLmq3aJIMjT2qQ"),
"oscilloscope", ("The EEVblog Forum", "http://www.eevblog.com/forum"),
"multimeter", ("EEVblog Merchandise (T-Shirts)", "http://www.eevblog.com/merch"),
"diy", ("EEVblog Donations", "http://www.eevblog.com/donations/"),
"hobby", ("Patreon", "https://www.patreon.com/eevblog"),
"review", ("SubscribeStar", "https://www.subscribestar.com/eevblog"),
"teardown", ("The AmpHour Radio Show", "http://www.theamphour.com/"),
"microcontroller", ("Flickr", "http://www.flickr.com/photos/eevblog"),
"arduino", ("EEVblog AMAZON Store", "http://www.amazon.com/gp/redirect.html?ie=UTF8&location=http%3A%2F%2Fwww.amazon.com%2F&tag=ee04-20&linkCode=ur2&camp=1789&creative=390957"),
"video", ("2nd EEVblog Channel", "http://www.youtube.com/EEVblog2"),
"blog",
"tutorial",
"how-to",
"interview",
"rant",
"industry",
"news",
"mailbag",
"dumpster diving",
"debunking",
], ],
vanity_url: Some("https://www.youtube.com/c/EevblogDave"),
banner: [
Thumbnail(
url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1060-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
width: 1060,
height: 175,
),
Thumbnail(
url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1138-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
width: 1138,
height: 188,
),
Thumbnail(
url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1707-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
width: 1707,
height: 283,
),
Thumbnail(
url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w2120-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
width: 2120,
height: 351,
),
Thumbnail(
url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w2276-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
width: 2276,
height: 377,
),
Thumbnail(
url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w2560-fcrop64=1,00005a57ffffa5a8-k-c0xffffffff-no-nd-rj",
width: 2560,
height: 424,
),
],
mobile_banner: [
Thumbnail(
url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w320-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
width: 320,
height: 88,
),
Thumbnail(
url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w640-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
width: 640,
height: 175,
),
Thumbnail(
url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w960-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
width: 960,
height: 263,
),
Thumbnail(
url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1280-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
width: 1280,
height: 351,
),
Thumbnail(
url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1440-fcrop64=1,32b75a57cd48a5a8-k-c0xffffffff-no-nd-rj",
width: 1440,
height: 395,
),
],
tv_banner: [
Thumbnail(
url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w320-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
width: 320,
height: 180,
),
Thumbnail(
url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w854-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
width: 854,
height: 480,
),
Thumbnail(
url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1280-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
width: 1280,
height: 720,
),
Thumbnail(
url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w1920-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
width: 1920,
height: 1080,
),
Thumbnail(
url: "https://yt3.ggpht.com/yIJ9ad80n49rK-YUcZLe_8bLmR-aGyg5ybDH_XKIc0GDWrC6s1Wzz8lxnq3_hux_5b6NHPZ9=w2120-fcrop64=1,00000000ffffffff-k-c0xffffffff-no-nd-rj",
width: 2120,
height: 1192,
),
],
has_shorts: false,
has_live: false,
visitor_data: Some("CgszMUUzZDlGLWxiRSipqr2ZBg%3D%3D"),
content: ChannelInfo(
create_date: Some("2009-04-04"),
view_count: Some(186854342),
links: [
("EEVblog Web Site", "http://www.eevblog.com/"),
("Twitter", "http://www.twitter.com/eevblog"),
("Facebook", "http://www.facebook.com/EEVblog"),
("EEVdiscover", "https://www.youtube.com/channel/UCkGvUEt8iQLmq3aJIMjT2qQ"),
("The EEVblog Forum", "http://www.eevblog.com/forum"),
("EEVblog Merchandise (T-Shirts)", "http://www.eevblog.com/merch"),
("EEVblog Donations", "http://www.eevblog.com/donations/"),
("Patreon", "https://www.patreon.com/eevblog"),
("SubscribeStar", "https://www.subscribestar.com/eevblog"),
("The AmpHour Radio Show", "http://www.theamphour.com/"),
("Flickr", "http://www.flickr.com/photos/eevblog"),
("EEVblog AMAZON Store", "http://www.amazon.com/gp/redirect.html?ie=UTF8&location=http%3A%2F%2Fwww.amazon.com%2F&tag=ee04-20&linkCode=ur2&camp=1789&creative=390957"),
("2nd EEVblog Channel", "http://www.youtube.com/EEVblog2"),
],
),
) )

File diff suppressed because one or more lines are too long

View file

@ -738,16 +738,31 @@ pub struct Channel<T> {
pub content: T, pub content: T,
} }
/// Additional channel metadata fetched from the "About" tab. /// Detailed channel information
#[serde_as] #[serde_as]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[non_exhaustive] #[non_exhaustive]
pub struct ChannelInfo { pub struct ChannelInfo {
/// Unique YouTube Channel-ID (e.g. `UC-lHJZR3Gqxm24_Vd_AJ5Yw`)
pub id: String,
/// Channel URL
pub url: String,
/// Channel description text
pub description: String,
/// Channel subscriber count
///
/// [`None`] if the subscriber count was hidden by the owner
/// or could not be parsed.
pub subscriber_count: Option<u64>,
/// Channel video count
pub video_count: Option<u64>,
/// Channel creation date /// Channel creation date
#[serde_as(as = "Option<DateYmd>")] #[serde_as(as = "Option<DateYmd>")]
pub create_date: Option<Date>, pub create_date: Option<Date>,
/// Channel view count /// Channel view count
pub view_count: Option<u64>, pub view_count: Option<u64>,
/// Channel origin country
pub country: Option<Country>,
/// Links to other websites or social media profiles /// Links to other websites or social media profiles
pub links: Vec<(String, String)>, pub links: Vec<(String, String)>,
} }

View file

@ -419,202 +419,207 @@ pub enum Country {
} }
/// Array of all available languages /// Array of all available languages
/// The languages are sorted by their native names. This array can be used to display
/// a language selection or to get the language code from a language name using binary search.
pub const LANGUAGES: [Language; 83] = [ pub const LANGUAGES: [Language; 83] = [
Language::Af, Language::Af,
Language::Am,
Language::Ar,
Language::As,
Language::Az, Language::Az,
Language::Be, Language::Id,
Language::Bg, Language::Ms,
Language::Bn,
Language::Bs, Language::Bs,
Language::Ca, Language::Ca,
Language::Cs,
Language::Da, Language::Da,
Language::De, Language::De,
Language::El, Language::Et,
Language::En,
Language::EnGb,
Language::EnIn, Language::EnIn,
Language::EnGb,
Language::En,
Language::Es, Language::Es,
Language::Es419, Language::Es419,
Language::EsUs, Language::EsUs,
Language::Et,
Language::Eu, Language::Eu,
Language::Fa,
Language::Fi,
Language::Fil, Language::Fil,
Language::Fr, Language::Fr,
Language::FrCa, Language::FrCa,
Language::Gl, Language::Gl,
Language::Gu,
Language::Hi,
Language::Hr, Language::Hr,
Language::Hu, Language::Zu,
Language::Hy,
Language::Id,
Language::Is,
Language::It, Language::It,
Language::Iw, Language::Sw,
Language::Ja,
Language::Ka,
Language::Kk,
Language::Km,
Language::Kn,
Language::Ko,
Language::Ky,
Language::Lo,
Language::Lt,
Language::Lv, Language::Lv,
Language::Mk, Language::Lt,
Language::Ml, Language::Hu,
Language::Mn,
Language::Mr,
Language::Ms,
Language::My,
Language::Ne,
Language::Nl, Language::Nl,
Language::No, Language::No,
Language::Or, Language::Uz,
Language::Pa,
Language::Pl, Language::Pl,
Language::Pt,
Language::PtPt, Language::PtPt,
Language::Pt,
Language::Ro, Language::Ro,
Language::Ru, Language::Sq,
Language::Si,
Language::Sk, Language::Sk,
Language::Sl, Language::Sl,
Language::Sq,
Language::Sr,
Language::SrLatn, Language::SrLatn,
Language::Fi,
Language::Sv, Language::Sv,
Language::Sw, Language::Vi,
Language::Tr,
Language::Is,
Language::Cs,
Language::El,
Language::Be,
Language::Bg,
Language::Ky,
Language::Mk,
Language::Mn,
Language::Ru,
Language::Sr,
Language::Uk,
Language::Kk,
Language::Hy,
Language::Iw,
Language::Ur,
Language::Ar,
Language::Fa,
Language::Ne,
Language::Mr,
Language::Hi,
Language::As,
Language::Bn,
Language::Pa,
Language::Gu,
Language::Or,
Language::Ta, Language::Ta,
Language::Te, Language::Te,
Language::Kn,
Language::Ml,
Language::Si,
Language::Th, Language::Th,
Language::Tr, Language::Lo,
Language::Uk, Language::My,
Language::Ur, Language::Ka,
Language::Uz, Language::Am,
Language::Vi, Language::Km,
Language::ZhCn, Language::ZhCn,
Language::ZhHk,
Language::ZhTw, Language::ZhTw,
Language::Zu, Language::ZhHk,
Language::Ja,
Language::Ko,
]; ];
/// Array of all available countries /// Array of all available countries
///
/// The countries are sorted by their english names. This array can be used to display
/// a country selection or to get the country code from a country name using binary search.
pub const COUNTRIES: [Country; 109] = [ pub const COUNTRIES: [Country; 109] = [
Country::Ae, Country::Dz,
Country::Ar, Country::Ar,
Country::At,
Country::Au, Country::Au,
Country::At,
Country::Az, Country::Az,
Country::Ba,
Country::Bd,
Country::Be,
Country::Bg,
Country::Bh, Country::Bh,
Country::Bo, Country::Bd,
Country::Br,
Country::By, Country::By,
Country::Be,
Country::Bo,
Country::Ba,
Country::Br,
Country::Bg,
Country::Kh,
Country::Ca, Country::Ca,
Country::Ch,
Country::Cl, Country::Cl,
Country::Co, Country::Co,
Country::Cr, Country::Cr,
Country::Hr,
Country::Cy, Country::Cy,
Country::Cz, Country::Cz,
Country::De,
Country::Dk, Country::Dk,
Country::Do, Country::Do,
Country::Dz,
Country::Ec, Country::Ec,
Country::Ee,
Country::Eg, Country::Eg,
Country::Es, Country::Sv,
Country::Ee,
Country::Fi, Country::Fi,
Country::Fr, Country::Fr,
Country::Gb,
Country::Ge, Country::Ge,
Country::De,
Country::Gh, Country::Gh,
Country::Gr, Country::Gr,
Country::Gt, Country::Gt,
Country::Hk,
Country::Hn, Country::Hn,
Country::Hr, Country::Hk,
Country::Hu, Country::Hu,
Country::Is,
Country::In,
Country::Id, Country::Id,
Country::Iq,
Country::Ie, Country::Ie,
Country::Il, Country::Il,
Country::In,
Country::Iq,
Country::Is,
Country::It, Country::It,
Country::Jm, Country::Jm,
Country::Jo,
Country::Jp, Country::Jp,
Country::Ke, Country::Jo,
Country::Kh,
Country::Kr,
Country::Kw,
Country::Kz, Country::Kz,
Country::Ke,
Country::Kw,
Country::La, Country::La,
Country::Lv,
Country::Lb, Country::Lb,
Country::Ly,
Country::Li, Country::Li,
Country::Lk,
Country::Lt, Country::Lt,
Country::Lu, Country::Lu,
Country::Lv, Country::My,
Country::Ly,
Country::Ma,
Country::Me,
Country::Mk,
Country::Mt, Country::Mt,
Country::Mx, Country::Mx,
Country::My, Country::Me,
Country::Ng, Country::Ma,
Country::Ni,
Country::Nl,
Country::No,
Country::Np, Country::Np,
Country::Nl,
Country::Nz, Country::Nz,
Country::Ni,
Country::Ng,
Country::Mk,
Country::No,
Country::Om, Country::Om,
Country::Pa,
Country::Pe,
Country::Pg,
Country::Ph,
Country::Pk, Country::Pk,
Country::Pl, Country::Pa,
Country::Pr, Country::Pg,
Country::Pt,
Country::Py, Country::Py,
Country::Pe,
Country::Ph,
Country::Pl,
Country::Pt,
Country::Pr,
Country::Qa, Country::Qa,
Country::Ro, Country::Ro,
Country::Rs,
Country::Ru, Country::Ru,
Country::Sa, Country::Sa,
Country::Se,
Country::Sg,
Country::Si,
Country::Sk,
Country::Sn, Country::Sn,
Country::Sv, Country::Rs,
Country::Sg,
Country::Sk,
Country::Si,
Country::Za,
Country::Kr,
Country::Es,
Country::Lk,
Country::Se,
Country::Ch,
Country::Tw,
Country::Tz,
Country::Th, Country::Th,
Country::Tn, Country::Tn,
Country::Tr, Country::Tr,
Country::Tw,
Country::Tz,
Country::Ua,
Country::Ug, Country::Ug,
Country::Ua,
Country::Ae,
Country::Gb,
Country::Us, Country::Us,
Country::Uy, Country::Uy,
Country::Ve, Country::Ve,
Country::Vn, Country::Vn,
Country::Ye, Country::Ye,
Country::Za,
Country::Zw, Country::Zw,
]; ];
@ -844,11 +849,7 @@ impl FromStr for Language {
Some(pos) => { Some(pos) => {
sub = &sub[..pos]; sub = &sub[..pos];
} }
None => { None => return Err(Error::Other("could not parse language `{s}`".into())),
return Err(Error::Other(
format!("could not parse language `{s}`").into(),
))
}
} }
} }
} }

View file

@ -6,7 +6,9 @@ use serde::{Deserialize, Deserializer};
use serde_with::{serde_as, DeserializeAs, VecSkipError}; use serde_with::{serde_as, DeserializeAs, VecSkipError};
use crate::{ use crate::{
client::response::url_endpoint::{MusicVideoType, NavigationEndpoint, PageType}, client::response::url_endpoint::{
MusicPage, MusicPageType, MusicVideoType, NavigationEndpoint, PageType,
},
model::UrlTarget, model::UrlTarget,
util, util,
}; };
@ -44,13 +46,14 @@ use crate::{
#[serde(untagged)] #[serde(untagged)]
pub(crate) enum Text { pub(crate) enum Text {
Simple { Simple {
#[serde(alias = "simpleText")] #[serde(alias = "simpleText", alias = "content")]
text: String, text: String,
}, },
Multiple { Multiple {
#[serde_as(as = "Vec<Text>")] #[serde_as(as = "Vec<Text>")]
runs: Vec<String>, runs: Vec<String>,
}, },
Str(String),
} }
impl<'de> DeserializeAs<'de, String> for Text { impl<'de> DeserializeAs<'de, String> for Text {
@ -60,7 +63,7 @@ impl<'de> DeserializeAs<'de, String> for Text {
{ {
let text = Text::deserialize(deserializer)?; let text = Text::deserialize(deserializer)?;
match text { match text {
Text::Simple { text } => Ok(text), Text::Simple { text } | Text::Str(text) => Ok(text),
Text::Multiple { runs } => Ok(runs.join("")), Text::Multiple { runs } => Ok(runs.join("")),
} }
} }
@ -73,7 +76,7 @@ impl<'de> DeserializeAs<'de, Vec<String>> for Text {
{ {
let text = Text::deserialize(deserializer)?; let text = Text::deserialize(deserializer)?;
match text { match text {
Text::Simple { text } => Ok(vec![text]), Text::Simple { text } | Text::Str(text) => Ok(vec![text]),
Text::Multiple { runs } => Ok(runs), Text::Multiple { runs } => Ok(runs),
} }
} }
@ -418,6 +421,23 @@ impl TextComponent {
| TextComponent::Text { text } => text, | TextComponent::Text { text } => text,
} }
} }
pub fn music_page(self) -> Option<MusicPage> {
match self {
TextComponent::Video {
video_id, vtype, ..
} => Some(MusicPage {
id: video_id,
typ: MusicPageType::Track { vtype },
}),
TextComponent::Browse {
page_type,
browse_id,
..
} => Some(MusicPage::from_browse(browse_id, page_type)),
_ => None,
}
}
} }
impl From<TextComponent> for String { impl From<TextComponent> for String {
@ -542,6 +562,7 @@ mod tests {
}"#, }"#,
vec!["Abo für ", "MBCkpop", " beenden?"] vec!["Abo für ", "MBCkpop", " beenden?"]
)] )]
#[case(r#"{"txt":"Hello World"}"#, vec!["Hello World"])]
fn t_deserialize_text(#[case] test_json: &str, #[case] exp: Vec<&str>) { fn t_deserialize_text(#[case] test_json: &str, #[case] exp: Vec<&str>) {
#[serde_as] #[serde_as]
#[derive(Deserialize)] #[derive(Deserialize)]

View file

@ -19,7 +19,11 @@ use rand::Rng;
use regex::Regex; use regex::Regex;
use url::Url; use url::Url;
use crate::{error::Error, param::Language, serializer::text::TextComponent}; use crate::{
error::Error,
param::{Country, Language, COUNTRIES},
serializer::text::TextComponent,
};
pub static VIDEO_ID_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"^[A-Za-z0-9_-]{11}$").unwrap()); pub static VIDEO_ID_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"^[A-Za-z0-9_-]{11}$").unwrap());
pub static CHANNEL_ID_REGEX: Lazy<Regex> = pub static CHANNEL_ID_REGEX: Lazy<Regex> =
@ -37,6 +41,8 @@ pub const DOT_SEPARATOR: &str = " • ";
pub const VARIOUS_ARTISTS: &str = "Various Artists"; pub const VARIOUS_ARTISTS: &str = "Various Artists";
pub const PLAYLIST_ID_ALBUM_PREFIX: &str = "OLAK"; pub const PLAYLIST_ID_ALBUM_PREFIX: &str = "OLAK";
pub const ARTIST_DISCOGRAPHY_PREFIX: &str = "MPAD"; pub const ARTIST_DISCOGRAPHY_PREFIX: &str = "MPAD";
pub const PODCAST_PLAYLIST_PREFIX: &str = "MPSP";
pub const PODCAST_EPISODE_PREFIX: &str = "MPED";
const CONTENT_PLAYBACK_NONCE_ALPHABET: &[u8; 64] = const CONTENT_PLAYBACK_NONCE_ALPHABET: &[u8; 64] =
b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
@ -462,6 +468,19 @@ pub fn b64_decode<T: AsRef<[u8]>>(input: T) -> Result<Vec<u8>, base64::DecodeErr
base64::engine::general_purpose::STANDARD.decode(input) base64::engine::general_purpose::STANDARD.decode(input)
} }
/// Get the country from its English name
pub fn country_from_name(name: &str) -> Option<Country> {
COUNTRIES
.binary_search_by_key(&name, Country::name)
.ok()
.map(|i| COUNTRIES[i])
}
/// Strip prefix from string if presend
pub fn strip_prefix(s: &str, prefix: &str) -> String {
s.strip_prefix(prefix).unwrap_or(s).to_string()
}
/// An iterator over the chars in a string (in str format) /// An iterator over the chars in a string (in str format)
pub struct SplitChar<'a> { pub struct SplitChar<'a> {
txt: &'a str, txt: &'a str,
@ -685,4 +704,13 @@ pub(crate) mod tests {
let res = Language::from_str(s).ok(); let res = Language::from_str(s).ok();
assert_eq!(res, expect); assert_eq!(res, expect);
} }
#[rstest]
#[case("United States", Some(Country::Us))]
#[case("Zimbabwe", Some(Country::Zw))]
#[case("foobar", None)]
fn t_country_from_name(#[case] name: &str, #[case] expect: Option<Country>) {
let res = country_from_name(name);
assert_eq!(res, expect);
}
} }

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -39,7 +39,10 @@ fn get_player_from_client(#[case] client_type: ClientType, rp: RustyPipe) {
// dbg!(&player_data); // dbg!(&player_data);
assert_eq!(player_data.details.id, "n4tK7LYFxI0"); assert_eq!(player_data.details.id, "n4tK7LYFxI0");
assert_eq!(player_data.details.name, "Spektrem - Shine [NCS Release]"); assert_eq!(
player_data.details.name,
"Spektrem - Shine | Progressive House | NCS - Copyright Free Music"
);
if client_type == ClientType::DesktopMusic { if client_type == ClientType::DesktopMusic {
assert!(player_data.details.description.is_none()); assert!(player_data.details.description.is_none());
} else { } else {
@ -68,9 +71,9 @@ fn get_player_from_client(#[case] client_type: ClientType, rp: RustyPipe) {
.unwrap(); .unwrap();
// Bitrates may change between requests // Bitrates may change between requests
assert_approx(f64::from(video.bitrate), 1_507_068.0); assert_approx(f64::from(video.bitrate), 1_851_854.0);
assert_eq!(video.average_bitrate, 1_345_149); assert_eq!(video.average_bitrate, 923_766);
assert_eq!(video.size.unwrap(), 43_553_412); assert_eq!(video.size.unwrap(), 29_909_835);
assert_eq!(video.width, 1280); assert_eq!(video.width, 1280);
assert_eq!(video.height, 720); assert_eq!(video.height, 720);
assert_eq!(video.fps, 30); assert_eq!(video.fps, 30);
@ -102,8 +105,8 @@ fn get_player_from_client(#[case] client_type: ClientType, rp: RustyPipe) {
.expect("audio stream not found"); .expect("audio stream not found");
assert_approx(f64::from(video.bitrate), 1_340_829.0); assert_approx(f64::from(video.bitrate), 1_340_829.0);
assert_approx(f64::from(video.average_bitrate), 1_233_444.0); assert_approx(f64::from(video.average_bitrate), 1_046_557.0);
assert_approx(video.size.unwrap() as f64, 39_936_630.0); assert_approx(video.size.unwrap() as f64, 33_885_572.0);
assert_eq!(video.width, 1280); assert_eq!(video.width, 1280);
assert_eq!(video.height, 720); assert_eq!(video.height, 720);
assert_eq!(video.fps, 30); assert_eq!(video.fps, 30);
@ -856,22 +859,15 @@ fn channel_playlists(rp: RustyPipe) {
#[rstest] #[rstest]
fn channel_info(rp: RustyPipe) { fn channel_info(rp: RustyPipe) {
let channel = let info = tokio_test::block_on(rp.query().channel_info("UC2DjFE7Xf11URZqWBigcVOQ")).unwrap();
tokio_test::block_on(rp.query().channel_info("UC2DjFE7Xf11URZqWBigcVOQ")).unwrap();
// dbg!(&channel); assert_eq!(info.create_date.unwrap(), date!(2009 - 4 - 4));
assert_channel_eevblog(&channel); assert_gte(info.view_count.unwrap(), 186_854_340, "channel views");
assert_gte(info.video_count.unwrap(), 1920, "channel videos");
assert_gte(info.subscriber_count.unwrap(), 920_000, "subscribers");
assert_eq!(info.country.unwrap(), Country::Au);
let created = channel.content.create_date.unwrap(); insta::assert_ron_snapshot!(info.links, @r###"
assert_eq!(created, date!(2009 - 4 - 4));
assert_gte(
channel.content.view_count.unwrap(),
186_854_340,
"channel views",
);
insta::assert_ron_snapshot!(channel.content.links, @r###"
[ [
("EEVblog Web Site", "http://www.eevblog.com/"), ("EEVblog Web Site", "http://www.eevblog.com/"),
("Twitter", "http://www.twitter.com/eevblog"), ("Twitter", "http://www.twitter.com/eevblog"),
@ -967,8 +963,8 @@ fn channel_more(
); );
} }
let channel_info = tokio_test::block_on(rp.query().channel_info(&id)).unwrap(); let info = tokio_test::block_on(rp.query().channel_info(&id)).unwrap();
assert_channel(&channel_info, id, name, unlocalized || name_unlocalized); assert_eq!(info.id, id);
} }
#[rstest] #[rstest]
@ -1668,7 +1664,9 @@ fn music_search_tracks(rp: RustyPipe, unlocalized: bool) {
.items .items
.iter() .iter()
.find(|a| a.id == "BL-aIpCLWnU") .find(|a| a.id == "BL-aIpCLWnU")
.unwrap(); .unwrap_or_else(|| {
panic!("could not find track, got {:#?}", &res.items.items);
});
assert_eq!(track.name, "Black Mamba"); assert_eq!(track.name, "Black Mamba");
assert!(!track.cover.is_empty(), "got no cover"); assert!(!track.cover.is_empty(), "got no cover");
@ -1703,7 +1701,9 @@ fn music_search_videos(rp: RustyPipe, unlocalized: bool) {
.items .items
.iter() .iter()
.find(|a| a.id == "ZeerrnuLi5E") .find(|a| a.id == "ZeerrnuLi5E")
.unwrap(); .unwrap_or_else(|| {
panic!("could not find video, got {:#?}", &res.items.items);
});
assert_eq!(track.name, "Black Mamba"); assert_eq!(track.name, "Black Mamba");
assert!(!track.cover.is_empty(), "got no cover"); assert!(!track.cover.is_empty(), "got no cover");
@ -1743,7 +1743,12 @@ fn music_search_episode(rp: RustyPipe, #[case] videos: bool) {
.tracks .tracks
}; };
let track = &tracks.iter().find(|a| a.id == "Zq_-LDy7AgE").unwrap(); let track = &tracks
.iter()
.find(|a| a.id == "Zq_-LDy7AgE")
.unwrap_or_else(|| {
panic!("could not find episode, got {:#?}", &tracks);
});
assert_eq!(track.artists.len(), 1); assert_eq!(track.artists.len(), 1);
let track_artist = &track.artists[0]; let track_artist = &track.artists[0];
@ -1809,7 +1814,14 @@ fn music_search_albums(
) { ) {
let res = tokio_test::block_on(rp.query().music_search_albums(query)).unwrap(); let res = tokio_test::block_on(rp.query().music_search_albums(query)).unwrap();
let album = &res.items.items.iter().find(|a| a.id == id).unwrap(); let album = &res
.items
.items
.iter()
.find(|a| a.id == id)
.unwrap_or_else(|| {
panic!("could not find album, got {:#?}", &res.items.items);
});
assert_eq!(album.name, name); assert_eq!(album.name, name);
assert_eq!(album.artists.len(), 1); assert_eq!(album.artists.len(), 1);
@ -1840,7 +1852,9 @@ fn music_search_artists(rp: RustyPipe, unlocalized: bool) {
.items .items
.iter() .iter()
.find(|a| a.id == "UCIh4j8fXWf2U0ro0qnGU8Mg") .find(|a| a.id == "UCIh4j8fXWf2U0ro0qnGU8Mg")
.unwrap(); .unwrap_or_else(|| {
panic!("could not find artist, got {:#?}", &res.items.items);
});
if unlocalized { if unlocalized {
assert_eq!(artist.name, "Namika"); assert_eq!(artist.name, "Namika");
} }
@ -1875,7 +1889,9 @@ fn music_search_playlists(rp: RustyPipe, unlocalized: bool) {
.items .items
.iter() .iter()
.find(|p| p.id == "RDCLAK5uy_nLtxizvEMkzYQUrA-bFf6MnBeR4bGYWUQ") .find(|p| p.id == "RDCLAK5uy_nLtxizvEMkzYQUrA-bFf6MnBeR4bGYWUQ")
.expect("no playlist"); .unwrap_or_else(|| {
panic!("could not find playlist, got {:#?}", &res.items.items);
});
if unlocalized { if unlocalized {
assert_eq!(playlist.name, "Today's Rock Hits"); assert_eq!(playlist.name, "Today's Rock Hits");
@ -1905,7 +1921,9 @@ fn music_search_playlists_community(rp: RustyPipe) {
.items .items
.iter() .iter()
.find(|p| p.id == "PLMC9KNkIncKtGvr2kFRuXBVmBev6cAJ2u") .find(|p| p.id == "PLMC9KNkIncKtGvr2kFRuXBVmBev6cAJ2u")
.expect("no playlist"); .unwrap_or_else(|| {
panic!("could not find playlist, got {:#?}", &res.items.items);
});
assert_eq!( assert_eq!(
playlist.name, playlist.name,
@ -2447,8 +2465,8 @@ fn assert_frameset(frameset: &Frameset) {
assert_gte(frameset.frame_height, 20, "frame width"); assert_gte(frameset.frame_height, 20, "frame width");
assert_gte(frameset.page_count, 1, "page count"); assert_gte(frameset.page_count, 1, "page count");
assert_gte(frameset.total_count, 50, "total count"); assert_gte(frameset.total_count, 50, "total count");
assert_gte(frameset.frames_per_page_x, 5, "frames per page x"); assert_gte(frameset.frames_per_page_x, 3, "frames per page x");
assert_gte(frameset.frames_per_page_y, 5, "frames per page y"); assert_gte(frameset.frames_per_page_y, 3, "frames per page y");
let n = frameset.urls().count() as u32; let n = frameset.urls().count() as u32;
assert_eq!(n, frameset.page_count); assert_eq!(n, frameset.page_count);