Compare commits

..

No commits in common. "182f9ebfb81f1d13909136c727c02a9fb8124e9b" and "da8b2a27fceeab07257f2f9c1da18c34a8da5d5f" have entirely different histories.

32 changed files with 1752 additions and 6496 deletions

View file

@ -4,8 +4,6 @@ use anyhow::{bail, Result};
use futures::{stream, StreamExt}; use futures::{stream, StreamExt};
use indicatif::{ProgressBar, ProgressStyle}; use indicatif::{ProgressBar, ProgressStyle};
use num_enum::TryFromPrimitive; use num_enum::TryFromPrimitive;
use once_cell::sync::Lazy;
use regex::Regex;
use rustypipe::client::{ClientType, RustyPipe, RustyPipeQuery, YTContext}; use rustypipe::client::{ClientType, RustyPipe, RustyPipeQuery, YTContext};
use rustypipe::model::YouTubeItem; use rustypipe::model::YouTubeItem;
use rustypipe::param::search_filter::{ItemType, SearchFilter}; use rustypipe::param::search_filter::{ItemType, SearchFilter};
@ -23,7 +21,6 @@ pub enum ABTest {
TrendsVideoTab = 4, TrendsVideoTab = 4,
TrendsPageHeaderRenderer = 5, TrendsPageHeaderRenderer = 5,
DiscographyPage = 6, DiscographyPage = 6,
ShortDateFormat = 7,
} }
const TESTS_TO_RUN: [ABTest; 3] = [ const TESTS_TO_RUN: [ABTest; 3] = [
@ -93,7 +90,6 @@ pub async fn run_test(
ABTest::TrendsVideoTab => trends_video_tab(&query).await, ABTest::TrendsVideoTab => trends_video_tab(&query).await,
ABTest::TrendsPageHeaderRenderer => trends_page_header_renderer(&query).await, ABTest::TrendsPageHeaderRenderer => trends_page_header_renderer(&query).await,
ABTest::DiscographyPage => discography_page(&query).await, ABTest::DiscographyPage => discography_page(&query).await,
ABTest::ShortDateFormat => short_date_format(&query).await,
} }
.unwrap(); .unwrap();
pb.inc(1); pb.inc(1);
@ -227,19 +223,10 @@ pub async fn trends_page_header_renderer(rp: &RustyPipeQuery) -> Result<bool> {
} }
pub async fn discography_page(rp: &RustyPipeQuery) -> Result<bool> { pub async fn discography_page(rp: &RustyPipeQuery) -> Result<bool> {
let artist = rp.music_artist("UC7cl4MmM6ZZ2TcFyMk_b4pg", false).await?; let artist = rp
.music_artist("UC7cl4MmM6ZZ2TcFyMk_b4pg", false)
.await
.unwrap();
Ok(artist.albums.len() <= 10) Ok(artist.albums.len() <= 10)
} }
pub async fn short_date_format(rp: &RustyPipeQuery) -> Result<bool> {
static SHORT_DATE: Lazy<Regex> = Lazy::new(|| Regex::new("\\d(?:y|mo|w|d|h|min) ").unwrap());
let channel = rp.channel_videos("UC2DjFE7Xf11URZqWBigcVOQ").await?;
Ok(channel.content.items.iter().any(|itm| {
itm.publish_date_txt
.as_deref()
.map(|d| SHORT_DATE.is_match(d))
.unwrap_or_default()
}))
}

View file

@ -1,83 +0,0 @@
use std::{
collections::{BTreeMap, HashSet},
fs::File,
};
use futures::{stream, StreamExt};
use path_macro::path;
use rustypipe::{
client::{RustyPipe, RustyPipeQuery},
param::{Language, LANGUAGES},
};
use crate::util::DICT_DIR;
pub async fn collect_video_dates(concurrency: usize) {
let json_path = path!(*DICT_DIR / "timeago_samples_short.json");
let rp = RustyPipe::builder()
.visitor_data("Cgtwel9tMkh2eHh0USiyzc6jBg%3D%3D")
.build();
let channels = [
"UCeY0bbntWzzVIaj2z3QigXg",
"UCcmpeVbSSQlZRvHfdC-CRwg",
"UC65afEgL62PGFWXY7n6CUbA",
"UCEOXxzW2vU0P-0THehuIIeg",
];
let mut lang_strings: BTreeMap<Language, Vec<String>> = BTreeMap::new();
for lang in LANGUAGES {
println!("{lang}");
let query = rp.query().lang(lang);
let strings = stream::iter(channels)
.map(|id| get_channel_datestrings(&query, id))
.buffered(concurrency)
.collect::<Vec<_>>()
.await
.into_iter()
.flatten()
.collect::<Vec<_>>();
lang_strings.insert(lang, strings);
}
let mut en_strings_uniq: HashSet<&str> = HashSet::new();
let mut uniq_ids: HashSet<usize> = HashSet::new();
lang_strings[&Language::En]
.iter()
.enumerate()
.for_each(|(n, s)| {
if en_strings_uniq.insert(s) {
uniq_ids.insert(n);
}
});
let strings_map = lang_strings
.iter()
.map(|(lang, strings)| {
(
lang,
strings
.iter()
.enumerate()
.filter(|(n, _)| uniq_ids.contains(n))
.map(|(_, s)| s)
.collect::<Vec<_>>(),
)
})
.collect::<BTreeMap<_, _>>();
let file = File::create(json_path).unwrap();
serde_json::to_writer_pretty(file, &strings_map).unwrap();
}
async fn get_channel_datestrings(rp: &RustyPipeQuery, id: &str) -> Vec<String> {
let channel = rp.channel_videos(id).await.unwrap();
channel
.content
.items
.into_iter()
.filter_map(|itm| itm.publish_date_txt)
.collect()
}

View file

@ -4,7 +4,6 @@ mod abtest;
mod collect_album_types; mod collect_album_types;
mod collect_large_numbers; mod collect_large_numbers;
mod collect_playlist_dates; mod collect_playlist_dates;
mod collect_video_dates;
mod collect_video_durations; mod collect_video_durations;
mod download_testfiles; mod download_testfiles;
mod gen_dictionary; mod gen_dictionary;
@ -28,7 +27,6 @@ enum Commands {
CollectLargeNumbers, CollectLargeNumbers,
CollectAlbumTypes, CollectAlbumTypes,
CollectVideoDurations, CollectVideoDurations,
CollectVideoDates,
ParsePlaylistDates, ParsePlaylistDates,
ParseLargeNumbers, ParseLargeNumbers,
ParseAlbumTypes, ParseAlbumTypes,
@ -62,9 +60,6 @@ async fn main() {
Commands::CollectVideoDurations => { Commands::CollectVideoDurations => {
collect_video_durations::collect_video_durations(cli.concurrency).await; collect_video_durations::collect_video_durations(cli.concurrency).await;
} }
Commands::CollectVideoDates => {
collect_video_dates::collect_video_dates(cli.concurrency).await;
}
Commands::ParsePlaylistDates => collect_playlist_dates::write_samples_to_dict(), Commands::ParsePlaylistDates => collect_playlist_dates::write_samples_to_dict(),
Commands::ParseLargeNumbers => collect_large_numbers::write_samples_to_dict(), Commands::ParseLargeNumbers => collect_large_numbers::write_samples_to_dict(),
Commands::ParseAlbumTypes => collect_album_types::write_samples_to_dict(), Commands::ParseAlbumTypes => collect_album_types::write_samples_to_dict(),

View file

@ -376,11 +376,3 @@ visitor data cookie to be set, as it was the case with the old system.
**NEW** **NEW**
![A/B test 4 old screenshot](./_img/ab_6_new.png) ![A/B test 4 old screenshot](./_img/ab_6_new.png)
## [7] Short timeago format
- **Encountered on:** 28.05.2023
- **Impact:** 🟡 Medium
YouTube changed their date format from the long format (*21 hours ago*, *3 days ago*) to
a short format (*21h ago*, *3d ago*).

View file

@ -200,20 +200,15 @@ impl MapResponse<Channel<Paginator<VideoItem>>> for response::Channel {
id: &str, id: &str,
lang: Language, lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
vdata: Option<&str>,
) -> Result<MapResult<Channel<Paginator<VideoItem>>>, ExtractionError> { ) -> Result<MapResult<Channel<Paginator<VideoItem>>>, ExtractionError> {
let content = map_channel_content(id, self.contents, self.alerts)?; let content = map_channel_content(id, self.contents, self.alerts)?;
let visitor_data = self
.response_context
.visitor_data
.or_else(|| vdata.map(str::to_owned));
let channel_data = map_channel( let channel_data = map_channel(
MapChannelData { MapChannelData {
header: self.header, header: self.header,
metadata: self.metadata, metadata: self.metadata,
microformat: self.microformat, microformat: self.microformat,
visitor_data: visitor_data.clone(), visitor_data: self.response_context.visitor_data.clone(),
has_shorts: content.has_shorts, has_shorts: content.has_shorts,
has_live: content.has_live, has_live: content.has_live,
}, },
@ -231,7 +226,7 @@ impl MapResponse<Channel<Paginator<VideoItem>>> for response::Channel {
None, None,
mapper.items, mapper.items,
mapper.ctoken, mapper.ctoken,
visitor_data, self.response_context.visitor_data,
crate::model::paginator::ContinuationEndpoint::Browse, crate::model::paginator::ContinuationEndpoint::Browse,
); );
@ -248,20 +243,15 @@ impl MapResponse<Channel<Paginator<PlaylistItem>>> for response::Channel {
id: &str, id: &str,
lang: Language, lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
vdata: Option<&str>,
) -> Result<MapResult<Channel<Paginator<PlaylistItem>>>, ExtractionError> { ) -> Result<MapResult<Channel<Paginator<PlaylistItem>>>, ExtractionError> {
let content = map_channel_content(id, self.contents, self.alerts)?; let content = map_channel_content(id, self.contents, self.alerts)?;
let visitor_data = self
.response_context
.visitor_data
.or_else(|| vdata.map(str::to_owned));
let channel_data = map_channel( let channel_data = map_channel(
MapChannelData { MapChannelData {
header: self.header, header: self.header,
metadata: self.metadata, metadata: self.metadata,
microformat: self.microformat, microformat: self.microformat,
visitor_data, visitor_data: self.response_context.visitor_data,
has_shorts: content.has_shorts, has_shorts: content.has_shorts,
has_live: content.has_live, has_live: content.has_live,
}, },
@ -290,7 +280,6 @@ impl MapResponse<Channel<ChannelInfo>> for response::Channel {
id: &str, id: &str,
lang: Language, lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
vdata: Option<&str>,
) -> Result<MapResult<Channel<ChannelInfo>>, ExtractionError> { ) -> Result<MapResult<Channel<ChannelInfo>>, ExtractionError> {
let content = map_channel_content(id, self.contents, self.alerts)?; let content = map_channel_content(id, self.contents, self.alerts)?;
let channel_data = map_channel( let channel_data = map_channel(
@ -298,10 +287,7 @@ impl MapResponse<Channel<ChannelInfo>> for response::Channel {
header: self.header, header: self.header,
metadata: self.metadata, metadata: self.metadata,
microformat: self.microformat, microformat: self.microformat,
visitor_data: self visitor_data: self.response_context.visitor_data,
.response_context
.visitor_data
.or_else(|| vdata.map(str::to_owned)),
has_shorts: content.has_shorts, has_shorts: content.has_shorts,
has_live: content.has_live, has_live: content.has_live,
}, },
@ -619,7 +605,7 @@ mod tests {
let channel: response::Channel = let channel: response::Channel =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Channel<Paginator<VideoItem>>> = let map_res: MapResult<Channel<Paginator<VideoItem>>> =
channel.map_response(id, Language::En, None, None).unwrap(); channel.map_response(id, Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),
@ -646,7 +632,7 @@ mod tests {
let channel: response::Channel = let channel: response::Channel =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Channel<Paginator<PlaylistItem>>> = channel let map_res: MapResult<Channel<Paginator<PlaylistItem>>> = channel
.map_response("UC2DjFE7Xf11URZqWBigcVOQ", Language::En, None, None) .map_response("UC2DjFE7Xf11URZqWBigcVOQ", Language::En, None)
.unwrap(); .unwrap();
assert!( assert!(
@ -665,7 +651,7 @@ mod tests {
let channel: response::Channel = let channel: response::Channel =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Channel<ChannelInfo>> = channel let map_res: MapResult<Channel<ChannelInfo>> = channel
.map_response("UC2DjFE7Xf11URZqWBigcVOQ", Language::En, None, None) .map_response("UC2DjFE7Xf11URZqWBigcVOQ", Language::En, None)
.unwrap(); .unwrap();
assert!( assert!(

View file

@ -1247,12 +1247,7 @@ impl RustyPipeQuery {
}) })
} else { } else {
match serde_json::from_str::<R>(&body) { match serde_json::from_str::<R>(&body) {
Ok(deserialized) => match deserialized.map_response( Ok(deserialized) => match deserialized.map_response(id, self.opts.lang, deobf) {
id,
self.opts.lang,
deobf,
self.opts.visitor_data.as_deref(),
) {
Ok(mapres) => Ok(mapres), Ok(mapres) => Ok(mapres),
Err(e) => Err(e.into()), Err(e) => Err(e.into()),
}, },
@ -1458,13 +1453,11 @@ trait MapResponse<T> {
/// that the returned entity matches this ID and return an error instead. /// that the returned entity matches this ID and return an error instead.
/// - `lang`: Language of the request. Used for mapping localized information like dates. /// - `lang`: Language of the request. Used for mapping localized information like dates.
/// - `deobf`: Deobfuscator (if passed to the `execute_request_deobf` method) /// - `deobf`: Deobfuscator (if passed to the `execute_request_deobf` method)
/// - `vdata`: Visitor data option of the client
fn map_response( fn map_response(
self, self,
id: &str, id: &str,
lang: Language, lang: Language,
deobf: Option<&DeobfData>, deobf: Option<&DeobfData>,
vdata: Option<&str>,
) -> Result<MapResult<T>, ExtractionError>; ) -> Result<MapResult<T>, ExtractionError>;
} }

View file

@ -4,7 +4,7 @@ use once_cell::sync::Lazy;
use regex::Regex; use regex::Regex;
use crate::{ use crate::{
client::response::url_endpoint::NavigationEndpoint, client::response::url_endpoint::{MusicPageType, NavigationEndpoint},
error::{Error, ExtractionError}, error::{Error, ExtractionError},
model::{AlbumItem, ArtistId, MusicArtist}, model::{AlbumItem, ArtistId, MusicArtist},
serializer::MapResult, serializer::MapResult,
@ -96,7 +96,6 @@ impl MapResponse<MusicArtist> for response::MusicArtist {
id: &str, id: &str,
lang: crate::param::Language, lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<MusicArtist>, ExtractionError> { ) -> Result<MapResult<MusicArtist>, ExtractionError> {
let mapped = map_artist_page(self, id, lang, false)?; let mapped = map_artist_page(self, id, lang, false)?;
Ok(MapResult { Ok(MapResult {
@ -112,7 +111,6 @@ impl MapResponse<(MusicArtist, bool)> for response::MusicArtist {
id: &str, id: &str,
lang: crate::param::Language, lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<(MusicArtist, bool)>, ExtractionError> { ) -> Result<MapResult<(MusicArtist, bool)>, ExtractionError> {
map_artist_page(self, id, lang, true) map_artist_page(self, id, lang, true)
} }
@ -191,29 +189,20 @@ fn map_artist_page(
.music_carousel_shelf_basic_header_renderer .music_carousel_shelf_basic_header_renderer
.more_content_button .more_content_button
{ {
if let NavigationEndpoint::Browse { match button.button_renderer.navigation_endpoint.music_page() {
browse_endpoint, ..
} = button.button_renderer.navigation_endpoint
{
// Music videos // Music videos
if browse_endpoint Some((MusicPageType::Playlist, id)) => {
.browse_endpoint_context_supported_configs
.map(|cfg| {
cfg.browse_endpoint_context_music_config.page_type
== PageType::Playlist
})
.unwrap_or_default()
{
if videos_playlist_id.is_none() { if videos_playlist_id.is_none() {
videos_playlist_id = Some(browse_endpoint.browse_id); videos_playlist_id = Some(id);
} }
} else if browse_endpoint }
.browse_id // Albums
.starts_with(util::ARTIST_DISCOGRAPHY_PREFIX) Some((MusicPageType::ArtistDiscography, _)) => {
{
can_fetch_more = true; can_fetch_more = true;
extendable_albums = true; extendable_albums = true;
} else { }
// Albums or playlists
Some((MusicPageType::Artist, _)) => {
// Peek at the first item to determine type // Peek at the first item to determine type
if let Some(response::music_item::MusicResponseItem::MusicTwoRowItemRenderer(item)) = shelf.contents.c.first() { if let Some(response::music_item::MusicResponseItem::MusicTwoRowItemRenderer(item)) = shelf.contents.c.first() {
if let Some(PageType::Album) = item.navigation_endpoint.page_type() { if let Some(PageType::Album) = item.navigation_endpoint.page_type() {
@ -222,6 +211,7 @@ fn map_artist_page(
} }
} }
} }
_ => {}
} }
} }
} }
@ -296,7 +286,6 @@ impl MapResponse<Vec<AlbumItem>> for response::MusicArtistAlbums {
id: &str, id: &str,
lang: crate::param::Language, lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<Vec<AlbumItem>>, ExtractionError> { ) -> Result<MapResult<Vec<AlbumItem>>, ExtractionError> {
// dbg!(&self); // dbg!(&self);
@ -367,7 +356,7 @@ mod tests {
let resp: response::MusicArtist = let resp: response::MusicArtist =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<(MusicArtist, bool)> = let map_res: MapResult<(MusicArtist, bool)> =
resp.map_response(id, Language::En, None, None).unwrap(); resp.map_response(id, Language::En, None).unwrap();
let (mut artist, can_fetch_more) = map_res.c; let (mut artist, can_fetch_more) = map_res.c;
assert!( assert!(
@ -382,7 +371,7 @@ mod tests {
let resp: response::MusicArtistAlbums = let resp: response::MusicArtistAlbums =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let mut map_res: MapResult<Vec<AlbumItem>> = let mut map_res: MapResult<Vec<AlbumItem>> =
resp.map_response(id, Language::En, None, None).unwrap(); resp.map_response(id, Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),
@ -403,7 +392,7 @@ mod tests {
let artist: response::MusicArtist = let artist: response::MusicArtist =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicArtist> = artist let map_res: MapResult<MusicArtist> = artist
.map_response("UClmXPfaYhXOYsNn_QUyheWQ", Language::En, None, None) .map_response("UClmXPfaYhXOYsNn_QUyheWQ", Language::En, None)
.unwrap(); .unwrap();
assert!( assert!(
@ -422,7 +411,7 @@ mod tests {
let artist: response::MusicArtist = let artist: response::MusicArtist =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let res: Result<MapResult<MusicArtist>, ExtractionError> = let res: Result<MapResult<MusicArtist>, ExtractionError> =
artist.map_response("UCLkAepWjdylmXSltofFvsYQ", Language::En, None, None); artist.map_response("UCLkAepWjdylmXSltofFvsYQ", Language::En, None);
let e = res.unwrap_err(); let e = res.unwrap_err();
match e { match e {

View file

@ -60,7 +60,6 @@ impl MapResponse<MusicCharts> for response::MusicCharts {
_id: &str, _id: &str,
lang: crate::param::Language, lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<crate::serializer::MapResult<MusicCharts>, crate::error::ExtractionError> { ) -> Result<crate::serializer::MapResult<MusicCharts>, crate::error::ExtractionError> {
let countries = self let countries = self
.framework_updates .framework_updates
@ -165,8 +164,7 @@ mod tests {
let charts: response::MusicCharts = let charts: response::MusicCharts =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicCharts> = let map_res: MapResult<MusicCharts> = charts.map_response("", Language::En, None).unwrap();
charts.map_response("", Language::En, None, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),

View file

@ -157,7 +157,6 @@ impl MapResponse<TrackDetails> for response::MusicDetails {
id: &str, id: &str,
lang: Language, lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<TrackDetails>, ExtractionError> { ) -> Result<MapResult<TrackDetails>, ExtractionError> {
let tabs = self let tabs = self
.contents .contents
@ -238,7 +237,6 @@ impl MapResponse<Paginator<TrackItem>> for response::MusicDetails {
id: &str, id: &str,
lang: Language, lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<Paginator<TrackItem>>, ExtractionError> { ) -> Result<MapResult<Paginator<TrackItem>>, ExtractionError> {
let tabs = self let tabs = self
.contents .contents
@ -299,7 +297,6 @@ impl MapResponse<Lyrics> for response::MusicLyrics {
id: &str, id: &str,
_lang: Language, _lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<Lyrics>, ExtractionError> { ) -> Result<MapResult<Lyrics>, ExtractionError> {
let lyrics = self let lyrics = self
.contents .contents
@ -333,7 +330,6 @@ impl MapResponse<MusicRelated> for response::MusicRelated {
_id: &str, _id: &str,
lang: Language, lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<MusicRelated>, ExtractionError> { ) -> Result<MapResult<MusicRelated>, ExtractionError> {
// Find artist // Find artist
let artist_id = self let artist_id = self
@ -426,7 +422,7 @@ mod tests {
let details: response::MusicDetails = let details: response::MusicDetails =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<model::TrackDetails> = let map_res: MapResult<model::TrackDetails> =
details.map_response(id, Language::En, None, None).unwrap(); details.map_response(id, Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),
@ -446,7 +442,7 @@ mod tests {
let radio: response::MusicDetails = let radio: response::MusicDetails =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Paginator<TrackItem>> = let map_res: MapResult<Paginator<TrackItem>> =
radio.map_response(id, Language::En, None, None).unwrap(); radio.map_response(id, Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),
@ -463,7 +459,7 @@ mod tests {
let lyrics: response::MusicLyrics = let lyrics: response::MusicLyrics =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Lyrics> = lyrics.map_response("", Language::En, None, None).unwrap(); let map_res: MapResult<Lyrics> = lyrics.map_response("", Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),
@ -480,8 +476,7 @@ mod tests {
let lyrics: response::MusicRelated = let lyrics: response::MusicRelated =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicRelated> = let map_res: MapResult<MusicRelated> = lyrics.map_response("", Language::En, None).unwrap();
lyrics.map_response("", Language::En, None, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),

View file

@ -57,7 +57,6 @@ impl MapResponse<Vec<MusicGenreItem>> for response::MusicGenres {
_id: &str, _id: &str,
_lang: crate::param::Language, _lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<crate::serializer::MapResult<Vec<MusicGenreItem>>, ExtractionError> { ) -> Result<crate::serializer::MapResult<Vec<MusicGenreItem>>, ExtractionError> {
let content = self let content = self
.contents .contents
@ -111,7 +110,6 @@ impl MapResponse<MusicGenre> for response::MusicGenre {
id: &str, id: &str,
lang: crate::param::Language, lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<crate::serializer::MapResult<MusicGenre>, ExtractionError> { ) -> Result<crate::serializer::MapResult<MusicGenre>, ExtractionError> {
// dbg!(&self); // dbg!(&self);
@ -216,7 +214,7 @@ mod tests {
let playlist: response::MusicGenres = let playlist: response::MusicGenres =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Vec<model::MusicGenreItem>> = let map_res: MapResult<Vec<model::MusicGenreItem>> =
playlist.map_response("", Language::En, None, None).unwrap(); playlist.map_response("", Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),
@ -236,7 +234,7 @@ mod tests {
let playlist: response::MusicGenre = let playlist: response::MusicGenre =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<model::MusicGenre> = let map_res: MapResult<model::MusicGenre> =
playlist.map_response(id, Language::En, None, None).unwrap(); playlist.map_response(id, Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),

View file

@ -52,7 +52,6 @@ impl<T: FromYtItem> MapResponse<Vec<T>> for response::MusicNew {
_id: &str, _id: &str,
lang: crate::param::Language, lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<crate::serializer::MapResult<Vec<T>>, ExtractionError> { ) -> Result<crate::serializer::MapResult<Vec<T>>, ExtractionError> {
let items = self let items = self
.contents .contents
@ -97,9 +96,8 @@ mod tests {
let new_albums: response::MusicNew = let new_albums: response::MusicNew =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Vec<AlbumItem>> = new_albums let map_res: MapResult<Vec<AlbumItem>> =
.map_response("", Language::En, None, None) new_albums.map_response("", Language::En, None).unwrap();
.unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),
@ -117,9 +115,8 @@ mod tests {
let new_albums: response::MusicNew = let new_albums: response::MusicNew =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Vec<TrackItem>> = new_albums let map_res: MapResult<Vec<TrackItem>> =
.map_response("", Language::En, None, None) new_albums.map_response("", Language::En, None).unwrap();
.unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),

View file

@ -122,7 +122,6 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
id: &str, id: &str,
lang: crate::param::Language, lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<MusicPlaylist>, ExtractionError> { ) -> Result<MapResult<MusicPlaylist>, ExtractionError> {
// dbg!(&self); // dbg!(&self);
@ -268,7 +267,6 @@ impl MapResponse<MusicAlbum> for response::MusicPlaylist {
id: &str, id: &str,
lang: crate::param::Language, lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<MusicAlbum>, ExtractionError> { ) -> Result<MapResult<MusicAlbum>, ExtractionError> {
// dbg!(&self); // dbg!(&self);
@ -420,7 +418,7 @@ mod tests {
let playlist: response::MusicPlaylist = let playlist: response::MusicPlaylist =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<model::MusicPlaylist> = let map_res: MapResult<model::MusicPlaylist> =
playlist.map_response(id, Language::En, None, None).unwrap(); playlist.map_response(id, Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),
@ -445,7 +443,7 @@ mod tests {
let playlist: response::MusicPlaylist = let playlist: response::MusicPlaylist =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<model::MusicAlbum> = let map_res: MapResult<model::MusicAlbum> =
playlist.map_response(id, Language::En, None, None).unwrap(); playlist.map_response(id, Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),

View file

@ -231,7 +231,6 @@ impl MapResponse<MusicSearchResult> for response::MusicSearch {
_id: &str, _id: &str,
lang: crate::param::Language, lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<MusicSearchResult>, crate::error::ExtractionError> { ) -> Result<MapResult<MusicSearchResult>, crate::error::ExtractionError> {
// dbg!(&self); // dbg!(&self);
@ -297,7 +296,6 @@ impl<T: FromYtItem> MapResponse<MusicSearchFiltered<T>> for response::MusicSearc
_id: &str, _id: &str,
lang: crate::param::Language, lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<MusicSearchFiltered<T>>, ExtractionError> { ) -> Result<MapResult<MusicSearchFiltered<T>>, ExtractionError> {
// dbg!(&self); // dbg!(&self);
@ -358,7 +356,6 @@ impl MapResponse<MusicSearchSuggestion> for response::MusicSearchSuggestion {
_id: &str, _id: &str,
lang: crate::param::Language, lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<MusicSearchSuggestion>, ExtractionError> { ) -> Result<MapResult<MusicSearchSuggestion>, ExtractionError> {
let mut mapper = MusicListMapper::new(lang); let mut mapper = MusicListMapper::new(lang);
let mut terms = Vec::new(); let mut terms = Vec::new();
@ -422,7 +419,7 @@ mod tests {
let search: response::MusicSearch = let search: response::MusicSearch =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicSearchResult> = let map_res: MapResult<MusicSearchResult> =
search.map_response("", Language::En, None, None).unwrap(); search.map_response("", Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),
@ -445,7 +442,7 @@ mod tests {
let search: response::MusicSearch = let search: response::MusicSearch =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicSearchFiltered<TrackItem>> = let map_res: MapResult<MusicSearchFiltered<TrackItem>> =
search.map_response("", Language::En, None, None).unwrap(); search.map_response("", Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),
@ -464,7 +461,7 @@ mod tests {
let search: response::MusicSearch = let search: response::MusicSearch =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicSearchFiltered<AlbumItem>> = let map_res: MapResult<MusicSearchFiltered<AlbumItem>> =
search.map_response("", Language::En, None, None).unwrap(); search.map_response("", Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),
@ -483,7 +480,7 @@ mod tests {
let search: response::MusicSearch = let search: response::MusicSearch =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicSearchFiltered<ArtistItem>> = let map_res: MapResult<MusicSearchFiltered<ArtistItem>> =
search.map_response("", Language::En, None, None).unwrap(); search.map_response("", Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),
@ -504,7 +501,7 @@ mod tests {
let search: response::MusicSearch = let search: response::MusicSearch =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicSearchFiltered<MusicPlaylistItem>> = let map_res: MapResult<MusicSearchFiltered<MusicPlaylistItem>> =
search.map_response("", Language::En, None, None).unwrap(); search.map_response("", Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),
@ -524,9 +521,8 @@ mod tests {
let suggestion: response::MusicSearchSuggestion = let suggestion: response::MusicSearchSuggestion =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicSearchSuggestion> = suggestion let map_res: MapResult<MusicSearchSuggestion> =
.map_response("", Language::En, None, None) suggestion.map_response("", Language::En, None).unwrap();
.unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),

View file

@ -96,7 +96,6 @@ impl MapResponse<Paginator<YouTubeItem>> for response::Continuation {
_id: &str, _id: &str,
lang: crate::param::Language, lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<Paginator<YouTubeItem>>, ExtractionError> { ) -> Result<MapResult<Paginator<YouTubeItem>>, ExtractionError> {
let items = self let items = self
.on_response_received_actions .on_response_received_actions
@ -132,7 +131,6 @@ impl MapResponse<Paginator<MusicItem>> for response::MusicContinuation {
_id: &str, _id: &str,
lang: crate::param::Language, lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<Paginator<MusicItem>>, ExtractionError> { ) -> Result<MapResult<Paginator<MusicItem>>, ExtractionError> {
let mut mapper = MusicListMapper::new(lang); let mut mapper = MusicListMapper::new(lang);
let mut continuations = Vec::new(); let mut continuations = Vec::new();
@ -355,7 +353,7 @@ mod tests {
let items: response::Continuation = let items: response::Continuation =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Paginator<YouTubeItem>> = let map_res: MapResult<Paginator<YouTubeItem>> =
items.map_response("", Language::En, None, None).unwrap(); items.map_response("", Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),
@ -377,7 +375,7 @@ mod tests {
let items: response::Continuation = let items: response::Continuation =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Paginator<YouTubeItem>> = let map_res: MapResult<Paginator<YouTubeItem>> =
items.map_response("", Language::En, None, None).unwrap(); items.map_response("", Language::En, None).unwrap();
let paginator: Paginator<VideoItem> = let paginator: Paginator<VideoItem> =
map_yt_paginator(map_res.c, None, ContinuationEndpoint::Browse); map_yt_paginator(map_res.c, None, ContinuationEndpoint::Browse);
@ -400,7 +398,7 @@ mod tests {
let items: response::Continuation = let items: response::Continuation =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Paginator<YouTubeItem>> = let map_res: MapResult<Paginator<YouTubeItem>> =
items.map_response("", Language::En, None, None).unwrap(); items.map_response("", Language::En, None).unwrap();
let paginator: Paginator<PlaylistItem> = let paginator: Paginator<PlaylistItem> =
map_yt_paginator(map_res.c, None, ContinuationEndpoint::Browse); map_yt_paginator(map_res.c, None, ContinuationEndpoint::Browse);
@ -423,7 +421,7 @@ mod tests {
let items: response::MusicContinuation = let items: response::MusicContinuation =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Paginator<MusicItem>> = let map_res: MapResult<Paginator<MusicItem>> =
items.map_response("", Language::En, None, None).unwrap(); items.map_response("", Language::En, None).unwrap();
let paginator: Paginator<TrackItem> = let paginator: Paginator<TrackItem> =
map_ytm_paginator(map_res.c, None, ContinuationEndpoint::MusicBrowse); map_ytm_paginator(map_res.c, None, ContinuationEndpoint::MusicBrowse);
@ -444,7 +442,7 @@ mod tests {
let items: response::MusicContinuation = let items: response::MusicContinuation =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Paginator<MusicItem>> = let map_res: MapResult<Paginator<MusicItem>> =
items.map_response("", Language::En, None, None).unwrap(); items.map_response("", Language::En, None).unwrap();
let paginator: Paginator<MusicPlaylistItem> = let paginator: Paginator<MusicPlaylistItem> =
map_ytm_paginator(map_res.c, None, ContinuationEndpoint::MusicBrowse); map_ytm_paginator(map_res.c, None, ContinuationEndpoint::MusicBrowse);

View file

@ -143,7 +143,6 @@ impl MapResponse<VideoPlayer> for response::Player {
id: &str, id: &str,
_lang: Language, _lang: Language,
deobf: Option<&crate::deobfuscate::DeobfData>, deobf: Option<&crate::deobfuscate::DeobfData>,
vdata: Option<&str>,
) -> Result<super::MapResult<VideoPlayer>, ExtractionError> { ) -> Result<super::MapResult<VideoPlayer>, ExtractionError> {
let deobf = Deobfuscator::new(deobf.unwrap())?; let deobf = Deobfuscator::new(deobf.unwrap())?;
let mut warnings = vec![]; let mut warnings = vec![];
@ -373,10 +372,7 @@ impl MapResponse<VideoPlayer> for response::Player {
hls_manifest_url: streaming_data.hls_manifest_url, hls_manifest_url: streaming_data.hls_manifest_url,
dash_manifest_url: streaming_data.dash_manifest_url, dash_manifest_url: streaming_data.dash_manifest_url,
preview_frames, preview_frames,
visitor_data: self visitor_data: self.response_context.visitor_data,
.response_context
.visitor_data
.or_else(|| vdata.map(str::to_owned)),
}, },
warnings, warnings,
}) })
@ -721,7 +717,7 @@ mod tests {
let resp: response::Player = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let resp: response::Player = serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res = resp let map_res = resp
.map_response("pPvd8UxmSbQ", Language::En, Some(&DEOBF_DATA), None) .map_response("pPvd8UxmSbQ", Language::En, Some(&DEOBF_DATA))
.unwrap(); .unwrap();
assert!( assert!(

View file

@ -37,7 +37,6 @@ impl MapResponse<Playlist> for response::Playlist {
id: &str, id: &str,
lang: crate::param::Language, lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
vdata: Option<&str>,
) -> Result<MapResult<Playlist>, ExtractionError> { ) -> Result<MapResult<Playlist>, ExtractionError> {
let (Some(contents), Some(header)) = (self.contents, self.header) else { let (Some(contents), Some(header)) = (self.contents, self.header) else {
return Err(response::alerts_to_err(id, self.alerts)); return Err(response::alerts_to_err(id, self.alerts));
@ -153,10 +152,7 @@ impl MapResponse<Playlist> for response::Playlist {
channel, channel,
last_update, last_update,
last_update_txt, last_update_txt,
visitor_data: self visitor_data: self.response_context.visitor_data,
.response_context
.visitor_data
.or_else(|| vdata.map(str::to_owned)),
}, },
warnings: mapper.warnings, warnings: mapper.warnings,
}) })
@ -185,7 +181,7 @@ mod tests {
let playlist: response::Playlist = let playlist: response::Playlist =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res = playlist.map_response(id, Language::En, None, None).unwrap(); let map_res = playlist.map_response(id, Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),

View file

@ -759,7 +759,7 @@ impl MusicListMapper {
})); }));
Ok(Some(MusicItemType::Playlist)) Ok(Some(MusicItemType::Playlist))
} }
MusicPageType::None => { MusicPageType::None | MusicPageType::ArtistDiscography => {
// There may be broken YT channels from the artist search. They can be skipped. // There may be broken YT channels from the artist search. They can be skipped.
Ok(None) Ok(None)
} }
@ -901,7 +901,7 @@ impl MusicListMapper {
})); }));
Ok(Some(MusicItemType::Playlist)) Ok(Some(MusicItemType::Playlist))
} }
MusicPageType::None => Ok(None), MusicPageType::None | MusicPageType::ArtistDiscography => Ok(None),
MusicPageType::Unknown => { MusicPageType::Unknown => {
self.has_unknown = true; self.has_unknown = true;
Ok(None) Ok(None)
@ -1039,7 +1039,7 @@ impl MusicListMapper {
})); }));
Some(MusicItemType::Playlist) Some(MusicItemType::Playlist)
} }
MusicPageType::None => None, MusicPageType::None | MusicPageType::ArtistDiscography => None,
MusicPageType::Unknown => { MusicPageType::Unknown => {
self.has_unknown = true; self.has_unknown = true;
None None

View file

@ -102,12 +102,9 @@ pub(crate) struct BrowseEndpointConfig {
pub browse_endpoint_context_music_config: BrowseEndpointMusicConfig, pub browse_endpoint_context_music_config: BrowseEndpointMusicConfig,
} }
#[serde_as]
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub(crate) struct BrowseEndpointMusicConfig { pub(crate) struct BrowseEndpointMusicConfig {
#[serde(default)]
#[serde_as(as = "DefaultOnError")]
pub page_type: PageType, pub page_type: PageType,
} }
@ -117,12 +114,9 @@ pub(crate) struct CommandMetadata {
pub web_command_metadata: WebCommandMetadata, pub web_command_metadata: WebCommandMetadata,
} }
#[serde_as]
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub(crate) struct WebCommandMetadata { pub(crate) struct WebCommandMetadata {
#[serde(default)]
#[serde_as(as = "DefaultOnError")]
pub web_page_type: PageType, pub web_page_type: PageType,
} }
@ -150,13 +144,15 @@ pub(crate) enum MusicVideoType {
Track, Track,
} }
#[derive(Default, Debug, Clone, Copy, Deserialize, PartialEq, Eq)] #[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
pub(crate) enum PageType { pub(crate) enum PageType {
#[serde( #[serde(
rename = "MUSIC_PAGE_TYPE_ARTIST", rename = "MUSIC_PAGE_TYPE_ARTIST",
alias = "MUSIC_PAGE_TYPE_AUDIOBOOK_ARTIST" alias = "MUSIC_PAGE_TYPE_AUDIOBOOK_ARTIST"
)] )]
Artist, Artist,
#[serde(rename = "MUSIC_PAGE_TYPE_ARTIST_DISCOGRAPHY")]
ArtistDiscography,
#[serde(rename = "MUSIC_PAGE_TYPE_ALBUM", alias = "MUSIC_PAGE_TYPE_AUDIOBOOK")] #[serde(rename = "MUSIC_PAGE_TYPE_ALBUM", alias = "MUSIC_PAGE_TYPE_AUDIOBOOK")]
Album, Album,
#[serde( #[serde(
@ -166,7 +162,7 @@ pub(crate) enum PageType {
Channel, Channel,
#[serde(rename = "MUSIC_PAGE_TYPE_PLAYLIST", alias = "WEB_PAGE_TYPE_PLAYLIST")] #[serde(rename = "MUSIC_PAGE_TYPE_PLAYLIST", alias = "WEB_PAGE_TYPE_PLAYLIST")]
Playlist, Playlist,
#[default] #[serde(rename = "MUSIC_PAGE_TYPE_UNKNOWN")]
Unknown, Unknown,
} }
@ -174,6 +170,9 @@ impl PageType {
pub(crate) fn to_url_target(self, id: String) -> Option<UrlTarget> { pub(crate) fn to_url_target(self, id: String) -> Option<UrlTarget> {
match self { match self {
PageType::Artist | PageType::Channel => Some(UrlTarget::Channel { id }), PageType::Artist | PageType::Channel => Some(UrlTarget::Channel { id }),
PageType::ArtistDiscography => id
.strip_prefix(util::ARTIST_DISCOGRAPHY_PREFIX)
.map(|id| UrlTarget::Channel { id: id.to_owned() }),
PageType::Album => Some(UrlTarget::Album { id }), PageType::Album => Some(UrlTarget::Album { id }),
PageType::Playlist => Some(UrlTarget::Playlist { id }), PageType::Playlist => Some(UrlTarget::Playlist { id }),
PageType::Unknown => None, PageType::Unknown => None,
@ -184,6 +183,7 @@ impl PageType {
#[derive(Debug, Copy, Clone, PartialEq, Eq)] #[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub(crate) enum MusicPageType { pub(crate) enum MusicPageType {
Artist, Artist,
ArtistDiscography,
Album, Album,
Playlist, Playlist,
Track { is_video: bool }, Track { is_video: bool },
@ -195,6 +195,7 @@ impl From<PageType> for MusicPageType {
fn from(t: PageType) -> Self { fn from(t: PageType) -> Self {
match t { match t {
PageType::Artist => MusicPageType::Artist, PageType::Artist => MusicPageType::Artist,
PageType::ArtistDiscography => MusicPageType::ArtistDiscography,
PageType::Album => MusicPageType::Album, PageType::Album => MusicPageType::Album,
PageType::Playlist => MusicPageType::Playlist, PageType::Playlist => MusicPageType::Playlist,
PageType::Channel => MusicPageType::None, PageType::Channel => MusicPageType::None,

View file

@ -92,7 +92,6 @@ impl MapResponse<SearchResult> for response::Search {
_id: &str, _id: &str,
lang: crate::param::Language, lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
vdata: Option<&str>,
) -> Result<MapResult<SearchResult>, ExtractionError> { ) -> Result<MapResult<SearchResult>, ExtractionError> {
let items = self let items = self
.contents .contents
@ -114,10 +113,7 @@ impl MapResponse<SearchResult> for response::Search {
crate::model::paginator::ContinuationEndpoint::Search, crate::model::paginator::ContinuationEndpoint::Search,
), ),
corrected_query: mapper.corrected_query, corrected_query: mapper.corrected_query,
visitor_data: self visitor_data: self.response_context.visitor_data,
.response_context
.visitor_data
.or_else(|| vdata.map(str::to_owned)),
}, },
warnings: mapper.warnings, warnings: mapper.warnings,
}) })
@ -149,8 +145,7 @@ mod tests {
let json_file = File::open(json_path).unwrap(); let json_file = File::open(json_path).unwrap();
let search: response::Search = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let search: response::Search = serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<SearchResult> = let map_res: MapResult<SearchResult> = search.map_response("", Language::En, None).unwrap();
search.map_response("", Language::En, None, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),

View file

@ -54,7 +54,6 @@ impl MapResponse<Paginator<VideoItem>> for response::Startpage {
_id: &str, _id: &str,
lang: crate::param::Language, lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
vdata: Option<&str>,
) -> Result<MapResult<Paginator<VideoItem>>, ExtractionError> { ) -> Result<MapResult<Paginator<VideoItem>>, ExtractionError> {
let grid = self let grid = self
.contents .contents
@ -71,9 +70,7 @@ impl MapResponse<Paginator<VideoItem>> for response::Startpage {
Ok(map_startpage_videos( Ok(map_startpage_videos(
grid, grid,
lang, lang,
self.response_context self.response_context.visitor_data,
.visitor_data
.or_else(|| vdata.map(str::to_owned)),
)) ))
} }
} }
@ -84,7 +81,6 @@ impl MapResponse<Vec<VideoItem>> for response::Trending {
_id: &str, _id: &str,
lang: crate::param::Language, lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<Vec<VideoItem>>, ExtractionError> { ) -> Result<MapResult<Vec<VideoItem>>, ExtractionError> {
let items = self let items = self
.contents .contents
@ -150,9 +146,8 @@ mod tests {
let startpage: response::Startpage = let startpage: response::Startpage =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Paginator<VideoItem>> = startpage let map_res: MapResult<Paginator<VideoItem>> =
.map_response("", Language::En, None, None) startpage.map_response("", Language::En, None).unwrap();
.unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),
@ -174,9 +169,8 @@ mod tests {
let startpage: response::Trending = let startpage: response::Trending =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Vec<VideoItem>> = startpage let map_res: MapResult<Vec<VideoItem>> =
.map_response("", Language::En, None, None) startpage.map_response("", Language::En, None).unwrap();
.unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),

View file

@ -328,7 +328,6 @@ impl MapResponse<UrlTarget> for response::ResolvedUrl {
_id: &str, _id: &str,
_lang: Language, _lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<UrlTarget>, ExtractionError> { ) -> Result<MapResult<UrlTarget>, ExtractionError> {
let pt = self.endpoint.page_type(); let pt = self.endpoint.page_type();
if let NavigationEndpoint::Browse { if let NavigationEndpoint::Browse {

View file

@ -82,7 +82,6 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
id: &str, id: &str,
lang: Language, lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
vdata: Option<&str>,
) -> Result<MapResult<VideoDetails>, ExtractionError> { ) -> Result<MapResult<VideoDetails>, ExtractionError> {
let mut warnings = Vec::new(); let mut warnings = Vec::new();
@ -257,10 +256,7 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
_ => return Err(ExtractionError::InvalidData("invalid channel link".into())), _ => return Err(ExtractionError::InvalidData("invalid channel link".into())),
}; };
let visitor_data = self let visitor_data = self.response_context.visitor_data;
.response_context
.visitor_data
.or_else(|| vdata.map(str::to_owned));
let recommended = contents let recommended = contents
.two_column_watch_next_results .two_column_watch_next_results
.secondary_results .secondary_results
@ -373,7 +369,6 @@ impl MapResponse<Paginator<Comment>> for response::VideoComments {
_id: &str, _id: &str,
lang: Language, lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>, _deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<Paginator<Comment>>, ExtractionError> { ) -> Result<MapResult<Paginator<Comment>>, ExtractionError> {
let received_endpoints = self.on_response_received_endpoints; let received_endpoints = self.on_response_received_endpoints;
let mut warnings = received_endpoints.warnings; let mut warnings = received_endpoints.warnings;
@ -566,7 +561,7 @@ mod tests {
let details: response::VideoDetails = let details: response::VideoDetails =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res = details.map_response(id, Language::En, None, None).unwrap(); let map_res = details.map_response(id, Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),
@ -586,9 +581,7 @@ mod tests {
let details: response::VideoDetails = let details: response::VideoDetails =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let err = details let err = details.map_response("", Language::En, None).unwrap_err();
.map_response("", Language::En, None, None)
.unwrap_err();
assert!(matches!( assert!(matches!(
err, err,
crate::error::ExtractionError::NotFound { .. } crate::error::ExtractionError::NotFound { .. }
@ -604,7 +597,7 @@ mod tests {
let comments: response::VideoComments = let comments: response::VideoComments =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res = comments.map_response("", Language::En, None, None).unwrap(); let map_res = comments.map_response("", Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),

View file

@ -80,42 +80,32 @@ SAttributed {
Text { Text {
text: "\n\n", text: "\n\n",
}, },
Browse { Text {
text: "#aespa", text: "#aespa",
page_type: Unknown,
browse_id: "FEhashtag",
}, },
Text { Text {
text: " ", text: " ",
}, },
Browse { Text {
text: "#æspa", text: "#æspa",
page_type: Unknown,
browse_id: "FEhashtag",
}, },
Text { Text {
text: " ", text: " ",
}, },
Browse { Text {
text: "#BlackMamba", text: "#BlackMamba",
page_type: Unknown,
browse_id: "FEhashtag",
}, },
Text { Text {
text: " ", text: " ",
}, },
Browse { Text {
text: "#블랙맘바", text: "#블랙맘바",
page_type: Unknown,
browse_id: "FEhashtag",
}, },
Text { Text {
text: " ", text: " ",
}, },
Browse { Text {
text: "#에스파", text: "#에스파",
page_type: Unknown,
browse_id: "FEhashtag",
}, },
Text { Text {
text: "\naespa 에스파 'Black Mamba' MV ℗ SM Entertainment", text: "\naespa 에스파 'Black Mamba' MV ℗ SM Entertainment",

File diff suppressed because it is too large Load diff

View file

@ -26,7 +26,7 @@ pub static VIDEO_ID_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"^[A-Za-z0-9_-
pub static CHANNEL_ID_REGEX: Lazy<Regex> = pub static CHANNEL_ID_REGEX: Lazy<Regex> =
Lazy::new(|| Regex::new(r"^UC[A-Za-z0-9_-]{22}$").unwrap()); Lazy::new(|| Regex::new(r"^UC[A-Za-z0-9_-]{22}$").unwrap());
pub static PLAYLIST_ID_REGEX: Lazy<Regex> = pub static PLAYLIST_ID_REGEX: Lazy<Regex> =
Lazy::new(|| Regex::new(r"^(?:PL|RD|OLAK|UU)[A-Za-z0-9_-]{16,50}$").unwrap()); Lazy::new(|| Regex::new(r"^(?:PL|RDCLAK|OLAK|UU)[A-Za-z0-9_-]{16,50}$").unwrap());
pub static ALBUM_ID_REGEX: Lazy<Regex> = pub static ALBUM_ID_REGEX: Lazy<Regex> =
Lazy::new(|| Regex::new(r"^MPREb_[A-Za-z0-9_-]{11}$").unwrap()); Lazy::new(|| Regex::new(r"^MPREb_[A-Za-z0-9_-]{11}$").unwrap());
pub static VANITY_PATH_REGEX: Lazy<Regex> = Lazy::new(|| { pub static VANITY_PATH_REGEX: Lazy<Regex> = Lazy::new(|| {
@ -128,35 +128,7 @@ where
buf.parse() buf.parse()
} }
/// Parse a string after removing all non-numeric characters. /// Parse all numbers occurring in a string and reurn them as a vec
///
/// If the string contains multiple numbers, it returns the product of them.
pub fn parse_numeric_prod<F>(string: &str) -> Option<F>
where
F: FromStr + Copy + std::ops::Mul<Output = F>,
{
let mut n = None;
let mut buf = String::new();
for c in string.chars() {
if c.is_ascii_digit() {
buf.push(c);
} else if !buf.is_empty() {
if let Ok(x) = buf.parse::<F>() {
n = n.map(|n| n * x).or(Some(x));
}
buf.clear();
}
}
if !buf.is_empty() {
if let Ok(x) = buf.parse::<F>() {
n = n.map(|n| n * x).or(Some(x));
}
}
n
}
/// Parse all numbers occurring in a string and return them as a vec
pub fn parse_numeric_vec<F>(string: &str) -> Vec<F> pub fn parse_numeric_vec<F>(string: &str) -> Vec<F>
where where
F: FromStr, F: FromStr,

View file

@ -199,20 +199,7 @@ pub fn parse_timeago(lang: Language, textual_date: &str) -> Option<TimeAgo> {
let entry = dictionary::entry(lang); let entry = dictionary::entry(lang);
let filtered_str = filter_str(textual_date); let filtered_str = filter_str(textual_date);
let qu: u8 = util::parse_numeric_prod(textual_date).unwrap_or(1); let qu: u8 = util::parse_numeric(textual_date).unwrap_or(1);
// French uses 'a' as a short form of years.
// Since 'a' is also a word in French, it cannot be parsed as a token.
if matches!(
lang,
Language::Fr | Language::FrCa | Language::Es | Language::Es419 | Language::EsUs
) && textual_date.ends_with(" a")
{
return Some(TimeAgo {
n: qu,
unit: TimeUnit::Year,
});
}
TaTokenParser::new(&entry, util::lang_by_char(lang), false, &filtered_str) TaTokenParser::new(&entry, util::lang_by_char(lang), false, &filtered_str)
.next() .next()
@ -416,10 +403,10 @@ mod tests {
use crate::util::tests::TESTFILES; use crate::util::tests::TESTFILES;
#[rstest] #[rstest]
#[case::de(Language::De, "vor 1 Sekunde", Some(TimeAgo { n: 1, unit: TimeUnit::Second }))] #[case(Language::De, "vor 1 Sekunde", Some(TimeAgo { n: 1, unit: TimeUnit::Second }))]
#[case::ar(Language::Ar, "قبل ساعة واحدة", Some(TimeAgo { n: 1, unit: TimeUnit::Hour }))] #[case(Language::Ar, "قبل ساعة واحدة", Some(TimeAgo { n: 1, unit: TimeUnit::Hour }))]
// No-break space // No-break space
#[case::nbsp(Language::De, "Vor 3\u{a0}Tagen aktualisiert", Some(TimeAgo { n: 3, unit: TimeUnit::Day }))] #[case(Language::De, "Vor 3\u{a0}Tagen aktualisiert", Some(TimeAgo { n: 3, unit: TimeUnit::Day }))]
fn t_parse( fn t_parse(
#[case] lang: Language, #[case] lang: Language,
#[case] textual_date: &str, #[case] textual_date: &str,
@ -594,196 +581,7 @@ mod tests {
assert_eq!( assert_eq!(
parse_timeago(*lang, s), parse_timeago(*lang, s),
Some(expect[n]), Some(expect[n]),
"Language: {lang}, txt: `{s}`" "Language: {lang}, n: {n}"
);
});
})
}
#[test]
fn t_testfile_short() {
let json_path = path!(*TESTFILES / "dict" / "timeago_samples_short.json");
let expect = [
TimeAgo {
n: 35,
unit: TimeUnit::Minute,
},
TimeAgo {
n: 50,
unit: TimeUnit::Minute,
},
TimeAgo {
n: 1,
unit: TimeUnit::Hour,
},
TimeAgo {
n: 2,
unit: TimeUnit::Hour,
},
TimeAgo {
n: 3,
unit: TimeUnit::Hour,
},
TimeAgo {
n: 4,
unit: TimeUnit::Hour,
},
TimeAgo {
n: 5,
unit: TimeUnit::Hour,
},
TimeAgo {
n: 6,
unit: TimeUnit::Hour,
},
TimeAgo {
n: 7,
unit: TimeUnit::Hour,
},
TimeAgo {
n: 8,
unit: TimeUnit::Hour,
},
TimeAgo {
n: 9,
unit: TimeUnit::Hour,
},
TimeAgo {
n: 12,
unit: TimeUnit::Hour,
},
TimeAgo {
n: 17,
unit: TimeUnit::Hour,
},
TimeAgo {
n: 18,
unit: TimeUnit::Hour,
},
TimeAgo {
n: 19,
unit: TimeUnit::Hour,
},
TimeAgo {
n: 20,
unit: TimeUnit::Hour,
},
TimeAgo {
n: 10,
unit: TimeUnit::Hour,
},
TimeAgo {
n: 11,
unit: TimeUnit::Hour,
},
TimeAgo {
n: 13,
unit: TimeUnit::Hour,
},
TimeAgo {
n: 1,
unit: TimeUnit::Day,
},
TimeAgo {
n: 2,
unit: TimeUnit::Day,
},
TimeAgo {
n: 3,
unit: TimeUnit::Day,
},
TimeAgo {
n: 4,
unit: TimeUnit::Day,
},
TimeAgo {
n: 6,
unit: TimeUnit::Day,
},
TimeAgo {
n: 8,
unit: TimeUnit::Day,
},
TimeAgo {
n: 10,
unit: TimeUnit::Day,
},
TimeAgo {
n: 11,
unit: TimeUnit::Day,
},
TimeAgo {
n: 12,
unit: TimeUnit::Day,
},
TimeAgo {
n: 13,
unit: TimeUnit::Day,
},
TimeAgo {
n: 2,
unit: TimeUnit::Week,
},
TimeAgo {
n: 3,
unit: TimeUnit::Week,
},
TimeAgo {
n: 1,
unit: TimeUnit::Month,
},
TimeAgo {
n: 4,
unit: TimeUnit::Week,
},
TimeAgo {
n: 7,
unit: TimeUnit::Month,
},
TimeAgo {
n: 10,
unit: TimeUnit::Month,
},
TimeAgo {
n: 1,
unit: TimeUnit::Year,
},
TimeAgo {
n: 2,
unit: TimeUnit::Year,
},
TimeAgo {
n: 3,
unit: TimeUnit::Year,
},
TimeAgo {
n: 4,
unit: TimeUnit::Year,
},
TimeAgo {
n: 5,
unit: TimeUnit::Year,
},
];
let json_file = File::open(json_path).unwrap();
let strings_map: BTreeMap<Language, Vec<String>> =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
strings_map.iter().for_each(|(lang, strings)| {
assert_eq!(strings.len(), expect.len(), "Language: {lang}");
strings.iter().enumerate().for_each(|(n, s)| {
let mut exp = expect[n];
if *lang == Language::Mn && exp.unit == TimeUnit::Week {
exp.unit = TimeUnit::Day;
exp.n *= 7;
}
assert_eq!(
parse_timeago(*lang, s),
Some(exp),
"Language: {lang}, txt: `{s}`"
); );
}); });
}) })

View file

@ -1,2 +0,0 @@
node_modules
package-lock.json

View file

@ -1,162 +0,0 @@
const fs = require("fs");
const DICT_PATH = "../dictionary.json";
function translateLang(lang) {
switch (lang) {
case "iw": // Hebrew
return "he";
case "zh-CN": // Simplified Chinese
return "zh-Hans";
case "zh-HK":
return "zh-Hant-HK";
case "zh-TW":
return "zh-Hant";
default:
return lang;
}
}
function prepString(s, by_char) {
const replaced = s.toLowerCase().replace("{0}", "").replace("-", " ");
if (by_char) {
return replaced.replace(/\s/, "").split("");
} else {
return replaced.split(/\s+/);
}
}
function storeToken(tokens, word, unit) {
if (word) {
if (word in tokens && tokens[word] != unit) {
tokens[word] = null;
} else {
tokens[word] = unit;
}
}
}
function validateTokens(tokens, lang) {
const units = { Y: 1, M: 1, W: 1, D: 1, h: 1, m: 1, s: 1 };
if (lang === "iw") {
tokens["שתי"] = "2";
}
for (const [key, val] of Object.entries(tokens)) {
if (val === null) {
delete tokens[key];
} else {
delete units[val];
}
}
if (Object.keys(units).length > 0) {
console.log(
`missing units ${JSON.stringify(
Object.keys(units)
)} for lang: ${lang}; tokens: ${JSON.stringify(tokens)}`
);
}
}
function validateNdTokens(tokens, lang) {
const units = { "0D": 1, "1D": 1 };
for (const [key, val] of Object.entries(tokens)) {
if (val === null) {
delete tokens[key];
} else {
delete units[val];
}
}
if (Object.keys(units).length > 0) {
console.log(
`missing nd tokens ${JSON.stringify(
Object.keys(units)
)} for lang: ${lang}; tokens: ${JSON.stringify(tokens)}`
);
} else if (Object.keys(tokens).length > 2) {
console.log(
`too many nd tokens for lang: ${lang}; tokens: ${JSON.stringify(tokens)}`
);
}
}
const sortObject = (obj) =>
Object.keys(obj)
.sort()
.reduce((res, key) => ((res[key] = obj[key]), res), {});
function collectTimeago(lang, by_char, timeagoTokens, timeagoNdTokens) {
const cldrLang = translateLang(lang);
const dates = require(`cldr-dates-modern/main/${cldrLang}/dateFields.json`);
const dateFields = dates.main[cldrLang].dates.fields;
for (const [unitStr, unit] of Object.entries(units)) {
for (const unitFields of [dateFields[unitStr], dateFields[`${unitStr}-short`]]) {
for (const [sKey, s] of Object.entries(unitFields["relativeTime-type-past"])) {
let u = unit;
if (s.indexOf("{0}") === -1) {
if (sKey.endsWith("-zero")) {
u = "0" + u;
} else if (sKey.endsWith("-one")) {
u = "1" + u;
} else if (sKey.endsWith("-two")) {
u = "2" + u;
} else {
throw new Error(`Invalid time pattern. lang: ${lang} key: ${sKey}`);
}
}
const words = prepString(s, by_char);
for (const word of words) {
storeToken(timeagoTokens, word, u);
}
}
}
}
if (dateFields.day["relative-type-0"]) {
const words = prepString(dateFields.day["relative-type-0"], by_char);
for (const word of words) {
storeToken(timeagoNdTokens, word, "0D");
}
}
if (dateFields.day["relative-type--1"]) {
const words = prepString(dateFields.day["relative-type--1"], by_char);
for (const word of words) {
storeToken(timeagoNdTokens, word, "1D");
}
}
}
const dict = JSON.parse(fs.readFileSync(DICT_PATH));
const units = {
second: "s",
minute: "m",
hour: "h",
day: "D",
week: "W",
month: "M",
year: "Y",
};
for (const [mainLang, entry] of Object.entries(dict)) {
const langs = [mainLang, ...entry["equivalent"]];
const timeagoTokens = {};
const timeagoNdTokens = {};
for (lang of langs) {
collectTimeago(lang, entry["by_char"], timeagoTokens, timeagoNdTokens);
}
validateTokens(timeagoTokens, mainLang);
// validateNdTokens(timeagoNdTokens, mainLang);
dict[mainLang]["timeago_tokens"] = timeagoTokens;
// dict[mainLang]["timeago_nd_tokens"] = timeagoNdTokens;
}
fs.writeFileSync(DICT_PATH, JSON.stringify(dict, null, 2));

View file

@ -1,12 +0,0 @@
{
"name": "cldr_data",
"version": "1.0.0",
"description": "Build the RustyPipe parsing dictionary using CLDR data",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"dependencies": {
"cldr-dates-modern": "^43.0.0",
"cldr-numbers-modern": "^43.0.0"
}
}

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -2336,12 +2336,7 @@ fn lang() -> Language {
/// Get a new RustyPipe instance /// Get a new RustyPipe instance
#[fixture] #[fixture]
fn rp(lang: Language) -> RustyPipe { fn rp(lang: Language) -> RustyPipe {
let vdata = std::env::var("YT_VDATA").ok(); RustyPipe::builder().strict().lang(lang).build()
RustyPipe::builder()
.strict()
.lang(lang)
.visitor_data_opt(vdata)
.build()
} }
/// Get a flag signaling if the language is set to English /// Get a flag signaling if the language is set to English