diff --git a/Cargo.lock b/Cargo.lock index 9156ad3..519d51d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,6 +90,28 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-stream" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad445822218ce64be7a341abfb0b1ea43b5c23aa83902542a4542e78309d8e5e" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.64" @@ -173,6 +195,12 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + [[package]] name = "bytes" version = "1.4.0" @@ -648,6 +676,18 @@ dependencies = [ "slab", ] +[[package]] +name = "futures_codec" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce54d63f8b0c75023ed920d46fd71d0cbbb830b0ee012726b5b4f506fb6dea5b" +dependencies = [ + "bytes 0.5.6", + "futures", + "memchr", + "pin-project", +] + [[package]] name = "fxhash" version = "0.2.1" @@ -694,7 +734,7 @@ version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" dependencies = [ - "bytes", + "bytes 1.4.0", "fnv", "futures-core", "futures-sink", @@ -721,7 +761,7 @@ checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" dependencies = [ "base64 0.13.1", "bitflags", - "bytes", + "bytes 1.4.0", "headers-core", "http", "httpdate", @@ -786,7 +826,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes", + "bytes 1.4.0", "fnv", "itoa", ] @@ -797,7 +837,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes", + "bytes 1.4.0", "http", "pin-project-lite", ] @@ -820,7 +860,7 @@ version = "0.14.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" dependencies = [ - "bytes", + "bytes 1.4.0", "futures-channel", "futures-core", "futures-util", @@ -1055,7 +1095,7 @@ version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ed4198ce7a4cbd2a57af78d28c6fbb57d81ac5f1d6ad79ac6c5587419cbdf22" dependencies = [ - "bytes", + "bytes 1.4.0", "encoding_rs", "futures-util", "http", @@ -1200,6 +1240,26 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +[[package]] +name = "pin-project" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ef0f924a5ee7ea9cbcea77529dba45f8a9ba9f622419fe3386ca581a3ae9d5a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851c8d0ce9bebe43790dedfc86614c23494ac9f423dd618d3a61fc693eafe61e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -1225,7 +1285,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0608069d4999c3c02d49dff261663f2e73a8f7b00b7cd364fb5e93e419dafa1" dependencies = [ "async-trait", - "bytes", + "bytes 1.4.0", "chrono", "cookie", "futures-util", @@ -1246,6 +1306,7 @@ dependencies = [ "serde_urlencoded", "serde_yaml", "smallvec", + "sse-codec", "tempfile", "thiserror", "time", @@ -1274,7 +1335,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1077defedfd8ff15990bb42993970ac75bc46dd8a5b3c0b452ab4e2041b825a4" dependencies = [ "base64 0.21.0", - "bytes", + "bytes 1.4.0", "derive_more", "futures-util", "mime", @@ -1686,6 +1747,18 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dccf47db1b41fa1573ed27ccf5e08e3ca771cb994f776668c5ebda893b248fc" +[[package]] +name = "sse-codec" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a59f811350c44b4a037aabeb72dc6a9591fc22aa95a036db9a96297c58085a" +dependencies = [ + "bytes 0.5.6", + "futures-io", + "futures_codec", + "memchr", +] + [[package]] name = "strsim" version = "0.10.0" @@ -1719,7 +1792,6 @@ dependencies = [ "flate2", "hex", "hex-literal", - "httpdate", "insta", "log", "mime_guess", @@ -1739,6 +1811,7 @@ dependencies = [ "thiserror", "time", "tokio", + "tokio-test", "toml", "tracing-subscriber", "zip", @@ -1853,7 +1926,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" dependencies = [ "autocfg", - "bytes", + "bytes 1.4.0", "libc", "memchr", "mio", @@ -1886,14 +1959,28 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-test" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53474327ae5e166530d17f2d956afcb4f8a004de581b3cae10f12006bc8163e3" +dependencies = [ + "async-stream", + "bytes 1.4.0", + "futures-core", + "tokio", + "tokio-stream", +] + [[package]] name = "tokio-util" version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" dependencies = [ - "bytes", + "bytes 1.4.0", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 585b375..543fa6e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,6 @@ mime_guess = { version = "2.0.4", default-features = false } compressible = "0.2.0" regex = "1.7.1" log = "0.4.17" -httpdate = "1.0.2" tracing-subscriber = "0.3.16" async-compression = { version = "0.3.15", features = [ "tokio", @@ -47,6 +46,8 @@ async-compression = { version = "0.3.15", features = [ [dev-dependencies] rstest = "0.16.0" +poem = { version = "1.3.55", features = ["test"] } +tokio-test = "0.4.2" temp_testdir = "0.2.3" insta = { version = "1.17.1", features = ["ron"] } hex-literal = "0.3.4" diff --git a/src/config.rs b/src/config.rs index a4ad846..4d1288a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -3,6 +3,8 @@ use std::{collections::BTreeMap, ops::Deref, path::Path, sync::Arc}; use regex::Regex; use serde::{Deserialize, Serialize}; +use crate::storage::CompressionAlg; + #[derive(Clone, Default)] pub struct Config { i: Arc, @@ -88,9 +90,9 @@ impl Default for ServerCfg { fn default() -> Self { Self { address: "0.0.0.0:3000".to_owned(), - root_domain: "localhost".to_owned(), + root_domain: "localhost:3000".to_owned(), internal_subdomain: "talon".to_owned(), - internal_url: "http://talon.localhost".to_owned(), + internal_url: "http://talon.localhost:3000".to_owned(), } } } @@ -111,9 +113,9 @@ pub struct CompressionCfg { impl Default for CompressionCfg { fn default() -> Self { Self { - gzip_en: false, + gzip_en: true, gzip_level: 6, - brotli_en: false, + brotli_en: true, brotli_level: 7, } } @@ -123,6 +125,17 @@ impl CompressionCfg { pub fn enabled(&self) -> bool { self.gzip_en || self.brotli_en } + + pub fn algs(&self) -> Vec { + let mut res = vec![CompressionAlg::None]; + if self.gzip_en { + res.push(CompressionAlg::Gzip) + } + if self.brotli_en { + res.push(CompressionAlg::Brotli) + } + res + } } #[derive(Debug, Default, Clone, Serialize, Deserialize)] diff --git a/src/page.rs b/src/page.rs index 6c612f8..10fd756 100644 --- a/src/page.rs +++ b/src/page.rs @@ -36,30 +36,30 @@ pub async fn page(request: &Request, talon: Data<&Talon>) -> Result { let ws = talon.db.get_website(subdomain)?; let vid = ws.latest_version.ok_or(PageError::NoVersion)?; - let (file, ok) = - match talon - .storage - .get_file(vid, request.original_uri().path(), request.headers()) - { - Ok(file) => (file, true), - Err(StorageError::NotFound(f)) => { - let version = talon.db.get_version(subdomain, vid)?; - if let Some(fallback) = &version.fallback { - ( - talon.storage.get_file(vid, fallback, request.headers())?, - version.spa, - ) - } else { - return Err(StorageError::NotFound(f).into()); - } + let (file, ok) = match talon + .storage + .get_file(vid, request.uri().path(), request.headers()) + { + Ok(file) => (file, true), + Err(StorageError::NotFound(f)) => { + let version = talon.db.get_version(subdomain, vid)?; + if let Some(fallback) = &version.fallback { + ( + talon.storage.get_file(vid, fallback, request.headers())?, + version.spa, + ) + } else { + return Err(StorageError::NotFound(f).into()); } - Err(e) => return Err(e.into()), - }; + } + Err(e) => return Err(e.into()), + }; Ok(match file.rd_path { Some(rd_path) => Redirect::moved_permanent(rd_path).into_response(), - None => file - .to_response(request.headers(), ok) + None => talon + .storage + .file_to_response(file, request.headers(), ok) .await? .into_response(), }) diff --git a/src/server.rs b/src/server.rs index 8af17f9..c5c9649 100644 --- a/src/server.rs +++ b/src/server.rs @@ -2,7 +2,10 @@ use std::{ops::Deref, path::Path, sync::Arc}; use crate::{api::TalonApi, config::Config, db::Db, page::page, storage::Storage, util}; use path_macro::path; -use poem::{listener::TcpListener, EndpointExt, Route, RouteDomain, Server}; +use poem::{ + http::header, listener::TcpListener, middleware, Endpoint, EndpointExt, Route, RouteDomain, + Server, +}; use poem_openapi::OpenApiService; #[derive(Clone)] @@ -54,7 +57,7 @@ impl Talon { }) } - pub async fn launch(&self) -> Result<()> { + pub fn endpoint(&self) -> impl Endpoint { let api_service = OpenApiService::new(TalonApi, "Talon", "0.1.0") .server(format!("{}/api", self.i.cfg.server.internal_url)); let swagger_ui = api_service.swagger_ui(); @@ -71,7 +74,7 @@ impl Talon { "/api/spec", poem::endpoint::make_sync(move |_| spec.clone()), ) - .with(poem::middleware::Cors::new()); + .with(middleware::Cors::new()); let internal_domain = format!( "{}.{}", @@ -79,15 +82,20 @@ impl Talon { ); let site_domains = format!("+.{}", self.i.cfg.server.root_domain); - let route = RouteDomain::new() - .at(&internal_domain, route_internal) - .at(&site_domains, page) + RouteDomain::new() + .at(internal_domain, route_internal) + .at(site_domains, page) .at(&self.i.cfg.server.root_domain, page) - .with(poem::middleware::Tracing) - .data(self.clone()); + .with(middleware::Tracing) + .with( + middleware::SetHeader::new().overriding(header::X_CONTENT_TYPE_OPTIONS, "nosniff"), + ) + .data(self.clone()) + } + pub async fn launch(&self) -> Result<()> { Server::new(TcpListener::bind(&self.i.cfg.server.address)) - .run(route) + .run(self.endpoint()) .await?; Ok(()) } diff --git a/src/storage.rs b/src/storage.rs index 3b968a3..9a59220 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -9,7 +9,6 @@ use std::{ use flate2::{read::GzDecoder, write::GzEncoder}; use hex::ToHex; -use httpdate::HttpDate; use mime_guess::Mime; use poem::{ error::{ResponseError, StaticFileError}, @@ -52,7 +51,7 @@ impl CompressionAlg { } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct GotFile { /// File hash pub hash: String, @@ -363,26 +362,25 @@ impl Storage { )), } } -} -impl GotFile { /// Convert the retrieved file to a HTTP response /// /// Adapted from: - pub async fn to_response( - self, + pub async fn file_to_response( + &self, + gf: GotFile, headers: &HeaderMap, ok: bool, ) -> std::result::Result { - let mut file = File::open(&self.file_path)?; + let mut file = File::open(&gf.file_path)?; let metadata = file.metadata()?; // etag and last modified - let etag = headers::ETag::from_str(&format!("\"{}\"", self.hash)).unwrap(); - let mut last_modified_str = String::new(); + let etag = headers::ETag::from_str(&format!("\"{}\"", gf.hash)).unwrap(); + let mut last_modified = None; if ok { - // handle if-match and if-(un)modified queries + // handle if-(none)-match and if-(un)modified queries let if_match = headers.typed_get::(); let if_unmodified_since = headers.typed_get::(); let if_none_match = headers.typed_get::(); @@ -411,11 +409,11 @@ impl GotFile { } } - last_modified_str = HttpDate::from(modified).to_string(); + last_modified = Some(headers::LastModified::from(modified)); } } - if self + if gf .mime .as_ref() .map(|m| m.essence_str() == "text/html") @@ -434,28 +432,21 @@ impl GotFile { } // Compress response if possible - let alg = util::parse_accept_encoding( - headers, - &[ - CompressionAlg::Brotli, - CompressionAlg::Gzip, - CompressionAlg::None, - ], - ) - .unwrap_or_default(); + let alg = util::parse_accept_encoding(headers, &self.cfg.compression.algs()) + .unwrap_or_default(); let body = match alg { CompressionAlg::None => Body::from(html), CompressionAlg::Gzip => { let enc = async_compression::tokio::bufread::GzipEncoder::with_quality( tokio::io::BufReader::new(Body::from(html).into_async_read()), - async_compression::Level::Precise(6), + async_compression::Level::Precise(self.cfg.compression.gzip_level.into()), ); Body::from_async_read(enc) } CompressionAlg::Brotli => { let enc = async_compression::tokio::bufread::BrotliEncoder::with_quality( tokio::io::BufReader::new(Body::from(html).into_async_read()), - async_compression::Level::Precise(7), + async_compression::Level::Precise(self.cfg.compression.brotli_level.into()), ); Body::from_async_read(enc) } @@ -469,8 +460,8 @@ impl GotFile { if let Some(encoding) = alg.encoding() { response = response.header(header::CONTENT_ENCODING, encoding) } - if !last_modified_str.is_empty() { - response = response.header(header::LAST_MODIFIED, last_modified_str); + if let Some(last_modified) = last_modified { + response = response.typed_header(last_modified); } Ok(response.body(body)) @@ -517,13 +508,13 @@ impl GotFile { } else { response = response.status(StatusCode::NOT_FOUND); } - if !last_modified_str.is_empty() { - response = response.header(header::LAST_MODIFIED, last_modified_str); + if let Some(last_modified) = last_modified { + response = response.typed_header(last_modified); } - if let Some(encoding) = self.encoding { + if let Some(encoding) = gf.encoding { response = response.header(header::CONTENT_ENCODING, encoding); } - if let Some(mime) = &self.mime { + if let Some(mime) = &gf.mime { response = response.header(header::CONTENT_TYPE, mime.essence_str()); } if let Some((range, size)) = content_range { diff --git a/tests/fixtures/mod.rs b/tests/fixtures/mod.rs index 691b13b..7804f88 100644 --- a/tests/fixtures/mod.rs +++ b/tests/fixtures/mod.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, ops::Deref}; +use std::{collections::BTreeMap, fs::File, ops::Deref}; use hex_literal::hex; use path_macro::path; @@ -7,27 +7,47 @@ use temp_testdir::TempDir; use time::macros::datetime; use talon::{ - config::{CompressionCfg, Config, ConfigInner}, db::{ model::{Version, Website}, Db, }, - storage::Storage, + Talon, }; pub const SUBDOMAIN_1: &str = "-"; pub const SUBDOMAIN_2: &str = "spotify-gender-ex"; pub const SUBDOMAIN_3: &str = "rustypipe"; +pub const SUBDOMAIN_4: &str = "spa"; pub const VERSION_1_1: u32 = 1; pub const VERSION_1_2: u32 = 2; pub const VERSION_2_1: u32 = 3; pub const VERSION_3_1: u32 = 4; +pub const VERSION_4_1: u32 = 5; pub const HASH_1_1_INDEX: [u8; 32] = hex!("3b5f6bad5376897435def176d0fe77e5b9b4f0deafc7491fc27262650744ad68"); pub const HASH_1_1_STYLE: [u8; 32] = hex!("356f131c825fbf604797c7e9c85352549d81db8af91fee834016d075110af026"); +pub const HASH_1_2_INDEX: [u8; 32] = + hex!("a44816e6c3b650bdf88e6532659ba07ef187c2113ae311da9709e056aec8eadb"); +pub const HASH_1_2_STYLE: [u8; 32] = + hex!("356f131c825fbf604797c7e9c85352549d81db8af91fee834016d075110af026"); +pub const HASH_2_1_INDEX: [u8; 32] = + hex!("6c5d37546616519e8973be51515b8a90898b4675f7b6d01f2d891edb686408a2"); +pub const HASH_2_1_STYLE: [u8; 32] = + hex!("fc825b409a49724af8f5b3c4ad15e175e68095ea746237a7b46152d3f383f541"); +pub const HASH_3_1_INDEX: [u8; 32] = + hex!("cc31423924cf1f124750825861ab1ccc675e755921fc2fa111c0a98e8c346a5e"); +pub const HASH_3_1_STYLE: [u8; 32] = + hex!("ee4fc4911a56e627c047a29ba3085131939d8d487759b9149d42aaab89ce8993"); +pub const HASH_3_1_PAGE2: [u8; 32] = + hex!("be4f409ca0adcb21cdc7130cde63031718406726f889ef97ac8870c90b330a75"); + +pub const HASH_SPA_INDEX: [u8; 32] = + hex!("90d454dd999b52486902e845f748ce7510a1fc8404421c8f44dadccc5b4b8e1d"); +pub const HASH_SPA_FALLBACK: [u8; 32] = + hex!("4ee0d3f7522f620a2a69b39b7443f8fe65029e1324cefaf797b8cad2b223cf7b"); pub struct DbTest { db: Db, @@ -91,6 +111,18 @@ fn insert_websites(db: &Db) { }, ) .unwrap(); + db.insert_website( + SUBDOMAIN_4, + &Website { + name: "SvelteKit SPA".to_owned(), + created_at: datetime!(2023-03-03 22:00 +0), + latest_version: Some(VERSION_4_1), + color: Some(16727552), + visibility: talon::model::Visibility::Hidden, + ..Default::default() + }, + ) + .unwrap(); let mut v1_data = BTreeMap::new(); v1_data.insert("Version".to_owned(), "v0.1.0".to_owned()); @@ -134,7 +166,6 @@ fn insert_websites(db: &Db) { VERSION_2_1, &Version { created_at: datetime!(2023-02-18 16:30 +0), - data: BTreeMap::new(), ..Default::default() }, ) @@ -145,7 +176,19 @@ fn insert_websites(db: &Db) { VERSION_3_1, &Version { created_at: datetime!(2023-02-20 18:30 +0), - data: BTreeMap::new(), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(db.new_version_id().unwrap(), VERSION_4_1); + db.insert_version( + SUBDOMAIN_4, + VERSION_4_1, + &Version { + created_at: datetime!(2023-03-03 22:00 +0), + fallback: Some("200.html".to_owned()), + spa: true, ..Default::default() }, ) @@ -163,18 +206,10 @@ pub fn db() -> DbTest { db.insert_file(VERSION_1_1, "style.css", &HASH_1_1_STYLE) .unwrap(); - db.insert_file( - VERSION_1_2, - "index.html", - &hex!("a44816e6c3b650bdf88e6532659ba07ef187c2113ae311da9709e056aec8eadb"), - ) - .unwrap(); - db.insert_file( - VERSION_1_2, - "assets/style.css", - &hex!("356f131c825fbf604797c7e9c85352549d81db8af91fee834016d075110af026"), - ) - .unwrap(); + db.insert_file(VERSION_1_2, "index.html", &HASH_1_2_INDEX) + .unwrap(); + db.insert_file(VERSION_1_2, "assets/style.css", &HASH_1_2_STYLE) + .unwrap(); db.insert_file( VERSION_1_2, "assets/image.jpg", @@ -194,80 +229,64 @@ pub fn db() -> DbTest { ) .unwrap(); - db.insert_file( - VERSION_2_1, - "index.html", - &hex!("6c5d37546616519e8973be51515b8a90898b4675f7b6d01f2d891edb686408a2"), - ) - .unwrap(); - db.insert_file( - VERSION_2_1, - "gex_style.css", - &hex!("fc825b409a49724af8f5b3c4ad15e175e68095ea746237a7b46152d3f383f541"), - ) - .unwrap(); + db.insert_file(VERSION_2_1, "index.html", &HASH_2_1_INDEX) + .unwrap(); + db.insert_file(VERSION_2_1, "gex_style.css", &HASH_2_1_STYLE) + .unwrap(); - db.insert_file( - VERSION_3_1, - "index.html", - &hex!("94a67cf13d752a9c1875ad999eb2be5a1b0f9746c66bca2631820b8186028811"), - ) - .unwrap(); - db.insert_file( - VERSION_3_1, - "rp_style.css", - &hex!("ee4fc4911a56e627c047a29ba3085131939d8d487759b9149d42aaab89ce8993"), - ) - .unwrap(); + db.insert_file(VERSION_3_1, "index.html", &HASH_3_1_INDEX) + .unwrap(); + db.insert_file(VERSION_3_1, "rp_style.css", &HASH_3_1_STYLE) + .unwrap(); + db.insert_file(VERSION_3_1, "page2/index.html", &HASH_3_1_PAGE2) + .unwrap(); DbTest { db, _temp: temp } } -pub struct StorageTest { - store: Storage, - _temp: TempDir, +pub struct TalonTest { + talon: Talon, + pub temp: TempDir, } -impl Deref for StorageTest { - type Target = Storage; +impl Deref for TalonTest { + type Target = Talon; fn deref(&self) -> &Self::Target { - &self.store + &self.talon } } #[fixture] -pub fn store() -> StorageTest { +pub fn tln() -> TalonTest { let temp = temp_testdir::TempDir::default(); - let db_path = path!(temp / "db"); - std::fs::create_dir(&db_path).unwrap(); + let talon = Talon::new(&temp).unwrap(); - let cfg = Config::new(ConfigInner { - compression: CompressionCfg { - gzip_en: true, - brotli_en: true, - ..Default::default() - }, - ..Default::default() - }); + insert_websites(&talon.db); - let db = Db::new(&db_path).unwrap(); - insert_websites(&db); - - let store = Storage::new(temp.to_path_buf(), db, cfg); - - store + talon + .storage .insert_dir(path!("tests" / "testfiles" / "ThetaDev0"), VERSION_1_1) .unwrap(); - store + talon + .storage .insert_dir(path!("tests" / "testfiles" / "ThetaDev1"), VERSION_1_2) .unwrap(); - store + talon + .storage .insert_dir(path!("tests" / "testfiles" / "GenderEx"), VERSION_2_1) .unwrap(); - store + talon + .storage .insert_dir(path!("tests" / "testfiles" / "RustyPipe"), VERSION_3_1) .unwrap(); + talon + .storage + .insert_tgz_archive( + File::open(path!("tests" / "testfiles" / "archive" / "spa.tar.gz")).unwrap(), + VERSION_4_1, + ) + .unwrap(); - StorageTest { store, _temp: temp } + TalonTest { talon, temp } } diff --git a/tests/snapshots/tests__config__sparse.snap b/tests/snapshots/tests__config__sparse.snap index a31db55..f9d11cb 100644 --- a/tests/snapshots/tests__config__sparse.snap +++ b/tests/snapshots/tests__config__sparse.snap @@ -5,14 +5,14 @@ expression: "&cfg" ConfigInner( server: ServerCfg( address: "0.0.0.0:3000", - root_domain: "localhost", + root_domain: "localhost:3000", internal_subdomain: "talon", - internal_url: "http://talon.localhost", + internal_url: "http://talon.localhost:3000", ), compression: CompressionCfg( gzip_en: true, gzip_level: 6, - brotli_en: false, + brotli_en: true, brotli_level: 7, ), keys: { diff --git a/tests/snapshots/tests__database__delete_website.snap b/tests/snapshots/tests__database__delete_website.snap index f7de527..6408673 100644 --- a/tests/snapshots/tests__database__delete_website.snap +++ b/tests/snapshots/tests__database__delete_website.snap @@ -3,10 +3,13 @@ source: tests/tests.rs expression: data --- {"type":"website","key":"rustypipe","value":{"name":"RustyPipe","created_at":[2023,51,18,30,0,0,0,0,0],"latest_version":4,"color":7943647,"visibility":"featured","source_url":"https://code.thetadev.de/ThetaDev/rustypipe","source_icon":"gitea"}} +{"type":"website","key":"spa","value":{"name":"SvelteKit SPA","created_at":[2023,62,22,0,0,0,0,0,0],"latest_version":5,"color":16727552,"visibility":"hidden","source_url":null,"source_icon":null}} {"type":"website","key":"spotify-gender-ex","value":{"name":"Spotify-Gender-Ex","created_at":[2023,49,16,30,0,0,0,0,0],"latest_version":3,"color":1947988,"visibility":"featured","source_url":"https://github.com/Theta-Dev/Spotify-Gender-Ex","source_icon":"github"}} {"type":"version","key":"rustypipe:4","value":{"created_at":[2023,51,18,30,0,0,0,0,0],"data":{},"fallback":null,"spa":false}} +{"type":"version","key":"spa:5","value":{"created_at":[2023,62,22,0,0,0,0,0,0],"data":{},"fallback":null,"spa":false}} {"type":"version","key":"spotify-gender-ex:3","value":{"created_at":[2023,49,16,30,0,0,0,0,0],"data":{},"fallback":null,"spa":false}} {"type":"file","key":"3:gex_style.css","value":"fc825b409a49724af8f5b3c4ad15e175e68095ea746237a7b46152d3f383f541"} {"type":"file","key":"3:index.html","value":"6c5d37546616519e8973be51515b8a90898b4675f7b6d01f2d891edb686408a2"} -{"type":"file","key":"4:index.html","value":"94a67cf13d752a9c1875ad999eb2be5a1b0f9746c66bca2631820b8186028811"} +{"type":"file","key":"4:index.html","value":"cc31423924cf1f124750825861ab1ccc675e755921fc2fa111c0a98e8c346a5e"} +{"type":"file","key":"4:page2/index.html","value":"be4f409ca0adcb21cdc7130cde63031718406726f889ef97ac8870c90b330a75"} {"type":"file","key":"4:rp_style.css","value":"ee4fc4911a56e627c047a29ba3085131939d8d487759b9149d42aaab89ce8993"} diff --git a/tests/snapshots/tests__database__export.snap b/tests/snapshots/tests__database__export.snap index d78a80b..5b1a231 100644 --- a/tests/snapshots/tests__database__export.snap +++ b/tests/snapshots/tests__database__export.snap @@ -4,10 +4,12 @@ expression: data --- {"type":"website","key":"-","value":{"name":"ThetaDev","created_at":[2023,49,16,30,0,0,0,0,0],"latest_version":2,"color":2068974,"visibility":"featured","source_url":null,"source_icon":null}} {"type":"website","key":"rustypipe","value":{"name":"RustyPipe","created_at":[2023,51,18,30,0,0,0,0,0],"latest_version":4,"color":7943647,"visibility":"featured","source_url":"https://code.thetadev.de/ThetaDev/rustypipe","source_icon":"gitea"}} +{"type":"website","key":"spa","value":{"name":"SvelteKit SPA","created_at":[2023,62,22,0,0,0,0,0,0],"latest_version":5,"color":16727552,"visibility":"hidden","source_url":null,"source_icon":null}} {"type":"website","key":"spotify-gender-ex","value":{"name":"Spotify-Gender-Ex","created_at":[2023,49,16,30,0,0,0,0,0],"latest_version":3,"color":1947988,"visibility":"featured","source_url":"https://github.com/Theta-Dev/Spotify-Gender-Ex","source_icon":"github"}} {"type":"version","key":"-:1","value":{"created_at":[2023,49,16,30,0,0,0,0,0],"data":{"Deployed by":"https://github.com/Theta-Dev/Talon/actions/runs/1352014628","Version":"v0.1.0"},"fallback":null,"spa":false}} {"type":"version","key":"-:2","value":{"created_at":[2023,49,16,52,0,0,0,0,0],"data":{"Deployed by":"https://github.com/Theta-Dev/Talon/actions/runs/1354755231","Version":"v0.1.1"},"fallback":null,"spa":false}} {"type":"version","key":"rustypipe:4","value":{"created_at":[2023,51,18,30,0,0,0,0,0],"data":{},"fallback":null,"spa":false}} +{"type":"version","key":"spa:5","value":{"created_at":[2023,62,22,0,0,0,0,0,0],"data":{},"fallback":null,"spa":false}} {"type":"version","key":"spotify-gender-ex:3","value":{"created_at":[2023,49,16,30,0,0,0,0,0],"data":{},"fallback":null,"spa":false}} {"type":"file","key":"1:index.html","value":"3b5f6bad5376897435def176d0fe77e5b9b4f0deafc7491fc27262650744ad68"} {"type":"file","key":"1:style.css","value":"356f131c825fbf604797c7e9c85352549d81db8af91fee834016d075110af026"} @@ -18,5 +20,6 @@ expression: data {"type":"file","key":"2:index.html","value":"a44816e6c3b650bdf88e6532659ba07ef187c2113ae311da9709e056aec8eadb"} {"type":"file","key":"3:gex_style.css","value":"fc825b409a49724af8f5b3c4ad15e175e68095ea746237a7b46152d3f383f541"} {"type":"file","key":"3:index.html","value":"6c5d37546616519e8973be51515b8a90898b4675f7b6d01f2d891edb686408a2"} -{"type":"file","key":"4:index.html","value":"94a67cf13d752a9c1875ad999eb2be5a1b0f9746c66bca2631820b8186028811"} +{"type":"file","key":"4:index.html","value":"cc31423924cf1f124750825861ab1ccc675e755921fc2fa111c0a98e8c346a5e"} +{"type":"file","key":"4:page2/index.html","value":"be4f409ca0adcb21cdc7130cde63031718406726f889ef97ac8870c90b330a75"} {"type":"file","key":"4:rp_style.css","value":"ee4fc4911a56e627c047a29ba3085131939d8d487759b9149d42aaab89ce8993"} diff --git a/tests/snapshots/tests__database__get_websites.snap b/tests/snapshots/tests__database__get_websites.snap index a5c5430..5693d0b 100644 --- a/tests/snapshots/tests__database__get_websites.snap +++ b/tests/snapshots/tests__database__get_websites.snap @@ -21,6 +21,15 @@ expression: websites source_url: Some("https://code.thetadev.de/ThetaDev/rustypipe"), source_icon: Some(gitea), )), + ("spa", Website( + name: "SvelteKit SPA", + created_at: (2023, 62, 22, 0, 0, 0, 0, 0, 0), + latest_version: Some(5), + color: Some(16727552), + visibility: hidden, + source_url: None, + source_icon: None, + )), ("spotify-gender-ex", Website( name: "Spotify-Gender-Ex", created_at: (2023, 49, 16, 30, 0, 0, 0, 0, 0), diff --git a/tests/testfiles/archive/spa.tar.gz b/tests/testfiles/archive/spa.tar.gz new file mode 100644 index 0000000..c4ad22f Binary files /dev/null and b/tests/testfiles/archive/spa.tar.gz differ diff --git a/tests/tests.rs b/tests/tests.rs index 22f825e..0071098 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -176,15 +176,22 @@ mod database { #[rstest] fn get_file_hashes(db: DbTest) { let hashes = db.get_file_hashes().unwrap(); - assert_eq!(hashes.len(), 10) + assert_eq!(hashes.len(), 11) } } mod storage { + use std::{str::FromStr, time::SystemTime}; + use hex::ToHex; - use poem::http::{header, HeaderMap}; + use poem::{ + error::StaticFileError, + http::{header, HeaderMap, StatusCode}, + web::headers::{self, HeaderMapExt}, + }; use talon::config::{CompressionCfg, Config, ConfigInner}; - use talon::storage::Storage; + use talon::storage::{GotFile, Storage}; + use time::OffsetDateTime; use super::*; @@ -297,7 +304,7 @@ mod storage { #[case::image("br", VERSION_1_2, "assets/image.jpg", false, "image/jpeg", None)] #[case::subdir("br", VERSION_3_1, "page2", false, "text/html", Some("/page2/"))] fn get_file( - store: StorageTest, + tln: TalonTest, #[case] encoding: &str, #[case] version: u32, #[case] path: &str, @@ -318,7 +325,8 @@ mod storage { None }; - let index_file = store.get_file(version, path, &headers).unwrap(); + let index_file = tln.storage.get_file(version, path, &headers).unwrap(); + dbg!(&index_file); assert!(index_file.file_path.is_file()); assert_eq!( index_file @@ -334,6 +342,213 @@ mod storage { assert_eq!(index_file.mime.unwrap().essence_str(), mime); assert_eq!(index_file.rd_path.as_deref(), rd_path); } + + #[rstest] + #[case::index(&HASH_1_2_INDEX, "text/html", "")] + #[case::index_gz(&HASH_1_2_INDEX, "text/html", "gzip")] + #[case::index_br(&HASH_1_2_INDEX, "text/html", "br")] + #[case::style(&HASH_1_2_STYLE, "text/css", "")] + fn file_to_response( + tln: TalonTest, + #[case] hash: &[u8], + #[case] mime: &str, + #[case] encoding: &str, + ) { + let gf = got_file(&tln, hash, mime); + + let file_date = std::fs::metadata(&gf.file_path) + .unwrap() + .modified() + .unwrap(); + + let mut headers = HeaderMap::new(); + if !encoding.is_empty() { + headers.insert(header::ACCEPT_ENCODING, encoding.parse().unwrap()); + } + + let resp = + tokio_test::block_on(tln.storage.file_to_response(gf.clone(), &headers, true)).unwrap(); + + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(resp.header(header::CONTENT_TYPE).unwrap(), mime); + assert_eq!( + resp.header(header::ETAG).unwrap(), + format!("\"{}\"", gf.hash) + ); + + let date = OffsetDateTime::from(SystemTime::from( + resp.headers().typed_get::().unwrap(), + )); + assert!(date - file_date < time::Duration::SECOND); + + // HTML files should get dynamically compressed + if mime == "text/html" && !encoding.is_empty() { + assert_eq!(resp.header(header::CONTENT_ENCODING).unwrap(), encoding); + } else { + assert!(resp.header(header::CONTENT_ENCODING).is_none()) + } + } + + fn got_file(tln: &TalonTest, hash: &[u8], mime: &str) -> GotFile { + let hash = hash.encode_hex::(); + let file_path = path!(tln.temp / "storage" / &hash[..2] / &hash); + GotFile { + hash: hash.clone(), + file_path, + encoding: None, + mime: Some(mime_guess::Mime::from_str(mime).unwrap()), + rd_path: None, + } + } + + fn got_file_html(tln: &TalonTest) -> GotFile { + got_file(tln, &HASH_1_2_INDEX, "text/html") + } + + #[rstest] + fn file_to_response_inject(tln: TalonTest) { + let gf = got_file_html(&tln); + let resp = tokio_test::block_on(tln.storage.file_to_response(gf, &HeaderMap::new(), true)) + .unwrap(); + let body = tokio_test::block_on(resp.into_body().into_string()).unwrap(); + assert!(body.contains("\n")); + } + + #[rstest] + #[case::unmodified(true)] + #[case::modified(true)] + fn file_to_response_if_modified(tln: TalonTest, #[case] modified: bool) { + let gf = got_file_html(&tln); + let mut file_date = std::fs::metadata(&gf.file_path) + .unwrap() + .modified() + .unwrap(); + if modified { + file_date -= std::time::Duration::from_secs(1); + } + + let mut headers = HeaderMap::new(); + headers.typed_insert(headers::IfModifiedSince::from(file_date)); + + let resp = tokio_test::block_on(tln.storage.file_to_response(gf, &headers, true)).unwrap(); + assert_eq!( + resp.status(), + if modified { + StatusCode::OK + } else { + StatusCode::NOT_MODIFIED + } + ); + assert_eq!(resp.into_body().is_empty(), !modified); + } + + #[rstest] + #[case::unmodified(true)] + #[case::modified(true)] + fn file_to_response_if_unmodified(tln: TalonTest, #[case] modified: bool) { + let gf = got_file_html(&tln); + let mut file_date = std::fs::metadata(&gf.file_path) + .unwrap() + .modified() + .unwrap(); + if modified { + file_date -= std::time::Duration::from_secs(1); + } + + let mut headers = HeaderMap::new(); + headers.typed_insert(headers::IfModifiedSince::from(file_date)); + + let res = tokio_test::block_on(tln.storage.file_to_response(gf, &headers, true)); + + if modified { + let resp = res.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert!(!resp.into_body().is_empty()); + } else { + assert!(matches!( + res.unwrap_err(), + StaticFileError::PreconditionFailed + )); + } + } + + #[rstest] + #[case::matched(true)] + #[case::no_match(false)] + fn file_to_response_if_match(tln: TalonTest, #[case] matched: bool) { + let gf = got_file_html(&tln); + let etag = format!( + "\"{}\"", + if matched { + gf.hash.clone() + } else { + HASH_2_1_INDEX.encode_hex() + } + ); + + let mut headers = HeaderMap::new(); + headers.typed_insert(headers::IfMatch::from( + headers::ETag::from_str(&etag).unwrap(), + )); + + let res = tokio_test::block_on(tln.storage.file_to_response(gf, &headers, true)); + + if matched { + let resp = res.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert!(!resp.into_body().is_empty()); + } else { + assert!(matches!( + res.unwrap_err(), + StaticFileError::PreconditionFailed + )); + } + } + + #[rstest] + #[case::matched(true)] + #[case::no_match(false)] + fn file_to_response_if_none_match(tln: TalonTest, #[case] matched: bool) { + let gf = got_file_html(&tln); + let etag = format!( + "\"{}\"", + if matched { + gf.hash.clone() + } else { + HASH_2_1_INDEX.encode_hex() + } + ); + + let mut headers = HeaderMap::new(); + headers.typed_insert(headers::IfNoneMatch::from( + headers::ETag::from_str(&etag).unwrap(), + )); + + let resp = tokio_test::block_on(tln.storage.file_to_response(gf, &headers, true)).unwrap(); + assert_eq!( + resp.status(), + if matched { + StatusCode::NOT_MODIFIED + } else { + StatusCode::OK + } + ); + assert_eq!(resp.into_body().is_empty(), matched); + } + + #[rstest] + fn file_to_response_range(tln: TalonTest) { + let gf = got_file(&tln, &HASH_2_1_STYLE, "text/css"); + + let mut headers = HeaderMap::new(); + headers.typed_insert(headers::Range::bytes(0..100).unwrap()); + + let resp = tokio_test::block_on(tln.storage.file_to_response(gf, &headers, true)).unwrap(); + assert_eq!(resp.status(), StatusCode::PARTIAL_CONTENT); + + let body = tokio_test::block_on(resp.into_body().into_bytes()).unwrap(); + assert_eq!(body.len(), 100); + } } mod config { @@ -351,3 +566,43 @@ mod config { insta::assert_ron_snapshot!(name, &cfg); } } + +mod page { + use hex::ToHex; + use poem::{http::header, test::TestClient}; + + use super::*; + + #[rstest] + #[case::index("", "/", &HASH_1_2_INDEX, "text/html")] + #[case::style("", "/assets/style.css", &HASH_1_2_STYLE, "text/css")] + #[case::rustypipe("rustypipe", "/", &HASH_3_1_INDEX, "text/html")] + #[case::rustypipe2("rustypipe", "/page2/index.html", &HASH_3_1_PAGE2, "text/html")] + #[case::spa_index("spa", "/", &HASH_SPA_INDEX, "text/html")] + #[case::spa_fallback("spa", "/user/2", &HASH_SPA_FALLBACK, "text/html")] + fn page( + tln: TalonTest, + #[case] subdomain: &str, + #[case] path: &str, + #[case] hash: &[u8], + #[case] mime: &str, + ) { + let host = if subdomain.is_empty() { + "localhost:3000".to_owned() + } else { + format!("{subdomain}.localhost:3000") + }; + + let resp = tokio_test::block_on( + TestClient::new(tln.endpoint()) + .get(format!("http://{host}{path}")) + .header(header::HOST, host) + .data(tln.clone()) + .send(), + ); + + resp.assert_status_is_ok(); + resp.assert_content_type(mime); + resp.assert_header(header::ETAG, format!("\"{}\"", hash.encode_hex::())) + } +}