From 8f129e44b89fce62bf4bb21286c75a5c95a2f1f7 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 1 Mar 2023 18:53:33 +0100 Subject: [PATCH 1/6] fix: dont compress html files, dont copy files if they exist --- src/storage.rs | 62 +++++++++++++++++++++++++++++++------------------- tests/tests.rs | 12 ++++++---- 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/src/storage.rs b/src/storage.rs index 13ec448..e8fbdaf 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -124,32 +124,34 @@ impl Storage { let hash = util::hash_file(file_path)?; let stored_file = self.file_path_mkdir(&hash)?; - fs::copy(file_path, &stored_file)?; + if !stored_file.is_file() { + fs::copy(file_path, &stored_file)?; - if self.cfg.compression.enabled() - && mime_guess::from_path(file_path) - .first() - .map(|t| compressible::is_compressible(t.essence_str())) - .unwrap_or_default() - { - if self.cfg.compression.gzip_en { - let mut encoder = GzEncoder::new( - fs::File::create(stored_file.with_extension("gz"))?, - flate2::Compression::new(self.cfg.compression.gzip_level.into()), - ); - let mut input = BufReader::new(fs::File::open(&stored_file)?); - std::io::copy(&mut input, &mut encoder)?; - } + if self.cfg.compression.enabled() + && mime_guess::from_path(file_path) + .first() + .map(|t| Self::is_compressible(t.essence_str())) + .unwrap_or_default() + { + if self.cfg.compression.gzip_en { + let mut encoder = GzEncoder::new( + fs::File::create(stored_file.with_extension("gz"))?, + flate2::Compression::new(self.cfg.compression.gzip_level.into()), + ); + let mut input = BufReader::new(fs::File::open(&stored_file)?); + std::io::copy(&mut input, &mut encoder)?; + } - if self.cfg.compression.brotli_en { - let mut encoder = brotli::CompressorWriter::new( - fs::File::create(stored_file.with_extension("br"))?, - 4096, - self.cfg.compression.brotli_level.into(), - 20, - ); - let mut input = BufReader::new(fs::File::open(&stored_file)?); - std::io::copy(&mut input, &mut encoder)?; + if self.cfg.compression.brotli_en { + let mut encoder = brotli::CompressorWriter::new( + fs::File::create(stored_file.with_extension("br"))?, + 4096, + self.cfg.compression.brotli_level.into(), + 20, + ); + let mut input = BufReader::new(fs::File::open(&stored_file)?); + std::io::copy(&mut input, &mut encoder)?; + } } } @@ -236,6 +238,8 @@ impl Storage { self.insert_dir(import_path, version) } + /// Get the path of a file with the given hash while creating the subdirectory + /// if necessary fn file_path_mkdir(&self, hash: &[u8]) -> Result { let hash_str = hash.encode_hex::(); @@ -246,12 +250,14 @@ impl Storage { Ok(subdir.join(&hash_str)) } + /// Get the path of a file with the given hash fn file_path(&self, hash: &[u8]) -> PathBuf { let hash_str = hash.encode_hex::(); let subdir = self.path.join(&hash_str[..2]); subdir.join(&hash_str) } + /// Get all compressed versions of a file fn files_compressed(&self, hash: &[u8]) -> BTreeMap { let path = self.file_path(hash); let mut res = BTreeMap::new(); @@ -274,6 +280,14 @@ impl Storage { res } + /// Check if a file with the given mime type should be compressed + /// + /// HTML files should not be compressed, since they need to be injected with the + /// UI code + fn is_compressible(mime: &str) -> bool { + mime != "text/html" && compressible::is_compressible(mime) + } + /// Get a file using the raw site path and the website version /// /// HTTP headers are used to determine if the compressed version of a file should be returned. diff --git a/tests/tests.rs b/tests/tests.rs index 0ff7890..22f825e 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -283,17 +283,19 @@ mod storage { // Images should not be compressed let expect = &hash_str != "901d291a47a8a9b55c06f84e5e5f82fd2dcee65cac1406d6e878b805d45c1e93" - && &hash_str != "9f7e7971b4bfdb75429e534dea461ed90340886925078cda252cada9aa0e25f7"; + && &hash_str != "9f7e7971b4bfdb75429e534dea461ed90340886925078cda252cada9aa0e25f7" + && &hash_str != "a44816e6c3b650bdf88e6532659ba07ef187c2113ae311da9709e056aec8eadb"; assert_eq!(path_compressed.is_file(), expect) } } #[rstest] - #[case::nocmp("", VERSION_1_2, "", true, "text/html", None)] - #[case::gzip("gzip", VERSION_1_2, "", true, "text/html", None)] - #[case::br("br", VERSION_1_2, "", true, "text/html", None)] + #[case::index("br", VERSION_1_2, "", false, "text/html", None)] + #[case::nocmp("", VERSION_1_2, "assets/style.css", true, "text/css", None)] + #[case::gzip("gzip", VERSION_1_2, "assets/style.css", true, "text/css", None)] + #[case::br("br", VERSION_1_2, "assets/style.css", true, "text/css", None)] #[case::image("br", VERSION_1_2, "assets/image.jpg", false, "image/jpeg", None)] - #[case::subdir("br", VERSION_3_1, "page2", true, "text/html", Some("/page2/"))] + #[case::subdir("br", VERSION_3_1, "page2", false, "text/html", Some("/page2/"))] fn get_file( store: StorageTest, #[case] encoding: &str, From c95dde6b0c2a0ba5e04c55468ce973a41786ec1f Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 1 Mar 2023 20:03:14 +0100 Subject: [PATCH 2/6] feat: inject client code into html --- src/page.rs | 4 +- src/storage.rs | 114 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 84 insertions(+), 34 deletions(-) diff --git a/src/page.rs b/src/page.rs index 82c0f2c..4ebb25a 100644 --- a/src/page.rs +++ b/src/page.rs @@ -23,7 +23,7 @@ impl ResponseError for PageError { } #[handler] -pub fn page(request: &Request, talon: Data<&Talon>) -> Result { +pub async fn page(request: &Request, talon: Data<&Talon>) -> Result { let host = request .header(header::HOST) .ok_or(PageError::InvalidSubdomain)?; @@ -42,6 +42,6 @@ pub fn page(request: &Request, talon: Data<&Talon>) -> Result { Ok(match file.rd_path { Some(rd_path) => Redirect::moved_permanent(rd_path).into_response(), - None => file.to_response(request.headers())?.into_response(), + None => file.to_response(request.headers()).await?.into_response(), }) } diff --git a/src/storage.rs b/src/storage.rs index e8fbdaf..20272d2 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -1,7 +1,7 @@ use std::{ borrow::Cow, collections::BTreeMap, - fs, + fs::{self, File}, io::{BufReader, Read, Seek, SeekFrom}, ops::Bound, path::{Path, PathBuf}, @@ -258,20 +258,26 @@ impl Storage { } /// Get all compressed versions of a file - fn files_compressed(&self, hash: &[u8]) -> BTreeMap { + fn files_compressed( + &self, + hash: &[u8], + is_compressible: bool, + ) -> BTreeMap { let path = self.file_path(hash); let mut res = BTreeMap::new(); - if self.cfg.compression.gzip_en { - let path_gz = path.with_extension("gz"); - if path_gz.is_file() { - res.insert(CompressionAlg::Gzip, path_gz); + if is_compressible { + if self.cfg.compression.gzip_en { + let path_gz = path.with_extension("gz"); + if path_gz.is_file() { + res.insert(CompressionAlg::Gzip, path_gz); + } } - } - if self.cfg.compression.brotli_en { - let path_br = path.with_extension("br"); - if path_br.is_file() { - res.insert(CompressionAlg::Brotli, path_br); + if self.cfg.compression.brotli_en { + let path_br = path.with_extension("br"); + if path_br.is_file() { + res.insert(CompressionAlg::Brotli, path_br); + } } } if path.is_file() { @@ -329,7 +335,12 @@ impl Storage { let mime = util::site_path_mime(&new_path); - let files = self.files_compressed(&hash); + let files = self.files_compressed( + &hash, + mime.as_ref() + .map(|m| Self::is_compressible(m.essence_str())) + .unwrap_or_default(), + ); let file = util::parse_accept_encoding(headers, &files); match file { @@ -350,20 +361,15 @@ impl Storage { impl GotFile { /// Convert the retrieved file to a HTTP response - /// - /// Adapted from: - pub fn to_response( + pub async fn to_response( self, headers: &HeaderMap, ) -> std::result::Result { - let path = self.file_path; - let mut file = std::fs::File::open(path)?; + let file = File::open(&self.file_path)?; let metadata = file.metadata()?; - // content length - let mut content_length = metadata.len(); - // etag and last modified + let etag = headers::ETag::from_str(&format!("\"{}\"", self.hash)).unwrap(); let mut last_modified_str = String::new(); // extract headers @@ -371,11 +377,9 @@ impl GotFile { let if_unmodified_since = headers.typed_get::(); let if_none_match = headers.typed_get::(); let if_modified_since = headers.typed_get::(); - let range = headers.typed_get::(); + // handle if-match and if-(un)modified queries if let Ok(modified) = metadata.modified() { - let etag = headers::ETag::from_str(&format!("\"{}\"", self.hash)).unwrap(); - if let Some(if_match) = if_match { if !if_match.precondition_passes(&etag) { return Err(StaticFileError::PreconditionFailed); @@ -401,6 +405,54 @@ impl GotFile { last_modified_str = HttpDate::from(modified).to_string(); } + if self + .mime + .as_ref() + .map(|m| m.essence_str() == "text/html") + .unwrap_or_default() + { + // Inject UI code into HTML + let to_inject = "\n"; + + let mut html = String::with_capacity(metadata.len() as usize); + tokio::fs::File::from_std(file) + .read_to_string(&mut html) + .await?; + + if let Some(ctag_pos) = html.rfind("") { + html.insert_str(ctag_pos, to_inject); + } + + let etag = headers::ETag::from_str(&format!("\"{}\"", self.hash)).unwrap(); + + let mut response = Response::builder() + .header(header::CONTENT_TYPE, "text/html") + .typed_header(etag); + + if !last_modified_str.is_empty() { + response = response.header(header::LAST_MODIFIED, last_modified_str); + } + + Ok(response.body(html)) + } else { + self.to_static_response(headers, file, metadata.len(), etag, last_modified_str) + } + } + + /// Convert the retrieved file to a static file response + /// + /// Adapted from: + fn to_static_response( + &self, + headers: &HeaderMap, + mut file: File, + size: u64, + etag: headers::ETag, + last_modified_str: String, + ) -> std::result::Result { + let range = headers.typed_get::(); + + let mut content_length = size; let mut content_range = None; let body = if let Some((start, end)) = range.and_then(|range| range.iter().next()) { @@ -412,16 +464,14 @@ impl GotFile { let end = match end { Bound::Included(n) => n + 1, Bound::Excluded(n) => n, - Bound::Unbounded => metadata.len(), + Bound::Unbounded => size, }; - if end < start || end > metadata.len() { - return Err(StaticFileError::RangeNotSatisfiable { - size: metadata.len(), - }); + if end < start || end > size { + return Err(StaticFileError::RangeNotSatisfiable { size }); } - if start != 0 || end != metadata.len() { - content_range = Some((start..end, metadata.len())); + if start != 0 || end != size { + content_range = Some((start..end, size)); } content_length = end - start; @@ -434,7 +484,7 @@ impl GotFile { let mut response = Response::builder() .header(header::ACCEPT_RANGES, "bytes") .header(header::CONTENT_LENGTH, content_length) - .header(header::ETAG, self.hash); + .typed_header(etag); if !last_modified_str.is_empty() { response = response.header(header::LAST_MODIFIED, last_modified_str); @@ -442,7 +492,7 @@ impl GotFile { if let Some(encoding) = self.encoding { response = response.header(header::CONTENT_ENCODING, encoding); } - if let Some(mime) = self.mime { + if let Some(mime) = &self.mime { response = response.header(header::CONTENT_TYPE, mime.essence_str()); } if let Some((range, size)) = content_range { From c4a5d7d17813c04f129e00d6162fa03ca8eb7ab7 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 1 Mar 2023 20:44:10 +0100 Subject: [PATCH 3/6] feat: add logging, compression --- Cargo.lock | 123 ++++++++++++++++++++++++++++++++++---------------- Cargo.toml | 4 +- src/main.rs | 5 ++ src/server.rs | 5 +- 4 files changed, 95 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bd474cc..73f434e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,6 +76,20 @@ dependencies = [ "libc", ] +[[package]] +name = "async-compression" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a" +dependencies = [ + "brotli", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-trait" version = "0.1.64" @@ -462,19 +476,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "env_logger" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" -dependencies = [ - "humantime", - "is-terminal", - "log", - "regex", - "termcolor", -] - [[package]] name = "errno" version = "0.2.8" @@ -746,12 +747,6 @@ dependencies = [ "libc", ] -[[package]] -name = "hermit-abi" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" - [[package]] name = "hex" version = "0.4.3" @@ -819,12 +814,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - [[package]] name = "hyper" version = "0.14.24" @@ -932,18 +921,6 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "is-terminal" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b6b32576413a8e69b90e952e4a026476040d81017b80445deda5f2d3921857" -dependencies = [ - "hermit-abi 0.3.1", - "io-lifetimes", - "rustix", - "windows-sys 0.45.0", -] - [[package]] name = "itoa" version = "1.0.5" @@ -1100,6 +1077,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -1125,7 +1112,7 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "libc", ] @@ -1141,6 +1128,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "parking_lot" version = "0.11.2" @@ -1231,6 +1224,7 @@ version = "1.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0608069d4999c3c02d49dff261663f2e73a8f7b00b7cd364fb5e93e419dafa1" dependencies = [ + "async-compression", "async-trait", "bytes", "chrono", @@ -1631,6 +1625,15 @@ dependencies = [ "digest", ] +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + [[package]] name = "similar" version = "2.2.1" @@ -1713,7 +1716,6 @@ version = "0.1.0" dependencies = [ "brotli", "compressible", - "env_logger", "flate2", "hex", "hex-literal", @@ -1738,6 +1740,7 @@ dependencies = [ "time", "tokio", "toml", + "tracing-subscriber", "zip", ] @@ -1806,6 +1809,16 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "time" version = "0.3.20" @@ -1974,6 +1987,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", ] [[package]] @@ -2034,6 +2073,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc7ed8ba44ca06be78ea1ad2c3682a43349126c8818054231ee6f4748012aed2" +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "version_check" version = "0.9.4" diff --git a/Cargo.toml b/Cargo.toml index fa355d9..3091c02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ license = "MIT" description = "Static site management system" [dependencies] -poem = "1.3.55" +poem = {version = "1.3.55", features = ["compression"]} poem-openapi = { version = "2.0.26", features = ["time", "swagger-ui"] } tokio = { version = "1.25.0", features = ["rt-multi-thread", "fs"] } sled = "0.34.7" @@ -38,7 +38,7 @@ compressible = "0.2.0" regex = "1.7.1" log = "0.4.17" httpdate = "1.0.2" -env_logger = "0.10.0" +tracing-subscriber = "0.3.16" [dev-dependencies] rstest = "0.16.0" diff --git a/src/main.rs b/src/main.rs index 65b55df..3c54f80 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,11 @@ use talon::{Result, Talon}; #[tokio::main] async fn main() -> Result<()> { + if std::env::var_os("RUST_LOG").is_none() { + std::env::set_var("RUST_LOG", "info"); + } + tracing_subscriber::fmt::init(); + let talon = Talon::new("tmp")?; talon.launch().await } diff --git a/src/server.rs b/src/server.rs index 516cf24..e0b5096 100644 --- a/src/server.rs +++ b/src/server.rs @@ -70,7 +70,8 @@ impl Talon { .at( "/api/spec", poem::endpoint::make_sync(move |_| spec.clone()), - ); + ) + .with(poem::middleware::Cors::new()); let internal_domain = format!( "{}.{}", @@ -82,6 +83,8 @@ impl Talon { .at(&internal_domain, route_internal) .at(&site_domains, page) .at(&self.i.cfg.server.root_domain, page) + .with(poem::middleware::Tracing) + .with(poem::middleware::Compression::new()) .data(self.clone()); Server::new(TcpListener::bind(&self.i.cfg.server.address)) From 6de6b21281a2b3f108d094aac0e9c98d6677ab53 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Thu, 2 Mar 2023 02:14:40 +0100 Subject: [PATCH 4/6] feat: add /websitesAll endpoint --- src/api.rs | 77 +++++++++++++++++++++++++++++++++++++++++++-------- src/config.rs | 22 +++++++++++++++ src/model.rs | 6 +++- 3 files changed, 92 insertions(+), 13 deletions(-) diff --git a/src/api.rs b/src/api.rs index b4cdb3c..703309b 100644 --- a/src/api.rs +++ b/src/api.rs @@ -8,12 +8,18 @@ use poem::{ }; use poem_openapi::{ auth::ApiKey, - param::Path, + param::{Path, Query}, payload::{Binary, Json}, OpenApi, SecurityScheme, }; -use crate::{config::KeyCfg, db, model::*, oai::DynParams, util, Talon}; +use crate::{ + config::{Access, KeyCfg}, + db, + model::*, + oai::DynParams, + util, Talon, +}; pub struct TalonApi; @@ -32,10 +38,10 @@ async fn api_key_checker(req: &Request, api_key: ApiKey) -> Option { } impl ApiKeyAuthorization { - fn check_subdomain(&self, subdomain: &str) -> Result<()> { + fn check_subdomain(&self, subdomain: &str, access: Access) -> Result<()> { if subdomain.is_empty() { Err(ApiError::InvalidSubdomain.into()) - } else if !self.0.domains.matches_domain(subdomain) { + } else if !self.0.domains.matches_domain(subdomain) || !self.0.allows(access) { Err(ApiError::NoAccess.into()) } else { Ok(()) @@ -85,7 +91,7 @@ impl TalonApi { subdomain: Path, website: Json, ) -> Result<()> { - auth.check_subdomain(&subdomain)?; + auth.check_subdomain(&subdomain, Access::Modify)?; if subdomain.as_str() == talon.cfg.server.internal_subdomain || !util::validate_subdomain(&subdomain) { @@ -105,7 +111,7 @@ impl TalonApi { subdomain: Path, website: Json, ) -> Result<()> { - auth.check_subdomain(&subdomain)?; + auth.check_subdomain(&subdomain, Access::Modify)?; talon.db.update_website(&subdomain, website.0.into())?; Ok(()) @@ -119,19 +125,66 @@ impl TalonApi { talon: Data<&Talon>, subdomain: Path, ) -> Result<()> { - auth.check_subdomain(&subdomain)?; + auth.check_subdomain(&subdomain, Access::Modify)?; talon.db.delete_website(&subdomain, true)?; Ok(()) } - /// Get all websites + /// Get all publicly listed websites + /// + /// Returns all publicly listed websites (visibility != `hidden`) #[oai(path = "/websites", method = "get")] - async fn websites_get(&self, talon: Data<&Talon>) -> Result>> { + async fn websites_get( + &self, + talon: Data<&Talon>, + /// Mimimum visibility of the websites + #[oai(default)] + visibility: Query, + ) -> Result>> { talon .db .get_websites() .map(|r| r.map(Website::from)) + .filter(|ws| match ws { + Ok(ws) => ws.visibility != Visibility::Hidden && ws.visibility <= visibility.0, + Err(_) => true, + }) + .collect::, _>>() + .map(Json) + .map_err(Error::from) + } + + /// Get all websites + /// + /// Returns all websites from Talon's database (including hidden ones, if the current user + /// has access to them). This endpoint requires authentication (use the `/websites` endpoint + /// for unauthenticated users). + #[oai(path = "/websitesAll", method = "get")] + async fn websites_get_all( + &self, + auth: ApiKeyAuthorization, + talon: Data<&Talon>, + /// Mimimum visibility of the websites + #[oai(default)] + visibility: Query, + ) -> Result>> { + talon + .db + .get_websites() + .map(|r| r.map(Website::from)) + .filter(|ws| match ws { + Ok(ws) => { + if ws.visibility == Visibility::Hidden + && auth.check_subdomain(&ws.subdomain, Access::Read).is_err() + { + false + } else { + ws.visibility <= visibility.0 + } + } + Err(_) => true, + }) .collect::, _>>() .map(Json) .map_err(Error::from) @@ -178,7 +231,7 @@ impl TalonApi { subdomain: Path, id: Path, ) -> Result<()> { - auth.check_subdomain(&subdomain)?; + auth.check_subdomain(&subdomain, Access::Modify)?; talon.db.delete_version(&subdomain, *id, true)?; Ok(()) @@ -199,7 +252,7 @@ impl TalonApi { /// zip archive with the website files data: Binary>, ) -> Result<()> { - auth.check_subdomain(&subdomain)?; + auth.check_subdomain(&subdomain, Access::Upload)?; let vid = talon.db.new_version_id()?; talon .storage @@ -238,7 +291,7 @@ impl TalonApi { /// tar.gz archive with the website files data: Binary>, ) -> Result<()> { - auth.check_subdomain(&subdomain)?; + auth.check_subdomain(&subdomain, Access::Upload)?; let vid = talon.db.new_version_id()?; talon.storage.insert_tgz_archive(data.as_slice(), vid)?; diff --git a/src/config.rs b/src/config.rs index 7b39ab6..a4ad846 100644 --- a/src/config.rs +++ b/src/config.rs @@ -130,6 +130,8 @@ impl CompressionCfg { pub struct KeyCfg { #[serde(skip_serializing_if = "Domains::is_none")] pub domains: Domains, + pub upload: bool, + pub modify: bool, } #[derive(Debug, Default, Clone, Serialize, Deserialize)] @@ -141,6 +143,16 @@ pub enum Domains { Multiple(Vec), } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Access { + /// Dont modify anything + Read, + /// Update a new website version + Upload, + /// Create, update or delete websites + Modify, +} + impl Domains { fn is_none(&self) -> bool { matches!(self, Domains::None) @@ -175,6 +187,16 @@ impl Domains { } } +impl KeyCfg { + pub fn allows(&self, access: Access) -> bool { + match access { + Access::Read => true, + Access::Upload => self.upload, + Access::Modify => self.modify, + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/model.rs b/src/model.rs index 54b9cdd..00d4c76 100644 --- a/src/model.rs +++ b/src/model.rs @@ -75,7 +75,10 @@ pub struct Version { pub data: BTreeMap, } -#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Enum, Serialize, Deserialize)] +#[derive( + Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Enum, Serialize, Deserialize, +)] +#[oai(rename_all = "snake_case")] #[serde(rename_all = "snake_case")] pub enum Visibility { Featured, @@ -85,6 +88,7 @@ pub enum Visibility { } #[derive(Debug, Copy, Clone, PartialEq, Eq, Enum, Serialize, Deserialize)] +#[oai(rename_all = "snake_case")] #[serde(rename_all = "snake_case")] pub enum SourceIcon { Link, From 16dd203018e16e8ddb1da68ec4d3c600a4c8712a Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Thu, 2 Mar 2023 13:16:30 +0100 Subject: [PATCH 5/6] feat: add fallback option --- src/api.rs | 97 +++++++++++++++--------- src/db/model.rs | 9 +++ src/oai.rs | 8 +- src/page.rs | 31 ++++++-- src/storage.rs | 171 ++++++++++++++++++++---------------------- tests/fixtures/mod.rs | 4 + 6 files changed, 186 insertions(+), 134 deletions(-) diff --git a/src/api.rs b/src/api.rs index 703309b..1b7e5e8 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1,4 +1,4 @@ -use std::io::Cursor; +use std::{collections::BTreeMap, io::Cursor}; use poem::{ error::{Error, ResponseError}, @@ -55,18 +55,21 @@ enum ApiError { InvalidSubdomain, #[error("you do not have access to this subdomain")] NoAccess, + #[error("invalid fallback: {0}")] + InvalidFallback(String), } impl ResponseError for ApiError { fn status(&self) -> StatusCode { match self { - ApiError::InvalidSubdomain => StatusCode::BAD_REQUEST, + ApiError::InvalidSubdomain | ApiError::InvalidFallback(_) => StatusCode::BAD_REQUEST, ApiError::NoAccess => StatusCode::UNAUTHORIZED, } } } #[OpenApi] +#[allow(clippy::too_many_arguments)] impl TalonApi { /// Get a website #[oai(path = "/website/:subdomain", method = "get")] @@ -237,6 +240,46 @@ impl TalonApi { Ok(()) } + fn insert_version( + talon: &Talon, + subdomain: &str, + id: u32, + fallback: Option, + spa: bool, + mut version_data: BTreeMap, + ) -> Result<()> { + version_data.remove("fallback"); + version_data.remove("spa"); + + // Validata fallback path + if let Some(fallback) = &fallback { + if let Err(e) = talon.storage.get_file(id, fallback, &Default::default()) { + // Remove the uploaded files of the bad version + let _ = talon.db.delete_version(subdomain, id, false); + return Err(ApiError::InvalidFallback(e.to_string()).into()); + } + } + + talon.db.insert_version( + subdomain, + id, + &db::model::Version { + data: version_data, + fallback, + spa, + ..Default::default() + }, + )?; + talon.db.update_website( + subdomain, + db::model::WebsiteUpdate { + latest_version: Some(Some(id)), + ..Default::default() + }, + )?; + Ok(()) + } + /// Upload a new version (.zip archive) #[oai(path = "/website/:subdomain/uploadZip", method = "post")] async fn version_upload_zip( @@ -244,6 +287,13 @@ impl TalonApi { auth: ApiKeyAuthorization, talon: Data<&Talon>, subdomain: Path, + /// Fallback page + /// + /// The fallback page gets returned when the requested page does not exist + fallback: Query>, + /// SPA mode (return fallback page with OK status) + #[oai(default)] + spa: Query, /// Associated version data /// /// This is an arbitrary string map that can hold build information and other stuff @@ -257,23 +307,7 @@ impl TalonApi { talon .storage .insert_zip_archive(Cursor::new(data.as_slice()), vid)?; - - talon.db.insert_version( - &subdomain, - vid, - &db::model::Version { - data: version_data.0, - ..Default::default() - }, - )?; - talon.db.update_website( - &subdomain, - db::model::WebsiteUpdate { - latest_version: Some(Some(vid)), - ..Default::default() - }, - )?; - Ok(()) + Self::insert_version(&talon, &subdomain, vid, fallback.0, spa.0, version_data.0) } /// Upload a new version (.tar.gz archive) @@ -283,6 +317,13 @@ impl TalonApi { auth: ApiKeyAuthorization, talon: Data<&Talon>, subdomain: Path, + /// Fallback page + /// + /// The fallback page gets returned when the requested page does not exist + fallback: Query>, + /// SPA mode (return fallback page with OK status) + #[oai(default)] + spa: Query, /// Associated version data /// /// This is an arbitrary string map that can hold build information and other stuff @@ -294,22 +335,6 @@ impl TalonApi { auth.check_subdomain(&subdomain, Access::Upload)?; let vid = talon.db.new_version_id()?; talon.storage.insert_tgz_archive(data.as_slice(), vid)?; - - talon.db.insert_version( - &subdomain, - vid, - &db::model::Version { - data: version_data.0, - ..Default::default() - }, - )?; - talon.db.update_website( - &subdomain, - db::model::WebsiteUpdate { - latest_version: Some(Some(vid)), - ..Default::default() - }, - )?; - Ok(()) + Self::insert_version(&talon, &subdomain, vid, fallback.0, spa.0, version_data.0) } } diff --git a/src/db/model.rs b/src/db/model.rs index b6e2760..89f8548 100644 --- a/src/db/model.rs +++ b/src/db/model.rs @@ -66,7 +66,14 @@ pub struct Version { /// /// This is an arbitrary string map that can hold build information and other stuff /// and will be displayed in the site info dialog. + #[serde(default)] pub data: BTreeMap, + /// Path of the fallback page which is returned if the requested path was not found + #[serde(default)] + pub fallback: Option, + /// SPA mode (return the fallback page with OK sta) + #[serde(default)] + pub spa: bool, } impl Default for Version { @@ -74,6 +81,8 @@ impl Default for Version { Self { created_at: OffsetDateTime::now_utc(), data: Default::default(), + fallback: Default::default(), + spa: Default::default(), } } } diff --git a/src/oai.rs b/src/oai.rs index 5f6fc8c..c637c9d 100644 --- a/src/oai.rs +++ b/src/oai.rs @@ -15,11 +15,11 @@ impl<'a> ApiExtractor<'a> for DynParams { const TYPE: ApiExtractorType = ApiExtractorType::Parameter; const PARAM_IS_REQUIRED: bool = false; - type ParamType = BTreeMap; - type ParamRawType = Self::ParamType; + type ParamType = Self; + type ParamRawType = BTreeMap; fn register(registry: &mut Registry) { - Self::ParamType::register(registry); + Self::ParamRawType::register(registry); } fn param_in() -> Option { @@ -27,7 +27,7 @@ impl<'a> ApiExtractor<'a> for DynParams { } fn param_schema_ref() -> Option { - Some(Self::ParamType::schema_ref()) + Some(Self::ParamRawType::schema_ref()) } fn param_raw_type(&self) -> Option<&Self::ParamRawType> { diff --git a/src/page.rs b/src/page.rs index 4ebb25a..6c612f8 100644 --- a/src/page.rs +++ b/src/page.rs @@ -6,7 +6,7 @@ use poem::{ IntoResponse, Request, Response, Result, }; -use crate::Talon; +use crate::{storage::StorageError, Talon}; #[derive(thiserror::Error, Debug)] pub enum PageError { @@ -35,13 +35,32 @@ pub async fn page(request: &Request, talon: Data<&Talon>) -> Result { }; let ws = talon.db.get_website(subdomain)?; - let version = ws.latest_version.ok_or(PageError::NoVersion)?; - let file = talon - .storage - .get_file(version, request.original_uri().path(), request.headers())?; + let vid = ws.latest_version.ok_or(PageError::NoVersion)?; + let (file, ok) = + match talon + .storage + .get_file(vid, request.original_uri().path(), request.headers()) + { + Ok(file) => (file, true), + Err(StorageError::NotFound(f)) => { + let version = talon.db.get_version(subdomain, vid)?; + if let Some(fallback) = &version.fallback { + ( + talon.storage.get_file(vid, fallback, request.headers())?, + version.spa, + ) + } else { + return Err(StorageError::NotFound(f).into()); + } + } + Err(e) => return Err(e.into()), + }; Ok(match file.rd_path { Some(rd_path) => Redirect::moved_permanent(rd_path).into_response(), - None => file.to_response(request.headers()).await?.into_response(), + None => file + .to_response(request.headers(), ok) + .await? + .into_response(), }) } diff --git a/src/storage.rs b/src/storage.rs index 20272d2..b539349 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -361,48 +361,52 @@ impl Storage { impl GotFile { /// Convert the retrieved file to a HTTP response + /// + /// Adapted from: pub async fn to_response( self, headers: &HeaderMap, + ok: bool, ) -> std::result::Result { - let file = File::open(&self.file_path)?; + let mut file = File::open(&self.file_path)?; let metadata = file.metadata()?; // etag and last modified let etag = headers::ETag::from_str(&format!("\"{}\"", self.hash)).unwrap(); let mut last_modified_str = String::new(); - // extract headers - let if_match = headers.typed_get::(); - let if_unmodified_since = headers.typed_get::(); - let if_none_match = headers.typed_get::(); - let if_modified_since = headers.typed_get::(); + if ok { + // handle if-match and if-(un)modified queries + let if_match = headers.typed_get::(); + let if_unmodified_since = headers.typed_get::(); + let if_none_match = headers.typed_get::(); + let if_modified_since = headers.typed_get::(); - // handle if-match and if-(un)modified queries - if let Ok(modified) = metadata.modified() { - if let Some(if_match) = if_match { - if !if_match.precondition_passes(&etag) { - return Err(StaticFileError::PreconditionFailed); + if let Ok(modified) = metadata.modified() { + if let Some(if_match) = if_match { + if !if_match.precondition_passes(&etag) { + return Err(StaticFileError::PreconditionFailed); + } } + + if let Some(if_unmodified_since) = if_unmodified_since { + if !if_unmodified_since.precondition_passes(modified) { + return Err(StaticFileError::PreconditionFailed); + } + } + + if let Some(if_non_match) = if_none_match { + if !if_non_match.precondition_passes(&etag) { + return Ok(StatusCode::NOT_MODIFIED.into()); + } + } else if let Some(if_modified_since) = if_modified_since { + if !if_modified_since.is_modified(modified) { + return Ok(StatusCode::NOT_MODIFIED.into()); + } + } + + last_modified_str = HttpDate::from(modified).to_string(); } - - if let Some(if_unmodified_since) = if_unmodified_since { - if !if_unmodified_since.precondition_passes(modified) { - return Err(StaticFileError::PreconditionFailed); - } - } - - if let Some(if_non_match) = if_none_match { - if !if_non_match.precondition_passes(&etag) { - return Ok(StatusCode::NOT_MODIFIED.into()); - } - } else if let Some(if_modified_since) = if_modified_since { - if !if_modified_since.is_modified(modified) { - return Ok(StatusCode::NOT_MODIFIED.into()); - } - } - - last_modified_str = HttpDate::from(modified).to_string(); } if self @@ -435,71 +439,62 @@ impl GotFile { Ok(response.body(html)) } else { - self.to_static_response(headers, file, metadata.len(), etag, last_modified_str) - } - } + let range = headers.typed_get::().filter(|_| ok); - /// Convert the retrieved file to a static file response - /// - /// Adapted from: - fn to_static_response( - &self, - headers: &HeaderMap, - mut file: File, - size: u64, - etag: headers::ETag, - last_modified_str: String, - ) -> std::result::Result { - let range = headers.typed_get::(); + let size = metadata.len(); + let mut content_length = size; + let mut content_range = None; - let mut content_length = size; - let mut content_range = None; + let body = if let Some((start, end)) = range.and_then(|range| range.iter().next()) { + let start = match start { + Bound::Included(n) => n, + Bound::Excluded(n) => n + 1, + Bound::Unbounded => 0, + }; + let end = match end { + Bound::Included(n) => n + 1, + Bound::Excluded(n) => n, + Bound::Unbounded => size, + }; + if end < start || end > size { + return Err(StaticFileError::RangeNotSatisfiable { size }); + } - let body = if let Some((start, end)) = range.and_then(|range| range.iter().next()) { - let start = match start { - Bound::Included(n) => n, - Bound::Excluded(n) => n + 1, - Bound::Unbounded => 0, + if start != 0 || end != size { + content_range = Some((start..end, size)); + } + + content_length = end - start; + file.seek(SeekFrom::Start(start))?; + Body::from_async_read(tokio::fs::File::from_std(file).take(end - start)) + } else { + Body::from_async_read(tokio::fs::File::from_std(file)) }; - let end = match end { - Bound::Included(n) => n + 1, - Bound::Excluded(n) => n, - Bound::Unbounded => size, - }; - if end < start || end > size { - return Err(StaticFileError::RangeNotSatisfiable { size }); + + let mut response = Response::builder().header(header::CONTENT_LENGTH, content_length); + + if ok { + response = response + .typed_header(etag) + .header(header::ACCEPT_RANGES, "bytes"); + } else { + response = response.status(StatusCode::NOT_FOUND); } - - if start != 0 || end != size { - content_range = Some((start..end, size)); + if !last_modified_str.is_empty() { + response = response.header(header::LAST_MODIFIED, last_modified_str); } - - content_length = end - start; - file.seek(SeekFrom::Start(start))?; - Body::from_async_read(tokio::fs::File::from_std(file).take(end - start)) - } else { - Body::from_async_read(tokio::fs::File::from_std(file)) - }; - - let mut response = Response::builder() - .header(header::ACCEPT_RANGES, "bytes") - .header(header::CONTENT_LENGTH, content_length) - .typed_header(etag); - - if !last_modified_str.is_empty() { - response = response.header(header::LAST_MODIFIED, last_modified_str); + if let Some(encoding) = self.encoding { + response = response.header(header::CONTENT_ENCODING, encoding); + } + if let Some(mime) = &self.mime { + response = response.header(header::CONTENT_TYPE, mime.essence_str()); + } + if let Some((range, size)) = content_range { + response = response + .status(StatusCode::PARTIAL_CONTENT) + .typed_header(headers::ContentRange::bytes(range, size).unwrap()); + } + Ok(response.body(body)) } - if let Some(encoding) = self.encoding { - response = response.header(header::CONTENT_ENCODING, encoding); - } - if let Some(mime) = &self.mime { - response = response.header(header::CONTENT_TYPE, mime.essence_str()); - } - if let Some((range, size)) = content_range { - response = response - .status(StatusCode::PARTIAL_CONTENT) - .typed_header(headers::ContentRange::bytes(range, size).unwrap()); - } - Ok(response.body(body)) } } diff --git a/tests/fixtures/mod.rs b/tests/fixtures/mod.rs index a947410..691b13b 100644 --- a/tests/fixtures/mod.rs +++ b/tests/fixtures/mod.rs @@ -105,6 +105,7 @@ fn insert_websites(db: &Db) { &Version { created_at: datetime!(2023-02-18 16:30 +0), data: v1_data, + ..Default::default() }, ) .unwrap(); @@ -122,6 +123,7 @@ fn insert_websites(db: &Db) { &Version { created_at: datetime!(2023-02-18 16:52 +0), data: v2_data, + ..Default::default() }, ) .unwrap(); @@ -133,6 +135,7 @@ fn insert_websites(db: &Db) { &Version { created_at: datetime!(2023-02-18 16:30 +0), data: BTreeMap::new(), + ..Default::default() }, ) .unwrap(); @@ -143,6 +146,7 @@ fn insert_websites(db: &Db) { &Version { created_at: datetime!(2023-02-20 18:30 +0), data: BTreeMap::new(), + ..Default::default() }, ) .unwrap(); From 8fc3d79abb01fb286b89b51eed488b04273a2b10 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Fri, 3 Mar 2023 19:16:47 +0100 Subject: [PATCH 6/6] feat: add compression for html pages --- Cargo.lock | 2 +- Cargo.toml | 7 +- src/server.rs | 1 - src/storage.rs | 79 ++++++++++++++----- src/util.rs | 53 ++++++------- tests/snapshots/tests__config__default.snap | 6 ++ tests/snapshots/tests__config__sparse.snap | 9 ++- .../tests__database__delete_website.snap | 4 +- tests/snapshots/tests__database__export.snap | 8 +- .../tests__database__get_version.snap | 4 +- ...tests__database__get_website_versions.snap | 6 +- 11 files changed, 119 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73f434e..9156ad3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1224,7 +1224,6 @@ version = "1.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0608069d4999c3c02d49dff261663f2e73a8f7b00b7cd364fb5e93e419dafa1" dependencies = [ - "async-compression", "async-trait", "bytes", "chrono", @@ -1714,6 +1713,7 @@ dependencies = [ name = "talon" version = "0.1.0" dependencies = [ + "async-compression", "brotli", "compressible", "flate2", diff --git a/Cargo.toml b/Cargo.toml index 3091c02..585b375 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ license = "MIT" description = "Static site management system" [dependencies] -poem = {version = "1.3.55", features = ["compression"]} +poem = "1.3.55" poem-openapi = { version = "2.0.26", features = ["time", "swagger-ui"] } tokio = { version = "1.25.0", features = ["rt-multi-thread", "fs"] } sled = "0.34.7" @@ -39,6 +39,11 @@ regex = "1.7.1" log = "0.4.17" httpdate = "1.0.2" tracing-subscriber = "0.3.16" +async-compression = { version = "0.3.15", features = [ + "tokio", + "gzip", + "brotli", +] } [dev-dependencies] rstest = "0.16.0" diff --git a/src/server.rs b/src/server.rs index e0b5096..8af17f9 100644 --- a/src/server.rs +++ b/src/server.rs @@ -84,7 +84,6 @@ impl Talon { .at(&site_domains, page) .at(&self.i.cfg.server.root_domain, page) .with(poem::middleware::Tracing) - .with(poem::middleware::Compression::new()) .data(self.clone()); Server::new(TcpListener::bind(&self.i.cfg.server.address)) diff --git a/src/storage.rs b/src/storage.rs index b539349..3b968a3 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -1,6 +1,5 @@ use std::{ borrow::Cow, - collections::BTreeMap, fs::{self, File}, io::{BufReader, Read, Seek, SeekFrom}, ops::Bound, @@ -34,7 +33,7 @@ pub struct Storage { cfg: Config, } -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum CompressionAlg { #[default] None, @@ -257,35 +256,42 @@ impl Storage { subdir.join(&hash_str) } - /// Get all compressed versions of a file - fn files_compressed( - &self, - hash: &[u8], - is_compressible: bool, - ) -> BTreeMap { + /// Get all available compression algorithms for a stored file + fn file_compressions(&self, hash: &[u8], is_compressible: bool) -> Vec { + let mut res = Vec::new(); let path = self.file_path(hash); - let mut res = BTreeMap::new(); if is_compressible { if self.cfg.compression.gzip_en { let path_gz = path.with_extension("gz"); if path_gz.is_file() { - res.insert(CompressionAlg::Gzip, path_gz); + res.push(CompressionAlg::Gzip); } } if self.cfg.compression.brotli_en { let path_br = path.with_extension("br"); if path_br.is_file() { - res.insert(CompressionAlg::Brotli, path_br); + res.push(CompressionAlg::Brotli); } } } if path.is_file() { - res.insert(CompressionAlg::None, path); + res.push(CompressionAlg::None); } + res } + /// Get the file path of a compressed file + fn file_path_compressed(&self, hash: &[u8], alg: CompressionAlg) -> PathBuf { + let path = self.file_path(hash); + match alg { + CompressionAlg::None => path, + CompressionAlg::Gzip => path.with_extension("gz"), + CompressionAlg::Brotli => path.with_extension("br"), + } + } + /// Check if a file with the given mime type should be compressed /// /// HTML files should not be compressed, since they need to be injected with the @@ -335,19 +341,19 @@ impl Storage { let mime = util::site_path_mime(&new_path); - let files = self.files_compressed( + let algorithms = self.file_compressions( &hash, mime.as_ref() .map(|m| Self::is_compressible(m.essence_str())) .unwrap_or_default(), ); - let file = util::parse_accept_encoding(headers, &files); + let alg = util::parse_accept_encoding(headers, &algorithms); - match file { - Some((compression, file)) => Ok(GotFile { + match alg { + Some(alg) => Ok(GotFile { hash: hash.encode_hex(), - file_path: file.to_owned(), - encoding: compression.encoding(), + file_path: self.file_path_compressed(&hash, alg), + encoding: alg.encoding(), mime, rd_path, }), @@ -427,20 +433,50 @@ impl GotFile { html.insert_str(ctag_pos, to_inject); } - let etag = headers::ETag::from_str(&format!("\"{}\"", self.hash)).unwrap(); + // Compress response if possible + let alg = util::parse_accept_encoding( + headers, + &[ + CompressionAlg::Brotli, + CompressionAlg::Gzip, + CompressionAlg::None, + ], + ) + .unwrap_or_default(); + let body = match alg { + CompressionAlg::None => Body::from(html), + CompressionAlg::Gzip => { + let enc = async_compression::tokio::bufread::GzipEncoder::with_quality( + tokio::io::BufReader::new(Body::from(html).into_async_read()), + async_compression::Level::Precise(6), + ); + Body::from_async_read(enc) + } + CompressionAlg::Brotli => { + let enc = async_compression::tokio::bufread::BrotliEncoder::with_quality( + tokio::io::BufReader::new(Body::from(html).into_async_read()), + async_compression::Level::Precise(7), + ); + Body::from_async_read(enc) + } + }; + // Build response let mut response = Response::builder() .header(header::CONTENT_TYPE, "text/html") .typed_header(etag); + if let Some(encoding) = alg.encoding() { + response = response.header(header::CONTENT_ENCODING, encoding) + } if !last_modified_str.is_empty() { response = response.header(header::LAST_MODIFIED, last_modified_str); } - Ok(response.body(html)) + Ok(response.body(body)) } else { + // Handle range requests let range = headers.typed_get::().filter(|_| ok); - let size = metadata.len(); let mut content_length = size; let mut content_range = None; @@ -471,6 +507,7 @@ impl GotFile { Body::from_async_read(tokio::fs::File::from_std(file)) }; + // Build response let mut response = Response::builder().header(header::CONTENT_LENGTH, content_length); if ok { diff --git a/src/util.rs b/src/util.rs index 0db1fbb..779544e 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, fs::File, path::Path, str::FromStr}; +use std::{fs::File, path::Path, str::FromStr}; use mime_guess::Mime; use poem::http::{header, HeaderMap}; @@ -53,14 +53,14 @@ impl FromStr for ContentCoding { } } -/// Parse Accept-Encoding header and return the compressed file with the preferred algorithm +/// Parse Accept-Encoding header and return the preferred algorithm /// /// Source: -pub fn parse_accept_encoding<'a, T>( +pub fn parse_accept_encoding( headers: &HeaderMap, - files: &'a BTreeMap, -) -> Option<(CompressionAlg, &'a T)> { - if files.is_empty() { + enabled_algorithms: &[CompressionAlg], +) -> Option { + if enabled_algorithms.is_empty() { return None; } @@ -75,23 +75,20 @@ pub fn parse_accept_encoding<'a, T>( None => (v, 1000), }; let coding: ContentCoding = e.parse().ok()?; - let alg_file = match coding { - ContentCoding::Brotli => { - (CompressionAlg::Brotli, files.get(&CompressionAlg::Brotli)?) - } - ContentCoding::Gzip => (CompressionAlg::Gzip, files.get(&CompressionAlg::Gzip)?), - ContentCoding::Star => { - files.iter().max_by_key(|(a, _)| *a).map(|(a, f)| (*a, f))? - } + let alg = match coding { + ContentCoding::Brotli => Some(CompressionAlg::Brotli) + .filter(|_| enabled_algorithms.contains(&CompressionAlg::Brotli)), + ContentCoding::Gzip => Some(CompressionAlg::Gzip) + .filter(|_| enabled_algorithms.contains(&CompressionAlg::Gzip)), + ContentCoding::Star => enabled_algorithms.iter().max().copied(), }; - Some((alg_file, q)) + alg.map(|alg| (alg, q)) }) - .max_by_key(|((a, _), q)| (*q, *a)) - .map(|(x, _)| x) + .max_by_key(|(a, q)| (*q, *a)) + .map(|(a, _)| a) .or_else(|| { - files - .get(&CompressionAlg::None) - .map(|f| (CompressionAlg::None, f)) + Some(CompressionAlg::None) + .filter(|_| enabled_algorithms.contains(&CompressionAlg::None)) }) } @@ -162,13 +159,15 @@ mod tests { let mut headers = HeaderMap::new(); headers.insert(header::ACCEPT_ENCODING, accept.parse().unwrap()); - let mut files = BTreeMap::new(); - files.insert(CompressionAlg::None, 0); - files.insert(CompressionAlg::Gzip, 1); - files.insert(CompressionAlg::Brotli, 2); - - let (compression, file) = parse_accept_encoding(&headers, &files).unwrap(); + let compression = parse_accept_encoding( + &headers, + &[ + CompressionAlg::Gzip, + CompressionAlg::Brotli, + CompressionAlg::None, + ], + ) + .unwrap(); assert_eq!(compression, expect); - assert_eq!(file, files.get(&compression).unwrap()); } } diff --git a/tests/snapshots/tests__config__default.snap b/tests/snapshots/tests__config__default.snap index ddadd88..7de5d4b 100644 --- a/tests/snapshots/tests__config__default.snap +++ b/tests/snapshots/tests__config__default.snap @@ -18,15 +18,21 @@ ConfigInner( keys: { "04e99561e3824f387a217d141d2a3b46375de6864afbedf9c9a2cc102bc946a4": KeyCfg( domains: "/^talon-\\d+/", + upload: false, + modify: false, ), "21bdac19ffd22870d561b1d55b35eddd9029497107edb7b926aa3e7856bb409b": KeyCfg( domains: [ "spotify-gender-ex", "rustypipe", ], + upload: false, + modify: false, ), "c32ff286c8ac1c3102625badf38ffd251ae0c4a56079d8ba490f320af63f1f47": KeyCfg( domains: "*", + upload: false, + modify: false, ), }, ) diff --git a/tests/snapshots/tests__config__sparse.snap b/tests/snapshots/tests__config__sparse.snap index a68b605..a31db55 100644 --- a/tests/snapshots/tests__config__sparse.snap +++ b/tests/snapshots/tests__config__sparse.snap @@ -16,15 +16,22 @@ ConfigInner( brotli_level: 7, ), keys: { - "04e99561e3824f387a217d141d2a3b46375de6864afbedf9c9a2cc102bc946a4": KeyCfg(), + "04e99561e3824f387a217d141d2a3b46375de6864afbedf9c9a2cc102bc946a4": KeyCfg( + upload: false, + modify: false, + ), "21bdac19ffd22870d561b1d55b35eddd9029497107edb7b926aa3e7856bb409b": KeyCfg( domains: [ "spotify-gender-ex", "rustypipe", ], + upload: false, + modify: false, ), "c32ff286c8ac1c3102625badf38ffd251ae0c4a56079d8ba490f320af63f1f47": KeyCfg( domains: "*", + upload: false, + modify: false, ), }, ) diff --git a/tests/snapshots/tests__database__delete_website.snap b/tests/snapshots/tests__database__delete_website.snap index c29d78f..f7de527 100644 --- a/tests/snapshots/tests__database__delete_website.snap +++ b/tests/snapshots/tests__database__delete_website.snap @@ -4,8 +4,8 @@ expression: data --- {"type":"website","key":"rustypipe","value":{"name":"RustyPipe","created_at":[2023,51,18,30,0,0,0,0,0],"latest_version":4,"color":7943647,"visibility":"featured","source_url":"https://code.thetadev.de/ThetaDev/rustypipe","source_icon":"gitea"}} {"type":"website","key":"spotify-gender-ex","value":{"name":"Spotify-Gender-Ex","created_at":[2023,49,16,30,0,0,0,0,0],"latest_version":3,"color":1947988,"visibility":"featured","source_url":"https://github.com/Theta-Dev/Spotify-Gender-Ex","source_icon":"github"}} -{"type":"version","key":"rustypipe:4","value":{"created_at":[2023,51,18,30,0,0,0,0,0],"data":{}}} -{"type":"version","key":"spotify-gender-ex:3","value":{"created_at":[2023,49,16,30,0,0,0,0,0],"data":{}}} +{"type":"version","key":"rustypipe:4","value":{"created_at":[2023,51,18,30,0,0,0,0,0],"data":{},"fallback":null,"spa":false}} +{"type":"version","key":"spotify-gender-ex:3","value":{"created_at":[2023,49,16,30,0,0,0,0,0],"data":{},"fallback":null,"spa":false}} {"type":"file","key":"3:gex_style.css","value":"fc825b409a49724af8f5b3c4ad15e175e68095ea746237a7b46152d3f383f541"} {"type":"file","key":"3:index.html","value":"6c5d37546616519e8973be51515b8a90898b4675f7b6d01f2d891edb686408a2"} {"type":"file","key":"4:index.html","value":"94a67cf13d752a9c1875ad999eb2be5a1b0f9746c66bca2631820b8186028811"} diff --git a/tests/snapshots/tests__database__export.snap b/tests/snapshots/tests__database__export.snap index 469053a..d78a80b 100644 --- a/tests/snapshots/tests__database__export.snap +++ b/tests/snapshots/tests__database__export.snap @@ -5,10 +5,10 @@ expression: data {"type":"website","key":"-","value":{"name":"ThetaDev","created_at":[2023,49,16,30,0,0,0,0,0],"latest_version":2,"color":2068974,"visibility":"featured","source_url":null,"source_icon":null}} {"type":"website","key":"rustypipe","value":{"name":"RustyPipe","created_at":[2023,51,18,30,0,0,0,0,0],"latest_version":4,"color":7943647,"visibility":"featured","source_url":"https://code.thetadev.de/ThetaDev/rustypipe","source_icon":"gitea"}} {"type":"website","key":"spotify-gender-ex","value":{"name":"Spotify-Gender-Ex","created_at":[2023,49,16,30,0,0,0,0,0],"latest_version":3,"color":1947988,"visibility":"featured","source_url":"https://github.com/Theta-Dev/Spotify-Gender-Ex","source_icon":"github"}} -{"type":"version","key":"-:1","value":{"created_at":[2023,49,16,30,0,0,0,0,0],"data":{"Deployed by":"https://github.com/Theta-Dev/Talon/actions/runs/1352014628","Version":"v0.1.0"}}} -{"type":"version","key":"-:2","value":{"created_at":[2023,49,16,52,0,0,0,0,0],"data":{"Deployed by":"https://github.com/Theta-Dev/Talon/actions/runs/1354755231","Version":"v0.1.1"}}} -{"type":"version","key":"rustypipe:4","value":{"created_at":[2023,51,18,30,0,0,0,0,0],"data":{}}} -{"type":"version","key":"spotify-gender-ex:3","value":{"created_at":[2023,49,16,30,0,0,0,0,0],"data":{}}} +{"type":"version","key":"-:1","value":{"created_at":[2023,49,16,30,0,0,0,0,0],"data":{"Deployed by":"https://github.com/Theta-Dev/Talon/actions/runs/1352014628","Version":"v0.1.0"},"fallback":null,"spa":false}} +{"type":"version","key":"-:2","value":{"created_at":[2023,49,16,52,0,0,0,0,0],"data":{"Deployed by":"https://github.com/Theta-Dev/Talon/actions/runs/1354755231","Version":"v0.1.1"},"fallback":null,"spa":false}} +{"type":"version","key":"rustypipe:4","value":{"created_at":[2023,51,18,30,0,0,0,0,0],"data":{},"fallback":null,"spa":false}} +{"type":"version","key":"spotify-gender-ex:3","value":{"created_at":[2023,49,16,30,0,0,0,0,0],"data":{},"fallback":null,"spa":false}} {"type":"file","key":"1:index.html","value":"3b5f6bad5376897435def176d0fe77e5b9b4f0deafc7491fc27262650744ad68"} {"type":"file","key":"1:style.css","value":"356f131c825fbf604797c7e9c85352549d81db8af91fee834016d075110af026"} {"type":"file","key":"2:assets/image.jpg","value":"901d291a47a8a9b55c06f84e5e5f82fd2dcee65cac1406d6e878b805d45c1e93"} diff --git a/tests/snapshots/tests__database__get_version.snap b/tests/snapshots/tests__database__get_version.snap index 9386db0..b4998aa 100644 --- a/tests/snapshots/tests__database__get_version.snap +++ b/tests/snapshots/tests__database__get_version.snap @@ -1,5 +1,5 @@ --- -source: src/db/mod.rs +source: tests/tests.rs expression: version --- Version( @@ -8,4 +8,6 @@ Version( "Deployed by": "https://github.com/Theta-Dev/Talon/actions/runs/1352014628", "Version": "v0.1.0", }, + fallback: None, + spa: false, ) diff --git a/tests/snapshots/tests__database__get_website_versions.snap b/tests/snapshots/tests__database__get_website_versions.snap index 1d32855..508b1c6 100644 --- a/tests/snapshots/tests__database__get_website_versions.snap +++ b/tests/snapshots/tests__database__get_website_versions.snap @@ -1,5 +1,5 @@ --- -source: src/db/mod.rs +source: tests/tests.rs expression: versions --- [ @@ -9,6 +9,8 @@ expression: versions "Deployed by": "https://github.com/Theta-Dev/Talon/actions/runs/1352014628", "Version": "v0.1.0", }, + fallback: None, + spa: false, )), (2, Version( created_at: (2023, 49, 16, 52, 0, 0, 0, 0, 0), @@ -16,5 +18,7 @@ expression: versions "Deployed by": "https://github.com/Theta-Dev/Talon/actions/runs/1354755231", "Version": "v0.1.1", }, + fallback: None, + spa: false, )), ]