feat: add compression for html pages

Cargo.lock

@@ -76,6 +76,20 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "async-compression"
+version = "0.3.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a"
+dependencies = [
+ "brotli",
+ "flate2",
+ "futures-core",
+ "memchr",
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "async-trait"
 version = "0.1.64"
@@ -462,19 +476,6 @@ dependencies = [
  "cfg-if",
 ]
 
-[[package]]
-name = "env_logger"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0"
-dependencies = [
- "humantime",
- "is-terminal",
- "log",
- "regex",
- "termcolor",
-]
-
 [[package]]
 name = "errno"
 version = "0.2.8"
@@ -746,12 +747,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "hermit-abi"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
-
 [[package]]
 name = "hex"
 version = "0.4.3"
@@ -819,12 +814,6 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
 
-[[package]]
-name = "humantime"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
-
 [[package]]
 name = "hyper"
 version = "0.14.24"
@@ -932,18 +921,6 @@ dependencies = [
  "windows-sys 0.45.0",
 ]
 
-[[package]]
-name = "is-terminal"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21b6b32576413a8e69b90e952e4a026476040d81017b80445deda5f2d3921857"
-dependencies = [
- "hermit-abi 0.3.1",
- "io-lifetimes",
- "rustix",
- "windows-sys 0.45.0",
-]
-
 [[package]]
 name = "itoa"
 version = "1.0.5"
@@ -1100,6 +1077,16 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "nu-ansi-term"
+version = "0.46.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
+dependencies = [
+ "overload",
+ "winapi",
+]
+
 [[package]]
 name = "num-integer"
 version = "0.1.45"
@@ -1125,7 +1112,7 @@ version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
 dependencies = [
- "hermit-abi 0.2.6",
+ "hermit-abi",
  "libc",
 ]
 
@@ -1141,6 +1128,12 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
+[[package]]
+name = "overload"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
+
 [[package]]
 name = "parking_lot"
 version = "0.11.2"
@@ -1631,6 +1624,15 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "sharded-slab"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31"
+dependencies = [
+ "lazy_static",
+]
+
 [[package]]
 name = "similar"
 version = "2.2.1"
@@ -1711,9 +1713,9 @@ dependencies = [
 name = "talon"
 version = "0.1.0"
 dependencies = [
+ "async-compression",
  "brotli",
  "compressible",
- "env_logger",
  "flate2",
  "hex",
  "hex-literal",
@@ -1738,6 +1740,7 @@ dependencies = [
  "time",
  "tokio",
  "toml",
+ "tracing-subscriber",
  "zip",
 ]
 
@@ -1806,6 +1809,16 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "thread_local"
+version = "1.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+]
+
 [[package]]
 name = "time"
 version = "0.3.20"
@@ -1974,6 +1987,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a"
 dependencies = [
  "once_cell",
+ "valuable",
+]
+
+[[package]]
+name = "tracing-log"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922"
+dependencies = [
+ "lazy_static",
+ "log",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-subscriber"
+version = "0.3.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70"
+dependencies = [
+ "nu-ansi-term",
+ "sharded-slab",
+ "smallvec",
+ "thread_local",
+ "tracing-core",
+ "tracing-log",
 ]
 
 [[package]]
@@ -2034,6 +2073,12 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bc7ed8ba44ca06be78ea1ad2c3682a43349126c8818054231ee6f4748012aed2"
 
+[[package]]
+name = "valuable"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
+
 [[package]]
 name = "version_check"
 version = "0.9.4"

Cargo.toml

@@ -38,7 +38,12 @@ compressible = "0.2.0"
 regex = "1.7.1"
 log = "0.4.17"
 httpdate = "1.0.2"
-env_logger = "0.10.0"
+tracing-subscriber = "0.3.16"
+async-compression = { version = "0.3.15", features = [
+    "tokio",
+    "gzip",
+    "brotli",
+] }
 
 [dev-dependencies]
 rstest = "0.16.0"

src/api.rs

@@ -1,4 +1,4 @@
-use std::io::Cursor;
+use std::{collections::BTreeMap, io::Cursor};
 
 use poem::{
     error::{Error, ResponseError},
@@ -8,12 +8,18 @@ use poem::{
 };
 use poem_openapi::{
     auth::ApiKey,
-    param::Path,
+    param::{Path, Query},
     payload::{Binary, Json},
     OpenApi, SecurityScheme,
 };
 
-use crate::{config::KeyCfg, db, model::*, oai::DynParams, util, Talon};
+use crate::{
+    config::{Access, KeyCfg},
+    db,
+    model::*,
+    oai::DynParams,
+    util, Talon,
+};
 
 pub struct TalonApi;
 
@@ -32,10 +38,10 @@ async fn api_key_checker(req: &Request, api_key: ApiKey) -> Option<KeyCfg> {
 }
 
 impl ApiKeyAuthorization {
-    fn check_subdomain(&self, subdomain: &str) -> Result<()> {
+    fn check_subdomain(&self, subdomain: &str, access: Access) -> Result<()> {
         if subdomain.is_empty() {
             Err(ApiError::InvalidSubdomain.into())
-        } else if !self.0.domains.matches_domain(subdomain) {
+        } else if !self.0.domains.matches_domain(subdomain) || !self.0.allows(access) {
             Err(ApiError::NoAccess.into())
         } else {
             Ok(())
@@ -49,18 +55,21 @@ enum ApiError {
     InvalidSubdomain,
     #[error("you do not have access to this subdomain")]
     NoAccess,
+    #[error("invalid fallback: {0}")]
+    InvalidFallback(String),
 }
 
 impl ResponseError for ApiError {
     fn status(&self) -> StatusCode {
         match self {
-            ApiError::InvalidSubdomain => StatusCode::BAD_REQUEST,
+            ApiError::InvalidSubdomain | ApiError::InvalidFallback(_) => StatusCode::BAD_REQUEST,
             ApiError::NoAccess => StatusCode::UNAUTHORIZED,
         }
     }
 }
 
 #[OpenApi]
+#[allow(clippy::too_many_arguments)]
 impl TalonApi {
     /// Get a website
     #[oai(path = "/website/:subdomain", method = "get")]
@@ -85,7 +94,7 @@ impl TalonApi {
         subdomain: Path<String>,
         website: Json<WebsiteNew>,
     ) -> Result<()> {
-        auth.check_subdomain(&subdomain)?;
+        auth.check_subdomain(&subdomain, Access::Modify)?;
         if subdomain.as_str() == talon.cfg.server.internal_subdomain
             || !util::validate_subdomain(&subdomain)
         {
@@ -105,7 +114,7 @@ impl TalonApi {
         subdomain: Path<String>,
         website: Json<WebsiteUpdate>,
     ) -> Result<()> {
-        auth.check_subdomain(&subdomain)?;
+        auth.check_subdomain(&subdomain, Access::Modify)?;
 
         talon.db.update_website(&subdomain, website.0.into())?;
         Ok(())
@@ -119,19 +128,66 @@ impl TalonApi {
         talon: Data<&Talon>,
         subdomain: Path<String>,
     ) -> Result<()> {
-        auth.check_subdomain(&subdomain)?;
+        auth.check_subdomain(&subdomain, Access::Modify)?;
 
         talon.db.delete_website(&subdomain, true)?;
         Ok(())
     }
 
-    /// Get all websites
+    /// Get all publicly listed websites
+    ///
+    /// Returns all publicly listed websites (visibility != `hidden`)
     #[oai(path = "/websites", method = "get")]
-    async fn websites_get(&self, talon: Data<&Talon>) -> Result<Json<Vec<Website>>> {
+    async fn websites_get(
+        &self,
+        talon: Data<&Talon>,
+        /// Mimimum visibility of the websites
+        #[oai(default)]
+        visibility: Query<Visibility>,
+    ) -> Result<Json<Vec<Website>>> {
         talon
             .db
             .get_websites()
             .map(|r| r.map(Website::from))
+            .filter(|ws| match ws {
+                Ok(ws) => ws.visibility != Visibility::Hidden && ws.visibility <= visibility.0,
+                Err(_) => true,
+            })
+            .collect::<Result<Vec<_>, _>>()
+            .map(Json)
+            .map_err(Error::from)
+    }
+
+    /// Get all websites
+    ///
+    /// Returns all websites from Talon's database (including hidden ones, if the current user
+    /// has access to them). This endpoint requires authentication (use the `/websites` endpoint
+    /// for unauthenticated users).
+    #[oai(path = "/websitesAll", method = "get")]
+    async fn websites_get_all(
+        &self,
+        auth: ApiKeyAuthorization,
+        talon: Data<&Talon>,
+        /// Mimimum visibility of the websites
+        #[oai(default)]
+        visibility: Query<Visibility>,
+    ) -> Result<Json<Vec<Website>>> {
+        talon
+            .db
+            .get_websites()
+            .map(|r| r.map(Website::from))
+            .filter(|ws| match ws {
+                Ok(ws) => {
+                    if ws.visibility == Visibility::Hidden
+                        && auth.check_subdomain(&ws.subdomain, Access::Read).is_err()
+                    {
+                        false
+                    } else {
+                        ws.visibility <= visibility.0
+                    }
+                }
+                Err(_) => true,
+            })
             .collect::<Result<Vec<_>, _>>()
             .map(Json)
             .map_err(Error::from)
@@ -178,12 +234,52 @@ impl TalonApi {
         subdomain: Path<String>,
         id: Path<u32>,
     ) -> Result<()> {
-        auth.check_subdomain(&subdomain)?;
+        auth.check_subdomain(&subdomain, Access::Modify)?;
 
         talon.db.delete_version(&subdomain, *id, true)?;
         Ok(())
     }
 
+    fn insert_version(
+        talon: &Talon,
+        subdomain: &str,
+        id: u32,
+        fallback: Option<String>,
+        spa: bool,
+        mut version_data: BTreeMap<String, String>,
+    ) -> Result<()> {
+        version_data.remove("fallback");
+        version_data.remove("spa");
+
+        // Validata fallback path
+        if let Some(fallback) = &fallback {
+            if let Err(e) = talon.storage.get_file(id, fallback, &Default::default()) {
+                // Remove the uploaded files of the bad version
+                let _ = talon.db.delete_version(subdomain, id, false);
+                return Err(ApiError::InvalidFallback(e.to_string()).into());
+            }
+        }
+
+        talon.db.insert_version(
+            subdomain,
+            id,
+            &db::model::Version {
+                data: version_data,
+                fallback,
+                spa,
+                ..Default::default()
+            },
+        )?;
+        talon.db.update_website(
+            subdomain,
+            db::model::WebsiteUpdate {
+                latest_version: Some(Some(id)),
+                ..Default::default()
+            },
+        )?;
+        Ok(())
+    }
+
     /// Upload a new version (.zip archive)
     #[oai(path = "/website/:subdomain/uploadZip", method = "post")]
     async fn version_upload_zip(
@@ -191,6 +287,13 @@ impl TalonApi {
         auth: ApiKeyAuthorization,
         talon: Data<&Talon>,
         subdomain: Path<String>,
+        /// Fallback page
+        ///
+        /// The fallback page gets returned when the requested page does not exist
+        fallback: Query<Option<String>>,
+        /// SPA mode (return fallback page with OK status)
+        #[oai(default)]
+        spa: Query<bool>,
         /// Associated version data
         ///
         /// This is an arbitrary string map that can hold build information and other stuff
@@ -199,28 +302,12 @@ impl TalonApi {
         /// zip archive with the website files
         data: Binary<Vec<u8>>,
     ) -> Result<()> {
-        auth.check_subdomain(&subdomain)?;
+        auth.check_subdomain(&subdomain, Access::Upload)?;
         let vid = talon.db.new_version_id()?;
         talon
             .storage
             .insert_zip_archive(Cursor::new(data.as_slice()), vid)?;
-
+        Self::insert_version(&talon, &subdomain, vid, fallback.0, spa.0, version_data.0)
-        talon.db.insert_version(
-            &subdomain,
-            vid,
-            &db::model::Version {
-                data: version_data.0,
-                ..Default::default()
-            },
-        )?;
-        talon.db.update_website(
-            &subdomain,
-            db::model::WebsiteUpdate {
-                latest_version: Some(Some(vid)),
-                ..Default::default()
-            },
-        )?;
-        Ok(())
     }
 
     /// Upload a new version (.tar.gz archive)
@@ -230,6 +317,13 @@ impl TalonApi {
         auth: ApiKeyAuthorization,
         talon: Data<&Talon>,
         subdomain: Path<String>,
+        /// Fallback page
+        ///
+        /// The fallback page gets returned when the requested page does not exist
+        fallback: Query<Option<String>>,
+        /// SPA mode (return fallback page with OK status)
+        #[oai(default)]
+        spa: Query<bool>,
         /// Associated version data
         ///
         /// This is an arbitrary string map that can hold build information and other stuff
@@ -238,25 +332,9 @@ impl TalonApi {
         /// tar.gz archive with the website files
         data: Binary<Vec<u8>>,
     ) -> Result<()> {
-        auth.check_subdomain(&subdomain)?;
+        auth.check_subdomain(&subdomain, Access::Upload)?;
         let vid = talon.db.new_version_id()?;
         talon.storage.insert_tgz_archive(data.as_slice(), vid)?;
-
+        Self::insert_version(&talon, &subdomain, vid, fallback.0, spa.0, version_data.0)
-        talon.db.insert_version(
-            &subdomain,
-            vid,
-            &db::model::Version {
-                data: version_data.0,
-                ..Default::default()
-            },
-        )?;
-        talon.db.update_website(
-            &subdomain,
-            db::model::WebsiteUpdate {
-                latest_version: Some(Some(vid)),
-                ..Default::default()
-            },
-        )?;
-        Ok(())
     }
 }

src/config.rs

@@ -130,6 +130,8 @@ impl CompressionCfg {
 pub struct KeyCfg {
     #[serde(skip_serializing_if = "Domains::is_none")]
     pub domains: Domains,
+    pub upload: bool,
+    pub modify: bool,
 }
 
 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
@@ -141,6 +143,16 @@ pub enum Domains {
     Multiple(Vec<String>),
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Access {
+    /// Dont modify anything
+    Read,
+    /// Update a new website version
+    Upload,
+    /// Create, update or delete websites
+    Modify,
+}
+
 impl Domains {
     fn is_none(&self) -> bool {
         matches!(self, Domains::None)
@@ -175,6 +187,16 @@ impl Domains {
     }
 }
 
+impl KeyCfg {
+    pub fn allows(&self, access: Access) -> bool {
+        match access {
+            Access::Read => true,
+            Access::Upload => self.upload,
+            Access::Modify => self.modify,
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/db/model.rs

@@ -66,7 +66,14 @@ pub struct Version {
     ///
     /// This is an arbitrary string map that can hold build information and other stuff
     /// and will be displayed in the site info dialog.
+    #[serde(default)]
     pub data: BTreeMap<String, String>,
+    /// Path of the fallback page which is returned if the requested path was not found
+    #[serde(default)]
+    pub fallback: Option<String>,
+    /// SPA mode (return the fallback page with OK sta)
+    #[serde(default)]
+    pub spa: bool,
 }
 
 impl Default for Version {
@@ -74,6 +81,8 @@ impl Default for Version {
         Self {
             created_at: OffsetDateTime::now_utc(),
             data: Default::default(),
+            fallback: Default::default(),
+            spa: Default::default(),
         }
     }
 }

src/main.rs

@@ -2,6 +2,11 @@ use talon::{Result, Talon};
 
 #[tokio::main]
 async fn main() -> Result<()> {
+    if std::env::var_os("RUST_LOG").is_none() {
+        std::env::set_var("RUST_LOG", "info");
+    }
+    tracing_subscriber::fmt::init();
+
     let talon = Talon::new("tmp")?;
     talon.launch().await
 }

src/model.rs

@@ -75,7 +75,10 @@ pub struct Version {
     pub data: BTreeMap<String, String>,
 }
 
-#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Enum, Serialize, Deserialize)]
+#[derive(
+    Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Enum, Serialize, Deserialize,
+)]
+#[oai(rename_all = "snake_case")]
 #[serde(rename_all = "snake_case")]
 pub enum Visibility {
     Featured,
@@ -85,6 +88,7 @@ pub enum Visibility {
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Enum, Serialize, Deserialize)]
+#[oai(rename_all = "snake_case")]
 #[serde(rename_all = "snake_case")]
 pub enum SourceIcon {
     Link,

src/oai.rs

@@ -15,11 +15,11 @@ impl<'a> ApiExtractor<'a> for DynParams {
     const TYPE: ApiExtractorType = ApiExtractorType::Parameter;
     const PARAM_IS_REQUIRED: bool = false;
 
-    type ParamType = BTreeMap<String, String>;
+    type ParamType = Self;
-    type ParamRawType = Self::ParamType;
+    type ParamRawType = BTreeMap<String, String>;
 
     fn register(registry: &mut Registry) {
-        Self::ParamType::register(registry);
+        Self::ParamRawType::register(registry);
     }
 
     fn param_in() -> Option<MetaParamIn> {
@@ -27,7 +27,7 @@ impl<'a> ApiExtractor<'a> for DynParams {
     }
 
     fn param_schema_ref() -> Option<MetaSchemaRef> {
-        Some(Self::ParamType::schema_ref())
+        Some(Self::ParamRawType::schema_ref())
     }
 
     fn param_raw_type(&self) -> Option<&Self::ParamRawType> {

src/page.rs

@@ -6,7 +6,7 @@ use poem::{
     IntoResponse, Request, Response, Result,
 };
 
-use crate::Talon;
+use crate::{storage::StorageError, Talon};
 
 #[derive(thiserror::Error, Debug)]
 pub enum PageError {
@@ -23,7 +23,7 @@ impl ResponseError for PageError {
 }
 
 #[handler]
-pub fn page(request: &Request, talon: Data<&Talon>) -> Result<Response> {
+pub async fn page(request: &Request, talon: Data<&Talon>) -> Result<Response> {
     let host = request
         .header(header::HOST)
         .ok_or(PageError::InvalidSubdomain)?;
@@ -35,13 +35,32 @@ pub fn page(request: &Request, talon: Data<&Talon>) -> Result<Response> {
     };
 
     let ws = talon.db.get_website(subdomain)?;
-    let version = ws.latest_version.ok_or(PageError::NoVersion)?;
+    let vid = ws.latest_version.ok_or(PageError::NoVersion)?;
-    let file = talon
+    let (file, ok) =
+        match talon
             .storage
-        .get_file(version, request.original_uri().path(), request.headers())?;
+            .get_file(vid, request.original_uri().path(), request.headers())
+        {
+            Ok(file) => (file, true),
+            Err(StorageError::NotFound(f)) => {
+                let version = talon.db.get_version(subdomain, vid)?;
+                if let Some(fallback) = &version.fallback {
+                    (
+                        talon.storage.get_file(vid, fallback, request.headers())?,
+                        version.spa,
+                    )
+                } else {
+                    return Err(StorageError::NotFound(f).into());
+                }
+            }
+            Err(e) => return Err(e.into()),
+        };
 
     Ok(match file.rd_path {
         Some(rd_path) => Redirect::moved_permanent(rd_path).into_response(),
-        None => file.to_response(request.headers())?.into_response(),
+        None => file
+            .to_response(request.headers(), ok)
+            .await?
+            .into_response(),
     })
 }

src/server.rs

@@ -70,7 +70,8 @@ impl Talon {
             .at(
                 "/api/spec",
                 poem::endpoint::make_sync(move |_| spec.clone()),
-            );
+            )
+            .with(poem::middleware::Cors::new());
 
         let internal_domain = format!(
             "{}.{}",
@@ -82,6 +83,7 @@ impl Talon {
             .at(&internal_domain, route_internal)
             .at(&site_domains, page)
             .at(&self.i.cfg.server.root_domain, page)
+            .with(poem::middleware::Tracing)
             .data(self.clone());
 
         Server::new(TcpListener::bind(&self.i.cfg.server.address))

src/storage.rs

@@ -1,7 +1,6 @@
 use std::{
     borrow::Cow,
-    collections::BTreeMap,
+    fs::{self, File},
-    fs,
     io::{BufReader, Read, Seek, SeekFrom},
     ops::Bound,
     path::{Path, PathBuf},
@@ -34,7 +33,7 @@ pub struct Storage {
     cfg: Config,
 }
 
-#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum CompressionAlg {
     #[default]
     None,
@@ -124,12 +123,13 @@ impl Storage {
         let hash = util::hash_file(file_path)?;
         let stored_file = self.file_path_mkdir(&hash)?;
 
+        if !stored_file.is_file() {
             fs::copy(file_path, &stored_file)?;
 
             if self.cfg.compression.enabled()
                 && mime_guess::from_path(file_path)
                     .first()
-                .map(|t| compressible::is_compressible(t.essence_str()))
+                    .map(|t| Self::is_compressible(t.essence_str()))
                     .unwrap_or_default()
             {
                 if self.cfg.compression.gzip_en {
@@ -152,6 +152,7 @@ impl Storage {
                     std::io::copy(&mut input, &mut encoder)?;
                 }
             }
+        }
 
         self.db.insert_file(version, site_path, &hash)?;
 
@@ -236,6 +237,8 @@ impl Storage {
         self.insert_dir(import_path, version)
     }
 
+    /// Get the path of a file with the given hash while creating the subdirectory
+    /// if necessary
     fn file_path_mkdir(&self, hash: &[u8]) -> Result<PathBuf> {
         let hash_str = hash.encode_hex::<String>();
 
@@ -246,34 +249,57 @@ impl Storage {
         Ok(subdir.join(&hash_str))
     }
 
+    /// Get the path of a file with the given hash
     fn file_path(&self, hash: &[u8]) -> PathBuf {
         let hash_str = hash.encode_hex::<String>();
         let subdir = self.path.join(&hash_str[..2]);
         subdir.join(&hash_str)
     }
 
-    fn files_compressed(&self, hash: &[u8]) -> BTreeMap<CompressionAlg, PathBuf> {
+    /// Get all available compression algorithms for a stored file
+    fn file_compressions(&self, hash: &[u8], is_compressible: bool) -> Vec<CompressionAlg> {
+        let mut res = Vec::new();
         let path = self.file_path(hash);
-        let mut res = BTreeMap::new();
 
+        if is_compressible {
             if self.cfg.compression.gzip_en {
                 let path_gz = path.with_extension("gz");
                 if path_gz.is_file() {
-                res.insert(CompressionAlg::Gzip, path_gz);
+                    res.push(CompressionAlg::Gzip);
                 }
             }
             if self.cfg.compression.brotli_en {
                 let path_br = path.with_extension("br");
                 if path_br.is_file() {
-                res.insert(CompressionAlg::Brotli, path_br);
+                    res.push(CompressionAlg::Brotli);
+                }
             }
         }
         if path.is_file() {
-            res.insert(CompressionAlg::None, path);
+            res.push(CompressionAlg::None);
         }
+
         res
     }
 
+    /// Get the file path of a compressed file
+    fn file_path_compressed(&self, hash: &[u8], alg: CompressionAlg) -> PathBuf {
+        let path = self.file_path(hash);
+        match alg {
+            CompressionAlg::None => path,
+            CompressionAlg::Gzip => path.with_extension("gz"),
+            CompressionAlg::Brotli => path.with_extension("br"),
+        }
+    }
+
+    /// Check if a file with the given mime type should be compressed
+    ///
+    /// HTML files should not be compressed, since they need to be injected with the
+    /// UI code
+    fn is_compressible(mime: &str) -> bool {
+        mime != "text/html" && compressible::is_compressible(mime)
+    }
+
     /// Get a file using the raw site path and the website version
     ///
     /// HTTP headers are used to determine if the compressed version of a file should be returned.
@@ -315,14 +341,19 @@ impl Storage {
 
         let mime = util::site_path_mime(&new_path);
 
-        let files = self.files_compressed(&hash);
+        let algorithms = self.file_compressions(
-        let file = util::parse_accept_encoding(headers, &files);
+            &hash,
+            mime.as_ref()
+                .map(|m| Self::is_compressible(m.essence_str()))
+                .unwrap_or_default(),
+        );
+        let alg = util::parse_accept_encoding(headers, &algorithms);
 
-        match file {
+        match alg {
-            Some((compression, file)) => Ok(GotFile {
+            Some(alg) => Ok(GotFile {
                 hash: hash.encode_hex(),
-                file_path: file.to_owned(),
+                file_path: self.file_path_compressed(&hash, alg),
-                encoding: compression.encoding(),
+                encoding: alg.encoding(),
                 mime,
                 rd_path,
             }),
@@ -338,30 +369,26 @@ impl GotFile {
     /// Convert the retrieved file to a HTTP response
     ///
     /// Adapted from: <https://github.com/poem-web/poem/blob/049215cf02c5d4b1ab76f290b4708f3142d6d61b/poem/src/web/static_file.rs#L175>
-    pub fn to_response(
+    pub async fn to_response(
         self,
         headers: &HeaderMap,
+        ok: bool,
     ) -> std::result::Result<Response, StaticFileError> {
-        let path = self.file_path;
+        let mut file = File::open(&self.file_path)?;
-        let mut file = std::fs::File::open(path)?;
         let metadata = file.metadata()?;
 
-        // content length
-        let mut content_length = metadata.len();
-
         // etag and last modified
+        let etag = headers::ETag::from_str(&format!("\"{}\"", self.hash)).unwrap();
         let mut last_modified_str = String::new();
 
-        // extract headers
+        if ok {
+            // handle if-match and if-(un)modified queries
             let if_match = headers.typed_get::<headers::IfMatch>();
             let if_unmodified_since = headers.typed_get::<headers::IfUnmodifiedSince>();
             let if_none_match = headers.typed_get::<headers::IfNoneMatch>();
             let if_modified_since = headers.typed_get::<headers::IfModifiedSince>();
-        let range = headers.typed_get::<headers::Range>();
 
             if let Ok(modified) = metadata.modified() {
-            let etag = headers::ETag::from_str(&format!("\"{}\"", self.hash)).unwrap();
-
                 if let Some(if_match) = if_match {
                     if !if_match.precondition_passes(&etag) {
                         return Err(StaticFileError::PreconditionFailed);
@@ -386,7 +413,72 @@ impl GotFile {
 
                 last_modified_str = HttpDate::from(modified).to_string();
             }
+        }
 
+        if self
+            .mime
+            .as_ref()
+            .map(|m| m.essence_str() == "text/html")
+            .unwrap_or_default()
+        {
+            // Inject UI code into HTML
+            let to_inject = "<!-- Hello World -->\n";
+
+            let mut html = String::with_capacity(metadata.len() as usize);
+            tokio::fs::File::from_std(file)
+                .read_to_string(&mut html)
+                .await?;
+
+            if let Some(ctag_pos) = html.rfind("</html>") {
+                html.insert_str(ctag_pos, to_inject);
+            }
+
+            // Compress response if possible
+            let alg = util::parse_accept_encoding(
+                headers,
+                &[
+                    CompressionAlg::Brotli,
+                    CompressionAlg::Gzip,
+                    CompressionAlg::None,
+                ],
+            )
+            .unwrap_or_default();
+            let body = match alg {
+                CompressionAlg::None => Body::from(html),
+                CompressionAlg::Gzip => {
+                    let enc = async_compression::tokio::bufread::GzipEncoder::with_quality(
+                        tokio::io::BufReader::new(Body::from(html).into_async_read()),
+                        async_compression::Level::Precise(6),
+                    );
+                    Body::from_async_read(enc)
+                }
+                CompressionAlg::Brotli => {
+                    let enc = async_compression::tokio::bufread::BrotliEncoder::with_quality(
+                        tokio::io::BufReader::new(Body::from(html).into_async_read()),
+                        async_compression::Level::Precise(7),
+                    );
+                    Body::from_async_read(enc)
+                }
+            };
+
+            // Build response
+            let mut response = Response::builder()
+                .header(header::CONTENT_TYPE, "text/html")
+                .typed_header(etag);
+
+            if let Some(encoding) = alg.encoding() {
+                response = response.header(header::CONTENT_ENCODING, encoding)
+            }
+            if !last_modified_str.is_empty() {
+                response = response.header(header::LAST_MODIFIED, last_modified_str);
+            }
+
+            Ok(response.body(body))
+        } else {
+            // Handle range requests
+            let range = headers.typed_get::<headers::Range>().filter(|_| ok);
+            let size = metadata.len();
+            let mut content_length = size;
             let mut content_range = None;
 
             let body = if let Some((start, end)) = range.and_then(|range| range.iter().next()) {
@@ -398,16 +490,14 @@ impl GotFile {
                 let end = match end {
                     Bound::Included(n) => n + 1,
                     Bound::Excluded(n) => n,
-                Bound::Unbounded => metadata.len(),
+                    Bound::Unbounded => size,
                 };
-            if end < start || end > metadata.len() {
+                if end < start || end > size {
-                return Err(StaticFileError::RangeNotSatisfiable {
+                    return Err(StaticFileError::RangeNotSatisfiable { size });
-                    size: metadata.len(),
-                });
                 }
 
-            if start != 0 || end != metadata.len() {
+                if start != 0 || end != size {
-                content_range = Some((start..end, metadata.len()));
+                    content_range = Some((start..end, size));
                 }
 
                 content_length = end - start;
@@ -417,18 +507,23 @@ impl GotFile {
                 Body::from_async_read(tokio::fs::File::from_std(file))
             };
 
-        let mut response = Response::builder()
+            // Build response
-            .header(header::ACCEPT_RANGES, "bytes")
+            let mut response = Response::builder().header(header::CONTENT_LENGTH, content_length);
-            .header(header::CONTENT_LENGTH, content_length)
-            .header(header::ETAG, self.hash);
 
+            if ok {
+                response = response
+                    .typed_header(etag)
+                    .header(header::ACCEPT_RANGES, "bytes");
+            } else {
+                response = response.status(StatusCode::NOT_FOUND);
+            }
             if !last_modified_str.is_empty() {
                 response = response.header(header::LAST_MODIFIED, last_modified_str);
             }
             if let Some(encoding) = self.encoding {
                 response = response.header(header::CONTENT_ENCODING, encoding);
             }
-        if let Some(mime) = self.mime {
+            if let Some(mime) = &self.mime {
                 response = response.header(header::CONTENT_TYPE, mime.essence_str());
             }
             if let Some((range, size)) = content_range {
@@ -438,4 +533,5 @@ impl GotFile {
             }
             Ok(response.body(body))
         }
+    }
 }

src/util.rs

@@ -1,4 +1,4 @@
-use std::{collections::BTreeMap, fs::File, path::Path, str::FromStr};
+use std::{fs::File, path::Path, str::FromStr};
 
 use mime_guess::Mime;
 use poem::http::{header, HeaderMap};
@@ -53,14 +53,14 @@ impl FromStr for ContentCoding {
     }
 }
 
-/// Parse Accept-Encoding header and return the compressed file with the preferred algorithm
+/// Parse Accept-Encoding header and return the preferred algorithm
 ///
 /// Source: <https://github.com/poem-web/poem/blob/049215cf02c5d4b1ab76f290b4708f3142d6d61b/poem/src/middleware/compression.rs#L36>
-pub fn parse_accept_encoding<'a, T>(
+pub fn parse_accept_encoding(
     headers: &HeaderMap,
-    files: &'a BTreeMap<CompressionAlg, T>,
+    enabled_algorithms: &[CompressionAlg],
-) -> Option<(CompressionAlg, &'a T)> {
+) -> Option<CompressionAlg> {
-    if files.is_empty() {
+    if enabled_algorithms.is_empty() {
         return None;
     }
 
@@ -75,23 +75,20 @@ pub fn parse_accept_encoding<'a, T>(
                 None => (v, 1000),
             };
             let coding: ContentCoding = e.parse().ok()?;
-            let alg_file = match coding {
+            let alg = match coding {
-                ContentCoding::Brotli => {
+                ContentCoding::Brotli => Some(CompressionAlg::Brotli)
-                    (CompressionAlg::Brotli, files.get(&CompressionAlg::Brotli)?)
+                    .filter(|_| enabled_algorithms.contains(&CompressionAlg::Brotli)),
-                }
+                ContentCoding::Gzip => Some(CompressionAlg::Gzip)
-                ContentCoding::Gzip => (CompressionAlg::Gzip, files.get(&CompressionAlg::Gzip)?),
+                    .filter(|_| enabled_algorithms.contains(&CompressionAlg::Gzip)),
-                ContentCoding::Star => {
+                ContentCoding::Star => enabled_algorithms.iter().max().copied(),
-                    files.iter().max_by_key(|(a, _)| *a).map(|(a, f)| (*a, f))?
-                }
             };
-            Some((alg_file, q))
+            alg.map(|alg| (alg, q))
         })
-        .max_by_key(|((a, _), q)| (*q, *a))
+        .max_by_key(|(a, q)| (*q, *a))
-        .map(|(x, _)| x)
+        .map(|(a, _)| a)
         .or_else(|| {
-            files
+            Some(CompressionAlg::None)
-                .get(&CompressionAlg::None)
+                .filter(|_| enabled_algorithms.contains(&CompressionAlg::None))
-                .map(|f| (CompressionAlg::None, f))
         })
 }
 
@@ -162,13 +159,15 @@ mod tests {
         let mut headers = HeaderMap::new();
         headers.insert(header::ACCEPT_ENCODING, accept.parse().unwrap());
 
-        let mut files = BTreeMap::new();
+        let compression = parse_accept_encoding(
-        files.insert(CompressionAlg::None, 0);
+            &headers,
-        files.insert(CompressionAlg::Gzip, 1);
+            &[
-        files.insert(CompressionAlg::Brotli, 2);
+                CompressionAlg::Gzip,
-
+                CompressionAlg::Brotli,
-        let (compression, file) = parse_accept_encoding(&headers, &files).unwrap();
+                CompressionAlg::None,
+            ],
+        )
+        .unwrap();
         assert_eq!(compression, expect);
-        assert_eq!(file, files.get(&compression).unwrap());
     }
 }

tests/fixtures/mod.rs

@@ -105,6 +105,7 @@ fn insert_websites(db: &Db) {
         &Version {
             created_at: datetime!(2023-02-18 16:30 +0),
             data: v1_data,
+            ..Default::default()
         },
     )
     .unwrap();
@@ -122,6 +123,7 @@ fn insert_websites(db: &Db) {
         &Version {
             created_at: datetime!(2023-02-18 16:52 +0),
             data: v2_data,
+            ..Default::default()
         },
     )
     .unwrap();
@@ -133,6 +135,7 @@ fn insert_websites(db: &Db) {
         &Version {
             created_at: datetime!(2023-02-18 16:30 +0),
             data: BTreeMap::new(),
+            ..Default::default()
         },
     )
     .unwrap();
@@ -143,6 +146,7 @@ fn insert_websites(db: &Db) {
         &Version {
             created_at: datetime!(2023-02-20 18:30 +0),
             data: BTreeMap::new(),
+            ..Default::default()
         },
     )
     .unwrap();

tests/snapshots/tests__config__default.snap

@@ -18,15 +18,21 @@ ConfigInner(
   keys: {
     "04e99561e3824f387a217d141d2a3b46375de6864afbedf9c9a2cc102bc946a4": KeyCfg(
       domains: "/^talon-\\d+/",
+      upload: false,
+      modify: false,
     ),
     "21bdac19ffd22870d561b1d55b35eddd9029497107edb7b926aa3e7856bb409b": KeyCfg(
       domains: [
         "spotify-gender-ex",
         "rustypipe",
       ],
+      upload: false,
+      modify: false,
     ),
     "c32ff286c8ac1c3102625badf38ffd251ae0c4a56079d8ba490f320af63f1f47": KeyCfg(
       domains: "*",
+      upload: false,
+      modify: false,
     ),
   },
 )

tests/snapshots/tests__config__sparse.snap

@@ -16,15 +16,22 @@ ConfigInner(
     brotli_level: 7,
   ),
   keys: {
-    "04e99561e3824f387a217d141d2a3b46375de6864afbedf9c9a2cc102bc946a4": KeyCfg(),
+    "04e99561e3824f387a217d141d2a3b46375de6864afbedf9c9a2cc102bc946a4": KeyCfg(
+      upload: false,
+      modify: false,
+    ),
     "21bdac19ffd22870d561b1d55b35eddd9029497107edb7b926aa3e7856bb409b": KeyCfg(
       domains: [
         "spotify-gender-ex",
         "rustypipe",
       ],
+      upload: false,
+      modify: false,
     ),
     "c32ff286c8ac1c3102625badf38ffd251ae0c4a56079d8ba490f320af63f1f47": KeyCfg(
       domains: "*",
+      upload: false,
+      modify: false,
     ),
   },
 )

tests/snapshots/tests__database__delete_website.snap

@@ -4,8 +4,8 @@ expression: data
 ---
 {"type":"website","key":"rustypipe","value":{"name":"RustyPipe","created_at":[2023,51,18,30,0,0,0,0,0],"latest_version":4,"color":7943647,"visibility":"featured","source_url":"https://code.thetadev.de/ThetaDev/rustypipe","source_icon":"gitea"}}
 {"type":"website","key":"spotify-gender-ex","value":{"name":"Spotify-Gender-Ex","created_at":[2023,49,16,30,0,0,0,0,0],"latest_version":3,"color":1947988,"visibility":"featured","source_url":"https://github.com/Theta-Dev/Spotify-Gender-Ex","source_icon":"github"}}
-{"type":"version","key":"rustypipe:4","value":{"created_at":[2023,51,18,30,0,0,0,0,0],"data":{}}}
+{"type":"version","key":"rustypipe:4","value":{"created_at":[2023,51,18,30,0,0,0,0,0],"data":{},"fallback":null,"spa":false}}
-{"type":"version","key":"spotify-gender-ex:3","value":{"created_at":[2023,49,16,30,0,0,0,0,0],"data":{}}}
+{"type":"version","key":"spotify-gender-ex:3","value":{"created_at":[2023,49,16,30,0,0,0,0,0],"data":{},"fallback":null,"spa":false}}
 {"type":"file","key":"3:gex_style.css","value":"fc825b409a49724af8f5b3c4ad15e175e68095ea746237a7b46152d3f383f541"}
 {"type":"file","key":"3:index.html","value":"6c5d37546616519e8973be51515b8a90898b4675f7b6d01f2d891edb686408a2"}
 {"type":"file","key":"4:index.html","value":"94a67cf13d752a9c1875ad999eb2be5a1b0f9746c66bca2631820b8186028811"}

tests/snapshots/tests__database__export.snap

@@ -5,10 +5,10 @@ expression: data
 {"type":"website","key":"-","value":{"name":"ThetaDev","created_at":[2023,49,16,30,0,0,0,0,0],"latest_version":2,"color":2068974,"visibility":"featured","source_url":null,"source_icon":null}}
 {"type":"website","key":"rustypipe","value":{"name":"RustyPipe","created_at":[2023,51,18,30,0,0,0,0,0],"latest_version":4,"color":7943647,"visibility":"featured","source_url":"https://code.thetadev.de/ThetaDev/rustypipe","source_icon":"gitea"}}
 {"type":"website","key":"spotify-gender-ex","value":{"name":"Spotify-Gender-Ex","created_at":[2023,49,16,30,0,0,0,0,0],"latest_version":3,"color":1947988,"visibility":"featured","source_url":"https://github.com/Theta-Dev/Spotify-Gender-Ex","source_icon":"github"}}
-{"type":"version","key":"-:1","value":{"created_at":[2023,49,16,30,0,0,0,0,0],"data":{"Deployed by":"https://github.com/Theta-Dev/Talon/actions/runs/1352014628","Version":"v0.1.0"}}}
+{"type":"version","key":"-:1","value":{"created_at":[2023,49,16,30,0,0,0,0,0],"data":{"Deployed by":"https://github.com/Theta-Dev/Talon/actions/runs/1352014628","Version":"v0.1.0"},"fallback":null,"spa":false}}
-{"type":"version","key":"-:2","value":{"created_at":[2023,49,16,52,0,0,0,0,0],"data":{"Deployed by":"https://github.com/Theta-Dev/Talon/actions/runs/1354755231","Version":"v0.1.1"}}}
+{"type":"version","key":"-:2","value":{"created_at":[2023,49,16,52,0,0,0,0,0],"data":{"Deployed by":"https://github.com/Theta-Dev/Talon/actions/runs/1354755231","Version":"v0.1.1"},"fallback":null,"spa":false}}
-{"type":"version","key":"rustypipe:4","value":{"created_at":[2023,51,18,30,0,0,0,0,0],"data":{}}}
+{"type":"version","key":"rustypipe:4","value":{"created_at":[2023,51,18,30,0,0,0,0,0],"data":{},"fallback":null,"spa":false}}
-{"type":"version","key":"spotify-gender-ex:3","value":{"created_at":[2023,49,16,30,0,0,0,0,0],"data":{}}}
+{"type":"version","key":"spotify-gender-ex:3","value":{"created_at":[2023,49,16,30,0,0,0,0,0],"data":{},"fallback":null,"spa":false}}
 {"type":"file","key":"1:index.html","value":"3b5f6bad5376897435def176d0fe77e5b9b4f0deafc7491fc27262650744ad68"}
 {"type":"file","key":"1:style.css","value":"356f131c825fbf604797c7e9c85352549d81db8af91fee834016d075110af026"}
 {"type":"file","key":"2:assets/image.jpg","value":"901d291a47a8a9b55c06f84e5e5f82fd2dcee65cac1406d6e878b805d45c1e93"}

tests/snapshots/tests__database__get_version.snap

@@ -1,5 +1,5 @@
 ---
-source: src/db/mod.rs
+source: tests/tests.rs
 expression: version
 ---
 Version(
@@ -8,4 +8,6 @@ Version(
     "Deployed by": "https://github.com/Theta-Dev/Talon/actions/runs/1352014628",
     "Version": "v0.1.0",
   },
+  fallback: None,
+  spa: false,
 )

tests/snapshots/tests__database__get_website_versions.snap

@@ -1,5 +1,5 @@
 ---
-source: src/db/mod.rs
+source: tests/tests.rs
 expression: versions
 ---
 [
@@ -9,6 +9,8 @@ expression: versions
       "Deployed by": "https://github.com/Theta-Dev/Talon/actions/runs/1352014628",
       "Version": "v0.1.0",
     },
+    fallback: None,
+    spa: false,
   )),
   (2, Version(
     created_at: (2023, 49, 16, 52, 0, 0, 0, 0, 0),
@@ -16,5 +18,7 @@ expression: versions
       "Deployed by": "https://github.com/Theta-Dev/Talon/actions/runs/1354755231",
       "Version": "v0.1.1",
     },
+    fallback: None,
+    spa: false,
   )),
 ]

tests/tests.rs

@@ -283,17 +283,19 @@ mod storage {
             // Images should not be compressed
             let expect = &hash_str
                 != "901d291a47a8a9b55c06f84e5e5f82fd2dcee65cac1406d6e878b805d45c1e93"
-                && &hash_str != "9f7e7971b4bfdb75429e534dea461ed90340886925078cda252cada9aa0e25f7";
+                && &hash_str != "9f7e7971b4bfdb75429e534dea461ed90340886925078cda252cada9aa0e25f7"
+                && &hash_str != "a44816e6c3b650bdf88e6532659ba07ef187c2113ae311da9709e056aec8eadb";
             assert_eq!(path_compressed.is_file(), expect)
         }
     }
 
     #[rstest]
-    #[case::nocmp("", VERSION_1_2, "", true, "text/html", None)]
+    #[case::index("br", VERSION_1_2, "", false, "text/html", None)]
-    #[case::gzip("gzip", VERSION_1_2, "", true, "text/html", None)]
+    #[case::nocmp("", VERSION_1_2, "assets/style.css", true, "text/css", None)]
-    #[case::br("br", VERSION_1_2, "", true, "text/html", None)]
+    #[case::gzip("gzip", VERSION_1_2, "assets/style.css", true, "text/css", None)]
+    #[case::br("br", VERSION_1_2, "assets/style.css", true, "text/css", None)]
     #[case::image("br", VERSION_1_2, "assets/image.jpg", false, "image/jpeg", None)]
-    #[case::subdir("br", VERSION_3_1, "page2", true, "text/html", Some("/page2/"))]
+    #[case::subdir("br", VERSION_3_1, "page2", false, "text/html", Some("/page2/"))]
     fn get_file(
         store: StorageTest,
         #[case] encoding: &str,