Merge pull request #25 from kkent030315/zstd

Add Zstd support as optional and compile-time low ratio report

.github/workflows/ci.yml

@@ -25,6 +25,10 @@ jobs:
         stability:
           - ""
           - "--release"
+        feature:
+          - ""
+          - "--no-default-features --features deflate"
+          - "--no-default-features --features zstd"
     steps:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
@@ -33,4 +37,4 @@ jobs:
           profile: default
           default: true
       - name: cargo clippy
-        run: "cargo clippy --all ${{matrix.stability}}"
+        run: "cargo clippy --all ${{matrix.feature}} ${{matrix.stability}}"

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = [".", "codegen"]
+members = [".", "codegen", "compress"]
 
 [package]
 name = "include-flate"
@@ -15,5 +15,13 @@ keywords = ["compression", "deflate", "macro", "include", "assets"]
 
 [dependencies]
 include-flate-codegen = { version = "0.2.0", path = "codegen" }
+include-flate-compress = { version = "0.1.0", path = "compress" }
 once_cell = "1.18.0"
 libflate = "2.0.0"
+zstd = "0.13.0"
+
+[features]
+default = ["deflate", "zstd"]
+deflate = ["include-flate-compress/deflate"]
+zstd = ["include-flate-compress/zstd"]
+no-compression-warnings = ["include-flate-codegen/no-compression-warnings"]

codegen/Cargo.toml

@@ -16,3 +16,10 @@ libflate = "2.0.0"
 proc-macro2 = "1.0.9"
 quote = "1.0.2"
 syn = { version = "2.0.2", features = ["full"] }
+zstd = "0.13.0"
+include-flate-compress = { version = "0.1.0", path = "../compress" }
+proc-macro-error = "1.0.4"
+
+[features]
+default = []
+no-compression-warnings = []

codegen/src/lib.rs

@@ -15,15 +15,17 @@
 
 extern crate proc_macro;
 
-use std::fs::File;
+use std::fs::{self, File};
+use std::io::{Read, Seek};
 use std::path::PathBuf;
-use std::str::from_utf8;
+use std::str::{from_utf8, FromStr};
 
-use libflate::deflate::Encoder;
+use include_flate_compress::{apply_compression, CompressionMethod};
 use proc_macro::TokenStream;
 use proc_macro2::Span;
+use proc_macro_error::{emit_warning, proc_macro_error};
 use quote::quote;
-use syn::{Error, LitByteStr, LitStr, Result};
+use syn::{Error, LitByteStr};
 
 /// `deflate_file!("file")` is equivalent to `include_bytes!("file.gz")`.
 ///
@@ -42,6 +44,7 @@ use syn::{Error, LitByteStr, LitStr, Result};
 /// - If the argument is not a single literal
 /// - If the referenced file does not exist or is not readable
 #[proc_macro]
+#[proc_macro_error]
 pub fn deflate_file(ts: TokenStream) -> TokenStream {
     match inner(ts, false) {
         Ok(ts) => ts.into(),
@@ -55,47 +58,122 @@ pub fn deflate_file(ts: TokenStream) -> TokenStream {
 /// - The compile errors in `deflate_file!`
 /// - If the file contents are not all valid UTF-8
 #[proc_macro]
+#[proc_macro_error]
 pub fn deflate_utf8_file(ts: TokenStream) -> TokenStream {
     match inner(ts, true) {
         Ok(ts) => ts.into(),
         Err(err) => err.to_compile_error().into(),
     }
 }
 
-fn inner(ts: TokenStream, utf8: bool) -> Result<impl Into<TokenStream>> {
+/// An arguments expected provided by the proc-macro.
+///
+/// ```ignore
+/// flate!(pub static DATA: [u8] from "assets/009f.dat"); // default, DEFLATE
+/// flate!(pub static DATA: [u8] from "assets/009f.dat" with zstd); // Use Zstd for this file spcifically
+/// flate!(pub static DATA: [u8] from "assets/009f.dat" with deflate); // Explicitly use DEFLATE.
+/// ```
+struct FlateArgs {
+    path: syn::LitStr,
+    algorithm: Option<CompressionMethodTy>,
+}
+
+impl syn::parse::Parse for FlateArgs {
+    fn parse(input: syn::parse::ParseStream) -> syn::Result<Self> {
+        let path = input.parse()?;
+
+        let algorithm = if input.is_empty() {
+            None
+        } else {
+            let lookahead = input.lookahead1();
+            if lookahead.peek(kw::deflate) {
+                input.parse::<kw::deflate>()?;
+                Some(CompressionMethodTy(CompressionMethod::Deflate))
+            } else if lookahead.peek(kw::zstd) {
+                input.parse::<kw::zstd>()?;
+                Some(CompressionMethodTy(CompressionMethod::Zstd))
+            } else {
+                return Err(lookahead.error());
+            }
+        };
+
+        Ok(Self { path, algorithm })
+    }
+}
+
+mod kw {
+    syn::custom_keyword!(deflate);
+    syn::custom_keyword!(zstd);
+}
+
+#[derive(Debug)]
+struct CompressionMethodTy(CompressionMethod);
+
+fn compression_ratio(original_size: u64, compressed_size: u64) -> f64 {
+    (compressed_size as f64 / original_size as f64) * 100.0
+}
+
+fn inner(ts: TokenStream, utf8: bool) -> syn::Result<impl Into<TokenStream>> {
     fn emap<E: std::fmt::Display>(error: E) -> Error {
         Error::new(Span::call_site(), error)
     }
 
-    let dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap());
+    let dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").map_err(emap)?);
 
-    let lit = syn::parse::<LitStr>(ts)?;
-    let path = PathBuf::from(lit.value());
+    let args: FlateArgs = syn::parse2::<FlateArgs>(ts.to_owned().into())?;
+    let path = PathBuf::from_str(&args.path.value()).map_err(emap)?;
+    let algo = args
+        .algorithm
+        .unwrap_or(CompressionMethodTy(CompressionMethod::Deflate));
 
     if path.is_absolute() {
         Err(emap("absolute paths are not supported"))?;
     }
 
-    let target = dir.join(path);
+    let target = dir.join(&path);
 
-    let mut file = File::open(target).map_err(emap)?;
+    let mut file = File::open(&target).map_err(emap)?;
 
-    let mut encoder = Encoder::new(Vec::<u8>::new());
+    let mut vec = Vec::<u8>::new();
     if utf8 {
-        use std::io::Write;
-
-        let mut vec = Vec::<u8>::new();
         std::io::copy(&mut file, &mut vec).map_err(emap)?;
         from_utf8(&vec).map_err(emap)?;
-        encoder.write_all(&vec).map_err(emap)?;
-    } else {
-        // no need to store the raw buffer; let's avoid storing two buffers
-        std::io::copy(&mut file, &mut encoder).map_err(emap)?;
     }
-    let bytes = encoder.finish().into_result().map_err(emap)?;
 
-    let bytes = LitByteStr::new(&bytes, Span::call_site());
+    let mut compressed_buffer = Vec::<u8>::new();
+
+    {
+        let mut compressed_cursor = std::io::Cursor::new(&mut compressed_buffer);
+        let mut source: Box<dyn Read> = if utf8 {
+            Box::new(std::io::Cursor::new(vec))
+        } else {
+            file.seek(std::io::SeekFrom::Start(0)).map_err(emap)?;
+            Box::new(&file)
+        };
+
+        apply_compression(&mut source, &mut compressed_cursor, algo.0).map_err(emap)?;
+    }
+
+    let bytes = LitByteStr::new(&compressed_buffer, Span::call_site());
     let result = quote!(#bytes);
 
+    #[cfg(not(feature = "no-compression-warnings"))]
+    {
+        let compression_ratio = compression_ratio(
+            fs::metadata(&target).map_err(emap)?.len(),
+            compressed_buffer.len() as u64,
+        );
+
+        if compression_ratio < 10.0f64 {
+            emit_warning!(
+            &args.path,
+            "Detected low compression ratio ({:.2}%) for file {:?} with `{:?}`. Consider using other compression methods.",
+            compression_ratio,
+            path.display(),
+            algo.0,
+        );
+        }
+    }
+
     Ok(result)
 }

compress/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "include-flate-compress"
+version = "0.1.0"
+authors = ["SOFe <sofe2038@gmail.com>", "Kento Oki <hrn832@protonmail.com>"]
+edition = "2021"
+license = "Apache-2.0"
+repository = "https://github.com/SOF3/include-flate.git"
+homepage = "https://github.com/SOF3/include-flate"
+description = "Compression algorithm provider"
+
+[dependencies]
+libflate = { version = "2.0.0", optional = true }
+zstd = { version = "0.13.0", optional = true }
+
+[features]
+default = ["deflate", "zstd"]
+deflate = ["dep:libflate"]
+zstd = ["dep:zstd"]