Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added support for zstd compression #1247

Open
wants to merge 18 commits into
base: master
Choose a base branch
from
Open

Added support for zstd compression #1247

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
899c383
updated meson_options.txt
EdgarModesto23 Sep 28, 2024
82c372f
added pistache_use_content_encoding_zstd to build and common http
EdgarModesto23 Oct 1, 2024
c91848e
Merge branch 'master' of github.com:pistacheio/pistache into edgar-ad…
EdgarModesto23 Oct 1, 2024
f046d8f
added zstd on http tests and http.cc
EdgarModesto23 Oct 2, 2024
255d16d
added support for zstd compression
EdgarModesto23 Oct 6, 2024
ba69362
Included zstd documentation on README
EdgarModesto23 Oct 6, 2024
dff96d1
bumped version number
EdgarModesto23 Oct 12, 2024
6f237fb
removed indentation on http server test file
EdgarModesto23 Oct 12, 2024
acb30d4
fixed meson build, use mid value on zstd compression
EdgarModesto23 Oct 12, 2024
35de6b2
Merge branch 'master' into edgar-add-support-zstd
kiplingw Oct 17, 2024
2b88038
fixed default value for zstd encoding, removed leading underscore
EdgarModesto23 Oct 20, 2024
72c748c
updated version date
EdgarModesto23 Oct 20, 2024
a0dda93
Merge branch 'master' into edgar-add-support-zstd
kiplingw Oct 20, 2024
f083b4a
removed zstd_bultra2
EdgarModesto23 Oct 20, 2024
fd4d77d
merge conflict solved
EdgarModesto23 Oct 20, 2024
6ff23bb
updated description of zstd meson option
EdgarModesto23 Oct 20, 2024
241a76f
use reinterpret_cast in zstd compress func
EdgarModesto23 Oct 20, 2024
342ceee
fix: bumped version to 12
EdgarModesto23 Oct 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@ $ meson setup build \
-DPISTACHE_BUILD_DOCS=false \
-DPISTACHE_USE_CONTENT_ENCODING_BROTLI=true \
-DPISTACHE_USE_CONTENT_ENCODING_DEFLATE=true \
-DPISTACHE_USE_CONTENT_ENCODING_ZSTD=true \
--prefix="$PWD/prefix"
$ meson compile -C build
$ meson install -C build
Expand All @@ -240,6 +241,7 @@ Some other Meson options:
| PISTACHE_BUILD_DOCS | False | Build Doxygen docs |
| PISTACHE_USE_CONTENT_ENCODING_BROTLI | False | Build with Brotli content encoding support |
| PISTACHE_USE_CONTENT_ENCODING_DEFLATE | False | Build with deflate content encoding support |
| PISTACHE_USE_CONTENT_ENCODING_ZSTD | False | Build with zstd content encoding support |

## Example

Expand Down
1 change: 1 addition & 0 deletions bldscripts/mesbuild.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ else
-DPISTACHE_BUILD_TESTS=true \
-DPISTACHE_BUILD_DOCS=false \
-DPISTACHE_USE_CONTENT_ENCODING_DEFLATE=true \
-DPISTACHE_USE_CONTENT_ENCODING_ZSTD=true \
--prefix="${MESON_PREFIX_DIR}" \
# -DPISTACHE_FORCE_LIBEVENT=true

Expand Down
1 change: 1 addition & 0 deletions bldscripts/mesbuilddebug.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ else
-DPISTACHE_BUILD_TESTS=true \
-DPISTACHE_BUILD_DOCS=false \
-DPISTACHE_USE_CONTENT_ENCODING_DEFLATE=true \
-DPISTACHE_USE_CONTENT_ENCODING_ZSTD=true \
-DPISTACHE_DEBUG=true \
--prefix="${MESON_PREFIX_DIR}" \
# -DPISTACHE_FORCE_LIBEVENT=true
Expand Down
16 changes: 16 additions & 0 deletions include/pistache/http.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@
#include <zlib.h>
#endif

#ifdef PISTACHE_USE_CONTENT_ENCODING_ZSTD
#include <zstd.h>
#endif

namespace Pistache
{
namespace Tcp
Expand Down Expand Up @@ -530,6 +534,14 @@ namespace Pistache
contentEncodingBrotliLevel_ = _contentEncodingBrotliLevel;
}
#endif
#ifdef PISTACHE_USE_CONTENT_ENCODING_ZSTD

void setCompressionZstdLevel(const int _contentEncodingZstdLevel)
{
contentEncodingZstdLevel = _contentEncodingZstdLevel;
}
EdgarModesto23 marked this conversation as resolved.
Show resolved Hide resolved

#endif

#ifdef PISTACHE_USE_CONTENT_ENCODING_DEFLATE
// Set the compression level for deflate algorithm. Defaults to
Expand Down Expand Up @@ -562,6 +574,10 @@ namespace Pistache
int contentEncodingBrotliLevel_ = BROTLI_DEFAULT_QUALITY;
#endif

#ifdef PISTACHE_USE_CONTENT_ENCODING_ZSTD
int contentEncodingZstdLevel = ZSTD_lazy2;
EdgarModesto23 marked this conversation as resolved.
Show resolved Hide resolved
#endif

#ifdef PISTACHE_USE_CONTENT_ENCODING_DEFLATE
int contentEncodingDeflateLevel_ = Z_DEFAULT_COMPRESSION;
#endif
Expand Down
1 change: 1 addition & 0 deletions include/pistache/http_header.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ namespace Pistache::Http::Header
Br,
Compress,
Deflate,
Zstd,
Identity,
Chunked,
Unknown };
Expand Down
9 changes: 9 additions & 0 deletions meson.build
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,15 @@ if get_option('PISTACHE_USE_RAPIDJSON')
public_deps += rapidjson_dep
endif

# Support Zstd compressed Content-Encoding responses...
if get_option('PISTACHE_USE_CONTENT_ENCODING_ZSTD')

#Need Zstd encoder for library...
zstd_dep = dependency('libzstd')
deps_libpistache += zstd_dep
public_deps += zstd_dep
endif

# Support Brotli compressed Content-Encoding responses...
if get_option('PISTACHE_USE_CONTENT_ENCODING_BROTLI')

Expand Down
1 change: 1 addition & 0 deletions meson_options.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ option('PISTACHE_INSTALL', type: 'boolean', value: true, description: 'add pista
option('PISTACHE_USE_SSL', type: 'boolean', value: false, description: 'add support for SSL server')
option('PISTACHE_USE_RAPIDJSON', type: 'boolean', value: true, description: 'add support for rapidjson')
option('PISTACHE_USE_CONTENT_ENCODING_BROTLI', type: 'boolean', value: false, description: 'add support for Brotli compressed content encoding')
option('PISTACHE_USE_CONTENT_ENCODING_ZSTD', type: 'boolean', value: false, description: 'add support for ZSTD compressed content encoding')
EdgarModesto23 marked this conversation as resolved.
Show resolved Hide resolved
option('PISTACHE_USE_CONTENT_ENCODING_DEFLATE', type: 'boolean', value: false, description: 'add support for deflate compressed content encoding')
option('PISTACHE_DEBUG', type: 'boolean', value: false, description: 'with debugging code')
option('PISTACHE_LOG_AND_STDOUT', type: 'boolean', value: false, description: 'send log msgs to stdout too')
Expand Down
34 changes: 34 additions & 0 deletions src/common/http.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
Http layer implementation
*/

#include "pistache/http_header.h"
EdgarModesto23 marked this conversation as resolved.
Show resolved Hide resolved
#include <pistache/config.h>
#include <pistache/eventmeth.h>
#include <pistache/http.h>
Expand Down Expand Up @@ -922,6 +923,33 @@ namespace Pistache::Http
}
#endif

#ifdef PISTACHE_USE_CONTENT_ENCODING_ZSTD

case Http::Header::Encoding::Zstd: {
// Get max compressed size
size_t estimated_size = ZSTD_compressBound(size);
// Allocate a smart buffer to contain compressed data...
std::unique_ptr compressedData = std::make_unique<std::byte[]>(estimated_size);

// Compress data using compresion_level = 5: https://facebook.github.io/zstd/zstd_manual.html#Chapter5
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We currently don't have a public API for the user to specify the compression level (each compressor can use a different range). Until we figure out an elegant way to do that, it might be best to set the compression level consistent with whatever we're using on the others (I think max).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe max is really max on zstd. We could also set compression level dynamically, or partially dynamically, taking into account available CPU bandwidth, available network bandwidth, number of cores (zstd is good with multithreading) etc.

But maybe a middling value is wisest until we do a well-thought-through API/algorithm for compression level?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dgreatwood @kiplingw I agree. Using a middle value makes the most sense to me. At least until we find a better solution as both of you guys mentioned.

auto compress_size = ZSTD_compress((void*)compressedData.get(), estimated_size,
data, size, ZSTD_lazy2);
if (ZSTD_isError(compress_size))
{
throw std::runtime_error(
std::string("failed to compress data to ZSTD on ZSTD_compress(), returning: ") + std::to_string(compress_size));
}
headers().add<Http::Header::ContentEncoding>(
Http::Header::Encoding::Zstd);

// Send compressed data back to client...
return putOnWire(
reinterpret_cast<const char*>(compressedData.get()),
compress_size);
}

#endif

#ifdef PISTACHE_USE_CONTENT_ENCODING_DEFLATE
// User requested deflate compression...
case Http::Header::Encoding::Deflate: {
Expand Down Expand Up @@ -1084,6 +1112,12 @@ namespace Pistache::Http
break;
#endif

#ifdef PISTACHE_USE_CONTENT_ENCODING_ZSTD
case Http::Header::Encoding::Zstd:
contentEncoding_ = Http::Header::Encoding::Zstd;
break;
#endif

#ifdef PISTACHE_USE_CONTENT_ENCODING_DEFLATE
// Application requested deflate compression...
case Http::Header::Encoding::Deflate:
Expand Down
12 changes: 12 additions & 0 deletions src/common/http_header.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ namespace Pistache::Http::Header
return "gzip";
case Encoding::Br:
return "br";
case Encoding::Zstd:
return "zstd";
case Encoding::Compress:
return "compress";
case Encoding::Deflate:
Expand All @@ -73,6 +75,11 @@ namespace Pistache::Http::Header
return Encoding::Unknown;
}

if (!strncasecmp(str.data(), "zstd", str.length()))
{
return Encoding::Zstd;
}

if (!strncasecmp(str.data(), "gzip", str.length()))
{
return Encoding::Gzip;
Expand Down Expand Up @@ -107,6 +114,11 @@ namespace Pistache::Http::Header
{
switch (encoding)
{

#ifdef PISTACHE_USE_CONTENT_ENCODING_ZSTD
case Encoding::Zstd:
/* @fallthrough@ */
#endif
#ifdef PISTACHE_USE_CONTENT_ENCODING_BROTLI
case Encoding::Br:
/* @fallthrough@ */
Expand Down
4 changes: 4 additions & 0 deletions src/meson.build
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ if get_option('PISTACHE_USE_CONTENT_ENCODING_BROTLI')
public_args += '-DPISTACHE_USE_CONTENT_ENCODING_BROTLI'
endif

if get_option('PISTACHE_USE_CONTENT_ENCODING_ZSTD')
public_args += '-DPISTACHE_USE_CONTENT_ENCODING_ZSTD'
endif

if get_option('PISTACHE_USE_CONTENT_ENCODING_DEFLATE')
public_args += '-DPISTACHE_USE_CONTENT_ENCODING_DEFLATE'
endif
Expand Down
Loading
Loading