Add support for native zlib-ng in the library and compression references

CMakeFilters.cmake

@@ -101,7 +101,11 @@ if (HDF5_ENABLE_Z_LIB_SUPPORT)
       endif ()
       set(H5_ZLIB_FOUND ${ZLIB_FOUND})
       if (H5_ZLIB_FOUND)
-        set (H5_ZLIB_HEADER "zlib.h")
+        if (HDF5_USE_ZLIB_NG)
+          set (H5_ZLIB_HEADER "zlib-ng.h")
+        else ()
+          set (H5_ZLIB_HEADER "zlib.h")
+        endif ()
         set (H5_ZLIB_INCLUDE_DIR_GEN ${ZLIB_INCLUDE_DIR})
         set (H5_ZLIB_INCLUDE_DIRS ${H5_ZLIB_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIR})
         # The FindZLIB.cmake module does not set an OUTPUT_NAME
@@ -125,6 +129,9 @@ if (HDF5_ENABLE_Z_LIB_SUPPORT)
   if (H5_ZLIB_FOUND)
     set (H5_HAVE_FILTER_DEFLATE 1)
     set (H5_HAVE_ZLIB_H 1)
+    if (HDF5_USE_ZLIB_NG)
+      set (H5_HAVE_ZLIBNG_H 1)
+    endif ()
     set (H5_HAVE_LIBZ 1)
     if (H5_HAVE_FILTER_DEFLATE)
       set (EXTERNAL_FILTERS "${EXTERNAL_FILTERS} DEFLATE")
@@ -135,6 +142,7 @@ if (HDF5_ENABLE_Z_LIB_SUPPORT)
     set (HDF5_ENABLE_Z_LIB_SUPPORT OFF CACHE BOOL "" FORCE)
     message (WARNING " ZLib support in HDF5 was enabled but not found")
   endif ()
+  message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}")
 endif ()
 
 #-----------------------------------------------------------------------------

config/cmake/H5pubconf.h.in

@@ -366,6 +366,9 @@
 /* Define to 1 if you have the <zlib.h> header file. */
 #cmakedefine H5_HAVE_ZLIB_H @H5_HAVE_ZLIB_H@
 
+/* Define to 1 if you have the <zlib.h> header file. */
+#cmakedefine H5_HAVE_ZLIBNG_H @H5_HAVE_ZLIBNG_H@
+
 /* Define to 1 if you have the `_getvideoconfig' function. */
 #cmakedefine H5_HAVE__GETVIDEOCONFIG @H5_HAVE__GETVIDEOCONFIG@
 

config/cmake/HDFLibMacros.cmake

@@ -40,6 +40,11 @@ macro (EXTERNAL_ZLIB_LIBRARY compress_type)
   add_library(${HDF_PACKAGE_NAMESPACE}zlib-static ALIAS zlib-static)
   set (H5_ZLIB_STATIC_LIBRARY "${HDF_PACKAGE_NAMESPACE}zlib-static")
   set (H5_ZLIB_LIBRARIES ${H5_ZLIB_STATIC_LIBRARY})
+  if (HDF5_USE_ZLIB_NG)
+    set (H5_ZLIB_HEADER "zlib-ng.h")
+  else ()
+    set (H5_ZLIB_HEADER "zlib.h")
+  endif ()
 
   set (H5_ZLIB_INCLUDE_DIR_GEN "${hdf5_zlib_BINARY_DIR}")
   set (H5_ZLIB_INCLUDE_DIR "${hdf5_zlib_SOURCE_DIR}")

config/cmake/ZLIBNG/CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.18)
-project(ZLIB VERSION ${ZLIB_HEADER_VERSION} LANGUAGES C)
+project(ZLIB LANGUAGES C)
 
 message(STATUS "Using CMake version ${CMAKE_VERSION}")
 
@@ -98,7 +98,7 @@ endif()
 # Options parsing
 #
 option(WITH_GZFILEOP "Compile with support for gzFile related functions" ON)
-option(ZLIB_COMPAT "Compile with zlib compatible API" ON)
+option(ZLIB_COMPAT "Compile with zlib compatible API" OFF)
 option(ZLIB_ENABLE_TESTS "Build test binaries" OFF)
 option(ZLIBNG_ENABLE_TESTS "Test zlib-ng specific API" OFF)
 option(WITH_GTEST "Build gtest_zlib" OFF)

release_docs/RELEASE.txt

@@ -47,6 +47,12 @@ New Features
 
     Configuration:
     -------------
+    - Added support for native zlib-ng compression.
+
+      Changed the zlib-ng CMake logic to prefer the native zlib-ng library. Added
+      #ifdef around the compression function calls. Added including the correct
+      header file with the same #ifdef.
+
     - Renamed remaining HDF5 library CMake options except for CMake BUILD* variables
 
       DEFAULT_API_VERSION to HDF5_DEFAULT_API_VERSION

src/CMakeLists.txt

@@ -727,8 +727,10 @@ set (H5Z_SOURCES
     ${HDF5_SRC_DIR}/H5Ztrans.c
 )
 if (H5_ZLIB_HEADER)
-  SET_PROPERTY(SOURCE ${HDF5_SRC_DIR}/H5Zdeflate.c PROPERTY
-      COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}")
+  message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}")
+  set_source_files_properties(${HDF5_SRC_DIR}/H5Zdeflate.c
+               PROPERTIES COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}"
+  )
 endif ()
 
 

src/H5Zdeflate.c

@@ -74,8 +74,12 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
 
     if (flags & H5Z_FLAG_REVERSE) {
         /* Input; uncompress */
-        z_stream z_strm;             /* zlib parameters */
-        size_t   nalloc = *buf_size; /* Number of bytes for output (compressed) buffer */
+#if defined(H5_HAVE_ZLIBNG_H)
+        zng_stream z_strm; /* zlib parameters */
+#else
+        z_stream z_strm; /* zlib parameters */
+#endif
+        size_t nalloc = *buf_size; /* Number of bytes for output (compressed) buffer */
 
         /* Allocate space for the compressed buffer */
         if (NULL == (outbuf = H5MM_malloc(nalloc)))
@@ -89,21 +93,34 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
         H5_CHECKED_ASSIGN(z_strm.avail_out, unsigned, nalloc, size_t);
 
         /* Initialize the uncompression routines */
+#if defined(H5_HAVE_ZLIBNG_H)
+        if (Z_OK != zng_inflateInit(&z_strm))
+            HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflateInit() failed");
+#else
         if (Z_OK != inflateInit(&z_strm))
             HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflateInit() failed");
+#endif
 
         /* Loop to uncompress the buffer */
         do {
             /* Uncompress some data */
+#if defined(H5_HAVE_ZLIBNG_H)
+            status = zng_inflate(&z_strm, Z_SYNC_FLUSH);
+#else
             status = inflate(&z_strm, Z_SYNC_FLUSH);
+#endif
 
             /* Check if we are done uncompressing data */
             if (Z_STREAM_END == status)
                 break; /*done*/
 
             /* Check for error */
             if (Z_OK != status) {
+#if defined(H5_HAVE_ZLIBNG_H)
+                (void)zng_inflateEnd(&z_strm);
+#else
                 (void)inflateEnd(&z_strm);
+#endif
                 HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflate() failed");
             }
             else {
@@ -114,7 +131,11 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
                     /* Allocate a buffer twice as big */
                     nalloc *= 2;
                     if (NULL == (new_outbuf = H5MM_realloc(outbuf, nalloc))) {
+#if defined(H5_HAVE_ZLIBNG_H)
+                        (void)zng_inflateEnd(&z_strm);
+#else
                         (void)inflateEnd(&z_strm);
+#endif
                         HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0,
                                     "memory allocation failed for deflate uncompression");
                     } /* end if */
@@ -137,7 +158,11 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
         ret_value = z_strm.total_out;
 
         /* Finish uncompressing the stream */
+#if defined(H5_HAVE_ZLIBNG_H)
+        (void)zng_inflateEnd(&z_strm);
+#else
         (void)inflateEnd(&z_strm);
+#endif
     } /* end if */
     else {
         /*
@@ -147,9 +172,13 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
          */
         const Bytef *z_src = (const Bytef *)(*buf);
         Bytef       *z_dst; /*destination buffer		*/
-        uLongf       z_dst_nbytes = (uLongf)compressBound(nbytes);
-        uLong        z_src_nbytes = (uLong)nbytes;
-        int          aggression; /* Compression aggression setting */
+#if defined(H5_HAVE_ZLIBNG_H)
+        uLongf z_dst_nbytes = (uLongf)zng_compressBound(nbytes);
+#else
+        uLongf z_dst_nbytes = (uLongf)compressBound(nbytes);
+#endif
+        uLong z_src_nbytes = (uLong)nbytes;
+        int   aggression; /* Compression aggression setting */
 
         /* Set the compression aggression level */
         H5_CHECKED_ASSIGN(aggression, int, cd_values[0], unsigned);
@@ -160,7 +189,11 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
         z_dst = (Bytef *)outbuf;
 
         /* Perform compression from the source to the destination buffer */
+#if defined(H5_HAVE_ZLIBNG_H)
+        status = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#else
         status = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#endif
 
         /* Check for various zlib errors */
         if (Z_BUF_ERROR == status)

test/CMakeLists.txt

@@ -483,6 +483,12 @@ endforeach ()
 ##############################################################################
 ######### Also special handling of link libs #############
 #-- Adding test for chunk_info
+if (H5_ZLIB_HEADER)
+  message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}")
+  set_source_files_properties(${HDF5_TEST_SOURCE_DIR}/chunk_info.c
+               PROPERTIES COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}"
+  )
+endif ()
 add_executable (chunk_info ${HDF5_TEST_SOURCE_DIR}/chunk_info.c)
 target_compile_options(chunk_info PRIVATE "${HDF5_CMAKE_C_FLAGS}")
 target_compile_definitions(chunk_info PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}")
@@ -504,6 +510,13 @@ if (HDF5_ENABLE_FORMATTERS)
 endif ()
 
 #-- Adding test for direct_chunk
+if (H5_ZLIB_HEADER)
+  message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}")
+
+  set_source_files_properties(${HDF5_TEST_SOURCE_DIR}/direct_chunk.c
+               PROPERTIES COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}"
+  )
+endif ()
 add_executable (direct_chunk ${HDF5_TEST_SOURCE_DIR}/direct_chunk.c)
 target_compile_options(direct_chunk PRIVATE "${HDF5_CMAKE_C_FLAGS}")
 target_compile_definitions(direct_chunk PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}")

test/chunk_info.c

@@ -40,7 +40,12 @@
 
 #include "h5test.h"
 #ifdef H5_HAVE_FILTER_DEFLATE
-#include "zlib.h"
+#if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER)
+#define H5_ZLIB_HEADER "zlib.h"
+#endif
+#if defined(H5_ZLIB_HEADER)
+#include H5_ZLIB_HEADER /* "zlib.h" */
+#endif
 #endif
 
 /* Test file names, using H5F_libver_t as indices */
@@ -530,7 +535,11 @@ test_get_chunk_info_highest_v18(hid_t fapl)
     z_dst = (Bytef *)inbuf;
 
     /* Perform compression from the source to the destination buffer */
+#if defined(H5_HAVE_ZLIBNG_H)
+    ret = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#else
     ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#endif
 
     /* Set the chunk size to the compressed chunk size */
     chunk_size = (hsize_t)z_dst_nbytes;

test/direct_chunk.c

@@ -12,12 +12,14 @@
 
 #include "h5test.h"
 
+#ifdef H5_HAVE_FILTER_DEFLATE
 #if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER)
 #define H5_ZLIB_HEADER "zlib.h"
 #endif
 #if defined(H5_ZLIB_HEADER)
 #include H5_ZLIB_HEADER /* "zlib.h" */
 #endif
+#endif
 
 #define FILE_NAME "direct_chunk.h5"
 
@@ -205,7 +207,11 @@ test_direct_chunk_write(hid_t file)
     z_dst  = (Bytef *)outbuf;
 
     /* Perform compression from the source to the destination buffer */
+#if defined(H5_HAVE_ZLIBNG_H)
+    ret = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#else
     ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#endif
 
     /* Check for various zlib errors */
     if (Z_BUF_ERROR == ret) {
@@ -284,7 +290,11 @@ test_direct_chunk_write(hid_t file)
     z_dst  = (Bytef *)outbuf;
 
     /* Perform compression from the source to the destination buffer */
+#if defined(H5_HAVE_ZLIBNG_H)
+    ret = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#else
     ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#endif
 
     /* Check for various zlib errors */
     if (Z_BUF_ERROR == ret) {
@@ -1522,8 +1532,12 @@ test_direct_chunk_read_no_cache(hid_t file)
             if (filter_mask != 0)
                 goto error;
 
-            /* Perform decompression from the source to the destination buffer */
+                /* Perform decompression from the source to the destination buffer */
+#if defined(H5_HAVE_ZLIBNG_H)
+            ret = zng_uncompress(z_dst, &z_dst_nbytes, z_src, z_src_nbytes);
+#else
             ret = uncompress(z_dst, &z_dst_nbytes, z_src, z_src_nbytes);
+#endif
 
             /* Check for various zlib errors */
             if (Z_BUF_ERROR == ret) {
@@ -1713,8 +1727,12 @@ test_direct_chunk_read_cache(hid_t file, bool flush)
             if (filter_mask != 0)
                 goto error;
 
-            /* Perform decompression from the source to the destination buffer */
+                /* Perform decompression from the source to the destination buffer */
+#if defined(H5_HAVE_ZLIBNG_H)
+            ret = zng_uncompress(z_dst, &z_dst_nbytes, z_src, z_src_nbytes);
+#else
             ret = uncompress(z_dst, &z_dst_nbytes, z_src, z_src_nbytes);
+#endif
 
             /* Check for various zlib errors */
             if (Z_BUF_ERROR == ret) {

tools/test/perform/CMakeLists.txt

@@ -112,6 +112,12 @@ endif ()
 set (zip_perf_SOURCES
     ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/zip_perf.c
 )
+if (H5_ZLIB_HEADER)
+  message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}")
+  if (HDF5_USE_ZLIB_NG)
+    add_compile_definitions(H5_HAVE_ZLIBNG_H=1 H5_ZLIB_HEADER="${H5_ZLIB_HEADER}")
+  endif ()
+endif ()
 add_executable (zip_perf ${zip_perf_SOURCES})
 target_include_directories (zip_perf PRIVATE "${HDF5_TEST_SRC_DIR};${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>")
 if (NOT BUILD_SHARED_LIBS)

tools/test/perform/direct_write_perf.c

@@ -18,7 +18,12 @@
 #include "hdf5.h"
 
 #ifdef H5_HAVE_FILTER_DEFLATE
-#include <zlib.h>
+#if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER)
+#define H5_ZLIB_HEADER "zlib.h"
+#endif
+#if defined(H5_ZLIB_HEADER)
+#include H5_ZLIB_HEADER /* "zlib.h" */
+#endif
 
 #if !defined(WIN32) && !defined(__MINGW32__)
 
@@ -242,7 +247,11 @@ create_file(hid_t fapl_id)
         z_dst     = (Bytef *)outbuf[i];
 
         /* Perform compression from the source to the destination buffer */
+#if defined(H5_HAVE_ZLIBNG_H)
+        ret = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#else
         ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#endif
 
         data_size[i] = (size_t)z_dst_nbytes;
         total_size += data_size[i];

tools/test/perform/zip_perf.c

@@ -24,8 +24,12 @@
 #include "h5tools_utils.h"
 
 #ifdef H5_HAVE_FILTER_DEFLATE
-
-#include <zlib.h>
+#if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER)
+#define H5_ZLIB_HEADER "zlib.h"
+#endif
+#if defined(H5_ZLIB_HEADER)
+#include H5_ZLIB_HEADER /* "zlib.h" */
+#endif
 
 #define ONE_KB 1024
 #define ONE_MB (ONE_KB * ONE_KB)
@@ -163,7 +167,11 @@ write_file(Bytef *source, uLongf sourceLen)
 static void
 compress_buffer(Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen)
 {
+#if defined(H5_HAVE_ZLIBNG_H)
+    int rc = zng_compress2(dest, destLen, source, sourceLen, compress_level);
+#else
     int rc = compress2(dest, destLen, source, sourceLen, compress_level);
+#endif
 
     if (rc != Z_OK) {
         /* compress2 failed - cleanup and tell why */