Some fixes to support fmt v10.

.jenkins/Jenkinsfile

@@ -137,7 +137,7 @@ pipeline {
                                     BUILD_PROFILE = "test"
                                 }
 
-                                sh "conan create -u ${BUILD_MISSING} -o sisl:prerelease=${PRERELEASE} -o ${PROJECT}:sanitize=${SANITIZE} ${BUILD_TEST_MODE} -pr ${BUILD_PROFILE} . ${PROJECT}/${TAG}"
+                                sh "conan create ${BUILD_MISSING} -o iomgr:testing=off -o sisl:prerelease=${PRERELEASE} -o ${PROJECT}:sanitize=${SANITIZE} ${BUILD_TEST_MODE} -pr ${BUILD_PROFILE} . ${PROJECT}/${TAG}"
                                 sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_load' -exec cp {} .jenkins/test_load \\;"
                                 sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_volume' -exec cp {} .jenkins/test_volume \\;"
                                 sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/check_btree' -exec cp {} .jenkins/check_btree \\;"

conanfile.py

@@ -53,14 +53,13 @@ def imports(self):
         self.copy(root_package="sisl", pattern="*", dst="bin/scripts/python/flip/", src="bindings/flip/python/", keep_path=False)
 
     def build_requirements(self):
-        self.build_requires("benchmark/1.7.1")
-        self.build_requires("gtest/1.13.0")
+        self.build_requires("benchmark/1.8.2")
+        self.build_requires("gtest/1.14.0")
 
     def requirements(self):
         self.requires("iomgr/[~=9, include_prerelease=True]@oss/master")
         self.requires("sisl/[~=10, include_prerelease=True]@oss/master")
 
-        # FOSS, rarely updated
         self.requires("farmhash/cci.20190513@")
         self.requires("isa-l/2.30.0")
         self.requires("spdk/21.07.y")

src/include/homestore/blk.h

@@ -13,8 +13,7 @@
  * specific language governing permissions and limitations under the License.
  *
  *********************************************************************************/
-#ifndef SRC_BLKALLOC_BLK_H_
-#define SRC_BLKALLOC_BLK_H_
+#pragma once
 
 #include <cassert>
 #include <cstdint>
@@ -128,6 +127,29 @@ static_assert(sizeof(BlkId8_t) == 8);
 inline blk_num_t begin_of(const BlkId& blkid) { return blkid.get_blk_num(); }
 inline blk_num_t end_of(const BlkId& blkid) { return blkid.get_blk_num() + blkid.get_nblks(); }
 inline size_t hash_value(const BlkId& blkid) { return std::hash< uint64_t >()(blkid.to_integer()); }
+} // namespace homestore
+
+// hash function definitions
+namespace std {
+template <>
+struct hash< homestore::BlkId > {
+    typedef homestore::BlkId argument_type;
+    typedef size_t result_type;
+    result_type operator()(const argument_type& bid) const noexcept {
+        return std::hash< uint64_t >()(bid.to_integer());
+    }
+};
+} // namespace std
+
+template < typename T >
+struct fmt::formatter< T, std::enable_if_t< std::is_base_of< homestore::BlkId, T >::value, char > >
+        : fmt::formatter< std::string > {
+    auto format(const homestore::BlkId& a, format_context& ctx) const {
+        return fmt::formatter< std::string >::format(a.to_string(), ctx);
+    }
+};
+
+namespace homestore {
 
 template < typename charT, typename traits >
 std::basic_ostream< charT, traits >& operator<<(std::basic_ostream< charT, traits >& outStream, const BlkId& blk) {
@@ -166,32 +188,3 @@ struct blk_alloc_hints {
 };
 
 } // namespace homestore
-
-// hash function definitions
-namespace std {
-template <>
-struct hash< homestore::BlkId > {
-    typedef homestore::BlkId argument_type;
-    typedef size_t result_type;
-    result_type operator()(const argument_type& bid) const noexcept {
-        return std::hash< uint64_t >()(bid.to_integer());
-    }
-};
-} // namespace std
-
-namespace fmt {
-template <>
-struct formatter< homestore::BlkId > {
-    template < typename ParseContext >
-    constexpr auto parse(ParseContext& ctx) {
-        return ctx.begin();
-    }
-
-    template < typename FormatContext >
-    auto format(const homestore::BlkId& s, FormatContext& ctx) {
-        return format_to(ctx.out(), s.to_string());
-    }
-};
-
-} // namespace fmt
-#endif /* SRC_BLKALLOC_BLK_H_ */

src/lib/blkalloc/varsize_blk_allocator.cpp

@@ -30,6 +30,14 @@
 
 SISL_LOGGING_DECL(blkalloc)
 
+template <>
+struct fmt::formatter< std::thread::id > {
+    constexpr auto parse(format_parse_context& ctx) -> format_parse_context::iterator { return ctx.begin(); }
+    auto format(const std::thread::id& i, format_context& ctx) const -> format_context::iterator {
+        return fmt::format_to(ctx.out(), "{}", std::hash< std::thread::id >{}(i));
+    }
+};
+
 namespace homestore {
 
 // initialize static variables

src/lib/common/resource_mgr.cpp

@@ -135,8 +135,8 @@ uint32_t ResourceMgr::get_dirty_buf_qd() const { return m_flush_dirty_buf_q_dept
 void ResourceMgr::increase_dirty_buf_qd() {
     auto qd = m_flush_dirty_buf_q_depth.load();
     if (qd < max_qd_multiplier * HS_DYNAMIC_CONFIG(generic.cache_max_throttle_cnt)) {
-        m_flush_dirty_buf_q_depth.fetch_add(2 * qd);
-        HS_PERIODIC_LOG(INFO, base, "q depth increased to {}", m_flush_dirty_buf_q_depth);
+        auto const nd = m_flush_dirty_buf_q_depth.fetch_add(2 * qd) + (2 * qd);
+        HS_PERIODIC_LOG(INFO, base, "q depth increased to {}", nd);
     }
 }
 

src/lib/logstore/log_dev.cpp

@@ -163,7 +163,7 @@ void LogDev::do_load(const off_t device_cursor) {
             break;
         }
 
-        HS_REL_ASSERT_EQ(header->start_idx(), m_log_idx, "log indx is not the expected one");
+        HS_REL_ASSERT_EQ(header->start_idx(), m_log_idx.load(), "log indx is not the expected one");
         if (loaded_from == -1) { loaded_from = header->start_idx(); }
 
         // Loop through each record within the log group and do a callback

src/lib/logstore/log_dev.hpp

@@ -187,7 +187,22 @@ struct log_group_footer {
     uint8_t padding[12];
 };
 #pragma pack()
+} // namespace homestore
 
+template <>
+struct fmt::formatter< homestore::log_group_header > {
+    constexpr auto parse(format_parse_context& ctx) -> format_parse_context::iterator { return ctx.begin(); }
+    auto format(const homestore::log_group_header& header, format_context& ctx) const -> format_context::iterator {
+        return fmt::format_to(
+            ctx.out(),
+            "magic = {} version={} n_log_records = {} start_log_idx = {} group_size = {} inline_data_offset = {} "
+            "oob_data_offset = {} prev_grp_crc = {} cur_grp_crc = {}",
+            header.magic, header.version, header.n_log_records, header.start_log_idx, header.group_size,
+            header.inline_data_offset, header.oob_data_offset, header.prev_grp_crc, header.cur_grp_crc);
+    }
+};
+
+namespace homestore {
 template < typename charT, typename traits >
 std::basic_ostream< charT, traits >& operator<<(std::basic_ostream< charT, traits >& out_stream,
                                                 const log_group_header& header) {
@@ -196,12 +211,7 @@ std::basic_ostream< charT, traits >& operator<<(std::basic_ostream< charT, trait
     out_string_stream.copyfmt(out_stream);
 
     // print the stream
-    const auto s{fmt::format(
-        "magic = {} version={} n_log_records = {} start_log_idx = {} group_size = {} inline_data_offset = {} "
-        "oob_data_offset = {} prev_grp_crc = {} cur_grp_crc = {}",
-        header.magic, header.version, header.n_log_records, header.start_log_idx, header.group_size,
-        header.inline_data_offset, header.oob_data_offset, header.prev_grp_crc, header.cur_grp_crc)};
-    out_string_stream << s;
+    out_string_stream << fmt::format("{}", header);
     out_stream << out_string_stream.str();
 
     return out_stream;
@@ -290,20 +300,30 @@ class LogGroup {
     log_group_footer* add_and_get_footer();
     bool new_iovec_for_footer() const;
 };
+} // namespace homestore
+
+template <>
+struct fmt::formatter< homestore::LogGroup > {
+    constexpr auto parse(format_parse_context& ctx) -> format_parse_context::iterator { return ctx.begin(); }
+    auto format(const homestore::LogGroup& lg, format_context& ctx) const -> format_context::iterator {
+        const auto* const header{lg.header()};
+        return fmt::format_to(ctx.out(),
+                              "Header:[{}]\nLog_idx_range: [{} - {}] DevOffset: {} Max_Records: {} IOVecSize: {}\n"
+                              "-----------------------------------------------------------------\n",
+                              *header, lg.flush_log_idx_from(), lg.flush_log_idx_upto(), lg.log_dev_offset(),
+                              lg.max_records(), lg.iovecs().size());
+    }
+};
 
+namespace homestore {
 template < typename charT, typename traits >
 std::basic_ostream< charT, traits >& operator<<(std::basic_ostream< charT, traits >& out_stream, const LogGroup& lg) {
     // copy the stream formatting
     std::basic_ostringstream< charT, traits > out_string_stream;
     out_string_stream.copyfmt(out_stream);
 
     // print the stream
-    const auto* const header{lg.header()};
-    const auto s{fmt::format("Header:[{}]\nLog_idx_range: [{} - {}] DevOffset: {} Max_Records: {} IOVecSize: {}\n"
-                             "-----------------------------------------------------------------\n",
-                             *header, lg.flush_log_idx_from(), lg.flush_log_idx_upto(), lg.log_dev_offset(),
-                             lg.max_records(), lg.iovecs().size())};
-    out_string_stream << s;
+    out_string_stream << fmt::format("{}", lg);
     out_stream << out_string_stream.str();
 
     return out_stream;
@@ -468,6 +488,7 @@ struct rollback_superblk {
 // This class represents the metadata of logdev providing methods to change/access log dev super block.
 class LogDevMetadata {
     friend class LogDev;
+
 public:
     LogDevMetadata(const std::string& logdev_name);
     LogDevMetadata(const LogDevMetadata&) = delete;
@@ -557,6 +578,7 @@ struct meta_blk;
 
 class LogDev {
     friend class HomeLogStore;
+
 public:
     // NOTE: Possibly change these in future to include constant correctness
     typedef std::function< void(logstore_id_t, logdev_key, logdev_key, uint32_t nremaining_in_batch, void*) >

src/lib/logstore/log_store_family.cpp

@@ -240,7 +240,7 @@ logdev_key LogStoreFamily::do_device_truncate(bool dry_run) {
             }
 
             fmt::format_to(std::back_inserter(dbg_str), "[{}:{}:{}:{}:{}] ", store_ptr->get_store_id(),
-                           trunc_info.seq_num, trunc_info.ld_key.idx, trunc_info.pending_dev_truncation,
+                           trunc_info.seq_num.load(), trunc_info.ld_key.idx, trunc_info.pending_dev_truncation,
                            trunc_info.active_writes_not_part_of_truncation);
             if (trunc_info.ld_key.idx > min_safe_ld_key.idx) { continue; }
 

src/lib/meta/meta_blk_service.cpp

@@ -29,6 +29,7 @@
 
 #include <homestore/meta_service.hpp>
 #include <homestore/homestore.hpp>
+#include "common/homestore_flip.hpp"
 #include "device/virtual_dev.hpp"
 #include "device/physical_dev.hpp"
 #include "blkalloc/blk_allocator.h"
@@ -1223,7 +1224,7 @@ void MetaBlkService::alloc_compress_buf(size_t size) {
     m_compress_info.size = size;
     m_compress_info.bytes = hs_utils::iobuf_alloc(size, sisl::buftag::compression, align_size());
 
-    HS_REL_ASSERT_NE(m_compress_info.bytes, nullptr, "fail to allocate iobuf for compression of size: {}", size);
+    HS_REL_ASSERT_NOTNULL(m_compress_info.bytes, "fail to allocate iobuf for compression of size: {}", size);
 }
 
 uint64_t MetaBlkService::meta_blk_context_sz() const { return block_size() - META_BLK_HDR_MAX_SZ; }

src/tests/CMakeLists.txt

@@ -41,6 +41,10 @@ if (${build_nonio_tests})
     target_link_libraries(test_blk_read_tracker ${COMMON_TEST_DEPS} GTest::gtest)
     add_test(NAME BlkReadTracker COMMAND test_blk_read_tracker)
 
+    add_executable(test_cp_mgr)
+    target_sources(test_cp_mgr PRIVATE test_cp_mgr.cpp)
+    target_link_libraries(test_cp_mgr homestore ${COMMON_TEST_DEPS} GTest::gtest)
+    add_test(NAME CPMgr COMMAND test_cp_mgr)
 
 endif()
 
@@ -64,11 +68,6 @@ if (${io_tests})
     add_executable(test_meta_blk_mgr ${TEST_METABLK_SOURCE_FILES})
     target_link_libraries(test_meta_blk_mgr homestore ${COMMON_TEST_DEPS} GTest::gmock)
 
-    add_executable(test_cp_mgr)
-    target_sources(test_cp_mgr PRIVATE test_cp_mgr.cpp)
-    target_link_libraries(test_cp_mgr homestore ${COMMON_TEST_DEPS} GTest::gtest)
-    add_test(NAME CPMgr COMMAND test_cp_mgr)
-
     can_build_epoll_io_tests(epoll_tests)
     if(${epoll_tests})
         add_test(NAME LogStore-Epoll COMMAND ${CMAKE_SOURCE_DIR}/test_wrap.sh ${CMAKE_BINARY_DIR}/bin/test_log_store)

src/tests/test_blkalloc.cpp

@@ -557,7 +557,8 @@ struct VarsizeBlkAllocatorTest : public ::testing::Test, BlkAllocatorTest {
             total_dealloc += freed_nblks;
             return !terminate_flag;
         });
-        LOGINFO("Total Alloced {} random blks and freed {} random blks in all thread", total_alloc, total_dealloc);
+        LOGINFO("Total Alloced {} random blks and freed {} random blks in all thread", total_alloc.load(),
+                total_dealloc.load());
         return {total_alloc, total_dealloc};
     }
 };

src/tests/test_common/homestore_test_common.hpp

@@ -21,8 +21,10 @@
 #pragma once
 #include <sisl/logging/logging.h>
 #include <sisl/options/options.h>
+#include <sisl/settings/settings.hpp>
 #include <homestore/homestore.hpp>
 #include <homestore/index_service.hpp>
+#include <iomgr/iomgr_config_generated.h>
 
 const std::string SPDK_ENV_VAR_STRING{"USER_WANT_SPDK"};
 const std::string HTTP_SVC_ENV_VAR_STRING{"USER_WANT_HTTP_OFF"};
@@ -44,14 +46,16 @@ SISL_OPTION_GROUP(test_common_setup,
                    ::cxxopts::value< int >()->default_value("-1"), "number"),
                   (spdk, "", "spdk", "spdk", ::cxxopts::value< bool >()->default_value("false"), "true or false"));
 
+SETTINGS_INIT(iomgrcfg::IomgrSettings, iomgr_config);
+
 using namespace homestore;
 
 namespace test_common {
 
 // Fix a port for http server
-inline static void set_fixed_http_port(uint32_t http_port){
-    IM_SETTINGS_FACTORY().modifiable_settings([http_port](auto& s) { s.io_env->http_port = http_port; });
-    IM_SETTINGS_FACTORY().save();
+inline static void set_fixed_http_port(uint32_t http_port) {
+    SETTINGS_FACTORY(iomgr_config).modifiable_settings([http_port](auto& s) { s.io_env->http_port = http_port; });
+    SETTINGS_FACTORY(iomgr_config).save();
     LOGINFO("http port = {}", http_port);
 }
 
@@ -138,7 +142,8 @@ class HSTestHelper {
 
         auto const http_port = SISL_OPTIONS["http_port"].as< int >();
         if (http_port != 0) {
-            ioenvironment.with_http_server((http_port == -1) ? generate_random_http_port() : uint32_cast(http_port));
+            set_fixed_http_port((http_port == -1) ? generate_random_http_port() : uint32_cast(http_port));
+            ioenvironment.with_http_server();
         }
 
         if (!index_svc_cb) { index_svc_cb = std::make_unique< TestIndexServiceCallbacks >(); }

src/tests/test_data_service.cpp

@@ -161,7 +161,7 @@ class BlkDataServiceTest : public testing::Test {
                         // trigger the free_blk cb firstly then send read complete cb back to caller;
                         m_read_blk_done = true;
                         LOGINFO("read completed;");
-                        HS_DBG_ASSERT_EQ(m_free_blk_done, true,
+                        HS_DBG_ASSERT_EQ(m_free_blk_done.load(), true,
                                          "free blk callback should not be called before read blk completes");
 
                         free_sg_buf(*sg_read_ptr);
@@ -171,7 +171,7 @@ class BlkDataServiceTest : public testing::Test {
                 LOGINFO("Step 3: started async_free_blk: {}", test_blkid_ptr->to_string());
                 inst().async_free_blk(*test_blkid_ptr).thenValue([this](auto) {
                     LOGINFO("completed async_free_blk");
-                    HS_DBG_ASSERT_EQ(m_free_blk_done, false, "Duplicate free blk completion");
+                    HS_DBG_ASSERT_EQ(m_free_blk_done.load(), false, "Duplicate free blk completion");
                     m_free_blk_done = true;
                 });
             });

src/tests/test_log_store.cpp

@@ -1266,9 +1266,7 @@ SISL_OPTION_GROUP(test_log_store,
                   (num_records, "", "num_records", "number of record to test",
                    ::cxxopts::value< uint32_t >()->default_value("10000"), "number"),
                   (iterations, "", "iterations", "Iterations", ::cxxopts::value< uint32_t >()->default_value("1"),
-                   "the number of iterations to run each test"),
-                  (http_port, "", "http_port", "http_port", ::cxxopts::value< uint32_t >()->default_value("5000"),
-                   "http server port"));
+                   "the number of iterations to run each test"));
 
 int main(int argc, char* argv[]) {
     int parsed_argc = argc;

src/tests/test_mem_btree.cpp

@@ -28,7 +28,7 @@
 
 static constexpr uint32_t g_node_size{4096};
 using namespace homestore;
-SISL_LOGGING_INIT(btree, iomgr, io_wd)
+SISL_LOGGING_INIT(btree, iomgr, io_wd, flip)
 
 SISL_OPTIONS_ENABLE(logging, test_mem_btree)
 SISL_OPTION_GROUP(test_mem_btree,
@@ -288,11 +288,10 @@ struct BtreeTest : public testing::Test {
     }
 
     btree_status_t get_num_elements_in_tree(uint32_t start_k, uint32_t end_k, K& out_key, V& out_value) const {
-        auto req = BtreeGetAnyRequest< K >{BtreeKeyRange< K >{K{start_k}, true, K{end_k}, true},
-                                           std::make_unique< K >(), std::make_unique< V >()};
+        auto req = BtreeGetAnyRequest< K >{BtreeKeyRange< K >{K{start_k}, true, K{end_k}, true}, new K(), new V()};
         auto ret = m_bt->get(req);
-        out_key = *((K*)req.m_outkey.get());
-        out_value = *((V*)req.m_outval.get());
+        out_key = *((K*)req.m_outkey);
+        out_value = *((V*)req.m_outval);
         return ret;
     }