From be0b38a4130d202582149c9739e20131b0d9847d Mon Sep 17 00:00:00 2001 From: crocdialer Date: Thu, 24 Oct 2024 13:39:18 +0200 Subject: [PATCH] small hashmap excourse, cough up non-atomic flavour --- include/vierkant/hash.hpp | 10 +- include/vierkant/linear_hashmap.hpp | 196 +++++++++++++++++++++++++++- src/PBRDeferred.cpp | 1 + src/object_overlay.cpp | 2 +- tests/TestLinearHashmap.cpp | 86 ++++++++---- 5 files changed, 261 insertions(+), 34 deletions(-) diff --git a/include/vierkant/hash.hpp b/include/vierkant/hash.hpp index 0032eae6..9c4d8f92 100644 --- a/include/vierkant/hash.hpp +++ b/include/vierkant/hash.hpp @@ -4,9 +4,9 @@ #pragma once -#include -#include #include +#include +#include namespace vierkant { @@ -90,11 +90,11 @@ static inline uint32_t murmur3_32(const K &key, uint32_t seed) if constexpr(num_hashes) { - auto ptr = reinterpret_cast(&key); + auto ptr = reinterpret_cast(&key), end = ptr + num_hashes; - for(uint32_t i = num_hashes; i; i--) + for(; ptr < end; ++ptr) { - h ^= murmur_32_scramble(ptr[i - 1]); + h ^= murmur_32_scramble(*ptr); h = (h << 13) | (h >> 19); h = h * 5 + 0xe6546b64; } diff --git a/include/vierkant/linear_hashmap.hpp b/include/vierkant/linear_hashmap.hpp index 040e11ad..4453f10f 100644 --- a/include/vierkant/linear_hashmap.hpp +++ b/include/vierkant/linear_hashmap.hpp @@ -41,6 +41,198 @@ class linear_hashmap clear(); } + [[nodiscard]] inline size_t size() const { return m_num_elements; } + + [[nodiscard]] inline size_t capacity() const { return m_capacity; } + + [[nodiscard]] inline bool empty() const { return size() == 0; } + + inline void clear() + { + m_num_elements = 0; + storage_item_t *ptr = m_storage.get(), *end = ptr + m_capacity; + for(; ptr != end; ++ptr) + { + ptr->key = key_t(); + ptr->value = std::optional(); + } + } + + inline uint32_t put(const key_t &key, const value_t &value) + { + check_load_factor(); + return internal_put(key, value); + } + + [[nodiscard]] std::optional get(const key_t &key) const + { + if(!m_capacity) { return {}; } + + for(uint32_t idx = m_hash_fn(key);; idx++) + { + idx &= m_capacity - 1; + auto &item = m_storage[idx]; + if(item.key == key_t()) { return {}; } + else if(key == item.key) + { + if(item.value) { return item.value; } + } + } + } + + void remove(const key_t &key) + { + if(!m_capacity) { return; } + + for(uint32_t idx = m_hash_fn(key);; idx++) + { + idx &= m_capacity - 1; + auto &item = m_storage[idx]; + if(item.key == key_t()) { return; } + else if(key == item.key && item.value) + { + item.value = {}; + m_num_elements--; + return; + } + } + } + + [[nodiscard]] inline bool contains(const key_t &key) const { return get(key) != std::nullopt; } + + size_t get_storage(void *dst) const + { + struct output_item_t + { + key_t key = {}; + value_t value = {}; + }; + + if(dst) + { + auto output_ptr = reinterpret_cast(dst); + storage_item_t *item = m_storage.get(), *end = item + m_capacity; + for(; item != end; ++item, ++output_ptr) + { + if(item->key != key_t()) + { + output_ptr->key = item->key; + output_ptr->value = item->value ? *item->value : value_t(); + } + else { *output_ptr = {}; } + } + } + return sizeof(output_item_t) * m_capacity; + } + + void reserve(size_t new_capacity) + { + auto new_linear_hashmap = linear_hashmap(new_capacity); + storage_item_t *ptr = m_storage.get(), *end = ptr + m_capacity; + for(; ptr != end; ++ptr) + { + if(ptr->key != key_t()) + { + if(ptr->value) { new_linear_hashmap.put(ptr->key, *ptr->value); } + } + } + swap(*this, new_linear_hashmap); + } + + [[nodiscard]] float load_factor() const { return static_cast(m_num_elements) / m_capacity; } + + [[nodiscard]] float max_load_factor() const { return m_max_load_factor; } + + void max_load_factor(float load_factor) + { + m_max_load_factor = std::clamp(load_factor, 0.01f, 1.f); + check_load_factor(); + } + + friend void swap(linear_hashmap &lhs, linear_hashmap &rhs) + { + std::swap(lhs.m_capacity, rhs.m_capacity); + std::swap(lhs.m_num_elements, rhs.m_num_elements); + std::swap(lhs.m_storage, rhs.m_storage); + std::swap(lhs.m_hash_fn, rhs.m_hash_fn); + std::swap(lhs.m_max_load_factor, rhs.m_max_load_factor); + std::swap(lhs.m_grow_factor, rhs.m_grow_factor); + } + +private: + struct storage_item_t + { + key_t key; + std::optional value; + }; + + inline void check_load_factor() + { + if(m_num_elements >= m_capacity * m_max_load_factor) + { + reserve(std::max(32, static_cast(m_grow_factor * m_capacity))); + } + } + + inline uint32_t internal_put(const key_t key, const value_t &value) + { + uint32_t probe_length = 0; + + for(uint64_t idx = m_hash_fn(key);; idx++, probe_length++) + { + idx &= m_capacity - 1; + auto &item = m_storage[idx]; + + // load previous key + key_t probed_key = item.key; + + if(probed_key != key) + { + // hit another valid entry, keep probing + if(probed_key != key_t() && item.value) { continue; } + item.key = key; + m_num_elements++; + } + item.value = value; + return probe_length; + } + } + + uint64_t m_capacity = 0; + uint64_t m_num_elements = 0; + std::unique_ptr m_storage; + hash32_fn m_hash_fn = std::bind(murmur3_32, std::placeholders::_1, 0); + + // reasonably low load-factor to keep average probe-lengths low + float m_max_load_factor = 0.5f; + float m_grow_factor = 2.f; +}; + +template +class linear_hashmap_mt +{ +public: + using key_t = K; + using value_t = V; + using hash32_fn = std::function; + static_assert(std::is_default_constructible_v, "key_t not default-constructible"); + static_assert(std::equality_comparable, "key_t not comparable"); + + linear_hashmap_mt() = default; + linear_hashmap_mt(const linear_hashmap_mt &) = delete; + linear_hashmap_mt(linear_hashmap_mt &other) : linear_hashmap_mt() { swap(*this, other); }; + linear_hashmap_mt &operator=(linear_hashmap_mt other) + { + swap(*this, other); + return *this; + } + + explicit linear_hashmap_mt(uint64_t min_capacity) + : m_capacity(crocore::next_pow_2(min_capacity)), m_storage(std::make_unique(m_capacity)) + { + clear(); + } + inline size_t size() const { return m_num_elements; } inline size_t capacity() const { return m_capacity; } @@ -133,7 +325,7 @@ class linear_hashmap void reserve(size_t new_capacity) { - auto new_linear_hashmap = linear_hashmap(new_capacity); + auto new_linear_hashmap = linear_hashmap_mt(new_capacity); storage_item_t *ptr = m_storage.get(), *end = ptr + m_capacity; for(; ptr != end; ++ptr) { @@ -155,7 +347,7 @@ class linear_hashmap check_load_factor(); } - friend void swap(linear_hashmap &lhs, linear_hashmap &rhs) + friend void swap(linear_hashmap_mt &lhs, linear_hashmap_mt &rhs) { std::lock(lhs.m_mutex, rhs.m_mutex); std::unique_lock lock_lhs(lhs.m_mutex, std::adopt_lock), lock_rhs(rhs.m_mutex, std::adopt_lock); diff --git a/src/PBRDeferred.cpp b/src/PBRDeferred.cpp index aa3e1d00..4ee9d8a9 100644 --- a/src/PBRDeferred.cpp +++ b/src/PBRDeferred.cpp @@ -879,6 +879,7 @@ vierkant::Framebuffer &PBRDeferred::geometry_pass(cull_result_t &cull_result) g_buffer_semaphore_submit_info_pre.signal_value = frame_context.current_semaphore_value + (use_gpu_culling ? SemaphoreValue::G_BUFFER_LAST_VISIBLE : SemaphoreValue::G_BUFFER_ALL); + g_buffer_semaphore_submit_info_pre.signal_stage = VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT; frame_context.g_buffer_main.submit({cmd_buffer_pre}, m_queue, {g_buffer_semaphore_submit_info_pre}); if(use_gpu_culling) diff --git a/src/object_overlay.cpp b/src/object_overlay.cpp index 628f2bff..27d973b7 100644 --- a/src/object_overlay.cpp +++ b/src/object_overlay.cpp @@ -9,7 +9,7 @@ namespace vierkant struct object_overlay_context_t { - vierkant::linear_hashmap id_map; + vierkant::linear_hashmap_mt id_map; vierkant::BufferPtr id_map_storage_buffer; vierkant::BufferPtr param_buffer; vierkant::BufferPtr staging_buffer; diff --git a/tests/TestLinearHashmap.cpp b/tests/TestLinearHashmap.cpp index 605b9bfe..4465b440 100644 --- a/tests/TestLinearHashmap.cpp +++ b/tests/TestLinearHashmap.cpp @@ -1,20 +1,21 @@ #include #include - -TEST(linear_hashmap, empty) +template class hashmap_t> +void test_empty() { - vierkant::linear_hashmap hashmap; + hashmap_t hashmap; EXPECT_TRUE(hashmap.empty()); hashmap.clear(); EXPECT_EQ(hashmap.capacity(), 0); EXPECT_EQ(hashmap.get_storage(nullptr), 0); -} +}; -TEST(linear_hashmap, basic) +template class hashmap_t> +void test_basic() { constexpr uint32_t test_capacity = 100; - vierkant::linear_hashmap hashmap(test_capacity); + hashmap_t hashmap(test_capacity); EXPECT_TRUE(hashmap.empty()); EXPECT_GT(hashmap.get_storage(nullptr), 0); @@ -44,7 +45,8 @@ TEST(linear_hashmap, basic) hashmap.get_storage(storage.get()); } -TEST(linear_hashmap, custom_key) +template class hashmap_t> +void test_custom_key() { // custom 32-byte key struct custom_key_t @@ -60,7 +62,7 @@ TEST(linear_hashmap, custom_key) } }; constexpr uint32_t test_capacity = 100; - auto hashmap = vierkant::linear_hashmap(test_capacity); + auto hashmap = hashmap_t(test_capacity); custom_key_t k1{{1, 2, 3, 4, 5, 6, 7, 8}}; hashmap.put(k1, 69); @@ -68,25 +70,10 @@ TEST(linear_hashmap, custom_key) EXPECT_FALSE(hashmap.contains(custom_key_t())); } -TEST(linear_hashmap, reserve) -{ - vierkant::linear_hashmap hashmap; - - // fix by resizing - hashmap.reserve(17); - EXPECT_TRUE(hashmap.empty()); - hashmap.put(13, 12); - EXPECT_TRUE(hashmap.contains(13)); - - // empty / no capacity specified -> triggers internal resize - hashmap = {}; - hashmap.put(13, 12); - EXPECT_TRUE(hashmap.contains(13)); -} - -TEST(linear_hashmap, probe_length) +template class hashmap_t> +void test_probe_length() { - vierkant::linear_hashmap hashmap; + hashmap_t hashmap; // default load_factor is 0.5 EXPECT_EQ(hashmap.max_load_factor(), 0.5f); @@ -107,4 +94,51 @@ TEST(linear_hashmap, probe_length) EXPECT_LE(avg_probe_length, expected_max_avg_probe_length); EXPECT_LE(hashmap.load_factor(), 0.25f); +} + +TEST(linear_hashmap, empty) +{ + test_empty(); + test_empty(); +} + +TEST(linear_hashmap, basic) +{ + test_basic(); + test_basic(); +} + +TEST(linear_hashmap, custom_key) +{ + test_custom_key(); + test_custom_key(); +} + +template class hashmap_t> +void test_reserve() +{ + hashmap_t hashmap; + + // fix by resizing + hashmap.reserve(17); + EXPECT_TRUE(hashmap.empty()); + hashmap.put(13, 12); + EXPECT_TRUE(hashmap.contains(13)); + + // empty / no capacity specified -> triggers internal resize + hashmap = {}; + hashmap.put(13, 12); + EXPECT_TRUE(hashmap.contains(13)); +} + +TEST(linear_hashmap, reserve) +{ + test_reserve(); + test_reserve(); +} + +TEST(linear_hashmap, probe_length) +{ + test_probe_length(); + test_probe_length(); } \ No newline at end of file