diff --git a/include/customizer/cell_customizer.hpp b/include/customizer/cell_customizer.hpp index 53570c51d..0fd6dcbc1 100644 --- a/include/customizer/cell_customizer.hpp +++ b/include/customizer/cell_customizer.hpp @@ -117,9 +117,7 @@ class CellCustomizer CellMetric &metric) const { const auto number_of_nodes = graph.GetNumberOfNodes(); - HeapPtr heaps([number_of_nodes]{ - return Heap{number_of_nodes}; - }); + HeapPtr heaps([number_of_nodes] { return Heap{number_of_nodes}; }); for (std::size_t level = 1; level < partition.GetNumberOfLevels(); ++level) { diff --git a/include/util/pool_allocator.hpp b/include/util/pool_allocator.hpp index 76bd27202..edd658f2b 100644 --- a/include/util/pool_allocator.hpp +++ b/include/util/pool_allocator.hpp @@ -6,21 +6,31 @@ #include #include #include -#include +#include #include #include #include -#include namespace osrm::util { +inline size_t align_up(size_t n, size_t alignment) +{ + return (n + alignment - 1) & ~(alignment - 1); +} + +inline size_t get_next_power_of_two_exponent(size_t n) +{ + BOOST_ASSERT(n > 0); + return (sizeof(size_t) * 8) - std::countl_zero(n - 1); +} + class MemoryPool { -private: + private: constexpr static size_t MIN_CHUNK_SIZE_BYTES = 4096; -public: + public: static std::shared_ptr instance() { static thread_local std::shared_ptr instance; @@ -31,10 +41,10 @@ public: return instance; } - template - T *allocate(std::size_t items_count) + template T *allocate(std::size_t items_count) { - static_assert(alignof(T) <= alignof(std::max_align_t), "Type is over-aligned for this allocator."); + static_assert(alignof(T) <= alignof(std::max_align_t), + "Type is over-aligned for this allocator."); size_t free_list_index = get_next_power_of_two_exponent(items_count * sizeof(T)); auto &free_list = free_lists_[free_list_index]; @@ -42,7 +52,7 @@ public: { size_t block_size_in_bytes = 1u << free_list_index; block_size_in_bytes = align_up(block_size_in_bytes, alignof(std::max_align_t)); - // Check if there is space in current memory chunk + // check if there is space in current memory chunk if (current_chunk_left_bytes_ < block_size_in_bytes) { allocate_chunk(block_size_in_bytes); @@ -52,13 +62,12 @@ public: current_chunk_left_bytes_ -= block_size_in_bytes; current_chunk_ptr_ += block_size_in_bytes; } - auto ptr = static_cast(free_list.back()); + auto ptr = static_cast(free_list.back()); free_list.pop_back(); return ptr; } - template - void deallocate(T *p, std::size_t n) noexcept + template void deallocate(T *p, std::size_t n) noexcept { size_t free_list_index = get_next_power_of_two_exponent(n * sizeof(T)); free_lists_[free_list_index].push_back(p); @@ -73,27 +82,11 @@ public: } } -private: + private: MemoryPool() = default; MemoryPool(const MemoryPool &) = delete; MemoryPool &operator=(const MemoryPool &) = delete; - inline size_t get_next_power_of_two_exponent(size_t n) const - { - BOOST_ASSERT(n > 0); - return (sizeof(size_t) * 8) - std::countl_zero(n - 1); - } - - inline size_t align_up(size_t n, size_t alignment) - { - return (n + alignment - 1) & ~(alignment - 1); - } - - inline void* align_pointer(void* ptr, size_t alignment) - { - return reinterpret_cast(align_up(reinterpret_cast(ptr), alignment)); - } - void allocate_chunk(size_t bytes) { auto chunk_size = std::max(bytes, MIN_CHUNK_SIZE_BYTES); @@ -104,49 +97,47 @@ private: throw std::bad_alloc(); } chunks_.push_back(chunk); - current_chunk_ptr_ = static_cast(chunk); + current_chunk_ptr_ = static_cast(chunk); current_chunk_left_bytes_ = chunk_size; } - std::array, 32> free_lists_; + // we have 64 free lists, one for each possible power of two + std::array, sizeof(std::size_t) * 8> free_lists_; + + // list of allocated memory chunks, we don't free them until the pool is destroyed std::vector chunks_; + uint8_t *current_chunk_ptr_ = nullptr; size_t current_chunk_left_bytes_ = 0; }; -template -class PoolAllocator +template class PoolAllocator { -public: + public: using value_type = T; - PoolAllocator() noexcept : pool(MemoryPool::instance()) {}; + PoolAllocator() noexcept : pool(MemoryPool::instance()){}; template - PoolAllocator(const PoolAllocator &) noexcept : pool(MemoryPool::instance()) {} + PoolAllocator(const PoolAllocator &) noexcept : pool(MemoryPool::instance()) + { + } - template - struct rebind + template struct rebind { using other = PoolAllocator; }; - T *allocate(std::size_t n) - { - return pool->allocate(n); - } + T *allocate(std::size_t n) { return pool->allocate(n); } - void deallocate(T *p, std::size_t n) noexcept - { - pool->deallocate(p, n); - } + void deallocate(T *p, std::size_t n) noexcept { pool->deallocate(p, n); } PoolAllocator(const PoolAllocator &) = default; PoolAllocator &operator=(const PoolAllocator &) = default; PoolAllocator(PoolAllocator &&) noexcept = default; PoolAllocator &operator=(PoolAllocator &&) noexcept = default; -private: + private: std::shared_ptr pool; }; template diff --git a/include/util/query_heap.hpp b/include/util/query_heap.hpp index 42818cb7f..c12a7d388 100644 --- a/include/util/query_heap.hpp +++ b/include/util/query_heap.hpp @@ -148,8 +148,6 @@ class QueryHeap } }; - - using HeapContainer = boost::heap::d_ary_heap, boost::heap::mutable_, @@ -169,8 +167,8 @@ class QueryHeap Data data; }; - QueryHeap(const QueryHeap& other) = delete; - QueryHeap(QueryHeap&& other) = delete; + QueryHeap(const QueryHeap &other) = delete; + QueryHeap(QueryHeap &&other) = delete; template explicit QueryHeap(StorageArgs... args) : node_index(args...) { diff --git a/src/benchmarks/bench.cpp b/src/benchmarks/bench.cpp index 45ef1d8fa..d11277db3 100644 --- a/src/benchmarks/bench.cpp +++ b/src/benchmarks/bench.cpp @@ -12,9 +12,9 @@ #include "osrm/coordinate.hpp" #include "osrm/engine_config.hpp" #include "osrm/json_container.hpp" -#include "util/meminfo.hpp" #include "osrm/osrm.hpp" #include "osrm/status.hpp" +#include "util/meminfo.hpp" #include @@ -656,7 +656,8 @@ try return EXIT_FAILURE; } - std::cout << "Peak RAM: " << osrm::util::PeakRAMUsedInBytes() / (1024 * 1024) << "MB" << std::endl; + std::cout << "Peak RAM: " << osrm::util::PeakRAMUsedInBytes() / (1024 * 1024) << "MB" + << std::endl; return EXIT_SUCCESS; } catch (const std::exception &e) diff --git a/unit_tests/util/pool_allocator.cpp b/unit_tests/util/pool_allocator.cpp index de013f788..7f88dfb85 100644 --- a/unit_tests/util/pool_allocator.cpp +++ b/unit_tests/util/pool_allocator.cpp @@ -9,7 +9,44 @@ BOOST_AUTO_TEST_SUITE(pool_allocator) using namespace osrm; using namespace osrm::util; -// in many of these tests we hope on address sanitizer to alert in the case if we are doing something wrong +BOOST_AUTO_TEST_CASE(test_align_up) +{ + BOOST_CHECK_EQUAL(align_up(5, 4), 8); + BOOST_CHECK_EQUAL(align_up(9, 8), 16); + BOOST_CHECK_EQUAL(align_up(17, 16), 32); + BOOST_CHECK_EQUAL(align_up(4, 4), 4); + BOOST_CHECK_EQUAL(align_up(8, 8), 8); + BOOST_CHECK_EQUAL(align_up(16, 16), 16); + BOOST_CHECK_EQUAL(align_up(32, 16), 32); + BOOST_CHECK_EQUAL(align_up(0, 4), 0); + BOOST_CHECK_EQUAL(align_up(0, 8), 0); + BOOST_CHECK_EQUAL(align_up(0, 16), 0); + BOOST_CHECK_EQUAL(align_up(1000000, 256), 1000192); + BOOST_CHECK_EQUAL(align_up(999999, 512), 1000448); + BOOST_CHECK_EQUAL(align_up(123456789, 1024), 123457536); + BOOST_CHECK_EQUAL(align_up(0, 1), 0); + BOOST_CHECK_EQUAL(align_up(5, 1), 5); + BOOST_CHECK_EQUAL(align_up(123456, 1), 123456); +} + +BOOST_AUTO_TEST_CASE(test_get_next_power_of_two_exponent) +{ + BOOST_CHECK_EQUAL(get_next_power_of_two_exponent(1), 0); + BOOST_CHECK_EQUAL(get_next_power_of_two_exponent(2), 1); + BOOST_CHECK_EQUAL(get_next_power_of_two_exponent(4), 2); + BOOST_CHECK_EQUAL(get_next_power_of_two_exponent(8), 3); + BOOST_CHECK_EQUAL(get_next_power_of_two_exponent(16), 4); + BOOST_CHECK_EQUAL(get_next_power_of_two_exponent(3), 2); + BOOST_CHECK_EQUAL(get_next_power_of_two_exponent(5), 3); + BOOST_CHECK_EQUAL(get_next_power_of_two_exponent(9), 4); + BOOST_CHECK_EQUAL(get_next_power_of_two_exponent(15), 4); + BOOST_CHECK_EQUAL(get_next_power_of_two_exponent(17), 5); + BOOST_CHECK_EQUAL(get_next_power_of_two_exponent(1), 0); + BOOST_CHECK_EQUAL(get_next_power_of_two_exponent(SIZE_MAX), sizeof(size_t) * 8); +} + +// in many of these tests we hope on address sanitizer to alert in the case if we are doing +// something wrong BOOST_AUTO_TEST_CASE(smoke) { PoolAllocator pool; @@ -74,7 +111,12 @@ BOOST_AUTO_TEST_CASE(move) BOOST_AUTO_TEST_CASE(unordered_map) { - std::unordered_map, std::equal_to, PoolAllocator>> map; + std::unordered_map, + std::equal_to, + PoolAllocator>> + map; map[1] = 42; BOOST_CHECK_EQUAL(map[1], 42); @@ -87,9 +129,9 @@ BOOST_AUTO_TEST_CASE(unordered_map) BOOST_AUTO_TEST_CASE(alignment) { - PoolAllocator pool_char; - PoolAllocator pool_double; - + PoolAllocator pool_char; + PoolAllocator pool_double; + auto ptr_char = pool_char.allocate(1); auto ptr_double = pool_double.allocate(1); @@ -98,9 +140,8 @@ BOOST_AUTO_TEST_CASE(alignment) BOOST_CHECK_NE(ptr_char, nullptr); BOOST_CHECK_EQUAL(reinterpret_cast(ptr_char) % alignof(char), 0); - pool_char.deallocate(ptr_char, 1); - pool_double.deallocate(ptr_double, 1); - + pool_char.deallocate(ptr_char, 1); + pool_double.deallocate(ptr_double, 1); ptr_char = pool_char.allocate(2); ptr_double = pool_double.allocate(1);