From 5395290fd55326d42932b9a6f4e4e72a4b6f3a75 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 26 Mar 2018 12:00:37 +0000 Subject: [PATCH] Add directory listing --- include/storage/shared_datatype.hpp | 55 +++++++++++++++---- .../datafacade/shared_memory_allocator.cpp | 1 + src/storage/storage.cpp | 8 ++- unit_tests/storage/data_layout.cpp | 38 +++++++++++++ 4 files changed, 89 insertions(+), 13 deletions(-) diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp index fbd6f579d..5d2b4893f 100644 --- a/include/storage/shared_datatype.hpp +++ b/include/storage/shared_datatype.hpp @@ -13,6 +13,7 @@ #include #include #include +#include namespace osrm { @@ -27,6 +28,32 @@ inline void read(io::BufferReader &reader, DataLayout &layout); inline void write(io::BufferWriter &writer, const DataLayout &layout); } +namespace detail +{ +// Removes the file name if name_prefix is a directory and name is not a file in that directory +inline std::string trimName(const std::string &name_prefix, const std::string &name) +{ + // list directory and + if (name_prefix.back() == '/') + { + auto directory_position = name.find_first_of("/", name_prefix.length()); + // this is a "file" in the directory of name_prefix + if (directory_position == std::string::npos) + { + return name; + } + else + { + return name.substr(0, directory_position); + } + } + else + { + return name; + } +} +} + // Added at the start and end of each block as sanity check const constexpr char CANARY[4] = {'O', 'S', 'R', 'M'}; @@ -42,10 +69,7 @@ class DataLayout return GetBlock(name).num_entries; } - inline uint64_t GetBlockSize(const std::string &name) const - { - return GetBlock(name).byte_size; - } + inline uint64_t GetBlockSize(const std::string &name) const { return GetBlock(name).byte_size; } inline uint64_t GetSizeOfLayout() const { @@ -68,7 +92,8 @@ class DataLayout { char *start_canary_ptr = ptr - sizeof(CANARY); char *end_canary_ptr = ptr + GetBlockSize(name); - std::cout << name << ": " << (long) (start_canary_ptr-shared_memory) << " -> " << (long) (end_canary_ptr-shared_memory) << std::endl; + std::cout << name << ": " << (long)(start_canary_ptr - shared_memory) << " -> " + << (long)(end_canary_ptr - shared_memory) << std::endl; std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr); std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr); } @@ -92,15 +117,25 @@ class DataLayout return (T *)ptr; } - template - void List(const std::string& name_prefix, OutIter out) const + // Depending on the name prefix this function either lists all blocks with the same prefix + // or all entries in the sub-directory. + // '/ch/edge' -> '/ch/edge_filter/0/blocks', '/ch/edge_filter/1/blocks' + // '/ch/edge_filters/' -> '/ch/edge_filter/0', '/ch/edge_filter/1' + template void List(const std::string &name_prefix, OutIter out) const { - for (const auto& pair : blocks) + std::unordered_set returned_name; + + for (const auto &pair : blocks) { // check if string begins with the name prefix if (pair.first.find(name_prefix) == 0) { - *out++ = pair.first; + auto trimmed_name = detail::trimName(name_prefix, pair.first); + auto ret = returned_name.insert(trimmed_name); + if (ret.second) + { + *out++ = trimmed_name; + } } } } @@ -109,7 +144,7 @@ class DataLayout friend void serialization::read(io::BufferReader &reader, DataLayout &layout); friend void serialization::write(io::BufferWriter &writer, const DataLayout &layout); - const Block& GetBlock(const std::string& name) const + const Block &GetBlock(const std::string &name) const { auto iter = blocks.find(name); if (iter == blocks.end()) diff --git a/src/engine/datafacade/shared_memory_allocator.cpp b/src/engine/datafacade/shared_memory_allocator.cpp index cb31d091e..91331acc0 100644 --- a/src/engine/datafacade/shared_memory_allocator.cpp +++ b/src/engine/datafacade/shared_memory_allocator.cpp @@ -23,6 +23,7 @@ SharedMemoryAllocator::SharedMemoryAllocator(storage::SharedDataType data_region storage::io::BufferReader reader(GetMemory()); storage::serialization::read(reader, data_layout); layout_size = reader.GetPosition(); + util::Log(logDEBUG) << "Data layout has size " << layout_size; } SharedMemoryAllocator::~SharedMemoryAllocator() {} diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 5971b6126..2295b8dd5 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -727,10 +727,12 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) std::vector metrics; - for (auto index : util::irange(0, NUM_METRICS)) + std::vector metric_prefix_names; + layout.List("/mld/metrics/", std::back_inserter(metric_prefix_names)); + for (const auto &prefix : metric_prefix_names) { - auto weights_block_id = "/mld/metrics/" + std::to_string(index) + "/weights"; - auto durations_block_id = "/mld/metrics/" + std::to_string(index) + "/durations"; + auto weights_block_id = prefix + "/weights"; + auto durations_block_id = prefix + "/durations"; auto weight_entries_count = layout.GetBlockEntries(weights_block_id); auto duration_entries_count = layout.GetBlockEntries(durations_block_id); diff --git a/unit_tests/storage/data_layout.cpp b/unit_tests/storage/data_layout.cpp index 6b628ecbe..b97a20206 100644 --- a/unit_tests/storage/data_layout.cpp +++ b/unit_tests/storage/data_layout.cpp @@ -63,4 +63,42 @@ BOOST_AUTO_TEST_CASE(layout_write_test) } } +BOOST_AUTO_TEST_CASE(layout_list_test) +{ + DataLayout layout; + + Block block_1{20, 8 * 20}; + Block block_2{1, 4 * 1}; + Block block_3{100, static_cast(std::ceil(100 / 64.))}; + + layout.SetBlock("/ch/edge_filter/block1", block_1); + layout.SetBlock("/ch/edge_filter/block2", block_2); + layout.SetBlock("/ch/edge_filter/block3", block_3); + layout.SetBlock("/mld/metrics/0/durations", block_2); + layout.SetBlock("/mld/metrics/0/weights", block_3); + layout.SetBlock("/mld/metrics/1/durations", block_2); + layout.SetBlock("/mld/metrics/1/weights", block_3); + + std::vector results_1; + std::vector results_2; + std::vector results_3; + layout.List("/ch/edge_filter", std::back_inserter(results_1)); + layout.List("/ch/edge_filter/", std::back_inserter(results_2)); + layout.List("/ch/", std::back_inserter(results_3)); + + std::vector results_4; + std::vector results_5; + std::vector results_6; + layout.List("/mld/metrics", std::back_inserter(results_4)); + layout.List("/mld/metrics/", std::back_inserter(results_5)); + layout.List("/mld/", std::back_inserter(results_6)); + + CHECK_EQUAL_RANGE(results_1, "/ch/edge_filter/block1", "/ch/edge_filter/block2", "/ch/edge_filter/block3"); + CHECK_EQUAL_RANGE(results_2, "/ch/edge_filter/block1", "/ch/edge_filter/block2", "/ch/edge_filter/block3"); + CHECK_EQUAL_RANGE(results_3, "/ch/edge_filter"); + CHECK_EQUAL_RANGE(results_4, "/mld/metrics/0/durations", "/mld/metrics/0/weights", "/mld/metrics/1/durations", "/mld/metrics/1/weights"); + CHECK_EQUAL_RANGE(results_5, "/mld/metrics/0", "/mld/metrics/1"); + CHECK_EQUAL_RANGE(results_6, "/mld/metrics"); +} + BOOST_AUTO_TEST_SUITE_END()