Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add directory listing

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-26 12:00:37 +00:00 committed by Patrick Niklaus
parent b3ef2a0383
commit 5395290fd5
4 changed files with 89 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
include/storage/shared_datatype.hpp
View File

@ -13,6 +13,7 @@
#include <array>
#include <cstdint>
#include <map>
#include <unordered_set>
namespace osrm
{
@ -27,6 +28,32 @@ inline void read(io::BufferReader &reader, DataLayout &layout);
inline void write(io::BufferWriter &writer, const DataLayout &layout);
}
namespace detail
{
// Removes the file name if name_prefix is a directory and name is not a file in that directory
inline std::string trimName(const std::string &name_prefix, const std::string &name)
{
// list directory and
if (name_prefix.back() == '/')
{
auto directory_position = name.find_first_of("/", name_prefix.length());
// this is a "file" in the directory of name_prefix
if (directory_position == std::string::npos)
{
return name;
}
else
{
return name.substr(0, directory_position);
}
}
else
{
return name;
}
}
}
// Added at the start and end of each block as sanity check
const constexpr char CANARY[4] = {'O', 'S', 'R', 'M'};
@ -42,10 +69,7 @@ class DataLayout
return GetBlock(name).num_entries;
}
inline uint64_t GetBlockSize(const std::string &name) const
{
return GetBlock(name).byte_size;
}
inline uint64_t GetBlockSize(const std::string &name) const { return GetBlock(name).byte_size; }
inline uint64_t GetSizeOfLayout() const
{
@ -68,7 +92,8 @@ class DataLayout
{
char *start_canary_ptr = ptr - sizeof(CANARY);
char *end_canary_ptr = ptr + GetBlockSize(name);
std::cout << name << ": " << (long) (start_canary_ptr-shared_memory) << " -> " << (long) (end_canary_ptr-shared_memory) << std::endl;
std::cout << name << ": " << (long)(start_canary_ptr - shared_memory) << " -> "
<< (long)(end_canary_ptr - shared_memory) << std::endl;
std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
}
@ -92,15 +117,25 @@ class DataLayout
return (T *)ptr;
}
template<typename OutIter>
void List(const std::string& name_prefix, OutIter out) const
// Depending on the name prefix this function either lists all blocks with the same prefix
// or all entries in the sub-directory.
// '/ch/edge' -> '/ch/edge_filter/0/blocks', '/ch/edge_filter/1/blocks'
// '/ch/edge_filters/' -> '/ch/edge_filter/0', '/ch/edge_filter/1'
template <typename OutIter> void List(const std::string &name_prefix, OutIter out) const
{
for (const auto& pair : blocks)
std::unordered_set<std::string> returned_name;
for (const auto &pair : blocks)
{
// check if string begins with the name prefix
if (pair.first.find(name_prefix) == 0)
{
*out++ = pair.first;
auto trimmed_name = detail::trimName(name_prefix, pair.first);
auto ret = returned_name.insert(trimmed_name);
if (ret.second)
{
*out++ = trimmed_name;
}
}
}
}
@ -109,7 +144,7 @@ class DataLayout
friend void serialization::read(io::BufferReader &reader, DataLayout &layout);
friend void serialization::write(io::BufferWriter &writer, const DataLayout &layout);
const Block& GetBlock(const std::string& name) const
const Block &GetBlock(const std::string &name) const
{
auto iter = blocks.find(name);
if (iter == blocks.end())

1
src/engine/datafacade/shared_memory_allocator.cpp
View File

@ -23,6 +23,7 @@ SharedMemoryAllocator::SharedMemoryAllocator(storage::SharedDataType data_region
storage::io::BufferReader reader(GetMemory());
storage::serialization::read(reader, data_layout);
layout_size = reader.GetPosition();
util::Log(logDEBUG) << "Data layout has size " << layout_size;
}
SharedMemoryAllocator::~SharedMemoryAllocator() {}

8
src/storage/storage.cpp
View File

@ -727,10 +727,12 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
std::vector<customizer::CellMetricView> metrics;
for (auto index : util::irange<std::size_t>(0, NUM_METRICS))
std::vector<std::string> metric_prefix_names;
layout.List("/mld/metrics/", std::back_inserter(metric_prefix_names));
for (const auto &prefix : metric_prefix_names)
{
auto weights_block_id = "/mld/metrics/" + std::to_string(index) + "/weights";
auto durations_block_id = "/mld/metrics/" + std::to_string(index) + "/durations";
auto weights_block_id = prefix + "/weights";
auto durations_block_id = prefix + "/durations";
auto weight_entries_count = layout.GetBlockEntries(weights_block_id);
auto duration_entries_count = layout.GetBlockEntries(durations_block_id);

38
unit_tests/storage/data_layout.cpp
View File

@ -63,4 +63,42 @@ BOOST_AUTO_TEST_CASE(layout_write_test)
}
}
BOOST_AUTO_TEST_CASE(layout_list_test)
{
DataLayout layout;
Block block_1{20, 8 * 20};
Block block_2{1, 4 * 1};
Block block_3{100, static_cast<std::uint64_t>(std::ceil(100 / 64.))};
layout.SetBlock("/ch/edge_filter/block1", block_1);
layout.SetBlock("/ch/edge_filter/block2", block_2);
layout.SetBlock("/ch/edge_filter/block3", block_3);
layout.SetBlock("/mld/metrics/0/durations", block_2);
layout.SetBlock("/mld/metrics/0/weights", block_3);
layout.SetBlock("/mld/metrics/1/durations", block_2);
layout.SetBlock("/mld/metrics/1/weights", block_3);
std::vector<std::string> results_1;
std::vector<std::string> results_2;
std::vector<std::string> results_3;
layout.List("/ch/edge_filter", std::back_inserter(results_1));
layout.List("/ch/edge_filter/", std::back_inserter(results_2));
layout.List("/ch/", std::back_inserter(results_3));
std::vector<std::string> results_4;
std::vector<std::string> results_5;
std::vector<std::string> results_6;
layout.List("/mld/metrics", std::back_inserter(results_4));
layout.List("/mld/metrics/", std::back_inserter(results_5));
layout.List("/mld/", std::back_inserter(results_6));
CHECK_EQUAL_RANGE(results_1, "/ch/edge_filter/block1", "/ch/edge_filter/block2", "/ch/edge_filter/block3");
CHECK_EQUAL_RANGE(results_2, "/ch/edge_filter/block1", "/ch/edge_filter/block2", "/ch/edge_filter/block3");
CHECK_EQUAL_RANGE(results_3, "/ch/edge_filter");
CHECK_EQUAL_RANGE(results_4, "/mld/metrics/0/durations", "/mld/metrics/0/weights", "/mld/metrics/1/durations", "/mld/metrics/1/weights");
CHECK_EQUAL_RANGE(results_5, "/mld/metrics/0", "/mld/metrics/1");
CHECK_EQUAL_RANGE(results_6, "/mld/metrics");
}
BOOST_AUTO_TEST_SUITE_END()