Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Port cell metrics to tar files

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-15 16:34:16 +00:00
parent 6a09d2aa9b
commit fed77c4066
3 changed files with 41 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
include/customizer/files.hpp
View File

@ -3,7 +3,7 @@
#include "customizer/serialization.hpp" #include "customizer/serialization.hpp"
#include "storage/io.hpp" #include "storage/tar.hpp"
#include "util/integer_range.hpp" #include "util/integer_range.hpp"
@ -22,15 +22,16 @@ inline void readCellMetrics(const boost::filesystem::path &path, std::vector<Cel
std::is_same<CellMetric, CellMetricT>::value, std::is_same<CellMetric, CellMetricT>::value,
""); "");
const auto fingerprint = storage::io::FileReader::VerifyFingerprint; const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint}; storage::tar::FileReader reader{path, fingerprint};
auto num_metrics = reader.ReadElementCount64(); auto num_metrics = reader.ReadElementCount64("/mld/metrics");
metrics.resize(num_metrics); metrics.resize(num_metrics);
auto id = 0;
for (auto &metric : metrics) for (auto &metric : metrics)
{ {
serialization::read(reader, metric); serialization::read(reader, "/mld/metrics/" + std::to_string(id++), metric);
} }
} }
@ -43,13 +44,15 @@ inline void writeCellMetrics(const boost::filesystem::path &path,
std::is_same<CellMetric, CellMetricT>::value, std::is_same<CellMetric, CellMetricT>::value,
""); "");
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint}; storage::tar::FileWriter writer{path, fingerprint};
writer.WriteElementCount64(metrics.size()); writer.WriteElementCount64("/mld/metrics", metrics.size());
auto id = 0;
for (const auto &metric : metrics) for (const auto &metric : metrics)
{ {
serialization::write(writer, metric); serialization::write(writer, "/mld/metrics/" + std::to_string(id++), metric);
} }
} }
} }

18
include/customizer/serialization.hpp
View File

@ -3,9 +3,9 @@
#include "partitioner/cell_storage.hpp" #include "partitioner/cell_storage.hpp"
#include "storage/io.hpp"
#include "storage/serialization.hpp" #include "storage/serialization.hpp"
#include "storage/shared_memory_ownership.hpp" #include "storage/shared_memory_ownership.hpp"
#include "storage/tar.hpp"
namespace osrm namespace osrm
{ {
@ -15,17 +15,21 @@ namespace serialization
{ {
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::CellMetricImpl<Ownership> &metric) inline void read(storage::tar::FileReader &reader,
const std::string &name,
detail::CellMetricImpl<Ownership> &metric)
{ {
storage::serialization::read(reader, metric.weights); storage::serialization::read(reader, name + "/weights", metric.weights);
storage::serialization::read(reader, metric.durations); storage::serialization::read(reader, name + "/durations", metric.durations);
} }
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer, const detail::CellMetricImpl<Ownership> &metric) inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::CellMetricImpl<Ownership> &metric)
{ {
storage::serialization::write(writer, metric.weights); storage::serialization::write(writer, name + "/weights", metric.weights);
storage::serialization::write(writer, metric.durations); storage::serialization::write(writer, name + "/durations", metric.durations);
} }
} }
} }

72
src/storage/storage.cpp
View File

@ -494,59 +494,6 @@ void Storage::PopulateLayout(DataLayout &layout)
} }
{ {
if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
{
io::FileReader reader(config.GetPath(".osrm.cell_metrics"),
io::FileReader::VerifyFingerprint);
auto num_metric = reader.ReadElementCount64();
if (num_metric > NUM_METRICS)
{
throw util::exception("Only " + std::to_string(NUM_METRICS) +
" metrics are supported at the same time.");
}
for (const auto index : util::irange<std::size_t>(0, num_metric))
{
const auto weights_count = reader.ReadVectorSize<EdgeWeight>();
layout.SetBlock(
static_cast<DataLayout::BlockID>(DataLayout::MLD_CELL_WEIGHTS_0 + index),
make_block<EdgeWeight>(weights_count));
const auto durations_count = reader.ReadVectorSize<EdgeDuration>();
layout.SetBlock(
static_cast<DataLayout::BlockID>(DataLayout::MLD_CELL_DURATIONS_0 + index),
make_block<EdgeDuration>(durations_count));
}
for (const auto index : util::irange<std::size_t>(num_metric, NUM_METRICS))
{
layout.SetBlock(
static_cast<DataLayout::BlockID>(DataLayout::MLD_CELL_WEIGHTS_0 + index),
make_block<EdgeWeight>(0));
layout.SetBlock(
static_cast<DataLayout::BlockID>(DataLayout::MLD_CELL_DURATIONS_0 + index),
make_block<EdgeDuration>(0));
}
}
else
{
layout.SetBlock(DataLayout::MLD_CELL_WEIGHTS_0, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_WEIGHTS_1, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_WEIGHTS_2, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_WEIGHTS_3, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_WEIGHTS_4, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_WEIGHTS_5, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_WEIGHTS_6, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_WEIGHTS_7, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_0, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_1, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_2, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_3, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_4, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_5, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_6, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_7, make_block<char>(0));
}
std::unordered_map<std::string, DataLayout::BlockID> name_to_block_id = { std::unordered_map<std::string, DataLayout::BlockID> name_to_block_id = {
{"/mld/multilevelgraph/node_array", DataLayout::MLD_GRAPH_NODE_LIST}, {"/mld/multilevelgraph/node_array", DataLayout::MLD_GRAPH_NODE_LIST},
{"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST}, {"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST},
@ -559,12 +506,29 @@ void Storage::PopulateLayout(DataLayout &layout)
{"/mld/cellstorage/destination_boundary", DataLayout::MLD_CELL_DESTINATION_BOUNDARY}, {"/mld/cellstorage/destination_boundary", DataLayout::MLD_CELL_DESTINATION_BOUNDARY},
{"/mld/cellstorage/cells", DataLayout::MLD_CELLS}, {"/mld/cellstorage/cells", DataLayout::MLD_CELLS},
{"/mld/cellstorage/level_to_cell_offset", DataLayout::MLD_CELL_LEVEL_OFFSETS}, {"/mld/cellstorage/level_to_cell_offset", DataLayout::MLD_CELL_LEVEL_OFFSETS},
{"/mld/metrics/0/weights", DataLayout::MLD_CELL_WEIGHTS_0},
{"/mld/metrics/1/weights", DataLayout::MLD_CELL_WEIGHTS_1},
{"/mld/metrics/2/weights", DataLayout::MLD_CELL_WEIGHTS_2},
{"/mld/metrics/3/weights", DataLayout::MLD_CELL_WEIGHTS_3},
{"/mld/metrics/4/weights", DataLayout::MLD_CELL_WEIGHTS_4},
{"/mld/metrics/5/weights", DataLayout::MLD_CELL_WEIGHTS_5},
{"/mld/metrics/6/weights", DataLayout::MLD_CELL_WEIGHTS_6},
{"/mld/metrics/7/weights", DataLayout::MLD_CELL_WEIGHTS_7},
{"/mld/metrics/0/durations", DataLayout::MLD_CELL_DURATIONS_0},
{"/mld/metrics/1/durations", DataLayout::MLD_CELL_DURATIONS_1},
{"/mld/metrics/2/durations", DataLayout::MLD_CELL_DURATIONS_2},
{"/mld/metrics/3/durations", DataLayout::MLD_CELL_DURATIONS_3},
{"/mld/metrics/4/durations", DataLayout::MLD_CELL_DURATIONS_4},
{"/mld/metrics/5/durations", DataLayout::MLD_CELL_DURATIONS_5},
{"/mld/metrics/6/durations", DataLayout::MLD_CELL_DURATIONS_6},
{"/mld/metrics/7/durations", DataLayout::MLD_CELL_DURATIONS_7},
}; };
std::vector<NamedBlock> blocks; std::vector<NamedBlock> blocks;
std::vector<boost::filesystem::path> optional_tar_files = {config.GetPath(".osrm.mldgr"), std::vector<boost::filesystem::path> optional_tar_files = {config.GetPath(".osrm.mldgr"),
config.GetPath(".osrm.cells"), config.GetPath(".osrm.cells"),
config.GetPath(".osrm.partition")}; config.GetPath(".osrm.partition"),
config.GetPath(".osrm.cell_metrics")};
for (const auto &path : optional_tar_files) for (const auto &path : optional_tar_files)
{ {