Port hsgr file to tar
This commit is contained in:
Patrick Niklaus
parent
fed77c4066
commit
cb31f9ec29
@ -5,7 +5,7 @@
|
||||
|
||||
#include "util/serialization.hpp"
|
||||
|
||||
#include "storage/io.hpp"
|
||||
#include "storage/tar.hpp"
|
||||
#include "storage/serialization.hpp"
|
||||
|
||||
namespace osrm
|
||||
@ -29,18 +29,20 @@ inline void readGraph(const boost::filesystem::path &path,
|
||||
std::is_same<EdgeFilterT, util::vector_view<bool>>::value,
|
||||
"edge_filter must be a container of vector<bool> or vector_view<bool>");
|
||||
|
||||
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
|
||||
storage::io::FileReader reader{path, fingerprint};
|
||||
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
|
||||
storage::tar::FileReader reader{path, fingerprint};
|
||||
|
||||
reader.ReadInto(checksum);
|
||||
util::serialization::read(reader, graph);
|
||||
auto count = reader.ReadElementCount64();
|
||||
checksum = reader.ReadOne<std::uint32_t>("/ch/checksum");
|
||||
util::serialization::read(reader, "/ch/contracted_graph", graph);
|
||||
|
||||
auto count = reader.ReadElementCount64("/ch/edge_filter");
|
||||
edge_filter.resize(count);
|
||||
for (const auto index : util::irange<std::size_t>(0, count))
|
||||
{
|
||||
storage::serialization::read(reader, edge_filter[index]);
|
||||
storage::serialization::read(reader, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]);
|
||||
}
|
||||
reader.ReadInto(connectivity_checksum);
|
||||
|
||||
connectivity_checksum = reader.ReadOne<std::uint32_t>("/ch/connectivity_checksum");
|
||||
}
|
||||
|
||||
// writes .osrm.hsgr file
|
||||
@ -57,17 +59,22 @@ inline void writeGraph(const boost::filesystem::path &path,
|
||||
static_assert(std::is_same<EdgeFilterT, std::vector<bool>>::value ||
|
||||
std::is_same<EdgeFilterT, util::vector_view<bool>>::value,
|
||||
"edge_filter must be a container of vector<bool> or vector_view<bool>");
|
||||
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
|
||||
storage::io::FileWriter writer{path, fingerprint};
|
||||
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
|
||||
storage::tar::FileWriter writer{path, fingerprint};
|
||||
|
||||
writer.WriteOne(checksum);
|
||||
util::serialization::write(writer, graph);
|
||||
writer.WriteElementCount64(edge_filter.size());
|
||||
writer.WriteElementCount64("/ch/checksum", 1);
|
||||
writer.WriteOne("/ch/checksum", checksum);
|
||||
util::serialization::write(writer, "/ch/contracted_graph", graph);
|
||||
|
||||
writer.WriteElementCount64("/ch/edge_filter", edge_filter.size());
|
||||
auto id = 0;
|
||||
for (const auto &filter : edge_filter)
|
||||
{
|
||||
storage::serialization::write(writer, filter);
|
||||
storage::serialization::write(writer, "/ch/edge_filter/" + std::to_string(id++), filter);
|
||||
}
|
||||
writer.WriteOne(connectivity_checksum);
|
||||
|
||||
writer.WriteElementCount64("/ch/connectivity_checksum", 1);
|
||||
writer.WriteOne("/ch/connectivity_checksum", connectivity_checksum);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -8,6 +8,9 @@
|
||||
#include "storage/io.hpp"
|
||||
#include "storage/tar.hpp"
|
||||
|
||||
#include <boost/function_output_iterator.hpp>
|
||||
#include <boost/iterator/function_input_iterator.hpp>
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
|
||||
@ -87,28 +90,32 @@ inline void write(storage::io::FileWriter &writer, const stxxl::vector<T> &vec)
|
||||
}
|
||||
#endif
|
||||
|
||||
template <typename T> void read(tar::FileReader &reader, const std::string& name, std::vector<T> &data)
|
||||
template <typename T>
|
||||
void read(tar::FileReader &reader, const std::string &name, std::vector<T> &data)
|
||||
{
|
||||
const auto count = reader.ReadElementCount64(name);
|
||||
data.resize(count);
|
||||
reader.ReadInto(name, data.data(), count);
|
||||
}
|
||||
|
||||
template <typename T> void write(tar::FileWriter &writer, const std::string& name, const std::vector<T> &data)
|
||||
template <typename T>
|
||||
void write(tar::FileWriter &writer, const std::string &name, const std::vector<T> &data)
|
||||
{
|
||||
const auto count = data.size();
|
||||
writer.WriteElementCount64(name, count);
|
||||
writer.WriteFrom(name, data.data(), count);
|
||||
}
|
||||
|
||||
template <typename T> void read(tar::FileReader &reader, const std::string& name, util::vector_view<T> &data)
|
||||
template <typename T>
|
||||
void read(tar::FileReader &reader, const std::string &name, util::vector_view<T> &data)
|
||||
{
|
||||
const auto count = reader.ReadElementCount64(name);
|
||||
BOOST_ASSERT(data.size() == count);
|
||||
reader.ReadInto(name, data.data(), count);
|
||||
}
|
||||
|
||||
template <typename T> void write(tar::FileWriter &writer, const std::string& name, const util::vector_view<T> &data)
|
||||
template <typename T>
|
||||
void write(tar::FileWriter &writer, const std::string &name, const util::vector_view<T> &data)
|
||||
{
|
||||
const auto count = data.size();
|
||||
writer.WriteElementCount64(name, count);
|
||||
@ -143,6 +150,8 @@ template <typename T> void write(io::FileWriter &writer, const util::vector_view
|
||||
writer.WriteFrom(data.data(), count);
|
||||
}
|
||||
|
||||
namespace detail
|
||||
{
|
||||
template <typename T>
|
||||
inline unsigned char packBits(const T &data, std::size_t index, std::size_t count)
|
||||
{
|
||||
@ -162,7 +171,7 @@ inline void unpackBits(T &data, std::size_t index, std::size_t count, unsigned c
|
||||
data[index] = value & mask;
|
||||
}
|
||||
|
||||
template <> inline void read<bool>(io::FileReader &reader, util::vector_view<bool> &data)
|
||||
template <typename VectorT> void readBoolVector(io::FileReader &reader, VectorT &data)
|
||||
{
|
||||
const auto count = reader.ReadElementCount64();
|
||||
BOOST_ASSERT(data.size() == count);
|
||||
@ -175,7 +184,7 @@ template <> inline void read<bool>(io::FileReader &reader, util::vector_view<boo
|
||||
unpackBits(data, index, count - index, reader.ReadOne<unsigned char>());
|
||||
}
|
||||
|
||||
template <> inline void write<bool>(io::FileWriter &writer, const util::vector_view<bool> &data)
|
||||
template <typename VectorT> void writeBoolVector(io::FileWriter &writer, const VectorT &data)
|
||||
{
|
||||
const auto count = data.size();
|
||||
writer.WriteElementCount64(count);
|
||||
@ -188,30 +197,87 @@ template <> inline void write<bool>(io::FileWriter &writer, const util::vector_v
|
||||
writer.WriteOne<unsigned char>(packBits(data, index, count - index));
|
||||
}
|
||||
|
||||
template <typename VectorT>
|
||||
void readBoolVector(tar::FileReader &reader, const std::string &name, VectorT &data)
|
||||
{
|
||||
const auto count = reader.ReadElementCount64(name);
|
||||
BOOST_ASSERT(data.size() == count);
|
||||
std::uint64_t index = 0;
|
||||
|
||||
const auto decode = [&data, &index, count](const char block) {
|
||||
auto read_size = std::min<std::size_t>(count - index, CHAR_BIT);
|
||||
unpackBits(data, index, read_size, block);
|
||||
index += CHAR_BIT;
|
||||
};
|
||||
|
||||
reader.ReadStreaming<unsigned char>(name, boost::make_function_output_iterator(decode));
|
||||
}
|
||||
|
||||
template <typename VectorT>
|
||||
void writeBoolVector(tar::FileWriter &writer, const std::string &name, const VectorT &data)
|
||||
{
|
||||
const auto count = data.size();
|
||||
writer.WriteElementCount64(name, count);
|
||||
std::uint64_t index = 0;
|
||||
|
||||
const auto encode = [&]() {
|
||||
auto write_size = std::min<std::size_t>(count - index, CHAR_BIT);
|
||||
auto packed = packBits(data, CHAR_BIT * index, write_size);
|
||||
index += CHAR_BIT;
|
||||
return packed;
|
||||
};
|
||||
|
||||
std::uint64_t number_of_blocks = std::ceil(count / CHAR_BIT);
|
||||
writer.WriteStreaming<unsigned char>(
|
||||
name, boost::make_function_input_iterator(encode, boost::infinite()), number_of_blocks);
|
||||
}
|
||||
}
|
||||
|
||||
template <> inline void read<bool>(io::FileReader &reader, util::vector_view<bool> &data)
|
||||
{
|
||||
detail::readBoolVector(reader, data);
|
||||
}
|
||||
|
||||
template <> inline void write<bool>(io::FileWriter &writer, const util::vector_view<bool> &data)
|
||||
{
|
||||
detail::writeBoolVector(writer, data);
|
||||
}
|
||||
|
||||
template <> inline void read<bool>(io::FileReader &reader, std::vector<bool> &data)
|
||||
{
|
||||
const auto count = reader.ReadElementCount64();
|
||||
data.resize(count);
|
||||
std::uint64_t index = 0;
|
||||
for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
|
||||
{
|
||||
unpackBits(data, index, CHAR_BIT, reader.ReadOne<unsigned char>());
|
||||
}
|
||||
if (count > index)
|
||||
unpackBits(data, index, count - index, reader.ReadOne<unsigned char>());
|
||||
detail::readBoolVector(reader, data);
|
||||
}
|
||||
|
||||
template <> inline void write<bool>(io::FileWriter &writer, const std::vector<bool> &data)
|
||||
{
|
||||
const auto count = data.size();
|
||||
writer.WriteElementCount64(count);
|
||||
std::uint64_t index = 0;
|
||||
for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
|
||||
{
|
||||
writer.WriteOne<unsigned char>(packBits(data, index, CHAR_BIT));
|
||||
}
|
||||
if (count > index)
|
||||
writer.WriteOne<unsigned char>(packBits(data, index, count - index));
|
||||
detail::writeBoolVector(writer, data);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline void
|
||||
read<bool>(tar::FileReader &reader, const std::string &name, util::vector_view<bool> &data)
|
||||
{
|
||||
detail::readBoolVector(reader, name, data);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline void
|
||||
write<bool>(tar::FileWriter &writer, const std::string &name, const util::vector_view<bool> &data)
|
||||
{
|
||||
detail::writeBoolVector(writer, name, data);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline void read<bool>(tar::FileReader &reader, const std::string &name, std::vector<bool> &data)
|
||||
{
|
||||
detail::readBoolVector(reader, name, data);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline void
|
||||
write<bool>(tar::FileWriter &writer, const std::string &name, const std::vector<bool> &data)
|
||||
{
|
||||
detail::writeBoolVector(writer, name, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
#include "util/exception.hpp"
|
||||
#include "util/exception_utils.hpp"
|
||||
#include "util/fingerprint.hpp"
|
||||
#include "util/integer_range.hpp"
|
||||
#include "util/version.hpp"
|
||||
|
||||
#include <boost/filesystem/path.hpp>
|
||||
@ -56,6 +57,36 @@ class FileReader
|
||||
return tmp;
|
||||
}
|
||||
|
||||
template <typename T, typename OutIter> void ReadStreaming(const std::string &name, OutIter out)
|
||||
{
|
||||
mtar_header_t header;
|
||||
auto ret = mtar_find(&handle, name.c_str(), &header);
|
||||
if (ret != MTAR_ESUCCESS)
|
||||
{
|
||||
throw util::exception(name + ": " + mtar_strerror(ret));
|
||||
}
|
||||
|
||||
auto number_of_elements = header.size / sizeof(T);
|
||||
auto expected_size = sizeof(T) * number_of_elements;
|
||||
if (header.size != expected_size)
|
||||
{
|
||||
throw util::exception(name + ": Datatype size does not match file size.");
|
||||
}
|
||||
|
||||
T tmp;
|
||||
for (auto index : util::irange<std::size_t>(0, number_of_elements))
|
||||
{
|
||||
(void) index;
|
||||
ret = mtar_read_data(&handle, reinterpret_cast<char *>(&tmp), sizeof(T));
|
||||
if (ret != MTAR_ESUCCESS)
|
||||
{
|
||||
throw util::exception(name + ": Failed reading data: " + mtar_strerror(ret));
|
||||
}
|
||||
|
||||
*out++ = tmp;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void ReadInto(const std::string &name, T *data, const std::size_t number_of_elements)
|
||||
{
|
||||
@ -160,6 +191,29 @@ class FileWriter
|
||||
WriteFrom(name, &data, 1);
|
||||
}
|
||||
|
||||
template <typename T, typename Iter>
|
||||
void WriteStreaming(const std::string &name, Iter iter, const std::uint64_t number_of_elements)
|
||||
{
|
||||
auto number_of_bytes = number_of_elements * sizeof(T);
|
||||
|
||||
auto ret = mtar_write_file_header(&handle, name.c_str(), number_of_bytes);
|
||||
if (ret != MTAR_ESUCCESS)
|
||||
{
|
||||
throw util::exception(name + ": Error writing header: " + mtar_strerror(ret));
|
||||
}
|
||||
|
||||
for (auto index : util::irange<std::size_t>(0, number_of_elements))
|
||||
{
|
||||
(void) index;
|
||||
T tmp = *iter++;
|
||||
ret = mtar_write_data(&handle, &tmp, sizeof(T));
|
||||
if (ret != MTAR_ESUCCESS)
|
||||
{
|
||||
throw util::exception(name + ": Error writing data : " + mtar_strerror(ret));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_elements)
|
||||
{
|
||||
@ -168,13 +222,13 @@ class FileWriter
|
||||
auto ret = mtar_write_file_header(&handle, name.c_str(), number_of_bytes);
|
||||
if (ret != MTAR_ESUCCESS)
|
||||
{
|
||||
throw util::exception(name + ": Error reading header: " + mtar_strerror(ret));
|
||||
throw util::exception(name + ": Error writing header: " + mtar_strerror(ret));
|
||||
}
|
||||
|
||||
ret = mtar_write_data(&handle, reinterpret_cast<const char *>(data), number_of_bytes);
|
||||
if (ret != MTAR_ESUCCESS)
|
||||
{
|
||||
throw util::exception(name + ": Error reading data : " + mtar_strerror(ret));
|
||||
throw util::exception(name + ": Error writing data : " + mtar_strerror(ret));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -86,6 +86,50 @@ inline void write(storage::io::FileWriter &writer, const DynamicGraph<EdgeDataT>
|
||||
writer.WriteOne(graph.edge_list[index]);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename EdgeDataT, storage::Ownership Ownership>
|
||||
inline void read(storage::tar::FileReader &reader,
|
||||
const std::string &name,
|
||||
StaticGraph<EdgeDataT, Ownership> &graph)
|
||||
{
|
||||
storage::serialization::read(reader, name + "/node_array", graph.node_array);
|
||||
storage::serialization::read(reader, name + "/edge_array", graph.edge_array);
|
||||
graph.number_of_nodes = graph.node_array.size() - 1;
|
||||
graph.number_of_edges = graph.edge_array.size();
|
||||
}
|
||||
|
||||
template <typename EdgeDataT, storage::Ownership Ownership>
|
||||
inline void write(storage::tar::FileWriter &writer,
|
||||
const std::string &name,
|
||||
const StaticGraph<EdgeDataT, Ownership> &graph)
|
||||
{
|
||||
storage::serialization::write(writer, name + "/node_array", graph.node_array);
|
||||
storage::serialization::write(writer, name + "/edge_array", graph.edge_array);
|
||||
}
|
||||
|
||||
template <typename EdgeDataT>
|
||||
inline void
|
||||
read(storage::tar::FileReader &reader, const std::string &name, DynamicGraph<EdgeDataT> &graph)
|
||||
{
|
||||
storage::serialization::read(reader, name + "/node_array", graph.node_array);
|
||||
const auto num_edges = reader.ReadElementCount64(name + "/edge_list");
|
||||
graph.edge_list.resize(num_edges);
|
||||
reader.ReadStreaming<typename std::remove_reference_t<decltype(graph)>::Edge>(
|
||||
name + "/edge_list", graph.edge_list.begin());
|
||||
graph.number_of_nodes = graph.node_array.size();
|
||||
graph.number_of_edges = num_edges;
|
||||
}
|
||||
|
||||
template <typename EdgeDataT>
|
||||
inline void write(storage::tar::FileWriter &writer,
|
||||
const std::string &name,
|
||||
const DynamicGraph<EdgeDataT> &graph)
|
||||
{
|
||||
storage::serialization::write(writer, name + "/node_array", graph.node_array);
|
||||
writer.WriteElementCount64(name + "/edge_list", graph.number_of_edges);
|
||||
writer.WriteStreaming<typename std::remove_reference_t<decltype(graph)>::Edge>(
|
||||
name + "/edge_list", graph.edge_list.begin(), graph.number_of_edges);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
|
||||
#include "storage/io_fwd.hpp"
|
||||
#include "storage/shared_memory_ownership.hpp"
|
||||
#include "storage/tar_fwd.hpp"
|
||||
|
||||
#include <boost/assert.hpp>
|
||||
|
||||
@ -32,6 +33,16 @@ void read(storage::io::FileReader &reader, StaticGraph<EdgeDataT, Ownership> &gr
|
||||
|
||||
template <typename EdgeDataT, storage::Ownership Ownership>
|
||||
void write(storage::io::FileWriter &writer, const StaticGraph<EdgeDataT, Ownership> &graph);
|
||||
|
||||
template <typename EdgeDataT, storage::Ownership Ownership>
|
||||
void read(storage::tar::FileReader &reader,
|
||||
const std::string &name,
|
||||
StaticGraph<EdgeDataT, Ownership> &graph);
|
||||
|
||||
template <typename EdgeDataT, storage::Ownership Ownership>
|
||||
void write(storage::tar::FileWriter &writer,
|
||||
const std::string &name,
|
||||
const StaticGraph<EdgeDataT, Ownership> &graph);
|
||||
}
|
||||
|
||||
namespace static_graph_details
|
||||
@ -278,6 +289,14 @@ class StaticGraph
|
||||
serialization::write<EdgeDataT, Ownership>(storage::io::FileWriter &writer,
|
||||
const StaticGraph<EdgeDataT, Ownership> &graph);
|
||||
|
||||
friend void serialization::read<EdgeDataT, Ownership>(storage::tar::FileReader &reader,
|
||||
const std::string &name,
|
||||
StaticGraph<EdgeDataT, Ownership> &graph);
|
||||
friend void
|
||||
serialization::write<EdgeDataT, Ownership>(storage::tar::FileWriter &writer,
|
||||
const std::string &name,
|
||||
const StaticGraph<EdgeDataT, Ownership> &graph);
|
||||
|
||||
protected:
|
||||
template <typename IterT>
|
||||
void InitializeFromSortedEdgeRange(const std::uint32_t nodes, IterT begin, IterT end)
|
||||
|
||||
@ -289,52 +289,6 @@ void Storage::PopulateLayout(DataLayout &layout)
|
||||
make_block<extractor::NodeBasedEdgeAnnotation>(annotations_number));
|
||||
}
|
||||
|
||||
if (boost::filesystem::exists(config.GetPath(".osrm.hsgr")))
|
||||
{
|
||||
io::FileReader reader(config.GetPath(".osrm.hsgr"), io::FileReader::VerifyFingerprint);
|
||||
|
||||
reader.Skip<std::uint32_t>(1); // checksum
|
||||
auto num_nodes = reader.ReadVectorSize<contractor::QueryGraph::NodeArrayEntry>();
|
||||
auto num_edges = reader.ReadVectorSize<contractor::QueryGraph::EdgeArrayEntry>();
|
||||
auto num_metrics = reader.ReadElementCount64();
|
||||
|
||||
if (num_metrics > NUM_METRICS)
|
||||
{
|
||||
throw util::exception("Only " + std::to_string(NUM_METRICS) +
|
||||
" metrics are supported at the same time.");
|
||||
}
|
||||
|
||||
layout.SetBlock(DataLayout::HSGR_CHECKSUM, make_block<unsigned>(1));
|
||||
layout.SetBlock(DataLayout::CH_GRAPH_NODE_LIST,
|
||||
make_block<contractor::QueryGraph::NodeArrayEntry>(num_nodes));
|
||||
layout.SetBlock(DataLayout::CH_GRAPH_EDGE_LIST,
|
||||
make_block<contractor::QueryGraph::EdgeArrayEntry>(num_edges));
|
||||
|
||||
for (const auto index : util::irange<std::size_t>(0, num_metrics))
|
||||
{
|
||||
layout.SetBlock(static_cast<DataLayout::BlockID>(DataLayout::CH_EDGE_FILTER_0 + index),
|
||||
make_block<unsigned>(num_edges));
|
||||
}
|
||||
for (const auto index : util::irange<std::size_t>(num_metrics, NUM_METRICS))
|
||||
{
|
||||
layout.SetBlock(static_cast<DataLayout::BlockID>(DataLayout::CH_EDGE_FILTER_0 + index),
|
||||
make_block<unsigned>(0));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
layout.SetBlock(DataLayout::HSGR_CHECKSUM, make_block<unsigned>(0));
|
||||
layout.SetBlock(DataLayout::CH_GRAPH_NODE_LIST,
|
||||
make_block<contractor::QueryGraph::NodeArrayEntry>(0));
|
||||
layout.SetBlock(DataLayout::CH_GRAPH_EDGE_LIST,
|
||||
make_block<contractor::QueryGraph::EdgeArrayEntry>(0));
|
||||
for (const auto index : util::irange<std::size_t>(0, NUM_METRICS))
|
||||
{
|
||||
layout.SetBlock(static_cast<DataLayout::BlockID>(DataLayout::CH_EDGE_FILTER_0 + index),
|
||||
make_block<unsigned>(0));
|
||||
}
|
||||
}
|
||||
|
||||
// load rsearch tree size
|
||||
{
|
||||
io::FileReader tree_node_file(config.GetPath(".osrm.ramIndex"),
|
||||
@ -493,61 +447,83 @@ void Storage::PopulateLayout(DataLayout &layout)
|
||||
make_block<NodeID>(number_of_nodes));
|
||||
}
|
||||
|
||||
std::unordered_map<std::string, DataLayout::BlockID> name_to_block_id = {
|
||||
{"/mld/multilevelgraph/node_array", DataLayout::MLD_GRAPH_NODE_LIST},
|
||||
{"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST},
|
||||
{"/mld/multilevelgraph/node_to_edge_offset", DataLayout::MLD_GRAPH_NODE_TO_OFFSET},
|
||||
{"/mld/multilevelgraph/connectivity_checksum", DataLayout::IGNORE_BLOCK},
|
||||
{"/mld/multilevelpartition/level_data", DataLayout::MLD_LEVEL_DATA},
|
||||
{"/mld/multilevelpartition/partition", DataLayout::MLD_PARTITION},
|
||||
{"/mld/multilevelpartition/cell_to_children", DataLayout::MLD_CELL_TO_CHILDREN},
|
||||
{"/mld/cellstorage/source_boundary", DataLayout::MLD_CELL_SOURCE_BOUNDARY},
|
||||
{"/mld/cellstorage/destination_boundary", DataLayout::MLD_CELL_DESTINATION_BOUNDARY},
|
||||
{"/mld/cellstorage/cells", DataLayout::MLD_CELLS},
|
||||
{"/mld/cellstorage/level_to_cell_offset", DataLayout::MLD_CELL_LEVEL_OFFSETS},
|
||||
{"/mld/metrics/0/weights", DataLayout::MLD_CELL_WEIGHTS_0},
|
||||
{"/mld/metrics/1/weights", DataLayout::MLD_CELL_WEIGHTS_1},
|
||||
{"/mld/metrics/2/weights", DataLayout::MLD_CELL_WEIGHTS_2},
|
||||
{"/mld/metrics/3/weights", DataLayout::MLD_CELL_WEIGHTS_3},
|
||||
{"/mld/metrics/4/weights", DataLayout::MLD_CELL_WEIGHTS_4},
|
||||
{"/mld/metrics/5/weights", DataLayout::MLD_CELL_WEIGHTS_5},
|
||||
{"/mld/metrics/6/weights", DataLayout::MLD_CELL_WEIGHTS_6},
|
||||
{"/mld/metrics/7/weights", DataLayout::MLD_CELL_WEIGHTS_7},
|
||||
{"/mld/metrics/0/durations", DataLayout::MLD_CELL_DURATIONS_0},
|
||||
{"/mld/metrics/1/durations", DataLayout::MLD_CELL_DURATIONS_1},
|
||||
{"/mld/metrics/2/durations", DataLayout::MLD_CELL_DURATIONS_2},
|
||||
{"/mld/metrics/3/durations", DataLayout::MLD_CELL_DURATIONS_3},
|
||||
{"/mld/metrics/4/durations", DataLayout::MLD_CELL_DURATIONS_4},
|
||||
{"/mld/metrics/5/durations", DataLayout::MLD_CELL_DURATIONS_5},
|
||||
{"/mld/metrics/6/durations", DataLayout::MLD_CELL_DURATIONS_6},
|
||||
{"/mld/metrics/7/durations", DataLayout::MLD_CELL_DURATIONS_7},
|
||||
{"/ch/checksum", DataLayout::HSGR_CHECKSUM},
|
||||
{"/ch/contracted_graph/node_array", DataLayout::CH_GRAPH_NODE_LIST},
|
||||
{"/ch/contracted_graph/edge_array", DataLayout::CH_GRAPH_EDGE_LIST},
|
||||
{"/ch/connectivity_checksum", DataLayout::IGNORE_BLOCK},
|
||||
{"/ch/edge_filter/0", DataLayout::CH_EDGE_FILTER_0},
|
||||
{"/ch/edge_filter/1", DataLayout::CH_EDGE_FILTER_1},
|
||||
{"/ch/edge_filter/2", DataLayout::CH_EDGE_FILTER_2},
|
||||
{"/ch/edge_filter/3", DataLayout::CH_EDGE_FILTER_3},
|
||||
{"/ch/edge_filter/4", DataLayout::CH_EDGE_FILTER_4},
|
||||
{"/ch/edge_filter/5", DataLayout::CH_EDGE_FILTER_5},
|
||||
{"/ch/edge_filter/6", DataLayout::CH_EDGE_FILTER_6},
|
||||
{"/ch/edge_filter/7", DataLayout::CH_EDGE_FILTER_7},
|
||||
};
|
||||
std::vector<NamedBlock> blocks;
|
||||
|
||||
constexpr bool REQUIRED = true;
|
||||
constexpr bool OPTIONAL = false;
|
||||
std::vector<std::tuple<bool, boost::filesystem::path>> tar_files = {
|
||||
{OPTIONAL, config.GetPath(".osrm.mldgr")},
|
||||
{OPTIONAL, config.GetPath(".osrm.cells")},
|
||||
{OPTIONAL, config.GetPath(".osrm.partition")},
|
||||
{OPTIONAL, config.GetPath(".osrm.cell_metrics")},
|
||||
{OPTIONAL, config.GetPath(".osrm.hsgr")}
|
||||
};
|
||||
|
||||
for (const auto &file : tar_files)
|
||||
{
|
||||
std::unordered_map<std::string, DataLayout::BlockID> name_to_block_id = {
|
||||
{"/mld/multilevelgraph/node_array", DataLayout::MLD_GRAPH_NODE_LIST},
|
||||
{"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST},
|
||||
{"/mld/multilevelgraph/node_to_edge_offset", DataLayout::MLD_GRAPH_NODE_TO_OFFSET},
|
||||
{"/mld/multilevelgraph/connectivity_checksum", DataLayout::IGNORE_BLOCK},
|
||||
{"/mld/multilevelpartition/level_data", DataLayout::MLD_LEVEL_DATA},
|
||||
{"/mld/multilevelpartition/partition", DataLayout::MLD_PARTITION},
|
||||
{"/mld/multilevelpartition/cell_to_children", DataLayout::MLD_CELL_TO_CHILDREN},
|
||||
{"/mld/cellstorage/source_boundary", DataLayout::MLD_CELL_SOURCE_BOUNDARY},
|
||||
{"/mld/cellstorage/destination_boundary", DataLayout::MLD_CELL_DESTINATION_BOUNDARY},
|
||||
{"/mld/cellstorage/cells", DataLayout::MLD_CELLS},
|
||||
{"/mld/cellstorage/level_to_cell_offset", DataLayout::MLD_CELL_LEVEL_OFFSETS},
|
||||
{"/mld/metrics/0/weights", DataLayout::MLD_CELL_WEIGHTS_0},
|
||||
{"/mld/metrics/1/weights", DataLayout::MLD_CELL_WEIGHTS_1},
|
||||
{"/mld/metrics/2/weights", DataLayout::MLD_CELL_WEIGHTS_2},
|
||||
{"/mld/metrics/3/weights", DataLayout::MLD_CELL_WEIGHTS_3},
|
||||
{"/mld/metrics/4/weights", DataLayout::MLD_CELL_WEIGHTS_4},
|
||||
{"/mld/metrics/5/weights", DataLayout::MLD_CELL_WEIGHTS_5},
|
||||
{"/mld/metrics/6/weights", DataLayout::MLD_CELL_WEIGHTS_6},
|
||||
{"/mld/metrics/7/weights", DataLayout::MLD_CELL_WEIGHTS_7},
|
||||
{"/mld/metrics/0/durations", DataLayout::MLD_CELL_DURATIONS_0},
|
||||
{"/mld/metrics/1/durations", DataLayout::MLD_CELL_DURATIONS_1},
|
||||
{"/mld/metrics/2/durations", DataLayout::MLD_CELL_DURATIONS_2},
|
||||
{"/mld/metrics/3/durations", DataLayout::MLD_CELL_DURATIONS_3},
|
||||
{"/mld/metrics/4/durations", DataLayout::MLD_CELL_DURATIONS_4},
|
||||
{"/mld/metrics/5/durations", DataLayout::MLD_CELL_DURATIONS_5},
|
||||
{"/mld/metrics/6/durations", DataLayout::MLD_CELL_DURATIONS_6},
|
||||
{"/mld/metrics/7/durations", DataLayout::MLD_CELL_DURATIONS_7},
|
||||
};
|
||||
std::vector<NamedBlock> blocks;
|
||||
|
||||
std::vector<boost::filesystem::path> optional_tar_files = {config.GetPath(".osrm.mldgr"),
|
||||
config.GetPath(".osrm.cells"),
|
||||
config.GetPath(".osrm.partition"),
|
||||
config.GetPath(".osrm.cell_metrics")};
|
||||
|
||||
for (const auto &path : optional_tar_files)
|
||||
if (boost::filesystem::exists(std::get<1>(file)))
|
||||
{
|
||||
if (boost::filesystem::exists(path))
|
||||
readBlocks(std::get<1>(file), std::back_inserter(blocks));
|
||||
}
|
||||
else
|
||||
{
|
||||
if (std::get<0>(file) == REQUIRED)
|
||||
{
|
||||
readBlocks(path, std::back_inserter(blocks));
|
||||
throw util::exception("Could not find required filed: " + std::get<1>(file).string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto &block : blocks)
|
||||
for (const auto &block : blocks)
|
||||
{
|
||||
auto id_iter = name_to_block_id.find(std::get<0>(block));
|
||||
if (id_iter == name_to_block_id.end())
|
||||
{
|
||||
auto id_iter = name_to_block_id.find(std::get<0>(block));
|
||||
if (id_iter == name_to_block_id.end())
|
||||
{
|
||||
throw util::exception("Could not map " + std::get<0>(block) +
|
||||
" to a region in memory.");
|
||||
}
|
||||
layout.SetBlock(id_iter->second, std::get<1>(block));
|
||||
throw util::exception("Could not map " + std::get<0>(block) +
|
||||
" to a region in memory.");
|
||||
}
|
||||
layout.SetBlock(id_iter->second, std::get<1>(block));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user