Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Switch over partition/ files to tar format

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-15 15:03:48 +00:00
parent 6d96a9a2e3
commit 15b53de056
5 changed files with 100 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
include/partitioner/cell_storage.hpp
View File

@ -38,9 +38,12 @@ using CellStorageView = detail::CellStorageImpl<storage::Ownership::View>;
namespace serialization namespace serialization
{ {
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::CellStorageImpl<Ownership> &storage); inline void read(storage::tar::FileReader &reader,
const std::string &name,
detail::CellStorageImpl<Ownership> &storage);
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer, inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::CellStorageImpl<Ownership> &storage); const detail::CellStorageImpl<Ownership> &storage);
} }
@ -92,10 +95,9 @@ template <storage::Ownership Ownership> class CellStorageImpl
WeightValueT, WeightValueT,
boost::random_access_traversal_tag> boost::random_access_traversal_tag>
{ {
typedef boost::iterator_facade<ColumnIterator, typedef boost::
WeightValueT, iterator_facade<ColumnIterator, WeightValueT, boost::random_access_traversal_tag>
boost::random_access_traversal_tag> base_t;
base_t;
public: public:
typedef typename base_t::value_type value_type; typedef typename base_t::value_type value_type;
@ -180,8 +182,8 @@ template <storage::Ownership Ownership> class CellStorageImpl
const NodeID *const all_destinations) const NodeID *const all_destinations)
: num_source_nodes{data.num_source_nodes}, : num_source_nodes{data.num_source_nodes},
num_destination_nodes{data.num_destination_nodes}, num_destination_nodes{data.num_destination_nodes},
weights{all_weights + data.value_offset}, weights{all_weights + data.value_offset}, durations{all_durations +
durations{all_durations + data.value_offset}, data.value_offset},
source_boundary{all_sources + data.source_boundary_offset}, source_boundary{all_sources + data.source_boundary_offset},
destination_boundary{all_destinations + data.destination_boundary_offset} destination_boundary{all_destinations + data.destination_boundary_offset}
{ {
@ -390,9 +392,11 @@ template <storage::Ownership Ownership> class CellStorageImpl
destination_boundary.data()}; destination_boundary.data()};
} }
friend void serialization::read<Ownership>(storage::io::FileReader &reader, friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
const std::string &name,
detail::CellStorageImpl<Ownership> &storage); detail::CellStorageImpl<Ownership> &storage);
friend void serialization::write<Ownership>(storage::io::FileWriter &writer, friend void serialization::write<Ownership>(storage::tar::FileWriter &writer,
const std::string &name,
const detail::CellStorageImpl<Ownership> &storage); const detail::CellStorageImpl<Ownership> &storage);
private: private:

24
include/partitioner/files.hpp
View File

@ -52,10 +52,10 @@ inline void readPartition(const boost::filesystem::path &path, MultiLevelPartiti
std::is_same<MultiLevelPartition, MultiLevelPartitionT>::value, std::is_same<MultiLevelPartition, MultiLevelPartitionT>::value,
""); "");
const auto fingerprint = storage::io::FileReader::VerifyFingerprint; const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint}; storage::tar::FileReader reader{path, fingerprint};
serialization::read(reader, mlp); serialization::read(reader, "/mld/multilevelpartition", mlp);
} }
// writes .osrm.partition file // writes .osrm.partition file
@ -66,10 +66,10 @@ inline void writePartition(const boost::filesystem::path &path, const MultiLevel
std::is_same<MultiLevelPartition, MultiLevelPartitionT>::value, std::is_same<MultiLevelPartition, MultiLevelPartitionT>::value,
""); "");
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint}; storage::tar::FileWriter writer{path, fingerprint};
serialization::write(writer, mlp); serialization::write(writer, "/mld/multilevelpartition", mlp);
} }
// reads .osrm.cells file // reads .osrm.cells file
@ -80,10 +80,10 @@ inline void readCells(const boost::filesystem::path &path, CellStorageT &storage
std::is_same<CellStorage, CellStorageT>::value, std::is_same<CellStorage, CellStorageT>::value,
""); "");
const auto fingerprint = storage::io::FileReader::VerifyFingerprint; const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint}; storage::tar::FileReader reader{path, fingerprint};
serialization::read(reader, storage); serialization::read(reader, "/mld/cellstorage", storage);
} }
// writes .osrm.cells file // writes .osrm.cells file
@ -94,10 +94,10 @@ inline void writeCells(const boost::filesystem::path &path, CellStorageT &storag
std::is_same<CellStorage, CellStorageT>::value, std::is_same<CellStorage, CellStorageT>::value,
""); "");
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint}; storage::tar::FileWriter writer{path, fingerprint};
serialization::write(writer, storage); serialization::write(writer, "/mld/cellstorage", storage);
} }
} }
} }

12
include/partitioner/multi_level_partition.hpp
View File

@ -7,7 +7,7 @@
#include "util/typedefs.hpp" #include "util/typedefs.hpp"
#include "util/vector_view.hpp" #include "util/vector_view.hpp"
#include "storage/io_fwd.hpp" #include "storage/tar_fwd.hpp"
#include "storage/shared_memory_ownership.hpp" #include "storage/shared_memory_ownership.hpp"
#include <algorithm> #include <algorithm>
@ -34,9 +34,9 @@ using MultiLevelPartitionView = detail::MultiLevelPartitionImpl<storage::Ownersh
namespace serialization namespace serialization
{ {
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl<Ownership> &mlp); void read(storage::tar::FileReader &reader, const std::string& name, detail::MultiLevelPartitionImpl<Ownership> &mlp);
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
void write(storage::io::FileWriter &writer, const detail::MultiLevelPartitionImpl<Ownership> &mlp); void write(storage::tar::FileWriter &writer, const std::string& name, const detail::MultiLevelPartitionImpl<Ownership> &mlp);
} }
namespace detail namespace detail
@ -136,9 +136,11 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
return cell_to_children[offset + cell + 1]; return cell_to_children[offset + cell + 1];
} }
friend void serialization::read<Ownership>(storage::io::FileReader &reader, friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
const std::string& name,
MultiLevelPartitionImpl &mlp); MultiLevelPartitionImpl &mlp);
friend void serialization::write<Ownership>(storage::io::FileWriter &writer, friend void serialization::write<Ownership>(storage::tar::FileWriter &writer,
const std::string& name,
const MultiLevelPartitionImpl &mlp); const MultiLevelPartitionImpl &mlp);
private: private:

53
include/partitioner/serialization.hpp
View File

@ -8,9 +8,9 @@
#include "storage/block.hpp" #include "storage/block.hpp"
#include "storage/io.hpp" #include "storage/io.hpp"
#include "storage/tar.hpp"
#include "storage/serialization.hpp" #include "storage/serialization.hpp"
#include "storage/shared_memory_ownership.hpp" #include "storage/shared_memory_ownership.hpp"
#include "storage/tar.hpp"
namespace osrm namespace osrm
{ {
@ -21,7 +21,7 @@ namespace serialization
template <typename EdgeDataT, storage::Ownership Ownership> template <typename EdgeDataT, storage::Ownership Ownership>
inline void read(storage::tar::FileReader &reader, inline void read(storage::tar::FileReader &reader,
const std::string& name, const std::string &name,
MultiLevelGraph<EdgeDataT, Ownership> &graph, MultiLevelGraph<EdgeDataT, Ownership> &graph,
std::uint32_t &connectivity_checksum) std::uint32_t &connectivity_checksum)
{ {
@ -33,7 +33,7 @@ inline void read(storage::tar::FileReader &reader,
template <typename EdgeDataT, storage::Ownership Ownership> template <typename EdgeDataT, storage::Ownership Ownership>
inline void write(storage::tar::FileWriter &writer, inline void write(storage::tar::FileWriter &writer,
const std::string& name, const std::string &name,
const MultiLevelGraph<EdgeDataT, Ownership> &graph, const MultiLevelGraph<EdgeDataT, Ownership> &graph,
const std::uint32_t connectivity_checksum) const std::uint32_t connectivity_checksum)
{ {
@ -45,39 +45,50 @@ inline void write(storage::tar::FileWriter &writer,
} }
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl<Ownership> &mlp) inline void read(storage::tar::FileReader &reader,
const std::string &name,
detail::MultiLevelPartitionImpl<Ownership> &mlp)
{ {
reader.ReadInto(*mlp.level_data); *mlp.level_data = reader.ReadOne<typename std::remove_reference_t<decltype(mlp)>::LevelData>(name + "/level_data");
storage::serialization::read(reader, mlp.partition); storage::serialization::read(reader, name + "/partition", mlp.partition);
storage::serialization::read(reader, mlp.cell_to_children); storage::serialization::read(reader, name + "/cell_to_children", mlp.cell_to_children);
} }
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer, inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::MultiLevelPartitionImpl<Ownership> &mlp) const detail::MultiLevelPartitionImpl<Ownership> &mlp)
{ {
writer.WriteOne(*mlp.level_data); writer.WriteElementCount64(name + "/level_data", 1);
storage::serialization::write(writer, mlp.partition); writer.WriteOne(name + "/level_data", *mlp.level_data);
storage::serialization::write(writer, mlp.cell_to_children); storage::serialization::write(writer, name + "/partition", mlp.partition);
storage::serialization::write(writer, name + "/cell_to_children", mlp.cell_to_children);
} }
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::CellStorageImpl<Ownership> &storage) inline void read(storage::tar::FileReader &reader,
const std::string &name,
detail::CellStorageImpl<Ownership> &storage)
{ {
storage::serialization::read(reader, storage.source_boundary); storage::serialization::read(reader, name + "/source_boundary", storage.source_boundary);
storage::serialization::read(reader, storage.destination_boundary); storage::serialization::read(
storage::serialization::read(reader, storage.cells); reader, name + "/destination_boundary", storage.destination_boundary);
storage::serialization::read(reader, storage.level_to_cell_offset); storage::serialization::read(reader, name + "/cells", storage.cells);
storage::serialization::read(
reader, name + "/level_to_cell_offset", storage.level_to_cell_offset);
} }
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer, inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::CellStorageImpl<Ownership> &storage) const detail::CellStorageImpl<Ownership> &storage)
{ {
storage::serialization::write(writer, storage.source_boundary); storage::serialization::write(writer, name + "/source_boundary", storage.source_boundary);
storage::serialization::write(writer, storage.destination_boundary); storage::serialization::write(
storage::serialization::write(writer, storage.cells); writer, name + "/destination_boundary", storage.destination_boundary);
storage::serialization::write(writer, storage.level_to_cell_offset); storage::serialization::write(writer, name + "/cells", storage.cells);
storage::serialization::write(
writer, name + "/level_to_cell_offset", storage.level_to_cell_offset);
} }
} }
} }

106
src/storage/storage.cpp
View File

@ -69,30 +69,28 @@ namespace storage
{ {
namespace namespace
{ {
template<typename OutIter> template <typename OutIter> void readBlocks(const boost::filesystem::path &path, OutIter out)
void readBlocks(const boost::filesystem::path& path, OutIter out) {
tar::FileReader reader(path, tar::FileReader::VerifyFingerprint);
std::vector<tar::FileReader::TarEntry> entries;
reader.List(std::back_inserter(entries));
for (const auto &entry : entries)
{ {
tar::FileReader reader(path, tar::FileReader::VerifyFingerprint); std::string name;
std::uint64_t size;
std::tie(name, size) = entry;
std::vector<tar::FileReader::TarEntry> entries; const auto name_end = name.rfind(".meta");
reader.List(std::back_inserter(entries)); if (name_end == std::string::npos)
for (const auto& entry : entries)
{ {
std::string name; auto number_of_elements = reader.ReadElementCount64(name);
std::uint64_t size; *out++ = NamedBlock{name, Block{number_of_elements, size}};
std::tie(name, size) = entry;
const auto name_end = name.rfind(".meta");
if (name_end == std::string::npos)
{
auto number_of_elements = reader.ReadElementCount64(name);
*out++ = NamedBlock {name, Block{number_of_elements, size}};
}
} }
} }
} }
}
static constexpr std::size_t NUM_METRICS = 8; static constexpr std::size_t NUM_METRICS = 8;
@ -496,55 +494,6 @@ void Storage::PopulateLayout(DataLayout &layout)
} }
{ {
// Loading MLD Data
if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
{
io::FileReader reader(config.GetPath(".osrm.partition"),
io::FileReader::VerifyFingerprint);
reader.Skip<partitioner::MultiLevelPartition::LevelData>(1);
layout.SetBlock(DataLayout::MLD_LEVEL_DATA,
make_block<partitioner::MultiLevelPartition::LevelData>(1));
const auto partition_entries_count = reader.ReadVectorSize<PartitionID>();
layout.SetBlock(DataLayout::MLD_PARTITION,
make_block<PartitionID>(partition_entries_count));
const auto children_entries_count = reader.ReadVectorSize<CellID>();
layout.SetBlock(DataLayout::MLD_CELL_TO_CHILDREN,
make_block<CellID>(children_entries_count));
}
else
{
layout.SetBlock(DataLayout::MLD_LEVEL_DATA,
make_block<partitioner::MultiLevelPartition::LevelData>(0));
layout.SetBlock(DataLayout::MLD_PARTITION, make_block<PartitionID>(0));
layout.SetBlock(DataLayout::MLD_CELL_TO_CHILDREN, make_block<CellID>(0));
}
if (boost::filesystem::exists(config.GetPath(".osrm.cells")))
{
io::FileReader reader(config.GetPath(".osrm.cells"), io::FileReader::VerifyFingerprint);
const auto source_node_count = reader.ReadVectorSize<NodeID>();
layout.SetBlock(DataLayout::MLD_CELL_SOURCE_BOUNDARY,
make_block<NodeID>(source_node_count));
const auto destination_node_count = reader.ReadVectorSize<NodeID>();
layout.SetBlock(DataLayout::MLD_CELL_DESTINATION_BOUNDARY,
make_block<NodeID>(destination_node_count));
const auto cell_count = reader.ReadVectorSize<partitioner::CellStorage::CellData>();
layout.SetBlock(DataLayout::MLD_CELLS,
make_block<partitioner::CellStorage::CellData>(cell_count));
const auto level_offsets_count = reader.ReadVectorSize<std::uint64_t>();
layout.SetBlock(DataLayout::MLD_CELL_LEVEL_OFFSETS,
make_block<std::uint64_t>(level_offsets_count));
}
else
{
layout.SetBlock(DataLayout::MLD_CELL_SOURCE_BOUNDARY, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_DESTINATION_BOUNDARY, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELLS, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_LEVEL_OFFSETS, make_block<char>(0));
}
if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics"))) if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
{ {
io::FileReader reader(config.GetPath(".osrm.cell_metrics"), io::FileReader reader(config.GetPath(".osrm.cell_metrics"),
@ -603,20 +552,35 @@ void Storage::PopulateLayout(DataLayout &layout)
{"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST}, {"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST},
{"/mld/multilevelgraph/node_to_edge_offset", DataLayout::MLD_GRAPH_NODE_TO_OFFSET}, {"/mld/multilevelgraph/node_to_edge_offset", DataLayout::MLD_GRAPH_NODE_TO_OFFSET},
{"/mld/multilevelgraph/connectivity_checksum", DataLayout::IGNORE_BLOCK}, {"/mld/multilevelgraph/connectivity_checksum", DataLayout::IGNORE_BLOCK},
{"/mld/multilevelpartition/level_data", DataLayout::MLD_LEVEL_DATA},
{"/mld/multilevelpartition/partition", DataLayout::MLD_PARTITION},
{"/mld/multilevelpartition/cell_to_children", DataLayout::MLD_CELL_TO_CHILDREN},
{"/mld/cellstorage/source_boundary", DataLayout::MLD_CELL_SOURCE_BOUNDARY},
{"/mld/cellstorage/destination_boundary", DataLayout::MLD_CELL_DESTINATION_BOUNDARY},
{"/mld/cellstorage/cells", DataLayout::MLD_CELLS},
{"/mld/cellstorage/level_to_cell_offset", DataLayout::MLD_CELL_LEVEL_OFFSETS},
}; };
std::vector<NamedBlock> blocks; std::vector<NamedBlock> blocks;
if (boost::filesystem::exists(config.GetPath(".osrm.mldgr"))) std::vector<boost::filesystem::path> optional_tar_files = {config.GetPath(".osrm.mldgr"),
config.GetPath(".osrm.cells"),
config.GetPath(".osrm.partition")};
for (const auto &path : optional_tar_files)
{ {
readBlocks(config.GetPath(".osrm.mldgr"), std::back_inserter(blocks)); if (boost::filesystem::exists(path))
{
readBlocks(path, std::back_inserter(blocks));
}
} }
for (const auto& block : blocks) for (const auto &block : blocks)
{ {
auto id_iter = name_to_block_id.find(std::get<0>(block)); auto id_iter = name_to_block_id.find(std::get<0>(block));
if (id_iter == name_to_block_id.end()) if (id_iter == name_to_block_id.end())
{ {
throw util::exception("Could not map " + std::get<0>(block) + " to a region in memory."); throw util::exception("Could not map " + std::get<0>(block) +
" to a region in memory.");
} }
layout.SetBlock(id_iter->second, std::get<1>(block)); layout.SetBlock(id_iter->second, std::get<1>(block));
} }