Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Switch over partition/ files to tar format

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-15 15:03:48 +00:00
parent 6d96a9a2e3
commit 15b53de056
5 changed files with 100 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
include/partitioner/cell_storage.hpp
View File

@ -38,9 +38,12 @@ using CellStorageView = detail::CellStorageImpl<storage::Ownership::View>;
namespace serialization
{
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::CellStorageImpl<Ownership> &storage);
inline void read(storage::tar::FileReader &reader,
const std::string &name,
detail::CellStorageImpl<Ownership> &storage);
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::CellStorageImpl<Ownership> &storage);
}
@ -92,10 +95,9 @@ template <storage::Ownership Ownership> class CellStorageImpl
WeightValueT,
boost::random_access_traversal_tag>
{
typedef boost::iterator_facade<ColumnIterator,
WeightValueT,
boost::random_access_traversal_tag>
base_t;
typedef boost::
iterator_facade<ColumnIterator, WeightValueT, boost::random_access_traversal_tag>
base_t;
public:
typedef typename base_t::value_type value_type;
@ -180,8 +182,8 @@ template <storage::Ownership Ownership> class CellStorageImpl
const NodeID *const all_destinations)
: num_source_nodes{data.num_source_nodes},
num_destination_nodes{data.num_destination_nodes},
weights{all_weights + data.value_offset},
durations{all_durations + data.value_offset},
weights{all_weights + data.value_offset}, durations{all_durations +
data.value_offset},
source_boundary{all_sources + data.source_boundary_offset},
destination_boundary{all_destinations + data.destination_boundary_offset}
{
@ -390,9 +392,11 @@ template <storage::Ownership Ownership> class CellStorageImpl
destination_boundary.data()};
}
friend void serialization::read<Ownership>(storage::io::FileReader &reader,
friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
const std::string &name,
detail::CellStorageImpl<Ownership> &storage);
friend void serialization::write<Ownership>(storage::io::FileWriter &writer,
friend void serialization::write<Ownership>(storage::tar::FileWriter &writer,
const std::string &name,
const detail::CellStorageImpl<Ownership> &storage);
private:

24
include/partitioner/files.hpp
View File

@ -52,10 +52,10 @@ inline void readPartition(const boost::filesystem::path &path, MultiLevelPartiti
std::is_same<MultiLevelPartition, MultiLevelPartitionT>::value,
"");
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
serialization::read(reader, mlp);
serialization::read(reader, "/mld/multilevelpartition", mlp);
}
// writes .osrm.partition file
@ -66,10 +66,10 @@ inline void writePartition(const boost::filesystem::path &path, const MultiLevel
std::is_same<MultiLevelPartition, MultiLevelPartitionT>::value,
"");
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
serialization::write(writer, mlp);
serialization::write(writer, "/mld/multilevelpartition", mlp);
}
// reads .osrm.cells file
@ -80,10 +80,10 @@ inline void readCells(const boost::filesystem::path &path, CellStorageT &storage
std::is_same<CellStorage, CellStorageT>::value,
"");
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
serialization::read(reader, storage);
serialization::read(reader, "/mld/cellstorage", storage);
}
// writes .osrm.cells file
@ -94,10 +94,10 @@ inline void writeCells(const boost::filesystem::path &path, CellStorageT &storag
std::is_same<CellStorage, CellStorageT>::value,
"");
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
serialization::write(writer, storage);
serialization::write(writer, "/mld/cellstorage", storage);
}
}
}

12
include/partitioner/multi_level_partition.hpp
View File

@ -7,7 +7,7 @@
#include "util/typedefs.hpp"
#include "util/vector_view.hpp"
#include "storage/io_fwd.hpp"
#include "storage/tar_fwd.hpp"
#include "storage/shared_memory_ownership.hpp"
#include <algorithm>
@ -34,9 +34,9 @@ using MultiLevelPartitionView = detail::MultiLevelPartitionImpl<storage::Ownersh
namespace serialization
{
template <storage::Ownership Ownership>
void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl<Ownership> &mlp);
void read(storage::tar::FileReader &reader, const std::string& name, detail::MultiLevelPartitionImpl<Ownership> &mlp);
template <storage::Ownership Ownership>
void write(storage::io::FileWriter &writer, const detail::MultiLevelPartitionImpl<Ownership> &mlp);
void write(storage::tar::FileWriter &writer, const std::string& name, const detail::MultiLevelPartitionImpl<Ownership> &mlp);
}
namespace detail
@ -136,9 +136,11 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
return cell_to_children[offset + cell + 1];
}
friend void serialization::read<Ownership>(storage::io::FileReader &reader,
friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
const std::string& name,
MultiLevelPartitionImpl &mlp);
friend void serialization::write<Ownership>(storage::io::FileWriter &writer,
friend void serialization::write<Ownership>(storage::tar::FileWriter &writer,
const std::string& name,
const MultiLevelPartitionImpl &mlp);
private:

53
include/partitioner/serialization.hpp
View File

@ -8,9 +8,9 @@
#include "storage/block.hpp"
#include "storage/io.hpp"
#include "storage/tar.hpp"
#include "storage/serialization.hpp"
#include "storage/shared_memory_ownership.hpp"
#include "storage/tar.hpp"
namespace osrm
{
@ -21,7 +21,7 @@ namespace serialization
template <typename EdgeDataT, storage::Ownership Ownership>
inline void read(storage::tar::FileReader &reader,
const std::string& name,
const std::string &name,
MultiLevelGraph<EdgeDataT, Ownership> &graph,
std::uint32_t &connectivity_checksum)
{
@ -33,7 +33,7 @@ inline void read(storage::tar::FileReader &reader,
template <typename EdgeDataT, storage::Ownership Ownership>
inline void write(storage::tar::FileWriter &writer,
const std::string& name,
const std::string &name,
const MultiLevelGraph<EdgeDataT, Ownership> &graph,
const std::uint32_t connectivity_checksum)
{
@ -45,39 +45,50 @@ inline void write(storage::tar::FileWriter &writer,
}
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl<Ownership> &mlp)
inline void read(storage::tar::FileReader &reader,
const std::string &name,
detail::MultiLevelPartitionImpl<Ownership> &mlp)
{
reader.ReadInto(*mlp.level_data);
storage::serialization::read(reader, mlp.partition);
storage::serialization::read(reader, mlp.cell_to_children);
*mlp.level_data = reader.ReadOne<typename std::remove_reference_t<decltype(mlp)>::LevelData>(name + "/level_data");
storage::serialization::read(reader, name + "/partition", mlp.partition);
storage::serialization::read(reader, name + "/cell_to_children", mlp.cell_to_children);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::MultiLevelPartitionImpl<Ownership> &mlp)
{
writer.WriteOne(*mlp.level_data);
storage::serialization::write(writer, mlp.partition);
storage::serialization::write(writer, mlp.cell_to_children);
writer.WriteElementCount64(name + "/level_data", 1);
writer.WriteOne(name + "/level_data", *mlp.level_data);
storage::serialization::write(writer, name + "/partition", mlp.partition);
storage::serialization::write(writer, name + "/cell_to_children", mlp.cell_to_children);
}
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::CellStorageImpl<Ownership> &storage)
inline void read(storage::tar::FileReader &reader,
const std::string &name,
detail::CellStorageImpl<Ownership> &storage)
{
storage::serialization::read(reader, storage.source_boundary);
storage::serialization::read(reader, storage.destination_boundary);
storage::serialization::read(reader, storage.cells);
storage::serialization::read(reader, storage.level_to_cell_offset);
storage::serialization::read(reader, name + "/source_boundary", storage.source_boundary);
storage::serialization::read(
reader, name + "/destination_boundary", storage.destination_boundary);
storage::serialization::read(reader, name + "/cells", storage.cells);
storage::serialization::read(
reader, name + "/level_to_cell_offset", storage.level_to_cell_offset);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::CellStorageImpl<Ownership> &storage)
{
storage::serialization::write(writer, storage.source_boundary);
storage::serialization::write(writer, storage.destination_boundary);
storage::serialization::write(writer, storage.cells);
storage::serialization::write(writer, storage.level_to_cell_offset);
storage::serialization::write(writer, name + "/source_boundary", storage.source_boundary);
storage::serialization::write(
writer, name + "/destination_boundary", storage.destination_boundary);
storage::serialization::write(writer, name + "/cells", storage.cells);
storage::serialization::write(
writer, name + "/level_to_cell_offset", storage.level_to_cell_offset);
}
}
}

106
src/storage/storage.cpp
View File

@ -69,30 +69,28 @@ namespace storage
{
namespace
{
template<typename OutIter>
void readBlocks(const boost::filesystem::path& path, OutIter out)
template <typename OutIter> void readBlocks(const boost::filesystem::path &path, OutIter out)
{
tar::FileReader reader(path, tar::FileReader::VerifyFingerprint);
std::vector<tar::FileReader::TarEntry> entries;
reader.List(std::back_inserter(entries));
for (const auto &entry : entries)
{
tar::FileReader reader(path, tar::FileReader::VerifyFingerprint);
std::string name;
std::uint64_t size;
std::tie(name, size) = entry;
std::vector<tar::FileReader::TarEntry> entries;
reader.List(std::back_inserter(entries));
for (const auto& entry : entries)
const auto name_end = name.rfind(".meta");
if (name_end == std::string::npos)
{
std::string name;
std::uint64_t size;
std::tie(name, size) = entry;
const auto name_end = name.rfind(".meta");
if (name_end == std::string::npos)
{
auto number_of_elements = reader.ReadElementCount64(name);
*out++ = NamedBlock {name, Block{number_of_elements, size}};
}
auto number_of_elements = reader.ReadElementCount64(name);
*out++ = NamedBlock{name, Block{number_of_elements, size}};
}
}
}
}
static constexpr std::size_t NUM_METRICS = 8;
@ -496,55 +494,6 @@ void Storage::PopulateLayout(DataLayout &layout)
}
{
// Loading MLD Data
if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
{
io::FileReader reader(config.GetPath(".osrm.partition"),
io::FileReader::VerifyFingerprint);
reader.Skip<partitioner::MultiLevelPartition::LevelData>(1);
layout.SetBlock(DataLayout::MLD_LEVEL_DATA,
make_block<partitioner::MultiLevelPartition::LevelData>(1));
const auto partition_entries_count = reader.ReadVectorSize<PartitionID>();
layout.SetBlock(DataLayout::MLD_PARTITION,
make_block<PartitionID>(partition_entries_count));
const auto children_entries_count = reader.ReadVectorSize<CellID>();
layout.SetBlock(DataLayout::MLD_CELL_TO_CHILDREN,
make_block<CellID>(children_entries_count));
}
else
{
layout.SetBlock(DataLayout::MLD_LEVEL_DATA,
make_block<partitioner::MultiLevelPartition::LevelData>(0));
layout.SetBlock(DataLayout::MLD_PARTITION, make_block<PartitionID>(0));
layout.SetBlock(DataLayout::MLD_CELL_TO_CHILDREN, make_block<CellID>(0));
}
if (boost::filesystem::exists(config.GetPath(".osrm.cells")))
{
io::FileReader reader(config.GetPath(".osrm.cells"), io::FileReader::VerifyFingerprint);
const auto source_node_count = reader.ReadVectorSize<NodeID>();
layout.SetBlock(DataLayout::MLD_CELL_SOURCE_BOUNDARY,
make_block<NodeID>(source_node_count));
const auto destination_node_count = reader.ReadVectorSize<NodeID>();
layout.SetBlock(DataLayout::MLD_CELL_DESTINATION_BOUNDARY,
make_block<NodeID>(destination_node_count));
const auto cell_count = reader.ReadVectorSize<partitioner::CellStorage::CellData>();
layout.SetBlock(DataLayout::MLD_CELLS,
make_block<partitioner::CellStorage::CellData>(cell_count));
const auto level_offsets_count = reader.ReadVectorSize<std::uint64_t>();
layout.SetBlock(DataLayout::MLD_CELL_LEVEL_OFFSETS,
make_block<std::uint64_t>(level_offsets_count));
}
else
{
layout.SetBlock(DataLayout::MLD_CELL_SOURCE_BOUNDARY, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_DESTINATION_BOUNDARY, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELLS, make_block<char>(0));
layout.SetBlock(DataLayout::MLD_CELL_LEVEL_OFFSETS, make_block<char>(0));
}
if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
{
io::FileReader reader(config.GetPath(".osrm.cell_metrics"),
@ -603,20 +552,35 @@ void Storage::PopulateLayout(DataLayout &layout)
{"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST},
{"/mld/multilevelgraph/node_to_edge_offset", DataLayout::MLD_GRAPH_NODE_TO_OFFSET},
{"/mld/multilevelgraph/connectivity_checksum", DataLayout::IGNORE_BLOCK},
{"/mld/multilevelpartition/level_data", DataLayout::MLD_LEVEL_DATA},
{"/mld/multilevelpartition/partition", DataLayout::MLD_PARTITION},
{"/mld/multilevelpartition/cell_to_children", DataLayout::MLD_CELL_TO_CHILDREN},
{"/mld/cellstorage/source_boundary", DataLayout::MLD_CELL_SOURCE_BOUNDARY},
{"/mld/cellstorage/destination_boundary", DataLayout::MLD_CELL_DESTINATION_BOUNDARY},
{"/mld/cellstorage/cells", DataLayout::MLD_CELLS},
{"/mld/cellstorage/level_to_cell_offset", DataLayout::MLD_CELL_LEVEL_OFFSETS},
};
std::vector<NamedBlock> blocks;
if (boost::filesystem::exists(config.GetPath(".osrm.mldgr")))
std::vector<boost::filesystem::path> optional_tar_files = {config.GetPath(".osrm.mldgr"),
config.GetPath(".osrm.cells"),
config.GetPath(".osrm.partition")};
for (const auto &path : optional_tar_files)
{
readBlocks(config.GetPath(".osrm.mldgr"), std::back_inserter(blocks));
if (boost::filesystem::exists(path))
{
readBlocks(path, std::back_inserter(blocks));
}
}
for (const auto& block : blocks)
for (const auto &block : blocks)
{
auto id_iter = name_to_block_id.find(std::get<0>(block));
if (id_iter == name_to_block_id.end())
{
throw util::exception("Could not map " + std::get<0>(block) + " to a region in memory.");
throw util::exception("Could not map " + std::get<0>(block) +
" to a region in memory.");
}
layout.SetBlock(id_iter->second, std::get<1>(block));
}