Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Switch over partition/ files to tar format

Browse Source
This commit is contained in:
Patrick Niklaus
2018-03-15 15:03:48 +00:00
parent 6d96a9a2e3
commit 15b53de056
5 changed files with 100 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/partitioner/cell_storage.hpp
+14 -10
View File
@@ -38,9 +38,12 @@ using CellStorageView = detail::CellStorageImpl<storage::Ownership::View>;
namespace serialization
{
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::CellStorageImpl<Ownership> &storage);
inline void read(storage::tar::FileReader &reader,
const std::string &name,
detail::CellStorageImpl<Ownership> &storage);
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::CellStorageImpl<Ownership> &storage);
}
@@ -92,10 +95,9 @@ template <storage::Ownership Ownership> class CellStorageImpl
WeightValueT,
boost::random_access_traversal_tag>
{
typedef boost::iterator_facade<ColumnIterator,
WeightValueT,
boost::random_access_traversal_tag>
base_t;
typedef boost::
iterator_facade<ColumnIterator, WeightValueT, boost::random_access_traversal_tag>
base_t;
public:
typedef typename base_t::value_type value_type;
@@ -180,8 +182,8 @@ template <storage::Ownership Ownership> class CellStorageImpl
const NodeID *const all_destinations)
: num_source_nodes{data.num_source_nodes},
num_destination_nodes{data.num_destination_nodes},
weights{all_weights + data.value_offset},
durations{all_durations + data.value_offset},
weights{all_weights + data.value_offset}, durations{all_durations +
data.value_offset},
source_boundary{all_sources + data.source_boundary_offset},
destination_boundary{all_destinations + data.destination_boundary_offset}
{
@@ -390,9 +392,11 @@ template <storage::Ownership Ownership> class CellStorageImpl
destination_boundary.data()};
}
friend void serialization::read<Ownership>(storage::io::FileReader &reader,
friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
const std::string &name,
detail::CellStorageImpl<Ownership> &storage);
friend void serialization::write<Ownership>(storage::io::FileWriter &writer,
friend void serialization::write<Ownership>(storage::tar::FileWriter &writer,
const std::string &name,
const detail::CellStorageImpl<Ownership> &storage);
private:
include/partitioner/files.hpp
+12 -12
View File
@@ -52,10 +52,10 @@ inline void readPartition(const boost::filesystem::path &path, MultiLevelPartiti
std::is_same<MultiLevelPartition, MultiLevelPartitionT>::value,
"");
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
serialization::read(reader, mlp);
serialization::read(reader, "/mld/multilevelpartition", mlp);
}
// writes .osrm.partition file
@@ -66,10 +66,10 @@ inline void writePartition(const boost::filesystem::path &path, const MultiLevel
std::is_same<MultiLevelPartition, MultiLevelPartitionT>::value,
"");
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
serialization::write(writer, mlp);
serialization::write(writer, "/mld/multilevelpartition", mlp);
}
// reads .osrm.cells file
@@ -80,10 +80,10 @@ inline void readCells(const boost::filesystem::path &path, CellStorageT &storage
std::is_same<CellStorage, CellStorageT>::value,
"");
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
serialization::read(reader, storage);
serialization::read(reader, "/mld/cellstorage", storage);
}
// writes .osrm.cells file
@@ -94,10 +94,10 @@ inline void writeCells(const boost::filesystem::path &path, CellStorageT &storag
std::is_same<CellStorage, CellStorageT>::value,
"");
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
serialization::write(writer, storage);
serialization::write(writer, "/mld/cellstorage", storage);
}
}
}
include/partitioner/multi_level_partition.hpp
+7 -5
View File
@@ -7,7 +7,7 @@
#include "util/typedefs.hpp"
#include "util/vector_view.hpp"
#include "storage/io_fwd.hpp"
#include "storage/tar_fwd.hpp"
#include "storage/shared_memory_ownership.hpp"
#include <algorithm>
@@ -34,9 +34,9 @@ using MultiLevelPartitionView = detail::MultiLevelPartitionImpl<storage::Ownersh
namespace serialization
{
template <storage::Ownership Ownership>
void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl<Ownership> &mlp);
void read(storage::tar::FileReader &reader, const std::string& name, detail::MultiLevelPartitionImpl<Ownership> &mlp);
template <storage::Ownership Ownership>
void write(storage::io::FileWriter &writer, const detail::MultiLevelPartitionImpl<Ownership> &mlp);
void write(storage::tar::FileWriter &writer, const std::string& name, const detail::MultiLevelPartitionImpl<Ownership> &mlp);
}
namespace detail
@@ -136,9 +136,11 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
return cell_to_children[offset + cell + 1];
}
friend void serialization::read<Ownership>(storage::io::FileReader &reader,
friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
const std::string& name,
MultiLevelPartitionImpl &mlp);
friend void serialization::write<Ownership>(storage::io::FileWriter &writer,
friend void serialization::write<Ownership>(storage::tar::FileWriter &writer,
const std::string& name,
const MultiLevelPartitionImpl &mlp);
private:
include/partitioner/serialization.hpp
+32 -21
View File
@@ -8,9 +8,9 @@
#include "storage/block.hpp"
#include "storage/io.hpp"
#include "storage/tar.hpp"
#include "storage/serialization.hpp"
#include "storage/shared_memory_ownership.hpp"
#include "storage/tar.hpp"
namespace osrm
{
@@ -21,7 +21,7 @@ namespace serialization
template <typename EdgeDataT, storage::Ownership Ownership>
inline void read(storage::tar::FileReader &reader,
const std::string& name,
const std::string &name,
MultiLevelGraph<EdgeDataT, Ownership> &graph,
std::uint32_t &connectivity_checksum)
{
@@ -33,7 +33,7 @@ inline void read(storage::tar::FileReader &reader,
template <typename EdgeDataT, storage::Ownership Ownership>
inline void write(storage::tar::FileWriter &writer,
const std::string& name,
const std::string &name,
const MultiLevelGraph<EdgeDataT, Ownership> &graph,
const std::uint32_t connectivity_checksum)
{
@@ -45,39 +45,50 @@ inline void write(storage::tar::FileWriter &writer,
}
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl<Ownership> &mlp)
inline void read(storage::tar::FileReader &reader,
const std::string &name,
detail::MultiLevelPartitionImpl<Ownership> &mlp)
{
reader.ReadInto(*mlp.level_data);
storage::serialization::read(reader, mlp.partition);
storage::serialization::read(reader, mlp.cell_to_children);
*mlp.level_data = reader.ReadOne<typename std::remove_reference_t<decltype(mlp)>::LevelData>(name + "/level_data");
storage::serialization::read(reader, name + "/partition", mlp.partition);
storage::serialization::read(reader, name + "/cell_to_children", mlp.cell_to_children);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::MultiLevelPartitionImpl<Ownership> &mlp)
{
writer.WriteOne(*mlp.level_data);
storage::serialization::write(writer, mlp.partition);
storage::serialization::write(writer, mlp.cell_to_children);
writer.WriteElementCount64(name + "/level_data", 1);
writer.WriteOne(name + "/level_data", *mlp.level_data);
storage::serialization::write(writer, name + "/partition", mlp.partition);
storage::serialization::write(writer, name + "/cell_to_children", mlp.cell_to_children);
}
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::CellStorageImpl<Ownership> &storage)
inline void read(storage::tar::FileReader &reader,
const std::string &name,
detail::CellStorageImpl<Ownership> &storage)
{
storage::serialization::read(reader, storage.source_boundary);
storage::serialization::read(reader, storage.destination_boundary);
storage::serialization::read(reader, storage.cells);
storage::serialization::read(reader, storage.level_to_cell_offset);
storage::serialization::read(reader, name + "/source_boundary", storage.source_boundary);
storage::serialization::read(
reader, name + "/destination_boundary", storage.destination_boundary);
storage::serialization::read(reader, name + "/cells", storage.cells);
storage::serialization::read(
reader, name + "/level_to_cell_offset", storage.level_to_cell_offset);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::CellStorageImpl<Ownership> &storage)
{
storage::serialization::write(writer, storage.source_boundary);
storage::serialization::write(writer, storage.destination_boundary);
storage::serialization::write(writer, storage.cells);
storage::serialization::write(writer, storage.level_to_cell_offset);
storage::serialization::write(writer, name + "/source_boundary", storage.source_boundary);
storage::serialization::write(
writer, name + "/destination_boundary", storage.destination_boundary);
storage::serialization::write(writer, name + "/cells", storage.cells);
storage::serialization::write(
writer, name + "/level_to_cell_offset", storage.level_to_cell_offset);
}
}
}