Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Split partition serialization logic

Browse Source
This commit is contained in:
Patrick Niklaus 2017-04-01 21:56:47 +00:00 committed by Patrick Niklaus
parent 08d62cd5e3
commit 865111bca9
8 changed files with 146 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
include/customizer/io.hpp
View File

@ -1,18 +0,0 @@
#ifndef OSRM_CUSTOMIZER_IO_HPP
#define OSRM_CUSTOMIZER_IO_HPP
#include "customizer/edge_based_graph.hpp"
#include "storage/io.hpp"
namespace osrm
{
namespace customizer
{
namespace io
{
}
}
}
#endif

24
include/partition/cell_storage.hpp
View File

@ -23,6 +23,15 @@
namespace osrm
{
namespace storage
{
namespace io
{
class FileReader;
class FileWriter;
}
}
namespace partition
{
namespace detail
@ -32,12 +41,12 @@ template <storage::Ownership Ownership> class CellStorageImpl;
using CellStorage = detail::CellStorageImpl<storage::Ownership::Container>;
using CellStorageView = detail::CellStorageImpl<storage::Ownership::View>;
namespace io
namespace serialization
{
template <storage::Ownership Ownership>
inline void read(const boost::filesystem::path &path, detail::CellStorageImpl<Ownership> &storage);
inline void read(storage::io::FileReader &reader, detail::CellStorageImpl<Ownership> &storage);
template <storage::Ownership Ownership>
inline void write(const boost::filesystem::path &path,
inline void write(storage::io::FileWriter &writer,
const detail::CellStorageImpl<Ownership> &storage);
}
@ -88,9 +97,8 @@ template <storage::Ownership Ownership> class CellStorageImpl
WeightValueT,
boost::random_access_traversal_tag>
{
typedef boost::iterator_facade<ColumnIterator,
WeightValueT,
boost::random_access_traversal_tag>
typedef boost::
iterator_facade<ColumnIterator, WeightValueT, boost::random_access_traversal_tag>
base_t;
public:
@ -352,9 +360,9 @@ template <storage::Ownership Ownership> class CellStorageImpl
cells[cell_index], weights.data(), source_boundary.data(), destination_boundary.data()};
}
friend void io::read<Ownership>(const boost::filesystem::path &path,
friend void serialization::read<Ownership>(storage::io::FileReader &reader,
detail::CellStorageImpl<Ownership> &storage);
friend void io::write<Ownership>(const boost::filesystem::path &path,
friend void serialization::write<Ownership>(storage::io::FileWriter &writer,
const detail::CellStorageImpl<Ownership> &storage);
private:

76
include/partition/files.hpp Normal file
View File

@ -0,0 +1,76 @@
#ifndef OSRM_PARTITION_SERILIZATION_HPP
#define OSRM_PARTITION_SERILIZATION_HPP
#include "partition/serialization.hpp"
#include "storage/io.hpp"
namespace osrm
{
namespace partition
{
namespace files
{
// reads .osrm.mldgr file
template <typename EdgeDataT, storage::Ownership Ownership>
inline void readGraph(const boost::filesystem::path &path,
MultiLevelGraph<EdgeDataT, Ownership> &graph)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
serialization::read(reader, graph);
}
// writes .osrm.mldgr file
template <typename EdgeDataT, storage::Ownership Ownership>
inline void writeGraph(const boost::filesystem::path &path,
const MultiLevelGraph<EdgeDataT, Ownership> &graph)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
serialization::write(writer, graph);
}
// read .osrm.partition file
inline void readPartition(const boost::filesystem::path &path, MultiLevelPartition &mlp)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
serialization::read(reader, mlp);
}
// writes .osrm.partition file
inline void writePartition(const boost::filesystem::path &path, const MultiLevelPartition &mlp)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
serialization::write(writer, mlp);
}
// reads .osrm.cells file
inline void readCells(const boost::filesystem::path &path, CellStorage &storage)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
serialization::read(reader, storage);
}
// writes .osrm.cells file
inline void writeCells(const boost::filesystem::path &path, const CellStorage &storage)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
serialization::write(writer, storage);
}
}
}
}
#endif

21
include/partition/multi_level_graph.hpp
View File

@ -14,17 +14,26 @@
namespace osrm
{
namespace storage
{
namespace io
{
class FileReader;
class FileWriter;
}
}
namespace partition
{
template <typename EdgeDataT, storage::Ownership Ownership> class MultiLevelGraph;
namespace io
namespace serialization
{
template <typename EdgeDataT, storage::Ownership Ownership>
void read(const boost::filesystem::path &path, MultiLevelGraph<EdgeDataT, Ownership> &graph);
void read(storage::io::FileReader &reader, MultiLevelGraph<EdgeDataT, Ownership> &graph);
template <typename EdgeDataT, storage::Ownership Ownership>
void write(const boost::filesystem::path &path, const MultiLevelGraph<EdgeDataT, Ownership> &graph);
void write(storage::io::FileWriter &writer, const MultiLevelGraph<EdgeDataT, Ownership> &graph);
}
template <typename EdgeDataT, storage::Ownership Ownership>
@ -190,9 +199,11 @@ class MultiLevelGraph : public util::StaticGraph<EdgeDataT, Ownership>
node_to_edge_offset.push_back(mlp.GetNumberOfLevels());
}
friend void io::read<EdgeDataT, Ownership>(const boost::filesystem::path &path,
friend void
serialization::read<EdgeDataT, Ownership>(storage::io::FileReader &reader,
MultiLevelGraph<EdgeDataT, Ownership> &graph);
friend void io::write<EdgeDataT, Ownership>(const boost::filesystem::path &path,
friend void
serialization::write<EdgeDataT, Ownership>(storage::io::FileWriter &writer,
const MultiLevelGraph<EdgeDataT, Ownership> &graph);
Vector<EdgeOffset> node_to_edge_offset;

19
include/partition/multi_level_partition.hpp
View File

@ -22,6 +22,14 @@
namespace osrm
{
namespace storage
{
namespace io
{
class FileReader;
class FileWriter;
}
}
namespace partition
{
namespace detail
@ -31,13 +39,12 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl;
using MultiLevelPartition = detail::MultiLevelPartitionImpl<storage::Ownership::Container>;
using MultiLevelPartitionView = detail::MultiLevelPartitionImpl<storage::Ownership::View>;
namespace io
namespace serialization
{
template <storage::Ownership Ownership>
void read(const boost::filesystem::path &file, detail::MultiLevelPartitionImpl<Ownership> &mlp);
void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl<Ownership> &mlp);
template <storage::Ownership Ownership>
void write(const boost::filesystem::path &file,
const detail::MultiLevelPartitionImpl<Ownership> &mlp);
void write(storage::io::FileWriter &writer, const detail::MultiLevelPartitionImpl<Ownership> &mlp);
}
namespace detail
@ -134,9 +141,9 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
return cell_to_children[offset + cell + 1];
}
friend void io::read<Ownership>(const boost::filesystem::path &file,
friend void serialization::read<Ownership>(storage::io::FileReader &reader,
MultiLevelPartitionImpl &mlp);
friend void io::write<Ownership>(const boost::filesystem::path &file,
friend void serialization::write<Ownership>(storage::io::FileWriter &writer,
const MultiLevelPartitionImpl &mlp);
private:

37
include/partition/io.hpp → include/partition/serialization.hpp
View File

@ -1,5 +1,5 @@
#ifndef OSRM_PARTITION_IO_HPP
#define OSRM_PARTITION_IO_HPP
#ifndef OSRM_PARTITION_SERIALIZATION_HPP
#define OSRM_PARTITION_SERIALIZATION_HPP
#include "partition/cell_storage.hpp"
#include "partition/edge_based_graph.hpp"
@ -13,57 +13,43 @@ namespace osrm
{
namespace partition
{
namespace io
namespace serialization
{
template <typename EdgeDataT, storage::Ownership Ownership>
inline void read(const boost::filesystem::path &path, MultiLevelGraph<EdgeDataT, Ownership> &graph)
inline void read(storage::io::FileReader &reader,
MultiLevelGraph<EdgeDataT, Ownership> &graph)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.DeserializeVector(graph.node_array);
reader.DeserializeVector(graph.edge_array);
reader.DeserializeVector(graph.edge_to_level);
}
template <typename EdgeDataT, storage::Ownership Ownership>
inline void write(const boost::filesystem::path &path,
inline void write(storage::io::FileWriter &writer,
const MultiLevelGraph<EdgeDataT, Ownership> &graph)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.SerializeVector(graph.node_array);
writer.SerializeVector(graph.edge_array);
writer.SerializeVector(graph.node_to_edge_offset);
}
template <> inline void read(const boost::filesystem::path &path, MultiLevelPartition &mlp)
template <> inline void read(storage::io::FileReader &reader, MultiLevelPartition &mlp)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.ReadInto<MultiLevelPartition::LevelData>(mlp.level_data);
reader.DeserializeVector(mlp.partition);
reader.DeserializeVector(mlp.cell_to_children);
}
template <> inline void write(const boost::filesystem::path &path, const MultiLevelPartition &mlp)
template <> inline void write(storage::io::FileWriter &writer, const MultiLevelPartition &mlp)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.WriteOne(mlp.level_data);
writer.SerializeVector(mlp.partition);
writer.SerializeVector(mlp.cell_to_children);
}
template <> inline void read(const boost::filesystem::path &path, CellStorage &storage)
template <> inline void read(storage::io::FileReader &reader, CellStorage &storage)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.DeserializeVector(storage.weights);
reader.DeserializeVector(storage.source_boundary);
reader.DeserializeVector(storage.destination_boundary);
@ -71,11 +57,8 @@ template <> inline void read(const boost::filesystem::path &path, CellStorage &s
reader.DeserializeVector(storage.level_to_cell_offset);
}
template <> inline void write(const boost::filesystem::path &path, const CellStorage &storage)
template <> inline void write(storage::io::FileWriter &writer, const CellStorage &storage)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.SerializeVector(storage.weights);
writer.SerializeVector(storage.source_boundary);
writer.SerializeVector(storage.destination_boundary);

11
src/customize/customizer.cpp
View File

@ -1,11 +1,10 @@
#include "customizer/customizer.hpp"
#include "customizer/cell_customizer.hpp"
#include "customizer/edge_based_graph.hpp"
#include "customizer/io.hpp"
#include "partition/cell_storage.hpp"
#include "partition/edge_based_graph_reader.hpp"
#include "partition/io.hpp"
#include "partition/files.hpp"
#include "partition/multi_level_partition.hpp"
#include "storage/shared_memory_ownership.hpp"
@ -100,12 +99,12 @@ int Customizer::Run(const CustomizationConfig &config)
TIMER_START(loading_data);
partition::MultiLevelPartition mlp;
partition::io::read(config.mld_partition_path, mlp);
partition::files::readPartition(config.mld_partition_path, mlp);
auto edge_based_graph = LoadAndUpdateEdgeExpandedGraph(config, mlp);
partition::CellStorage storage;
partition::io::read(config.mld_storage_path, storage);
partition::files::readCells(config.mld_storage_path, storage);
TIMER_STOP(loading_data);
util::Log() << "Loading partition data took " << TIMER_SEC(loading_data) << " seconds";
@ -116,12 +115,12 @@ int Customizer::Run(const CustomizationConfig &config)
util::Log() << "Cells customization took " << TIMER_SEC(cell_customize) << " seconds";
TIMER_START(writing_mld_data);
partition::io::write(config.mld_storage_path, storage);
partition::files::writeCells(config.mld_storage_path, storage);
TIMER_STOP(writing_mld_data);
util::Log() << "MLD customization writing took " << TIMER_SEC(writing_mld_data) << " seconds";
TIMER_START(writing_graph);
partition::io::write(config.mld_graph_path, *edge_based_graph);
partition::files::writeGraph(config.mld_graph_path, *edge_based_graph);
TIMER_STOP(writing_graph);
util::Log() << "Graph writing took " << TIMER_SEC(writing_graph) << " seconds";

6
src/partition/partitioner.cpp
View File

@ -4,7 +4,7 @@
#include "partition/cell_storage.hpp"
#include "partition/compressed_node_based_graph_reader.hpp"
#include "partition/edge_based_graph_reader.hpp"
#include "partition/io.hpp"
#include "partition/files.hpp"
#include "partition/multi_level_partition.hpp"
#include "partition/recursive_bisection.hpp"
#include "partition/remove_unconnected.hpp"
@ -184,8 +184,8 @@ int Partitioner::Run(const PartitionConfig &config)
util::Log() << "CellStorage constructed in " << TIMER_SEC(cell_storage) << " seconds";
TIMER_START(writing_mld_data);
io::write(config.mld_partition_path, mlp);
io::write(config.mld_storage_path, storage);
files::writePartition(config.mld_partition_path, mlp);
files::writeCells(config.mld_storage_path, storage);
TIMER_STOP(writing_mld_data);
util::Log() << "MLD data writing took " << TIMER_SEC(writing_mld_data) << " seconds";