Split partition serialization logic

include/customizer/io.hpp

@@ -1,18 +0,0 @@
-#ifndef OSRM_CUSTOMIZER_IO_HPP
-#define OSRM_CUSTOMIZER_IO_HPP
-
-#include "customizer/edge_based_graph.hpp"
-
-#include "storage/io.hpp"
-
-namespace osrm
-{
-namespace customizer
-{
-namespace io
-{
-}
-}
-}
-
-#endif

include/partition/cell_storage.hpp

@@ -23,6 +23,15 @@
 
 namespace osrm
 {
+namespace storage
+{
+namespace io
+{
+class FileReader;
+class FileWriter;
+}
+}
+
 namespace partition
 {
 namespace detail
@@ -32,12 +41,12 @@ template <storage::Ownership Ownership> class CellStorageImpl;
 using CellStorage = detail::CellStorageImpl<storage::Ownership::Container>;
 using CellStorageView = detail::CellStorageImpl<storage::Ownership::View>;
 
-namespace io
+namespace serialization
 {
 template <storage::Ownership Ownership>
-inline void read(const boost::filesystem::path &path, detail::CellStorageImpl<Ownership> &storage);
+inline void read(storage::io::FileReader &reader, detail::CellStorageImpl<Ownership> &storage);
 template <storage::Ownership Ownership>
-inline void write(const boost::filesystem::path &path,
+inline void write(storage::io::FileWriter &writer,
                   const detail::CellStorageImpl<Ownership> &storage);
 }
 
@@ -88,10 +97,9 @@ template <storage::Ownership Ownership> class CellStorageImpl
                                                              WeightValueT,
                                                              boost::random_access_traversal_tag>
         {
-            typedef boost::iterator_facade<ColumnIterator,
-                                           WeightValueT,
-                                           boost::random_access_traversal_tag>
-                base_t;
+            typedef boost::
+                iterator_facade<ColumnIterator, WeightValueT, boost::random_access_traversal_tag>
+                    base_t;
 
           public:
             typedef typename base_t::value_type value_type;
@@ -352,10 +360,10 @@ template <storage::Ownership Ownership> class CellStorageImpl
             cells[cell_index], weights.data(), source_boundary.data(), destination_boundary.data()};
     }
 
-    friend void io::read<Ownership>(const boost::filesystem::path &path,
-                                    detail::CellStorageImpl<Ownership> &storage);
-    friend void io::write<Ownership>(const boost::filesystem::path &path,
-                                     const detail::CellStorageImpl<Ownership> &storage);
+    friend void serialization::read<Ownership>(storage::io::FileReader &reader,
+                                               detail::CellStorageImpl<Ownership> &storage);
+    friend void serialization::write<Ownership>(storage::io::FileWriter &writer,
+                                                const detail::CellStorageImpl<Ownership> &storage);
 
   private:
     Vector<EdgeWeight> weights;

include/partition/files.hpp

@@ -0,0 +1,76 @@
+#ifndef OSRM_PARTITION_SERILIZATION_HPP
+#define OSRM_PARTITION_SERILIZATION_HPP
+
+#include "partition/serialization.hpp"
+
+#include "storage/io.hpp"
+
+namespace osrm
+{
+namespace partition
+{
+namespace files
+{
+
+// reads .osrm.mldgr file
+template <typename EdgeDataT, storage::Ownership Ownership>
+inline void readGraph(const boost::filesystem::path &path,
+                      MultiLevelGraph<EdgeDataT, Ownership> &graph)
+{
+    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
+    storage::io::FileReader reader{path, fingerprint};
+
+    serialization::read(reader, graph);
+}
+
+// writes .osrm.mldgr file
+template <typename EdgeDataT, storage::Ownership Ownership>
+inline void writeGraph(const boost::filesystem::path &path,
+                       const MultiLevelGraph<EdgeDataT, Ownership> &graph)
+{
+    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
+    storage::io::FileWriter writer{path, fingerprint};
+
+    serialization::write(writer, graph);
+}
+
+// read .osrm.partition file
+inline void readPartition(const boost::filesystem::path &path, MultiLevelPartition &mlp)
+{
+    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
+    storage::io::FileReader reader{path, fingerprint};
+
+    serialization::read(reader, mlp);
+}
+
+// writes .osrm.partition file
+inline void writePartition(const boost::filesystem::path &path, const MultiLevelPartition &mlp)
+{
+    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
+    storage::io::FileWriter writer{path, fingerprint};
+
+    serialization::write(writer, mlp);
+}
+
+// reads .osrm.cells file
+inline void readCells(const boost::filesystem::path &path, CellStorage &storage)
+{
+    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
+    storage::io::FileReader reader{path, fingerprint};
+
+    serialization::read(reader, storage);
+}
+
+// writes .osrm.cells file
+inline void writeCells(const boost::filesystem::path &path, const CellStorage &storage)
+{
+    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
+    storage::io::FileWriter writer{path, fingerprint};
+
+    serialization::write(writer, storage);
+}
+}
+}
+}
+
+#endif

include/partition/multi_level_graph.hpp

@@ -14,17 +14,26 @@
 
 namespace osrm
 {
+namespace storage
+{
+namespace io
+{
+class FileReader;
+class FileWriter;
+}
+}
+
 namespace partition
 {
 template <typename EdgeDataT, storage::Ownership Ownership> class MultiLevelGraph;
 
-namespace io
+namespace serialization
 {
 template <typename EdgeDataT, storage::Ownership Ownership>
-void read(const boost::filesystem::path &path, MultiLevelGraph<EdgeDataT, Ownership> &graph);
+void read(storage::io::FileReader &reader, MultiLevelGraph<EdgeDataT, Ownership> &graph);
 
 template <typename EdgeDataT, storage::Ownership Ownership>
-void write(const boost::filesystem::path &path, const MultiLevelGraph<EdgeDataT, Ownership> &graph);
+void write(storage::io::FileWriter &writer, const MultiLevelGraph<EdgeDataT, Ownership> &graph);
 }
 
 template <typename EdgeDataT, storage::Ownership Ownership>
@@ -190,10 +199,12 @@ class MultiLevelGraph : public util::StaticGraph<EdgeDataT, Ownership>
         node_to_edge_offset.push_back(mlp.GetNumberOfLevels());
     }
 
-    friend void io::read<EdgeDataT, Ownership>(const boost::filesystem::path &path,
-                                               MultiLevelGraph<EdgeDataT, Ownership> &graph);
-    friend void io::write<EdgeDataT, Ownership>(const boost::filesystem::path &path,
-                                                const MultiLevelGraph<EdgeDataT, Ownership> &graph);
+    friend void
+    serialization::read<EdgeDataT, Ownership>(storage::io::FileReader &reader,
+                                              MultiLevelGraph<EdgeDataT, Ownership> &graph);
+    friend void
+    serialization::write<EdgeDataT, Ownership>(storage::io::FileWriter &writer,
+                                               const MultiLevelGraph<EdgeDataT, Ownership> &graph);
 
     Vector<EdgeOffset> node_to_edge_offset;
 };

include/partition/multi_level_partition.hpp

@@ -22,6 +22,14 @@
 
 namespace osrm
 {
+namespace storage
+{
+namespace io
+{
+class FileReader;
+class FileWriter;
+}
+}
 namespace partition
 {
 namespace detail
@@ -31,13 +39,12 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl;
 using MultiLevelPartition = detail::MultiLevelPartitionImpl<storage::Ownership::Container>;
 using MultiLevelPartitionView = detail::MultiLevelPartitionImpl<storage::Ownership::View>;
 
-namespace io
+namespace serialization
 {
 template <storage::Ownership Ownership>
-void read(const boost::filesystem::path &file, detail::MultiLevelPartitionImpl<Ownership> &mlp);
+void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl<Ownership> &mlp);
 template <storage::Ownership Ownership>
-void write(const boost::filesystem::path &file,
-           const detail::MultiLevelPartitionImpl<Ownership> &mlp);
+void write(storage::io::FileWriter &writer, const detail::MultiLevelPartitionImpl<Ownership> &mlp);
 }
 
 namespace detail
@@ -134,10 +141,10 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
         return cell_to_children[offset + cell + 1];
     }
 
-    friend void io::read<Ownership>(const boost::filesystem::path &file,
-                                    MultiLevelPartitionImpl &mlp);
-    friend void io::write<Ownership>(const boost::filesystem::path &file,
-                                     const MultiLevelPartitionImpl &mlp);
+    friend void serialization::read<Ownership>(storage::io::FileReader &reader,
+                                               MultiLevelPartitionImpl &mlp);
+    friend void serialization::write<Ownership>(storage::io::FileWriter &writer,
+                                                const MultiLevelPartitionImpl &mlp);
 
   private:
     auto MakeLevelData(const std::vector<std::uint32_t> &lidx_to_num_cells)

include/partition/serialization.hpp

@@ -1,5 +1,5 @@
-#ifndef OSRM_PARTITION_IO_HPP
-#define OSRM_PARTITION_IO_HPP
+#ifndef OSRM_PARTITION_SERIALIZATION_HPP
+#define OSRM_PARTITION_SERIALIZATION_HPP
 
 #include "partition/cell_storage.hpp"
 #include "partition/edge_based_graph.hpp"
@@ -13,57 +13,43 @@ namespace osrm
 {
 namespace partition
 {
-namespace io
+namespace serialization
 {
 
 template <typename EdgeDataT, storage::Ownership Ownership>
-inline void read(const boost::filesystem::path &path, MultiLevelGraph<EdgeDataT, Ownership> &graph)
+inline void read(storage::io::FileReader &reader,
+                 MultiLevelGraph<EdgeDataT, Ownership> &graph)
 {
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
-
     reader.DeserializeVector(graph.node_array);
     reader.DeserializeVector(graph.edge_array);
     reader.DeserializeVector(graph.edge_to_level);
 }
 
 template <typename EdgeDataT, storage::Ownership Ownership>
-inline void write(const boost::filesystem::path &path,
+inline void write(storage::io::FileWriter &writer,
                   const MultiLevelGraph<EdgeDataT, Ownership> &graph)
 {
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
-
     writer.SerializeVector(graph.node_array);
     writer.SerializeVector(graph.edge_array);
     writer.SerializeVector(graph.node_to_edge_offset);
 }
 
-template <> inline void read(const boost::filesystem::path &path, MultiLevelPartition &mlp)
+template <> inline void read(storage::io::FileReader &reader, MultiLevelPartition &mlp)
 {
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
-
     reader.ReadInto<MultiLevelPartition::LevelData>(mlp.level_data);
     reader.DeserializeVector(mlp.partition);
     reader.DeserializeVector(mlp.cell_to_children);
 }
 
-template <> inline void write(const boost::filesystem::path &path, const MultiLevelPartition &mlp)
+template <> inline void write(storage::io::FileWriter &writer, const MultiLevelPartition &mlp)
 {
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
-
     writer.WriteOne(mlp.level_data);
     writer.SerializeVector(mlp.partition);
     writer.SerializeVector(mlp.cell_to_children);
 }
 
-template <> inline void read(const boost::filesystem::path &path, CellStorage &storage)
+template <> inline void read(storage::io::FileReader &reader, CellStorage &storage)
 {
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
-
     reader.DeserializeVector(storage.weights);
     reader.DeserializeVector(storage.source_boundary);
     reader.DeserializeVector(storage.destination_boundary);
@@ -71,11 +57,8 @@ template <> inline void read(const boost::filesystem::path &path, CellStorage &s
     reader.DeserializeVector(storage.level_to_cell_offset);
 }
 
-template <> inline void write(const boost::filesystem::path &path, const CellStorage &storage)
+template <> inline void write(storage::io::FileWriter &writer, const CellStorage &storage)
 {
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
-
     writer.SerializeVector(storage.weights);
     writer.SerializeVector(storage.source_boundary);
     writer.SerializeVector(storage.destination_boundary);

src/customize/customizer.cpp

@@ -1,11 +1,10 @@
 #include "customizer/customizer.hpp"
 #include "customizer/cell_customizer.hpp"
 #include "customizer/edge_based_graph.hpp"
-#include "customizer/io.hpp"
 
 #include "partition/cell_storage.hpp"
 #include "partition/edge_based_graph_reader.hpp"
-#include "partition/io.hpp"
+#include "partition/files.hpp"
 #include "partition/multi_level_partition.hpp"
 
 #include "storage/shared_memory_ownership.hpp"
@@ -100,12 +99,12 @@ int Customizer::Run(const CustomizationConfig &config)
     TIMER_START(loading_data);
 
     partition::MultiLevelPartition mlp;
-    partition::io::read(config.mld_partition_path, mlp);
+    partition::files::readPartition(config.mld_partition_path, mlp);
 
     auto edge_based_graph = LoadAndUpdateEdgeExpandedGraph(config, mlp);
 
     partition::CellStorage storage;
-    partition::io::read(config.mld_storage_path, storage);
+    partition::files::readCells(config.mld_storage_path, storage);
     TIMER_STOP(loading_data);
     util::Log() << "Loading partition data took " << TIMER_SEC(loading_data) << " seconds";
 
@@ -116,12 +115,12 @@ int Customizer::Run(const CustomizationConfig &config)
     util::Log() << "Cells customization took " << TIMER_SEC(cell_customize) << " seconds";
 
     TIMER_START(writing_mld_data);
-    partition::io::write(config.mld_storage_path, storage);
+    partition::files::writeCells(config.mld_storage_path, storage);
     TIMER_STOP(writing_mld_data);
     util::Log() << "MLD customization writing took " << TIMER_SEC(writing_mld_data) << " seconds";
 
     TIMER_START(writing_graph);
-    partition::io::write(config.mld_graph_path, *edge_based_graph);
+    partition::files::writeGraph(config.mld_graph_path, *edge_based_graph);
     TIMER_STOP(writing_graph);
     util::Log() << "Graph writing took " << TIMER_SEC(writing_graph) << " seconds";
 

src/partition/partitioner.cpp

@@ -4,7 +4,7 @@
 #include "partition/cell_storage.hpp"
 #include "partition/compressed_node_based_graph_reader.hpp"
 #include "partition/edge_based_graph_reader.hpp"
-#include "partition/io.hpp"
+#include "partition/files.hpp"
 #include "partition/multi_level_partition.hpp"
 #include "partition/recursive_bisection.hpp"
 #include "partition/remove_unconnected.hpp"
@@ -184,8 +184,8 @@ int Partitioner::Run(const PartitionConfig &config)
     util::Log() << "CellStorage constructed in " << TIMER_SEC(cell_storage) << " seconds";
 
     TIMER_START(writing_mld_data);
-    io::write(config.mld_partition_path, mlp);
-    io::write(config.mld_storage_path, storage);
+    files::writePartition(config.mld_partition_path, mlp);
+    files::writeCells(config.mld_storage_path, storage);
     TIMER_STOP(writing_mld_data);
     util::Log() << "MLD data writing took " << TIMER_SEC(writing_mld_data) << " seconds";