Separate serialization and files in extractor

include/extractor/files.hpp

@@ -0,0 +1,72 @@
+#ifndef OSRM_EXTRACTOR_FILES_HPP
+#define OSRM_EXTRACTOR_FILES_HPP
+
+#include "extractor/seralization.hpp"
+
+#include <boost/assert.hpp>
+
+namespace osrm
+{
+namespace extractor
+{
+namespace files
+{
+
+// reads .osrm.cnbg_to_ebg
+inline void readNBGMapping(const boost::filesystem::path &path, std::vector<NBGToEBG> &mapping)
+{
+    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
+    storage::io::FileReader reader{path, fingerprint};
+
+    serialization::read(reader, mapping);
+}
+
+// writes .osrm.cnbg_to_ebg
+inline void writeNBGMapping(const boost::filesystem::path &path, const std::vector<NBGToEBG> &mapping)
+{
+    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
+    storage::io::FileWriter writer{path, fingerprint};
+
+    serialization::write(writer, mapping);
+}
+
+// reads .osrm.datasource_names
+inline void readDatasources(const boost::filesystem::path &path, Datasources &sources)
+{
+    const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
+    storage::io::FileReader reader{path, fingerprint};
+
+    serialization::read(reader, sources);
+}
+
+// writes .osrm.datasource_names
+inline void writeDatasources(const boost::filesystem::path &path, Datasources &sources)
+{
+    const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
+    storage::io::FileWriter writer{path, fingerprint};
+
+    serialization::write(writer, sources);
+}
+
+// reads .osrm.geometry
+inline void readSegmentData(const boost::filesystem::path &path, SegmentDataContainer &segment_data)
+{
+    const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
+    storage::io::FileReader reader{path, fingerprint};
+
+    serialization::read(reader, segment_data);
+}
+
+// writes .osrm.geometry
+inline void writeSegmentData(const boost::filesystem::path &path, const SegmentDataContainer &segment_data)
+{
+    const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
+    storage::io::FileWriter writer{path, fingerprint};
+
+    serialization::write(writer, segment_data);
+}
+}
+}
+}
+
+#endif

include/extractor/segment_data_container.hpp

@@ -17,6 +17,15 @@
 
 namespace osrm
 {
+namespace storage
+{
+namespace io
+{
+class FileReader;
+class FileWriter;
+}
+}
+
 namespace extractor
 {
 
@@ -27,13 +36,13 @@ namespace detail
 template <storage::Ownership Ownership> class SegmentDataContainerImpl;
 }
 
-namespace io
+namespace serialization
 {
 template <storage::Ownership Ownership>
-inline void read(const boost::filesystem::path &path,
+inline void read(storage::io::FileReader &reader,
                  detail::SegmentDataContainerImpl<Ownership> &segment_data);
 template <storage::Ownership Ownership>
-inline void write(const boost::filesystem::path &path,
+inline void write(storage::io::FileWriter &writer,
                   const detail::SegmentDataContainerImpl<Ownership> &segment_data);
 }
 
@@ -190,11 +199,12 @@ template <storage::Ownership Ownership> class SegmentDataContainerImpl
     auto GetNumberOfGeometries() const { return index.size() - 1; }
     auto GetNumberOfSegments() const { return fwd_weights.size(); }
 
-    friend void io::read<Ownership>(const boost::filesystem::path &path,
-                                    detail::SegmentDataContainerImpl<Ownership> &segment_data);
     friend void
-    io::write<Ownership>(const boost::filesystem::path &path,
+    serialization::read<Ownership>(storage::io::FileReader &reader,
-                         const detail::SegmentDataContainerImpl<Ownership> &segment_data);
+                             detail::SegmentDataContainerImpl<Ownership> &segment_data);
+    friend void
+    serialization::write<Ownership>(storage::io::FileWriter &writer,
+                              const detail::SegmentDataContainerImpl<Ownership> &segment_data);
 
   private:
     Vector<std::uint32_t> index;

include/extractor/seralization.hpp

@@ -13,47 +13,32 @@ namespace osrm
 {
 namespace extractor
 {
-namespace io
+namespace serialization
 {
 
-inline void read(const boost::filesystem::path &path, std::vector<NBGToEBG> &mapping)
+inline void read(storage::io::FileReader &reader, std::vector<NBGToEBG> &mapping)
 {
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
-
     reader.DeserializeVector(mapping);
 }
 
-inline void write(const boost::filesystem::path &path, const std::vector<NBGToEBG> &mapping)
+inline void write(storage::io::FileWriter &writer, const std::vector<NBGToEBG> &mapping)
 {
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
+    writer.SerializeVector(mapping);
-    storage::io::FileWriter reader{path, fingerprint};
-
-    reader.SerializeVector(mapping);
 }
 
-inline void read(const boost::filesystem::path &path, Datasources &sources)
+inline void read(storage::io::FileReader &reader, Datasources &sources)
 {
-    const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
-
     reader.ReadInto(sources);
 }
 
-inline void write(const boost::filesystem::path &path, Datasources &sources)
+inline void write(storage::io::FileWriter &writer, Datasources &sources)
 {
-    const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
-
     writer.WriteFrom(sources);
 }
 
 template <>
-inline void read(const boost::filesystem::path &path, SegmentDataContainer &segment_data)
+inline void read(storage::io::FileReader &reader, SegmentDataContainer &segment_data)
 {
-    const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
-
     auto num_indices = reader.ReadElementCount32();
     segment_data.index.resize(num_indices);
     reader.ReadInto(segment_data.index.data(), num_indices);
@@ -75,11 +60,8 @@ inline void read(const boost::filesystem::path &path, SegmentDataContainer &segm
 }
 
 template <>
-inline void write(const boost::filesystem::path &path, const SegmentDataContainer &segment_data)
+inline void write(storage::io::FileWriter &writer, const SegmentDataContainer &segment_data)
 {
-    const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
-
     writer.WriteElementCount32(segment_data.index.size());
     writer.WriteFrom(segment_data.index);
 

src/extractor/edge_based_graph_factory.cpp

@@ -2,7 +2,7 @@
 #include "extractor/edge_based_edge.hpp"
 #include "extractor/guidance/turn_analysis.hpp"
 #include "extractor/guidance/turn_lane_handler.hpp"
-#include "extractor/io.hpp"
+#include "extractor/files.hpp"
 #include "extractor/scripting_environment.hpp"
 #include "extractor/suffix_table.hpp"
 
@@ -205,7 +205,7 @@ void EdgeBasedGraphFactory::Run(ScriptingEnvironment &scripting_environment,
     m_edge_based_node_weights.reserve(m_max_edge_id + 1);
     {
         auto mapping = GenerateEdgeExpandedNodes();
-        io::write(cnbg_ebg_mapping_path, mapping);
+        files::writeNBGMapping(cnbg_ebg_mapping_path, mapping);
     }
     TIMER_STOP(generate_nodes);
 

src/extractor/extractor.cpp

@@ -5,7 +5,7 @@
 #include "extractor/extraction_node.hpp"
 #include "extractor/extraction_way.hpp"
 #include "extractor/extractor_callbacks.hpp"
-#include "extractor/io.hpp"
+#include "extractor/files.hpp"
 #include "extractor/raster_source.hpp"
 #include "extractor/restriction_parser.hpp"
 #include "extractor/scripting_environment.hpp"
@@ -512,7 +512,7 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment,
     });
 
     WriteTurnLaneData(config.turn_lane_descriptions_file_name);
-    io::write(config.geometry_output_path, *compressed_edge_container.ToSegmentData());
+    files::writeSegmentData(config.geometry_output_path, *compressed_edge_container.ToSegmentData());
 
     edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list);
     edge_based_graph_factory.GetEdgeBasedNodes(node_based_edge_list);

src/partition/partitioner.cpp

@@ -9,7 +9,7 @@
 #include "partition/recursive_bisection.hpp"
 #include "partition/remove_unconnected.hpp"
 
-#include "extractor/io.hpp"
+#include "extractor/files.hpp"
 
 #include "util/coordinate.hpp"
 #include "util/geojson_debug_logger.hpp"
@@ -126,7 +126,7 @@ int Partitioner::Run(const PartitionConfig &config)
     // For details see #3205
 
     std::vector<extractor::NBGToEBG> mapping;
-    extractor::io::read(config.cnbg_ebg_mapping_path.string(), mapping);
+    extractor::files::readNBGMapping(config.cnbg_ebg_mapping_path.string(), mapping);
     util::Log() << "Loaded node based graph to edge based graph mapping";
 
     auto edge_based_graph = LoadEdgeBasedGraph(config.edge_based_graph_path.string());

src/storage/storage.cpp

@@ -4,7 +4,7 @@
 #include "extractor/compressed_edge_container.hpp"
 #include "extractor/edge_based_edge.hpp"
 #include "extractor/guidance/turn_instruction.hpp"
-#include "extractor/io.hpp"
+#include "extractor/files.hpp"
 #include "extractor/original_edge_data.hpp"
 #include "extractor/profile_properties.hpp"
 #include "extractor/query_node.hpp"
@@ -686,7 +686,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     {
         const auto datasources_names_ptr = layout.GetBlockPtr<extractor::Datasources, true>(
             memory_ptr, DataLayout::DATASOURCES_NAMES);
-        extractor::io::read(config.datasource_names_path, *datasources_names_ptr);
+        extractor::files::readDatasources(config.datasource_names_path, *datasources_names_ptr);
     }
 
     // Loading list of coordinates

src/updater/updater.cpp

@@ -2,12 +2,14 @@
 
 #include "updater/csv_source.hpp"
 
+#include "extractor/files.hpp"
 #include "extractor/compressed_edge_container.hpp"
 #include "extractor/edge_based_graph_factory.hpp"
-#include "extractor/io.hpp"
+#include "extractor/seralization.hpp"
 #include "extractor/node_based_edge.hpp"
 
 #include "storage/io.hpp"
+
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
 #include "util/for_each_pair.hpp"
@@ -85,7 +87,7 @@ void checkWeightsConsistency(
     using OriginalEdgeData = osrm::extractor::OriginalEdgeData;
 
     extractor::SegmentDataContainer segment_data;
-    extractor::io::read(config.geometry_path, segment_data);
+    extractor::files::readSegmentData(config.geometry_path, segment_data);
 
     Reader edges_input_file(config.osrm_input_path.string() + ".edges", Reader::HasNoFingerprint);
     std::vector<OriginalEdgeData> current_edge_data(edges_input_file.ReadElementCount64());
@@ -361,7 +363,7 @@ void saveDatasourcesNames(const UpdaterConfig &config)
         source++;
     }
 
-    extractor::io::write(config.datasource_names_path, sources);
+    extractor::files::writeDatasources(config.datasource_names_path, sources);
 }
 
 std::vector<std::uint64_t>
@@ -459,7 +461,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
     if (update_edge_weights || update_turn_penalties)
     {
         const auto load_segment_data = [&] {
-            extractor::io::read(config.geometry_path, segment_data);
+            extractor::files::readSegmentData(config.geometry_path, segment_data);
         };
 
         const auto load_edge_data = [&] {
@@ -503,7 +505,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
         updated_segments =
             updateSegmentData(config, profile_properties, segment_speed_lookup, segment_data);
         // Now save out the updated compressed geometries
-        extractor::io::write(config.geometry_path, segment_data);
+        extractor::files::writeSegmentData(config.geometry_path, segment_data);
         TIMER_STOP(segment);
         util::Log() << "Updating segment data took " << TIMER_MSEC(segment) << "ms.";
     }