Port .nbg_nodes file to tar

include/extractor/files.hpp

@@ -116,11 +116,11 @@ inline void readNodes(const boost::filesystem::path &path,
     static_assert(std::is_same<typename CoordinatesT::value_type, util::Coordinate>::value, "");
     static_assert(std::is_same<typename PackedOSMIDsT::value_type, OSMNodeID>::value, "");
 
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
+    storage::tar::FileReader reader{path, fingerprint};
 
-    storage::serialization::read(reader, coordinates);
+    storage::serialization::read(reader, "/common/coordinates", coordinates);
-    util::serialization::read(reader, osm_node_ids);
+    util::serialization::read(reader, "/common/osm_node_ids", osm_node_ids);
 }
 
 // writes .osrm.nodes
@@ -132,11 +132,11 @@ inline void writeNodes(const boost::filesystem::path &path,
     static_assert(std::is_same<typename CoordinatesT::value_type, util::Coordinate>::value, "");
     static_assert(std::is_same<typename PackedOSMIDsT::value_type, OSMNodeID>::value, "");
 
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
+    storage::tar::FileWriter writer{path, fingerprint};
 
-    storage::serialization::write(writer, coordinates);
+    storage::serialization::write(writer, "/common/coordinates", coordinates);
-    util::serialization::write(writer, osm_node_ids);
+    util::serialization::write(writer, "/common/osm_node_ids", osm_node_ids);
 }
 
 // reads .osrm.cnbg_to_ebg

include/util/packed_vector.hpp

@@ -7,6 +7,7 @@
 
 #include "storage/io_fwd.hpp"
 #include "storage/shared_memory_ownership.hpp"
+#include "storage/tar_fwd.hpp"
 
 #include <boost/iterator/iterator_facade.hpp>
 #include <boost/iterator/reverse_iterator.hpp>
@@ -34,6 +35,16 @@ inline void read(storage::io::FileReader &reader, detail::PackedVector<T, Bits,
 template <typename T, std::size_t Bits, storage::Ownership Ownership>
 inline void write(storage::io::FileWriter &writer,
                   const detail::PackedVector<T, Bits, Ownership> &vec);
+
+template <typename T, std::size_t Bits, storage::Ownership Ownership>
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 detail::PackedVector<T, Bits, Ownership> &vec);
+
+template <typename T, std::size_t Bits, storage::Ownership Ownership>
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
+                  const detail::PackedVector<T, Bits, Ownership> &vec);
 }
 
 namespace detail
@@ -451,6 +462,15 @@ template <typename T, std::size_t Bits, storage::Ownership Ownership> class Pack
 
     friend void serialization::write<T, Bits, Ownership>(storage::io::FileWriter &writer,
                                                          const PackedVector &vec);
+
+    friend void serialization::read<T, Bits, Ownership>(storage::tar::FileReader &reader,
+                                                        const std::string &name,
+                                                        PackedVector &vec);
+
+    friend void serialization::write<T, Bits, Ownership>(storage::tar::FileWriter &writer,
+                                                         const std::string &name,
+                                                         const PackedVector &vec);
+
     inline void swap(PackedVector &other) noexcept
     {
         std::swap(vec, other.vec);

include/util/serialization.hpp

@@ -32,7 +32,9 @@ void read(storage::io::FileReader &reader, util::RangeTable<BlockSize, Ownership
 }
 
 template <unsigned BlockSize, storage::Ownership Ownership>
-void write(storage::tar::FileWriter &writer, const std::string& name, const util::RangeTable<BlockSize, Ownership> &table)
+void write(storage::tar::FileWriter &writer,
+           const std::string &name,
+           const util::RangeTable<BlockSize, Ownership> &table)
 {
     writer.WriteOne(name + "/sum_lengths.meta", table.sum_lengths);
     storage::serialization::write(writer, name + "/block_offsets", table.block_offsets);
@@ -40,7 +42,9 @@ void write(storage::tar::FileWriter &writer, const std::string& name, const util
 }
 
 template <unsigned BlockSize, storage::Ownership Ownership>
-void read(storage::tar::FileReader &reader, const std::string& name, util::RangeTable<BlockSize, Ownership> &table)
+void read(storage::tar::FileReader &reader,
+          const std::string &name,
+          util::RangeTable<BlockSize, Ownership> &table)
 {
     table.sum_lengths = reader.ReadOne<unsigned>(name + "/sum_lengths.meta");
     storage::serialization::read(reader, name + "/block_offsets", table.block_offsets);
@@ -62,6 +66,24 @@ inline void write(storage::io::FileWriter &writer,
     storage::serialization::write(writer, vec.vec);
 }
 
+template <typename T, std::size_t Bits, storage::Ownership Ownership>
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 detail::PackedVector<T, Bits, Ownership> &vec)
+{
+    vec.num_elements = reader.ReadOne<std::uint64_t>(name + "/number_of_elements.meta");
+    storage::serialization::read(reader, name + "/packed", vec.vec);
+}
+
+template <typename T, std::size_t Bits, storage::Ownership Ownership>
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
+                  const detail::PackedVector<T, Bits, Ownership> &vec)
+{
+    writer.WriteOne(name + "/number_of_elements.meta", vec.num_elements);
+    storage::serialization::write(writer, name + "/packed", vec.vec);
+}
+
 template <typename EdgeDataT, storage::Ownership Ownership>
 inline void read(storage::io::FileReader &reader, StaticGraph<EdgeDataT, Ownership> &graph)
 {

src/storage/storage.cpp

@@ -328,23 +328,6 @@ void Storage::PopulateLayout(DataLayout &layout)
                         make_block<TurnPenalty>(number_of_penalties));
     }
 
-    // load coordinate size
-    {
-        io::FileReader node_file(config.GetPath(".osrm.nbg_nodes"),
-                                 io::FileReader::VerifyFingerprint);
-        const auto coordinate_list_size = node_file.ReadElementCount64();
-        layout.SetBlock(DataLayout::COORDINATE_LIST,
-                        make_block<util::Coordinate>(coordinate_list_size));
-        node_file.Skip<util::Coordinate>(coordinate_list_size);
-        // skip number of elements
-        node_file.Skip<std::uint64_t>(1);
-        const auto num_id_blocks = node_file.ReadElementCount64();
-        // we'll read a list of OSM node IDs from the same data, so set the block size for the same
-        // number of items:
-        layout.SetBlock(DataLayout::OSM_NODE_ID_LIST,
-                        make_block<extractor::PackedOSMIDsView::block_type>(num_id_blocks));
-    }
-
     // load geometries sizes
     {
         io::FileReader reader(config.GetPath(".osrm.geometry"), io::FileReader::VerifyFingerprint);
@@ -462,6 +445,8 @@ void Storage::PopulateLayout(DataLayout &layout)
         {"/common/intersection_bearings/class_id_to_ranges/diff_blocks", DataLayout::BEARING_BLOCKS},
         {"/common/entry_classes", DataLayout::ENTRY_CLASS},
         {"/common/properties", DataLayout::PROPERTIES},
+        {"/common/coordinates", DataLayout::COORDINATE_LIST},
+        {"/common/osm_node_ids/packed", DataLayout::OSM_NODE_ID_LIST},
     };
     std::vector<NamedBlock> blocks;
 
@@ -474,7 +459,8 @@ void Storage::PopulateLayout(DataLayout &layout)
         {OPTIONAL, config.GetPath(".osrm.cell_metrics")},
         {OPTIONAL, config.GetPath(".osrm.hsgr")},
         {REQUIRED, config.GetPath(".osrm.icd")},
-        {REQUIRED, config.GetPath(".osrm.properties")}
+        {REQUIRED, config.GetPath(".osrm.properties")},
+        {REQUIRED, config.GetPath(".osrm.nbg_nodes")}
     };
 
     for (const auto &file : tar_files)