Port .osrm.icd file over to tar

CMakeLists.txt

@@ -169,7 +169,7 @@ add_executable(osrm-routed src/tools/routed.cpp $<TARGET_OBJECTS:SERVER> $<TARGE
 add_executable(osrm-datastore src/tools/store.cpp $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
 add_library(osrm src/osrm/osrm.cpp $<TARGET_OBJECTS:ENGINE> $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL> )
 add_library(osrm_contract src/osrm/contractor.cpp $<TARGET_OBJECTS:CONTRACTOR> $<TARGET_OBJECTS:UTIL>)
-add_library(osrm_extract src/osrm/extractor.cpp $<TARGET_OBJECTS:EXTRACTOR> $<TARGET_OBJECTS:UTIL>)
+add_library(osrm_extract src/osrm/extractor.cpp $<TARGET_OBJECTS:EXTRACTOR> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
 add_library(osrm_guidance $<TARGET_OBJECTS:GUIDANCE> $<TARGET_OBJECTS:UTIL>)
 add_library(osrm_partition src/osrm/partitioner.cpp $<TARGET_OBJECTS:PARTITIONER> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
 add_library(osrm_customize src/osrm/customizer.cpp $<TARGET_OBJECTS:CUSTOMIZER> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)

include/extractor/files.hpp

@@ -33,10 +33,10 @@ inline void writeIntersections(const boost::filesystem::path &path,
                       std::is_same<IntersectionBearingsView, IntersectionBearingsT>::value,
                   "");
 
-    storage::io::FileWriter writer(path, storage::io::FileWriter::GenerateFingerprint);
+    storage::tar::FileWriter writer(path, storage::tar::FileWriter::GenerateFingerprint);
 
-    serialization::write(writer, intersection_bearings);
+    serialization::write(writer, "/common/intersection_bearings", intersection_bearings);
-    storage::serialization::write(writer, entry_classes);
+    storage::serialization::write(writer, "/common/entry_classes", entry_classes);
 }
 
 // read the .osrm.icd file
@@ -49,10 +49,10 @@ inline void readIntersections(const boost::filesystem::path &path,
                       std::is_same<IntersectionBearingsView, IntersectionBearingsT>::value,
                   "");
 
-    storage::io::FileReader reader(path, storage::io::FileReader::VerifyFingerprint);
+    storage::tar::FileReader reader(path, storage::tar::FileReader::VerifyFingerprint);
 
-    serialization::read(reader, intersection_bearings);
+    serialization::read(reader, "/common/intersection_bearings", intersection_bearings);
-    storage::serialization::read(reader, entry_classes);
+    storage::serialization::read(reader, "/common/entry_classes", entry_classes);
 }
 
 // reads .osrm.properties

include/extractor/intersection_bearings_container.hpp

@@ -2,6 +2,7 @@
 #define OSRM_EXTRACTOR_BEARING_CONTAINER_HPP
 
 #include "storage/shared_memory_ownership.hpp"
+#include "storage/tar_fwd.hpp"
 
 #include "util/guidance/bearing_class.hpp"
 #include "util/range_table.hpp"
@@ -21,11 +22,13 @@ template <storage::Ownership Ownership> class IntersectionBearingsContainer;
 namespace serialization
 {
 template <storage::Ownership Ownership>
-void read(storage::io::FileReader &reader,
+void read(storage::tar::FileReader &reader,
+          const std::string& name,
           detail::IntersectionBearingsContainer<Ownership> &turn_data);
 
 template <storage::Ownership Ownership>
-void write(storage::io::FileWriter &writer,
+void write(storage::tar::FileWriter &writer,
+          const std::string& name,
            const detail::IntersectionBearingsContainer<Ownership> &turn_data);
 }
 
@@ -83,10 +86,12 @@ template <storage::Ownership Ownership> class IntersectionBearingsContainer
         return result;
     }
 
-    friend void serialization::read<Ownership>(storage::io::FileReader &reader,
+    friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
+                                               const std::string &name,
                                                IntersectionBearingsContainer &turn_data_container);
     friend void
-    serialization::write<Ownership>(storage::io::FileWriter &writer,
+    serialization::write<Ownership>(storage::tar::FileWriter &writer,
+                                    const std::string &name,
                                     const IntersectionBearingsContainer &turn_data_container);
 
   private:

include/extractor/serialization.hpp

@@ -25,21 +25,23 @@ namespace serialization
 
 // read/write for bearing data
 template <storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader,
+inline void read(storage::tar::FileReader &reader,
+                 const std::string& name,
                  detail::IntersectionBearingsContainer<Ownership> &intersection_bearings)
 {
-    storage::serialization::read(reader, intersection_bearings.values);
+    storage::serialization::read(reader, name + "/bearing_values", intersection_bearings.values);
-    storage::serialization::read(reader, intersection_bearings.node_to_class_id);
+    storage::serialization::read(reader, name + "/node_to_class_id", intersection_bearings.node_to_class_id);
-    util::serialization::read(reader, intersection_bearings.class_id_to_ranges_table);
+    util::serialization::read(reader, name + "/class_id_to_ranges", intersection_bearings.class_id_to_ranges_table);
 }
 
 template <storage::Ownership Ownership>
-inline void write(storage::io::FileWriter &writer,
+inline void write(storage::tar::FileWriter &writer,
+                 const std::string& name,
                   const detail::IntersectionBearingsContainer<Ownership> &intersection_bearings)
 {
-    storage::serialization::write(writer, intersection_bearings.values);
+    storage::serialization::write(writer, name + "/bearing_values", intersection_bearings.values);
-    storage::serialization::write(writer, intersection_bearings.node_to_class_id);
+    storage::serialization::write(writer, name + "/node_to_class_id", intersection_bearings.node_to_class_id);
-    util::serialization::write(writer, intersection_bearings.class_id_to_ranges_table);
+    util::serialization::write(writer, name + "/class_id_to_ranges", intersection_bearings.class_id_to_ranges_table);
 }
 
 // read/write for properties file

include/util/range_table.hpp

@@ -1,8 +1,8 @@
 #ifndef RANGE_TABLE_HPP
 #define RANGE_TABLE_HPP
 
-#include "storage/io.hpp"
 #include "storage/shared_memory_ownership.hpp"
+#include "storage/tar_fwd.hpp"
 #include "util/integer_range.hpp"
 #include "util/vector_view.hpp"
 
@@ -25,10 +25,14 @@ class RangeTable;
 namespace serialization
 {
 template <unsigned BlockSize, storage::Ownership Ownership>
-void write(storage::io::FileWriter &writer, const util::RangeTable<BlockSize, Ownership> &table);
+void write(storage::tar::FileWriter &writer,
+           const std::string &name,
+           const util::RangeTable<BlockSize, Ownership> &table);
 
 template <unsigned BlockSize, storage::Ownership Ownership>
-void read(storage::io::FileReader &reader, util::RangeTable<BlockSize, Ownership> &table);
+void read(storage::tar::FileReader &reader,
+          const std::string &name,
+          util::RangeTable<BlockSize, Ownership> &table);
 }
 
 /**
@@ -177,9 +181,11 @@ template <unsigned BLOCK_SIZE, storage::Ownership Ownership> class RangeTable
         return irange(begin_idx, end_idx);
     }
 
-    friend void serialization::write<BLOCK_SIZE, Ownership>(storage::io::FileWriter &writer,
+    friend void serialization::write<BLOCK_SIZE, Ownership>(storage::tar::FileWriter &writer,
+                                                            const std::string &name,
                                                             const RangeTable &table);
-    friend void serialization::read<BLOCK_SIZE, Ownership>(storage::io::FileReader &reader,
+    friend void serialization::read<BLOCK_SIZE, Ownership>(storage::tar::FileReader &reader,
+                                                           const std::string &name,
                                                            RangeTable &table);
 
   private:

include/util/serialization.hpp

@@ -31,6 +31,22 @@ void read(storage::io::FileReader &reader, util::RangeTable<BlockSize, Ownership
     storage::serialization::read(reader, table.diff_blocks);
 }
 
+template <unsigned BlockSize, storage::Ownership Ownership>
+void write(storage::tar::FileWriter &writer, const std::string& name, const util::RangeTable<BlockSize, Ownership> &table)
+{
+    writer.WriteOne(name + "/sum_lengths.meta", table.sum_lengths);
+    storage::serialization::write(writer, name + "/block_offsets", table.block_offsets);
+    storage::serialization::write(writer, name + "/diff_blocks", table.diff_blocks);
+}
+
+template <unsigned BlockSize, storage::Ownership Ownership>
+void read(storage::tar::FileReader &reader, const std::string& name, util::RangeTable<BlockSize, Ownership> &table)
+{
+    table.sum_lengths = reader.ReadOne<unsigned>(name + "/sum_lengths.meta");
+    storage::serialization::read(reader, name + "/block_offsets", table.block_offsets);
+    storage::serialization::read(reader, name + "/diff_blocks", table.diff_blocks);
+}
+
 template <typename T, std::size_t Bits, storage::Ownership Ownership>
 inline void read(storage::io::FileReader &reader, detail::PackedVector<T, Bits, Ownership> &vec)
 {

src/storage/storage.cpp

@@ -399,30 +399,6 @@ void Storage::PopulateLayout(DataLayout &layout)
     }
 
     {
-        io::FileReader reader(config.GetPath(".osrm.icd"), io::FileReader::VerifyFingerprint);
-
-        auto num_discreate_bearings = reader.ReadVectorSize<DiscreteBearing>();
-        layout.SetBlock(DataLayout::BEARING_VALUES,
-                        make_block<DiscreteBearing>(num_discreate_bearings));
-
-        auto num_bearing_classes = reader.ReadVectorSize<BearingClassID>();
-        layout.SetBlock(DataLayout::BEARING_CLASSID,
-                        make_block<BearingClassID>(num_bearing_classes));
-
-        reader.Skip<std::uint32_t>(1); // sum_lengths
-        const auto bearing_blocks = reader.ReadVectorSize<unsigned>();
-        const auto bearing_offsets =
-            reader
-                .ReadVectorSize<typename util::RangeTable<16, storage::Ownership::View>::BlockT>();
-
-        layout.SetBlock(DataLayout::BEARING_OFFSETS, make_block<unsigned>(bearing_blocks));
-        layout.SetBlock(DataLayout::BEARING_BLOCKS,
-                        make_block<typename util::RangeTable<16, storage::Ownership::View>::BlockT>(
-                            bearing_offsets));
-
-        auto num_entry_classes = reader.ReadVectorSize<util::guidance::EntryClass>();
-        layout.SetBlock(DataLayout::ENTRY_CLASS,
-                        make_block<util::guidance::EntryClass>(num_entry_classes));
     }
 
     {
@@ -487,6 +463,11 @@ void Storage::PopulateLayout(DataLayout &layout)
         {"/ch/edge_filter/5", DataLayout::CH_EDGE_FILTER_5},
         {"/ch/edge_filter/6", DataLayout::CH_EDGE_FILTER_6},
         {"/ch/edge_filter/7", DataLayout::CH_EDGE_FILTER_7},
+        {"/common/intersection_bearings/bearing_values", DataLayout::BEARING_VALUES},
+        {"/common/intersection_bearings/node_to_class_id", DataLayout::BEARING_CLASSID},
+        {"/common/intersection_bearings/class_id_to_ranges/block_offsets", DataLayout::BEARING_OFFSETS},
+        {"/common/intersection_bearings/class_id_to_ranges/diff_blocks", DataLayout::BEARING_BLOCKS},
+        {"/common/entry_classes", DataLayout::ENTRY_CLASS},
     };
     std::vector<NamedBlock> blocks;
 
@@ -497,7 +478,8 @@ void Storage::PopulateLayout(DataLayout &layout)
         {OPTIONAL, config.GetPath(".osrm.cells")},
         {OPTIONAL, config.GetPath(".osrm.partition")},
         {OPTIONAL, config.GetPath(".osrm.cell_metrics")},
-        {OPTIONAL, config.GetPath(".osrm.hsgr")}
+        {OPTIONAL, config.GetPath(".osrm.hsgr")},
+        {REQUIRED, config.GetPath(".osrm.icd")}
     };
 
     for (const auto &file : tar_files)

unit_tests/util/serialization.cpp

@@ -0,0 +1,38 @@
+#include "util/serialization.hpp"
+
+#include "../common/range_tools.hpp"
+#include "../common/temporary_file.hpp"
+
+#include <boost/filesystem.hpp>
+#include <boost/test/unit_test.hpp>
+
+BOOST_AUTO_TEST_SUITE(serialization)
+
+using namespace osrm;
+using namespace osrm::util;
+
+BOOST_AUTO_TEST_CASE(tar_serialize_range_table)
+{
+    TemporaryFile tmp;
+    {
+        constexpr unsigned BLOCK_SIZE = 16;
+        using TestRangeTable = RangeTable<BLOCK_SIZE, osrm::storage::Ownership::Container>;
+
+        std::vector<TestRangeTable> data = {
+        };
+
+        for (const auto &v : data)
+        {
+            {
+                storage::tar::FileWriter writer(tmp.path, storage::tar::FileWriter::GenerateFingerprint);
+                util::serialization::write(writer, "my_range_table", v);
+            }
+
+            TestRangeTable result;
+            storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint);
+            util::serialization::read(reader, "my_range_table", result);
+        }
+    }
+}
+
+BOOST_AUTO_TEST_SUITE_END()