Serialize mldgr using the new tar writer

CMakeLists.txt

@@ -167,13 +167,13 @@ add_executable(osrm-customize src/tools/customize.cpp)
 add_executable(osrm-contract src/tools/contract.cpp)
 add_executable(osrm-routed src/tools/routed.cpp $<TARGET_OBJECTS:SERVER> $<TARGET_OBJECTS:UTIL>)
 add_executable(osrm-datastore src/tools/store.cpp $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
-add_library(osrm src/osrm/osrm.cpp $<TARGET_OBJECTS:ENGINE> $<TARGET_OBJECTS:UTIL> $<TARGET_OBJECTS:STORAGE>)
+add_library(osrm src/osrm/osrm.cpp $<TARGET_OBJECTS:ENGINE> $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL> )
 add_library(osrm_contract src/osrm/contractor.cpp $<TARGET_OBJECTS:CONTRACTOR> $<TARGET_OBJECTS:UTIL>)
 add_library(osrm_extract src/osrm/extractor.cpp $<TARGET_OBJECTS:EXTRACTOR> $<TARGET_OBJECTS:UTIL>)
 add_library(osrm_guidance $<TARGET_OBJECTS:GUIDANCE> $<TARGET_OBJECTS:UTIL>)
-add_library(osrm_partition src/osrm/partitioner.cpp $<TARGET_OBJECTS:PARTITIONER> $<TARGET_OBJECTS:UTIL>)
+add_library(osrm_partition src/osrm/partitioner.cpp $<TARGET_OBJECTS:PARTITIONER> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
-add_library(osrm_customize src/osrm/customizer.cpp $<TARGET_OBJECTS:CUSTOMIZER> $<TARGET_OBJECTS:UTIL>)
+add_library(osrm_customize src/osrm/customizer.cpp $<TARGET_OBJECTS:CUSTOMIZER> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
-add_library(osrm_update $<TARGET_OBJECTS:UPDATER> $<TARGET_OBJECTS:UTIL>)
+add_library(osrm_update $<TARGET_OBJECTS:UPDATER> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
 add_library(osrm_store $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
 
 if(ENABLE_GOLD_LINKER)
@@ -560,6 +560,7 @@ include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
 set(MICROTAR_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/microtar/src")
 include_directories(SYSTEM ${MICROTAR_INCLUDE_DIR})
 add_library(MICROTAR OBJECT "${CMAKE_CURRENT_SOURCE_DIR}/third_party/microtar/src/microtar.c")
+set_property(TARGET MICROTAR PROPERTY POSITION_INDEPENDENT_CODE ON)
 
 # prefix compilation with ccache by default if available and on clang or gcc
 if(ENABLE_CCACHE AND (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU"))

include/partitioner/files.hpp

@@ -24,7 +24,7 @@ inline void readGraph(const boost::filesystem::path &path,
                       std::is_same<customizer::MultiLevelEdgeBasedGraph, MultiLevelGraphT>::value,
                   "");
 
-    storage::tar::FileReader reader{path};
+    storage::tar::FileReader reader{path, storage::tar::FileReader::VerifyFingerprint};
 
     serialization::read(reader, "/mld/multilevelgraph", graph, connectivity_checksum);
 }
@@ -39,7 +39,7 @@ inline void writeGraph(const boost::filesystem::path &path,
                       std::is_same<customizer::MultiLevelEdgeBasedGraph, MultiLevelGraphT>::value,
                   "");
 
-    storage::tar::FileWriter writer{path};
+    storage::tar::FileWriter writer{path, storage::tar::FileWriter::GenerateFingerprint};
 
     serialization::write(writer, "/mld/multilevelgraph", graph, connectivity_checksum);
 }

include/partitioner/multi_level_graph.hpp

@@ -3,7 +3,7 @@
 
 #include "partitioner/multi_level_partition.hpp"
 
-#include "storage/io_fwd.hpp"
+#include "storage/tar_fwd.hpp"
 #include "storage/shared_memory_ownership.hpp"
 
 #include "util/static_graph.hpp"
@@ -24,12 +24,14 @@ template <typename EdgeDataT, storage::Ownership Ownership> class MultiLevelGrap
 namespace serialization
 {
 template <typename EdgeDataT, storage::Ownership Ownership>
-void read(storage::io::FileReader &reader,
+void read(storage::tar::FileReader &reader,
+          const std::string& name,
           MultiLevelGraph<EdgeDataT, Ownership> &graph,
           std::uint32_t &connectivity_checksum);
 
 template <typename EdgeDataT, storage::Ownership Ownership>
-void write(storage::io::FileWriter &writer,
+void write(storage::tar::FileWriter &writer,
+          const std::string& name,
            const MultiLevelGraph<EdgeDataT, Ownership> &graph,
            const std::uint32_t connectivity_checksum);
 }
@@ -202,11 +204,13 @@ class MultiLevelGraph : public util::StaticGraph<EdgeDataT, Ownership>
     }
 
     friend void
-    serialization::read<EdgeDataT, Ownership>(storage::io::FileReader &reader,
+    serialization::read<EdgeDataT, Ownership>(storage::tar::FileReader &reader,
+                                              const std::string& name,
                                               MultiLevelGraph<EdgeDataT, Ownership> &graph,
                                               std::uint32_t &connectivity_checksum);
     friend void
-    serialization::write<EdgeDataT, Ownership>(storage::io::FileWriter &writer,
+    serialization::write<EdgeDataT, Ownership>(storage::tar::FileWriter &writer,
+                                               const std::string& name,
                                                const MultiLevelGraph<EdgeDataT, Ownership> &graph,
                                                const std::uint32_t connectivity_checksum);
 

include/partitioner/serialization.hpp

@@ -40,6 +40,7 @@ inline void write(storage::tar::FileWriter &writer,
     storage::serialization::write(writer, name + "/node_array", graph.node_array);
     storage::serialization::write(writer, name + "/edge_array", graph.edge_array);
     storage::serialization::write(writer, name + "/node_to_edge_offset", graph.node_to_edge_offset);
+    writer.WriteElementCount64(name + "/connectivity_checksum", 1);
     writer.WriteOne(name + "/connectivity_checksum", connectivity_checksum);
 }
 

include/storage/block.hpp

@@ -14,10 +14,12 @@ namespace storage
 
 struct Block
 {
-    std::uint64_t num_entries;
+    std::uint64_t num_entries = 0;
-    std::uint64_t byte_size;
+    std::uint64_t byte_size = 0;
 };
 
+using NamedBlock = std::tuple<std::string, Block>;
+
 template <typename T> Block make_block(uint64_t num_entries)
 {
     static_assert(sizeof(T) % alignof(T) == 0, "aligned T* can't be used as an array pointer");

include/storage/shared_datatype.hpp

@@ -92,7 +92,8 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
                                             "MLD_GRAPH_EDGE_LIST",
                                             "MLD_GRAPH_NODE_TO_OFFSET",
                                             "MANEUVER_OVERRIDES",
-                                            "MANEUVER_OVERRIDE_NODE_SEQUENCES"};
+                                            "MANEUVER_OVERRIDE_NODE_SEQUENCES",
+                                            "IGNORE_BLOCK"};
 
 class DataLayout
 {
@@ -172,6 +173,7 @@ class DataLayout
         MLD_GRAPH_NODE_TO_OFFSET,
         MANEUVER_OVERRIDES,
         MANEUVER_OVERRIDE_NODE_SEQUENCES,
+        IGNORE_BLOCK,
         NUM_BLOCKS
     };
 

include/storage/tar.hpp

@@ -22,23 +22,31 @@ namespace tar
 class FileReader
 {
   public:
-    FileReader(const boost::filesystem::path &path) : path(path)
+    enum FingerprintFlag
+    {
+        VerifyFingerprint,
+        HasNoFingerprint
+    };
+
+    FileReader(const boost::filesystem::path &path, FingerprintFlag flag) : path(path)
     {
         auto ret = mtar_open(&handle, path.c_str(), "r");
         if (ret != MTAR_ESUCCESS)
         {
             throw util::exception(mtar_strerror(ret));
         }
+
+        if (flag == VerifyFingerprint)
+        {
+            ReadAndCheckFingerprint();
+        }
     }
 
-    ~FileReader()
+    ~FileReader() { mtar_close(&handle); }
-    {
-        mtar_close(&handle);
-    }
 
     std::uint64_t ReadElementCount64(const std::string &name)
     {
-        return ReadOne<std::uint64_t>(name + "_count");
+        return ReadOne<std::uint64_t>(name + ".meta");
     }
 
     template <typename T> T ReadOne(const std::string &name)
@@ -49,24 +57,25 @@ class FileReader
     }
 
     template <typename T>
-    void ReadInto(const std::string &name, T *data, const std::size_t number_of_entries)
+    void ReadInto(const std::string &name, T *data, const std::size_t number_of_elements)
     {
         mtar_header_t header;
         auto ret = mtar_find(&handle, name.c_str(), &header);
         if (ret != MTAR_ESUCCESS)
         {
-            throw util::exception(mtar_strerror(ret));
+            throw util::exception(name + ": " + mtar_strerror(ret));
         }
 
-        if (header.size != sizeof(T) * number_of_entries)
+        auto expected_size = sizeof(T) * number_of_elements;
+        if (header.size != expected_size)
         {
-            throw util::exception("Datatype size does not match file size.");
+            throw util::exception(name + ": Datatype size does not match file size.");
         }
 
         ret = mtar_read_data(&handle, reinterpret_cast<char *>(data), header.size);
         if (ret != MTAR_ESUCCESS)
         {
-            throw util::exception(mtar_strerror(ret));
+            throw util::exception(name + ": Failed reading data: " + mtar_strerror(ret));
         }
     }
 
@@ -87,7 +96,7 @@ class FileReader
   private:
     bool ReadAndCheckFingerprint()
     {
-        auto loaded_fingerprint = ReadOne<util::FingerPrint>("osrm_fingerprint");
+        auto loaded_fingerprint = ReadOne<util::FingerPrint>("osrm_fingerprint.meta");
         const auto expected_fingerprint = util::FingerPrint::GetValid();
 
         if (!loaded_fingerprint.IsValid())
@@ -117,13 +126,23 @@ class FileReader
 class FileWriter
 {
   public:
-    FileWriter(const boost::filesystem::path &path) : path(path)
+    enum FingerprintFlag
+    {
+        GenerateFingerprint,
+        HasNoFingerprint
+    };
+
+    FileWriter(const boost::filesystem::path &path, FingerprintFlag flag) : path(path)
     {
         auto ret = mtar_open(&handle, path.c_str(), "w");
         if (ret != MTAR_ESUCCESS)
             throw util::exception(mtar_strerror(ret));
+
+        if (flag == GenerateFingerprint)
+        {
             WriteFingerprint();
         }
+    }
 
     ~FileWriter()
     {
@@ -133,7 +152,7 @@ class FileWriter
 
     void WriteElementCount64(const std::string &name, const std::uint64_t count)
     {
-        WriteOne(name + "_count", count);
+        WriteOne(name + ".meta", count);
     }
 
     template <typename T> void WriteOne(const std::string &name, const T &data)
@@ -142,18 +161,20 @@ class FileWriter
     }
 
     template <typename T>
-    void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_entries)
+    void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_elements)
     {
-        auto number_of_bytes = number_of_entries * sizeof(T);
+        auto number_of_bytes = number_of_elements * sizeof(T);
+
         auto ret = mtar_write_file_header(&handle, name.c_str(), number_of_bytes);
         if (ret != MTAR_ESUCCESS)
         {
-            throw util::exception(mtar_strerror(ret));
+            throw util::exception(name + ": Error reading header: " + mtar_strerror(ret));
         }
+
         ret = mtar_write_data(&handle, reinterpret_cast<const char *>(data), number_of_bytes);
         if (ret != MTAR_ESUCCESS)
         {
-            throw util::exception(mtar_strerror(ret));
+            throw util::exception(name + ": Error reading data : " + mtar_strerror(ret));
         }
     }
 
@@ -161,13 +182,12 @@ class FileWriter
     void WriteFingerprint()
     {
         const auto fingerprint = util::FingerPrint::GetValid();
-        WriteOne("osrm_fingerprint", fingerprint);
+        WriteOne("osrm_fingerprint.meta", fingerprint);
     }
 
     boost::filesystem::path path;
     mtar_t handle;
 };
-
 }
 }
 }

include/storage/tar_fwd.hpp

@@ -0,0 +1,18 @@
+#ifndef OSRM_STORAGE_TAR_FWD_HPP_
+#define OSRM_STORAGE_TAR_FWD_HPP_
+
+namespace osrm
+{
+namespace storage
+{
+namespace tar
+{
+
+class FileReader;
+class FileWriter;
+
+} // ns io
+} // ns storage
+} // ns osrm
+
+#endif

src/storage/storage.cpp

@@ -67,6 +67,32 @@ namespace osrm
 {
 namespace storage
 {
+namespace
+{
+    template<typename OutIter>
+    void readBlocks(const boost::filesystem::path& path, OutIter out)
+    {
+        tar::FileReader reader(path, tar::FileReader::VerifyFingerprint);
+
+        std::vector<tar::FileReader::TarEntry> entries;
+        reader.List(std::back_inserter(entries));
+
+        for (const auto& entry : entries)
+        {
+            std::string name;
+            std::uint64_t size;
+            std::tie(name, size) = entry;
+
+            const auto name_end = name.rfind(".meta");
+            if (name_end == std::string::npos)
+            {
+                auto number_of_elements = reader.ReadElementCount64(name);
+                *out++ = NamedBlock {name, Block{number_of_elements, size}};
+            }
+        }
+
+    }
+}
 
 static constexpr std::size_t NUM_METRICS = 8;
 
@@ -572,35 +598,27 @@ void Storage::PopulateLayout(DataLayout &layout)
             layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_7, make_block<char>(0));
         }
 
+        std::unordered_map<std::string, DataLayout::BlockID> name_to_block_id = {
+            {"/mld/multilevelgraph/node_array", DataLayout::MLD_GRAPH_NODE_LIST},
+            {"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST},
+            {"/mld/multilevelgraph/node_to_edge_offset", DataLayout::MLD_GRAPH_NODE_TO_OFFSET},
+            {"/mld/multilevelgraph/connectivity_checksum", DataLayout::IGNORE_BLOCK},
+        };
+        std::vector<NamedBlock> blocks;
+
         if (boost::filesystem::exists(config.GetPath(".osrm.mldgr")))
         {
-            io::FileReader reader(config.GetPath(".osrm.mldgr"), io::FileReader::VerifyFingerprint);
+            readBlocks(config.GetPath(".osrm.mldgr"), std::back_inserter(blocks));
-
-            const auto num_nodes =
-                reader.ReadVectorSize<customizer::MultiLevelEdgeBasedGraph::NodeArrayEntry>();
-            const auto num_edges =
-                reader.ReadVectorSize<customizer::MultiLevelEdgeBasedGraph::EdgeArrayEntry>();
-            const auto num_node_offsets =
-                reader.ReadVectorSize<customizer::MultiLevelEdgeBasedGraph::EdgeOffset>();
-
-            layout.SetBlock(
-                DataLayout::MLD_GRAPH_NODE_LIST,
-                make_block<customizer::MultiLevelEdgeBasedGraph::NodeArrayEntry>(num_nodes));
-            layout.SetBlock(
-                DataLayout::MLD_GRAPH_EDGE_LIST,
-                make_block<customizer::MultiLevelEdgeBasedGraph::EdgeArrayEntry>(num_edges));
-            layout.SetBlock(
-                DataLayout::MLD_GRAPH_NODE_TO_OFFSET,
-                make_block<customizer::MultiLevelEdgeBasedGraph::EdgeOffset>(num_node_offsets));
         }
-        else
+
+        for (const auto& block : blocks)
         {
-            layout.SetBlock(DataLayout::MLD_GRAPH_NODE_LIST,
+            auto id_iter = name_to_block_id.find(std::get<0>(block));
-                            make_block<customizer::MultiLevelEdgeBasedGraph::NodeArrayEntry>(0));
+            if (id_iter == name_to_block_id.end())
-            layout.SetBlock(DataLayout::MLD_GRAPH_EDGE_LIST,
+            {
-                            make_block<customizer::MultiLevelEdgeBasedGraph::EdgeArrayEntry>(0));
+                throw util::exception("Could not map " + std::get<0>(block) + " to a region in memory.");
-            layout.SetBlock(DataLayout::MLD_GRAPH_NODE_TO_OFFSET,
+            }
-                            make_block<customizer::MultiLevelEdgeBasedGraph::EdgeOffset>(0));
+            layout.SetBlock(id_iter->second, std::get<1>(block));
         }
     }
 }

unit_tests/storage/tar.cpp

@@ -10,7 +10,8 @@ using namespace osrm;
 
 BOOST_AUTO_TEST_CASE(list_tar_file)
 {
-    storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar");
+    storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar",
+                                    storage::tar::FileReader::HasNoFingerprint);
 
     std::vector<storage::tar::FileReader::TarEntry> file_list;
     reader.List(std::back_inserter(file_list));
@@ -29,7 +30,8 @@ BOOST_AUTO_TEST_CASE(list_tar_file)
 
 BOOST_AUTO_TEST_CASE(read_tar_file)
 {
-    storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar");
+    storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar",
+                                    storage::tar::FileReader::HasNoFingerprint);
 
     char result_0[4];
     reader.ReadInto("foo_1.txt", result_0, 4);
@@ -58,7 +60,7 @@ BOOST_AUTO_TEST_CASE(write_tar_file)
         0, 1, 2, 3, 4, 1ULL << 62, 0, 1 << 22, 0xFFFFFFFFFFFFFFFF};
 
     {
-        storage::tar::FileWriter writer(tmp_path);
+        storage::tar::FileWriter writer(tmp_path, storage::tar::FileWriter::GenerateFingerprint);
         writer.WriteOne("foo/single_64bit_integer", single_64bit_integer);
         writer.WriteOne("bar/single_32bit_integer", single_32bit_integer);
         writer.WriteElementCount64("baz/bla/64bit_vector", vector_64bit.size());
@@ -67,7 +69,7 @@ BOOST_AUTO_TEST_CASE(write_tar_file)
         writer.WriteFrom("32bit_vector", vector_32bit.data(), vector_32bit.size());
     }
 
-    storage::tar::FileReader reader(tmp_path);
+    storage::tar::FileReader reader(tmp_path, storage::tar::FileReader::VerifyFingerprint);
 
     BOOST_CHECK_EQUAL(reader.ReadOne<std::uint32_t>("bar/single_32bit_integer"),
                       single_32bit_integer);