Merge pull request #4960 from Project-OSRM/refactor/tar_files

Use tar-format to encapsulate data

CHANGELOG.md

@@ -17,6 +17,7 @@
       - `OSRM` object accepts a new option `memory_file` that stores the memory in a file on disk.
     - Internals
       - CHANGED #4845 #4968: Updated segregated intersection identification
+      - REMOVED: Remove `.timestamp` file since it was unused.
     - Documentation:
       - ADDED: Add documentation about OSM node ids in nearest service response [#4436](https://github.com/Project-OSRM/osrm-backend/pull/4436)
     - Performance

CMakeLists.txt

@@ -166,15 +166,15 @@ add_executable(osrm-partition src/tools/partition.cpp)
 add_executable(osrm-customize src/tools/customize.cpp)
 add_executable(osrm-contract src/tools/contract.cpp)
 add_executable(osrm-routed src/tools/routed.cpp $<TARGET_OBJECTS:SERVER> $<TARGET_OBJECTS:UTIL>)
-add_executable(osrm-datastore src/tools/store.cpp $<TARGET_OBJECTS:UTIL>)
-add_library(osrm src/osrm/osrm.cpp $<TARGET_OBJECTS:ENGINE> $<TARGET_OBJECTS:UTIL> $<TARGET_OBJECTS:STORAGE>)
+add_executable(osrm-datastore src/tools/store.cpp $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
+add_library(osrm src/osrm/osrm.cpp $<TARGET_OBJECTS:ENGINE> $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL> )
 add_library(osrm_contract src/osrm/contractor.cpp $<TARGET_OBJECTS:CONTRACTOR> $<TARGET_OBJECTS:UTIL>)
-add_library(osrm_extract src/osrm/extractor.cpp $<TARGET_OBJECTS:EXTRACTOR> $<TARGET_OBJECTS:UTIL>)
+add_library(osrm_extract src/osrm/extractor.cpp $<TARGET_OBJECTS:EXTRACTOR> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
 add_library(osrm_guidance $<TARGET_OBJECTS:GUIDANCE> $<TARGET_OBJECTS:UTIL>)
-add_library(osrm_partition src/osrm/partitioner.cpp $<TARGET_OBJECTS:PARTITIONER> $<TARGET_OBJECTS:UTIL>)
-add_library(osrm_customize src/osrm/customizer.cpp $<TARGET_OBJECTS:CUSTOMIZER> $<TARGET_OBJECTS:UTIL>)
-add_library(osrm_update $<TARGET_OBJECTS:UPDATER> $<TARGET_OBJECTS:UTIL>)
-add_library(osrm_store $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:UTIL>)
+add_library(osrm_partition src/osrm/partitioner.cpp $<TARGET_OBJECTS:PARTITIONER> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
+add_library(osrm_customize src/osrm/customizer.cpp $<TARGET_OBJECTS:CUSTOMIZER> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
+add_library(osrm_update $<TARGET_OBJECTS:UPDATER> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
+add_library(osrm_store $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
 
 if(ENABLE_GOLD_LINKER)
     execute_process(COMMAND ${CMAKE_C_COMPILER} -fuse-ld=gold -Wl,--version ERROR_QUIET OUTPUT_VARIABLE LD_VERSION)
@@ -495,9 +495,6 @@ if(ENABLE_MASON)
   # note: we avoid calling find_package(Osmium ...) here to ensure that the
   # expat and bzip2 are used from mason rather than the system
   include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/libosmium/include)
-
-  include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/rapidjson/include)
-
 else()
 
   find_package(Boost 1.54 REQUIRED COMPONENTS ${BOOST_COMPONENTS})
@@ -555,12 +552,16 @@ else()
   endif()
   find_package(Osmium REQUIRED COMPONENTS io)
   include_directories(SYSTEM ${OSMIUM_INCLUDE_DIR})
-
-  set(RAPIDJSON_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/rapidjson/include")
-  include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
-
 endif()
 
+set(RAPIDJSON_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/rapidjson/include")
+include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
+
+set(MICROTAR_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/microtar/src")
+include_directories(SYSTEM ${MICROTAR_INCLUDE_DIR})
+add_library(MICROTAR OBJECT "${CMAKE_CURRENT_SOURCE_DIR}/third_party/microtar/src/microtar.c")
+set_property(TARGET MICROTAR PROPERTY POSITION_INDEPENDENT_CODE ON)
+
 # prefix compilation with ccache by default if available and on clang or gcc
 if(ENABLE_CCACHE AND (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU"))
   find_program(CCACHE_FOUND ccache)
@@ -699,7 +700,7 @@ target_link_libraries(osrm_customize ${CUSTOMIZER_LIBRARIES} osrm_update osrm_st
 target_link_libraries(osrm_store ${STORAGE_LIBRARIES})
 
 # BUILD_COMPONENTS
-add_executable(osrm-components src/tools/components.cpp $<TARGET_OBJECTS:UTIL>)
+add_executable(osrm-components src/tools/components.cpp $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
 target_link_libraries(osrm-components ${TBB_LIBRARIES} ${BOOST_BASE_LIBRARIES} ${UTIL_LIBRARIES})
 install(TARGETS osrm-components DESTINATION bin)
 

include/contractor/files.hpp

@@ -5,8 +5,8 @@
 
 #include "util/serialization.hpp"
 
-#include "storage/io.hpp"
 #include "storage/serialization.hpp"
+#include "storage/tar.hpp"
 
 namespace osrm
 {
@@ -29,18 +29,21 @@ inline void readGraph(const boost::filesystem::path &path,
                       std::is_same<EdgeFilterT, util::vector_view<bool>>::value,
                   "edge_filter must be a container of vector<bool> or vector_view<bool>");
 
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
 
-    reader.ReadInto(checksum);
-    util::serialization::read(reader, graph);
-    auto count = reader.ReadElementCount64();
+    reader.ReadInto("/ch/checksum", checksum);
+    util::serialization::read(reader, "/ch/contracted_graph", graph);
+
+    auto count = reader.ReadElementCount64("/ch/edge_filter");
     edge_filter.resize(count);
     for (const auto index : util::irange<std::size_t>(0, count))
     {
-        storage::serialization::read(reader, edge_filter[index]);
+        storage::serialization::read(
+            reader, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]);
     }
-    reader.ReadInto(connectivity_checksum);
+
+    reader.ReadInto("/ch/connectivity_checksum", connectivity_checksum);
 }
 
 // writes .osrm.hsgr file
@@ -57,17 +60,22 @@ inline void writeGraph(const boost::filesystem::path &path,
     static_assert(std::is_same<EdgeFilterT, std::vector<bool>>::value ||
                       std::is_same<EdgeFilterT, util::vector_view<bool>>::value,
                   "edge_filter must be a container of vector<bool> or vector_view<bool>");
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
 
-    writer.WriteOne(checksum);
-    util::serialization::write(writer, graph);
-    writer.WriteElementCount64(edge_filter.size());
-    for (const auto &filter : edge_filter)
+    writer.WriteElementCount64("/ch/checksum", 1);
+    writer.WriteFrom("/ch/checksum", checksum);
+    util::serialization::write(writer, "/ch/contracted_graph", graph);
+
+    writer.WriteElementCount64("/ch/edge_filter", edge_filter.size());
+    for (const auto index : util::irange<std::size_t>(0, edge_filter.size()))
     {
-        storage::serialization::write(writer, filter);
+        storage::serialization::write(
+            writer, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]);
     }
-    writer.WriteOne(connectivity_checksum);
+
+    writer.WriteElementCount64("/ch/connectivity_checksum", 1);
+    writer.WriteFrom("/ch/connectivity_checksum", connectivity_checksum);
 }
 }
 }

include/customizer/files.hpp

@@ -3,7 +3,7 @@
 
 #include "customizer/serialization.hpp"
 
-#include "storage/io.hpp"
+#include "storage/tar.hpp"
 
 #include "util/integer_range.hpp"
 
@@ -22,15 +22,16 @@ inline void readCellMetrics(const boost::filesystem::path &path, std::vector<Cel
                       std::is_same<CellMetric, CellMetricT>::value,
                   "");
 
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
 
-    auto num_metrics = reader.ReadElementCount64();
+    auto num_metrics = reader.ReadElementCount64("/mld/metrics");
     metrics.resize(num_metrics);
 
+    auto id = 0;
     for (auto &metric : metrics)
     {
-        serialization::read(reader, metric);
+        serialization::read(reader, "/mld/metrics/" + std::to_string(id++), metric);
     }
 }
 
@@ -43,13 +44,15 @@ inline void writeCellMetrics(const boost::filesystem::path &path,
                       std::is_same<CellMetric, CellMetricT>::value,
                   "");
 
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
 
-    writer.WriteElementCount64(metrics.size());
+    writer.WriteElementCount64("/mld/metrics", metrics.size());
+
+    auto id = 0;
     for (const auto &metric : metrics)
     {
-        serialization::write(writer, metric);
+        serialization::write(writer, "/mld/metrics/" + std::to_string(id++), metric);
     }
 }
 }

include/customizer/serialization.hpp

@@ -3,9 +3,9 @@
 
 #include "partitioner/cell_storage.hpp"
 
-#include "storage/io.hpp"
 #include "storage/serialization.hpp"
 #include "storage/shared_memory_ownership.hpp"
+#include "storage/tar.hpp"
 
 namespace osrm
 {
@@ -15,17 +15,21 @@ namespace serialization
 {
 
 template <storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader, detail::CellMetricImpl<Ownership> &metric)
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 detail::CellMetricImpl<Ownership> &metric)
 {
-    storage::serialization::read(reader, metric.weights);
-    storage::serialization::read(reader, metric.durations);
+    storage::serialization::read(reader, name + "/weights", metric.weights);
+    storage::serialization::read(reader, name + "/durations", metric.durations);
 }
 
 template <storage::Ownership Ownership>
-inline void write(storage::io::FileWriter &writer, const detail::CellMetricImpl<Ownership> &metric)
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
+                  const detail::CellMetricImpl<Ownership> &metric)
 {
-    storage::serialization::write(writer, metric.weights);
-    storage::serialization::write(writer, metric.durations);
+    storage::serialization::write(writer, name + "/weights", metric.weights);
+    storage::serialization::write(writer, name + "/durations", metric.durations);
 }
 }
 }

include/engine/datafacade/contiguous_internalmem_datafacade.hpp

@@ -15,6 +15,7 @@
 #include "extractor/edge_based_node.hpp"
 #include "extractor/intersection_bearings_container.hpp"
 #include "extractor/maneuver_override.hpp"
+#include "extractor/name_table.hpp"
 #include "extractor/node_data_container.hpp"
 #include "extractor/packed_osm_ids.hpp"
 #include "extractor/profile_properties.hpp"
@@ -40,7 +41,6 @@
 #include "util/guidance/entry_class.hpp"
 #include "util/guidance/turn_lanes.hpp"
 #include "util/log.hpp"
-#include "util/name_table.hpp"
 #include "util/packed_vector.hpp"
 #include "util/range_table.hpp"
 #include "util/rectangle.hpp"
@@ -94,7 +94,8 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
         auto filter_block_id = static_cast<storage::DataLayout::BlockID>(
             storage::DataLayout::CH_EDGE_FILTER_0 + exclude_index);
 
-        auto edge_filter_ptr = data_layout.GetBlockPtr<unsigned>(memory_block, filter_block_id);
+        auto edge_filter_ptr =
+            data_layout.GetBlockPtr<util::vector_view<bool>::Word>(memory_block, filter_block_id);
 
         util::vector_view<GraphNode> node_list(
             graph_nodes_ptr, data_layout.GetBlockEntries(storage::DataLayout::CH_GRAPH_NODE_LIST));
@@ -215,7 +216,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
 
     extractor::IntersectionBearingsView intersection_bearings_view;
 
-    util::NameTable m_name_table;
+    extractor::NameTableView m_name_table;
     // the look-up table for entry classes. An entry class lists the possibility of entry for all
     // available turns. Such a class id is stored with every edge.
     util::vector_view<util::guidance::EntryClass> m_entry_class_table;
@@ -233,16 +234,6 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
         exclude_mask = m_profile_properties->excludable_classes[exclude_index];
     }
 
-    void InitializeTimestampPointer(storage::DataLayout &data_layout, char *memory_block)
-    {
-        auto timestamp_ptr =
-            data_layout.GetBlockPtr<char>(memory_block, storage::DataLayout::TIMESTAMP);
-        m_timestamp.resize(data_layout.GetBlockSize(storage::DataLayout::TIMESTAMP));
-        std::copy(timestamp_ptr,
-                  timestamp_ptr + data_layout.GetBlockSize(storage::DataLayout::TIMESTAMP),
-                  m_timestamp.begin());
-    }
-
     void InitializeChecksumPointer(storage::DataLayout &data_layout, char *memory_block)
     {
         m_check_sum =
@@ -264,17 +255,21 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
                                   "Is any data loaded into shared memory?" + SOURCE_REF);
         }
 
-        auto tree_nodes_ptr =
+        const auto rtree_ptr =
             data_layout.GetBlockPtr<RTreeNode>(memory_block, storage::DataLayout::R_SEARCH_TREE);
-        auto tree_level_sizes_ptr = data_layout.GetBlockPtr<std::uint64_t>(
-            memory_block, storage::DataLayout::R_SEARCH_TREE_LEVELS);
-        m_static_rtree.reset(
-            new SharedRTree(tree_nodes_ptr,
-                            data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE),
-                            tree_level_sizes_ptr,
-                            data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE_LEVELS),
-                            file_index_path,
-                            m_coordinate_list));
+        util::vector_view<RTreeNode> search_tree(
+            rtree_ptr, data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE));
+
+        const auto rtree_levelstarts_ptr = data_layout.GetBlockPtr<std::uint64_t>(
+            memory_block, storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS);
+        util::vector_view<std::uint64_t> rtree_level_starts(
+            rtree_levelstarts_ptr,
+            data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS));
+
+        m_static_rtree.reset(new SharedRTree{std::move(search_tree),
+                                             std::move(rtree_level_starts),
+                                             file_index_path,
+                                             m_coordinate_list});
         m_geospatial_query.reset(
             new SharedGeospatialQuery(*m_static_rtree, m_coordinate_list, *this));
     }
@@ -353,11 +348,20 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
 
     void InitializeNamePointers(storage::DataLayout &data_layout, char *memory_block)
     {
-        auto name_data_ptr =
-            data_layout.GetBlockPtr<char>(memory_block, storage::DataLayout::NAME_CHAR_DATA);
-        const auto name_data_size =
-            data_layout.GetBlockEntries(storage::DataLayout::NAME_CHAR_DATA);
-        m_name_table.reset(name_data_ptr, name_data_ptr + name_data_size);
+        const auto name_blocks_ptr =
+            data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::BlockReference>(
+                memory_block, storage::DataLayout::NAME_BLOCKS);
+        const auto name_values_ptr =
+            data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::ValueType>(
+                memory_block, storage::DataLayout::NAME_VALUES);
+
+        util::vector_view<extractor::NameTableView::IndexedData::BlockReference> blocks(
+            name_blocks_ptr, data_layout.GetBlockEntries(storage::DataLayout::NAME_BLOCKS));
+        util::vector_view<extractor::NameTableView::IndexedData::ValueType> values(
+            name_values_ptr, data_layout.GetBlockEntries(storage::DataLayout::NAME_VALUES));
+
+        extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
+        m_name_table = extractor::NameTableView{std::move(index_data_view)};
     }
 
     void InitializeTurnLaneDescriptionsPointers(storage::DataLayout &data_layout,
@@ -534,7 +538,6 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
         InitializeEdgeInformationPointers(data_layout, memory_block);
         InitializeTurnPenalties(data_layout, memory_block);
         InitializeGeometryPointers(data_layout, memory_block);
-        InitializeTimestampPointer(data_layout, memory_block);
         InitializeNamePointers(data_layout, memory_block);
         InitializeTurnLaneDescriptionsPointers(data_layout, memory_block);
         InitializeProfilePropertiesPointer(data_layout, memory_block, exclude_index);
@@ -842,8 +845,6 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
         return m_datasources->GetSourceName(id);
     }
 
-    std::string GetTimestamp() const override final { return m_timestamp; }
-
     bool GetContinueStraightDefault() const override final
     {
         return m_profile_properties->continue_straight_at_waypoint;

include/engine/datafacade/datafacade_base.hpp

@@ -173,8 +173,6 @@ class BaseDataFacade
 
     virtual StringView GetExitsForID(const NameID id) const = 0;
 
-    virtual std::string GetTimestamp() const = 0;
-
     virtual bool GetContinueStraightDefault() const = 0;
 
     virtual double GetMapMatchingMaxSpeed() const = 0;

include/engine/engine.hpp

@@ -157,13 +157,7 @@ bool Engine<routing_algorithms::ch::Algorithm>::CheckCompatibility(const EngineC
     }
     else
     {
-        if (!boost::filesystem::exists(config.storage_config.GetPath(".osrm.hsgr")))
-            return false;
-        storage::io::FileReader in(config.storage_config.GetPath(".osrm.hsgr"),
-                                   storage::io::FileReader::VerifyFingerprint);
-
-        auto size = in.GetSize();
-        return size > 0;
+        return boost::filesystem::exists(config.storage_config.GetPath(".osrm.hsgr"));
     }
 }
 
@@ -196,16 +190,10 @@ bool Engine<routing_algorithms::mld::Algorithm>::CheckCompatibility(const Engine
     }
     else
     {
-        if (!boost::filesystem::exists(config.storage_config.GetPath(".osrm.partition")) ||
-            !boost::filesystem::exists(config.storage_config.GetPath(".osrm.cells")) ||
-            !boost::filesystem::exists(config.storage_config.GetPath(".osrm.mldgr")) ||
-            !boost::filesystem::exists(config.storage_config.GetPath(".osrm.cell_metrics")))
-            return false;
-        storage::io::FileReader in(config.storage_config.GetPath(".osrm.partition"),
-                                   storage::io::FileReader::VerifyFingerprint);
-
-        auto size = in.GetSize();
-        return size > 0;
+        return boost::filesystem::exists(config.storage_config.GetPath(".osrm.partition")) &&
+               boost::filesystem::exists(config.storage_config.GetPath(".osrm.cells")) &&
+               boost::filesystem::exists(config.storage_config.GetPath(".osrm.mldgr")) &&
+               boost::filesystem::exists(config.storage_config.GetPath(".osrm.cell_metrics"));
     }
 }
 }

include/extractor/compressed_node_based_graph_edge.hpp

@@ -0,0 +1,20 @@
+#ifndef OSRM_EXTRACTOR_COMPRESSED_NODE_BASED_GRAPH_EDGE_HPP
+#define OSRM_EXTRACTOR_COMPRESSED_NODE_BASED_GRAPH_EDGE_HPP
+
+#include "util/typedefs.hpp"
+
+namespace osrm
+{
+namespace extractor
+{
+
+// We encode the cnbg graph only using its topology as edge list
+struct CompressedNodeBasedGraphEdge
+{
+    NodeID source;
+    NodeID target;
+};
+}
+}
+
+#endif // OSRM_EXTRACTOR_COMPRESSED_NODE_BASED_GRAPH_EDGE_HPP

include/extractor/edge_based_graph_factory.hpp

@@ -9,6 +9,7 @@
 #include "extractor/edge_based_node_segment.hpp"
 #include "extractor/extraction_turn.hpp"
 #include "extractor/maneuver_override.hpp"
+#include "extractor/name_table.hpp"
 #include "extractor/nbg_to_ebg.hpp"
 #include "extractor/node_data_container.hpp"
 #include "extractor/query_node.hpp"
@@ -18,7 +19,6 @@
 
 #include "util/concurrent_id_map.hpp"
 #include "util/deallocating_vector.hpp"
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 #include "util/typedefs.hpp"
 
@@ -70,7 +70,7 @@ class EdgeBasedGraphFactory
                                    const std::unordered_set<NodeID> &barrier_nodes,
                                    const std::unordered_set<NodeID> &traffic_lights,
                                    const std::vector<util::Coordinate> &coordinates,
-                                   const util::NameTable &name_table,
+                                   const NameTable &name_table,
                                    const std::unordered_set<EdgeID> &segregated_edges,
                                    const LaneDescriptionMap &lane_description_map);
 
@@ -138,7 +138,7 @@ class EdgeBasedGraphFactory
     const std::unordered_set<NodeID> &m_traffic_lights;
     const CompressedEdgeContainer &m_compressed_edge_container;
 
-    const util::NameTable &name_table;
+    const NameTable &name_table;
     const std::unordered_set<EdgeID> &segregated_edges;
     const LaneDescriptionMap &lane_description_map;
 

include/extractor/extraction_containers.hpp

@@ -7,7 +7,7 @@
 #include "extractor/restriction.hpp"
 #include "extractor/scripting_environment.hpp"
 
-#include "storage/io.hpp"
+#include "storage/tar_fwd.hpp"
 
 namespace osrm
 {
@@ -27,9 +27,9 @@ class ExtractionContainers
     void PrepareRestrictions();
     void PrepareEdges(ScriptingEnvironment &scripting_environment);
 
-    void WriteNodes(storage::io::FileWriter &file_out) const;
-    void WriteEdges(storage::io::FileWriter &file_out) const;
-    void WriteMetadata(storage::io::FileWriter &file_out) const;
+    void WriteNodes(storage::tar::FileWriter &file_out) const;
+    void WriteEdges(storage::tar::FileWriter &file_out) const;
+    void WriteMetadata(storage::tar::FileWriter &file_out) const;
     void WriteCharData(const std::string &file_name);
 
   public:

include/extractor/extractor.hpp

@@ -77,7 +77,7 @@ class Extractor
         const std::vector<TurnRestriction> &turn_restrictions,
         const std::vector<ConditionalTurnRestriction> &conditional_turn_restrictions,
         const std::unordered_set<EdgeID> &segregated_edges,
-        const util::NameTable &name_table,
+        const NameTable &name_table,
         const std::vector<UnresolvedManeuverOverride> &maneuver_overrides,
         const LaneDescriptionMap &turn_lane_map,
         // for calculating turn penalties
@@ -99,11 +99,6 @@ class Extractor
                     const std::vector<util::Coordinate> &coordinates);
     std::shared_ptr<RestrictionMap> LoadRestrictionMap();
 
-    // Writes compressed node based graph and its embedding into a file for osrm-partition to use.
-    static void WriteCompressedNodeBasedGraph(const std::string &path,
-                                              const util::NodeBasedDynamicGraph &graph,
-                                              const std::vector<util::Coordinate> &coordiantes);
-
     void WriteConditionalRestrictions(
         const std::string &path,
         std::vector<ConditionalTurnRestriction> &conditional_turn_restrictions);
@@ -116,7 +111,7 @@ class Extractor
         const std::unordered_set<NodeID> &barrier_nodes,
         const std::vector<TurnRestriction> &turn_restrictions,
         const std::vector<ConditionalTurnRestriction> &conditional_turn_restrictions,
-        const util::NameTable &name_table,
+        const NameTable &name_table,
         LaneDescriptionMap lane_description_map,
         ScriptingEnvironment &scripting_environment);
 };

include/extractor/extractor_config.hpp

@@ -52,7 +52,6 @@ struct ExtractorConfig final : storage::IOConfig
                                       ".osrm.names",
                                       ".osrm.tls",
                                       ".osrm.tld",
-                                      ".osrm.timestamp",
                                       ".osrm.geometry",
                                       ".osrm.nbg_nodes",
                                       ".osrm.ebg_nodes",

include/extractor/files.hpp

@@ -4,12 +4,14 @@
 #include "extractor/edge_based_edge.hpp"
 #include "extractor/node_data_container.hpp"
 #include "extractor/profile_properties.hpp"
+#include "extractor/query_node.hpp"
 #include "extractor/serialization.hpp"
 #include "extractor/turn_lane_types.hpp"
 
 #include "util/coordinate.hpp"
 #include "util/guidance/bearing_class.hpp"
 #include "util/guidance/entry_class.hpp"
+#include "util/guidance/turn_lanes.hpp"
 #include "util/packed_vector.hpp"
 #include "util/range_table.hpp"
 #include "util/serialization.hpp"
@@ -33,10 +35,10 @@ inline void writeIntersections(const boost::filesystem::path &path,
                       std::is_same<IntersectionBearingsView, IntersectionBearingsT>::value,
                   "");
 
-    storage::io::FileWriter writer(path, storage::io::FileWriter::GenerateFingerprint);
+    storage::tar::FileWriter writer(path, storage::tar::FileWriter::GenerateFingerprint);
 
-    serialization::write(writer, intersection_bearings);
-    storage::serialization::write(writer, entry_classes);
+    serialization::write(writer, "/common/intersection_bearings", intersection_bearings);
+    storage::serialization::write(writer, "/common/entry_classes", entry_classes);
 }
 
 // read the .osrm.icd file
@@ -49,30 +51,30 @@ inline void readIntersections(const boost::filesystem::path &path,
                       std::is_same<IntersectionBearingsView, IntersectionBearingsT>::value,
                   "");
 
-    storage::io::FileReader reader(path, storage::io::FileReader::VerifyFingerprint);
+    storage::tar::FileReader reader(path, storage::tar::FileReader::VerifyFingerprint);
 
-    serialization::read(reader, intersection_bearings);
-    storage::serialization::read(reader, entry_classes);
+    serialization::read(reader, "/common/intersection_bearings", intersection_bearings);
+    storage::serialization::read(reader, "/common/entry_classes", entry_classes);
 }
 
 // reads .osrm.properties
 inline void readProfileProperties(const boost::filesystem::path &path,
                                   ProfileProperties &properties)
 {
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
 
-    serialization::read(reader, properties);
+    serialization::read(reader, "/common/properties", properties);
 }
 
 // writes .osrm.properties
 inline void writeProfileProperties(const boost::filesystem::path &path,
                                    const ProfileProperties &properties)
 {
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
 
-    serialization::write(writer, properties);
+    serialization::write(writer, "/common/properties", properties);
 }
 
 template <typename EdgeBasedEdgeVector>
@@ -83,13 +85,16 @@ void writeEdgeBasedGraph(const boost::filesystem::path &path,
 {
     static_assert(std::is_same<typename EdgeBasedEdgeVector::value_type, EdgeBasedEdge>::value, "");
 
-    storage::io::FileWriter writer(path, storage::io::FileWriter::GenerateFingerprint);
+    storage::tar::FileWriter writer(path, storage::tar::FileWriter::GenerateFingerprint);
 
-    writer.WriteElementCount64(number_of_edge_based_nodes);
-    storage::serialization::write(writer, edge_based_edge_list);
-    writer.WriteOne(connectivity_checksum);
+    writer.WriteElementCount64("/common/number_of_edge_based_nodes", 1);
+    writer.WriteFrom("/common/number_of_edge_based_nodes", number_of_edge_based_nodes);
+    storage::serialization::write(writer, "/common/edge_based_edge_list", edge_based_edge_list);
+    writer.WriteElementCount64("/common/connectivity_checksum", 1);
+    writer.WriteFrom("/common/connectivity_checksum", connectivity_checksum);
 }
 
+// reads .osrm.ebg file
 template <typename EdgeBasedEdgeVector>
 void readEdgeBasedGraph(const boost::filesystem::path &path,
                         EdgeID &number_of_edge_based_nodes,
@@ -98,14 +103,14 @@ void readEdgeBasedGraph(const boost::filesystem::path &path,
 {
     static_assert(std::is_same<typename EdgeBasedEdgeVector::value_type, EdgeBasedEdge>::value, "");
 
-    storage::io::FileReader reader(path, storage::io::FileReader::VerifyFingerprint);
+    storage::tar::FileReader reader(path, storage::tar::FileReader::VerifyFingerprint);
 
-    number_of_edge_based_nodes = reader.ReadElementCount64();
-    storage::serialization::read(reader, edge_based_edge_list);
-    reader.ReadInto(connectivity_checksum);
+    reader.ReadInto("/common/number_of_edge_based_nodes", number_of_edge_based_nodes);
+    storage::serialization::read(reader, "/common/edge_based_edge_list", edge_based_edge_list);
+    reader.ReadInto("/common/connectivity_checksum", connectivity_checksum);
 }
 
-// reads .osrm.nodes
+// reads .osrm.nbg_nodes
 template <typename CoordinatesT, typename PackedOSMIDsT>
 inline void readNodes(const boost::filesystem::path &path,
                       CoordinatesT &coordinates,
@@ -114,14 +119,26 @@ inline void readNodes(const boost::filesystem::path &path,
     static_assert(std::is_same<typename CoordinatesT::value_type, util::Coordinate>::value, "");
     static_assert(std::is_same<typename PackedOSMIDsT::value_type, OSMNodeID>::value, "");
 
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
 
-    storage::serialization::read(reader, coordinates);
-    util::serialization::read(reader, osm_node_ids);
+    storage::serialization::read(reader, "/common/coordinates", coordinates);
+    util::serialization::read(reader, "/common/osm_node_ids", osm_node_ids);
 }
 
-// writes .osrm.nodes
+// reads only coordinates from .osrm.nbg_nodes
+template <typename CoordinatesT>
+inline void readNodeCoordinates(const boost::filesystem::path &path, CoordinatesT &coordinates)
+{
+    static_assert(std::is_same<typename CoordinatesT::value_type, util::Coordinate>::value, "");
+
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
+
+    storage::serialization::read(reader, "/common/coordinates", coordinates);
+}
+
+// writes .osrm.nbg_nodes
 template <typename CoordinatesT, typename PackedOSMIDsT>
 inline void writeNodes(const boost::filesystem::path &path,
                        const CoordinatesT &coordinates,
@@ -130,48 +147,48 @@ inline void writeNodes(const boost::filesystem::path &path,
     static_assert(std::is_same<typename CoordinatesT::value_type, util::Coordinate>::value, "");
     static_assert(std::is_same<typename PackedOSMIDsT::value_type, OSMNodeID>::value, "");
 
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
 
-    storage::serialization::write(writer, coordinates);
-    util::serialization::write(writer, osm_node_ids);
+    storage::serialization::write(writer, "/common/coordinates", coordinates);
+    util::serialization::write(writer, "/common/osm_node_ids", osm_node_ids);
 }
 
 // reads .osrm.cnbg_to_ebg
 inline void readNBGMapping(const boost::filesystem::path &path, std::vector<NBGToEBG> &mapping)
 {
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
 
-    storage::serialization::read(reader, mapping);
+    storage::serialization::read(reader, "/common/cnbg_to_ebg", mapping);
 }
 
 // writes .osrm.cnbg_to_ebg
 inline void writeNBGMapping(const boost::filesystem::path &path,
                             const std::vector<NBGToEBG> &mapping)
 {
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
 
-    storage::serialization::write(writer, mapping);
+    storage::serialization::write(writer, "/common/cnbg_to_ebg", mapping);
 }
 
 // reads .osrm.datasource_names
 inline void readDatasources(const boost::filesystem::path &path, Datasources &sources)
 {
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
 
-    serialization::read(reader, sources);
+    serialization::read(reader, "/common/data_sources_names", sources);
 }
 
 // writes .osrm.datasource_names
 inline void writeDatasources(const boost::filesystem::path &path, Datasources &sources)
 {
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
 
-    serialization::write(writer, sources);
+    serialization::write(writer, "/common/data_sources_names", sources);
 }
 
 // reads .osrm.geometry
@@ -181,10 +198,10 @@ inline void readSegmentData(const boost::filesystem::path &path, SegmentDataT &s
     static_assert(std::is_same<SegmentDataContainer, SegmentDataT>::value ||
                       std::is_same<SegmentDataView, SegmentDataT>::value,
                   "");
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
 
-    serialization::read(reader, segment_data);
+    serialization::read(reader, "/common/segment_data", segment_data);
 }
 
 // writes .osrm.geometry
@@ -194,10 +211,10 @@ inline void writeSegmentData(const boost::filesystem::path &path, const SegmentD
     static_assert(std::is_same<SegmentDataContainer, SegmentDataT>::value ||
                       std::is_same<SegmentDataView, SegmentDataT>::value,
                   "");
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
 
-    serialization::write(writer, segment_data);
+    serialization::write(writer, "/common/segment_data", segment_data);
 }
 
 // reads .osrm.ebg_nodes
@@ -208,10 +225,10 @@ inline void readNodeData(const boost::filesystem::path &path, NodeDataT &node_da
                       std::is_same<EdgeBasedNodeDataView, NodeDataT>::value ||
                       std::is_same<EdgeBasedNodeDataExternalContainer, NodeDataT>::value,
                   "");
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
 
-    serialization::read(reader, node_data);
+    serialization::read(reader, "/common/ebg_node_data", node_data);
 }
 
 // writes .osrm.ebg_nodes
@@ -222,10 +239,10 @@ inline void writeNodeData(const boost::filesystem::path &path, const NodeDataT &
                       std::is_same<EdgeBasedNodeDataView, NodeDataT>::value ||
                       std::is_same<EdgeBasedNodeDataExternalContainer, NodeDataT>::value,
                   "");
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
 
-    serialization::write(writer, node_data);
+    serialization::write(writer, "/common/ebg_node_data", node_data);
 }
 
 // reads .osrm.tls
@@ -238,11 +255,11 @@ inline void readTurnLaneDescriptions(const boost::filesystem::path &path,
                   "");
     static_assert(std::is_same<typename OffsetsT::value_type, std::uint32_t>::value, "");
 
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
 
-    storage::serialization::read(reader, turn_offsets);
-    storage::serialization::read(reader, turn_masks);
+    storage::serialization::read(reader, "/common/turn_lanes/offsets", turn_offsets);
+    storage::serialization::read(reader, "/common/turn_lanes/masks", turn_masks);
 }
 
 // writes .osrm.tls
@@ -255,11 +272,40 @@ inline void writeTurnLaneDescriptions(const boost::filesystem::path &path,
                   "");
     static_assert(std::is_same<typename OffsetsT::value_type, std::uint32_t>::value, "");
 
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
 
-    storage::serialization::write(writer, turn_offsets);
-    storage::serialization::write(writer, turn_masks);
+    storage::serialization::write(writer, "/common/turn_lanes/offsets", turn_offsets);
+    storage::serialization::write(writer, "/common/turn_lanes/masks", turn_masks);
+}
+
+// reads .osrm.tld
+template <typename TurnLaneDataT>
+inline void readTurnLaneData(const boost::filesystem::path &path, TurnLaneDataT &turn_lane_data)
+{
+    static_assert(
+        std::is_same<typename TurnLaneDataT::value_type, util::guidance::LaneTupleIdPair>::value,
+        "");
+
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
+
+    storage::serialization::read(reader, "/common/turn_lanes/data", turn_lane_data);
+}
+
+// writes .osrm.tld
+template <typename TurnLaneDataT>
+inline void writeTurnLaneData(const boost::filesystem::path &path,
+                              const TurnLaneDataT &turn_lane_data)
+{
+    static_assert(
+        std::is_same<typename TurnLaneDataT::value_type, util::guidance::LaneTupleIdPair>::value,
+        "");
+
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
+
+    storage::serialization::write(writer, "/common/turn_lanes/data", turn_lane_data);
 }
 
 // reads .osrm.maneuver_overrides
@@ -268,10 +314,13 @@ inline void readManeuverOverrides(const boost::filesystem::path &path,
                                   StorageManeuverOverrideT &maneuver_overrides,
                                   NodeSequencesT &node_sequences)
 {
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
 
-    serialization::read(reader, maneuver_overrides, node_sequences);
+    storage::serialization::read(
+        reader, "/common/maneuver_overrides/overrides", maneuver_overrides);
+    storage::serialization::read(
+        reader, "/common/maneuver_overrides/node_sequences", node_sequences);
 }
 
 // writes .osrm.maneuver_overrides
@@ -279,10 +328,183 @@ inline void writeManeuverOverrides(const boost::filesystem::path &path,
                                    const std::vector<StorageManeuverOverride> &maneuver_overrides,
                                    const std::vector<NodeID> &node_sequences)
 {
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
 
-    serialization::write(writer, maneuver_overrides, node_sequences);
+    storage::serialization::write(
+        writer, "/common/maneuver_overrides/overrides", maneuver_overrides);
+    storage::serialization::write(
+        writer, "/common/maneuver_overrides/node_sequences", node_sequences);
+}
+
+// writes .osrm.turn_weight_penalties
+template <typename TurnPenaltyT>
+inline void writeTurnWeightPenalty(const boost::filesystem::path &path,
+                                   const TurnPenaltyT &turn_penalty)
+{
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
+
+    storage::serialization::write(writer, "/common/turn_penalty/weight", turn_penalty);
+}
+
+// read .osrm.turn_weight_penalties
+template <typename TurnPenaltyT>
+inline void readTurnWeightPenalty(const boost::filesystem::path &path, TurnPenaltyT &turn_penalty)
+{
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
+
+    storage::serialization::read(reader, "/common/turn_penalty/weight", turn_penalty);
+}
+
+// writes .osrm.turn_duration_penalties
+template <typename TurnPenaltyT>
+inline void writeTurnDurationPenalty(const boost::filesystem::path &path,
+                                     const TurnPenaltyT &turn_penalty)
+{
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
+
+    storage::serialization::write(writer, "/common/turn_penalty/duration", turn_penalty);
+}
+
+// read .osrm.turn_weight_penalties
+template <typename TurnPenaltyT>
+inline void readTurnDurationPenalty(const boost::filesystem::path &path, TurnPenaltyT &turn_penalty)
+{
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
+
+    storage::serialization::read(reader, "/common/turn_penalty/duration", turn_penalty);
+}
+
+// writes .osrm.restrictions
+template <typename ConditionalRestrictionsT>
+inline void writeConditionalRestrictions(const boost::filesystem::path &path,
+                                         const ConditionalRestrictionsT &conditional_restrictions)
+{
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
+
+    serialization::write(writer, "/common/conditional_restrictions", conditional_restrictions);
+}
+
+// read .osrm.restrictions
+template <typename ConditionalRestrictionsT>
+inline void readConditionalRestrictions(const boost::filesystem::path &path,
+                                        ConditionalRestrictionsT &conditional_restrictions)
+{
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
+
+    serialization::read(reader, "/common/conditional_restrictions", conditional_restrictions);
+}
+
+// reads .osrm file which is a temporary file of osrm-extract
+template <typename BarrierOutIter, typename TrafficSignalsOutIter, typename PackedOSMIDsT>
+void readRawNBGraph(const boost::filesystem::path &path,
+                    BarrierOutIter barriers,
+                    TrafficSignalsOutIter traffic_signals,
+                    std::vector<util::Coordinate> &coordinates,
+                    PackedOSMIDsT &osm_node_ids,
+                    std::vector<extractor::NodeBasedEdge> &edge_list,
+                    std::vector<extractor::NodeBasedEdgeAnnotation> &annotations)
+{
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
+
+    auto number_of_nodes = reader.ReadElementCount64("/extractor/nodes");
+    coordinates.resize(number_of_nodes);
+    osm_node_ids.reserve(number_of_nodes);
+    auto index = 0;
+    auto decode = [&](const auto &current_node) {
+        coordinates[index].lon = current_node.lon;
+        coordinates[index].lat = current_node.lat;
+        osm_node_ids.push_back(current_node.node_id);
+        index++;
+    };
+    reader.ReadStreaming<extractor::QueryNode>("/extractor/nodes",
+                                               boost::make_function_output_iterator(decode));
+
+    reader.ReadStreaming<NodeID>("/extractor/barriers", barriers);
+
+    reader.ReadStreaming<NodeID>("/extractor/traffic_lights", traffic_signals);
+
+    storage::serialization::read(reader, "/extractor/edges", edge_list);
+    storage::serialization::read(reader, "/extractor/annotations", annotations);
+}
+
+template <typename NameTableT>
+void readNames(const boost::filesystem::path &path, NameTableT &table)
+{
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
+
+    serialization::read(reader, "/common/names", table);
+}
+
+template <typename NameTableT>
+void writeNames(const boost::filesystem::path &path, const NameTableT &table)
+{
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
+
+    serialization::write(writer, "/common/names", table);
+}
+
+template <typename NodeWeigtsVectorT>
+void readEdgeBasedNodeWeights(const boost::filesystem::path &path, NodeWeigtsVectorT &weights)
+{
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
+
+    storage::serialization::read(reader, "/extractor/edge_based_node_weights", weights);
+}
+
+template <typename NodeWeigtsVectorT>
+void writeEdgeBasedNodeWeights(const boost::filesystem::path &path,
+                               const NodeWeigtsVectorT &weights)
+{
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
+
+    storage::serialization::write(writer, "/extractor/edge_based_node_weights", weights);
+}
+
+template <typename RTreeT>
+void writeRamIndex(const boost::filesystem::path &path, const RTreeT &rtree)
+{
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
+
+    util::serialization::write(writer, "/common/rtree", rtree);
+}
+
+template <typename RTreeT> void readRamIndex(const boost::filesystem::path &path, RTreeT &rtree)
+{
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
+
+    util::serialization::read(reader, "/common/rtree", rtree);
+}
+
+template <typename EdgeListT>
+void writeCompressedNodeBasedGraph(const boost::filesystem::path &path, const EdgeListT &edge_list)
+{
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
+
+    storage::serialization::write(writer, "/extractor/cnbg", edge_list);
+}
+
+template <typename EdgeListT>
+void readCompressedNodeBasedGraph(const boost::filesystem::path &path, EdgeListT &edge_list)
+{
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
+
+    storage::serialization::read(reader, "/extractor/cnbg", edge_list);
 }
 }
 }

include/extractor/intersection/have_identical_names.hpp

@@ -1,9 +1,9 @@
 #ifndef OSRM_EXTRACTOR_INTERSECTION_HAVE_IDENTICAL_NAMES_HPP_
 #define OSRM_EXTRACTOR_INTERSECTION_HAVE_IDENTICAL_NAMES_HPP_
 
+#include "extractor/name_table.hpp"
 #include "extractor/suffix_table.hpp"
 #include "guidance/constants.hpp"
-#include "util/name_table.hpp"
 
 namespace osrm
 {
@@ -17,7 +17,7 @@ namespace intersection
 // rhs->lhs)
 bool HaveIdenticalNames(const NameID lhs,
                         const NameID rhs,
-                        const util::NameTable &name_table,
+                        const NameTable &name_table,
                         const SuffixTable &street_name_suffix_table);
 
 } // namespace intersection

include/extractor/intersection/mergable_road_detector.hpp

@@ -4,9 +4,12 @@
 #include "extractor/compressed_edge_container.hpp"
 #include "extractor/intersection/coordinate_extractor.hpp"
 #include "extractor/intersection/have_identical_names.hpp"
+#include "extractor/name_table.hpp"
 #include "extractor/restriction_index.hpp"
 #include "extractor/turn_lane_types.hpp"
+
 #include "guidance/intersection.hpp"
+
 #include "util/coordinate.hpp"
 #include "util/node_based_graph.hpp"
 #include "util/typedefs.hpp"
@@ -49,7 +52,7 @@ class MergableRoadDetector
                          const RestrictionMap &node_restriction_map,
                          const std::unordered_set<NodeID> &barrier_nodes,
                          const TurnLanesIndexedArray &turn_lanes_data,
-                         const util::NameTable &name_table,
+                         const extractor::NameTable &name_table,
                          const SuffixTable &street_name_suffix_table);
 
     // OSM ways tend to be modelled as separate ways for different directions. This is often due to
@@ -168,7 +171,7 @@ class MergableRoadDetector
     const TurnLanesIndexedArray &turn_lanes_data;
 
     // name detection
-    const util::NameTable &name_table;
+    const extractor::NameTable &name_table;
     const SuffixTable &street_name_suffix_table;
 
     const CoordinateExtractor coordinate_extractor;

include/extractor/intersection_bearings_container.hpp

@@ -2,6 +2,7 @@
 #define OSRM_EXTRACTOR_BEARING_CONTAINER_HPP
 
 #include "storage/shared_memory_ownership.hpp"
+#include "storage/tar_fwd.hpp"
 
 #include "util/guidance/bearing_class.hpp"
 #include "util/range_table.hpp"
@@ -21,11 +22,13 @@ template <storage::Ownership Ownership> class IntersectionBearingsContainer;
 namespace serialization
 {
 template <storage::Ownership Ownership>
-void read(storage::io::FileReader &reader,
+void read(storage::tar::FileReader &reader,
+          const std::string &name,
           detail::IntersectionBearingsContainer<Ownership> &turn_data);
 
 template <storage::Ownership Ownership>
-void write(storage::io::FileWriter &writer,
+void write(storage::tar::FileWriter &writer,
+           const std::string &name,
            const detail::IntersectionBearingsContainer<Ownership> &turn_data);
 }
 
@@ -83,10 +86,12 @@ template <storage::Ownership Ownership> class IntersectionBearingsContainer
         return result;
     }
 
-    friend void serialization::read<Ownership>(storage::io::FileReader &reader,
+    friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
+                                               const std::string &name,
                                                IntersectionBearingsContainer &turn_data_container);
     friend void
-    serialization::write<Ownership>(storage::io::FileWriter &writer,
+    serialization::write<Ownership>(storage::tar::FileWriter &writer,
+                                    const std::string &name,
                                     const IntersectionBearingsContainer &turn_data_container);
 
   private:

include/extractor/name_table.hpp

@@ -0,0 +1,121 @@
+#ifndef OSRM_EXTRACTOR_NAME_TABLE_HPP
+#define OSRM_EXTRACTOR_NAME_TABLE_HPP
+
+#include "util/indexed_data.hpp"
+#include "util/string_view.hpp"
+#include "util/typedefs.hpp"
+
+#include <string>
+
+namespace osrm
+{
+namespace extractor
+{
+
+namespace detail
+{
+template <storage::Ownership Ownership> class NameTableImpl;
+}
+
+namespace serialization
+{
+template <storage::Ownership Ownership>
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 detail::NameTableImpl<Ownership> &index_data);
+
+template <storage::Ownership Ownership>
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
+                  const detail::NameTableImpl<Ownership> &index_data);
+}
+
+namespace detail
+{
+// This class provides a limited view over all the string data we serialize out.
+// The following functions are a subset of what is available.
+// See the data facades for they provide full access to this serialized string data.
+// Way string data is stored in blocks based on `id` as follows:
+//
+// | name | destination | pronunciation | ref | exits
+//                      ^               ^
+//                      [range)
+//                       ^ id + 2
+//
+// `id + offset` gives us the range of chars.
+//
+// Offset 0 is name, 1 is destination, 2 is pronunciation, 3 is ref, 4 is exits
+// See datafacades and extractor callbacks for details.
+template <storage::Ownership Ownership> class NameTableImpl
+{
+  public:
+    using IndexedData =
+        util::detail::IndexedDataImpl<util::VariableGroupBlock<16, util::StringView>, Ownership>;
+    using ResultType = typename IndexedData::ResultType;
+    using ValueType = typename IndexedData::ValueType;
+
+    NameTableImpl() {}
+
+    NameTableImpl(IndexedData indexed_data_) : indexed_data{std::move(indexed_data_)} {}
+
+    util::StringView GetNameForID(const NameID id) const
+    {
+        if (id == INVALID_NAMEID)
+            return {};
+
+        return indexed_data.at(id + 0);
+    }
+
+    util::StringView GetDestinationsForID(const NameID id) const
+    {
+        if (id == INVALID_NAMEID)
+            return {};
+
+        return indexed_data.at(id + 1);
+    }
+
+    util::StringView GetExitsForID(const NameID id) const
+    {
+        if (id == INVALID_NAMEID)
+            return {};
+
+        return indexed_data.at(id + 4);
+    }
+
+    util::StringView GetRefForID(const NameID id) const
+    {
+        if (id == INVALID_NAMEID)
+            return {};
+
+        const constexpr auto OFFSET_REF = 3u;
+        return indexed_data.at(id + OFFSET_REF);
+    }
+
+    util::StringView GetPronunciationForID(const NameID id) const
+    {
+        if (id == INVALID_NAMEID)
+            return {};
+
+        const constexpr auto OFFSET_PRONUNCIATION = 2u;
+        return indexed_data.at(id + OFFSET_PRONUNCIATION);
+    }
+
+    friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
+                                               const std::string &name,
+                                               NameTableImpl &index_data);
+
+    friend void serialization::write<Ownership>(storage::tar::FileWriter &writer,
+                                                const std::string &name,
+                                                const NameTableImpl &index_data);
+
+  private:
+    IndexedData indexed_data;
+};
+}
+
+using NameTable = detail::NameTableImpl<storage::Ownership::Container>;
+using NameTableView = detail::NameTableImpl<storage::Ownership::View>;
+} // namespace extractor
+} // namespace osrm
+
+#endif // OSRM_EXTRACTOR_NAME_TABLE_HPP

include/extractor/node_data_container.hpp

@@ -6,8 +6,8 @@
 #include "extractor/node_based_edge.hpp"
 #include "extractor/travel_mode.hpp"
 
-#include "storage/io_fwd.hpp"
 #include "storage/shared_memory_ownership.hpp"
+#include "storage/tar_fwd.hpp"
 
 #include "util/permutation.hpp"
 #include "util/typedefs.hpp"
@@ -29,11 +29,13 @@ template <storage::Ownership Ownership> class EdgeBasedNodeDataContainerImpl;
 namespace serialization
 {
 template <storage::Ownership Ownership>
-void read(storage::io::FileReader &reader,
+void read(storage::tar::FileReader &reader,
+          const std::string &name,
           detail::EdgeBasedNodeDataContainerImpl<Ownership> &ebn_data);
 
 template <storage::Ownership Ownership>
-void write(storage::io::FileWriter &writer,
+void write(storage::tar::FileWriter &writer,
+           const std::string &name,
            const detail::EdgeBasedNodeDataContainerImpl<Ownership> &ebn_data);
 }
 
@@ -89,10 +91,12 @@ template <storage::Ownership Ownership> class EdgeBasedNodeDataContainerImpl
         return annotation_data[nodes[node_id].annotation_id].classes;
     }
 
-    friend void serialization::read<Ownership>(storage::io::FileReader &reader,
+    friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
+                                               const std::string &name,
                                                EdgeBasedNodeDataContainerImpl &ebn_data_container);
     friend void
-    serialization::write<Ownership>(storage::io::FileWriter &writer,
+    serialization::write<Ownership>(storage::tar::FileWriter &writer,
+                                    const std::string &name,
                                     const EdgeBasedNodeDataContainerImpl &ebn_data_container);
 
     template <typename = std::enable_if<Ownership == storage::Ownership::Container>>

include/extractor/segment_data_container.hpp

@@ -5,8 +5,8 @@
 #include "util/typedefs.hpp"
 #include "util/vector_view.hpp"
 
-#include "storage/io_fwd.hpp"
 #include "storage/shared_memory_ownership.hpp"
+#include "storage/tar_fwd.hpp"
 
 #include <boost/filesystem/path.hpp>
 #include <boost/range/adaptor/reversed.hpp>
@@ -32,10 +32,12 @@ template <storage::Ownership Ownership> class SegmentDataContainerImpl;
 namespace serialization
 {
 template <storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader,
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
                  detail::SegmentDataContainerImpl<Ownership> &segment_data);
 template <storage::Ownership Ownership>
-inline void write(storage::io::FileWriter &writer,
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
                   const detail::SegmentDataContainerImpl<Ownership> &segment_data);
 }
 
@@ -200,10 +202,12 @@ template <storage::Ownership Ownership> class SegmentDataContainerImpl
     auto GetNumberOfSegments() const { return fwd_weights.size(); }
 
     friend void
-    serialization::read<Ownership>(storage::io::FileReader &reader,
+    serialization::read<Ownership>(storage::tar::FileReader &reader,
+                                   const std::string &name,
                                    detail::SegmentDataContainerImpl<Ownership> &segment_data);
     friend void serialization::write<Ownership>(
-        storage::io::FileWriter &writer,
+        storage::tar::FileWriter &writer,
+        const std::string &name,
         const detail::SegmentDataContainerImpl<Ownership> &segment_data);
 
   private:

include/extractor/serialization.hpp

@@ -1,10 +1,11 @@
 #ifndef OSRM_EXTRACTOR_IO_HPP
 #define OSRM_EXTRACTOR_IO_HPP
 
-#include "conditional_turn_penalty.hpp"
+#include "extractor/conditional_turn_penalty.hpp"
 #include "extractor/datasources.hpp"
 #include "extractor/intersection_bearings_container.hpp"
 #include "extractor/maneuver_override.hpp"
+#include "extractor/name_table.hpp"
 #include "extractor/nbg_to_ebg.hpp"
 #include "extractor/node_data_container.hpp"
 #include "extractor/profile_properties.hpp"
@@ -25,234 +26,113 @@ namespace serialization
 
 // read/write for bearing data
 template <storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader,
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
                  detail::IntersectionBearingsContainer<Ownership> &intersection_bearings)
 {
-    storage::serialization::read(reader, intersection_bearings.values);
-    storage::serialization::read(reader, intersection_bearings.node_to_class_id);
-    util::serialization::read(reader, intersection_bearings.class_id_to_ranges_table);
+    storage::serialization::read(reader, name + "/bearing_values", intersection_bearings.values);
+    storage::serialization::read(
+        reader, name + "/node_to_class_id", intersection_bearings.node_to_class_id);
+    util::serialization::read(
+        reader, name + "/class_id_to_ranges", intersection_bearings.class_id_to_ranges_table);
 }
 
 template <storage::Ownership Ownership>
-inline void write(storage::io::FileWriter &writer,
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
                   const detail::IntersectionBearingsContainer<Ownership> &intersection_bearings)
 {
-    storage::serialization::write(writer, intersection_bearings.values);
-    storage::serialization::write(writer, intersection_bearings.node_to_class_id);
-    util::serialization::write(writer, intersection_bearings.class_id_to_ranges_table);
+    storage::serialization::write(writer, name + "/bearing_values", intersection_bearings.values);
+    storage::serialization::write(
+        writer, name + "/node_to_class_id", intersection_bearings.node_to_class_id);
+    util::serialization::write(
+        writer, name + "/class_id_to_ranges", intersection_bearings.class_id_to_ranges_table);
 }
 
 // read/write for properties file
-inline void read(storage::io::FileReader &reader, ProfileProperties &properties)
+inline void
+read(storage::tar::FileReader &reader, const std::string &name, ProfileProperties &properties)
 {
-    reader.ReadInto(properties);
+    reader.ReadInto(name, properties);
 }
 
-inline void write(storage::io::FileWriter &writer, const ProfileProperties &properties)
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
+                  const ProfileProperties &properties)
 {
-    writer.WriteFrom(properties);
+    writer.WriteElementCount64(name, 1);
+    writer.WriteFrom(name, properties);
 }
 
 // read/write for datasources file
-inline void read(storage::io::FileReader &reader, Datasources &sources)
+inline void read(storage::tar::FileReader &reader, const std::string &name, Datasources &sources)
 {
-    reader.ReadInto(sources);
+    reader.ReadInto(name, sources);
 }
 
-inline void write(storage::io::FileWriter &writer, Datasources &sources)
+inline void write(storage::tar::FileWriter &writer, const std::string &name, Datasources &sources)
 {
-    writer.WriteFrom(sources);
+    writer.WriteElementCount64(name, 1);
+    writer.WriteFrom(name, sources);
 }
 
 // read/write for segment data file
 template <storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader,
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
                  detail::SegmentDataContainerImpl<Ownership> &segment_data)
 {
-    storage::serialization::read(reader, segment_data.index);
-    storage::serialization::read(reader, segment_data.nodes);
-    util::serialization::read(reader, segment_data.fwd_weights);
-    util::serialization::read(reader, segment_data.rev_weights);
-    util::serialization::read(reader, segment_data.fwd_durations);
-    util::serialization::read(reader, segment_data.rev_durations);
-    storage::serialization::read(reader, segment_data.fwd_datasources);
-    storage::serialization::read(reader, segment_data.rev_datasources);
+    storage::serialization::read(reader, name + "/index", segment_data.index);
+    storage::serialization::read(reader, name + "/nodes", segment_data.nodes);
+    util::serialization::read(reader, name + "/forward_weights", segment_data.fwd_weights);
+    util::serialization::read(reader, name + "/reverse_weights", segment_data.rev_weights);
+    util::serialization::read(reader, name + "/forward_durations", segment_data.fwd_durations);
+    util::serialization::read(reader, name + "/reverse_durations", segment_data.rev_durations);
+    storage::serialization::read(
+        reader, name + "/forward_data_sources", segment_data.fwd_datasources);
+    storage::serialization::read(
+        reader, name + "/reverse_data_sources", segment_data.rev_datasources);
 }
 
 template <storage::Ownership Ownership>
-inline void write(storage::io::FileWriter &writer,
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
                   const detail::SegmentDataContainerImpl<Ownership> &segment_data)
 {
-    storage::serialization::write(writer, segment_data.index);
-    storage::serialization::write(writer, segment_data.nodes);
-    util::serialization::write(writer, segment_data.fwd_weights);
-    util::serialization::write(writer, segment_data.rev_weights);
-    util::serialization::write(writer, segment_data.fwd_durations);
-    util::serialization::write(writer, segment_data.rev_durations);
-    storage::serialization::write(writer, segment_data.fwd_datasources);
-    storage::serialization::write(writer, segment_data.rev_datasources);
+    storage::serialization::write(writer, name + "/index", segment_data.index);
+    storage::serialization::write(writer, name + "/nodes", segment_data.nodes);
+    util::serialization::write(writer, name + "/forward_weights", segment_data.fwd_weights);
+    util::serialization::write(writer, name + "/reverse_weights", segment_data.rev_weights);
+    util::serialization::write(writer, name + "/forward_durations", segment_data.fwd_durations);
+    util::serialization::write(writer, name + "/reverse_durations", segment_data.rev_durations);
+    storage::serialization::write(
+        writer, name + "/forward_data_sources", segment_data.fwd_datasources);
+    storage::serialization::write(
+        writer, name + "/reverse_data_sources", segment_data.rev_datasources);
 }
 
 template <storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader,
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
                  detail::EdgeBasedNodeDataContainerImpl<Ownership> &node_data_container)
 {
-    // read header (separate sizes for both vectors)
-    reader.ReadElementCount64();
-    reader.ReadElementCount64();
     // read actual data
-    storage::serialization::read(reader, node_data_container.nodes);
-    storage::serialization::read(reader, node_data_container.annotation_data);
+    storage::serialization::read(reader, name + "/nodes", node_data_container.nodes);
+    storage::serialization::read(
+        reader, name + "/annotations", node_data_container.annotation_data);
 }
 
 template <storage::Ownership Ownership>
-inline void write(storage::io::FileWriter &writer,
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
                   const detail::EdgeBasedNodeDataContainerImpl<Ownership> &node_data_container)
 {
-    writer.WriteElementCount64(node_data_container.NumberOfNodes());
-    writer.WriteElementCount64(node_data_container.NumberOfAnnotations());
-    storage::serialization::write(writer, node_data_container.nodes);
-    storage::serialization::write(writer, node_data_container.annotation_data);
+    storage::serialization::write(writer, name + "/nodes", node_data_container.nodes);
+    storage::serialization::write(
+        writer, name + "/annotations", node_data_container.annotation_data);
 }
 
-inline void read(storage::io::FileReader &reader, NodeRestriction &restriction)
-{
-    reader.ReadInto(restriction.from);
-    reader.ReadInto(restriction.via);
-    reader.ReadInto(restriction.to);
-}
-
-inline void write(storage::io::FileWriter &writer, const NodeRestriction &restriction)
-{
-    writer.WriteOne(restriction.from);
-    writer.WriteOne(restriction.via);
-    writer.WriteOne(restriction.to);
-}
-
-inline void read(storage::io::FileReader &reader, WayRestriction &restriction)
-{
-    read(reader, restriction.in_restriction);
-    read(reader, restriction.out_restriction);
-}
-
-inline void write(storage::io::FileWriter &writer, const WayRestriction &restriction)
-{
-    write(writer, restriction.in_restriction);
-    write(writer, restriction.out_restriction);
-}
-
-inline void read(storage::io::FileReader &reader, TurnRestriction &restriction)
-{
-    reader.ReadInto(restriction.is_only);
-    std::uint32_t restriction_type;
-    reader.ReadInto(restriction_type);
-    if (restriction_type == RestrictionType::WAY_RESTRICTION)
-    {
-        WayRestriction way_restriction;
-        read(reader, way_restriction);
-        restriction.node_or_way = std::move(way_restriction);
-    }
-    else
-    {
-        BOOST_ASSERT(restriction_type == RestrictionType::NODE_RESTRICTION);
-        NodeRestriction node_restriction;
-        read(reader, node_restriction);
-        restriction.node_or_way = std::move(node_restriction);
-    }
-}
-
-inline void write(storage::io::FileWriter &writer, const TurnRestriction &restriction)
-{
-    writer.WriteOne(restriction.is_only);
-    const std::uint32_t restriction_type = restriction.Type();
-    writer.WriteOne(restriction_type);
-    if (restriction.Type() == RestrictionType::WAY_RESTRICTION)
-    {
-        write(writer, mapbox::util::get<WayRestriction>(restriction.node_or_way));
-    }
-    else
-    {
-        BOOST_ASSERT(restriction.Type() == RestrictionType::NODE_RESTRICTION);
-        write(writer, mapbox::util::get<NodeRestriction>(restriction.node_or_way));
-    }
-}
-
-inline void write(storage::io::FileWriter &writer, const ConditionalTurnRestriction &restriction)
-{
-    write(writer, static_cast<const TurnRestriction &>(restriction));
-    writer.WriteElementCount64(restriction.condition.size());
-    for (const auto &c : restriction.condition)
-    {
-        writer.WriteOne(c.modifier);
-        storage::serialization::write(writer, c.times);
-        storage::serialization::write(writer, c.weekdays);
-        storage::serialization::write(writer, c.monthdays);
-    }
-}
-
-inline void read(storage::io::FileReader &reader, ConditionalTurnRestriction &restriction)
-{
-    read(reader, static_cast<TurnRestriction &>(restriction));
-    const auto num_conditions = reader.ReadElementCount64();
-    restriction.condition.resize(num_conditions);
-    for (uint64_t i = 0; i < num_conditions; i++)
-    {
-        reader.ReadInto(restriction.condition[i].modifier);
-        storage::serialization::read(reader, restriction.condition[i].times);
-        storage::serialization::read(reader, restriction.condition[i].weekdays);
-        storage::serialization::read(reader, restriction.condition[i].monthdays);
-    }
-}
-
-// read/write for conditional turn restrictions file
-inline void read(storage::io::FileReader &reader, std::vector<TurnRestriction> &restrictions)
-{
-    auto num_indices = reader.ReadElementCount64();
-    restrictions.reserve(num_indices);
-    TurnRestriction restriction;
-    while (num_indices-- > 0)
-    {
-        read(reader, restriction);
-        restrictions.push_back(std::move(restriction));
-    }
-}
-
-inline void write(storage::io::FileWriter &writer, const std::vector<TurnRestriction> &restrictions)
-{
-    const auto num_indices = restrictions.size();
-    writer.WriteElementCount64(num_indices);
-    const auto write_restriction = [&writer](const auto &restriction) {
-        write(writer, restriction);
-    };
-    std::for_each(restrictions.begin(), restrictions.end(), write_restriction);
-}
-
-// read/write for conditional turn restrictions file
-inline void read(storage::io::FileReader &reader,
-                 std::vector<ConditionalTurnRestriction> &restrictions)
-{
-    auto num_indices = reader.ReadElementCount64();
-    restrictions.reserve(num_indices);
-    ConditionalTurnRestriction restriction;
-    while (num_indices-- > 0)
-    {
-        read(reader, restriction);
-        restrictions.push_back(std::move(restriction));
-    }
-}
-
-inline void write(storage::io::FileWriter &writer,
-                  const std::vector<ConditionalTurnRestriction> &restrictions)
-{
-    const auto num_indices = restrictions.size();
-    writer.WriteElementCount64(num_indices);
-    const auto write_restriction = [&writer](const auto &restriction) {
-        write(writer, restriction);
-    };
-    std::for_each(restrictions.begin(), restrictions.end(), write_restriction);
-}
-
-inline void read(storage::io::FileReader &reader, ConditionalTurnPenalty &turn_penalty)
+inline void read(storage::io::BufferReader &reader, ConditionalTurnPenalty &turn_penalty)
 {
     reader.ReadInto(turn_penalty.turn_offset);
     reader.ReadInto(turn_penalty.location.lat);
@@ -268,53 +148,79 @@ inline void read(storage::io::FileReader &reader, ConditionalTurnPenalty &turn_p
     }
 }
 
-inline void write(storage::io::FileWriter &writer, const ConditionalTurnPenalty &turn_penalty)
+inline void write(storage::io::BufferWriter &writer, const ConditionalTurnPenalty &turn_penalty)
 {
-    writer.WriteOne(turn_penalty.turn_offset);
-    writer.WriteOne(static_cast<util::FixedLatitude::value_type>(turn_penalty.location.lat));
-    writer.WriteOne(static_cast<util::FixedLongitude::value_type>(turn_penalty.location.lon));
+    writer.WriteFrom(turn_penalty.turn_offset);
+    writer.WriteFrom(static_cast<util::FixedLatitude::value_type>(turn_penalty.location.lat));
+    writer.WriteFrom(static_cast<util::FixedLongitude::value_type>(turn_penalty.location.lon));
     writer.WriteElementCount64(turn_penalty.conditions.size());
     for (const auto &c : turn_penalty.conditions)
     {
-        writer.WriteOne(c.modifier);
+        writer.WriteFrom(c.modifier);
         storage::serialization::write(writer, c.times);
         storage::serialization::write(writer, c.weekdays);
         storage::serialization::write(writer, c.monthdays);
     }
 }
 
-inline void write(storage::io::FileWriter &writer,
+inline void write(storage::io::BufferWriter &writer,
                   const std::vector<ConditionalTurnPenalty> &conditional_penalties)
 {
     writer.WriteElementCount64(conditional_penalties.size());
     for (const auto &penalty : conditional_penalties)
+    {
         write(writer, penalty);
+    }
 }
 
-inline void read(storage::io::FileReader &reader,
+inline void read(storage::io::BufferReader &reader,
                  std::vector<ConditionalTurnPenalty> &conditional_penalties)
 {
-    auto const num_elements = reader.ReadElementCount64();
+    auto num_elements = reader.ReadElementCount64();
     conditional_penalties.resize(num_elements);
     for (auto &penalty : conditional_penalties)
+    {
         read(reader, penalty);
+    }
 }
 
-inline void write(storage::io::FileWriter &writer,
-                  const std::vector<StorageManeuverOverride> &maneuver_overrides,
-                  const std::vector<NodeID> &node_sequences)
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
+                  const std::vector<ConditionalTurnPenalty> &conditional_penalties)
 {
-    storage::serialization::write(writer, maneuver_overrides);
-    storage::serialization::write(writer, node_sequences);
+    storage::io::BufferWriter buffer_writer;
+    write(buffer_writer, conditional_penalties);
+
+    storage::serialization::write(writer, name, buffer_writer.GetBuffer());
 }
 
-template <typename ManeuverOverridesT, typename NodeSequenceT>
-inline void read(storage::io::FileReader &reader,
-                 ManeuverOverridesT &maneuver_overrides,
-                 NodeSequenceT &node_sequences)
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 std::vector<ConditionalTurnPenalty> &conditional_penalties)
 {
-    storage::serialization::read(reader, maneuver_overrides);
-    storage::serialization::read(reader, node_sequences);
+    std::string buffer;
+    storage::serialization::read(reader, name, buffer);
+
+    storage::io::BufferReader buffer_reader{buffer};
+    read(buffer_reader, conditional_penalties);
+}
+
+template <storage::Ownership Ownership>
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
+                  const detail::NameTableImpl<Ownership> &name_table)
+{
+    storage::io::BufferWriter buffer_writer;
+    util::serialization::write(writer, name, name_table.indexed_data);
+}
+
+template <storage::Ownership Ownership>
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 detail::NameTableImpl<Ownership> &name_table)
+{
+    std::string buffer;
+    util::serialization::read(reader, name, name_table.indexed_data);
 }
 }
 }

include/guidance/driveway_handler.hpp

@@ -19,7 +19,7 @@ class DrivewayHandler final : public IntersectionHandler
                     const extractor::RestrictionMap &node_restriction_map,
                     const std::unordered_set<NodeID> &barrier_nodes,
                     const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                    const util::NameTable &name_table,
+                    const extractor::NameTable &name_table,
                     const extractor::SuffixTable &street_name_suffix_table);
 
     ~DrivewayHandler() override final = default;

include/guidance/files.hpp

@@ -27,10 +27,10 @@ inline void readTurnData(const boost::filesystem::path &path,
                       std::is_same<guidance::TurnDataView, TurnDataT>::value ||
                       std::is_same<guidance::TurnDataExternalContainer, TurnDataT>::value,
                   "");
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
 
-    serialization::read(reader, turn_data, connectivity_checksum);
+    serialization::read(reader, "/common/turn_data", turn_data, connectivity_checksum);
 }
 
 // writes .osrm.edges
@@ -43,10 +43,10 @@ inline void writeTurnData(const boost::filesystem::path &path,
                       std::is_same<guidance::TurnDataView, TurnDataT>::value ||
                       std::is_same<guidance::TurnDataExternalContainer, TurnDataT>::value,
                   "");
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
 
-    serialization::write(writer, turn_data, connectivity_checksum);
+    serialization::write(writer, "/common/turn_data", turn_data, connectivity_checksum);
 }
 }
 }

include/guidance/guidance_processing.hpp

@@ -4,6 +4,7 @@
 #include "guidance/turn_data_container.hpp"
 
 #include "extractor/compressed_edge_container.hpp"
+#include "extractor/name_table.hpp"
 #include "extractor/node_data_container.hpp"
 #include "extractor/suffix_table.hpp"
 #include "extractor/turn_lane_types.hpp"
@@ -13,7 +14,6 @@
 #include "util/guidance/bearing_class.hpp"
 #include "util/guidance/entry_class.hpp"
 #include "util/guidance/turn_lanes.hpp"
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 
 #include <unordered_set>
@@ -33,7 +33,7 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
                    const std::unordered_set<NodeID> &barrier_nodes,
                    const extractor::RestrictionMap &node_restriction_map,
                    const extractor::WayRestrictionMap &way_restriction_map,
-                   const util::NameTable &name_table,
+                   const extractor::NameTable &name_table,
                    const extractor::SuffixTable &suffix_table,
                    const extractor::TurnLanesIndexedArray &turn_lanes_data,
                    extractor::LaneDescriptionMap &lane_description_map,

include/guidance/intersection_handler.hpp

@@ -3,6 +3,7 @@
 
 #include "extractor/intersection/intersection_analysis.hpp"
 #include "extractor/intersection/node_based_graph_walker.hpp"
+#include "extractor/name_table.hpp"
 #include "extractor/suffix_table.hpp"
 #include "guidance/constants.hpp"
 #include "guidance/intersection.hpp"
@@ -10,7 +11,6 @@
 #include "util/assert.hpp"
 #include "util/coordinate_calculation.hpp"
 #include "util/guidance/name_announcements.hpp"
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 
 #include <algorithm>
@@ -38,7 +38,7 @@ class IntersectionHandler
                         const extractor::RestrictionMap &node_restriction_map,
                         const std::unordered_set<NodeID> &barrier_nodes,
                         const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                        const util::NameTable &name_table,
+                        const extractor::NameTable &name_table,
                         const extractor::SuffixTable &street_name_suffix_table);
 
     virtual ~IntersectionHandler() = default;
@@ -59,7 +59,7 @@ class IntersectionHandler
     const extractor::RestrictionMap &node_restriction_map;
     const std::unordered_set<NodeID> &barrier_nodes;
     const extractor::TurnLanesIndexedArray &turn_lanes_data;
-    const util::NameTable &name_table;
+    const extractor::NameTable &name_table;
     const extractor::SuffixTable &street_name_suffix_table;
     const extractor::intersection::NodeBasedGraphWalker
         graph_walker; // for skipping traffic signal, distances etc.

include/guidance/is_through_street.hpp

@@ -7,6 +7,9 @@
 #include "extractor/node_data_container.hpp"
 #include "extractor/suffix_table.hpp"
 
+#include "util/bearing.hpp"
+#include "util/node_based_graph.hpp"
+
 #include "util/guidance/name_announcements.hpp"
 
 namespace osrm
@@ -19,7 +22,7 @@ inline bool isThroughStreet(const std::size_t index,
                             const IntersectionType &intersection,
                             const util::NodeBasedDynamicGraph &node_based_graph,
                             const extractor::EdgeBasedNodeDataContainer &node_data_container,
-                            const util::NameTable &name_table,
+                            const extractor::NameTable &name_table,
                             const extractor::SuffixTable &street_name_suffix_table)
 {
     using osrm::util::angularDeviation;

include/guidance/motorway_handler.hpp

@@ -1,12 +1,13 @@
 #ifndef OSRM_GUIDANCE_MOTORWAY_HANDLER_HPP_
 #define OSRM_GUIDANCE_MOTORWAY_HANDLER_HPP_
 
+#include "extractor/name_table.hpp"
+
 #include "guidance/intersection.hpp"
 #include "guidance/intersection_handler.hpp"
 #include "guidance/is_through_street.hpp"
 
 #include "util/attributes.hpp"
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 
 #include <vector>
@@ -27,7 +28,7 @@ class MotorwayHandler : public IntersectionHandler
                     const extractor::RestrictionMap &node_restriction_map,
                     const std::unordered_set<NodeID> &barrier_nodes,
                     const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                    const util::NameTable &name_table,
+                    const extractor::NameTable &name_table,
                     const extractor::SuffixTable &street_name_suffix_table);
 
     ~MotorwayHandler() override final = default;

include/guidance/roundabout_handler.hpp

@@ -3,13 +3,14 @@
 
 #include "extractor/compressed_edge_container.hpp"
 #include "extractor/intersection/coordinate_extractor.hpp"
+#include "extractor/name_table.hpp"
 #include "extractor/query_node.hpp"
+
 #include "guidance/intersection.hpp"
 #include "guidance/intersection_handler.hpp"
 #include "guidance/is_through_street.hpp"
 #include "guidance/roundabout_type.hpp"
 
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 #include "util/typedefs.hpp"
 
@@ -44,7 +45,7 @@ class RoundaboutHandler : public IntersectionHandler
                       const extractor::RestrictionMap &node_restriction_map,
                       const std::unordered_set<NodeID> &barrier_nodes,
                       const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                      const util::NameTable &name_table,
+                      const extractor::NameTable &name_table,
                       const extractor::SuffixTable &street_name_suffix_table);
 
     ~RoundaboutHandler() override final = default;

include/guidance/segregated_intersection_classification.hpp

@@ -1,14 +1,11 @@
+#include "extractor/name_table.hpp"
+
 #include "util/typedefs.hpp"
 
 #include <unordered_set>
 
 namespace osrm
 {
-namespace util
-{
-class NameTable;
-}
-
 namespace extractor
 {
 class NodeBasedGraphFactory;
@@ -22,6 +19,6 @@ namespace guidance
 // - staggered intersections (X-cross)
 // - square/circle intersections
 std::unordered_set<EdgeID> findSegregatedNodes(const extractor::NodeBasedGraphFactory &factory,
-                                               const util::NameTable &names);
+                                               const extractor::NameTable &names);
 }
 }

include/guidance/serialization.hpp

@@ -3,8 +3,8 @@
 
 #include "guidance/turn_data_container.hpp"
 
-#include "storage/io.hpp"
 #include "storage/serialization.hpp"
+#include "storage/tar.hpp"
 
 #include <boost/assert.hpp>
 
@@ -17,29 +17,42 @@ namespace serialization
 
 // read/write for turn data file
 template <storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader,
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
                  guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container,
                  std::uint32_t &connectivity_checksum)
 {
-    storage::serialization::read(reader, turn_data_container.turn_instructions);
-    storage::serialization::read(reader, turn_data_container.lane_data_ids);
-    storage::serialization::read(reader, turn_data_container.entry_class_ids);
-    storage::serialization::read(reader, turn_data_container.pre_turn_bearings);
-    storage::serialization::read(reader, turn_data_container.post_turn_bearings);
-    reader.ReadInto(connectivity_checksum);
+    storage::serialization::read(
+        reader, name + "/turn_instructions", turn_data_container.turn_instructions);
+    storage::serialization::read(
+        reader, name + "/lane_data_ids", turn_data_container.lane_data_ids);
+    storage::serialization::read(
+        reader, name + "/entry_class_ids", turn_data_container.entry_class_ids);
+    storage::serialization::read(
+        reader, name + "/pre_turn_bearings", turn_data_container.pre_turn_bearings);
+    storage::serialization::read(
+        reader, name + "/post_turn_bearings", turn_data_container.post_turn_bearings);
+    reader.ReadInto(name + "/connectivity_checksum", connectivity_checksum);
 }
 
 template <storage::Ownership Ownership>
-inline void write(storage::io::FileWriter &writer,
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
                   const guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container,
                   const std::uint32_t connectivity_checksum)
 {
-    storage::serialization::write(writer, turn_data_container.turn_instructions);
-    storage::serialization::write(writer, turn_data_container.lane_data_ids);
-    storage::serialization::write(writer, turn_data_container.entry_class_ids);
-    storage::serialization::write(writer, turn_data_container.pre_turn_bearings);
-    storage::serialization::write(writer, turn_data_container.post_turn_bearings);
-    writer.WriteOne(connectivity_checksum);
+    storage::serialization::write(
+        writer, name + "/turn_instructions", turn_data_container.turn_instructions);
+    storage::serialization::write(
+        writer, name + "/lane_data_ids", turn_data_container.lane_data_ids);
+    storage::serialization::write(
+        writer, name + "/entry_class_ids", turn_data_container.entry_class_ids);
+    storage::serialization::write(
+        writer, name + "/pre_turn_bearings", turn_data_container.pre_turn_bearings);
+    storage::serialization::write(
+        writer, name + "/post_turn_bearings", turn_data_container.post_turn_bearings);
+    writer.WriteElementCount64(name + "/connectivity_checksum", 1);
+    writer.WriteFrom(name + "/connectivity_checksum", connectivity_checksum);
 }
 }
 }

include/guidance/sliproad_handler.hpp

@@ -1,11 +1,12 @@
 #ifndef OSRM_GUIDANCE_SLIPROAD_HANDLER_HPP_
 #define OSRM_GUIDANCE_SLIPROAD_HANDLER_HPP_
 
+#include "extractor/name_table.hpp"
+
 #include "guidance/intersection.hpp"
 #include "guidance/intersection_handler.hpp"
 #include "guidance/is_through_street.hpp"
 
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 
 #include <vector>
@@ -28,7 +29,7 @@ class SliproadHandler final : public IntersectionHandler
                     const extractor::RestrictionMap &node_restriction_map,
                     const std::unordered_set<NodeID> &barrier_nodes,
                     const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                    const util::NameTable &name_table,
+                    const extractor::NameTable &name_table,
                     const extractor::SuffixTable &street_name_suffix_table);
 
     ~SliproadHandler() override final = default;

include/guidance/statistics_handler.hpp

@@ -32,7 +32,7 @@ class StatisticsHandler final : public IntersectionHandler
                       const extractor::RestrictionMap &node_restriction_map,
                       const std::unordered_set<NodeID> &barrier_nodes,
                       const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                      const util::NameTable &name_table,
+                      const extractor::NameTable &name_table,
                       const extractor::SuffixTable &street_name_suffix_table)
         : IntersectionHandler(node_based_graph,
                               node_data_container,

include/guidance/suppress_mode_handler.hpp

@@ -1,8 +1,11 @@
 #ifndef OSRM_GUIDANCE_SUPPRESS_MODE_HANDLER_HPP_
 #define OSRM_GUIDANCE_SUPPRESS_MODE_HANDLER_HPP_
 
+#include "extractor/name_table.hpp"
+
 #include "guidance/intersection.hpp"
 #include "guidance/intersection_handler.hpp"
+
 #include "util/node_based_graph.hpp"
 
 namespace osrm
@@ -23,7 +26,7 @@ class SuppressModeHandler final : public IntersectionHandler
                         const extractor::RestrictionMap &node_restriction_map,
                         const std::unordered_set<NodeID> &barrier_nodes,
                         const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                        const util::NameTable &name_table,
+                        const extractor::NameTable &name_table,
                         const extractor::SuffixTable &street_name_suffix_table);
 
     ~SuppressModeHandler() override final = default;

include/guidance/turn_analysis.hpp

@@ -3,8 +3,10 @@
 
 #include "extractor/compressed_edge_container.hpp"
 #include "extractor/intersection/intersection_view.hpp"
+#include "extractor/name_table.hpp"
 #include "extractor/restriction_index.hpp"
 #include "extractor/suffix_table.hpp"
+
 #include "guidance/driveway_handler.hpp"
 #include "guidance/intersection.hpp"
 #include "guidance/motorway_handler.hpp"
@@ -16,7 +18,6 @@
 #include "guidance/turn_handler.hpp"
 
 #include "util/attributes.hpp"
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 
 #include <cstdint>
@@ -43,7 +44,7 @@ class TurnAnalysis
                  const extractor::RestrictionMap &restriction_map,
                  const std::unordered_set<NodeID> &barrier_nodes,
                  const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                 const util::NameTable &name_table,
+                 const extractor::NameTable &name_table,
                  const extractor::SuffixTable &street_name_suffix_table);
 
     /* Full Analysis Process for a single node/edge combination. Use with caution, as the process is

include/guidance/turn_data_container.hpp

@@ -5,8 +5,8 @@
 #include "guidance/turn_bearing.hpp"
 #include "guidance/turn_instruction.hpp"
 
-#include "storage/io_fwd.hpp"
 #include "storage/shared_memory_ownership.hpp"
+#include "storage/tar_fwd.hpp"
 
 #include "util/vector_view.hpp"
 
@@ -24,12 +24,14 @@ template <storage::Ownership Ownership> class TurnDataContainerImpl;
 namespace serialization
 {
 template <storage::Ownership Ownership>
-void read(storage::io::FileReader &reader,
+void read(storage::tar::FileReader &reader,
+          const std::string &name,
           detail::TurnDataContainerImpl<Ownership> &turn_data,
           std::uint32_t &connectivity_checksum);
 
 template <storage::Ownership Ownership>
-void write(storage::io::FileWriter &writer,
+void write(storage::tar::FileWriter &writer,
+           const std::string &name,
            const detail::TurnDataContainerImpl<Ownership> &turn_data,
            const std::uint32_t connectivity_checksum);
 }
@@ -97,10 +99,12 @@ template <storage::Ownership Ownership> class TurnDataContainerImpl
             others.begin(), others.end(), [this](const TurnData &other) { push_back(other); });
     }
 
-    friend void serialization::read<Ownership>(storage::io::FileReader &reader,
+    friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
+                                               const std::string &name,
                                                TurnDataContainerImpl &turn_data_container,
                                                std::uint32_t &connectivity_checksum);
-    friend void serialization::write<Ownership>(storage::io::FileWriter &writer,
+    friend void serialization::write<Ownership>(storage::tar::FileWriter &writer,
+                                                const std::string &name,
                                                 const TurnDataContainerImpl &turn_data_container,
                                                 const std::uint32_t connectivity_checksum);
 

include/guidance/turn_handler.hpp

@@ -1,13 +1,14 @@
 #ifndef OSRM_GUIDANCE_TURN_HANDLER_HPP_
 #define OSRM_GUIDANCE_TURN_HANDLER_HPP_
 
+#include "extractor/name_table.hpp"
 #include "extractor/query_node.hpp"
+
 #include "guidance/intersection.hpp"
 #include "guidance/intersection_handler.hpp"
 #include "guidance/is_through_street.hpp"
 
 #include "util/attributes.hpp"
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 
 #include <boost/optional.hpp>
@@ -32,7 +33,7 @@ class TurnHandler : public IntersectionHandler
                 const extractor::RestrictionMap &node_restriction_map,
                 const std::unordered_set<NodeID> &barrier_nodes,
                 const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                const util::NameTable &name_table,
+                const extractor::NameTable &name_table,
                 const extractor::SuffixTable &street_name_suffix_table);
 
     ~TurnHandler() override final = default;

include/guidance/turn_lane_handler.hpp

@@ -1,15 +1,16 @@
 #ifndef OSRM_GUIDANCE_TURN_LANE_HANDLER_HPP_
 #define OSRM_GUIDANCE_TURN_LANE_HANDLER_HPP_
 
+#include "extractor/name_table.hpp"
 #include "extractor/query_node.hpp"
 #include "extractor/turn_lane_types.hpp"
+
 #include "guidance/intersection.hpp"
 #include "guidance/turn_analysis.hpp"
 #include "guidance/turn_lane_data.hpp"
 
 #include "util/attributes.hpp"
 #include "util/guidance/turn_lanes.hpp"
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 #include "util/typedefs.hpp"
 

include/partitioner/cell_storage.hpp

@@ -38,9 +38,12 @@ using CellStorageView = detail::CellStorageImpl<storage::Ownership::View>;
 namespace serialization
 {
 template <storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader, detail::CellStorageImpl<Ownership> &storage);
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 detail::CellStorageImpl<Ownership> &storage);
 template <storage::Ownership Ownership>
-inline void write(storage::io::FileWriter &writer,
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
                   const detail::CellStorageImpl<Ownership> &storage);
 }
 
@@ -390,9 +393,11 @@ template <storage::Ownership Ownership> class CellStorageImpl
                     destination_boundary.data()};
     }
 
-    friend void serialization::read<Ownership>(storage::io::FileReader &reader,
+    friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
+                                               const std::string &name,
                                                detail::CellStorageImpl<Ownership> &storage);
-    friend void serialization::write<Ownership>(storage::io::FileWriter &writer,
+    friend void serialization::write<Ownership>(storage::tar::FileWriter &writer,
+                                                const std::string &name,
                                                 const detail::CellStorageImpl<Ownership> &storage);
 
   private:

include/partitioner/compressed_node_based_graph_reader.hpp

@@ -1,60 +0,0 @@
-#ifndef OSRM_PARTITIONER_COMPRESSED_NODE_BASED_GRAPH_READER_HPP
-#define OSRM_PARTITIONER_COMPRESSED_NODE_BASED_GRAPH_READER_HPP
-
-#include "storage/io.hpp"
-#include "util/coordinate.hpp"
-#include "util/typedefs.hpp"
-
-#include <string>
-#include <vector>
-
-namespace osrm
-{
-namespace partitioner
-{
-
-struct CompressedNodeBasedGraphEdge
-{
-    NodeID source;
-    NodeID target;
-};
-
-struct CompressedNodeBasedGraph
-{
-    CompressedNodeBasedGraph(storage::io::FileReader &reader)
-    {
-        // Reads:  | Fingerprint | #e | #n | edges | coordinates |
-        // - uint64: number of edges (from, to) pairs
-        // - uint64: number of nodes and therefore also coordinates
-        // - (uint32_t, uint32_t): num_edges * edges
-        // - (int32_t, int32_t: num_nodes * coordinates (lon, lat)
-        //
-        // Gets written in Extractor::WriteCompressedNodeBasedGraph
-
-        const auto num_edges = reader.ReadElementCount64();
-        const auto num_nodes = reader.ReadElementCount64();
-
-        edges.resize(num_edges);
-        coordinates.resize(num_nodes);
-
-        reader.ReadInto(edges);
-        reader.ReadInto(coordinates);
-    }
-
-    std::vector<CompressedNodeBasedGraphEdge> edges;
-    std::vector<util::Coordinate> coordinates;
-};
-
-inline CompressedNodeBasedGraph LoadCompressedNodeBasedGraph(const std::string &path)
-{
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader(path, fingerprint);
-
-    CompressedNodeBasedGraph graph{reader};
-    return graph;
-}
-
-} // ns partition
-} // ns osrm
-
-#endif

include/partitioner/files.hpp

@@ -24,10 +24,9 @@ inline void readGraph(const boost::filesystem::path &path,
                       std::is_same<customizer::MultiLevelEdgeBasedGraph, MultiLevelGraphT>::value,
                   "");
 
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
+    storage::tar::FileReader reader{path, storage::tar::FileReader::VerifyFingerprint};
 
-    serialization::read(reader, graph, connectivity_checksum);
+    serialization::read(reader, "/mld/multilevelgraph", graph, connectivity_checksum);
 }
 
 // writes .osrm.mldgr file
@@ -40,10 +39,9 @@ inline void writeGraph(const boost::filesystem::path &path,
                       std::is_same<customizer::MultiLevelEdgeBasedGraph, MultiLevelGraphT>::value,
                   "");
 
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
+    storage::tar::FileWriter writer{path, storage::tar::FileWriter::GenerateFingerprint};
 
-    serialization::write(writer, graph, connectivity_checksum);
+    serialization::write(writer, "/mld/multilevelgraph", graph, connectivity_checksum);
 }
 
 // read .osrm.partition file
@@ -54,10 +52,10 @@ inline void readPartition(const boost::filesystem::path &path, MultiLevelPartiti
                       std::is_same<MultiLevelPartition, MultiLevelPartitionT>::value,
                   "");
 
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
 
-    serialization::read(reader, mlp);
+    serialization::read(reader, "/mld/multilevelpartition", mlp);
 }
 
 // writes .osrm.partition file
@@ -68,10 +66,10 @@ inline void writePartition(const boost::filesystem::path &path, const MultiLevel
                       std::is_same<MultiLevelPartition, MultiLevelPartitionT>::value,
                   "");
 
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
 
-    serialization::write(writer, mlp);
+    serialization::write(writer, "/mld/multilevelpartition", mlp);
 }
 
 // reads .osrm.cells file
@@ -82,10 +80,10 @@ inline void readCells(const boost::filesystem::path &path, CellStorageT &storage
                       std::is_same<CellStorage, CellStorageT>::value,
                   "");
 
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
 
-    serialization::read(reader, storage);
+    serialization::read(reader, "/mld/cellstorage", storage);
 }
 
 // writes .osrm.cells file
@@ -96,10 +94,10 @@ inline void writeCells(const boost::filesystem::path &path, CellStorageT &storag
                       std::is_same<CellStorage, CellStorageT>::value,
                   "");
 
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
 
-    serialization::write(writer, storage);
+    serialization::write(writer, "/mld/cellstorage", storage);
 }
 }
 }

include/partitioner/multi_level_graph.hpp

@@ -3,8 +3,8 @@
 
 #include "partitioner/multi_level_partition.hpp"
 
-#include "storage/io_fwd.hpp"
 #include "storage/shared_memory_ownership.hpp"
+#include "storage/tar_fwd.hpp"
 
 #include "util/static_graph.hpp"
 #include "util/vector_view.hpp"
@@ -24,12 +24,14 @@ template <typename EdgeDataT, storage::Ownership Ownership> class MultiLevelGrap
 namespace serialization
 {
 template <typename EdgeDataT, storage::Ownership Ownership>
-void read(storage::io::FileReader &reader,
+void read(storage::tar::FileReader &reader,
+          const std::string &name,
           MultiLevelGraph<EdgeDataT, Ownership> &graph,
           std::uint32_t &connectivity_checksum);
 
 template <typename EdgeDataT, storage::Ownership Ownership>
-void write(storage::io::FileWriter &writer,
+void write(storage::tar::FileWriter &writer,
+           const std::string &name,
            const MultiLevelGraph<EdgeDataT, Ownership> &graph,
            const std::uint32_t connectivity_checksum);
 }
@@ -202,11 +204,13 @@ class MultiLevelGraph : public util::StaticGraph<EdgeDataT, Ownership>
     }
 
     friend void
-    serialization::read<EdgeDataT, Ownership>(storage::io::FileReader &reader,
+    serialization::read<EdgeDataT, Ownership>(storage::tar::FileReader &reader,
+                                              const std::string &name,
                                               MultiLevelGraph<EdgeDataT, Ownership> &graph,
                                               std::uint32_t &connectivity_checksum);
     friend void
-    serialization::write<EdgeDataT, Ownership>(storage::io::FileWriter &writer,
+    serialization::write<EdgeDataT, Ownership>(storage::tar::FileWriter &writer,
+                                               const std::string &name,
                                                const MultiLevelGraph<EdgeDataT, Ownership> &graph,
                                                const std::uint32_t connectivity_checksum);
 

include/partitioner/multi_level_partition.hpp

@@ -7,8 +7,8 @@
 #include "util/typedefs.hpp"
 #include "util/vector_view.hpp"
 
-#include "storage/io_fwd.hpp"
 #include "storage/shared_memory_ownership.hpp"
+#include "storage/tar_fwd.hpp"
 
 #include <algorithm>
 #include <array>
@@ -34,9 +34,13 @@ using MultiLevelPartitionView = detail::MultiLevelPartitionImpl<storage::Ownersh
 namespace serialization
 {
 template <storage::Ownership Ownership>
-void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl<Ownership> &mlp);
+void read(storage::tar::FileReader &reader,
+          const std::string &name,
+          detail::MultiLevelPartitionImpl<Ownership> &mlp);
 template <storage::Ownership Ownership>
-void write(storage::io::FileWriter &writer, const detail::MultiLevelPartitionImpl<Ownership> &mlp);
+void write(storage::tar::FileWriter &writer,
+           const std::string &name,
+           const detail::MultiLevelPartitionImpl<Ownership> &mlp);
 }
 
 namespace detail
@@ -136,9 +140,11 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
         return cell_to_children[offset + cell + 1];
     }
 
-    friend void serialization::read<Ownership>(storage::io::FileReader &reader,
+    friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
+                                               const std::string &name,
                                                MultiLevelPartitionImpl &mlp);
-    friend void serialization::write<Ownership>(storage::io::FileWriter &writer,
+    friend void serialization::write<Ownership>(storage::tar::FileWriter &writer,
+                                                const std::string &name,
                                                 const MultiLevelPartitionImpl &mlp);
 
   private:

include/partitioner/partitioner_config.hpp

@@ -21,6 +21,7 @@ struct PartitionerConfig final : storage::IOConfig
                    {".osrm.ebg",
                     ".osrm.cnbg",
                     ".osrm.cnbg_to_ebg",
+                    ".osrm.nbg_nodes",
                     ".osrm.partition",
                     ".osrm.cells",
                     ".osrm.maneuver_overrides"}),

include/partitioner/serialization.hpp

@@ -10,6 +10,7 @@
 #include "storage/io.hpp"
 #include "storage/serialization.hpp"
 #include "storage/shared_memory_ownership.hpp"
+#include "storage/tar.hpp"
 
 namespace osrm
 {
@@ -19,61 +20,75 @@ namespace serialization
 {
 
 template <typename EdgeDataT, storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader,
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
                  MultiLevelGraph<EdgeDataT, Ownership> &graph,
                  std::uint32_t &connectivity_checksum)
 {
-    storage::serialization::read(reader, graph.node_array);
-    storage::serialization::read(reader, graph.edge_array);
-    storage::serialization::read(reader, graph.node_to_edge_offset);
-    reader.ReadInto(connectivity_checksum);
+    storage::serialization::read(reader, name + "/node_array", graph.node_array);
+    storage::serialization::read(reader, name + "/edge_array", graph.edge_array);
+    storage::serialization::read(reader, name + "/node_to_edge_offset", graph.node_to_edge_offset);
+    reader.ReadInto(name + "/connectivity_checksum", connectivity_checksum);
 }
 
 template <typename EdgeDataT, storage::Ownership Ownership>
-inline void write(storage::io::FileWriter &writer,
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
                   const MultiLevelGraph<EdgeDataT, Ownership> &graph,
                   const std::uint32_t connectivity_checksum)
 {
-    storage::serialization::write(writer, graph.node_array);
-    storage::serialization::write(writer, graph.edge_array);
-    storage::serialization::write(writer, graph.node_to_edge_offset);
-    writer.WriteOne(connectivity_checksum);
+    storage::serialization::write(writer, name + "/node_array", graph.node_array);
+    storage::serialization::write(writer, name + "/edge_array", graph.edge_array);
+    storage::serialization::write(writer, name + "/node_to_edge_offset", graph.node_to_edge_offset);
+    writer.WriteElementCount64(name + "/connectivity_checksum", 1);
+    writer.WriteFrom(name + "/connectivity_checksum", connectivity_checksum);
 }
 
 template <storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl<Ownership> &mlp)
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 detail::MultiLevelPartitionImpl<Ownership> &mlp)
 {
-    reader.ReadInto(*mlp.level_data);
-    storage::serialization::read(reader, mlp.partition);
-    storage::serialization::read(reader, mlp.cell_to_children);
+    reader.ReadInto(name + "/level_data", *mlp.level_data);
+    storage::serialization::read(reader, name + "/partition", mlp.partition);
+    storage::serialization::read(reader, name + "/cell_to_children", mlp.cell_to_children);
 }
 
 template <storage::Ownership Ownership>
-inline void write(storage::io::FileWriter &writer,
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
                   const detail::MultiLevelPartitionImpl<Ownership> &mlp)
 {
-    writer.WriteOne(*mlp.level_data);
-    storage::serialization::write(writer, mlp.partition);
-    storage::serialization::write(writer, mlp.cell_to_children);
+    writer.WriteElementCount64(name + "/level_data", 1);
+    writer.WriteFrom(name + "/level_data", *mlp.level_data);
+    storage::serialization::write(writer, name + "/partition", mlp.partition);
+    storage::serialization::write(writer, name + "/cell_to_children", mlp.cell_to_children);
 }
 
 template <storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader, detail::CellStorageImpl<Ownership> &storage)
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 detail::CellStorageImpl<Ownership> &storage)
 {
-    storage::serialization::read(reader, storage.source_boundary);
-    storage::serialization::read(reader, storage.destination_boundary);
-    storage::serialization::read(reader, storage.cells);
-    storage::serialization::read(reader, storage.level_to_cell_offset);
+    storage::serialization::read(reader, name + "/source_boundary", storage.source_boundary);
+    storage::serialization::read(
+        reader, name + "/destination_boundary", storage.destination_boundary);
+    storage::serialization::read(reader, name + "/cells", storage.cells);
+    storage::serialization::read(
+        reader, name + "/level_to_cell_offset", storage.level_to_cell_offset);
 }
 
 template <storage::Ownership Ownership>
-inline void write(storage::io::FileWriter &writer,
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
                   const detail::CellStorageImpl<Ownership> &storage)
 {
-    storage::serialization::write(writer, storage.source_boundary);
-    storage::serialization::write(writer, storage.destination_boundary);
-    storage::serialization::write(writer, storage.cells);
-    storage::serialization::write(writer, storage.level_to_cell_offset);
+    storage::serialization::write(writer, name + "/source_boundary", storage.source_boundary);
+    storage::serialization::write(
+        writer, name + "/destination_boundary", storage.destination_boundary);
+    storage::serialization::write(writer, name + "/cells", storage.cells);
+    storage::serialization::write(
+        writer, name + "/level_to_cell_offset", storage.level_to_cell_offset);
 }
 }
 }

include/storage/block.hpp

@@ -16,14 +16,20 @@ struct Block
 {
     std::uint64_t num_entries;
     std::uint64_t byte_size;
-    std::uint64_t entry_size;
-    std::uint64_t entry_align;
+
+    Block() : num_entries(0), byte_size(0) {}
+    Block(std::uint64_t num_entries, std::uint64_t byte_size)
+        : num_entries(num_entries), byte_size(byte_size)
+    {
+    }
 };
 
+using NamedBlock = std::tuple<std::string, Block>;
+
 template <typename T> Block make_block(uint64_t num_entries)
 {
     static_assert(sizeof(T) % alignof(T) == 0, "aligned T* can't be used as an array pointer");
-    return Block{num_entries, sizeof(T) * num_entries, sizeof(T), alignof(T)};
+    return Block{num_entries, sizeof(T) * num_entries};
 }
 }
 }

include/storage/io.hpp

@@ -6,6 +6,7 @@
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
 #include "util/fingerprint.hpp"
+#include "util/integer_range.hpp"
 #include "util/log.hpp"
 #include "util/version.hpp"
 
@@ -27,16 +28,6 @@ namespace io
 class FileReader
 {
   public:
-    class LineWrapper : public std::string
-    {
-        friend std::istream &operator>>(std::istream &is, LineWrapper &line)
-        {
-            return std::getline(is, line);
-        }
-    };
-    auto GetLineIteratorBegin() { return std::istream_iterator<LineWrapper>(input_stream); }
-    auto GetLineIteratorEnd() { return std::istream_iterator<LineWrapper>(); }
-
     enum FingerprintFlag
     {
         VerifyFingerprint,
@@ -120,6 +111,38 @@ class FileReader
         }
     }
 
+    template <typename T, typename OutIter> void ReadStreaming(OutIter out, const std::size_t count)
+    {
+#if !defined(__GNUC__) || (__GNUC__ > 4)
+        static_assert(!std::is_pointer<T>::value, "saving pointer types is not allowed");
+        static_assert(std::is_trivially_copyable<T>::value,
+                      "bytewise reading requires trivially copyable type");
+#endif
+
+        if (count == 0)
+            return;
+
+        T tmp;
+        for (auto index : util::irange<std::size_t>(0, count))
+        {
+            (void)index;
+            const auto &result = input_stream.read(reinterpret_cast<char *>(&tmp), sizeof(T));
+            const std::size_t bytes_read = input_stream.gcount();
+
+            if (bytes_read != sizeof(T) && !result)
+            {
+                if (result.eof())
+                {
+                    throw util::RuntimeError(
+                        filepath.string(), ErrorCode::UnexpectedEndOfFile, SOURCE_REF);
+                }
+                throw util::RuntimeError(
+                    filepath.string(), ErrorCode::FileReadError, SOURCE_REF, std::strerror(errno));
+            }
+            *out++ = tmp;
+        }
+    }
+
     template <typename T> void ReadInto(std::vector<T> &target)
     {
         ReadInto(target.data(), target.size());
@@ -127,13 +150,6 @@ class FileReader
 
     template <typename T> void ReadInto(T &target) { ReadInto(&target, 1); }
 
-    template <typename T> T ReadOne()
-    {
-        T tmp;
-        ReadInto(tmp);
-        return tmp;
-    }
-
     template <typename T> void Skip(const std::size_t element_count)
     {
         boost::iostreams::seek(input_stream, element_count * sizeof(T), BOOST_IOS::cur);
@@ -141,7 +157,12 @@ class FileReader
 
     /*******************************************/
 
-    std::uint64_t ReadElementCount64() { return ReadOne<std::uint64_t>(); }
+    std::uint64_t ReadElementCount64()
+    {
+        std::uint64_t count;
+        ReadInto(count);
+        return count;
+    }
 
     template <typename T> std::size_t ReadVectorSize()
     {
@@ -152,7 +173,8 @@ class FileReader
 
     bool ReadAndCheckFingerprint()
     {
-        auto loaded_fingerprint = ReadOne<util::FingerPrint>();
+        util::FingerPrint loaded_fingerprint;
+        ReadInto(loaded_fingerprint);
         const auto expected_fingerprint = util::FingerPrint::GetValid();
 
         if (!loaded_fingerprint.IsValid())
@@ -239,14 +261,12 @@ class FileWriter
 
     template <typename T> void WriteFrom(const T &src) { WriteFrom(&src, 1); }
 
-    template <typename T> void WriteOne(const T &tmp) { WriteFrom(tmp); }
-
-    void WriteElementCount64(const std::uint64_t count) { WriteOne<std::uint64_t>(count); }
+    void WriteElementCount64(const std::uint64_t count) { WriteFrom(count); }
 
     void WriteFingerprint()
     {
         const auto fingerprint = util::FingerPrint::GetValid();
-        return WriteOne(fingerprint);
+        return WriteFrom(fingerprint);
     }
 
     template <typename T> void Skip(const std::size_t element_count)
@@ -270,6 +290,98 @@ class FileWriter
     boost::filesystem::ofstream output_stream;
     FingerprintFlag fingerprint;
 };
+
+class BufferReader
+{
+  public:
+    BufferReader(const std::string &buffer) : input_stream(buffer, std::ios::binary)
+    {
+        if (!input_stream)
+        {
+            throw util::RuntimeError(
+                "<buffer>", ErrorCode::FileOpenError, SOURCE_REF, std::strerror(errno));
+        }
+    }
+
+    template <typename T> void ReadInto(T *dest, const std::size_t count)
+    {
+#if !defined(__GNUC__) || (__GNUC__ > 4)
+        static_assert(!std::is_pointer<T>::value, "saving pointer types is not allowed");
+        static_assert(std::is_trivially_copyable<T>::value,
+                      "bytewise reading requires trivially copyable type");
+#endif
+
+        if (count == 0)
+            return;
+
+        const auto &result = input_stream.read(reinterpret_cast<char *>(dest), count * sizeof(T));
+        const std::size_t bytes_read = input_stream.gcount();
+
+        if (bytes_read != count * sizeof(T) && !result)
+        {
+            if (result.eof())
+            {
+                throw util::RuntimeError("<buffer>", ErrorCode::UnexpectedEndOfFile, SOURCE_REF);
+            }
+            throw util::RuntimeError(
+                "<buffer>", ErrorCode::FileReadError, SOURCE_REF, std::strerror(errno));
+        }
+    }
+
+    template <typename T> void ReadInto(T &tmp) { ReadInto(&tmp, 1); }
+
+    std::uint64_t ReadElementCount64()
+    {
+        std::uint64_t count;
+        ReadInto(count);
+        return count;
+    }
+
+  private:
+    std::istringstream input_stream;
+};
+
+class BufferWriter
+{
+  public:
+    BufferWriter() : output_stream(std::ios::binary)
+    {
+        if (!output_stream)
+        {
+            throw util::RuntimeError(
+                "<buffer>", ErrorCode::FileOpenError, SOURCE_REF, std::strerror(errno));
+        }
+    }
+
+    template <typename T> void WriteFrom(const T *src, const std::size_t count)
+    {
+#if !defined(__GNUC__) || (__GNUC__ > 4)
+        static_assert(std::is_trivially_copyable<T>::value,
+                      "bytewise writing requires trivially copyable type");
+#endif
+
+        if (count == 0)
+            return;
+
+        const auto &result =
+            output_stream.write(reinterpret_cast<const char *>(src), count * sizeof(T));
+
+        if (!result)
+        {
+            throw util::RuntimeError(
+                "<buffer>", ErrorCode::FileWriteError, SOURCE_REF, std::strerror(errno));
+        }
+    }
+
+    template <typename T> void WriteFrom(const T &tmp) { WriteFrom(&tmp, 1); }
+
+    void WriteElementCount64(const std::uint64_t count) { WriteFrom(count); }
+
+    std::string GetBuffer() const { return output_stream.str(); }
+
+  private:
+    std::ostringstream output_stream;
+};
 } // ns io
 } // ns storage
 } // ns osrm

include/storage/serialization.hpp

@@ -6,6 +6,10 @@
 #include "util/vector_view.hpp"
 
 #include "storage/io.hpp"
+#include "storage/tar.hpp"
+
+#include <boost/function_output_iterator.hpp>
+#include <boost/iterator/function_input_iterator.hpp>
 
 #include <cmath>
 #include <cstdint>
@@ -29,160 +33,190 @@ namespace serialization
  */
 
 template <typename T>
-inline void read(storage::io::FileReader &reader, util::DeallocatingVector<T> &vec)
+inline void
+read(storage::tar::FileReader &reader, const std::string &name, util::DeallocatingVector<T> &vec)
 {
-    vec.current_size = reader.ReadElementCount64(vec.current_size);
-    std::size_t num_blocks =
-        std::ceil(vec.current_size / util::DeallocatingVector<T>::ELEMENTS_PER_BLOCK);
-    vec.bucket_list.resize(num_blocks);
-    // Read all but the last block which can be partital
-    for (auto bucket_index : util::irange<std::size_t>(0, num_blocks - 1))
-    {
-        vec.bucket_list[bucket_index] = new T[util::DeallocatingVector<T>::ELEMENTS_PER_BLOCK];
-        reader.ReadInto(vec.bucket_list[bucket_index],
-                        util::DeallocatingVector<T>::ELEMENTS_PER_BLOCK);
-    }
-    std::size_t last_block_size =
-        vec.current_size % util::DeallocatingVector<T>::ELEMENTS_PER_BLOCK;
-    vec.bucket_list.back() = new T[util::DeallocatingVector<T>::ELEMENTS_PER_BLOCK];
-    reader.ReadInto(vec.bucket_list.back(), last_block_size);
+    vec.resize(reader.ReadElementCount64(name));
+    reader.ReadStreaming<T>(name, vec.begin(), vec.size());
 }
 
 template <typename T>
-inline void write(storage::io::FileWriter &writer, const util::DeallocatingVector<T> &vec)
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
+                  const util::DeallocatingVector<T> &vec)
 {
-    writer.WriteElementCount64(vec.current_size);
-    // Write all but the last block which can be partially filled
-    for (auto bucket_index : util::irange<std::size_t>(0, vec.bucket_list.size() - 1))
-    {
-        writer.WriteFrom(vec.bucket_list[bucket_index],
-                         util::DeallocatingVector<T>::ELEMENTS_PER_BLOCK);
-    }
-    std::size_t last_block_size =
-        vec.current_size % util::DeallocatingVector<T>::ELEMENTS_PER_BLOCK;
-    writer.WriteFrom(vec.bucket_list.back(), last_block_size);
+    writer.WriteElementCount64(name, vec.size());
+    writer.WriteStreaming<T>(name, vec.begin(), vec.size());
 }
 
 #if USE_STXXL_LIBRARY
-template <typename T> inline void read(storage::io::FileReader &reader, stxxl::vector<T> &vec)
+template <typename T>
+inline void read(storage::tar::FileReader &reader, const std::string &name, stxxl::vector<T> &vec)
 {
-    auto size = reader.ReadOne<std::uint64_t>();
+    auto size = reader.ReadElementCount64(name);
     vec.reserve(size);
-    for (auto idx : util::irange<std::size_t>(0, size))
-    {
-        (void)idx;
-        vec.push_back(reader.ReadOne<T>());
-    }
+    reader.ReadStreaming<T>(name, std::back_inserter(vec), size);
 }
 
 template <typename T>
-inline void write(storage::io::FileWriter &writer, const stxxl::vector<T> &vec)
+inline void
+write(storage::tar::FileWriter &writer, const std::string &name, const stxxl::vector<T> &vec)
 {
-    writer.WriteOne(vec.size());
-    for (auto idx : util::irange<std::size_t>(0, vec.size()))
-    {
-        writer.WriteOne<T>(vec[idx]);
-    }
+    writer.WriteElementCount64(name, vec.size());
+    writer.WriteStreaming<T>(name, vec.begin(), vec.size());
 }
 #endif
 
-template <typename T> void read(io::FileReader &reader, std::vector<T> &data)
+template <typename T> void read(io::BufferReader &reader, std::vector<T> &data)
 {
     const auto count = reader.ReadElementCount64();
     data.resize(count);
     reader.ReadInto(data.data(), count);
 }
 
-template <typename T> void write(io::FileWriter &writer, const std::vector<T> &data)
+template <typename T> void write(io::BufferWriter &writer, const std::vector<T> &data)
 {
     const auto count = data.size();
     writer.WriteElementCount64(count);
     writer.WriteFrom(data.data(), count);
 }
 
-template <typename T> void read(io::FileReader &reader, util::vector_view<T> &data)
-{
-    const auto count = reader.ReadElementCount64();
-    BOOST_ASSERT(data.size() == count);
-    reader.ReadInto(data.data(), count);
-}
-
-template <typename T> void write(io::FileWriter &writer, const util::vector_view<T> &data)
+inline void write(tar::FileWriter &writer, const std::string &name, const std::string &data)
 {
     const auto count = data.size();
-    writer.WriteElementCount64(count);
-    writer.WriteFrom(data.data(), count);
+    writer.WriteElementCount64(name, count);
+    writer.WriteFrom(name, data.data(), count);
+}
+
+inline void read(tar::FileReader &reader, const std::string &name, std::string &data)
+{
+    const auto count = reader.ReadElementCount64(name);
+    data.resize(count);
+    reader.ReadInto(name, const_cast<char *>(data.data()), count);
 }
 
 template <typename T>
-inline unsigned char packBits(const T &data, std::size_t index, std::size_t count)
+inline void read(tar::FileReader &reader, const std::string &name, std::vector<T> &data)
+{
+    const auto count = reader.ReadElementCount64(name);
+    data.resize(count);
+    reader.ReadInto(name, data.data(), count);
+}
+
+template <typename T>
+void write(tar::FileWriter &writer, const std::string &name, const std::vector<T> &data)
+{
+    const auto count = data.size();
+    writer.WriteElementCount64(name, count);
+    writer.WriteFrom(name, data.data(), count);
+}
+
+template <typename T>
+void read(tar::FileReader &reader, const std::string &name, util::vector_view<T> &data)
+{
+    const auto count = reader.ReadElementCount64(name);
+    data.resize(count);
+    reader.ReadInto(name, data.data(), count);
+}
+
+template <typename T>
+void write(tar::FileWriter &writer, const std::string &name, const util::vector_view<T> &data)
+{
+    const auto count = data.size();
+    writer.WriteElementCount64(name, count);
+    writer.WriteFrom(name, data.data(), count);
+}
+
+namespace detail
+{
+template <typename T, typename BlockT = unsigned char>
+inline BlockT packBits(const T &data, std::size_t index, std::size_t count)
 {
     static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
-    unsigned char value = 0;
+    BlockT value = 0;
     for (std::size_t bit = 0; bit < count; ++bit, ++index)
         value = (value << 1) | data[index];
     return value;
 }
 
-template <typename T>
-inline void unpackBits(T &data, std::size_t index, std::size_t count, unsigned char value)
+template <typename T, typename BlockT = unsigned char>
+inline void unpackBits(T &data, std::size_t index, std::size_t count, BlockT value)
 {
     static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
-    const unsigned char mask = 1 << (count - 1);
+    const BlockT mask = BlockT{1} << (count - 1);
     for (std::size_t bit = 0; bit < count; value <<= 1, ++bit, ++index)
         data[index] = value & mask;
 }
 
-template <> inline void read<bool>(io::FileReader &reader, util::vector_view<bool> &data)
+template <typename VectorT>
+void readBoolVector(tar::FileReader &reader, const std::string &name, VectorT &data)
 {
-    const auto count = reader.ReadElementCount64();
-    BOOST_ASSERT(data.size() == count);
-    std::uint64_t index = 0;
-    for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
-    {
-        unpackBits(data, index, CHAR_BIT, reader.ReadOne<unsigned char>());
-    }
-    if (count > index)
-        unpackBits(data, index, count - index, reader.ReadOne<unsigned char>());
-}
-
-template <> inline void write<bool>(io::FileWriter &writer, const util::vector_view<bool> &data)
-{
-    const auto count = data.size();
-    writer.WriteElementCount64(count);
-    std::uint64_t index = 0;
-    for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
-    {
-        writer.WriteOne<unsigned char>(packBits(data, CHAR_BIT * index, CHAR_BIT));
-    }
-    if (count > index)
-        writer.WriteOne<unsigned char>(packBits(data, index, count - index));
-}
-
-template <> inline void read<bool>(io::FileReader &reader, std::vector<bool> &data)
-{
-    const auto count = reader.ReadElementCount64();
+    const auto count = reader.ReadElementCount64(name);
     data.resize(count);
     std::uint64_t index = 0;
-    for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
-    {
-        unpackBits(data, index, CHAR_BIT, reader.ReadOne<unsigned char>());
-    }
-    if (count > index)
-        unpackBits(data, index, count - index, reader.ReadOne<unsigned char>());
+
+    constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
+
+    const auto decode = [&](const std::uint64_t block) {
+        auto read_size = std::min<std::size_t>(count - index, WORD_BITS);
+        unpackBits<VectorT, std::uint64_t>(data, index, read_size, block);
+        index += WORD_BITS;
+    };
+
+    reader.ReadStreaming<std::uint64_t>(name, boost::make_function_output_iterator(decode));
 }
 
-template <> inline void write<bool>(io::FileWriter &writer, const std::vector<bool> &data)
+template <typename VectorT>
+void writeBoolVector(tar::FileWriter &writer, const std::string &name, const VectorT &data)
 {
     const auto count = data.size();
-    writer.WriteElementCount64(count);
+    writer.WriteElementCount64(name, count);
     std::uint64_t index = 0;
-    for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
-    {
-        writer.WriteOne<unsigned char>(packBits(data, index, CHAR_BIT));
-    }
-    if (count > index)
-        writer.WriteOne<unsigned char>(packBits(data, index, count - index));
+
+    constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
+
+    // FIXME on old boost version the function_input_iterator does not work with lambdas
+    // so we need to wrap it in a function here.
+    const std::function<std::uint64_t()> encode_function = [&]() -> std::uint64_t {
+        auto write_size = std::min<std::size_t>(count - index, WORD_BITS);
+        auto packed = packBits<VectorT, std::uint64_t>(data, index, write_size);
+        index += WORD_BITS;
+        return packed;
+    };
+
+    std::uint64_t number_of_blocks = (count + WORD_BITS - 1) / WORD_BITS;
+    writer.WriteStreaming<std::uint64_t>(
+        name,
+        boost::make_function_input_iterator(encode_function, boost::infinite()),
+        number_of_blocks);
+}
+}
+
+template <>
+inline void
+read<bool>(tar::FileReader &reader, const std::string &name, util::vector_view<bool> &data)
+{
+    detail::readBoolVector(reader, name, data);
+}
+
+template <>
+inline void
+write<bool>(tar::FileWriter &writer, const std::string &name, const util::vector_view<bool> &data)
+{
+    detail::writeBoolVector(writer, name, data);
+}
+
+template <>
+inline void read<bool>(tar::FileReader &reader, const std::string &name, std::vector<bool> &data)
+{
+    detail::readBoolVector(reader, name, data);
+}
+
+template <>
+inline void
+write<bool>(tar::FileWriter &writer, const std::string &name, const std::vector<bool> &data)
+{
+    detail::writeBoolVector(writer, name, data);
 }
 }
 }

include/storage/shared_datatype.hpp

@@ -20,7 +20,9 @@ namespace storage
 // Added at the start and end of each block as sanity check
 const constexpr char CANARY[4] = {'O', 'S', 'R', 'M'};
 
-const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
+const constexpr char *block_id_to_name[] = {"IGNORE_BLOCK",
+                                            "NAME_BLOCKS",
+                                            "NAME_VALUES",
                                             "EDGE_BASED_NODE_DATA",
                                             "ANNOTATION_DATA",
                                             "CH_GRAPH_NODE_LIST",
@@ -38,7 +40,7 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
                                             "TURN_INSTRUCTION",
                                             "ENTRY_CLASSID",
                                             "R_SEARCH_TREE",
-                                            "R_SEARCH_TREE_LEVELS",
+                                            "R_SEARCH_TREE_LEVEL_STARTS",
                                             "GEOMETRIES_INDEX",
                                             "GEOMETRIES_NODE_LIST",
                                             "GEOMETRIES_FWD_WEIGHT_LIST",
@@ -48,7 +50,6 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
                                             "GEOMETRIES_FWD_DATASOURCES_LIST",
                                             "GEOMETRIES_REV_DATASOURCES_LIST",
                                             "HSGR_CHECKSUM",
-                                            "TIMESTAMP",
                                             "FILE_INDEX_PATH",
                                             "DATASOURCES_NAMES",
                                             "PROPERTIES",
@@ -94,11 +95,14 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
                                             "MANEUVER_OVERRIDES",
                                             "MANEUVER_OVERRIDE_NODE_SEQUENCES"};
 
-struct DataLayout
+class DataLayout
 {
+  public:
     enum BlockID
     {
-        NAME_CHAR_DATA = 0,
+        IGNORE_BLOCK = 0,
+        NAME_BLOCKS,
+        NAME_VALUES,
         EDGE_BASED_NODE_DATA_LIST,
         ANNOTATION_DATA_LIST,
         CH_GRAPH_NODE_LIST,
@@ -116,7 +120,7 @@ struct DataLayout
         TURN_INSTRUCTION,
         ENTRY_CLASSID,
         R_SEARCH_TREE,
-        R_SEARCH_TREE_LEVELS,
+        R_SEARCH_TREE_LEVEL_STARTS,
         GEOMETRIES_INDEX,
         GEOMETRIES_NODE_LIST,
         GEOMETRIES_FWD_WEIGHT_LIST,
@@ -126,7 +130,6 @@ struct DataLayout
         GEOMETRIES_FWD_DATASOURCES_LIST,
         GEOMETRIES_REV_DATASOURCES_LIST,
         HSGR_CHECKSUM,
-        TIMESTAMP,
         FILE_INDEX_PATH,
         DATASOURCES_NAMES,
         PROPERTIES,
@@ -174,8 +177,6 @@ struct DataLayout
         NUM_BLOCKS
     };
 
-    std::array<Block, NUM_BLOCKS> blocks;
-
     DataLayout() : blocks{} {}
 
     inline void SetBlock(BlockID bid, Block block) { blocks[bid] = std::move(block); }
@@ -189,47 +190,17 @@ struct DataLayout
         uint64_t result = 0;
         for (auto i = 0; i < NUM_BLOCKS; i++)
         {
-            BOOST_ASSERT(blocks[i].entry_align > 0);
-            result += 2 * sizeof(CANARY) + GetBlockSize((BlockID)i) + blocks[i].entry_align;
+            result += 2 * sizeof(CANARY) + GetBlockSize(static_cast<BlockID>(i)) + BLOCK_ALIGNMENT;
         }
         return result;
     }
 
-    // \brief Fit aligned storage in buffer.
-    // Interface Similar to [ptr.align] but omits space computation.
-    // The method can be removed and changed directly to an std::align
-    // function call after dropping gcc < 5 support.
-    inline void *align(std::size_t align, std::size_t, void *&ptr) const noexcept
-    {
-        const auto intptr = reinterpret_cast<uintptr_t>(ptr);
-        const auto aligned = (intptr - 1u + align) & -align;
-        return ptr = reinterpret_cast<void *>(aligned);
-    }
-
-    inline void *GetAlignedBlockPtr(void *ptr, BlockID bid) const
-    {
-        for (auto i = 0; i < bid; i++)
-        {
-            ptr = static_cast<char *>(ptr) + sizeof(CANARY);
-            ptr = align(blocks[i].entry_align, blocks[i].entry_size, ptr);
-            ptr = static_cast<char *>(ptr) + GetBlockSize((BlockID)i);
-            ptr = static_cast<char *>(ptr) + sizeof(CANARY);
-        }
-
-        ptr = static_cast<char *>(ptr) + sizeof(CANARY);
-        ptr = align(blocks[bid].entry_align, blocks[bid].entry_size, ptr);
-        return ptr;
-    }
-
-    template <typename T> inline T *GetBlockEnd(char *shared_memory, BlockID bid) const
-    {
-        auto begin = GetBlockPtr<T>(shared_memory, bid);
-        return begin + GetBlockEntries(bid);
-    }
-
     template <typename T, bool WRITE_CANARY = false>
     inline T *GetBlockPtr(char *shared_memory, BlockID bid) const
     {
+        static_assert(BLOCK_ALIGNMENT % std::alignment_of<T>::value == 0,
+                      "Datatype does not fit alignment constraints.");
+
         char *ptr = (char *)GetAlignedBlockPtr(shared_memory, bid);
         if (WRITE_CANARY)
         {
@@ -258,6 +229,39 @@ struct DataLayout
 
         return (T *)ptr;
     }
+
+  private:
+    // Fit aligned storage in buffer to 64 bytes to conform with AVX 512 types
+    inline void *align(void *&ptr) const noexcept
+    {
+        const auto intptr = reinterpret_cast<uintptr_t>(ptr);
+        const auto aligned = (intptr - 1u + BLOCK_ALIGNMENT) & -BLOCK_ALIGNMENT;
+        return ptr = reinterpret_cast<void *>(aligned);
+    }
+
+    inline void *GetAlignedBlockPtr(void *ptr, BlockID bid) const
+    {
+        for (auto i = 0; i < bid; i++)
+        {
+            ptr = static_cast<char *>(ptr) + sizeof(CANARY);
+            ptr = align(ptr);
+            ptr = static_cast<char *>(ptr) + GetBlockSize((BlockID)i);
+            ptr = static_cast<char *>(ptr) + sizeof(CANARY);
+        }
+
+        ptr = static_cast<char *>(ptr) + sizeof(CANARY);
+        ptr = align(ptr);
+        return ptr;
+    }
+
+    template <typename T> inline T *GetBlockEnd(char *shared_memory, BlockID bid) const
+    {
+        auto begin = GetBlockPtr<T>(shared_memory, bid);
+        return begin + GetBlockEntries(bid);
+    }
+
+    static constexpr std::size_t BLOCK_ALIGNMENT = 64;
+    std::array<Block, NUM_BLOCKS> blocks;
 };
 
 enum SharedDataType

include/storage/storage_config.hpp

@@ -54,7 +54,6 @@ struct StorageConfig final : IOConfig
                     ".osrm.fileIndex",
                     ".osrm.edges",
                     ".osrm.geometry",
-                    ".osrm.timestamp",
                     ".osrm.turn_weight_penalties",
                     ".osrm.turn_duration_penalties",
                     ".osrm.datasource_names",

include/storage/tar.hpp

@@ -0,0 +1,319 @@
+#ifndef OSRM_STORAGE_TAR_HPP
+#define OSRM_STORAGE_TAR_HPP
+
+#include "util/exception.hpp"
+#include "util/exception_utils.hpp"
+#include "util/fingerprint.hpp"
+#include "util/integer_range.hpp"
+#include "util/version.hpp"
+
+#include <boost/filesystem/path.hpp>
+
+extern "C" {
+#include "microtar.h"
+}
+
+namespace osrm
+{
+namespace storage
+{
+namespace tar
+{
+namespace detail
+{
+inline void
+checkMTarError(int error_code, const boost::filesystem::path &filepath, const std::string &name)
+{
+    switch (error_code)
+    {
+    case MTAR_ESUCCESS:
+        return;
+    case MTAR_EFAILURE:
+        throw util::RuntimeError(filepath.string() + " : " + name,
+                                 ErrorCode::FileIOError,
+                                 SOURCE_REF,
+                                 std::strerror(errno));
+    case MTAR_EOPENFAIL:
+        throw util::RuntimeError(filepath.string() + " : " + name,
+                                 ErrorCode::FileOpenError,
+                                 SOURCE_REF,
+                                 std::strerror(errno));
+    case MTAR_EREADFAIL:
+        throw util::RuntimeError(filepath.string() + " : " + name,
+                                 ErrorCode::FileReadError,
+                                 SOURCE_REF,
+                                 std::strerror(errno));
+    case MTAR_EWRITEFAIL:
+        throw util::RuntimeError(filepath.string() + " : " + name,
+                                 ErrorCode::FileWriteError,
+                                 SOURCE_REF,
+                                 std::strerror(errno));
+    case MTAR_ESEEKFAIL:
+        throw util::RuntimeError(filepath.string() + " : " + name,
+                                 ErrorCode::FileIOError,
+                                 SOURCE_REF,
+                                 std::strerror(errno));
+    case MTAR_EBADCHKSUM:
+        throw util::RuntimeError(filepath.string() + " : " + name,
+                                 ErrorCode::FileIOError,
+                                 SOURCE_REF,
+                                 std::strerror(errno));
+    case MTAR_ENULLRECORD:
+        throw util::RuntimeError(filepath.string() + " : " + name,
+                                 ErrorCode::UnexpectedEndOfFile,
+                                 SOURCE_REF,
+                                 std::strerror(errno));
+    case MTAR_ENOTFOUND:
+        throw util::RuntimeError(filepath.string() + " : " + name,
+                                 ErrorCode::FileIOError,
+                                 SOURCE_REF,
+                                 std::strerror(errno));
+    default:
+        throw util::exception(filepath.string() + " : " + name + ":" + mtar_strerror(error_code));
+    }
+}
+}
+
+class FileReader
+{
+  public:
+    enum FingerprintFlag
+    {
+        VerifyFingerprint,
+        HasNoFingerprint
+    };
+
+    FileReader(const boost::filesystem::path &path, FingerprintFlag flag) : path(path)
+    {
+        auto ret = mtar_open(&handle, path.string().c_str(), "r");
+        detail::checkMTarError(ret, path, "");
+
+        if (flag == VerifyFingerprint)
+        {
+            ReadAndCheckFingerprint();
+        }
+    }
+
+    ~FileReader() { mtar_close(&handle); }
+
+    std::uint64_t ReadElementCount64(const std::string &name)
+    {
+        std::uint64_t size;
+        ReadInto(name + ".meta", size);
+        return size;
+    }
+
+    template <typename T> void ReadInto(const std::string &name, T &tmp)
+    {
+        ReadInto(name, &tmp, 1);
+    }
+
+    template <typename T, typename OutIter> void ReadStreaming(const std::string &name, OutIter out)
+    {
+        mtar_header_t header;
+        auto ret = mtar_find(&handle, name.c_str(), &header);
+        detail::checkMTarError(ret, path, name);
+
+        auto number_of_elements = header.size / sizeof(T);
+        auto expected_size = sizeof(T) * number_of_elements;
+        if (header.size != expected_size)
+        {
+            throw util::RuntimeError(name + ": Datatype size does not match file size.",
+                                     ErrorCode::UnexpectedEndOfFile,
+                                     SOURCE_REF);
+        }
+
+        T tmp;
+        for (auto index : util::irange<std::size_t>(0, number_of_elements))
+        {
+            (void)index;
+            ret = mtar_read_data(&handle, reinterpret_cast<char *>(&tmp), sizeof(T));
+            detail::checkMTarError(ret, path, name);
+
+            *out++ = tmp;
+        }
+    }
+
+    template <typename T>
+    void ReadInto(const std::string &name, T *data, const std::size_t number_of_elements)
+    {
+        mtar_header_t header;
+        auto ret = mtar_find(&handle, name.c_str(), &header);
+        detail::checkMTarError(ret, path, name);
+
+        auto expected_size = sizeof(T) * number_of_elements;
+        if (header.size != expected_size)
+        {
+            throw util::RuntimeError(name + ": Datatype size does not match file size.",
+                                     ErrorCode::UnexpectedEndOfFile,
+                                     SOURCE_REF);
+        }
+
+        ret = mtar_read_data(&handle, reinterpret_cast<char *>(data), header.size);
+        detail::checkMTarError(ret, path, name);
+    }
+
+    struct FileEntry
+    {
+        std::string name;
+        std::size_t size;
+        std::size_t offset;
+    };
+
+    template <typename OutIter> void List(OutIter out)
+    {
+        mtar_header_t header;
+        while (mtar_read_header(&handle, &header) != MTAR_ENULLRECORD)
+        {
+            if (header.type == MTAR_TREG)
+            {
+                int ret = mtar_read_data(&handle, nullptr, 0);
+                detail::checkMTarError(ret, path, header.name);
+
+                auto offset = handle.pos;
+                // seek back to the header
+                handle.remaining_data = 0;
+                ret = mtar_seek(&handle, handle.last_header);
+                detail::checkMTarError(ret, path, header.name);
+
+                *out++ = FileEntry{header.name, header.size, offset};
+            }
+            mtar_next(&handle);
+        }
+    }
+
+  private:
+    bool ReadAndCheckFingerprint()
+    {
+        util::FingerPrint loaded_fingerprint;
+        ReadInto("osrm_fingerprint.meta", loaded_fingerprint);
+        const auto expected_fingerprint = util::FingerPrint::GetValid();
+
+        if (!loaded_fingerprint.IsValid())
+        {
+            throw util::RuntimeError(path.string(), ErrorCode::InvalidFingerprint, SOURCE_REF);
+        }
+
+        if (!expected_fingerprint.IsDataCompatible(loaded_fingerprint))
+        {
+            const std::string fileversion =
+                std::to_string(loaded_fingerprint.GetMajorVersion()) + "." +
+                std::to_string(loaded_fingerprint.GetMinorVersion()) + "." +
+                std::to_string(loaded_fingerprint.GetPatchVersion());
+            throw util::RuntimeError(std::string(path.string()) + " prepared with OSRM " +
+                                         fileversion + " but this is " + OSRM_VERSION,
+                                     ErrorCode::IncompatibleFileVersion,
+                                     SOURCE_REF);
+        }
+
+        return true;
+    }
+
+    boost::filesystem::path path;
+    mtar_t handle;
+};
+
+class FileWriter
+{
+  public:
+    enum FingerprintFlag
+    {
+        GenerateFingerprint,
+        HasNoFingerprint
+    };
+
+    FileWriter(const boost::filesystem::path &path, FingerprintFlag flag) : path(path)
+    {
+        auto ret = mtar_open(&handle, path.string().c_str(), "w");
+        detail::checkMTarError(ret, path, "");
+
+        if (flag == GenerateFingerprint)
+        {
+            WriteFingerprint();
+        }
+    }
+
+    ~FileWriter()
+    {
+        mtar_finalize(&handle);
+        mtar_close(&handle);
+    }
+
+    void WriteElementCount64(const std::string &name, const std::uint64_t count)
+    {
+        WriteFrom(name + ".meta", count);
+    }
+
+    template <typename T> void WriteFrom(const std::string &name, const T &data)
+    {
+        WriteFrom(name, &data, 1);
+    }
+
+    template <typename T, typename Iter>
+    void WriteStreaming(const std::string &name, Iter iter, const std::uint64_t number_of_elements)
+    {
+        auto number_of_bytes = number_of_elements * sizeof(T);
+
+        auto ret = mtar_write_file_header(&handle, name.c_str(), number_of_bytes);
+        detail::checkMTarError(ret, path, name);
+
+        for (auto index : util::irange<std::size_t>(0, number_of_elements))
+        {
+            (void)index;
+            T tmp = *iter++;
+            ret = mtar_write_data(&handle, &tmp, sizeof(T));
+            detail::checkMTarError(ret, path, name);
+        }
+    }
+
+    // Continue writing an existing file, overwrites all data after the file!
+    template <typename T>
+    void ContinueFrom(const std::string &name, const T *data, const std::size_t number_of_elements)
+    {
+        auto number_of_bytes = number_of_elements * sizeof(T);
+
+        mtar_header_t header;
+        auto ret = mtar_find(&handle, name.c_str(), &header);
+        detail::checkMTarError(ret, path, name);
+
+        // update header to reflect increased tar size
+        auto old_size = header.size;
+        header.size += number_of_bytes;
+        mtar_write_header(&handle, &header);
+
+        // now seek to the end of the old record
+        handle.remaining_data = number_of_bytes;
+        ret = mtar_seek(&handle, handle.pos + old_size);
+        detail::checkMTarError(ret, path, name);
+
+        ret = mtar_write_data(&handle, data, number_of_bytes);
+        detail::checkMTarError(ret, path, name);
+    }
+
+    template <typename T>
+    void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_elements)
+    {
+        auto number_of_bytes = number_of_elements * sizeof(T);
+
+        auto ret = mtar_write_file_header(&handle, name.c_str(), number_of_bytes);
+        detail::checkMTarError(ret, path, name);
+
+        ret = mtar_write_data(&handle, reinterpret_cast<const char *>(data), number_of_bytes);
+        detail::checkMTarError(ret, path, name);
+    }
+
+  private:
+    void WriteFingerprint()
+    {
+        const auto fingerprint = util::FingerPrint::GetValid();
+        WriteFrom("osrm_fingerprint.meta", fingerprint);
+    }
+
+    boost::filesystem::path path;
+    mtar_t handle;
+};
+}
+}
+}
+
+#endif

include/storage/tar_fwd.hpp

@@ -0,0 +1,18 @@
+#ifndef OSRM_STORAGE_TAR_FWD_HPP_
+#define OSRM_STORAGE_TAR_FWD_HPP_
+
+namespace osrm
+{
+namespace storage
+{
+namespace tar
+{
+
+class FileReader;
+class FileWriter;
+
+} // ns io
+} // ns storage
+} // ns osrm
+
+#endif

include/util/deallocating_vector.hpp

@@ -13,23 +13,6 @@
 
 namespace osrm
 {
-namespace util
-{
-template <typename ElementT> class DeallocatingVector;
-}
-
-namespace storage
-{
-namespace serialization
-{
-template <typename T>
-inline void read(storage::io::FileReader &reader, util::DeallocatingVector<T> &vec);
-
-template <typename T>
-inline void write(storage::io::FileWriter &writer, const util::DeallocatingVector<T> &vec);
-}
-}
-
 namespace util
 {
 template <typename ElementT> struct ConstDeallocatingVectorIteratorState
@@ -183,6 +166,8 @@ class DeallocatingVectorIterator
     }
 };
 
+template <typename ElementT> class DeallocatingVector;
+
 template <typename T> void swap(DeallocatingVector<T> &lhs, DeallocatingVector<T> &rhs);
 
 template <typename ElementT> class DeallocatingVector
@@ -228,6 +213,14 @@ template <typename ElementT> class DeallocatingVector
         return *this;
     }
 
+    DeallocatingVector(std::initializer_list<ElementT> elements) : DeallocatingVector()
+    {
+        for (auto &&elem : elements)
+        {
+            emplace_back(std::move(elem));
+        }
+    }
+
     ~DeallocatingVector() { clear(); }
 
     friend void swap<>(DeallocatingVector<ElementT> &lhs, DeallocatingVector<ElementT> &rhs);
@@ -351,11 +344,6 @@ template <typename ElementT> class DeallocatingVector
             ++position;
         }
     }
-
-    friend void storage::serialization::read<ElementT>(storage::io::FileReader &reader,
-                                                       DeallocatingVector &vec);
-    friend void storage::serialization::write<ElementT>(storage::io::FileWriter &writer,
-                                                        const DeallocatingVector &vec);
 };
 
 template <typename T> void swap(DeallocatingVector<T> &lhs, DeallocatingVector<T> &rhs)

include/util/dynamic_graph.hpp

@@ -6,8 +6,6 @@
 #include "util/permutation.hpp"
 #include "util/typedefs.hpp"
 
-#include "storage/io_fwd.hpp"
-
 #include <boost/assert.hpp>
 
 #include <cstdint>
@@ -22,17 +20,6 @@ namespace osrm
 {
 namespace util
 {
-template <typename EdgeDataT> class DynamicGraph;
-
-namespace serialization
-{
-template <typename EdgeDataT, bool UseSharedMemory>
-void read(storage::io::FileReader &reader, DynamicGraph<EdgeDataT> &graph);
-
-template <typename EdgeDataT, bool UseSharedMemory>
-void write(storage::io::FileWriter &writer, const DynamicGraph<EdgeDataT> &graph);
-}
-
 namespace detail
 {
 // These types need to live outside of DynamicGraph

include/util/graph_loader.hpp

@@ -1,131 +0,0 @@
-#ifndef GRAPH_LOADER_HPP
-#define GRAPH_LOADER_HPP
-
-#include "extractor/node_based_edge.hpp"
-#include "extractor/packed_osm_ids.hpp"
-#include "extractor/query_node.hpp"
-#include "extractor/restriction.hpp"
-#include "storage/io.hpp"
-#include "util/exception.hpp"
-#include "util/fingerprint.hpp"
-#include "util/log.hpp"
-#include "util/packed_vector.hpp"
-#include "util/typedefs.hpp"
-
-#include <boost/assert.hpp>
-#include <boost/filesystem.hpp>
-#include <boost/filesystem/fstream.hpp>
-
-#include <tbb/parallel_sort.h>
-
-#include <cmath>
-
-#include <fstream>
-#include <ios>
-#include <unordered_set>
-#include <vector>
-
-namespace osrm
-{
-namespace util
-{
-
-/**
- * Reads the beginning of an .osrm file and produces:
- *  - barrier nodes
- *  - traffic lights
- *  - nodes indexed by their internal (non-osm) id
- */
-template <typename BarrierOutIter, typename TrafficSignalsOutIter>
-NodeID loadNodesFromFile(storage::io::FileReader &file_reader,
-                         BarrierOutIter barriers,
-                         TrafficSignalsOutIter traffic_signals,
-                         std::vector<util::Coordinate> &coordinates,
-                         extractor::PackedOSMIDs &osm_node_ids)
-{
-    auto number_of_nodes = file_reader.ReadElementCount64();
-    Log() << "Importing number_of_nodes new = " << number_of_nodes << " nodes ";
-
-    coordinates.resize(number_of_nodes);
-    osm_node_ids.reserve(number_of_nodes);
-
-    extractor::QueryNode current_node;
-    for (NodeID i = 0; i < number_of_nodes; ++i)
-    {
-        file_reader.ReadInto(&current_node, 1);
-
-        coordinates[i].lon = current_node.lon;
-        coordinates[i].lat = current_node.lat;
-        osm_node_ids.push_back(current_node.node_id);
-    }
-
-    auto num_barriers = file_reader.ReadElementCount64();
-    for (auto index = 0UL; index < num_barriers; ++index)
-    {
-        *barriers++ = file_reader.ReadOne<NodeID>();
-    }
-
-    auto num_lights = file_reader.ReadElementCount64();
-    for (auto index = 0UL; index < num_lights; ++index)
-    {
-        *traffic_signals++ = file_reader.ReadOne<NodeID>();
-    }
-
-    return number_of_nodes;
-}
-
-/**
- * Reads a .osrm file and produces the edges.
- */
-inline EdgeID loadEdgesFromFile(storage::io::FileReader &file_reader,
-                                std::vector<extractor::NodeBasedEdge> &edge_list)
-{
-    auto number_of_edges = file_reader.ReadElementCount64();
-
-    edge_list.resize(number_of_edges);
-    Log() << " and " << number_of_edges << " edges ";
-
-    file_reader.ReadInto(edge_list.data(), number_of_edges);
-
-    BOOST_ASSERT(edge_list.size() > 0);
-
-#ifndef NDEBUG
-    Log() << "Validating loaded edges...";
-    tbb::parallel_sort(
-        edge_list.begin(),
-        edge_list.end(),
-        [](const extractor::NodeBasedEdge &lhs, const extractor::NodeBasedEdge &rhs) {
-            return (lhs.source < rhs.source) ||
-                   (lhs.source == rhs.source && lhs.target < rhs.target);
-        });
-    for (auto i = 1u; i < edge_list.size(); ++i)
-    {
-        const auto &edge = edge_list[i];
-        const auto &prev_edge = edge_list[i - 1];
-
-        BOOST_ASSERT_MSG(edge.weight > 0, "loaded null weight");
-        BOOST_ASSERT_MSG(edge.flags.forward, "edge must be oriented in forward direction");
-
-        BOOST_ASSERT_MSG(edge.source != edge.target, "loaded edges contain a loop");
-        BOOST_ASSERT_MSG(edge.source != prev_edge.source || edge.target != prev_edge.target,
-                         "loaded edges contain a multi edge");
-    }
-#endif
-
-    Log() << "Graph loaded ok and has " << edge_list.size() << " edges";
-
-    return number_of_edges;
-}
-
-inline EdgeID loadAnnotationData(storage::io::FileReader &file_reader,
-                                 std::vector<extractor::NodeBasedEdgeAnnotation> &metadata)
-{
-    auto const meta_data_count = file_reader.ReadElementCount64();
-    metadata.resize(meta_data_count);
-    file_reader.ReadInto(metadata.data(), meta_data_count);
-    return meta_data_count;
-}
-}
-}
-
-#endif // GRAPH_LOADER_HPP

include/util/guidance/name_announcements.hpp

@@ -3,9 +3,10 @@
 
 /* A set of tools required for guidance in both pre and post-processing */
 
+#include "extractor/name_table.hpp"
 #include "extractor/suffix_table.hpp"
+
 #include "util/attributes.hpp"
-#include "util/name_table.hpp"
 #include "util/typedefs.hpp"
 
 #include <algorithm>
@@ -216,7 +217,7 @@ inline bool requiresNameAnnounced(const std::string &from_name,
 
 inline bool requiresNameAnnounced(const NameID from_name_id,
                                   const NameID to_name_id,
-                                  const util::NameTable &name_table,
+                                  const extractor::NameTable &name_table,
                                   const extractor::SuffixTable &suffix_table)
 {
     if (from_name_id == to_name_id)

include/util/indexed_data.hpp

@@ -1,12 +1,14 @@
 #ifndef OSRM_INDEXED_DATA_HPP
 #define OSRM_INDEXED_DATA_HPP
 
-#include "storage/io.hpp"
+#include "storage/tar_fwd.hpp"
 
 #include "util/exception.hpp"
 #include "util/string_view.hpp"
+#include "util/vector_view.hpp"
 
 #include <boost/assert.hpp>
+#include <boost/function_output_iterator.hpp>
 
 #include <array>
 #include <iterator>
@@ -18,6 +20,23 @@ namespace osrm
 {
 namespace util
 {
+namespace detail
+{
+template <typename GroupBlockPolicy, storage::Ownership Ownership> struct IndexedDataImpl;
+}
+
+namespace serialization
+{
+template <typename BlockPolicy, storage::Ownership Ownership>
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 detail::IndexedDataImpl<BlockPolicy, Ownership> &index_data);
+
+template <typename BlockPolicy, storage::Ownership Ownership>
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
+                  const detail::IndexedDataImpl<BlockPolicy, Ownership> &index_data);
+}
 
 template <int N, typename T = std::string> struct VariableGroupBlock
 {
@@ -85,11 +104,11 @@ template <int N, typename T = std::string> struct VariableGroupBlock
     /// of prefix length. sum(descriptor) equals to the block
     /// prefix length.
     /// Returns the block prefix length.
-    template <typename Offset, typename OffsetIterator>
-    Offset WriteBlockReference(storage::io::FileWriter &out,
-                               Offset data_offset,
-                               OffsetIterator first,
-                               OffsetIterator last) const
+    template <typename Offset, typename OffsetIterator, typename OutIter>
+    OutIter WriteBlockReference(OffsetIterator first,
+                                OffsetIterator last,
+                                Offset &data_offset,
+                                OutIter out) const
     {
         BOOST_ASSERT(data_offset <= std::numeric_limits<decltype(BlockReference::offset)>::max());
 
@@ -106,9 +125,9 @@ template <int N, typename T = std::string> struct VariableGroupBlock
             prefix_length += byte_length;
         }
 
-        out.WriteOne(refernce);
-
-        return prefix_length;
+        data_offset += prefix_length;
+        *out++ = refernce;
+        return out;
     }
 
     /// Write a block prefix that is an array of variable encoded data lengths:
@@ -118,9 +137,8 @@ template <int N, typename T = std::string> struct VariableGroupBlock
     ///   65536..16777215 is 3 bytes.
     /// [first..last] is an inclusive range of block data.
     /// The length of the last item in the block is not stored.
-    template <typename OffsetIterator>
-    void
-    WriteBlockPrefix(storage::io::FileWriter &out, OffsetIterator first, OffsetIterator last) const
+    template <typename OffsetIterator, typename OutByteIter>
+    OutByteIter WriteBlockPrefix(OffsetIterator first, OffsetIterator last, OutByteIter out) const
     {
         for (OffsetIterator curr = first, next = std::next(first); curr != last; ++curr, ++next)
         {
@@ -131,8 +149,9 @@ template <int N, typename T = std::string> struct VariableGroupBlock
 
             // Here, we're only writing a few bytes from the 4-byte std::uint32_t,
             // so we need to cast to (char *)
-            out.WriteFrom((const char *)&data_length, byte_length);
+            out = std::copy_n((const char *)&data_length, byte_length, out);
         }
+        return out;
     }
 
     /// Advances the range to an item stored in the referenced block.
@@ -178,36 +197,41 @@ template <int N, typename T = std::string> struct FixedGroupBlock
 
     /// Write a block reference {offset}, where offset is a global block offset
     /// Returns the fixed block prefix length.
-    template <typename Offset, typename OffsetIterator>
-    Offset WriteBlockReference(storage::io::FileWriter &out,
-                               Offset data_offset,
-                               OffsetIterator,
-                               OffsetIterator) const
+    template <typename Offset, typename OffsetIterator, typename OutIterator>
+    OutIterator
+    WriteBlockReference(OffsetIterator, OffsetIterator, Offset &data_offset, OutIterator out) const
     {
         BOOST_ASSERT(data_offset <= std::numeric_limits<decltype(BlockReference::offset)>::max());
 
         BlockReference refernce{static_cast<decltype(BlockReference::offset)>(data_offset)};
-        out.WriteOne(refernce);
+        data_offset += BLOCK_SIZE;
+        *out++ = refernce;
 
-        return BLOCK_SIZE;
+        return out;
     }
 
     /// Write a fixed length block prefix.
-    template <typename OffsetIterator>
-    void
-    WriteBlockPrefix(storage::io::FileWriter &out, OffsetIterator first, OffsetIterator last) const
+    template <typename OffsetIterator, typename OutByteIter>
+    OutByteIter WriteBlockPrefix(OffsetIterator first, OffsetIterator last, OutByteIter out) const
     {
-        std::uint32_t index = 0;
-        std::array<ValueType, BLOCK_SIZE> block_prefix;
+        constexpr std::size_t MAX_LENGTH =
+            std::numeric_limits<std::make_unsigned_t<ValueType>>::max();
+
+        auto index = 0;
+        std::array<ValueType, BLOCK_SIZE> prefix;
+
         for (OffsetIterator curr = first, next = std::next(first); curr != last; ++curr, ++next)
         {
             const std::uint32_t data_length = *next - *curr;
-            if (data_length >= 0x100)
-                throw util::exception(boost::format("too large data length %1%") % data_length);
+            if (data_length > MAX_LENGTH)
+                throw util::exception(boost::format("too large data length %1% > %2%") %
+                                      data_length % MAX_LENGTH);
 
-            block_prefix[index++] = static_cast<ValueType>(data_length);
+            prefix[index++] = data_length;
         }
-        out.WriteFrom(block_prefix.data(), block_prefix.size());
+
+        out = std::copy_n((const char *)prefix.data(), sizeof(ValueType) * BLOCK_SIZE, out);
+        return out;
     }
 
     /// Advances the range to an item stored in the referenced block.
@@ -233,28 +257,31 @@ template <int N, typename T = std::string> struct FixedGroupBlock
     }
 };
 
-template <typename GroupBlock> struct IndexedData
+namespace detail
 {
-    static constexpr std::uint32_t BLOCK_SIZE = GroupBlock::BLOCK_SIZE;
+template <typename GroupBlockPolicy, storage::Ownership Ownership> struct IndexedDataImpl
+{
+    static constexpr std::uint32_t BLOCK_SIZE = GroupBlockPolicy::BLOCK_SIZE;
 
     using BlocksNumberType = std::uint32_t;
     using DataSizeType = std::uint64_t;
 
-    using BlockReference = typename GroupBlock::BlockReference;
-    using ResultType = typename GroupBlock::ResultType;
-    using ValueType = typename GroupBlock::ValueType;
+    using BlockReference = typename GroupBlockPolicy::BlockReference;
+    using ResultType = typename GroupBlockPolicy::ResultType;
+    using ValueType = typename GroupBlockPolicy::ValueType;
 
     static_assert(sizeof(ValueType) == 1, "data basic type must char");
 
-    IndexedData() : blocks_number{0}, block_references{nullptr}, begin{nullptr}, end{nullptr} {}
+    IndexedDataImpl() = default;
+    IndexedDataImpl(util::vector_view<BlockReference> blocks_, util::vector_view<ValueType> values_)
+        : blocks(std::move(blocks_)), values(std::move(values_))
+    {
+    }
 
-    bool empty() const { return blocks_number == 0; }
+    bool empty() const { return blocks.empty(); }
 
     template <typename OffsetIterator, typename DataIterator>
-    void write(storage::io::FileWriter &out,
-               OffsetIterator first,
-               OffsetIterator last,
-               DataIterator data) const
+    IndexedDataImpl(OffsetIterator first, OffsetIterator last, DataIterator data)
     {
         static_assert(sizeof(typename DataIterator::value_type) == 1, "data basic type must char");
 
@@ -268,69 +295,39 @@ template <typename GroupBlock> struct IndexedData
         const BlocksNumberType number_of_blocks =
             number_of_elements == 0 ? 0
                                     : 1 + (std::distance(first, sentinel) - 1) / (BLOCK_SIZE + 1);
-        out.WriteOne(number_of_blocks);
+        blocks.resize(number_of_blocks);
 
         // Write block references and compute the total data size that includes prefix and data
-        const GroupBlock block;
+        const GroupBlockPolicy block;
+
+        auto block_iter = blocks.begin();
         DataSizeType data_size = 0;
         for (OffsetIterator curr = first, next = first; next != sentinel; curr = next)
         {
             std::advance(next, std::min<diff_type>(BLOCK_SIZE, std::distance(next, sentinel)));
-            data_size += block.WriteBlockReference(out, data_size, curr, next);
+            block_iter = block.WriteBlockReference(curr, next, data_size, block_iter);
             std::advance(next, std::min<diff_type>(1, std::distance(next, sentinel)));
             data_size += *next - *curr;
         }
 
-        // Write the total data size
-        out.WriteOne(data_size);
-
+        values.resize(data_size);
+        auto values_byte_iter = reinterpret_cast<char *>(values.data());
         // Write data blocks that are (prefix, data)
         for (OffsetIterator curr = first, next = first; next != sentinel; curr = next)
         {
             std::advance(next, std::min<diff_type>(BLOCK_SIZE, std::distance(next, sentinel)));
-            block.WriteBlockPrefix(out, curr, next);
+            values_byte_iter = block.WriteBlockPrefix(curr, next, values_byte_iter);
             std::advance(next, std::min<diff_type>(1, std::distance(next, sentinel)));
-            std::for_each(
-                data + *curr, data + *next, [&out](const auto &element) { out.WriteOne(element); });
+
+            auto to_bytes = [&](const auto &data) {
+                values_byte_iter = std::copy_n(&data, sizeof(ValueType), values_byte_iter);
+            };
+            std::copy(data + *curr,
+                      data + *next,
+                      boost::make_function_output_iterator(std::cref(to_bytes)));
         }
     }
 
-    /// Set internal pointers from the buffer [first, last).
-    /// Data buffer pointed by ptr must exists during IndexedData life-time.
-    /// No ownership is transferred.
-    void reset(const ValueType *first, const ValueType *last)
-    {
-        // Read blocks number
-        if (first + sizeof(BlocksNumberType) > last)
-            throw util::exception("incorrect memory block");
-
-        blocks_number = *reinterpret_cast<const BlocksNumberType *>(first);
-        first += sizeof(BlocksNumberType);
-
-        // Get block references pointer
-        if (first + sizeof(BlockReference) * blocks_number > last)
-            throw util::exception("incorrect memory block");
-
-        block_references = reinterpret_cast<const BlockReference *>(first);
-        first += sizeof(BlockReference) * blocks_number;
-
-        // Read total data size
-        if (first + sizeof(DataSizeType) > last)
-            throw util::exception("incorrect memory block");
-
-        auto data_size = *reinterpret_cast<const DataSizeType *>(first);
-        first += sizeof(DataSizeType);
-
-        // Get data blocks begin and end iterators
-        begin = reinterpret_cast<const ValueType *>(first);
-        first += sizeof(ValueType) * data_size;
-
-        if (first > last)
-            throw util::exception("incorrect memory block");
-
-        end = reinterpret_cast<const ValueType *>(first);
-    }
-
     // Return value at the given index
     ResultType at(std::uint32_t index) const
     {
@@ -338,20 +335,29 @@ template <typename GroupBlock> struct IndexedData
         const BlocksNumberType block_idx = index / (BLOCK_SIZE + 1);
         const std::uint32_t internal_idx = index % (BLOCK_SIZE + 1);
 
-        if (block_idx >= blocks_number)
+        if (block_idx >= blocks.size())
             return ResultType();
 
         // Get block first and last iterators
-        auto first = begin + block_references[block_idx].offset;
-        auto last =
-            block_idx + 1 == blocks_number ? end : begin + block_references[block_idx + 1].offset;
+        auto first = values.begin() + blocks[block_idx].offset;
+        auto last = block_idx + 1 == blocks.size() ? values.end()
+                                                   : values.begin() + blocks[block_idx + 1].offset;
 
-        const GroupBlock block;
-        block.ReadRefrencedBlock(block_references[block_idx], internal_idx, first, last);
+        const GroupBlockPolicy block;
+        block.ReadRefrencedBlock(blocks[block_idx], internal_idx, first, last);
 
-        return adapt(first, last);
+        return adapt(&*first, &*last);
     }
 
+    friend void serialization::read<GroupBlockPolicy, Ownership>(storage::tar::FileReader &reader,
+                                                                 const std::string &name,
+                                                                 IndexedDataImpl &index_data);
+
+    friend void
+    serialization::write<GroupBlockPolicy, Ownership>(storage::tar::FileWriter &writer,
+                                                      const std::string &name,
+                                                      const IndexedDataImpl &index_data);
+
   private:
     template <class T = ResultType>
     typename std::enable_if<!std::is_same<T, StringView>::value, T>::type
@@ -367,10 +373,16 @@ template <typename GroupBlock> struct IndexedData
         return ResultType(first, std::distance(first, last));
     }
 
-    BlocksNumberType blocks_number;
-    const BlockReference *block_references;
-    const ValueType *begin, *end;
+    template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
+    Vector<BlockReference> blocks;
+    Vector<ValueType> values;
 };
 }
+
+template <typename GroupBlockPolicy>
+using IndexedData = detail::IndexedDataImpl<GroupBlockPolicy, storage::Ownership::Container>;
+template <typename GroupBlockPolicy>
+using IndexedDataView = detail::IndexedDataImpl<GroupBlockPolicy, storage::Ownership::View>;
+}
 }
 #endif // OSRM_INDEXED_DATA_HPP

include/util/mmap_tar.hpp

@@ -0,0 +1,44 @@
+#ifndef OSRM_UTIL_MMAP_TAR_HPP
+#define OSRM_UTIL_MMAP_TAR_HPP
+
+#include "storage/tar.hpp"
+
+#include "util/mmap_file.hpp"
+
+#include <boost/iostreams/device/mapped_file.hpp>
+
+#include <tuple>
+#include <unordered_map>
+
+namespace osrm
+{
+namespace util
+{
+using DataRange = std::pair<const char *, const char *>;
+using DataMap = std::unordered_map<std::string, DataRange>;
+
+inline DataMap mmapTarFile(const boost::filesystem::path &path,
+                           boost::iostreams::mapped_file_source &region)
+{
+    DataMap map;
+
+    storage::tar::FileReader reader{path, storage::tar::FileReader::VerifyFingerprint};
+
+    std::vector<storage::tar::FileReader::FileEntry> entries;
+    reader.List(std::back_inserter(entries));
+
+    auto raw_file = mmapFile<char>(path, region);
+
+    for (const auto &entry : entries)
+    {
+        auto begin = raw_file.data() + entry.offset;
+        auto end = begin + entry.size;
+        map[entry.name] = DataRange{begin, end};
+    }
+
+    return map;
+}
+}
+}
+
+#endif

include/util/name_table.hpp

@@ -1,51 +0,0 @@
-#ifndef OSRM_UTIL_NAME_TABLE_HPP
-#define OSRM_UTIL_NAME_TABLE_HPP
-
-#include "util/indexed_data.hpp"
-#include "util/string_view.hpp"
-#include "util/typedefs.hpp"
-
-#include <string>
-
-namespace osrm
-{
-namespace util
-{
-
-// While this could, theoretically, hold any names in the fitting format,
-// the NameTable allows access to a part of the Datafacade to allow
-// processing based on name indices.
-class NameTable
-{
-  public:
-    using IndexedData = util::IndexedData<util::VariableGroupBlock<16, util::StringView>>;
-    using ResultType = IndexedData::ResultType;
-    using ValueType = IndexedData::ValueType;
-
-    NameTable() {}
-
-    // Read filename and store own data in m_buffer
-    NameTable(const std::string &filename);
-
-    // Keep pointers only in m_name_table and don't own data in m_buffer
-    void reset(ValueType *begin, ValueType *end);
-
-    // This class provides a limited view over all the string data we serialize out.
-    // The following functions are a subset of what is available.
-    // See the data facades for they provide full access to this serialized string data.
-    util::StringView GetNameForID(const NameID id) const;
-    util::StringView GetDestinationsForID(const NameID id) const;
-    util::StringView GetExitsForID(const NameID id) const;
-    util::StringView GetRefForID(const NameID id) const;
-    util::StringView GetPronunciationForID(const NameID id) const;
-
-  private:
-    using BufferType = std::unique_ptr<ValueType, std::function<void(void *)>>;
-
-    BufferType m_buffer;
-    IndexedData m_name_table;
-};
-} // namespace util
-} // namespace osrm
-
-#endif // OSRM_UTIL_NAME_TABLE_HPP

include/util/packed_vector.hpp

@@ -5,8 +5,8 @@
 #include "util/typedefs.hpp"
 #include "util/vector_view.hpp"
 
-#include "storage/io_fwd.hpp"
 #include "storage/shared_memory_ownership.hpp"
+#include "storage/tar_fwd.hpp"
 
 #include <boost/iterator/iterator_facade.hpp>
 #include <boost/iterator/reverse_iterator.hpp>
@@ -29,10 +29,13 @@ template <typename T, std::size_t Bits, storage::Ownership Ownership> class Pack
 namespace serialization
 {
 template <typename T, std::size_t Bits, storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader, detail::PackedVector<T, Bits, Ownership> &vec);
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 detail::PackedVector<T, Bits, Ownership> &vec);
 
 template <typename T, std::size_t Bits, storage::Ownership Ownership>
-inline void write(storage::io::FileWriter &writer,
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
                   const detail::PackedVector<T, Bits, Ownership> &vec);
 }
 
@@ -433,7 +436,7 @@ template <typename T, std::size_t Bits, storage::Ownership Ownership> class Pack
     void resize(std::size_t elements)
     {
         num_elements = elements;
-        auto num_blocks = std::ceil(static_cast<double>(elements) / BLOCK_ELEMENTS);
+        auto num_blocks = (elements + BLOCK_ELEMENTS - 1) / BLOCK_ELEMENTS;
         vec.resize(num_blocks * BLOCK_WORDS + 1);
     }
 
@@ -442,15 +445,18 @@ template <typename T, std::size_t Bits, storage::Ownership Ownership> class Pack
     template <bool enabled = (Ownership == storage::Ownership::View)>
     void reserve(typename std::enable_if<!enabled, std::size_t>::type capacity)
     {
-        auto num_blocks = std::ceil(static_cast<double>(capacity) / BLOCK_ELEMENTS);
+        auto num_blocks = (capacity + BLOCK_ELEMENTS - 1) / BLOCK_ELEMENTS;
         vec.reserve(num_blocks * BLOCK_WORDS + 1);
     }
 
-    friend void serialization::read<T, Bits, Ownership>(storage::io::FileReader &reader,
+    friend void serialization::read<T, Bits, Ownership>(storage::tar::FileReader &reader,
+                                                        const std::string &name,
                                                         PackedVector &vec);
 
-    friend void serialization::write<T, Bits, Ownership>(storage::io::FileWriter &writer,
+    friend void serialization::write<T, Bits, Ownership>(storage::tar::FileWriter &writer,
+                                                         const std::string &name,
                                                          const PackedVector &vec);
+
     inline void swap(PackedVector &other) noexcept
     {
         std::swap(vec, other.vec);

include/util/range_table.hpp

@@ -1,8 +1,8 @@
 #ifndef RANGE_TABLE_HPP
 #define RANGE_TABLE_HPP
 
-#include "storage/io.hpp"
 #include "storage/shared_memory_ownership.hpp"
+#include "storage/tar_fwd.hpp"
 #include "util/integer_range.hpp"
 #include "util/vector_view.hpp"
 
@@ -25,10 +25,14 @@ class RangeTable;
 namespace serialization
 {
 template <unsigned BlockSize, storage::Ownership Ownership>
-void write(storage::io::FileWriter &writer, const util::RangeTable<BlockSize, Ownership> &table);
+void write(storage::tar::FileWriter &writer,
+           const std::string &name,
+           const util::RangeTable<BlockSize, Ownership> &table);
 
 template <unsigned BlockSize, storage::Ownership Ownership>
-void read(storage::io::FileReader &reader, util::RangeTable<BlockSize, Ownership> &table);
+void read(storage::tar::FileReader &reader,
+          const std::string &name,
+          util::RangeTable<BlockSize, Ownership> &table);
 }
 
 /**
@@ -177,9 +181,11 @@ template <unsigned BLOCK_SIZE, storage::Ownership Ownership> class RangeTable
         return irange(begin_idx, end_idx);
     }
 
-    friend void serialization::write<BLOCK_SIZE, Ownership>(storage::io::FileWriter &writer,
+    friend void serialization::write<BLOCK_SIZE, Ownership>(storage::tar::FileWriter &writer,
+                                                            const std::string &name,
                                                             const RangeTable &table);
-    friend void serialization::read<BLOCK_SIZE, Ownership>(storage::io::FileReader &reader,
+    friend void serialization::read<BLOCK_SIZE, Ownership>(storage::tar::FileReader &reader,
+                                                           const std::string &name,
                                                            RangeTable &table);
 
   private:

include/util/serialization.hpp

@@ -2,9 +2,11 @@
 #define OSMR_UTIL_SERIALIZATION_HPP
 
 #include "util/dynamic_graph.hpp"
+#include "util/indexed_data.hpp"
 #include "util/packed_vector.hpp"
 #include "util/range_table.hpp"
 #include "util/static_graph.hpp"
+#include "util/static_rtree.hpp"
 
 #include "storage/io.hpp"
 #include "storage/serialization.hpp"
@@ -15,76 +17,131 @@ namespace util
 {
 namespace serialization
 {
+
 template <unsigned BlockSize, storage::Ownership Ownership>
-void write(storage::io::FileWriter &writer, const util::RangeTable<BlockSize, Ownership> &table)
+void write(storage::tar::FileWriter &writer,
+           const std::string &name,
+           const util::RangeTable<BlockSize, Ownership> &table)
 {
-    writer.WriteOne(table.sum_lengths);
-    storage::serialization::write(writer, table.block_offsets);
-    storage::serialization::write(writer, table.diff_blocks);
+    writer.WriteFrom(name + "/sum_lengths.meta", table.sum_lengths);
+    storage::serialization::write(writer, name + "/block_offsets", table.block_offsets);
+    storage::serialization::write(writer, name + "/diff_blocks", table.diff_blocks);
 }
 
 template <unsigned BlockSize, storage::Ownership Ownership>
-void read(storage::io::FileReader &reader, util::RangeTable<BlockSize, Ownership> &table)
+void read(storage::tar::FileReader &reader,
+          const std::string &name,
+          util::RangeTable<BlockSize, Ownership> &table)
 {
-    table.sum_lengths = reader.ReadOne<unsigned>();
-    storage::serialization::read(reader, table.block_offsets);
-    storage::serialization::read(reader, table.diff_blocks);
+    reader.ReadInto(name + "/sum_lengths.meta", table.sum_lengths);
+    storage::serialization::read(reader, name + "/block_offsets", table.block_offsets);
+    storage::serialization::read(reader, name + "/diff_blocks", table.diff_blocks);
 }
 
 template <typename T, std::size_t Bits, storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader, detail::PackedVector<T, Bits, Ownership> &vec)
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 detail::PackedVector<T, Bits, Ownership> &vec)
 {
-    vec.num_elements = reader.ReadOne<std::uint64_t>();
-    storage::serialization::read(reader, vec.vec);
+    reader.ReadInto(name + "/number_of_elements.meta", vec.num_elements);
+    storage::serialization::read(reader, name + "/packed", vec.vec);
 }
 
 template <typename T, std::size_t Bits, storage::Ownership Ownership>
-inline void write(storage::io::FileWriter &writer,
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
                   const detail::PackedVector<T, Bits, Ownership> &vec)
 {
-    writer.WriteOne(vec.num_elements);
-    storage::serialization::write(writer, vec.vec);
+    writer.WriteFrom(name + "/number_of_elements.meta", vec.num_elements);
+    storage::serialization::write(writer, name + "/packed", vec.vec);
 }
 
 template <typename EdgeDataT, storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader, StaticGraph<EdgeDataT, Ownership> &graph)
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 StaticGraph<EdgeDataT, Ownership> &graph)
 {
-    storage::serialization::read(reader, graph.node_array);
-    storage::serialization::read(reader, graph.edge_array);
+    storage::serialization::read(reader, name + "/node_array", graph.node_array);
+    storage::serialization::read(reader, name + "/edge_array", graph.edge_array);
     graph.number_of_nodes = graph.node_array.size() - 1;
     graph.number_of_edges = graph.edge_array.size();
 }
 
 template <typename EdgeDataT, storage::Ownership Ownership>
-inline void write(storage::io::FileWriter &writer, const StaticGraph<EdgeDataT, Ownership> &graph)
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
+                  const StaticGraph<EdgeDataT, Ownership> &graph)
 {
-    storage::serialization::write(writer, graph.node_array);
-    storage::serialization::write(writer, graph.edge_array);
+    storage::serialization::write(writer, name + "/node_array", graph.node_array);
+    storage::serialization::write(writer, name + "/edge_array", graph.edge_array);
 }
 
 template <typename EdgeDataT>
-inline void read(storage::io::FileReader &reader, DynamicGraph<EdgeDataT> &graph)
+inline void
+read(storage::tar::FileReader &reader, const std::string &name, DynamicGraph<EdgeDataT> &graph)
 {
-    storage::serialization::read(reader, graph.node_array);
-    const auto num_edges = reader.ReadElementCount64();
+    storage::serialization::read(reader, name + "/node_array", graph.node_array);
+    const auto num_edges = reader.ReadElementCount64(name + "/edge_list");
     graph.edge_list.resize(num_edges);
-    for (auto index : irange<std::size_t>(0, num_edges))
-    {
-        reader.ReadOne(graph.edge_list[index]);
-    }
+    reader.ReadStreaming<typename std::remove_reference_t<decltype(graph)>::Edge>(
+        name + "/edge_list", graph.edge_list.begin());
     graph.number_of_nodes = graph.node_array.size();
     graph.number_of_edges = num_edges;
 }
 
 template <typename EdgeDataT>
-inline void write(storage::io::FileWriter &writer, const DynamicGraph<EdgeDataT> &graph)
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
+                  const DynamicGraph<EdgeDataT> &graph)
 {
-    storage::serialization::write(writer, graph.node_array);
-    writer.WriteElementCount64(graph.number_of_edges);
-    for (auto index : irange<std::size_t>(0, graph.number_of_edges))
-    {
-        writer.WriteOne(graph.edge_list[index]);
-    }
+    storage::serialization::write(writer, name + "/node_array", graph.node_array);
+    writer.WriteElementCount64(name + "/edge_list", graph.number_of_edges);
+    writer.WriteStreaming<typename std::remove_reference_t<decltype(graph)>::Edge>(
+        name + "/edge_list", graph.edge_list.begin(), graph.number_of_edges);
+}
+
+template <typename BlockPolicy, storage::Ownership Ownership>
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 detail::IndexedDataImpl<BlockPolicy, Ownership> &index_data)
+{
+    storage::serialization::read(reader, name + "/blocks", index_data.blocks);
+    storage::serialization::read(reader, name + "/values", index_data.values);
+}
+
+template <typename BlockPolicy, storage::Ownership Ownership>
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
+                  const detail::IndexedDataImpl<BlockPolicy, Ownership> &index_data)
+{
+    storage::serialization::write(writer, name + "/blocks", index_data.blocks);
+    storage::serialization::write(writer, name + "/values", index_data.values);
+}
+
+template <class EdgeDataT,
+          storage::Ownership Ownership,
+          std::uint32_t BRANCHING_FACTOR,
+          std::uint32_t LEAF_PAGE_SIZE>
+void read(storage::tar::FileReader &reader,
+          const std::string &name,
+          util::StaticRTree<EdgeDataT, Ownership, BRANCHING_FACTOR, LEAF_PAGE_SIZE> &rtree)
+{
+    storage::serialization::read(reader, name + "/search_tree", rtree.m_search_tree);
+    storage::serialization::read(
+        reader, name + "/search_tree_level_starts", rtree.m_tree_level_starts);
+}
+
+template <class EdgeDataT,
+          storage::Ownership Ownership,
+          std::uint32_t BRANCHING_FACTOR,
+          std::uint32_t LEAF_PAGE_SIZE>
+void write(storage::tar::FileWriter &writer,
+           const std::string &name,
+           const util::StaticRTree<EdgeDataT, Ownership, BRANCHING_FACTOR, LEAF_PAGE_SIZE> &rtree)
+{
+    storage::serialization::write(writer, name + "/search_tree", rtree.m_search_tree);
+    storage::serialization::write(
+        writer, name + "/search_tree_level_starts", rtree.m_tree_level_starts);
 }
 }
 }

include/util/static_graph.hpp

@@ -8,8 +8,8 @@
 #include "util/typedefs.hpp"
 #include "util/vector_view.hpp"
 
-#include "storage/io_fwd.hpp"
 #include "storage/shared_memory_ownership.hpp"
+#include "storage/tar_fwd.hpp"
 
 #include <boost/assert.hpp>
 
@@ -28,10 +28,14 @@ template <typename EdgeDataT, storage::Ownership Ownership> class StaticGraph;
 namespace serialization
 {
 template <typename EdgeDataT, storage::Ownership Ownership>
-void read(storage::io::FileReader &reader, StaticGraph<EdgeDataT, Ownership> &graph);
+void read(storage::tar::FileReader &reader,
+          const std::string &name,
+          StaticGraph<EdgeDataT, Ownership> &graph);
 
 template <typename EdgeDataT, storage::Ownership Ownership>
-void write(storage::io::FileWriter &writer, const StaticGraph<EdgeDataT, Ownership> &graph);
+void write(storage::tar::FileWriter &writer,
+           const std::string &name,
+           const StaticGraph<EdgeDataT, Ownership> &graph);
 }
 
 namespace static_graph_details
@@ -272,10 +276,12 @@ class StaticGraph
         util::inplacePermutation(edge_array.begin(), edge_array.end(), old_to_new_edge);
     }
 
-    friend void serialization::read<EdgeDataT, Ownership>(storage::io::FileReader &reader,
+    friend void serialization::read<EdgeDataT, Ownership>(storage::tar::FileReader &reader,
+                                                          const std::string &name,
                                                           StaticGraph<EdgeDataT, Ownership> &graph);
     friend void
-    serialization::write<EdgeDataT, Ownership>(storage::io::FileWriter &writer,
+    serialization::write<EdgeDataT, Ownership>(storage::tar::FileWriter &writer,
+                                               const std::string &name,
                                                const StaticGraph<EdgeDataT, Ownership> &graph);
 
   protected:

include/util/static_rtree.hpp

@@ -1,7 +1,8 @@
 #ifndef STATIC_RTREE_HPP
 #define STATIC_RTREE_HPP
 
-#include "storage/io.hpp"
+#include "storage/tar_fwd.hpp"
+
 #include "util/bearing.hpp"
 #include "util/coordinate_calculation.hpp"
 #include "util/deallocating_vector.hpp"
@@ -35,20 +36,35 @@
 #include <string>
 #include <vector>
 
-// An extended alignment is implementation-defined, so use compiler attributes
-// until alignas(LEAF_PAGE_SIZE) is compiler-independent.
-#if defined(_MSC_VER)
-#define ALIGNED(x) __declspec(align(x))
-#elif defined(__GNUC__)
-#define ALIGNED(x) __attribute__((aligned(x)))
-#else
-#define ALIGNED(x)
-#endif
-
 namespace osrm
 {
 namespace util
 {
+template <class EdgeDataT,
+          storage::Ownership Ownership = storage::Ownership::Container,
+          std::uint32_t BRANCHING_FACTOR = 64,
+          std::uint32_t LEAF_PAGE_SIZE = 4096>
+class StaticRTree;
+
+namespace serialization
+{
+template <class EdgeDataT,
+          storage::Ownership Ownership,
+          std::uint32_t BRANCHING_FACTOR,
+          std::uint32_t LEAF_PAGE_SIZE>
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 util::StaticRTree<EdgeDataT, Ownership, BRANCHING_FACTOR, LEAF_PAGE_SIZE> &rtree);
+
+template <class EdgeDataT,
+          storage::Ownership Ownership,
+          std::uint32_t BRANCHING_FACTOR,
+          std::uint32_t LEAF_PAGE_SIZE>
+inline void
+write(storage::tar::FileWriter &writer,
+      const std::string &name,
+      const util::StaticRTree<EdgeDataT, Ownership, BRANCHING_FACTOR, LEAF_PAGE_SIZE> &rtree);
+}
 
 /***
  * Static RTree for serving nearest neighbour queries
@@ -57,9 +73,9 @@ namespace util
  */
 
 template <class EdgeDataT,
-          storage::Ownership Ownership = storage::Ownership::Container,
-          std::uint32_t BRANCHING_FACTOR = 64,
-          std::uint32_t LEAF_PAGE_SIZE = 4096>
+          storage::Ownership Ownership,
+          std::uint32_t BRANCHING_FACTOR,
+          std::uint32_t LEAF_PAGE_SIZE>
 class StaticRTree
 {
     /**********************************************************
@@ -236,24 +252,12 @@ class StaticRTree
         std::uint32_t segment_index;
     };
 
-    // We use a const view type when we don't own the data, otherwise
-    // we use a mutable type (usually becase we're building the tree)
-    using TreeViewType = typename std::conditional<Ownership == storage::Ownership::View,
-                                                   const Vector<const TreeNode>,
-                                                   Vector<TreeNode>>::type;
-    TreeViewType m_search_tree;
-
+    // Representation of the in-memory search tree
+    Vector<TreeNode> m_search_tree;
     // Reference to the actual lon/lat data we need for doing math
     const Vector<Coordinate> &m_coordinate_list;
-
-    // Holds the number of TreeNodes in each level.
-    // We always start with the root node, so
-    // m_tree_level_sizes[0] should always be 1
-    std::vector<std::uint64_t> m_tree_level_sizes;
-
     // Holds the start indexes of each level in m_search_tree
-    std::vector<std::uint64_t> m_tree_level_starts;
-
+    Vector<std::uint64_t> m_tree_level_starts;
     // mmap'd .fileIndex file
     boost::iostreams::mapped_file_source m_objects_region;
     // This is a view of the EdgeDataT data mmap'd from the .fileIndex file
@@ -262,12 +266,13 @@ class StaticRTree
   public:
     StaticRTree(const StaticRTree &) = delete;
     StaticRTree &operator=(const StaticRTree &) = delete;
+    StaticRTree(StaticRTree &&) = default;
+    StaticRTree &operator=(StaticRTree &&) = default;
 
     // Construct a packed Hilbert-R-Tree with Kamel-Faloutsos algorithm [1]
     explicit StaticRTree(const std::vector<EdgeDataT> &input_data_vector,
-                         const std::string &tree_node_filename,
-                         const std::string &leaf_node_filename,
-                         const Vector<Coordinate> &coordinate_list)
+                         const Vector<Coordinate> &coordinate_list,
+                         const boost::filesystem::path &on_disk_file_name)
         : m_coordinate_list(coordinate_list)
     {
         const auto element_count = input_data_vector.size();
@@ -304,8 +309,11 @@ class StaticRTree
         // sort the hilbert-value representatives
         tbb::parallel_sort(input_wrapper_vector.begin(), input_wrapper_vector.end());
         {
-            storage::io::FileWriter leaf_node_file(leaf_node_filename,
-                                                   storage::io::FileWriter::HasNoFingerprint);
+            boost::iostreams::mapped_file out_objects_region;
+            auto out_objects = mmapFile<EdgeDataT>(on_disk_file_name,
+                                                   out_objects_region,
+                                                   input_data_vector.size() * sizeof(EdgeDataT));
+
             // Note, we can't just write everything in one go, because the input_data_vector
             // is not sorted by hilbert code, only the input_wrapper_vector is in the correct
             // order.  Instead, we iterate over input_wrapper_vector, copy the hilbert-indexed
@@ -315,13 +323,12 @@ class StaticRTree
             // Create the first level of TreeNodes - each bounding LEAF_NODE_COUNT EdgeDataT
             // objects.
             std::size_t wrapped_element_index = 0;
+            auto objects_iter = out_objects.begin();
+
             while (wrapped_element_index < element_count)
             {
                 TreeNode current_node;
 
-                std::array<EdgeDataT, LEAF_NODE_SIZE> objects;
-                std::uint32_t object_count = 0;
-
                 // Loop over the next block of EdgeDataT, calculate the bounding box
                 // for the block, and save the data to write to disk in the correct
                 // order.
@@ -333,8 +340,7 @@ class StaticRTree
                         input_wrapper_vector[wrapped_element_index].m_original_index;
                     const EdgeDataT &object = input_data_vector[input_object_index];
 
-                    object_count += 1;
-                    objects[object_index] = object;
+                    *objects_iter++ = object;
 
                     Coordinate projected_u{
                         web_mercator::fromWGS84(Coordinate{m_coordinate_list[object.u]})};
@@ -361,19 +367,21 @@ class StaticRTree
                     current_node.minimum_bounding_rectangle.MergeBoundingBoxes(rectangle);
                 }
 
-                // Write out our EdgeDataT block to the leaf node file
-                leaf_node_file.WriteFrom(objects.data(), object_count);
-
                 m_search_tree.emplace_back(current_node);
             }
-
-            // leaf_node_file wil be RAII closed at this point
         }
+        // mmap as read-only now
+        m_objects = mmapFile<EdgeDataT>(on_disk_file_name, m_objects_region);
 
         // Should hold the number of nodes at the lowest level of the graph (closest
         // to the data)
         std::uint32_t nodes_in_previous_level = m_search_tree.size();
-        m_tree_level_sizes.push_back(nodes_in_previous_level);
+
+        // Holds the number of TreeNodes in each level.
+        // We always start with the root node, so
+        // m_tree_level_sizes[0] should always be 1
+        std::vector<std::uint64_t> tree_level_sizes;
+        tree_level_sizes.push_back(nodes_in_previous_level);
 
         // Now, repeatedly create levels of nodes that contain BRANCHING_FACTOR
         // nodes from the previous level.
@@ -408,7 +416,7 @@ class StaticRTree
                 m_search_tree.emplace_back(parent_node);
             }
             nodes_in_previous_level = nodes_in_current_level;
-            m_tree_level_sizes.push_back(nodes_in_previous_level);
+            tree_level_sizes.push_back(nodes_in_previous_level);
         }
         // At this point, we've got our tree built, but the nodes are in a weird order.
         // Next thing we'll do is flip it around so that we don't end up with a lot of
@@ -419,14 +427,15 @@ class StaticRTree
         std::reverse(m_search_tree.begin(), m_search_tree.end());
 
         // Same for the level sizes - root node / base level is at 0
-        std::reverse(m_tree_level_sizes.begin(), m_tree_level_sizes.end());
+        std::reverse(tree_level_sizes.begin(), tree_level_sizes.end());
 
         // The first level starts at 0
         m_tree_level_starts = {0};
         // The remaining levels start at the partial sum of the preceeding level sizes
-        std::partial_sum(m_tree_level_sizes.begin(),
-                         m_tree_level_sizes.end() - 1,
+        std::partial_sum(tree_level_sizes.begin(),
+                         tree_level_sizes.end(),
                          std::back_inserter(m_tree_level_starts));
+        BOOST_ASSERT(m_tree_level_starts.size() >= 2);
 
         // Now we have to flip the coordinates within each level so that math is easier
         // later on.  The workflow here is:
@@ -438,58 +447,22 @@ class StaticRTree
         // 0 12 345 6789
         // This ordering keeps the position math easy to understand during later
         // searches
-        for (auto i : irange<std::size_t>(0, m_tree_level_sizes.size()))
+        for (auto i : irange<std::size_t>(0, tree_level_sizes.size()))
         {
             std::reverse(m_search_tree.begin() + m_tree_level_starts[i],
-                         m_search_tree.begin() + m_tree_level_starts[i] + m_tree_level_sizes[i]);
+                         m_search_tree.begin() + m_tree_level_starts[i] + tree_level_sizes[i]);
         }
-
-        // Write all the TreeNode data to disk
-        {
-            storage::io::FileWriter tree_node_file(tree_node_filename,
-                                                   storage::io::FileWriter::GenerateFingerprint);
-
-            std::uint64_t size_of_tree = m_search_tree.size();
-            BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty");
-
-            tree_node_file.WriteOne(size_of_tree);
-            tree_node_file.WriteFrom(m_search_tree);
-
-            tree_node_file.WriteOne(static_cast<std::uint64_t>(m_tree_level_sizes.size()));
-            tree_node_file.WriteFrom(m_tree_level_sizes);
-        }
-
-        m_objects = mmapFile<EdgeDataT>(leaf_node_filename, m_objects_region);
     }
 
     /**
-     * Constructs an r-tree from already prepared files on disk (generated by the previous
-     * constructor)
+     * Constructs an empty RTree for de-serialization.
      */
-    explicit StaticRTree(const boost::filesystem::path &node_file,
-                         const boost::filesystem::path &leaf_file,
+    template <typename = std::enable_if<Ownership == storage::Ownership::Container>>
+    explicit StaticRTree(const boost::filesystem::path &on_disk_file_name,
                          const Vector<Coordinate> &coordinate_list)
         : m_coordinate_list(coordinate_list)
     {
-        storage::io::FileReader tree_node_file(node_file,
-                                               storage::io::FileReader::VerifyFingerprint);
-
-        const auto tree_size = tree_node_file.ReadElementCount64();
-        m_search_tree.resize(tree_size);
-        tree_node_file.ReadInto(m_search_tree);
-
-        const auto levels_array_size = tree_node_file.ReadElementCount64();
-        m_tree_level_sizes.resize(levels_array_size);
-        tree_node_file.ReadInto(m_tree_level_sizes);
-
-        // The first level always starts at 0
-        m_tree_level_starts = {0};
-        // The remaining levels start at the partial sum of the preceeding level sizes
-        std::partial_sum(m_tree_level_sizes.begin(),
-                         m_tree_level_sizes.end() - 1,
-                         std::back_inserter(m_tree_level_starts));
-
-        m_objects = mmapFile<EdgeDataT>(leaf_file, m_objects_region);
+        m_objects = mmapFile<EdgeDataT>(on_disk_file_name, m_objects_region);
     }
 
     /**
@@ -498,22 +471,15 @@ class StaticRTree
      * These memory blocks basically just contain the files read into RAM,
      * excep the .fileIndex file always stays on disk, and we mmap() it as usual
      */
-    explicit StaticRTree(const TreeNode *tree_node_ptr,
-                         const uint64_t number_of_nodes,
-                         const std::uint64_t *level_sizes_ptr,
-                         const std::size_t number_of_levels,
-                         const boost::filesystem::path &leaf_file,
+    explicit StaticRTree(Vector<TreeNode> search_tree_,
+                         Vector<std::uint64_t> tree_level_starts,
+                         const boost::filesystem::path &on_disk_file_name,
                          const Vector<Coordinate> &coordinate_list)
-        : m_search_tree(tree_node_ptr, number_of_nodes), m_coordinate_list(coordinate_list),
-          m_tree_level_sizes(level_sizes_ptr, level_sizes_ptr + number_of_levels)
+        : m_search_tree(std::move(search_tree_)), m_coordinate_list(coordinate_list),
+          m_tree_level_starts(std::move(tree_level_starts))
     {
-        // The first level starts at 0
-        m_tree_level_starts = {0};
-        // The remaining levels start at the partial sum of the preceeding level sizes
-        std::partial_sum(m_tree_level_sizes.begin(),
-                         m_tree_level_sizes.end() - 1,
-                         std::back_inserter(m_tree_level_starts));
-        m_objects = mmapFile<EdgeDataT>(leaf_file, m_objects_region);
+        BOOST_ASSERT(m_tree_level_starts.size() >= 2);
+        m_objects = mmapFile<EdgeDataT>(on_disk_file_name, m_objects_region);
     }
 
     /* Returns all features inside the bounding box.
@@ -734,6 +700,13 @@ class StaticRTree
         }
     }
 
+    std::uint64_t GetLevelSize(const std::size_t level) const
+    {
+        BOOST_ASSERT(m_tree_level_starts.size() > level + 1);
+        BOOST_ASSERT(m_tree_level_starts[level + 1] >= m_tree_level_starts[level]);
+        return m_tree_level_starts[level + 1] - m_tree_level_starts[level];
+    }
+
     /**
      * Calculates the absolute position of child data in our packed data
      * vectors.
@@ -769,22 +742,29 @@ class StaticRTree
             const std::uint64_t first_child_index =
                 m_tree_level_starts[parent.level + 1] + parent.offset * BRANCHING_FACTOR;
 
-            const std::uint64_t end_child_index = std::min(
-                first_child_index + BRANCHING_FACTOR,
-                m_tree_level_starts[parent.level + 1] + m_tree_level_sizes[parent.level + 1]);
+            const std::uint64_t end_child_index =
+                std::min(first_child_index + BRANCHING_FACTOR,
+                         m_tree_level_starts[parent.level + 1] + GetLevelSize(parent.level + 1));
             BOOST_ASSERT(first_child_index < std::numeric_limits<std::uint32_t>::max());
             BOOST_ASSERT(end_child_index < std::numeric_limits<std::uint32_t>::max());
             BOOST_ASSERT(end_child_index <= m_search_tree.size());
-            BOOST_ASSERT(end_child_index <= m_tree_level_starts[parent.level + 1] +
-                                                m_tree_level_sizes[parent.level + 1]);
+            BOOST_ASSERT(end_child_index <=
+                         m_tree_level_starts[parent.level + 1] + GetLevelSize(parent.level + 1));
             return irange<std::size_t>(first_child_index, end_child_index);
         }
     }
 
     bool is_leaf(const TreeIndex &treeindex) const
     {
-        return treeindex.level == m_tree_level_starts.size() - 1;
+        BOOST_ASSERT(m_tree_level_starts.size() >= 2);
+        return treeindex.level == m_tree_level_starts.size() - 2;
     }
+
+    friend void serialization::read<EdgeDataT, Ownership, BRANCHING_FACTOR, LEAF_PAGE_SIZE>(
+        storage::tar::FileReader &reader, const std::string &name, StaticRTree &rtree);
+
+    friend void serialization::write<EdgeDataT, Ownership, BRANCHING_FACTOR, LEAF_PAGE_SIZE>(
+        storage::tar::FileWriter &writer, const std::string &name, const StaticRTree &rtree);
 };
 
 //[1] "On Packing R-Trees"; I. Kamel, C. Faloutsos; 1993; DOI: 10.1145/170088.170403

include/util/vector_view.hpp

@@ -108,6 +108,15 @@ template <typename DataT> class vector_view
 
     std::size_t size() const { return m_size; }
 
+    void resize(const size_t size)
+    {
+        if (size > m_size)
+        {
+            throw util::exception("Trying to resize a view to a larger size.");
+        }
+        m_size = size;
+    }
+
     bool empty() const { return 0 == size(); }
 
     DataT &operator[](const unsigned index)
@@ -141,11 +150,14 @@ template <typename DataT> class vector_view
 
 template <> class vector_view<bool>
 {
-  private:
-    unsigned *m_ptr;
-    std::size_t m_size;
+  public:
+    using Word = std::uint64_t;
 
-    static constexpr std::size_t UNSIGNED_BITS = CHAR_BIT * sizeof(unsigned);
+  private:
+    static constexpr std::size_t WORD_BITS = CHAR_BIT * sizeof(Word);
+
+    Word *m_ptr;
+    std::size_t m_size;
 
   public:
     using value_type = bool;
@@ -169,36 +181,49 @@ template <> class vector_view<bool>
             return os << static_cast<bool>(rhs);
         }
 
-        unsigned *m_ptr;
-        const unsigned mask;
+        Word *m_ptr;
+        const Word mask;
     };
 
     vector_view() : m_ptr(nullptr), m_size(0) {}
 
-    vector_view(unsigned *ptr, std::size_t size) : m_ptr(ptr), m_size(size) {}
+    vector_view(Word *ptr, std::size_t size) : m_ptr(ptr), m_size(size) {}
 
     bool at(const std::size_t index) const
     {
         BOOST_ASSERT_MSG(index < m_size, "invalid size");
-        const std::size_t bucket = index / UNSIGNED_BITS;
-        const unsigned offset = index % UNSIGNED_BITS;
-        return m_ptr[bucket] & (1u << offset);
+        const std::size_t bucket = index / WORD_BITS;
+        const auto offset = index % WORD_BITS;
+        return m_ptr[bucket] & (static_cast<Word>(1) << offset);
     }
 
-    void reset(unsigned *, std::size_t size) { m_size = size; }
+    void reset(std::uint64_t *ptr, std::size_t size)
+    {
+        m_ptr = ptr;
+        m_size = size;
+    }
+
+    void resize(const size_t size)
+    {
+        if (size > m_size)
+        {
+            throw util::exception("Trying to resize a view to a larger size.");
+        }
+        m_size = size;
+    }
 
     std::size_t size() const { return m_size; }
 
     bool empty() const { return 0 == size(); }
 
-    bool operator[](const unsigned index) const { return at(index); }
+    bool operator[](const std::size_t index) const { return at(index); }
 
-    reference operator[](const unsigned index)
+    reference operator[](const std::size_t index)
     {
         BOOST_ASSERT(index < m_size);
-        const std::size_t bucket = index / UNSIGNED_BITS;
-        const unsigned offset = index % UNSIGNED_BITS;
-        return reference{m_ptr + bucket, 1u << offset};
+        const auto bucket = index / WORD_BITS;
+        const auto offset = index % WORD_BITS;
+        return reference{m_ptr + bucket, static_cast<Word>(1) << offset};
     }
 
     template <typename T> friend void swap(vector_view<T> &, vector_view<T> &) noexcept;

scripts/update_dependencies.sh

@@ -21,15 +21,20 @@ SOL_TAG=v2.17.5
 RAPIDJSON_REPO="https://github.com/miloyip/rapidjson.git"
 RAPIDJSON_TAG=v1.1.0
 
+MICROTAR_REPO="https://github.com/rxi/microtar"
+MICROTAR_TAG=v0.1.0
+
 VARIANT_LATEST=$(curl "https://api.github.com/repos/mapbox/variant/releases/latest" | jq ".tag_name")
 OSMIUM_LATEST=$(curl "https://api.github.com/repos/osmcode/libosmium/releases/latest" | jq ".tag_name")
 SOL_LATEST=$(curl "https://api.github.com/repos/ThePhD/sol2/releases/latest" | jq ".tag_name")
 RAPIDJSON_LATEST=$(curl "https://api.github.com/repos/miloyip/rapidjson/releases/latest" | jq ".tag_name")
+MICROTAR_LATEST=$(curl "https://api.github.com/repos/rxi/microtar/releases/latest" | jq ".tag_name")
 
 echo "Latest osmium release is $OSMIUM_LATEST, pulling in \"$OSMIUM_TAG\""
 echo "Latest variant release is $VARIANT_LATEST, pulling in \"$VARIANT_TAG\""
 echo "Latest sol2 release is $SOL_LATEST, pulling in \"$SOL_TAG\""
 echo "Latest rapidjson release is $RAPIDJSON_LATEST, pulling in \"$RAPIDJSON_TAG\""
+echo "Latest microtar release is $MICROTAR_LATEST, pulling in \"$MICROTAR_TAG\""
 
 read -p "Update osmium (y/n) " ok
 if [[ $ok =~ [yY] ]]
@@ -70,3 +75,13 @@ then
     git subtree add -P third_party/rapidjson/ $RAPIDJSON_REPO $RAPIDJSON_TAG --squash
   fi
 fi
+
+read -p "Update microtar (y/n) " ok
+if [[ $ok =~ [yY] ]]
+then
+  if [ -d "third_party/microtar" ]; then
+    git subtree pull -P third_party/microtar/ $MICROTAR_REPO $MICROTAR_TAG --squash
+  else
+    git subtree add -P third_party/microtar/ $MICROTAR_REPO $MICROTAR_TAG --squash
+  fi
+fi

src/benchmarks/CMakeLists.txt

@@ -6,7 +6,7 @@ file(GLOB PackedVectorBenchmarkSources packed_vector.cpp)
 add_executable(rtree-bench
 	EXCLUDE_FROM_ALL
 	${RTreeBenchmarkSources}
-	$<TARGET_OBJECTS:UTIL>)
+    $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
 
 target_include_directories(rtree-bench
 	PUBLIC

src/benchmarks/static_rtree.cpp

@@ -1,6 +1,9 @@
 #include "util/static_rtree.hpp"
 #include "extractor/edge_based_node_segment.hpp"
+#include "extractor/files.hpp"
+#include "extractor/packed_osm_ids.hpp"
 #include "extractor/query_node.hpp"
+
 #include "mocks/mock_datafacade.hpp"
 #include "storage/io.hpp"
 #include "engine/geospatial_query.hpp"
@@ -30,16 +33,6 @@ constexpr int32_t WORLD_MAX_LON = 180 * COORDINATE_PRECISION;
 using RTreeLeaf = extractor::EdgeBasedNodeSegment;
 using BenchStaticRTree = util::StaticRTree<RTreeLeaf, storage::Ownership::Container>;
 
-std::vector<util::Coordinate> loadCoordinates(const boost::filesystem::path &nodes_file)
-{
-    storage::io::FileReader nodes_path_file_reader(nodes_file,
-                                                   storage::io::FileReader::VerifyFingerprint);
-
-    std::vector<util::Coordinate> coords;
-    storage::serialization::read(nodes_path_file_reader, coords);
-    return coords;
-}
-
 template <typename QueryT>
 void benchmarkQuery(const std::vector<util::Coordinate> &queries,
                     const std::string &name,
@@ -97,9 +90,11 @@ int main(int argc, char **argv)
     const char *file_path = argv[2];
     const char *nodes_path = argv[3];
 
-    auto coords = osrm::benchmarks::loadCoordinates(nodes_path);
+    std::vector<osrm::util::Coordinate> coords;
+    osrm::extractor::files::readNodeCoordinates(nodes_path, coords);
 
-    osrm::benchmarks::BenchStaticRTree rtree(ram_path, file_path, coords);
+    osrm::benchmarks::BenchStaticRTree rtree(file_path, coords);
+    osrm::extractor::files::readRamIndex(ram_path, rtree);
 
     osrm::benchmarks::benchmark(rtree, 10000);
 

src/contractor/contractor.cpp

@@ -19,7 +19,6 @@
 #include "util/exception_utils.hpp"
 #include "util/exclude_flag.hpp"
 #include "util/filtered_graph.hpp"
-#include "util/graph_loader.hpp"
 #include "util/integer_range.hpp"
 #include "util/log.hpp"
 #include "util/static_graph.hpp"
@@ -64,11 +63,7 @@ int Contractor::Run()
 
     util::Log() << "Reading node weights.";
     std::vector<EdgeWeight> node_weights;
-    {
-        storage::io::FileReader reader(config.GetPath(".osrm.enw"),
-                                       storage::io::FileReader::VerifyFingerprint);
-        storage::serialization::read(reader, node_weights);
-    }
+    extractor::files::readEdgeBasedNodeWeights(config.GetPath(".osrm.enw"), node_weights);
     util::Log() << "Done reading node weights.";
 
     util::Log() << "Loading edge-expanded graph representation";

src/extractor/edge_based_graph_factory.cpp

@@ -69,7 +69,7 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory(
     const std::unordered_set<NodeID> &barrier_nodes,
     const std::unordered_set<NodeID> &traffic_lights,
     const std::vector<util::Coordinate> &coordinates,
-    const util::NameTable &name_table,
+    const NameTable &name_table,
     const std::unordered_set<EdgeID> &segregated_edges,
     const extractor::LaneDescriptionMap &lane_description_map)
     : m_edge_based_node_container(node_data_container), m_connectivity_checksum(0),
@@ -423,8 +423,9 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
 
     std::size_t node_based_edge_counter = 0;
 
-    storage::io::FileWriter turn_penalties_index_file(turn_penalties_index_filename,
-                                                      storage::io::FileWriter::HasNoFingerprint);
+    storage::tar::FileWriter turn_penalties_index_file(
+        turn_penalties_index_filename, storage::tar::FileWriter::GenerateFingerprint);
+    turn_penalties_index_file.WriteFrom("/extractor/turn_index", (char *)nullptr, 0);
 
     SuffixTable street_name_suffix_table(scripting_environment);
     const auto &turn_lanes_data = transformTurnLaneMapIntoArrays(lane_description_map);
@@ -1029,8 +1030,9 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
                 // Buffer writes to reduce syscall count
                 if (turn_indexes_write_buffer.size() >= TURN_INDEX_WRITE_BUFFER_SIZE)
                 {
-                    turn_penalties_index_file.WriteFrom(turn_indexes_write_buffer.data(),
-                                                        turn_indexes_write_buffer.size());
+                    turn_penalties_index_file.ContinueFrom("/extractor/turn_index",
+                                                           turn_indexes_write_buffer.data(),
+                                                           turn_indexes_write_buffer.size());
                     turn_indexes_write_buffer.clear();
                 }
 
@@ -1093,8 +1095,9 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
         // Flush the turn_indexes_write_buffer if it's not empty
         if (!turn_indexes_write_buffer.empty())
         {
-            turn_penalties_index_file.WriteFrom(turn_indexes_write_buffer.data(),
-                                                turn_indexes_write_buffer.size());
+            turn_penalties_index_file.ContinueFrom("/extractor/turn_index",
+                                                   turn_indexes_write_buffer.data(),
+                                                   turn_indexes_write_buffer.size());
             turn_indexes_write_buffer.clear();
         }
     }
@@ -1127,28 +1130,14 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
     // do not really have a choice but to index the conditional penalties and walk over all
     // edge-based-edges to find the ID of the edge
     auto const indexed_conditionals = IndexConditionals(std::move(conditionals));
-    {
-        util::Log() << "Writing " << indexed_conditionals.size()
-                    << " conditional turn penalties...";
-        // write conditional turn penalties into the restrictions file
-        storage::io::FileWriter writer(conditional_penalties_filename,
-                                       storage::io::FileWriter::GenerateFingerprint);
-        extractor::serialization::write(writer, indexed_conditionals);
-    }
+    util::Log() << "Writing " << indexed_conditionals.size() << " conditional turn penalties...";
+    extractor::files::writeConditionalRestrictions(conditional_penalties_filename,
+                                                   indexed_conditionals);
 
     // write weight penalties per turn
     BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size());
-    {
-        storage::io::FileWriter writer(turn_weight_penalties_filename,
-                                       storage::io::FileWriter::GenerateFingerprint);
-        storage::serialization::write(writer, turn_weight_penalties);
-    }
-
-    {
-        storage::io::FileWriter writer(turn_duration_penalties_filename,
-                                       storage::io::FileWriter::GenerateFingerprint);
-        storage::serialization::write(writer, turn_duration_penalties);
-    }
+    files::writeTurnWeightPenalty(turn_weight_penalties_filename, turn_weight_penalties);
+    files::writeTurnDurationPenalty(turn_duration_penalties_filename, turn_duration_penalties);
 
     util::Log() << "Generated " << m_edge_based_node_segments.size() << " edge based node segments";
     util::Log() << "Node-based graph contains " << node_based_edge_counter << " edges";

src/extractor/extraction_containers.cpp

@@ -1,6 +1,8 @@
 #include "extractor/extraction_containers.hpp"
 #include "extractor/extraction_segment.hpp"
 #include "extractor/extraction_way.hpp"
+#include "extractor/files.hpp"
+#include "extractor/name_table.hpp"
 #include "extractor/restriction.hpp"
 #include "extractor/serialization.hpp"
 
@@ -10,7 +12,6 @@
 #include "util/exception_utils.hpp"
 #include "util/fingerprint.hpp"
 #include "util/log.hpp"
-#include "util/name_table.hpp"
 #include "util/timing_util.hpp"
 
 #include "storage/io.hpp"
@@ -131,15 +132,15 @@ void ExtractionContainers::PrepareData(ScriptingEnvironment &scripting_environme
                                        const std::string &osrm_path,
                                        const std::string &name_file_name)
 {
-    storage::io::FileWriter file_out(osrm_path, storage::io::FileWriter::GenerateFingerprint);
+    storage::tar::FileWriter writer(osrm_path, storage::tar::FileWriter::GenerateFingerprint);
 
     PrepareNodes();
-    WriteNodes(file_out);
+    WriteNodes(writer);
     PrepareEdges(scripting_environment);
     all_nodes_list.clear(); // free all_nodes_list before allocation of normal_edges
     all_nodes_list.shrink_to_fit();
-    WriteEdges(file_out);
-    WriteMetadata(file_out);
+    WriteEdges(writer);
+    WriteMetadata(writer);
 
     /* Sort these so that searching is a bit faster later on */
     {
@@ -163,10 +164,10 @@ void ExtractionContainers::WriteCharData(const std::string &file_name)
     util::UnbufferedLog log;
     log << "writing street name index ... ";
     TIMER_START(write_index);
-    storage::io::FileWriter file(file_name, storage::io::FileWriter::GenerateFingerprint);
 
-    const util::NameTable::IndexedData indexed_data;
-    indexed_data.write(file, name_offsets.begin(), name_offsets.end(), name_char_data.begin());
+    files::writeNames(file_name,
+                      NameTable{NameTable::IndexedData(
+                          name_offsets.begin(), name_offsets.end(), name_char_data.begin())});
 
     TIMER_STOP(write_index);
     log << "ok, after " << TIMER_SEC(write_index) << "s";
@@ -531,7 +532,7 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm
     }
 }
 
-void ExtractionContainers::WriteEdges(storage::io::FileWriter &file_out) const
+void ExtractionContainers::WriteEdges(storage::tar::FileWriter &writer) const
 {
     std::vector<NodeBasedEdge> normal_edges;
     normal_edges.reserve(all_edges_list.size());
@@ -558,8 +559,7 @@ void ExtractionContainers::WriteEdges(storage::io::FileWriter &file_out) const
             throw util::exception("There are too many edges, OSRM only supports 2^32" + SOURCE_REF);
         }
 
-        file_out.WriteElementCount64(normal_edges.size());
-        file_out.WriteFrom(normal_edges.data(), normal_edges.size());
+        storage::serialization::write(writer, "/extractor/edges", normal_edges);
 
         TIMER_STOP(write_edges);
         log << "ok, after " << TIMER_SEC(write_edges) << "s";
@@ -567,31 +567,21 @@ void ExtractionContainers::WriteEdges(storage::io::FileWriter &file_out) const
     }
 }
 
-void ExtractionContainers::WriteMetadata(storage::io::FileWriter &file_out) const
+void ExtractionContainers::WriteMetadata(storage::tar::FileWriter &writer) const
 {
     util::UnbufferedLog log;
     log << "Writing way meta-data     ... " << std::flush;
     TIMER_START(write_meta_data);
 
-    file_out.WriteElementCount64(all_edges_annotation_data_list.size());
-    file_out.WriteFrom(all_edges_annotation_data_list.data(),
-                       all_edges_annotation_data_list.size());
+    storage::serialization::write(writer, "/extractor/annotations", all_edges_annotation_data_list);
 
     TIMER_STOP(write_meta_data);
     log << "ok, after " << TIMER_SEC(write_meta_data) << "s";
     log << " -- Metadata contains << " << all_edges_annotation_data_list.size() << " entries.";
 }
 
-void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const
+void ExtractionContainers::WriteNodes(storage::tar::FileWriter &writer) const
 {
-    {
-        // write dummy value, will be overwritten later
-        util::UnbufferedLog log;
-        log << "setting number of nodes   ... " << std::flush;
-        file_out.WriteElementCount64(max_internal_node_id);
-        log << "ok";
-    }
-
     {
         util::UnbufferedLog log;
         log << "Confirming/Writing used nodes     ... ";
@@ -599,28 +589,35 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const
         // identify all used nodes by a merging step of two sorted lists
         auto node_iterator = all_nodes_list.begin();
         auto node_id_iterator = used_node_id_list.begin();
-        const auto used_node_id_list_end = used_node_id_list.end();
         const auto all_nodes_list_end = all_nodes_list.end();
 
-        while (node_id_iterator != used_node_id_list_end && node_iterator != all_nodes_list_end)
-        {
-            if (*node_id_iterator < node_iterator->node_id)
-            {
-                ++node_id_iterator;
-                continue;
-            }
-            if (*node_id_iterator > node_iterator->node_id)
+        const std::function<QueryNode()> encode_function = [&]() -> QueryNode {
+            BOOST_ASSERT(node_id_iterator != used_node_id_list.end());
+            BOOST_ASSERT(node_iterator != all_nodes_list_end);
+            BOOST_ASSERT(*node_id_iterator >= node_iterator->node_id);
+            while (*node_id_iterator > node_iterator->node_id &&
+                   node_iterator != all_nodes_list_end)
             {
                 ++node_iterator;
-                continue;
+            }
+            if (node_iterator == all_nodes_list_end || *node_id_iterator < node_iterator->node_id)
+            {
+                throw util::exception(
+                    "Invalid OSM data: Referenced non-existing node with ID " +
+                    std::to_string(static_cast<std::uint64_t>(*node_id_iterator)));
             }
             BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
 
-            file_out.WriteOne((*node_iterator));
-
             ++node_id_iterator;
-            ++node_iterator;
-        }
+            return *node_iterator++;
+        };
+
+        writer.WriteElementCount64("/extractor/nodes", used_node_id_list.size());
+        writer.WriteStreaming<QueryNode>(
+            "/extractor/nodes",
+            boost::make_function_input_iterator(encode_function, boost::infinite()),
+            used_node_id_list.size());
+
         TIMER_STOP(write_nodes);
         log << "ok, after " << TIMER_SEC(write_nodes) << "s";
     }
@@ -639,7 +636,7 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const
                 internal_barrier_nodes.push_back(node_id);
             }
         }
-        storage::serialization::write(file_out, internal_barrier_nodes);
+        storage::serialization::write(writer, "/extractor/barriers", internal_barrier_nodes);
         log << "ok, after " << TIMER_SEC(write_nodes) << "s";
     }
 
@@ -657,7 +654,8 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const
                 internal_traffic_signals.push_back(node_id);
             }
         }
-        storage::serialization::write(file_out, internal_traffic_signals);
+        storage::serialization::write(
+            writer, "/extractor/traffic_lights", internal_traffic_signals);
         log << "ok, after " << TIMER_SEC(write_nodes) << "s";
     }
 

src/extractor/extractor.cpp

@@ -1,5 +1,6 @@
 #include "extractor/extractor.hpp"
 
+#include "extractor/compressed_node_based_graph_edge.hpp"
 #include "extractor/edge_based_edge.hpp"
 #include "extractor/extraction_containers.hpp"
 #include "extractor/extraction_node.hpp"
@@ -8,6 +9,7 @@
 #include "extractor/extractor_callbacks.hpp"
 #include "extractor/files.hpp"
 #include "extractor/maneuver_override_relation_parser.hpp"
+#include "extractor/name_table.hpp"
 #include "extractor/node_based_graph_factory.hpp"
 #include "extractor/raster_source.hpp"
 #include "extractor/restriction_filter.hpp"
@@ -23,10 +25,8 @@
 
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
-#include "util/graph_loader.hpp"
 #include "util/integer_range.hpp"
 #include "util/log.hpp"
-#include "util/name_table.hpp"
 #include "util/range_table.hpp"
 #include "util/timing_util.hpp"
 
@@ -169,6 +169,25 @@ void SetExcludableClasses(const ExtractorCallbacks::ClassesMap &classes_map,
         }
     }
 }
+
+std::vector<CompressedNodeBasedGraphEdge> toEdgeList(const util::NodeBasedDynamicGraph &graph)
+{
+    std::vector<CompressedNodeBasedGraphEdge> edges;
+    edges.reserve(graph.GetNumberOfEdges());
+
+    // For all nodes iterate over its edges and dump (from, to) pairs
+    for (const NodeID from_node : util::irange(0u, graph.GetNumberOfNodes()))
+    {
+        for (const EdgeID edge : graph.GetAdjacentEdgeRange(from_node))
+        {
+            const auto to_node = graph.GetTarget(edge);
+
+            edges.push_back({from_node, to_node});
+        }
+    }
+
+    return edges;
+}
 }
 
 /**
@@ -231,11 +250,13 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
                                                    conditional_turn_restrictions,
                                                    unresolved_maneuver_overrides);
 
+    NameTable name_table;
+    files::readNames(config.GetPath(".osrm.names"), name_table);
+
     util::Log() << "Find segregated edges in node-based graph ..." << std::flush;
     TIMER_START(segregated);
 
-    util::NameTable names(config.GetPath(".osrm.names").string());
-    auto segregated_edges = guidance::findSegregatedNodes(node_based_graph_factory, names);
+    auto segregated_edges = guidance::findSegregatedNodes(node_based_graph_factory, name_table);
 
     TIMER_STOP(segregated);
     util::Log() << "ok, after " << TIMER_SEC(segregated) << "s";
@@ -266,10 +287,8 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
             compressed_node_based_graph_writing.wait();
     };
 
-    compressed_node_based_graph_writing = std::async(std::launch::async, [&] {
-        WriteCompressedNodeBasedGraph(
-            config.GetPath(".osrm.cnbg").string(), node_based_graph, coordinates);
-    });
+    files::writeCompressedNodeBasedGraph(config.GetPath(".osrm.cnbg").string(),
+                                         toEdgeList(node_based_graph));
 
     node_based_graph_factory.GetCompressedEdges().PrintStatistics();
 
@@ -284,8 +303,6 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
 
     const auto number_of_node_based_nodes = node_based_graph.GetNumberOfNodes();
 
-    const util::NameTable name_table(config.GetPath(".osrm.names").string());
-
     const auto number_of_edge_based_nodes =
         BuildEdgeExpandedGraph(node_based_graph,
                                coordinates,
@@ -326,11 +343,8 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
 
     util::Log() << "Saving edge-based node weights to file.";
     TIMER_START(timer_write_node_weights);
-    {
-        storage::io::FileWriter writer(config.GetPath(".osrm.enw"),
-                                       storage::io::FileWriter::GenerateFingerprint);
-        storage::serialization::write(writer, edge_based_node_weights);
-    }
+    extractor::files::writeEdgeBasedNodeWeights(config.GetPath(".osrm.enw"),
+                                                edge_based_node_weights);
     TIMER_STOP(timer_write_node_weights);
     util::Log() << "Done writing. (" << TIMER_SEC(timer_write_node_weights) << ")";
 
@@ -412,11 +426,6 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
             timestamp = "n/a";
         }
         util::Log() << "timestamp: " << timestamp;
-
-        storage::io::FileWriter timestamp_file(config.GetPath(".osrm.timestamp"),
-                                               storage::io::FileWriter::GenerateFingerprint);
-
-        timestamp_file.WriteFrom(timestamp.c_str(), timestamp.length());
     }
 
     // Extraction containers and restriction parser
@@ -714,7 +723,7 @@ EdgeID Extractor::BuildEdgeExpandedGraph(
     const std::vector<TurnRestriction> &turn_restrictions,
     const std::vector<ConditionalTurnRestriction> &conditional_turn_restrictions,
     const std::unordered_set<EdgeID> &segregated_edges,
-    const util::NameTable &name_table,
+    const NameTable &name_table,
     const std::vector<UnresolvedManeuverOverride> &maneuver_overrides,
     const LaneDescriptionMap &turn_lane_map,
     // for calculating turn penalties
@@ -815,57 +824,15 @@ void Extractor::BuildRTree(std::vector<EdgeBasedNodeSegment> edge_based_node_seg
     edge_based_node_segments.resize(new_size);
 
     TIMER_START(construction);
-    util::StaticRTree<EdgeBasedNodeSegment> rtree(edge_based_node_segments,
-                                                  config.GetPath(".osrm.ramIndex").string(),
-                                                  config.GetPath(".osrm.fileIndex").string(),
-                                                  coordinates);
+    util::StaticRTree<EdgeBasedNodeSegment> rtree(
+        edge_based_node_segments, coordinates, config.GetPath(".osrm.fileIndex"));
+
+    files::writeRamIndex(config.GetPath(".osrm.ramIndex"), rtree);
 
     TIMER_STOP(construction);
     util::Log() << "finished r-tree construction in " << TIMER_SEC(construction) << " seconds";
 }
 
-void Extractor::WriteCompressedNodeBasedGraph(const std::string &path,
-                                              const util::NodeBasedDynamicGraph &graph,
-                                              const std::vector<util::Coordinate> &coordinates)
-{
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-
-    storage::io::FileWriter writer{path, fingerprint};
-
-    // Writes:  | Fingerprint | #e | #n | edges | coordinates |
-    // - uint64: number of edges (from, to) pairs
-    // - uint64: number of nodes and therefore also coordinates
-    // - (uint32_t, uint32_t): num_edges * edges
-    // - (int32_t, int32_t: num_nodes * coordinates (lon, lat)
-
-    const auto num_edges = graph.GetNumberOfEdges();
-    const auto num_nodes = graph.GetNumberOfNodes();
-
-    BOOST_ASSERT_MSG(num_nodes == coordinates.size(), "graph and embedding out of sync");
-
-    writer.WriteElementCount64(num_edges);
-    writer.WriteElementCount64(num_nodes);
-
-    // For all nodes iterate over its edges and dump (from, to) pairs
-    for (const NodeID from_node : util::irange(0u, num_nodes))
-    {
-        for (const EdgeID edge : graph.GetAdjacentEdgeRange(from_node))
-        {
-            const auto to_node = graph.GetTarget(edge);
-
-            writer.WriteOne(from_node);
-            writer.WriteOne(to_node);
-        }
-    }
-
-    // FIXME this is unneccesary: We have this data
-    for (const auto &qnode : coordinates)
-    {
-        writer.WriteOne(qnode.lon);
-        writer.WriteOne(qnode.lat);
-    }
-}
-
 template <typename Map> auto convertIDMapToVector(const Map &map)
 {
     std::vector<typename Map::key_type> result(map.size());
@@ -885,7 +852,7 @@ void Extractor::ProcessGuidanceTurns(
     const std::unordered_set<NodeID> &barrier_nodes,
     const std::vector<TurnRestriction> &turn_restrictions,
     const std::vector<ConditionalTurnRestriction> &conditional_turn_restrictions,
-    const util::NameTable &name_table,
+    const NameTable &name_table,
     LaneDescriptionMap lane_description_map,
     ScriptingEnvironment &scripting_environment)
 {
@@ -945,9 +912,11 @@ void Extractor::ProcessGuidanceTurns(
 
     util::Log() << "Writing Turns and Lane Data...";
     TIMER_START(write_guidance_data);
-    storage::io::FileWriter writer(config.GetPath(".osrm.tld").string(),
-                                   storage::io::FileWriter::GenerateFingerprint);
-    storage::serialization::write(writer, convertIDMapToVector(lane_data_map.data));
+
+    {
+        auto turn_lane_data = convertIDMapToVector(lane_data_map.data);
+        files::writeTurnLaneData(config.GetPath(".osrm.tld"), turn_lane_data);
+    }
 
     { // Turn lanes handler modifies lane_description_map, so another transformation is needed
         std::vector<std::uint32_t> turn_lane_offsets;

src/extractor/intersection/have_identical_names.cpp

@@ -1,3 +1,5 @@
+#include "extractor/intersection/have_identical_names.hpp"
+
 #include "util/guidance/name_announcements.hpp"
 
 namespace osrm
@@ -12,7 +14,7 @@ namespace intersection
 // rhs->lhs)
 bool HaveIdenticalNames(const NameID lhs,
                         const NameID rhs,
-                        const util::NameTable &name_table,
+                        const NameTable &name_table,
                         const extractor::SuffixTable &street_name_suffix_table)
 {
     const auto non_empty = (lhs != EMPTY_NAMEID) && (rhs != EMPTY_NAMEID);

src/extractor/intersection/mergable_road_detector.cpp

@@ -1,6 +1,7 @@
 #include "extractor/intersection/mergable_road_detector.hpp"
 #include "extractor/intersection/intersection_analysis.hpp"
 #include "extractor/intersection/node_based_graph_walker.hpp"
+#include "extractor/name_table.hpp"
 #include "extractor/query_node.hpp"
 #include "extractor/suffix_table.hpp"
 #include "guidance/constants.hpp"
@@ -8,7 +9,6 @@
 #include "util/bearing.hpp"
 #include "util/coordinate_calculation.hpp"
 #include "util/guidance/name_announcements.hpp"
-#include "util/name_table.hpp"
 
 using osrm::util::angularDeviation;
 
@@ -27,7 +27,7 @@ namespace
 inline auto makeCheckRoadForName(const NameID name_id,
                                  const util::NodeBasedDynamicGraph &node_based_graph,
                                  const EdgeBasedNodeDataContainer &node_data_container,
-                                 const util::NameTable &name_table,
+                                 const NameTable &name_table,
                                  const SuffixTable &suffix_table)
 {
     return [name_id, &node_based_graph, &node_data_container, &name_table, &suffix_table](
@@ -59,7 +59,7 @@ MergableRoadDetector::MergableRoadDetector(
     const RestrictionMap &node_restriction_map,
     const std::unordered_set<NodeID> &barrier_nodes,
     const extractor::TurnLanesIndexedArray &turn_lanes_data,
-    const util::NameTable &name_table,
+    const NameTable &name_table,
     const SuffixTable &street_name_suffix_table)
     : node_based_graph(node_based_graph), node_data_container(node_data_container),
       node_coordinates(node_coordinates), compressed_geometries(compressed_geometries),

src/extractor/node_based_graph_factory.cpp

@@ -1,7 +1,7 @@
 #include "extractor/node_based_graph_factory.hpp"
+#include "extractor/files.hpp"
 #include "extractor/graph_compressor.hpp"
 #include "storage/io.hpp"
-#include "util/graph_loader.hpp"
 
 #include "util/log.hpp"
 #include "util/timing_util.hpp"
@@ -34,28 +34,25 @@ NodeBasedGraphFactory::NodeBasedGraphFactory(
 // load the data serialised during the extraction run
 void NodeBasedGraphFactory::LoadDataFromFile(const boost::filesystem::path &input_file)
 {
-    // the extraction_containers serialise all data necessary to create the node-based graph into a
-    // single file, the *.osrm file. It contains nodes, basic information about which of these nodes
-    // are traffic signals/stop signs. It also contains Edges and purely annotative meta-data
-    storage::io::FileReader file_reader(input_file, storage::io::FileReader::VerifyFingerprint);
-
     auto barriers_iter = inserter(barriers, end(barriers));
     auto traffic_signals_iter = inserter(traffic_signals, end(traffic_signals));
-
-    const auto number_of_node_based_nodes = util::loadNodesFromFile(
-        file_reader, barriers_iter, traffic_signals_iter, coordinates, osm_node_ids);
-
     std::vector<NodeBasedEdge> edge_list;
-    util::loadEdgesFromFile(file_reader, edge_list);
 
+    files::readRawNBGraph(input_file,
+                          barriers_iter,
+                          traffic_signals_iter,
+                          coordinates,
+                          osm_node_ids,
+                          edge_list,
+                          annotation_data);
+
+    const auto number_of_node_based_nodes = coordinates.size();
     if (edge_list.empty())
     {
         throw util::exception("Node-based-graph (" + input_file.string() + ") contains no edges." +
                               SOURCE_REF);
     }
 
-    util::loadAnnotationData(file_reader, annotation_data);
-
     // at this point, the data isn't compressed, but since we update the graph in-place, we assign
     // it here.
     compressed_output_graph =

src/guidance/driveway_handler.cpp

@@ -17,7 +17,7 @@ DrivewayHandler::DrivewayHandler(const util::NodeBasedDynamicGraph &node_based_g
                                  const extractor::RestrictionMap &node_restriction_map,
                                  const std::unordered_set<NodeID> &barrier_nodes,
                                  const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                                 const util::NameTable &name_table,
+                                 const extractor::NameTable &name_table,
                                  const extractor::SuffixTable &street_name_suffix_table)
     : IntersectionHandler(node_based_graph,
                           node_data_container,

src/guidance/guidance_processing.cpp

@@ -24,7 +24,7 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
                    const std::unordered_set<NodeID> &barrier_nodes,
                    const extractor::RestrictionMap &node_restriction_map,
                    const extractor::WayRestrictionMap &way_restriction_map,
-                   const util::NameTable &name_table,
+                   const extractor::NameTable &name_table,
                    const extractor::SuffixTable &suffix_table,
                    const extractor::TurnLanesIndexedArray &turn_lanes_data,
                    extractor::LaneDescriptionMap &lane_description_map,

src/guidance/intersection_handler.cpp

@@ -52,7 +52,7 @@ IntersectionHandler::IntersectionHandler(
     const extractor::RestrictionMap &node_restriction_map,
     const std::unordered_set<NodeID> &barrier_nodes,
     const extractor::TurnLanesIndexedArray &turn_lanes_data,
-    const util::NameTable &name_table,
+    const extractor::NameTable &name_table,
     const extractor::SuffixTable &street_name_suffix_table)
     : node_based_graph(node_based_graph), node_data_container(node_data_container),
       node_coordinates(node_coordinates), compressed_geometries(compressed_geometries),

src/guidance/motorway_handler.cpp

@@ -49,7 +49,7 @@ MotorwayHandler::MotorwayHandler(const util::NodeBasedDynamicGraph &node_based_g
                                  const extractor::RestrictionMap &node_restriction_map,
                                  const std::unordered_set<NodeID> &barrier_nodes,
                                  const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                                 const util::NameTable &name_table,
+                                 const extractor::NameTable &name_table,
                                  const extractor::SuffixTable &street_name_suffix_table)
     : IntersectionHandler(node_based_graph,
                           node_data_container,

src/guidance/roundabout_handler.cpp

@@ -30,7 +30,7 @@ RoundaboutHandler::RoundaboutHandler(
     const extractor::RestrictionMap &node_restriction_map,
     const std::unordered_set<NodeID> &barrier_nodes,
     const extractor::TurnLanesIndexedArray &turn_lanes_data,
-    const util::NameTable &name_table,
+    const extractor::NameTable &name_table,
     const extractor::SuffixTable &street_name_suffix_table)
     : IntersectionHandler(node_based_graph,
                           node_data_container,

src/guidance/segregated_intersection_classification.cpp

@@ -4,7 +4,6 @@
 #include "guidance/turn_instruction.hpp"
 
 #include "util/coordinate_calculation.hpp"
-#include "util/name_table.hpp"
 #include <set>
 
 using osrm::guidance::getTurnDirection;
@@ -42,7 +41,7 @@ struct EdgeInfo
 };
 
 std::unordered_set<EdgeID> findSegregatedNodes(const extractor::NodeBasedGraphFactory &factory,
-                                               const util::NameTable &names)
+                                               const extractor::NameTable &names)
 {
     auto const &graph = factory.GetGraph();
     auto const &annotation = factory.GetAnnotationData();

src/guidance/sliproad_handler.cpp

@@ -27,7 +27,7 @@ SliproadHandler::SliproadHandler(const util::NodeBasedDynamicGraph &node_based_g
                                  const extractor::RestrictionMap &node_restriction_map,
                                  const std::unordered_set<NodeID> &barrier_nodes,
                                  const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                                 const util::NameTable &name_table,
+                                 const extractor::NameTable &name_table,
                                  const extractor::SuffixTable &street_name_suffix_table)
     : IntersectionHandler(node_based_graph,
                           node_data_container,

src/guidance/suppress_mode_handler.cpp

@@ -17,7 +17,7 @@ SuppressModeHandler::SuppressModeHandler(
     const extractor::RestrictionMap &node_restriction_map,
     const std::unordered_set<NodeID> &barrier_nodes,
     const extractor::TurnLanesIndexedArray &turn_lanes_data,
-    const util::NameTable &name_table,
+    const extractor::NameTable &name_table,
     const extractor::SuffixTable &street_name_suffix_table)
     : IntersectionHandler(node_based_graph,
                           node_data_container,

src/guidance/turn_analysis.cpp

@@ -26,7 +26,7 @@ TurnAnalysis::TurnAnalysis(const util::NodeBasedDynamicGraph &node_based_graph,
                            const extractor::RestrictionMap &restriction_map,
                            const std::unordered_set<NodeID> &barrier_nodes,
                            const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                           const util::NameTable &name_table,
+                           const extractor::NameTable &name_table,
                            const extractor::SuffixTable &street_name_suffix_table)
     : node_based_graph(node_based_graph), roundabout_handler(node_based_graph,
                                                              node_data_container,

src/guidance/turn_handler.cpp

@@ -115,7 +115,7 @@ TurnHandler::TurnHandler(const util::NodeBasedDynamicGraph &node_based_graph,
                          const extractor::RestrictionMap &node_restriction_map,
                          const std::unordered_set<NodeID> &barrier_nodes,
                          const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                         const util::NameTable &name_table,
+                         const extractor::NameTable &name_table,
                          const extractor::SuffixTable &street_name_suffix_table)
     : IntersectionHandler(node_based_graph,
                           node_data_container,

src/osrm/osrm.cpp

@@ -35,7 +35,7 @@ OSRM::OSRM(engine::EngineConfig &config)
 
         auto mem = storage::makeSharedMemory(barrier.data().region);
         auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr());
-        if (layout->GetBlockSize(storage::DataLayout::NAME_CHAR_DATA) == 0)
+        if (layout->GetBlockSize(storage::DataLayout::NAME_VALUES) == 0)
             throw util::exception(
                 "No name data loaded, cannot continue.  Have you run osrm-datastore to load data?");
     }

src/partitioner/partitioner.cpp

@@ -2,7 +2,6 @@
 #include "partitioner/bisection_graph.hpp"
 #include "partitioner/bisection_to_partition.hpp"
 #include "partitioner/cell_storage.hpp"
-#include "partitioner/compressed_node_based_graph_reader.hpp"
 #include "partitioner/edge_based_graph_reader.hpp"
 #include "partitioner/files.hpp"
 #include "partitioner/multi_level_partition.hpp"
@@ -10,6 +9,7 @@
 #include "partitioner/remove_unconnected.hpp"
 #include "partitioner/renumber.hpp"
 
+#include "extractor/compressed_node_based_graph_edge.hpp"
 #include "extractor/files.hpp"
 
 #include "util/coordinate.hpp"
@@ -40,19 +40,17 @@ namespace partitioner
 {
 auto getGraphBisection(const PartitionerConfig &config)
 {
-    auto compressed_node_based_graph =
-        LoadCompressedNodeBasedGraph(config.GetPath(".osrm.cnbg").string());
+    std::vector<extractor::CompressedNodeBasedGraphEdge> edges;
+    extractor::files::readCompressedNodeBasedGraph(config.GetPath(".osrm.cnbg"), edges);
+    groupEdgesBySource(begin(edges), end(edges));
 
-    util::Log() << "Loaded compressed node based graph: "
-                << compressed_node_based_graph.edges.size() << " edges, "
-                << compressed_node_based_graph.coordinates.size() << " nodes";
+    std::vector<util::Coordinate> coordinates;
+    extractor::files::readNodeCoordinates(config.GetPath(".osrm.nbg_nodes"), coordinates);
 
-    groupEdgesBySource(begin(compressed_node_based_graph.edges),
-                       end(compressed_node_based_graph.edges));
+    util::Log() << "Loaded compressed node based graph: " << edges.size() << " edges, "
+                << coordinates.size() << " nodes";
 
-    auto graph =
-        makeBisectionGraph(compressed_node_based_graph.coordinates,
-                           adaptToBisectionEdge(std::move(compressed_node_based_graph.edges)));
+    auto graph = makeBisectionGraph(coordinates, adaptToBisectionEdge(std::move(edges)));
 
     util::Log() << " running partition: " << config.max_cell_sizes.front() << " " << config.balance
                 << " " << config.boundary_factor << " " << config.num_optimizing_cuts << " "

src/storage/storage.cpp

@@ -18,6 +18,7 @@
 #include "extractor/edge_based_node.hpp"
 #include "extractor/files.hpp"
 #include "extractor/maneuver_override.hpp"
+#include "extractor/name_table.hpp"
 #include "extractor/packed_osm_ids.hpp"
 #include "extractor/profile_properties.hpp"
 #include "extractor/query_node.hpp"
@@ -67,6 +68,26 @@ namespace osrm
 {
 namespace storage
 {
+namespace
+{
+template <typename OutIter> void readBlocks(const boost::filesystem::path &path, OutIter out)
+{
+    tar::FileReader reader(path, tar::FileReader::VerifyFingerprint);
+
+    std::vector<tar::FileReader::FileEntry> entries;
+    reader.List(std::back_inserter(entries));
+
+    for (const auto &entry : entries)
+    {
+        const auto name_end = entry.name.rfind(".meta");
+        if (name_end == std::string::npos)
+        {
+            auto number_of_elements = reader.ReadElementCount64(entry.name);
+            *out++ = NamedBlock{entry.name, Block{number_of_elements, entry.size}};
+        }
+    }
+}
+}
 
 static constexpr std::size_t NUM_METRICS = 8;
 
@@ -219,389 +240,136 @@ void Storage::PopulateLayout(DataLayout &layout)
                         make_block<char>(absolute_file_index_path.string().length() + 1));
     }
 
+    std::unordered_map<std::string, DataLayout::BlockID> name_to_block_id = {
+        {"/mld/multilevelgraph/node_array", DataLayout::MLD_GRAPH_NODE_LIST},
+        {"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST},
+        {"/mld/multilevelgraph/node_to_edge_offset", DataLayout::MLD_GRAPH_NODE_TO_OFFSET},
+        {"/mld/multilevelgraph/connectivity_checksum", DataLayout::IGNORE_BLOCK},
+        {"/mld/multilevelpartition/level_data", DataLayout::MLD_LEVEL_DATA},
+        {"/mld/multilevelpartition/partition", DataLayout::MLD_PARTITION},
+        {"/mld/multilevelpartition/cell_to_children", DataLayout::MLD_CELL_TO_CHILDREN},
+        {"/mld/cellstorage/source_boundary", DataLayout::MLD_CELL_SOURCE_BOUNDARY},
+        {"/mld/cellstorage/destination_boundary", DataLayout::MLD_CELL_DESTINATION_BOUNDARY},
+        {"/mld/cellstorage/cells", DataLayout::MLD_CELLS},
+        {"/mld/cellstorage/level_to_cell_offset", DataLayout::MLD_CELL_LEVEL_OFFSETS},
+        {"/mld/metrics/0/weights", DataLayout::MLD_CELL_WEIGHTS_0},
+        {"/mld/metrics/1/weights", DataLayout::MLD_CELL_WEIGHTS_1},
+        {"/mld/metrics/2/weights", DataLayout::MLD_CELL_WEIGHTS_2},
+        {"/mld/metrics/3/weights", DataLayout::MLD_CELL_WEIGHTS_3},
+        {"/mld/metrics/4/weights", DataLayout::MLD_CELL_WEIGHTS_4},
+        {"/mld/metrics/5/weights", DataLayout::MLD_CELL_WEIGHTS_5},
+        {"/mld/metrics/6/weights", DataLayout::MLD_CELL_WEIGHTS_6},
+        {"/mld/metrics/7/weights", DataLayout::MLD_CELL_WEIGHTS_7},
+        {"/mld/metrics/0/durations", DataLayout::MLD_CELL_DURATIONS_0},
+        {"/mld/metrics/1/durations", DataLayout::MLD_CELL_DURATIONS_1},
+        {"/mld/metrics/2/durations", DataLayout::MLD_CELL_DURATIONS_2},
+        {"/mld/metrics/3/durations", DataLayout::MLD_CELL_DURATIONS_3},
+        {"/mld/metrics/4/durations", DataLayout::MLD_CELL_DURATIONS_4},
+        {"/mld/metrics/5/durations", DataLayout::MLD_CELL_DURATIONS_5},
+        {"/mld/metrics/6/durations", DataLayout::MLD_CELL_DURATIONS_6},
+        {"/mld/metrics/7/durations", DataLayout::MLD_CELL_DURATIONS_7},
+        {"/ch/checksum", DataLayout::HSGR_CHECKSUM},
+        {"/ch/contracted_graph/node_array", DataLayout::CH_GRAPH_NODE_LIST},
+        {"/ch/contracted_graph/edge_array", DataLayout::CH_GRAPH_EDGE_LIST},
+        {"/ch/connectivity_checksum", DataLayout::IGNORE_BLOCK},
+        {"/ch/edge_filter/0", DataLayout::CH_EDGE_FILTER_0},
+        {"/ch/edge_filter/1", DataLayout::CH_EDGE_FILTER_1},
+        {"/ch/edge_filter/2", DataLayout::CH_EDGE_FILTER_2},
+        {"/ch/edge_filter/3", DataLayout::CH_EDGE_FILTER_3},
+        {"/ch/edge_filter/4", DataLayout::CH_EDGE_FILTER_4},
+        {"/ch/edge_filter/5", DataLayout::CH_EDGE_FILTER_5},
+        {"/ch/edge_filter/6", DataLayout::CH_EDGE_FILTER_6},
+        {"/ch/edge_filter/7", DataLayout::CH_EDGE_FILTER_7},
+        {"/common/intersection_bearings/bearing_values", DataLayout::BEARING_VALUES},
+        {"/common/intersection_bearings/node_to_class_id", DataLayout::BEARING_CLASSID},
+        {"/common/intersection_bearings/class_id_to_ranges/block_offsets",
+         DataLayout::BEARING_OFFSETS},
+        {"/common/intersection_bearings/class_id_to_ranges/diff_blocks",
+         DataLayout::BEARING_BLOCKS},
+        {"/common/entry_classes", DataLayout::ENTRY_CLASS},
+        {"/common/properties", DataLayout::PROPERTIES},
+        {"/common/coordinates", DataLayout::COORDINATE_LIST},
+        {"/common/osm_node_ids/packed", DataLayout::OSM_NODE_ID_LIST},
+        {"/common/data_sources_names", DataLayout::DATASOURCES_NAMES},
+        {"/common/segment_data/index", DataLayout::GEOMETRIES_INDEX},
+        {"/common/segment_data/nodes", DataLayout::GEOMETRIES_NODE_LIST},
+        {"/common/segment_data/forward_weights/packed", DataLayout::GEOMETRIES_FWD_WEIGHT_LIST},
+        {"/common/segment_data/reverse_weights/packed", DataLayout::GEOMETRIES_REV_WEIGHT_LIST},
+        {"/common/segment_data/forward_durations/packed", DataLayout::GEOMETRIES_FWD_DURATION_LIST},
+        {"/common/segment_data/reverse_durations/packed", DataLayout::GEOMETRIES_REV_DURATION_LIST},
+        {"/common/segment_data/forward_data_sources", DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST},
+        {"/common/segment_data/reverse_data_sources", DataLayout::GEOMETRIES_REV_DATASOURCES_LIST},
+        {"/common/ebg_node_data/nodes", DataLayout::EDGE_BASED_NODE_DATA_LIST},
+        {"/common/ebg_node_data/annotations", DataLayout::ANNOTATION_DATA_LIST},
+        {"/common/turn_lanes/offsets", DataLayout::LANE_DESCRIPTION_OFFSETS},
+        {"/common/turn_lanes/masks", DataLayout::LANE_DESCRIPTION_MASKS},
+        {"/common/turn_lanes/data", DataLayout::TURN_LANE_DATA},
+        {"/common/maneuver_overrides/overrides", DataLayout::MANEUVER_OVERRIDES},
+        {"/common/maneuver_overrides/node_sequences", DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES},
+        {"/common/turn_penalty/weight", DataLayout::TURN_WEIGHT_PENALTIES},
+        {"/common/turn_penalty/duration", DataLayout::TURN_DURATION_PENALTIES},
+        {"/common/turn_data/pre_turn_bearings", DataLayout::PRE_TURN_BEARING},
+        {"/common/turn_data/post_turn_bearings", DataLayout::POST_TURN_BEARING},
+        {"/common/turn_data/turn_instructions", DataLayout::TURN_INSTRUCTION},
+        {"/common/turn_data/lane_data_ids", DataLayout::LANE_DATA_ID},
+        {"/common/turn_data/entry_class_ids", DataLayout::ENTRY_CLASSID},
+        {"/common/turn_data/connectivity_checksum", DataLayout::IGNORE_BLOCK},
+        {"/common/names/blocks", DataLayout::NAME_BLOCKS},
+        {"/common/names/values", DataLayout::NAME_VALUES},
+        {"/common/rtree/search_tree", DataLayout::R_SEARCH_TREE},
+        {"/common/rtree/search_tree_level_starts", DataLayout::R_SEARCH_TREE_LEVEL_STARTS},
+    };
+    std::vector<NamedBlock> blocks;
+
+    constexpr bool REQUIRED = true;
+    constexpr bool OPTIONAL = false;
+    std::vector<std::pair<bool, boost::filesystem::path>> tar_files = {
+        {OPTIONAL, config.GetPath(".osrm.mldgr")},
+        {OPTIONAL, config.GetPath(".osrm.cells")},
+        {OPTIONAL, config.GetPath(".osrm.partition")},
+        {OPTIONAL, config.GetPath(".osrm.cell_metrics")},
+        {OPTIONAL, config.GetPath(".osrm.hsgr")},
+        {REQUIRED, config.GetPath(".osrm.icd")},
+        {REQUIRED, config.GetPath(".osrm.properties")},
+        {REQUIRED, config.GetPath(".osrm.nbg_nodes")},
+        {REQUIRED, config.GetPath(".osrm.datasource_names")},
+        {REQUIRED, config.GetPath(".osrm.geometry")},
+        {REQUIRED, config.GetPath(".osrm.ebg_nodes")},
+        {REQUIRED, config.GetPath(".osrm.tls")},
+        {REQUIRED, config.GetPath(".osrm.tld")},
+        {REQUIRED, config.GetPath(".osrm.maneuver_overrides")},
+        {REQUIRED, config.GetPath(".osrm.turn_weight_penalties")},
+        {REQUIRED, config.GetPath(".osrm.turn_duration_penalties")},
+        {REQUIRED, config.GetPath(".osrm.edges")},
+        {REQUIRED, config.GetPath(".osrm.names")},
+        {REQUIRED, config.GetPath(".osrm.ramIndex")},
+    };
+
+    for (const auto &file : tar_files)
     {
-        util::Log() << "load names from: " << config.GetPath(".osrm.names");
-        // number of entries in name index
-        io::FileReader name_file(config.GetPath(".osrm.names"), io::FileReader::VerifyFingerprint);
-        layout.SetBlock(DataLayout::NAME_CHAR_DATA, make_block<char>(name_file.GetSize()));
-    }
-
-    {
-        io::FileReader reader(config.GetPath(".osrm.tls"), io::FileReader::VerifyFingerprint);
-        auto num_offsets = reader.ReadVectorSize<std::uint32_t>();
-        auto num_masks = reader.ReadVectorSize<extractor::TurnLaneType::Mask>();
-
-        layout.SetBlock(DataLayout::LANE_DESCRIPTION_OFFSETS,
-                        make_block<std::uint32_t>(num_offsets));
-        layout.SetBlock(DataLayout::LANE_DESCRIPTION_MASKS,
-                        make_block<extractor::TurnLaneType::Mask>(num_masks));
-    }
-
-    // Loading information for original edges
-    {
-        io::FileReader edges_file(config.GetPath(".osrm.edges"), io::FileReader::VerifyFingerprint);
-        const auto number_of_original_edges = edges_file.ReadElementCount64();
-
-        // note: settings this all to the same size is correct, we extract them from the same struct
-        layout.SetBlock(DataLayout::PRE_TURN_BEARING,
-                        make_block<guidance::TurnBearing>(number_of_original_edges));
-        layout.SetBlock(DataLayout::POST_TURN_BEARING,
-                        make_block<guidance::TurnBearing>(number_of_original_edges));
-        layout.SetBlock(DataLayout::TURN_INSTRUCTION,
-                        make_block<guidance::TurnInstruction>(number_of_original_edges));
-        layout.SetBlock(DataLayout::LANE_DATA_ID, make_block<LaneDataID>(number_of_original_edges));
-        layout.SetBlock(DataLayout::ENTRY_CLASSID,
-                        make_block<EntryClassID>(number_of_original_edges));
-    }
-
-    {
-        io::FileReader nodes_data_file(config.GetPath(".osrm.ebg_nodes"),
-                                       io::FileReader::VerifyFingerprint);
-        const auto nodes_number = nodes_data_file.ReadElementCount64();
-        const auto annotations_number = nodes_data_file.ReadElementCount64();
-        layout.SetBlock(DataLayout::EDGE_BASED_NODE_DATA_LIST,
-                        make_block<extractor::EdgeBasedNode>(nodes_number));
-        layout.SetBlock(DataLayout::ANNOTATION_DATA_LIST,
-                        make_block<extractor::NodeBasedEdgeAnnotation>(annotations_number));
-    }
-
-    if (boost::filesystem::exists(config.GetPath(".osrm.hsgr")))
-    {
-        io::FileReader reader(config.GetPath(".osrm.hsgr"), io::FileReader::VerifyFingerprint);
-
-        reader.Skip<std::uint32_t>(1); // checksum
-        auto num_nodes = reader.ReadVectorSize<contractor::QueryGraph::NodeArrayEntry>();
-        auto num_edges = reader.ReadVectorSize<contractor::QueryGraph::EdgeArrayEntry>();
-        auto num_metrics = reader.ReadElementCount64();
-
-        if (num_metrics > NUM_METRICS)
+        if (boost::filesystem::exists(file.second))
         {
-            throw util::exception("Only " + std::to_string(NUM_METRICS) +
-                                  " metrics are supported at the same time.");
-        }
-
-        layout.SetBlock(DataLayout::HSGR_CHECKSUM, make_block<unsigned>(1));
-        layout.SetBlock(DataLayout::CH_GRAPH_NODE_LIST,
-                        make_block<contractor::QueryGraph::NodeArrayEntry>(num_nodes));
-        layout.SetBlock(DataLayout::CH_GRAPH_EDGE_LIST,
-                        make_block<contractor::QueryGraph::EdgeArrayEntry>(num_edges));
-
-        for (const auto index : util::irange<std::size_t>(0, num_metrics))
-        {
-            layout.SetBlock(static_cast<DataLayout::BlockID>(DataLayout::CH_EDGE_FILTER_0 + index),
-                            make_block<unsigned>(num_edges));
-        }
-        for (const auto index : util::irange<std::size_t>(num_metrics, NUM_METRICS))
-        {
-            layout.SetBlock(static_cast<DataLayout::BlockID>(DataLayout::CH_EDGE_FILTER_0 + index),
-                            make_block<unsigned>(0));
-        }
-    }
-    else
-    {
-        layout.SetBlock(DataLayout::HSGR_CHECKSUM, make_block<unsigned>(0));
-        layout.SetBlock(DataLayout::CH_GRAPH_NODE_LIST,
-                        make_block<contractor::QueryGraph::NodeArrayEntry>(0));
-        layout.SetBlock(DataLayout::CH_GRAPH_EDGE_LIST,
-                        make_block<contractor::QueryGraph::EdgeArrayEntry>(0));
-        for (const auto index : util::irange<std::size_t>(0, NUM_METRICS))
-        {
-            layout.SetBlock(static_cast<DataLayout::BlockID>(DataLayout::CH_EDGE_FILTER_0 + index),
-                            make_block<unsigned>(0));
-        }
-    }
-
-    // load rsearch tree size
-    {
-        io::FileReader tree_node_file(config.GetPath(".osrm.ramIndex"),
-                                      io::FileReader::VerifyFingerprint);
-
-        const auto tree_size = tree_node_file.ReadElementCount64();
-        layout.SetBlock(DataLayout::R_SEARCH_TREE, make_block<RTreeNode>(tree_size));
-        tree_node_file.Skip<RTreeNode>(tree_size);
-        const auto tree_levels_size = tree_node_file.ReadElementCount64();
-        layout.SetBlock(DataLayout::R_SEARCH_TREE_LEVELS,
-                        make_block<std::uint64_t>(tree_levels_size));
-    }
-
-    {
-        layout.SetBlock(DataLayout::PROPERTIES, make_block<extractor::ProfileProperties>(1));
-    }
-
-    // read timestampsize
-    {
-        io::FileReader timestamp_file(config.GetPath(".osrm.timestamp"),
-                                      io::FileReader::VerifyFingerprint);
-        const auto timestamp_size = timestamp_file.GetSize();
-        layout.SetBlock(DataLayout::TIMESTAMP, make_block<char>(timestamp_size));
-    }
-
-    // load turn weight penalties
-    {
-        io::FileReader turn_weight_penalties_file(config.GetPath(".osrm.turn_weight_penalties"),
-                                                  io::FileReader::VerifyFingerprint);
-        const auto number_of_penalties = turn_weight_penalties_file.ReadElementCount64();
-        layout.SetBlock(DataLayout::TURN_WEIGHT_PENALTIES,
-                        make_block<TurnPenalty>(number_of_penalties));
-    }
-
-    // load turn duration penalties
-    {
-        io::FileReader turn_duration_penalties_file(config.GetPath(".osrm.turn_duration_penalties"),
-                                                    io::FileReader::VerifyFingerprint);
-        const auto number_of_penalties = turn_duration_penalties_file.ReadElementCount64();
-        layout.SetBlock(DataLayout::TURN_DURATION_PENALTIES,
-                        make_block<TurnPenalty>(number_of_penalties));
-    }
-
-    // load coordinate size
-    {
-        io::FileReader node_file(config.GetPath(".osrm.nbg_nodes"),
-                                 io::FileReader::VerifyFingerprint);
-        const auto coordinate_list_size = node_file.ReadElementCount64();
-        layout.SetBlock(DataLayout::COORDINATE_LIST,
-                        make_block<util::Coordinate>(coordinate_list_size));
-        node_file.Skip<util::Coordinate>(coordinate_list_size);
-        // skip number of elements
-        node_file.Skip<std::uint64_t>(1);
-        const auto num_id_blocks = node_file.ReadElementCount64();
-        // we'll read a list of OSM node IDs from the same data, so set the block size for the same
-        // number of items:
-        layout.SetBlock(DataLayout::OSM_NODE_ID_LIST,
-                        make_block<extractor::PackedOSMIDsView::block_type>(num_id_blocks));
-    }
-
-    // load geometries sizes
-    {
-        io::FileReader reader(config.GetPath(".osrm.geometry"), io::FileReader::VerifyFingerprint);
-
-        const auto number_of_geometries_indices = reader.ReadVectorSize<unsigned>();
-        layout.SetBlock(DataLayout::GEOMETRIES_INDEX,
-                        make_block<unsigned>(number_of_geometries_indices));
-
-        const auto number_of_compressed_geometries = reader.ReadVectorSize<NodeID>();
-        layout.SetBlock(DataLayout::GEOMETRIES_NODE_LIST,
-                        make_block<NodeID>(number_of_compressed_geometries));
-
-        reader.ReadElementCount64(); // number of segments
-        const auto number_of_segment_weight_blocks =
-            reader.ReadVectorSize<extractor::SegmentDataView::SegmentWeightVector::block_type>();
-
-        reader.ReadElementCount64(); // number of segments
-        auto number_of_rev_weight_blocks =
-            reader.ReadVectorSize<extractor::SegmentDataView::SegmentWeightVector::block_type>();
-        BOOST_ASSERT(number_of_rev_weight_blocks == number_of_segment_weight_blocks);
-        (void)number_of_rev_weight_blocks;
-
-        reader.ReadElementCount64(); // number of segments
-        const auto number_of_segment_duration_blocks =
-            reader.ReadVectorSize<extractor::SegmentDataView::SegmentDurationVector::block_type>();
-
-        layout.SetBlock(DataLayout::GEOMETRIES_FWD_WEIGHT_LIST,
-                        make_block<extractor::SegmentDataView::SegmentWeightVector::block_type>(
-                            number_of_segment_weight_blocks));
-        layout.SetBlock(DataLayout::GEOMETRIES_REV_WEIGHT_LIST,
-                        make_block<extractor::SegmentDataView::SegmentWeightVector::block_type>(
-                            number_of_segment_weight_blocks));
-        layout.SetBlock(DataLayout::GEOMETRIES_FWD_DURATION_LIST,
-                        make_block<extractor::SegmentDataView::SegmentDurationVector::block_type>(
-                            number_of_segment_duration_blocks));
-        layout.SetBlock(DataLayout::GEOMETRIES_REV_DURATION_LIST,
-                        make_block<extractor::SegmentDataView::SegmentDurationVector::block_type>(
-                            number_of_segment_duration_blocks));
-        layout.SetBlock(DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST,
-                        make_block<DatasourceID>(number_of_compressed_geometries));
-        layout.SetBlock(DataLayout::GEOMETRIES_REV_DATASOURCES_LIST,
-                        make_block<DatasourceID>(number_of_compressed_geometries));
-    }
-
-    // Load datasource name sizes.
-    {
-        layout.SetBlock(DataLayout::DATASOURCES_NAMES, make_block<extractor::Datasources>(1));
-    }
-
-    {
-        io::FileReader reader(config.GetPath(".osrm.icd"), io::FileReader::VerifyFingerprint);
-
-        auto num_discreate_bearings = reader.ReadVectorSize<DiscreteBearing>();
-        layout.SetBlock(DataLayout::BEARING_VALUES,
-                        make_block<DiscreteBearing>(num_discreate_bearings));
-
-        auto num_bearing_classes = reader.ReadVectorSize<BearingClassID>();
-        layout.SetBlock(DataLayout::BEARING_CLASSID,
-                        make_block<BearingClassID>(num_bearing_classes));
-
-        reader.Skip<std::uint32_t>(1); // sum_lengths
-        const auto bearing_blocks = reader.ReadVectorSize<unsigned>();
-        const auto bearing_offsets =
-            reader
-                .ReadVectorSize<typename util::RangeTable<16, storage::Ownership::View>::BlockT>();
-
-        layout.SetBlock(DataLayout::BEARING_OFFSETS, make_block<unsigned>(bearing_blocks));
-        layout.SetBlock(DataLayout::BEARING_BLOCKS,
-                        make_block<typename util::RangeTable<16, storage::Ownership::View>::BlockT>(
-                            bearing_offsets));
-
-        auto num_entry_classes = reader.ReadVectorSize<util::guidance::EntryClass>();
-        layout.SetBlock(DataLayout::ENTRY_CLASS,
-                        make_block<util::guidance::EntryClass>(num_entry_classes));
-    }
-
-    {
-        // Loading turn lane data
-        io::FileReader lane_data_file(config.GetPath(".osrm.tld"),
-                                      io::FileReader::VerifyFingerprint);
-        const auto lane_tuple_count = lane_data_file.ReadElementCount64();
-        layout.SetBlock(DataLayout::TURN_LANE_DATA,
-                        make_block<util::guidance::LaneTupleIdPair>(lane_tuple_count));
-    }
-
-    // load maneuver overrides
-    {
-        io::FileReader maneuver_overrides_file(config.GetPath(".osrm.maneuver_overrides"),
-                                               io::FileReader::VerifyFingerprint);
-        const auto number_of_overrides =
-            maneuver_overrides_file.ReadVectorSize<extractor::StorageManeuverOverride>();
-        layout.SetBlock(DataLayout::MANEUVER_OVERRIDES,
-                        make_block<extractor::StorageManeuverOverride>(number_of_overrides));
-        const auto number_of_nodes = maneuver_overrides_file.ReadVectorSize<NodeID>();
-        layout.SetBlock(DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES,
-                        make_block<NodeID>(number_of_nodes));
-    }
-
-    {
-        // Loading MLD Data
-        if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
-        {
-            io::FileReader reader(config.GetPath(".osrm.partition"),
-                                  io::FileReader::VerifyFingerprint);
-
-            reader.Skip<partitioner::MultiLevelPartition::LevelData>(1);
-            layout.SetBlock(DataLayout::MLD_LEVEL_DATA,
-                            make_block<partitioner::MultiLevelPartition::LevelData>(1));
-            const auto partition_entries_count = reader.ReadVectorSize<PartitionID>();
-            layout.SetBlock(DataLayout::MLD_PARTITION,
-                            make_block<PartitionID>(partition_entries_count));
-            const auto children_entries_count = reader.ReadVectorSize<CellID>();
-            layout.SetBlock(DataLayout::MLD_CELL_TO_CHILDREN,
-                            make_block<CellID>(children_entries_count));
+            readBlocks(file.second, std::back_inserter(blocks));
         }
         else
         {
-            layout.SetBlock(DataLayout::MLD_LEVEL_DATA,
-                            make_block<partitioner::MultiLevelPartition::LevelData>(0));
-            layout.SetBlock(DataLayout::MLD_PARTITION, make_block<PartitionID>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_TO_CHILDREN, make_block<CellID>(0));
-        }
-
-        if (boost::filesystem::exists(config.GetPath(".osrm.cells")))
-        {
-            io::FileReader reader(config.GetPath(".osrm.cells"), io::FileReader::VerifyFingerprint);
-
-            const auto source_node_count = reader.ReadVectorSize<NodeID>();
-            layout.SetBlock(DataLayout::MLD_CELL_SOURCE_BOUNDARY,
-                            make_block<NodeID>(source_node_count));
-            const auto destination_node_count = reader.ReadVectorSize<NodeID>();
-            layout.SetBlock(DataLayout::MLD_CELL_DESTINATION_BOUNDARY,
-                            make_block<NodeID>(destination_node_count));
-            const auto cell_count = reader.ReadVectorSize<partitioner::CellStorage::CellData>();
-            layout.SetBlock(DataLayout::MLD_CELLS,
-                            make_block<partitioner::CellStorage::CellData>(cell_count));
-            const auto level_offsets_count = reader.ReadVectorSize<std::uint64_t>();
-            layout.SetBlock(DataLayout::MLD_CELL_LEVEL_OFFSETS,
-                            make_block<std::uint64_t>(level_offsets_count));
-        }
-        else
-        {
-            layout.SetBlock(DataLayout::MLD_CELL_SOURCE_BOUNDARY, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_DESTINATION_BOUNDARY, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELLS, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_LEVEL_OFFSETS, make_block<char>(0));
-        }
-
-        if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
-        {
-            io::FileReader reader(config.GetPath(".osrm.cell_metrics"),
-                                  io::FileReader::VerifyFingerprint);
-            auto num_metric = reader.ReadElementCount64();
-
-            if (num_metric > NUM_METRICS)
+            if (file.first == REQUIRED)
             {
-                throw util::exception("Only " + std::to_string(NUM_METRICS) +
-                                      " metrics are supported at the same time.");
-            }
-
-            for (const auto index : util::irange<std::size_t>(0, num_metric))
-            {
-                const auto weights_count = reader.ReadVectorSize<EdgeWeight>();
-                layout.SetBlock(
-                    static_cast<DataLayout::BlockID>(DataLayout::MLD_CELL_WEIGHTS_0 + index),
-                    make_block<EdgeWeight>(weights_count));
-                const auto durations_count = reader.ReadVectorSize<EdgeDuration>();
-                layout.SetBlock(
-                    static_cast<DataLayout::BlockID>(DataLayout::MLD_CELL_DURATIONS_0 + index),
-                    make_block<EdgeDuration>(durations_count));
-            }
-            for (const auto index : util::irange<std::size_t>(num_metric, NUM_METRICS))
-            {
-                layout.SetBlock(
-                    static_cast<DataLayout::BlockID>(DataLayout::MLD_CELL_WEIGHTS_0 + index),
-                    make_block<EdgeWeight>(0));
-                layout.SetBlock(
-                    static_cast<DataLayout::BlockID>(DataLayout::MLD_CELL_DURATIONS_0 + index),
-                    make_block<EdgeDuration>(0));
+                throw util::exception("Could not find required filed: " +
+                                      std::get<1>(file).string());
             }
         }
-        else
-        {
-            layout.SetBlock(DataLayout::MLD_CELL_WEIGHTS_0, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_WEIGHTS_1, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_WEIGHTS_2, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_WEIGHTS_3, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_WEIGHTS_4, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_WEIGHTS_5, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_WEIGHTS_6, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_WEIGHTS_7, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_0, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_1, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_2, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_3, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_4, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_5, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_6, make_block<char>(0));
-            layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_7, make_block<char>(0));
-        }
+    }
 
-        if (boost::filesystem::exists(config.GetPath(".osrm.mldgr")))
+    for (const auto &block : blocks)
+    {
+        auto id_iter = name_to_block_id.find(std::get<0>(block));
+        if (id_iter == name_to_block_id.end())
         {
-            io::FileReader reader(config.GetPath(".osrm.mldgr"), io::FileReader::VerifyFingerprint);
-
-            const auto num_nodes =
-                reader.ReadVectorSize<customizer::MultiLevelEdgeBasedGraph::NodeArrayEntry>();
-            const auto num_edges =
-                reader.ReadVectorSize<customizer::MultiLevelEdgeBasedGraph::EdgeArrayEntry>();
-            const auto num_node_offsets =
-                reader.ReadVectorSize<customizer::MultiLevelEdgeBasedGraph::EdgeOffset>();
-
-            layout.SetBlock(
-                DataLayout::MLD_GRAPH_NODE_LIST,
-                make_block<customizer::MultiLevelEdgeBasedGraph::NodeArrayEntry>(num_nodes));
-            layout.SetBlock(
-                DataLayout::MLD_GRAPH_EDGE_LIST,
-                make_block<customizer::MultiLevelEdgeBasedGraph::EdgeArrayEntry>(num_edges));
-            layout.SetBlock(
-                DataLayout::MLD_GRAPH_NODE_TO_OFFSET,
-                make_block<customizer::MultiLevelEdgeBasedGraph::EdgeOffset>(num_node_offsets));
-        }
-        else
-        {
-            layout.SetBlock(DataLayout::MLD_GRAPH_NODE_LIST,
-                            make_block<customizer::MultiLevelEdgeBasedGraph::NodeArrayEntry>(0));
-            layout.SetBlock(DataLayout::MLD_GRAPH_EDGE_LIST,
-                            make_block<customizer::MultiLevelEdgeBasedGraph::EdgeArrayEntry>(0));
-            layout.SetBlock(DataLayout::MLD_GRAPH_NODE_TO_OFFSET,
-                            make_block<customizer::MultiLevelEdgeBasedGraph::EdgeOffset>(0));
+            throw util::exception("Could not map " + std::get<0>(block) +
+                                  " to a region in memory.");
         }
+        layout.SetBlock(id_iter->second, std::get<1>(block));
     }
 }
 
@@ -632,30 +400,31 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // Name data
     {
-        io::FileReader name_file(config.GetPath(".osrm.names"), io::FileReader::VerifyFingerprint);
-        std::size_t name_file_size = name_file.GetSize();
+        const auto name_blocks_ptr =
+            layout.GetBlockPtr<extractor::NameTableView::IndexedData::BlockReference, true>(
+                memory_ptr, DataLayout::NAME_BLOCKS);
+        const auto name_values_ptr =
+            layout.GetBlockPtr<extractor::NameTableView::IndexedData::ValueType, true>(
+                memory_ptr, DataLayout::NAME_VALUES);
 
-        BOOST_ASSERT(name_file_size == layout.GetBlockSize(DataLayout::NAME_CHAR_DATA));
-        const auto name_char_ptr =
-            layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::NAME_CHAR_DATA);
+        util::vector_view<extractor::NameTableView::IndexedData::BlockReference> blocks(
+            name_blocks_ptr, layout.GetBlockEntries(storage::DataLayout::NAME_BLOCKS));
+        util::vector_view<extractor::NameTableView::IndexedData::ValueType> values(
+            name_values_ptr, layout.GetBlockEntries(storage::DataLayout::NAME_VALUES));
 
-        name_file.ReadInto<char>(name_char_ptr, name_file_size);
+        extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
+        extractor::NameTableView name_table{index_data_view};
+        extractor::files::readNames(config.GetPath(".osrm.names"), name_table);
     }
 
     // Turn lane data
     {
-        io::FileReader lane_data_file(config.GetPath(".osrm.tld"),
-                                      io::FileReader::VerifyFingerprint);
-
-        const auto lane_tuple_count = lane_data_file.ReadElementCount64();
-
-        // Need to call GetBlockPtr -> it write the memory canary, even if no data needs to be
-        // loaded.
         const auto turn_lane_data_ptr = layout.GetBlockPtr<util::guidance::LaneTupleIdPair, true>(
             memory_ptr, DataLayout::TURN_LANE_DATA);
-        BOOST_ASSERT(lane_tuple_count * sizeof(util::guidance::LaneTupleIdPair) ==
-                     layout.GetBlockSize(DataLayout::TURN_LANE_DATA));
-        lane_data_file.ReadInto(turn_lane_data_ptr, lane_tuple_count);
+        util::vector_view<util::guidance::LaneTupleIdPair> turn_lane_data(
+            turn_lane_data_ptr, layout.GetBlockEntries(storage::DataLayout::TURN_LANE_DATA));
+
+        extractor::files::readTurnLaneData(config.GetPath(".osrm.tld"), turn_lane_data);
     }
 
     // Turn lane descriptions
@@ -831,54 +600,49 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // load turn weight penalties
     {
-        io::FileReader turn_weight_penalties_file(config.GetPath(".osrm.turn_weight_penalties"),
-                                                  io::FileReader::VerifyFingerprint);
-        const auto number_of_penalties = turn_weight_penalties_file.ReadElementCount64();
-        const auto turn_weight_penalties_ptr =
-            layout.GetBlockPtr<TurnPenalty, true>(memory_ptr, DataLayout::TURN_WEIGHT_PENALTIES);
-        turn_weight_penalties_file.ReadInto(turn_weight_penalties_ptr, number_of_penalties);
+        auto turn_duration_penalties_ptr = layout.GetBlockPtr<TurnPenalty, true>(
+            memory_ptr, storage::DataLayout::TURN_WEIGHT_PENALTIES);
+        util::vector_view<TurnPenalty> turn_duration_penalties(
+            turn_duration_penalties_ptr,
+            layout.GetBlockEntries(storage::DataLayout::TURN_WEIGHT_PENALTIES));
+        extractor::files::readTurnWeightPenalty(config.GetPath(".osrm.turn_weight_penalties"),
+                                                turn_duration_penalties);
     }
 
     // load turn duration penalties
     {
-        io::FileReader turn_duration_penalties_file(config.GetPath(".osrm.turn_duration_penalties"),
-                                                    io::FileReader::VerifyFingerprint);
-        const auto number_of_penalties = turn_duration_penalties_file.ReadElementCount64();
-        const auto turn_duration_penalties_ptr =
-            layout.GetBlockPtr<TurnPenalty, true>(memory_ptr, DataLayout::TURN_DURATION_PENALTIES);
-        turn_duration_penalties_file.ReadInto(turn_duration_penalties_ptr, number_of_penalties);
-    }
-
-    // store timestamp
-    {
-        io::FileReader timestamp_file(config.GetPath(".osrm.timestamp"),
-                                      io::FileReader::VerifyFingerprint);
-        const auto timestamp_size = timestamp_file.GetSize();
-
-        const auto timestamp_ptr =
-            layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::TIMESTAMP);
-        BOOST_ASSERT(timestamp_size == layout.GetBlockEntries(DataLayout::TIMESTAMP));
-        timestamp_file.ReadInto(timestamp_ptr, timestamp_size);
+        auto turn_duration_penalties_ptr = layout.GetBlockPtr<TurnPenalty, true>(
+            memory_ptr, storage::DataLayout::TURN_DURATION_PENALTIES);
+        util::vector_view<TurnPenalty> turn_duration_penalties(
+            turn_duration_penalties_ptr,
+            layout.GetBlockEntries(storage::DataLayout::TURN_DURATION_PENALTIES));
+        extractor::files::readTurnDurationPenalty(config.GetPath(".osrm.turn_duration_penalties"),
+                                                  turn_duration_penalties);
     }
 
     // store search tree portion of rtree
     {
-        io::FileReader tree_node_file(config.GetPath(".osrm.ramIndex"),
-                                      io::FileReader::VerifyFingerprint);
-        // perform this read so that we're at the right stream position for the next
-        // read.
-        tree_node_file.Skip<std::uint64_t>(1);
+
         const auto rtree_ptr =
             layout.GetBlockPtr<RTreeNode, true>(memory_ptr, DataLayout::R_SEARCH_TREE);
+        util::vector_view<RTreeNode> search_tree(
+            rtree_ptr, layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE));
 
-        tree_node_file.ReadInto(rtree_ptr, layout.GetBlockEntries(DataLayout::R_SEARCH_TREE));
+        const auto rtree_levelstarts_ptr = layout.GetBlockPtr<std::uint64_t, true>(
+            memory_ptr, DataLayout::R_SEARCH_TREE_LEVEL_STARTS);
+        util::vector_view<std::uint64_t> rtree_level_starts(
+            rtree_levelstarts_ptr,
+            layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS));
 
-        tree_node_file.Skip<std::uint64_t>(1);
-        const auto rtree_levelsizes_ptr =
-            layout.GetBlockPtr<std::uint64_t, true>(memory_ptr, DataLayout::R_SEARCH_TREE_LEVELS);
+        // we need this purely for the interface
+        util::vector_view<util::Coordinate> empty_coords;
 
-        tree_node_file.ReadInto(rtree_levelsizes_ptr,
-                                layout.GetBlockEntries(DataLayout::R_SEARCH_TREE_LEVELS));
+        util::StaticRTree<RTreeLeaf, storage::Ownership::View> rtree{
+            std::move(search_tree),
+            std::move(rtree_level_starts),
+            config.GetPath(".osrm.fileIndex"),
+            empty_coords};
+        extractor::files::readRamIndex(config.GetPath(".osrm.ramIndex"), rtree);
     }
 
     // load profile properties
@@ -948,7 +712,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
             {
                 auto block_id =
                     static_cast<DataLayout::BlockID>(storage::DataLayout::CH_EDGE_FILTER_0 + index);
-                auto data_ptr = layout.GetBlockPtr<unsigned, true>(memory_ptr, block_id);
+                auto data_ptr =
+                    layout.GetBlockPtr<util::vector_view<bool>::Word, true>(memory_ptr, block_id);
                 auto num_entries = layout.GetBlockEntries(block_id);
                 edge_filter.emplace_back(data_ptr, num_entries);
             }

src/tools/components.cpp

@@ -1,9 +1,11 @@
+#include "extractor/files.hpp"
+#include "extractor/packed_osm_ids.hpp"
 #include "extractor/tarjan_scc.hpp"
+
 #include "util/coordinate.hpp"
 #include "util/coordinate_calculation.hpp"
 #include "util/dynamic_graph.hpp"
 #include "util/fingerprint.hpp"
-#include "util/graph_loader.hpp"
 #include "util/log.hpp"
 #include "util/static_graph.hpp"
 #include "util/typedefs.hpp"
@@ -36,16 +38,13 @@ std::size_t loadGraph(const std::string &path,
                       extractor::PackedOSMIDs &osm_node_ids,
                       std::vector<TarjanEdge> &graph_edge_list)
 {
-    storage::io::FileReader file_reader(path, storage::io::FileReader::VerifyFingerprint);
-
     std::vector<extractor::NodeBasedEdge> edge_list;
+    std::vector<extractor::NodeBasedEdgeAnnotation> annotation_data;
 
     auto nop = boost::make_function_output_iterator([](auto) {});
 
-    const auto number_of_nodes =
-        util::loadNodesFromFile(file_reader, nop, nop, coordinate_list, osm_node_ids);
-
-    util::loadEdgesFromFile(file_reader, edge_list);
+    extractor::files::readRawNBGraph(
+        path, nop, nop, coordinate_list, osm_node_ids, edge_list, annotation_data);
 
     // Building a node-based graph
     for (const auto &input_edge : edge_list)
@@ -66,7 +65,7 @@ std::size_t loadGraph(const std::string &path,
         }
     }
 
-    return number_of_nodes;
+    return osm_node_ids.size();
 }
 
 struct FeatureWriter

src/updater/updater.cpp

@@ -16,9 +16,9 @@
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
 #include "util/for_each_pair.hpp"
-#include "util/graph_loader.hpp"
 #include "util/integer_range.hpp"
 #include "util/log.hpp"
+#include "util/mmap_tar.hpp"
 #include "util/opening_hours.hpp"
 #include "util/static_graph.hpp"
 #include "util/static_rtree.hpp"
@@ -434,8 +434,13 @@ updateTurnPenalties(const UpdaterConfig &config,
 
     // Mapped file pointer for turn indices
     boost::iostreams::mapped_file_source turn_index_region;
-    auto turn_index_blocks = util::mmapFile<extractor::lookup::TurnIndexBlock>(
-        config.GetPath(".osrm.turn_penalties_index"), turn_index_region);
+    const extractor::lookup::TurnIndexBlock *turn_index_blocks;
+    {
+        auto map =
+            util::mmapTarFile(config.GetPath(".osrm.turn_penalties_index"), turn_index_region);
+        turn_index_blocks = reinterpret_cast<const extractor::lookup::TurnIndexBlock *>(
+            map["/extractor/turn_index"].first);
+    }
 
     // Get the turn penalty and update to the new value if required
     std::vector<std::uint64_t> updated_turns;
@@ -571,48 +576,31 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
     std::vector<TurnPenalty> turn_duration_penalties;
     if (update_edge_weights || update_turn_penalties || update_conditional_turns)
     {
-        const auto load_segment_data = [&] {
-            extractor::files::readSegmentData(config.GetPath(".osrm.geometry"), segment_data);
-        };
+        tbb::parallel_invoke(
+            [&] {
+                extractor::files::readSegmentData(config.GetPath(".osrm.geometry"), segment_data);
+            },
+            [&] { extractor::files::readNodeData(config.GetPath(".osrm.ebg_nodes"), node_data); },
 
-        const auto load_node_data = [&] {
-            extractor::files::readNodeData(config.GetPath(".osrm.ebg_nodes"), node_data);
-        };
-
-        const auto load_turn_weight_penalties = [&] {
-            using storage::io::FileReader;
-            FileReader reader(config.GetPath(".osrm.turn_weight_penalties"),
-                              FileReader::VerifyFingerprint);
-            storage::serialization::read(reader, turn_weight_penalties);
-        };
-
-        const auto load_turn_duration_penalties = [&] {
-            using storage::io::FileReader;
-            FileReader reader(config.GetPath(".osrm.turn_duration_penalties"),
-                              FileReader::VerifyFingerprint);
-            storage::serialization::read(reader, turn_duration_penalties);
-        };
-
-        const auto load_profile_properties = [&] {
-            // Propagate profile properties to contractor configuration structure
-            storage::io::FileReader profile_properties_file(
-                config.GetPath(".osrm.properties"), storage::io::FileReader::VerifyFingerprint);
-            profile_properties = profile_properties_file.ReadOne<extractor::ProfileProperties>();
-        };
-
-        tbb::parallel_invoke(load_node_data,
-                             load_segment_data,
-                             load_turn_weight_penalties,
-                             load_turn_duration_penalties,
-                             load_profile_properties);
+            [&] {
+                extractor::files::readTurnWeightPenalty(
+                    config.GetPath(".osrm.turn_weight_penalties"), turn_weight_penalties);
+            },
+            [&] {
+                extractor::files::readTurnDurationPenalty(
+                    config.GetPath(".osrm.turn_duration_penalties"), turn_duration_penalties);
+            },
+            [&] {
+                extractor::files::readProfileProperties(config.GetPath(".osrm.properties"),
+                                                        profile_properties);
+            });
     }
 
     std::vector<extractor::ConditionalTurnPenalty> conditional_turns;
     if (update_conditional_turns)
     {
-        using storage::io::FileReader;
-        FileReader reader(config.GetPath(".osrm.restrictions"), FileReader::VerifyFingerprint);
-        extractor::serialization::read(reader, conditional_turns);
+        extractor::files::readConditionalRestrictions(config.GetPath(".osrm.restrictions"),
+                                                      conditional_turns);
     }
 
     tbb::concurrent_vector<GeometryID> updated_segments;
@@ -811,19 +799,14 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
 
     if (update_turn_penalties || update_conditional_turns)
     {
-        const auto save_penalties = [](const auto &filename, const auto &data) -> void {
-            storage::io::FileWriter writer(filename, storage::io::FileWriter::GenerateFingerprint);
-            storage::serialization::write(writer, data);
-        };
-
         tbb::parallel_invoke(
             [&] {
-                save_penalties(config.GetPath(".osrm.turn_weight_penalties"),
-                               turn_weight_penalties);
+                extractor::files::writeTurnWeightPenalty(
+                    config.GetPath(".osrm.turn_weight_penalties"), turn_weight_penalties);
             },
             [&] {
-                save_penalties(config.GetPath(".osrm.turn_duration_penalties"),
-                               turn_duration_penalties);
+                extractor::files::writeTurnDurationPenalty(
+                    config.GetPath(".osrm.turn_duration_penalties"), turn_duration_penalties);
             });
     }
 

src/util/name_table.cpp

@@ -1,100 +0,0 @@
-#include "util/name_table.hpp"
-#include "storage/io.hpp"
-#include "util/log.hpp"
-
-namespace osrm
-{
-namespace util
-{
-
-NameTable::NameTable(const std::string &file_name)
-{
-    using FileReader = storage::io::FileReader;
-
-    FileReader name_stream_file_reader(file_name, FileReader::VerifyFingerprint);
-    const auto file_size = name_stream_file_reader.GetSize();
-
-    m_buffer = BufferType(static_cast<ValueType *>(::operator new(file_size)),
-                          [](void *ptr) { ::operator delete(ptr); });
-    name_stream_file_reader.ReadInto<char>(m_buffer.get(), file_size);
-    m_name_table.reset(m_buffer.get(), m_buffer.get() + file_size);
-
-    if (m_name_table.empty())
-    {
-        util::Log() << "list of street names is empty in construction of name table from: \""
-                    << file_name << "\"";
-    }
-}
-
-void NameTable::reset(ValueType *begin, ValueType *end)
-{
-    m_buffer.reset();
-    m_name_table.reset(begin, end);
-}
-
-StringView NameTable::GetNameForID(const NameID id) const
-{
-    if (id == INVALID_NAMEID)
-        return {};
-
-    return m_name_table.at(id + 0);
-}
-
-StringView NameTable::GetDestinationsForID(const NameID id) const
-{
-    if (id == INVALID_NAMEID)
-        return {};
-
-    return m_name_table.at(id + 1);
-}
-
-StringView NameTable::GetExitsForID(const NameID id) const
-{
-    if (id == INVALID_NAMEID)
-        return {};
-
-    return m_name_table.at(id + 4);
-}
-
-StringView NameTable::GetRefForID(const NameID id) const
-{
-    if (id == INVALID_NAMEID)
-        return {};
-
-    // Way string data is stored in blocks based on `id` as follows:
-    //
-    // | name | destination | pronunciation | ref | exits
-    //                                      ^     ^
-    //                                      [range)
-    //                                       ^ id + 3
-    //
-    // `id + offset` gives us the range of chars.
-    //
-    // Offset 0 is name, 1 is destination, 2 is pronunciation, 3 is ref, 4 is exits
-    // See datafacades and extractor callbacks for details.
-    const constexpr auto OFFSET_REF = 3u;
-    return m_name_table.at(id + OFFSET_REF);
-}
-
-StringView NameTable::GetPronunciationForID(const NameID id) const
-{
-    if (id == INVALID_NAMEID)
-        return {};
-
-    // Way string data is stored in blocks based on `id` as follows:
-    //
-    // | name | destination | pronunciation | ref | exits
-    //                      ^               ^
-    //                      [range)
-    //                       ^ id + 2
-    //
-    // `id + offset` gives us the range of chars.
-    //
-    // Offset 0 is name, 1 is destination, 2 is pronunciation, 3 is ref, 4 is exits
-    // See datafacades and extractor callbacks for details.
-    const constexpr auto OFFSET_PRONUNCIATION = 2u;
-    return m_name_table.at(id + OFFSET_PRONUNCIATION);
-}
-
-} // namespace util
-} // namespace osrm

third_party/microtar/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2017 rxi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

third_party/microtar/README.md

@@ -0,0 +1,99 @@
+# microtar
+A lightweight tar library written in ANSI C
+
+
+## Basic Usage
+The library consists of `microtar.c` and `microtar.h`. These two files can be
+dropped into an existing project and compiled along with it.
+
+
+#### Reading
+```c
+mtar_t tar;
+mtar_header_t h;
+char *p;
+
+/* Open archive for reading */
+mtar_open(&tar, "test.tar", "r");
+
+/* Print all file names and sizes */
+while ( (mtar_read_header(&tar, &h)) != MTAR_ENULLRECORD ) {
+  printf("%s (%d bytes)\n", h.name, h.size);
+  mtar_next(&tar);
+}
+
+/* Load and print contents of file "test.txt" */
+mtar_find(&tar, "test.txt", &h);
+p = calloc(1, h.size + 1);
+mtar_read_data(&tar, p, h.size);
+printf("%s", p);
+free(p);
+
+/* Close archive */
+mtar_close(&tar);
+```
+
+#### Writing
+```c
+mtar_t tar;
+const char *str1 = "Hello world";
+const char *str2 = "Goodbye world";
+
+/* Open archive for writing */
+mtar_open(&tar, "test.tar", "w");
+
+/* Write strings to files `test1.txt` and `test2.txt` */
+mtar_write_file_header(&tar, "test1.txt", strlen(str1));
+mtar_write_data(&tar, str1, strlen(str1));
+mtar_write_file_header(&tar, "test2.txt", strlen(str2));
+mtar_write_data(&tar, str2, strlen(str2));
+
+/* Finalize -- this needs to be the last thing done before closing */
+mtar_finalize(&tar);
+
+/* Close archive */
+mtar_close(&tar);
+```
+
+
+## Error handling
+All functions which return an `int` will return `MTAR_ESUCCESS` if the operation
+is successful. If an error occurs an error value less-than-zero will be
+returned; this value can be passed to the function `mtar_strerror()` to get its
+corresponding error string.
+
+
+## Wrapping a stream
+If you want to read or write from something other than a file, the `mtar_t`
+struct can be manually initialized with your own callback functions and a
+`stream` pointer.
+
+All callback functions are passed a pointer to the `mtar_t` struct as their
+first argument. They should return `MTAR_ESUCCESS` if the operation succeeds
+without an error, or an integer below zero if an error occurs.
+
+After the `stream` field has been set, all required callbacks have been set and
+all unused fields have been zeroset the `mtar_t` struct can be safely used with
+the microtar functions. `mtar_open` *should not* be called if the `mtar_t`
+struct was initialized manually.
+
+#### Reading
+The following callbacks should be set for reading an archive from a stream:
+
+Name    | Arguments                                | Description
+--------|------------------------------------------|---------------------------
+`read`  | `mtar_t *tar, void *data, unsigned size` | Read data from the stream
+`seek`  | `mtar_t *tar, unsigned pos`              | Set the position indicator
+`close` | `mtar_t *tar`                            | Close the stream
+
+#### Writing
+The following callbacks should be set for writing an archive to a stream:
+
+Name    | Arguments                                      | Description
+--------|------------------------------------------------|---------------------
+`write` | `mtar_t *tar, const void *data, unsigned size` | Write data to the stream
+
+
+## License
+This library is free software; you can redistribute it and/or modify it under
+the terms of the MIT license. See [LICENSE](LICENSE) for details.

third_party/microtar/src/microtar.c

@@ -0,0 +1,376 @@
+/*
+ * Copyright (c) 2017 rxi
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <string.h>
+
+#include "microtar.h"
+
+typedef struct {
+  char name[100];
+  char mode[8];
+  char owner[8];
+  char group[8];
+  char size[12];
+  char mtime[12];
+  char checksum[8];
+  char type;
+  char linkname[100];
+  char _padding[255];
+} mtar_raw_header_t;
+
+
+static unsigned round_up(unsigned n, unsigned incr) {
+  return n + (incr - n % incr) % incr;
+}
+
+
+static unsigned checksum(const mtar_raw_header_t* rh) {
+  unsigned i;
+  unsigned char *p = (unsigned char*) rh;
+  unsigned res = 256;
+  for (i = 0; i < offsetof(mtar_raw_header_t, checksum); i++) {
+    res += p[i];
+  }
+  for (i = offsetof(mtar_raw_header_t, type); i < sizeof(*rh); i++) {
+    res += p[i];
+  }
+  return res;
+}
+
+
+static int tread(mtar_t *tar, void *data, unsigned size) {
+  int err = tar->read(tar, data, size);
+  tar->pos += size;
+  return err;
+}
+
+
+static int twrite(mtar_t *tar, const void *data, unsigned size) {
+  int err = tar->write(tar, data, size);
+  tar->pos += size;
+  return err;
+}
+
+
+static int write_null_bytes(mtar_t *tar, int n) {
+  int i, err;
+  char nul = '\0';
+  for (i = 0; i < n; i++) {
+    err = twrite(tar, &nul, 1);
+    if (err) {
+      return err;
+    }
+  }
+  return MTAR_ESUCCESS;
+}
+
+
+static int raw_to_header(mtar_header_t *h, const mtar_raw_header_t *rh) {
+  unsigned chksum1, chksum2;
+
+  /* If the checksum starts with a null byte we assume the record is NULL */
+  if (*rh->checksum == '\0') {
+    return MTAR_ENULLRECORD;
+  }
+
+  /* Build and compare checksum */
+  chksum1 = checksum(rh);
+  sscanf(rh->checksum, "%o", &chksum2);
+  if (chksum1 != chksum2) {
+    return MTAR_EBADCHKSUM;
+  }
+
+  /* Load raw header into header */
+  sscanf(rh->mode, "%o", &h->mode);
+  sscanf(rh->owner, "%o", &h->owner);
+  sscanf(rh->size, "%o", &h->size);
+  sscanf(rh->mtime, "%o", &h->mtime);
+  h->type = rh->type;
+  strcpy(h->name, rh->name);
+  strcpy(h->linkname, rh->linkname);
+
+  return MTAR_ESUCCESS;
+}
+
+
+static int header_to_raw(mtar_raw_header_t *rh, const mtar_header_t *h) {
+  unsigned chksum;
+
+  /* Load header into raw header */
+  memset(rh, 0, sizeof(*rh));
+  sprintf(rh->mode, "%o", h->mode);
+  sprintf(rh->owner, "%o", h->owner);
+  sprintf(rh->size, "%o", h->size);
+  sprintf(rh->mtime, "%o", h->mtime);
+  rh->type = h->type ? h->type : MTAR_TREG;
+  strcpy(rh->name, h->name);
+  strcpy(rh->linkname, h->linkname);
+
+  /* Calculate and write checksum */
+  chksum = checksum(rh);
+  sprintf(rh->checksum, "%06o", chksum);
+  rh->checksum[7] = ' ';
+
+  return MTAR_ESUCCESS;
+}
+
+
+const char* mtar_strerror(int err) {
+  switch (err) {
+    case MTAR_ESUCCESS     : return "success";
+    case MTAR_EFAILURE     : return "failure";
+    case MTAR_EOPENFAIL    : return "could not open";
+    case MTAR_EREADFAIL    : return "could not read";
+    case MTAR_EWRITEFAIL   : return "could not write";
+    case MTAR_ESEEKFAIL    : return "could not seek";
+    case MTAR_EBADCHKSUM   : return "bad checksum";
+    case MTAR_ENULLRECORD  : return "null record";
+    case MTAR_ENOTFOUND    : return "file not found";
+  }
+  return "unknown error";
+}
+
+
+static int file_write(mtar_t *tar, const void *data, unsigned size) {
+  unsigned res = fwrite(data, 1, size, tar->stream);
+  return (res == size) ? MTAR_ESUCCESS : MTAR_EWRITEFAIL;
+}
+
+static int file_read(mtar_t *tar, void *data, unsigned size) {
+  unsigned res = fread(data, 1, size, tar->stream);
+  return (res == size) ? MTAR_ESUCCESS : MTAR_EREADFAIL;
+}
+
+static int file_seek(mtar_t *tar, unsigned offset) {
+  int res = fseek(tar->stream, offset, SEEK_SET);
+  return (res == 0) ? MTAR_ESUCCESS : MTAR_ESEEKFAIL;
+}
+
+static int file_close(mtar_t *tar) {
+  fclose(tar->stream);
+  return MTAR_ESUCCESS;
+}
+
+
+int mtar_open(mtar_t *tar, const char *filename, const char *mode) {
+  int err;
+  mtar_header_t h;
+
+  /* Init tar struct and functions */
+  memset(tar, 0, sizeof(*tar));
+  tar->write = file_write;
+  tar->read = file_read;
+  tar->seek = file_seek;
+  tar->close = file_close;
+
+  /* Assure mode is always binary */
+  if ( strchr(mode, 'r') ) mode = "rb";
+  if ( strchr(mode, 'w') ) mode = "w+b";
+  if ( strchr(mode, 'a') ) mode = "ab";
+  /* Open file */
+  tar->stream = fopen(filename, mode);
+  if (!tar->stream) {
+    return MTAR_EOPENFAIL;
+  }
+  /* Read first header to check it is valid if mode is `r` */
+  if (*mode == 'r') {
+    err = mtar_read_header(tar, &h);
+    if (err != MTAR_ESUCCESS) {
+      mtar_close(tar);
+      return err;
+    }
+  }
+
+  /* Return ok */
+  return MTAR_ESUCCESS;
+}
+
+
+int mtar_close(mtar_t *tar) {
+  return tar->close(tar);
+}
+
+
+int mtar_seek(mtar_t *tar, unsigned pos) {
+  int err = tar->seek(tar, pos);
+  tar->pos = pos;
+  return err;
+}
+
+
+int mtar_rewind(mtar_t *tar) {
+  tar->remaining_data = 0;
+  tar->last_header = 0;
+  return mtar_seek(tar, 0);
+}
+
+
+int mtar_next(mtar_t *tar) {
+  int err, n;
+  mtar_header_t h;
+  /* Load header */
+  err = mtar_read_header(tar, &h);
+  if (err) {
+    return err;
+  }
+  /* Seek to next record */
+  n = round_up(h.size, 512) + sizeof(mtar_raw_header_t);
+  return mtar_seek(tar, tar->pos + n);
+}
+
+
+int mtar_find(mtar_t *tar, const char *name, mtar_header_t *h) {
+  int err;
+  mtar_header_t header;
+  /* Start at beginning */
+  err = mtar_rewind(tar);
+  if (err) {
+    return err;
+  }
+  /* Iterate all files until we hit an error or find the file */
+  while ( (err = mtar_read_header(tar, &header)) == MTAR_ESUCCESS ) {
+    if ( !strcmp(header.name, name) ) {
+      if (h) {
+        *h = header;
+      }
+      return MTAR_ESUCCESS;
+    }
+    mtar_next(tar);
+  }
+  /* Return error */
+  if (err == MTAR_ENULLRECORD) {
+    err = MTAR_ENOTFOUND;
+  }
+  return err;
+}
+
+
+int mtar_read_header(mtar_t *tar, mtar_header_t *h) {
+  int err;
+  mtar_raw_header_t rh;
+  /* Save header position */
+  tar->last_header = tar->pos;
+  /* Read raw header */
+  err = tread(tar, &rh, sizeof(rh));
+  if (err) {
+    return err;
+  }
+  /* Seek back to start of header */
+  err = mtar_seek(tar, tar->last_header);
+  if (err) {
+    return err;
+  }
+  /* Load raw header into header struct and return */
+  return raw_to_header(h, &rh);
+}
+
+
+int mtar_read_data(mtar_t *tar, void *ptr, unsigned size) {
+  int err;
+  /* If we have no remaining data then this is the first read, we get the size,
+   * set the remaining data and seek to the beginning of the data */
+  if (tar->remaining_data == 0) {
+    mtar_header_t h;
+    /* Read header */
+    err = mtar_read_header(tar, &h);
+    if (err) {
+      return err;
+    }
+    /* Seek past header and init remaining data */
+    err = mtar_seek(tar, tar->pos + sizeof(mtar_raw_header_t));
+    if (err) {
+      return err;
+    }
+    tar->remaining_data = h.size;
+  }
+  /* Read data */
+  err = tread(tar, ptr, size);
+  if (err) {
+    return err;
+  }
+  tar->remaining_data -= size;
+  /* If there is no remaining data we've finished reading and seek back to the
+   * header */
+  if (tar->remaining_data == 0) {
+    return mtar_seek(tar, tar->last_header);
+  }
+  return MTAR_ESUCCESS;
+}
+
+
+int mtar_write_header(mtar_t *tar, const mtar_header_t *h) {
+  mtar_raw_header_t rh;
+  /* Build raw header and write */
+  header_to_raw(&rh, h);
+  tar->remaining_data = h->size;
+  return twrite(tar, &rh, sizeof(rh));
+}
+
+
+int mtar_write_file_header(mtar_t *tar, const char *name, unsigned size) {
+  mtar_header_t h;
+  /* Build header */
+  memset(&h, 0, sizeof(h));
+  strcpy(h.name, name);
+  h.size = size;
+  h.type = MTAR_TREG;
+  h.mode = 0664;
+  /* Write header */
+  return mtar_write_header(tar, &h);
+}
+
+
+int mtar_write_dir_header(mtar_t *tar, const char *name) {
+  mtar_header_t h;
+  /* Build header */
+  memset(&h, 0, sizeof(h));
+  strcpy(h.name, name);
+  h.type = MTAR_TDIR;
+  h.mode = 0775;
+  /* Write header */
+  return mtar_write_header(tar, &h);
+}
+
+
+int mtar_write_data(mtar_t *tar, const void *data, unsigned size) {
+  int err;
+  /* Write data */
+  err = twrite(tar, data, size);
+  if (err) {
+    return err;
+  }
+  tar->remaining_data -= size;
+  /* Write padding if we've written all the data for this file */
+  if (tar->remaining_data == 0) {
+    return write_null_bytes(tar, round_up(tar->pos, 512) - tar->pos);
+  }
+  return MTAR_ESUCCESS;
+}
+
+
+int mtar_finalize(mtar_t *tar) {
+  /* Write two NULL records */
+  return write_null_bytes(tar, sizeof(mtar_raw_header_t) * 2);
+}

third_party/microtar/src/microtar.h

@@ -0,0 +1,82 @@
+/**
+ * Copyright (c) 2017 rxi
+ *
+ * This library is free software; you can redistribute it and/or modify it
+ * under the terms of the MIT license. See `microtar.c` for details.
+ */
+
+#ifndef MICROTAR_H
+#define MICROTAR_H
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define MTAR_VERSION "0.1.0"
+
+enum {
+  MTAR_ESUCCESS     =  0,
+  MTAR_EFAILURE     = -1,
+  MTAR_EOPENFAIL    = -2,
+  MTAR_EREADFAIL    = -3,
+  MTAR_EWRITEFAIL   = -4,
+  MTAR_ESEEKFAIL    = -5,
+  MTAR_EBADCHKSUM   = -6,
+  MTAR_ENULLRECORD  = -7,
+  MTAR_ENOTFOUND    = -8
+};
+
+enum {
+  MTAR_TREG   = '0',
+  MTAR_TLNK   = '1',
+  MTAR_TSYM   = '2',
+  MTAR_TCHR   = '3',
+  MTAR_TBLK   = '4',
+  MTAR_TDIR   = '5',
+  MTAR_TFIFO  = '6'
+};
+
+typedef struct {
+  unsigned mode;
+  unsigned owner;
+  unsigned size;
+  unsigned mtime;
+  unsigned type;
+  char name[100];
+  char linkname[100];
+} mtar_header_t;
+
+
+typedef struct mtar_t mtar_t;
+
+struct mtar_t {
+  int (*read)(mtar_t *tar, void *data, unsigned size);
+  int (*write)(mtar_t *tar, const void *data, unsigned size);
+  int (*seek)(mtar_t *tar, unsigned pos);
+  int (*close)(mtar_t *tar);
+  void *stream;
+  unsigned pos;
+  unsigned remaining_data;
+  unsigned last_header;
+};
+
+
+const char* mtar_strerror(int err);
+
+int mtar_open(mtar_t *tar, const char *filename, const char *mode);
+int mtar_close(mtar_t *tar);
+
+int mtar_seek(mtar_t *tar, unsigned pos);
+int mtar_rewind(mtar_t *tar);
+int mtar_next(mtar_t *tar);
+int mtar_find(mtar_t *tar, const char *name, mtar_header_t *h);
+int mtar_read_header(mtar_t *tar, mtar_header_t *h);
+int mtar_read_data(mtar_t *tar, void *ptr, unsigned size);
+
+int mtar_write_header(mtar_t *tar, const mtar_header_t *h);
+int mtar_write_file_header(mtar_t *tar, const char *name, unsigned size);
+int mtar_write_dir_header(mtar_t *tar, const char *name);
+int mtar_write_data(mtar_t *tar, const void *data, unsigned size);
+int mtar_finalize(mtar_t *tar);
+
+
+#endif

unit_tests/CMakeLists.txt

@@ -22,6 +22,10 @@ file(GLOB UpdaterTestsSources
     updater_tests.cpp
     updater/*.cpp)
 
+file(GLOB StorageTestsSources
+    storage_tests.cpp
+    storage/*.cpp)
+
 file(GLOB LibraryTestsSources
     library_tests.cpp
     library/*.cpp)
@@ -85,6 +89,11 @@ add_executable(updater-tests
     ${UpdaterTestsSources}
     $<TARGET_OBJECTS:UPDATER> $<TARGET_OBJECTS:UPDATER> $<TARGET_OBJECTS:UTIL>)
 
+add_executable(storage-tests
+	EXCLUDE_FROM_ALL
+    ${StorageTestsSources}
+    $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:UTIL>)
+
 add_executable(library-tests
 	EXCLUDE_FROM_ALL
 	${LibraryTestsSources})
@@ -136,6 +145,7 @@ target_compile_definitions(library-customize-tests PRIVATE COMPILE_DEFINITIONS O
 target_compile_definitions(library-partition-tests PRIVATE COMPILE_DEFINITIONS OSRM_TEST_DATA_DIR="${TEST_DATA_DIR}")
 target_compile_definitions(updater-tests PRIVATE COMPILE_DEFINITIONS TEST_DATA_DIR="${UPDATER_TEST_DATA_DIR}")
 target_compile_definitions(contractor-tests PRIVATE COMPILE_DEFINITIONS TEST_DATA_DIR="${TEST_DATA_DIR}")
+target_compile_definitions(storage-tests PRIVATE COMPILE_DEFINITIONS TEST_DATA_DIR="${TEST_DATA_DIR}")
 
 target_include_directories(engine-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_include_directories(library-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
@@ -148,20 +158,22 @@ target_include_directories(partitioner-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_include_directories(customizer-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_include_directories(updater-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_include_directories(contractor-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+target_include_directories(storage-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 
-target_link_libraries(engine-tests ${ENGINE_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
+target_link_libraries(engine-tests ${ENGINE_LIBRARIES} $<TARGET_OBJECTS:MICROTAR> ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(extractor-tests osrm_extract osrm_guidance ${EXTRACTOR_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
-target_link_libraries(partitioner-tests ${PARTITIONER_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
-target_link_libraries(customizer-tests ${CUSTOMIZER_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
-target_link_libraries(updater-tests ${UPDATER_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
+target_link_libraries(partitioner-tests osrm_partition ${PARTITIONER_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
+target_link_libraries(customizer-tests osrm_customize ${CUSTOMIZER_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
+target_link_libraries(updater-tests osrm_update ${UPDATER_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(library-tests osrm ${ENGINE_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(library-extract-tests osrm_extract osrm_guidance ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(library-contract-tests osrm_contract ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(library-customize-tests osrm_customize ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(library-partition-tests osrm_partition ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(server-tests osrm ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
-target_link_libraries(util-tests ${UTIL_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
-target_link_libraries(contractor-tests ${CONTRACTOR_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
+target_link_libraries(util-tests ${UTIL_LIBRARIES} $<TARGET_OBJECTS:MICROTAR> ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
+target_link_libraries(contractor-tests osrm_contract ${CONTRACTOR_LIBRARIES} $<TARGET_OBJECTS:MICROTAR> ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
+target_link_libraries(storage-tests osrm_store ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 
 add_custom_target(tests
-	DEPENDS engine-tests extractor-tests contractor-tests partitioner-tests updater-tests customizer-tests library-tests library-extract-tests library-contract-tests library-customize-tests library-partition-tests server-tests util-tests)
+	DEPENDS engine-tests extractor-tests contractor-tests partitioner-tests updater-tests customizer-tests library-tests library-extract-tests library-contract-tests library-customize-tests library-partition-tests server-tests util-tests storage-tests)

unit_tests/common/temporary_file.hpp

@@ -6,6 +6,7 @@
 struct TemporaryFile
 {
     TemporaryFile() : path(boost::filesystem::unique_path()) {}
+    TemporaryFile(const std::string &path) : path(path) {}
 
     ~TemporaryFile() { boost::filesystem::remove(path); }
 

unit_tests/contractor/files.cpp

@@ -0,0 +1,57 @@
+#include "contractor/files.hpp"
+#include "contractor/graph_contractor_adaptors.hpp"
+
+#include "../common/range_tools.hpp"
+#include "../common/temporary_file.hpp"
+#include "helper.hpp"
+
+#include <boost/test/unit_test.hpp>
+
+BOOST_AUTO_TEST_SUITE(tar)
+
+using namespace osrm;
+using namespace osrm::contractor;
+using namespace osrm::unit_test;
+
+BOOST_AUTO_TEST_CASE(read_write_hsgr)
+{
+    auto reference_checksum = 0xFF00FF00;
+    auto reference_connectivity_checksum = 0xDEADBEEF;
+    std::vector<TestEdge> edges = {TestEdge{0, 1, 3},
+                                   TestEdge{0, 5, 1},
+                                   TestEdge{1, 3, 3},
+                                   TestEdge{1, 4, 1},
+                                   TestEdge{3, 1, 1},
+                                   TestEdge{4, 3, 1},
+                                   TestEdge{5, 1, 1}};
+    auto reference_graph = QueryGraph{6, toEdges<QueryEdge>(makeGraph(edges))};
+    std::vector<std::vector<bool>> reference_filters = {
+        {false, false, true, true, false, false, true},
+        {true, false, true, false, true, false, true},
+        {false, false, false, false, false, false, false},
+        {true, true, true, true, true, true, true},
+    };
+
+    TemporaryFile tmp{TEST_DATA_DIR "/read_write_hsgr_test.osrm.hsgr"};
+    contractor::files::writeGraph(tmp.path,
+                                  reference_checksum,
+                                  reference_graph,
+                                  reference_filters,
+                                  reference_connectivity_checksum);
+
+    unsigned checksum;
+    unsigned connectivity_checksum;
+    QueryGraph graph;
+    std::vector<std::vector<bool>> filters;
+    contractor::files::readGraph(tmp.path, checksum, graph, filters, connectivity_checksum);
+
+    BOOST_CHECK_EQUAL(checksum, reference_checksum);
+    BOOST_CHECK_EQUAL(connectivity_checksum, reference_connectivity_checksum);
+    BOOST_CHECK_EQUAL(filters.size(), reference_filters.size());
+    CHECK_EQUAL_COLLECTIONS(filters[0], reference_filters[0]);
+    CHECK_EQUAL_COLLECTIONS(filters[1], reference_filters[1]);
+    CHECK_EQUAL_COLLECTIONS(filters[2], reference_filters[2]);
+    CHECK_EQUAL_COLLECTIONS(filters[3], reference_filters[3]);
+}
+
+BOOST_AUTO_TEST_SUITE_END()

unit_tests/contractor/graph_contractor.cpp

@@ -1,6 +1,7 @@
 #include "contractor/graph_contractor.hpp"
 
 #include "../common/range_tools.hpp"
+#include "helper.hpp"
 
 #include <boost/test/test_case_template.hpp>
 #include <boost/test/unit_test.hpp>
@@ -9,32 +10,10 @@
 
 using namespace osrm;
 using namespace osrm::contractor;
+using namespace osrm::unit_test;
 
 BOOST_AUTO_TEST_SUITE(graph_contractor)
 
-using TestEdge = std::tuple<unsigned, unsigned, int>;
-ContractorGraph makeGraph(const std::vector<TestEdge> &edges)
-{
-    std::vector<ContractorEdge> input_edges;
-    auto id = 0u;
-    auto max_id = 0u;
-    for (const auto &edge : edges)
-    {
-        unsigned start;
-        unsigned target;
-        int weight;
-        std::tie(start, target, weight) = edge;
-        max_id = std::max(std::max(start, target), max_id);
-        input_edges.push_back(ContractorEdge{
-            start, target, ContractorEdgeData{weight, weight * 2, id++, 0, false, true, false}});
-        input_edges.push_back(ContractorEdge{
-            target, start, ContractorEdgeData{weight, weight * 2, id++, 0, false, false, true}});
-    }
-    std::sort(input_edges.begin(), input_edges.end());
-
-    return ContractorGraph{max_id + 1, std::move(input_edges)};
-}
-
 BOOST_AUTO_TEST_CASE(contract_graph)
 {
     tbb::task_scheduler_init scheduler(1);

unit_tests/contractor/helper.hpp

@@ -0,0 +1,40 @@
+#ifndef OSRM_UNIT_TESTS_CONTRACTOR_HELPER_HPP_
+#define OSRM_UNIT_TESTS_CONTRACTOR_HELPER_HPP_
+
+#include "contractor/contractor_graph.hpp"
+
+namespace osrm
+{
+namespace unit_test
+{
+
+using TestEdge = std::tuple<unsigned, unsigned, int>;
+inline contractor::ContractorGraph makeGraph(const std::vector<TestEdge> &edges)
+{
+    std::vector<contractor::ContractorEdge> input_edges;
+    auto id = 0u;
+    auto max_id = 0u;
+    for (const auto &edge : edges)
+    {
+        unsigned start;
+        unsigned target;
+        int weight;
+        std::tie(start, target, weight) = edge;
+        max_id = std::max(std::max(start, target), max_id);
+        input_edges.push_back(contractor::ContractorEdge{
+            start,
+            target,
+            contractor::ContractorEdgeData{weight, weight * 2, id++, 0, false, true, false}});
+        input_edges.push_back(contractor::ContractorEdge{
+            target,
+            start,
+            contractor::ContractorEdgeData{weight, weight * 2, id++, 0, false, false, true}});
+    }
+    std::sort(input_edges.begin(), input_edges.end());
+
+    return contractor::ContractorGraph{max_id + 1, std::move(input_edges)};
+}
+}
+}
+
+#endif