Move .osrm file to tar format

CMakeLists.txt

@@ -700,7 +700,7 @@ target_link_libraries(osrm_customize ${CUSTOMIZER_LIBRARIES} osrm_update osrm_st
 target_link_libraries(osrm_store ${STORAGE_LIBRARIES})
 
 # BUILD_COMPONENTS
-add_executable(osrm-components src/tools/components.cpp $<TARGET_OBJECTS:UTIL>)
+add_executable(osrm-components src/tools/components.cpp $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
 target_link_libraries(osrm-components ${TBB_LIBRARIES} ${BOOST_BASE_LIBRARIES} ${UTIL_LIBRARIES})
 install(TARGETS osrm-components DESTINATION bin)
 

include/extractor/extraction_containers.hpp

@@ -7,7 +7,7 @@
 #include "extractor/restriction.hpp"
 #include "extractor/scripting_environment.hpp"
 
-#include "storage/io.hpp"
+#include "storage/tar_fwd.hpp"
 
 namespace osrm
 {
@@ -27,9 +27,9 @@ class ExtractionContainers
     void PrepareRestrictions();
     void PrepareEdges(ScriptingEnvironment &scripting_environment);
 
-    void WriteNodes(storage::io::FileWriter &file_out) const;
+    void WriteNodes(storage::tar::FileWriter &file_out) const;
-    void WriteEdges(storage::io::FileWriter &file_out) const;
+    void WriteEdges(storage::tar::FileWriter &file_out) const;
-    void WriteMetadata(storage::io::FileWriter &file_out) const;
+    void WriteMetadata(storage::tar::FileWriter &file_out) const;
     void WriteCharData(const std::string &file_name);
 
   public:

include/extractor/files.hpp

@@ -4,6 +4,7 @@
 #include "extractor/edge_based_edge.hpp"
 #include "extractor/node_data_container.hpp"
 #include "extractor/profile_properties.hpp"
+#include "extractor/query_node.hpp"
 #include "extractor/serialization.hpp"
 #include "extractor/turn_lane_types.hpp"
 
@@ -368,7 +369,7 @@ inline void readTurnDurationPenalty(const boost::filesystem::path &path, TurnPen
 // writes .osrm.restrictions
 template <typename ConditionalRestrictionsT>
 inline void writeConditionalRestrictions(const boost::filesystem::path &path,
-                                     const ConditionalRestrictionsT &conditional_restrictions)
+                                         const ConditionalRestrictionsT &conditional_restrictions)
 {
     const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
     storage::tar::FileWriter writer{path, fingerprint};
@@ -378,13 +379,47 @@ inline void writeConditionalRestrictions(const boost::filesystem::path &path,
 
 // read .osrm.restrictions
 template <typename ConditionalRestrictionsT>
-inline void readConditionalRestrictions(const boost::filesystem::path &path, ConditionalRestrictionsT &conditional_restrictions)
+inline void readConditionalRestrictions(const boost::filesystem::path &path,
+                                        ConditionalRestrictionsT &conditional_restrictions)
 {
     const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
     storage::tar::FileReader reader{path, fingerprint};
 
     serialization::read(reader, "/common/conditional_restrictions", conditional_restrictions);
 }
+
+// reads .osrm file which is a temporary file of osrm-extract
+template <typename BarrierOutIter, typename TrafficSignalsOutIter, typename PackedOSMIDsT>
+void readRawNBGraph(const boost::filesystem::path &path,
+                    BarrierOutIter barriers,
+                    TrafficSignalsOutIter traffic_signals,
+                    std::vector<util::Coordinate> &coordinates,
+                    PackedOSMIDsT &osm_node_ids,
+                    std::vector<extractor::NodeBasedEdge> &edge_list,
+                    std::vector<extractor::NodeBasedEdgeAnnotation> &annotations)
+{
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
+
+    auto number_of_nodes = reader.ReadElementCount64("/extractor/nodes");
+    coordinates.resize(number_of_nodes);
+    osm_node_ids.reserve(number_of_nodes);
+    auto index = 0;
+    auto decode = [&](const auto &current_node) {
+        coordinates[index].lon = current_node.lon;
+        coordinates[index].lat = current_node.lat;
+        osm_node_ids.push_back(current_node.node_id);
+        index++;
+    };
+    reader.ReadStreaming<extractor::QueryNode>("/extractor/nodes", boost::make_function_output_iterator(decode));
+
+    reader.ReadStreaming<NodeID>("/extractor/barriers", barriers);
+
+    reader.ReadStreaming<NodeID>("/extractor/traffic_lights", traffic_signals);
+
+    storage::serialization::read(reader, "/extractor/edges", edge_list);
+    storage::serialization::read(reader, "/extractor/annotations", annotations);
+}
 }
 }
 }

include/util/graph_loader.hpp

@@ -1,127 +0,0 @@
-#ifndef GRAPH_LOADER_HPP
-#define GRAPH_LOADER_HPP
-
-#include "extractor/node_based_edge.hpp"
-#include "extractor/packed_osm_ids.hpp"
-#include "extractor/query_node.hpp"
-#include "extractor/restriction.hpp"
-#include "storage/io.hpp"
-#include "util/exception.hpp"
-#include "util/fingerprint.hpp"
-#include "util/log.hpp"
-#include "util/packed_vector.hpp"
-#include "util/typedefs.hpp"
-
-#include <boost/assert.hpp>
-#include <boost/filesystem.hpp>
-#include <boost/filesystem/fstream.hpp>
-#include <boost/function_output_iterator.hpp>
-
-#include <tbb/parallel_sort.h>
-
-#include <cmath>
-
-#include <fstream>
-#include <ios>
-#include <unordered_set>
-#include <vector>
-
-namespace osrm
-{
-namespace util
-{
-
-/**
- * Reads the beginning of an .osrm file and produces:
- *  - barrier nodes
- *  - traffic lights
- *  - nodes indexed by their internal (non-osm) id
- */
-template <typename BarrierOutIter, typename TrafficSignalsOutIter>
-NodeID loadNodesFromFile(storage::io::FileReader &file_reader,
-                         BarrierOutIter barriers,
-                         TrafficSignalsOutIter traffic_signals,
-                         std::vector<util::Coordinate> &coordinates,
-                         extractor::PackedOSMIDs &osm_node_ids)
-{
-    auto number_of_nodes = file_reader.ReadElementCount64();
-    Log() << "Importing number_of_nodes new = " << number_of_nodes << " nodes ";
-
-    coordinates.resize(number_of_nodes);
-    osm_node_ids.reserve(number_of_nodes);
-
-    auto index = 0;
-    auto decode = [&](const extractor::QueryNode &current_node) {
-        coordinates[index].lon = current_node.lon;
-        coordinates[index].lat = current_node.lat;
-        osm_node_ids.push_back(current_node.node_id);
-        index++;
-    };
-
-    file_reader.ReadStreaming<extractor::QueryNode>(boost::make_function_output_iterator(decode),
-                                                    number_of_nodes);
-
-    auto num_barriers = file_reader.ReadElementCount64();
-    file_reader.ReadStreaming<NodeID>(barriers, num_barriers);
-
-    auto num_lights = file_reader.ReadElementCount64();
-    file_reader.ReadStreaming<NodeID>(traffic_signals, num_lights);
-
-    return number_of_nodes;
-}
-
-/**
- * Reads a .osrm file and produces the edges.
- */
-inline EdgeID loadEdgesFromFile(storage::io::FileReader &file_reader,
-                                std::vector<extractor::NodeBasedEdge> &edge_list)
-{
-    auto number_of_edges = file_reader.ReadElementCount64();
-
-    edge_list.resize(number_of_edges);
-    Log() << " and " << number_of_edges << " edges ";
-
-    file_reader.ReadInto(edge_list.data(), number_of_edges);
-
-    BOOST_ASSERT(edge_list.size() > 0);
-
-#ifndef NDEBUG
-    Log() << "Validating loaded edges...";
-    tbb::parallel_sort(
-        edge_list.begin(),
-        edge_list.end(),
-        [](const extractor::NodeBasedEdge &lhs, const extractor::NodeBasedEdge &rhs) {
-            return (lhs.source < rhs.source) ||
-                   (lhs.source == rhs.source && lhs.target < rhs.target);
-        });
-    for (auto i = 1u; i < edge_list.size(); ++i)
-    {
-        const auto &edge = edge_list[i];
-        const auto &prev_edge = edge_list[i - 1];
-
-        BOOST_ASSERT_MSG(edge.weight > 0, "loaded null weight");
-        BOOST_ASSERT_MSG(edge.flags.forward, "edge must be oriented in forward direction");
-
-        BOOST_ASSERT_MSG(edge.source != edge.target, "loaded edges contain a loop");
-        BOOST_ASSERT_MSG(edge.source != prev_edge.source || edge.target != prev_edge.target,
-                         "loaded edges contain a multi edge");
-    }
-#endif
-
-    Log() << "Graph loaded ok and has " << edge_list.size() << " edges";
-
-    return number_of_edges;
-}
-
-inline EdgeID loadAnnotationData(storage::io::FileReader &file_reader,
-                                 std::vector<extractor::NodeBasedEdgeAnnotation> &metadata)
-{
-    auto const meta_data_count = file_reader.ReadElementCount64();
-    metadata.resize(meta_data_count);
-    file_reader.ReadInto(metadata.data(), meta_data_count);
-    return meta_data_count;
-}
-}
-}
-
-#endif // GRAPH_LOADER_HPP

src/contractor/contractor.cpp

@@ -19,7 +19,6 @@
 #include "util/exception_utils.hpp"
 #include "util/exclude_flag.hpp"
 #include "util/filtered_graph.hpp"
-#include "util/graph_loader.hpp"
 #include "util/integer_range.hpp"
 #include "util/log.hpp"
 #include "util/static_graph.hpp"

src/extractor/extraction_containers.cpp

@@ -131,15 +131,15 @@ void ExtractionContainers::PrepareData(ScriptingEnvironment &scripting_environme
                                        const std::string &osrm_path,
                                        const std::string &name_file_name)
 {
-    storage::io::FileWriter file_out(osrm_path, storage::io::FileWriter::GenerateFingerprint);
+    storage::tar::FileWriter writer(osrm_path, storage::tar::FileWriter::GenerateFingerprint);
 
     PrepareNodes();
-    WriteNodes(file_out);
+    WriteNodes(writer);
     PrepareEdges(scripting_environment);
     all_nodes_list.clear(); // free all_nodes_list before allocation of normal_edges
     all_nodes_list.shrink_to_fit();
-    WriteEdges(file_out);
+    WriteEdges(writer);
-    WriteMetadata(file_out);
+    WriteMetadata(writer);
 
     /* Sort these so that searching is a bit faster later on */
     {
@@ -276,8 +276,8 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm
         {
             if (edge_iterator->result.osm_source_id < node_iterator->node_id)
             {
-                util::Log(logDEBUG) << "Found invalid node reference "
+                util::Log(logDEBUG)
-                                    << edge_iterator->result.source;
+                    << "Found invalid node reference " << edge_iterator->result.source;
                 edge_iterator->result.source = SPECIAL_NODEID;
                 ++edge_iterator;
                 continue;
@@ -531,7 +531,7 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm
     }
 }
 
-void ExtractionContainers::WriteEdges(storage::io::FileWriter &file_out) const
+void ExtractionContainers::WriteEdges(storage::tar::FileWriter &writer) const
 {
     std::vector<NodeBasedEdge> normal_edges;
     normal_edges.reserve(all_edges_list.size());
@@ -558,8 +558,7 @@ void ExtractionContainers::WriteEdges(storage::io::FileWriter &file_out) const
             throw util::exception("There are too many edges, OSRM only supports 2^32" + SOURCE_REF);
         }
 
-        file_out.WriteElementCount64(normal_edges.size());
+        storage::serialization::write(writer, "/extractor/edges", normal_edges);
-        file_out.WriteFrom(normal_edges.data(), normal_edges.size());
 
         TIMER_STOP(write_edges);
         log << "ok, after " << TIMER_SEC(write_edges) << "s";
@@ -567,31 +566,21 @@ void ExtractionContainers::WriteEdges(storage::io::FileWriter &file_out) const
     }
 }
 
-void ExtractionContainers::WriteMetadata(storage::io::FileWriter &file_out) const
+void ExtractionContainers::WriteMetadata(storage::tar::FileWriter &writer) const
 {
     util::UnbufferedLog log;
     log << "Writing way meta-data     ... " << std::flush;
     TIMER_START(write_meta_data);
 
-    file_out.WriteElementCount64(all_edges_annotation_data_list.size());
+    storage::serialization::write(writer, "/extractor/annotations", all_edges_annotation_data_list);
-    file_out.WriteFrom(all_edges_annotation_data_list.data(),
-                       all_edges_annotation_data_list.size());
 
     TIMER_STOP(write_meta_data);
     log << "ok, after " << TIMER_SEC(write_meta_data) << "s";
     log << " -- Metadata contains << " << all_edges_annotation_data_list.size() << " entries.";
 }
 
-void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const
+void ExtractionContainers::WriteNodes(storage::tar::FileWriter &writer) const
 {
-    {
-        // write dummy value, will be overwritten later
-        util::UnbufferedLog log;
-        log << "setting number of nodes   ... " << std::flush;
-        file_out.WriteElementCount64(max_internal_node_id);
-        log << "ok";
-    }
-
     {
         util::UnbufferedLog log;
         log << "Confirming/Writing used nodes     ... ";
@@ -599,28 +588,35 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const
         // identify all used nodes by a merging step of two sorted lists
         auto node_iterator = all_nodes_list.begin();
         auto node_id_iterator = used_node_id_list.begin();
-        const auto used_node_id_list_end = used_node_id_list.end();
         const auto all_nodes_list_end = all_nodes_list.end();
 
-        while (node_id_iterator != used_node_id_list_end && node_iterator != all_nodes_list_end)
+        const auto encode = [&]() {
-        {
+            BOOST_ASSERT(node_id_iterator != used_node_id_list.end());
-            if (*node_id_iterator < node_iterator->node_id)
+            BOOST_ASSERT(node_iterator != all_nodes_list_end);
-            {
+            BOOST_ASSERT(*node_id_iterator >= node_iterator->node_id);
-                ++node_id_iterator;
+            while (*node_id_iterator > node_iterator->node_id &&
-                continue;
+                   node_iterator != all_nodes_list_end)
-            }
-            if (*node_id_iterator > node_iterator->node_id)
             {
                 ++node_iterator;
-                continue;
+            }
+            if (node_iterator == all_nodes_list_end || *node_id_iterator < node_iterator->node_id)
+            {
+                throw util::exception(
+                    "Invalid OSM data: Referenced non-existing node with ID " +
+                    std::to_string(static_cast<std::uint64_t>(*node_id_iterator)));
             }
             BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
 
-            file_out.WriteFrom((*node_iterator));
-
             ++node_id_iterator;
-            ++node_iterator;
+            return *node_iterator++;
-        }
+        };
+
+        writer.WriteElementCount64("/extractor/nodes", used_node_id_list.size());
+        writer.WriteStreaming<QueryNode>(
+            "/extractor/nodes",
+            boost::make_function_input_iterator(encode, boost::infinite()),
+            used_node_id_list.size());
+
         TIMER_STOP(write_nodes);
         log << "ok, after " << TIMER_SEC(write_nodes) << "s";
     }
@@ -639,7 +635,7 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const
                 internal_barrier_nodes.push_back(node_id);
             }
         }
-        storage::serialization::write(file_out, internal_barrier_nodes);
+        storage::serialization::write(writer, "/extractor/barriers", internal_barrier_nodes);
         log << "ok, after " << TIMER_SEC(write_nodes) << "s";
     }
 
@@ -657,7 +653,8 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const
                 internal_traffic_signals.push_back(node_id);
             }
         }
-        storage::serialization::write(file_out, internal_traffic_signals);
+        storage::serialization::write(
+            writer, "/extractor/traffic_lights", internal_traffic_signals);
         log << "ok, after " << TIMER_SEC(write_nodes) << "s";
     }
 
@@ -1035,8 +1032,9 @@ void ExtractionContainers::PrepareRestrictions()
     // translate the turn from one segment onto another into a node restriction (the ways can
     // only
     // be connected at a single location)
-    auto const get_node_restriction_from_OSM_ids = [&](
+    auto const get_node_restriction_from_OSM_ids = [&](auto const from_id,
-        auto const from_id, auto const to_id, const OSMNodeID via_node) {
+                                                       auto const to_id,
+                                                       const OSMNodeID via_node) {
         auto const from_segment_itr = referenced_ways.find(from_id);
         if (from_segment_itr->second.way_id != from_id)
         {

src/extractor/extractor.cpp

@@ -23,7 +23,6 @@
 
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
-#include "util/graph_loader.hpp"
 #include "util/integer_range.hpp"
 #include "util/log.hpp"
 #include "util/name_table.hpp"

src/extractor/node_based_graph_factory.cpp

@@ -1,7 +1,7 @@
 #include "extractor/node_based_graph_factory.hpp"
 #include "extractor/graph_compressor.hpp"
+#include "extractor/files.hpp"
 #include "storage/io.hpp"
-#include "util/graph_loader.hpp"
 
 #include "util/log.hpp"
 #include "util/timing_util.hpp"
@@ -34,28 +34,19 @@ NodeBasedGraphFactory::NodeBasedGraphFactory(
 // load the data serialised during the extraction run
 void NodeBasedGraphFactory::LoadDataFromFile(const boost::filesystem::path &input_file)
 {
-    // the extraction_containers serialise all data necessary to create the node-based graph into a
-    // single file, the *.osrm file. It contains nodes, basic information about which of these nodes
-    // are traffic signals/stop signs. It also contains Edges and purely annotative meta-data
-    storage::io::FileReader file_reader(input_file, storage::io::FileReader::VerifyFingerprint);
-
     auto barriers_iter = inserter(barriers, end(barriers));
     auto traffic_signals_iter = inserter(traffic_signals, end(traffic_signals));
-
-    const auto number_of_node_based_nodes = util::loadNodesFromFile(
-        file_reader, barriers_iter, traffic_signals_iter, coordinates, osm_node_ids);
-
     std::vector<NodeBasedEdge> edge_list;
-    util::loadEdgesFromFile(file_reader, edge_list);
 
+    files::readRawNBGraph(input_file, barriers_iter, traffic_signals_iter, coordinates, osm_node_ids, edge_list, annotation_data);
+
+    const auto number_of_node_based_nodes = coordinates.size();
     if (edge_list.empty())
     {
         throw util::exception("Node-based-graph (" + input_file.string() + ") contains no edges." +
                               SOURCE_REF);
     }
 
-    util::loadAnnotationData(file_reader, annotation_data);
-
     // at this point, the data isn't compressed, but since we update the graph in-place, we assign
     // it here.
     compressed_output_graph =

src/tools/components.cpp

@@ -1,9 +1,11 @@
+#include "extractor/files.hpp"
+#include "extractor/packed_osm_ids.hpp"
 #include "extractor/tarjan_scc.hpp"
+
 #include "util/coordinate.hpp"
 #include "util/coordinate_calculation.hpp"
 #include "util/dynamic_graph.hpp"
 #include "util/fingerprint.hpp"
-#include "util/graph_loader.hpp"
 #include "util/log.hpp"
 #include "util/static_graph.hpp"
 #include "util/typedefs.hpp"
@@ -36,16 +38,13 @@ std::size_t loadGraph(const std::string &path,
                       extractor::PackedOSMIDs &osm_node_ids,
                       std::vector<TarjanEdge> &graph_edge_list)
 {
-    storage::io::FileReader file_reader(path, storage::io::FileReader::VerifyFingerprint);
-
     std::vector<extractor::NodeBasedEdge> edge_list;
+    std::vector<extractor::NodeBasedEdgeAnnotation> annotation_data;
 
     auto nop = boost::make_function_output_iterator([](auto) {});
 
-    const auto number_of_nodes =
+    extractor::files::readRawNBGraph(
-        util::loadNodesFromFile(file_reader, nop, nop, coordinate_list, osm_node_ids);
+        path, nop, nop, coordinate_list, osm_node_ids, edge_list, annotation_data);
-
-    util::loadEdgesFromFile(file_reader, edge_list);
 
     // Building a node-based graph
     for (const auto &input_edge : edge_list)
@@ -66,7 +65,7 @@ std::size_t loadGraph(const std::string &path,
         }
     }
 
-    return number_of_nodes;
+    return osm_node_ids.size();
 }
 
 struct FeatureWriter

src/updater/updater.cpp

@@ -16,7 +16,6 @@
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
 #include "util/for_each_pair.hpp"
-#include "util/graph_loader.hpp"
 #include "util/integer_range.hpp"
 #include "util/log.hpp"
 #include "util/opening_hours.hpp"