diff --git a/include/extractor/extraction_containers.hpp b/include/extractor/extraction_containers.hpp index b630763fb..b4ff21aa1 100644 --- a/include/extractor/extraction_containers.hpp +++ b/include/extractor/extraction_containers.hpp @@ -43,8 +43,8 @@ class ExtractionContainers void PrepareTrafficSignals(const ReferencedTrafficSignals &referenced_traffic_signals); void PrepareEdges(ScriptingEnvironment &scripting_environment); - void WriteNodes(storage::tar::FileWriter &file_out) const; - void WriteEdges(storage::tar::FileWriter &file_out) const; + void WriteNodes(storage::tar::FileWriter &file_out); + void WriteEdges(storage::tar::FileWriter &file_out); void WriteMetadata(storage::tar::FileWriter &file_out) const; void WriteCharData(const std::string &file_name); @@ -75,6 +75,8 @@ class ExtractionContainers std::vector external_traffic_signals; TrafficSignals internal_traffic_signals; + std::vector normal_edges; + // List of restrictions (conditional and unconditional) before we transform them into the // output types. Input containers reference OSMNodeIDs. We can only transform them to the // correct internal IDs after we've read everything. Without a multi-parse approach, @@ -84,6 +86,7 @@ class ExtractionContainers std::vector external_maneuver_overrides_list; std::vector internal_maneuver_overrides; + std::unordered_set internal_barrier_nodes; ExtractionContainers(); diff --git a/include/extractor/extractor.hpp b/include/extractor/extractor.hpp index fc103fe69..bb5403302 100644 --- a/include/extractor/extractor.hpp +++ b/include/extractor/extractor.hpp @@ -66,7 +66,12 @@ class Extractor std::tuple, std::vector, - TrafficSignals> + TrafficSignals, + std::unordered_set, + std::vector, + extractor::PackedOSMIDs, + std::vector, + std::vector> ParseOSMData(ScriptingEnvironment &scripting_environment, const unsigned number_of_threads); EdgeID BuildEdgeExpandedGraph( diff --git a/include/extractor/node_based_graph_factory.hpp b/include/extractor/node_based_graph_factory.hpp index 9e07513c2..59363a5dc 100644 --- a/include/extractor/node_based_graph_factory.hpp +++ b/include/extractor/node_based_graph_factory.hpp @@ -38,11 +38,15 @@ class NodeBasedGraphFactory // node-based graph to represent the OSM network. This includes geometry compression, annotation // data optimisation and many other aspects. After this step, the edge-based graph factory can // turn the graph into the routing graph to be used with the navigation algorithms. - NodeBasedGraphFactory(const boost::filesystem::path &input_file, - ScriptingEnvironment &scripting_environment, + NodeBasedGraphFactory(ScriptingEnvironment &scripting_environment, std::vector &turn_restrictions, std::vector &maneuver_overrides, - const TrafficSignals &traffic_signals); + const TrafficSignals &traffic_signals, + std::unordered_set barriers, + std::vector coordinates, + extractor::PackedOSMIDs osm_node_ids, + const std::vector &edge_list, + std::vector annotation_data); auto const &GetGraph() const { return compressed_output_graph; } auto const &GetBarriers() const { return barriers; } @@ -60,9 +64,8 @@ class NodeBasedGraphFactory void ReleaseOsmNodes(); private: - // Get the information from the *.osrm file (direct product of the extractor callback/extraction - // containers) and prepare the graph creation process - void LoadDataFromFile(const boost::filesystem::path &input_file); + // Build and validate compressed output graph + void BuildCompressedOutputGraph(const std::vector &edge_list); // Compress the node-based graph into a compact representation of itself. This removes storing a // single edge for every part of the geometry and might also combine meta-data for multiple diff --git a/src/extractor/extraction_containers.cpp b/src/extractor/extraction_containers.cpp index 4730441b1..5c1b57c18 100644 --- a/src/extractor/extraction_containers.cpp +++ b/src/extractor/extraction_containers.cpp @@ -806,9 +806,9 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm } } -void ExtractionContainers::WriteEdges(storage::tar::FileWriter &writer) const +void ExtractionContainers::WriteEdges(storage::tar::FileWriter &writer) { - std::vector normal_edges; + // std::vector normal_edges; normal_edges.reserve(all_edges_list.size()); { util::UnbufferedLog log; @@ -854,7 +854,7 @@ void ExtractionContainers::WriteMetadata(storage::tar::FileWriter &writer) const log << " -- Metadata contains << " << all_edges_annotation_data_list.size() << " entries."; } -void ExtractionContainers::WriteNodes(storage::tar::FileWriter &writer) const +void ExtractionContainers::WriteNodes(storage::tar::FileWriter &writer) { { util::UnbufferedLog log; @@ -900,17 +900,16 @@ void ExtractionContainers::WriteNodes(storage::tar::FileWriter &writer) const util::UnbufferedLog log; log << "Writing barrier nodes ... "; TIMER_START(write_nodes); - std::vector internal_barrier_nodes; for (const auto osm_id : barrier_nodes) { const auto node_id = mapExternalToInternalNodeID( used_node_id_list.begin(), used_node_id_list.end(), osm_id); if (node_id != SPECIAL_NODEID) { - internal_barrier_nodes.push_back(node_id); + internal_barrier_nodes.emplace(node_id); } } - storage::serialization::write(writer, "/extractor/barriers", internal_barrier_nodes); + // storage::serialization::write(writer, "/extractor/barriers", internal_barrier_nodes); log << "ok, after " << TIMER_SEC(write_nodes) << "s"; } diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index 388ee866d..fe2d26bce 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -208,8 +208,21 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) std::vector turn_restrictions; std::vector unresolved_maneuver_overrides; TrafficSignals traffic_signals; - std::tie(turn_lane_map, turn_restrictions, unresolved_maneuver_overrides, traffic_signals) = - ParseOSMData(scripting_environment, number_of_threads); + std::unordered_set barriers; + std::vector osm_coordinates; + extractor::PackedOSMIDs osm_node_ids; + std::vector edge_list; + std::vector annotation_data; + + std::tie(turn_lane_map, + turn_restrictions, + unresolved_maneuver_overrides, + traffic_signals, + barriers, + osm_coordinates, + osm_node_ids, + edge_list, + annotation_data) = ParseOSMData(scripting_environment, number_of_threads); // Transform the node-based graph that OSM is based on into an edge-based graph // that is better for routing. Every edge becomes a node, and every valid @@ -227,11 +240,15 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) std::uint32_t ebg_connectivity_checksum = 0; // Create a node-based graph from the OSRM file - NodeBasedGraphFactory node_based_graph_factory(config.GetPath(".osrm"), - scripting_environment, + NodeBasedGraphFactory node_based_graph_factory(scripting_environment, turn_restrictions, unresolved_maneuver_overrides, - traffic_signals); + traffic_signals, + std::move(barriers), + std::move(osm_coordinates), + std::move(osm_node_ids), + edge_list, + std::move(annotation_data)); NameTable name_table; files::readNames(config.GetPath(".osrm.names"), name_table); @@ -364,7 +381,12 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) std::tuple, std::vector, - TrafficSignals> + TrafficSignals, + std::unordered_set, + std::vector, + extractor::PackedOSMIDs, + std::vector, + std::vector> Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, const unsigned number_of_threads) { @@ -629,10 +651,31 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, TIMER_STOP(extracting); util::Log() << "extraction finished after " << TIMER_SEC(extracting) << "s"; + std::unordered_set barriers; + std::vector osm_coordinates; + extractor::PackedOSMIDs osm_node_ids; + std::vector edge_list; + std::vector annotation_data; + + osm_coordinates.resize(extraction_containers.all_nodes_list.size()); + osm_node_ids.reserve(extraction_containers.all_nodes_list.size()); + for (size_t index = 0; index < extraction_containers.all_nodes_list.size(); ++index) + { + const auto ¤t_node = extraction_containers.all_nodes_list[index]; + osm_coordinates[index].lon = current_node.lon; + osm_coordinates[index].lat = current_node.lat; + osm_node_ids.push_back(current_node.node_id); + } + return std::make_tuple(std::move(turn_lane_map), std::move(extraction_containers.turn_restrictions), std::move(extraction_containers.internal_maneuver_overrides), - std::move(extraction_containers.internal_traffic_signals)); + std::move(extraction_containers.internal_traffic_signals), + std::move(extraction_containers.internal_barrier_nodes), + std::move(osm_coordinates), + std::move(osm_node_ids), + std::move(extraction_containers.normal_edges), + std::move(extraction_containers.all_edges_annotation_data_list)); } void Extractor::FindComponents(unsigned number_of_edge_based_nodes, diff --git a/src/extractor/node_based_graph_factory.cpp b/src/extractor/node_based_graph_factory.cpp index 887523de9..500ee69f6 100644 --- a/src/extractor/node_based_graph_factory.cpp +++ b/src/extractor/node_based_graph_factory.cpp @@ -16,32 +16,30 @@ namespace extractor { NodeBasedGraphFactory::NodeBasedGraphFactory( - const boost::filesystem::path &input_file, ScriptingEnvironment &scripting_environment, std::vector &turn_restrictions, std::vector &maneuver_overrides, - const TrafficSignals &traffic_signals) + const TrafficSignals &traffic_signals, + std::unordered_set barriers, + std::vector coordinates, + extractor::PackedOSMIDs osm_node_ids, + const std::vector &edge_list, + std::vector annotation_data) + : annotation_data(std::move(annotation_data)), barriers(std::move(barriers)), + coordinates(std::move(coordinates)), osm_node_ids(std::move(osm_node_ids)) { - LoadDataFromFile(input_file); + BuildCompressedOutputGraph(edge_list); Compress(scripting_environment, turn_restrictions, maneuver_overrides, traffic_signals); CompressGeometry(); CompressAnnotationData(); } -// load the data serialised during the extraction run -void NodeBasedGraphFactory::LoadDataFromFile(const boost::filesystem::path &input_file) +void NodeBasedGraphFactory::BuildCompressedOutputGraph(const std::vector &edge_list) { - auto barriers_iter = inserter(barriers, end(barriers)); - std::vector edge_list; - - files::readRawNBGraph( - input_file, barriers_iter, coordinates, osm_node_ids, edge_list, annotation_data); - const auto number_of_node_based_nodes = coordinates.size(); if (edge_list.empty()) { - throw util::exception("Node-based-graph (" + input_file.string() + ") contains no edges." + - SOURCE_REF); + throw util::exception("Node-based-graph contains no edges." + SOURCE_REF); } // at this point, the data isn't compressed, but since we update the graph in-place, we assign