Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Do not generate intermediate .osrm file in osrm-extract. (#6354)

Browse Source
This commit is contained in:
Siarhei Fedartsou
2022-09-30 14:29:10 +02:00
committed by GitHub
parent 395cc6e9df
commit 21888334dd
19 changed files with 193 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/extractor/extraction_containers.cpp
+63 -95
View File
@@ -5,8 +5,8 @@
#include "extractor/name_table.hpp"
#include "extractor/restriction.hpp"
#include "extractor/serialization.hpp"
#include "util/coordinate_calculation.hpp"
#include "util/integer_range.hpp"
#include "util/exception.hpp"
#include "util/exception_utils.hpp"
@@ -16,6 +16,7 @@
#include "util/timing_util.hpp"
#include <boost/assert.hpp>
#include <boost/core/ignore_unused.hpp>
#include <boost/numeric/conversion/cast.hpp>
#include <tbb/parallel_sort.h>
@@ -407,22 +408,14 @@ ExtractionContainers::ExtractionContainers()
*
*/
void ExtractionContainers::PrepareData(ScriptingEnvironment &scripting_environment,
const std::string &osrm_path,
const std::string &name_file_name)
{
storage::tar::FileWriter writer(osrm_path, storage::tar::FileWriter::GenerateFingerprint);
const auto restriction_ways = IdentifyRestrictionWays();
const auto maneuver_override_ways = IdentifyManeuverOverrideWays();
const auto traffic_signals = IdentifyTrafficSignals();
PrepareNodes();
WriteNodes(writer);
PrepareEdges(scripting_environment);
all_nodes_list.clear(); // free all_nodes_list before allocation of normal_edges
all_nodes_list.shrink_to_fit();
WriteEdges(writer);
WriteMetadata(writer);
PrepareTrafficSignals(traffic_signals);
PrepareManeuverOverrides(maneuver_override_ways);
@@ -519,6 +512,60 @@ void ExtractionContainers::PrepareNodes()
TIMER_STOP(id_map);
log << "ok, after " << TIMER_SEC(id_map) << "s";
}
{
util::UnbufferedLog log;
log << "Confirming/Writing used nodes ... ";
TIMER_START(write_nodes);
// identify all used nodes by a merging step of two sorted lists
auto node_iterator = all_nodes_list.begin();
auto node_id_iterator = used_node_id_list.begin();
const auto all_nodes_list_end = all_nodes_list.end();
for (const auto index : util::irange<NodeID>(0, used_node_id_list.size()))
{
boost::ignore_unused(index);
BOOST_ASSERT(node_id_iterator != used_node_id_list.end());
BOOST_ASSERT(node_iterator != all_nodes_list_end);
BOOST_ASSERT(*node_id_iterator >= node_iterator->node_id);
while (*node_id_iterator > node_iterator->node_id &&
node_iterator != all_nodes_list_end)
{
++node_iterator;
}
if (node_iterator == all_nodes_list_end || *node_id_iterator < node_iterator->node_id)
{
throw util::exception(
"Invalid OSM data: Referenced non-existing node with ID " +
std::to_string(static_cast<std::uint64_t>(*node_id_iterator)));
}
BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
++node_id_iterator;
used_nodes.emplace_back(*node_iterator++);
}
TIMER_STOP(write_nodes);
log << "ok, after " << TIMER_SEC(write_nodes) << "s";
}
{
util::UnbufferedLog log;
log << "Writing barrier nodes ... ";
TIMER_START(write_nodes);
for (const auto osm_id : barrier_nodes)
{
const auto node_id = mapExternalToInternalNodeID(
used_node_id_list.begin(), used_node_id_list.end(), osm_id);
if (node_id != SPECIAL_NODEID)
{
used_barrier_nodes.emplace(node_id);
}
}
log << "ok, after " << TIMER_SEC(write_nodes) << "s";
}
util::Log() << "Processed " << max_internal_node_id << " nodes";
}
void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environment)
@@ -804,12 +851,11 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm
all_edges_list[j].result.target = SPECIAL_NODEID;
}
}
}
void ExtractionContainers::WriteEdges(storage::tar::FileWriter &writer) const
{
std::vector<NodeBasedEdge> normal_edges;
normal_edges.reserve(all_edges_list.size());
all_nodes_list.clear(); // free all_nodes_list before allocation of used_edges
all_nodes_list.shrink_to_fit();
used_edges.reserve(all_edges_list.size());
{
util::UnbufferedLog log;
log << "Writing used edges ... " << std::flush;
@@ -825,98 +871,20 @@ void ExtractionContainers::WriteEdges(storage::tar::FileWriter &writer) const
// IMPORTANT: here, we're using slicing to only write the data from the base
// class of NodeBasedEdgeWithOSM
normal_edges.push_back(edge.result);
used_edges.push_back(edge.result);
}
if (normal_edges.size() > std::numeric_limits<uint32_t>::max())
if (used_edges.size() > std::numeric_limits<uint32_t>::max())
{
throw util::exception("There are too many edges, OSRM only supports 2^32" + SOURCE_REF);
}
storage::serialization::write(writer, "/extractor/edges", normal_edges);
TIMER_STOP(write_edges);
log << "ok, after " << TIMER_SEC(write_edges) << "s";
log << " -- Processed " << normal_edges.size() << " edges";
log << " -- Processed " << used_edges.size() << " edges";
}
}
void ExtractionContainers::WriteMetadata(storage::tar::FileWriter &writer) const
{
util::UnbufferedLog log;
log << "Writing way meta-data ... " << std::flush;
TIMER_START(write_meta_data);
storage::serialization::write(writer, "/extractor/annotations", all_edges_annotation_data_list);
TIMER_STOP(write_meta_data);
log << "ok, after " << TIMER_SEC(write_meta_data) << "s";
log << " -- Metadata contains << " << all_edges_annotation_data_list.size() << " entries.";
}
void ExtractionContainers::WriteNodes(storage::tar::FileWriter &writer) const
{
{
util::UnbufferedLog log;
log << "Confirming/Writing used nodes ... ";
TIMER_START(write_nodes);
// identify all used nodes by a merging step of two sorted lists
auto node_iterator = all_nodes_list.begin();
auto node_id_iterator = used_node_id_list.begin();
const auto all_nodes_list_end = all_nodes_list.end();
const std::function<QueryNode()> encode_function = [&]() -> QueryNode {
BOOST_ASSERT(node_id_iterator != used_node_id_list.end());
BOOST_ASSERT(node_iterator != all_nodes_list_end);
BOOST_ASSERT(*node_id_iterator >= node_iterator->node_id);
while (*node_id_iterator > node_iterator->node_id &&
node_iterator != all_nodes_list_end)
{
++node_iterator;
}
if (node_iterator == all_nodes_list_end || *node_id_iterator < node_iterator->node_id)
{
throw util::exception(
"Invalid OSM data: Referenced non-existing node with ID " +
std::to_string(static_cast<std::uint64_t>(*node_id_iterator)));
}
BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
++node_id_iterator;
return *node_iterator++;
};
writer.WriteElementCount64("/extractor/nodes", used_node_id_list.size());
writer.WriteStreaming<QueryNode>(
"/extractor/nodes",
boost::make_function_input_iterator(encode_function, boost::infinite()),
used_node_id_list.size());
TIMER_STOP(write_nodes);
log << "ok, after " << TIMER_SEC(write_nodes) << "s";
}
{
util::UnbufferedLog log;
log << "Writing barrier nodes ... ";
TIMER_START(write_nodes);
std::vector<NodeID> internal_barrier_nodes;
for (const auto osm_id : barrier_nodes)
{
const auto node_id = mapExternalToInternalNodeID(
used_node_id_list.begin(), used_node_id_list.end(), osm_id);
if (node_id != SPECIAL_NODEID)
{
internal_barrier_nodes.push_back(node_id);
}
}
storage::serialization::write(writer, "/extractor/barriers", internal_barrier_nodes);
log << "ok, after " << TIMER_SEC(write_nodes) << "s";
}
util::Log() << "Processed " << max_internal_node_id << " nodes";
}
ExtractionContainers::ReferencedWays ExtractionContainers::IdentifyManeuverOverrideWays()
{
ReferencedWays maneuver_override_ways;