Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Do not generate intermediate .osrm file in osrm-extract. (#6354)

Browse Source
This commit is contained in:
Siarhei Fedartsou 2022-09-30 14:29:10 +02:00 committed by GitHub
parent 395cc6e9df
commit 21888334dd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 193 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.github/workflows/osrm-backend.yml vendored
View File

@ -124,7 +124,17 @@ jobs:
# when `--memory-swap` value equals `--memory` it means container won't use swap
# see https://docs.docker.com/config/containers/resource_constraints/#--memory-swap-details
MEMORY_ARGS="--memory=1g --memory-swap=1g"
docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-extract -p /opt/car.lua /data/berlin-latest.osm.pbf
docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-extract --dump-nbg-graph -p /opt/car.lua /data/berlin-latest.osm.pbf
docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-components /data/berlin-latest.osrm.nbg /data/berlin-latest.geojson
if [ ! -s "${PWD}/berlin-latest.geojson" ]
then
>&2 echo "No berlin-latest.geojson found"
exit 1
fi
# removing `.osrm.nbg` to check that whole pipeline works without it
rm -rf "${PWD}/berlin-latest.osrm.nbg"
docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-partition /data/berlin-latest.osrm
docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-customize /data/berlin-latest.osrm
docker run $MEMORY_ARGS --name=osrm-container -t -p 5000:5000 -v "${PWD}:/data" "${TAG}" osrm-routed --algorithm mld /data/berlin-latest.osrm &

3
CHANGELOG.md
View File

@ -64,7 +64,8 @@
- FIXED: Ensure u-turn exists in intersection view. [#6376](https://github.com/Project-OSRM/osrm-backend/pull/6376)
- Profile:
- CHANGED: Bicycle surface speeds [#6212](https://github.com/Project-OSRM/osrm-backend/pull/6212)
- Tools:
- CHANGED: Do not generate intermediate .osrm file in osrm-extract. [#6354](https://github.com/Project-OSRM/osrm-backend/pull/6354)
# 5.26.0
- Changes from 5.25.0

2
README.md
View File

@ -65,7 +65,7 @@ The flag `-v "${PWD}:/data"` creates the directory `/data` inside the docker con
docker run -t -v "${PWD}:/data" osrm/osrm-backend osrm-partition /data/berlin-latest.osrm
docker run -t -v "${PWD}:/data" osrm/osrm-backend osrm-customize /data/berlin-latest.osrm
Note that `berlin-latest.osrm` has a different file extension.
Note there is no `berlin-latest.osrm` file, but multiple `berlin-latest.osrm.*` files, i.e. `berlin-latest.osrm` is not file path, but "base" path referring to set of files and there is an option to omit this `.osrm` suffix completely(e.g. `osrm-partition /data/berlin-latest`).
docker run -t -i -p 5000:5000 -v "${PWD}:/data" osrm/osrm-backend osrm-routed --algorithm mld /data/berlin-latest.osrm

6
docs/nodejs/api.md
View File

@ -3,8 +3,8 @@
## OSRM
The `OSRM` method is the main constructor for creating an OSRM instance.
An OSRM instance requires a `.osrm` dataset, which is prepared by the OSRM toolchain.
You can create such a `.osrm` file by running the OSRM binaries we ship in `node_modules/osrm/lib/binding/` and default
An OSRM instance requires a `.osrm.*` dataset(`.osrm.*` because it contains several files), which is prepared by the OSRM toolchain.
You can create such a `.osrm.*` dataset by running the OSRM binaries we ship in `node_modules/osrm/lib/binding/` and default
profiles (e.g. for setting speeds and determining road types to route on) in `node_modules/osrm/profiles/`:
node_modules/osrm/lib/binding/osrm-extract data.osm.pbf -p node_modules/osrm/profiles/car.lua
@ -12,7 +12,7 @@ profiles (e.g. for setting speeds and determining road types to route on) in `no
Consult the [osrm-backend](https://github.com/Project-OSRM/osrm-backend) documentation for further details.
Once you have a complete `network.osrm` file, you can calculate routes in javascript with this object.
Once you have a complete `network.osrm.*` dataset, you can calculate routes in javascript with this object.
```javascript
var osrm = new OSRM('network.osrm');

8
include/extractor/extraction_containers.hpp
View File

@ -43,9 +43,6 @@ class ExtractionContainers
void PrepareTrafficSignals(const ReferencedTrafficSignals &referenced_traffic_signals);
void PrepareEdges(ScriptingEnvironment &scripting_environment);
void WriteNodes(storage::tar::FileWriter &file_out) const;
void WriteEdges(storage::tar::FileWriter &file_out) const;
void WriteMetadata(storage::tar::FileWriter &file_out) const;
void WriteCharData(const std::string &file_name);
public:
@ -75,6 +72,8 @@ class ExtractionContainers
std::vector<InputTrafficSignal> external_traffic_signals;
TrafficSignals internal_traffic_signals;
std::vector<NodeBasedEdge> used_edges;
// List of restrictions (conditional and unconditional) before we transform them into the
// output types. Input containers reference OSMNodeIDs. We can only transform them to the
// correct internal IDs after we've read everything. Without a multi-parse approach,
@ -84,11 +83,12 @@ class ExtractionContainers
std::vector<InputManeuverOverride> external_maneuver_overrides_list;
std::vector<UnresolvedManeuverOverride> internal_maneuver_overrides;
std::unordered_set<NodeID> used_barrier_nodes;
NodeVector used_nodes;
ExtractionContainers();
void PrepareData(ScriptingEnvironment &scripting_environment,
const std::string &osrm_path,
const std::string &names_data_path);
};
} // namespace extractor

21
include/extractor/extractor.hpp
View File

@ -60,14 +60,25 @@ class Extractor
Extractor(ExtractorConfig extractor_config) : config(std::move(extractor_config)) {}
int run(ScriptingEnvironment &scripting_environment);
private:
struct ParsedOSMData
{
LaneDescriptionMap turn_lane_map;
std::vector<TurnRestriction> turn_restrictions;
std::vector<UnresolvedManeuverOverride> unresolved_maneuver_overrides;
TrafficSignals traffic_signals;
std::unordered_set<NodeID> barriers;
std::vector<util::Coordinate> osm_coordinates;
extractor::PackedOSMIDs osm_node_ids;
std::vector<NodeBasedEdge> edge_list;
std::vector<NodeBasedEdgeAnnotation> annotation_data;
};
private:
ExtractorConfig config;
std::tuple<LaneDescriptionMap,
std::vector<TurnRestriction>,
std::vector<UnresolvedManeuverOverride>,
TrafficSignals>
ParseOSMData(ScriptingEnvironment &scripting_environment, const unsigned number_of_threads);
ParsedOSMData ParseOSMData(ScriptingEnvironment &scripting_environment,
const unsigned number_of_threads);
EdgeID BuildEdgeExpandedGraph(
// input data

3
include/extractor/extractor_config.hpp
View File

@ -48,7 +48,7 @@ struct ExtractorConfig final : storage::IOConfig
"",
},
{},
{".osrm",
{".osrm.nbg",
".osrm.restrictions",
".osrm.names",
".osrm.tls",
@ -89,6 +89,7 @@ struct ExtractorConfig final : storage::IOConfig
bool use_metadata = false;
bool parse_conditionals = false;
bool use_locations_cache = true;
bool dump_nbg_graph = false;
};
} // namespace extractor
} // namespace osrm

9
include/extractor/files.hpp
View File

@ -444,13 +444,11 @@ inline void readConditionalRestrictions(const boost::filesystem::path &path,
}
// reads .osrm file which is a temporary file of osrm-extract
template <typename BarrierOutIter, typename PackedOSMIDsT>
template <typename PackedOSMIDsT>
void readRawNBGraph(const boost::filesystem::path &path,
BarrierOutIter barriers,
std::vector<util::Coordinate> &coordinates,
PackedOSMIDsT &osm_node_ids,
std::vector<extractor::NodeBasedEdge> &edge_list,
std::vector<extractor::NodeBasedEdgeAnnotation> &annotations)
std::vector<extractor::NodeBasedEdge> &edge_list)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
@ -468,10 +466,7 @@ void readRawNBGraph(const boost::filesystem::path &path,
reader.ReadStreaming<extractor::QueryNode>("/extractor/nodes",
boost::make_function_output_iterator(decode));
reader.ReadStreaming<NodeID>("/extractor/barriers", barriers);
storage::serialization::read(reader, "/extractor/edges", edge_list);
storage::serialization::read(reader, "/extractor/annotations", annotations);
}
template <typename NameTableT>

17
include/extractor/node_based_graph_factory.hpp
View File

@ -34,15 +34,19 @@ namespace extractor
class NodeBasedGraphFactory
{
public:
// The node-based graph factory loads the *.osrm file and transforms the data within into the
// The node-based graph factory transforms the graph data into the
// node-based graph to represent the OSM network. This includes geometry compression, annotation
// data optimisation and many other aspects. After this step, the edge-based graph factory can
// turn the graph into the routing graph to be used with the navigation algorithms.
NodeBasedGraphFactory(const boost::filesystem::path &input_file,
ScriptingEnvironment &scripting_environment,
NodeBasedGraphFactory(ScriptingEnvironment &scripting_environment,
std::vector<TurnRestriction> &turn_restrictions,
std::vector<UnresolvedManeuverOverride> &maneuver_overrides,
const TrafficSignals &traffic_signals);
const TrafficSignals &traffic_signals,
std::unordered_set<NodeID> &&barriers,
std::vector<util::Coordinate> &&coordinates,
extractor::PackedOSMIDs &&osm_node_ids,
const std::vector<NodeBasedEdge> &edge_list,
std::vector<NodeBasedEdgeAnnotation> &&annotation_data);
auto const &GetGraph() const { return compressed_output_graph; }
auto const &GetBarriers() const { return barriers; }
@ -60,9 +64,8 @@ class NodeBasedGraphFactory
void ReleaseOsmNodes();
private:
// Get the information from the *.osrm file (direct product of the extractor callback/extraction
// containers) and prepare the graph creation process
void LoadDataFromFile(const boost::filesystem::path &input_file);
// Build and validate compressed output graph
void BuildCompressedOutputGraph(const std::vector<NodeBasedEdge> &edge_list);
// Compress the node-based graph into a compact representation of itself. This removes storing a
// single edge for every part of the geometry and might also combine meta-data for multiple

2
include/partitioner/partitioner_config.hpp
View File

@ -16,7 +16,7 @@ namespace partitioner
struct PartitionerConfig final : storage::IOConfig
{
PartitionerConfig()
: IOConfig({".osrm", ".osrm.fileIndex", ".osrm.ebg_nodes", ".osrm.enw"},
: IOConfig({".osrm.fileIndex", ".osrm.ebg_nodes", ".osrm.enw"},
{".osrm.hsgr", ".osrm.cnbg"},
{".osrm.ebg",
".osrm.cnbg",

158
src/extractor/extraction_containers.cpp
View File

@ -5,8 +5,8 @@
#include "extractor/name_table.hpp"
#include "extractor/restriction.hpp"
#include "extractor/serialization.hpp"
#include "util/coordinate_calculation.hpp"
#include "util/integer_range.hpp"
#include "util/exception.hpp"
#include "util/exception_utils.hpp"
@ -16,6 +16,7 @@
#include "util/timing_util.hpp"
#include <boost/assert.hpp>
#include <boost/core/ignore_unused.hpp>
#include <boost/numeric/conversion/cast.hpp>
#include <tbb/parallel_sort.h>
@ -407,22 +408,14 @@ ExtractionContainers::ExtractionContainers()
*
*/
void ExtractionContainers::PrepareData(ScriptingEnvironment &scripting_environment,
const std::string &osrm_path,
const std::string &name_file_name)
{
storage::tar::FileWriter writer(osrm_path, storage::tar::FileWriter::GenerateFingerprint);
const auto restriction_ways = IdentifyRestrictionWays();
const auto maneuver_override_ways = IdentifyManeuverOverrideWays();
const auto traffic_signals = IdentifyTrafficSignals();
PrepareNodes();
WriteNodes(writer);
PrepareEdges(scripting_environment);
all_nodes_list.clear(); // free all_nodes_list before allocation of normal_edges
all_nodes_list.shrink_to_fit();
WriteEdges(writer);
WriteMetadata(writer);
PrepareTrafficSignals(traffic_signals);
PrepareManeuverOverrides(maneuver_override_ways);
@ -519,6 +512,60 @@ void ExtractionContainers::PrepareNodes()
TIMER_STOP(id_map);
log << "ok, after " << TIMER_SEC(id_map) << "s";
}
{
util::UnbufferedLog log;
log << "Confirming/Writing used nodes ... ";
TIMER_START(write_nodes);
// identify all used nodes by a merging step of two sorted lists
auto node_iterator = all_nodes_list.begin();
auto node_id_iterator = used_node_id_list.begin();
const auto all_nodes_list_end = all_nodes_list.end();
for (const auto index : util::irange<NodeID>(0, used_node_id_list.size()))
{
boost::ignore_unused(index);
BOOST_ASSERT(node_id_iterator != used_node_id_list.end());
BOOST_ASSERT(node_iterator != all_nodes_list_end);
BOOST_ASSERT(*node_id_iterator >= node_iterator->node_id);
while (*node_id_iterator > node_iterator->node_id &&
node_iterator != all_nodes_list_end)
{
++node_iterator;
}
if (node_iterator == all_nodes_list_end || *node_id_iterator < node_iterator->node_id)
{
throw util::exception(
"Invalid OSM data: Referenced non-existing node with ID " +
std::to_string(static_cast<std::uint64_t>(*node_id_iterator)));
}
BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
++node_id_iterator;
used_nodes.emplace_back(*node_iterator++);
}
TIMER_STOP(write_nodes);
log << "ok, after " << TIMER_SEC(write_nodes) << "s";
}
{
util::UnbufferedLog log;
log << "Writing barrier nodes ... ";
TIMER_START(write_nodes);
for (const auto osm_id : barrier_nodes)
{
const auto node_id = mapExternalToInternalNodeID(
used_node_id_list.begin(), used_node_id_list.end(), osm_id);
if (node_id != SPECIAL_NODEID)
{
used_barrier_nodes.emplace(node_id);
}
}
log << "ok, after " << TIMER_SEC(write_nodes) << "s";
}
util::Log() << "Processed " << max_internal_node_id << " nodes";
}
void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environment)
@ -804,12 +851,11 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm
all_edges_list[j].result.target = SPECIAL_NODEID;
}
}
}
void ExtractionContainers::WriteEdges(storage::tar::FileWriter &writer) const
{
std::vector<NodeBasedEdge> normal_edges;
normal_edges.reserve(all_edges_list.size());
all_nodes_list.clear(); // free all_nodes_list before allocation of used_edges
all_nodes_list.shrink_to_fit();
used_edges.reserve(all_edges_list.size());
{
util::UnbufferedLog log;
log << "Writing used edges ... " << std::flush;
@ -825,98 +871,20 @@ void ExtractionContainers::WriteEdges(storage::tar::FileWriter &writer) const
// IMPORTANT: here, we're using slicing to only write the data from the base
// class of NodeBasedEdgeWithOSM
normal_edges.push_back(edge.result);
used_edges.push_back(edge.result);
}
if (normal_edges.size() > std::numeric_limits<uint32_t>::max())
if (used_edges.size() > std::numeric_limits<uint32_t>::max())
{
throw util::exception("There are too many edges, OSRM only supports 2^32" + SOURCE_REF);
}
storage::serialization::write(writer, "/extractor/edges", normal_edges);
TIMER_STOP(write_edges);
log << "ok, after " << TIMER_SEC(write_edges) << "s";
log << " -- Processed " << normal_edges.size() << " edges";
log << " -- Processed " << used_edges.size() << " edges";
}
}
void ExtractionContainers::WriteMetadata(storage::tar::FileWriter &writer) const
{
util::UnbufferedLog log;
log << "Writing way meta-data ... " << std::flush;
TIMER_START(write_meta_data);
storage::serialization::write(writer, "/extractor/annotations", all_edges_annotation_data_list);
TIMER_STOP(write_meta_data);
log << "ok, after " << TIMER_SEC(write_meta_data) << "s";
log << " -- Metadata contains << " << all_edges_annotation_data_list.size() << " entries.";
}
void ExtractionContainers::WriteNodes(storage::tar::FileWriter &writer) const
{
{
util::UnbufferedLog log;
log << "Confirming/Writing used nodes ... ";
TIMER_START(write_nodes);
// identify all used nodes by a merging step of two sorted lists
auto node_iterator = all_nodes_list.begin();
auto node_id_iterator = used_node_id_list.begin();
const auto all_nodes_list_end = all_nodes_list.end();
const std::function<QueryNode()> encode_function = [&]() -> QueryNode {
BOOST_ASSERT(node_id_iterator != used_node_id_list.end());
BOOST_ASSERT(node_iterator != all_nodes_list_end);
BOOST_ASSERT(*node_id_iterator >= node_iterator->node_id);
while (*node_id_iterator > node_iterator->node_id &&
node_iterator != all_nodes_list_end)
{
++node_iterator;
}
if (node_iterator == all_nodes_list_end || *node_id_iterator < node_iterator->node_id)
{
throw util::exception(
"Invalid OSM data: Referenced non-existing node with ID " +
std::to_string(static_cast<std::uint64_t>(*node_id_iterator)));
}
BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
++node_id_iterator;
return *node_iterator++;
};
writer.WriteElementCount64("/extractor/nodes", used_node_id_list.size());
writer.WriteStreaming<QueryNode>(
"/extractor/nodes",
boost::make_function_input_iterator(encode_function, boost::infinite()),
used_node_id_list.size());
TIMER_STOP(write_nodes);
log << "ok, after " << TIMER_SEC(write_nodes) << "s";
}
{
util::UnbufferedLog log;
log << "Writing barrier nodes ... ";
TIMER_START(write_nodes);
std::vector<NodeID> internal_barrier_nodes;
for (const auto osm_id : barrier_nodes)
{
const auto node_id = mapExternalToInternalNodeID(
used_node_id_list.begin(), used_node_id_list.end(), osm_id);
if (node_id != SPECIAL_NODEID)
{
internal_barrier_nodes.push_back(node_id);
}
}
storage::serialization::write(writer, "/extractor/barriers", internal_barrier_nodes);
log << "ok, after " << TIMER_SEC(write_nodes) << "s";
}
util::Log() << "Processed " << max_internal_node_id << " nodes";
}
ExtractionContainers::ReferencedWays ExtractionContainers::IdentifyManeuverOverrideWays()
{
ReferencedWays maneuver_override_ways;

83
src/extractor/extractor.cpp
View File

@ -204,12 +204,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
tbb::global_control gc(tbb::global_control::max_allowed_parallelism,
config.requested_num_threads);
LaneDescriptionMap turn_lane_map;
std::vector<TurnRestriction> turn_restrictions;
std::vector<UnresolvedManeuverOverride> unresolved_maneuver_overrides;
TrafficSignals traffic_signals;
std::tie(turn_lane_map, turn_restrictions, unresolved_maneuver_overrides, traffic_signals) =
ParseOSMData(scripting_environment, number_of_threads);
auto parsed_osm_data = ParseOSMData(scripting_environment, number_of_threads);
// Transform the node-based graph that OSM is based on into an edge-based graph
// that is better for routing. Every edge becomes a node, and every valid
@ -227,11 +222,15 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
std::uint32_t ebg_connectivity_checksum = 0;
// Create a node-based graph from the OSRM file
NodeBasedGraphFactory node_based_graph_factory(config.GetPath(".osrm"),
scripting_environment,
turn_restrictions,
unresolved_maneuver_overrides,
traffic_signals);
NodeBasedGraphFactory node_based_graph_factory(scripting_environment,
parsed_osm_data.turn_restrictions,
parsed_osm_data.unresolved_maneuver_overrides,
parsed_osm_data.traffic_signals,
std::move(parsed_osm_data.barriers),
std::move(parsed_osm_data.osm_coordinates),
std::move(parsed_osm_data.osm_node_ids),
parsed_osm_data.edge_list,
std::move(parsed_osm_data.annotation_data));
NameTable name_table;
files::readNames(config.GetPath(".osrm.names"), name_table);
@ -270,10 +269,11 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
edge_based_nodes_container =
EdgeBasedNodeDataContainer({}, std::move(node_based_graph_factory.GetAnnotationData()));
turn_restrictions = removeInvalidTurnPaths(std::move(turn_restrictions), node_based_graph);
unresolved_maneuver_overrides =
removeInvalidTurnPaths(std::move(unresolved_maneuver_overrides), node_based_graph);
auto restriction_graph = constructRestrictionGraph(turn_restrictions);
parsed_osm_data.turn_restrictions =
removeInvalidTurnPaths(std::move(parsed_osm_data.turn_restrictions), node_based_graph);
parsed_osm_data.unresolved_maneuver_overrides = removeInvalidTurnPaths(
std::move(parsed_osm_data.unresolved_maneuver_overrides), node_based_graph);
auto restriction_graph = constructRestrictionGraph(parsed_osm_data.turn_restrictions);
const auto number_of_node_based_nodes = node_based_graph.GetNumberOfNodes();
@ -282,12 +282,12 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
coordinates,
node_based_graph_factory.GetCompressedEdges(),
barrier_nodes,
traffic_signals,
parsed_osm_data.traffic_signals,
restriction_graph,
segregated_edges,
name_table,
unresolved_maneuver_overrides,
turn_lane_map,
parsed_osm_data.unresolved_maneuver_overrides,
parsed_osm_data.turn_lane_map,
scripting_environment,
edge_based_nodes_container,
edge_based_node_segments,
@ -304,7 +304,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
barrier_nodes,
restriction_graph,
name_table,
std::move(turn_lane_map),
std::move(parsed_osm_data.turn_lane_map),
scripting_environment);
TIMER_STOP(expansion);
@ -356,17 +356,13 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
util::Log() << "Expansion: " << nodes_per_second << " nodes/sec and " << edges_per_second
<< " edges/sec";
util::Log() << "To prepare the data for routing, run: "
<< "./osrm-contract " << config.GetPath(".osrm");
<< "./osrm-contract " << config.base_path;
return 0;
}
std::tuple<LaneDescriptionMap,
std::vector<TurnRestriction>,
std::vector<UnresolvedManeuverOverride>,
TrafficSignals>
Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
const unsigned number_of_threads)
Extractor::ParsedOSMData Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
const unsigned number_of_threads)
{
TIMER_START(extracting);
@ -617,7 +613,6 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
}
extraction_containers.PrepareData(scripting_environment,
config.GetPath(".osrm").string(),
config.GetPath(".osrm.names").string());
auto profile_properties = scripting_environment.GetProfileProperties();
@ -629,10 +624,36 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
TIMER_STOP(extracting);
util::Log() << "extraction finished after " << TIMER_SEC(extracting) << "s";
return std::make_tuple(std::move(turn_lane_map),
std::move(extraction_containers.turn_restrictions),
std::move(extraction_containers.internal_maneuver_overrides),
std::move(extraction_containers.internal_traffic_signals));
std::vector<util::Coordinate> osm_coordinates;
extractor::PackedOSMIDs osm_node_ids;
osm_coordinates.resize(extraction_containers.used_nodes.size());
osm_node_ids.reserve(extraction_containers.used_nodes.size());
for (size_t index = 0; index < extraction_containers.used_nodes.size(); ++index)
{
const auto &current_node = extraction_containers.used_nodes[index];
osm_coordinates[index].lon = current_node.lon;
osm_coordinates[index].lat = current_node.lat;
osm_node_ids.push_back(current_node.node_id);
}
if (config.dump_nbg_graph)
{
storage::tar::FileWriter writer(config.GetPath(".osrm.nbg").string(),
storage::tar::FileWriter::GenerateFingerprint);
storage::serialization::write(writer, "/extractor/nodes", extraction_containers.used_nodes);
storage::serialization::write(writer, "/extractor/edges", extraction_containers.used_edges);
}
return ParsedOSMData{std::move(turn_lane_map),
std::move(extraction_containers.turn_restrictions),
std::move(extraction_containers.internal_maneuver_overrides),
std::move(extraction_containers.internal_traffic_signals),
std::move(extraction_containers.used_barrier_nodes),
std::move(osm_coordinates),
std::move(osm_node_ids),
std::move(extraction_containers.used_edges),
std::move(extraction_containers.all_edges_annotation_data_list)};
}
void Extractor::FindComponents(unsigned number_of_edge_based_nodes,

24
src/extractor/node_based_graph_factory.cpp
View File

@ -16,32 +16,30 @@ namespace extractor
{
NodeBasedGraphFactory::NodeBasedGraphFactory(
const boost::filesystem::path &input_file,
ScriptingEnvironment &scripting_environment,
std::vector<TurnRestriction> &turn_restrictions,
std::vector<UnresolvedManeuverOverride> &maneuver_overrides,
const TrafficSignals &traffic_signals)
const TrafficSignals &traffic_signals,
std::unordered_set<NodeID> &&barriers,
std::vector<util::Coordinate> &&coordinates,
extractor::PackedOSMIDs &&osm_node_ids,
const std::vector<NodeBasedEdge> &edge_list,
std::vector<NodeBasedEdgeAnnotation> &&annotation_data)
: annotation_data(std::move(annotation_data)), barriers(std::move(barriers)),
coordinates(std::move(coordinates)), osm_node_ids(std::move(osm_node_ids))
{
LoadDataFromFile(input_file);
BuildCompressedOutputGraph(edge_list);
Compress(scripting_environment, turn_restrictions, maneuver_overrides, traffic_signals);
CompressGeometry();
CompressAnnotationData();
}
// load the data serialised during the extraction run
void NodeBasedGraphFactory::LoadDataFromFile(const boost::filesystem::path &input_file)
void NodeBasedGraphFactory::BuildCompressedOutputGraph(const std::vector<NodeBasedEdge> &edge_list)
{
auto barriers_iter = inserter(barriers, end(barriers));
std::vector<NodeBasedEdge> edge_list;
files::readRawNBGraph(
input_file, barriers_iter, coordinates, osm_node_ids, edge_list, annotation_data);
const auto number_of_node_based_nodes = coordinates.size();
if (edge_list.empty())
{
throw util::exception("Node-based-graph (" + input_file.string() + ") contains no edges." +
SOURCE_REF);
throw util::exception("Node-based-graph contains no edges." + SOURCE_REF);
}
// at this point, the data isn't compressed, but since we update the graph in-place, we assign

6
src/nodejs/node_osrm.cpp
View File

@ -55,8 +55,8 @@ NAN_MODULE_INIT(Engine::Init)
// clang-format off
/**
* The `OSRM` method is the main constructor for creating an OSRM instance.
* An OSRM instance requires a `.osrm` dataset, which is prepared by the OSRM toolchain.
* You can create such a `.osrm` file by running the OSRM binaries we ship in `node_modules/osrm/lib/binding/` and default
* An OSRM instance requires a `.osrm.*` dataset(`.osrm.*` because it contains several files), which is prepared by the OSRM toolchain.
* You can create such a `.osrm.*` dataset by running the OSRM binaries we ship in `node_modules/osrm/lib/binding/` and default
* profiles (e.g. for setting speeds and determining road types to route on) in `node_modules/osrm/profiles/`:
*
* node_modules/osrm/lib/binding/osrm-extract data.osm.pbf -p node_modules/osrm/profiles/car.lua
@ -64,7 +64,7 @@ NAN_MODULE_INIT(Engine::Init)
*
* Consult the [osrm-backend](https://github.com/Project-OSRM/osrm-backend) documentation for further details.
*
* Once you have a complete `network.osrm` file, you can calculate routes in javascript with this object.
* Once you have a complete `network.osrm.*` dataset, you can calculate routes in javascript with this object.
*
* ```javascript
* var osrm = new OSRM('network.osrm');

6
src/tools/components.cpp
View File

@ -39,12 +39,8 @@ std::size_t loadGraph(const std::string &path,
std::vector<TarjanEdge> &graph_edge_list)
{
std::vector<extractor::NodeBasedEdge> edge_list;
std::vector<extractor::NodeBasedEdgeAnnotation> annotation_data;
auto nop = boost::make_function_output_iterator([](auto) {});
extractor::files::readRawNBGraph(
path, nop, coordinate_list, osm_node_ids, edge_list, annotation_data);
extractor::files::readRawNBGraph(path, coordinate_list, osm_node_ids, edge_list);
// Building a node-based graph
for (const auto &input_edge : edge_list)

2
src/tools/customize.cpp
View File

@ -77,7 +77,7 @@ return_code parseArguments(int argc,
hidden_options.add_options()(
"input,i",
boost::program_options::value<boost::filesystem::path>(&customization_config.base_path),
"Input file in .osrm format");
"Input base file path");
// positional option
boost::program_options::positional_options_description positional_options;

7
src/tools/extract.cpp
View File

@ -74,7 +74,12 @@ return_code parseArguments(int argc,
boost::program_options::bool_switch(&extractor_config.use_locations_cache)
->implicit_value(false)
->default_value(true),
"Use internal nodes locations cache for location-dependent data lookups");
"Use internal nodes locations cache for location-dependent data lookups")(
"dump-nbg-graph",
boost::program_options::bool_switch(&extractor_config.dump_nbg_graph)
->implicit_value(true)
->default_value(false),
"Dump raw node-based graph to *.osrm file for debug purposes.");
bool dummy;
// hidden options, will be allowed on command line, but will not be

2
src/tools/partition.cpp
View File

@ -119,7 +119,7 @@ return_code parseArguments(int argc,
hidden_options.add_options()(
"input,i",
boost::program_options::value<boost::filesystem::path>(&config.base_path),
"Input file in .osrm format");
"Input base file path");
// positional option
boost::program_options::positional_options_description positional_options;

6
test/data/Makefile
View File

@ -22,15 +22,15 @@ clean:
ch/$(DATA_NAME).osrm: $(DATA_NAME).osrm
mkdir -p ch
cp $(DATA_NAME).osrm $(DATA_NAME).osrm.* ch/
cp $(DATA_NAME).osrm.* ch/
corech/$(DATA_NAME).osrm: $(DATA_NAME).osrm
mkdir -p corech
cp $(DATA_NAME).osrm $(DATA_NAME).osrm.* corech/
cp $(DATA_NAME).osrm.* corech/
mld/$(DATA_NAME).osrm: $(DATA_NAME).osrm
mkdir -p mld
cp $(DATA_NAME).osrm $(DATA_NAME).osrm.* mld/
cp $(DATA_NAME).osrm.* mld/
$(DATA_NAME).osrm: $(DATA_NAME).osm.pbf $(DATA_NAME).poly $(PROFILE) $(OSRM_EXTRACT)
@echo "Verifiyng data file integrity..."