Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

write distances out in a seperate file for mld

Browse Source
This commit is contained in:
Kajari Ghosh 2018-09-26 01:25:06 -04:00
parent 9b7bb7b458
commit 1e2c40337c
9 changed files with 360 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/extractor/edge_based_graph_factory.hpp
View File

@ -92,6 +92,7 @@ class EdgeBasedGraphFactory
void GetStartPointMarkers(std::vector<bool> &node_is_startpoint);
void GetEdgeBasedNodeWeights(std::vector<EdgeWeight> &output_node_weights);
void GetEdgeBasedNodeDurations(std::vector<EdgeWeight> &output_node_durations);
void GetEdgeBasedNodeDistances(std::vector<EdgeDistance> &output_node_distances);
std::uint32_t GetConnectivityChecksum() const;
std::uint64_t GetNumberOfEdgeBasedNodes() const;
@ -119,6 +120,7 @@ class EdgeBasedGraphFactory
//! edge-based node
std::vector<EdgeWeight> m_edge_based_node_weights;
std::vector<EdgeDuration> m_edge_based_node_durations;
std::vector<EdgeDistance> m_edge_based_node_distances;
//! list of edge based nodes (compressed segments)
std::vector<EdgeBasedNodeSegment> m_edge_based_node_segments;

1
include/extractor/extractor.hpp
View File

@ -88,6 +88,7 @@ class Extractor
std::vector<bool> &node_is_startpoint,
std::vector<EdgeWeight> &edge_based_node_weights,
std::vector<EdgeDuration> &edge_based_node_durations,
std::vector<EdgeDistance> &edge_based_node_distances,
util::DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list,
std::uint32_t &connectivity_checksum);

28
include/extractor/files.hpp
View File

@ -462,6 +462,34 @@ void readEdgeBasedNodeWeights(const boost::filesystem::path &path, NodeWeigtsVec
storage::serialization::read(reader, "/extractor/edge_based_node_weights", weights);
}
template <typename NodeWeigtsVectorT, typename NodeDurationsVectorT, typename NodeDistancesVectorT>
void readEdgeBasedNodeWeightsDurationsDistances(const boost::filesystem::path &path,
NodeWeigtsVectorT &weights,
NodeDurationsVectorT &durations,
NodeDistancesVectorT &distances)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
storage::serialization::read(reader, "/extractor/edge_based_node_weights", weights);
storage::serialization::read(reader, "/extractor/edge_based_node_durations", durations);
storage::serialization::read(reader, "/extractor/edge_based_node_distances", distances);
}
template <typename NodeWeigtsVectorT, typename NodeDurationsVectorT, typename NodeDistancesVectorT>
void writeEdgeBasedNodeWeightsDurationsDistances(const boost::filesystem::path &path,
const NodeWeigtsVectorT &weights,
const NodeDurationsVectorT &durations,
const NodeDistancesVectorT &distances)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
storage::serialization::write(writer, "/extractor/edge_based_node_weights", weights);
storage::serialization::write(writer, "/extractor/edge_based_node_durations", durations);
storage::serialization::write(writer, "/extractor/edge_based_node_distances", distances);
}
template <typename NodeWeigtsVectorT, typename NodeDurationsVectorT>
void readEdgeBasedNodeWeightsDurations(const boost::filesystem::path &path,
NodeWeigtsVectorT &weights,

6
include/updater/updater.hpp
View File

@ -27,6 +27,12 @@ class Updater
std::vector<EdgeWeight> &node_weights,
std::vector<EdgeDuration> &node_durations, // TODO: to be deleted
std::uint32_t &connectivity_checksum) const;
EdgeID
LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &edge_based_edge_list,
std::vector<EdgeWeight> &node_weights,
std::vector<EdgeDuration> &node_durations, // TODO: to be deleted
std::vector<EdgeDistance> &node_distances, // TODO: to be deleted
std::uint32_t &connectivity_checksum) const;
private:
UpdaterConfig config;

5
src/customize/customizer.cpp
View File

@ -76,13 +76,14 @@ auto LoadAndUpdateEdgeExpandedGraph(const CustomizationConfig &config,
const partitioner::MultiLevelPartition &mlp,
std::vector<EdgeWeight> &node_weights,
std::vector<EdgeDuration> &node_durations,
std::vector<EdgeDistance> &node_distances,
std::uint32_t &connectivity_checksum)
{
updater::Updater updater(config.updater_config);
std::vector<extractor::EdgeBasedEdge> edge_based_edge_list;
EdgeID num_nodes = updater.LoadAndUpdateEdgeExpandedGraph(
edge_based_edge_list, node_weights, node_durations, connectivity_checksum);
edge_based_edge_list, node_weights, node_durations, node_distances, connectivity_checksum);
auto directed = partitioner::splitBidirectionalEdges(edge_based_edge_list);
@ -129,7 +130,7 @@ int Customizer::Run(const CustomizationConfig &config)
node_distances; // TODO: is this and the above still to be removed later? dont think so
std::uint32_t connectivity_checksum = 0;
auto graph = LoadAndUpdateEdgeExpandedGraph(
config, mlp, node_weights, node_durations, connectivity_checksum);
config, mlp, node_weights, node_durations, node_distances, connectivity_checksum);
BOOST_ASSERT(graph.GetNumberOfNodes() == node_weights.size());
std::for_each(node_weights.begin(), node_weights.end(), [](auto &w) { w &= 0x7fffffff; });
util::Log() << "Loaded edge based graph: " << graph.GetNumberOfEdges() << " edges, "

12
src/extractor/edge_based_graph_factory.cpp
View File

@ -114,6 +114,13 @@ void EdgeBasedGraphFactory::GetEdgeBasedNodeDurations(
swap(m_edge_based_node_durations, output_node_durations);
}
void EdgeBasedGraphFactory::GetEdgeBasedNodeDistances(
std::vector<EdgeDistance> &output_node_distances)
{
using std::swap; // Koenig swap
swap(m_edge_based_node_distances, output_node_distances);
}
std::uint32_t EdgeBasedGraphFactory::GetConnectivityChecksum() const
{
return m_connectivity_checksum;
@ -293,6 +300,7 @@ unsigned EdgeBasedGraphFactory::LabelEdgeBasedNodes()
// (edge-based nodes)
m_edge_based_node_weights.reserve(4 * m_node_based_graph.GetNumberOfNodes());
m_edge_based_node_durations.reserve(4 * m_node_based_graph.GetNumberOfNodes());
m_edge_based_node_distances.reserve(4 * m_node_based_graph.GetNumberOfNodes());
nbe_to_ebn_mapping.resize(m_node_based_graph.GetEdgeCapacity(), SPECIAL_NODEID);
// renumber edge based node of outgoing edges
@ -310,6 +318,7 @@ unsigned EdgeBasedGraphFactory::LabelEdgeBasedNodes()
m_edge_based_node_weights.push_back(edge_data.weight);
m_edge_based_node_durations.push_back(edge_data.duration);
m_edge_based_node_distances.push_back(edge_data.distance);
BOOST_ASSERT(numbered_edges_count < m_node_based_graph.GetNumberOfEdges());
nbe_to_ebn_mapping[current_edge] = numbered_edges_count;
@ -407,6 +416,8 @@ EdgeBasedGraphFactory::GenerateEdgeExpandedNodes(const WayRestrictionMap &way_re
m_edge_based_node_weights.push_back(ebn_weight);
m_edge_based_node_durations.push_back(
m_edge_based_node_durations[nbe_to_ebn_mapping[eid]]);
m_edge_based_node_distances.push_back(
m_edge_based_node_distances[nbe_to_ebn_mapping[eid]]);
edge_based_node_id++;
progress.PrintStatus(progress_counter++);
@ -416,6 +427,7 @@ EdgeBasedGraphFactory::GenerateEdgeExpandedNodes(const WayRestrictionMap &way_re
BOOST_ASSERT(m_edge_based_node_segments.size() == m_edge_based_node_is_startpoint.size());
BOOST_ASSERT(m_number_of_edge_based_nodes == m_edge_based_node_weights.size());
BOOST_ASSERT(m_number_of_edge_based_nodes == m_edge_based_node_durations.size());
BOOST_ASSERT(m_number_of_edge_based_nodes == m_edge_based_node_distances.size());
util::Log() << "Generated " << m_number_of_edge_based_nodes << " nodes ("
<< way_restriction_map.NumberOfDuplicatedNodes()

10
src/extractor/extractor.cpp
View File

@ -242,6 +242,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
std::vector<bool> node_is_startpoint;
std::vector<EdgeWeight> edge_based_node_weights;
std::vector<EdgeDuration> edge_based_node_durations;
std::vector<EdgeDistance> edge_based_node_distances;
std::uint32_t ebg_connectivity_checksum = 0;
// Create a node-based graph from the OSRM file
@ -322,6 +323,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
node_is_startpoint,
edge_based_node_weights,
edge_based_node_durations,
edge_based_node_distances,
edge_based_edge_list,
ebg_connectivity_checksum);
@ -345,8 +347,10 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
util::Log() << "Saving edge-based node weights to file.";
TIMER_START(timer_write_node_weights);
extractor::files::writeEdgeBasedNodeWeightsDurations(
config.GetPath(".osrm.enw"), edge_based_node_weights, edge_based_node_durations);
extractor::files::writeEdgeBasedNodeWeightsDurationsDistances(config.GetPath(".osrm.enw"),
edge_based_node_weights,
edge_based_node_durations,
edge_based_node_distances);
TIMER_STOP(timer_write_node_weights);
util::Log() << "Done writing. (" << TIMER_SEC(timer_write_node_weights) << ")";
@ -736,6 +740,7 @@ EdgeID Extractor::BuildEdgeExpandedGraph(
std::vector<bool> &node_is_startpoint,
std::vector<EdgeWeight> &edge_based_node_weights,
std::vector<EdgeDuration> &edge_based_node_durations,
std::vector<EdgeDistance> &edge_based_node_distances,
util::DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list,
std::uint32_t &connectivity_checksum)
{
@ -786,6 +791,7 @@ EdgeID Extractor::BuildEdgeExpandedGraph(
edge_based_graph_factory.GetStartPointMarkers(node_is_startpoint);
edge_based_graph_factory.GetEdgeBasedNodeWeights(edge_based_node_weights);
edge_based_graph_factory.GetEdgeBasedNodeDurations(edge_based_node_durations);
edge_based_graph_factory.GetEdgeBasedNodeDistances(edge_based_node_distances);
connectivity_checksum = edge_based_graph_factory.GetConnectivityChecksum();
return number_of_edge_based_nodes;

10
src/partitioner/partitioner.cpp
View File

@ -147,12 +147,14 @@ int Partitioner::Run(const PartitionerConfig &config)
{
std::vector<EdgeWeight> node_weights;
std::vector<EdgeDuration> node_durations;
extractor::files::readEdgeBasedNodeWeightsDurations(
config.GetPath(".osrm.enw"), node_weights, node_durations);
std::vector<EdgeDuration> node_distances;
extractor::files::readEdgeBasedNodeWeightsDurationsDistances(
config.GetPath(".osrm.enw"), node_weights, node_durations, node_distances);
util::inplacePermutation(node_weights.begin(), node_weights.end(), permutation);
util::inplacePermutation(node_durations.begin(), node_durations.end(), permutation);
extractor::files::writeEdgeBasedNodeWeightsDurations(
config.GetPath(".osrm.enw"), node_weights, node_durations);
util::inplacePermutation(node_distances.begin(), node_distances.end(), permutation);
extractor::files::writeEdgeBasedNodeWeightsDurationsDistances(
config.GetPath(".osrm.enw"), node_weights, node_durations, node_distances);
}
{
const auto &filename = config.GetPath(".osrm.maneuver_overrides");

294
src/updater/updater.cpp
View File

@ -541,6 +541,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
std::vector<EdgeDuration> &node_durations,
std::uint32_t &connectivity_checksum) const
{
TIMER_START(load_edges);
EdgeID number_of_edge_based_nodes = 0;
@ -812,6 +813,299 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
});
}
#if !defined(NDEBUG)
if (config.turn_penalty_lookup_paths.empty())
{ // don't check weights consistency with turn updates that can break assertion
// condition with turn weight penalties negative updates
checkWeightsConsistency(config, edge_based_edge_list);
}
#endif
saveDatasourcesNames(config);
TIMER_STOP(load_edges);
util::Log() << "Done reading edges in " << TIMER_MSEC(load_edges) << "ms.";
return number_of_edge_based_nodes;
}
EdgeID
Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &edge_based_edge_list,
std::vector<EdgeWeight> &node_weights,
std::vector<EdgeDuration> &node_durations,
std::vector<EdgeDistance> &node_distances,
std::uint32_t &connectivity_checksum) const
{
TIMER_START(load_edges);
EdgeID number_of_edge_based_nodes = 0;
std::vector<util::Coordinate> coordinates;
extractor::PackedOSMIDs osm_node_ids;
extractor::files::readEdgeBasedNodeWeightsDurationsDistances(
config.GetPath(".osrm.enw"), node_weights, node_durations, node_distances);
extractor::files::readEdgeBasedGraph(config.GetPath(".osrm.ebg"),
number_of_edge_based_nodes,
edge_based_edge_list,
connectivity_checksum);
extractor::files::readNodes(config.GetPath(".osrm.nbg_nodes"), coordinates, osm_node_ids);
const bool update_conditional_turns =
!config.GetPath(".osrm.restrictions").empty() && config.valid_now;
const bool update_edge_weights = !config.segment_speed_lookup_paths.empty();
const bool update_turn_penalties = !config.turn_penalty_lookup_paths.empty();
if (!update_edge_weights && !update_turn_penalties && !update_conditional_turns)
{
saveDatasourcesNames(config);
return number_of_edge_based_nodes;
}
if (config.segment_speed_lookup_paths.size() + config.turn_penalty_lookup_paths.size() > 255)
throw util::exception("Limit of 255 segment speed and turn penalty files each reached" +
SOURCE_REF);
extractor::EdgeBasedNodeDataContainer node_data;
extractor::SegmentDataContainer segment_data;
extractor::ProfileProperties profile_properties;
std::vector<TurnPenalty> turn_weight_penalties;
std::vector<TurnPenalty> turn_duration_penalties;
if (update_edge_weights || update_turn_penalties || update_conditional_turns)
{
tbb::parallel_invoke(
[&] {
extractor::files::readSegmentData(config.GetPath(".osrm.geometry"), segment_data);
},
[&] { extractor::files::readNodeData(config.GetPath(".osrm.ebg_nodes"), node_data); },
[&] {
extractor::files::readTurnWeightPenalty(
config.GetPath(".osrm.turn_weight_penalties"), turn_weight_penalties);
},
[&] {
extractor::files::readTurnDurationPenalty(
config.GetPath(".osrm.turn_duration_penalties"), turn_duration_penalties);
},
[&] {
extractor::files::readProfileProperties(config.GetPath(".osrm.properties"),
profile_properties);
});
}
std::vector<extractor::ConditionalTurnPenalty> conditional_turns;
if (update_conditional_turns)
{
extractor::files::readConditionalRestrictions(config.GetPath(".osrm.restrictions"),
conditional_turns);
}
tbb::concurrent_vector<GeometryID> updated_segments;
if (update_edge_weights)
{
auto segment_speed_lookup = csv::readSegmentValues(config.segment_speed_lookup_paths);
TIMER_START(segment);
updated_segments = updateSegmentData(config,
profile_properties,
segment_speed_lookup,
segment_data,
coordinates,
osm_node_ids);
// Now save out the updated compressed geometries
extractor::files::writeSegmentData(config.GetPath(".osrm.geometry"), segment_data);
TIMER_STOP(segment);
util::Log() << "Updating segment data took " << TIMER_MSEC(segment) << "ms.";
}
auto turn_penalty_lookup = csv::readTurnValues(config.turn_penalty_lookup_paths);
if (update_turn_penalties)
{
auto updated_turn_penalties = updateTurnPenalties(config,
profile_properties,
turn_penalty_lookup,
turn_weight_penalties,
turn_duration_penalties,
osm_node_ids);
const auto offset = updated_segments.size();
updated_segments.resize(offset + updated_turn_penalties.size());
// we need to re-compute all edges that have updated turn penalties.
// this marks it for re-computation
std::transform(updated_turn_penalties.begin(),
updated_turn_penalties.end(),
updated_segments.begin() + offset,
[&node_data, &edge_based_edge_list](const std::uint64_t turn_id) {
const auto node_id = edge_based_edge_list[turn_id].source;
return node_data.GetGeometryID(node_id);
});
}
if (update_conditional_turns)
{
// initialize instance of class that handles time zone resolution
if (config.valid_now <= 0)
{
util::Log(logERROR) << "Given UTC time is invalid: " << config.valid_now;
throw;
}
const Timezoner time_zone_handler = Timezoner(config.tz_file_path, config.valid_now);
auto updated_turn_penalties =
updateConditionalTurns(turn_weight_penalties, conditional_turns, time_zone_handler);
const auto offset = updated_segments.size();
updated_segments.resize(offset + updated_turn_penalties.size());
// we need to re-compute all edges that have updated turn penalties.
// this marks it for re-computation
std::transform(updated_turn_penalties.begin(),
updated_turn_penalties.end(),
updated_segments.begin() + offset,
[&node_data, &edge_based_edge_list](const std::uint64_t turn_id) {
const auto node_id = edge_based_edge_list[turn_id].source;
return node_data.GetGeometryID(node_id);
});
}
tbb::parallel_sort(updated_segments.begin(),
updated_segments.end(),
[](const GeometryID lhs, const GeometryID rhs) {
return std::tie(lhs.id, lhs.forward) < std::tie(rhs.id, rhs.forward);
});
using WeightAndDuration = std::tuple<EdgeWeight, EdgeWeight>;
const auto compute_new_weight_and_duration =
[&](const GeometryID geometry_id) -> WeightAndDuration {
EdgeWeight new_weight = 0;
EdgeWeight new_duration = 0;
if (geometry_id.forward)
{
const auto weights = segment_data.GetForwardWeights(geometry_id.id);
for (const auto weight : weights)
{
if (weight == INVALID_SEGMENT_WEIGHT)
{
new_weight = INVALID_EDGE_WEIGHT;
break;
}
new_weight += weight;
}
const auto durations = segment_data.GetForwardDurations(geometry_id.id);
new_duration = std::accumulate(durations.begin(), durations.end(), EdgeWeight{0});
}
else
{
const auto weights = segment_data.GetReverseWeights(geometry_id.id);
for (const auto weight : weights)
{
if (weight == INVALID_SEGMENT_WEIGHT)
{
new_weight = INVALID_EDGE_WEIGHT;
break;
}
new_weight += weight;
}
const auto durations = segment_data.GetReverseDurations(geometry_id.id);
new_duration = std::accumulate(durations.begin(), durations.end(), EdgeWeight{0});
}
return std::make_tuple(new_weight, new_duration);
};
std::vector<WeightAndDuration> accumulated_segment_data(updated_segments.size());
tbb::parallel_for(tbb::blocked_range<std::size_t>(0, updated_segments.size()),
[&](const auto &range) {
for (auto index = range.begin(); index < range.end(); ++index)
{
accumulated_segment_data[index] =
compute_new_weight_and_duration(updated_segments[index]);
}
});
const auto update_edge = [&](extractor::EdgeBasedEdge &edge) {
const auto node_id = edge.source;
const auto geometry_id = node_data.GetGeometryID(node_id);
auto updated_iter = std::lower_bound(updated_segments.begin(),
updated_segments.end(),
geometry_id,
[](const GeometryID lhs, const GeometryID rhs) {
return std::tie(lhs.id, lhs.forward) <
std::tie(rhs.id, rhs.forward);
});
if (updated_iter != updated_segments.end() && updated_iter->id == geometry_id.id &&
updated_iter->forward == geometry_id.forward)
{
// Find a segment with zero speed and simultaneously compute the new edge
// weight
EdgeWeight new_weight;
EdgeWeight new_duration;
std::tie(new_weight, new_duration) =
accumulated_segment_data[updated_iter - updated_segments.begin()];
// Update the node-weight cache. This is the weight of the edge-based-node
// only, it doesn't include the turn. We may visit the same node multiple times,
// but we should always assign the same value here.
BOOST_ASSERT(edge.source < node_weights.size());
node_weights[edge.source] =
node_weights[edge.source] & 0x80000000 ? new_weight | 0x80000000 : new_weight;
node_durations[edge.source] = new_duration;
// We found a zero-speed edge, so we'll skip this whole edge-based-edge
// which
// effectively removes it from the routing network.
if (new_weight == INVALID_EDGE_WEIGHT)
{
edge.data.weight = INVALID_EDGE_WEIGHT;
return;
}
// Get the turn penalty and update to the new value if required
auto turn_weight_penalty = turn_weight_penalties[edge.data.turn_id];
auto turn_duration_penalty = turn_duration_penalties[edge.data.turn_id];
const auto num_nodes = segment_data.GetForwardGeometry(geometry_id.id).size();
const auto weight_min_value = static_cast<EdgeWeight>(num_nodes);
if (turn_weight_penalty + new_weight < weight_min_value)
{
if (turn_weight_penalty < 0)
{
util::Log(logWARNING) << "turn penalty " << turn_weight_penalty
<< " is too negative: clamping turn weight to "
<< weight_min_value;
turn_weight_penalty = weight_min_value - new_weight;
turn_weight_penalties[edge.data.turn_id] = turn_weight_penalty;
}
else
{
new_weight = weight_min_value;
}
}
// Update edge weight
edge.data.weight = new_weight + turn_weight_penalty;
edge.data.duration = new_duration + turn_duration_penalty;
}
};
if (updated_segments.size() > 0)
{
tbb::parallel_for(tbb::blocked_range<std::size_t>(0, edge_based_edge_list.size()),
[&](const auto &range) {
for (auto index = range.begin(); index < range.end(); ++index)
{
update_edge(edge_based_edge_list[index]);
}
});
}
if (update_turn_penalties || update_conditional_turns)
{
tbb::parallel_invoke(
[&] {
extractor::files::writeTurnWeightPenalty(
config.GetPath(".osrm.turn_weight_penalties"), turn_weight_penalties);
},
[&] {
extractor::files::writeTurnDurationPenalty(
config.GetPath(".osrm.turn_duration_penalties"), turn_duration_penalties);
});
}
#if !defined(NDEBUG)
if (config.turn_penalty_lookup_paths.empty())
{ // don't check weights consistency with turn updates that can break assertion