Add CRC checksums to EBG and turns data

This commit is contained in:
Michael Krasnyk
2018-02-01 10:00:15 -05:00
parent 10de243556
commit 9e93f198ae
25 changed files with 432 additions and 110 deletions
+5 -3
View File
@@ -76,8 +76,9 @@ int Contractor::Run()
std::vector<extractor::EdgeBasedEdge> edge_based_edge_list;
updater::Updater updater(config.updater_config);
EdgeID number_of_edge_based_nodes =
updater.LoadAndUpdateEdgeExpandedGraph(edge_based_edge_list, node_weights);
std::uint32_t connectivity_checksum = 0;
EdgeID number_of_edge_based_nodes = updater.LoadAndUpdateEdgeExpandedGraph(
edge_based_edge_list, node_weights, connectivity_checksum);
// Contracting the edge-expanded graph
@@ -109,7 +110,8 @@ int Contractor::Run()
util::Log() << "Contracted graph has " << query_graph.GetNumberOfEdges() << " edges.";
util::Log() << "Contraction took " << TIMER_SEC(contraction) << " sec";
files::writeGraph(config.GetPath(".osrm.hsgr"), checksum, query_graph, edge_filters);
files::writeGraph(
config.GetPath(".osrm.hsgr"), checksum, query_graph, edge_filters, connectivity_checksum);
TIMER_STOP(preparing);
+7 -4
View File
@@ -83,13 +83,15 @@ void CellStorageStatistics(const Graph &graph,
}
auto LoadAndUpdateEdgeExpandedGraph(const CustomizationConfig &config,
const partitioner::MultiLevelPartition &mlp)
const partitioner::MultiLevelPartition &mlp,
std::uint32_t &connectivity_checksum)
{
updater::Updater updater(config.updater_config);
EdgeID num_nodes;
std::vector<extractor::EdgeBasedEdge> edge_based_edge_list;
std::tie(num_nodes, edge_based_edge_list) = updater.LoadAndUpdateEdgeExpandedGraph();
std::tie(num_nodes, edge_based_edge_list, connectivity_checksum) =
updater.LoadAndUpdateEdgeExpandedGraph();
auto directed = partitioner::splitBidirectionalEdges(edge_based_edge_list);
auto tidied =
@@ -127,7 +129,8 @@ int Customizer::Run(const CustomizationConfig &config)
partitioner::MultiLevelPartition mlp;
partitioner::files::readPartition(config.GetPath(".osrm.partition"), mlp);
auto graph = LoadAndUpdateEdgeExpandedGraph(config, mlp);
std::uint32_t connectivity_checksum = 0;
auto graph = LoadAndUpdateEdgeExpandedGraph(config, mlp, connectivity_checksum);
util::Log() << "Loaded edge based graph: " << graph.GetNumberOfEdges() << " edges, "
<< graph.GetNumberOfNodes() << " nodes";
@@ -155,7 +158,7 @@ int Customizer::Run(const CustomizationConfig &config)
util::Log() << "MLD customization writing took " << TIMER_SEC(writing_mld_data) << " seconds";
TIMER_START(writing_graph);
partitioner::files::writeGraph(config.GetPath(".osrm.mldgr"), graph);
partitioner::files::writeGraph(config.GetPath(".osrm.mldgr"), graph, connectivity_checksum);
TIMER_STOP(writing_graph);
util::Log() << "Graph writing took " << TIMER_SEC(writing_graph) << " seconds";
+35 -16
View File
@@ -11,6 +11,7 @@
#include "util/assert.hpp"
#include "util/bearing.hpp"
#include "util/connectivity_checksum.hpp"
#include "util/coordinate.hpp"
#include "util/coordinate_calculation.hpp"
#include "util/exception.hpp"
@@ -20,10 +21,10 @@
#include "util/timing_util.hpp"
#include <boost/assert.hpp>
#include <boost/crc.hpp>
#include <boost/functional/hash.hpp>
#include <boost/numeric/conversion/cast.hpp>
#include "boost/unordered_map.hpp"
#include <algorithm>
#include <cmath>
#include <iomanip>
@@ -70,11 +71,12 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory(
const util::NameTable &name_table,
const std::unordered_set<EdgeID> &segregated_edges,
const extractor::LaneDescriptionMap &lane_description_map)
: m_edge_based_node_container(node_data_container), m_number_of_edge_based_nodes(0),
m_coordinates(coordinates), m_node_based_graph(std::move(node_based_graph)),
m_barrier_nodes(barrier_nodes), m_traffic_lights(traffic_lights),
m_compressed_edge_container(compressed_edge_container), name_table(name_table),
segregated_edges(segregated_edges), lane_description_map(lane_description_map)
: m_edge_based_node_container(node_data_container), m_connectivity_checksum(0),
m_number_of_edge_based_nodes(0), m_coordinates(coordinates),
m_node_based_graph(std::move(node_based_graph)), m_barrier_nodes(barrier_nodes),
m_traffic_lights(traffic_lights), m_compressed_edge_container(compressed_edge_container),
name_table(name_table), segregated_edges(segregated_edges),
lane_description_map(lane_description_map)
{
}
@@ -104,6 +106,11 @@ void EdgeBasedGraphFactory::GetEdgeBasedNodeWeights(std::vector<EdgeWeight> &out
swap(m_edge_based_node_weights, output_node_weights);
}
std::uint32_t EdgeBasedGraphFactory::GetConnectivityChecksum() const
{
return m_connectivity_checksum;
}
std::uint64_t EdgeBasedGraphFactory::GetNumberOfEdgeBasedNodes() const
{
return m_number_of_edge_based_nodes;
@@ -474,9 +481,13 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
std::vector<EdgeWithData> continuous_data; // may need this
std::vector<EdgeWithData> delayed_data; // may need this
std::vector<Conditional> conditionals;
util::ConnectivityChecksum checksum;
};
using EdgesPipelineBufferPtr = std::shared_ptr<EdgesPipelineBuffer>;
m_connectivity_checksum = 0;
// going over all nodes (which form the center of an intersection), we compute all possible
// turns along these intersections.
NodeID current_node = 0;
@@ -486,8 +497,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
// serial final stage time to complete its tasks.
const constexpr unsigned GRAINSIZE = 100;
// First part of the pipeline generates iterator ranges of IDs in sets of
// GRAINSIZE
// First part of the pipeline generates iterator ranges of IDs in sets of GRAINSIZE
tbb::filter_t<void, tbb::blocked_range<NodeID>> generator_stage(
tbb::filter::serial_in_order, [&](tbb::flow_control &fc) {
if (current_node < node_count)
@@ -669,6 +679,9 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
mergable_road_detector,
intersection_node);
buffer->checksum.process_byte(incoming_edges.size());
buffer->checksum.process_byte(outgoing_edges.size());
// all nodes in the graph are connected in both directions. We check all
// outgoing nodes to find the incoming edge. This is a larger search overhead,
// but the cost we need to pay to generate edges here is worth the additional
@@ -709,14 +722,18 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
for (const auto &outgoing_edge : outgoing_edges)
{
if (!intersection::isTurnAllowed(m_node_based_graph,
m_edge_based_node_container,
node_restriction_map,
m_barrier_nodes,
edge_geometries,
turn_lanes_data,
incoming_edge,
outgoing_edge))
auto is_turn_allowed =
intersection::isTurnAllowed(m_node_based_graph,
m_edge_based_node_container,
node_restriction_map,
m_barrier_nodes,
edge_geometries,
turn_lanes_data,
incoming_edge,
outgoing_edge);
buffer->checksum.process_bit(is_turn_allowed);
if (!is_turn_allowed)
continue;
const auto turn =
@@ -954,6 +971,8 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
routing_progress.PrintAddition(buffer->nodes_processed);
m_connectivity_checksum = buffer->checksum.update_checksum(m_connectivity_checksum);
// Copy data from local buffers into global EBG data
std::for_each(
buffer->continuous_data.begin(), buffer->continuous_data.end(), transfer_data);
+15 -7
View File
@@ -217,6 +217,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
util::DeallocatingVector<EdgeBasedEdge> edge_based_edge_list;
std::vector<bool> node_is_startpoint;
std::vector<EdgeWeight> edge_based_node_weights;
std::uint32_t ebg_connectivity_checksum = 0;
// Create a node-based graph from the OSRM file
NodeBasedGraphFactory node_based_graph_factory(config.GetPath(".osrm"),
@@ -295,7 +296,8 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
edge_based_node_segments,
node_is_startpoint,
edge_based_node_weights,
edge_based_edge_list);
edge_based_edge_list,
ebg_connectivity_checksum);
ProcessGuidanceTurns(node_based_graph,
edge_based_nodes_container,
@@ -341,8 +343,10 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
util::Log() << "Writing edge-based-graph edges ... " << std::flush;
TIMER_START(write_edges);
files::writeEdgeBasedGraph(
config.GetPath(".osrm.ebg"), number_of_edge_based_nodes, edge_based_edge_list);
files::writeEdgeBasedGraph(config.GetPath(".osrm.ebg"),
number_of_edge_based_nodes,
edge_based_edge_list,
ebg_connectivity_checksum);
TIMER_STOP(write_edges);
util::Log() << "ok, after " << TIMER_SEC(write_edges) << "s";
@@ -697,7 +701,8 @@ EdgeID Extractor::BuildEdgeExpandedGraph(
std::vector<EdgeBasedNodeSegment> &edge_based_node_segments,
std::vector<bool> &node_is_startpoint,
std::vector<EdgeWeight> &edge_based_node_weights,
util::DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list)
util::DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list,
std::uint32_t &connectivity_checksum)
{
EdgeBasedGraphFactory edge_based_graph_factory(node_based_graph,
edge_based_nodes_container,
@@ -743,6 +748,7 @@ EdgeID Extractor::BuildEdgeExpandedGraph(
edge_based_graph_factory.GetEdgeBasedNodeSegments(edge_based_node_segments);
edge_based_graph_factory.GetStartPointMarkers(node_is_startpoint);
edge_based_graph_factory.GetEdgeBasedNodeWeights(edge_based_node_weights);
connectivity_checksum = edge_based_graph_factory.GetConnectivityChecksum();
return number_of_edge_based_nodes;
}
@@ -864,6 +870,7 @@ void Extractor::ProcessGuidanceTurns(
osrm::guidance::BearingClassesVector bearing_class_by_node_based_node;
osrm::guidance::BearingClassesMap bearing_class_hash;
osrm::guidance::EntryClassesMap entry_class_hash;
std::uint32_t connectivity_checksum = 0;
TIMER_START(turn_annotations);
@@ -894,7 +901,8 @@ void Extractor::ProcessGuidanceTurns(
turn_data_container,
bearing_class_by_node_based_node,
bearing_class_hash,
entry_class_hash);
entry_class_hash,
connectivity_checksum);
}
TIMER_STOP(turn_annotations);
@@ -925,8 +933,8 @@ void Extractor::ProcessGuidanceTurns(
config.GetPath(".osrm.tls"), turn_lane_offsets, turn_lane_masks);
}
osrm::guidance::files::writeTurnData(config.GetPath(".osrm.edges").string(),
turn_data_container);
osrm::guidance::files::writeTurnData(
config.GetPath(".osrm.edges").string(), turn_data_container, connectivity_checksum);
TIMER_STOP(write_guidance_data);
util::Log() << "ok, after " << TIMER_SEC(write_guidance_data) << "s";
}
+14 -1
View File
@@ -5,6 +5,7 @@
#include "extractor/intersection/intersection_analysis.hpp"
#include "util/assert.hpp"
#include "util/connectivity_checksum.hpp"
#include "util/percent.hpp"
#include <tbb/blocked_range.h>
@@ -31,7 +32,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
guidance::TurnDataExternalContainer &turn_data_container,
BearingClassesVector &bearing_class_by_node_based_node,
BearingClassesMap &bearing_class_hash,
EntryClassesMap &entry_class_hash)
EntryClassesMap &entry_class_hash,
std::uint32_t &connectivity_checksum)
{
util::Log() << "Generating guidance turns ";
@@ -75,6 +77,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
std::vector<guidance::TurnData> continuous_turn_data; // populate answers from guidance
std::vector<guidance::TurnData> delayed_turn_data; // populate answers from guidance
util::ConnectivityChecksum checksum;
};
using TurnsPipelineBufferPtr = std::shared_ptr<TurnsPipelineBuffer>;
@@ -84,6 +88,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
const NodeID node_count = node_based_graph.GetNumberOfNodes();
NodeID current_node = 0;
connectivity_checksum = 0;
// Handle intersections in sets of 100. The pipeline below has a serial bottleneck
// during the writing phase, so we want to make the parallel workers do more work
// to give the serial final stage time to complete its tasks.
@@ -136,6 +142,9 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
const auto &edge_geometries = edge_geometries_and_merged_edges.first;
const auto &merged_edge_ids = edge_geometries_and_merged_edges.second;
buffer->checksum.process_byte(incoming_edges.size());
buffer->checksum.process_byte(outgoing_edges.size());
// all nodes in the graph are connected in both directions. We check all
// outgoing nodes to find the incoming edge. This is a larger search overhead,
// but the cost we need to pay to generate edges here is worth the additional
@@ -208,6 +217,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
incoming_edge,
outgoing_edge);
buffer->checksum.process_bit(is_turn_allowed);
if (!is_turn_allowed)
continue;
@@ -291,6 +302,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
guidance_progress.PrintAddition(buffer->nodes_processed);
connectivity_checksum = buffer->checksum.update_checksum(connectivity_checksum);
// Guidance data
std::for_each(buffer->continuous_turn_data.begin(),
buffer->continuous_turn_data.end(),
+2 -1
View File
@@ -228,7 +228,8 @@ int Partitioner::Run(const PartitionerConfig &config)
files::writeCells(config.GetPath(".osrm.cells"), storage);
extractor::files::writeEdgeBasedGraph(config.GetPath(".osrm.ebg"),
edge_based_graph.GetNumberOfNodes(),
graphToEdges(edge_based_graph));
graphToEdges(edge_based_graph),
edge_based_graph.connectivity_checksum);
TIMER_STOP(writing_mld_data);
util::Log() << "MLD data writing took " << TIMER_SEC(writing_mld_data) << " seconds";
+71 -41
View File
@@ -572,45 +572,11 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
{
BOOST_ASSERT(memory_ptr != nullptr);
// Connectivity matrix checksum
std::uint32_t turns_connectivity_checksum = 0;
// read actual data into shared memory object //
// Load the HSGR file
if (boost::filesystem::exists(config.GetPath(".osrm.hsgr")))
{
auto graph_nodes_ptr = layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
memory_ptr, storage::DataLayout::CH_GRAPH_NODE_LIST);
auto graph_edges_ptr = layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
memory_ptr, storage::DataLayout::CH_GRAPH_EDGE_LIST);
auto checksum = layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
util::vector_view<contractor::QueryGraphView::NodeArrayEntry> node_list(
graph_nodes_ptr, layout.num_entries[storage::DataLayout::CH_GRAPH_NODE_LIST]);
util::vector_view<contractor::QueryGraphView::EdgeArrayEntry> edge_list(
graph_edges_ptr, layout.num_entries[storage::DataLayout::CH_GRAPH_EDGE_LIST]);
std::vector<util::vector_view<bool>> edge_filter;
for (auto index : util::irange<std::size_t>(0, NUM_METRICS))
{
auto block_id =
static_cast<DataLayout::BlockID>(storage::DataLayout::CH_EDGE_FILTER_0 + index);
auto data_ptr = layout.GetBlockPtr<unsigned, true>(memory_ptr, block_id);
auto num_entries = layout.num_entries[block_id];
edge_filter.emplace_back(data_ptr, num_entries);
}
contractor::QueryGraphView graph_view(std::move(node_list), std::move(edge_list));
contractor::files::readGraph(
config.GetPath(".osrm.hsgr"), *checksum, graph_view, edge_filter);
}
else
{
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
memory_ptr, DataLayout::CH_GRAPH_NODE_LIST);
layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
memory_ptr, DataLayout::CH_GRAPH_EDGE_LIST);
}
// store the filename of the on-disk portion of the RTree
{
const auto file_index_path_ptr =
@@ -724,7 +690,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
std::move(pre_turn_bearings),
std::move(post_turn_bearings));
guidance::files::readTurnData(config.GetPath(".osrm.edges"), turn_data);
guidance::files::readTurnData(
config.GetPath(".osrm.edges"), turn_data, turns_connectivity_checksum);
}
// load compressed geometry
@@ -921,8 +888,60 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
config.GetPath(".osrm.icd"), intersection_bearings_view, entry_classes);
}
{
// Loading MLD Data
{ // Load the HSGR file
if (boost::filesystem::exists(config.GetPath(".osrm.hsgr")))
{
auto graph_nodes_ptr =
layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
memory_ptr, storage::DataLayout::CH_GRAPH_NODE_LIST);
auto graph_edges_ptr =
layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
memory_ptr, storage::DataLayout::CH_GRAPH_EDGE_LIST);
auto checksum =
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
util::vector_view<contractor::QueryGraphView::NodeArrayEntry> node_list(
graph_nodes_ptr, layout.num_entries[storage::DataLayout::CH_GRAPH_NODE_LIST]);
util::vector_view<contractor::QueryGraphView::EdgeArrayEntry> edge_list(
graph_edges_ptr, layout.num_entries[storage::DataLayout::CH_GRAPH_EDGE_LIST]);
std::vector<util::vector_view<bool>> edge_filter;
for (auto index : util::irange<std::size_t>(0, NUM_METRICS))
{
auto block_id =
static_cast<DataLayout::BlockID>(storage::DataLayout::CH_EDGE_FILTER_0 + index);
auto data_ptr = layout.GetBlockPtr<unsigned, true>(memory_ptr, block_id);
auto num_entries = layout.num_entries[block_id];
edge_filter.emplace_back(data_ptr, num_entries);
}
std::uint32_t graph_connectivity_checksum = 0;
contractor::QueryGraphView graph_view(std::move(node_list), std::move(edge_list));
contractor::files::readGraph(config.GetPath(".osrm.hsgr"),
*checksum,
graph_view,
edge_filter,
graph_connectivity_checksum);
if (turns_connectivity_checksum != graph_connectivity_checksum)
{
throw util::exception(
"Connectivity checksum " + std::to_string(graph_connectivity_checksum) +
" in " + config.GetPath(".osrm.hsgr").string() +
" does not equal to checksum " + std::to_string(turns_connectivity_checksum) +
" in " + config.GetPath(".osrm.edges").string());
}
}
else
{
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
memory_ptr, DataLayout::CH_GRAPH_NODE_LIST);
layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
memory_ptr, DataLayout::CH_GRAPH_EDGE_LIST);
}
}
{ // Loading MLD Data
if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
{
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_LEVEL_DATA) > 0);
@@ -1040,9 +1059,20 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
graph_node_to_offset_ptr,
layout.num_entries[storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET]);
std::uint32_t graph_connectivity_checksum = 0;
customizer::MultiLevelEdgeBasedGraphView graph_view(
std::move(node_list), std::move(edge_list), std::move(node_to_offset));
partitioner::files::readGraph(config.GetPath(".osrm.mldgr"), graph_view);
partitioner::files::readGraph(
config.GetPath(".osrm.mldgr"), graph_view, graph_connectivity_checksum);
if (turns_connectivity_checksum != graph_connectivity_checksum)
{
throw util::exception(
"Connectivity checksum " + std::to_string(graph_connectivity_checksum) +
" in " + config.GetPath(".osrm.mldgr").string() +
" does not equal to checksum " + std::to_string(turns_connectivity_checksum) +
" in " + config.GetPath(".osrm.edges").string());
}
}
}
}
+11 -6
View File
@@ -524,14 +524,17 @@ Updater::NumNodesAndEdges Updater::LoadAndUpdateEdgeExpandedGraph() const
{
std::vector<EdgeWeight> node_weights;
std::vector<extractor::EdgeBasedEdge> edge_based_edge_list;
auto number_of_edge_based_nodes =
Updater::LoadAndUpdateEdgeExpandedGraph(edge_based_edge_list, node_weights);
return std::make_tuple(number_of_edge_based_nodes, std::move(edge_based_edge_list));
std::uint32_t connectivity_checksum;
auto number_of_edge_based_nodes = Updater::LoadAndUpdateEdgeExpandedGraph(
edge_based_edge_list, node_weights, connectivity_checksum);
return std::make_tuple(
number_of_edge_based_nodes, std::move(edge_based_edge_list), connectivity_checksum);
}
EdgeID
Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &edge_based_edge_list,
std::vector<EdgeWeight> &node_weights) const
std::vector<EdgeWeight> &node_weights,
std::uint32_t &connectivity_checksum) const
{
TIMER_START(load_edges);
@@ -539,8 +542,10 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
std::vector<util::Coordinate> coordinates;
extractor::PackedOSMIDs osm_node_ids;
extractor::files::readEdgeBasedGraph(
config.GetPath(".osrm.ebg"), number_of_edge_based_nodes, edge_based_edge_list);
extractor::files::readEdgeBasedGraph(config.GetPath(".osrm.ebg"),
number_of_edge_based_nodes,
edge_based_edge_list,
connectivity_checksum);
extractor::files::readNodes(config.GetPath(".osrm.nbg_nodes"), coordinates, osm_node_ids);
const bool update_conditional_turns =