Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add CRC checksums to EBG and turns data

Browse Source
This commit is contained in:
Michael Krasnyk 2018-02-01 10:00:15 -05:00
parent 10de243556
commit 9e93f198ae
25 changed files with 432 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CMakeLists.txt
View File

@ -679,7 +679,8 @@ set(UTIL_LIBRARIES
${CMAKE_THREAD_LIBS_INIT}
${MAYBE_STXXL_LIBRARY}
${TBB_LIBRARIES}
${MAYBE_COVERAGE_LIBRARIES})
${MAYBE_COVERAGE_LIBRARIES}
${ZLIB_LIBRARY})
# Libraries
target_link_libraries(osrm ${ENGINE_LIBRARIES})

8
include/contractor/files.hpp
View File

@ -19,7 +19,8 @@ template <typename QueryGraphT, typename EdgeFilterT>
inline void readGraph(const boost::filesystem::path &path,
unsigned &checksum,
QueryGraphT &graph,
std::vector<EdgeFilterT> &edge_filter)
std::vector<EdgeFilterT> &edge_filter,
std::uint32_t &connectivity_checksum)
{
static_assert(std::is_same<QueryGraphView, QueryGraphT>::value ||
std::is_same<QueryGraph, QueryGraphT>::value,
@ -39,6 +40,7 @@ inline void readGraph(const boost::filesystem::path &path,
{
storage::serialization::read(reader, edge_filter[index]);
}
reader.ReadInto(connectivity_checksum);
}
// writes .osrm.hsgr file
@ -46,7 +48,8 @@ template <typename QueryGraphT, typename EdgeFilterT>
inline void writeGraph(const boost::filesystem::path &path,
unsigned checksum,
const QueryGraphT &graph,
const std::vector<EdgeFilterT> &edge_filter)
const std::vector<EdgeFilterT> &edge_filter,
const std::uint32_t connectivity_checksum)
{
static_assert(std::is_same<QueryGraphView, QueryGraphT>::value ||
std::is_same<QueryGraph, QueryGraphT>::value,
@ -64,6 +67,7 @@ inline void writeGraph(const boost::filesystem::path &path,
{
storage::serialization::write(writer, filter);
}
writer.WriteOne(connectivity_checksum);
}
}
}

2
include/extractor/edge_based_graph_factory.hpp
View File

@ -92,6 +92,7 @@ class EdgeBasedGraphFactory
void GetEdgeBasedNodeSegments(std::vector<EdgeBasedNodeSegment> &nodes);
void GetStartPointMarkers(std::vector<bool> &node_is_startpoint);
void GetEdgeBasedNodeWeights(std::vector<EdgeWeight> &output_node_weights);
std::uint32_t GetConnectivityChecksum() const;
std::uint64_t GetNumberOfEdgeBasedNodes() const;
@ -133,6 +134,7 @@ class EdgeBasedGraphFactory
std::vector<EdgeBasedNodeSegment> m_edge_based_node_segments;
EdgeBasedNodeDataContainer &m_edge_based_node_container;
util::DeallocatingVector<EdgeBasedEdge> m_edge_based_edge_list;
std::uint32_t m_connectivity_checksum;
// The number of edge-based nodes is mostly made up out of the edges in the node-based graph.
// Any edge in the node-based graph represents a node in the edge-based graph. In addition, we

3
include/extractor/extractor.hpp
View File

@ -84,7 +84,8 @@ class Extractor
std::vector<EdgeBasedNodeSegment> &edge_based_node_segments,
std::vector<bool> &node_is_startpoint,
std::vector<EdgeWeight> &edge_based_node_weights,
util::DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list);
util::DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list,
std::uint32_t &connectivity_checksum);
void FindComponents(unsigned max_edge_id,
const util::DeallocatingVector<EdgeBasedEdge> &input_edge_list,

8
include/extractor/files.hpp
View File

@ -78,7 +78,8 @@ inline void writeProfileProperties(const boost::filesystem::path &path,
template <typename EdgeBasedEdgeVector>
void writeEdgeBasedGraph(const boost::filesystem::path &path,
EdgeID const number_of_edge_based_nodes,
const EdgeBasedEdgeVector &edge_based_edge_list)
const EdgeBasedEdgeVector &edge_based_edge_list,
const std::uint32_t connectivity_checksum)
{
static_assert(std::is_same<typename EdgeBasedEdgeVector::value_type, EdgeBasedEdge>::value, "");
@ -86,12 +87,14 @@ void writeEdgeBasedGraph(const boost::filesystem::path &path,
writer.WriteElementCount64(number_of_edge_based_nodes);
storage::serialization::write(writer, edge_based_edge_list);
writer.WriteOne(connectivity_checksum);
}
template <typename EdgeBasedEdgeVector>
void readEdgeBasedGraph(const boost::filesystem::path &path,
EdgeID &number_of_edge_based_nodes,
EdgeBasedEdgeVector &edge_based_edge_list)
EdgeBasedEdgeVector &edge_based_edge_list,
std::uint32_t &connectivity_checksum)
{
static_assert(std::is_same<typename EdgeBasedEdgeVector::value_type, EdgeBasedEdge>::value, "");
@ -99,6 +102,7 @@ void readEdgeBasedGraph(const boost::filesystem::path &path,
number_of_edge_based_nodes = reader.ReadElementCount64();
storage::serialization::read(reader, edge_based_edge_list);
reader.ReadInto(connectivity_checksum);
}
// reads .osrm.nodes

12
include/guidance/files.hpp
View File

@ -19,7 +19,9 @@ namespace files
// reads .osrm.edges
template <typename TurnDataT>
inline void readTurnData(const boost::filesystem::path &path, TurnDataT &turn_data)
inline void readTurnData(const boost::filesystem::path &path,
TurnDataT &turn_data,
std::uint32_t &connectivity_checksum)
{
static_assert(std::is_same<guidance::TurnDataContainer, TurnDataT>::value ||
std::is_same<guidance::TurnDataView, TurnDataT>::value ||
@ -28,12 +30,14 @@ inline void readTurnData(const boost::filesystem::path &path, TurnDataT &turn_da
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
serialization::read(reader, turn_data);
serialization::read(reader, turn_data, connectivity_checksum);
}
// writes .osrm.edges
template <typename TurnDataT>
inline void writeTurnData(const boost::filesystem::path &path, const TurnDataT &turn_data)
inline void writeTurnData(const boost::filesystem::path &path,
const TurnDataT &turn_data,
const std::uint32_t connectivity_checksum)
{
static_assert(std::is_same<guidance::TurnDataContainer, TurnDataT>::value ||
std::is_same<guidance::TurnDataView, TurnDataT>::value ||
@ -42,7 +46,7 @@ inline void writeTurnData(const boost::filesystem::path &path, const TurnDataT &
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
serialization::write(writer, turn_data);
serialization::write(writer, turn_data, connectivity_checksum);
}
}
}

3
include/guidance/guidance_processing.hpp
View File

@ -41,7 +41,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
guidance::TurnDataExternalContainer &turn_data_container,
BearingClassesVector &bearing_class_by_node_based_node,
BearingClassesMap &bearing_class_hash,
EntryClassesMap &entry_class_hash);
EntryClassesMap &entry_class_hash,
std::uint32_t &connectivity_checksum);
} // namespace customizer
} // namespace osrm

8
include/guidance/serialization.hpp
View File

@ -18,24 +18,28 @@ namespace serialization
// read/write for turn data file
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader,
guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container)
guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container,
std::uint32_t &connectivity_checksum)
{
storage::serialization::read(reader, turn_data_container.turn_instructions);
storage::serialization::read(reader, turn_data_container.lane_data_ids);
storage::serialization::read(reader, turn_data_container.entry_class_ids);
storage::serialization::read(reader, turn_data_container.pre_turn_bearings);
storage::serialization::read(reader, turn_data_container.post_turn_bearings);
reader.ReadInto(connectivity_checksum);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container)
const guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container,
const std::uint32_t connectivity_checksum)
{
storage::serialization::write(writer, turn_data_container.turn_instructions);
storage::serialization::write(writer, turn_data_container.lane_data_ids);
storage::serialization::write(writer, turn_data_container.entry_class_ids);
storage::serialization::write(writer, turn_data_container.pre_turn_bearings);
storage::serialization::write(writer, turn_data_container.post_turn_bearings);
writer.WriteOne(connectivity_checksum);
}
}
}

13
include/guidance/turn_data_container.hpp
View File

@ -24,11 +24,14 @@ template <storage::Ownership Ownership> class TurnDataContainerImpl;
namespace serialization
{
template <storage::Ownership Ownership>
void read(storage::io::FileReader &reader, detail::TurnDataContainerImpl<Ownership> &turn_data);
void read(storage::io::FileReader &reader,
detail::TurnDataContainerImpl<Ownership> &turn_data,
std::uint32_t &connectivity_checksum);
template <storage::Ownership Ownership>
void write(storage::io::FileWriter &writer,
const detail::TurnDataContainerImpl<Ownership> &turn_data);
const detail::TurnDataContainerImpl<Ownership> &turn_data,
const std::uint32_t connectivity_checksum);
}
struct TurnData
@ -95,9 +98,11 @@ template <storage::Ownership Ownership> class TurnDataContainerImpl
}
friend void serialization::read<Ownership>(storage::io::FileReader &reader,
TurnDataContainerImpl &turn_data_container);
TurnDataContainerImpl &turn_data_container,
std::uint32_t &connectivity_checksum);
friend void serialization::write<Ownership>(storage::io::FileWriter &writer,
const TurnDataContainerImpl &turn_data_container);
const TurnDataContainerImpl &turn_data_container,
const std::uint32_t connectivity_checksum);
private:
Vector<TurnInstruction> turn_instructions;

10
include/partitioner/edge_based_graph.hpp
View File

@ -36,6 +36,16 @@ struct DynamicEdgeBasedGraph : util::DynamicGraph<EdgeBasedGraphEdgeData>
{
using Base = util::DynamicGraph<EdgeBasedGraphEdgeData>;
using Base::Base;
template <class ContainerT>
DynamicEdgeBasedGraph(const NodeIterator nodes,
const ContainerT &graph,
std::uint32_t connectivity_checksum)
: Base(nodes, graph), connectivity_checksum(connectivity_checksum)
{
}
std::uint32_t connectivity_checksum;
};
struct DynamicEdgeBasedGraphEdge : DynamicEdgeBasedGraph::InputEdge

5
include/partitioner/edge_based_graph_reader.hpp
View File

@ -187,12 +187,13 @@ inline DynamicEdgeBasedGraph LoadEdgeBasedGraph(const boost::filesystem::path &p
{
EdgeID number_of_edge_based_nodes;
std::vector<extractor::EdgeBasedEdge> edges;
extractor::files::readEdgeBasedGraph(path, number_of_edge_based_nodes, edges);
std::uint32_t checksum;
extractor::files::readEdgeBasedGraph(path, number_of_edge_based_nodes, edges, checksum);
auto directed = splitBidirectionalEdges(edges);
auto tidied = prepareEdgesForUsageInGraph<DynamicEdgeBasedGraphEdge>(std::move(directed));
return DynamicEdgeBasedGraph(number_of_edge_based_nodes, std::move(tidied));
return DynamicEdgeBasedGraph(number_of_edge_based_nodes, std::move(tidied), checksum);
}
} // ns partition

12
include/partitioner/files.hpp
View File

@ -16,7 +16,9 @@ namespace files
// reads .osrm.mldgr file
template <typename MultiLevelGraphT>
inline void readGraph(const boost::filesystem::path &path, MultiLevelGraphT &graph)
inline void readGraph(const boost::filesystem::path &path,
MultiLevelGraphT &graph,
std::uint32_t &connectivity_checksum)
{
static_assert(std::is_same<customizer::MultiLevelEdgeBasedGraphView, MultiLevelGraphT>::value ||
std::is_same<customizer::MultiLevelEdgeBasedGraph, MultiLevelGraphT>::value,
@ -25,12 +27,14 @@ inline void readGraph(const boost::filesystem::path &path, MultiLevelGraphT &gra
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
serialization::read(reader, graph);
serialization::read(reader, graph, connectivity_checksum);
}
// writes .osrm.mldgr file
template <typename MultiLevelGraphT>
inline void writeGraph(const boost::filesystem::path &path, const MultiLevelGraphT &graph)
inline void writeGraph(const boost::filesystem::path &path,
const MultiLevelGraphT &graph,
const std::uint32_t connectivity_checksum)
{
static_assert(std::is_same<customizer::MultiLevelEdgeBasedGraphView, MultiLevelGraphT>::value ||
std::is_same<customizer::MultiLevelEdgeBasedGraph, MultiLevelGraphT>::value,
@ -39,7 +43,7 @@ inline void writeGraph(const boost::filesystem::path &path, const MultiLevelGrap
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
serialization::write(writer, graph);
serialization::write(writer, graph, connectivity_checksum);
}
// read .osrm.partition file

15
include/partitioner/multi_level_graph.hpp
View File

@ -24,10 +24,14 @@ template <typename EdgeDataT, storage::Ownership Ownership> class MultiLevelGrap
namespace serialization
{
template <typename EdgeDataT, storage::Ownership Ownership>
void read(storage::io::FileReader &reader, MultiLevelGraph<EdgeDataT, Ownership> &graph);
void read(storage::io::FileReader &reader,
MultiLevelGraph<EdgeDataT, Ownership> &graph,
std::uint32_t &connectivity_checksum);
template <typename EdgeDataT, storage::Ownership Ownership>
void write(storage::io::FileWriter &writer, const MultiLevelGraph<EdgeDataT, Ownership> &graph);
void write(storage::io::FileWriter &writer,
const MultiLevelGraph<EdgeDataT, Ownership> &graph,
const std::uint32_t connectivity_checksum);
}
template <typename EdgeDataT, storage::Ownership Ownership>
@ -199,12 +203,15 @@ class MultiLevelGraph : public util::StaticGraph<EdgeDataT, Ownership>
friend void
serialization::read<EdgeDataT, Ownership>(storage::io::FileReader &reader,
MultiLevelGraph<EdgeDataT, Ownership> &graph);
MultiLevelGraph<EdgeDataT, Ownership> &graph,
std::uint32_t &connectivity_checksum);
friend void
serialization::write<EdgeDataT, Ownership>(storage::io::FileWriter &writer,
const MultiLevelGraph<EdgeDataT, Ownership> &graph);
const MultiLevelGraph<EdgeDataT, Ownership> &graph,
const std::uint32_t connectivity_checksum);
Vector<EdgeOffset> node_to_edge_offset;
std::uint32_t connectivity_checksum;
};
}
}

9
include/partitioner/serialization.hpp
View File

@ -18,20 +18,25 @@ namespace serialization
{
template <typename EdgeDataT, storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, MultiLevelGraph<EdgeDataT, Ownership> &graph)
inline void read(storage::io::FileReader &reader,
MultiLevelGraph<EdgeDataT, Ownership> &graph,
std::uint32_t &connectivity_checksum)
{
storage::serialization::read(reader, graph.node_array);
storage::serialization::read(reader, graph.edge_array);
storage::serialization::read(reader, graph.node_to_edge_offset);
reader.ReadInto(connectivity_checksum);
}
template <typename EdgeDataT, storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const MultiLevelGraph<EdgeDataT, Ownership> &graph)
const MultiLevelGraph<EdgeDataT, Ownership> &graph,
const std::uint32_t connectivity_checksum)
{
storage::serialization::write(writer, graph.node_array);
storage::serialization::write(writer, graph.edge_array);
storage::serialization::write(writer, graph.node_to_edge_offset);
writer.WriteOne(connectivity_checksum);
}
template <storage::Ownership Ownership>

6
include/updater/updater.hpp
View File

@ -17,12 +17,14 @@ class Updater
public:
Updater(UpdaterConfig config_) : config(std::move(config_)) {}
using NumNodesAndEdges = std::tuple<EdgeID, std::vector<extractor::EdgeBasedEdge>>;
using NumNodesAndEdges =
std::tuple<EdgeID, std::vector<extractor::EdgeBasedEdge>, std::uint32_t>;
NumNodesAndEdges LoadAndUpdateEdgeExpandedGraph() const;
EdgeID
LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &edge_based_edge_list,
std::vector<EdgeWeight> &node_weights) const;
std::vector<EdgeWeight> &node_weights,
std::uint32_t &connectivity_checksum) const;
private:
UpdaterConfig config;

90
include/util/connectivity_checksum.hpp Normal file
View File

@ -0,0 +1,90 @@
#ifndef CONNECTIVITY_CHECKSUM_HPP
#define CONNECTIVITY_CHECKSUM_HPP
#include <zlib.h>
#include <boost/assert.hpp>
#include <array>
#include <climits>
#include <cstdint>
namespace osrm
{
namespace util
{
struct ConnectivityChecksum
{
ConnectivityChecksum() : checksum(0), length(0), byte_number(0), bit_number(0) {}
void process_byte(unsigned char byte)
{
BOOST_ASSERT(byte_number < buffer.size());
if (bit_number > 0)
{
bit_number = 0;
++byte_number;
++length;
}
flush_bytes();
buffer[byte_number] = byte;
++byte_number;
++length;
flush_bytes();
buffer[byte_number] = 0;
}
void process_bit(bool bit)
{
BOOST_ASSERT(byte_number < buffer.size());
BOOST_ASSERT(bit_number < CHAR_BIT);
buffer[byte_number] = (buffer[byte_number] << 1) | static_cast<unsigned char>(bit);
if (++bit_number >= CHAR_BIT)
{
bit_number = 0;
++byte_number;
++length;
flush_bytes();
buffer[byte_number] = 0;
}
}
std::uint32_t update_checksum(std::uint32_t current)
{
if (bit_number > 0)
{
++byte_number;
++length;
}
checksum = crc32(checksum, buffer.data(), byte_number);
checksum = crc32_combine(current, checksum, length);
length = byte_number = bit_number = 0;
buffer[byte_number] = 0;
return checksum;
}
private:
void flush_bytes()
{
if (byte_number >= buffer.size())
{
checksum = crc32(checksum, buffer.data(), buffer.size());
byte_number = 0;
}
}
std::array<unsigned char, 64> buffer;
std::uint32_t checksum;
std::size_t length;
std::size_t byte_number;
unsigned char bit_number;
};
}
}
#endif

8
src/contractor/contractor.cpp
View File

@ -76,8 +76,9 @@ int Contractor::Run()
std::vector<extractor::EdgeBasedEdge> edge_based_edge_list;
updater::Updater updater(config.updater_config);
EdgeID number_of_edge_based_nodes =
updater.LoadAndUpdateEdgeExpandedGraph(edge_based_edge_list, node_weights);
std::uint32_t connectivity_checksum = 0;
EdgeID number_of_edge_based_nodes = updater.LoadAndUpdateEdgeExpandedGraph(
edge_based_edge_list, node_weights, connectivity_checksum);
// Contracting the edge-expanded graph
@ -109,7 +110,8 @@ int Contractor::Run()
util::Log() << "Contracted graph has " << query_graph.GetNumberOfEdges() << " edges.";
util::Log() << "Contraction took " << TIMER_SEC(contraction) << " sec";
files::writeGraph(config.GetPath(".osrm.hsgr"), checksum, query_graph, edge_filters);
files::writeGraph(
config.GetPath(".osrm.hsgr"), checksum, query_graph, edge_filters, connectivity_checksum);
TIMER_STOP(preparing);

11
src/customize/customizer.cpp
View File

@ -83,13 +83,15 @@ void CellStorageStatistics(const Graph &graph,
}
auto LoadAndUpdateEdgeExpandedGraph(const CustomizationConfig &config,
const partitioner::MultiLevelPartition &mlp)
const partitioner::MultiLevelPartition &mlp,
std::uint32_t &connectivity_checksum)
{
updater::Updater updater(config.updater_config);
EdgeID num_nodes;
std::vector<extractor::EdgeBasedEdge> edge_based_edge_list;
std::tie(num_nodes, edge_based_edge_list) = updater.LoadAndUpdateEdgeExpandedGraph();
std::tie(num_nodes, edge_based_edge_list, connectivity_checksum) =
updater.LoadAndUpdateEdgeExpandedGraph();
auto directed = partitioner::splitBidirectionalEdges(edge_based_edge_list);
auto tidied =
@ -127,7 +129,8 @@ int Customizer::Run(const CustomizationConfig &config)
partitioner::MultiLevelPartition mlp;
partitioner::files::readPartition(config.GetPath(".osrm.partition"), mlp);
auto graph = LoadAndUpdateEdgeExpandedGraph(config, mlp);
std::uint32_t connectivity_checksum = 0;
auto graph = LoadAndUpdateEdgeExpandedGraph(config, mlp, connectivity_checksum);
util::Log() << "Loaded edge based graph: " << graph.GetNumberOfEdges() << " edges, "
<< graph.GetNumberOfNodes() << " nodes";
@ -155,7 +158,7 @@ int Customizer::Run(const CustomizationConfig &config)
util::Log() << "MLD customization writing took " << TIMER_SEC(writing_mld_data) << " seconds";
TIMER_START(writing_graph);
partitioner::files::writeGraph(config.GetPath(".osrm.mldgr"), graph);
partitioner::files::writeGraph(config.GetPath(".osrm.mldgr"), graph, connectivity_checksum);
TIMER_STOP(writing_graph);
util::Log() << "Graph writing took " << TIMER_SEC(writing_graph) << " seconds";

51
src/extractor/edge_based_graph_factory.cpp
View File

@ -11,6 +11,7 @@
#include "util/assert.hpp"
#include "util/bearing.hpp"
#include "util/connectivity_checksum.hpp"
#include "util/coordinate.hpp"
#include "util/coordinate_calculation.hpp"
#include "util/exception.hpp"
@ -20,10 +21,10 @@
#include "util/timing_util.hpp"
#include <boost/assert.hpp>
#include <boost/crc.hpp>
#include <boost/functional/hash.hpp>
#include <boost/numeric/conversion/cast.hpp>
#include "boost/unordered_map.hpp"
#include <algorithm>
#include <cmath>
#include <iomanip>
@ -70,11 +71,12 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory(
const util::NameTable &name_table,
const std::unordered_set<EdgeID> &segregated_edges,
const extractor::LaneDescriptionMap &lane_description_map)
: m_edge_based_node_container(node_data_container), m_number_of_edge_based_nodes(0),
m_coordinates(coordinates), m_node_based_graph(std::move(node_based_graph)),
m_barrier_nodes(barrier_nodes), m_traffic_lights(traffic_lights),
m_compressed_edge_container(compressed_edge_container), name_table(name_table),
segregated_edges(segregated_edges), lane_description_map(lane_description_map)
: m_edge_based_node_container(node_data_container), m_connectivity_checksum(0),
m_number_of_edge_based_nodes(0), m_coordinates(coordinates),
m_node_based_graph(std::move(node_based_graph)), m_barrier_nodes(barrier_nodes),
m_traffic_lights(traffic_lights), m_compressed_edge_container(compressed_edge_container),
name_table(name_table), segregated_edges(segregated_edges),
lane_description_map(lane_description_map)
{
}
@ -104,6 +106,11 @@ void EdgeBasedGraphFactory::GetEdgeBasedNodeWeights(std::vector<EdgeWeight> &out
swap(m_edge_based_node_weights, output_node_weights);
}
std::uint32_t EdgeBasedGraphFactory::GetConnectivityChecksum() const
{
return m_connectivity_checksum;
}
std::uint64_t EdgeBasedGraphFactory::GetNumberOfEdgeBasedNodes() const
{
return m_number_of_edge_based_nodes;
@ -474,9 +481,13 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
std::vector<EdgeWithData> continuous_data; // may need this
std::vector<EdgeWithData> delayed_data; // may need this
std::vector<Conditional> conditionals;
util::ConnectivityChecksum checksum;
};
using EdgesPipelineBufferPtr = std::shared_ptr<EdgesPipelineBuffer>;
m_connectivity_checksum = 0;
// going over all nodes (which form the center of an intersection), we compute all possible
// turns along these intersections.
NodeID current_node = 0;
@ -486,8 +497,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
// serial final stage time to complete its tasks.
const constexpr unsigned GRAINSIZE = 100;
// First part of the pipeline generates iterator ranges of IDs in sets of
// GRAINSIZE
// First part of the pipeline generates iterator ranges of IDs in sets of GRAINSIZE
tbb::filter_t<void, tbb::blocked_range<NodeID>> generator_stage(
tbb::filter::serial_in_order, [&](tbb::flow_control &fc) {
if (current_node < node_count)
@ -669,6 +679,9 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
mergable_road_detector,
intersection_node);
buffer->checksum.process_byte(incoming_edges.size());
buffer->checksum.process_byte(outgoing_edges.size());
// all nodes in the graph are connected in both directions. We check all
// outgoing nodes to find the incoming edge. This is a larger search overhead,
// but the cost we need to pay to generate edges here is worth the additional
@ -709,14 +722,18 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
for (const auto &outgoing_edge : outgoing_edges)
{
if (!intersection::isTurnAllowed(m_node_based_graph,
m_edge_based_node_container,
node_restriction_map,
m_barrier_nodes,
edge_geometries,
turn_lanes_data,
incoming_edge,
outgoing_edge))
auto is_turn_allowed =
intersection::isTurnAllowed(m_node_based_graph,
m_edge_based_node_container,
node_restriction_map,
m_barrier_nodes,
edge_geometries,
turn_lanes_data,
incoming_edge,
outgoing_edge);
buffer->checksum.process_bit(is_turn_allowed);
if (!is_turn_allowed)
continue;
const auto turn =
@ -954,6 +971,8 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
routing_progress.PrintAddition(buffer->nodes_processed);
m_connectivity_checksum = buffer->checksum.update_checksum(m_connectivity_checksum);
// Copy data from local buffers into global EBG data
std::for_each(
buffer->continuous_data.begin(), buffer->continuous_data.end(), transfer_data);

22
src/extractor/extractor.cpp
View File

@ -217,6 +217,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
util::DeallocatingVector<EdgeBasedEdge> edge_based_edge_list;
std::vector<bool> node_is_startpoint;
std::vector<EdgeWeight> edge_based_node_weights;
std::uint32_t ebg_connectivity_checksum = 0;
// Create a node-based graph from the OSRM file
NodeBasedGraphFactory node_based_graph_factory(config.GetPath(".osrm"),
@ -295,7 +296,8 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
edge_based_node_segments,
node_is_startpoint,
edge_based_node_weights,
edge_based_edge_list);
edge_based_edge_list,
ebg_connectivity_checksum);
ProcessGuidanceTurns(node_based_graph,
edge_based_nodes_container,
@ -341,8 +343,10 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
util::Log() << "Writing edge-based-graph edges ... " << std::flush;
TIMER_START(write_edges);
files::writeEdgeBasedGraph(
config.GetPath(".osrm.ebg"), number_of_edge_based_nodes, edge_based_edge_list);
files::writeEdgeBasedGraph(config.GetPath(".osrm.ebg"),
number_of_edge_based_nodes,
edge_based_edge_list,
ebg_connectivity_checksum);
TIMER_STOP(write_edges);
util::Log() << "ok, after " << TIMER_SEC(write_edges) << "s";
@ -697,7 +701,8 @@ EdgeID Extractor::BuildEdgeExpandedGraph(
std::vector<EdgeBasedNodeSegment> &edge_based_node_segments,
std::vector<bool> &node_is_startpoint,
std::vector<EdgeWeight> &edge_based_node_weights,
util::DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list)
util::DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list,
std::uint32_t &connectivity_checksum)
{
EdgeBasedGraphFactory edge_based_graph_factory(node_based_graph,
edge_based_nodes_container,
@ -743,6 +748,7 @@ EdgeID Extractor::BuildEdgeExpandedGraph(
edge_based_graph_factory.GetEdgeBasedNodeSegments(edge_based_node_segments);
edge_based_graph_factory.GetStartPointMarkers(node_is_startpoint);
edge_based_graph_factory.GetEdgeBasedNodeWeights(edge_based_node_weights);
connectivity_checksum = edge_based_graph_factory.GetConnectivityChecksum();
return number_of_edge_based_nodes;
}
@ -864,6 +870,7 @@ void Extractor::ProcessGuidanceTurns(
osrm::guidance::BearingClassesVector bearing_class_by_node_based_node;
osrm::guidance::BearingClassesMap bearing_class_hash;
osrm::guidance::EntryClassesMap entry_class_hash;
std::uint32_t connectivity_checksum = 0;
TIMER_START(turn_annotations);
@ -894,7 +901,8 @@ void Extractor::ProcessGuidanceTurns(
turn_data_container,
bearing_class_by_node_based_node,
bearing_class_hash,
entry_class_hash);
entry_class_hash,
connectivity_checksum);
}
TIMER_STOP(turn_annotations);
@ -925,8 +933,8 @@ void Extractor::ProcessGuidanceTurns(
config.GetPath(".osrm.tls"), turn_lane_offsets, turn_lane_masks);
}
osrm::guidance::files::writeTurnData(config.GetPath(".osrm.edges").string(),
turn_data_container);
osrm::guidance::files::writeTurnData(
config.GetPath(".osrm.edges").string(), turn_data_container, connectivity_checksum);
TIMER_STOP(write_guidance_data);
util::Log() << "ok, after " << TIMER_SEC(write_guidance_data) << "s";
}

15
src/guidance/guidance_processing.cpp
View File

@ -5,6 +5,7 @@
#include "extractor/intersection/intersection_analysis.hpp"
#include "util/assert.hpp"
#include "util/connectivity_checksum.hpp"
#include "util/percent.hpp"
#include <tbb/blocked_range.h>
@ -31,7 +32,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
guidance::TurnDataExternalContainer &turn_data_container,
BearingClassesVector &bearing_class_by_node_based_node,
BearingClassesMap &bearing_class_hash,
EntryClassesMap &entry_class_hash)
EntryClassesMap &entry_class_hash,
std::uint32_t &connectivity_checksum)
{
util::Log() << "Generating guidance turns ";
@ -75,6 +77,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
std::vector<guidance::TurnData> continuous_turn_data; // populate answers from guidance
std::vector<guidance::TurnData> delayed_turn_data; // populate answers from guidance
util::ConnectivityChecksum checksum;
};
using TurnsPipelineBufferPtr = std::shared_ptr<TurnsPipelineBuffer>;
@ -84,6 +88,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
const NodeID node_count = node_based_graph.GetNumberOfNodes();
NodeID current_node = 0;
connectivity_checksum = 0;
// Handle intersections in sets of 100. The pipeline below has a serial bottleneck
// during the writing phase, so we want to make the parallel workers do more work
// to give the serial final stage time to complete its tasks.
@ -136,6 +142,9 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
const auto &edge_geometries = edge_geometries_and_merged_edges.first;
const auto &merged_edge_ids = edge_geometries_and_merged_edges.second;
buffer->checksum.process_byte(incoming_edges.size());
buffer->checksum.process_byte(outgoing_edges.size());
// all nodes in the graph are connected in both directions. We check all
// outgoing nodes to find the incoming edge. This is a larger search overhead,
// but the cost we need to pay to generate edges here is worth the additional
@ -208,6 +217,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
incoming_edge,
outgoing_edge);
buffer->checksum.process_bit(is_turn_allowed);
if (!is_turn_allowed)
continue;
@ -291,6 +302,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
guidance_progress.PrintAddition(buffer->nodes_processed);
connectivity_checksum = buffer->checksum.update_checksum(connectivity_checksum);
// Guidance data
std::for_each(buffer->continuous_turn_data.begin(),
buffer->continuous_turn_data.end(),

3
src/partitioner/partitioner.cpp
View File

@ -228,7 +228,8 @@ int Partitioner::Run(const PartitionerConfig &config)
files::writeCells(config.GetPath(".osrm.cells"), storage);
extractor::files::writeEdgeBasedGraph(config.GetPath(".osrm.ebg"),
edge_based_graph.GetNumberOfNodes(),
graphToEdges(edge_based_graph));
graphToEdges(edge_based_graph),
edge_based_graph.connectivity_checksum);
TIMER_STOP(writing_mld_data);
util::Log() << "MLD data writing took " << TIMER_SEC(writing_mld_data) << " seconds";

112
src/storage/storage.cpp
View File

@ -572,45 +572,11 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
{
BOOST_ASSERT(memory_ptr != nullptr);
// Connectivity matrix checksum
std::uint32_t turns_connectivity_checksum = 0;
// read actual data into shared memory object //
// Load the HSGR file
if (boost::filesystem::exists(config.GetPath(".osrm.hsgr")))
{
auto graph_nodes_ptr = layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
memory_ptr, storage::DataLayout::CH_GRAPH_NODE_LIST);
auto graph_edges_ptr = layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
memory_ptr, storage::DataLayout::CH_GRAPH_EDGE_LIST);
auto checksum = layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
util::vector_view<contractor::QueryGraphView::NodeArrayEntry> node_list(
graph_nodes_ptr, layout.num_entries[storage::DataLayout::CH_GRAPH_NODE_LIST]);
util::vector_view<contractor::QueryGraphView::EdgeArrayEntry> edge_list(
graph_edges_ptr, layout.num_entries[storage::DataLayout::CH_GRAPH_EDGE_LIST]);
std::vector<util::vector_view<bool>> edge_filter;
for (auto index : util::irange<std::size_t>(0, NUM_METRICS))
{
auto block_id =
static_cast<DataLayout::BlockID>(storage::DataLayout::CH_EDGE_FILTER_0 + index);
auto data_ptr = layout.GetBlockPtr<unsigned, true>(memory_ptr, block_id);
auto num_entries = layout.num_entries[block_id];
edge_filter.emplace_back(data_ptr, num_entries);
}
contractor::QueryGraphView graph_view(std::move(node_list), std::move(edge_list));
contractor::files::readGraph(
config.GetPath(".osrm.hsgr"), *checksum, graph_view, edge_filter);
}
else
{
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
memory_ptr, DataLayout::CH_GRAPH_NODE_LIST);
layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
memory_ptr, DataLayout::CH_GRAPH_EDGE_LIST);
}
// store the filename of the on-disk portion of the RTree
{
const auto file_index_path_ptr =
@ -724,7 +690,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
std::move(pre_turn_bearings),
std::move(post_turn_bearings));
guidance::files::readTurnData(config.GetPath(".osrm.edges"), turn_data);
guidance::files::readTurnData(
config.GetPath(".osrm.edges"), turn_data, turns_connectivity_checksum);
}
// load compressed geometry
@ -921,8 +888,60 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
config.GetPath(".osrm.icd"), intersection_bearings_view, entry_classes);
}
{
// Loading MLD Data
{ // Load the HSGR file
if (boost::filesystem::exists(config.GetPath(".osrm.hsgr")))
{
auto graph_nodes_ptr =
layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
memory_ptr, storage::DataLayout::CH_GRAPH_NODE_LIST);
auto graph_edges_ptr =
layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
memory_ptr, storage::DataLayout::CH_GRAPH_EDGE_LIST);
auto checksum =
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
util::vector_view<contractor::QueryGraphView::NodeArrayEntry> node_list(
graph_nodes_ptr, layout.num_entries[storage::DataLayout::CH_GRAPH_NODE_LIST]);
util::vector_view<contractor::QueryGraphView::EdgeArrayEntry> edge_list(
graph_edges_ptr, layout.num_entries[storage::DataLayout::CH_GRAPH_EDGE_LIST]);
std::vector<util::vector_view<bool>> edge_filter;
for (auto index : util::irange<std::size_t>(0, NUM_METRICS))
{
auto block_id =
static_cast<DataLayout::BlockID>(storage::DataLayout::CH_EDGE_FILTER_0 + index);
auto data_ptr = layout.GetBlockPtr<unsigned, true>(memory_ptr, block_id);
auto num_entries = layout.num_entries[block_id];
edge_filter.emplace_back(data_ptr, num_entries);
}
std::uint32_t graph_connectivity_checksum = 0;
contractor::QueryGraphView graph_view(std::move(node_list), std::move(edge_list));
contractor::files::readGraph(config.GetPath(".osrm.hsgr"),
*checksum,
graph_view,
edge_filter,
graph_connectivity_checksum);
if (turns_connectivity_checksum != graph_connectivity_checksum)
{
throw util::exception(
"Connectivity checksum " + std::to_string(graph_connectivity_checksum) +
" in " + config.GetPath(".osrm.hsgr").string() +
" does not equal to checksum " + std::to_string(turns_connectivity_checksum) +
" in " + config.GetPath(".osrm.edges").string());
}
}
else
{
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
memory_ptr, DataLayout::CH_GRAPH_NODE_LIST);
layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
memory_ptr, DataLayout::CH_GRAPH_EDGE_LIST);
}
}
{ // Loading MLD Data
if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
{
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_LEVEL_DATA) > 0);
@ -1040,9 +1059,20 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
graph_node_to_offset_ptr,
layout.num_entries[storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET]);
std::uint32_t graph_connectivity_checksum = 0;
customizer::MultiLevelEdgeBasedGraphView graph_view(
std::move(node_list), std::move(edge_list), std::move(node_to_offset));
partitioner::files::readGraph(config.GetPath(".osrm.mldgr"), graph_view);
partitioner::files::readGraph(
config.GetPath(".osrm.mldgr"), graph_view, graph_connectivity_checksum);
if (turns_connectivity_checksum != graph_connectivity_checksum)
{
throw util::exception(
"Connectivity checksum " + std::to_string(graph_connectivity_checksum) +
" in " + config.GetPath(".osrm.mldgr").string() +
" does not equal to checksum " + std::to_string(turns_connectivity_checksum) +
" in " + config.GetPath(".osrm.edges").string());
}
}
}
}

17
src/updater/updater.cpp
View File

@ -524,14 +524,17 @@ Updater::NumNodesAndEdges Updater::LoadAndUpdateEdgeExpandedGraph() const
{
std::vector<EdgeWeight> node_weights;
std::vector<extractor::EdgeBasedEdge> edge_based_edge_list;
auto number_of_edge_based_nodes =
Updater::LoadAndUpdateEdgeExpandedGraph(edge_based_edge_list, node_weights);
return std::make_tuple(number_of_edge_based_nodes, std::move(edge_based_edge_list));
std::uint32_t connectivity_checksum;
auto number_of_edge_based_nodes = Updater::LoadAndUpdateEdgeExpandedGraph(
edge_based_edge_list, node_weights, connectivity_checksum);
return std::make_tuple(
number_of_edge_based_nodes, std::move(edge_based_edge_list), connectivity_checksum);
}
EdgeID
Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &edge_based_edge_list,
std::vector<EdgeWeight> &node_weights) const
std::vector<EdgeWeight> &node_weights,
std::uint32_t &connectivity_checksum) const
{
TIMER_START(load_edges);
@ -539,8 +542,10 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
std::vector<util::Coordinate> coordinates;
extractor::PackedOSMIDs osm_node_ids;
extractor::files::readEdgeBasedGraph(
config.GetPath(".osrm.ebg"), number_of_edge_based_nodes, edge_based_edge_list);
extractor::files::readEdgeBasedGraph(config.GetPath(".osrm.ebg"),
number_of_edge_based_nodes,
edge_based_edge_list,
connectivity_checksum);
extractor::files::readNodes(config.GetPath(".osrm.nbg_nodes"), coordinates, osm_node_ids);
const bool update_conditional_turns =

96
unit_tests/util/connectivity_checksum.cpp Normal file
View File

@ -0,0 +1,96 @@
#include "util/connectivity_checksum.hpp"
#include <boost/test/test_case_template.hpp>
#include <boost/test/unit_test.hpp>
BOOST_AUTO_TEST_SUITE(connectivity_checksum)
using namespace osrm;
using namespace osrm::util;
BOOST_AUTO_TEST_CASE(connectivity_checksum)
{
ConnectivityChecksum checksum;
BOOST_CHECK_EQUAL(checksum.update_checksum(42), 42);
checksum.process_bit(1);
BOOST_CHECK_EQUAL(checksum.update_checksum(0), 0x7ebe16cd);
for (std::size_t i = 0; i < CHAR_BIT; ++i)
checksum.process_bit(1);
BOOST_CHECK_EQUAL(checksum.update_checksum(0), 0x1aab001b);
checksum.process_byte(1);
BOOST_CHECK_EQUAL(checksum.update_checksum(0), 0x2f7abdf7);
}
BOOST_AUTO_TEST_CASE(connectivity_checksum_bytes_bits)
{
ConnectivityChecksum checksum1, checksum2;
checksum1.process_bit(1); // A
checksum1.process_bit(0);
checksum1.process_bit(1);
checksum1.process_bit(0);
checksum1.process_bit(1); // 9
checksum1.process_bit(0);
checksum1.process_bit(0);
checksum1.process_bit(1);
checksum1.process_bit(1); // 5
checksum1.process_bit(0);
checksum1.process_bit(1);
checksum2.process_byte(0xA9);
checksum2.process_byte(0x05);
BOOST_CHECK_EQUAL(checksum1.update_checksum(0), checksum2.update_checksum(0));
}
BOOST_AUTO_TEST_CASE(connectivity_checksum_diff_size)
{
ConnectivityChecksum checksum1, checksum2;
checksum1.process_byte(1);
checksum1.process_byte(2);
checksum1.process_byte(3);
checksum1.process_byte(4);
checksum1.process_byte(5);
checksum1.process_bit(1);
checksum2.process_byte(1);
checksum2.process_byte(2);
checksum2.process_byte(3);
checksum2.update_checksum(0);
checksum2.process_byte(4);
checksum2.process_byte(5);
checksum2.process_bit(1);
BOOST_CHECK_EQUAL(checksum1.update_checksum(0), checksum2.update_checksum(0));
}
BOOST_AUTO_TEST_CASE(connectivity_checksum_parallel)
{
ConnectivityChecksum expected, checksum1, checksum2;
expected.process_byte(1);
expected.process_byte(2);
expected.process_byte(3);
expected.process_byte(4);
expected.process_byte(5);
expected.process_bit(1);
checksum1.process_byte(1);
checksum1.process_byte(2);
checksum1.process_byte(3);
checksum2.process_byte(4);
checksum2.process_byte(5);
checksum2.process_bit(1);
BOOST_CHECK_EQUAL(expected.update_checksum(0),
checksum2.update_checksum(checksum1.update_checksum(0)));
}
BOOST_AUTO_TEST_SUITE_END()