Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add CRC checksums to EBG and turns data

Browse Source
This commit is contained in:
Michael Krasnyk
2018-02-01 10:00:15 -05:00
parent 10de243556
commit 9e93f198ae
25 changed files with 432 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/contractor/files.hpp
+6 -2
View File
@@ -19,7 +19,8 @@ template <typename QueryGraphT, typename EdgeFilterT>
inline void readGraph(const boost::filesystem::path &path,
unsigned &checksum,
QueryGraphT &graph,
std::vector<EdgeFilterT> &edge_filter)
std::vector<EdgeFilterT> &edge_filter,
std::uint32_t &connectivity_checksum)
{
static_assert(std::is_same<QueryGraphView, QueryGraphT>::value ||
std::is_same<QueryGraph, QueryGraphT>::value,
@@ -39,6 +40,7 @@ inline void readGraph(const boost::filesystem::path &path,
{
storage::serialization::read(reader, edge_filter[index]);
}
reader.ReadInto(connectivity_checksum);
}
// writes .osrm.hsgr file
@@ -46,7 +48,8 @@ template <typename QueryGraphT, typename EdgeFilterT>
inline void writeGraph(const boost::filesystem::path &path,
unsigned checksum,
const QueryGraphT &graph,
const std::vector<EdgeFilterT> &edge_filter)
const std::vector<EdgeFilterT> &edge_filter,
const std::uint32_t connectivity_checksum)
{
static_assert(std::is_same<QueryGraphView, QueryGraphT>::value ||
std::is_same<QueryGraph, QueryGraphT>::value,
@@ -64,6 +67,7 @@ inline void writeGraph(const boost::filesystem::path &path,
{
storage::serialization::write(writer, filter);
}
writer.WriteOne(connectivity_checksum);
}
}
}
include/extractor/edge_based_graph_factory.hpp
+2
View File
@@ -92,6 +92,7 @@ class EdgeBasedGraphFactory
void GetEdgeBasedNodeSegments(std::vector<EdgeBasedNodeSegment> &nodes);
void GetStartPointMarkers(std::vector<bool> &node_is_startpoint);
void GetEdgeBasedNodeWeights(std::vector<EdgeWeight> &output_node_weights);
std::uint32_t GetConnectivityChecksum() const;
std::uint64_t GetNumberOfEdgeBasedNodes() const;
@@ -133,6 +134,7 @@ class EdgeBasedGraphFactory
std::vector<EdgeBasedNodeSegment> m_edge_based_node_segments;
EdgeBasedNodeDataContainer &m_edge_based_node_container;
util::DeallocatingVector<EdgeBasedEdge> m_edge_based_edge_list;
std::uint32_t m_connectivity_checksum;
// The number of edge-based nodes is mostly made up out of the edges in the node-based graph.
// Any edge in the node-based graph represents a node in the edge-based graph. In addition, we
include/extractor/extractor.hpp
+2 -1
View File
@@ -84,7 +84,8 @@ class Extractor
std::vector<EdgeBasedNodeSegment> &edge_based_node_segments,
std::vector<bool> &node_is_startpoint,
std::vector<EdgeWeight> &edge_based_node_weights,
util::DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list);
util::DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list,
std::uint32_t &connectivity_checksum);
void FindComponents(unsigned max_edge_id,
const util::DeallocatingVector<EdgeBasedEdge> &input_edge_list,
include/extractor/files.hpp
+6 -2
View File
@@ -78,7 +78,8 @@ inline void writeProfileProperties(const boost::filesystem::path &path,
template <typename EdgeBasedEdgeVector>
void writeEdgeBasedGraph(const boost::filesystem::path &path,
EdgeID const number_of_edge_based_nodes,
const EdgeBasedEdgeVector &edge_based_edge_list)
const EdgeBasedEdgeVector &edge_based_edge_list,
const std::uint32_t connectivity_checksum)
{
static_assert(std::is_same<typename EdgeBasedEdgeVector::value_type, EdgeBasedEdge>::value, "");
@@ -86,12 +87,14 @@ void writeEdgeBasedGraph(const boost::filesystem::path &path,
writer.WriteElementCount64(number_of_edge_based_nodes);
storage::serialization::write(writer, edge_based_edge_list);
writer.WriteOne(connectivity_checksum);
}
template <typename EdgeBasedEdgeVector>
void readEdgeBasedGraph(const boost::filesystem::path &path,
EdgeID &number_of_edge_based_nodes,
EdgeBasedEdgeVector &edge_based_edge_list)
EdgeBasedEdgeVector &edge_based_edge_list,
std::uint32_t &connectivity_checksum)
{
static_assert(std::is_same<typename EdgeBasedEdgeVector::value_type, EdgeBasedEdge>::value, "");
@@ -99,6 +102,7 @@ void readEdgeBasedGraph(const boost::filesystem::path &path,
number_of_edge_based_nodes = reader.ReadElementCount64();
storage::serialization::read(reader, edge_based_edge_list);
reader.ReadInto(connectivity_checksum);
}
// reads .osrm.nodes
include/guidance/files.hpp
+8 -4
View File
@@ -19,7 +19,9 @@ namespace files
// reads .osrm.edges
template <typename TurnDataT>
inline void readTurnData(const boost::filesystem::path &path, TurnDataT &turn_data)
inline void readTurnData(const boost::filesystem::path &path,
TurnDataT &turn_data,
std::uint32_t &connectivity_checksum)
{
static_assert(std::is_same<guidance::TurnDataContainer, TurnDataT>::value ||
std::is_same<guidance::TurnDataView, TurnDataT>::value ||
@@ -28,12 +30,14 @@ inline void readTurnData(const boost::filesystem::path &path, TurnDataT &turn_da
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
serialization::read(reader, turn_data);
serialization::read(reader, turn_data, connectivity_checksum);
}
// writes .osrm.edges
template <typename TurnDataT>
inline void writeTurnData(const boost::filesystem::path &path, const TurnDataT &turn_data)
inline void writeTurnData(const boost::filesystem::path &path,
const TurnDataT &turn_data,
const std::uint32_t connectivity_checksum)
{
static_assert(std::is_same<guidance::TurnDataContainer, TurnDataT>::value ||
std::is_same<guidance::TurnDataView, TurnDataT>::value ||
@@ -42,7 +46,7 @@ inline void writeTurnData(const boost::filesystem::path &path, const TurnDataT &
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
serialization::write(writer, turn_data);
serialization::write(writer, turn_data, connectivity_checksum);
}
}
}
include/guidance/guidance_processing.hpp
+2 -1
View File
@@ -41,7 +41,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
guidance::TurnDataExternalContainer &turn_data_container,
BearingClassesVector &bearing_class_by_node_based_node,
BearingClassesMap &bearing_class_hash,
EntryClassesMap &entry_class_hash);
EntryClassesMap &entry_class_hash,
std::uint32_t &connectivity_checksum);
} // namespace customizer
} // namespace osrm
include/guidance/serialization.hpp
+6 -2
View File
@@ -18,24 +18,28 @@ namespace serialization
// read/write for turn data file
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader,
guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container)
guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container,
std::uint32_t &connectivity_checksum)
{
storage::serialization::read(reader, turn_data_container.turn_instructions);
storage::serialization::read(reader, turn_data_container.lane_data_ids);
storage::serialization::read(reader, turn_data_container.entry_class_ids);
storage::serialization::read(reader, turn_data_container.pre_turn_bearings);
storage::serialization::read(reader, turn_data_container.post_turn_bearings);
reader.ReadInto(connectivity_checksum);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container)
const guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container,
const std::uint32_t connectivity_checksum)
{
storage::serialization::write(writer, turn_data_container.turn_instructions);
storage::serialization::write(writer, turn_data_container.lane_data_ids);
storage::serialization::write(writer, turn_data_container.entry_class_ids);
storage::serialization::write(writer, turn_data_container.pre_turn_bearings);
storage::serialization::write(writer, turn_data_container.post_turn_bearings);
writer.WriteOne(connectivity_checksum);
}
}
}
include/guidance/turn_data_container.hpp
+9 -4
View File
@@ -24,11 +24,14 @@ template <storage::Ownership Ownership> class TurnDataContainerImpl;
namespace serialization
{
template <storage::Ownership Ownership>
void read(storage::io::FileReader &reader, detail::TurnDataContainerImpl<Ownership> &turn_data);
void read(storage::io::FileReader &reader,
detail::TurnDataContainerImpl<Ownership> &turn_data,
std::uint32_t &connectivity_checksum);
template <storage::Ownership Ownership>
void write(storage::io::FileWriter &writer,
const detail::TurnDataContainerImpl<Ownership> &turn_data);
const detail::TurnDataContainerImpl<Ownership> &turn_data,
const std::uint32_t connectivity_checksum);
}
struct TurnData
@@ -95,9 +98,11 @@ template <storage::Ownership Ownership> class TurnDataContainerImpl
}
friend void serialization::read<Ownership>(storage::io::FileReader &reader,
TurnDataContainerImpl &turn_data_container);
TurnDataContainerImpl &turn_data_container,
std::uint32_t &connectivity_checksum);
friend void serialization::write<Ownership>(storage::io::FileWriter &writer,
const TurnDataContainerImpl &turn_data_container);
const TurnDataContainerImpl &turn_data_container,
const std::uint32_t connectivity_checksum);
private:
Vector<TurnInstruction> turn_instructions;
include/partitioner/edge_based_graph.hpp
+10
View File
@@ -36,6 +36,16 @@ struct DynamicEdgeBasedGraph : util::DynamicGraph<EdgeBasedGraphEdgeData>
{
using Base = util::DynamicGraph<EdgeBasedGraphEdgeData>;
using Base::Base;
template <class ContainerT>
DynamicEdgeBasedGraph(const NodeIterator nodes,
const ContainerT &graph,
std::uint32_t connectivity_checksum)
: Base(nodes, graph), connectivity_checksum(connectivity_checksum)
{
}
std::uint32_t connectivity_checksum;
};
struct DynamicEdgeBasedGraphEdge : DynamicEdgeBasedGraph::InputEdge
include/partitioner/edge_based_graph_reader.hpp
+3 -2
View File
@@ -187,12 +187,13 @@ inline DynamicEdgeBasedGraph LoadEdgeBasedGraph(const boost::filesystem::path &p
{
EdgeID number_of_edge_based_nodes;
std::vector<extractor::EdgeBasedEdge> edges;
extractor::files::readEdgeBasedGraph(path, number_of_edge_based_nodes, edges);
std::uint32_t checksum;
extractor::files::readEdgeBasedGraph(path, number_of_edge_based_nodes, edges, checksum);
auto directed = splitBidirectionalEdges(edges);
auto tidied = prepareEdgesForUsageInGraph<DynamicEdgeBasedGraphEdge>(std::move(directed));
return DynamicEdgeBasedGraph(number_of_edge_based_nodes, std::move(tidied));
return DynamicEdgeBasedGraph(number_of_edge_based_nodes, std::move(tidied), checksum);
}
} // ns partition
include/partitioner/files.hpp
+8 -4
View File
@@ -16,7 +16,9 @@ namespace files
// reads .osrm.mldgr file
template <typename MultiLevelGraphT>
inline void readGraph(const boost::filesystem::path &path, MultiLevelGraphT &graph)
inline void readGraph(const boost::filesystem::path &path,
MultiLevelGraphT &graph,
std::uint32_t &connectivity_checksum)
{
static_assert(std::is_same<customizer::MultiLevelEdgeBasedGraphView, MultiLevelGraphT>::value ||
std::is_same<customizer::MultiLevelEdgeBasedGraph, MultiLevelGraphT>::value,
@@ -25,12 +27,14 @@ inline void readGraph(const boost::filesystem::path &path, MultiLevelGraphT &gra
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
serialization::read(reader, graph);
serialization::read(reader, graph, connectivity_checksum);
}
// writes .osrm.mldgr file
template <typename MultiLevelGraphT>
inline void writeGraph(const boost::filesystem::path &path, const MultiLevelGraphT &graph)
inline void writeGraph(const boost::filesystem::path &path,
const MultiLevelGraphT &graph,
const std::uint32_t connectivity_checksum)
{
static_assert(std::is_same<customizer::MultiLevelEdgeBasedGraphView, MultiLevelGraphT>::value ||
std::is_same<customizer::MultiLevelEdgeBasedGraph, MultiLevelGraphT>::value,
@@ -39,7 +43,7 @@ inline void writeGraph(const boost::filesystem::path &path, const MultiLevelGrap
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
serialization::write(writer, graph);
serialization::write(writer, graph, connectivity_checksum);
}
// read .osrm.partition file
include/partitioner/multi_level_graph.hpp
+11 -4
View File
@@ -24,10 +24,14 @@ template <typename EdgeDataT, storage::Ownership Ownership> class MultiLevelGrap
namespace serialization
{
template <typename EdgeDataT, storage::Ownership Ownership>
void read(storage::io::FileReader &reader, MultiLevelGraph<EdgeDataT, Ownership> &graph);
void read(storage::io::FileReader &reader,
MultiLevelGraph<EdgeDataT, Ownership> &graph,
std::uint32_t &connectivity_checksum);
template <typename EdgeDataT, storage::Ownership Ownership>
void write(storage::io::FileWriter &writer, const MultiLevelGraph<EdgeDataT, Ownership> &graph);
void write(storage::io::FileWriter &writer,
const MultiLevelGraph<EdgeDataT, Ownership> &graph,
const std::uint32_t connectivity_checksum);
}
template <typename EdgeDataT, storage::Ownership Ownership>
@@ -199,12 +203,15 @@ class MultiLevelGraph : public util::StaticGraph<EdgeDataT, Ownership>
friend void
serialization::read<EdgeDataT, Ownership>(storage::io::FileReader &reader,
MultiLevelGraph<EdgeDataT, Ownership> &graph);
MultiLevelGraph<EdgeDataT, Ownership> &graph,
std::uint32_t &connectivity_checksum);
friend void
serialization::write<EdgeDataT, Ownership>(storage::io::FileWriter &writer,
const MultiLevelGraph<EdgeDataT, Ownership> &graph);
const MultiLevelGraph<EdgeDataT, Ownership> &graph,
const std::uint32_t connectivity_checksum);
Vector<EdgeOffset> node_to_edge_offset;
std::uint32_t connectivity_checksum;
};
}
}
include/partitioner/serialization.hpp
+7 -2
View File
@@ -18,20 +18,25 @@ namespace serialization
{
template <typename EdgeDataT, storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, MultiLevelGraph<EdgeDataT, Ownership> &graph)
inline void read(storage::io::FileReader &reader,
MultiLevelGraph<EdgeDataT, Ownership> &graph,
std::uint32_t &connectivity_checksum)
{
storage::serialization::read(reader, graph.node_array);
storage::serialization::read(reader, graph.edge_array);
storage::serialization::read(reader, graph.node_to_edge_offset);
reader.ReadInto(connectivity_checksum);
}
template <typename EdgeDataT, storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const MultiLevelGraph<EdgeDataT, Ownership> &graph)
const MultiLevelGraph<EdgeDataT, Ownership> &graph,
const std::uint32_t connectivity_checksum)
{
storage::serialization::write(writer, graph.node_array);
storage::serialization::write(writer, graph.edge_array);
storage::serialization::write(writer, graph.node_to_edge_offset);
writer.WriteOne(connectivity_checksum);
}
template <storage::Ownership Ownership>
include/updater/updater.hpp
+4 -2
View File
@@ -17,12 +17,14 @@ class Updater
public:
Updater(UpdaterConfig config_) : config(std::move(config_)) {}
using NumNodesAndEdges = std::tuple<EdgeID, std::vector<extractor::EdgeBasedEdge>>;
using NumNodesAndEdges =
std::tuple<EdgeID, std::vector<extractor::EdgeBasedEdge>, std::uint32_t>;
NumNodesAndEdges LoadAndUpdateEdgeExpandedGraph() const;
EdgeID
LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &edge_based_edge_list,
std::vector<EdgeWeight> &node_weights) const;
std::vector<EdgeWeight> &node_weights,
std::uint32_t &connectivity_checksum) const;
private:
UpdaterConfig config;
include/util/connectivity_checksum.hpp
+90
View File
@@ -0,0 +1,90 @@
#ifndef CONNECTIVITY_CHECKSUM_HPP
#define CONNECTIVITY_CHECKSUM_HPP
#include <zlib.h>
#include <boost/assert.hpp>
#include <array>
#include <climits>
#include <cstdint>
namespace osrm
{
namespace util
{
struct ConnectivityChecksum
{
ConnectivityChecksum() : checksum(0), length(0), byte_number(0), bit_number(0) {}
void process_byte(unsigned char byte)
{
BOOST_ASSERT(byte_number < buffer.size());
if (bit_number > 0)
{
bit_number = 0;
++byte_number;
++length;
}
flush_bytes();
buffer[byte_number] = byte;
++byte_number;
++length;
flush_bytes();
buffer[byte_number] = 0;
}
void process_bit(bool bit)
{
BOOST_ASSERT(byte_number < buffer.size());
BOOST_ASSERT(bit_number < CHAR_BIT);
buffer[byte_number] = (buffer[byte_number] << 1) | static_cast<unsigned char>(bit);
if (++bit_number >= CHAR_BIT)
{
bit_number = 0;
++byte_number;
++length;
flush_bytes();
buffer[byte_number] = 0;
}
}
std::uint32_t update_checksum(std::uint32_t current)
{
if (bit_number > 0)
{
++byte_number;
++length;
}
checksum = crc32(checksum, buffer.data(), byte_number);
checksum = crc32_combine(current, checksum, length);
length = byte_number = bit_number = 0;
buffer[byte_number] = 0;
return checksum;
}
private:
void flush_bytes()
{
if (byte_number >= buffer.size())
{
checksum = crc32(checksum, buffer.data(), buffer.size());
byte_number = 0;
}
}
std::array<unsigned char, 64> buffer;
std::uint32_t checksum;
std::size_t length;
std::size_t byte_number;
unsigned char bit_number;
};
}
}
#endif