Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Fix errors when loading an incompatible dataset.

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-28 23:18:09 +00:00 committed by Patrick Niklaus
parent 24e0028afb
commit 9fc2c32408
13 changed files with 40 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
include/contractor/files.hpp
View File

@ -14,7 +14,6 @@ namespace files
// reads .osrm.hsgr file
template <typename ContractedMetricT>
inline void readGraph(const boost::filesystem::path &path,
unsigned &checksum,
std::unordered_map<std::string, ContractedMetricT> &metrics,
std::uint32_t &connectivity_checksum)
{
@ -25,7 +24,6 @@ inline void readGraph(const boost::filesystem::path &path,
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
reader.ReadInto("/ch/checksum", checksum);
reader.ReadInto("/ch/connectivity_checksum", connectivity_checksum);
for (auto &pair : metrics)
@ -38,7 +36,6 @@ inline void readGraph(const boost::filesystem::path &path,
// writes .osrm.hsgr file
template <typename ContractedMetricT>
inline void writeGraph(const boost::filesystem::path &path,
unsigned checksum,
const std::unordered_map<std::string, ContractedMetricT> &metrics,
const std::uint32_t connectivity_checksum)
{
@ -48,8 +45,6 @@ inline void writeGraph(const boost::filesystem::path &path,
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
writer.WriteElementCount64("/ch/checksum", 1);
writer.WriteFrom("/ch/checksum", checksum);
writer.WriteElementCount64("/ch/connectivity_checksum", 1);
writer.WriteFrom("/ch/connectivity_checksum", connectivity_checksum);

6
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
View File

@ -194,7 +194,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
extractor::ProfileProperties *m_profile_properties;
extractor::Datasources *m_datasources;
unsigned m_check_sum;
std::uint32_t m_check_sum;
util::vector_view<util::Coordinate> m_coordinate_list;
extractor::PackedOSMIDsView m_osmnodeid_list;
util::vector_view<std::uint32_t> m_lane_description_offsets;
@ -242,7 +242,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
void InitializeChecksumPointer(const storage::DataLayout &data_layout, char *memory_block)
{
m_check_sum = *data_layout.GetBlockPtr<unsigned>(memory_block, "/ch/checksum");
m_check_sum = *data_layout.GetBlockPtr<std::uint32_t>(memory_block, "/common/connectivity_checksum");
util::Log() << "set checksum: " << m_check_sum;
}
@ -779,7 +779,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
input_coordinate, bearing, bearing_range, approach);
}
unsigned GetCheckSum() const override final { return m_check_sum; }
std::uint32_t GetCheckSum() const override final { return m_check_sum; }
GeometryID GetGeometryIndex(const NodeID id) const override final
{

2
include/engine/datafacade/datafacade_base.hpp
View File

@ -53,7 +53,7 @@ class BaseDataFacade
BaseDataFacade() {}
virtual ~BaseDataFacade() {}
virtual unsigned GetCheckSum() const = 0;
virtual std::uint32_t GetCheckSum() const = 0;
// node and edge information access
virtual util::Coordinate GetCoordinateOfNode(const NodeID id) const = 0;

7
include/engine/datafacade_factory.hpp
View File

@ -57,6 +57,13 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
layout.List(exclude_path, std::back_inserter(exclude_prefixes));
facades.resize(exclude_prefixes.size());
if (facades.empty())
{
throw util::exception(std::string("Could not find any metrics for ") +
routing_algorithms::name<AlgorithmT>() +
" in the data. Did you load the right dataset?");
}
for (const auto &exclude_prefix : exclude_prefixes)
{
auto index_begin = exclude_prefix.find_last_of("/");

7
include/guidance/files.hpp
View File

@ -30,7 +30,8 @@ inline void readTurnData(const boost::filesystem::path &path,
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
serialization::read(reader, "/common/turn_data", turn_data, connectivity_checksum);
reader.ReadInto("/common/connectivity_checksum", connectivity_checksum);
serialization::read(reader, "/common/turn_data", turn_data);
}
// writes .osrm.edges
@ -46,7 +47,9 @@ inline void writeTurnData(const boost::filesystem::path &path,
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
serialization::write(writer, "/common/turn_data", turn_data, connectivity_checksum);
writer.WriteElementCount64("/common/connectivity_checksum", 1);
writer.WriteFrom("/common/connectivity_checksum", connectivity_checksum);
serialization::write(writer, "/common/turn_data", turn_data);
}
}
}

9
include/guidance/serialization.hpp
View File

@ -19,8 +19,7 @@ namespace serialization
template <storage::Ownership Ownership>
inline void read(storage::tar::FileReader &reader,
const std::string &name,
guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container,
std::uint32_t &connectivity_checksum)
guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container)
{
storage::serialization::read(
reader, name + "/turn_instructions", turn_data_container.turn_instructions);
@ -32,14 +31,12 @@ inline void read(storage::tar::FileReader &reader,
reader, name + "/pre_turn_bearings", turn_data_container.pre_turn_bearings);
storage::serialization::read(
reader, name + "/post_turn_bearings", turn_data_container.post_turn_bearings);
reader.ReadInto(name + "/connectivity_checksum", connectivity_checksum);
}
template <storage::Ownership Ownership>
inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container,
const std::uint32_t connectivity_checksum)
const guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container)
{
storage::serialization::write(
writer, name + "/turn_instructions", turn_data_container.turn_instructions);
@ -51,8 +48,6 @@ inline void write(storage::tar::FileWriter &writer,
writer, name + "/pre_turn_bearings", turn_data_container.pre_turn_bearings);
storage::serialization::write(
writer, name + "/post_turn_bearings", turn_data_container.post_turn_bearings);
writer.WriteElementCount64(name + "/connectivity_checksum", 1);
writer.WriteFrom(name + "/connectivity_checksum", connectivity_checksum);
}
}
}

12
include/guidance/turn_data_container.hpp
View File

@ -26,14 +26,12 @@ namespace serialization
template <storage::Ownership Ownership>
void read(storage::tar::FileReader &reader,
const std::string &name,
detail::TurnDataContainerImpl<Ownership> &turn_data,
std::uint32_t &connectivity_checksum);
detail::TurnDataContainerImpl<Ownership> &turn_data);
template <storage::Ownership Ownership>
void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::TurnDataContainerImpl<Ownership> &turn_data,
const std::uint32_t connectivity_checksum);
const detail::TurnDataContainerImpl<Ownership> &turn_data);
}
struct TurnData
@ -101,12 +99,10 @@ template <storage::Ownership Ownership> class TurnDataContainerImpl
friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
const std::string &name,
TurnDataContainerImpl &turn_data_container,
std::uint32_t &connectivity_checksum);
TurnDataContainerImpl &turn_data_container);
friend void serialization::write<Ownership>(storage::tar::FileWriter &writer,
const std::string &name,
const TurnDataContainerImpl &turn_data_container,
const std::uint32_t connectivity_checksum);
const TurnDataContainerImpl &turn_data_container);
private:
Vector<TurnInstruction> turn_instructions;

7
include/partitioner/files.hpp
View File

@ -26,7 +26,8 @@ inline void readGraph(const boost::filesystem::path &path,
storage::tar::FileReader reader{path, storage::tar::FileReader::VerifyFingerprint};
serialization::read(reader, "/mld/multilevelgraph", graph, connectivity_checksum);
reader.ReadInto("/mld/connectivity_checksum", connectivity_checksum);
serialization::read(reader, "/mld/multilevelgraph", graph);
}
// writes .osrm.mldgr file
@ -41,7 +42,9 @@ inline void writeGraph(const boost::filesystem::path &path,
storage::tar::FileWriter writer{path, storage::tar::FileWriter::GenerateFingerprint};
serialization::write(writer, "/mld/multilevelgraph", graph, connectivity_checksum);
writer.WriteElementCount64("/mld/connectivity_checksum", 1);
writer.WriteFrom("/mld/connectivity_checksum", connectivity_checksum);
serialization::write(writer, "/mld/multilevelgraph", graph);
}
// read .osrm.partition file

12
include/partitioner/multi_level_graph.hpp
View File

@ -26,14 +26,12 @@ namespace serialization
template <typename EdgeDataT, storage::Ownership Ownership>
void read(storage::tar::FileReader &reader,
const std::string &name,
MultiLevelGraph<EdgeDataT, Ownership> &graph,
std::uint32_t &connectivity_checksum);
MultiLevelGraph<EdgeDataT, Ownership> &graph);
template <typename EdgeDataT, storage::Ownership Ownership>
void write(storage::tar::FileWriter &writer,
const std::string &name,
const MultiLevelGraph<EdgeDataT, Ownership> &graph,
const std::uint32_t connectivity_checksum);
const MultiLevelGraph<EdgeDataT, Ownership> &graph);
}
template <typename EdgeDataT, storage::Ownership Ownership>
@ -206,13 +204,11 @@ class MultiLevelGraph : public util::StaticGraph<EdgeDataT, Ownership>
friend void
serialization::read<EdgeDataT, Ownership>(storage::tar::FileReader &reader,
const std::string &name,
MultiLevelGraph<EdgeDataT, Ownership> &graph,
std::uint32_t &connectivity_checksum);
MultiLevelGraph<EdgeDataT, Ownership> &graph);
friend void
serialization::write<EdgeDataT, Ownership>(storage::tar::FileWriter &writer,
const std::string &name,
const MultiLevelGraph<EdgeDataT, Ownership> &graph,
const std::uint32_t connectivity_checksum);
const MultiLevelGraph<EdgeDataT, Ownership> &graph);
Vector<EdgeOffset> node_to_edge_offset;
std::uint32_t connectivity_checksum;

9
include/partitioner/serialization.hpp
View File

@ -22,26 +22,21 @@ namespace serialization
template <typename EdgeDataT, storage::Ownership Ownership>
inline void read(storage::tar::FileReader &reader,
const std::string &name,
MultiLevelGraph<EdgeDataT, Ownership> &graph,
std::uint32_t &connectivity_checksum)
MultiLevelGraph<EdgeDataT, Ownership> &graph)
{
storage::serialization::read(reader, name + "/node_array", graph.node_array);
storage::serialization::read(reader, name + "/edge_array", graph.edge_array);
storage::serialization::read(reader, name + "/node_to_edge_offset", graph.node_to_edge_offset);
reader.ReadInto(name + "/connectivity_checksum", connectivity_checksum);
}
template <typename EdgeDataT, storage::Ownership Ownership>
inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const MultiLevelGraph<EdgeDataT, Ownership> &graph,
const std::uint32_t connectivity_checksum)
const MultiLevelGraph<EdgeDataT, Ownership> &graph)
{
storage::serialization::write(writer, name + "/node_array", graph.node_array);
storage::serialization::write(writer, name + "/edge_array", graph.edge_array);
storage::serialization::write(writer, name + "/node_to_edge_offset", graph.node_to_edge_offset);
writer.WriteElementCount64(name + "/connectivity_checksum", 1);
writer.WriteFrom(name + "/connectivity_checksum", connectivity_checksum);
}
template <storage::Ownership Ownership>

5
src/contractor/contractor.cpp
View File

@ -93,9 +93,6 @@ int Contractor::Run()
util::excludeFlagsToNodeFilter(number_of_edge_based_nodes, node_data, properties);
}
RangebasedCRC32 crc32_calculator;
const unsigned checksum = crc32_calculator(edge_based_edge_list);
QueryGraph query_graph;
std::vector<std::vector<bool>> edge_filters;
std::vector<std::vector<bool>> cores;
@ -110,7 +107,7 @@ int Contractor::Run()
std::unordered_map<std::string, ContractedMetric> metrics = {
{metric_name, {std::move(query_graph), std::move(edge_filters)}}};
files::writeGraph(config.GetPath(".osrm.hsgr"), checksum, metrics, connectivity_checksum);
files::writeGraph(config.GetPath(".osrm.hsgr"), metrics, connectivity_checksum);
TIMER_STOP(preparing);

8
src/storage/storage.cpp
View File

@ -405,9 +405,12 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
std::move(entry_class_ids),
std::move(pre_turn_bearings),
std::move(post_turn_bearings));
auto connectivity_checksum_ptr = layout.GetBlockPtr<std::uint32_t, true>(memory_ptr, "/common/connectivity_checksum");
guidance::files::readTurnData(
config.GetPath(".osrm.edges"), turn_data, turns_connectivity_checksum);
config.GetPath(".osrm.edges"), turn_data, *connectivity_checksum_ptr);
turns_connectivity_checksum = *connectivity_checksum_ptr;
}
// load compressed geometry
@ -613,7 +616,6 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
memory_ptr, metric_prefix + "/contracted_graph/node_array");
auto graph_edges_ptr = layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
memory_ptr, metric_prefix + "/contracted_graph/edge_array");
auto checksum = layout.GetBlockPtr<unsigned, true>(memory_ptr, "/ch/checksum");
util::vector_view<contractor::QueryGraphView::NodeArrayEntry> node_list(
graph_nodes_ptr,
@ -636,7 +638,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
{metric_name, {{std::move(node_list), std::move(edge_list)}, std::move(edge_filter)}}};
contractor::files::readGraph(
config.GetPath(".osrm.hsgr"), *checksum, metrics, graph_connectivity_checksum);
config.GetPath(".osrm.hsgr"), metrics, graph_connectivity_checksum);
if (turns_connectivity_checksum != graph_connectivity_checksum)
{
throw util::exception(

2
unit_tests/mocks/mock_datafacade.hpp
View File

@ -197,7 +197,7 @@ class MockBaseDataFacade : public engine::datafacade::BaseDataFacade
return {};
}
unsigned GetCheckSum() const override { return 0; }
std::uint32_t GetCheckSum() const override { return 0; }
extractor::TravelMode GetTravelMode(const NodeID /* id */) const override
{