Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add edge-based graph loading in MLD facade

Browse Source
This commit is contained in:
Michael Krasnyk 2017-03-06 23:00:11 +01:00 committed by Patrick Niklaus
parent 53b0417e36
commit f42136637d
15 changed files with 406 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
include/contractor/graph_contractor_adaptors.hpp
View File

@ -22,33 +22,33 @@ std::vector<ContractorEdge> adaptToContractorInput(InputEdgeContainer input_edge
{
#ifndef NDEBUG
const unsigned int constexpr DAY_IN_DECI_SECONDS = 24 * 60 * 60 * 10;
if (static_cast<unsigned int>(std::max(input_edge.weight, 1)) > DAY_IN_DECI_SECONDS)
if (static_cast<unsigned int>(std::max(input_edge.data.weight, 1)) > DAY_IN_DECI_SECONDS)
{
util::Log(logWARNING) << "Edge weight large -> "
<< static_cast<unsigned int>(std::max(input_edge.weight, 1))
<< static_cast<unsigned int>(std::max(input_edge.data.weight, 1))
<< " : " << static_cast<unsigned int>(input_edge.source) << " -> "
<< static_cast<unsigned int>(input_edge.target);
}
#endif
edges.emplace_back(input_edge.source,
input_edge.target,
std::max(input_edge.weight, 1),
input_edge.duration,
std::max(input_edge.data.weight, 1),
input_edge.data.duration,
1,
input_edge.edge_id,
input_edge.data.edge_id,
false,
input_edge.forward ? true : false,
input_edge.backward ? true : false);
input_edge.data.forward ? true : false,
input_edge.data.backward ? true : false);
edges.emplace_back(input_edge.target,
input_edge.source,
std::max(input_edge.weight, 1),
input_edge.duration,
std::max(input_edge.data.weight, 1),
input_edge.data.duration,
1,
input_edge.edge_id,
input_edge.data.edge_id,
false,
input_edge.backward ? true : false,
input_edge.forward ? true : false);
input_edge.data.backward ? true : false,
input_edge.data.forward ? true : false);
}
// FIXME not sure if we need this
edges.shrink_to_fit();

20
include/engine/datafacade/algorithm_datafacade.hpp
View File

@ -2,6 +2,7 @@
#define OSRM_ENGINE_DATAFACADE_ALGORITHM_DATAFACADE_HPP
#include "contractor/query_edge.hpp"
#include "extractor/edge_based_edge.hpp"
#include "engine/algorithm.hpp"
#include "partition/cell_storage.hpp"
@ -66,6 +67,25 @@ template <> class AlgorithmDataFacade<algorithm::CoreCH>
template <> class AlgorithmDataFacade<algorithm::MLD>
{
public:
using EdgeData = extractor::EdgeBasedEdge::EdgeData;
// search graph access
virtual unsigned GetNumberOfNodes() const = 0;
virtual unsigned GetNumberOfEdges() const = 0;
virtual unsigned GetOutDegree(const NodeID n) const = 0;
virtual NodeID GetTarget(const EdgeID e) const = 0;
virtual const EdgeData &GetEdgeData(const EdgeID e) const = 0;
virtual EdgeID BeginEdges(const NodeID n) const = 0;
virtual EdgeID EndEdges(const NodeID n) const = 0;
virtual EdgeRange GetAdjacentEdgeRange(const NodeID node) const = 0;
virtual const partition::MultiLevelPartitionView &GetMultiLevelPartition() const = 0;
virtual const partition::CellStorageView &GetCellStorage() const = 0;

55
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
View File

@ -1200,12 +1200,67 @@ class ContiguousInternalMemoryDataFacade<algorithm::MLD>
: public ContiguousInternalMemoryDataFacadeBase,
public ContiguousInternalMemoryAlgorithmDataFacade<algorithm::MLD>
{
private:
using QueryGraph = util::StaticGraph<EdgeData, true>;
using GraphNode = QueryGraph::NodeArrayEntry;
using GraphEdge = QueryGraph::EdgeArrayEntry;
std::unique_ptr<QueryGraph> m_query_graph;
void InitializeGraphPointer(storage::DataLayout &data_layout, char *memory_block)
{
auto graph_nodes_ptr = data_layout.GetBlockPtr<GraphNode>(
memory_block, storage::DataLayout::MLD_GRAPH_NODE_LIST);
auto graph_edges_ptr = data_layout.GetBlockPtr<GraphEdge>(
memory_block, storage::DataLayout::MLD_GRAPH_EDGE_LIST);
util::ShM<GraphNode, true>::vector node_list(
graph_nodes_ptr, data_layout.num_entries[storage::DataLayout::MLD_GRAPH_NODE_LIST]);
util::ShM<GraphEdge, true>::vector edge_list(
graph_edges_ptr, data_layout.num_entries[storage::DataLayout::MLD_GRAPH_EDGE_LIST]);
m_query_graph.reset(new QueryGraph(node_list, edge_list));
}
public:
ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator)
: ContiguousInternalMemoryDataFacadeBase(allocator),
ContiguousInternalMemoryAlgorithmDataFacade<algorithm::MLD>(allocator)
{
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory());
}
void InitializeInternalPointers(storage::DataLayout &data_layout, char *memory_block)
{
InitializeGraphPointer(data_layout, memory_block);
}
// search graph access
unsigned GetNumberOfNodes() const override final { return m_query_graph->GetNumberOfNodes(); }
unsigned GetNumberOfEdges() const override final { return m_query_graph->GetNumberOfEdges(); }
unsigned GetOutDegree(const NodeID n) const override final
{
return m_query_graph->GetOutDegree(n);
}
NodeID GetTarget(const EdgeID e) const override final { return m_query_graph->GetTarget(e); }
EdgeData &GetEdgeData(const EdgeID e) const override final
{
return m_query_graph->GetEdgeData(e);
}
EdgeID BeginEdges(const NodeID n) const override final { return m_query_graph->BeginEdges(n); }
EdgeID EndEdges(const NodeID n) const override final { return m_query_graph->EndEdges(n); }
EdgeRange GetAdjacentEdgeRange(const NodeID node) const override final
{
return m_query_graph->GetAdjacentEdgeRange(node);
}
};
}

44
include/extractor/edge_based_edge.hpp
View File

@ -29,11 +29,29 @@ struct EdgeBasedEdge
NodeID source;
NodeID target;
NodeID edge_id;
EdgeWeight weight;
EdgeWeight duration : 30;
std::uint32_t forward : 1;
std::uint32_t backward : 1;
struct EdgeData
{
EdgeData() : edge_id(0), weight(0), duration(0), forward(false), backward(false) {}
EdgeData(const NodeID edge_id,
const EdgeWeight weight,
const EdgeWeight duration,
const bool forward,
const bool backward)
: edge_id(edge_id), weight(weight), duration(duration), forward(forward),
backward(backward)
{
}
NodeID edge_id;
EdgeWeight weight;
EdgeWeight duration : 30;
std::uint32_t forward : 1;
std::uint32_t backward : 1;
auto is_unidirectional() const { return !forward || !backward; }
} data;
};
static_assert(sizeof(extractor::EdgeBasedEdge) == 20,
"Size of extractor::EdgeBasedEdge type is "
@ -42,10 +60,7 @@ static_assert(sizeof(extractor::EdgeBasedEdge) == 20,
// Impl.
inline EdgeBasedEdge::EdgeBasedEdge()
: source(0), target(0), edge_id(0), weight(0), duration(0), forward(false), backward(false)
{
}
inline EdgeBasedEdge::EdgeBasedEdge() : source(0), target(0) {}
inline EdgeBasedEdge::EdgeBasedEdge(const NodeID source,
const NodeID target,
@ -54,19 +69,18 @@ inline EdgeBasedEdge::EdgeBasedEdge(const NodeID source,
const EdgeWeight duration,
const bool forward,
const bool backward)
: source(source), target(target), edge_id(edge_id), weight(weight), duration(duration),
forward(forward), backward(backward)
: source(source), target(target), data{edge_id, weight, duration, forward, backward}
{
}
inline bool EdgeBasedEdge::operator<(const EdgeBasedEdge &other) const
{
const auto unidirectional = (!forward || !backward);
const auto other_is_unidirectional = (!other.forward || !other.backward);
const auto unidirectional = data.is_unidirectional();
const auto other_is_unidirectional = other.data.is_unidirectional();
// if all items are the same, we want to keep bidirectional edges. due to the `<` operator,
// preferring 0 (false) over 1 (true), we need to compare the inverse of `bidirectional`
return std::tie(source, target, weight, unidirectional) <
std::tie(other.source, other.target, other.weight, other_is_unidirectional);
return std::tie(source, target, data.weight, unidirectional) <
std::tie(other.source, other.target, other.data.weight, other_is_unidirectional);
}
} // ns extractor
} // ns osrm

204
include/partition/edge_based_graph_reader.hpp
View File

@ -19,7 +19,7 @@ namespace osrm
namespace partition
{
struct EdgeBasedGraphEdgeData : extractor::EdgeBasedEdge
struct EdgeBasedGraphEdgeData : extractor::EdgeBasedEdge::EdgeData
{
// We need to write out the full edge based graph again.
@ -38,6 +38,107 @@ struct EdgeBasedGraphEdge : EdgeBasedGraph::InputEdge
using Base::Base;
};
// Bidirectional (s,t) to (s,t) and (t,s)
std::vector<extractor::EdgeBasedEdge>
SplitBidirectionalEdges(const std::vector<extractor::EdgeBasedEdge> &edges)
{
std::vector<extractor::EdgeBasedEdge> directed;
directed.reserve(edges.size() * 2);
for (const auto &edge : edges)
{
directed.emplace_back(edge.source,
edge.target,
edge.data.edge_id,
std::max(edge.data.weight, 1),
edge.data.duration,
edge.data.forward,
edge.data.backward);
directed.emplace_back(edge.target,
edge.source,
edge.data.edge_id,
std::max(edge.data.weight, 1),
edge.data.duration,
edge.data.backward,
edge.data.forward);
}
return directed;
}
std::vector<EdgeBasedGraphEdge>
PrepareEdgesForUsageInGraph(std::vector<extractor::EdgeBasedEdge> edges)
{
std::sort(begin(edges), end(edges));
std::vector<EdgeBasedGraphEdge> graph_edges;
graph_edges.reserve(edges.size());
for (NodeID i = 0; i < edges.size();)
{
const NodeID source = edges[i].source;
const NodeID target = edges[i].target;
// remove eigenloops
if (source == target)
{
++i;
continue;
}
EdgeBasedGraphEdge forward_edge;
EdgeBasedGraphEdge reverse_edge;
forward_edge.source = reverse_edge.source = source;
forward_edge.target = reverse_edge.target = target;
forward_edge.data.edge_id = reverse_edge.data.edge_id = edges[i].data.edge_id;
forward_edge.data.weight = reverse_edge.data.weight = INVALID_EDGE_WEIGHT;
forward_edge.data.duration = reverse_edge.data.duration = MAXIMAL_EDGE_DURATION_INT_30;
forward_edge.data.forward = reverse_edge.data.backward = true;
forward_edge.data.backward = reverse_edge.data.forward = false;
// remove parallel edges
while (i < edges.size() && edges[i].source == source && edges[i].target == target)
{
if (edges[i].data.forward)
{
forward_edge.data.weight = std::min(edges[i].data.weight, forward_edge.data.weight);
forward_edge.data.duration =
std::min(edges[i].data.duration, forward_edge.data.duration);
}
if (edges[i].data.backward)
{
reverse_edge.data.weight = std::min(edges[i].data.weight, reverse_edge.data.weight);
reverse_edge.data.duration =
std::min(edges[i].data.duration, reverse_edge.data.duration);
}
++i;
}
// merge edges (s,t) and (t,s) into bidirectional edge
if (forward_edge.data.weight == reverse_edge.data.weight)
{
if ((int)forward_edge.data.weight != INVALID_EDGE_WEIGHT)
{
forward_edge.data.backward = true;
graph_edges.push_back(forward_edge);
}
}
else
{ // insert seperate edges
if (((int)forward_edge.data.weight) != INVALID_EDGE_WEIGHT)
{
graph_edges.push_back(forward_edge);
}
if ((int)reverse_edge.data.weight != INVALID_EDGE_WEIGHT)
{
graph_edges.push_back(reverse_edge);
}
}
}
return graph_edges;
}
struct EdgeBasedGraphReader
{
EdgeBasedGraphReader(storage::io::FileReader &reader)
@ -74,107 +175,6 @@ struct EdgeBasedGraphReader
}
private:
// Bidirectional (s,t) to (s,t) and (t,s)
std::vector<extractor::EdgeBasedEdge>
SplitBidirectionalEdges(const std::vector<extractor::EdgeBasedEdge> &edges)
{
std::vector<extractor::EdgeBasedEdge> directed;
directed.reserve(edges.size() * 2);
for (const auto &edge : edges)
{
directed.emplace_back(edge.source,
edge.target,
edge.edge_id,
std::max(edge.weight, 1),
edge.duration,
edge.forward,
edge.backward);
directed.emplace_back(edge.target,
edge.source,
edge.edge_id,
std::max(edge.weight, 1),
edge.duration,
edge.backward,
edge.forward);
}
return directed;
}
std::vector<EdgeBasedGraphEdge>
PrepareEdgesForUsageInGraph(std::vector<extractor::EdgeBasedEdge> edges)
{
std::sort(begin(edges), end(edges));
std::vector<EdgeBasedGraphEdge> graph_edges;
graph_edges.reserve(edges.size());
for (NodeID i = 0; i < edges.size();)
{
const NodeID source = edges[i].source;
const NodeID target = edges[i].target;
// remove eigenloops
if (source == target)
{
++i;
continue;
}
EdgeBasedGraphEdge forward_edge;
EdgeBasedGraphEdge reverse_edge;
forward_edge.source = reverse_edge.source = source;
forward_edge.target = reverse_edge.target = target;
forward_edge.data.edge_id = reverse_edge.data.edge_id = edges[i].edge_id;
forward_edge.data.weight = reverse_edge.data.weight = INVALID_EDGE_WEIGHT;
forward_edge.data.duration = reverse_edge.data.duration = MAXIMAL_EDGE_DURATION_INT_30;
forward_edge.data.forward = reverse_edge.data.backward = true;
forward_edge.data.backward = reverse_edge.data.forward = false;
// remove parallel edges
while (i < edges.size() && edges[i].source == source && edges[i].target == target)
{
if (edges[i].forward)
{
forward_edge.data.weight = std::min(edges[i].weight, forward_edge.data.weight);
forward_edge.data.duration =
std::min(edges[i].duration, forward_edge.data.duration);
}
if (edges[i].backward)
{
reverse_edge.data.weight = std::min(edges[i].weight, reverse_edge.data.weight);
reverse_edge.data.duration =
std::min(edges[i].duration, reverse_edge.data.duration);
}
++i;
}
// merge edges (s,t) and (t,s) into bidirectional edge
if (forward_edge.data.weight == reverse_edge.data.weight)
{
if ((int)forward_edge.data.weight != INVALID_EDGE_WEIGHT)
{
forward_edge.data.backward = true;
graph_edges.push_back(forward_edge);
}
}
else
{ // insert seperate edges
if (((int)forward_edge.data.weight) != INVALID_EDGE_WEIGHT)
{
graph_edges.push_back(forward_edge);
}
if ((int)reverse_edge.data.weight != INVALID_EDGE_WEIGHT)
{
graph_edges.push_back(reverse_edge);
}
}
}
return graph_edges;
}
std::vector<extractor::EdgeBasedEdge> edges;
std::size_t num_nodes;
};

6
include/storage/shared_datatype.hpp
View File

@ -64,7 +64,9 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
"MLD_CELL_SOURCE_BOUNDARY",
"MLD_CELL_DESTINATION_BOUNDARY",
"MLD_CELLS",
"MLD_CELL_LEVEL_OFFSETS"};
"MLD_CELL_LEVEL_OFFSETS",
"MLD_GRAPH_NODE_LIST",
"MLD_GRAPH_EDGE_LIST"};
struct DataLayout
{
@ -117,6 +119,8 @@ struct DataLayout
MLD_CELL_DESTINATION_BOUNDARY,
MLD_CELLS,
MLD_CELL_LEVEL_OFFSETS,
MLD_GRAPH_NODE_LIST,
MLD_GRAPH_EDGE_LIST,
NUM_BLOCKS
};

1
include/storage/storage_config.hpp
View File

@ -71,6 +71,7 @@ struct StorageConfig final
boost::filesystem::path turn_lane_description_path;
boost::filesystem::path mld_partition_path;
boost::filesystem::path mld_storage_path;
boost::filesystem::path edge_based_graph_path;
};
}
}

12
include/util/shared_memory_vector_wrapper.hpp
View File

@ -106,6 +106,18 @@ template <typename DataT> class SharedMemoryWrapper
return m_ptr[index];
}
const DataT &front() const {
BOOST_ASSERT_MSG(m_size > 0, "invalid size");
return m_ptr[0];
}
const DataT &back() const {
BOOST_ASSERT_MSG(m_size > 0, "invalid size");
return m_ptr[m_size - 1];
}
auto data() const { return m_ptr; }
template <typename T>
friend void swap(SharedMemoryWrapper<T> &, SharedMemoryWrapper<T> &) noexcept;
};

5
include/util/static_graph.hpp
View File

@ -134,8 +134,11 @@ template <typename EdgeDataT, bool UseSharedMemory = false> class StaticGraph
StaticGraph(typename ShM<NodeArrayEntry, UseSharedMemory>::vector &nodes,
typename ShM<EdgeArrayEntry, UseSharedMemory>::vector &edges)
{
BOOST_ASSERT(!nodes.empty());
number_of_nodes = static_cast<decltype(number_of_nodes)>(nodes.size() - 1);
number_of_edges = static_cast<decltype(number_of_edges)>(edges.size());
number_of_edges = static_cast<decltype(number_of_edges)>(nodes.back().first_edge);
BOOST_ASSERT(number_of_edges <= edges.size());
using std::swap;
swap(node_array, nodes);

10
src/contractor/contractor.cpp
View File

@ -147,8 +147,8 @@ void CheckWeightsConsistency(
for (auto &edge : edge_based_edge_list)
{
BOOST_ASSERT(edge.edge_id < current_edge_data.size());
auto geometry_id = current_edge_data[edge.edge_id].via_geometry;
BOOST_ASSERT(edge.data.edge_id < current_edge_data.size());
auto geometry_id = current_edge_data[edge.data.edge_id].via_geometry;
BOOST_ASSERT(geometry_id.id < geometry_indices.size());
const auto &weights = geometry_id.forward ? forward_weight_list : reverse_weight_list;
@ -157,7 +157,7 @@ void CheckWeightsConsistency(
const auto last = weights.begin() + geometry_indices.at(geometry_id.id + 1) - 1 + shift;
EdgeWeight weight = std::accumulate(first, last, 0);
BOOST_ASSERT(weight <= edge.weight);
BOOST_ASSERT(weight <= edge.data.weight);
}
}
@ -908,8 +908,8 @@ Contractor::LoadEdgeExpandedGraph(const ContractorConfig &config,
}
// Update edge weight
inbuffer.weight = new_weight + turn_weight_penalty;
inbuffer.duration = new_duration + turn_duration_penalty;
inbuffer.data.weight = new_weight + turn_weight_penalty;
inbuffer.data.duration = new_duration + turn_duration_penalty;
}
edge_based_edge_list.emplace_back(std::move(inbuffer));

6
src/extractor/extractor.cpp
View File

@ -333,16 +333,16 @@ void Extractor::FindComponents(unsigned max_edge_id,
for (const auto &edge : input_edge_list)
{
BOOST_ASSERT_MSG(static_cast<unsigned int>(std::max(edge.weight, 1)) > 0,
BOOST_ASSERT_MSG(static_cast<unsigned int>(std::max(edge.data.weight, 1)) > 0,
"edge distance < 1");
BOOST_ASSERT(edge.source <= max_edge_id);
BOOST_ASSERT(edge.target <= max_edge_id);
if (edge.forward)
if (edge.data.forward)
{
edges.push_back({edge.source, edge.target});
}
if (edge.backward)
if (edge.data.backward)
{
edges.push_back({edge.target, edge.source});
}

7
src/partition/partitioner.cpp
View File

@ -169,6 +169,13 @@ int Partitioner::Run(const PartitionConfig &config)
config.minimum_cell_size * 32 * 16,
config.minimum_cell_size * 32 * 16 * 32});
util::Log() << "Edge-based-graph annotation:";
for (std::size_t level = 0; level < level_to_num_cells.size(); ++level)
{
util::Log() << " level " << level + 1 << " #cells " << level_to_num_cells[level]
<< " bit size " << std::ceil(std::log2(level_to_num_cells[level] + 1));
}
TIMER_START(packed_mlp);
MultiLevelPartition mlp{partitions, level_to_num_cells};
TIMER_STOP(packed_mlp);

124
src/storage/storage.cpp
View File

@ -1,12 +1,14 @@
#include "storage/storage.hpp"
#include "contractor/query_edge.hpp"
#include "extractor/compressed_edge_container.hpp"
#include "extractor/edge_based_edge.hpp"
#include "extractor/guidance/turn_instruction.hpp"
#include "extractor/original_edge_data.hpp"
#include "extractor/profile_properties.hpp"
#include "extractor/query_node.hpp"
#include "extractor/travel_mode.hpp"
#include "partition/cell_storage.hpp"
#include "partition/edge_based_graph_reader.hpp"
#include "partition/multi_level_partition.hpp"
#include "storage/io.hpp"
#include "storage/serialization.hpp"
@ -54,6 +56,7 @@ using RTreeLeaf = engine::datafacade::BaseDataFacade::RTreeLeaf;
using RTreeNode =
util::StaticRTree<RTreeLeaf, util::ShM<util::Coordinate, true>::vector, true>::TreeNode;
using QueryGraph = util::StaticGraph<contractor::QueryEdge::EdgeData>;
using EdgeBasedGraph = util::StaticGraph<extractor::EdgeBasedEdge::EdgeData>;
using Monitor = SharedMonitor<SharedDataTimestamp>;
@ -237,6 +240,7 @@ void Storage::PopulateLayout(DataLayout &layout)
layout.SetBlockSize<EntryClassID>(DataLayout::ENTRY_CLASSID, number_of_original_edges);
}
if (boost::filesystem::exists(config.hsgr_data_path))
{
io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::VerifyFingerprint);
@ -247,6 +251,12 @@ void Storage::PopulateLayout(DataLayout &layout)
layout.SetBlockSize<QueryGraph::EdgeArrayEntry>(DataLayout::CH_GRAPH_EDGE_LIST,
hsgr_header.number_of_edges);
}
else
{
layout.SetBlockSize<unsigned>(DataLayout::HSGR_CHECKSUM, 0);
layout.SetBlockSize<QueryGraph::NodeArrayEntry>(DataLayout::CH_GRAPH_NODE_LIST, 0);
layout.SetBlockSize<QueryGraph::EdgeArrayEntry>(DataLayout::CH_GRAPH_EDGE_LIST, 0);
}
// load rsearch tree size
{
@ -270,11 +280,16 @@ void Storage::PopulateLayout(DataLayout &layout)
}
// load core marker size
if (boost::filesystem::exists(config.core_data_path))
{
io::FileReader core_marker_file(config.core_data_path, io::FileReader::HasNoFingerprint);
const auto number_of_core_markers = core_marker_file.ReadElementCount32();
layout.SetBlockSize<unsigned>(DataLayout::CH_CORE_MARKER, number_of_core_markers);
}
else
{
layout.SetBlockSize<unsigned>(DataLayout::CH_CORE_MARKER, 0);
}
// load turn weight penalties
{
@ -326,24 +341,25 @@ void Storage::PopulateLayout(DataLayout &layout)
number_of_compressed_geometries);
}
// load datasource sizes. This file is optional, and it's non-fatal if it doesn't
// exist.
// load datasource sizes. This file is optional, and it's non-fatal if it doesn't exist.
if (boost::filesystem::exists(config.datasource_indexes_path))
{
io::FileReader geometry_datasource_file(config.datasource_indexes_path,
io::FileReader::HasNoFingerprint);
const auto number_of_compressed_datasources = geometry_datasource_file.ReadElementCount64();
layout.SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST,
number_of_compressed_datasources);
io::FileReader reader(config.datasource_indexes_path, io::FileReader::HasNoFingerprint);
const auto number_of_datasources = reader.ReadElementCount64();
layout.SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST, number_of_datasources);
}
else
{
layout.SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST, 0);
}
// Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't
// exist
// Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't exist
if (boost::filesystem::exists(config.datasource_names_path))
{
io::FileReader datasource_names_file(config.datasource_names_path,
io::FileReader::HasNoFingerprint);
io::FileReader reader(config.datasource_names_path, io::FileReader::HasNoFingerprint);
const serialization::DatasourceNamesData datasource_names_data =
serialization::readDatasourceNames(datasource_names_file);
serialization::readDatasourceNames(reader);
layout.SetBlockSize<char>(DataLayout::DATASOURCE_NAME_DATA,
datasource_names_data.names.size());
@ -352,6 +368,12 @@ void Storage::PopulateLayout(DataLayout &layout)
layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_LENGTHS,
datasource_names_data.lengths.size());
}
else
{
layout.SetBlockSize<char>(DataLayout::DATASOURCE_NAME_DATA, 0);
layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_OFFSETS, 0);
layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_LENGTHS, 0);
}
{
io::FileReader intersection_file(config.intersection_class_path,
@ -446,6 +468,25 @@ void Storage::PopulateLayout(DataLayout &layout)
layout.SetBlockSize<char>(DataLayout::MLD_CELLS, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_LEVEL_OFFSETS, 0);
}
if (boost::filesystem::exists(config.edge_based_graph_path))
{
io::FileReader ebg_file(config.edge_based_graph_path,
io::FileReader::VerifyFingerprint);
const auto num_edges = ebg_file.ReadElementCount64();
const auto num_nodes = ebg_file.ReadOne<EdgeID>() + 1;
layout.SetBlockSize<EdgeBasedGraph::NodeArrayEntry>(DataLayout::MLD_GRAPH_NODE_LIST,
num_nodes + 1);
layout.SetBlockSize<EdgeBasedGraph::EdgeArrayEntry>(DataLayout::MLD_GRAPH_EDGE_LIST,
2 * num_edges);
}
else
{
layout.SetBlockSize<char>(DataLayout::MLD_GRAPH_NODE_LIST, 0);
layout.SetBlockSize<char>(DataLayout::MLD_GRAPH_EDGE_LIST, 0);
}
}
}
@ -456,6 +497,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// read actual data into shared memory object //
// Load the HSGR file
if (boost::filesystem::exists(config.hsgr_data_path))
{
io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::VerifyFingerprint);
auto hsgr_header = serialization::readHSGRHeader(hsgr_file);
@ -479,6 +521,14 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
graph_edge_list_ptr,
hsgr_header.number_of_edges);
}
else
{
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
layout.GetBlockPtr<QueryGraph::NodeArrayEntry, true>(memory_ptr,
DataLayout::CH_GRAPH_NODE_LIST);
layout.GetBlockPtr<QueryGraph::EdgeArrayEntry, true>(memory_ptr,
DataLayout::CH_GRAPH_EDGE_LIST);
}
// store the filename of the on-disk portion of the RTree
{
@ -643,6 +693,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
geometry_input_file.ReadInto(geometries_rev_duration_list_ptr, geometry_node_lists_count);
}
if (boost::filesystem::exists(config.datasource_indexes_path))
{
io::FileReader geometry_datasource_file(config.datasource_indexes_path,
io::FileReader::HasNoFingerprint);
@ -657,9 +708,13 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
geometry_datasource_file, datasources_list_ptr, number_of_compressed_datasources);
}
}
else
{
layout.GetBlockPtr<uint8_t, true>(memory_ptr, DataLayout::DATASOURCES_LIST);
}
if (boost::filesystem::exists(config.datasource_names_path))
{
/* Load names */
io::FileReader datasource_names_file(config.datasource_names_path,
io::FileReader::HasNoFingerprint);
@ -706,6 +761,12 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
datasource_name_lengths_ptr);
}
}
else
{
layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::DATASOURCE_NAME_DATA);
layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_OFFSETS);
layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_LENGTHS);
}
// Loading list of coordinates
{
@ -768,6 +829,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
tree_node_file.ReadInto(rtree_ptr, layout.num_entries[DataLayout::R_SEARCH_TREE]);
}
if (boost::filesystem::exists(config.core_data_path))
{
io::FileReader core_marker_file(config.core_data_path, io::FileReader::HasNoFingerprint);
const auto number_of_core_markers = core_marker_file.ReadElementCount32();
@ -939,6 +1001,40 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
reader.ReadInto(mld_cell_level_offsets_ptr,
layout.GetBlockEntries(DataLayout::MLD_CELL_LEVEL_OFFSETS));
}
if (boost::filesystem::exists(config.edge_based_graph_path))
{
io::FileReader reader(config.edge_based_graph_path, io::FileReader::VerifyFingerprint);
const auto number_of_edges = reader.ReadElementCount64();
const auto number_of_nodes = reader.ReadOne<EdgeID>() + 1;
std::vector<extractor::EdgeBasedEdge> original_edges(number_of_edges);
reader.ReadInto(original_edges);
original_edges = partition::SplitBidirectionalEdges(std::move(original_edges));
auto edges = partition::PrepareEdgesForUsageInGraph(std::move(original_edges));
BOOST_ASSERT(edges.size() <= 2 * number_of_edges);
auto nodes_ptr = layout.GetBlockPtr<EdgeBasedGraph::NodeArrayEntry, true>(
memory_ptr, DataLayout::MLD_GRAPH_NODE_LIST);
auto edges_ptr = layout.GetBlockPtr<EdgeBasedGraph::EdgeArrayEntry, true>(
memory_ptr, DataLayout::MLD_GRAPH_EDGE_LIST);
EdgeBasedGraph::EdgeIterator edge = 0;
for (const auto node : util::irange(0u, number_of_nodes + 1))
{
EdgeBasedGraph::EdgeIterator last_edge = edge;
while (edge < edges.size() && edges[edge].source == node)
{
edges_ptr[edge].target = edges[edge].target;
edges_ptr[edge].data = edges[edge].data;
++edge;
}
nodes_ptr[node].first_edge = last_edge;
}
BOOST_ASSERT(edge == edges.size());
}
}
}
}

64
src/storage/storage_config.cpp
View File

@ -7,6 +7,22 @@ namespace osrm
{
namespace storage
{
namespace
{
bool CheckFileList(const std::vector<boost::filesystem::path> &files)
{
bool success = true;
for (auto &path : files)
{
if (!boost::filesystem::is_regular_file(path))
{
util::Log(logWARNING) << "Missing/Broken File: " << path.string();
success = false;
}
}
return success;
}
}
StorageConfig::StorageConfig(const boost::filesystem::path &base)
: ram_index_path{base.string() + ".ramIndex"}, file_index_path{base.string() + ".fileIndex"},
@ -20,40 +36,40 @@ StorageConfig::StorageConfig(const boost::filesystem::path &base)
names_data_path{base.string() + ".names"}, properties_path{base.string() + ".properties"},
intersection_class_path{base.string() + ".icd"}, turn_lane_data_path{base.string() + ".tld"},
turn_lane_description_path{base.string() + ".tls"},
mld_partition_path{base.string() + ".partition"}, mld_storage_path{base.string() + ".cells"}
mld_partition_path{base.string() + ".partition"}, mld_storage_path{base.string() + ".cells"},
edge_based_graph_path{base.string() + ".ebg"}
{
}
bool StorageConfig::IsValid() const
{
const constexpr auto num_files = 15;
const boost::filesystem::path paths[num_files] = {ram_index_path,
file_index_path,
hsgr_data_path,
nodes_data_path,
edges_data_path,
core_data_path,
geometries_path,
timestamp_path,
turn_weight_penalties_path,
turn_duration_penalties_path,
datasource_names_path,
datasource_indexes_path,
names_data_path,
properties_path,
intersection_class_path};
bool success = true;
for (auto path = paths; path != paths + num_files; ++path)
// Common files
if (!CheckFileList({ram_index_path,
file_index_path,
nodes_data_path,
edges_data_path,
geometries_path,
timestamp_path,
turn_weight_penalties_path,
turn_duration_penalties_path,
names_data_path,
properties_path,
intersection_class_path}))
{
if (!boost::filesystem::is_regular_file(*path))
{
util::Log(logWARNING) << "Missing/Broken File: " << path->string();
success = false;
}
return false;
}
return success;
// TODO: add algorithm checks
// CH files
CheckFileList({hsgr_data_path, core_data_path, datasource_names_path, datasource_indexes_path});
// MLD files
CheckFileList({mld_partition_path, mld_storage_path, edge_based_graph_path});
return true;
}
}
}

2
unit_tests/util/static_graph.cpp
View File

@ -42,7 +42,7 @@ template <unsigned NUM_NODES, unsigned NUM_EDGES> struct RandomArrayEntryFixture
}
std::sort(offsets.begin(), offsets.end());
// add sentinel
offsets.push_back(offsets.back());
offsets.push_back(TEST_NUM_EDGES);
// extract interval lengths
for (unsigned i = 0; i < offsets.size() - 1; i++)