Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add edge-based graph loading in MLD facade

Browse Source
This commit is contained in:
Michael Krasnyk
2017-03-06 23:00:11 +01:00
committed by Patrick Niklaus
parent 53b0417e36
commit f42136637d
15 changed files with 406 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/contractor/contractor.cpp
+5 -5
View File
@@ -147,8 +147,8 @@ void CheckWeightsConsistency(
for (auto &edge : edge_based_edge_list)
{
BOOST_ASSERT(edge.edge_id < current_edge_data.size());
auto geometry_id = current_edge_data[edge.edge_id].via_geometry;
BOOST_ASSERT(edge.data.edge_id < current_edge_data.size());
auto geometry_id = current_edge_data[edge.data.edge_id].via_geometry;
BOOST_ASSERT(geometry_id.id < geometry_indices.size());
const auto &weights = geometry_id.forward ? forward_weight_list : reverse_weight_list;
@@ -157,7 +157,7 @@ void CheckWeightsConsistency(
const auto last = weights.begin() + geometry_indices.at(geometry_id.id + 1) - 1 + shift;
EdgeWeight weight = std::accumulate(first, last, 0);
BOOST_ASSERT(weight <= edge.weight);
BOOST_ASSERT(weight <= edge.data.weight);
}
}
@@ -908,8 +908,8 @@ Contractor::LoadEdgeExpandedGraph(const ContractorConfig &config,
}
// Update edge weight
inbuffer.weight = new_weight + turn_weight_penalty;
inbuffer.duration = new_duration + turn_duration_penalty;
inbuffer.data.weight = new_weight + turn_weight_penalty;
inbuffer.data.duration = new_duration + turn_duration_penalty;
}
edge_based_edge_list.emplace_back(std::move(inbuffer));
src/extractor/extractor.cpp
+3 -3
View File
@@ -333,16 +333,16 @@ void Extractor::FindComponents(unsigned max_edge_id,
for (const auto &edge : input_edge_list)
{
BOOST_ASSERT_MSG(static_cast<unsigned int>(std::max(edge.weight, 1)) > 0,
BOOST_ASSERT_MSG(static_cast<unsigned int>(std::max(edge.data.weight, 1)) > 0,
"edge distance < 1");
BOOST_ASSERT(edge.source <= max_edge_id);
BOOST_ASSERT(edge.target <= max_edge_id);
if (edge.forward)
if (edge.data.forward)
{
edges.push_back({edge.source, edge.target});
}
if (edge.backward)
if (edge.data.backward)
{
edges.push_back({edge.target, edge.source});
}
src/partition/partitioner.cpp
+7
View File
@@ -169,6 +169,13 @@ int Partitioner::Run(const PartitionConfig &config)
config.minimum_cell_size * 32 * 16,
config.minimum_cell_size * 32 * 16 * 32});
util::Log() << "Edge-based-graph annotation:";
for (std::size_t level = 0; level < level_to_num_cells.size(); ++level)
{
util::Log() << " level " << level + 1 << " #cells " << level_to_num_cells[level]
<< " bit size " << std::ceil(std::log2(level_to_num_cells[level] + 1));
}
TIMER_START(packed_mlp);
MultiLevelPartition mlp{partitions, level_to_num_cells};
TIMER_STOP(packed_mlp);
src/storage/storage.cpp
+110 -14
View File
@@ -1,12 +1,14 @@
#include "storage/storage.hpp"
#include "contractor/query_edge.hpp"
#include "extractor/compressed_edge_container.hpp"
#include "extractor/edge_based_edge.hpp"
#include "extractor/guidance/turn_instruction.hpp"
#include "extractor/original_edge_data.hpp"
#include "extractor/profile_properties.hpp"
#include "extractor/query_node.hpp"
#include "extractor/travel_mode.hpp"
#include "partition/cell_storage.hpp"
#include "partition/edge_based_graph_reader.hpp"
#include "partition/multi_level_partition.hpp"
#include "storage/io.hpp"
#include "storage/serialization.hpp"
@@ -54,6 +56,7 @@ using RTreeLeaf = engine::datafacade::BaseDataFacade::RTreeLeaf;
using RTreeNode =
util::StaticRTree<RTreeLeaf, util::ShM<util::Coordinate, true>::vector, true>::TreeNode;
using QueryGraph = util::StaticGraph<contractor::QueryEdge::EdgeData>;
using EdgeBasedGraph = util::StaticGraph<extractor::EdgeBasedEdge::EdgeData>;
using Monitor = SharedMonitor<SharedDataTimestamp>;
@@ -237,6 +240,7 @@ void Storage::PopulateLayout(DataLayout &layout)
layout.SetBlockSize<EntryClassID>(DataLayout::ENTRY_CLASSID, number_of_original_edges);
}
if (boost::filesystem::exists(config.hsgr_data_path))
{
io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::VerifyFingerprint);
@@ -247,6 +251,12 @@ void Storage::PopulateLayout(DataLayout &layout)
layout.SetBlockSize<QueryGraph::EdgeArrayEntry>(DataLayout::CH_GRAPH_EDGE_LIST,
hsgr_header.number_of_edges);
}
else
{
layout.SetBlockSize<unsigned>(DataLayout::HSGR_CHECKSUM, 0);
layout.SetBlockSize<QueryGraph::NodeArrayEntry>(DataLayout::CH_GRAPH_NODE_LIST, 0);
layout.SetBlockSize<QueryGraph::EdgeArrayEntry>(DataLayout::CH_GRAPH_EDGE_LIST, 0);
}
// load rsearch tree size
{
@@ -270,11 +280,16 @@ void Storage::PopulateLayout(DataLayout &layout)
}
// load core marker size
if (boost::filesystem::exists(config.core_data_path))
{
io::FileReader core_marker_file(config.core_data_path, io::FileReader::HasNoFingerprint);
const auto number_of_core_markers = core_marker_file.ReadElementCount32();
layout.SetBlockSize<unsigned>(DataLayout::CH_CORE_MARKER, number_of_core_markers);
}
else
{
layout.SetBlockSize<unsigned>(DataLayout::CH_CORE_MARKER, 0);
}
// load turn weight penalties
{
@@ -326,24 +341,25 @@ void Storage::PopulateLayout(DataLayout &layout)
number_of_compressed_geometries);
}
// load datasource sizes. This file is optional, and it's non-fatal if it doesn't
// exist.
// load datasource sizes. This file is optional, and it's non-fatal if it doesn't exist.
if (boost::filesystem::exists(config.datasource_indexes_path))
{
io::FileReader geometry_datasource_file(config.datasource_indexes_path,
io::FileReader::HasNoFingerprint);
const auto number_of_compressed_datasources = geometry_datasource_file.ReadElementCount64();
layout.SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST,
number_of_compressed_datasources);
io::FileReader reader(config.datasource_indexes_path, io::FileReader::HasNoFingerprint);
const auto number_of_datasources = reader.ReadElementCount64();
layout.SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST, number_of_datasources);
}
else
{
layout.SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST, 0);
}
// Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't
// exist
// Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't exist
if (boost::filesystem::exists(config.datasource_names_path))
{
io::FileReader datasource_names_file(config.datasource_names_path,
io::FileReader::HasNoFingerprint);
io::FileReader reader(config.datasource_names_path, io::FileReader::HasNoFingerprint);
const serialization::DatasourceNamesData datasource_names_data =
serialization::readDatasourceNames(datasource_names_file);
serialization::readDatasourceNames(reader);
layout.SetBlockSize<char>(DataLayout::DATASOURCE_NAME_DATA,
datasource_names_data.names.size());
@@ -352,6 +368,12 @@ void Storage::PopulateLayout(DataLayout &layout)
layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_LENGTHS,
datasource_names_data.lengths.size());
}
else
{
layout.SetBlockSize<char>(DataLayout::DATASOURCE_NAME_DATA, 0);
layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_OFFSETS, 0);
layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_LENGTHS, 0);
}
{
io::FileReader intersection_file(config.intersection_class_path,
@@ -446,6 +468,25 @@ void Storage::PopulateLayout(DataLayout &layout)
layout.SetBlockSize<char>(DataLayout::MLD_CELLS, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_LEVEL_OFFSETS, 0);
}
if (boost::filesystem::exists(config.edge_based_graph_path))
{
io::FileReader ebg_file(config.edge_based_graph_path,
io::FileReader::VerifyFingerprint);
const auto num_edges = ebg_file.ReadElementCount64();
const auto num_nodes = ebg_file.ReadOne<EdgeID>() + 1;
layout.SetBlockSize<EdgeBasedGraph::NodeArrayEntry>(DataLayout::MLD_GRAPH_NODE_LIST,
num_nodes + 1);
layout.SetBlockSize<EdgeBasedGraph::EdgeArrayEntry>(DataLayout::MLD_GRAPH_EDGE_LIST,
2 * num_edges);
}
else
{
layout.SetBlockSize<char>(DataLayout::MLD_GRAPH_NODE_LIST, 0);
layout.SetBlockSize<char>(DataLayout::MLD_GRAPH_EDGE_LIST, 0);
}
}
}
@@ -456,6 +497,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// read actual data into shared memory object //
// Load the HSGR file
if (boost::filesystem::exists(config.hsgr_data_path))
{
io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::VerifyFingerprint);
auto hsgr_header = serialization::readHSGRHeader(hsgr_file);
@@ -479,6 +521,14 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
graph_edge_list_ptr,
hsgr_header.number_of_edges);
}
else
{
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
layout.GetBlockPtr<QueryGraph::NodeArrayEntry, true>(memory_ptr,
DataLayout::CH_GRAPH_NODE_LIST);
layout.GetBlockPtr<QueryGraph::EdgeArrayEntry, true>(memory_ptr,
DataLayout::CH_GRAPH_EDGE_LIST);
}
// store the filename of the on-disk portion of the RTree
{
@@ -643,6 +693,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
geometry_input_file.ReadInto(geometries_rev_duration_list_ptr, geometry_node_lists_count);
}
if (boost::filesystem::exists(config.datasource_indexes_path))
{
io::FileReader geometry_datasource_file(config.datasource_indexes_path,
io::FileReader::HasNoFingerprint);
@@ -657,9 +708,13 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
geometry_datasource_file, datasources_list_ptr, number_of_compressed_datasources);
}
}
else
{
layout.GetBlockPtr<uint8_t, true>(memory_ptr, DataLayout::DATASOURCES_LIST);
}
if (boost::filesystem::exists(config.datasource_names_path))
{
/* Load names */
io::FileReader datasource_names_file(config.datasource_names_path,
io::FileReader::HasNoFingerprint);
@@ -706,6 +761,12 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
datasource_name_lengths_ptr);
}
}
else
{
layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::DATASOURCE_NAME_DATA);
layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_OFFSETS);
layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_LENGTHS);
}
// Loading list of coordinates
{
@@ -768,6 +829,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
tree_node_file.ReadInto(rtree_ptr, layout.num_entries[DataLayout::R_SEARCH_TREE]);
}
if (boost::filesystem::exists(config.core_data_path))
{
io::FileReader core_marker_file(config.core_data_path, io::FileReader::HasNoFingerprint);
const auto number_of_core_markers = core_marker_file.ReadElementCount32();
@@ -939,6 +1001,40 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
reader.ReadInto(mld_cell_level_offsets_ptr,
layout.GetBlockEntries(DataLayout::MLD_CELL_LEVEL_OFFSETS));
}
if (boost::filesystem::exists(config.edge_based_graph_path))
{
io::FileReader reader(config.edge_based_graph_path, io::FileReader::VerifyFingerprint);
const auto number_of_edges = reader.ReadElementCount64();
const auto number_of_nodes = reader.ReadOne<EdgeID>() + 1;
std::vector<extractor::EdgeBasedEdge> original_edges(number_of_edges);
reader.ReadInto(original_edges);
original_edges = partition::SplitBidirectionalEdges(std::move(original_edges));
auto edges = partition::PrepareEdgesForUsageInGraph(std::move(original_edges));
BOOST_ASSERT(edges.size() <= 2 * number_of_edges);
auto nodes_ptr = layout.GetBlockPtr<EdgeBasedGraph::NodeArrayEntry, true>(
memory_ptr, DataLayout::MLD_GRAPH_NODE_LIST);
auto edges_ptr = layout.GetBlockPtr<EdgeBasedGraph::EdgeArrayEntry, true>(
memory_ptr, DataLayout::MLD_GRAPH_EDGE_LIST);
EdgeBasedGraph::EdgeIterator edge = 0;
for (const auto node : util::irange(0u, number_of_nodes + 1))
{
EdgeBasedGraph::EdgeIterator last_edge = edge;
while (edge < edges.size() && edges[edge].source == node)
{
edges_ptr[edge].target = edges[edge].target;
edges_ptr[edge].data = edges[edge].data;
++edge;
}
nodes_ptr[node].first_edge = last_edge;
}
BOOST_ASSERT(edge == edges.size());
}
}
}
}
src/storage/storage_config.cpp
+40 -24
View File
@@ -7,6 +7,22 @@ namespace osrm
{
namespace storage
{
namespace
{
bool CheckFileList(const std::vector<boost::filesystem::path> &files)
{
bool success = true;
for (auto &path : files)
{
if (!boost::filesystem::is_regular_file(path))
{
util::Log(logWARNING) << "Missing/Broken File: " << path.string();
success = false;
}
}
return success;
}
}
StorageConfig::StorageConfig(const boost::filesystem::path &base)
: ram_index_path{base.string() + ".ramIndex"}, file_index_path{base.string() + ".fileIndex"},
@@ -20,40 +36,40 @@ StorageConfig::StorageConfig(const boost::filesystem::path &base)
names_data_path{base.string() + ".names"}, properties_path{base.string() + ".properties"},
intersection_class_path{base.string() + ".icd"}, turn_lane_data_path{base.string() + ".tld"},
turn_lane_description_path{base.string() + ".tls"},
mld_partition_path{base.string() + ".partition"}, mld_storage_path{base.string() + ".cells"}
mld_partition_path{base.string() + ".partition"}, mld_storage_path{base.string() + ".cells"},
edge_based_graph_path{base.string() + ".ebg"}
{
}
bool StorageConfig::IsValid() const
{
const constexpr auto num_files = 15;
const boost::filesystem::path paths[num_files] = {ram_index_path,
file_index_path,
hsgr_data_path,
nodes_data_path,
edges_data_path,
core_data_path,
geometries_path,
timestamp_path,
turn_weight_penalties_path,
turn_duration_penalties_path,
datasource_names_path,
datasource_indexes_path,
names_data_path,
properties_path,
intersection_class_path};
bool success = true;
for (auto path = paths; path != paths + num_files; ++path)
// Common files
if (!CheckFileList({ram_index_path,
file_index_path,
nodes_data_path,
edges_data_path,
geometries_path,
timestamp_path,
turn_weight_penalties_path,
turn_duration_penalties_path,
names_data_path,
properties_path,
intersection_class_path}))
{
if (!boost::filesystem::is_regular_file(*path))
{
util::Log(logWARNING) << "Missing/Broken File: " << path->string();
success = false;
}
return false;
}
return success;
// TODO: add algorithm checks
// CH files
CheckFileList({hsgr_data_path, core_data_path, datasource_names_path, datasource_indexes_path});
// MLD files
CheckFileList({mld_partition_path, mld_storage_path, edge_based_graph_path});
return true;
}
}
}