Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Move geometry ids, name_ids and travel_modes to EdgeBasedNodeData

Browse Source
This commit is contained in:
Michael Krasnyk
2017-04-25 11:36:34 +02:00
committed by Patrick Niklaus
parent 1f701341db
commit f1e4349c82
20 changed files with 355 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/extractor/edge_based_graph_factory.cpp
+54 -33
View File
@@ -168,6 +168,7 @@ NBGToEBG EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const N
}
void EdgeBasedGraphFactory::Run(ScriptingEnvironment &scripting_environment,
const std::string &nodes_data_filename,
const std::string &turn_data_filename,
const std::string &turn_lane_data_filename,
const std::string &turn_weight_penalties_filename,
@@ -186,8 +187,13 @@ void EdgeBasedGraphFactory::Run(ScriptingEnvironment &scripting_environment,
}
TIMER_STOP(generate_nodes);
TIMER_START(generate_nodes_data);
auto index_nbg_edgeid_to_ebg_nodeid = GenerateEdgeExpandedNodesData(nodes_data_filename);
TIMER_STOP(generate_nodes_data);
TIMER_START(generate_edges);
GenerateEdgeExpandedEdges(scripting_environment,
index_nbg_edgeid_to_ebg_nodeid,
turn_data_filename,
turn_lane_data_filename,
turn_weight_penalties_filename,
@@ -292,9 +298,48 @@ std::vector<NBGToEBG> EdgeBasedGraphFactory::GenerateEdgeExpandedNodes()
return mapping;
}
/// Actually it also generates turn data and serializes them...
std::unordered_map<EdgeID, NodeID>
EdgeBasedGraphFactory::GenerateEdgeExpandedNodesData(const std::string &node_data_filename) const
{
// parallel_for( blocked_range<int>( 0, n ), avg, auto_partitioner( ) );
NodeID ebg_node_id = 0;
std::unordered_map<EdgeID, NodeID> index;
EdgeBasedNodeDataExternalContainer node_data_container;
for (auto node_id : util::irange(0u, m_node_based_graph->GetNumberOfNodes()))
{
for (auto edge_id : m_node_based_graph->GetAdjacentEdgeRange(node_id))
{
const EdgeData &edge_data = m_node_based_graph->GetEdgeData(edge_id);
const bool is_encoded_forwards =
m_compressed_edge_container.HasZippedEntryForForwardID(edge_id);
const bool is_encoded_backwards =
m_compressed_edge_container.HasZippedEntryForReverseID(edge_id);
if (is_encoded_forwards || is_encoded_backwards)
{
auto geometry_id =
is_encoded_forwards
? m_compressed_edge_container.GetZippedPositionForForwardID(edge_id)
: m_compressed_edge_container.GetZippedPositionForReverseID(edge_id);
BOOST_ASSERT(index.find(edge_id) == index.end());
index.insert({edge_id, ebg_node_id++});
node_data_container.push_back(GeometryID{geometry_id, is_encoded_forwards},
edge_data.name_id,
edge_data.travel_mode);
}
}
}
files::writeNodeData(node_data_filename, node_data_container);
return index;
}
/// Actually it also generates turn data and serializes them...
void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
ScriptingEnvironment &scripting_environment,
const std::unordered_map<EdgeID, NodeID> &nbg_edgeid_to_ebg_nodeid,
const std::string &turn_data_filename,
const std::string &turn_lane_data_filename,
const std::string &turn_weight_penalties_filename,
@@ -454,39 +499,15 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
BOOST_ASSERT(!edge_data2.reversed);
// the following is the core of the loop.
const bool is_encoded_forwards =
m_compressed_edge_container.HasZippedEntryForForwardID(incoming_edge);
const bool is_encoded_backwards =
m_compressed_edge_container.HasZippedEntryForReverseID(incoming_edge);
BOOST_ASSERT(is_encoded_forwards || is_encoded_backwards);
if (is_encoded_forwards)
{
turn_data_container.push_back(
GeometryID{m_compressed_edge_container.GetZippedPositionForForwardID(
incoming_edge),
true},
edge_data1.name_id,
turn.instruction,
turn.lane_data_id,
entry_class_id,
edge_data1.travel_mode,
util::guidance::TurnBearing(intersection[0].bearing),
util::guidance::TurnBearing(turn.bearing));
}
else if (is_encoded_backwards)
{
turn_data_container.push_back(
GeometryID{m_compressed_edge_container.GetZippedPositionForReverseID(
incoming_edge),
false},
edge_data1.name_id,
turn.instruction,
turn.lane_data_id,
entry_class_id,
edge_data1.travel_mode,
util::guidance::TurnBearing(intersection[0].bearing),
util::guidance::TurnBearing(turn.bearing));
}
BOOST_ASSERT(nbg_edgeid_to_ebg_nodeid.find(incoming_edge) !=
nbg_edgeid_to_ebg_nodeid.end());
turn_data_container.push_back(
nbg_edgeid_to_ebg_nodeid.find(incoming_edge)->second,
turn.instruction,
turn.lane_data_id,
entry_class_id,
util::guidance::TurnBearing(intersection[0].bearing),
util::guidance::TurnBearing(turn.bearing));
// compute weight and duration penalties
auto is_traffic_light = m_traffic_lights.count(node_at_center_of_intersection);
src/extractor/extractor.cpp
+1
View File
@@ -484,6 +484,7 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment,
turn_lane_map);
edge_based_graph_factory.Run(scripting_environment,
config.edge_based_nodes_data_path,
config.edge_output_path,
config.turn_lane_data_file_name,
config.turn_weight_penalties_path,
src/storage/storage.cpp
+39 -21
View File
@@ -234,10 +234,7 @@ void Storage::PopulateLayout(DataLayout &layout)
const auto number_of_original_edges = edges_file.ReadElementCount64();
// note: settings this all to the same size is correct, we extract them from the same struct
layout.SetBlockSize<NodeID>(DataLayout::VIA_NODE_LIST, number_of_original_edges);
layout.SetBlockSize<unsigned>(DataLayout::NAME_ID_LIST, number_of_original_edges);
layout.SetBlockSize<extractor::TravelMode>(DataLayout::TRAVEL_MODE,
number_of_original_edges);
layout.SetBlockSize<NodeID>(DataLayout::EDGE_BASED_NODE_ID_LIST, number_of_original_edges);
layout.SetBlockSize<util::guidance::TurnBearing>(DataLayout::PRE_TURN_BEARING,
number_of_original_edges);
layout.SetBlockSize<util::guidance::TurnBearing>(DataLayout::POST_TURN_BEARING,
@@ -248,6 +245,16 @@ void Storage::PopulateLayout(DataLayout &layout)
layout.SetBlockSize<EntryClassID>(DataLayout::ENTRY_CLASSID, number_of_original_edges);
}
{
io::FileReader nodes_data_file(config.edge_based_nodes_data_path,
io::FileReader::VerifyFingerprint);
const auto nodes_number = nodes_data_file.ReadElementCount64();
layout.SetBlockSize<NodeID>(DataLayout::GEOMETRY_ID_LIST, nodes_number);
layout.SetBlockSize<unsigned>(DataLayout::NAME_ID_LIST, nodes_number);
layout.SetBlockSize<extractor::TravelMode>(DataLayout::TRAVEL_MODE_LIST, nodes_number);
}
if (boost::filesystem::exists(config.hsgr_data_path))
{
io::FileReader reader(config.hsgr_data_path, io::FileReader::VerifyFingerprint);
@@ -573,17 +580,35 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
config.turn_lane_description_path, offsets, masks);
}
// Load edge-based nodes data
{
auto geometry_id_list_ptr =
layout.GetBlockPtr<GeometryID, true>(memory_ptr, storage::DataLayout::GEOMETRY_ID_LIST);
util::vector_view<GeometryID> geometry_ids(
geometry_id_list_ptr, layout.num_entries[storage::DataLayout::GEOMETRY_ID_LIST]);
auto name_id_list_ptr =
layout.GetBlockPtr<NameID, true>(memory_ptr, storage::DataLayout::NAME_ID_LIST);
util::vector_view<NameID> name_ids(name_id_list_ptr,
layout.num_entries[storage::DataLayout::NAME_ID_LIST]);
auto travel_mode_list_ptr = layout.GetBlockPtr<extractor::TravelMode, true>(
memory_ptr, storage::DataLayout::TRAVEL_MODE_LIST);
util::vector_view<extractor::TravelMode> travel_modes(
travel_mode_list_ptr, layout.num_entries[storage::DataLayout::TRAVEL_MODE_LIST]);
extractor::EdgeBasedNodeDataView node_data(
std::move(geometry_ids), std::move(name_ids), std::move(travel_modes));
extractor::files::readNodeData(config.edge_based_nodes_data_path, node_data);
}
// Load original edge data
{
auto via_geometry_list_ptr =
layout.GetBlockPtr<GeometryID, true>(memory_ptr, storage::DataLayout::VIA_NODE_LIST);
util::vector_view<GeometryID> geometry_ids(
via_geometry_list_ptr, layout.num_entries[storage::DataLayout::VIA_NODE_LIST]);
const auto travel_mode_list_ptr = layout.GetBlockPtr<extractor::TravelMode, true>(
memory_ptr, storage::DataLayout::TRAVEL_MODE);
util::vector_view<extractor::TravelMode> travel_modes(
travel_mode_list_ptr, layout.num_entries[storage::DataLayout::TRAVEL_MODE]);
auto node_id_list_ptr = layout.GetBlockPtr<NodeID, true>(
memory_ptr, storage::DataLayout::EDGE_BASED_NODE_ID_LIST);
util::vector_view<NodeID> node_ids(
node_id_list_ptr, layout.num_entries[storage::DataLayout::EDGE_BASED_NODE_ID_LIST]);
const auto lane_data_id_ptr =
layout.GetBlockPtr<LaneDataID, true>(memory_ptr, storage::DataLayout::LANE_DATA_ID);
@@ -596,11 +621,6 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
util::vector_view<extractor::guidance::TurnInstruction> turn_instructions(
turn_instruction_list_ptr, layout.num_entries[storage::DataLayout::TURN_INSTRUCTION]);
const auto name_id_list_ptr =
layout.GetBlockPtr<NameID, true>(memory_ptr, storage::DataLayout::NAME_ID_LIST);
util::vector_view<NameID> name_ids(name_id_list_ptr,
layout.num_entries[storage::DataLayout::NAME_ID_LIST]);
const auto entry_class_id_list_ptr =
layout.GetBlockPtr<EntryClassID, true>(memory_ptr, storage::DataLayout::ENTRY_CLASSID);
util::vector_view<EntryClassID> entry_class_ids(
@@ -616,11 +636,9 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
util::vector_view<util::guidance::TurnBearing> post_turn_bearings(
post_turn_bearing_ptr, layout.num_entries[storage::DataLayout::POST_TURN_BEARING]);
extractor::TurnDataView turn_data(std::move(geometry_ids),
std::move(name_ids),
extractor::TurnDataView turn_data(std::move(node_ids),
std::move(turn_instructions),
std::move(lane_data_ids),
std::move(travel_modes),
std::move(entry_class_ids),
std::move(pre_turn_bearings),
std::move(post_turn_bearings));
src/storage/storage_config.cpp
+4 -2
View File
@@ -27,8 +27,10 @@ bool CheckFileList(const std::vector<boost::filesystem::path> &files)
StorageConfig::StorageConfig(const boost::filesystem::path &base)
: ram_index_path{base.string() + ".ramIndex"}, file_index_path{base.string() + ".fileIndex"},
hsgr_data_path{base.string() + ".hsgr"}, nodes_data_path{base.string() + ".nodes"},
edges_data_path{base.string() + ".edges"}, core_data_path{base.string() + ".core"},
geometries_path{base.string() + ".geometry"}, timestamp_path{base.string() + ".timestamp"},
edges_data_path{base.string() + ".edges"},
edge_based_nodes_data_path{base.string() + ".nodes_data"},
core_data_path{base.string() + ".core"}, geometries_path{base.string() + ".geometry"},
timestamp_path{base.string() + ".timestamp"},
turn_weight_penalties_path{base.string() + ".turn_weight_penalties"},
turn_duration_penalties_path{base.string() + ".turn_duration_penalties"},
datasource_names_path{base.string() + ".datasource_names"},
src/updater/updater.cpp
+21 -8
View File
@@ -76,12 +76,16 @@ void checkWeightsConsistency(
extractor::SegmentDataContainer segment_data;
extractor::files::readSegmentData(config.geometry_path, segment_data);
extractor::EdgeBasedNodeDataContainer node_data;
extractor::files::readNodeData(config.osrm_input_path.string() + ".nodes_data", node_data);
extractor::TurnDataContainer turn_data;
extractor::files::readTurnData(config.osrm_input_path.string() + ".edges", turn_data);
for (auto &edge : edge_based_edge_list)
{
auto geometry_id = turn_data.GetGeometryID(edge.data.turn_id);
const auto node_id = turn_data.GetNodeID(edge.data.turn_id);
const auto geometry_id = node_data.GetGeometryID(node_id);
if (geometry_id.forward)
{
@@ -457,6 +461,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
throw util::exception("Limit of 255 segment speed and turn penalty files each reached" +
SOURCE_REF);
extractor::EdgeBasedNodeDataContainer node_data;
extractor::TurnDataContainer turn_data;
extractor::SegmentDataContainer segment_data;
extractor::ProfileProperties profile_properties;
@@ -468,6 +473,10 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
extractor::files::readSegmentData(config.geometry_path, segment_data);
};
const auto load_node_data = [&] {
extractor::files::readNodeData(config.edge_based_nodes_data_path, node_data);
};
const auto load_edge_data = [&] {
extractor::files::readTurnData(config.edge_data_path, turn_data);
};
@@ -491,7 +500,8 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
profile_properties = profile_properties_file.ReadOne<extractor::ProfileProperties>();
};
tbb::parallel_invoke(load_edge_data,
tbb::parallel_invoke(load_node_data,
load_edge_data,
load_segment_data,
load_turn_weight_penalties,
load_turn_duration_penalties,
@@ -524,11 +534,13 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
updated_segments.resize(offset + updated_turn_penalties.size());
// we need to re-compute all edges that have updated turn penalties.
// this marks it for re-computation
std::transform(
updated_turn_penalties.begin(),
updated_turn_penalties.end(),
updated_segments.begin() + offset,
[&turn_data](const std::uint64_t turn_id) { return turn_data.GetGeometryID(turn_id); });
std::transform(updated_turn_penalties.begin(),
updated_turn_penalties.end(),
updated_segments.begin() + offset,
[&node_data, &turn_data](const std::uint64_t turn_id) {
const auto node_id = turn_data.GetNodeID(turn_id);
return node_data.GetGeometryID(node_id);
});
}
tbb::parallel_sort(updated_segments.begin(),
@@ -586,7 +598,8 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
});
const auto update_edge = [&](extractor::EdgeBasedEdge &edge) {
const auto geometry_id = turn_data.GetGeometryID(edge.data.turn_id);
const auto node_id = turn_data.GetNodeID(edge.data.turn_id);
const auto geometry_id = node_data.GetGeometryID(node_id);
auto updated_iter = std::lower_bound(updated_segments.begin(),
updated_segments.end(),
geometry_id,
src/util/assert.cpp
+1
View File
@@ -15,6 +15,7 @@ namespace
std::cerr << "[assert][" << tid << "] " << file << ":" << line << "\nin: " << function << ": "
<< expr << "\n"
<< msg;
std::terminate();
}
}