Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

feat: unify turn_penalties_index dump same with turn_weight_penalties and turn_duration_penalties

Browse Source
This commit is contained in:
Jay Zhang 2020-10-23 09:38:56 +00:00
parent 1ba8aba466
commit 7d28b7aa3a
3 changed files with 28 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
include/extractor/files.hpp
View File

@ -399,6 +399,27 @@ inline void readTurnDurationPenalty(const boost::filesystem::path &path, TurnPen
storage::serialization::read(reader, "/common/turn_penalty/duration", turn_penalty);
}
// writes .osrm.turn_penalties_index
template <typename TurnIndexT>
inline void writeTurnPenaltiesIndex(const boost::filesystem::path &path,
const TurnIndexT &turn_penalties_index)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
storage::serialization::write(writer, "/extractor/turn_index", turn_penalties_index);
}
// read .osrm.turn_penalties_index
template <typename TurnIndexT>
inline void readTurnPenaltiesIndex(const boost::filesystem::path &path, TurnIndexT &turn_penalties_index)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
storage::serialization::read(reader, "/extractor/turn_index", turn_penalties_index);
}
// writes .osrm.restrictions
template <typename ConditionalRestrictionsT>
inline void writeConditionalRestrictions(const boost::filesystem::path &path,

39
src/extractor/edge_based_graph_factory.cpp
View File

@ -53,9 +53,6 @@ template <> struct hash<std::pair<NodeID, NodeID>>
};
}
// Buffer size of turn_indexes_write_buffer to reduce number of write(v) syscals
const constexpr int TURN_INDEX_WRITE_BUFFER_SIZE = 1000;
namespace osrm
{
namespace extractor
@ -449,10 +446,6 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
std::size_t node_based_edge_counter = 0;
storage::tar::FileWriter turn_penalties_index_file(
turn_penalties_index_filename, storage::tar::FileWriter::GenerateFingerprint);
turn_penalties_index_file.WriteFrom("/extractor/turn_index", (char *)nullptr, 0);
SuffixTable street_name_suffix_table(scripting_environment);
const auto &turn_lanes_data = transformTurnLaneMapIntoArrays(lane_description_map);
intersection::MergableRoadDetector mergable_road_detector(m_node_based_graph,
@ -468,6 +461,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
// FIXME these need to be tuned in pre-allocated size
std::vector<TurnPenalty> turn_weight_penalties;
std::vector<TurnPenalty> turn_duration_penalties;
std::vector<lookup::TurnIndexBlock> turn_penalties_index;
// Now, renumber all our maneuver overrides to use edge-based-nodes
std::vector<StorageManeuverOverride> storage_maneuver_overrides;
@ -487,14 +481,6 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
{
const NodeID node_count = m_node_based_graph.GetNumberOfNodes();
// Because we write TurnIndexBlock data as we go, we'll
// buffer them into groups of 1000 to reduce the syscall
// count by 1000x. This doesn't need much memory, but
// greatly reduces the syscall overhead of writing lots
// of small objects
std::vector<lookup::TurnIndexBlock> turn_indexes_write_buffer;
turn_indexes_write_buffer.reserve(TURN_INDEX_WRITE_BUFFER_SIZE);
// This struct is the buffered output of the `processor_stage`. This data is
// appended to the various output arrays/files by the `output_stage`.
// same as IntersectionData, but grouped with edge to allow sorting after creating.
@ -510,7 +496,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
m_edge_based_edge_list.push_back(edge_with_data.edge);
turn_weight_penalties.push_back(edge_with_data.turn_weight_penalty);
turn_duration_penalties.push_back(edge_with_data.turn_duration_penalty);
turn_indexes_write_buffer.push_back(edge_with_data.turn_index);
turn_penalties_index.push_back(edge_with_data.turn_index);
};
struct EdgesPipelineBuffer
@ -1054,15 +1040,6 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
// NOTE: potential overflow here if we hit 2^32 routable edges
BOOST_ASSERT(m_edge_based_edge_list.size() <= std::numeric_limits<NodeID>::max());
// Buffer writes to reduce syscall count
if (turn_indexes_write_buffer.size() >= TURN_INDEX_WRITE_BUFFER_SIZE)
{
turn_penalties_index_file.ContinueFrom("/extractor/turn_index",
turn_indexes_write_buffer.data(),
turn_indexes_write_buffer.size());
turn_indexes_write_buffer.clear();
}
// Copy via-way restrictions delayed data
delayed_data.insert(
delayed_data.end(), buffer->delayed_data.begin(), buffer->delayed_data.end());
@ -1118,15 +1095,6 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
storage_maneuver_overrides.push_back(storage_override);
}
// Flush the turn_indexes_write_buffer if it's not empty
if (!turn_indexes_write_buffer.empty())
{
turn_penalties_index_file.ContinueFrom("/extractor/turn_index",
turn_indexes_write_buffer.data(),
turn_indexes_write_buffer.size());
turn_indexes_write_buffer.clear();
}
}
{
util::Log() << "Sorting and writing " << storage_maneuver_overrides.size()
@ -1162,9 +1130,10 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
indexed_conditionals);
// write weight penalties per turn
BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size());
BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size() && turn_weight_penalties.size() == turn_penalties_index.size());
files::writeTurnWeightPenalty(turn_weight_penalties_filename, turn_weight_penalties);
files::writeTurnDurationPenalty(turn_duration_penalties_filename, turn_duration_penalties);
files::writeTurnPenaltiesIndex(turn_penalties_index_filename, turn_penalties_index);
util::Log() << "Generated " << m_edge_based_node_segments.size() << " edge based node segments";
util::Log() << "Node-based graph contains " << node_based_edge_counter << " edges";

3
src/updater/updater.cpp
View File

@ -432,6 +432,9 @@ updateTurnPenalties(const UpdaterConfig &config,
{
const auto weight_multiplier = profile_properties.GetWeightMultiplier();
// [NOTE] turn_index_blocks could be simply loaded by `files::readTurnPenaltiesIndex()`,
// however, we leave the below mmap to keep compatiblity.
// Use `files::readTurnPenaltiesIndex()` instead once the compatiblity is not that important.
// Mapped file pointer for turn indices
boost::iostreams::mapped_file_source turn_index_region;
const extractor::lookup::TurnIndexBlock *turn_index_blocks;