Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Make CSV parsing function generalized

Browse Source
This commit is contained in:
Michael Krasnyk 2017-01-09 15:34:15 +01:00 committed by Patrick Niklaus
parent bb4f13a47c
commit 29f736f1c8
1 changed files with 252 additions and 409 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

661
src/contractor/contractor.cpp
View File

@ -22,9 +22,13 @@
#include <boost/assert.hpp> #include <boost/assert.hpp>
#include <boost/filesystem/fstream.hpp> #include <boost/filesystem/fstream.hpp>
#include <boost/functional/hash.hpp> #include <boost/functional/hash.hpp>
#include <boost/fusion/adapted/std_pair.hpp>
#include <boost/fusion/include/adapt_adt.hpp>
#include <boost/interprocess/file_mapping.hpp> #include <boost/interprocess/file_mapping.hpp>
#include <boost/interprocess/mapped_region.hpp> #include <boost/interprocess/mapped_region.hpp>
#include <boost/spirit/include/phoenix.hpp>
#include <boost/spirit/include/qi.hpp> #include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/support_line_pos_iterator.hpp>
#include <tbb/blocked_range.h> #include <tbb/blocked_range.h>
#include <tbb/concurrent_unordered_map.h> #include <tbb/concurrent_unordered_map.h>
@ -45,46 +49,191 @@
#include <tuple> #include <tuple>
#include <vector> #include <vector>
namespace std
{
template <> struct hash<std::pair<OSMNodeID, OSMNodeID>>
{
std::size_t operator()(const std::pair<OSMNodeID, OSMNodeID> &k) const noexcept
{
return static_cast<uint64_t>(k.first) ^ (static_cast<uint64_t>(k.second) << 12);
}
};
template <> struct hash<std::tuple<OSMNodeID, OSMNodeID, OSMNodeID>>
{
std::size_t operator()(const std::tuple<OSMNodeID, OSMNodeID, OSMNodeID> &k) const noexcept
{
std::size_t seed = 0;
boost::hash_combine(seed, static_cast<uint64_t>(std::get<0>(k)));
boost::hash_combine(seed, static_cast<uint64_t>(std::get<1>(k)));
boost::hash_combine(seed, static_cast<uint64_t>(std::get<2>(k)));
return seed;
}
};
}
namespace namespace
{ {
struct Segment final
{
std::uint64_t from, to;
Segment() : from(0), to(0) {}
Segment(const std::uint64_t from, const std::uint64_t to) : from(from), to(to) {}
Segment(const OSMNodeID from, const OSMNodeID to)
: from(static_cast<std::uint64_t>(from)), to(static_cast<std::uint64_t>(to))
{
}
bool operator<(const Segment &rhs) const
{
return std::tie(from, to) < std::tie(rhs.from, rhs.to);
}
bool operator==(const Segment &rhs) const
{
return std::tie(from, to) == std::tie(rhs.from, rhs.to);
}
};
struct SpeedSource final
{
unsigned speed;
std::uint8_t source;
};
struct Turn final
{
std::uint64_t from, via, to;
Turn() : from(0), via(0), to(0) {}
Turn(const std::uint64_t from, const std::uint64_t via, const std::uint64_t to)
: from(from), via(via), to(to)
{
}
Turn(const OSMNodeID from, const OSMNodeID via, const OSMNodeID to)
: from(static_cast<std::uint64_t>(from)), via(static_cast<std::uint64_t>(via)),
to(static_cast<std::uint64_t>(to))
{
}
bool operator<(const Turn &rhs) const
{
return std::tie(from, via, to) < std::tie(rhs.from, rhs.via, rhs.to);
}
bool operator==(const Turn &rhs) const
{
return std::tie(from, via, to) == std::tie(rhs.from, rhs.via, rhs.to);
}
};
struct PenaltySource final
{
double penalty;
std::uint8_t source;
};
template <typename T> inline bool is_aligned(const void *pointer) template <typename T> inline bool is_aligned(const void *pointer)
{ {
static_assert(sizeof(T) % alignof(T) == 0, "pointer can not be used as an array pointer"); static_assert(sizeof(T) % alignof(T) == 0, "pointer can not be used as an array pointer");
return reinterpret_cast<uintptr_t>(pointer) % alignof(T) == 0; return reinterpret_cast<uintptr_t>(pointer) % alignof(T) == 0;
} }
} } // anon ns
BOOST_FUSION_ADAPT_STRUCT(Segment, (decltype(Segment::from), from)(decltype(Segment::to), to))
BOOST_FUSION_ADAPT_STRUCT(SpeedSource, (decltype(SpeedSource::speed), speed))
BOOST_FUSION_ADAPT_STRUCT(Turn,
(decltype(Turn::from), from)(decltype(Turn::via), via)(decltype(Turn::to),
to))
BOOST_FUSION_ADAPT_STRUCT(PenaltySource, (decltype(PenaltySource::penalty), penalty))
namespace osrm namespace osrm
{ {
namespace contractor namespace contractor
{ {
namespace
{
namespace qi = boost::spirit::qi;
}
// Functor to parse a list of CSV files using "key,value,comment" grammar.
// Key and Value structures must be a model of Random Access Sequence.
// Also the Value structure must have source member that will be filled
// with the corresponding file index in the CSV filenames vector.
template <typename Key, typename Value> struct CSVFilesParser
{
using Iterator = boost::spirit::line_pos_iterator<boost::spirit::istream_iterator>;
using KeyRule = qi::rule<Iterator, Key()>;
using ValueRule = qi::rule<Iterator, Value()>;
CSVFilesParser(const KeyRule &key_rule, const ValueRule &value_rule)
: key_rule(key_rule), value_rule(value_rule)
{
}
// Operator returns a lambda function that maps input Key to boost::optional<Value>.
auto operator()(const std::vector<std::string> &csv_filenames) const
{
try
{
tbb::spin_mutex mutex;
std::vector<std::pair<Key, Value>> lookup;
tbb::parallel_for(std::size_t{0},
csv_filenames.size(),
[&](const std::size_t idx) {
// id starts at one, 0 means we assigned the weight from profile
auto local = ParseCSVFile(csv_filenames[idx], idx + 1);
{ // Merge local CSV results into a flat global vector
tbb::spin_mutex::scoped_lock _{mutex};
lookup.insert(end(lookup),
std::make_move_iterator(begin(local)),
std::make_move_iterator(end(local)));
}
});
// With flattened map-ish view of all the files, make a stable sort on key and source
// and unique them on key to keep only the value with the largest file index
// and the largest line number in a file.
// The operands order is swapped to make descending ordering on (key, source)
std::stable_sort(begin(lookup), end(lookup), [](const auto &lhs, const auto &rhs) {
return rhs.first < lhs.first ||
(rhs.first == lhs.first && rhs.second.source < lhs.second.source);
});
// Unique only on key to take the source precedence into account and remove duplicates.
const auto it =
std::unique(begin(lookup), end(lookup), [](const auto &lhs, const auto &rhs) {
return lhs.first == rhs.first;
});
lookup.erase(it, end(lookup));
osrm::util::Log() << "In total loaded " << csv_filenames.size()
<< " file(s) with a total of " << lookup.size() << " unique values";
return [lookup](const Key &key) {
using Result = boost::optional<Value>;
const auto it = std::lower_bound(
lookup.begin(), lookup.end(), key, [](const auto &lhs, const auto &rhs) {
return rhs < lhs.first;
});
return it != std::end(lookup) && !(it->first < key) ? Result(it->second) : Result();
};
}
catch (const tbb::captured_exception &e)
{
throw osrm::util::exception(e.what() + SOURCE_REF);
}
}
private:
// Parse a single CSV file and return result as a vector<Key, Value>
auto ParseCSVFile(const std::string &filename, std::size_t file_id) const
{
std::ifstream input_stream(filename, std::ios::binary);
input_stream.unsetf(std::ios::skipws);
boost::spirit::istream_iterator sfirst(input_stream), slast;
Iterator first(sfirst), last(slast);
ValueRule value_source =
value_rule[qi::_val = qi::_1, boost::phoenix::bind(&Value::source, qi::_val) = file_id];
qi::rule<Iterator, std::pair<Key, Value>()> csv_line =
(key_rule >> ',' >> value_source) >> -(',' >> *(qi::char_ - qi::eol));
std::vector<std::pair<Key, Value>> result;
const auto ok = qi::parse(first, last, (csv_line % qi::eol) >> *qi::eol, result);
if (!ok || first != last)
{
const auto message =
boost::format("CSV file %1% malformed on line %2%") % filename % first.position();
throw osrm::util::exception(message.str() + SOURCE_REF);
}
osrm::util::Log() << "Loaded " << filename << " with " << result.size() << "values";
return std::move(result);
}
KeyRule key_rule;
ValueRule value_rule;
};
// Returns duration in deci-seconds // Returns duration in deci-seconds
inline EdgeWeight distanceAndSpeedToWeight(double distance_in_meters, double speed_in_kmh) inline EdgeWeight ConvertToDuration(double distance_in_meters, double speed_in_kmh)
{ {
BOOST_ASSERT(speed_in_kmh > 0); BOOST_ASSERT(speed_in_kmh > 0);
const double speed_in_ms = speed_in_kmh / 3.6; const double speed_in_ms = speed_in_kmh / 3.6;
@ -93,19 +242,18 @@ inline EdgeWeight distanceAndSpeedToWeight(double distance_in_meters, double spe
} }
// Returns updated edge weight // Returns updated edge weight
template <class IterType> EdgeWeight GetNewWeight(const SpeedSource &value,
EdgeWeight getNewWeight(IterType speed_iter,
const double &segment_length, const double &segment_length,
const std::vector<std::string> &segment_speed_filenames, const std::vector<std::string> &segment_speed_filenames,
const EdgeWeight old_weight, const EdgeWeight old_weight,
const double log_edge_updates_factor) const double log_edge_updates_factor,
const OSMNodeID &from,
const OSMNodeID &to)
{ {
const auto new_segment_weight = const auto new_segment_weight =
(speed_iter->speed_source.speed > 0) (value.speed > 0) ? ConvertToDuration(segment_length, value.speed) : INVALID_EDGE_WEIGHT;
? distanceAndSpeedToWeight(segment_length, speed_iter->speed_source.speed) // the check here is enabled by the `--edge-weight-updates-over-factor` flag it logs
: INVALID_EDGE_WEIGHT; // a warning if the new weight exceeds a heuristic of what a reasonable weight update is
// the check here is enabled by the `--edge-weight-updates-over-factor` flag
// it logs a warning if the new weight exceeds a heuristic of what a reasonable weight update is
if (log_edge_updates_factor > 0 && old_weight != 0) if (log_edge_updates_factor > 0 && old_weight != 0)
{ {
auto new_secs = new_segment_weight / 10.0; auto new_secs = new_segment_weight / 10.0;
@ -113,14 +261,14 @@ EdgeWeight getNewWeight(IterType speed_iter,
auto approx_original_speed = (segment_length / old_secs) * 3.6; auto approx_original_speed = (segment_length / old_secs) * 3.6;
if (old_weight >= (new_segment_weight * log_edge_updates_factor)) if (old_weight >= (new_segment_weight * log_edge_updates_factor))
{ {
auto speed_file = segment_speed_filenames.at(speed_iter->speed_source.source - 1); auto speed_file = segment_speed_filenames.at(value.source - 1);
util::Log(logWARNING) << "[weight updates] Edge weight update from " << old_secs util::Log(logWARNING) << "[weight updates] Edge weight update from " << old_secs
<< "s to " << new_secs << "s to " << new_secs << "s New speed: " << value.speed
<< "s New speed: " << speed_iter->speed_source.speed << " kph" << " kph"
<< ". Old speed: " << approx_original_speed << " kph" << ". Old speed: " << approx_original_speed << " kph"
<< ". Segment length: " << segment_length << " m" << ". Segment length: " << segment_length << " m"
<< ". Segment: " << speed_iter->segment.from << "," << ". Segment: " << from << "," << to << " based on "
<< speed_iter->segment.to << " based on " << speed_file; << speed_file;
} }
} }
@ -219,293 +367,6 @@ int Contractor::Run()
} }
// Utilities for LoadEdgeExpandedGraph to restore my sanity // Utilities for LoadEdgeExpandedGraph to restore my sanity
namespace
{
struct Segment final
{
OSMNodeID from, to;
bool operator==(const Segment &other) const
{
return std::tie(from, to) == std::tie(other.from, other.to);
}
};
struct SpeedSource final
{
unsigned speed;
std::uint8_t source;
};
struct SegmentSpeedSource final
{
Segment segment;
SpeedSource speed_source;
// < operator is overloaded here to return a > comparison to be used by the
// std::lower_bound() call in the find() function
bool operator<(const SegmentSpeedSource &other) const
{
return std::tie(segment.from, segment.to) > std::tie(other.segment.from, other.segment.to);
}
};
struct Turn final
{
OSMNodeID from, via, to;
bool operator==(const Turn &other) const
{
return std::tie(from, via, to) == std::tie(other.from, other.via, other.to);
}
};
struct PenaltySource final
{
double penalty;
std::uint8_t source;
};
struct TurnPenaltySource final
{
Turn segment;
PenaltySource penalty_source;
// < operator is overloaded here to return a > comparison to be used by the
// std::lower_bound() call in the find() function
bool operator<(const TurnPenaltySource &other) const
{
return std::tie(segment.from, segment.via, segment.to) >
std::tie(other.segment.from, other.segment.via, other.segment.to);
}
};
using TurnPenaltySourceFlatMap = std::vector<TurnPenaltySource>;
using SegmentSpeedSourceFlatMap = std::vector<SegmentSpeedSource>;
// Find is a binary Search over a flattened key,val Segment storage
// It takes the flat map and a Segment/PenaltySource object that has an overloaded
// `==` operator, to make the std::lower_bound call work generically
template <typename FlatMap, typename SegmentKey>
auto find(const FlatMap &map, const SegmentKey &key)
{
const auto last = end(map);
auto it = std::lower_bound(begin(map), last, key);
if (it != last && (it->segment == key.segment))
return it;
return last;
}
// Functions for parsing files and creating lookup tables
SegmentSpeedSourceFlatMap
parse_segment_lookup_from_csv_files(const std::vector<std::string> &segment_speed_filenames)
{
// TODO: shares code with turn penalty lookup parse function
using Mutex = tbb::spin_mutex;
// Loaded and parsed in parallel, at the end we combine results in a flattened map-ish view
SegmentSpeedSourceFlatMap flatten;
Mutex flatten_mutex;
const auto parse_segment_speed_file = [&](const std::size_t idx) {
const auto file_id = idx + 1; // starts at one, zero means we assigned the weight
const auto filename = segment_speed_filenames[idx];
storage::io::FileReader segment_speed_file_reader(
filename, storage::io::FileReader::HasNoFingerprint);
SegmentSpeedSourceFlatMap local;
std::uint64_t from_node_id{};
std::uint64_t to_node_id{};
unsigned speed{};
std::size_t line_number = 0;
std::for_each(
segment_speed_file_reader.GetLineIteratorBegin(),
segment_speed_file_reader.GetLineIteratorEnd(),
[&](const std::string &line) {
++line_number;
using namespace boost::spirit::qi;
auto it = begin(line);
const auto last = end(line);
// The ulong_long -> uint64_t will likely break on 32bit platforms
const auto ok =
parse(it,
last, //
(ulong_long >> ',' >> ulong_long >> ',' >> uint_ >> *(',' >> *char_)), //
from_node_id,
to_node_id,
speed); //
if (!ok || it != last)
{
const std::string message{"Segment speed file " + filename +
" malformed on line " + std::to_string(line_number)};
throw util::exception(message + SOURCE_REF);
}
SegmentSpeedSource val{{OSMNodeID{from_node_id}, OSMNodeID{to_node_id}},
{speed, static_cast<std::uint8_t>(file_id)}};
local.push_back(std::move(val));
});
util::Log() << "Loaded speed file " << filename << " with " << local.size() << " speeds";
{
Mutex::scoped_lock _{flatten_mutex};
flatten.insert(end(flatten),
std::make_move_iterator(begin(local)),
std::make_move_iterator(end(local)));
}
};
try
{
tbb::parallel_for(std::size_t{0}, segment_speed_filenames.size(), parse_segment_speed_file);
}
catch (const tbb::captured_exception &e)
{
throw util::exception(e.what() + SOURCE_REF);
}
// With flattened map-ish view of all the files, sort and unique them on from,to,source
// The greater '>' is used here since we want to give files later on higher precedence
const auto sort_by = [](const SegmentSpeedSource &lhs, const SegmentSpeedSource &rhs) {
return std::tie(lhs.segment.from, lhs.segment.to, lhs.speed_source.source) >
std::tie(rhs.segment.from, rhs.segment.to, rhs.speed_source.source);
};
std::stable_sort(begin(flatten), end(flatten), sort_by);
// Unique only on from,to to take the source precedence into account and remove duplicates
const auto unique_by = [](const SegmentSpeedSource &lhs, const SegmentSpeedSource &rhs) {
return std::tie(lhs.segment.from, lhs.segment.to) ==
std::tie(rhs.segment.from, rhs.segment.to);
};
const auto it = std::unique(begin(flatten), end(flatten), unique_by);
flatten.erase(it, end(flatten));
util::Log() << "In total loaded " << segment_speed_filenames.size()
<< " speed file(s) with a total of " << flatten.size() << " unique values";
return flatten;
}
TurnPenaltySourceFlatMap
parse_turn_penalty_lookup_from_csv_files(const std::vector<std::string> &turn_penalty_filenames)
{
using Mutex = tbb::spin_mutex;
// TODO: shares code with turn penalty lookup parse function
TurnPenaltySourceFlatMap map;
Mutex flatten_mutex;
const auto parse_turn_penalty_file = [&](const std::size_t idx) {
const auto file_id = idx + 1; // starts at one, zero means we assigned the weight
const auto filename = turn_penalty_filenames[idx];
storage::io::FileReader turn_penalty_file_reader(filename,
storage::io::FileReader::HasNoFingerprint);
TurnPenaltySourceFlatMap local;
std::uint64_t from_node_id{};
std::uint64_t via_node_id{};
std::uint64_t to_node_id{};
double penalty{};
std::size_t line_number = 0;
std::for_each(
turn_penalty_file_reader.GetLineIteratorBegin(),
turn_penalty_file_reader.GetLineIteratorEnd(),
[&](const std::string &line) {
++line_number;
using namespace boost::spirit::qi;
auto it = begin(line);
const auto last = end(line);
// The ulong_long -> uint64_t will likely break on 32bit platforms
const auto ok = parse(it,
last, //
(ulong_long >> ',' >> ulong_long >> ',' >> ulong_long >>
',' >> double_ >> *(',' >> *char_)), //
from_node_id,
via_node_id,
to_node_id,
penalty); //
if (!ok || it != last)
{
const std::string message{"Turn penalty file " + filename +
" malformed on line " + std::to_string(line_number)};
throw util::exception(message + SOURCE_REF);
}
TurnPenaltySource val{
{OSMNodeID{from_node_id}, OSMNodeID{via_node_id}, OSMNodeID{to_node_id}},
{penalty, static_cast<std::uint8_t>(file_id)}};
local.push_back(std::move(val));
});
util::Log() << "Loaded penalty file " << filename << " with " << local.size()
<< " turn penalties";
{
Mutex::scoped_lock _{flatten_mutex};
map.insert(end(map),
std::make_move_iterator(begin(local)),
std::make_move_iterator(end(local)));
}
};
try
{
tbb::parallel_for(std::size_t{0}, turn_penalty_filenames.size(), parse_turn_penalty_file);
}
catch (const tbb::captured_exception &e)
{
throw util::exception(e.what() + SOURCE_REF);
}
// With flattened map-ish view of all the files, sort and unique them on from,to,source
// The greater '>' is used here since we want to give files later on higher precedence
const auto sort_by = [](const TurnPenaltySource &lhs, const TurnPenaltySource &rhs) {
return std::tie(
lhs.segment.from, lhs.segment.via, lhs.segment.to, lhs.penalty_source.source) >
std::tie(
rhs.segment.from, rhs.segment.via, rhs.segment.to, rhs.penalty_source.source);
};
std::stable_sort(begin(map), end(map), sort_by);
// Unique only on from,to to take the source precedence into account and remove duplicates
const auto unique_by = [](const TurnPenaltySource &lhs, const TurnPenaltySource &rhs) {
return std::tie(lhs.segment.from, lhs.segment.via, lhs.segment.to) ==
std::tie(rhs.segment.from, rhs.segment.via, rhs.segment.to);
};
const auto it = std::unique(begin(map), end(map), unique_by);
map.erase(it, end(map));
util::Log() << "In total loaded " << turn_penalty_filenames.size()
<< " turn penalty file(s) with a total of " << map.size() << " unique values";
return map;
}
} // anon ns
EdgeID Contractor::LoadEdgeExpandedGraph( EdgeID Contractor::LoadEdgeExpandedGraph(
std::string const &edge_based_graph_filename, std::string const &edge_based_graph_filename,
util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list, util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list,
@ -607,28 +468,22 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
edge_based_edge_list.resize(graph_header.number_of_edges); edge_based_edge_list.resize(graph_header.number_of_edges);
util::Log() << "Reading " << graph_header.number_of_edges << " edges from the edge based graph"; util::Log() << "Reading " << graph_header.number_of_edges << " edges from the edge based graph";
SegmentSpeedSourceFlatMap segment_speed_lookup; auto segment_speed_lookup = CSVFilesParser<Segment, SpeedSource>(
TurnPenaltySourceFlatMap turn_penalty_lookup; qi::ulong_long >> ',' >> qi::ulong_long, qi::uint_)(segment_speed_filenames);
const auto parse_segment_speeds = [&] { auto turn_penalty_lookup = CSVFilesParser<Turn, PenaltySource>(
if (update_edge_weights) qi::ulong_long >> ',' >> qi::ulong_long >> ',' >> qi::ulong_long,
segment_speed_lookup = parse_segment_lookup_from_csv_files(segment_speed_filenames); qi::double_)(turn_penalty_filenames);
};
const auto parse_turn_penalties = [&] {
if (update_turn_penalties)
turn_penalty_lookup = parse_turn_penalty_lookup_from_csv_files(turn_penalty_filenames);
};
// If we update the edge weights, this file will hold the datasource information for each // If we update the edge weights, this file will hold the datasource information for each
// segment; the other files will also be conditionally filled concurrently if we make an update // segment; the other files will also be conditionally filled concurrently if we make an update
std::vector<uint8_t> m_geometry_datasource; std::vector<uint8_t> geometry_datasource;
std::vector<extractor::QueryNode> internal_to_external_node_map; std::vector<extractor::QueryNode> internal_to_external_node_map;
std::vector<unsigned> m_geometry_indices; std::vector<unsigned> geometry_indices;
std::vector<NodeID> m_geometry_node_list; std::vector<NodeID> geometry_node_list;
std::vector<EdgeWeight> m_geometry_fwd_weight_list; std::vector<EdgeWeight> geometry_fwd_weight_list;
std::vector<EdgeWeight> m_geometry_rev_weight_list; std::vector<EdgeWeight> geometry_rev_weight_list;
const auto maybe_load_internal_to_external_node_map = [&] { const auto maybe_load_internal_to_external_node_map = [&] {
if (!(update_edge_weights || update_turn_penalties)) if (!(update_edge_weights || update_turn_penalties))
@ -648,31 +503,28 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
storage::io::FileReader geometry_file(geometry_filename, storage::io::FileReader geometry_file(geometry_filename,
storage::io::FileReader::HasNoFingerprint); storage::io::FileReader::HasNoFingerprint);
const auto number_of_indices = geometry_file.ReadElementCount32(); const auto number_of_indices = geometry_file.ReadElementCount32();
m_geometry_indices.resize(number_of_indices); geometry_indices.resize(number_of_indices);
geometry_file.ReadInto(m_geometry_indices.data(), number_of_indices); geometry_file.ReadInto(geometry_indices.data(), number_of_indices);
const auto number_of_compressed_geometries = geometry_file.ReadElementCount32(); const auto number_of_compressed_geometries = geometry_file.ReadElementCount32();
BOOST_ASSERT(m_geometry_indices.back() == number_of_compressed_geometries); BOOST_ASSERT(geometry_indices.back() == number_of_compressed_geometries);
m_geometry_node_list.resize(number_of_compressed_geometries); geometry_node_list.resize(number_of_compressed_geometries);
m_geometry_fwd_weight_list.resize(number_of_compressed_geometries); geometry_fwd_weight_list.resize(number_of_compressed_geometries);
m_geometry_rev_weight_list.resize(number_of_compressed_geometries); geometry_rev_weight_list.resize(number_of_compressed_geometries);
if (number_of_compressed_geometries > 0) if (number_of_compressed_geometries > 0)
{ {
geometry_file.ReadInto(m_geometry_node_list.data(), number_of_compressed_geometries); geometry_file.ReadInto(geometry_node_list.data(), number_of_compressed_geometries);
geometry_file.ReadInto(m_geometry_fwd_weight_list.data(), geometry_file.ReadInto(geometry_fwd_weight_list.data(),
number_of_compressed_geometries); number_of_compressed_geometries);
geometry_file.ReadInto(m_geometry_rev_weight_list.data(), geometry_file.ReadInto(geometry_rev_weight_list.data(),
number_of_compressed_geometries); number_of_compressed_geometries);
} }
}; };
// Folds all our actions into independently concurrently executing lambdas // Folds all our actions into independently concurrently executing lambdas
tbb::parallel_invoke(parse_segment_speeds, tbb::parallel_invoke(maybe_load_internal_to_external_node_map, maybe_load_geometries);
parse_turn_penalties, //
maybe_load_internal_to_external_node_map,
maybe_load_geometries);
if (update_edge_weights || update_turn_penalties) if (update_edge_weights || update_turn_penalties)
{ {
@ -685,7 +537,7 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
// CSV file that supplied the value that gets used for that segment, then // CSV file that supplied the value that gets used for that segment, then
// we write out this list so that it can be returned by the debugging // we write out this list so that it can be returned by the debugging
// vector tiles later on. // vector tiles later on.
m_geometry_datasource.resize(m_geometry_fwd_weight_list.size(), 0); geometry_datasource.resize(geometry_fwd_weight_list.size(), 0);
// Now, we iterate over all the segments stored in the StaticRTree, updating // Now, we iterate over all the segments stored in the StaticRTree, updating
// the packed geometry weights in the `.geometries` file (note: we do not // the packed geometry weights in the `.geometries` file (note: we do not
@ -718,38 +570,36 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
extractor::QueryNode *u; extractor::QueryNode *u;
extractor::QueryNode *v; extractor::QueryNode *v;
const unsigned forward_begin = const unsigned forward_begin = geometry_indices.at(leaf_object.packed_geometry_id);
m_geometry_indices.at(leaf_object.packed_geometry_id);
const auto current_fwd_weight = const auto current_fwd_weight =
m_geometry_fwd_weight_list[forward_begin + leaf_object.fwd_segment_position]; geometry_fwd_weight_list[forward_begin + leaf_object.fwd_segment_position];
u = &(internal_to_external_node_map u = &(internal_to_external_node_map
[m_geometry_node_list[forward_begin + leaf_object.fwd_segment_position]]); [geometry_node_list[forward_begin + leaf_object.fwd_segment_position]]);
v = &(internal_to_external_node_map v = &(
[m_geometry_node_list[forward_begin + leaf_object.fwd_segment_position + internal_to_external_node_map
1]]); [geometry_node_list[forward_begin + leaf_object.fwd_segment_position + 1]]);
const double segment_length = util::coordinate_calculation::greatCircleDistance( const double segment_length = util::coordinate_calculation::greatCircleDistance(
util::Coordinate{u->lon, u->lat}, util::Coordinate{v->lon, v->lat}); util::Coordinate{u->lon, u->lat}, util::Coordinate{v->lon, v->lat});
auto forward_speed_iter = find( if (auto value = segment_speed_lookup({u->node_id, v->node_id}))
segment_speed_lookup, SegmentSpeedSource{{u->node_id, v->node_id}, {0, 0}});
if (forward_speed_iter != segment_speed_lookup.end())
{ {
const auto new_segment_weight = getNewWeight(forward_speed_iter, const auto new_segment_weight = GetNewWeight(*value,
segment_length, segment_length,
segment_speed_filenames, segment_speed_filenames,
current_fwd_weight, current_fwd_weight,
log_edge_updates_factor); log_edge_updates_factor,
u->node_id,
v->node_id);
m_geometry_fwd_weight_list[forward_begin + 1 + geometry_fwd_weight_list[forward_begin + 1 + leaf_object.fwd_segment_position] =
leaf_object.fwd_segment_position] =
new_segment_weight; new_segment_weight;
m_geometry_datasource[forward_begin + 1 + leaf_object.fwd_segment_position] = geometry_datasource[forward_begin + 1 + leaf_object.fwd_segment_position] =
forward_speed_iter->speed_source.source; value->source;
// count statistics for logging // count statistics for logging
counters[forward_speed_iter->speed_source.source] += 1; counters[value->source] += 1;
} }
else else
{ {
@ -758,25 +608,24 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
} }
const auto current_rev_weight = const auto current_rev_weight =
m_geometry_rev_weight_list[forward_begin + leaf_object.fwd_segment_position]; geometry_rev_weight_list[forward_begin + leaf_object.fwd_segment_position];
const auto reverse_speed_iter = find( if (auto value = segment_speed_lookup({v->node_id, u->node_id}))
segment_speed_lookup, SegmentSpeedSource{{v->node_id, u->node_id}, {0, 0}});
if (reverse_speed_iter != segment_speed_lookup.end())
{ {
const auto new_segment_weight = getNewWeight(reverse_speed_iter, const auto new_segment_weight = GetNewWeight(*value,
segment_length, segment_length,
segment_speed_filenames, segment_speed_filenames,
current_rev_weight, current_rev_weight,
log_edge_updates_factor); log_edge_updates_factor,
m_geometry_rev_weight_list[forward_begin + leaf_object.fwd_segment_position] = v->node_id,
u->node_id);
geometry_rev_weight_list[forward_begin + leaf_object.fwd_segment_position] =
new_segment_weight; new_segment_weight;
m_geometry_datasource[forward_begin + leaf_object.fwd_segment_position] = geometry_datasource[forward_begin + leaf_object.fwd_segment_position] =
reverse_speed_iter->speed_source.source; value->source;
// count statistics for logging // count statistics for logging
counters[reverse_speed_iter->speed_source.source] += 1; counters[value->source] += 1;
} }
else else
{ {
@ -822,18 +671,18 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
const std::string message{"Failed to open " + geometry_filename + " for writing"}; const std::string message{"Failed to open " + geometry_filename + " for writing"};
throw util::exception(message + SOURCE_REF); throw util::exception(message + SOURCE_REF);
} }
const unsigned number_of_indices = m_geometry_indices.size(); const unsigned number_of_indices = geometry_indices.size();
const unsigned number_of_compressed_geometries = m_geometry_node_list.size(); const unsigned number_of_compressed_geometries = geometry_node_list.size();
geometry_stream.write(reinterpret_cast<const char *>(&number_of_indices), sizeof(unsigned)); geometry_stream.write(reinterpret_cast<const char *>(&number_of_indices), sizeof(unsigned));
geometry_stream.write(reinterpret_cast<char *>(&(m_geometry_indices[0])), geometry_stream.write(reinterpret_cast<char *>(&(geometry_indices[0])),
number_of_indices * sizeof(unsigned)); number_of_indices * sizeof(unsigned));
geometry_stream.write(reinterpret_cast<const char *>(&number_of_compressed_geometries), geometry_stream.write(reinterpret_cast<const char *>(&number_of_compressed_geometries),
sizeof(unsigned)); sizeof(unsigned));
geometry_stream.write(reinterpret_cast<char *>(&(m_geometry_node_list[0])), geometry_stream.write(reinterpret_cast<char *>(&(geometry_node_list[0])),
number_of_compressed_geometries * sizeof(NodeID)); number_of_compressed_geometries * sizeof(NodeID));
geometry_stream.write(reinterpret_cast<char *>(&(m_geometry_fwd_weight_list[0])), geometry_stream.write(reinterpret_cast<char *>(&(geometry_fwd_weight_list[0])),
number_of_compressed_geometries * sizeof(EdgeWeight)); number_of_compressed_geometries * sizeof(EdgeWeight));
geometry_stream.write(reinterpret_cast<char *>(&(m_geometry_rev_weight_list[0])), geometry_stream.write(reinterpret_cast<char *>(&(geometry_rev_weight_list[0])),
number_of_compressed_geometries * sizeof(EdgeWeight)); number_of_compressed_geometries * sizeof(EdgeWeight));
}; };
@ -845,12 +694,12 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
" for writing"}; " for writing"};
throw util::exception(message + SOURCE_REF); throw util::exception(message + SOURCE_REF);
} }
std::uint64_t number_of_datasource_entries = m_geometry_datasource.size(); std::uint64_t number_of_datasource_entries = geometry_datasource.size();
datasource_stream.write(reinterpret_cast<const char *>(&number_of_datasource_entries), datasource_stream.write(reinterpret_cast<const char *>(&number_of_datasource_entries),
sizeof(number_of_datasource_entries)); sizeof(number_of_datasource_entries));
if (number_of_datasource_entries > 0) if (number_of_datasource_entries > 0)
{ {
datasource_stream.write(reinterpret_cast<char *>(&(m_geometry_datasource[0])), datasource_stream.write(reinterpret_cast<char *>(&(geometry_datasource[0])),
number_of_datasource_entries * sizeof(uint8_t)); number_of_datasource_entries * sizeof(uint8_t));
} }
}; };
@ -920,16 +769,13 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
const auto num_segments = header->num_osm_nodes - 1; const auto num_segments = header->num_osm_nodes - 1;
for (auto i : util::irange<std::size_t>(0, num_segments)) for (auto i : util::irange<std::size_t>(0, num_segments))
{ {
auto speed_iter = if (auto value = segment_speed_lookup(
find(segment_speed_lookup, {previous_osm_node_id, segmentblocks[i].this_osm_node_id}))
SegmentSpeedSource{
{previous_osm_node_id, segmentblocks[i].this_osm_node_id}, {0, 0}});
if (speed_iter != segment_speed_lookup.end())
{ {
if (speed_iter->speed_source.speed > 0) if (value->speed > 0)
{ {
const auto new_segment_weight = distanceAndSpeedToWeight( const auto new_segment_weight =
segmentblocks[i].segment_length, speed_iter->speed_source.speed); ConvertToDuration(segmentblocks[i].segment_length, value->speed);
new_weight += new_segment_weight; new_weight += new_segment_weight;
} }
else else
@ -964,19 +810,16 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
continue; continue;
} }
auto turn_iter = find( if (auto value = turn_penalty_lookup(
turn_penalty_lookup, {penaltyblock->from_id, penaltyblock->via_id, penaltyblock->to_id}))
TurnPenaltySource{
{penaltyblock->from_id, penaltyblock->via_id, penaltyblock->to_id}, {0, 0}});
if (turn_iter != turn_penalty_lookup.end())
{ {
int new_turn_weight = static_cast<int>(turn_iter->penalty_source.penalty * 10); int new_turn_weight = static_cast<int>(value->penalty * 10);
if (new_turn_weight + new_weight < compressed_edge_nodes) if (new_turn_weight + new_weight < compressed_edge_nodes)
{ {
util::Log(logWARNING) << "turn penalty " << turn_iter->penalty_source.penalty util::Log(logWARNING) << "turn penalty " << value->penalty << " for turn "
<< " for turn " << penaltyblock->from_id << ", " << penaltyblock->from_id << ", " << penaltyblock->via_id
<< penaltyblock->via_id << ", " << penaltyblock->to_id << ", " << penaltyblock->to_id
<< " is too negative: clamping turn weight to " << " is too negative: clamping turn weight to "
<< compressed_edge_nodes; << compressed_edge_nodes;
} }