Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Switch code to use packed vector for segment weights/durations

Browse Source
This commit is contained in:
Patrick Niklaus 2017-05-10 23:04:09 +00:00 committed by Patrick Niklaus
parent 5c61f00ffa
commit 3599d1db8e
14 changed files with 242 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
features/testbot/ferry.feature
View File

@ -12,6 +12,7 @@ Feature: Testbot - Handle ferry routes
i j k l i j k l
m n o p m n o p
q r s t q r s t
u v w x
""" """
And the ways And the ways
@ -26,11 +27,14 @@ Feature: Testbot - Handle ferry routes
| op | primary | | | | op | primary | | |
| qr | primary | | | | qr | primary | | |
| st | primary | | | | st | primary | | |
| uv | primary | | |
| wx | primary | | |
| bc | | ferry | 0:01 | | bc | | ferry | 0:01 |
| fg | | ferry | 0:10 | | fg | | ferry | 0:10 |
| jk | | ferry | 1:00 | | jk | | ferry | 1:00 |
| no | | ferry | 24:00 | | no | | ferry | 24:00 |
| rs | | ferry | 96:00 | | rs | | ferry | 96:00 |
| vw | | ferry | P5D |
When I route I should get When I route I should get
| from | to | route | time | | from | to | route | time |
@ -39,6 +43,7 @@ Feature: Testbot - Handle ferry routes
| j | k | jk,jk | 3600s +-1 | | j | k | jk,jk | 3600s +-1 |
| n | o | no,no | 86400s +-1 | | n | o | no,no | 86400s +-1 |
| r | s | rs,rs | 345600s +-1 | | r | s | rs,rs | 345600s +-1 |
| v | w | vw,vw | 419430s +-1|
@todo @todo
Scenario: Testbot - Week long ferry routes Scenario: Testbot - Week long ferry routes

26
features/testbot/traffic_speeds.feature
View File

@ -2,6 +2,13 @@
Feature: Traffic - speeds Feature: Traffic - speeds
Background: Use specific speeds Background: Use specific speeds
# __ a
# / /
#c----b / g
# \ |\/
# \ e/\_.f
# \ | /
# d./
Given the node locations Given the node locations
| node | lat | lon | | node | lat | lon |
| a | 0.1 | 0.1 | | a | 0.1 | 0.1 |
@ -82,7 +89,7 @@ Feature: Traffic - speeds
api_version = 1 api_version = 1
properties.traffic_signal_penalty = 0 properties.traffic_signal_penalty = 0
properties.u_turn_penalty = 0 properties.u_turn_penalty = 0
properties.weight_precision = 3 properties.weight_precision = 2
""" """
And the contract extra arguments "--segment-speed-file {speeds_file}" And the contract extra arguments "--segment-speed-file {speeds_file}"
And the customize extra arguments "--segment-speed-file {speeds_file}" And the customize extra arguments "--segment-speed-file {speeds_file}"
@ -100,14 +107,14 @@ Feature: Traffic - speeds
When I route I should get When I route I should get
| from | to | route | speed | weights | a:datasources | | from | to | route | speed | weights | a:datasources |
| a | b | ab,ab | 1 km/h | 20020.735,0 | 1:0 | | a | b | ab,ab | 1 km/h | 20020.73,0 | 1:0 |
| a | c | ab,bc,bc | 2 km/h | 20020.735,741.509,0 | 1:1:0 | | a | c | ab,bc,bc | 2 km/h | 20020.73,741.51,0 | 1:1:0 |
| b | c | bc,bc | 27 km/h | 741.509,0 | 1:0 | | b | c | bc,bc | 27 km/h | 741.51,0 | 1:0 |
| a | d | ab,eb,de,de | 2 km/h | 20020.735,378.169,400.415,0 | 1:0:0 | | a | d | ab,eb,de,de | 2 km/h | 20020.73,378.17,400.41,0 | 1:0:0 |
| d | c | dc,dc | 36 km/h | 956.805,0 | 0 | | d | c | dc,dc | 36 km/h | 956.8,0 | 0 |
| g | b | ab,ab | 1 km/h | 10010.365,0 | 1:0 | | g | b | ab,ab | 1 km/h | 10010.37,0 | 1:0 |
| a | g | ab,ab | 1 km/h | 10010.37,0 | 1 | | a | g | ab,ab | 1 km/h | 10010.36,0 | 1 |
| g | a | ab,ab | 1 km/h | 10010.37,0 | 1:1 | | g | a | ab,ab | 1 km/h | 10010.36,0 | 1:1 |
Scenario: Speeds that isolate a single node (a) Scenario: Speeds that isolate a single node (a)
@ -161,3 +168,4 @@ Feature: Traffic - speeds
And the data has been extracted And the data has been extracted
When I try to run "osrm-contract --segment-speed-file {speeds_file} {processed_file}" When I try to run "osrm-contract --segment-speed-file {speeds_file} {processed_file}"
And it should exit successfully And it should exit successfully

30
features/testbot/traffic_speeds_overflow.feature Normal file
View File

@ -0,0 +1,30 @@
@routing @speed @traffic
Feature: Traffic - speeds edge cases
Scenario: Weighting based on speed file weights that cause segment weight overflows
Given the node map
"""
a-----b
"""
And the ways
| nodes | highway |
| ab | primary |
And the profile file "testbot" extended with
"""
api_version = 1
properties.traffic_signal_penalty = 0
properties.u_turn_penalty = 0
properties.weight_precision = 2
"""
And the contract extra arguments "--segment-speed-file {speeds_file}"
And the customize extra arguments "--segment-speed-file {speeds_file}"
And the speed file
"""
1,2,1,0.001
2,1,1,0.001
"""
And the query options
| annotations | datasources |
When I route I should get
| from | to | route | speed | weights | a:datasources |
| a | b | ab,ab | 1 km/h | 41943.02,0 | 1 |

42
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
View File

@ -434,35 +434,47 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
util::vector_view<unsigned> geometry_begin_indices( util::vector_view<unsigned> geometry_begin_indices(
geometries_index_ptr, data_layout.num_entries[storage::DataLayout::GEOMETRIES_INDEX]); geometries_index_ptr, data_layout.num_entries[storage::DataLayout::GEOMETRIES_INDEX]);
auto num_entries = data_layout.num_entries[storage::DataLayout::GEOMETRIES_NODE_LIST];
auto geometries_node_list_ptr = data_layout.GetBlockPtr<NodeID>( auto geometries_node_list_ptr = data_layout.GetBlockPtr<NodeID>(
memory_block, storage::DataLayout::GEOMETRIES_NODE_LIST); memory_block, storage::DataLayout::GEOMETRIES_NODE_LIST);
util::vector_view<NodeID> geometry_node_list( util::vector_view<NodeID> geometry_node_list(geometries_node_list_ptr, num_entries);
geometries_node_list_ptr,
data_layout.num_entries[storage::DataLayout::GEOMETRIES_NODE_LIST]);
auto geometries_fwd_weight_list_ptr = data_layout.GetBlockPtr<SegmentWeight>( auto geometries_fwd_weight_list_ptr =
data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type>(
memory_block, storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST); memory_block, storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST);
util::vector_view<SegmentWeight> geometry_fwd_weight_list( extractor::SegmentDataView::SegmentWeightVector geometry_fwd_weight_list(
util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
geometries_fwd_weight_list_ptr, geometries_fwd_weight_list_ptr,
data_layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST]); data_layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST]),
num_entries);
auto geometries_rev_weight_list_ptr = data_layout.GetBlockPtr<SegmentWeight>( auto geometries_rev_weight_list_ptr =
data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type>(
memory_block, storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST); memory_block, storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST);
util::vector_view<SegmentWeight> geometry_rev_weight_list( extractor::SegmentDataView::SegmentWeightVector geometry_rev_weight_list(
util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
geometries_rev_weight_list_ptr, geometries_rev_weight_list_ptr,
data_layout.num_entries[storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST]); data_layout.num_entries[storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST]),
num_entries);
auto geometries_fwd_duration_list_ptr = data_layout.GetBlockPtr<SegmentDuration>( auto geometries_fwd_duration_list_ptr =
data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type>(
memory_block, storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST); memory_block, storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST);
util::vector_view<SegmentDuration> geometry_fwd_duration_list( extractor::SegmentDataView::SegmentDurationVector geometry_fwd_duration_list(
util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
geometries_fwd_duration_list_ptr, geometries_fwd_duration_list_ptr,
data_layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST]); data_layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST]),
num_entries);
auto geometries_rev_duration_list_ptr = data_layout.GetBlockPtr<SegmentDuration>( auto geometries_rev_duration_list_ptr =
data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type>(
memory_block, storage::DataLayout::GEOMETRIES_REV_DURATION_LIST); memory_block, storage::DataLayout::GEOMETRIES_REV_DURATION_LIST);
util::vector_view<SegmentDuration> geometry_rev_duration_list( extractor::SegmentDataView::SegmentDurationVector geometry_rev_duration_list(
util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
geometries_rev_duration_list_ptr, geometries_rev_duration_list_ptr,
data_layout.num_entries[storage::DataLayout::GEOMETRIES_REV_DURATION_LIST]); data_layout.num_entries[storage::DataLayout::GEOMETRIES_REV_DURATION_LIST]),
num_entries);
auto datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>( auto datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>(
memory_block, storage::DataLayout::DATASOURCES_LIST); memory_block, storage::DataLayout::DATASOURCES_LIST);

8
include/engine/geospatial_query.hpp
View File

@ -425,7 +425,7 @@ template <typename RTreeT, typename DataFacadeT> class GeospatialQuery
// check phantom node segments validity // check phantom node segments validity
auto areSegmentsValid = [](auto first, auto last) -> bool { auto areSegmentsValid = [](auto first, auto last) -> bool {
return std::find(first, last, INVALID_EDGE_WEIGHT) == last; return std::find(first, last, INVALID_SEGMENT_WEIGHT) == last;
}; };
bool is_forward_valid_source = bool is_forward_valid_source =
areSegmentsValid(forward_weight_vector.begin(), forward_weight_vector.end()); areSegmentsValid(forward_weight_vector.begin(), forward_weight_vector.end());
@ -503,7 +503,7 @@ template <typename RTreeT, typename DataFacadeT> class GeospatialQuery
/** /**
* Checks to see if the edge weights are valid. We might have an edge, * Checks to see if the edge weights are valid. We might have an edge,
* but a traffic update might set the speed to 0 (weight == INVALID_EDGE_WEIGHT). * but a traffic update might set the speed to 0 (weight == INVALID_SEGMENT_WEIGHT).
* which means that this edge is not currently traversible. If this is the case, * which means that this edge is not currently traversible. If this is the case,
* then we shouldn't snap to this edge. * then we shouldn't snap to this edge.
*/ */
@ -521,7 +521,7 @@ template <typename RTreeT, typename DataFacadeT> class GeospatialQuery
const std::vector<EdgeWeight> forward_weight_vector = const std::vector<EdgeWeight> forward_weight_vector =
datafacade.GetUncompressedForwardWeights(geometry_id); datafacade.GetUncompressedForwardWeights(geometry_id);
if (forward_weight_vector[data.fwd_segment_position] != INVALID_EDGE_WEIGHT) if (forward_weight_vector[data.fwd_segment_position] != INVALID_SEGMENT_WEIGHT)
{ {
forward_edge_valid = data.forward_segment_id.enabled; forward_edge_valid = data.forward_segment_id.enabled;
} }
@ -529,7 +529,7 @@ template <typename RTreeT, typename DataFacadeT> class GeospatialQuery
const std::vector<EdgeWeight> reverse_weight_vector = const std::vector<EdgeWeight> reverse_weight_vector =
datafacade.GetUncompressedReverseWeights(geometry_id); datafacade.GetUncompressedReverseWeights(geometry_id);
if (reverse_weight_vector[reverse_weight_vector.size() - data.fwd_segment_position - 1] != if (reverse_weight_vector[reverse_weight_vector.size() - data.fwd_segment_position - 1] !=
INVALID_EDGE_WEIGHT) INVALID_SEGMENT_WEIGHT)
{ {
reverse_edge_valid = data.reverse_segment_id.enabled; reverse_edge_valid = data.reverse_segment_id.enabled;
} }

7
include/extractor/compressed_edge_container.hpp
View File

@ -63,9 +63,12 @@ class CompressedEdgeContainer
std::unique_ptr<SegmentDataContainer> ToSegmentData(); std::unique_ptr<SegmentDataContainer> ToSegmentData();
private: private:
SegmentWeight ClipWeight(const SegmentWeight weight);
SegmentDuration ClipDuration(const SegmentDuration duration);
int free_list_maximum = 0; int free_list_maximum = 0;
std::atomic_size_t clipped_weights; std::atomic_size_t clipped_weights{0};
std::atomic_size_t clipped_durations; std::atomic_size_t clipped_durations{0};
void IncreaseFreeList(); void IncreaseFreeList();
std::vector<OnewayEdgeBucket> m_compressed_oneway_geometries; std::vector<OnewayEdgeBucket> m_compressed_oneway_geometries;

21
include/extractor/segment_data_container.hpp
View File

@ -1,6 +1,7 @@
#ifndef OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_ #ifndef OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_
#define OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_ #define OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_
#include "util/packed_vector.hpp"
#include "util/typedefs.hpp" #include "util/typedefs.hpp"
#include "util/vector_view.hpp" #include "util/vector_view.hpp"
@ -43,6 +44,8 @@ namespace detail
template <storage::Ownership Ownership> class SegmentDataContainerImpl template <storage::Ownership Ownership> class SegmentDataContainerImpl
{ {
template <typename T> using Vector = util::ViewOrVector<T, Ownership>; template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
template <typename T, std::size_t Bits>
using PackedVector = util::detail::PackedVector<T, Bits, Ownership>;
friend CompressedEdgeContainer; friend CompressedEdgeContainer;
@ -50,15 +53,17 @@ template <storage::Ownership Ownership> class SegmentDataContainerImpl
// FIXME We should change the indexing to Edge-Based-Node id // FIXME We should change the indexing to Edge-Based-Node id
using DirectionalGeometryID = std::uint32_t; using DirectionalGeometryID = std::uint32_t;
using SegmentOffset = std::uint32_t; using SegmentOffset = std::uint32_t;
using SegmentWeightVector = PackedVector<SegmentWeight, SEGMENT_WEIGHT_BITS>;
using SegmentDurationVector = PackedVector<SegmentDuration, SEGMENT_DURAITON_BITS>;
SegmentDataContainerImpl() = default; SegmentDataContainerImpl() = default;
SegmentDataContainerImpl(Vector<std::uint32_t> index_, SegmentDataContainerImpl(Vector<std::uint32_t> index_,
Vector<NodeID> nodes_, Vector<NodeID> nodes_,
Vector<SegmentWeight> fwd_weights_, SegmentWeightVector fwd_weights_,
Vector<SegmentWeight> rev_weights_, SegmentWeightVector rev_weights_,
Vector<SegmentDuration> fwd_durations_, SegmentDurationVector fwd_durations_,
Vector<SegmentDuration> rev_durations_, SegmentDurationVector rev_durations_,
Vector<DatasourceID> datasources_) Vector<DatasourceID> datasources_)
: index(std::move(index_)), nodes(std::move(nodes_)), fwd_weights(std::move(fwd_weights_)), : index(std::move(index_)), nodes(std::move(nodes_)), fwd_weights(std::move(fwd_weights_)),
rev_weights(std::move(rev_weights_)), fwd_durations(std::move(fwd_durations_)), rev_weights(std::move(rev_weights_)), fwd_durations(std::move(fwd_durations_)),
@ -201,10 +206,10 @@ template <storage::Ownership Ownership> class SegmentDataContainerImpl
private: private:
Vector<std::uint32_t> index; Vector<std::uint32_t> index;
Vector<NodeID> nodes; Vector<NodeID> nodes;
Vector<SegmentWeight> fwd_weights; SegmentWeightVector fwd_weights;
Vector<SegmentWeight> rev_weights; SegmentWeightVector rev_weights;
Vector<SegmentDuration> fwd_durations; SegmentDurationVector fwd_durations;
Vector<SegmentDuration> rev_durations; SegmentDurationVector rev_durations;
Vector<DatasourceID> datasources; Vector<DatasourceID> datasources;
}; };
} }

16
include/extractor/serialization.hpp
View File

@ -38,10 +38,10 @@ inline void read(storage::io::FileReader &reader,
{ {
storage::serialization::read(reader, segment_data.index); storage::serialization::read(reader, segment_data.index);
storage::serialization::read(reader, segment_data.nodes); storage::serialization::read(reader, segment_data.nodes);
storage::serialization::read(reader, segment_data.fwd_weights); util::serialization::read(reader, segment_data.fwd_weights);
storage::serialization::read(reader, segment_data.rev_weights); util::serialization::read(reader, segment_data.rev_weights);
storage::serialization::read(reader, segment_data.fwd_durations); util::serialization::read(reader, segment_data.fwd_durations);
storage::serialization::read(reader, segment_data.rev_durations); util::serialization::read(reader, segment_data.rev_durations);
storage::serialization::read(reader, segment_data.datasources); storage::serialization::read(reader, segment_data.datasources);
} }
@ -51,10 +51,10 @@ inline void write(storage::io::FileWriter &writer,
{ {
storage::serialization::write(writer, segment_data.index); storage::serialization::write(writer, segment_data.index);
storage::serialization::write(writer, segment_data.nodes); storage::serialization::write(writer, segment_data.nodes);
storage::serialization::write(writer, segment_data.fwd_weights); util::serialization::write(writer, segment_data.fwd_weights);
storage::serialization::write(writer, segment_data.rev_weights); util::serialization::write(writer, segment_data.rev_weights);
storage::serialization::write(writer, segment_data.fwd_durations); util::serialization::write(writer, segment_data.fwd_durations);
storage::serialization::write(writer, segment_data.rev_durations); util::serialization::write(writer, segment_data.rev_durations);
storage::serialization::write(writer, segment_data.datasources); storage::serialization::write(writer, segment_data.datasources);
} }

2
include/util/packed_vector.hpp
View File

@ -411,6 +411,8 @@ template <typename T, std::size_t Bits, storage::Ownership Ownership> class Pack
void push_back(const T value) void push_back(const T value)
{ {
BOOST_ASSERT_MSG(value <= T{(1ULL << Bits) - 1}, "Value too big for packed storage.");
auto internal_index = get_internal_index(num_elements); auto internal_index = get_internal_index(num_elements);
while (internal_index.lower_word + 1 >= vec.size()) while (internal_index.lower_word + 1 >= vec.size())

8
include/util/typedefs.hpp
View File

@ -96,8 +96,12 @@ static const EdgeID SPECIAL_EDGEID = std::numeric_limits<EdgeID>::max();
static const NameID INVALID_NAMEID = std::numeric_limits<NameID>::max(); static const NameID INVALID_NAMEID = std::numeric_limits<NameID>::max();
static const NameID EMPTY_NAMEID = 0; static const NameID EMPTY_NAMEID = 0;
static const unsigned INVALID_COMPONENTID = 0; static const unsigned INVALID_COMPONENTID = 0;
static const SegmentWeight INVALID_SEGMENT_WEIGHT = (1u << 20) - 1; static const std::size_t SEGMENT_WEIGHT_BITS = 22;
static const SegmentDuration INVALID_SEGMENT_DURATION = (1u << 20) - 1; static const std::size_t SEGMENT_DURAITON_BITS = 22;
static const SegmentWeight INVALID_SEGMENT_WEIGHT = (1u << SEGMENT_WEIGHT_BITS) - 1;
static const SegmentDuration INVALID_SEGMENT_DURATION = (1u << SEGMENT_DURAITON_BITS) - 1;
static const SegmentWeight MAX_SEGMENT_WEIGHT = INVALID_SEGMENT_WEIGHT - 1;
static const SegmentDuration MAX_SEGMENT_DURATION = INVALID_SEGMENT_DURATION - 1;
static const EdgeWeight INVALID_EDGE_WEIGHT = std::numeric_limits<EdgeWeight>::max(); static const EdgeWeight INVALID_EDGE_WEIGHT = std::numeric_limits<EdgeWeight>::max();
static const EdgeDuration MAXIMAL_EDGE_DURATION = std::numeric_limits<EdgeDuration>::max(); static const EdgeDuration MAXIMAL_EDGE_DURATION = std::numeric_limits<EdgeDuration>::max();
static const TurnPenalty INVALID_TURN_PENALTY = std::numeric_limits<TurnPenalty>::max(); static const TurnPenalty INVALID_TURN_PENALTY = std::numeric_limits<TurnPenalty>::max();

70
src/extractor/compressed_edge_container.cpp
View File

@ -74,6 +74,26 @@ unsigned CompressedEdgeContainer::GetZippedPositionForReverseID(const EdgeID edg
return map_iterator->second; return map_iterator->second;
} }
SegmentWeight CompressedEdgeContainer::ClipWeight(const SegmentWeight weight)
{
if (weight >= INVALID_SEGMENT_WEIGHT)
{
clipped_weights++;
return MAX_SEGMENT_WEIGHT;
}
return weight;
}
SegmentDuration CompressedEdgeContainer::ClipDuration(const SegmentDuration duration)
{
if (duration >= INVALID_SEGMENT_DURATION)
{
clipped_weights++;
return MAX_SEGMENT_DURATION;
}
return duration;
}
// Adds info for a compressed edge to the container. edge_id_2 // Adds info for a compressed edge to the container. edge_id_2
// has been removed from the graph, so we have to save These edges/nodes // has been removed from the graph, so we have to save These edges/nodes
// have already been trimmed from the graph, this function just stores // have already been trimmed from the graph, this function just stores
@ -87,10 +107,10 @@ void CompressedEdgeContainer::CompressEdge(const EdgeID edge_id_1,
const EdgeID edge_id_2, const EdgeID edge_id_2,
const NodeID via_node_id, const NodeID via_node_id,
const NodeID target_node_id, const NodeID target_node_id,
EdgeWeight weight1, const EdgeWeight weight1,
EdgeWeight weight2, const EdgeWeight weight2,
EdgeDuration duration1, const EdgeDuration duration1,
EdgeDuration duration2) const EdgeDuration duration2)
{ {
// remove super-trivial geometries // remove super-trivial geometries
BOOST_ASSERT(SPECIAL_EDGEID != edge_id_1); BOOST_ASSERT(SPECIAL_EDGEID != edge_id_1);
@ -131,34 +151,12 @@ void CompressedEdgeContainer::CompressEdge(const EdgeID edge_id_1,
std::vector<OnewayCompressedEdge> &edge_bucket_list1 = std::vector<OnewayCompressedEdge> &edge_bucket_list1 =
m_compressed_oneway_geometries[edge_bucket_id1]; m_compressed_oneway_geometries[edge_bucket_id1];
const auto clip_weight = [&](const SegmentWeight weight) {
if (weight >= INVALID_SEGMENT_WEIGHT)
{
clipped_weights++;
return INVALID_SEGMENT_WEIGHT - 1;
}
return weight;
};
const auto clip_duration = [&](const SegmentDuration duration) {
if (duration >= INVALID_SEGMENT_DURATION)
{
clipped_weights++;
return INVALID_SEGMENT_DURATION - 1;
}
return duration;
};
// note we don't save the start coordinate: it is implicitly given by edge 1 // note we don't save the start coordinate: it is implicitly given by edge 1
// weight1 is the distance to the (currently) last coordinate in the bucket // weight1 is the distance to the (currently) last coordinate in the bucket
if (edge_bucket_list1.empty()) if (edge_bucket_list1.empty())
{ {
weight1 = clip_weight(weight1);
duration1 = clip_duration(duration1);
edge_bucket_list1.emplace_back( edge_bucket_list1.emplace_back(
OnewayCompressedEdge{via_node_id, OnewayCompressedEdge{via_node_id, ClipWeight(weight1), ClipDuration(duration1)});
static_cast<SegmentWeight>(weight1),
static_cast<SegmentDuration>(duration1)});
} }
BOOST_ASSERT(0 < edge_bucket_list1.size()); BOOST_ASSERT(0 < edge_bucket_list1.size());
@ -188,14 +186,9 @@ void CompressedEdgeContainer::CompressEdge(const EdgeID edge_id_1,
} }
else else
{ {
weight2 = clip_weight(weight2);
duration2 = clip_duration(duration2);
// we are certain that the second edge is atomic. // we are certain that the second edge is atomic.
edge_bucket_list1.emplace_back( edge_bucket_list1.emplace_back(
OnewayCompressedEdge{target_node_id, OnewayCompressedEdge{target_node_id, ClipWeight(weight2), ClipDuration(duration2)});
static_cast<SegmentWeight>(weight2),
static_cast<SegmentDuration>(duration2)});
} }
} }
@ -238,7 +231,8 @@ void CompressedEdgeContainer::AddUncompressedEdge(const EdgeID edge_id,
// Don't re-add this if it's already in there. // Don't re-add this if it's already in there.
if (edge_bucket_list.empty()) if (edge_bucket_list.empty())
{ {
edge_bucket_list.emplace_back(OnewayCompressedEdge{target_node_id, weight, duration}); edge_bucket_list.emplace_back(
OnewayCompressedEdge{target_node_id, ClipWeight(weight), ClipDuration(duration)});
} }
} }
@ -272,9 +266,9 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID
constexpr DatasourceID LUA_SOURCE = 0; constexpr DatasourceID LUA_SOURCE = 0;
segment_data->nodes.emplace_back(first_node.node_id); segment_data->nodes.emplace_back(first_node.node_id);
segment_data->fwd_weights.emplace_back(INVALID_EDGE_WEIGHT); segment_data->fwd_weights.emplace_back(INVALID_SEGMENT_WEIGHT);
segment_data->rev_weights.emplace_back(first_node.weight); segment_data->rev_weights.emplace_back(first_node.weight);
segment_data->fwd_durations.emplace_back(INVALID_EDGE_WEIGHT); segment_data->fwd_durations.emplace_back(INVALID_SEGMENT_DURATION);
segment_data->rev_durations.emplace_back(first_node.duration); segment_data->rev_durations.emplace_back(first_node.duration);
segment_data->datasources.emplace_back(LUA_SOURCE); segment_data->datasources.emplace_back(LUA_SOURCE);
@ -297,9 +291,9 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID
segment_data->nodes.emplace_back(last_node.node_id); segment_data->nodes.emplace_back(last_node.node_id);
segment_data->fwd_weights.emplace_back(last_node.weight); segment_data->fwd_weights.emplace_back(last_node.weight);
segment_data->rev_weights.emplace_back(INVALID_EDGE_WEIGHT); segment_data->rev_weights.emplace_back(INVALID_SEGMENT_WEIGHT);
segment_data->fwd_durations.emplace_back(last_node.duration); segment_data->fwd_durations.emplace_back(last_node.duration);
segment_data->rev_durations.emplace_back(INVALID_EDGE_WEIGHT); segment_data->rev_durations.emplace_back(INVALID_SEGMENT_DURATION);
segment_data->datasources.emplace_back(LUA_SOURCE); segment_data->datasources.emplace_back(LUA_SOURCE);
return zipped_geometry_id; return zipped_geometry_id;

2
src/extractor/extractor.cpp
View File

@ -487,6 +487,8 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment,
config.turn_penalties_index_path, config.turn_penalties_index_path,
config.cnbg_ebg_graph_mapping_output_path); config.cnbg_ebg_graph_mapping_output_path);
compressed_edge_container.PrintStatistics();
// The osrm-partition tool requires the compressed node based graph with an embedding. // The osrm-partition tool requires the compressed node based graph with an embedding.
// //
// The `Run` function above re-numbers non-reverse compressed node based graph edges // The `Run` function above re-numbers non-reverse compressed node based graph edges

76
src/storage/storage.cpp
View File

@ -351,14 +351,29 @@ void Storage::PopulateLayout(DataLayout &layout)
const auto number_of_compressed_geometries = reader.ReadVectorSize<NodeID>(); const auto number_of_compressed_geometries = reader.ReadVectorSize<NodeID>();
layout.SetBlockSize<NodeID>(DataLayout::GEOMETRIES_NODE_LIST, layout.SetBlockSize<NodeID>(DataLayout::GEOMETRIES_NODE_LIST,
number_of_compressed_geometries); number_of_compressed_geometries);
layout.SetBlockSize<SegmentWeight>(DataLayout::GEOMETRIES_FWD_WEIGHT_LIST,
number_of_compressed_geometries); reader.ReadElementCount64(); // number of segments
layout.SetBlockSize<SegmentWeight>(DataLayout::GEOMETRIES_REV_WEIGHT_LIST, const auto number_of_segment_weight_blocks =
number_of_compressed_geometries); reader.ReadVectorSize<extractor::SegmentDataView::SegmentWeightVector::block_type>();
layout.SetBlockSize<SegmentDuration>(DataLayout::GEOMETRIES_FWD_DURATION_LIST,
number_of_compressed_geometries); reader.ReadElementCount64(); // number of segments
layout.SetBlockSize<SegmentDuration>(DataLayout::GEOMETRIES_REV_DURATION_LIST, auto number_of_rev_weight_blocks =
number_of_compressed_geometries); reader.ReadVectorSize<extractor::SegmentDataView::SegmentWeightVector::block_type>();
BOOST_ASSERT(number_of_rev_weight_blocks == number_of_segment_weight_blocks);
(void)number_of_rev_weight_blocks;
reader.ReadElementCount64(); // number of segments
const auto number_of_segment_duration_blocks =
reader.ReadVectorSize<extractor::SegmentDataView::SegmentDurationVector::block_type>();
layout.SetBlockSize<extractor::SegmentDataView::SegmentWeightVector::block_type>(
DataLayout::GEOMETRIES_FWD_WEIGHT_LIST, number_of_segment_weight_blocks);
layout.SetBlockSize<extractor::SegmentDataView::SegmentWeightVector::block_type>(
DataLayout::GEOMETRIES_REV_WEIGHT_LIST, number_of_segment_weight_blocks);
layout.SetBlockSize<extractor::SegmentDataView::SegmentDurationVector::block_type>(
DataLayout::GEOMETRIES_FWD_DURATION_LIST, number_of_segment_duration_blocks);
layout.SetBlockSize<extractor::SegmentDataView::SegmentDurationVector::block_type>(
DataLayout::GEOMETRIES_REV_DURATION_LIST, number_of_segment_duration_blocks);
layout.SetBlockSize<DatasourceID>(DataLayout::DATASOURCES_LIST, layout.SetBlockSize<DatasourceID>(DataLayout::DATASOURCES_LIST,
number_of_compressed_geometries); number_of_compressed_geometries);
} }
@ -658,35 +673,47 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
util::vector_view<unsigned> geometry_begin_indices( util::vector_view<unsigned> geometry_begin_indices(
geometries_index_ptr, layout.num_entries[storage::DataLayout::GEOMETRIES_INDEX]); geometries_index_ptr, layout.num_entries[storage::DataLayout::GEOMETRIES_INDEX]);
auto num_entries = layout.num_entries[storage::DataLayout::GEOMETRIES_NODE_LIST];
auto geometries_node_list_ptr = auto geometries_node_list_ptr =
layout.GetBlockPtr<NodeID, true>(memory_ptr, storage::DataLayout::GEOMETRIES_NODE_LIST); layout.GetBlockPtr<NodeID, true>(memory_ptr, storage::DataLayout::GEOMETRIES_NODE_LIST);
util::vector_view<NodeID> geometry_node_list( util::vector_view<NodeID> geometry_node_list(geometries_node_list_ptr, num_entries);
geometries_node_list_ptr,
layout.num_entries[storage::DataLayout::GEOMETRIES_NODE_LIST]);
auto geometries_fwd_weight_list_ptr = layout.GetBlockPtr<SegmentWeight, true>( auto geometries_fwd_weight_list_ptr =
layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type, true>(
memory_ptr, storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST); memory_ptr, storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST);
util::vector_view<SegmentWeight> geometry_fwd_weight_list( extractor::SegmentDataView::SegmentWeightVector geometry_fwd_weight_list(
util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
geometries_fwd_weight_list_ptr, geometries_fwd_weight_list_ptr,
layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST]); layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST]),
num_entries);
auto geometries_rev_weight_list_ptr = layout.GetBlockPtr<SegmentWeight, true>( auto geometries_rev_weight_list_ptr =
layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type, true>(
memory_ptr, storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST); memory_ptr, storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST);
util::vector_view<SegmentWeight> geometry_rev_weight_list( extractor::SegmentDataView::SegmentWeightVector geometry_rev_weight_list(
util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
geometries_rev_weight_list_ptr, geometries_rev_weight_list_ptr,
layout.num_entries[storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST]); layout.num_entries[storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST]),
num_entries);
auto geometries_fwd_duration_list_ptr = layout.GetBlockPtr<SegmentDuration, true>( auto geometries_fwd_duration_list_ptr =
layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type, true>(
memory_ptr, storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST); memory_ptr, storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST);
util::vector_view<SegmentDuration> geometry_fwd_duration_list( extractor::SegmentDataView::SegmentDurationVector geometry_fwd_duration_list(
util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
geometries_fwd_duration_list_ptr, geometries_fwd_duration_list_ptr,
layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST]); layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST]),
num_entries);
auto geometries_rev_duration_list_ptr = layout.GetBlockPtr<SegmentDuration, true>( auto geometries_rev_duration_list_ptr =
layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type, true>(
memory_ptr, storage::DataLayout::GEOMETRIES_REV_DURATION_LIST); memory_ptr, storage::DataLayout::GEOMETRIES_REV_DURATION_LIST);
util::vector_view<SegmentDuration> geometry_rev_duration_list( extractor::SegmentDataView::SegmentDurationVector geometry_rev_duration_list(
util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
geometries_rev_duration_list_ptr, geometries_rev_duration_list_ptr,
layout.num_entries[storage::DataLayout::GEOMETRIES_REV_DURATION_LIST]); layout.num_entries[storage::DataLayout::GEOMETRIES_REV_DURATION_LIST]),
num_entries);
auto datasources_list_ptr = layout.GetBlockPtr<DatasourceID, true>( auto datasources_list_ptr = layout.GetBlockPtr<DatasourceID, true>(
memory_ptr, storage::DataLayout::DATASOURCES_LIST); memory_ptr, storage::DataLayout::DATASOURCES_LIST);
@ -722,8 +749,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
extractor::PackedOSMIDsView osm_node_ids( extractor::PackedOSMIDsView osm_node_ids(
util::vector_view<extractor::PackedOSMIDsView::block_type>( util::vector_view<extractor::PackedOSMIDsView::block_type>(
osmnodeid_ptr, layout.num_entries[DataLayout::OSM_NODE_ID_LIST]), osmnodeid_ptr, layout.num_entries[DataLayout::OSM_NODE_ID_LIST]),
layout.num_entries[DataLayout::OSM_NODE_ID_LIST] * layout.num_entries[DataLayout::COORDINATE_LIST]);
extractor::PackedOSMIDsView::BLOCK_ELEMENTS);
extractor::files::readNodes(config.node_based_nodes_data_path, coordinates, osm_node_ids); extractor::files::readNodes(config.node_based_nodes_data_path, coordinates, osm_node_ids);
} }

30
src/updater/updater.cpp
View File

@ -81,14 +81,22 @@ template <typename T> inline bool is_aligned(const void *pointer)
} }
// Returns duration in deci-seconds // Returns duration in deci-seconds
inline EdgeWeight convertToDuration(double speed_in_kmh, double distance_in_meters) inline SegmentDuration convertToDuration(double speed_in_kmh, double distance_in_meters)
{ {
if (speed_in_kmh <= 0.) if (speed_in_kmh <= 0.)
return MAXIMAL_EDGE_DURATION; return INVALID_SEGMENT_DURATION;
const auto speed_in_ms = speed_in_kmh / 3.6; const auto speed_in_ms = speed_in_kmh / 3.6;
const auto duration = distance_in_meters / speed_in_ms; const auto duration = distance_in_meters / speed_in_ms;
return std::max(1, boost::numeric_cast<EdgeWeight>(std::round(duration * 10.))); auto segment_duration = std::max<SegmentDuration>(
1, boost::numeric_cast<SegmentDuration>(std::round(duration * 10.)));
if (segment_duration >= INVALID_SEGMENT_DURATION)
{
util::Log(logWARNING) << "Clamping segment duration " << segment_duration << " to "
<< MAX_SEGMENT_DURATION;
segment_duration = MAX_SEGMENT_DURATION;
}
return segment_duration;
} }
#if !defined(NDEBUG) #if !defined(NDEBUG)
@ -182,11 +190,19 @@ updateSegmentData(const UpdaterConfig &config,
} }
if (rate <= 0.) if (rate <= 0.)
return INVALID_EDGE_WEIGHT; return INVALID_SEGMENT_WEIGHT;
const auto weight_multiplier = profile_properties.GetWeightMultiplier(); const auto weight_multiplier = profile_properties.GetWeightMultiplier();
const auto weight = distance_in_meters / rate; const auto weight = distance_in_meters / rate;
return std::max(1, boost::numeric_cast<EdgeWeight>(std::round(weight * weight_multiplier))); auto segment_weight = std::max<SegmentWeight>(
1, boost::numeric_cast<SegmentWeight>(std::round(weight * weight_multiplier)));
if (segment_weight >= INVALID_SEGMENT_WEIGHT)
{
util::Log(logWARNING) << "Clamping segment weight " << segment_weight << " to "
<< MAX_SEGMENT_WEIGHT;
segment_weight = MAX_SEGMENT_WEIGHT;
}
return segment_weight;
}; };
// The check here is enabled by the `--edge-weight-updates-over-factor` flag it logs a // The check here is enabled by the `--edge-weight-updates-over-factor` flag it logs a
@ -739,7 +755,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
const auto weights = segment_data.GetForwardWeights(geometry_id.id); const auto weights = segment_data.GetForwardWeights(geometry_id.id);
for (const auto weight : weights) for (const auto weight : weights)
{ {
if (weight == INVALID_EDGE_WEIGHT) if (weight == INVALID_SEGMENT_WEIGHT)
{ {
new_weight = INVALID_EDGE_WEIGHT; new_weight = INVALID_EDGE_WEIGHT;
break; break;
@ -754,7 +770,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
const auto weights = segment_data.GetReverseWeights(geometry_id.id); const auto weights = segment_data.GetReverseWeights(geometry_id.id);
for (const auto weight : weights) for (const auto weight : weights)
{ {
if (weight == INVALID_EDGE_WEIGHT) if (weight == INVALID_SEGMENT_WEIGHT)
{ {
new_weight = INVALID_EDGE_WEIGHT; new_weight = INVALID_EDGE_WEIGHT;
break; break;