From 3599d1db8e3e72dbbd8f7ee22bc440edf9c57239 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 10 May 2017 23:04:09 +0000 Subject: [PATCH] Switch code to use packed vector for segment weights/durations --- features/testbot/ferry.feature | 5 + features/testbot/traffic_speeds.feature | 28 ++++-- .../testbot/traffic_speeds_overflow.feature | 30 ++++++ .../contiguous_internalmem_datafacade.hpp | 58 +++++++----- include/engine/geospatial_query.hpp | 8 +- .../extractor/compressed_edge_container.hpp | 7 +- include/extractor/segment_data_container.hpp | 21 +++-- include/extractor/serialization.hpp | 16 ++-- include/util/packed_vector.hpp | 2 + include/util/typedefs.hpp | 8 +- src/extractor/compressed_edge_container.cpp | 70 +++++++------- src/extractor/extractor.cpp | 2 + src/storage/storage.cpp | 92 ++++++++++++------- src/updater/updater.cpp | 30 ++++-- 14 files changed, 242 insertions(+), 135 deletions(-) create mode 100644 features/testbot/traffic_speeds_overflow.feature diff --git a/features/testbot/ferry.feature b/features/testbot/ferry.feature index 42401c82d..6c5f67cc1 100644 --- a/features/testbot/ferry.feature +++ b/features/testbot/ferry.feature @@ -12,6 +12,7 @@ Feature: Testbot - Handle ferry routes i j k l m n o p q r s t + u v w x """ And the ways @@ -26,11 +27,14 @@ Feature: Testbot - Handle ferry routes | op | primary | | | | qr | primary | | | | st | primary | | | + | uv | primary | | | + | wx | primary | | | | bc | | ferry | 0:01 | | fg | | ferry | 0:10 | | jk | | ferry | 1:00 | | no | | ferry | 24:00 | | rs | | ferry | 96:00 | + | vw | | ferry | P5D | When I route I should get | from | to | route | time | @@ -39,6 +43,7 @@ Feature: Testbot - Handle ferry routes | j | k | jk,jk | 3600s +-1 | | n | o | no,no | 86400s +-1 | | r | s | rs,rs | 345600s +-1 | + | v | w | vw,vw | 419430s +-1| @todo Scenario: Testbot - Week long ferry routes diff --git a/features/testbot/traffic_speeds.feature b/features/testbot/traffic_speeds.feature index 32f850c0f..1dfbbc27e 100644 --- a/features/testbot/traffic_speeds.feature +++ b/features/testbot/traffic_speeds.feature @@ -2,6 +2,13 @@ Feature: Traffic - speeds Background: Use specific speeds + # __ a + # / / + #c----b / g + # \ |\/ + # \ e/\_.f + # \ | / + # d./ Given the node locations | node | lat | lon | | a | 0.1 | 0.1 | @@ -82,7 +89,7 @@ Feature: Traffic - speeds api_version = 1 properties.traffic_signal_penalty = 0 properties.u_turn_penalty = 0 - properties.weight_precision = 3 + properties.weight_precision = 2 """ And the contract extra arguments "--segment-speed-file {speeds_file}" And the customize extra arguments "--segment-speed-file {speeds_file}" @@ -99,15 +106,15 @@ Feature: Traffic - speeds | annotations | datasources | When I route I should get - | from | to | route | speed | weights | a:datasources | - | a | b | ab,ab | 1 km/h | 20020.735,0 | 1:0 | - | a | c | ab,bc,bc | 2 km/h | 20020.735,741.509,0 | 1:1:0 | - | b | c | bc,bc | 27 km/h | 741.509,0 | 1:0 | - | a | d | ab,eb,de,de | 2 km/h | 20020.735,378.169,400.415,0 | 1:0:0 | - | d | c | dc,dc | 36 km/h | 956.805,0 | 0 | - | g | b | ab,ab | 1 km/h | 10010.365,0 | 1:0 | - | a | g | ab,ab | 1 km/h | 10010.37,0 | 1 | - | g | a | ab,ab | 1 km/h | 10010.37,0 | 1:1 | + | from | to | route | speed | weights | a:datasources | + | a | b | ab,ab | 1 km/h | 20020.73,0 | 1:0 | + | a | c | ab,bc,bc | 2 km/h | 20020.73,741.51,0 | 1:1:0 | + | b | c | bc,bc | 27 km/h | 741.51,0 | 1:0 | + | a | d | ab,eb,de,de | 2 km/h | 20020.73,378.17,400.41,0 | 1:0:0 | + | d | c | dc,dc | 36 km/h | 956.8,0 | 0 | + | g | b | ab,ab | 1 km/h | 10010.37,0 | 1:0 | + | a | g | ab,ab | 1 km/h | 10010.36,0 | 1 | + | g | a | ab,ab | 1 km/h | 10010.36,0 | 1:1 | Scenario: Speeds that isolate a single node (a) @@ -161,3 +168,4 @@ Feature: Traffic - speeds And the data has been extracted When I try to run "osrm-contract --segment-speed-file {speeds_file} {processed_file}" And it should exit successfully + diff --git a/features/testbot/traffic_speeds_overflow.feature b/features/testbot/traffic_speeds_overflow.feature new file mode 100644 index 000000000..f747b85fb --- /dev/null +++ b/features/testbot/traffic_speeds_overflow.feature @@ -0,0 +1,30 @@ +@routing @speed @traffic +Feature: Traffic - speeds edge cases + Scenario: Weighting based on speed file weights that cause segment weight overflows + Given the node map + """ + a-----b + """ + And the ways + | nodes | highway | + | ab | primary | + And the profile file "testbot" extended with + """ + api_version = 1 + properties.traffic_signal_penalty = 0 + properties.u_turn_penalty = 0 + properties.weight_precision = 2 + """ + And the contract extra arguments "--segment-speed-file {speeds_file}" + And the customize extra arguments "--segment-speed-file {speeds_file}" + And the speed file + """ + 1,2,1,0.001 + 2,1,1,0.001 + """ + And the query options + | annotations | datasources | + + When I route I should get + | from | to | route | speed | weights | a:datasources | + | a | b | ab,ab | 1 km/h | 41943.02,0 | 1 | diff --git a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp index 81fe6c30c..68501d2c4 100644 --- a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp +++ b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp @@ -434,35 +434,47 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade util::vector_view geometry_begin_indices( geometries_index_ptr, data_layout.num_entries[storage::DataLayout::GEOMETRIES_INDEX]); + auto num_entries = data_layout.num_entries[storage::DataLayout::GEOMETRIES_NODE_LIST]; + auto geometries_node_list_ptr = data_layout.GetBlockPtr( memory_block, storage::DataLayout::GEOMETRIES_NODE_LIST); - util::vector_view geometry_node_list( - geometries_node_list_ptr, - data_layout.num_entries[storage::DataLayout::GEOMETRIES_NODE_LIST]); + util::vector_view geometry_node_list(geometries_node_list_ptr, num_entries); - auto geometries_fwd_weight_list_ptr = data_layout.GetBlockPtr( - memory_block, storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST); - util::vector_view geometry_fwd_weight_list( - geometries_fwd_weight_list_ptr, - data_layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST]); + auto geometries_fwd_weight_list_ptr = + data_layout.GetBlockPtr( + memory_block, storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST); + extractor::SegmentDataView::SegmentWeightVector geometry_fwd_weight_list( + util::vector_view( + geometries_fwd_weight_list_ptr, + data_layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST]), + num_entries); - auto geometries_rev_weight_list_ptr = data_layout.GetBlockPtr( - memory_block, storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST); - util::vector_view geometry_rev_weight_list( - geometries_rev_weight_list_ptr, - data_layout.num_entries[storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST]); + auto geometries_rev_weight_list_ptr = + data_layout.GetBlockPtr( + memory_block, storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST); + extractor::SegmentDataView::SegmentWeightVector geometry_rev_weight_list( + util::vector_view( + geometries_rev_weight_list_ptr, + data_layout.num_entries[storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST]), + num_entries); - auto geometries_fwd_duration_list_ptr = data_layout.GetBlockPtr( - memory_block, storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST); - util::vector_view geometry_fwd_duration_list( - geometries_fwd_duration_list_ptr, - data_layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST]); + auto geometries_fwd_duration_list_ptr = + data_layout.GetBlockPtr( + memory_block, storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST); + extractor::SegmentDataView::SegmentDurationVector geometry_fwd_duration_list( + util::vector_view( + geometries_fwd_duration_list_ptr, + data_layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST]), + num_entries); - auto geometries_rev_duration_list_ptr = data_layout.GetBlockPtr( - memory_block, storage::DataLayout::GEOMETRIES_REV_DURATION_LIST); - util::vector_view geometry_rev_duration_list( - geometries_rev_duration_list_ptr, - data_layout.num_entries[storage::DataLayout::GEOMETRIES_REV_DURATION_LIST]); + auto geometries_rev_duration_list_ptr = + data_layout.GetBlockPtr( + memory_block, storage::DataLayout::GEOMETRIES_REV_DURATION_LIST); + extractor::SegmentDataView::SegmentDurationVector geometry_rev_duration_list( + util::vector_view( + geometries_rev_duration_list_ptr, + data_layout.num_entries[storage::DataLayout::GEOMETRIES_REV_DURATION_LIST]), + num_entries); auto datasources_list_ptr = data_layout.GetBlockPtr( memory_block, storage::DataLayout::DATASOURCES_LIST); diff --git a/include/engine/geospatial_query.hpp b/include/engine/geospatial_query.hpp index 21c7014b2..ae984d969 100644 --- a/include/engine/geospatial_query.hpp +++ b/include/engine/geospatial_query.hpp @@ -425,7 +425,7 @@ template class GeospatialQuery // check phantom node segments validity auto areSegmentsValid = [](auto first, auto last) -> bool { - return std::find(first, last, INVALID_EDGE_WEIGHT) == last; + return std::find(first, last, INVALID_SEGMENT_WEIGHT) == last; }; bool is_forward_valid_source = areSegmentsValid(forward_weight_vector.begin(), forward_weight_vector.end()); @@ -503,7 +503,7 @@ template class GeospatialQuery /** * Checks to see if the edge weights are valid. We might have an edge, - * but a traffic update might set the speed to 0 (weight == INVALID_EDGE_WEIGHT). + * but a traffic update might set the speed to 0 (weight == INVALID_SEGMENT_WEIGHT). * which means that this edge is not currently traversible. If this is the case, * then we shouldn't snap to this edge. */ @@ -521,7 +521,7 @@ template class GeospatialQuery const std::vector forward_weight_vector = datafacade.GetUncompressedForwardWeights(geometry_id); - if (forward_weight_vector[data.fwd_segment_position] != INVALID_EDGE_WEIGHT) + if (forward_weight_vector[data.fwd_segment_position] != INVALID_SEGMENT_WEIGHT) { forward_edge_valid = data.forward_segment_id.enabled; } @@ -529,7 +529,7 @@ template class GeospatialQuery const std::vector reverse_weight_vector = datafacade.GetUncompressedReverseWeights(geometry_id); if (reverse_weight_vector[reverse_weight_vector.size() - data.fwd_segment_position - 1] != - INVALID_EDGE_WEIGHT) + INVALID_SEGMENT_WEIGHT) { reverse_edge_valid = data.reverse_segment_id.enabled; } diff --git a/include/extractor/compressed_edge_container.hpp b/include/extractor/compressed_edge_container.hpp index 90c0da583..ca00be864 100644 --- a/include/extractor/compressed_edge_container.hpp +++ b/include/extractor/compressed_edge_container.hpp @@ -63,9 +63,12 @@ class CompressedEdgeContainer std::unique_ptr ToSegmentData(); private: + SegmentWeight ClipWeight(const SegmentWeight weight); + SegmentDuration ClipDuration(const SegmentDuration duration); + int free_list_maximum = 0; - std::atomic_size_t clipped_weights; - std::atomic_size_t clipped_durations; + std::atomic_size_t clipped_weights{0}; + std::atomic_size_t clipped_durations{0}; void IncreaseFreeList(); std::vector m_compressed_oneway_geometries; diff --git a/include/extractor/segment_data_container.hpp b/include/extractor/segment_data_container.hpp index dd0f2120c..c93d97c96 100644 --- a/include/extractor/segment_data_container.hpp +++ b/include/extractor/segment_data_container.hpp @@ -1,6 +1,7 @@ #ifndef OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_ #define OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_ +#include "util/packed_vector.hpp" #include "util/typedefs.hpp" #include "util/vector_view.hpp" @@ -43,6 +44,8 @@ namespace detail template class SegmentDataContainerImpl { template using Vector = util::ViewOrVector; + template + using PackedVector = util::detail::PackedVector; friend CompressedEdgeContainer; @@ -50,15 +53,17 @@ template class SegmentDataContainerImpl // FIXME We should change the indexing to Edge-Based-Node id using DirectionalGeometryID = std::uint32_t; using SegmentOffset = std::uint32_t; + using SegmentWeightVector = PackedVector; + using SegmentDurationVector = PackedVector; SegmentDataContainerImpl() = default; SegmentDataContainerImpl(Vector index_, Vector nodes_, - Vector fwd_weights_, - Vector rev_weights_, - Vector fwd_durations_, - Vector rev_durations_, + SegmentWeightVector fwd_weights_, + SegmentWeightVector rev_weights_, + SegmentDurationVector fwd_durations_, + SegmentDurationVector rev_durations_, Vector datasources_) : index(std::move(index_)), nodes(std::move(nodes_)), fwd_weights(std::move(fwd_weights_)), rev_weights(std::move(rev_weights_)), fwd_durations(std::move(fwd_durations_)), @@ -201,10 +206,10 @@ template class SegmentDataContainerImpl private: Vector index; Vector nodes; - Vector fwd_weights; - Vector rev_weights; - Vector fwd_durations; - Vector rev_durations; + SegmentWeightVector fwd_weights; + SegmentWeightVector rev_weights; + SegmentDurationVector fwd_durations; + SegmentDurationVector rev_durations; Vector datasources; }; } diff --git a/include/extractor/serialization.hpp b/include/extractor/serialization.hpp index a69a45172..49211e9e0 100644 --- a/include/extractor/serialization.hpp +++ b/include/extractor/serialization.hpp @@ -38,10 +38,10 @@ inline void read(storage::io::FileReader &reader, { storage::serialization::read(reader, segment_data.index); storage::serialization::read(reader, segment_data.nodes); - storage::serialization::read(reader, segment_data.fwd_weights); - storage::serialization::read(reader, segment_data.rev_weights); - storage::serialization::read(reader, segment_data.fwd_durations); - storage::serialization::read(reader, segment_data.rev_durations); + util::serialization::read(reader, segment_data.fwd_weights); + util::serialization::read(reader, segment_data.rev_weights); + util::serialization::read(reader, segment_data.fwd_durations); + util::serialization::read(reader, segment_data.rev_durations); storage::serialization::read(reader, segment_data.datasources); } @@ -51,10 +51,10 @@ inline void write(storage::io::FileWriter &writer, { storage::serialization::write(writer, segment_data.index); storage::serialization::write(writer, segment_data.nodes); - storage::serialization::write(writer, segment_data.fwd_weights); - storage::serialization::write(writer, segment_data.rev_weights); - storage::serialization::write(writer, segment_data.fwd_durations); - storage::serialization::write(writer, segment_data.rev_durations); + util::serialization::write(writer, segment_data.fwd_weights); + util::serialization::write(writer, segment_data.rev_weights); + util::serialization::write(writer, segment_data.fwd_durations); + util::serialization::write(writer, segment_data.rev_durations); storage::serialization::write(writer, segment_data.datasources); } diff --git a/include/util/packed_vector.hpp b/include/util/packed_vector.hpp index c9cd836ba..18b1e7bb8 100644 --- a/include/util/packed_vector.hpp +++ b/include/util/packed_vector.hpp @@ -411,6 +411,8 @@ template class Pack void push_back(const T value) { + BOOST_ASSERT_MSG(value <= T{(1ULL << Bits) - 1}, "Value too big for packed storage."); + auto internal_index = get_internal_index(num_elements); while (internal_index.lower_word + 1 >= vec.size()) diff --git a/include/util/typedefs.hpp b/include/util/typedefs.hpp index bba5c5412..9d4c859c5 100644 --- a/include/util/typedefs.hpp +++ b/include/util/typedefs.hpp @@ -96,8 +96,12 @@ static const EdgeID SPECIAL_EDGEID = std::numeric_limits::max(); static const NameID INVALID_NAMEID = std::numeric_limits::max(); static const NameID EMPTY_NAMEID = 0; static const unsigned INVALID_COMPONENTID = 0; -static const SegmentWeight INVALID_SEGMENT_WEIGHT = (1u << 20) - 1; -static const SegmentDuration INVALID_SEGMENT_DURATION = (1u << 20) - 1; +static const std::size_t SEGMENT_WEIGHT_BITS = 22; +static const std::size_t SEGMENT_DURAITON_BITS = 22; +static const SegmentWeight INVALID_SEGMENT_WEIGHT = (1u << SEGMENT_WEIGHT_BITS) - 1; +static const SegmentDuration INVALID_SEGMENT_DURATION = (1u << SEGMENT_DURAITON_BITS) - 1; +static const SegmentWeight MAX_SEGMENT_WEIGHT = INVALID_SEGMENT_WEIGHT - 1; +static const SegmentDuration MAX_SEGMENT_DURATION = INVALID_SEGMENT_DURATION - 1; static const EdgeWeight INVALID_EDGE_WEIGHT = std::numeric_limits::max(); static const EdgeDuration MAXIMAL_EDGE_DURATION = std::numeric_limits::max(); static const TurnPenalty INVALID_TURN_PENALTY = std::numeric_limits::max(); diff --git a/src/extractor/compressed_edge_container.cpp b/src/extractor/compressed_edge_container.cpp index f9d597ad8..d7b109c8d 100644 --- a/src/extractor/compressed_edge_container.cpp +++ b/src/extractor/compressed_edge_container.cpp @@ -74,6 +74,26 @@ unsigned CompressedEdgeContainer::GetZippedPositionForReverseID(const EdgeID edg return map_iterator->second; } +SegmentWeight CompressedEdgeContainer::ClipWeight(const SegmentWeight weight) +{ + if (weight >= INVALID_SEGMENT_WEIGHT) + { + clipped_weights++; + return MAX_SEGMENT_WEIGHT; + } + return weight; +} + +SegmentDuration CompressedEdgeContainer::ClipDuration(const SegmentDuration duration) +{ + if (duration >= INVALID_SEGMENT_DURATION) + { + clipped_weights++; + return MAX_SEGMENT_DURATION; + } + return duration; +} + // Adds info for a compressed edge to the container. edge_id_2 // has been removed from the graph, so we have to save These edges/nodes // have already been trimmed from the graph, this function just stores @@ -87,10 +107,10 @@ void CompressedEdgeContainer::CompressEdge(const EdgeID edge_id_1, const EdgeID edge_id_2, const NodeID via_node_id, const NodeID target_node_id, - EdgeWeight weight1, - EdgeWeight weight2, - EdgeDuration duration1, - EdgeDuration duration2) + const EdgeWeight weight1, + const EdgeWeight weight2, + const EdgeDuration duration1, + const EdgeDuration duration2) { // remove super-trivial geometries BOOST_ASSERT(SPECIAL_EDGEID != edge_id_1); @@ -131,34 +151,12 @@ void CompressedEdgeContainer::CompressEdge(const EdgeID edge_id_1, std::vector &edge_bucket_list1 = m_compressed_oneway_geometries[edge_bucket_id1]; - const auto clip_weight = [&](const SegmentWeight weight) { - if (weight >= INVALID_SEGMENT_WEIGHT) - { - clipped_weights++; - return INVALID_SEGMENT_WEIGHT - 1; - } - return weight; - }; - const auto clip_duration = [&](const SegmentDuration duration) { - if (duration >= INVALID_SEGMENT_DURATION) - { - clipped_weights++; - return INVALID_SEGMENT_DURATION - 1; - } - return duration; - }; - // note we don't save the start coordinate: it is implicitly given by edge 1 // weight1 is the distance to the (currently) last coordinate in the bucket if (edge_bucket_list1.empty()) { - weight1 = clip_weight(weight1); - duration1 = clip_duration(duration1); - edge_bucket_list1.emplace_back( - OnewayCompressedEdge{via_node_id, - static_cast(weight1), - static_cast(duration1)}); + OnewayCompressedEdge{via_node_id, ClipWeight(weight1), ClipDuration(duration1)}); } BOOST_ASSERT(0 < edge_bucket_list1.size()); @@ -188,14 +186,9 @@ void CompressedEdgeContainer::CompressEdge(const EdgeID edge_id_1, } else { - weight2 = clip_weight(weight2); - duration2 = clip_duration(duration2); - // we are certain that the second edge is atomic. edge_bucket_list1.emplace_back( - OnewayCompressedEdge{target_node_id, - static_cast(weight2), - static_cast(duration2)}); + OnewayCompressedEdge{target_node_id, ClipWeight(weight2), ClipDuration(duration2)}); } } @@ -238,7 +231,8 @@ void CompressedEdgeContainer::AddUncompressedEdge(const EdgeID edge_id, // Don't re-add this if it's already in there. if (edge_bucket_list.empty()) { - edge_bucket_list.emplace_back(OnewayCompressedEdge{target_node_id, weight, duration}); + edge_bucket_list.emplace_back( + OnewayCompressedEdge{target_node_id, ClipWeight(weight), ClipDuration(duration)}); } } @@ -272,9 +266,9 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID constexpr DatasourceID LUA_SOURCE = 0; segment_data->nodes.emplace_back(first_node.node_id); - segment_data->fwd_weights.emplace_back(INVALID_EDGE_WEIGHT); + segment_data->fwd_weights.emplace_back(INVALID_SEGMENT_WEIGHT); segment_data->rev_weights.emplace_back(first_node.weight); - segment_data->fwd_durations.emplace_back(INVALID_EDGE_WEIGHT); + segment_data->fwd_durations.emplace_back(INVALID_SEGMENT_DURATION); segment_data->rev_durations.emplace_back(first_node.duration); segment_data->datasources.emplace_back(LUA_SOURCE); @@ -297,9 +291,9 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID segment_data->nodes.emplace_back(last_node.node_id); segment_data->fwd_weights.emplace_back(last_node.weight); - segment_data->rev_weights.emplace_back(INVALID_EDGE_WEIGHT); + segment_data->rev_weights.emplace_back(INVALID_SEGMENT_WEIGHT); segment_data->fwd_durations.emplace_back(last_node.duration); - segment_data->rev_durations.emplace_back(INVALID_EDGE_WEIGHT); + segment_data->rev_durations.emplace_back(INVALID_SEGMENT_DURATION); segment_data->datasources.emplace_back(LUA_SOURCE); return zipped_geometry_id; diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index 5c90b28df..d2c8a1620 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -487,6 +487,8 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment, config.turn_penalties_index_path, config.cnbg_ebg_graph_mapping_output_path); + compressed_edge_container.PrintStatistics(); + // The osrm-partition tool requires the compressed node based graph with an embedding. // // The `Run` function above re-numbers non-reverse compressed node based graph edges diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index f97073ede..998e45e65 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -351,14 +351,29 @@ void Storage::PopulateLayout(DataLayout &layout) const auto number_of_compressed_geometries = reader.ReadVectorSize(); layout.SetBlockSize(DataLayout::GEOMETRIES_NODE_LIST, number_of_compressed_geometries); - layout.SetBlockSize(DataLayout::GEOMETRIES_FWD_WEIGHT_LIST, - number_of_compressed_geometries); - layout.SetBlockSize(DataLayout::GEOMETRIES_REV_WEIGHT_LIST, - number_of_compressed_geometries); - layout.SetBlockSize(DataLayout::GEOMETRIES_FWD_DURATION_LIST, - number_of_compressed_geometries); - layout.SetBlockSize(DataLayout::GEOMETRIES_REV_DURATION_LIST, - number_of_compressed_geometries); + + reader.ReadElementCount64(); // number of segments + const auto number_of_segment_weight_blocks = + reader.ReadVectorSize(); + + reader.ReadElementCount64(); // number of segments + auto number_of_rev_weight_blocks = + reader.ReadVectorSize(); + BOOST_ASSERT(number_of_rev_weight_blocks == number_of_segment_weight_blocks); + (void)number_of_rev_weight_blocks; + + reader.ReadElementCount64(); // number of segments + const auto number_of_segment_duration_blocks = + reader.ReadVectorSize(); + + layout.SetBlockSize( + DataLayout::GEOMETRIES_FWD_WEIGHT_LIST, number_of_segment_weight_blocks); + layout.SetBlockSize( + DataLayout::GEOMETRIES_REV_WEIGHT_LIST, number_of_segment_weight_blocks); + layout.SetBlockSize( + DataLayout::GEOMETRIES_FWD_DURATION_LIST, number_of_segment_duration_blocks); + layout.SetBlockSize( + DataLayout::GEOMETRIES_REV_DURATION_LIST, number_of_segment_duration_blocks); layout.SetBlockSize(DataLayout::DATASOURCES_LIST, number_of_compressed_geometries); } @@ -658,35 +673,47 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) util::vector_view geometry_begin_indices( geometries_index_ptr, layout.num_entries[storage::DataLayout::GEOMETRIES_INDEX]); + auto num_entries = layout.num_entries[storage::DataLayout::GEOMETRIES_NODE_LIST]; + auto geometries_node_list_ptr = layout.GetBlockPtr(memory_ptr, storage::DataLayout::GEOMETRIES_NODE_LIST); - util::vector_view geometry_node_list( - geometries_node_list_ptr, - layout.num_entries[storage::DataLayout::GEOMETRIES_NODE_LIST]); + util::vector_view geometry_node_list(geometries_node_list_ptr, num_entries); - auto geometries_fwd_weight_list_ptr = layout.GetBlockPtr( - memory_ptr, storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST); - util::vector_view geometry_fwd_weight_list( - geometries_fwd_weight_list_ptr, - layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST]); + auto geometries_fwd_weight_list_ptr = + layout.GetBlockPtr( + memory_ptr, storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST); + extractor::SegmentDataView::SegmentWeightVector geometry_fwd_weight_list( + util::vector_view( + geometries_fwd_weight_list_ptr, + layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST]), + num_entries); - auto geometries_rev_weight_list_ptr = layout.GetBlockPtr( - memory_ptr, storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST); - util::vector_view geometry_rev_weight_list( - geometries_rev_weight_list_ptr, - layout.num_entries[storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST]); + auto geometries_rev_weight_list_ptr = + layout.GetBlockPtr( + memory_ptr, storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST); + extractor::SegmentDataView::SegmentWeightVector geometry_rev_weight_list( + util::vector_view( + geometries_rev_weight_list_ptr, + layout.num_entries[storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST]), + num_entries); - auto geometries_fwd_duration_list_ptr = layout.GetBlockPtr( - memory_ptr, storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST); - util::vector_view geometry_fwd_duration_list( - geometries_fwd_duration_list_ptr, - layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST]); + auto geometries_fwd_duration_list_ptr = + layout.GetBlockPtr( + memory_ptr, storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST); + extractor::SegmentDataView::SegmentDurationVector geometry_fwd_duration_list( + util::vector_view( + geometries_fwd_duration_list_ptr, + layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST]), + num_entries); - auto geometries_rev_duration_list_ptr = layout.GetBlockPtr( - memory_ptr, storage::DataLayout::GEOMETRIES_REV_DURATION_LIST); - util::vector_view geometry_rev_duration_list( - geometries_rev_duration_list_ptr, - layout.num_entries[storage::DataLayout::GEOMETRIES_REV_DURATION_LIST]); + auto geometries_rev_duration_list_ptr = + layout.GetBlockPtr( + memory_ptr, storage::DataLayout::GEOMETRIES_REV_DURATION_LIST); + extractor::SegmentDataView::SegmentDurationVector geometry_rev_duration_list( + util::vector_view( + geometries_rev_duration_list_ptr, + layout.num_entries[storage::DataLayout::GEOMETRIES_REV_DURATION_LIST]), + num_entries); auto datasources_list_ptr = layout.GetBlockPtr( memory_ptr, storage::DataLayout::DATASOURCES_LIST); @@ -722,8 +749,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) extractor::PackedOSMIDsView osm_node_ids( util::vector_view( osmnodeid_ptr, layout.num_entries[DataLayout::OSM_NODE_ID_LIST]), - layout.num_entries[DataLayout::OSM_NODE_ID_LIST] * - extractor::PackedOSMIDsView::BLOCK_ELEMENTS); + layout.num_entries[DataLayout::COORDINATE_LIST]); extractor::files::readNodes(config.node_based_nodes_data_path, coordinates, osm_node_ids); } diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index 97c86e81d..2efd00428 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -81,14 +81,22 @@ template inline bool is_aligned(const void *pointer) } // Returns duration in deci-seconds -inline EdgeWeight convertToDuration(double speed_in_kmh, double distance_in_meters) +inline SegmentDuration convertToDuration(double speed_in_kmh, double distance_in_meters) { if (speed_in_kmh <= 0.) - return MAXIMAL_EDGE_DURATION; + return INVALID_SEGMENT_DURATION; const auto speed_in_ms = speed_in_kmh / 3.6; const auto duration = distance_in_meters / speed_in_ms; - return std::max(1, boost::numeric_cast(std::round(duration * 10.))); + auto segment_duration = std::max( + 1, boost::numeric_cast(std::round(duration * 10.))); + if (segment_duration >= INVALID_SEGMENT_DURATION) + { + util::Log(logWARNING) << "Clamping segment duration " << segment_duration << " to " + << MAX_SEGMENT_DURATION; + segment_duration = MAX_SEGMENT_DURATION; + } + return segment_duration; } #if !defined(NDEBUG) @@ -182,11 +190,19 @@ updateSegmentData(const UpdaterConfig &config, } if (rate <= 0.) - return INVALID_EDGE_WEIGHT; + return INVALID_SEGMENT_WEIGHT; const auto weight_multiplier = profile_properties.GetWeightMultiplier(); const auto weight = distance_in_meters / rate; - return std::max(1, boost::numeric_cast(std::round(weight * weight_multiplier))); + auto segment_weight = std::max( + 1, boost::numeric_cast(std::round(weight * weight_multiplier))); + if (segment_weight >= INVALID_SEGMENT_WEIGHT) + { + util::Log(logWARNING) << "Clamping segment weight " << segment_weight << " to " + << MAX_SEGMENT_WEIGHT; + segment_weight = MAX_SEGMENT_WEIGHT; + } + return segment_weight; }; // The check here is enabled by the `--edge-weight-updates-over-factor` flag it logs a @@ -739,7 +755,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e const auto weights = segment_data.GetForwardWeights(geometry_id.id); for (const auto weight : weights) { - if (weight == INVALID_EDGE_WEIGHT) + if (weight == INVALID_SEGMENT_WEIGHT) { new_weight = INVALID_EDGE_WEIGHT; break; @@ -754,7 +770,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e const auto weights = segment_data.GetReverseWeights(geometry_id.id); for (const auto weight : weights) { - if (weight == INVALID_EDGE_WEIGHT) + if (weight == INVALID_SEGMENT_WEIGHT) { new_weight = INVALID_EDGE_WEIGHT; break;