Remove deduplication of unpacked_path_segments in MM collapsing (#4911)

CHANGELOG.md

@@ -1,5 +1,10 @@
-# 5.16.0 RC 6
-  - Changes from 5.15.1:
+# 5.16.1
+  - Changes from 5.16.0:
+    - Bugfixes
+      - FIXED #4909: deduplication of route steps when waypoints are used [#4909](https://github.com/Project-OSRM/osrm-backend/issues/4909)
+
+# 5.16.0
+  - Changes from 5.15.2:
     - Guidance
       - ADDED #4676: Support for maneuver override relation, allowing data-driven overrides for turn-by-turn instructions [#4676](https://github.com/Project-OSRM/osrm-backend/pull/4676)
       - CHANGED #4830: Announce reference change if names are empty
@@ -14,6 +19,13 @@
       - FIXED #4763: Add support for non-numerical units in car profile for maxheight [#4763](https://github.com/Project-OSRM/osrm-backend/issues/4763)
       - ADDED #4872: Handling of `barrier=height_restrictor` nodes [#4872](https://github.com/Project-OSRM/osrm-backend/pull/4872)
 
+# 5.15.2
+  - Changes from 5.15.1:
+    - Features:
+        - ADDED: Exposed the waypoints parameter in the node bindings interface
+    - Bugfixes:
+        - FIXED: Segfault causing bug in leg collapsing map matching when traversing edges in reverse
+
 # 5.15.1
   - Changes from 5.15.0:
     - Bugfixes:

CMakeLists.txt

@@ -61,7 +61,7 @@ endif()
 project(OSRM C CXX)
 set(OSRM_VERSION_MAJOR 5)
 set(OSRM_VERSION_MINOR 16)
-set(OSRM_VERSION_PATCH 0)
+set(OSRM_VERSION_PATCH 1)
 set(OSRM_VERSION "${OSRM_VERSION_MAJOR}.${OSRM_VERSION_MINOR}.${OSRM_VERSION_PATCH}")
 
 add_definitions(-DOSRM_PROJECT_DIR="${CMAKE_CURRENT_SOURCE_DIR}")

features/testbot/matching.feature

@@ -626,3 +626,61 @@ Feature: Basic Map Matching
         | trace    | timestamps                                   | matchings  | code |
         | abbecd   | 10 11 27 1516914902 1516914913 1516914952    | ab,ecd     | Ok   |
 
+    Scenario: Regression test - waypoints trimming too much geometry
+        # fixes bug in map matching collapsing that was dropping path geometries
+        # after segments that had 0 distance in internal route results
+        Given the node map
+            """
+            ad
+            |
+            |
+            |
+            |
+            |e   g
+            b--------------c
+            f              h
+            """
+
+        And the ways
+            | nodes |
+            | ab    |
+            | bc    |
+
+        Given the query options
+            | waypoints | 0;4   |
+            | overview  | full  |
+
+        When I match I should get
+            | trace    | geometry                           | code |
+            | defgh    | 1,1,1,0.999461,1.000674,0.999461   | Ok   |
+
+    @match @testbot
+    Scenario: Regression test - waypoints trimming too much geometry
+        Given the profile "testbot"
+        Given a grid size of 10 meters
+        Given the query options
+          | geometries | geojson |
+        Given the node map
+          """
+            bh
+             |
+             |
+             |
+             c
+             g\
+               \k
+                \
+                 \
+                  \
+                 j f
+          """
+        And the ways
+          | nodes |
+          | hc    |
+          | cf    |
+        Given the query options
+          | waypoints | 0;3  |
+          | overview  | full |
+        When I match I should get
+          | trace | geometry                                      | code |
+          | bgkj  | 1.000135,1,1.000135,0.99964,1.000387,0.999137 | Ok   |

include/engine/internal_route_result.hpp

@@ -139,11 +139,6 @@ inline InternalRouteResult CollapseInternalRouteResult(const InternalRouteResult
         {
             BOOST_ASSERT(!collapsed.unpacked_path_segments.empty());
             auto &last_segment = collapsed.unpacked_path_segments.back();
-            // deduplicate last segment (needs to be checked for empty for the same node query edge
-            // case)
-            if (!last_segment.empty())
-                last_segment.pop_back();
-            // update target phantom node of leg
             BOOST_ASSERT(!collapsed.segment_end_coordinates.empty());
             collapsed.segment_end_coordinates.back().target_phantom =
                 leggy_result.segment_end_coordinates[i].target_phantom;

include/extractor/files.hpp

@@ -261,6 +261,29 @@ inline void writeTurnLaneDescriptions(const boost::filesystem::path &path,
     storage::serialization::write(writer, turn_offsets);
     storage::serialization::write(writer, turn_masks);
 }
+
+// reads .osrm.maneuver_overrides
+template <typename StorageManeuverOverrideT, typename NodeSequencesT>
+inline void readManeuverOverrides(const boost::filesystem::path &path,
+                                  StorageManeuverOverrideT &maneuver_overrides,
+                                  NodeSequencesT &node_sequences)
+{
+    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
+    storage::io::FileReader reader{path, fingerprint};
+
+    serialization::read(reader, maneuver_overrides, node_sequences);
+}
+
+// writes .osrm.maneuver_overrides
+inline void writeManeuverOverrides(const boost::filesystem::path &path,
+                                   const std::vector<StorageManeuverOverride> &maneuver_overrides,
+                                   const std::vector<NodeID> &node_sequences)
+{
+    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
+    storage::io::FileWriter writer{path, fingerprint};
+
+    serialization::write(writer, maneuver_overrides, node_sequences);
+}
 }
 }
 }

include/extractor/serialization.hpp

@@ -304,10 +304,17 @@ inline void write(storage::io::FileWriter &writer,
                   const std::vector<StorageManeuverOverride> &maneuver_overrides,
                   const std::vector<NodeID> &node_sequences)
 {
-    writer.WriteElementCount64(maneuver_overrides.size());
-    writer.WriteElementCount64(node_sequences.size());
-    writer.WriteFrom(maneuver_overrides);
-    writer.WriteFrom(node_sequences);
+    storage::serialization::write(writer, maneuver_overrides);
+    storage::serialization::write(writer, node_sequences);
+}
+
+template <typename ManeuverOverridesT, typename NodeSequenceT>
+inline void read(storage::io::FileReader &reader,
+                 ManeuverOverridesT &maneuver_overrides,
+                 NodeSequenceT &node_sequences)
+{
+    storage::serialization::read(reader, maneuver_overrides);
+    storage::serialization::read(reader, node_sequences);
 }
 }
 }

include/partitioner/partitioner_config.hpp

@@ -16,10 +16,14 @@ namespace partitioner
 struct PartitionerConfig final : storage::IOConfig
 {
     PartitionerConfig()
-        : IOConfig(
-              {".osrm", ".osrm.fileIndex", ".osrm.ebg_nodes"},
-              {".osrm.hsgr", ".osrm.cnbg"},
-              {".osrm.ebg", ".osrm.cnbg", ".osrm.cnbg_to_ebg", ".osrm.partition", ".osrm.cells"}),
+        : IOConfig({".osrm", ".osrm.fileIndex", ".osrm.ebg_nodes"},
+                   {".osrm.hsgr", ".osrm.cnbg"},
+                   {".osrm.ebg",
+                    ".osrm.cnbg",
+                    ".osrm.cnbg_to_ebg",
+                    ".osrm.partition",
+                    ".osrm.cells",
+                    ".osrm.maneuver_overrides"}),
           requested_num_threads(0), balance(1.2), boundary_factor(0.25), num_optimizing_cuts(10),
           small_component_size(1000),
           max_cell_sizes({128, 128 * 32, 128 * 32 * 16, 128 * 32 * 16 * 32})

include/partitioner/renumber.hpp

@@ -2,6 +2,7 @@
 #define OSRM_PARTITIONER_RENUMBER_HPP
 
 #include "extractor/edge_based_node_segment.hpp"
+#include "extractor/maneuver_override.hpp"
 #include "extractor/nbg_to_ebg.hpp"
 #include "extractor/node_data_container.hpp"
 
@@ -73,6 +74,25 @@ inline void renumber(std::vector<extractor::NBGToEBG> &mapping,
     }
 }
 
+inline void renumber(std::vector<NodeID> &node_ids, const std::vector<std::uint32_t> &permutation)
+{
+    for (auto &node_id : node_ids)
+    {
+        if (node_id != SPECIAL_NODEID)
+            node_id = permutation[node_id];
+    }
+}
+
+inline void renumber(std::vector<extractor::StorageManeuverOverride> &maneuver_overrides,
+                     const std::vector<std::uint32_t> &permutation)
+{
+    for (auto &maneuver_override : maneuver_overrides)
+    {
+        if (maneuver_override.start_node != SPECIAL_NODEID)
+            maneuver_override.start_node = permutation[maneuver_override.start_node];
+    }
+}
+
 } // namespace partitioner
 } // namespace osrm
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "osrm",
-  "version": "5.16.0-rc.6",
+  "version": "5.16.1",
   "private": false,
   "description": "The Open Source Routing Machine is a high performance routing engine written in C++14 designed to run on OpenStreetMap data.",
   "dependencies": {

src/extractor/edge_based_graph_factory.cpp

@@ -1106,11 +1106,9 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
         std::sort(storage_maneuver_overrides.begin(),
                   storage_maneuver_overrides.end(),
                   [](const auto &a, const auto &b) { return a.start_node < b.start_node; });
-        // write conditional turn penalties into the restrictions file
-        storage::io::FileWriter writer(maneuver_overrides_filename,
-                                       storage::io::FileWriter::GenerateFingerprint);
-        extractor::serialization::write(
-            writer, storage_maneuver_overrides, maneuver_override_sequences);
+
+        files::writeManeuverOverrides(
+            maneuver_overrides_filename, storage_maneuver_overrides, maneuver_override_sequences);
     }
 
     util::Log() << "done.";

src/partitioner/partitioner.cpp

@@ -145,6 +145,15 @@ int Partitioner::Run(const PartitionerConfig &config)
         renumber(node_data, permutation);
         extractor::files::writeNodeData(config.GetPath(".osrm.ebg_nodes"), node_data);
     }
+    {
+        const auto &filename = config.GetPath(".osrm.maneuver_overrides");
+        std::vector<extractor::StorageManeuverOverride> maneuver_overrides;
+        std::vector<NodeID> node_sequences;
+        extractor::files::readManeuverOverrides(filename, maneuver_overrides, node_sequences);
+        renumber(maneuver_overrides, permutation);
+        renumber(node_sequences, permutation);
+        extractor::files::writeManeuverOverrides(filename, maneuver_overrides, node_sequences);
+    }
     if (boost::filesystem::exists(config.GetPath(".osrm.hsgr")))
     {
         util::Log(logWARNING) << "Found existing .osrm.hsgr file, removing. You need to re-run "

src/storage/storage.cpp

@@ -445,10 +445,11 @@ void Storage::PopulateLayout(DataLayout &layout)
     {
         io::FileReader maneuver_overrides_file(config.GetPath(".osrm.maneuver_overrides"),
                                                io::FileReader::VerifyFingerprint);
-        const auto number_of_overrides = maneuver_overrides_file.ReadElementCount64();
+        const auto number_of_overrides =
+            maneuver_overrides_file.ReadVectorSize<extractor::StorageManeuverOverride>();
         layout.SetBlockSize<extractor::StorageManeuverOverride>(DataLayout::MANEUVER_OVERRIDES,
                                                                 number_of_overrides);
-        const auto number_of_nodes = maneuver_overrides_file.ReadElementCount64();
+        const auto number_of_nodes = maneuver_overrides_file.ReadVectorSize<NodeID>();
         layout.SetBlockSize<NodeID>(DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES, number_of_nodes);
     }
 
@@ -1089,18 +1090,21 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
         // load maneuver overrides
         {
-            io::FileReader maneuver_overrides_file(config.GetPath(".osrm.maneuver_overrides"),
-                                                   io::FileReader::VerifyFingerprint);
-            const auto number_of_overrides = maneuver_overrides_file.ReadElementCount64();
-            const auto number_of_nodes = maneuver_overrides_file.ReadElementCount64();
             const auto maneuver_overrides_ptr =
                 layout.GetBlockPtr<extractor::StorageManeuverOverride, true>(
                     memory_ptr, DataLayout::MANEUVER_OVERRIDES);
-            maneuver_overrides_file.ReadInto(maneuver_overrides_ptr, number_of_overrides);
-
             const auto maneuver_override_node_sequences_ptr = layout.GetBlockPtr<NodeID, true>(
                 memory_ptr, DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES);
-            maneuver_overrides_file.ReadInto(maneuver_override_node_sequences_ptr, number_of_nodes);
+
+            util::vector_view<extractor::StorageManeuverOverride> maneuver_overrides(
+                maneuver_overrides_ptr, layout.num_entries[DataLayout::MANEUVER_OVERRIDES]);
+            util::vector_view<NodeID> maneuver_override_node_sequences(
+                maneuver_override_node_sequences_ptr,
+                layout.num_entries[DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES]);
+
+            extractor::files::readManeuverOverrides(config.GetPath(".osrm.maneuver_overrides"),
+                                                    maneuver_overrides,
+                                                    maneuver_override_node_sequences);
         }
     }
 }

unit_tests/engine/collapse_internal_route_result.cpp

@@ -38,6 +38,10 @@ BOOST_AUTO_TEST_CASE(unchanged_collapse_route_result)
 
 BOOST_AUTO_TEST_CASE(two_legs_to_one_leg)
 {
+    // from_edge_based_node, turn_via_node, name_id, is_segregated, weight_until_turn,
+    // weight_of_turn,
+    // duration_until_turn, duration_of_turn, turn_instruction, lane_data, travel_mode, classes,
+    // entry_class, datasource_id, pre_turn_bearing, post_turn_bearing, left_hand
     PathData pathy{0, 2, 17, false, 2, 3, 4, 5, 0, {}, 4, 2, {}, 2, {1.0}, {1.0}, false};
     PathData kathy{0, 1, 16, false, 1, 2, 3, 4, 1, {}, 3, 1, {}, 1, {2.0}, {3.0}, false};
     PathData cathy{0, 3, 16, false, 1, 2, 3, 4, 1, {}, 3, 1, {}, 1, {2.0}, {3.0}, false};
@@ -61,10 +65,11 @@ BOOST_AUTO_TEST_CASE(two_legs_to_one_leg)
     BOOST_CHECK_EQUAL(collapsed.segment_end_coordinates[0].target_phantom.forward_segment_id.id,
                       12);
     BOOST_CHECK_EQUAL(collapsed.segment_end_coordinates[0].source_phantom.forward_segment_id.id, 1);
-    BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[0].size(), 3);
+    BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[0].size(), 4);
     BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[0][0].turn_via_node, 2);
     BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[0][1].turn_via_node, 1);
-    BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[0][2].turn_via_node, 3);
+    BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[0][2].turn_via_node, 1);
+    BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[0][3].turn_via_node, 3);
 }
 
 BOOST_AUTO_TEST_CASE(three_legs_to_two_legs)
@@ -101,13 +106,14 @@ BOOST_AUTO_TEST_CASE(three_legs_to_two_legs)
     BOOST_CHECK_EQUAL(collapsed.segment_end_coordinates[1].target_phantom.forward_segment_id.id,
                       18);
     BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[0].size(), 2);
-    BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[1].size(), 4);
+    BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[1].size(), 5);
     BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[0][0].turn_via_node, 2);
     BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[0][1].turn_via_node, 1);
     BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[1][0].turn_via_node, 1);
     BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[1][1].turn_via_node, 5);
     BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[1][2].turn_via_node, 3);
-    BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[1][3].turn_via_node, 4);
+    BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[1][3].turn_via_node, 3);
+    BOOST_CHECK_EQUAL(collapsed.unpacked_path_segments[1][4].turn_via_node, 4);
 }
 
 BOOST_AUTO_TEST_CASE(two_legs_to_two_legs)