Transparently Tidy Traces in Map Matching, resolves #2840.

The Map Matching plugin currently has issues with:
- high frequency traces and (performance)
- blobs, think noise at traffic signals (correctness)

This changeset implements trace-tidying transparently for the user.

We hopefully will see both performance gains as well as better matches!

include/engine/api/match_parameters.hpp

@@ -56,7 +56,7 @@ struct MatchParameters : public RouteParameters
                           false,
                           RouteParameters::GeometriesType::Polyline,
                           RouteParameters::OverviewType::Simplified,
-                          {})
+                          {}), use_tidying(true)
     {
     }
 
@@ -67,7 +67,8 @@ struct MatchParameters : public RouteParameters
     }
 
     std::vector<unsigned> timestamps;
-    bool use_tidying = true;
+    bool use_tidying;
+
     bool IsValid() const
     {
         return RouteParameters::IsValid() &&

include/engine/api/match_parameters_tidy.hpp

@@ -0,0 +1,138 @@
+#ifndef COORDINATE_TIDY
+#define COORDINATE_TIDY
+
+#include <algorithm>
+#include <cstdint>
+#include <iterator>
+
+#include "engine/api/match_parameters.hpp"
+#include "util/coordinate_calculation.hpp"
+
+#include <boost/assert.hpp>
+#include <boost/dynamic_bitset.hpp>
+
+namespace osrm
+{
+namespace engine
+{
+namespace api
+{
+namespace tidy
+{
+
+struct Thresholds
+{
+    double distance_in_meters;
+    std::int32_t duration_in_seconds;
+};
+
+using Mask = boost::dynamic_bitset<>;
+using Mapping = std::vector<std::size_t>;
+
+struct Result
+{
+    // Tidied parameters
+    MatchParameters parameters;
+    // Masking the MatchParameter parallel arrays for items which should be removed.
+    Mask can_be_removed;
+    // Maps the MatchParameter's original items to items which should not be removed.
+    Mapping original_to_tidied;
+};
+
+inline Result tidy(const MatchParameters &params, Thresholds cfg = {15., 5})
+{
+    Result result;
+
+    result.can_be_removed.reserve(params.coordinates.size());
+    result.original_to_tidied.reserve(params.coordinates.size());
+
+    result.can_be_removed.push_back(false);
+    result.original_to_tidied.push_back(0);
+    std::size_t last_good = 0;
+
+    const auto uses_timestamps = !params.timestamps.empty();
+
+    Thresholds running{0., 0};
+
+    // Walk over adjacent (coord, ts)-pairs, with rhs being the candidate to discard or keep
+    for (std::size_t current = 0; current < params.coordinates.size() - 1; ++current)
+    {
+        const auto next = current + 1;
+
+        auto distance_delta = util::coordinate_calculation::haversineDistance(
+            params.coordinates[current], params.coordinates[next]);
+        running.distance_in_meters += distance_delta;
+        const auto over_distance = running.distance_in_meters >= cfg.distance_in_meters;
+
+        if (uses_timestamps)
+        {
+            auto duration_delta = params.timestamps[next] - params.timestamps[current];
+            running.duration_in_seconds += duration_delta;
+            const auto over_duration = running.duration_in_seconds >= cfg.duration_in_seconds;
+
+            if (over_distance && over_duration)
+            {
+                result.can_be_removed.push_back(false);
+                last_good = next;
+                running = {0., 0}; // reset running distance and time
+            }
+            else
+            {
+                result.can_be_removed.push_back(true);
+            }
+        }
+        else
+        {
+            if (over_distance)
+            {
+                result.can_be_removed.push_back(false);
+                last_good = next;
+                running = {0., 0}; // reset running distance and time
+            }
+            else
+            {
+                result.can_be_removed.push_back(true);
+            }
+        }
+
+        result.original_to_tidied.push_back(last_good);
+    }
+
+    BOOST_ASSERT(result.original_to_tidied.size() == params.coordinates.size());
+    BOOST_ASSERT(result.can_be_removed.size() == params.coordinates.size());
+    BOOST_ASSERT(std::is_sorted(begin(result.original_to_tidied), end(result.original_to_tidied)));
+
+    // We have to filter parallel arrays that may be empty or the exact same size.
+    // result.parameters contains an empty MatchParameters at this point: conditionally fill.
+
+    const auto to_keep = result.can_be_removed.size() - result.can_be_removed.count();
+
+    for (std::size_t i = 0; i < result.can_be_removed.size(); ++i)
+    {
+        if (!result.can_be_removed[i])
+        {
+            result.parameters.coordinates.push_back(params.coordinates[i]);
+
+            if (!params.hints.empty())
+                result.parameters.hints.push_back(params.hints[i]);
+
+            if (!params.radiuses.empty())
+                result.parameters.radiuses.push_back(params.radiuses[i]);
+
+            if (!params.bearings.empty())
+                result.parameters.bearings.push_back(params.bearings[i]);
+
+            if (!params.timestamps.empty())
+                result.parameters.timestamps.push_back(params.timestamps[i]);
+        }
+    }
+
+    return result;
+}
+
+} // ns tidy
+} // ns api
+} // ns engine
+} // ns osrm
+
+#endif

src/engine/plugins/match.cpp

@@ -3,6 +3,7 @@
 
 #include "engine/api/match_api.hpp"
 #include "engine/api/match_parameters.hpp"
+#include "engine/api/match_parameters_tidy.hpp"
 #include "engine/map_matching/bayes_classifier.hpp"
 #include "engine/map_matching/sub_matching.hpp"
 #include "util/coordinate_calculation.hpp"
@@ -173,9 +174,13 @@ Status MatchPlugin::HandleRequest(const datafacade::ContiguousInternalMemoryData
                        });
     }
 
-    auto candidates_lists = GetPhantomNodesInRange(facade, parameters, search_radiuses);
+    // Transparently tidy match parameters, do map matching on tidied parameters.
+    // Then use the mapping to restore the original <-> tidied relationship.
+    auto tidied = api::tidy::tidy(parameters);
 
-    filterCandidates(parameters.coordinates, candidates_lists);
+    auto candidates_lists = GetPhantomNodesInRange(facade, tidied.parameters, search_radiuses);
+
+    filterCandidates(tidied.parameters.coordinates, candidates_lists);
     if (std::all_of(candidates_lists.begin(),
                     candidates_lists.end(),
                     [](const std::vector<PhantomNodeWithDistance> &candidates) {
@@ -189,7 +194,11 @@ Status MatchPlugin::HandleRequest(const datafacade::ContiguousInternalMemoryData
 
     // call the actual map matching
     SubMatchingList sub_matchings = algorithms.MapMatching(
-        candidates_lists, parameters.coordinates, parameters.timestamps, parameters.radiuses, parameters.use_tidying);
+                                                 candidates_lists,
+                                                 tidied.parameters.coordinates,
+                                                 tidied.parameters.timestamps,
+                                                 tidied.parameters.radiuses,
+                                                 parameters.use_tidying);
 
     if (sub_matchings.size() == 0)
     {
@@ -220,6 +229,8 @@ Status MatchPlugin::HandleRequest(const datafacade::ContiguousInternalMemoryData
         BOOST_ASSERT(sub_routes[index].shortest_path_length != INVALID_EDGE_WEIGHT);
     }
 
+    // TODO: restore original coordinates
+
     api::MatchAPI match_api{facade, parameters};
     match_api.MakeResponse(sub_matchings, sub_routes, json_result);
 

unit_tests/engine/tidy.cpp

@@ -0,0 +1,156 @@
+#include "engine/api/match_parameters_tidy.hpp"
+
+#include <boost/test/test_case_template.hpp>
+#include <boost/test/unit_test.hpp>
+
+#include <algorithm>
+#include <iterator>
+#include <vector>
+
+BOOST_AUTO_TEST_SUITE(tidy_test)
+
+using namespace osrm;
+using namespace osrm::util;
+using namespace osrm::engine::api;
+
+BOOST_AUTO_TEST_CASE(two_item_trace_already_tidied_test)
+{
+    MatchParameters params;
+    params.coordinates.emplace_back(FloatLongitude{13.207993}, FloatLatitude{52.446379});
+    params.coordinates.emplace_back(FloatLongitude{13.231658}, FloatLatitude{52.465416});
+
+    params.timestamps.emplace_back(1477090402);
+    params.timestamps.emplace_back(1477090663);
+
+    tidy::Thresholds thresholds;
+    thresholds.distance_in_meters = 15.;
+    thresholds.duration_in_seconds = 5;
+
+    auto result = tidy::tidy(params, thresholds);
+
+    BOOST_CHECK_EQUAL(result.can_be_removed.size(), 2);
+    BOOST_CHECK_EQUAL(result.original_to_tidied.size(), 2);
+
+    BOOST_CHECK(result.can_be_removed[0] == false);
+    BOOST_CHECK(result.can_be_removed[1] == false);
+    BOOST_CHECK_EQUAL(result.original_to_tidied[0], 0);
+    BOOST_CHECK_EQUAL(result.original_to_tidied[1], 1);
+}
+
+BOOST_AUTO_TEST_CASE(two_item_trace_needs_tidiying_test)
+{
+    MatchParameters params;
+    params.coordinates.emplace_back(FloatLongitude{13.207993}, FloatLatitude{52.446379});
+    params.coordinates.emplace_back(FloatLongitude{13.231658}, FloatLatitude{52.465416});
+
+    params.timestamps.emplace_back(1477090402);
+    params.timestamps.emplace_back(1477090663);
+
+    tidy::Thresholds thresholds;
+    thresholds.distance_in_meters = 5000;
+    thresholds.duration_in_seconds = 5 * 60;
+
+    auto result = tidy::tidy(params, thresholds);
+
+    BOOST_CHECK_EQUAL(result.can_be_removed.size(), 2);
+    BOOST_CHECK_EQUAL(result.original_to_tidied.size(), 2);
+
+    BOOST_CHECK_EQUAL(result.can_be_removed[0], false);
+    BOOST_CHECK_EQUAL(result.can_be_removed[1], true);
+    BOOST_CHECK_EQUAL(result.original_to_tidied[0], 0);
+    BOOST_CHECK_EQUAL(result.original_to_tidied[1], 0);
+}
+
+BOOST_AUTO_TEST_CASE(two_blobs_in_traces_needs_tidiying_test)
+{
+    MatchParameters params;
+
+    params.coordinates.emplace_back(FloatLongitude{13.207993}, FloatLatitude{52.446379});
+    params.coordinates.emplace_back(FloatLongitude{13.207994}, FloatLatitude{52.446380});
+    params.coordinates.emplace_back(FloatLongitude{13.207995}, FloatLatitude{52.446381});
+
+    params.coordinates.emplace_back(FloatLongitude{13.231658}, FloatLatitude{52.465416});
+    params.coordinates.emplace_back(FloatLongitude{13.231659}, FloatLatitude{52.465417});
+    params.coordinates.emplace_back(FloatLongitude{13.231660}, FloatLatitude{52.465417});
+
+    params.timestamps.emplace_back(1477090402);
+    params.timestamps.emplace_back(1477090403);
+    params.timestamps.emplace_back(1477090404);
+
+    params.timestamps.emplace_back(1477090661);
+    params.timestamps.emplace_back(1477090662);
+    params.timestamps.emplace_back(1477090663);
+
+    tidy::Thresholds thresholds;
+    thresholds.distance_in_meters = 15;
+    thresholds.duration_in_seconds = 5;
+
+    auto result = tidy::tidy(params, thresholds);
+
+    BOOST_CHECK_EQUAL(result.can_be_removed.size(), params.coordinates.size());
+    BOOST_CHECK_EQUAL(result.original_to_tidied.size(), params.coordinates.size());
+
+    auto valid = [](auto index) { return index == 0 || index == 3; };
+    auto ok = std::all_of(begin(result.original_to_tidied), end(result.original_to_tidied), valid);
+    BOOST_CHECK(ok);
+
+    BOOST_CHECK(std::is_sorted(begin(result.original_to_tidied), end(result.original_to_tidied)));
+
+    const auto redundant = result.can_be_removed.count();
+    BOOST_CHECK_EQUAL(redundant, params.coordinates.size() - 2);
+
+    BOOST_CHECK_EQUAL(result.can_be_removed[0], false);
+    BOOST_CHECK_EQUAL(result.can_be_removed[3], false);
+
+    auto second_blob_start = std::partition_point(begin(result.original_to_tidied),
+                                                  end(result.original_to_tidied),
+                                                  [](auto index) { return index == 0; });
+
+    BOOST_CHECK(begin(result.original_to_tidied) < second_blob_start);
+    BOOST_CHECK(second_blob_start < end(result.original_to_tidied));
+    BOOST_CHECK(second_blob_start == begin(result.original_to_tidied) + 3); // 3 in first blob
+}
+
+BOOST_AUTO_TEST_CASE(two_blobs_in_traces_needs_tidiying_no_timestamps_test)
+{
+    MatchParameters params;
+
+    params.coordinates.emplace_back(FloatLongitude{13.207993}, FloatLatitude{52.446379});
+    params.coordinates.emplace_back(FloatLongitude{13.207994}, FloatLatitude{52.446380});
+    params.coordinates.emplace_back(FloatLongitude{13.207995}, FloatLatitude{52.446381});
+
+    params.coordinates.emplace_back(FloatLongitude{13.231658}, FloatLatitude{52.465416});
+    params.coordinates.emplace_back(FloatLongitude{13.231659}, FloatLatitude{52.465417});
+    params.coordinates.emplace_back(FloatLongitude{13.231660}, FloatLatitude{52.465417});
+
+    tidy::Thresholds thresholds;
+    thresholds.distance_in_meters = 15;
+    thresholds.duration_in_seconds = 5;
+
+    auto result = tidy::tidy(params, thresholds);
+
+    BOOST_CHECK_EQUAL(result.can_be_removed.size(), params.coordinates.size());
+    BOOST_CHECK_EQUAL(result.original_to_tidied.size(), params.coordinates.size());
+
+    auto valid = [](auto index) { return index == 0 || index == 3; };
+    auto ok = std::all_of(begin(result.original_to_tidied), end(result.original_to_tidied), valid);
+    BOOST_CHECK(ok);
+
+    BOOST_CHECK(std::is_sorted(begin(result.original_to_tidied), end(result.original_to_tidied)));
+
+    const auto redundant = result.can_be_removed.count();
+    BOOST_CHECK_EQUAL(redundant, params.coordinates.size() - 2);
+
+    BOOST_CHECK_EQUAL(result.can_be_removed[0], false);
+    BOOST_CHECK_EQUAL(result.can_be_removed[3], false);
+
+    auto second_blob_start = std::partition_point(begin(result.original_to_tidied),
+                                                  end(result.original_to_tidied),
+                                                  [](auto index) { return index == 0; });
+
+    BOOST_CHECK(begin(result.original_to_tidied) < second_blob_start);
+    BOOST_CHECK(second_blob_start < end(result.original_to_tidied));
+    BOOST_CHECK(second_blob_start == begin(result.original_to_tidied) + 3); // 3 in first blob
+}
+
+BOOST_AUTO_TEST_SUITE_END()