Bump version field and update CHANGELOG.

This change fixes two bugs:

1. A dead-lock that occurs between osrm-datastore and libosrm when an
   old dataset is free during a data update. This happened because the
   mutexes where acquired in a different order.

2. A region is deleted eventhough it is still in use. This happens when
   libosrm gets overtaken by osrm-datastore, so the new dataset is in
   the same region the old one was.

CHANGELOG.md

@@ -1,3 +1,8 @@
+# 5.5.1
+  - Changes from 5.5.0
+    - Bugfixes
+      - Fixes #3455 where a deadlock could occur if re-loading new data under heavy load with multiple consumers osrm-datastore
+
 # 5.5.0
   - Changes from 5.4.0
     - API:

CMakeLists.txt

@@ -53,7 +53,7 @@ endif()
 project(OSRM C CXX)
 set(OSRM_VERSION_MAJOR 5)
 set(OSRM_VERSION_MINOR 5)
-set(OSRM_VERSION_PATCH 0)
+set(OSRM_VERSION_PATCH 1)
 set(OSRM_VERSION "${OSRM_VERSION_MAJOR}.${OSRM_VERSION_MINOR}.${OSRM_VERSION_PATCH}")
 
 add_definitions(-DOSRM_PROJECT_DIR="${CMAKE_CURRENT_SOURCE_DIR}")

features/car/access.feature

@@ -220,3 +220,15 @@ Feature: Car - Restricted access
         Then routability should be
             | highway | toll        | bothw |
             | primary | yes         |       |
+
+    Scenario: Car - directional access tags
+        Then routability should be
+            | highway | access | access:forward | access:backward | forw | backw |
+            | primary | yes    | yes            | yes             | x    | x     |
+            | primary | yes    |                | no              | x    |       |
+            | primary | yes    | no             |                 |      | x     |
+            | primary | yes    | no             | no              |      |       |
+            | primary | no     | no             | no              |      |       |
+            | primary | no     |                | yes             |      | x     |
+            | primary | no     | yes            |                 | x    |       |
+            | primary | no     | yes            | yes             | x    | x     |

include/contractor/contractor.hpp

@@ -81,6 +81,7 @@ class Contractor
     EdgeID
     LoadEdgeExpandedGraph(const std::string &edge_based_graph_path,
                           util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list,
+                          std::vector<EdgeWeight> &node_weights,
                           const std::string &edge_segment_lookup_path,
                           const std::string &edge_penalty_path,
                           const std::vector<std::string> &segment_speed_path,

include/engine/datafacade/shared_memory_datafacade.hpp

@@ -38,29 +38,33 @@ class SharedMemoryDataFacade : public ContiguousInternalMemoryDataFacadeBase
     // used anymore.  We crash hard here if something goes wrong (noexcept).
     virtual ~SharedMemoryDataFacade() noexcept
     {
+        // Now check if this is still the newest dataset
+        boost::interprocess::sharable_lock<boost::interprocess::named_upgradable_mutex>
+            current_regions_lock(shared_barriers->current_regions_mutex,
+                                 boost::interprocess::defer_lock);
+
         boost::interprocess::scoped_lock<boost::interprocess::named_sharable_mutex> exclusive_lock(
             data_region == storage::DATA_1 ? shared_barriers->regions_1_mutex
                                            : shared_barriers->regions_2_mutex,
             boost::interprocess::defer_lock);
 
         // if this returns false this is still in use
-        if (exclusive_lock.try_lock())
+        if (current_regions_lock.try_lock() && exclusive_lock.try_lock())
         {
             if (storage::SharedMemory::RegionExists(data_region))
             {
                 BOOST_ASSERT(storage::SharedMemory::RegionExists(layout_region));
 
-                // Now check if this is still the newest dataset
-                const boost::interprocess::sharable_lock<boost::interprocess::named_upgradable_mutex>
-                    lock(shared_barriers->current_regions_mutex);
-
                 auto shared_regions = storage::makeSharedMemory(storage::CURRENT_REGIONS);
                 const auto current_timestamp =
                     static_cast<const storage::SharedDataTimestamp *>(shared_regions->Ptr());
 
-                if (current_timestamp->timestamp == shared_timestamp)
+                // check if the memory region referenced by this facade needs cleanup
+                if (current_timestamp->data == data_region)
                 {
-                    util::Log(logDEBUG) << "Retaining data with shared timestamp " << shared_timestamp;
+                    BOOST_ASSERT(current_timestamp->layout == layout_region);
+                    util::Log(logDEBUG) << "Retaining data with shared timestamp "
+                                        << shared_timestamp;
                 }
                 else
                 {

include/util/meminfo.hpp

@@ -0,0 +1,43 @@
+#ifndef MEMINFO_HPP
+#define MEMINFO_HPP
+
+#include "util/log.hpp"
+
+#include <stxxl/mng>
+#ifndef _WIN32
+#include <sys/resource.h>
+#endif
+
+namespace osrm
+{
+namespace util
+{
+inline void DumpMemoryStats()
+{
+#if STXXL_VERSION_MAJOR > 1 || (STXXL_VERSION_MAJOR == 1 && STXXL_VERSION_MINOR >= 4)
+    auto manager = stxxl::block_manager::get_instance();
+    util::Log() << "STXXL: peak bytes used: " << manager->get_maximum_allocation();
+    util::Log() << "STXXL: total disk allocated: " << manager->get_total_bytes();
+#else
+#warning STXXL 1.4+ recommended - STXXL memory summary will not be available
+    util::Log() << "STXXL: memory summary not available, needs STXXL 1.4 or higher";
+#endif
+
+#ifndef _WIN32
+    rusage usage;
+    getrusage(RUSAGE_SELF, &usage);
+#ifdef __linux__
+    // Under linux, ru.maxrss is in kb
+    util::Log() << "RAM: peak bytes used: " << usage.ru_maxrss * 1024;
+#else  // __linux__
+    // Under BSD systems (OSX), it's in bytes
+    util::Log() << "RAM: peak bytes used: " << usage.ru_maxrss;
+#endif // __linux__
+#else  // _WIN32
+    util::Log() << "RAM: peak bytes used: <not implemented on Windows>";
+#endif // _WIN32
+}
+}
+}
+
+#endif

src/contractor/contractor.cpp

@@ -136,12 +136,24 @@ int Contractor::Run()
 
     TIMER_START(preparing);
 
+    util::Log() << "Reading node weights.";
+    std::vector<EdgeWeight> node_weights;
+    std::string node_file_name = config.osrm_input_path.string() + ".enw";
+
+    {
+        storage::io::FileReader node_file(node_file_name,
+                                          storage::io::FileReader::VerifyFingerprint);
+        node_file.DeserializeVector(node_weights);
+    }
+    util::Log() << "Done reading node weights.";
+
     util::Log() << "Loading edge-expanded graph representation";
 
     util::DeallocatingVector<extractor::EdgeBasedEdge> edge_based_edge_list;
 
     EdgeID max_edge_id = LoadEdgeExpandedGraph(config.edge_based_graph_path,
                                                edge_based_edge_list,
+                                               node_weights,
                                                config.edge_segment_lookup_path,
                                                config.edge_penalty_path,
                                                config.segment_speed_lookup_paths,
@@ -163,17 +175,6 @@ int Contractor::Run()
         ReadNodeLevels(node_levels);
     }
 
-    util::Log() << "Reading node weights.";
-    std::vector<EdgeWeight> node_weights;
-    std::string node_file_name = config.osrm_input_path.string() + ".enw";
-
-    {
-        storage::io::FileReader node_file(node_file_name,
-                                          storage::io::FileReader::VerifyFingerprint);
-        node_file.DeserializeVector(node_weights);
-    }
-    util::Log() << "Done reading node weights.";
-
     util::DeallocatingVector<QueryEdge> contracted_edge_list;
     ContractGraph(max_edge_id,
                   edge_based_edge_list,
@@ -499,6 +500,7 @@ parse_turn_penalty_lookup_from_csv_files(const std::vector<std::string> &turn_pe
 EdgeID Contractor::LoadEdgeExpandedGraph(
     std::string const &edge_based_graph_filename,
     util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list,
+    std::vector<EdgeWeight> &node_weights,
     const std::string &edge_segment_lookup_filename,
     const std::string &edge_penalty_filename,
     const std::vector<std::string> &segment_speed_filenames,
@@ -914,6 +916,11 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
                 previous_osm_node_id = segmentblocks[i].this_osm_node_id;
             }
 
+            // Update the node-weight cache.  This is the weight of the edge-based-node only,
+            // it doesn't include the turn.  We may visit the same node multiple times, but
+            // we should always assign the same value here.
+            node_weights[inbuffer.source] = new_weight;
+
             // We found a zero-speed edge, so we'll skip this whole edge-based-edge which
             // effectively removes it from the routing network.
             if (skip_this_edge)

src/extractor/guidance/coordinate_extractor.cpp

@@ -132,7 +132,7 @@ util::Coordinate CoordinateExtractor::ExtractRepresentativeCoordinate(
     {
         const auto result = ExtractCoordinateAtLength(
             skipping_inaccuracies_distance, coordinates);
-        BOOST_ASSERT(is_valid_result(coordinates.back()));
+        BOOST_ASSERT(is_valid_result(result));
         return result;
     }
 

src/extractor/guidance/sliproad_handler.cpp

@@ -186,11 +186,9 @@ operator()(const NodeID /*nid*/, const EdgeID source_edge_id, Intersection inter
             if (is_left_sliproad_turn)
                 return main_road_intersection->intersection.getLeftmostRoad();
 
-            if (is_right_sliproad_turn)
-                return main_road_intersection->intersection.getRightmostRoad();
-
-            BOOST_ASSERT_MSG(false, "Sliproad is neither a left nor right of obvious main road");
-            return main_road_intersection->intersection.getLeftmostRoad();
+            BOOST_ASSERT_MSG(is_right_sliproad_turn,
+                             "Sliproad is neither a left nor right of obvious main road");
+            return main_road_intersection->intersection.getRightmostRoad();
         }();
 
         const auto &crossing_road_data = node_based_graph.GetEdgeData(crossing_road.eid);

src/tools/contract.cpp

@@ -15,6 +15,8 @@
 #include <new>
 #include <ostream>
 
+#include "util/meminfo.hpp"
+
 using namespace osrm;
 
 enum class return_code : unsigned
@@ -166,7 +168,11 @@ int main(int argc, char *argv[]) try
 
     tbb::task_scheduler_init init(contractor_config.requested_num_threads);
 
-    return contractor::Contractor(contractor_config).Run();
+    auto exitcode = contractor::Contractor(contractor_config).Run();
+
+    util::DumpMemoryStats();
+
+    return exitcode;
 }
 catch (const std::bad_alloc &e)
 {

src/tools/extract.cpp

@@ -13,6 +13,8 @@
 #include <exception>
 #include <new>
 
+#include "util/meminfo.hpp"
+
 using namespace osrm;
 
 enum class return_code : unsigned
@@ -153,7 +155,11 @@ int main(int argc, char *argv[]) try
     // setup scripting environment
     extractor::LuaScriptingEnvironment scripting_environment(
         extractor_config.profile_path.string().c_str());
-    return extractor::Extractor(extractor_config).run(scripting_environment);
+    auto exitcode = extractor::Extractor(extractor_config).run(scripting_environment);
+
+    util::DumpMemoryStats();
+
+    return exitcode;
 }
 catch (const std::bad_alloc &e)
 {