Update changelog

This fixes issue #3016.

CHANGELOG.md

@@ -1,3 +1,35 @@
+# 5.4.1
+  - Changes from 5.4.0
+    - Bugfixes
+      - #3016: Fixes shared memory updates while queries are running
+
+# 5.4.0
+  - Changes from 5.3.0
+    - Profiles
+      - includes library guidance.lua that offers preliminary configuration on guidance.
+      - added left_hand_driving flag in global profile properties
+      - modified turn penalty function for car profile - better fit to real data
+      - return `ref` and `name` as separate fields. Do no use ref or destination as fallback for name value
+    - Guidance
+      - Handle Access tags for lanes, only considering valid lanes in lane-guidance (think car | car | bike | car)
+    - API:
+      - `annotations=true` now returns the data source id for each segment as `datasources`
+      - Reduced semantic of merge to refer only to merges from a lane onto a motorway-like road
+      - new `ref` field in the `RouteStep` object. It contains the reference code or name of a way. Previously merged into the `name` property like `name (ref)` and are now separate fields.
+    - Bugfixes
+      - Fixed an issue that would result in segfaults for viaroutes with an invalid intermediate segment when u-turns were allowed at the via-location
+      - Invalid only_* restrictions could result in loss of connectivity. As a fallback, we assume all turns allowed when the restriction is not valid
+      - Fixed a bug that could result in an infinite loop when finding information about an upcoming intersection
+      - Fixed a bug that led to not discovering if a road simply looses a considered prefix
+      - BREAKING: Fixed a bug that could crash postprocessing of instructions on invalid roundabout taggings. This change requires reprocessing datasets with osrm-extract and osrm-contract
+      - Fixed an issue that could emit `invalid` as instruction when ending on a sliproad after a traffic-light
+      - Fixed an issue that would detect turning circles as sliproads
+      - Fixed a bug where post-processing instructions (e.g. left + left -> uturn) could result in false pronunciations
+      - Fixes a bug where a bearing range of zero would cause exhaustive graph traversals
+      - Fixes a bug where certain looped geometries could cause an infinite loop during extraction
+    - Infrastructure:
+      - Adds a feature to limit results in nearest service with a default of 100 in `osrm-routed`
+
 # 5.4.0-rc.7
   - Chages from 5.4.0-rc.6
     - Bugfixes re-introduce space between two entries in summaries
@@ -33,34 +65,6 @@
     - Trip Plugin
       - changed internal behaviour to prefer the smallest lexicographic result over the largest one
 
-# 5.4.0
-  - Changes from 5.3.0
-    - Profiles
-      - includes library guidance.lua that offers preliminary configuration on guidance.
-      - added left_hand_driving flag in global profile properties
-      - modified turn penalty function for car profile - better fit to real data
-      - return `ref` and `name` as separate fields. Do no use ref or destination as fallback for name value
-    - Guidance
-      - Handle Access tags for lanes, only considering valid lanes in lane-guidance (think car | car | bike | car)
-    - API:
-      - `annotations=true` now returns the data source id for each segment as `datasources`
-      - Reduced semantic of merge to refer only to merges from a lane onto a motorway-like road
-      - new `ref` field in the `RouteStep` object. It contains the reference code or name of a way. Previously merged into the `name` property like `name (ref)` and are now separate fields.
-    - Bugfixes
-      - Fixed an issue that would result in segfaults for viaroutes with an invalid intermediate segment when u-turns were allowed at the via-location
-      - Invalid only_* restrictions could result in loss of connectivity. As a fallback, we assume all turns allowed when the restriction is not valid
-      - Fixed a bug that could result in an infinite loop when finding information about an upcoming intersection
-      - Fixed a bug that led to not discovering if a road simply looses a considered prefix
-      - BREAKING: Fixed a bug that could crash postprocessing of instructions on invalid roundabout taggings. This change requires reprocessing datasets with osrm-extract and osrm-contract
-      - Fixed an issue that could emit `invalid` as instruction when ending on a sliproad after a traffic-light
-      - Fixed an issue that would detect turning circles as sliproads
-      - Fixed a bug where post-processing instructions (e.g. left + left -> uturn) could result in false pronunciations
-      - Fixes a bug where a bearing range of zero would cause exhaustive graph traversals
-      - Fixes a bug where certain looped geometries could cause an infinite loop during extraction
-
-    - Infrastructure:
-      - Adds a feature to limit results in nearest service with a default of 100 in `osrm-routed`
-
 # 5.3.0
   - Changes from 5.3.0-rc.3
     - Guidance

CMakeLists.txt

@@ -127,8 +127,8 @@ if(ENABLE_GOLD_LINKER)
 
         # Issue 2785: check gold binutils version and don't use gc-sections for versions prior 2.25
         string(REGEX REPLACE ".*\\(GNU Binutils[^\\)0-9]+([0-9]+\\.[0-9]+)[^\\)]*\\).*" "\\1" GOLD_BINUTILS_VERSION "${LD_VERSION}")
-        if ("${GOLD_BINUTILS_VERSION}" VERSION_LESS "2.25")
+        if ("${GOLD_BINUTILS_VERSION}" VERSION_LESS "2.26")
-          message(STATUS "Disabling gc-sections on gold binutils < 2.25, see: https://sourceware.org/bugzilla/show_bug.cgi?id=17639")
+          message(STATUS "Disabling gc-sections on gold binutils < 2.26, see: https://sourceware.org/bugzilla/show_bug.cgi?id=17639")
           set(LD_AVOID_GC_SECTIONS TRUE)
         endif()
     else()

example/example.cpp

@@ -34,7 +34,7 @@ int main(int argc, const char *argv[])
     config.use_shared_memory = false;
 
     // Routing machine with several services (such as Route, Table, Nearest, Trip, Match)
-    OSRM osrm{config};
+    const OSRM osrm{config};
 
     // The following shows how to use the Route service; configure this service
     RouteParameters params;

features/support/env.js

@@ -51,8 +51,26 @@ module.exports = function () {
         } else {
             this.TERMSIGNAL = 'SIGTERM';
             this.EXE = '';
-            // TODO autodetect if this was build with shared or static libraries
+
-            this.LIB = process.env.BUILD_SHARED_LIBS && '.so' || '.a';
+            // heuristically detect .so/.a suffix
+            this.LIB = null;
+
+            try {
+                const dot_a = util.format('%s/libosrm%s', this.BIN_PATH, '.a');
+                fs.accessSync(dot_a, fs.F_OK);
+                this.LIB = '.a';
+            } catch(e) { /*nop*/ }
+
+            try {
+                const dot_so = util.format('%s/libosrm%s', this.BIN_PATH, '.so');
+                fs.accessSync(dot_so, fs.F_OK);
+                this.LIB = '.so';
+            } catch(e) { /*nop*/ }
+
+            if (!this.LIB) {
+                throw new Error('*** Unable to detect dynamic or static libosrm libraries');
+            }
+
             this.QQ = '';
         }
 
@@ -65,7 +83,7 @@ module.exports = function () {
 
         // eslint-disable-next-line no-console
         console.info(util.format('Node Version', process.version));
-        if (parseInt(process.version.match(/v(\d)/)[1]) < 4) throw new Error('*** PLease upgrade to Node 4.+ to run OSRM cucumber tests');
+        if (parseInt(process.version.match(/v(\d)/)[1]) < 4) throw new Error('*** Please upgrade to Node 4.+ to run OSRM cucumber tests');
 
         fs.exists(this.TEST_PATH, (exists) => {
             if (exists)

include/engine/api/route_parameters.hpp

@@ -69,6 +69,21 @@ struct RouteParameters : public BaseParameters
 
     RouteParameters() = default;
 
+    template <typename... Args>
+    RouteParameters(const bool steps_,
+                    const bool alternatives_,
+                    const GeometriesType geometries_,
+                    const OverviewType overview_,
+                    const boost::optional<bool> continue_straight_,
+                    Args... args_)
+        : BaseParameters{std::forward<Args>(args_)...}, steps{steps_}, alternatives{alternatives_},
+          annotations{false}, geometries{geometries_}, overview{overview_},
+          continue_straight{continue_straight_}
+    // Once we perfectly-forward `args` (see #2990) this constructor can delegate to the one below.
+    {
+    }
+
+    // RouteParameters constructor adding the `annotations` setting in a API-compatible way.
     template <typename... Args>
     RouteParameters(const bool steps_,
                     const bool alternatives_,

include/engine/engine.hpp

@@ -1,7 +1,6 @@
 #ifndef ENGINE_HPP
 #define ENGINE_HPP
 
-#include "storage/shared_barriers.hpp"
 #include "engine/status.hpp"
 #include "util/json_container.hpp"
 
@@ -20,6 +19,11 @@ struct Object;
 }
 }
 
+namespace storage
+{
+struct SharedBarriers;
+}
+
 // Fwd decls
 namespace engine
 {
@@ -52,10 +56,7 @@ class BaseDataFacade;
 class Engine final
 {
   public:
-    // Needs to be public
+    explicit Engine(const EngineConfig &config);
-    struct EngineLock;
-
-    explicit Engine(EngineConfig &config);
 
     Engine(Engine &&) noexcept;
     Engine &operator=(Engine &&) noexcept;
@@ -63,15 +64,15 @@ class Engine final
     // Impl. in cpp since for unique_ptr of incomplete types
     ~Engine();
 
-    Status Route(const api::RouteParameters &parameters, util::json::Object &result);
+    Status Route(const api::RouteParameters &parameters, util::json::Object &result) const;
-    Status Table(const api::TableParameters &parameters, util::json::Object &result);
+    Status Table(const api::TableParameters &parameters, util::json::Object &result) const;
-    Status Nearest(const api::NearestParameters &parameters, util::json::Object &result);
+    Status Nearest(const api::NearestParameters &parameters, util::json::Object &result) const;
-    Status Trip(const api::TripParameters &parameters, util::json::Object &result);
+    Status Trip(const api::TripParameters &parameters, util::json::Object &result) const;
-    Status Match(const api::MatchParameters &parameters, util::json::Object &result);
+    Status Match(const api::MatchParameters &parameters, util::json::Object &result) const;
-    Status Tile(const api::TileParameters &parameters, std::string &result);
+    Status Tile(const api::TileParameters &parameters, std::string &result) const;
 
   private:
-    std::unique_ptr<EngineLock> lock;
+    std::unique_ptr<storage::SharedBarriers> lock;
 
     std::unique_ptr<plugins::ViaRoutePlugin> route_plugin;
     std::unique_ptr<plugins::TablePlugin> table_plugin;

include/osrm/osrm.hpp

@@ -83,7 +83,7 @@ class OSRM final
      * \return Status indicating success for the query or failure
      * \see Status, RouteParameters and json::Object
      */
-    Status Route(const RouteParameters &parameters, json::Object &result);
+    Status Route(const RouteParameters &parameters, json::Object &result) const;
 
     /**
      * Distance tables for coordinates.
@@ -92,7 +92,7 @@ class OSRM final
      * \return Status indicating success for the query or failure
      * \see Status, TableParameters and json::Object
      */
-    Status Table(const TableParameters &parameters, json::Object &result);
+    Status Table(const TableParameters &parameters, json::Object &result) const;
 
     /**
      * Nearest street segment for coordinate.
@@ -101,7 +101,7 @@ class OSRM final
      * \return Status indicating success for the query or failure
      * \see Status, NearestParameters and json::Object
      */
-    Status Nearest(const NearestParameters &parameters, json::Object &result);
+    Status Nearest(const NearestParameters &parameters, json::Object &result) const;
 
     /**
      * Trip: shortest round trip between coordinates.
@@ -110,7 +110,7 @@ class OSRM final
      * \return Status indicating success for the query or failure
      * \see Status, TripParameters and json::Object
      */
-    Status Trip(const TripParameters &parameters, json::Object &result);
+    Status Trip(const TripParameters &parameters, json::Object &result) const;
 
     /**
      * Match: snaps noisy coordinate traces to the road network
@@ -119,7 +119,7 @@ class OSRM final
      * \return Status indicating success for the query or failure
      * \see Status, MatchParameters and json::Object
      */
-    Status Match(const MatchParameters &parameters, json::Object &result);
+    Status Match(const MatchParameters &parameters, json::Object &result) const;
 
     /**
      * Tile: vector tiles with internal graph representation
@@ -128,7 +128,7 @@ class OSRM final
      * \return Status indicating success for the query or failure
      * \see Status, TileParameters and json::Object
      */
-    Status Tile(const TileParameters &parameters, std::string &result);
+    Status Tile(const TileParameters &parameters, std::string &result) const;
 
   private:
     std::unique_ptr<engine::Engine> engine_;

include/storage/shared_barriers.hpp

@@ -3,6 +3,7 @@
 
 #include <boost/interprocess/sync/named_condition.hpp>
 #include <boost/interprocess/sync/named_mutex.hpp>
+#include <boost/interprocess/sync/named_sharable_mutex.hpp>
 
 namespace osrm
 {
@@ -13,25 +14,13 @@ struct SharedBarriers
 
     SharedBarriers()
         : pending_update_mutex(boost::interprocess::open_or_create, "pending_update"),
-          update_mutex(boost::interprocess::open_or_create, "update"),
+          query_mutex(boost::interprocess::open_or_create, "query")
-          query_mutex(boost::interprocess::open_or_create, "query"),
-          no_running_queries_condition(boost::interprocess::open_or_create, "no_running_queries"),
-          update_ongoing(false), number_of_queries(0)
     {
     }
 
     // Mutex to protect access to the boolean variable
     boost::interprocess::named_mutex pending_update_mutex;
-    boost::interprocess::named_mutex update_mutex;
+    boost::interprocess::named_sharable_mutex query_mutex;
-    boost::interprocess::named_mutex query_mutex;
-
-    // Condition that no update is running
-    boost::interprocess::named_condition no_running_queries_condition;
-
-    // Is there an ongoing update?
-    bool update_ongoing;
-    // Is there any query?
-    int number_of_queries;
 };
 }
 }

src/engine/engine.cpp

@@ -1,5 +1,5 @@
-#include "engine/engine.hpp"
 #include "engine/api/route_parameters.hpp"
+#include "engine/engine.hpp"
 #include "engine/engine_config.hpp"
 #include "engine/status.hpp"
 
@@ -21,6 +21,7 @@
 #include <boost/assert.hpp>
 #include <boost/interprocess/sync/named_condition.hpp>
 #include <boost/interprocess/sync/scoped_lock.hpp>
+#include <boost/interprocess/sync/sharable_lock.hpp>
 #include <boost/thread/lock_types.hpp>
 
 #include <algorithm>
@@ -28,68 +29,17 @@
 #include <utility>
 #include <vector>
 
-namespace osrm
-{
-namespace engine
-{
-struct Engine::EngineLock
-{
-    // will only be initialized if shared memory is used
-    storage::SharedBarriers barrier;
-    // decrease number of concurrent queries
-    void DecreaseQueryCount();
-    // increase number of concurrent queries
-    void IncreaseQueryCount();
-};
-
-// decrease number of concurrent queries
-void Engine::EngineLock::DecreaseQueryCount()
-{
-    // lock query
-    boost::interprocess::scoped_lock<boost::interprocess::named_mutex> query_lock(
-        barrier.query_mutex);
-
-    // decrement query count
-    --(barrier.number_of_queries);
-    BOOST_ASSERT_MSG(0 <= barrier.number_of_queries, "invalid number of queries");
-
-    // notify all processes that were waiting for this condition
-    if (0 == barrier.number_of_queries)
-    {
-        barrier.no_running_queries_condition.notify_all();
-    }
-}
-
-// increase number of concurrent queries
-void Engine::EngineLock::IncreaseQueryCount()
-{
-    // lock update pending
-    boost::interprocess::scoped_lock<boost::interprocess::named_mutex> pending_lock(
-        barrier.pending_update_mutex);
-
-    // lock query
-    boost::interprocess::scoped_lock<boost::interprocess::named_mutex> query_lock(
-        barrier.query_mutex);
-
-    // unlock update pending
-    pending_lock.unlock();
-
-    // increment query count
-    ++(barrier.number_of_queries);
-}
-} // ns engine
-} // ns osrm
-
 namespace
 {
 // Abstracted away the query locking into a template function
 // Works the same for every plugin.
 template <typename ParameterT, typename PluginT, typename ResultT>
-osrm::engine::Status RunQuery(const std::unique_ptr<osrm::engine::Engine::EngineLock> &lock,
+osrm::engine::Status
-                              osrm::engine::datafacade::BaseDataFacade &facade,
+RunQuery(const std::unique_ptr<osrm::storage::SharedBarriers> &lock,
-                              const ParameterT &parameters,
+         osrm::engine::datafacade::BaseDataFacade &facade,
-                              PluginT &plugin,
+         const ParameterT &parameters,
-                              ResultT &result)
+         PluginT &plugin,
+         ResultT &result)
 {
     if (!lock)
     {
@@ -97,7 +47,10 @@ osrm::engine::Status RunQuery(const std::unique_ptr<osrm::engine::Engine::Engine
     }
 
     BOOST_ASSERT(lock);
-    lock->IncreaseQueryCount();
+    // this locks aquires shared ownership of the query mutex: other requets are allowed
+    // to run, but data updates need to wait for all queries to finish until they can aquire an exclusive lock
+    boost::interprocess::sharable_lock<boost::interprocess::named_sharable_mutex> query_lock(
+        lock->query_mutex);
 
     auto &shared_facade = static_cast<osrm::engine::datafacade::SharedDataFacade &>(facade);
     shared_facade.CheckAndReloadFacade();
@@ -107,7 +60,6 @@ osrm::engine::Status RunQuery(const std::unique_ptr<osrm::engine::Engine::Engine
 
     osrm::engine::Status status = plugin.HandleRequest(parameters, result);
 
-    lock->DecreaseQueryCount();
     return status;
 }
 
@@ -124,11 +76,12 @@ namespace osrm
 namespace engine
 {
 
-Engine::Engine(EngineConfig &config)
+Engine::Engine(const EngineConfig &config)
+    : lock(config.use_shared_memory ? std::make_unique<storage::SharedBarriers>()
+                                    : std::unique_ptr<storage::SharedBarriers>())
 {
     if (config.use_shared_memory)
     {
-        lock = util::make_unique<EngineLock>();
         query_data_facade = util::make_unique<datafacade::SharedDataFacade>();
     }
     else
@@ -157,32 +110,32 @@ Engine::~Engine() = default;
 Engine::Engine(Engine &&) noexcept = default;
 Engine &Engine::operator=(Engine &&) noexcept = default;
 
-Status Engine::Route(const api::RouteParameters &params, util::json::Object &result)
+Status Engine::Route(const api::RouteParameters &params, util::json::Object &result) const
 {
     return RunQuery(lock, *query_data_facade, params, *route_plugin, result);
 }
 
-Status Engine::Table(const api::TableParameters &params, util::json::Object &result)
+Status Engine::Table(const api::TableParameters &params, util::json::Object &result) const
 {
     return RunQuery(lock, *query_data_facade, params, *table_plugin, result);
 }
 
-Status Engine::Nearest(const api::NearestParameters &params, util::json::Object &result)
+Status Engine::Nearest(const api::NearestParameters &params, util::json::Object &result) const
 {
     return RunQuery(lock, *query_data_facade, params, *nearest_plugin, result);
 }
 
-Status Engine::Trip(const api::TripParameters &params, util::json::Object &result)
+Status Engine::Trip(const api::TripParameters &params, util::json::Object &result) const
 {
     return RunQuery(lock, *query_data_facade, params, *trip_plugin, result);
 }
 
-Status Engine::Match(const api::MatchParameters &params, util::json::Object &result)
+Status Engine::Match(const api::MatchParameters &params, util::json::Object &result) const
 {
     return RunQuery(lock, *query_data_facade, params, *match_plugin, result);
 }
 
-Status Engine::Tile(const api::TileParameters &params, std::string &result)
+Status Engine::Tile(const api::TileParameters &params, std::string &result) const
 {
     return RunQuery(lock, *query_data_facade, params, *tile_plugin, result);
 }

src/osrm/osrm.cpp

@@ -21,32 +21,32 @@ OSRM &OSRM::operator=(OSRM &&) noexcept = default;
 
 // Forward to implementation
 
-engine::Status OSRM::Route(const engine::api::RouteParameters &params, util::json::Object &result)
+engine::Status OSRM::Route(const engine::api::RouteParameters &params, util::json::Object &result) const
 {
     return engine_->Route(params, result);
 }
 
-engine::Status OSRM::Table(const engine::api::TableParameters &params, json::Object &result)
+engine::Status OSRM::Table(const engine::api::TableParameters &params, json::Object &result) const
 {
     return engine_->Table(params, result);
 }
 
-engine::Status OSRM::Nearest(const engine::api::NearestParameters &params, json::Object &result)
+engine::Status OSRM::Nearest(const engine::api::NearestParameters &params, json::Object &result) const
 {
     return engine_->Nearest(params, result);
 }
 
-engine::Status OSRM::Trip(const engine::api::TripParameters &params, json::Object &result)
+engine::Status OSRM::Trip(const engine::api::TripParameters &params, json::Object &result) const
 {
     return engine_->Trip(params, result);
 }
 
-engine::Status OSRM::Match(const engine::api::MatchParameters &params, json::Object &result)
+engine::Status OSRM::Match(const engine::api::MatchParameters &params, json::Object &result) const
 {
     return engine_->Match(params, result);
 }
 
-engine::Status OSRM::Tile(const engine::api::TileParameters &params, std::string &result)
+engine::Status OSRM::Tile(const engine::api::TileParameters &params, std::string &result) const
 {
     return engine_->Tile(params, result);
 }

src/storage/storage.cpp

@@ -42,11 +42,6 @@ namespace osrm
 namespace storage
 {
 
-using RTreeLeaf = engine::datafacade::BaseDataFacade::RTreeLeaf;
-using RTreeNode =
-    util::StaticRTree<RTreeLeaf, util::ShM<util::Coordinate, true>::vector, true>::TreeNode;
-using QueryGraph = util::StaticGraph<contractor::QueryEdge::EdgeData>;
-
 // delete a shared memory region. report warning if it could not be deleted
 void deleteRegion(const SharedDataType region)
 {
@@ -76,6 +71,11 @@ void deleteRegion(const SharedDataType region)
     }
 }
 
+using RTreeLeaf = engine::datafacade::BaseDataFacade::RTreeLeaf;
+using RTreeNode =
+    util::StaticRTree<RTreeLeaf, util::ShM<util::Coordinate, true>::vector, true>::TreeNode;
+using QueryGraph = util::StaticGraph<contractor::QueryEdge::EdgeData>;
+
 Storage::Storage(StorageConfig config_) : config(std::move(config_)) {}
 
 int Storage::Run()
@@ -738,20 +738,15 @@ int Storage::Run()
     SharedDataTimestamp *data_timestamp_ptr =
         static_cast<SharedDataTimestamp *>(data_type_memory->Ptr());
 
-    boost::interprocess::scoped_lock<boost::interprocess::named_mutex> query_lock(
-        barrier.query_mutex);
-
-    // notify all processes that were waiting for this condition
-    if (0 < barrier.number_of_queries)
     {
-        barrier.no_running_queries_condition.wait(query_lock);
+        boost::interprocess::scoped_lock<boost::interprocess::named_sharable_mutex> query_lock(barrier.query_mutex);
-    }
 
-    data_timestamp_ptr->layout = layout_region;
+        data_timestamp_ptr->layout = layout_region;
-    data_timestamp_ptr->data = data_region;
+        data_timestamp_ptr->data = data_region;
-    data_timestamp_ptr->timestamp += 1;
+        data_timestamp_ptr->timestamp += 1;
-    deleteRegion(previous_data_region);
+        deleteRegion(previous_data_region);
-    deleteRegion(previous_layout_region);
+        deleteRegion(previous_layout_region);
+    }
     util::SimpleLogger().Write() << "all data loaded";
 
     return EXIT_SUCCESS;

src/tools/unlock_all_mutexes.cpp

@@ -10,6 +10,5 @@ int main()
     osrm::storage::SharedBarriers barrier;
     barrier.pending_update_mutex.unlock();
     barrier.query_mutex.unlock();
-    barrier.update_mutex.unlock();
     return 0;
 }