update changelog for 5.17.2

* Do not combine a segregated edge with a roundabout, add test

.travis.yml

@@ -15,6 +15,7 @@ branches:
     - master
     # enable building tags
     - /^v\d+\.\d+(\.\d+)?(-\S*)?$/
+    - "5.17"
 
 cache:
   yarn: true

CHANGELOG.md

@@ -1,4 +1,9 @@
-# UNRELEASED
+# 5.17.2
+  - Changes from 5.17.0:
+    - Bugfixes:
+      - FIXED: Do not combine a segregated edge with a roundabout [#5039](https://github.com/Project-OSRM/osrm-backend/issues/5039)
+
+# 5.17.0
   - Changes from 5.16.0:
     - Bugfixes:
       - FIXED: deduplication of route steps when waypoints are used [#4909](https://github.com/Project-OSRM/osrm-backend/issues/4909)
@@ -6,9 +11,11 @@
       - FIXED: Remove the last short annotation segment in `trimShortSegments` [#4946](https://github.com/Project-OSRM/osrm-backend/pull/4946)
       - FIXED: Properly calculate annotations for speeds, durations and distances when waypoints are used with mapmatching [#4949](https://github.com/Project-OSRM/osrm-backend/pull/4949)
       - FIXED: Don't apply unimplemented SH and PH conditions in OpeningHours and add inversed date ranges [#4992](https://github.com/Project-OSRM/osrm-backend/issues/4992)
+      - FIXED: integer overflow in `DynamicGraph::Renumber` [#5021](https://github.com/Project-OSRM/osrm-backend/pull/5021)
     - Profile:
       - CHANGED: Handle oneways in get_forward_backward_by_key [#4929](https://github.com/Project-OSRM/osrm-backend/pull/4929)
       - FIXED: Do not route against oneway road if there is a cycleway in the wrong direction; also review bike profile [#4943](https://github.com/Project-OSRM/osrm-backend/issues/4943)
+      - CHANGED: Make cyclability weighting of the bike profile prefer safer routes more strongly [#5015](https://github.com/Project-OSRM/osrm-backend/issues/5015)
     - Guidance:
       - CHANGED: Don't use obviousness for links bifurcations [#4929](https://github.com/Project-OSRM/osrm-backend/pull/4929)
       - FIXED: Adjust Straight direction modifiers of side roads in driveway handler [#4929](https://github.com/Project-OSRM/osrm-backend/pull/4929)

features/bicycle/safety.feature

@@ -11,11 +11,11 @@ Feature: Bicycle - Adds penalties to unsafe roads
         Then routability should be
             | highway        | cycleway | forw       | backw      | forw_rate | backw_rate |
             | motorway       |          |            |            |           |            |
-            | primary        |          | 15 km/h    | 15 km/h    |       2.9 |        2.9 |
+            | primary        |          | 15 km/h    | 15 km/h    |       2.1 |        2.1 |
-            | secondary      |          | 15 km/h    | 15 km/h    |       3.1 |        3.1 |
+            | secondary      |          | 15 km/h    | 15 km/h    |       2.7 |        2.7 |
             | tertiary       |          | 15 km/h    | 15 km/h    |       3.3 |        3.3 |
-            | primary_link   |          | 15 km/h    | 15 km/h    |       2.9 |        2.9 |
+            | primary_link   |          | 15 km/h    | 15 km/h    |       2.1 |        2.1 |
-            | secondary_link |          | 15 km/h    | 15 km/h    |       3.1 |        3.1 |
+            | secondary_link |          | 15 km/h    | 15 km/h    |       2.7 |        2.7 |
             | tertiary_link  |          | 15 km/h    | 15 km/h    |       3.3 |        3.3 |
             | residential    |          | 15 km/h    | 15 km/h    |       4.2 |        4.2 |
             | cycleway       |          | 15 km/h    | 15 km/h    |       4.2 |        4.2 |
@@ -51,49 +51,49 @@ Feature: Bicycle - Adds penalties to unsafe roads
         Then routability should be
             | highway        | cycleway:right | cycleway:left | forw        | backw       | forw_rate | backw_rate |
             | motorway       | track          |               | 15 km/h     |             |       4.2 |            |
-            | primary        | track          |               | 15 km/h     | 15 km/h     |       4.2 |        2.9 |
+            | primary        | track          |               | 15 km/h     | 15 km/h     |       4.2 |        2.1 |
-            | secondary      | track          |               | 15 km/h     | 15 km/h     |       4.2 |        3.1 |
+            | secondary      | track          |               | 15 km/h     | 15 km/h     |       4.2 |        2.7 |
             | tertiary       | track          |               | 15 km/h     | 15 km/h     |       4.2 |        3.3 |
-            | primary_link   | track          |               | 15 km/h     | 15 km/h     |       4.2 |        2.9 |
+            | primary_link   | track          |               | 15 km/h     | 15 km/h     |       4.2 |        2.1 |
-            | secondary_link | track          |               | 15 km/h     | 15 km/h     |       4.2 |        3.1 |
+            | secondary_link | track          |               | 15 km/h     | 15 km/h     |       4.2 |        2.7 |
             | tertiary_link  | track          |               | 15 km/h     | 15 km/h     |       4.2 |        3.3 |
             | residential    | track          |               | 15 km/h     | 15 km/h     |       4.2 |        4.2 |
             | cycleway       | track          |               | 15 km/h     | 15 km/h     |       4.2 |        4.2 |
             | footway        | track          |               | 15 km/h     | 4 km/h +-1  |       4.2 |        1.1 |
             | motorway       |                | track         | 15 km/h     |             |       4.2 |            |
-            | primary        |                | track         | 15 km/h     | 15 km/h     |       2.9 |        4.2 |
+            | primary        |                | track         | 15 km/h     | 15 km/h     |       2.1 |        4.2 |
-            | secondary      |                | track         | 15 km/h     | 15 km/h     |       3.1 |        4.2 |
+            | secondary      |                | track         | 15 km/h     | 15 km/h     |       2.7 |        4.2 |
             | tertiary       |                | track         | 15 km/h     | 15 km/h     |       3.3 |        4.2 |
-            | primary_link   |                | track         | 15 km/h     | 15 km/h     |       2.9 |        4.2 |
+            | primary_link   |                | track         | 15 km/h     | 15 km/h     |       2.1 |        4.2 |
-            | secondary_link |                | track         | 15 km/h     | 15 km/h     |       3.1 |        4.2 |
+            | secondary_link |                | track         | 15 km/h     | 15 km/h     |       2.7 |        4.2 |
             | tertiary_link  |                | track         | 15 km/h     | 15 km/h     |       3.3 |        4.2 |
             | residential    |                | track         | 15 km/h     | 15 km/h     |       4.2 |        4.2 |
             | cycleway       |                | track         | 15 km/h     | 15 km/h     |       4.2 |        4.2 |
             | footway        |                | track         | 4 km/h +-1  | 15 km/h     |       1.1 |        4.2 |
             | motorway       | lane           |               | 15 km/h     |             |       4.2 |            |
-            | primary        | lane           |               | 15 km/h     | 15 km/h     |       4.2 |        2.9 |
+            | primary        | lane           |               | 15 km/h     | 15 km/h     |       4.2 |        2.1 |
-            | secondary      | lane           |               | 15 km/h     | 15 km/h     |       4.2 |        3.1 |
+            | secondary      | lane           |               | 15 km/h     | 15 km/h     |       4.2 |        2.7 |
             | tertiary       | lane           |               | 15 km/h     | 15 km/h     |       4.2 |        3.3 |
-            | primary_link   | lane           |               | 15 km/h     | 15 km/h     |       4.2 |        2.9 |
+            | primary_link   | lane           |               | 15 km/h     | 15 km/h     |       4.2 |        2.1 |
-            | secondary_link | lane           |               | 15 km/h     | 15 km/h     |       4.2 |        3.1 |
+            | secondary_link | lane           |               | 15 km/h     | 15 km/h     |       4.2 |        2.7 |
             | tertiary_link  | lane           |               | 15 km/h     | 15 km/h     |       4.2 |        3.3 |
             | residential    | lane           |               | 15 km/h +-1 | 15 km/h +-1 |       4.2 |        4.2 |
             | cycleway       | lane           |               | 15 km/h     | 15 km/h     |       4.2 |        4.2 |
             | footway        | lane           |               | 15 km/h     | 4 km/h +-1  |       4.2 |        1.1 |
             | motorway       |                | lane          | 15 km/h     |             |       4.2 |            |
-            | primary        |                | lane          | 15 km/h     | 15 km/h     |       2.9 |        4.2 |
+            | primary        |                | lane          | 15 km/h     | 15 km/h     |       2.1 |        4.2 |
-            | secondary      |                | lane          | 15 km/h +-1 | 15 km/h +-1 |       3.1 |        4.2 |
+            | secondary      |                | lane          | 15 km/h +-1 | 15 km/h +-1 |       2.7 |        4.2 |
             | tertiary       |                | lane          | 15 km/h     | 15 km/h     |       3.3 |        4.2 |
-            | primary_link   |                | lane          | 15 km/h     | 15 km/h     |       2.9 |        4.2 |
+            | primary_link   |                | lane          | 15 km/h     | 15 km/h     |       2.1 |        4.2 |
-            | secondary_link |                | lane          | 15 km/h     | 15 km/h     |       3.1 |        4.2 |
+            | secondary_link |                | lane          | 15 km/h     | 15 km/h     |       2.7 |        4.2 |
             | tertiary_link  |                | lane          | 15 km/h     | 15 km/h     |       3.3 |        4.2 |
             | residential    |                | lane          | 15 km/h     | 15 km/h     |       4.2 |        4.2 |
             | cycleway       |                | lane          | 15 km/h     | 15 km/h     |       4.2 |        4.2 |
             | footway        |                | lane          | 4 km/h +-1  | 15 km/h     |       1.1 |        4.2 |
             | motorway       | shared_lane    |               | 15 km/h     |             |       4.2 |            |
-            | primary        | shared_lane    |               | 15 km/h     | 15 km/h     |       4.2 |        2.9 |
+            | primary        | shared_lane    |               | 15 km/h     | 15 km/h     |       4.2 |        2.1 |
             | motorway       |                | shared_lane   | 15 km/h     |             |       4.2 |            |
-            | primary        |                | shared_lane   | 15 km/h     | 15 km/h     |       2.9 |        4.2 |
+            | primary        |                | shared_lane   | 15 km/h     | 15 km/h     |       2.1 |        4.2 |
 
 
     Scenario: Bike - Don't apply penalties for all kind of cycleways

features/testbot/fixed.feature

@@ -39,5 +39,64 @@ Feature: Fixed bugs, kept to check for regressions
             | de    | yes    |
 
         When I route I should get
-            | from | to | route     |
+            | from | to | route   |
-            | 1    | 2  | bcd,bcd   |
+            | 1    | 2  | bcd,bcd |
+
+    #############################
+    # This test models the OSM map at the location for
+    #    https://github.com/Project-OSRM/osrm-backend/issues/5039
+    #############################
+    Scenario: Mixed Entry and Exit and segregated
+        Given the profile file "car" initialized with
+            """
+            profile.properties.left_hand_driving = true
+            """
+            Given the node locations
+            | node | lon             | lat             |
+            | a    | 171.12889297029 | -42.58425289548 |
+            | b    | 171.1299357     | -42.5849295     |
+            | c    | 171.1295427     | -42.5849385     |
+            | d    | 171.1297356     | -42.5852029     |
+            | e    | 171.1296909     | -42.5851986     |
+            | f    | 171.1295097     | -42.585007      |
+            | g    | 171.1298225     | -42.5851928     |
+            | h    | 171.1300262     | -42.5859122     |
+            | i    | 171.1292651     | -42.584698      |
+            | j    | 171.1297209     | -42.5848569     |
+            | k    | 171.1297188     | -42.5854056     |
+            | l    | 171.1298326     | -42.5857266     |
+            | m    | 171.1298871     | -42.5848922     |
+            | n    | 171.1296505     | -42.585189      |
+            | o    | 171.1295206     | -42.5850862     |
+            | p    | 171.1296106     | -42.5848862     |
+            | q    | 171.1299784     | -42.5850191     |
+            | r    | 171.1298867     | -42.5851671     |
+            | s    | 171.1306955     | -42.5845812     |
+            | t    | 171.129525      | -42.584807      |
+            | u    | 171.1299705     | -42.584984      |
+            | v    | 171.1299067     | -42.5849073     |
+            | w    | 171.1302061     | -42.5848173     |
+            | x    | 171.12975       | -42.5855753     |
+            | y    | 171.129969      | -42.585079      |
+            | 1    | 171.131511926651| -42.584306746421966 |
+            | 2    | 171.128743886947| -42.58414875714669 |
+        And the ways
+            | nodes     | highway | maxspeed | name                    | ref   | surface | junction   | oneway |
+            | ws        | primary | 100      | Taramakau Highway       | SH 6  | asphalt |            |        |
+            | kxlh      | trunk   |          | Otira Highway           | SH 73 |         |            |        |
+            | ai        | primary | 100      | Kumara Junction Highway | SH 6  | asphalt |            |        |
+            | qyrgdenof | primary | 100      | Kumara Junction         |       |         | roundabout | yes    |
+            | ke        | trunk   |          | Otira Highway           | SH 73 |         |            | yes    |
+            | itj       | primary | 100      | Kumara Junction Highway | SH 6  |         |            | yes    |
+            | gk        | trunk   |          | Otira Highway           | SH 73 |         |            | yes    |
+            | fi        | primary | 100      | Kumara Junction Highway | SH 6  |         |            | yes    |
+            | wq        | primary | 100      | Taramakau Highway       | SH 6  |         |            | yes    |
+            | vw        | primary | 100      | Taramakau Highway       | SH 6  |         |            | yes    |
+            | vbuq      | primary | 100      | Kumara Junction         |       |         | roundabout | yes    |
+            | jmv       | primary | 100      | Kumara Junction         |       |         | roundabout | yes    |
+            | fcpj      | primary | 100      | Kumara Junction         |       |         | roundabout | yes    |
+
+        When I route I should get
+            | waypoints | route           | turns                                                    |
+            | 1,2       | Taramakau Highway,Kumara Junction Highway,Kumara Junction Highway,Kumara Junction Highway | depart,Kumara Junction-exit-2,exit rotary slight left,arrive |
+

include/storage/shared_monitor.hpp

@@ -117,6 +117,19 @@ template <typename Data> struct SharedMonitor
 #endif
 
     static void remove() { bi::shared_memory_object::remove(Data::name); }
+    static bool exists()
+    {
+        try
+        {
+            bi::shared_memory_object shmem_open =
+                bi::shared_memory_object(bi::open_only, Data::name, bi::read_only);
+        }
+        catch (const bi::interprocess_exception &exception)
+        {
+            return false;
+        }
+        return true;
+    }
 
   private:
 #if USE_BOOST_INTERPROCESS_CONDITION

include/storage/tar.hpp

@@ -29,10 +29,8 @@ checkMTarError(int error_code, const boost::filesystem::path &filepath, const st
     case MTAR_ESUCCESS:
         return;
     case MTAR_EFAILURE:
-        throw util::RuntimeError(filepath.string() + " : " + name,
+        throw util::RuntimeError(
-                                 ErrorCode::FileIOError,
+            filepath.string() + " : " + name, ErrorCode::FileIOError, SOURCE_REF);
-                                 SOURCE_REF,
-                                 std::strerror(errno));
     case MTAR_EOPENFAIL:
         throw util::RuntimeError(filepath.string() + " : " + name,
                                  ErrorCode::FileOpenError,

include/util/dynamic_graph.hpp

@@ -2,6 +2,8 @@
 #define DYNAMICGRAPH_HPP
 
 #include "util/deallocating_vector.hpp"
+#include "util/exception.hpp"
+#include "util/exception_utils.hpp"
 #include "util/integer_range.hpp"
 #include "util/permutation.hpp"
 #include "util/typedefs.hpp"
@@ -411,7 +413,7 @@ template <typename EdgeDataT> class DynamicGraph
         util::inplacePermutation(node_array.begin(), node_array.end(), old_to_new_node);
 
         // Build up edge permutation
-        auto new_edge_index = 0;
+        EdgeID new_edge_index = 0;
         std::vector<EdgeID> old_to_new_edge(edge_list.size(), SPECIAL_EDGEID);
         for (auto node : util::irange<NodeID>(0, number_of_nodes))
         {
@@ -419,6 +421,11 @@ template <typename EdgeDataT> class DynamicGraph
             // move all filled edges
             for (auto edge : GetAdjacentEdgeRange(node))
             {
+                if (new_edge_index == std::numeric_limits<EdgeID>::max())
+                {
+                    throw util::exception("There are too many edges, OSRM only supports 2^32" +
+                                          SOURCE_REF);
+                }
                 edge_list[edge].target = old_to_new_node[edge_list[edge].target];
                 BOOST_ASSERT(edge_list[edge].target != SPECIAL_NODEID);
                 old_to_new_edge[edge] = new_edge_index++;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "osrm",
-  "version": "5.17.0-latest.1",
+  "version": "5.17.2",
   "private": false,
   "description": "The Open Source Routing Machine is a high performance routing engine written in C++14 designed to run on OpenStreetMap data.",
   "dependencies": {

profiles/bicycle.lua

@@ -109,11 +109,11 @@ function setup()
     -- reduce the driving speed by 30% for unsafe roads
     -- only used for cyclability metric
     unsafe_highway_list = {
-      primary = 0.7,
+      primary = 0.5,
-      secondary = 0.75,
+      secondary = 0.65,
       tertiary = 0.8,
-      primary_link = 0.7,
+      primary_link = 0.5,
-      secondary_link = 0.75,
+      secondary_link = 0.65,
       tertiary_link = 0.8,
     },
 
@@ -250,204 +250,237 @@ end
 
 function handle_bicycle_tags(profile,way,result,data)
     -- initial routability check, filters out buildings, boundaries, etc
-  local route = way:get_value_by_key("route")
+  data.route = way:get_value_by_key("route")
-  local man_made = way:get_value_by_key("man_made")
+  data.man_made = way:get_value_by_key("man_made")
-  local railway = way:get_value_by_key("railway")
+  data.railway = way:get_value_by_key("railway")
-  local amenity = way:get_value_by_key("amenity")
+  data.amenity = way:get_value_by_key("amenity")
-  local public_transport = way:get_value_by_key("public_transport")
+  data.public_transport = way:get_value_by_key("public_transport")
-  local bridge = way:get_value_by_key("bridge")
+  data.bridge = way:get_value_by_key("bridge")
 
   if (not data.highway or data.highway == '') and
-  (not route or route == '') and
+  (not data.route or data.route == '') and
-  (not profile.use_public_transport or not railway or railway=='') and
+  (not profile.use_public_transport or not data.railway or data.railway=='') and
-  (not amenity or amenity=='') and
+  (not data.amenity or data.amenity=='') and
-  (not man_made or man_made=='') and
+  (not data.man_made or data.man_made=='') and
-  (not public_transport or public_transport=='') and
+  (not data.public_transport or data.public_transport=='') and
-  (not bridge or bridge=='')
+  (not data.bridge or data.bridge=='')
   then
     return false
   end
 
   -- access
-  local access = find_access_tag(way, profile.access_tags_hierarchy)
+  data.access = find_access_tag(way, profile.access_tags_hierarchy)
-  if access and profile.access_tag_blacklist[access] then
+  if data.access and profile.access_tag_blacklist[data.access] then
     return false
   end
 
   -- other tags
-  local junction = way:get_value_by_key("junction")
+  data.junction = way:get_value_by_key("junction")
-  local maxspeed = parse_maxspeed(way:get_value_by_key ( "maxspeed") )
+  data.maxspeed = parse_maxspeed(way:get_value_by_key ( "maxspeed") )
-  local maxspeed_forward = parse_maxspeed(way:get_value_by_key( "maxspeed:forward"))
+  data.maxspeed_forward = parse_maxspeed(way:get_value_by_key( "maxspeed:forward"))
-  local maxspeed_backward = parse_maxspeed(way:get_value_by_key( "maxspeed:backward"))
+  data.maxspeed_backward = parse_maxspeed(way:get_value_by_key( "maxspeed:backward"))
-  local barrier = way:get_value_by_key("barrier")
+  data.barrier = way:get_value_by_key("barrier")
-  local oneway = way:get_value_by_key("oneway")
+  data.oneway = way:get_value_by_key("oneway")
-  local oneway_bicycle = way:get_value_by_key("oneway:bicycle")
+  data.oneway_bicycle = way:get_value_by_key("oneway:bicycle")
-  local cycleway = way:get_value_by_key("cycleway")
+  data.cycleway = way:get_value_by_key("cycleway")
-  local cycleway_left = way:get_value_by_key("cycleway:left")
+  data.cycleway_left = way:get_value_by_key("cycleway:left")
-  local cycleway_right = way:get_value_by_key("cycleway:right")
+  data.cycleway_right = way:get_value_by_key("cycleway:right")
-  local duration = way:get_value_by_key("duration")
+  data.duration = way:get_value_by_key("duration")
-  local service = way:get_value_by_key("service")
+  data.service = way:get_value_by_key("service")
-  local foot = way:get_value_by_key("foot")
+  data.foot = way:get_value_by_key("foot")
-  local foot_forward = way:get_value_by_key("foot:forward")
+  data.foot_forward = way:get_value_by_key("foot:forward")
-  local foot_backward = way:get_value_by_key("foot:backward")
+  data.foot_backward = way:get_value_by_key("foot:backward")
-  local bicycle = way:get_value_by_key("bicycle")
+  data.bicycle = way:get_value_by_key("bicycle")
+
+  speed_handler(profile,way,result,data)
+
+  oneway_handler(profile,way,result,data)
+
+  cycleway_handler(profile,way,result,data)
+
+  bike_push_handler(profile,way,result,data)
 
 
-  local way_type_allows_pushing = false
+  -- maxspeed
+  limit( result, data.maxspeed, data.maxspeed_forward, data.maxspeed_backward )
+
+  -- not routable if no speed assigned
+  -- this avoid assertions in debug builds
+  if result.forward_speed <= 0 and result.duration <= 0 then
+    result.forward_mode = mode.inaccessible
+  end
+  if result.backward_speed <= 0 and result.duration <= 0 then
+    result.backward_mode = mode.inaccessible
+  end
+
+  safety_handler(profile,way,result,data)
+end
+
+
+
+function speed_handler(profile,way,result,data)
+
+  data.way_type_allows_pushing = false
 
   -- speed
-  local bridge_speed = profile.bridge_speeds[bridge]
+  local bridge_speed = profile.bridge_speeds[data.bridge]
   if (bridge_speed and bridge_speed > 0) then
-    data.highway = bridge
+    data.highway = data.bridge
-    if duration and durationIsValid(duration) then
+    if data.duration and durationIsValid(data.duration) then
-      result.duration = math.max( parseDuration(duration), 1 )
+      result.duration = math.max( parseDuration(data.duration), 1 )
     end
     result.forward_speed = bridge_speed
     result.backward_speed = bridge_speed
-    way_type_allows_pushing = true
+    data.way_type_allows_pushing = true
-  elseif profile.route_speeds[route] then
+  elseif profile.route_speeds[data.route] then
     -- ferries (doesn't cover routes tagged using relations)
     result.forward_mode = mode.ferry
     result.backward_mode = mode.ferry
-    if duration and durationIsValid(duration) then
+    if data.duration and durationIsValid(data.duration) then
-      result.duration = math.max( 1, parseDuration(duration) )
+      result.duration = math.max( 1, parseDuration(data.duration) )
     else
-       result.forward_speed = profile.route_speeds[route]
+       result.forward_speed = profile.route_speeds[data.route]
-       result.backward_speed = profile.route_speeds[route]
+       result.backward_speed = profile.route_speeds[data.route]
     end
   -- railway platforms (old tagging scheme)
-  elseif railway and profile.platform_speeds[railway] then
+  elseif data.railway and profile.platform_speeds[data.railway] then
-    result.forward_speed = profile.platform_speeds[railway]
+    result.forward_speed = profile.platform_speeds[data.railway]
-    result.backward_speed = profile.platform_speeds[railway]
+    result.backward_speed = profile.platform_speeds[data.railway]
-    way_type_allows_pushing = true
+    data.way_type_allows_pushing = true
   -- public_transport platforms (new tagging platform)
-  elseif public_transport and profile.platform_speeds[public_transport] then
+  elseif data.public_transport and profile.platform_speeds[data.public_transport] then
-    result.forward_speed = profile.platform_speeds[public_transport]
+    result.forward_speed = profile.platform_speeds[data.public_transport]
-    result.backward_speed = profile.platform_speeds[public_transport]
+    result.backward_speed = profile.platform_speeds[data.public_transport]
-    way_type_allows_pushing = true
+    data.way_type_allows_pushing = true
   -- railways
-  elseif profile.use_public_transport and railway and profile.railway_speeds[railway] and profile.access_tag_whitelist[access] then
+  elseif profile.use_public_transport and data.railway and profile.railway_speeds[data.railway] and profile.access_tag_whitelist[data.access] then
     result.forward_mode = mode.train
     result.backward_mode = mode.train
-    result.forward_speed = profile.railway_speeds[railway]
+    result.forward_speed = profile.railway_speeds[data.railway]
-    result.backward_speed = profile.railway_speeds[railway]
+    result.backward_speed = profile.railway_speeds[data.railway]
-  elseif amenity and profile.amenity_speeds[amenity] then
+  elseif data.amenity and profile.amenity_speeds[data.amenity] then
     -- parking areas
-    result.forward_speed = profile.amenity_speeds[amenity]
+    result.forward_speed = profile.amenity_speeds[data.amenity]
-    result.backward_speed = profile.amenity_speeds[amenity]
+    result.backward_speed = profile.amenity_speeds[data.amenity]
-    way_type_allows_pushing = true
+    data.way_type_allows_pushing = true
   elseif profile.bicycle_speeds[data.highway] then
     -- regular ways
     result.forward_speed = profile.bicycle_speeds[data.highway]
     result.backward_speed = profile.bicycle_speeds[data.highway]
-    way_type_allows_pushing = true
+    data.way_type_allows_pushing = true
-  elseif access and profile.access_tag_whitelist[access]  then
+  elseif data.access and profile.access_tag_whitelist[data.access]  then
     -- unknown way, but valid access tag
     result.forward_speed = profile.default_speed
     result.backward_speed = profile.default_speed
-    way_type_allows_pushing = true
+    data.way_type_allows_pushing = true
   end
+end
 
+function oneway_handler(profile,way,result,data)
   -- oneway
-  local implied_oneway = junction == "roundabout" or junction == "circular" or data.highway == "motorway"
+  data.implied_oneway = data.junction == "roundabout" or data.junction == "circular" or data.highway == "motorway"
-  local reverse = false
+  data.reverse = false
 
-  if oneway_bicycle == "yes" or oneway_bicycle == "1" or oneway_bicycle == "true" then
+  if data.oneway_bicycle == "yes" or data.oneway_bicycle == "1" or data.oneway_bicycle == "true" then
     result.backward_mode = mode.inaccessible
-  elseif oneway_bicycle == "no" or oneway_bicycle == "0" or oneway_bicycle == "false" then
+  elseif data.oneway_bicycle == "no" or data.oneway_bicycle == "0" or data.oneway_bicycle == "false" then
    -- prevent other cases
-  elseif oneway_bicycle == "-1" then
+  elseif data.oneway_bicycle == "-1" then
     result.forward_mode = mode.inaccessible
-    reverse = true
+    data.reverse = true
-  elseif oneway == "yes" or oneway == "1" or oneway == "true" then
+  elseif data.oneway == "yes" or data.oneway == "1" or data.oneway == "true" then
     result.backward_mode = mode.inaccessible
-  elseif oneway == "no" or oneway == "0" or oneway == "false" then
+  elseif data.oneway == "no" or data.oneway == "0" or data.oneway == "false" then
     -- prevent other cases
-  elseif oneway == "-1" then
+  elseif data.oneway == "-1" then
     result.forward_mode = mode.inaccessible
-    reverse = true
+    data.reverse = true
-  elseif implied_oneway then
+  elseif data.implied_oneway then
     result.backward_mode = mode.inaccessible
   end
+end
 
+function cycleway_handler(profile,way,result,data)
   -- cycleway
-  local has_cycleway_forward = false
+  data.has_cycleway_forward = false
-  local has_cycleway_backward = false
+  data.has_cycleway_backward = false
-  local is_oneway = result.forward_mode ~= mode.inaccessible and result.backward_mode ~= mode.inaccessible and not implied_oneway
+  data.is_twoway = result.forward_mode ~= mode.inaccessible and result.backward_mode ~= mode.inaccessible and not data.implied_oneway
 
   -- cycleways on normal roads
-  if is_oneway then
+  if data.is_twoway then
-    if cycleway and profile.cycleway_tags[cycleway] then
+    if data.cycleway and profile.cycleway_tags[data.cycleway] then
-      has_cycleway_backward = true
+      data.has_cycleway_backward = true
-      has_cycleway_forward = true
+      data.has_cycleway_forward = true
     end
-    if (cycleway_right and profile.cycleway_tags[cycleway_right]) or (cycleway_left and profile.opposite_cycleway_tags[cycleway_left]) then
+    if (data.cycleway_right and profile.cycleway_tags[data.cycleway_right]) or (data.cycleway_left and profile.opposite_cycleway_tags[data.cycleway_left]) then
-      has_cycleway_forward = true
+      data.has_cycleway_forward = true
     end
-    if (cycleway_left and profile.cycleway_tags[cycleway_left]) or (cycleway_right and profile.opposite_cycleway_tags[cycleway_right]) then
+    if (data.cycleway_left and profile.cycleway_tags[data.cycleway_left]) or (data.cycleway_right and profile.opposite_cycleway_tags[data.cycleway_right]) then
-      has_cycleway_backward = true
+      data.has_cycleway_backward = true
     end
   else
-    local has_twoway_cycleway = (cycleway and profile.opposite_cycleway_tags[cycleway]) or (cycleway_right and profile.opposite_cycleway_tags[cycleway_right]) or (cycleway_left and profile.opposite_cycleway_tags[cycleway_left])
+    local has_twoway_cycleway = (data.cycleway and profile.opposite_cycleway_tags[data.cycleway]) or (data.cycleway_right and profile.opposite_cycleway_tags[data.cycleway_right]) or (data.cycleway_left and profile.opposite_cycleway_tags[data.cycleway_left])
-    local has_opposite_cycleway = (cycleway_left and profile.opposite_cycleway_tags[cycleway_left]) or (cycleway_right and profile.opposite_cycleway_tags[cycleway_right])
+    local has_opposite_cycleway = (data.cycleway_left and profile.opposite_cycleway_tags[data.cycleway_left]) or (data.cycleway_right and profile.opposite_cycleway_tags[data.cycleway_right])
-    local has_oneway_cycleway = (cycleway and profile.cycleway_tags[cycleway]) or (cycleway_right and profile.cycleway_tags[cycleway_right]) or (cycleway_left and profile.cycleway_tags[cycleway_left])
+    local has_oneway_cycleway = (data.cycleway and profile.cycleway_tags[data.cycleway]) or (data.cycleway_right and profile.cycleway_tags[data.cycleway_right]) or (data.cycleway_left and profile.cycleway_tags[data.cycleway_left])
 
     -- set cycleway even though it is an one-way if opposite is tagged
     if has_twoway_cycleway then
-      has_cycleway_backward = true
+      data.has_cycleway_backward = true
-      has_cycleway_forward = true
+      data.has_cycleway_forward = true
     elseif has_opposite_cycleway then
-      if not reverse then
+      if not data.reverse then
-        has_cycleway_backward = true
+        data.has_cycleway_backward = true
       else
-        has_cycleway_forward = true
+        data.has_cycleway_forward = true
       end
     elseif has_oneway_cycleway then
-      if not reverse then
+      if not data.reverse then
-        has_cycleway_forward = true
+        data.has_cycleway_forward = true
       else
-        has_cycleway_backward = true
+        data.has_cycleway_backward = true
       end
 
     end
   end
 
-  if has_cycleway_backward then
+  if data.has_cycleway_backward then
     result.backward_mode = mode.cycling
     result.backward_speed = profile.bicycle_speeds["cycleway"]
   end
 
-  if has_cycleway_forward then
+  if data.has_cycleway_forward then
     result.forward_mode = mode.cycling
     result.forward_speed = profile.bicycle_speeds["cycleway"]
   end
+end
 
+function bike_push_handler(profile,way,result,data)
   -- pushing bikes - if no other mode found
   if result.forward_mode == mode.inaccessible or result.backward_mode == mode.inaccessible or
     result.forward_speed == -1 or result.backward_speed == -1 then
-    if foot ~= 'no' then
+    if data.foot ~= 'no' then
       local push_forward_speed = nil
       local push_backward_speed = nil
 
       if profile.pedestrian_speeds[data.highway] then
         push_forward_speed = profile.pedestrian_speeds[data.highway]
         push_backward_speed = profile.pedestrian_speeds[data.highway]
-      elseif man_made and profile.man_made_speeds[man_made] then
+      elseif data.man_made and profile.man_made_speeds[data.man_made] then
-        push_forward_speed = profile.man_made_speeds[man_made]
+        push_forward_speed = profile.man_made_speeds[data.man_made]
-        push_backward_speed = profile.man_made_speeds[man_made]
+        push_backward_speed = profile.man_made_speeds[data.man_made]
       else
-        if foot == 'yes' then
+        if data.foot == 'yes' then
           push_forward_speed = profile.walking_speed
-          if not implied_oneway then
+          if not data.implied_oneway then
             push_backward_speed = profile.walking_speed
           end
-        elseif foot_forward == 'yes' then
+        elseif data.foot_forward == 'yes' then
           push_forward_speed = profile.walking_speed
-        elseif foot_backward == 'yes' then
+        elseif data.foot_backward == 'yes' then
           push_backward_speed = profile.walking_speed
-        elseif way_type_allows_pushing then
+        elseif data.way_type_allows_pushing then
           push_forward_speed = profile.walking_speed
-          if not implied_oneway then
+          if not data.implied_oneway then
             push_backward_speed = profile.walking_speed
           end
         end
@@ -466,65 +499,77 @@ function handle_bicycle_tags(profile,way,result,data)
 
   end
 
-
   -- dismount
-  if bicycle == "dismount" then
+  if data.bicycle == "dismount" then
     result.forward_mode = mode.pushing_bike
     result.backward_mode = mode.pushing_bike
     result.forward_speed = profile.walking_speed
     result.backward_speed = profile.walking_speed
   end
+end
 
-
+function safety_handler(profile,way,result,data)
-
-  -- maxspeed
-  limit( result, maxspeed, maxspeed_forward, maxspeed_backward )
-
-
-  -- not routable if no speed assigned
-  -- this avoid assertions in debug builds
-  if result.forward_speed <= 0 and result.duration <= 0 then
-    result.forward_mode = mode.inaccessible
-  end
-  if result.backward_speed <= 0 and result.duration <= 0 then
-    result.backward_mode = mode.inaccessible
-  end
-
-
   -- convert duration into cyclability
   if profile.properties.weight_name == 'cyclability' then
-      local safety_penalty = profile.unsafe_highway_list[data.highway] or 1.
+    local safety_penalty = profile.unsafe_highway_list[data.highway] or 1.
-      local is_unsafe = safety_penalty < 1
+    local is_unsafe = safety_penalty < 1
-      local forward_is_unsafe = is_unsafe and not has_cycleway_forward
-      local backward_is_unsafe = is_unsafe and not has_cycleway_backward
-      local is_undesireable = data.highway == "service" and profile.service_penalties[service]
-      local forward_penalty = 1.
-      local backward_penalty = 1.
-      if forward_is_unsafe then
-        forward_penalty = math.min(forward_penalty, safety_penalty)
-      end
-      if backward_is_unsafe then
-         backward_penalty = math.min(backward_penalty, safety_penalty)
-      end
 
-      if is_undesireable then
+    -- primaries that are one ways are probably huge primaries where the lanes need to be separated
-         forward_penalty = math.min(forward_penalty, profile.service_penalties[service])
+    if is_unsafe and data.highway == 'primary' and not data.is_twoway then
-         backward_penalty = math.min(backward_penalty, profile.service_penalties[service])
+      safety_penalty = safety_penalty * 0.5
-      end
+    end
+    if is_unsafe and data.highway == 'secondary' and not data.is_twoway then
+      safety_penalty = safety_penalty * 0.6
+    end
 
+    local forward_is_unsafe = is_unsafe and not data.has_cycleway_forward
+    local backward_is_unsafe = is_unsafe and not data.has_cycleway_backward
+    local is_undesireable = data.highway == "service" and profile.service_penalties[data.service]
+    local forward_penalty = 1.
+    local backward_penalty = 1.
+    if forward_is_unsafe then
+      forward_penalty = math.min(forward_penalty, safety_penalty)
+    end
+    if backward_is_unsafe then
+       backward_penalty = math.min(backward_penalty, safety_penalty)
+    end
+
+    if is_undesireable then
+       forward_penalty = math.min(forward_penalty, profile.service_penalties[data.service])
+       backward_penalty = math.min(backward_penalty, profile.service_penalties[data.service])
+    end
+
+    if result.forward_speed > 0 then
+      -- convert from km/h to m/s
+      result.forward_rate = result.forward_speed / 3.6 * forward_penalty
+    end
+    if result.backward_speed > 0 then
+      -- convert from km/h to m/s
+      result.backward_rate = result.backward_speed / 3.6 * backward_penalty
+    end
+    if result.duration > 0 then
+      result.weight = result.duration / forward_penalty
+    end
+
+    if data.highway == "bicycle" then
+      safety_bonus = safety_bonus + 0.2
       if result.forward_speed > 0 then
         -- convert from km/h to m/s
-        result.forward_rate = result.forward_speed / 3.6 * forward_penalty
+        result.forward_rate = result.forward_speed / 3.6 * safety_bonus
       end
       if result.backward_speed > 0 then
         -- convert from km/h to m/s
-        result.backward_rate = result.backward_speed / 3.6 * backward_penalty
+        result.backward_rate = result.backward_speed / 3.6 * safety_bonus
       end
       if result.duration > 0 then
-        result.weight = result.duration / forward_penalty
+        result.weight = result.duration / safety_bonus
       end
+    end
   end
 end
+
+
+
 function process_way(profile, way, result)
   -- the initial filtering of ways based on presence of tags
   -- affects processing times significantly, because all ways
@@ -542,6 +587,39 @@ function process_way(profile, way, result)
   local data = {
     -- prefetch tags
     highway = way:get_value_by_key('highway'),
+
+    route = nil,
+    man_made = nil,
+    railway = nil,
+    amenity = nil,
+    public_transport = nil,
+    bridge = nil,
+
+    access = nil,
+
+    junction = nil,
+    maxspeed = nil,
+    maxspeed_forward = nil,
+    maxspeed_backward = nil,
+    barrier = nil,
+    oneway = nil,
+    oneway_bicycle = nil,
+    cycleway = nil,
+    cycleway_left = nil,
+    cycleway_right = nil,
+    duration = nil,
+    service = nil,
+    foot = nil,
+    foot_forward = nil,
+    foot_backward = nil,
+    bicycle = nil,
+
+    way_type_allows_pushing = false,
+    has_cycleway_forward = false,
+    has_cycleway_backward = false,
+    is_twoway = true,
+    reverse = false,
+    implied_oneway = false
   }
 
   local handlers = Sequence {

scripts/osm2cucumber.js

@@ -0,0 +1,43 @@
+/*********************************
+ * Takes an XML `.osm` file and converts it into a cucumber scenario definition like
+ *    Given the node locations
+ *       | node | lon | lat |
+ *       .....
+ *    Given the ways
+ *       | nodes | tag1 | tag2 | tag3 |
+ *       .....
+ *
+ * Note that cucumber tests are limited to 26 nodes (labelled a-z), so
+ * you'll need to use pretty small OSM extracts to get this to work.
+ *****************************************/
+var fs = require('fs');
+var parseString = require('xml2js').parseString;
+
+var data = fs.readFileSync('filename.osm', 'utf8');
+
+const items = parseString(data, (err, result) => {
+var idmap = {};
+
+console.log('Given the node locations');
+console.log('    | node | lon       | lat       |');
+result.osm.node.filter(n => !n.$.action || n.$.action !== 'delete').forEach(i => {
+  var code = String.fromCharCode(97 + Object.keys(idmap).length)
+  idmap[i.$.id] = code;
+  console.log(`    | ${code} | ${i.$.lon} | ${i.$.lat} |`);
+});
+
+var allkeys = {};
+var waytags = {};
+
+result.osm.way.filter(n => !n.$.action || n.$.action !== 'delete').forEach(w => {
+  if (!waytags[w.$.id]) waytags[w.$.id] = {};
+  w.tag.forEach(t => { allkeys[t.$.k] = t.$.v; waytags[w.$.id][t.$.k] = t.$.v; });
+});
+
+console.log('And the ways');
+console.log(`    | nodes |  ${Object.keys(allkeys).join(' | ')} |`);
+
+result.osm.way.filter(n => !n.$.action || n.$.action !== 'delete').forEach(w => {
+  console.log(`    | ${w.nd.map(n => idmap[n.$.ref]).join('')} | ${Object.keys(allkeys).map(k => waytags[w.$.id][k] || '').join(' | ')} |`);
+});
+});

src/engine/guidance/collapse_turns.cpp

@@ -618,9 +618,10 @@ RouteSteps collapseSegregatedTurnInstructions(RouteSteps steps)
                               TransferLanesStrategy());
             ++next_step;
         }
-        // else if the current step is segregated and the next step is not then combine with turn
+        // else if the current step is segregated and the next step is not segregated
-        // adjustment
+        // and the next step is not a roundabout then combine with turn adjustment
-        else if (curr_step->is_segregated && !next_step->is_segregated)
+        else if (curr_step->is_segregated && !next_step->is_segregated &&
+                 !hasRoundaboutType(next_step->maneuver.instruction))
         {
             // Determine if u-turn
             if (bearingsAreReversed(

src/tools/store.cpp

@@ -27,12 +27,15 @@ void deleteRegion(const storage::SharedRegionRegister::ShmKey key)
 
 void listRegions()
 {
-
+    osrm::util::Log() << "name\tshm key\ttimestamp\tsize";
+    if (!storage::SharedMonitor<storage::SharedRegionRegister>::exists())
+    {
+        return;
+    }
     storage::SharedMonitor<storage::SharedRegionRegister> monitor;
     std::vector<std::string> names;
     const auto &shared_register = monitor.data();
     shared_register.List(std::back_inserter(names));
-    osrm::util::Log() << "name\tshm key\ttimestamp\tsize";
     for (const auto &name : names)
     {
         auto id = shared_register.Find(name);
@@ -105,8 +108,7 @@ bool generateDataStoreOptions(const int argc,
          boost::program_options::value<bool>(&list_datasets)
              ->default_value(false)
              ->implicit_value(true),
-         "Name of the dataset to load into memory. This allows having multiple datasets in memory "
+         "List all OSRM datasets currently in memory") //
-         "at the same time.") //
         ("only-metric",
          boost::program_options::value<bool>(&only_metric)
              ->default_value(false)

third_party/microtar/src/microtar.c

@@ -41,7 +41,7 @@ typedef struct {
 } mtar_raw_header_t;
 
 
-static unsigned round_up(unsigned n, unsigned incr) {
+static mtar_size_t round_up(mtar_size_t n, unsigned incr) {
   return n + (incr - n % incr) % incr;
 }
 
@@ -60,14 +60,14 @@ static unsigned checksum(const mtar_raw_header_t* rh) {
 }
 
 
-static int tread(mtar_t *tar, void *data, unsigned size) {
+static int tread(mtar_t *tar, void *data, mtar_size_t size) {
   int err = tar->read(tar, data, size);
   tar->pos += size;
   return err;
 }
 
 
-static int twrite(mtar_t *tar, const void *data, unsigned size) {
+static int twrite(mtar_t *tar, const void *data, mtar_size_t size) {
   int err = tar->write(tar, data, size);
   tar->pos += size;
   return err;
@@ -89,6 +89,7 @@ static int write_null_bytes(mtar_t *tar, int n) {
 
 static int raw_to_header(mtar_header_t *h, const mtar_raw_header_t *rh) {
   unsigned chksum1, chksum2;
+  mtar_size_t filesize;
 
   /* If the checksum starts with a null byte we assume the record is NULL */
   if (*rh->checksum == '\0') {
@@ -105,24 +106,68 @@ static int raw_to_header(mtar_header_t *h, const mtar_raw_header_t *rh) {
   /* Load raw header into header */
   sscanf(rh->mode, "%o", &h->mode);
   sscanf(rh->owner, "%o", &h->owner);
-  sscanf(rh->size, "%o", &h->size);
   sscanf(rh->mtime, "%o", &h->mtime);
   h->type = rh->type;
   strcpy(h->name, rh->name);
   strcpy(h->linkname, rh->linkname);
 
+  /* Load size field */
+  if ((rh->size[0] & 0x80) == 0) {
+#ifdef _MSC_VER
+      sscanf(rh->size, "%12llo", &h->size);
+#else
+      sscanf(rh->size, "%12lo", &h->size);
+#endif
+  } else {
+      h->size = (rh->size[0] & 0x7f) | (rh->size[0] & 0x40 ? 0x80 : 0);
+      uint8_t *p8 = (uint8_t *)&rh->size + 1;
+      while (p8 != (uint8_t *)&rh->size + sizeof(rh->size)) {
+          if (h->size >= ((mtar_size_t)1 << (sizeof(h->size) - 1) * 8)) {
+              return MTAR_EFAILURE;
+          }
+          h->size = ((mtar_size_t)h->size << 8) + *p8++;
+      }
+  }
+
   return MTAR_ESUCCESS;
 }
 
 
 static int header_to_raw(mtar_raw_header_t *rh, const mtar_header_t *h) {
   unsigned chksum;
+  mtar_size_t filesize = h->size;
 
   /* Load header into raw header */
   memset(rh, 0, sizeof(*rh));
+
+  /* Store size in ASCII octal digits or base-256 formats */
+  if (sizeof(mtar_size_t) <= 4 || filesize <= (mtar_size_t)077777777777LL) {
+#ifdef _MSC_VER
+      sprintf(rh->size, "%llo", h->size);
+#else
+      sprintf(rh->size, "%lo", h->size);
+#endif
+  } else if (sizeof(filesize) < sizeof(rh->size)) {
+      /* GNU tar uses "base-256 encoding" for very large numbers.
+       * Encoding is binary, with highest bit always set as a marker
+       * and sign in next-highest bit:
+       * 80 00 .. 00 - zero
+       * bf ff .. ff - largest positive number
+       * ff ff .. ff - minus 1
+       * c0 00 .. 00 - smallest negative number
+       */
+      uint8_t *p8 = (uint8_t *)&rh->size + sizeof(rh->size);
+      do {
+          *--p8 = (uint8_t)filesize;
+          filesize >>= 8;
+      } while (p8 != (uint8_t *)&rh->size);
+      *p8 |= 0x80;
+  } else {
+      return MTAR_EFAILURE;
+  }
+
   sprintf(rh->mode, "%o", h->mode);
   sprintf(rh->owner, "%o", h->owner);
-  sprintf(rh->size, "%o", h->size);
   sprintf(rh->mtime, "%o", h->mtime);
   rh->type = h->type ? h->type : MTAR_TREG;
   strcpy(rh->name, h->name);
@@ -153,17 +198,17 @@ const char* mtar_strerror(int err) {
 }
 
 
-static int file_write(mtar_t *tar, const void *data, unsigned size) {
+static int file_write(mtar_t *tar, const void *data, mtar_size_t size) {
-  unsigned res = fwrite(data, 1, size, tar->stream);
+  mtar_size_t res = fwrite(data, 1, size, tar->stream);
   return (res == size) ? MTAR_ESUCCESS : MTAR_EWRITEFAIL;
 }
 
-static int file_read(mtar_t *tar, void *data, unsigned size) {
+static int file_read(mtar_t *tar, void *data, mtar_size_t size) {
-  unsigned res = fread(data, 1, size, tar->stream);
+  mtar_size_t res = fread(data, 1, size, tar->stream);
   return (res == size) ? MTAR_ESUCCESS : MTAR_EREADFAIL;
 }
 
-static int file_seek(mtar_t *tar, unsigned offset) {
+static int file_seek(mtar_t *tar, mtar_size_t offset) {
   int res = fseek(tar->stream, offset, SEEK_SET);
   return (res == 0) ? MTAR_ESUCCESS : MTAR_ESEEKFAIL;
 }
@@ -213,7 +258,7 @@ int mtar_close(mtar_t *tar) {
 }
 
 
-int mtar_seek(mtar_t *tar, unsigned pos) {
+int mtar_seek(mtar_t *tar, mtar_size_t pos) {
   int err = tar->seek(tar, pos);
   tar->pos = pos;
   return err;
@@ -228,7 +273,8 @@ int mtar_rewind(mtar_t *tar) {
 
 
 int mtar_next(mtar_t *tar) {
-  int err, n;
+  mtar_size_t n;
+  int err;
   mtar_header_t h;
   /* Load header */
   err = mtar_read_header(tar, &h);
@@ -287,7 +333,7 @@ int mtar_read_header(mtar_t *tar, mtar_header_t *h) {
 }
 
 
-int mtar_read_data(mtar_t *tar, void *ptr, unsigned size) {
+int mtar_read_data(mtar_t *tar, void *ptr, mtar_size_t size) {
   int err;
   /* If we have no remaining data then this is the first read, we get the size,
    * set the remaining data and seek to the beginning of the data */
@@ -329,7 +375,7 @@ int mtar_write_header(mtar_t *tar, const mtar_header_t *h) {
 }
 
 
-int mtar_write_file_header(mtar_t *tar, const char *name, unsigned size) {
+int mtar_write_file_header(mtar_t *tar, const char *name, mtar_size_t size) {
   mtar_header_t h;
   /* Build header */
   memset(&h, 0, sizeof(h));
@@ -354,7 +400,7 @@ int mtar_write_dir_header(mtar_t *tar, const char *name) {
 }
 
 
-int mtar_write_data(mtar_t *tar, const void *data, unsigned size) {
+int mtar_write_data(mtar_t *tar, const void *data, mtar_size_t size) {
   int err;
   /* Write data */
   err = twrite(tar, data, size);

third_party/microtar/src/microtar.h

@@ -10,9 +10,12 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <stdint.h>
 
 #define MTAR_VERSION "0.1.0"
 
+typedef size_t mtar_size_t;
+
 enum {
   MTAR_ESUCCESS     =  0,
   MTAR_EFAILURE     = -1,
@@ -38,7 +41,7 @@ enum {
 typedef struct {
   unsigned mode;
   unsigned owner;
-  unsigned size;
+  mtar_size_t size;
   unsigned mtime;
   unsigned type;
   char name[100];
@@ -49,14 +52,14 @@ typedef struct {
 typedef struct mtar_t mtar_t;
 
 struct mtar_t {
-  int (*read)(mtar_t *tar, void *data, unsigned size);
+  int (*read)(mtar_t *tar, void *data, mtar_size_t size);
-  int (*write)(mtar_t *tar, const void *data, unsigned size);
+  int (*write)(mtar_t *tar, const void *data, mtar_size_t size);
-  int (*seek)(mtar_t *tar, unsigned pos);
+  int (*seek)(mtar_t *tar, mtar_size_t pos);
   int (*close)(mtar_t *tar);
   void *stream;
-  unsigned pos;
+  mtar_size_t pos;
-  unsigned remaining_data;
+  mtar_size_t remaining_data;
-  unsigned last_header;
+  mtar_size_t last_header;
 };
 
 
@@ -65,17 +68,17 @@ const char* mtar_strerror(int err);
 int mtar_open(mtar_t *tar, const char *filename, const char *mode);
 int mtar_close(mtar_t *tar);
 
-int mtar_seek(mtar_t *tar, unsigned pos);
+int mtar_seek(mtar_t *tar, mtar_size_t pos);
 int mtar_rewind(mtar_t *tar);
 int mtar_next(mtar_t *tar);
 int mtar_find(mtar_t *tar, const char *name, mtar_header_t *h);
 int mtar_read_header(mtar_t *tar, mtar_header_t *h);
-int mtar_read_data(mtar_t *tar, void *ptr, unsigned size);
+int mtar_read_data(mtar_t *tar, void *ptr, mtar_size_t size);
 
 int mtar_write_header(mtar_t *tar, const mtar_header_t *h);
-int mtar_write_file_header(mtar_t *tar, const char *name, unsigned size);
+int mtar_write_file_header(mtar_t *tar, const char *name, mtar_size_t size);
 int mtar_write_dir_header(mtar_t *tar, const char *name);
-int mtar_write_data(mtar_t *tar, const void *data, unsigned size);
+int mtar_write_data(mtar_t *tar, const void *data, mtar_size_t size);
 int mtar_finalize(mtar_t *tar);
 
 

unit_tests/storage/tar.cpp

@@ -3,6 +3,8 @@
 #include "../common/range_tools.hpp"
 #include "../common/temporary_file.hpp"
 
+#include <boost/function_output_iterator.hpp>
+#include <boost/iterator/function_input_iterator.hpp>
 #include <boost/test/unit_test.hpp>
 
 BOOST_AUTO_TEST_SUITE(tar)
@@ -187,4 +189,39 @@ BOOST_AUTO_TEST_CASE(continue_write_tar_file)
     CHECK_EQUAL_COLLECTIONS(result_64bit_vector, vector_64bit);
 }
 
+// Boost test only supports disabling was only introduced in 1.59
+#if BOOST_VERSION >= 105900
+// This test case is disabled by default because it needs 10 GiB of storage
+// Enable with ./storage-tests --run_test=tar/write_huge_tar_file
+BOOST_AUTO_TEST_CASE(write_huge_tar_file, *boost::unit_test::disabled())
+{
+    TemporaryFile tmp{TEST_DATA_DIR "/tar_huge_write_test.tar"};
+
+    std::uint64_t reference_checksum = 0;
+    {
+        storage::tar::FileWriter writer(tmp.path, storage::tar::FileWriter::GenerateFingerprint);
+        std::uint64_t value = 0;
+        const std::function<std::uint64_t()> encode_function = [&]() -> std::uint64_t {
+            reference_checksum += value;
+            return value++;
+        };
+        std::uint64_t num_elements = (10ULL * 1024ULL * 1024ULL * 1024ULL) / sizeof(std::uint64_t);
+        writer.WriteStreaming<std::uint64_t>(
+            "huge_data",
+            boost::make_function_input_iterator(encode_function, boost::infinite()),
+            num_elements);
+    }
+
+    std::uint64_t checksum = 0;
+    {
+        storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint);
+        reader.ReadStreaming<std::uint64_t>(
+            "huge_data",
+            boost::make_function_output_iterator([&](const auto &value) { checksum += value; }));
+    }
+
+    BOOST_CHECK_EQUAL(checksum, reference_checksum);
+}
+#endif
+
 BOOST_AUTO_TEST_SUITE_END()