Bump RC to 4

This test case is disabled by default because it needs too much storage,
but serves as documentation for the future.

.travis.yml

@@ -15,6 +15,7 @@ branches:
     - master
     # enable building tags
     - /^v\d+\.\d+(\.\d+)?(-\S*)?$/
+    - "5.17"
 
 cache:
   yarn: true

CHANGELOG.md

@@ -1,4 +1,4 @@
-# UNRELEASED
+# 5.17.0
   - Changes from 5.16.0:
     - Bugfixes:
       - FIXED: deduplication of route steps when waypoints are used [#4909](https://github.com/Project-OSRM/osrm-backend/issues/4909)
@@ -6,6 +6,7 @@
       - FIXED: Remove the last short annotation segment in `trimShortSegments` [#4946](https://github.com/Project-OSRM/osrm-backend/pull/4946)
       - FIXED: Properly calculate annotations for speeds, durations and distances when waypoints are used with mapmatching [#4949](https://github.com/Project-OSRM/osrm-backend/pull/4949)
       - FIXED: Don't apply unimplemented SH and PH conditions in OpeningHours and add inversed date ranges [#4992](https://github.com/Project-OSRM/osrm-backend/issues/4992)
+      - FIXED: integer overflow in `DynamicGraph::Renumber` [#5021](https://github.com/Project-OSRM/osrm-backend/pull/5021)
     - Profile:
       - CHANGED: Handle oneways in get_forward_backward_by_key [#4929](https://github.com/Project-OSRM/osrm-backend/pull/4929)
       - FIXED: Do not route against oneway road if there is a cycleway in the wrong direction; also review bike profile [#4943](https://github.com/Project-OSRM/osrm-backend/issues/4943)

include/storage/shared_monitor.hpp

@@ -117,6 +117,19 @@ template <typename Data> struct SharedMonitor
 #endif
 
     static void remove() { bi::shared_memory_object::remove(Data::name); }
+    static bool exists()
+    {
+        try
+        {
+            bi::shared_memory_object shmem_open =
+                bi::shared_memory_object(bi::open_only, Data::name, bi::read_only);
+        }
+        catch (const bi::interprocess_exception &exception)
+        {
+            return false;
+        }
+        return true;
+    }
 
   private:
 #if USE_BOOST_INTERPROCESS_CONDITION

include/storage/tar.hpp

@@ -29,10 +29,8 @@ checkMTarError(int error_code, const boost::filesystem::path &filepath, const st
     case MTAR_ESUCCESS:
         return;
     case MTAR_EFAILURE:
-        throw util::RuntimeError(filepath.string() + " : " + name,
+        throw util::RuntimeError(
-                                 ErrorCode::FileIOError,
+            filepath.string() + " : " + name, ErrorCode::FileIOError, SOURCE_REF);
-                                 SOURCE_REF,
-                                 std::strerror(errno));
     case MTAR_EOPENFAIL:
         throw util::RuntimeError(filepath.string() + " : " + name,
                                  ErrorCode::FileOpenError,

include/util/dynamic_graph.hpp

@@ -2,6 +2,8 @@
 #define DYNAMICGRAPH_HPP
 
 #include "util/deallocating_vector.hpp"
+#include "util/exception.hpp"
+#include "util/exception_utils.hpp"
 #include "util/integer_range.hpp"
 #include "util/permutation.hpp"
 #include "util/typedefs.hpp"
@@ -411,7 +413,7 @@ template <typename EdgeDataT> class DynamicGraph
         util::inplacePermutation(node_array.begin(), node_array.end(), old_to_new_node);
 
         // Build up edge permutation
-        auto new_edge_index = 0;
+        EdgeID new_edge_index = 0;
         std::vector<EdgeID> old_to_new_edge(edge_list.size(), SPECIAL_EDGEID);
         for (auto node : util::irange<NodeID>(0, number_of_nodes))
         {
@@ -419,6 +421,11 @@ template <typename EdgeDataT> class DynamicGraph
             // move all filled edges
             for (auto edge : GetAdjacentEdgeRange(node))
             {
+                if (new_edge_index == std::numeric_limits<EdgeID>::max())
+                {
+                    throw util::exception("There are too many edges, OSRM only supports 2^32" +
+                                          SOURCE_REF);
+                }
                 edge_list[edge].target = old_to_new_node[edge_list[edge].target];
                 BOOST_ASSERT(edge_list[edge].target != SPECIAL_NODEID);
                 old_to_new_edge[edge] = new_edge_index++;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "osrm",
-  "version": "5.17.0-latest.1",
+  "version": "5.17.0-rc.4",
   "private": false,
   "description": "The Open Source Routing Machine is a high performance routing engine written in C++14 designed to run on OpenStreetMap data.",
   "dependencies": {

src/tools/store.cpp

@@ -27,12 +27,15 @@ void deleteRegion(const storage::SharedRegionRegister::ShmKey key)
 
 void listRegions()
 {
-
+    osrm::util::Log() << "name\tshm key\ttimestamp\tsize";
+    if (!storage::SharedMonitor<storage::SharedRegionRegister>::exists())
+    {
+        return;
+    }
     storage::SharedMonitor<storage::SharedRegionRegister> monitor;
     std::vector<std::string> names;
     const auto &shared_register = monitor.data();
     shared_register.List(std::back_inserter(names));
-    osrm::util::Log() << "name\tshm key\ttimestamp\tsize";
     for (const auto &name : names)
     {
         auto id = shared_register.Find(name);
@@ -105,8 +108,7 @@ bool generateDataStoreOptions(const int argc,
          boost::program_options::value<bool>(&list_datasets)
              ->default_value(false)
              ->implicit_value(true),
-         "Name of the dataset to load into memory. This allows having multiple datasets in memory "
+         "List all OSRM datasets currently in memory") //
-         "at the same time.") //
         ("only-metric",
          boost::program_options::value<bool>(&only_metric)
              ->default_value(false)

third_party/microtar/src/microtar.c

@@ -41,7 +41,7 @@ typedef struct {
 } mtar_raw_header_t;
 
 
-static unsigned round_up(unsigned n, unsigned incr) {
+static mtar_size_t round_up(mtar_size_t n, unsigned incr) {
   return n + (incr - n % incr) % incr;
 }
 
@@ -60,14 +60,14 @@ static unsigned checksum(const mtar_raw_header_t* rh) {
 }
 
 
-static int tread(mtar_t *tar, void *data, unsigned size) {
+static int tread(mtar_t *tar, void *data, mtar_size_t size) {
   int err = tar->read(tar, data, size);
   tar->pos += size;
   return err;
 }
 
 
-static int twrite(mtar_t *tar, const void *data, unsigned size) {
+static int twrite(mtar_t *tar, const void *data, mtar_size_t size) {
   int err = tar->write(tar, data, size);
   tar->pos += size;
   return err;
@@ -89,6 +89,7 @@ static int write_null_bytes(mtar_t *tar, int n) {
 
 static int raw_to_header(mtar_header_t *h, const mtar_raw_header_t *rh) {
   unsigned chksum1, chksum2;
+  mtar_size_t filesize;
 
   /* If the checksum starts with a null byte we assume the record is NULL */
   if (*rh->checksum == '\0') {
@@ -105,24 +106,68 @@ static int raw_to_header(mtar_header_t *h, const mtar_raw_header_t *rh) {
   /* Load raw header into header */
   sscanf(rh->mode, "%o", &h->mode);
   sscanf(rh->owner, "%o", &h->owner);
-  sscanf(rh->size, "%o", &h->size);
   sscanf(rh->mtime, "%o", &h->mtime);
   h->type = rh->type;
   strcpy(h->name, rh->name);
   strcpy(h->linkname, rh->linkname);
 
+  /* Load size field */
+  if ((rh->size[0] & 0x80) == 0) {
+#ifdef _MSC_VER
+      sscanf(rh->size, "%12llo", &h->size);
+#else
+      sscanf(rh->size, "%12lo", &h->size);
+#endif
+  } else {
+      h->size = (rh->size[0] & 0x7f) | (rh->size[0] & 0x40 ? 0x80 : 0);
+      uint8_t *p8 = (uint8_t *)&rh->size + 1;
+      while (p8 != (uint8_t *)&rh->size + sizeof(rh->size)) {
+          if (h->size >= ((mtar_size_t)1 << (sizeof(h->size) - 1) * 8)) {
+              return MTAR_EFAILURE;
+          }
+          h->size = ((mtar_size_t)h->size << 8) + *p8++;
+      }
+  }
+
   return MTAR_ESUCCESS;
 }
 
 
 static int header_to_raw(mtar_raw_header_t *rh, const mtar_header_t *h) {
   unsigned chksum;
+  mtar_size_t filesize = h->size;
 
   /* Load header into raw header */
   memset(rh, 0, sizeof(*rh));
+
+  /* Store size in ASCII octal digits or base-256 formats */
+  if (sizeof(mtar_size_t) <= 4 || filesize <= (mtar_size_t)077777777777LL) {
+#ifdef _MSC_VER
+      sprintf(rh->size, "%llo", h->size);
+#else
+      sprintf(rh->size, "%lo", h->size);
+#endif
+  } else if (sizeof(filesize) < sizeof(rh->size)) {
+      /* GNU tar uses "base-256 encoding" for very large numbers.
+       * Encoding is binary, with highest bit always set as a marker
+       * and sign in next-highest bit:
+       * 80 00 .. 00 - zero
+       * bf ff .. ff - largest positive number
+       * ff ff .. ff - minus 1
+       * c0 00 .. 00 - smallest negative number
+       */
+      uint8_t *p8 = (uint8_t *)&rh->size + sizeof(rh->size);
+      do {
+          *--p8 = (uint8_t)filesize;
+          filesize >>= 8;
+      } while (p8 != (uint8_t *)&rh->size);
+      *p8 |= 0x80;
+  } else {
+      return MTAR_EFAILURE;
+  }
+
   sprintf(rh->mode, "%o", h->mode);
   sprintf(rh->owner, "%o", h->owner);
-  sprintf(rh->size, "%o", h->size);
   sprintf(rh->mtime, "%o", h->mtime);
   rh->type = h->type ? h->type : MTAR_TREG;
   strcpy(rh->name, h->name);
@@ -153,17 +198,17 @@ const char* mtar_strerror(int err) {
 }
 
 
-static int file_write(mtar_t *tar, const void *data, unsigned size) {
+static int file_write(mtar_t *tar, const void *data, mtar_size_t size) {
-  unsigned res = fwrite(data, 1, size, tar->stream);
+  mtar_size_t res = fwrite(data, 1, size, tar->stream);
   return (res == size) ? MTAR_ESUCCESS : MTAR_EWRITEFAIL;
 }
 
-static int file_read(mtar_t *tar, void *data, unsigned size) {
+static int file_read(mtar_t *tar, void *data, mtar_size_t size) {
-  unsigned res = fread(data, 1, size, tar->stream);
+  mtar_size_t res = fread(data, 1, size, tar->stream);
   return (res == size) ? MTAR_ESUCCESS : MTAR_EREADFAIL;
 }
 
-static int file_seek(mtar_t *tar, unsigned offset) {
+static int file_seek(mtar_t *tar, mtar_size_t offset) {
   int res = fseek(tar->stream, offset, SEEK_SET);
   return (res == 0) ? MTAR_ESUCCESS : MTAR_ESEEKFAIL;
 }
@@ -213,7 +258,7 @@ int mtar_close(mtar_t *tar) {
 }
 
 
-int mtar_seek(mtar_t *tar, unsigned pos) {
+int mtar_seek(mtar_t *tar, mtar_size_t pos) {
   int err = tar->seek(tar, pos);
   tar->pos = pos;
   return err;
@@ -228,7 +273,8 @@ int mtar_rewind(mtar_t *tar) {
 
 
 int mtar_next(mtar_t *tar) {
-  int err, n;
+  mtar_size_t n;
+  int err;
   mtar_header_t h;
   /* Load header */
   err = mtar_read_header(tar, &h);
@@ -287,7 +333,7 @@ int mtar_read_header(mtar_t *tar, mtar_header_t *h) {
 }
 
 
-int mtar_read_data(mtar_t *tar, void *ptr, unsigned size) {
+int mtar_read_data(mtar_t *tar, void *ptr, mtar_size_t size) {
   int err;
   /* If we have no remaining data then this is the first read, we get the size,
    * set the remaining data and seek to the beginning of the data */
@@ -329,7 +375,7 @@ int mtar_write_header(mtar_t *tar, const mtar_header_t *h) {
 }
 
 
-int mtar_write_file_header(mtar_t *tar, const char *name, unsigned size) {
+int mtar_write_file_header(mtar_t *tar, const char *name, mtar_size_t size) {
   mtar_header_t h;
   /* Build header */
   memset(&h, 0, sizeof(h));
@@ -354,7 +400,7 @@ int mtar_write_dir_header(mtar_t *tar, const char *name) {
 }
 
 
-int mtar_write_data(mtar_t *tar, const void *data, unsigned size) {
+int mtar_write_data(mtar_t *tar, const void *data, mtar_size_t size) {
   int err;
   /* Write data */
   err = twrite(tar, data, size);

third_party/microtar/src/microtar.h

@@ -10,9 +10,12 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <stdint.h>
 
 #define MTAR_VERSION "0.1.0"
 
+typedef size_t mtar_size_t;
+
 enum {
   MTAR_ESUCCESS     =  0,
   MTAR_EFAILURE     = -1,
@@ -38,7 +41,7 @@ enum {
 typedef struct {
   unsigned mode;
   unsigned owner;
-  unsigned size;
+  mtar_size_t size;
   unsigned mtime;
   unsigned type;
   char name[100];
@@ -49,14 +52,14 @@ typedef struct {
 typedef struct mtar_t mtar_t;
 
 struct mtar_t {
-  int (*read)(mtar_t *tar, void *data, unsigned size);
+  int (*read)(mtar_t *tar, void *data, mtar_size_t size);
-  int (*write)(mtar_t *tar, const void *data, unsigned size);
+  int (*write)(mtar_t *tar, const void *data, mtar_size_t size);
-  int (*seek)(mtar_t *tar, unsigned pos);
+  int (*seek)(mtar_t *tar, mtar_size_t pos);
   int (*close)(mtar_t *tar);
   void *stream;
-  unsigned pos;
+  mtar_size_t pos;
-  unsigned remaining_data;
+  mtar_size_t remaining_data;
-  unsigned last_header;
+  mtar_size_t last_header;
 };
 
 
@@ -65,17 +68,17 @@ const char* mtar_strerror(int err);
 int mtar_open(mtar_t *tar, const char *filename, const char *mode);
 int mtar_close(mtar_t *tar);
 
-int mtar_seek(mtar_t *tar, unsigned pos);
+int mtar_seek(mtar_t *tar, mtar_size_t pos);
 int mtar_rewind(mtar_t *tar);
 int mtar_next(mtar_t *tar);
 int mtar_find(mtar_t *tar, const char *name, mtar_header_t *h);
 int mtar_read_header(mtar_t *tar, mtar_header_t *h);
-int mtar_read_data(mtar_t *tar, void *ptr, unsigned size);
+int mtar_read_data(mtar_t *tar, void *ptr, mtar_size_t size);
 
 int mtar_write_header(mtar_t *tar, const mtar_header_t *h);
-int mtar_write_file_header(mtar_t *tar, const char *name, unsigned size);
+int mtar_write_file_header(mtar_t *tar, const char *name, mtar_size_t size);
 int mtar_write_dir_header(mtar_t *tar, const char *name);
-int mtar_write_data(mtar_t *tar, const void *data, unsigned size);
+int mtar_write_data(mtar_t *tar, const void *data, mtar_size_t size);
 int mtar_finalize(mtar_t *tar);
 
 

unit_tests/storage/tar.cpp

@@ -3,6 +3,8 @@
 #include "../common/range_tools.hpp"
 #include "../common/temporary_file.hpp"
 
+#include <boost/function_output_iterator.hpp>
+#include <boost/iterator/function_input_iterator.hpp>
 #include <boost/test/unit_test.hpp>
 
 BOOST_AUTO_TEST_SUITE(tar)
@@ -187,4 +189,39 @@ BOOST_AUTO_TEST_CASE(continue_write_tar_file)
     CHECK_EQUAL_COLLECTIONS(result_64bit_vector, vector_64bit);
 }
 
+// Boost test only supports disabling was only introduced in 1.59
+#if BOOST_VERSION >= 105900
+// This test case is disabled by default because it needs 10 GiB of storage
+// Enable with ./storage-tests --run_test=tar/write_huge_tar_file
+BOOST_AUTO_TEST_CASE(write_huge_tar_file, *boost::unit_test::disabled())
+{
+    TemporaryFile tmp{TEST_DATA_DIR "/tar_huge_write_test.tar"};
+
+    std::uint64_t reference_checksum = 0;
+    {
+        storage::tar::FileWriter writer(tmp.path, storage::tar::FileWriter::GenerateFingerprint);
+        std::uint64_t value = 0;
+        const std::function<std::uint64_t()> encode_function = [&]() -> std::uint64_t {
+            reference_checksum += value;
+            return value++;
+        };
+        std::uint64_t num_elements = (10ULL * 1024ULL * 1024ULL * 1024ULL) / sizeof(std::uint64_t);
+        writer.WriteStreaming<std::uint64_t>(
+            "huge_data",
+            boost::make_function_input_iterator(encode_function, boost::infinite()),
+            num_elements);
+    }
+
+    std::uint64_t checksum = 0;
+    {
+        storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint);
+        reader.ReadStreaming<std::uint64_t>(
+            "huge_data",
+            boost::make_function_output_iterator([&](const auto &value) { checksum += value; }));
+    }
+
+    BOOST_CHECK_EQUAL(checksum, reference_checksum);
+}
+#endif
+
 BOOST_AUTO_TEST_SUITE_END()