Bump version to 5.17.0-rc.2

This test case is disabled by default because it needs too much storage,
but serves as documentation for the future.

.travis.yml

@@ -15,6 +15,7 @@ branches:
     - master
     # enable building tags
     - /^v\d+\.\d+(\.\d+)?(-\S*)?$/
+    - "5.17"
 
 cache:
   yarn: true

CHANGELOG.md

@@ -1,4 +1,4 @@
-# UNRELEASED
+# 5.17.0
   - Changes from 5.16.0:
     - Bugfixes:
       - FIXED: deduplication of route steps when waypoints are used [#4909](https://github.com/Project-OSRM/osrm-backend/issues/4909)

include/storage/tar.hpp

@@ -29,10 +29,8 @@ checkMTarError(int error_code, const boost::filesystem::path &filepath, const st
     case MTAR_ESUCCESS:
         return;
     case MTAR_EFAILURE:
-        throw util::RuntimeError(filepath.string() + " : " + name,
-                                 ErrorCode::FileIOError,
-                                 SOURCE_REF,
-                                 std::strerror(errno));
+        throw util::RuntimeError(
+            filepath.string() + " : " + name, ErrorCode::FileIOError, SOURCE_REF);
     case MTAR_EOPENFAIL:
         throw util::RuntimeError(filepath.string() + " : " + name,
                                  ErrorCode::FileOpenError,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "osrm",
-  "version": "5.17.0-latest.1",
+  "version": "5.17.0-rc.2",
   "private": false,
   "description": "The Open Source Routing Machine is a high performance routing engine written in C++14 designed to run on OpenStreetMap data.",
   "dependencies": {

third_party/microtar/src/microtar.c

@@ -41,7 +41,7 @@ typedef struct {
 } mtar_raw_header_t;
 
 
-static unsigned round_up(unsigned n, unsigned incr) {
+static mtar_size_t round_up(mtar_size_t n, unsigned incr) {
   return n + (incr - n % incr) % incr;
 }
 
@@ -60,14 +60,14 @@ static unsigned checksum(const mtar_raw_header_t* rh) {
 }
 
 
-static int tread(mtar_t *tar, void *data, unsigned size) {
+static int tread(mtar_t *tar, void *data, mtar_size_t size) {
   int err = tar->read(tar, data, size);
   tar->pos += size;
   return err;
 }
 
 
-static int twrite(mtar_t *tar, const void *data, unsigned size) {
+static int twrite(mtar_t *tar, const void *data, mtar_size_t size) {
   int err = tar->write(tar, data, size);
   tar->pos += size;
   return err;
@@ -89,6 +89,7 @@ static int write_null_bytes(mtar_t *tar, int n) {
 
 static int raw_to_header(mtar_header_t *h, const mtar_raw_header_t *rh) {
   unsigned chksum1, chksum2;
+  mtar_size_t filesize;
 
   /* If the checksum starts with a null byte we assume the record is NULL */
   if (*rh->checksum == '\0') {
@@ -105,24 +106,68 @@ static int raw_to_header(mtar_header_t *h, const mtar_raw_header_t *rh) {
   /* Load raw header into header */
   sscanf(rh->mode, "%o", &h->mode);
   sscanf(rh->owner, "%o", &h->owner);
-  sscanf(rh->size, "%o", &h->size);
   sscanf(rh->mtime, "%o", &h->mtime);
   h->type = rh->type;
   strcpy(h->name, rh->name);
   strcpy(h->linkname, rh->linkname);
 
+  /* Load size field */
+  if ((rh->size[0] & 0x80) == 0) {
+#ifdef _MSC_VER
+      sscanf(rh->size, "%12llo", &h->size);
+#else
+      sscanf(rh->size, "%12lo", &h->size);
+#endif
+  } else {
+      h->size = (rh->size[0] & 0x7f) | (rh->size[0] & 0x40 ? 0x80 : 0);
+      uint8_t *p8 = (uint8_t *)&rh->size + 1;
+      while (p8 != (uint8_t *)&rh->size + sizeof(rh->size)) {
+          if (h->size >= ((mtar_size_t)1 << (sizeof(h->size) - 1) * 8)) {
+              return MTAR_EFAILURE;
+          }
+          h->size = ((mtar_size_t)h->size << 8) + *p8++;
+      }
+  }
+
   return MTAR_ESUCCESS;
 }
 
 
 static int header_to_raw(mtar_raw_header_t *rh, const mtar_header_t *h) {
   unsigned chksum;
+  mtar_size_t filesize = h->size;
 
   /* Load header into raw header */
   memset(rh, 0, sizeof(*rh));
+
+  /* Store size in ASCII octal digits or base-256 formats */
+  if (sizeof(mtar_size_t) <= 4 || filesize <= (mtar_size_t)077777777777LL) {
+#ifdef _MSC_VER
+      sprintf(rh->size, "%llo", h->size);
+#else
+      sprintf(rh->size, "%lo", h->size);
+#endif
+  } else if (sizeof(filesize) < sizeof(rh->size)) {
+      /* GNU tar uses "base-256 encoding" for very large numbers.
+       * Encoding is binary, with highest bit always set as a marker
+       * and sign in next-highest bit:
+       * 80 00 .. 00 - zero
+       * bf ff .. ff - largest positive number
+       * ff ff .. ff - minus 1
+       * c0 00 .. 00 - smallest negative number
+       */
+      uint8_t *p8 = (uint8_t *)&rh->size + sizeof(rh->size);
+      do {
+          *--p8 = (uint8_t)filesize;
+          filesize >>= 8;
+      } while (p8 != (uint8_t *)&rh->size);
+      *p8 |= 0x80;
+  } else {
+      return MTAR_EFAILURE;
+  }
+
   sprintf(rh->mode, "%o", h->mode);
   sprintf(rh->owner, "%o", h->owner);
-  sprintf(rh->size, "%o", h->size);
   sprintf(rh->mtime, "%o", h->mtime);
   rh->type = h->type ? h->type : MTAR_TREG;
   strcpy(rh->name, h->name);
@@ -153,17 +198,17 @@ const char* mtar_strerror(int err) {
 }
 
 
-static int file_write(mtar_t *tar, const void *data, unsigned size) {
-  unsigned res = fwrite(data, 1, size, tar->stream);
+static int file_write(mtar_t *tar, const void *data, mtar_size_t size) {
+  mtar_size_t res = fwrite(data, 1, size, tar->stream);
   return (res == size) ? MTAR_ESUCCESS : MTAR_EWRITEFAIL;
 }
 
-static int file_read(mtar_t *tar, void *data, unsigned size) {
-  unsigned res = fread(data, 1, size, tar->stream);
+static int file_read(mtar_t *tar, void *data, mtar_size_t size) {
+  mtar_size_t res = fread(data, 1, size, tar->stream);
   return (res == size) ? MTAR_ESUCCESS : MTAR_EREADFAIL;
 }
 
-static int file_seek(mtar_t *tar, unsigned offset) {
+static int file_seek(mtar_t *tar, mtar_size_t offset) {
   int res = fseek(tar->stream, offset, SEEK_SET);
   return (res == 0) ? MTAR_ESUCCESS : MTAR_ESEEKFAIL;
 }
@@ -213,7 +258,7 @@ int mtar_close(mtar_t *tar) {
 }
 
 
-int mtar_seek(mtar_t *tar, unsigned pos) {
+int mtar_seek(mtar_t *tar, mtar_size_t pos) {
   int err = tar->seek(tar, pos);
   tar->pos = pos;
   return err;
@@ -228,7 +273,8 @@ int mtar_rewind(mtar_t *tar) {
 
 
 int mtar_next(mtar_t *tar) {
-  int err, n;
+  mtar_size_t n;
+  int err;
   mtar_header_t h;
   /* Load header */
   err = mtar_read_header(tar, &h);
@@ -287,7 +333,7 @@ int mtar_read_header(mtar_t *tar, mtar_header_t *h) {
 }
 
 
-int mtar_read_data(mtar_t *tar, void *ptr, unsigned size) {
+int mtar_read_data(mtar_t *tar, void *ptr, mtar_size_t size) {
   int err;
   /* If we have no remaining data then this is the first read, we get the size,
    * set the remaining data and seek to the beginning of the data */
@@ -329,7 +375,7 @@ int mtar_write_header(mtar_t *tar, const mtar_header_t *h) {
 }
 
 
-int mtar_write_file_header(mtar_t *tar, const char *name, unsigned size) {
+int mtar_write_file_header(mtar_t *tar, const char *name, mtar_size_t size) {
   mtar_header_t h;
   /* Build header */
   memset(&h, 0, sizeof(h));
@@ -354,7 +400,7 @@ int mtar_write_dir_header(mtar_t *tar, const char *name) {
 }
 
 
-int mtar_write_data(mtar_t *tar, const void *data, unsigned size) {
+int mtar_write_data(mtar_t *tar, const void *data, mtar_size_t size) {
   int err;
   /* Write data */
   err = twrite(tar, data, size);

third_party/microtar/src/microtar.h

@@ -10,9 +10,12 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <stdint.h>
 
 #define MTAR_VERSION "0.1.0"
 
+typedef size_t mtar_size_t;
+
 enum {
   MTAR_ESUCCESS     =  0,
   MTAR_EFAILURE     = -1,
@@ -38,7 +41,7 @@ enum {
 typedef struct {
   unsigned mode;
   unsigned owner;
-  unsigned size;
+  mtar_size_t size;
   unsigned mtime;
   unsigned type;
   char name[100];
@@ -49,14 +52,14 @@ typedef struct {
 typedef struct mtar_t mtar_t;
 
 struct mtar_t {
-  int (*read)(mtar_t *tar, void *data, unsigned size);
-  int (*write)(mtar_t *tar, const void *data, unsigned size);
-  int (*seek)(mtar_t *tar, unsigned pos);
+  int (*read)(mtar_t *tar, void *data, mtar_size_t size);
+  int (*write)(mtar_t *tar, const void *data, mtar_size_t size);
+  int (*seek)(mtar_t *tar, mtar_size_t pos);
   int (*close)(mtar_t *tar);
   void *stream;
-  unsigned pos;
-  unsigned remaining_data;
-  unsigned last_header;
+  mtar_size_t pos;
+  mtar_size_t remaining_data;
+  mtar_size_t last_header;
 };
 
 
@@ -65,17 +68,17 @@ const char* mtar_strerror(int err);
 int mtar_open(mtar_t *tar, const char *filename, const char *mode);
 int mtar_close(mtar_t *tar);
 
-int mtar_seek(mtar_t *tar, unsigned pos);
+int mtar_seek(mtar_t *tar, mtar_size_t pos);
 int mtar_rewind(mtar_t *tar);
 int mtar_next(mtar_t *tar);
 int mtar_find(mtar_t *tar, const char *name, mtar_header_t *h);
 int mtar_read_header(mtar_t *tar, mtar_header_t *h);
-int mtar_read_data(mtar_t *tar, void *ptr, unsigned size);
+int mtar_read_data(mtar_t *tar, void *ptr, mtar_size_t size);
 
 int mtar_write_header(mtar_t *tar, const mtar_header_t *h);
-int mtar_write_file_header(mtar_t *tar, const char *name, unsigned size);
+int mtar_write_file_header(mtar_t *tar, const char *name, mtar_size_t size);
 int mtar_write_dir_header(mtar_t *tar, const char *name);
-int mtar_write_data(mtar_t *tar, const void *data, unsigned size);
+int mtar_write_data(mtar_t *tar, const void *data, mtar_size_t size);
 int mtar_finalize(mtar_t *tar);
 
 

unit_tests/storage/tar.cpp

@@ -3,6 +3,8 @@
 #include "../common/range_tools.hpp"
 #include "../common/temporary_file.hpp"
 
+#include <boost/function_output_iterator.hpp>
+#include <boost/iterator/function_input_iterator.hpp>
 #include <boost/test/unit_test.hpp>
 
 BOOST_AUTO_TEST_SUITE(tar)
@@ -187,4 +189,39 @@ BOOST_AUTO_TEST_CASE(continue_write_tar_file)
     CHECK_EQUAL_COLLECTIONS(result_64bit_vector, vector_64bit);
 }
 
+// Boost test only supports disabling was only introduced in 1.59
+#if BOOST_VERSION >= 105900
+// This test case is disabled by default because it needs 10 GiB of storage
+// Enable with ./storage-tests --run_test=tar/write_huge_tar_file
+BOOST_AUTO_TEST_CASE(write_huge_tar_file, *boost::unit_test::disabled())
+{
+    TemporaryFile tmp{TEST_DATA_DIR "/tar_huge_write_test.tar"};
+
+    std::uint64_t reference_checksum = 0;
+    {
+        storage::tar::FileWriter writer(tmp.path, storage::tar::FileWriter::GenerateFingerprint);
+        std::uint64_t value = 0;
+        const std::function<std::uint64_t()> encode_function = [&]() -> std::uint64_t {
+            reference_checksum += value;
+            return value++;
+        };
+        std::uint64_t num_elements = (10ULL * 1024ULL * 1024ULL * 1024ULL) / sizeof(std::uint64_t);
+        writer.WriteStreaming<std::uint64_t>(
+            "huge_data",
+            boost::make_function_input_iterator(encode_function, boost::infinite()),
+            num_elements);
+    }
+
+    std::uint64_t checksum = 0;
+    {
+        storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint);
+        reader.ReadStreaming<std::uint64_t>(
+            "huge_data",
+            boost::make_function_output_iterator([&](const auto &value) { checksum += value; }));
+    }
+
+    BOOST_CHECK_EQUAL(checksum, reference_checksum);
+}
+#endif
+
 BOOST_AUTO_TEST_SUITE_END()