First step towards reading/writing tar

CMakeLists.txt

@@ -166,7 +166,7 @@ add_executable(osrm-partition src/tools/partition.cpp)
 add_executable(osrm-customize src/tools/customize.cpp)
 add_executable(osrm-contract src/tools/contract.cpp)
 add_executable(osrm-routed src/tools/routed.cpp $<TARGET_OBJECTS:SERVER> $<TARGET_OBJECTS:UTIL>)
-add_executable(osrm-datastore src/tools/store.cpp $<TARGET_OBJECTS:UTIL>)
+add_executable(osrm-datastore src/tools/store.cpp $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
 add_library(osrm src/osrm/osrm.cpp $<TARGET_OBJECTS:ENGINE> $<TARGET_OBJECTS:UTIL> $<TARGET_OBJECTS:STORAGE>)
 add_library(osrm_contract src/osrm/contractor.cpp $<TARGET_OBJECTS:CONTRACTOR> $<TARGET_OBJECTS:UTIL>)
 add_library(osrm_extract src/osrm/extractor.cpp $<TARGET_OBJECTS:EXTRACTOR> $<TARGET_OBJECTS:UTIL>)
@@ -174,7 +174,7 @@ add_library(osrm_guidance $<TARGET_OBJECTS:GUIDANCE> $<TARGET_OBJECTS:UTIL>)
 add_library(osrm_partition src/osrm/partitioner.cpp $<TARGET_OBJECTS:PARTITIONER> $<TARGET_OBJECTS:UTIL>)
 add_library(osrm_customize src/osrm/customizer.cpp $<TARGET_OBJECTS:CUSTOMIZER> $<TARGET_OBJECTS:UTIL>)
 add_library(osrm_update $<TARGET_OBJECTS:UPDATER> $<TARGET_OBJECTS:UTIL>)
-add_library(osrm_store $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:UTIL>)
+add_library(osrm_store $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
 
 if(ENABLE_GOLD_LINKER)
     execute_process(COMMAND ${CMAKE_C_COMPILER} -fuse-ld=gold -Wl,--version ERROR_QUIET OUTPUT_VARIABLE LD_VERSION)
@@ -495,9 +495,6 @@ if(ENABLE_MASON)
   # note: we avoid calling find_package(Osmium ...) here to ensure that the
   # expat and bzip2 are used from mason rather than the system
   include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/libosmium/include)
-
-  include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/rapidjson/include)
-
 else()
 
   find_package(Boost 1.54 REQUIRED COMPONENTS ${BOOST_COMPONENTS})
@@ -555,12 +552,15 @@ else()
   endif()
   find_package(Osmium REQUIRED COMPONENTS io)
   include_directories(SYSTEM ${OSMIUM_INCLUDE_DIR})
-
-  set(RAPIDJSON_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/rapidjson/include")
-  include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
-
 endif()
 
+set(RAPIDJSON_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/rapidjson/include")
+include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
+
+set(MICROTAR_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/microtar/src")
+include_directories(SYSTEM ${MICROTAR_INCLUDE_DIR})
+add_library(MICROTAR OBJECT "${CMAKE_CURRENT_SOURCE_DIR}/third_party/microtar/src/microtar.c")
+
 # prefix compilation with ccache by default if available and on clang or gcc
 if(ENABLE_CCACHE AND (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU"))
   find_program(CCACHE_FOUND ccache)

include/storage/tar.hpp

@@ -0,0 +1,130 @@
+#ifndef OSRM_STORAGE_TAR_HPP
+#define OSRM_STORAGE_TAR_HPP
+
+#include "util/exception.hpp"
+#include "util/exception_utils.hpp"
+#include "util/fingerprint.hpp"
+#include "util/version.hpp"
+
+#include <boost/filesystem/path.hpp>
+
+extern "C" {
+#include "microtar.h"
+}
+
+namespace osrm
+{
+namespace storage
+{
+
+class TarFileReader
+{
+  public:
+    TarFileReader(const boost::filesystem::path &path) : path(path)
+    {
+        mtar_open(&handle, path.c_str(), "r");
+    }
+
+    template <typename T> T ReadOne(const std::string &name)
+    {
+        mtar_header_t header;
+        mtar_find(&handle, name.c_str(), &header);
+        if (header.size != sizeof(T))
+        {
+            throw util::exception("Datatype size does not match file size.");
+        }
+
+        T tmp;
+        mtar_read_data(&handle, reinterpret_cast<char *>(&tmp), header.size);
+        return tmp;
+    }
+
+    template <typename T>
+    void ReadInto(const std::string &name, T &data, const std::size_t number_of_entries)
+    {
+        mtar_header_t header;
+        mtar_find(&handle, name.c_str(), &header);
+        if (header.size != sizeof(T) * number_of_entries)
+        {
+            throw util::exception("Datatype size does not match file size.");
+        }
+
+        mtar_read_data(&handle, reinterpret_cast<char *>(&data), header.size);
+    }
+
+    using TarEntry = std::tuple<std::string, std::size_t>;
+    template <typename OutIter> void List(OutIter out)
+    {
+        mtar_header_t header;
+        while ((mtar_read_header(&handle, &header)) != MTAR_ENULLRECORD)
+        {
+            *out++ = std::tuple<std::string, std::uint64_t>(header.name, header.size);
+        }
+    }
+
+  private:
+    bool ReadAndCheckFingerprint()
+    {
+        auto loaded_fingerprint = ReadOne<util::FingerPrint>("osrm_fingerprint");
+        const auto expected_fingerprint = util::FingerPrint::GetValid();
+
+        if (!loaded_fingerprint.IsValid())
+        {
+            throw util::RuntimeError(path.string(), ErrorCode::InvalidFingerprint, SOURCE_REF);
+        }
+
+        if (!expected_fingerprint.IsDataCompatible(loaded_fingerprint))
+        {
+            const std::string fileversion =
+                std::to_string(loaded_fingerprint.GetMajorVersion()) + "." +
+                std::to_string(loaded_fingerprint.GetMinorVersion()) + "." +
+                std::to_string(loaded_fingerprint.GetPatchVersion());
+            throw util::RuntimeError(std::string(path.string()) + " prepared with OSRM " +
+                                         fileversion + " but this is " + OSRM_VERSION,
+                                     ErrorCode::IncompatibleFileVersion,
+                                     SOURCE_REF);
+        }
+
+        return true;
+    }
+
+    boost::filesystem::path path;
+    mtar_t handle;
+};
+
+class TarFileWriter
+{
+  public:
+    TarFileWriter(const boost::filesystem::path &path) : path(path)
+    {
+        mtar_open(&handle, path.c_str(), "w");
+        WriteFingerprint();
+    }
+
+    template <typename T> void WriteOne(const std::string &name, const T &data)
+    {
+        mtar_write_file_header(&handle, name.c_str(), name.size());
+        mtar_write_data(&handle, reinterpret_cast<const char *>(&data), sizeof(T));
+    }
+
+    template <typename T>
+    void WriteFrom(const std::string &name, const T &data, const std::size_t number_of_entries)
+    {
+        mtar_write_file_header(&handle, name.c_str(), name.size());
+        mtar_write_data(&handle, reinterpret_cast<const char *>(&data), number_of_entries * sizeof(T));
+    }
+
+  private:
+    void WriteFingerprint()
+    {
+        const auto fingerprint = util::FingerPrint::GetValid();
+        WriteOne("osrm_fingerprint", fingerprint);
+    }
+
+    boost::filesystem::path path;
+    mtar_t handle;
+};
+}
+}
+
+#endif

unit_tests/CMakeLists.txt

@@ -22,6 +22,10 @@ file(GLOB UpdaterTestsSources
     updater_tests.cpp
     updater/*.cpp)
 
+file(GLOB StorageTestsSources
+    storage_tests.cpp
+    storage/*.cpp)
+
 file(GLOB LibraryTestsSources
     library_tests.cpp
     library/*.cpp)
@@ -85,6 +89,11 @@ add_executable(updater-tests
     ${UpdaterTestsSources}
     $<TARGET_OBJECTS:UPDATER> $<TARGET_OBJECTS:UPDATER> $<TARGET_OBJECTS:UTIL>)
 
+add_executable(storage-tests
+	EXCLUDE_FROM_ALL
+    ${StorageTestsSources}
+    $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:UTIL>)
+
 add_executable(library-tests
 	EXCLUDE_FROM_ALL
 	${LibraryTestsSources})
@@ -136,6 +145,7 @@ target_compile_definitions(library-customize-tests PRIVATE COMPILE_DEFINITIONS O
 target_compile_definitions(library-partition-tests PRIVATE COMPILE_DEFINITIONS OSRM_TEST_DATA_DIR="${TEST_DATA_DIR}")
 target_compile_definitions(updater-tests PRIVATE COMPILE_DEFINITIONS TEST_DATA_DIR="${UPDATER_TEST_DATA_DIR}")
 target_compile_definitions(contractor-tests PRIVATE COMPILE_DEFINITIONS TEST_DATA_DIR="${TEST_DATA_DIR}")
+target_compile_definitions(storage-tests PRIVATE COMPILE_DEFINITIONS TEST_DATA_DIR="${TEST_DATA_DIR}")
 
 target_include_directories(engine-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_include_directories(library-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
@@ -148,6 +158,7 @@ target_include_directories(partitioner-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_include_directories(customizer-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_include_directories(updater-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_include_directories(contractor-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+target_include_directories(storage-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 
 target_link_libraries(engine-tests ${ENGINE_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(extractor-tests osrm_extract osrm_guidance ${EXTRACTOR_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
@@ -162,6 +173,7 @@ target_link_libraries(library-partition-tests osrm_partition ${Boost_UNIT_TEST_F
 target_link_libraries(server-tests osrm ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(util-tests ${UTIL_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(contractor-tests ${CONTRACTOR_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
+target_link_libraries(storage-tests osrm_store ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 
 add_custom_target(tests
-	DEPENDS engine-tests extractor-tests contractor-tests partitioner-tests updater-tests customizer-tests library-tests library-extract-tests library-contract-tests library-customize-tests library-partition-tests server-tests util-tests)
+	DEPENDS engine-tests extractor-tests contractor-tests partitioner-tests updater-tests customizer-tests library-tests library-extract-tests library-contract-tests library-customize-tests library-partition-tests server-tests util-tests storage-tests)

unit_tests/storage_tests.cpp

@@ -0,0 +1,7 @@
+#define BOOST_TEST_MODULE storage tests
+
+#include <boost/test/unit_test.hpp>
+
+/*
+ * This file will contain an automatically generated main function.
+ */