From 86bfe1ede102351eaf652d4d6ebd1952c6deb379 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 14 Mar 2018 09:51:41 +0000 Subject: [PATCH] First step towards reading/writing tar --- CMakeLists.txt | 18 ++--- include/storage/tar.hpp | 130 +++++++++++++++++++++++++++++++++++ unit_tests/CMakeLists.txt | 14 +++- unit_tests/storage_tests.cpp | 7 ++ 4 files changed, 159 insertions(+), 10 deletions(-) create mode 100644 include/storage/tar.hpp create mode 100644 unit_tests/storage_tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f26c061c..e922c9ddb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -166,7 +166,7 @@ add_executable(osrm-partition src/tools/partition.cpp) add_executable(osrm-customize src/tools/customize.cpp) add_executable(osrm-contract src/tools/contract.cpp) add_executable(osrm-routed src/tools/routed.cpp $ $) -add_executable(osrm-datastore src/tools/store.cpp $) +add_executable(osrm-datastore src/tools/store.cpp $ $) add_library(osrm src/osrm/osrm.cpp $ $ $) add_library(osrm_contract src/osrm/contractor.cpp $ $) add_library(osrm_extract src/osrm/extractor.cpp $ $) @@ -174,7 +174,7 @@ add_library(osrm_guidance $ $) add_library(osrm_partition src/osrm/partitioner.cpp $ $) add_library(osrm_customize src/osrm/customizer.cpp $ $) add_library(osrm_update $ $) -add_library(osrm_store $ $) +add_library(osrm_store $ $ $) if(ENABLE_GOLD_LINKER) execute_process(COMMAND ${CMAKE_C_COMPILER} -fuse-ld=gold -Wl,--version ERROR_QUIET OUTPUT_VARIABLE LD_VERSION) @@ -495,9 +495,6 @@ if(ENABLE_MASON) # note: we avoid calling find_package(Osmium ...) here to ensure that the # expat and bzip2 are used from mason rather than the system include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/libosmium/include) - - include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/rapidjson/include) - else() find_package(Boost 1.54 REQUIRED COMPONENTS ${BOOST_COMPONENTS}) @@ -555,12 +552,15 @@ else() endif() find_package(Osmium REQUIRED COMPONENTS io) include_directories(SYSTEM ${OSMIUM_INCLUDE_DIR}) - - set(RAPIDJSON_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/rapidjson/include") - include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR}) - endif() +set(RAPIDJSON_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/rapidjson/include") +include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR}) + +set(MICROTAR_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/microtar/src") +include_directories(SYSTEM ${MICROTAR_INCLUDE_DIR}) +add_library(MICROTAR OBJECT "${CMAKE_CURRENT_SOURCE_DIR}/third_party/microtar/src/microtar.c") + # prefix compilation with ccache by default if available and on clang or gcc if(ENABLE_CCACHE AND (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU")) find_program(CCACHE_FOUND ccache) diff --git a/include/storage/tar.hpp b/include/storage/tar.hpp new file mode 100644 index 000000000..8a3ce92e7 --- /dev/null +++ b/include/storage/tar.hpp @@ -0,0 +1,130 @@ +#ifndef OSRM_STORAGE_TAR_HPP +#define OSRM_STORAGE_TAR_HPP + +#include "util/exception.hpp" +#include "util/exception_utils.hpp" +#include "util/fingerprint.hpp" +#include "util/version.hpp" + +#include + +extern "C" { +#include "microtar.h" +} + +namespace osrm +{ +namespace storage +{ + +class TarFileReader +{ + public: + TarFileReader(const boost::filesystem::path &path) : path(path) + { + mtar_open(&handle, path.c_str(), "r"); + } + + template T ReadOne(const std::string &name) + { + mtar_header_t header; + mtar_find(&handle, name.c_str(), &header); + if (header.size != sizeof(T)) + { + throw util::exception("Datatype size does not match file size."); + } + + T tmp; + mtar_read_data(&handle, reinterpret_cast(&tmp), header.size); + return tmp; + } + + template + void ReadInto(const std::string &name, T &data, const std::size_t number_of_entries) + { + mtar_header_t header; + mtar_find(&handle, name.c_str(), &header); + if (header.size != sizeof(T) * number_of_entries) + { + throw util::exception("Datatype size does not match file size."); + } + + mtar_read_data(&handle, reinterpret_cast(&data), header.size); + } + + using TarEntry = std::tuple; + template void List(OutIter out) + { + mtar_header_t header; + while ((mtar_read_header(&handle, &header)) != MTAR_ENULLRECORD) + { + *out++ = std::tuple(header.name, header.size); + } + } + + private: + bool ReadAndCheckFingerprint() + { + auto loaded_fingerprint = ReadOne("osrm_fingerprint"); + const auto expected_fingerprint = util::FingerPrint::GetValid(); + + if (!loaded_fingerprint.IsValid()) + { + throw util::RuntimeError(path.string(), ErrorCode::InvalidFingerprint, SOURCE_REF); + } + + if (!expected_fingerprint.IsDataCompatible(loaded_fingerprint)) + { + const std::string fileversion = + std::to_string(loaded_fingerprint.GetMajorVersion()) + "." + + std::to_string(loaded_fingerprint.GetMinorVersion()) + "." + + std::to_string(loaded_fingerprint.GetPatchVersion()); + throw util::RuntimeError(std::string(path.string()) + " prepared with OSRM " + + fileversion + " but this is " + OSRM_VERSION, + ErrorCode::IncompatibleFileVersion, + SOURCE_REF); + } + + return true; + } + + boost::filesystem::path path; + mtar_t handle; +}; + +class TarFileWriter +{ + public: + TarFileWriter(const boost::filesystem::path &path) : path(path) + { + mtar_open(&handle, path.c_str(), "w"); + WriteFingerprint(); + } + + template void WriteOne(const std::string &name, const T &data) + { + mtar_write_file_header(&handle, name.c_str(), name.size()); + mtar_write_data(&handle, reinterpret_cast(&data), sizeof(T)); + } + + template + void WriteFrom(const std::string &name, const T &data, const std::size_t number_of_entries) + { + mtar_write_file_header(&handle, name.c_str(), name.size()); + mtar_write_data(&handle, reinterpret_cast(&data), number_of_entries * sizeof(T)); + } + + private: + void WriteFingerprint() + { + const auto fingerprint = util::FingerPrint::GetValid(); + WriteOne("osrm_fingerprint", fingerprint); + } + + boost::filesystem::path path; + mtar_t handle; +}; +} +} + +#endif diff --git a/unit_tests/CMakeLists.txt b/unit_tests/CMakeLists.txt index ba39a71f0..ac4bbdf85 100644 --- a/unit_tests/CMakeLists.txt +++ b/unit_tests/CMakeLists.txt @@ -22,6 +22,10 @@ file(GLOB UpdaterTestsSources updater_tests.cpp updater/*.cpp) +file(GLOB StorageTestsSources + storage_tests.cpp + storage/*.cpp) + file(GLOB LibraryTestsSources library_tests.cpp library/*.cpp) @@ -85,6 +89,11 @@ add_executable(updater-tests ${UpdaterTestsSources} $ $ $) +add_executable(storage-tests + EXCLUDE_FROM_ALL + ${StorageTestsSources} + $ $) + add_executable(library-tests EXCLUDE_FROM_ALL ${LibraryTestsSources}) @@ -136,6 +145,7 @@ target_compile_definitions(library-customize-tests PRIVATE COMPILE_DEFINITIONS O target_compile_definitions(library-partition-tests PRIVATE COMPILE_DEFINITIONS OSRM_TEST_DATA_DIR="${TEST_DATA_DIR}") target_compile_definitions(updater-tests PRIVATE COMPILE_DEFINITIONS TEST_DATA_DIR="${UPDATER_TEST_DATA_DIR}") target_compile_definitions(contractor-tests PRIVATE COMPILE_DEFINITIONS TEST_DATA_DIR="${TEST_DATA_DIR}") +target_compile_definitions(storage-tests PRIVATE COMPILE_DEFINITIONS TEST_DATA_DIR="${TEST_DATA_DIR}") target_include_directories(engine-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(library-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) @@ -148,6 +158,7 @@ target_include_directories(partitioner-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(customizer-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(updater-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(contractor-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories(storage-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(engine-tests ${ENGINE_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY}) target_link_libraries(extractor-tests osrm_extract osrm_guidance ${EXTRACTOR_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY}) @@ -162,6 +173,7 @@ target_link_libraries(library-partition-tests osrm_partition ${Boost_UNIT_TEST_F target_link_libraries(server-tests osrm ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY}) target_link_libraries(util-tests ${UTIL_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY}) target_link_libraries(contractor-tests ${CONTRACTOR_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY}) +target_link_libraries(storage-tests osrm_store ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY}) add_custom_target(tests - DEPENDS engine-tests extractor-tests contractor-tests partitioner-tests updater-tests customizer-tests library-tests library-extract-tests library-contract-tests library-customize-tests library-partition-tests server-tests util-tests) + DEPENDS engine-tests extractor-tests contractor-tests partitioner-tests updater-tests customizer-tests library-tests library-extract-tests library-contract-tests library-customize-tests library-partition-tests server-tests util-tests storage-tests) diff --git a/unit_tests/storage_tests.cpp b/unit_tests/storage_tests.cpp new file mode 100644 index 000000000..05d43ddbe --- /dev/null +++ b/unit_tests/storage_tests.cpp @@ -0,0 +1,7 @@ +#define BOOST_TEST_MODULE storage tests + +#include + +/* + * This file will contain an automatically generated main function. + */