Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

First step towards reading/writing tar

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-14 09:51:41 +00:00
parent c04e1b2ded
commit 86bfe1ede1
4 changed files with 159 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
CMakeLists.txt
View File

@ -166,7 +166,7 @@ add_executable(osrm-partition src/tools/partition.cpp)
add_executable(osrm-customize src/tools/customize.cpp)
add_executable(osrm-contract src/tools/contract.cpp)
add_executable(osrm-routed src/tools/routed.cpp $<TARGET_OBJECTS:SERVER> $<TARGET_OBJECTS:UTIL>)
add_executable(osrm-datastore src/tools/store.cpp $<TARGET_OBJECTS:UTIL>)
add_executable(osrm-datastore src/tools/store.cpp $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
add_library(osrm src/osrm/osrm.cpp $<TARGET_OBJECTS:ENGINE> $<TARGET_OBJECTS:UTIL> $<TARGET_OBJECTS:STORAGE>)
add_library(osrm_contract src/osrm/contractor.cpp $<TARGET_OBJECTS:CONTRACTOR> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_extract src/osrm/extractor.cpp $<TARGET_OBJECTS:EXTRACTOR> $<TARGET_OBJECTS:UTIL>)
@ -174,7 +174,7 @@ add_library(osrm_guidance $<TARGET_OBJECTS:GUIDANCE> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_partition src/osrm/partitioner.cpp $<TARGET_OBJECTS:PARTITIONER> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_customize src/osrm/customizer.cpp $<TARGET_OBJECTS:CUSTOMIZER> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_update $<TARGET_OBJECTS:UPDATER> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_store $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_store $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
if(ENABLE_GOLD_LINKER)
execute_process(COMMAND ${CMAKE_C_COMPILER} -fuse-ld=gold -Wl,--version ERROR_QUIET OUTPUT_VARIABLE LD_VERSION)
@ -495,9 +495,6 @@ if(ENABLE_MASON)
# note: we avoid calling find_package(Osmium ...) here to ensure that the
# expat and bzip2 are used from mason rather than the system
include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/libosmium/include)
include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/rapidjson/include)
else()
find_package(Boost 1.54 REQUIRED COMPONENTS ${BOOST_COMPONENTS})
@ -555,12 +552,15 @@ else()
endif()
find_package(Osmium REQUIRED COMPONENTS io)
include_directories(SYSTEM ${OSMIUM_INCLUDE_DIR})
set(RAPIDJSON_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/rapidjson/include")
include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
endif()
set(RAPIDJSON_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/rapidjson/include")
include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
set(MICROTAR_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/microtar/src")
include_directories(SYSTEM ${MICROTAR_INCLUDE_DIR})
add_library(MICROTAR OBJECT "${CMAKE_CURRENT_SOURCE_DIR}/third_party/microtar/src/microtar.c")
# prefix compilation with ccache by default if available and on clang or gcc
if(ENABLE_CCACHE AND (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU"))
find_program(CCACHE_FOUND ccache)

130
include/storage/tar.hpp Normal file
View File

@ -0,0 +1,130 @@
#ifndef OSRM_STORAGE_TAR_HPP
#define OSRM_STORAGE_TAR_HPP
#include "util/exception.hpp"
#include "util/exception_utils.hpp"
#include "util/fingerprint.hpp"
#include "util/version.hpp"
#include <boost/filesystem/path.hpp>
extern "C" {
#include "microtar.h"
}
namespace osrm
{
namespace storage
{
class TarFileReader
{
public:
TarFileReader(const boost::filesystem::path &path) : path(path)
{
mtar_open(&handle, path.c_str(), "r");
}
template <typename T> T ReadOne(const std::string &name)
{
mtar_header_t header;
mtar_find(&handle, name.c_str(), &header);
if (header.size != sizeof(T))
{
throw util::exception("Datatype size does not match file size.");
}
T tmp;
mtar_read_data(&handle, reinterpret_cast<char *>(&tmp), header.size);
return tmp;
}
template <typename T>
void ReadInto(const std::string &name, T &data, const std::size_t number_of_entries)
{
mtar_header_t header;
mtar_find(&handle, name.c_str(), &header);
if (header.size != sizeof(T) * number_of_entries)
{
throw util::exception("Datatype size does not match file size.");
}
mtar_read_data(&handle, reinterpret_cast<char *>(&data), header.size);
}
using TarEntry = std::tuple<std::string, std::size_t>;
template <typename OutIter> void List(OutIter out)
{
mtar_header_t header;
while ((mtar_read_header(&handle, &header)) != MTAR_ENULLRECORD)
{
*out++ = std::tuple<std::string, std::uint64_t>(header.name, header.size);
}
}
private:
bool ReadAndCheckFingerprint()
{
auto loaded_fingerprint = ReadOne<util::FingerPrint>("osrm_fingerprint");
const auto expected_fingerprint = util::FingerPrint::GetValid();
if (!loaded_fingerprint.IsValid())
{
throw util::RuntimeError(path.string(), ErrorCode::InvalidFingerprint, SOURCE_REF);
}
if (!expected_fingerprint.IsDataCompatible(loaded_fingerprint))
{
const std::string fileversion =
std::to_string(loaded_fingerprint.GetMajorVersion()) + "." +
std::to_string(loaded_fingerprint.GetMinorVersion()) + "." +
std::to_string(loaded_fingerprint.GetPatchVersion());
throw util::RuntimeError(std::string(path.string()) + " prepared with OSRM " +
fileversion + " but this is " + OSRM_VERSION,
ErrorCode::IncompatibleFileVersion,
SOURCE_REF);
}
return true;
}
boost::filesystem::path path;
mtar_t handle;
};
class TarFileWriter
{
public:
TarFileWriter(const boost::filesystem::path &path) : path(path)
{
mtar_open(&handle, path.c_str(), "w");
WriteFingerprint();
}
template <typename T> void WriteOne(const std::string &name, const T &data)
{
mtar_write_file_header(&handle, name.c_str(), name.size());
mtar_write_data(&handle, reinterpret_cast<const char *>(&data), sizeof(T));
}
template <typename T>
void WriteFrom(const std::string &name, const T &data, const std::size_t number_of_entries)
{
mtar_write_file_header(&handle, name.c_str(), name.size());
mtar_write_data(&handle, reinterpret_cast<const char *>(&data), number_of_entries * sizeof(T));
}
private:
void WriteFingerprint()
{
const auto fingerprint = util::FingerPrint::GetValid();
WriteOne("osrm_fingerprint", fingerprint);
}
boost::filesystem::path path;
mtar_t handle;
};
}
}
#endif

14
unit_tests/CMakeLists.txt
View File

@ -22,6 +22,10 @@ file(GLOB UpdaterTestsSources
updater_tests.cpp
updater/*.cpp)
file(GLOB StorageTestsSources
storage_tests.cpp
storage/*.cpp)
file(GLOB LibraryTestsSources
library_tests.cpp
library/*.cpp)
@ -85,6 +89,11 @@ add_executable(updater-tests
${UpdaterTestsSources}
$<TARGET_OBJECTS:UPDATER> $<TARGET_OBJECTS:UPDATER> $<TARGET_OBJECTS:UTIL>)
add_executable(storage-tests
EXCLUDE_FROM_ALL
${StorageTestsSources}
$<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:UTIL>)
add_executable(library-tests
EXCLUDE_FROM_ALL
${LibraryTestsSources})
@ -136,6 +145,7 @@ target_compile_definitions(library-customize-tests PRIVATE COMPILE_DEFINITIONS O
target_compile_definitions(library-partition-tests PRIVATE COMPILE_DEFINITIONS OSRM_TEST_DATA_DIR="${TEST_DATA_DIR}")
target_compile_definitions(updater-tests PRIVATE COMPILE_DEFINITIONS TEST_DATA_DIR="${UPDATER_TEST_DATA_DIR}")
target_compile_definitions(contractor-tests PRIVATE COMPILE_DEFINITIONS TEST_DATA_DIR="${TEST_DATA_DIR}")
target_compile_definitions(storage-tests PRIVATE COMPILE_DEFINITIONS TEST_DATA_DIR="${TEST_DATA_DIR}")
target_include_directories(engine-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(library-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
@ -148,6 +158,7 @@ target_include_directories(partitioner-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(customizer-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(updater-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(contractor-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(storage-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(engine-tests ${ENGINE_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
target_link_libraries(extractor-tests osrm_extract osrm_guidance ${EXTRACTOR_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
@ -162,6 +173,7 @@ target_link_libraries(library-partition-tests osrm_partition ${Boost_UNIT_TEST_F
target_link_libraries(server-tests osrm ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
target_link_libraries(util-tests ${UTIL_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
target_link_libraries(contractor-tests ${CONTRACTOR_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
target_link_libraries(storage-tests osrm_store ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
add_custom_target(tests
DEPENDS engine-tests extractor-tests contractor-tests partitioner-tests updater-tests customizer-tests library-tests library-extract-tests library-contract-tests library-customize-tests library-partition-tests server-tests util-tests)
DEPENDS engine-tests extractor-tests contractor-tests partitioner-tests updater-tests customizer-tests library-tests library-extract-tests library-contract-tests library-customize-tests library-partition-tests server-tests util-tests storage-tests)

7
unit_tests/storage_tests.cpp Normal file
View File

@ -0,0 +1,7 @@
#define BOOST_TEST_MODULE storage tests
#include <boost/test/unit_test.hpp>
/*
* This file will contain an automatically generated main function.
*/