diff --git a/CMakeLists.txt b/CMakeLists.txt index e922c9ddb..8838aa2f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -167,13 +167,13 @@ add_executable(osrm-customize src/tools/customize.cpp) add_executable(osrm-contract src/tools/contract.cpp) add_executable(osrm-routed src/tools/routed.cpp $ $) add_executable(osrm-datastore src/tools/store.cpp $ $) -add_library(osrm src/osrm/osrm.cpp $ $ $) +add_library(osrm src/osrm/osrm.cpp $ $ $ $ ) add_library(osrm_contract src/osrm/contractor.cpp $ $) add_library(osrm_extract src/osrm/extractor.cpp $ $) add_library(osrm_guidance $ $) -add_library(osrm_partition src/osrm/partitioner.cpp $ $) -add_library(osrm_customize src/osrm/customizer.cpp $ $) -add_library(osrm_update $ $) +add_library(osrm_partition src/osrm/partitioner.cpp $ $ $) +add_library(osrm_customize src/osrm/customizer.cpp $ $ $) +add_library(osrm_update $ $ $) add_library(osrm_store $ $ $) if(ENABLE_GOLD_LINKER) @@ -560,6 +560,7 @@ include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR}) set(MICROTAR_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/microtar/src") include_directories(SYSTEM ${MICROTAR_INCLUDE_DIR}) add_library(MICROTAR OBJECT "${CMAKE_CURRENT_SOURCE_DIR}/third_party/microtar/src/microtar.c") +set_property(TARGET MICROTAR PROPERTY POSITION_INDEPENDENT_CODE ON) # prefix compilation with ccache by default if available and on clang or gcc if(ENABLE_CCACHE AND (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU")) diff --git a/include/partitioner/files.hpp b/include/partitioner/files.hpp index adfef3c2d..f28862b08 100644 --- a/include/partitioner/files.hpp +++ b/include/partitioner/files.hpp @@ -24,7 +24,7 @@ inline void readGraph(const boost::filesystem::path &path, std::is_same::value, ""); - storage::tar::FileReader reader{path}; + storage::tar::FileReader reader{path, storage::tar::FileReader::VerifyFingerprint}; serialization::read(reader, "/mld/multilevelgraph", graph, connectivity_checksum); } @@ -39,7 +39,7 @@ inline void writeGraph(const boost::filesystem::path &path, std::is_same::value, ""); - storage::tar::FileWriter writer{path}; + storage::tar::FileWriter writer{path, storage::tar::FileWriter::GenerateFingerprint}; serialization::write(writer, "/mld/multilevelgraph", graph, connectivity_checksum); } diff --git a/include/partitioner/multi_level_graph.hpp b/include/partitioner/multi_level_graph.hpp index a987ae7b5..72702d0d4 100644 --- a/include/partitioner/multi_level_graph.hpp +++ b/include/partitioner/multi_level_graph.hpp @@ -3,7 +3,7 @@ #include "partitioner/multi_level_partition.hpp" -#include "storage/io_fwd.hpp" +#include "storage/tar_fwd.hpp" #include "storage/shared_memory_ownership.hpp" #include "util/static_graph.hpp" @@ -24,12 +24,14 @@ template class MultiLevelGrap namespace serialization { template -void read(storage::io::FileReader &reader, +void read(storage::tar::FileReader &reader, + const std::string& name, MultiLevelGraph &graph, std::uint32_t &connectivity_checksum); template -void write(storage::io::FileWriter &writer, +void write(storage::tar::FileWriter &writer, + const std::string& name, const MultiLevelGraph &graph, const std::uint32_t connectivity_checksum); } @@ -202,11 +204,13 @@ class MultiLevelGraph : public util::StaticGraph } friend void - serialization::read(storage::io::FileReader &reader, + serialization::read(storage::tar::FileReader &reader, + const std::string& name, MultiLevelGraph &graph, std::uint32_t &connectivity_checksum); friend void - serialization::write(storage::io::FileWriter &writer, + serialization::write(storage::tar::FileWriter &writer, + const std::string& name, const MultiLevelGraph &graph, const std::uint32_t connectivity_checksum); diff --git a/include/partitioner/serialization.hpp b/include/partitioner/serialization.hpp index 355d50550..4b33b8dbc 100644 --- a/include/partitioner/serialization.hpp +++ b/include/partitioner/serialization.hpp @@ -40,6 +40,7 @@ inline void write(storage::tar::FileWriter &writer, storage::serialization::write(writer, name + "/node_array", graph.node_array); storage::serialization::write(writer, name + "/edge_array", graph.edge_array); storage::serialization::write(writer, name + "/node_to_edge_offset", graph.node_to_edge_offset); + writer.WriteElementCount64(name + "/connectivity_checksum", 1); writer.WriteOne(name + "/connectivity_checksum", connectivity_checksum); } diff --git a/include/storage/block.hpp b/include/storage/block.hpp index 2b895c44f..ccd2ce70d 100644 --- a/include/storage/block.hpp +++ b/include/storage/block.hpp @@ -14,10 +14,12 @@ namespace storage struct Block { - std::uint64_t num_entries; - std::uint64_t byte_size; + std::uint64_t num_entries = 0; + std::uint64_t byte_size = 0; }; +using NamedBlock = std::tuple; + template Block make_block(uint64_t num_entries) { static_assert(sizeof(T) % alignof(T) == 0, "aligned T* can't be used as an array pointer"); diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp index 573708288..5788c9996 100644 --- a/include/storage/shared_datatype.hpp +++ b/include/storage/shared_datatype.hpp @@ -92,7 +92,8 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA", "MLD_GRAPH_EDGE_LIST", "MLD_GRAPH_NODE_TO_OFFSET", "MANEUVER_OVERRIDES", - "MANEUVER_OVERRIDE_NODE_SEQUENCES"}; + "MANEUVER_OVERRIDE_NODE_SEQUENCES", + "IGNORE_BLOCK"}; class DataLayout { @@ -172,6 +173,7 @@ class DataLayout MLD_GRAPH_NODE_TO_OFFSET, MANEUVER_OVERRIDES, MANEUVER_OVERRIDE_NODE_SEQUENCES, + IGNORE_BLOCK, NUM_BLOCKS }; diff --git a/include/storage/tar.hpp b/include/storage/tar.hpp index 5af2514a8..3b5f58041 100644 --- a/include/storage/tar.hpp +++ b/include/storage/tar.hpp @@ -22,23 +22,31 @@ namespace tar class FileReader { public: - FileReader(const boost::filesystem::path &path) : path(path) + enum FingerprintFlag + { + VerifyFingerprint, + HasNoFingerprint + }; + + FileReader(const boost::filesystem::path &path, FingerprintFlag flag) : path(path) { auto ret = mtar_open(&handle, path.c_str(), "r"); if (ret != MTAR_ESUCCESS) { throw util::exception(mtar_strerror(ret)); } + + if (flag == VerifyFingerprint) + { + ReadAndCheckFingerprint(); + } } - ~FileReader() - { - mtar_close(&handle); - } + ~FileReader() { mtar_close(&handle); } std::uint64_t ReadElementCount64(const std::string &name) { - return ReadOne(name + "_count"); + return ReadOne(name + ".meta"); } template T ReadOne(const std::string &name) @@ -49,24 +57,25 @@ class FileReader } template - void ReadInto(const std::string &name, T *data, const std::size_t number_of_entries) + void ReadInto(const std::string &name, T *data, const std::size_t number_of_elements) { mtar_header_t header; auto ret = mtar_find(&handle, name.c_str(), &header); if (ret != MTAR_ESUCCESS) { - throw util::exception(mtar_strerror(ret)); + throw util::exception(name + ": " + mtar_strerror(ret)); } - if (header.size != sizeof(T) * number_of_entries) + auto expected_size = sizeof(T) * number_of_elements; + if (header.size != expected_size) { - throw util::exception("Datatype size does not match file size."); + throw util::exception(name + ": Datatype size does not match file size."); } ret = mtar_read_data(&handle, reinterpret_cast(data), header.size); if (ret != MTAR_ESUCCESS) { - throw util::exception(mtar_strerror(ret)); + throw util::exception(name + ": Failed reading data: " + mtar_strerror(ret)); } } @@ -78,7 +87,7 @@ class FileReader { if (header.type == MTAR_TREG) { - *out++ = TarEntry {header.name, header.size}; + *out++ = TarEntry{header.name, header.size}; } mtar_next(&handle); } @@ -87,7 +96,7 @@ class FileReader private: bool ReadAndCheckFingerprint() { - auto loaded_fingerprint = ReadOne("osrm_fingerprint"); + auto loaded_fingerprint = ReadOne("osrm_fingerprint.meta"); const auto expected_fingerprint = util::FingerPrint::GetValid(); if (!loaded_fingerprint.IsValid()) @@ -117,12 +126,22 @@ class FileReader class FileWriter { public: - FileWriter(const boost::filesystem::path &path) : path(path) + enum FingerprintFlag + { + GenerateFingerprint, + HasNoFingerprint + }; + + FileWriter(const boost::filesystem::path &path, FingerprintFlag flag) : path(path) { auto ret = mtar_open(&handle, path.c_str(), "w"); if (ret != MTAR_ESUCCESS) throw util::exception(mtar_strerror(ret)); - WriteFingerprint(); + + if (flag == GenerateFingerprint) + { + WriteFingerprint(); + } } ~FileWriter() @@ -133,7 +152,7 @@ class FileWriter void WriteElementCount64(const std::string &name, const std::uint64_t count) { - WriteOne(name + "_count", count); + WriteOne(name + ".meta", count); } template void WriteOne(const std::string &name, const T &data) @@ -142,18 +161,20 @@ class FileWriter } template - void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_entries) + void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_elements) { - auto number_of_bytes = number_of_entries * sizeof(T); + auto number_of_bytes = number_of_elements * sizeof(T); + auto ret = mtar_write_file_header(&handle, name.c_str(), number_of_bytes); if (ret != MTAR_ESUCCESS) { - throw util::exception(mtar_strerror(ret)); + throw util::exception(name + ": Error reading header: " + mtar_strerror(ret)); } + ret = mtar_write_data(&handle, reinterpret_cast(data), number_of_bytes); if (ret != MTAR_ESUCCESS) { - throw util::exception(mtar_strerror(ret)); + throw util::exception(name + ": Error reading data : " + mtar_strerror(ret)); } } @@ -161,13 +182,12 @@ class FileWriter void WriteFingerprint() { const auto fingerprint = util::FingerPrint::GetValid(); - WriteOne("osrm_fingerprint", fingerprint); + WriteOne("osrm_fingerprint.meta", fingerprint); } boost::filesystem::path path; mtar_t handle; }; - } } } diff --git a/include/storage/tar_fwd.hpp b/include/storage/tar_fwd.hpp new file mode 100644 index 000000000..cc19ad484 --- /dev/null +++ b/include/storage/tar_fwd.hpp @@ -0,0 +1,18 @@ +#ifndef OSRM_STORAGE_TAR_FWD_HPP_ +#define OSRM_STORAGE_TAR_FWD_HPP_ + +namespace osrm +{ +namespace storage +{ +namespace tar +{ + +class FileReader; +class FileWriter; + +} // ns io +} // ns storage +} // ns osrm + +#endif diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 10304d39f..343d0ba94 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -67,6 +67,32 @@ namespace osrm { namespace storage { +namespace +{ + template + void readBlocks(const boost::filesystem::path& path, OutIter out) + { + tar::FileReader reader(path, tar::FileReader::VerifyFingerprint); + + std::vector entries; + reader.List(std::back_inserter(entries)); + + for (const auto& entry : entries) + { + std::string name; + std::uint64_t size; + std::tie(name, size) = entry; + + const auto name_end = name.rfind(".meta"); + if (name_end == std::string::npos) + { + auto number_of_elements = reader.ReadElementCount64(name); + *out++ = NamedBlock {name, Block{number_of_elements, size}}; + } + } + + } +} static constexpr std::size_t NUM_METRICS = 8; @@ -572,35 +598,27 @@ void Storage::PopulateLayout(DataLayout &layout) layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_7, make_block(0)); } + std::unordered_map name_to_block_id = { + {"/mld/multilevelgraph/node_array", DataLayout::MLD_GRAPH_NODE_LIST}, + {"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST}, + {"/mld/multilevelgraph/node_to_edge_offset", DataLayout::MLD_GRAPH_NODE_TO_OFFSET}, + {"/mld/multilevelgraph/connectivity_checksum", DataLayout::IGNORE_BLOCK}, + }; + std::vector blocks; + if (boost::filesystem::exists(config.GetPath(".osrm.mldgr"))) { - io::FileReader reader(config.GetPath(".osrm.mldgr"), io::FileReader::VerifyFingerprint); - - const auto num_nodes = - reader.ReadVectorSize(); - const auto num_edges = - reader.ReadVectorSize(); - const auto num_node_offsets = - reader.ReadVectorSize(); - - layout.SetBlock( - DataLayout::MLD_GRAPH_NODE_LIST, - make_block(num_nodes)); - layout.SetBlock( - DataLayout::MLD_GRAPH_EDGE_LIST, - make_block(num_edges)); - layout.SetBlock( - DataLayout::MLD_GRAPH_NODE_TO_OFFSET, - make_block(num_node_offsets)); + readBlocks(config.GetPath(".osrm.mldgr"), std::back_inserter(blocks)); } - else + + for (const auto& block : blocks) { - layout.SetBlock(DataLayout::MLD_GRAPH_NODE_LIST, - make_block(0)); - layout.SetBlock(DataLayout::MLD_GRAPH_EDGE_LIST, - make_block(0)); - layout.SetBlock(DataLayout::MLD_GRAPH_NODE_TO_OFFSET, - make_block(0)); + auto id_iter = name_to_block_id.find(std::get<0>(block)); + if (id_iter == name_to_block_id.end()) + { + throw util::exception("Could not map " + std::get<0>(block) + " to a region in memory."); + } + layout.SetBlock(id_iter->second, std::get<1>(block)); } } } diff --git a/unit_tests/storage/tar.cpp b/unit_tests/storage/tar.cpp index b4cfe9cde..24aad2866 100644 --- a/unit_tests/storage/tar.cpp +++ b/unit_tests/storage/tar.cpp @@ -10,7 +10,8 @@ using namespace osrm; BOOST_AUTO_TEST_CASE(list_tar_file) { - storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar"); + storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar", + storage::tar::FileReader::HasNoFingerprint); std::vector file_list; reader.List(std::back_inserter(file_list)); @@ -29,7 +30,8 @@ BOOST_AUTO_TEST_CASE(list_tar_file) BOOST_AUTO_TEST_CASE(read_tar_file) { - storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar"); + storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar", + storage::tar::FileReader::HasNoFingerprint); char result_0[4]; reader.ReadInto("foo_1.txt", result_0, 4); @@ -58,7 +60,7 @@ BOOST_AUTO_TEST_CASE(write_tar_file) 0, 1, 2, 3, 4, 1ULL << 62, 0, 1 << 22, 0xFFFFFFFFFFFFFFFF}; { - storage::tar::FileWriter writer(tmp_path); + storage::tar::FileWriter writer(tmp_path, storage::tar::FileWriter::GenerateFingerprint); writer.WriteOne("foo/single_64bit_integer", single_64bit_integer); writer.WriteOne("bar/single_32bit_integer", single_32bit_integer); writer.WriteElementCount64("baz/bla/64bit_vector", vector_64bit.size()); @@ -67,7 +69,7 @@ BOOST_AUTO_TEST_CASE(write_tar_file) writer.WriteFrom("32bit_vector", vector_32bit.data(), vector_32bit.size()); } - storage::tar::FileReader reader(tmp_path); + storage::tar::FileReader reader(tmp_path, storage::tar::FileReader::VerifyFingerprint); BOOST_CHECK_EQUAL(reader.ReadOne("bar/single_32bit_integer"), single_32bit_integer);