Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Serialize mldgr using the new tar writer

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-15 13:55:06 +00:00
parent 653f647fee
commit 6d96a9a2e3
10 changed files with 133 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CMakeLists.txt
View File

@ -167,13 +167,13 @@ add_executable(osrm-customize src/tools/customize.cpp)
add_executable(osrm-contract src/tools/contract.cpp)
add_executable(osrm-routed src/tools/routed.cpp $<TARGET_OBJECTS:SERVER> $<TARGET_OBJECTS:UTIL>)
add_executable(osrm-datastore src/tools/store.cpp $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
add_library(osrm src/osrm/osrm.cpp $<TARGET_OBJECTS:ENGINE> $<TARGET_OBJECTS:UTIL> $<TARGET_OBJECTS:STORAGE>)
add_library(osrm src/osrm/osrm.cpp $<TARGET_OBJECTS:ENGINE> $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL> )
add_library(osrm_contract src/osrm/contractor.cpp $<TARGET_OBJECTS:CONTRACTOR> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_extract src/osrm/extractor.cpp $<TARGET_OBJECTS:EXTRACTOR> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_guidance $<TARGET_OBJECTS:GUIDANCE> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_partition src/osrm/partitioner.cpp $<TARGET_OBJECTS:PARTITIONER> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_customize src/osrm/customizer.cpp $<TARGET_OBJECTS:CUSTOMIZER> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_update $<TARGET_OBJECTS:UPDATER> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_partition src/osrm/partitioner.cpp $<TARGET_OBJECTS:PARTITIONER> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_customize src/osrm/customizer.cpp $<TARGET_OBJECTS:CUSTOMIZER> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_update $<TARGET_OBJECTS:UPDATER> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_store $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
if(ENABLE_GOLD_LINKER)
@ -560,6 +560,7 @@ include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
set(MICROTAR_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/microtar/src")
include_directories(SYSTEM ${MICROTAR_INCLUDE_DIR})
add_library(MICROTAR OBJECT "${CMAKE_CURRENT_SOURCE_DIR}/third_party/microtar/src/microtar.c")
set_property(TARGET MICROTAR PROPERTY POSITION_INDEPENDENT_CODE ON)
# prefix compilation with ccache by default if available and on clang or gcc
if(ENABLE_CCACHE AND (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU"))

4
include/partitioner/files.hpp
View File

@ -24,7 +24,7 @@ inline void readGraph(const boost::filesystem::path &path,
std::is_same<customizer::MultiLevelEdgeBasedGraph, MultiLevelGraphT>::value,
"");
storage::tar::FileReader reader{path};
storage::tar::FileReader reader{path, storage::tar::FileReader::VerifyFingerprint};
serialization::read(reader, "/mld/multilevelgraph", graph, connectivity_checksum);
}
@ -39,7 +39,7 @@ inline void writeGraph(const boost::filesystem::path &path,
std::is_same<customizer::MultiLevelEdgeBasedGraph, MultiLevelGraphT>::value,
"");
storage::tar::FileWriter writer{path};
storage::tar::FileWriter writer{path, storage::tar::FileWriter::GenerateFingerprint};
serialization::write(writer, "/mld/multilevelgraph", graph, connectivity_checksum);
}

14
include/partitioner/multi_level_graph.hpp
View File

@ -3,7 +3,7 @@
#include "partitioner/multi_level_partition.hpp"
#include "storage/io_fwd.hpp"
#include "storage/tar_fwd.hpp"
#include "storage/shared_memory_ownership.hpp"
#include "util/static_graph.hpp"
@ -24,12 +24,14 @@ template <typename EdgeDataT, storage::Ownership Ownership> class MultiLevelGrap
namespace serialization
{
template <typename EdgeDataT, storage::Ownership Ownership>
void read(storage::io::FileReader &reader,
void read(storage::tar::FileReader &reader,
const std::string& name,
MultiLevelGraph<EdgeDataT, Ownership> &graph,
std::uint32_t &connectivity_checksum);
template <typename EdgeDataT, storage::Ownership Ownership>
void write(storage::io::FileWriter &writer,
void write(storage::tar::FileWriter &writer,
const std::string& name,
const MultiLevelGraph<EdgeDataT, Ownership> &graph,
const std::uint32_t connectivity_checksum);
}
@ -202,11 +204,13 @@ class MultiLevelGraph : public util::StaticGraph<EdgeDataT, Ownership>
}
friend void
serialization::read<EdgeDataT, Ownership>(storage::io::FileReader &reader,
serialization::read<EdgeDataT, Ownership>(storage::tar::FileReader &reader,
const std::string& name,
MultiLevelGraph<EdgeDataT, Ownership> &graph,
std::uint32_t &connectivity_checksum);
friend void
serialization::write<EdgeDataT, Ownership>(storage::io::FileWriter &writer,
serialization::write<EdgeDataT, Ownership>(storage::tar::FileWriter &writer,
const std::string& name,
const MultiLevelGraph<EdgeDataT, Ownership> &graph,
const std::uint32_t connectivity_checksum);

1
include/partitioner/serialization.hpp
View File

@ -40,6 +40,7 @@ inline void write(storage::tar::FileWriter &writer,
storage::serialization::write(writer, name + "/node_array", graph.node_array);
storage::serialization::write(writer, name + "/edge_array", graph.edge_array);
storage::serialization::write(writer, name + "/node_to_edge_offset", graph.node_to_edge_offset);
writer.WriteElementCount64(name + "/connectivity_checksum", 1);
writer.WriteOne(name + "/connectivity_checksum", connectivity_checksum);
}

6
include/storage/block.hpp
View File

@ -14,10 +14,12 @@ namespace storage
struct Block
{
std::uint64_t num_entries;
std::uint64_t byte_size;
std::uint64_t num_entries = 0;
std::uint64_t byte_size = 0;
};
using NamedBlock = std::tuple<std::string, Block>;
template <typename T> Block make_block(uint64_t num_entries)
{
static_assert(sizeof(T) % alignof(T) == 0, "aligned T* can't be used as an array pointer");

4
include/storage/shared_datatype.hpp
View File

@ -92,7 +92,8 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
"MLD_GRAPH_EDGE_LIST",
"MLD_GRAPH_NODE_TO_OFFSET",
"MANEUVER_OVERRIDES",
"MANEUVER_OVERRIDE_NODE_SEQUENCES"};
"MANEUVER_OVERRIDE_NODE_SEQUENCES",
"IGNORE_BLOCK"};
class DataLayout
{
@ -172,6 +173,7 @@ class DataLayout
MLD_GRAPH_NODE_TO_OFFSET,
MANEUVER_OVERRIDES,
MANEUVER_OVERRIDE_NODE_SEQUENCES,
IGNORE_BLOCK,
NUM_BLOCKS
};

64
include/storage/tar.hpp
View File

@ -22,23 +22,31 @@ namespace tar
class FileReader
{
public:
FileReader(const boost::filesystem::path &path) : path(path)
enum FingerprintFlag
{
VerifyFingerprint,
HasNoFingerprint
};
FileReader(const boost::filesystem::path &path, FingerprintFlag flag) : path(path)
{
auto ret = mtar_open(&handle, path.c_str(), "r");
if (ret != MTAR_ESUCCESS)
{
throw util::exception(mtar_strerror(ret));
}
if (flag == VerifyFingerprint)
{
ReadAndCheckFingerprint();
}
}
~FileReader()
{
mtar_close(&handle);
}
~FileReader() { mtar_close(&handle); }
std::uint64_t ReadElementCount64(const std::string &name)
{
return ReadOne<std::uint64_t>(name + "_count");
return ReadOne<std::uint64_t>(name + ".meta");
}
template <typename T> T ReadOne(const std::string &name)
@ -49,24 +57,25 @@ class FileReader
}
template <typename T>
void ReadInto(const std::string &name, T *data, const std::size_t number_of_entries)
void ReadInto(const std::string &name, T *data, const std::size_t number_of_elements)
{
mtar_header_t header;
auto ret = mtar_find(&handle, name.c_str(), &header);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(mtar_strerror(ret));
throw util::exception(name + ": " + mtar_strerror(ret));
}
if (header.size != sizeof(T) * number_of_entries)
auto expected_size = sizeof(T) * number_of_elements;
if (header.size != expected_size)
{
throw util::exception("Datatype size does not match file size.");
throw util::exception(name + ": Datatype size does not match file size.");
}
ret = mtar_read_data(&handle, reinterpret_cast<char *>(data), header.size);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(mtar_strerror(ret));
throw util::exception(name + ": Failed reading data: " + mtar_strerror(ret));
}
}
@ -78,7 +87,7 @@ class FileReader
{
if (header.type == MTAR_TREG)
{
*out++ = TarEntry {header.name, header.size};
*out++ = TarEntry{header.name, header.size};
}
mtar_next(&handle);
}
@ -87,7 +96,7 @@ class FileReader
private:
bool ReadAndCheckFingerprint()
{
auto loaded_fingerprint = ReadOne<util::FingerPrint>("osrm_fingerprint");
auto loaded_fingerprint = ReadOne<util::FingerPrint>("osrm_fingerprint.meta");
const auto expected_fingerprint = util::FingerPrint::GetValid();
if (!loaded_fingerprint.IsValid())
@ -117,12 +126,22 @@ class FileReader
class FileWriter
{
public:
FileWriter(const boost::filesystem::path &path) : path(path)
enum FingerprintFlag
{
GenerateFingerprint,
HasNoFingerprint
};
FileWriter(const boost::filesystem::path &path, FingerprintFlag flag) : path(path)
{
auto ret = mtar_open(&handle, path.c_str(), "w");
if (ret != MTAR_ESUCCESS)
throw util::exception(mtar_strerror(ret));
WriteFingerprint();
if (flag == GenerateFingerprint)
{
WriteFingerprint();
}
}
~FileWriter()
@ -133,7 +152,7 @@ class FileWriter
void WriteElementCount64(const std::string &name, const std::uint64_t count)
{
WriteOne(name + "_count", count);
WriteOne(name + ".meta", count);
}
template <typename T> void WriteOne(const std::string &name, const T &data)
@ -142,18 +161,20 @@ class FileWriter
}
template <typename T>
void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_entries)
void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_elements)
{
auto number_of_bytes = number_of_entries * sizeof(T);
auto number_of_bytes = number_of_elements * sizeof(T);
auto ret = mtar_write_file_header(&handle, name.c_str(), number_of_bytes);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(mtar_strerror(ret));
throw util::exception(name + ": Error reading header: " + mtar_strerror(ret));
}
ret = mtar_write_data(&handle, reinterpret_cast<const char *>(data), number_of_bytes);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(mtar_strerror(ret));
throw util::exception(name + ": Error reading data : " + mtar_strerror(ret));
}
}
@ -161,13 +182,12 @@ class FileWriter
void WriteFingerprint()
{
const auto fingerprint = util::FingerPrint::GetValid();
WriteOne("osrm_fingerprint", fingerprint);
WriteOne("osrm_fingerprint.meta", fingerprint);
}
boost::filesystem::path path;
mtar_t handle;
};
}
}
}

18
include/storage/tar_fwd.hpp Normal file
View File

@ -0,0 +1,18 @@
#ifndef OSRM_STORAGE_TAR_FWD_HPP_
#define OSRM_STORAGE_TAR_FWD_HPP_
namespace osrm
{
namespace storage
{
namespace tar
{
class FileReader;
class FileWriter;
} // ns io
} // ns storage
} // ns osrm
#endif

68
src/storage/storage.cpp
View File

@ -67,6 +67,32 @@ namespace osrm
{
namespace storage
{
namespace
{
template<typename OutIter>
void readBlocks(const boost::filesystem::path& path, OutIter out)
{
tar::FileReader reader(path, tar::FileReader::VerifyFingerprint);
std::vector<tar::FileReader::TarEntry> entries;
reader.List(std::back_inserter(entries));
for (const auto& entry : entries)
{
std::string name;
std::uint64_t size;
std::tie(name, size) = entry;
const auto name_end = name.rfind(".meta");
if (name_end == std::string::npos)
{
auto number_of_elements = reader.ReadElementCount64(name);
*out++ = NamedBlock {name, Block{number_of_elements, size}};
}
}
}
}
static constexpr std::size_t NUM_METRICS = 8;
@ -572,35 +598,27 @@ void Storage::PopulateLayout(DataLayout &layout)
layout.SetBlock(DataLayout::MLD_CELL_DURATIONS_7, make_block<char>(0));
}
std::unordered_map<std::string, DataLayout::BlockID> name_to_block_id = {
{"/mld/multilevelgraph/node_array", DataLayout::MLD_GRAPH_NODE_LIST},
{"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST},
{"/mld/multilevelgraph/node_to_edge_offset", DataLayout::MLD_GRAPH_NODE_TO_OFFSET},
{"/mld/multilevelgraph/connectivity_checksum", DataLayout::IGNORE_BLOCK},
};
std::vector<NamedBlock> blocks;
if (boost::filesystem::exists(config.GetPath(".osrm.mldgr")))
{
io::FileReader reader(config.GetPath(".osrm.mldgr"), io::FileReader::VerifyFingerprint);
const auto num_nodes =
reader.ReadVectorSize<customizer::MultiLevelEdgeBasedGraph::NodeArrayEntry>();
const auto num_edges =
reader.ReadVectorSize<customizer::MultiLevelEdgeBasedGraph::EdgeArrayEntry>();
const auto num_node_offsets =
reader.ReadVectorSize<customizer::MultiLevelEdgeBasedGraph::EdgeOffset>();
layout.SetBlock(
DataLayout::MLD_GRAPH_NODE_LIST,
make_block<customizer::MultiLevelEdgeBasedGraph::NodeArrayEntry>(num_nodes));
layout.SetBlock(
DataLayout::MLD_GRAPH_EDGE_LIST,
make_block<customizer::MultiLevelEdgeBasedGraph::EdgeArrayEntry>(num_edges));
layout.SetBlock(
DataLayout::MLD_GRAPH_NODE_TO_OFFSET,
make_block<customizer::MultiLevelEdgeBasedGraph::EdgeOffset>(num_node_offsets));
readBlocks(config.GetPath(".osrm.mldgr"), std::back_inserter(blocks));
}
else
for (const auto& block : blocks)
{
layout.SetBlock(DataLayout::MLD_GRAPH_NODE_LIST,
make_block<customizer::MultiLevelEdgeBasedGraph::NodeArrayEntry>(0));
layout.SetBlock(DataLayout::MLD_GRAPH_EDGE_LIST,
make_block<customizer::MultiLevelEdgeBasedGraph::EdgeArrayEntry>(0));
layout.SetBlock(DataLayout::MLD_GRAPH_NODE_TO_OFFSET,
make_block<customizer::MultiLevelEdgeBasedGraph::EdgeOffset>(0));
auto id_iter = name_to_block_id.find(std::get<0>(block));
if (id_iter == name_to_block_id.end())
{
throw util::exception("Could not map " + std::get<0>(block) + " to a region in memory.");
}
layout.SetBlock(id_iter->second, std::get<1>(block));
}
}
}

10
unit_tests/storage/tar.cpp
View File

@ -10,7 +10,8 @@ using namespace osrm;
BOOST_AUTO_TEST_CASE(list_tar_file)
{
storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar");
storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar",
storage::tar::FileReader::HasNoFingerprint);
std::vector<storage::tar::FileReader::TarEntry> file_list;
reader.List(std::back_inserter(file_list));
@ -29,7 +30,8 @@ BOOST_AUTO_TEST_CASE(list_tar_file)
BOOST_AUTO_TEST_CASE(read_tar_file)
{
storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar");
storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar",
storage::tar::FileReader::HasNoFingerprint);
char result_0[4];
reader.ReadInto("foo_1.txt", result_0, 4);
@ -58,7 +60,7 @@ BOOST_AUTO_TEST_CASE(write_tar_file)
0, 1, 2, 3, 4, 1ULL << 62, 0, 1 << 22, 0xFFFFFFFFFFFFFFFF};
{
storage::tar::FileWriter writer(tmp_path);
storage::tar::FileWriter writer(tmp_path, storage::tar::FileWriter::GenerateFingerprint);
writer.WriteOne("foo/single_64bit_integer", single_64bit_integer);
writer.WriteOne("bar/single_32bit_integer", single_32bit_integer);
writer.WriteElementCount64("baz/bla/64bit_vector", vector_64bit.size());
@ -67,7 +69,7 @@ BOOST_AUTO_TEST_CASE(write_tar_file)
writer.WriteFrom("32bit_vector", vector_32bit.data(), vector_32bit.size());
}
storage::tar::FileReader reader(tmp_path);
storage::tar::FileReader reader(tmp_path, storage::tar::FileReader::VerifyFingerprint);
BOOST_CHECK_EQUAL(reader.ReadOne<std::uint32_t>("bar/single_32bit_integer"),
single_32bit_integer);