Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Serialize mldgr using the new tar writer

Browse Source
This commit is contained in:
Patrick Niklaus
2018-03-15 13:55:06 +00:00
parent 653f647fee
commit 6d96a9a2e3
10 changed files with 133 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/partitioner/files.hpp
+2 -2
View File
@@ -24,7 +24,7 @@ inline void readGraph(const boost::filesystem::path &path,
std::is_same<customizer::MultiLevelEdgeBasedGraph, MultiLevelGraphT>::value,
"");
storage::tar::FileReader reader{path};
storage::tar::FileReader reader{path, storage::tar::FileReader::VerifyFingerprint};
serialization::read(reader, "/mld/multilevelgraph", graph, connectivity_checksum);
}
@@ -39,7 +39,7 @@ inline void writeGraph(const boost::filesystem::path &path,
std::is_same<customizer::MultiLevelEdgeBasedGraph, MultiLevelGraphT>::value,
"");
storage::tar::FileWriter writer{path};
storage::tar::FileWriter writer{path, storage::tar::FileWriter::GenerateFingerprint};
serialization::write(writer, "/mld/multilevelgraph", graph, connectivity_checksum);
}
include/partitioner/multi_level_graph.hpp
+9 -5
View File
@@ -3,7 +3,7 @@
#include "partitioner/multi_level_partition.hpp"
#include "storage/io_fwd.hpp"
#include "storage/tar_fwd.hpp"
#include "storage/shared_memory_ownership.hpp"
#include "util/static_graph.hpp"
@@ -24,12 +24,14 @@ template <typename EdgeDataT, storage::Ownership Ownership> class MultiLevelGrap
namespace serialization
{
template <typename EdgeDataT, storage::Ownership Ownership>
void read(storage::io::FileReader &reader,
void read(storage::tar::FileReader &reader,
const std::string& name,
MultiLevelGraph<EdgeDataT, Ownership> &graph,
std::uint32_t &connectivity_checksum);
template <typename EdgeDataT, storage::Ownership Ownership>
void write(storage::io::FileWriter &writer,
void write(storage::tar::FileWriter &writer,
const std::string& name,
const MultiLevelGraph<EdgeDataT, Ownership> &graph,
const std::uint32_t connectivity_checksum);
}
@@ -202,11 +204,13 @@ class MultiLevelGraph : public util::StaticGraph<EdgeDataT, Ownership>
}
friend void
serialization::read<EdgeDataT, Ownership>(storage::io::FileReader &reader,
serialization::read<EdgeDataT, Ownership>(storage::tar::FileReader &reader,
const std::string& name,
MultiLevelGraph<EdgeDataT, Ownership> &graph,
std::uint32_t &connectivity_checksum);
friend void
serialization::write<EdgeDataT, Ownership>(storage::io::FileWriter &writer,
serialization::write<EdgeDataT, Ownership>(storage::tar::FileWriter &writer,
const std::string& name,
const MultiLevelGraph<EdgeDataT, Ownership> &graph,
const std::uint32_t connectivity_checksum);
include/partitioner/serialization.hpp
+1
View File
@@ -40,6 +40,7 @@ inline void write(storage::tar::FileWriter &writer,
storage::serialization::write(writer, name + "/node_array", graph.node_array);
storage::serialization::write(writer, name + "/edge_array", graph.edge_array);
storage::serialization::write(writer, name + "/node_to_edge_offset", graph.node_to_edge_offset);
writer.WriteElementCount64(name + "/connectivity_checksum", 1);
writer.WriteOne(name + "/connectivity_checksum", connectivity_checksum);
}
include/storage/block.hpp
+4 -2
View File
@@ -14,10 +14,12 @@ namespace storage
struct Block
{
std::uint64_t num_entries;
std::uint64_t byte_size;
std::uint64_t num_entries = 0;
std::uint64_t byte_size = 0;
};
using NamedBlock = std::tuple<std::string, Block>;
template <typename T> Block make_block(uint64_t num_entries)
{
static_assert(sizeof(T) % alignof(T) == 0, "aligned T* can't be used as an array pointer");
include/storage/shared_datatype.hpp
+3 -1
View File
@@ -92,7 +92,8 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
"MLD_GRAPH_EDGE_LIST",
"MLD_GRAPH_NODE_TO_OFFSET",
"MANEUVER_OVERRIDES",
"MANEUVER_OVERRIDE_NODE_SEQUENCES"};
"MANEUVER_OVERRIDE_NODE_SEQUENCES",
"IGNORE_BLOCK"};
class DataLayout
{
@@ -172,6 +173,7 @@ class DataLayout
MLD_GRAPH_NODE_TO_OFFSET,
MANEUVER_OVERRIDES,
MANEUVER_OVERRIDE_NODE_SEQUENCES,
IGNORE_BLOCK,
NUM_BLOCKS
};
include/storage/tar.hpp
+42 -22
View File
@@ -22,23 +22,31 @@ namespace tar
class FileReader
{
public:
FileReader(const boost::filesystem::path &path) : path(path)
enum FingerprintFlag
{
VerifyFingerprint,
HasNoFingerprint
};
FileReader(const boost::filesystem::path &path, FingerprintFlag flag) : path(path)
{
auto ret = mtar_open(&handle, path.c_str(), "r");
if (ret != MTAR_ESUCCESS)
{
throw util::exception(mtar_strerror(ret));
}
if (flag == VerifyFingerprint)
{
ReadAndCheckFingerprint();
}
}
~FileReader()
{
mtar_close(&handle);
}
~FileReader() { mtar_close(&handle); }
std::uint64_t ReadElementCount64(const std::string &name)
{
return ReadOne<std::uint64_t>(name + "_count");
return ReadOne<std::uint64_t>(name + ".meta");
}
template <typename T> T ReadOne(const std::string &name)
@@ -49,24 +57,25 @@ class FileReader
}
template <typename T>
void ReadInto(const std::string &name, T *data, const std::size_t number_of_entries)
void ReadInto(const std::string &name, T *data, const std::size_t number_of_elements)
{
mtar_header_t header;
auto ret = mtar_find(&handle, name.c_str(), &header);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(mtar_strerror(ret));
throw util::exception(name + ": " + mtar_strerror(ret));
}
if (header.size != sizeof(T) * number_of_entries)
auto expected_size = sizeof(T) * number_of_elements;
if (header.size != expected_size)
{
throw util::exception("Datatype size does not match file size.");
throw util::exception(name + ": Datatype size does not match file size.");
}
ret = mtar_read_data(&handle, reinterpret_cast<char *>(data), header.size);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(mtar_strerror(ret));
throw util::exception(name + ": Failed reading data: " + mtar_strerror(ret));
}
}
@@ -78,7 +87,7 @@ class FileReader
{
if (header.type == MTAR_TREG)
{
*out++ = TarEntry {header.name, header.size};
*out++ = TarEntry{header.name, header.size};
}
mtar_next(&handle);
}
@@ -87,7 +96,7 @@ class FileReader
private:
bool ReadAndCheckFingerprint()
{
auto loaded_fingerprint = ReadOne<util::FingerPrint>("osrm_fingerprint");
auto loaded_fingerprint = ReadOne<util::FingerPrint>("osrm_fingerprint.meta");
const auto expected_fingerprint = util::FingerPrint::GetValid();
if (!loaded_fingerprint.IsValid())
@@ -117,12 +126,22 @@ class FileReader
class FileWriter
{
public:
FileWriter(const boost::filesystem::path &path) : path(path)
enum FingerprintFlag
{
GenerateFingerprint,
HasNoFingerprint
};
FileWriter(const boost::filesystem::path &path, FingerprintFlag flag) : path(path)
{
auto ret = mtar_open(&handle, path.c_str(), "w");
if (ret != MTAR_ESUCCESS)
throw util::exception(mtar_strerror(ret));
WriteFingerprint();
if (flag == GenerateFingerprint)
{
WriteFingerprint();
}
}
~FileWriter()
@@ -133,7 +152,7 @@ class FileWriter
void WriteElementCount64(const std::string &name, const std::uint64_t count)
{
WriteOne(name + "_count", count);
WriteOne(name + ".meta", count);
}
template <typename T> void WriteOne(const std::string &name, const T &data)
@@ -142,18 +161,20 @@ class FileWriter
}
template <typename T>
void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_entries)
void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_elements)
{
auto number_of_bytes = number_of_entries * sizeof(T);
auto number_of_bytes = number_of_elements * sizeof(T);
auto ret = mtar_write_file_header(&handle, name.c_str(), number_of_bytes);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(mtar_strerror(ret));
throw util::exception(name + ": Error reading header: " + mtar_strerror(ret));
}
ret = mtar_write_data(&handle, reinterpret_cast<const char *>(data), number_of_bytes);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(mtar_strerror(ret));
throw util::exception(name + ": Error reading data : " + mtar_strerror(ret));
}
}
@@ -161,13 +182,12 @@ class FileWriter
void WriteFingerprint()
{
const auto fingerprint = util::FingerPrint::GetValid();
WriteOne("osrm_fingerprint", fingerprint);
WriteOne("osrm_fingerprint.meta", fingerprint);
}
boost::filesystem::path path;
mtar_t handle;
};
}
}
}
include/storage/tar_fwd.hpp
+18
View File
@@ -0,0 +1,18 @@
#ifndef OSRM_STORAGE_TAR_FWD_HPP_
#define OSRM_STORAGE_TAR_FWD_HPP_
namespace osrm
{
namespace storage
{
namespace tar
{
class FileReader;
class FileWriter;
} // ns io
} // ns storage
} // ns osrm
#endif