Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Port hsgr file to tar

Browse Source
This commit is contained in:
Patrick Niklaus
2018-03-15 20:10:21 +00:00
parent fed77c4066
commit cb31f9ec29
6 changed files with 300 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/contractor/files.hpp
+22 -15
View File
@@ -5,7 +5,7 @@
#include "util/serialization.hpp"
#include "storage/io.hpp"
#include "storage/tar.hpp"
#include "storage/serialization.hpp"
namespace osrm
@@ -29,18 +29,20 @@ inline void readGraph(const boost::filesystem::path &path,
std::is_same<EdgeFilterT, util::vector_view<bool>>::value,
"edge_filter must be a container of vector<bool> or vector_view<bool>");
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
reader.ReadInto(checksum);
util::serialization::read(reader, graph);
auto count = reader.ReadElementCount64();
checksum = reader.ReadOne<std::uint32_t>("/ch/checksum");
util::serialization::read(reader, "/ch/contracted_graph", graph);
auto count = reader.ReadElementCount64("/ch/edge_filter");
edge_filter.resize(count);
for (const auto index : util::irange<std::size_t>(0, count))
{
storage::serialization::read(reader, edge_filter[index]);
storage::serialization::read(reader, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]);
}
reader.ReadInto(connectivity_checksum);
connectivity_checksum = reader.ReadOne<std::uint32_t>("/ch/connectivity_checksum");
}
// writes .osrm.hsgr file
@@ -57,17 +59,22 @@ inline void writeGraph(const boost::filesystem::path &path,
static_assert(std::is_same<EdgeFilterT, std::vector<bool>>::value ||
std::is_same<EdgeFilterT, util::vector_view<bool>>::value,
"edge_filter must be a container of vector<bool> or vector_view<bool>");
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
writer.WriteOne(checksum);
util::serialization::write(writer, graph);
writer.WriteElementCount64(edge_filter.size());
writer.WriteElementCount64("/ch/checksum", 1);
writer.WriteOne("/ch/checksum", checksum);
util::serialization::write(writer, "/ch/contracted_graph", graph);
writer.WriteElementCount64("/ch/edge_filter", edge_filter.size());
auto id = 0;
for (const auto &filter : edge_filter)
{
storage::serialization::write(writer, filter);
storage::serialization::write(writer, "/ch/edge_filter/" + std::to_string(id++), filter);
}
writer.WriteOne(connectivity_checksum);
writer.WriteElementCount64("/ch/connectivity_checksum", 1);
writer.WriteOne("/ch/connectivity_checksum", connectivity_checksum);
}
}
}
include/storage/serialization.hpp
+90 -24
View File
@@ -8,6 +8,9 @@
#include "storage/io.hpp"
#include "storage/tar.hpp"
#include <boost/function_output_iterator.hpp>
#include <boost/iterator/function_input_iterator.hpp>
#include <cmath>
#include <cstdint>
@@ -87,28 +90,32 @@ inline void write(storage::io::FileWriter &writer, const stxxl::vector<T> &vec)
}
#endif
template <typename T> void read(tar::FileReader &reader, const std::string& name, std::vector<T> &data)
template <typename T>
void read(tar::FileReader &reader, const std::string &name, std::vector<T> &data)
{
const auto count = reader.ReadElementCount64(name);
data.resize(count);
reader.ReadInto(name, data.data(), count);
}
template <typename T> void write(tar::FileWriter &writer, const std::string& name, const std::vector<T> &data)
template <typename T>
void write(tar::FileWriter &writer, const std::string &name, const std::vector<T> &data)
{
const auto count = data.size();
writer.WriteElementCount64(name, count);
writer.WriteFrom(name, data.data(), count);
}
template <typename T> void read(tar::FileReader &reader, const std::string& name, util::vector_view<T> &data)
template <typename T>
void read(tar::FileReader &reader, const std::string &name, util::vector_view<T> &data)
{
const auto count = reader.ReadElementCount64(name);
BOOST_ASSERT(data.size() == count);
reader.ReadInto(name, data.data(), count);
}
template <typename T> void write(tar::FileWriter &writer, const std::string& name, const util::vector_view<T> &data)
template <typename T>
void write(tar::FileWriter &writer, const std::string &name, const util::vector_view<T> &data)
{
const auto count = data.size();
writer.WriteElementCount64(name, count);
@@ -143,6 +150,8 @@ template <typename T> void write(io::FileWriter &writer, const util::vector_view
writer.WriteFrom(data.data(), count);
}
namespace detail
{
template <typename T>
inline unsigned char packBits(const T &data, std::size_t index, std::size_t count)
{
@@ -162,7 +171,7 @@ inline void unpackBits(T &data, std::size_t index, std::size_t count, unsigned c
data[index] = value & mask;
}
template <> inline void read<bool>(io::FileReader &reader, util::vector_view<bool> &data)
template <typename VectorT> void readBoolVector(io::FileReader &reader, VectorT &data)
{
const auto count = reader.ReadElementCount64();
BOOST_ASSERT(data.size() == count);
@@ -175,7 +184,7 @@ template <> inline void read<bool>(io::FileReader &reader, util::vector_view<boo
unpackBits(data, index, count - index, reader.ReadOne<unsigned char>());
}
template <> inline void write<bool>(io::FileWriter &writer, const util::vector_view<bool> &data)
template <typename VectorT> void writeBoolVector(io::FileWriter &writer, const VectorT &data)
{
const auto count = data.size();
writer.WriteElementCount64(count);
@@ -188,30 +197,87 @@ template <> inline void write<bool>(io::FileWriter &writer, const util::vector_v
writer.WriteOne<unsigned char>(packBits(data, index, count - index));
}
template <typename VectorT>
void readBoolVector(tar::FileReader &reader, const std::string &name, VectorT &data)
{
const auto count = reader.ReadElementCount64(name);
BOOST_ASSERT(data.size() == count);
std::uint64_t index = 0;
const auto decode = [&data, &index, count](const char block) {
auto read_size = std::min<std::size_t>(count - index, CHAR_BIT);
unpackBits(data, index, read_size, block);
index += CHAR_BIT;
};
reader.ReadStreaming<unsigned char>(name, boost::make_function_output_iterator(decode));
}
template <typename VectorT>
void writeBoolVector(tar::FileWriter &writer, const std::string &name, const VectorT &data)
{
const auto count = data.size();
writer.WriteElementCount64(name, count);
std::uint64_t index = 0;
const auto encode = [&]() {
auto write_size = std::min<std::size_t>(count - index, CHAR_BIT);
auto packed = packBits(data, CHAR_BIT * index, write_size);
index += CHAR_BIT;
return packed;
};
std::uint64_t number_of_blocks = std::ceil(count / CHAR_BIT);
writer.WriteStreaming<unsigned char>(
name, boost::make_function_input_iterator(encode, boost::infinite()), number_of_blocks);
}
}
template <> inline void read<bool>(io::FileReader &reader, util::vector_view<bool> &data)
{
detail::readBoolVector(reader, data);
}
template <> inline void write<bool>(io::FileWriter &writer, const util::vector_view<bool> &data)
{
detail::writeBoolVector(writer, data);
}
template <> inline void read<bool>(io::FileReader &reader, std::vector<bool> &data)
{
const auto count = reader.ReadElementCount64();
data.resize(count);
std::uint64_t index = 0;
for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
{
unpackBits(data, index, CHAR_BIT, reader.ReadOne<unsigned char>());
}
if (count > index)
unpackBits(data, index, count - index, reader.ReadOne<unsigned char>());
detail::readBoolVector(reader, data);
}
template <> inline void write<bool>(io::FileWriter &writer, const std::vector<bool> &data)
{
const auto count = data.size();
writer.WriteElementCount64(count);
std::uint64_t index = 0;
for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
{
writer.WriteOne<unsigned char>(packBits(data, index, CHAR_BIT));
}
if (count > index)
writer.WriteOne<unsigned char>(packBits(data, index, count - index));
detail::writeBoolVector(writer, data);
}
template <>
inline void
read<bool>(tar::FileReader &reader, const std::string &name, util::vector_view<bool> &data)
{
detail::readBoolVector(reader, name, data);
}
template <>
inline void
write<bool>(tar::FileWriter &writer, const std::string &name, const util::vector_view<bool> &data)
{
detail::writeBoolVector(writer, name, data);
}
template <>
inline void read<bool>(tar::FileReader &reader, const std::string &name, std::vector<bool> &data)
{
detail::readBoolVector(reader, name, data);
}
template <>
inline void
write<bool>(tar::FileWriter &writer, const std::string &name, const std::vector<bool> &data)
{
detail::writeBoolVector(writer, name, data);
}
}
}
include/storage/tar.hpp
+56 -2
View File
@@ -4,6 +4,7 @@
#include "util/exception.hpp"
#include "util/exception_utils.hpp"
#include "util/fingerprint.hpp"
#include "util/integer_range.hpp"
#include "util/version.hpp"
#include <boost/filesystem/path.hpp>
@@ -56,6 +57,36 @@ class FileReader
return tmp;
}
template <typename T, typename OutIter> void ReadStreaming(const std::string &name, OutIter out)
{
mtar_header_t header;
auto ret = mtar_find(&handle, name.c_str(), &header);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(name + ": " + mtar_strerror(ret));
}
auto number_of_elements = header.size / sizeof(T);
auto expected_size = sizeof(T) * number_of_elements;
if (header.size != expected_size)
{
throw util::exception(name + ": Datatype size does not match file size.");
}
T tmp;
for (auto index : util::irange<std::size_t>(0, number_of_elements))
{
(void) index;
ret = mtar_read_data(&handle, reinterpret_cast<char *>(&tmp), sizeof(T));
if (ret != MTAR_ESUCCESS)
{
throw util::exception(name + ": Failed reading data: " + mtar_strerror(ret));
}
*out++ = tmp;
}
}
template <typename T>
void ReadInto(const std::string &name, T *data, const std::size_t number_of_elements)
{
@@ -160,6 +191,29 @@ class FileWriter
WriteFrom(name, &data, 1);
}
template <typename T, typename Iter>
void WriteStreaming(const std::string &name, Iter iter, const std::uint64_t number_of_elements)
{
auto number_of_bytes = number_of_elements * sizeof(T);
auto ret = mtar_write_file_header(&handle, name.c_str(), number_of_bytes);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(name + ": Error writing header: " + mtar_strerror(ret));
}
for (auto index : util::irange<std::size_t>(0, number_of_elements))
{
(void) index;
T tmp = *iter++;
ret = mtar_write_data(&handle, &tmp, sizeof(T));
if (ret != MTAR_ESUCCESS)
{
throw util::exception(name + ": Error writing data : " + mtar_strerror(ret));
}
}
}
template <typename T>
void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_elements)
{
@@ -168,13 +222,13 @@ class FileWriter
auto ret = mtar_write_file_header(&handle, name.c_str(), number_of_bytes);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(name + ": Error reading header: " + mtar_strerror(ret));
throw util::exception(name + ": Error writing header: " + mtar_strerror(ret));
}
ret = mtar_write_data(&handle, reinterpret_cast<const char *>(data), number_of_bytes);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(name + ": Error reading data : " + mtar_strerror(ret));
throw util::exception(name + ": Error writing data : " + mtar_strerror(ret));
}
}
include/util/serialization.hpp
+44
View File
@@ -86,6 +86,50 @@ inline void write(storage::io::FileWriter &writer, const DynamicGraph<EdgeDataT>
writer.WriteOne(graph.edge_list[index]);
}
}
template <typename EdgeDataT, storage::Ownership Ownership>
inline void read(storage::tar::FileReader &reader,
const std::string &name,
StaticGraph<EdgeDataT, Ownership> &graph)
{
storage::serialization::read(reader, name + "/node_array", graph.node_array);
storage::serialization::read(reader, name + "/edge_array", graph.edge_array);
graph.number_of_nodes = graph.node_array.size() - 1;
graph.number_of_edges = graph.edge_array.size();
}
template <typename EdgeDataT, storage::Ownership Ownership>
inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const StaticGraph<EdgeDataT, Ownership> &graph)
{
storage::serialization::write(writer, name + "/node_array", graph.node_array);
storage::serialization::write(writer, name + "/edge_array", graph.edge_array);
}
template <typename EdgeDataT>
inline void
read(storage::tar::FileReader &reader, const std::string &name, DynamicGraph<EdgeDataT> &graph)
{
storage::serialization::read(reader, name + "/node_array", graph.node_array);
const auto num_edges = reader.ReadElementCount64(name + "/edge_list");
graph.edge_list.resize(num_edges);
reader.ReadStreaming<typename std::remove_reference_t<decltype(graph)>::Edge>(
name + "/edge_list", graph.edge_list.begin());
graph.number_of_nodes = graph.node_array.size();
graph.number_of_edges = num_edges;
}
template <typename EdgeDataT>
inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const DynamicGraph<EdgeDataT> &graph)
{
storage::serialization::write(writer, name + "/node_array", graph.node_array);
writer.WriteElementCount64(name + "/edge_list", graph.number_of_edges);
writer.WriteStreaming<typename std::remove_reference_t<decltype(graph)>::Edge>(
name + "/edge_list", graph.edge_list.begin(), graph.number_of_edges);
}
}
}
}
include/util/static_graph.hpp
+19
View File
@@ -10,6 +10,7 @@
#include "storage/io_fwd.hpp"
#include "storage/shared_memory_ownership.hpp"
#include "storage/tar_fwd.hpp"
#include <boost/assert.hpp>
@@ -32,6 +33,16 @@ void read(storage::io::FileReader &reader, StaticGraph<EdgeDataT, Ownership> &gr
template <typename EdgeDataT, storage::Ownership Ownership>
void write(storage::io::FileWriter &writer, const StaticGraph<EdgeDataT, Ownership> &graph);
template <typename EdgeDataT, storage::Ownership Ownership>
void read(storage::tar::FileReader &reader,
const std::string &name,
StaticGraph<EdgeDataT, Ownership> &graph);
template <typename EdgeDataT, storage::Ownership Ownership>
void write(storage::tar::FileWriter &writer,
const std::string &name,
const StaticGraph<EdgeDataT, Ownership> &graph);
}
namespace static_graph_details
@@ -278,6 +289,14 @@ class StaticGraph
serialization::write<EdgeDataT, Ownership>(storage::io::FileWriter &writer,
const StaticGraph<EdgeDataT, Ownership> &graph);
friend void serialization::read<EdgeDataT, Ownership>(storage::tar::FileReader &reader,
const std::string &name,
StaticGraph<EdgeDataT, Ownership> &graph);
friend void
serialization::write<EdgeDataT, Ownership>(storage::tar::FileWriter &writer,
const std::string &name,
const StaticGraph<EdgeDataT, Ownership> &graph);
protected:
template <typename IterT>
void InitializeFromSortedEdgeRange(const std::uint32_t nodes, IterT begin, IterT end)