Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Refactor metric storage

Browse Source
This commit is contained in:
Patrick Niklaus
2018-03-27 09:44:13 +00:00
committed by Patrick Niklaus
parent aec9b6a178
commit c334d11e95
14 changed files with 505 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/contractor/contracted_metric.hpp
+26
View File
@@ -0,0 +1,26 @@
#ifndef OSMR_CONTRACTOR_CONTRACTED_METRIC_HPP
#define OSMR_CONTRACTOR_CONTRACTED_METRIC_HPP
#include "contractor/query_graph.hpp"
namespace osrm
{
namespace contractor
{
namespace detail
{
template<storage::Ownership Ownership>
struct ContractedMetric
{
detail::QueryGraph<Ownership> graph;
std::vector<util::ViewOrVector<bool, Ownership>> edge_filter;
};
}
using ContractedMetric = detail::ContractedMetric<storage::Ownership::Container>;
using ContractedMetricView = detail::ContractedMetric<storage::Ownership::View>;
}
}
#endif
include/contractor/files.hpp
+20 -39
View File
@@ -1,12 +1,9 @@
#ifndef OSRM_CONTRACTOR_FILES_HPP
#define OSRM_CONTRACTOR_FILES_HPP
#include "contractor/query_graph.hpp"
#include "contractor/serialization.hpp"
#include "util/serialization.hpp"
#include "storage/serialization.hpp"
#include "storage/tar.hpp"
#include <unordered_map>
namespace osrm
{
@@ -15,67 +12,51 @@ namespace contractor
namespace files
{
// reads .osrm.hsgr file
template <typename QueryGraphT, typename EdgeFilterT>
template <typename ContractedMetricT>
inline void readGraph(const boost::filesystem::path &path,
unsigned &checksum,
QueryGraphT &graph,
std::vector<EdgeFilterT> &edge_filter,
std::unordered_map<std::string, ContractedMetricT> &metrics,
std::uint32_t &connectivity_checksum)
{
static_assert(std::is_same<QueryGraphView, QueryGraphT>::value ||
std::is_same<QueryGraph, QueryGraphT>::value,
"graph must be of type QueryGraph<>");
static_assert(std::is_same<EdgeFilterT, std::vector<bool>>::value ||
std::is_same<EdgeFilterT, util::vector_view<bool>>::value,
"edge_filter must be a container of vector<bool> or vector_view<bool>");
static_assert(std::is_same<ContractedMetric, ContractedMetricT>::value ||
std::is_same<ContractedMetricView, ContractedMetricT>::value,
"metric must be of type ContractedMetric<>");
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
reader.ReadInto("/ch/checksum", checksum);
util::serialization::read(reader, "/ch/contracted_graph", graph);
reader.ReadInto("/ch/connectivity_checksum", connectivity_checksum);
auto count = reader.ReadElementCount64("/ch/edge_filter");
edge_filter.resize(count);
for (const auto index : util::irange<std::size_t>(0, count))
for (auto &pair : metrics)
{
storage::serialization::read(
reader, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]);
serialization::read(reader, "/ch/metrics/" + pair.first, pair.second);
}
reader.ReadInto("/ch/connectivity_checksum", connectivity_checksum);
}
// writes .osrm.hsgr file
template <typename QueryGraphT, typename EdgeFilterT>
template <typename ContractedMetricT>
inline void writeGraph(const boost::filesystem::path &path,
unsigned checksum,
const QueryGraphT &graph,
const std::vector<EdgeFilterT> &edge_filter,
const std::unordered_map<std::string, ContractedMetricT> &metrics,
const std::uint32_t connectivity_checksum)
{
static_assert(std::is_same<QueryGraphView, QueryGraphT>::value ||
std::is_same<QueryGraph, QueryGraphT>::value,
"graph must be of type QueryGraph<>");
static_assert(std::is_same<EdgeFilterT, std::vector<bool>>::value ||
std::is_same<EdgeFilterT, util::vector_view<bool>>::value,
"edge_filter must be a container of vector<bool> or vector_view<bool>");
static_assert(std::is_same<ContractedMetric, ContractedMetricT>::value ||
std::is_same<ContractedMetricView, ContractedMetricT>::value,
"metric must be of type ContractedMetric<>");
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
writer.WriteElementCount64("/ch/checksum", 1);
writer.WriteFrom("/ch/checksum", checksum);
util::serialization::write(writer, "/ch/contracted_graph", graph);
writer.WriteElementCount64("/ch/edge_filter", edge_filter.size());
for (const auto index : util::irange<std::size_t>(0, edge_filter.size()))
{
storage::serialization::write(
writer, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]);
}
writer.WriteElementCount64("/ch/connectivity_checksum", 1);
writer.WriteFrom("/ch/connectivity_checksum", connectivity_checksum);
for (const auto &pair : metrics)
{
serialization::write(writer, "/ch/metrics/" + pair.first, pair.second);
}
}
}
}
include/contractor/serialization.hpp
+53
View File
@@ -0,0 +1,53 @@
#ifndef OSRM_CONTRACTOR_SERIALIZATION_HPP
#define OSRM_CONTRACTOR_SERIALIZATION_HPP
#include "contractor/contracted_metric.hpp"
#include "util/serialization.hpp"
#include "storage/serialization.hpp"
#include "storage/tar.hpp"
namespace osrm
{
namespace contractor
{
namespace serialization
{
template <storage::Ownership Ownership>
void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::ContractedMetric<Ownership> &metric)
{
util::serialization::write(writer, name + "/contracted_graph", metric.graph);
writer.WriteElementCount64(name + "/exclude", metric.edge_filter.size());
for (const auto index : util::irange<std::size_t>(0, metric.edge_filter.size()))
{
storage::serialization::write(writer,
name + "/exclude/" + std::to_string(index) + "/edge_filter",
metric.edge_filter[index]);
}
}
template <storage::Ownership Ownership>
void read(storage::tar::FileReader &reader,
const std::string &name,
detail::ContractedMetric<Ownership> &metric)
{
util::serialization::read(reader, name + "/contracted_graph", metric.graph);
metric.edge_filter.resize(reader.ReadElementCount64(name + "/exclude"));
for (const auto index : util::irange<std::size_t>(0, metric.edge_filter.size()))
{
storage::serialization::read(reader,
name + "/exclude/" + std::to_string(index) + "/edge_filter",
metric.edge_filter[index]);
}
}
}
}
}
#endif
include/customizer/files.hpp
+29 -13
View File
@@ -7,6 +7,8 @@
#include "util/integer_range.hpp"
#include <unordered_map>
namespace osrm
{
namespace customizer
@@ -16,7 +18,7 @@ namespace files
// reads .osrm.cell_metrics file
template <typename CellMetricT>
inline void readCellMetrics(const boost::filesystem::path &path, std::vector<CellMetricT> &metrics)
inline void readCellMetrics(const boost::filesystem::path &path, std::unordered_map<std::string, std::vector<CellMetricT>> &metrics)
{
static_assert(std::is_same<CellMetricView, CellMetricT>::value ||
std::is_same<CellMetric, CellMetricT>::value,
@@ -25,20 +27,27 @@ inline void readCellMetrics(const boost::filesystem::path &path, std::vector<Cel
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
auto num_metrics = reader.ReadElementCount64("/mld/metrics");
metrics.resize(num_metrics);
auto id = 0;
for (auto &metric : metrics)
for (auto& pair : metrics)
{
serialization::read(reader, "/mld/metrics/" + std::to_string(id++), metric);
const auto& metric_name = pair.first;
auto& metric_exclude_classes = pair.second;
auto prefix = "/mld/metrics/" + metric_name + "/exclude";
auto num_exclude_classes = reader.ReadElementCount64(prefix);
metric_exclude_classes.resize(num_exclude_classes);
auto id = 0;
for (auto &metric : metric_exclude_classes)
{
serialization::read(reader, prefix + "/" + std::to_string(id++), metric);
}
}
}
// writes .osrm.cell_metrics file
template <typename CellMetricT>
inline void writeCellMetrics(const boost::filesystem::path &path,
const std::vector<CellMetricT> &metrics)
const std::unordered_map<std::string, std::vector<CellMetricT>> &metrics)
{
static_assert(std::is_same<CellMetricView, CellMetricT>::value ||
std::is_same<CellMetric, CellMetricT>::value,
@@ -47,12 +56,19 @@ inline void writeCellMetrics(const boost::filesystem::path &path,
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
writer.WriteElementCount64("/mld/metrics", metrics.size());
auto id = 0;
for (const auto &metric : metrics)
for (const auto& pair : metrics)
{
serialization::write(writer, "/mld/metrics/" + std::to_string(id++), metric);
const auto& metric_name = pair.first;
const auto& metric_exclude_classes = pair.second;
auto prefix = "/mld/metrics/" + metric_name + "/exclude";
writer.WriteElementCount64(prefix, metric_exclude_classes.size());
auto id = 0;
for (auto &exclude_metric : metric_exclude_classes)
{
serialization::write(writer, prefix + "/" + std::to_string(id++), exclude_metric);
}
}
}
}
include/engine/algorithm.hpp
+5
View File
@@ -30,6 +30,11 @@ template <typename AlgorithmT> const char *name();
template <> inline const char *name<ch::Algorithm>() { return "CH"; }
template <> inline const char *name<mld::Algorithm>() { return "MLD"; }
// Algorithm identifier
template <typename AlgorithmT> const char *identifier();
template <> inline const char *identifier<ch::Algorithm>() { return "ch"; }
template <> inline const char *identifier<mld::Algorithm>() { return "mld"; }
template <typename AlgorithmT> struct HasAlternativePathSearch final : std::false_type
{
};
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
+35 -22
View File
@@ -83,23 +83,26 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
void InitializeGraphPointer(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
const std::string metric_prefix = "/ch/metrics/" + metric_name;
auto graph_nodes_ptr =
data_layout.GetBlockPtr<GraphNode>(memory_block, "/ch/contracted_graph/node_array");
data_layout.GetBlockPtr<GraphNode>(memory_block, metric_prefix + "/contracted_graph/node_array");
auto graph_edges_ptr =
data_layout.GetBlockPtr<GraphEdge>(memory_block, "/ch/contracted_graph/edge_array");
data_layout.GetBlockPtr<GraphEdge>(memory_block, metric_prefix + "/contracted_graph/edge_array");
auto filter_block_id = "/ch/edge_filter/" + std::to_string(exclude_index);
auto exclude_prefix = metric_prefix + "/exclude/" + std::to_string(exclude_index);
auto filter_block_id = exclude_prefix + "/edge_filter";
auto edge_filter_ptr =
data_layout.GetBlockPtr<util::vector_view<bool>::Word>(memory_block, filter_block_id);
util::vector_view<GraphNode> node_list(
graph_nodes_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/node_array"));
graph_nodes_ptr, data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/node_array"));
util::vector_view<GraphEdge> edge_list(
graph_edges_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/edge_array"));
graph_edges_ptr, data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/edge_array"));
util::vector_view<bool> edge_filter(edge_filter_ptr,
data_layout.GetBlockEntries(filter_block_id));
@@ -108,17 +111,18 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
public:
ContiguousInternalMemoryAlgorithmDataFacade(
std::shared_ptr<ContiguousBlockAllocator> allocator_, std::size_t exclude_index)
std::shared_ptr<ContiguousBlockAllocator> allocator_, const std::string& metric_name, std::size_t exclude_index)
: allocator(std::move(allocator_))
{
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
}
void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
InitializeGraphPointer(data_layout, memory_block, exclude_index);
InitializeGraphPointer(data_layout, memory_block, metric_name, exclude_index);
}
// search graph access
@@ -225,8 +229,11 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
void InitializeProfilePropertiesPointer(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
// TODO: For multi-metric support we need to have separate exclude classes per metric
(void) metric_name;
m_profile_properties = data_layout.GetBlockPtr<extractor::ProfileProperties>(
memory_block, "/common/properties");
@@ -528,6 +535,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
InitializeChecksumPointer(data_layout, memory_block);
@@ -538,7 +546,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
InitializeGeometryPointers(data_layout, memory_block);
InitializeNamePointers(data_layout, memory_block);
InitializeTurnLaneDescriptionsPointers(data_layout, memory_block);
InitializeProfilePropertiesPointer(data_layout, memory_block, exclude_index);
InitializeProfilePropertiesPointer(data_layout, memory_block, metric_name, exclude_index);
InitializeRTreePointers(data_layout, memory_block);
InitializeIntersectionClassPointers(data_layout, memory_block);
InitializeManeuverOverridePointers(data_layout, memory_block);
@@ -548,10 +556,11 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
// allows switching between process_memory/shared_memory datafacade, based on the type of
// allocator
ContiguousInternalMemoryDataFacadeBase(std::shared_ptr<ContiguousBlockAllocator> allocator_,
const std::string& metric_name,
const std::size_t exclude_index)
: allocator(std::move(allocator_))
{
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
}
// node and edge information access
@@ -959,9 +968,10 @@ class ContiguousInternalMemoryDataFacade<CH>
{
public:
ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator,
const std::string& metric_name,
const std::size_t exclude_index)
: ContiguousInternalMemoryDataFacadeBase(allocator, exclude_index),
ContiguousInternalMemoryAlgorithmDataFacade<CH>(allocator, exclude_index)
: ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index),
ContiguousInternalMemoryAlgorithmDataFacade<CH>(allocator, metric_name, exclude_index)
{
}
@@ -981,14 +991,16 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
InitializeMLDDataPointers(data_layout, memory_block, exclude_index);
InitializeMLDDataPointers(data_layout, memory_block, metric_name, exclude_index);
InitializeGraphPointer(data_layout, memory_block);
}
void InitializeMLDDataPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
if (data_layout.GetBlockSize("/mld/multilevelpartition/partition") > 0)
@@ -1016,9 +1028,9 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
partitioner::MultiLevelPartitionView{level_data, partition, cell_to_children};
}
const auto weights_block_id = "/mld/metrics/" + std::to_string(exclude_index) + "/weights";
const auto durations_block_id =
"/mld/metrics/" + std::to_string(exclude_index) + "/durations";
const auto exclude_prefix = "/mld/metrics/" + metric_name + "/exclude/" + std::to_string(exclude_index);
const auto weights_block_id = exclude_prefix + "/weights";
const auto durations_block_id = exclude_prefix + "/durations";
if (data_layout.GetBlockSize(weights_block_id) > 0)
{
@@ -1026,8 +1038,8 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
data_layout.GetBlockPtr<EdgeWeight>(memory_block, weights_block_id);
auto mld_cell_durations_ptr =
data_layout.GetBlockPtr<EdgeDuration>(memory_block, durations_block_id);
auto weight_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/weights");
auto duration_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/durations");
auto weight_entries_count = data_layout.GetBlockEntries(weights_block_id);
auto duration_entries_count = data_layout.GetBlockEntries(durations_block_id);
BOOST_ASSERT(weight_entries_count == duration_entries_count);
util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
util::vector_view<EdgeDuration> durations(mld_cell_durations_ptr,
@@ -1099,10 +1111,10 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
public:
ContiguousInternalMemoryAlgorithmDataFacade(
std::shared_ptr<ContiguousBlockAllocator> allocator_, const std::size_t exclude_index)
std::shared_ptr<ContiguousBlockAllocator> allocator_, const std::string& metric_name, const std::size_t exclude_index)
: allocator(std::move(allocator_))
{
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
}
const partitioner::MultiLevelPartitionView &GetMultiLevelPartition() const override
@@ -1156,9 +1168,10 @@ class ContiguousInternalMemoryDataFacade<MLD> final
private:
public:
ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator,
const std::string& metric_name,
const std::size_t exclude_index)
: ContiguousInternalMemoryDataFacadeBase(allocator, exclude_index),
ContiguousInternalMemoryAlgorithmDataFacade<MLD>(allocator, exclude_index)
: ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index),
ContiguousInternalMemoryAlgorithmDataFacade<MLD>(allocator, metric_name, exclude_index)
{
}
include/engine/datafacade_factory.hpp
+27 -8
View File
@@ -33,6 +33,7 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
DataFacadeFactory(std::shared_ptr<AllocatorT> allocator)
: DataFacadeFactory(allocator, has_exclude_flags)
{
BOOST_ASSERT_MSG(facades.size() >= 1, "At least one datafacade is needed");
}
template <typename ParameterT> std::shared_ptr<const Facade> Get(const ParameterT &params) const
@@ -45,13 +46,27 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
template <typename AllocatorT>
DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::true_type)
{
for (const auto index : util::irange<std::size_t>(0, facades.size()))
{
facades[index] = std::make_shared<const Facade>(allocator, index);
}
properties = allocator->GetLayout().template GetBlockPtr<extractor::ProfileProperties>(
const auto &layout = allocator->GetLayout();
properties = layout.template GetBlockPtr<extractor::ProfileProperties>(
allocator->GetMemory(), "/common/properties");
const auto &metric_name = properties->GetWeightName();
std::vector<std::string> exclude_prefixes;
auto exclude_path = std::string("/") + routing_algorithms::identifier<AlgorithmT>() +
std::string("/metrics/") + metric_name + "/exclude/";
layout.List(exclude_path, std::back_inserter(exclude_prefixes));
facades.resize(exclude_prefixes.size());
for (const auto &exclude_prefix : exclude_prefixes)
{
auto index_begin = exclude_prefix.find_last_of("/");
BOOST_ASSERT_MSG(index_begin != std::string::npos,
"The exclude prefix needs to be a valid data path.");
std::size_t index =
std::stoi(exclude_prefix.substr(index_begin + 1, exclude_prefix.size()));
BOOST_ASSERT(index >= 0 && index < facades.size());
facades[index] = std::make_shared<const Facade>(allocator, metric_name, index);
}
for (const auto index : util::irange<std::size_t>(0, properties->class_names.size()))
{
@@ -67,7 +82,11 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
template <typename AllocatorT>
DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::false_type)
{
facades[0] = std::make_shared<const Facade>(allocator, 0);
const auto &layout = allocator->GetLayout();
properties = layout.template GetBlockPtr<extractor::ProfileProperties>(
allocator->GetMemory(), "/common/properties");
const auto &metric_name = properties->GetWeightName();
facades.push_back(std::make_shared<const Facade>(allocator, metric_name, 0));
}
std::shared_ptr<const Facade> Get(const api::TileParameters &, std::false_type) const
@@ -124,7 +143,7 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
return {};
}
std::array<std::shared_ptr<const Facade>, extractor::MAX_EXCLUDABLE_CLASSES> facades;
std::vector<std::shared_ptr<const Facade>> facades;
std::unordered_map<std::string, extractor::ClassData> name_to_class;
const extractor::ProfileProperties *properties = nullptr;
};
include/engine/engine.hpp
+32 -15
View File
@@ -155,10 +155,19 @@ bool Engine<routing_algorithms::ch::Algorithm>::CheckCompatibility(const EngineC
auto mem = storage::makeSharedMemory(barrier.data().region);
storage::DataLayout layout;
storage::io::BufferReader reader(reinterpret_cast<const char*>(mem->Ptr()), mem->Size());
storage::io::BufferReader reader(reinterpret_cast<const char *>(mem->Ptr()), mem->Size());
storage::serialization::read(reader, layout);
return layout.HasBlock("/ch/contracted_graph/node_array") &&
layout.HasBlock("/ch/contracted_graph/edge_array");
std::vector<std::string> metric_prefixes;
layout.List("/ch/metrics/", std::back_inserter(metric_prefixes));
bool has_graph = false;
for (const auto &metric_prefix : metric_prefixes)
{
has_graph |= layout.HasBlock(metric_prefix + "/contracted_graph/node_array") &&
layout.HasBlock(metric_prefix + "/contracted_graph/edge_array");
}
return has_graph;
}
else
{
@@ -177,24 +186,32 @@ bool Engine<routing_algorithms::mld::Algorithm>::CheckCompatibility(const Engine
auto mem = storage::makeSharedMemory(barrier.data().region);
storage::DataLayout layout;
storage::io::BufferReader reader(reinterpret_cast<const char*>(mem->Ptr()), mem->Size());
storage::io::BufferReader reader(reinterpret_cast<const char *>(mem->Ptr()), mem->Size());
storage::serialization::read(reader, layout);
std::vector<std::string> metric_prefixes;
layout.List("/mld/metrics/", std::back_inserter(metric_prefixes));
bool has_cells = false;
for (const auto &metric_prefix : metric_prefixes)
{
has_cells |= layout.HasBlock(metric_prefix + "/exclude/0/durations") &&
layout.HasBlock(metric_prefix + "/exclude/0/weights");
}
// checks that all the needed memory blocks are populated
// "/mld/cellstorage/source_boundary" and "/mld/cellstorage/destination_boundary"
// are not checked, because in situations where there are so few nodes in the graph that
// they all fit into one cell, they can be empty.
bool empty_data = layout.HasBlock("/mld/multilevelpartition/level_data") &&
layout.HasBlock("/mld/multilevelpartition/partition") &&
layout.HasBlock("/mld/multilevelpartition/cell_to_children") &&
layout.HasBlock("/mld/cellstorage/cells") &&
layout.HasBlock("/mld/cellstorage/level_to_cell_offset") &&
layout.HasBlock("/mld/multilevelgraph/node_array") &&
layout.HasBlock("/mld/multilevelgraph/edge_array") &&
layout.HasBlock("/mld/metrics/0/weights") &&
layout.HasBlock("/mld/metrics/0/durations") &&
layout.HasBlock("/mld/multilevelgraph/node_to_edge_offset");
return empty_data;
bool has_data = has_cells && layout.HasBlock("/mld/multilevelpartition/level_data") &&
layout.HasBlock("/mld/multilevelpartition/partition") &&
layout.HasBlock("/mld/multilevelpartition/cell_to_children") &&
layout.HasBlock("/mld/cellstorage/cells") &&
layout.HasBlock("/mld/cellstorage/level_to_cell_offset") &&
layout.HasBlock("/mld/multilevelgraph/node_array") &&
layout.HasBlock("/mld/multilevelgraph/edge_array") &&
layout.HasBlock("/mld/multilevelgraph/node_to_edge_offset");
return has_data;
}
else
{
include/storage/serialization.hpp
+66 -65
View File
@@ -27,6 +27,72 @@ namespace storage
namespace serialization
{
namespace detail
{
template <typename T, typename BlockT = unsigned char>
inline BlockT packBits(const T &data, std::size_t index, std::size_t count)
{
static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
BlockT value = 0;
for (std::size_t bit = 0; bit < count; ++bit, ++index)
value = (value << 1) | data[index];
return value;
}
template <typename T, typename BlockT = unsigned char>
inline void unpackBits(T &data, std::size_t index, std::size_t count, BlockT value)
{
static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
const BlockT mask = BlockT{1} << (count - 1);
for (std::size_t bit = 0; bit < count; value <<= 1, ++bit, ++index)
data[index] = value & mask;
}
template <typename VectorT>
void readBoolVector(tar::FileReader &reader, const std::string &name, VectorT &data)
{
const auto count = reader.ReadElementCount64(name);
data.resize(count);
std::uint64_t index = 0;
constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
const auto decode = [&](const std::uint64_t block) {
auto read_size = std::min<std::size_t>(count - index, WORD_BITS);
unpackBits<VectorT, std::uint64_t>(data, index, read_size, block);
index += WORD_BITS;
};
reader.ReadStreaming<std::uint64_t>(name, boost::make_function_output_iterator(decode));
}
template <typename VectorT>
void writeBoolVector(tar::FileWriter &writer, const std::string &name, const VectorT &data)
{
const auto count = data.size();
writer.WriteElementCount64(name, count);
std::uint64_t index = 0;
constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
// FIXME on old boost version the function_input_iterator does not work with lambdas
// so we need to wrap it in a function here.
const std::function<std::uint64_t()> encode_function = [&]() -> std::uint64_t {
auto write_size = std::min<std::size_t>(count - index, WORD_BITS);
auto packed = packBits<VectorT, std::uint64_t>(data, index, write_size);
index += WORD_BITS;
return packed;
};
std::uint64_t number_of_blocks = (count + WORD_BITS - 1) / WORD_BITS;
writer.WriteStreaming<std::uint64_t>(
name,
boost::make_function_input_iterator(encode_function, boost::infinite()),
number_of_blocks);
}
}
/* All vector formats here use the same on-disk format.
* This is important because we want to be able to write from a vector
* of one kind, but read it into a vector of another kind.
@@ -155,71 +221,6 @@ void write(tar::FileWriter &writer, const std::string &name, const util::vector_
writer.WriteFrom(name, data.data(), count);
}
namespace detail
{
template <typename T, typename BlockT = unsigned char>
inline BlockT packBits(const T &data, std::size_t index, std::size_t count)
{
static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
BlockT value = 0;
for (std::size_t bit = 0; bit < count; ++bit, ++index)
value = (value << 1) | data[index];
return value;
}
template <typename T, typename BlockT = unsigned char>
inline void unpackBits(T &data, std::size_t index, std::size_t count, BlockT value)
{
static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
const BlockT mask = BlockT{1} << (count - 1);
for (std::size_t bit = 0; bit < count; value <<= 1, ++bit, ++index)
data[index] = value & mask;
}
template <typename VectorT>
void readBoolVector(tar::FileReader &reader, const std::string &name, VectorT &data)
{
const auto count = reader.ReadElementCount64(name);
data.resize(count);
std::uint64_t index = 0;
constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
const auto decode = [&](const std::uint64_t block) {
auto read_size = std::min<std::size_t>(count - index, WORD_BITS);
unpackBits<VectorT, std::uint64_t>(data, index, read_size, block);
index += WORD_BITS;
};
reader.ReadStreaming<std::uint64_t>(name, boost::make_function_output_iterator(decode));
}
template <typename VectorT>
void writeBoolVector(tar::FileWriter &writer, const std::string &name, const VectorT &data)
{
const auto count = data.size();
writer.WriteElementCount64(name, count);
std::uint64_t index = 0;
constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
// FIXME on old boost version the function_input_iterator does not work with lambdas
// so we need to wrap it in a function here.
const std::function<std::uint64_t()> encode_function = [&]() -> std::uint64_t {
auto write_size = std::min<std::size_t>(count - index, WORD_BITS);
auto packed = packBits<VectorT, std::uint64_t>(data, index, write_size);
index += WORD_BITS;
return packed;
};
std::uint64_t number_of_blocks = (count + WORD_BITS - 1) / WORD_BITS;
writer.WriteStreaming<std::uint64_t>(
name,
boost::make_function_input_iterator(encode_function, boost::infinite()),
number_of_blocks);
}
}
template <>
inline void
read<bool>(tar::FileReader &reader, const std::string &name, util::vector_view<bool> &data)
include/storage/shared_datatype.hpp
-2
View File
@@ -94,8 +94,6 @@ class DataLayout
{
char *start_canary_ptr = ptr - sizeof(CANARY);
char *end_canary_ptr = ptr + GetBlockSize(name);
std::cout << name << ": " << (long)(start_canary_ptr - shared_memory) << " -> "
<< (long)(end_canary_ptr - shared_memory) << std::endl;
std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
}