Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Remove fixed block identifiers

Browse Source
This commit is contained in:
Patrick Niklaus
2018-03-22 19:16:19 +00:00
committed by Patrick Niklaus
parent f3b7ab92ff
commit 4a9fdca7b2
14 changed files with 483 additions and 576 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/storage/io.hpp
+4
View File
@@ -303,6 +303,10 @@ class BufferReader
}
}
std::size_t GetPosition() {
return input_stream.tellg();
}
template <typename T> void ReadInto(T *dest, const std::size_t count)
{
#if !defined(__GNUC__) || (__GNUC__ > 4)
include/storage/serialization.hpp
+64
View File
@@ -7,12 +7,14 @@
#include "storage/io.hpp"
#include "storage/tar.hpp"
#include "storage/shared_datatype.hpp"
#include <boost/function_output_iterator.hpp>
#include <boost/iterator/function_input_iterator.hpp>
#include <cmath>
#include <cstdint>
#include <tuple>
#if USE_STXXL_LIBRARY
#include <stxxl/vector>
@@ -81,6 +83,32 @@ template <typename T> void write(io::BufferWriter &writer, const std::vector<T>
writer.WriteFrom(data.data(), count);
}
template<typename T>
inline void write(io::BufferWriter &writer, const T &data)
{
writer.WriteFrom(data);
}
template<typename T>
inline void read(io::BufferReader &reader, T &data)
{
reader.ReadInto(data);
}
inline void write(io::BufferWriter &writer, const std::string &data)
{
const auto count = data.size();
writer.WriteElementCount64(count);
writer.WriteFrom(data.data(), count);
}
inline void read(io::BufferReader &reader, std::string &data)
{
const auto count = reader.ReadElementCount64();
data.resize(count);
reader.ReadInto(const_cast<char *>(data.data()), count);
}
inline void write(tar::FileWriter &writer, const std::string &name, const std::string &data)
{
const auto count = data.size();
@@ -218,6 +246,42 @@ write<bool>(tar::FileWriter &writer, const std::string &name, const std::vector<
{
detail::writeBoolVector(writer, name, data);
}
template <typename K, typename V> void read(io::BufferReader &reader, std::map<K, V> &data)
{
const auto count = reader.ReadElementCount64();
std::pair<K, V> pair;
for (auto index : util::irange<std::size_t>(0, count))
{
(void) index;
read(reader, pair.first);
read(reader, pair.second);
data.insert(pair);
}
}
template <typename K, typename V> void write(io::BufferWriter &writer, const std::map<K, V> &data)
{
const auto count = data.size();
writer.WriteElementCount64(count);
for (const auto &pair : data)
{
write(writer, pair.first);
write(writer, pair.second);
}
}
inline void read(io::BufferReader &reader, DataLayout &layout)
{
read(reader, layout.blocks);
}
inline void write(io::BufferWriter &writer, const DataLayout &layout)
{
write(writer, layout.blocks);
}
}
}
}
include/storage/shared_datatype.hpp
+50 -177
View File
@@ -2,6 +2,7 @@
#define SHARED_DATA_TYPE_HPP
#include "storage/block.hpp"
#include "storage/io_fwd.hpp"
#include "util/exception.hpp"
#include "util/exception_utils.hpp"
@@ -11,219 +12,91 @@
#include <array>
#include <cstdint>
#include <map>
namespace osrm
{
namespace storage
{
class DataLayout;
namespace serialization
{
inline void read(io::BufferReader &reader, DataLayout &layout);
inline void write(io::BufferWriter &writer, const DataLayout &layout);
}
// Added at the start and end of each block as sanity check
const constexpr char CANARY[4] = {'O', 'S', 'R', 'M'};
const constexpr char *block_id_to_name[] = {"IGNORE_BLOCK",
"NAME_BLOCKS",
"NAME_VALUES",
"EDGE_BASED_NODE_DATA",
"ANNOTATION_DATA",
"CH_GRAPH_NODE_LIST",
"CH_GRAPH_EDGE_LIST",
"CH_EDGE_FILTER_0",
"CH_EDGE_FILTER_1",
"CH_EDGE_FILTER_2",
"CH_EDGE_FILTER_3",
"CH_EDGE_FILTER_4",
"CH_EDGE_FILTER_5",
"CH_EDGE_FILTER_6",
"CH_EDGE_FILTER_7",
"COORDINATE_LIST",
"OSM_NODE_ID_LIST",
"TURN_INSTRUCTION",
"ENTRY_CLASSID",
"R_SEARCH_TREE",
"R_SEARCH_TREE_LEVEL_STARTS",
"GEOMETRIES_INDEX",
"GEOMETRIES_NODE_LIST",
"GEOMETRIES_FWD_WEIGHT_LIST",
"GEOMETRIES_REV_WEIGHT_LIST",
"GEOMETRIES_FWD_DURATION_LIST",
"GEOMETRIES_REV_DURATION_LIST",
"GEOMETRIES_FWD_DATASOURCES_LIST",
"GEOMETRIES_REV_DATASOURCES_LIST",
"HSGR_CHECKSUM",
"FILE_INDEX_PATH",
"DATASOURCES_NAMES",
"PROPERTIES",
"BEARING_CLASSID",
"BEARING_OFFSETS",
"BEARING_BLOCKS",
"BEARING_VALUES",
"ENTRY_CLASS",
"LANE_DATA_ID",
"PRE_TURN_BEARING",
"POST_TURN_BEARING",
"TURN_LANE_DATA",
"LANE_DESCRIPTION_OFFSETS",
"LANE_DESCRIPTION_MASKS",
"TURN_WEIGHT_PENALTIES",
"TURN_DURATION_PENALTIES",
"MLD_LEVEL_DATA",
"MLD_PARTITION",
"MLD_CELL_TO_CHILDREN",
"MLD_CELL_WEIGHTS_0",
"MLD_CELL_WEIGHTS_1",
"MLD_CELL_WEIGHTS_2",
"MLD_CELL_WEIGHTS_3",
"MLD_CELL_WEIGHTS_4",
"MLD_CELL_WEIGHTS_5",
"MLD_CELL_WEIGHTS_6",
"MLD_CELL_WEIGHTS_7",
"MLD_CELL_DURATIONS_0",
"MLD_CELL_DURATIONS_1",
"MLD_CELL_DURATIONS_2",
"MLD_CELL_DURATIONS_3",
"MLD_CELL_DURATIONS_4",
"MLD_CELL_DURATIONS_5",
"MLD_CELL_DURATIONS_6",
"MLD_CELL_DURATIONS_7",
"MLD_CELL_SOURCE_BOUNDARY",
"MLD_CELL_DESTINATION_BOUNDARY",
"MLD_CELLS",
"MLD_CELL_LEVEL_OFFSETS",
"MLD_GRAPH_NODE_LIST",
"MLD_GRAPH_EDGE_LIST",
"MLD_GRAPH_NODE_TO_OFFSET",
"MANEUVER_OVERRIDES",
"MANEUVER_OVERRIDE_NODE_SEQUENCES"};
class DataLayout
{
public:
enum BlockID
{
IGNORE_BLOCK = 0,
NAME_BLOCKS,
NAME_VALUES,
EDGE_BASED_NODE_DATA_LIST,
ANNOTATION_DATA_LIST,
CH_GRAPH_NODE_LIST,
CH_GRAPH_EDGE_LIST,
CH_EDGE_FILTER_0,
CH_EDGE_FILTER_1,
CH_EDGE_FILTER_2,
CH_EDGE_FILTER_3,
CH_EDGE_FILTER_4,
CH_EDGE_FILTER_5,
CH_EDGE_FILTER_6,
CH_EDGE_FILTER_7,
COORDINATE_LIST,
OSM_NODE_ID_LIST,
TURN_INSTRUCTION,
ENTRY_CLASSID,
R_SEARCH_TREE,
R_SEARCH_TREE_LEVEL_STARTS,
GEOMETRIES_INDEX,
GEOMETRIES_NODE_LIST,
GEOMETRIES_FWD_WEIGHT_LIST,
GEOMETRIES_REV_WEIGHT_LIST,
GEOMETRIES_FWD_DURATION_LIST,
GEOMETRIES_REV_DURATION_LIST,
GEOMETRIES_FWD_DATASOURCES_LIST,
GEOMETRIES_REV_DATASOURCES_LIST,
HSGR_CHECKSUM,
FILE_INDEX_PATH,
DATASOURCES_NAMES,
PROPERTIES,
BEARING_CLASSID,
BEARING_OFFSETS,
BEARING_BLOCKS,
BEARING_VALUES,
ENTRY_CLASS,
LANE_DATA_ID,
PRE_TURN_BEARING,
POST_TURN_BEARING,
TURN_LANE_DATA,
LANE_DESCRIPTION_OFFSETS,
LANE_DESCRIPTION_MASKS,
TURN_WEIGHT_PENALTIES,
TURN_DURATION_PENALTIES,
MLD_LEVEL_DATA,
MLD_PARTITION,
MLD_CELL_TO_CHILDREN,
MLD_CELL_WEIGHTS_0,
MLD_CELL_WEIGHTS_1,
MLD_CELL_WEIGHTS_2,
MLD_CELL_WEIGHTS_3,
MLD_CELL_WEIGHTS_4,
MLD_CELL_WEIGHTS_5,
MLD_CELL_WEIGHTS_6,
MLD_CELL_WEIGHTS_7,
MLD_CELL_DURATIONS_0,
MLD_CELL_DURATIONS_1,
MLD_CELL_DURATIONS_2,
MLD_CELL_DURATIONS_3,
MLD_CELL_DURATIONS_4,
MLD_CELL_DURATIONS_5,
MLD_CELL_DURATIONS_6,
MLD_CELL_DURATIONS_7,
MLD_CELL_SOURCE_BOUNDARY,
MLD_CELL_DESTINATION_BOUNDARY,
MLD_CELLS,
MLD_CELL_LEVEL_OFFSETS,
MLD_GRAPH_NODE_LIST,
MLD_GRAPH_EDGE_LIST,
MLD_GRAPH_NODE_TO_OFFSET,
MANEUVER_OVERRIDES,
MANEUVER_OVERRIDE_NODE_SEQUENCES,
NUM_BLOCKS
};
DataLayout() : blocks{} {}
inline void SetBlock(BlockID bid, Block block) { blocks[bid] = std::move(block); }
inline void SetBlock(const std::string &name, Block block) { blocks[name] = std::move(block); }
inline uint64_t GetBlockEntries(BlockID bid) const { return blocks[bid].num_entries; }
inline uint64_t GetBlockEntries(const std::string &name) const
{
auto iter = blocks.find(name);
if (iter == blocks.end())
{
throw util::exception("Could not find block " + name);
}
inline uint64_t GetBlockSize(BlockID bid) const { return blocks[bid].byte_size; }
return iter->second.num_entries;
}
inline uint64_t GetBlockSize(const std::string &name) const
{
auto iter = blocks.find(name);
if (iter == blocks.end())
{
throw util::exception("Could not find block " + name);
}
return iter->second.byte_size;
}
inline uint64_t GetSizeOfLayout() const
{
uint64_t result = 0;
for (auto i = 0; i < NUM_BLOCKS; i++)
for (const auto &name_and_block : blocks)
{
result += 2 * sizeof(CANARY) + GetBlockSize(static_cast<BlockID>(i)) + BLOCK_ALIGNMENT;
result += 2 * sizeof(CANARY) + GetBlockSize(name_and_block.first) + BLOCK_ALIGNMENT;
}
return result;
}
template <typename T, bool WRITE_CANARY = false>
inline T *GetBlockPtr(char *shared_memory, BlockID bid) const
inline T *GetBlockPtr(char *shared_memory, const std::string &name) const
{
static_assert(BLOCK_ALIGNMENT % std::alignment_of<T>::value == 0,
"Datatype does not fit alignment constraints.");
char *ptr = (char *)GetAlignedBlockPtr(shared_memory, bid);
char *ptr = (char *)GetAlignedBlockPtr(shared_memory, name);
if (WRITE_CANARY)
{
char *start_canary_ptr = ptr - sizeof(CANARY);
char *end_canary_ptr = ptr + GetBlockSize(bid);
char *end_canary_ptr = ptr + GetBlockSize(name);
std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
}
else
{
char *start_canary_ptr = ptr - sizeof(CANARY);
char *end_canary_ptr = ptr + GetBlockSize(bid);
char *end_canary_ptr = ptr + GetBlockSize(name);
bool start_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
bool end_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
if (!start_canary_alive)
{
throw util::exception("Start canary of block corrupted. (" +
std::string(block_id_to_name[bid]) + ")" + SOURCE_REF);
throw util::exception("Start canary of block corrupted. (" + name + ")" +
SOURCE_REF);
}
if (!end_canary_alive)
{
throw util::exception("End canary of block corrupted. (" +
std::string(block_id_to_name[bid]) + ")" + SOURCE_REF);
throw util::exception("End canary of block corrupted. (" + name + ")" + SOURCE_REF);
}
}
@@ -231,6 +104,9 @@ class DataLayout
}
private:
friend void serialization::read(io::BufferReader &reader, DataLayout &layout);
friend void serialization::write(io::BufferWriter &writer, const DataLayout &layout);
// Fit aligned storage in buffer to 64 bytes to conform with AVX 512 types
inline void *align(void *&ptr) const noexcept
{
@@ -239,13 +115,13 @@ class DataLayout
return ptr = reinterpret_cast<void *>(aligned);
}
inline void *GetAlignedBlockPtr(void *ptr, BlockID bid) const
inline void *GetAlignedBlockPtr(void *ptr, const std::string &name) const
{
for (auto i = 0; i < bid; i++)
for (auto iter = blocks.begin(); iter != blocks.end() && iter->first != name; ++iter)
{
ptr = static_cast<char *>(ptr) + sizeof(CANARY);
ptr = align(ptr);
ptr = static_cast<char *>(ptr) + GetBlockSize((BlockID)i);
ptr = static_cast<char *>(ptr) + GetBlockSize(name);
ptr = static_cast<char *>(ptr) + sizeof(CANARY);
}
@@ -254,14 +130,14 @@ class DataLayout
return ptr;
}
template <typename T> inline T *GetBlockEnd(char *shared_memory, BlockID bid) const
template <typename T> inline T *GetBlockEnd(char *shared_memory, const std::string &name) const
{
auto begin = GetBlockPtr<T>(shared_memory, bid);
return begin + GetBlockEntries(bid);
auto begin = GetBlockPtr<T>(shared_memory, name);
return begin + GetBlockEntries(name);
}
static constexpr std::size_t BLOCK_ALIGNMENT = 64;
std::array<Block, NUM_BLOCKS> blocks;
std::map<std::string, Block> blocks;
};
enum SharedDataType
@@ -298,9 +174,6 @@ inline std::string regionToString(const SharedDataType region)
return "INVALID_REGION";
}
}
static_assert(sizeof(block_id_to_name) / sizeof(*block_id_to_name) == DataLayout::NUM_BLOCKS,
"Number of blocks needs to match the number of Block names.");
}
}