Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Remove fixed block identifiers

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-22 19:16:19 +00:00 committed by Patrick Niklaus
parent f3b7ab92ff
commit 4a9fdca7b2
14 changed files with 483 additions and 576 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/engine/datafacade/contiguous_block_allocator.hpp
View File

@ -16,7 +16,7 @@ class ContiguousBlockAllocator
virtual ~ContiguousBlockAllocator() = default;
// interface to give access to the datafacades
virtual storage::DataLayout &GetLayout() = 0;
virtual const storage::DataLayout &GetLayout() = 0;
virtual char *GetMemory() = 0;
};

291
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
View File

@ -81,26 +81,25 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
// allocator that keeps the allocation data
std::shared_ptr<ContiguousBlockAllocator> allocator;
void InitializeGraphPointer(storage::DataLayout &data_layout,
void InitializeGraphPointer(const storage::DataLayout &data_layout,
char *memory_block,
const std::size_t exclude_index)
{
auto graph_nodes_ptr = data_layout.GetBlockPtr<GraphNode>(
memory_block, storage::DataLayout::CH_GRAPH_NODE_LIST);
auto graph_nodes_ptr =
data_layout.GetBlockPtr<GraphNode>(memory_block, "/ch/contracted_graph/node_array");
auto graph_edges_ptr = data_layout.GetBlockPtr<GraphEdge>(
memory_block, storage::DataLayout::CH_GRAPH_EDGE_LIST);
auto graph_edges_ptr =
data_layout.GetBlockPtr<GraphEdge>(memory_block, "/ch/contracted_graph/edge_array");
auto filter_block_id = static_cast<storage::DataLayout::BlockID>(
storage::DataLayout::CH_EDGE_FILTER_0 + exclude_index);
auto filter_block_id = "/ch/edge_filter/" + std::to_string(exclude_index);
auto edge_filter_ptr =
data_layout.GetBlockPtr<util::vector_view<bool>::Word>(memory_block, filter_block_id);
util::vector_view<GraphNode> node_list(
graph_nodes_ptr, data_layout.GetBlockEntries(storage::DataLayout::CH_GRAPH_NODE_LIST));
graph_nodes_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/node_array"));
util::vector_view<GraphEdge> edge_list(
graph_edges_ptr, data_layout.GetBlockEntries(storage::DataLayout::CH_GRAPH_EDGE_LIST));
graph_edges_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/edge_array"));
util::vector_view<bool> edge_filter(edge_filter_ptr,
data_layout.GetBlockEntries(filter_block_id));
@ -115,7 +114,7 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
}
void InitializeInternalPointers(storage::DataLayout &data_layout,
void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::size_t exclude_index)
{
@ -224,29 +223,28 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
// allocator that keeps the allocation data
std::shared_ptr<ContiguousBlockAllocator> allocator;
void InitializeProfilePropertiesPointer(storage::DataLayout &data_layout,
void InitializeProfilePropertiesPointer(const storage::DataLayout &data_layout,
char *memory_block,
const std::size_t exclude_index)
{
m_profile_properties = data_layout.GetBlockPtr<extractor::ProfileProperties>(
memory_block, storage::DataLayout::PROPERTIES);
memory_block, "/common/properties");
exclude_mask = m_profile_properties->excludable_classes[exclude_index];
}
void InitializeChecksumPointer(storage::DataLayout &data_layout, char *memory_block)
void InitializeChecksumPointer(const storage::DataLayout &data_layout, char *memory_block)
{
m_check_sum =
*data_layout.GetBlockPtr<unsigned>(memory_block, storage::DataLayout::HSGR_CHECKSUM);
m_check_sum = *data_layout.GetBlockPtr<unsigned>(memory_block, "/ch/checksum");
util::Log() << "set checksum: " << m_check_sum;
}
void InitializeRTreePointers(storage::DataLayout &data_layout, char *memory_block)
void InitializeRTreePointers(const storage::DataLayout &data_layout, char *memory_block)
{
BOOST_ASSERT_MSG(!m_coordinate_list.empty(), "coordinates must be loaded before r-tree");
const auto file_index_ptr =
data_layout.GetBlockPtr<char>(memory_block, storage::DataLayout::FILE_INDEX_PATH);
data_layout.GetBlockPtr<char>(memory_block, "/common/rtree/file_index_path");
file_index_path = boost::filesystem::path(file_index_ptr);
if (!boost::filesystem::exists(file_index_path))
{
@ -256,15 +254,15 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
}
const auto rtree_ptr =
data_layout.GetBlockPtr<RTreeNode>(memory_block, storage::DataLayout::R_SEARCH_TREE);
data_layout.GetBlockPtr<RTreeNode>(memory_block, "/common/rtree/search_tree");
util::vector_view<RTreeNode> search_tree(
rtree_ptr, data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE));
rtree_ptr, data_layout.GetBlockEntries("/common/rtree/search_tree"));
const auto rtree_levelstarts_ptr = data_layout.GetBlockPtr<std::uint64_t>(
memory_block, storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS);
memory_block, "/common/rtree/search_tree_level_starts");
util::vector_view<std::uint64_t> rtree_level_starts(
rtree_levelstarts_ptr,
data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS));
data_layout.GetBlockEntries("/common/rtree/search_tree_level_starts"));
m_static_rtree.reset(new SharedRTree{std::move(search_tree),
std::move(rtree_level_starts),
@ -274,70 +272,67 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
new SharedGeospatialQuery(*m_static_rtree, m_coordinate_list, *this));
}
void InitializeNodeInformationPointers(storage::DataLayout &layout, char *memory_ptr)
void InitializeNodeInformationPointers(const storage::DataLayout &layout, char *memory_ptr)
{
const auto coordinate_list_ptr =
layout.GetBlockPtr<util::Coordinate>(memory_ptr, storage::DataLayout::COORDINATE_LIST);
m_coordinate_list.reset(coordinate_list_ptr,
layout.GetBlockEntries(storage::DataLayout::COORDINATE_LIST));
layout.GetBlockPtr<util::Coordinate>(memory_ptr, "/common/coordinates");
m_coordinate_list.reset(coordinate_list_ptr, layout.GetBlockEntries("/common/coordinates"));
const auto osmnodeid_ptr = layout.GetBlockPtr<extractor::PackedOSMIDsView::block_type>(
memory_ptr, storage::DataLayout::OSM_NODE_ID_LIST);
memory_ptr, "/common/osm_node_ids/packed");
m_osmnodeid_list = extractor::PackedOSMIDsView(
util::vector_view<extractor::PackedOSMIDsView::block_type>(
osmnodeid_ptr, layout.GetBlockEntries(storage::DataLayout::OSM_NODE_ID_LIST)),
osmnodeid_ptr, layout.GetBlockEntries("/common/osm_node_ids/packed")),
// We (ab)use the number of coordinates here because we know we have the same amount of
// ids
layout.GetBlockEntries(storage::DataLayout::COORDINATE_LIST));
layout.GetBlockEntries("/common/coordinates"));
}
void InitializeEdgeBasedNodeDataInformationPointers(storage::DataLayout &layout,
void InitializeEdgeBasedNodeDataInformationPointers(const storage::DataLayout &layout,
char *memory_ptr)
{
const auto edge_based_node_list_ptr = layout.GetBlockPtr<extractor::EdgeBasedNode>(
memory_ptr, storage::DataLayout::EDGE_BASED_NODE_DATA_LIST);
const auto edge_based_node_list_ptr =
layout.GetBlockPtr<extractor::EdgeBasedNode>(memory_ptr, "/common/ebg_node_data/nodes");
util::vector_view<extractor::EdgeBasedNode> edge_based_node_data_list(
edge_based_node_list_ptr,
layout.GetBlockEntries(storage::DataLayout::EDGE_BASED_NODE_DATA_LIST));
edge_based_node_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/nodes"));
const auto annotation_data_list_ptr =
layout.GetBlockPtr<extractor::NodeBasedEdgeAnnotation>(
memory_ptr, storage::DataLayout::ANNOTATION_DATA_LIST);
memory_ptr, "/common/ebg_node_data/annotations");
util::vector_view<extractor::NodeBasedEdgeAnnotation> annotation_data(
annotation_data_list_ptr,
layout.GetBlockEntries(storage::DataLayout::ANNOTATION_DATA_LIST));
annotation_data_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/annotations"));
edge_based_node_data = extractor::EdgeBasedNodeDataView(
std::move(edge_based_node_data_list), std::move(annotation_data));
}
void InitializeEdgeInformationPointers(storage::DataLayout &layout, char *memory_ptr)
void InitializeEdgeInformationPointers(const storage::DataLayout &layout, char *memory_ptr)
{
const auto lane_data_id_ptr =
layout.GetBlockPtr<LaneDataID>(memory_ptr, storage::DataLayout::LANE_DATA_ID);
layout.GetBlockPtr<LaneDataID>(memory_ptr, "/common/turn_data/lane_data_ids");
util::vector_view<LaneDataID> lane_data_ids(
lane_data_id_ptr, layout.GetBlockEntries(storage::DataLayout::LANE_DATA_ID));
lane_data_id_ptr, layout.GetBlockEntries("/common/turn_data/lane_data_ids"));
const auto turn_instruction_list_ptr = layout.GetBlockPtr<guidance::TurnInstruction>(
memory_ptr, storage::DataLayout::TURN_INSTRUCTION);
memory_ptr, "/common/turn_data/turn_instructions");
util::vector_view<guidance::TurnInstruction> turn_instructions(
turn_instruction_list_ptr,
layout.GetBlockEntries(storage::DataLayout::TURN_INSTRUCTION));
layout.GetBlockEntries("/common/turn_data/turn_instructions"));
const auto entry_class_id_list_ptr =
layout.GetBlockPtr<EntryClassID>(memory_ptr, storage::DataLayout::ENTRY_CLASSID);
layout.GetBlockPtr<EntryClassID>(memory_ptr, "/common/turn_data/entry_class_ids");
util::vector_view<EntryClassID> entry_class_ids(
entry_class_id_list_ptr, layout.GetBlockEntries(storage::DataLayout::ENTRY_CLASSID));
entry_class_id_list_ptr, layout.GetBlockEntries("/common/turn_data/entry_class_ids"));
const auto pre_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing>(
memory_ptr, storage::DataLayout::PRE_TURN_BEARING);
memory_ptr, "/common/turn_data/pre_turn_bearings");
util::vector_view<guidance::TurnBearing> pre_turn_bearings(
pre_turn_bearing_ptr, layout.GetBlockEntries(storage::DataLayout::PRE_TURN_BEARING));
pre_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/pre_turn_bearings"));
const auto post_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing>(
memory_ptr, storage::DataLayout::POST_TURN_BEARING);
memory_ptr, "/common/turn_data/post_turn_bearings");
util::vector_view<guidance::TurnBearing> post_turn_bearings(
post_turn_bearing_ptr, layout.GetBlockEntries(storage::DataLayout::POST_TURN_BEARING));
post_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/post_turn_bearings"));
turn_data = guidance::TurnDataView(std::move(turn_instructions),
std::move(lane_data_ids),
@ -346,124 +341,120 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
std::move(post_turn_bearings));
}
void InitializeNamePointers(storage::DataLayout &data_layout, char *memory_block)
void InitializeNamePointers(const storage::DataLayout &data_layout, char *memory_block)
{
const auto name_blocks_ptr =
data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::BlockReference>(
memory_block, storage::DataLayout::NAME_BLOCKS);
memory_block, "/common/names/blocks");
const auto name_values_ptr =
data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::ValueType>(
memory_block, storage::DataLayout::NAME_VALUES);
memory_block, "/common/names/values");
util::vector_view<extractor::NameTableView::IndexedData::BlockReference> blocks(
name_blocks_ptr, data_layout.GetBlockEntries(storage::DataLayout::NAME_BLOCKS));
name_blocks_ptr, data_layout.GetBlockEntries("/common/names/blocks"));
util::vector_view<extractor::NameTableView::IndexedData::ValueType> values(
name_values_ptr, data_layout.GetBlockEntries(storage::DataLayout::NAME_VALUES));
name_values_ptr, data_layout.GetBlockEntries("/common/names/values"));
extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
m_name_table = extractor::NameTableView{std::move(index_data_view)};
}
void InitializeTurnLaneDescriptionsPointers(storage::DataLayout &data_layout,
void InitializeTurnLaneDescriptionsPointers(const storage::DataLayout &data_layout,
char *memory_block)
{
auto offsets_ptr = data_layout.GetBlockPtr<std::uint32_t>(
memory_block, storage::DataLayout::LANE_DESCRIPTION_OFFSETS);
auto offsets_ptr =
data_layout.GetBlockPtr<std::uint32_t>(memory_block, "/common/turn_lanes/offsets");
util::vector_view<std::uint32_t> offsets(
offsets_ptr,
data_layout.GetBlockEntries(storage::DataLayout::LANE_DESCRIPTION_OFFSETS));
offsets_ptr, data_layout.GetBlockEntries("/common/turn_lanes/offsets"));
m_lane_description_offsets = std::move(offsets);
auto masks_ptr = data_layout.GetBlockPtr<extractor::TurnLaneType::Mask>(
memory_block, storage::DataLayout::LANE_DESCRIPTION_MASKS);
memory_block, "/common/turn_lanes/masks");
util::vector_view<extractor::TurnLaneType::Mask> masks(
masks_ptr, data_layout.GetBlockEntries(storage::DataLayout::LANE_DESCRIPTION_MASKS));
masks_ptr, data_layout.GetBlockEntries("/common/turn_lanes/masks"));
m_lane_description_masks = std::move(masks);
const auto lane_tupel_id_pair_ptr =
data_layout.GetBlockPtr<util::guidance::LaneTupleIdPair>(
memory_block, storage::DataLayout::TURN_LANE_DATA);
data_layout.GetBlockPtr<util::guidance::LaneTupleIdPair>(memory_block,
"/common/turn_lanes/data");
util::vector_view<util::guidance::LaneTupleIdPair> lane_tupel_id_pair(
lane_tupel_id_pair_ptr,
data_layout.GetBlockEntries(storage::DataLayout::TURN_LANE_DATA));
lane_tupel_id_pair_ptr, data_layout.GetBlockEntries("/common/turn_lanes/data"));
m_lane_tupel_id_pairs = std::move(lane_tupel_id_pair);
}
void InitializeTurnPenalties(storage::DataLayout &data_layout, char *memory_block)
void InitializeTurnPenalties(const storage::DataLayout &data_layout, char *memory_block)
{
auto turn_weight_penalties_ptr = data_layout.GetBlockPtr<TurnPenalty>(
memory_block, storage::DataLayout::TURN_WEIGHT_PENALTIES);
auto turn_weight_penalties_ptr =
data_layout.GetBlockPtr<TurnPenalty>(memory_block, "/common/turn_penalty/weight");
m_turn_weight_penalties = util::vector_view<TurnPenalty>(
turn_weight_penalties_ptr,
data_layout.GetBlockEntries(storage::DataLayout::TURN_WEIGHT_PENALTIES));
auto turn_duration_penalties_ptr = data_layout.GetBlockPtr<TurnPenalty>(
memory_block, storage::DataLayout::TURN_DURATION_PENALTIES);
turn_weight_penalties_ptr, data_layout.GetBlockEntries("/common/turn_penalty/weight"));
auto turn_duration_penalties_ptr =
data_layout.GetBlockPtr<TurnPenalty>(memory_block, "/common/turn_penalty/duration");
m_turn_duration_penalties = util::vector_view<TurnPenalty>(
turn_duration_penalties_ptr,
data_layout.GetBlockEntries(storage::DataLayout::TURN_DURATION_PENALTIES));
data_layout.GetBlockEntries("/common/turn_penalty/duration"));
}
void InitializeGeometryPointers(storage::DataLayout &data_layout, char *memory_block)
void InitializeGeometryPointers(const storage::DataLayout &data_layout, char *memory_block)
{
auto geometries_index_ptr =
data_layout.GetBlockPtr<unsigned>(memory_block, storage::DataLayout::GEOMETRIES_INDEX);
data_layout.GetBlockPtr<unsigned>(memory_block, "/common/segment_data/index");
util::vector_view<unsigned> geometry_begin_indices(
geometries_index_ptr,
data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_INDEX));
geometries_index_ptr, data_layout.GetBlockEntries("/common/segment_data/index"));
auto num_entries = data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_NODE_LIST);
auto geometries_node_list_ptr = data_layout.GetBlockPtr<NodeID>(
memory_block, storage::DataLayout::GEOMETRIES_NODE_LIST);
auto num_entries = data_layout.GetBlockEntries("/common/segment_data/nodes");
auto geometries_node_list_ptr =
data_layout.GetBlockPtr<NodeID>(memory_block, "/common/segment_data/nodes");
util::vector_view<NodeID> geometry_node_list(geometries_node_list_ptr, num_entries);
auto geometries_fwd_weight_list_ptr =
data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type>(
memory_block, storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST);
memory_block, "/common/segment_data/forward_weights/packed");
extractor::SegmentDataView::SegmentWeightVector geometry_fwd_weight_list(
util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
geometries_fwd_weight_list_ptr,
data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST)),
data_layout.GetBlockEntries("/common/segment_data/forward_weights/packed")),
num_entries);
auto geometries_rev_weight_list_ptr =
data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type>(
memory_block, storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST);
memory_block, "/common/segment_data/reverse_weights/packed");
extractor::SegmentDataView::SegmentWeightVector geometry_rev_weight_list(
util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
geometries_rev_weight_list_ptr,
data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST)),
data_layout.GetBlockEntries("/common/segment_data/reverse_weights/packed")),
num_entries);
auto geometries_fwd_duration_list_ptr =
data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type>(
memory_block, storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST);
memory_block, "/common/segment_data/forward_durations/packed");
extractor::SegmentDataView::SegmentDurationVector geometry_fwd_duration_list(
util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
geometries_fwd_duration_list_ptr,
data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST)),
data_layout.GetBlockEntries("/common/segment_data/forward_durations/packed")),
num_entries);
auto geometries_rev_duration_list_ptr =
data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type>(
memory_block, storage::DataLayout::GEOMETRIES_REV_DURATION_LIST);
memory_block, "/common/segment_data/reverse_durations/packed");
extractor::SegmentDataView::SegmentDurationVector geometry_rev_duration_list(
util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
geometries_rev_duration_list_ptr,
data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_DURATION_LIST)),
data_layout.GetBlockEntries("/common/segment_data/reverse_durations/packed")),
num_entries);
auto geometries_fwd_datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>(
memory_block, storage::DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST);
memory_block, "/common/segment_data/forward_data_sources");
util::vector_view<DatasourceID> geometry_fwd_datasources_list(
geometries_fwd_datasources_list_ptr,
data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST));
data_layout.GetBlockEntries("/common/segment_data/forward_data_sources"));
auto geometries_rev_datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>(
memory_block, storage::DataLayout::GEOMETRIES_REV_DATASOURCES_LIST);
memory_block, "/common/segment_data/reverse_data_sources");
util::vector_view<DatasourceID> geometry_rev_datasources_list(
geometries_rev_datasources_list_ptr,
data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_DATASOURCES_LIST));
data_layout.GetBlockEntries("/common/segment_data/reverse_data_sources"));
segment_data = extractor::SegmentDataView{std::move(geometry_begin_indices),
std::move(geometry_node_list),
@ -475,30 +466,36 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
std::move(geometry_rev_datasources_list)};
m_datasources = data_layout.GetBlockPtr<extractor::Datasources>(
memory_block, storage::DataLayout::DATASOURCES_NAMES);
memory_block, "/common/data_sources_names");
}
void InitializeIntersectionClassPointers(storage::DataLayout &data_layout, char *memory_block)
void InitializeIntersectionClassPointers(const storage::DataLayout &data_layout,
char *memory_block)
{
auto bearing_class_id_ptr = data_layout.GetBlockPtr<BearingClassID>(
memory_block, storage::DataLayout::BEARING_CLASSID);
memory_block, "/common/intersection_bearings/node_to_class_id");
util::vector_view<BearingClassID> bearing_class_id(
bearing_class_id_ptr,
data_layout.GetBlockEntries(storage::DataLayout::BEARING_CLASSID));
data_layout.GetBlockEntries("/common/intersection_bearings/node_to_class_id"));
auto bearing_values_ptr = data_layout.GetBlockPtr<DiscreteBearing>(
memory_block, storage::DataLayout::BEARING_VALUES);
memory_block, "/common/intersection_bearings/bearing_values");
util::vector_view<DiscreteBearing> bearing_values(
bearing_values_ptr, data_layout.GetBlockEntries(storage::DataLayout::BEARING_VALUES));
bearing_values_ptr,
data_layout.GetBlockEntries("/common/intersection_bearings/bearing_values"));
auto offsets_ptr =
data_layout.GetBlockPtr<unsigned>(memory_block, storage::DataLayout::BEARING_OFFSETS);
auto blocks_ptr =
data_layout.GetBlockPtr<IndexBlock>(memory_block, storage::DataLayout::BEARING_BLOCKS);
auto offsets_ptr = data_layout.GetBlockPtr<unsigned>(
memory_block, "/common/intersection_bearings/class_id_to_ranges/block_offsets");
auto blocks_ptr = data_layout.GetBlockPtr<IndexBlock>(
memory_block, "/common/intersection_bearings/class_id_to_ranges/diff_blocks");
util::vector_view<unsigned> bearing_offsets(
offsets_ptr, data_layout.GetBlockEntries(storage::DataLayout::BEARING_OFFSETS));
offsets_ptr,
data_layout.GetBlockEntries(
"/common/intersection_bearings/class_id_to_ranges/block_offsets"));
util::vector_view<IndexBlock> bearing_blocks(
blocks_ptr, data_layout.GetBlockEntries(storage::DataLayout::BEARING_BLOCKS));
blocks_ptr,
data_layout.GetBlockEntries(
"/common/intersection_bearings/class_id_to_ranges/diff_blocks"));
util::RangeTable<16, storage::Ownership::View> bearing_range_table(
bearing_offsets, bearing_blocks, static_cast<unsigned>(bearing_values.size()));
@ -507,28 +504,29 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)};
auto entry_class_ptr = data_layout.GetBlockPtr<util::guidance::EntryClass>(
memory_block, storage::DataLayout::ENTRY_CLASS);
memory_block, "/common/entry_classes");
util::vector_view<util::guidance::EntryClass> entry_class_table(
entry_class_ptr, data_layout.GetBlockEntries(storage::DataLayout::ENTRY_CLASS));
entry_class_ptr, data_layout.GetBlockEntries("/common/entry_classes"));
m_entry_class_table = std::move(entry_class_table);
}
void InitializeManeuverOverridePointers(storage::DataLayout &data_layout, char *memory_block)
void InitializeManeuverOverridePointers(const storage::DataLayout &data_layout,
char *memory_block)
{
auto maneuver_overrides_ptr = data_layout.GetBlockPtr<extractor::StorageManeuverOverride>(
memory_block, storage::DataLayout::MANEUVER_OVERRIDES);
memory_block, "/common/maneuver_overrides/overrides");
m_maneuver_overrides = util::vector_view<extractor::StorageManeuverOverride>(
maneuver_overrides_ptr,
data_layout.GetBlockEntries(storage::DataLayout::MANEUVER_OVERRIDES));
data_layout.GetBlockEntries("/common/maneuver_overrides/overrides"));
auto maneuver_override_node_sequences_ptr = data_layout.GetBlockPtr<NodeID>(
memory_block, storage::DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES);
memory_block, "/common/maneuver_overrides/node_sequences");
m_maneuver_override_node_sequences = util::vector_view<NodeID>(
maneuver_override_node_sequences_ptr,
data_layout.GetBlockEntries(storage::DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES));
data_layout.GetBlockEntries("/common/maneuver_overrides/node_sequences"));
}
void InitializeInternalPointers(storage::DataLayout &data_layout,
void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::size_t exclude_index)
{
@ -939,7 +937,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
auto found_range = std::equal_range(
m_maneuver_overrides.begin(), m_maneuver_overrides.end(), edge_based_node_id, Comp{});
std::for_each(found_range.first, found_range.second, [&](const auto & override) {
std::for_each(found_range.first, found_range.second, [&](const auto &override) {
std::vector<NodeID> sequence(
m_maneuver_override_node_sequences.begin() + override.node_sequence_offset_begin,
m_maneuver_override_node_sequences.begin() + override.node_sequence_offset_end);
@ -981,7 +979,7 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
QueryGraph query_graph;
void InitializeInternalPointers(storage::DataLayout &data_layout,
void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::size_t exclude_index)
{
@ -989,39 +987,38 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
InitializeGraphPointer(data_layout, memory_block);
}
void InitializeMLDDataPointers(storage::DataLayout &data_layout,
void InitializeMLDDataPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::size_t exclude_index)
{
if (data_layout.GetBlockSize(storage::DataLayout::MLD_PARTITION) > 0)
if (data_layout.GetBlockSize("/mld/multilevelpartition/partition") > 0)
{
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_LEVEL_DATA) > 0);
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_TO_CHILDREN) > 0);
BOOST_ASSERT(data_layout.GetBlockSize("/mld/multilevelpartition/level_data") > 0);
BOOST_ASSERT(data_layout.GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0);
auto level_data =
data_layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData>(
memory_block, storage::DataLayout::MLD_LEVEL_DATA);
memory_block, "/mld/multilevelpartition/level_data");
auto mld_partition_ptr = data_layout.GetBlockPtr<PartitionID>(
memory_block, storage::DataLayout::MLD_PARTITION);
memory_block, "/mld/multilevelpartition/partition");
auto partition_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_PARTITION);
data_layout.GetBlockEntries("/mld/multilevelpartition/partition");
util::vector_view<PartitionID> partition(mld_partition_ptr, partition_entries_count);
auto mld_chilren_ptr = data_layout.GetBlockPtr<CellID>(
memory_block, storage::DataLayout::MLD_CELL_TO_CHILDREN);
memory_block, "/mld/multilevelpartition/cell_to_children");
auto children_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_TO_CHILDREN);
data_layout.GetBlockEntries("/mld/multilevelpartition/cell_to_children");
util::vector_view<CellID> cell_to_children(mld_chilren_ptr, children_entries_count);
mld_partition =
partitioner::MultiLevelPartitionView{level_data, partition, cell_to_children};
}
const auto weights_block_id = static_cast<storage::DataLayout::BlockID>(
storage::DataLayout::MLD_CELL_WEIGHTS_0 + exclude_index);
const auto durations_block_id = static_cast<storage::DataLayout::BlockID>(
storage::DataLayout::MLD_CELL_DURATIONS_0 + exclude_index);
const auto weights_block_id = "/mld/metrics/" + std::to_string(exclude_index) + "/weights";
const auto durations_block_id =
"/mld/metrics/" + std::to_string(exclude_index) + "/durations";
if (data_layout.GetBlockSize(weights_block_id) > 0)
{
@ -1029,10 +1026,8 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
data_layout.GetBlockPtr<EdgeWeight>(memory_block, weights_block_id);
auto mld_cell_durations_ptr =
data_layout.GetBlockPtr<EdgeDuration>(memory_block, durations_block_id);
auto weight_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_WEIGHTS_0);
auto duration_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_DURATIONS_0);
auto weight_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/weights");
auto duration_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/durations");
BOOST_ASSERT(weight_entries_count == duration_entries_count);
util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
util::vector_view<EdgeDuration> durations(mld_cell_durations_ptr,
@ -1041,25 +1036,25 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
mld_cell_metric = customizer::CellMetricView{std::move(weights), std::move(durations)};
}
if (data_layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0)
if (data_layout.GetBlockSize("/mld/cellstorage/cells") > 0)
{
auto mld_source_boundary_ptr = data_layout.GetBlockPtr<NodeID>(
memory_block, storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
auto mld_source_boundary_ptr =
data_layout.GetBlockPtr<NodeID>(memory_block, "/mld/cellstorage/source_boundary");
auto mld_destination_boundary_ptr = data_layout.GetBlockPtr<NodeID>(
memory_block, storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY);
memory_block, "/mld/cellstorage/destination_boundary");
auto mld_cells_ptr = data_layout.GetBlockPtr<partitioner::CellStorageView::CellData>(
memory_block, storage::DataLayout::MLD_CELLS);
memory_block, "/mld/cellstorage/cells");
auto mld_cell_level_offsets_ptr = data_layout.GetBlockPtr<std::uint64_t>(
memory_block, storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
memory_block, "/mld/cellstorage/level_to_cell_offset");
auto source_boundary_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
data_layout.GetBlockEntries("/mld/cellstorage/source_boundary");
auto destination_boundary_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY);
auto cells_entries_counts = data_layout.GetBlockEntries(storage::DataLayout::MLD_CELLS);
data_layout.GetBlockEntries("/mld/cellstorage/destination_boundary");
auto cells_entries_counts = data_layout.GetBlockEntries("/mld/cellstorage/cells");
auto cell_level_offsets_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
data_layout.GetBlockEntries("/mld/cellstorage/level_to_cell_offset");
util::vector_view<NodeID> source_boundary(mld_source_boundary_ptr,
source_boundary_entries_count);
@ -1076,24 +1071,24 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
std::move(level_offsets)};
}
}
void InitializeGraphPointer(storage::DataLayout &data_layout, char *memory_block)
void InitializeGraphPointer(const storage::DataLayout &data_layout, char *memory_block)
{
auto graph_nodes_ptr = data_layout.GetBlockPtr<GraphNode>(
memory_block, storage::DataLayout::MLD_GRAPH_NODE_LIST);
auto graph_nodes_ptr =
data_layout.GetBlockPtr<GraphNode>(memory_block, "/mld/multilevelgraph/node_array");
auto graph_edges_ptr = data_layout.GetBlockPtr<GraphEdge>(
memory_block, storage::DataLayout::MLD_GRAPH_EDGE_LIST);
auto graph_edges_ptr =
data_layout.GetBlockPtr<GraphEdge>(memory_block, "/mld/multilevelgraph/edge_array");
auto graph_node_to_offset_ptr = data_layout.GetBlockPtr<QueryGraph::EdgeOffset>(
memory_block, storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET);
memory_block, "/mld/multilevelgraph/node_to_edge_offset");
util::vector_view<GraphNode> node_list(
graph_nodes_ptr, data_layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_NODE_LIST));
graph_nodes_ptr, data_layout.GetBlockEntries("/mld/multilevelgraph/node_array"));
util::vector_view<GraphEdge> edge_list(
graph_edges_ptr, data_layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_EDGE_LIST));
graph_edges_ptr, data_layout.GetBlockEntries("/mld/multilevelgraph/edge_array"));
util::vector_view<QueryGraph::EdgeOffset> node_to_offset(
graph_node_to_offset_ptr,
data_layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET));
data_layout.GetBlockEntries("/mld/multilevelgraph/node_to_edge_offset"));
query_graph =
QueryGraph(std::move(node_list), std::move(edge_list), std::move(node_to_offset));

4
include/engine/datafacade/process_memory_allocator.hpp
View File

@ -27,12 +27,12 @@ class ProcessMemoryAllocator : public ContiguousBlockAllocator
~ProcessMemoryAllocator() override final;
// interface to give access to the datafacades
storage::DataLayout &GetLayout() override final;
const storage::DataLayout &GetLayout() override final;
char *GetMemory() override final;
private:
storage::DataLayout internal_layout;
std::unique_ptr<char[]> internal_memory;
std::unique_ptr<storage::DataLayout> internal_layout;
};
} // namespace datafacade

4
include/engine/datafacade/shared_memory_allocator.hpp
View File

@ -27,10 +27,12 @@ class SharedMemoryAllocator : public ContiguousBlockAllocator
~SharedMemoryAllocator() override final;
// interface to give access to the datafacades
storage::DataLayout &GetLayout() override final;
const storage::DataLayout &GetLayout() override final;
char *GetMemory() override final;
private:
std::size_t layout_size;
storage::DataLayout data_layout;
std::unique_ptr<storage::SharedMemory> m_large_memory;
};

2
include/engine/datafacade_factory.hpp
View File

@ -51,7 +51,7 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
}
properties = allocator->GetLayout().template GetBlockPtr<extractor::ProfileProperties>(
allocator->GetMemory(), storage::DataLayout::PROPERTIES);
allocator->GetMemory(), "/common/properties");
for (const auto index : util::irange<std::size_t>(0, properties->class_names.size()))
{

26
include/engine/engine.hpp
View File

@ -152,8 +152,8 @@ bool Engine<routing_algorithms::ch::Algorithm>::CheckCompatibility(const EngineC
auto mem = storage::makeSharedMemory(barrier.data().region);
auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr());
return layout->GetBlockSize(storage::DataLayout::CH_GRAPH_NODE_LIST) > 4 &&
layout->GetBlockSize(storage::DataLayout::CH_GRAPH_EDGE_LIST) > 4;
return layout->GetBlockSize("/ch/contracted_graph/node_array") > 4 &&
layout->GetBlockSize("/ch/contracted_graph/edge_array") > 4;
}
else
{
@ -173,19 +173,19 @@ bool Engine<routing_algorithms::mld::Algorithm>::CheckCompatibility(const Engine
auto mem = storage::makeSharedMemory(barrier.data().region);
auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr());
// checks that all the needed memory blocks are populated
// DataLayout::MLD_CELL_SOURCE_BOUNDARY and DataLayout::MLD_CELL_DESTINATION_BOUNDARY
// "/mld/cellstorage/source_boundary" and "/mld/cellstorage/destination_boundary"
// are not checked, because in situations where there are so few nodes in the graph that
// they all fit into one cell, they can be empty.
bool empty_data = layout->GetBlockSize(storage::DataLayout::MLD_LEVEL_DATA) > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_PARTITION) > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_CELL_TO_CHILDREN) > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_CELLS) > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS) > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_GRAPH_NODE_LIST) > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_GRAPH_EDGE_LIST) > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_CELL_WEIGHTS_0) > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_CELL_DURATIONS_0) > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET) > 0;
bool empty_data = layout->GetBlockSize("/mld/multilevelpartition/level_data") > 0 &&
layout->GetBlockSize("/mld/multilevelpartition/partition") > 0 &&
layout->GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0 &&
layout->GetBlockSize("/mld/cellstorage/cells") > 0 &&
layout->GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0 &&
layout->GetBlockSize("/mld/multilevelgraph/node_array") > 0 &&
layout->GetBlockSize("/mld/multilevelgraph/edge_array") > 0 &&
layout->GetBlockSize("/mld/metrics/0/weights") > 0 &&
layout->GetBlockSize("/mld/metrics/0/durations") > 0 &&
layout->GetBlockSize("/mld/multilevelgraph/node_to_edge_offset") > 0;
return empty_data;
}
else

4
include/storage/io.hpp
View File

@ -303,6 +303,10 @@ class BufferReader
}
}
std::size_t GetPosition() {
return input_stream.tellg();
}
template <typename T> void ReadInto(T *dest, const std::size_t count)
{
#if !defined(__GNUC__) || (__GNUC__ > 4)

64
include/storage/serialization.hpp
View File

@ -7,12 +7,14 @@
#include "storage/io.hpp"
#include "storage/tar.hpp"
#include "storage/shared_datatype.hpp"
#include <boost/function_output_iterator.hpp>
#include <boost/iterator/function_input_iterator.hpp>
#include <cmath>
#include <cstdint>
#include <tuple>
#if USE_STXXL_LIBRARY
#include <stxxl/vector>
@ -81,6 +83,32 @@ template <typename T> void write(io::BufferWriter &writer, const std::vector<T>
writer.WriteFrom(data.data(), count);
}
template<typename T>
inline void write(io::BufferWriter &writer, const T &data)
{
writer.WriteFrom(data);
}
template<typename T>
inline void read(io::BufferReader &reader, T &data)
{
reader.ReadInto(data);
}
inline void write(io::BufferWriter &writer, const std::string &data)
{
const auto count = data.size();
writer.WriteElementCount64(count);
writer.WriteFrom(data.data(), count);
}
inline void read(io::BufferReader &reader, std::string &data)
{
const auto count = reader.ReadElementCount64();
data.resize(count);
reader.ReadInto(const_cast<char *>(data.data()), count);
}
inline void write(tar::FileWriter &writer, const std::string &name, const std::string &data)
{
const auto count = data.size();
@ -218,6 +246,42 @@ write<bool>(tar::FileWriter &writer, const std::string &name, const std::vector<
{
detail::writeBoolVector(writer, name, data);
}
template <typename K, typename V> void read(io::BufferReader &reader, std::map<K, V> &data)
{
const auto count = reader.ReadElementCount64();
std::pair<K, V> pair;
for (auto index : util::irange<std::size_t>(0, count))
{
(void) index;
read(reader, pair.first);
read(reader, pair.second);
data.insert(pair);
}
}
template <typename K, typename V> void write(io::BufferWriter &writer, const std::map<K, V> &data)
{
const auto count = data.size();
writer.WriteElementCount64(count);
for (const auto &pair : data)
{
write(writer, pair.first);
write(writer, pair.second);
}
}
inline void read(io::BufferReader &reader, DataLayout &layout)
{
read(reader, layout.blocks);
}
inline void write(io::BufferWriter &writer, const DataLayout &layout)
{
write(writer, layout.blocks);
}
}
}
}

227
include/storage/shared_datatype.hpp
View File

@ -2,6 +2,7 @@
#define SHARED_DATA_TYPE_HPP
#include "storage/block.hpp"
#include "storage/io_fwd.hpp"
#include "util/exception.hpp"
#include "util/exception_utils.hpp"
@ -11,219 +12,91 @@
#include <array>
#include <cstdint>
#include <map>
namespace osrm
{
namespace storage
{
class DataLayout;
namespace serialization
{
inline void read(io::BufferReader &reader, DataLayout &layout);
inline void write(io::BufferWriter &writer, const DataLayout &layout);
}
// Added at the start and end of each block as sanity check
const constexpr char CANARY[4] = {'O', 'S', 'R', 'M'};
const constexpr char *block_id_to_name[] = {"IGNORE_BLOCK",
"NAME_BLOCKS",
"NAME_VALUES",
"EDGE_BASED_NODE_DATA",
"ANNOTATION_DATA",
"CH_GRAPH_NODE_LIST",
"CH_GRAPH_EDGE_LIST",
"CH_EDGE_FILTER_0",
"CH_EDGE_FILTER_1",
"CH_EDGE_FILTER_2",
"CH_EDGE_FILTER_3",
"CH_EDGE_FILTER_4",
"CH_EDGE_FILTER_5",
"CH_EDGE_FILTER_6",
"CH_EDGE_FILTER_7",
"COORDINATE_LIST",
"OSM_NODE_ID_LIST",
"TURN_INSTRUCTION",
"ENTRY_CLASSID",
"R_SEARCH_TREE",
"R_SEARCH_TREE_LEVEL_STARTS",
"GEOMETRIES_INDEX",
"GEOMETRIES_NODE_LIST",
"GEOMETRIES_FWD_WEIGHT_LIST",
"GEOMETRIES_REV_WEIGHT_LIST",
"GEOMETRIES_FWD_DURATION_LIST",
"GEOMETRIES_REV_DURATION_LIST",
"GEOMETRIES_FWD_DATASOURCES_LIST",
"GEOMETRIES_REV_DATASOURCES_LIST",
"HSGR_CHECKSUM",
"FILE_INDEX_PATH",
"DATASOURCES_NAMES",
"PROPERTIES",
"BEARING_CLASSID",
"BEARING_OFFSETS",
"BEARING_BLOCKS",
"BEARING_VALUES",
"ENTRY_CLASS",
"LANE_DATA_ID",
"PRE_TURN_BEARING",
"POST_TURN_BEARING",
"TURN_LANE_DATA",
"LANE_DESCRIPTION_OFFSETS",
"LANE_DESCRIPTION_MASKS",
"TURN_WEIGHT_PENALTIES",
"TURN_DURATION_PENALTIES",
"MLD_LEVEL_DATA",
"MLD_PARTITION",
"MLD_CELL_TO_CHILDREN",
"MLD_CELL_WEIGHTS_0",
"MLD_CELL_WEIGHTS_1",
"MLD_CELL_WEIGHTS_2",
"MLD_CELL_WEIGHTS_3",
"MLD_CELL_WEIGHTS_4",
"MLD_CELL_WEIGHTS_5",
"MLD_CELL_WEIGHTS_6",
"MLD_CELL_WEIGHTS_7",
"MLD_CELL_DURATIONS_0",
"MLD_CELL_DURATIONS_1",
"MLD_CELL_DURATIONS_2",
"MLD_CELL_DURATIONS_3",
"MLD_CELL_DURATIONS_4",
"MLD_CELL_DURATIONS_5",
"MLD_CELL_DURATIONS_6",
"MLD_CELL_DURATIONS_7",
"MLD_CELL_SOURCE_BOUNDARY",
"MLD_CELL_DESTINATION_BOUNDARY",
"MLD_CELLS",
"MLD_CELL_LEVEL_OFFSETS",
"MLD_GRAPH_NODE_LIST",
"MLD_GRAPH_EDGE_LIST",
"MLD_GRAPH_NODE_TO_OFFSET",
"MANEUVER_OVERRIDES",
"MANEUVER_OVERRIDE_NODE_SEQUENCES"};
class DataLayout
{
public:
enum BlockID
{
IGNORE_BLOCK = 0,
NAME_BLOCKS,
NAME_VALUES,
EDGE_BASED_NODE_DATA_LIST,
ANNOTATION_DATA_LIST,
CH_GRAPH_NODE_LIST,
CH_GRAPH_EDGE_LIST,
CH_EDGE_FILTER_0,
CH_EDGE_FILTER_1,
CH_EDGE_FILTER_2,
CH_EDGE_FILTER_3,
CH_EDGE_FILTER_4,
CH_EDGE_FILTER_5,
CH_EDGE_FILTER_6,
CH_EDGE_FILTER_7,
COORDINATE_LIST,
OSM_NODE_ID_LIST,
TURN_INSTRUCTION,
ENTRY_CLASSID,
R_SEARCH_TREE,
R_SEARCH_TREE_LEVEL_STARTS,
GEOMETRIES_INDEX,
GEOMETRIES_NODE_LIST,
GEOMETRIES_FWD_WEIGHT_LIST,
GEOMETRIES_REV_WEIGHT_LIST,
GEOMETRIES_FWD_DURATION_LIST,
GEOMETRIES_REV_DURATION_LIST,
GEOMETRIES_FWD_DATASOURCES_LIST,
GEOMETRIES_REV_DATASOURCES_LIST,
HSGR_CHECKSUM,
FILE_INDEX_PATH,
DATASOURCES_NAMES,
PROPERTIES,
BEARING_CLASSID,
BEARING_OFFSETS,
BEARING_BLOCKS,
BEARING_VALUES,
ENTRY_CLASS,
LANE_DATA_ID,
PRE_TURN_BEARING,
POST_TURN_BEARING,
TURN_LANE_DATA,
LANE_DESCRIPTION_OFFSETS,
LANE_DESCRIPTION_MASKS,
TURN_WEIGHT_PENALTIES,
TURN_DURATION_PENALTIES,
MLD_LEVEL_DATA,
MLD_PARTITION,
MLD_CELL_TO_CHILDREN,
MLD_CELL_WEIGHTS_0,
MLD_CELL_WEIGHTS_1,
MLD_CELL_WEIGHTS_2,
MLD_CELL_WEIGHTS_3,
MLD_CELL_WEIGHTS_4,
MLD_CELL_WEIGHTS_5,
MLD_CELL_WEIGHTS_6,
MLD_CELL_WEIGHTS_7,
MLD_CELL_DURATIONS_0,
MLD_CELL_DURATIONS_1,
MLD_CELL_DURATIONS_2,
MLD_CELL_DURATIONS_3,
MLD_CELL_DURATIONS_4,
MLD_CELL_DURATIONS_5,
MLD_CELL_DURATIONS_6,
MLD_CELL_DURATIONS_7,
MLD_CELL_SOURCE_BOUNDARY,
MLD_CELL_DESTINATION_BOUNDARY,
MLD_CELLS,
MLD_CELL_LEVEL_OFFSETS,
MLD_GRAPH_NODE_LIST,
MLD_GRAPH_EDGE_LIST,
MLD_GRAPH_NODE_TO_OFFSET,
MANEUVER_OVERRIDES,
MANEUVER_OVERRIDE_NODE_SEQUENCES,
NUM_BLOCKS
};
DataLayout() : blocks{} {}
inline void SetBlock(BlockID bid, Block block) { blocks[bid] = std::move(block); }
inline void SetBlock(const std::string &name, Block block) { blocks[name] = std::move(block); }
inline uint64_t GetBlockEntries(BlockID bid) const { return blocks[bid].num_entries; }
inline uint64_t GetBlockEntries(const std::string &name) const
{
auto iter = blocks.find(name);
if (iter == blocks.end())
{
throw util::exception("Could not find block " + name);
}
inline uint64_t GetBlockSize(BlockID bid) const { return blocks[bid].byte_size; }
return iter->second.num_entries;
}
inline uint64_t GetBlockSize(const std::string &name) const
{
auto iter = blocks.find(name);
if (iter == blocks.end())
{
throw util::exception("Could not find block " + name);
}
return iter->second.byte_size;
}
inline uint64_t GetSizeOfLayout() const
{
uint64_t result = 0;
for (auto i = 0; i < NUM_BLOCKS; i++)
for (const auto &name_and_block : blocks)
{
result += 2 * sizeof(CANARY) + GetBlockSize(static_cast<BlockID>(i)) + BLOCK_ALIGNMENT;
result += 2 * sizeof(CANARY) + GetBlockSize(name_and_block.first) + BLOCK_ALIGNMENT;
}
return result;
}
template <typename T, bool WRITE_CANARY = false>
inline T *GetBlockPtr(char *shared_memory, BlockID bid) const
inline T *GetBlockPtr(char *shared_memory, const std::string &name) const
{
static_assert(BLOCK_ALIGNMENT % std::alignment_of<T>::value == 0,
"Datatype does not fit alignment constraints.");
char *ptr = (char *)GetAlignedBlockPtr(shared_memory, bid);
char *ptr = (char *)GetAlignedBlockPtr(shared_memory, name);
if (WRITE_CANARY)
{
char *start_canary_ptr = ptr - sizeof(CANARY);
char *end_canary_ptr = ptr + GetBlockSize(bid);
char *end_canary_ptr = ptr + GetBlockSize(name);
std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
}
else
{
char *start_canary_ptr = ptr - sizeof(CANARY);
char *end_canary_ptr = ptr + GetBlockSize(bid);
char *end_canary_ptr = ptr + GetBlockSize(name);
bool start_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
bool end_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
if (!start_canary_alive)
{
throw util::exception("Start canary of block corrupted. (" +
std::string(block_id_to_name[bid]) + ")" + SOURCE_REF);
throw util::exception("Start canary of block corrupted. (" + name + ")" +
SOURCE_REF);
}
if (!end_canary_alive)
{
throw util::exception("End canary of block corrupted. (" +
std::string(block_id_to_name[bid]) + ")" + SOURCE_REF);
throw util::exception("End canary of block corrupted. (" + name + ")" + SOURCE_REF);
}
}
@ -231,6 +104,9 @@ class DataLayout
}
private:
friend void serialization::read(io::BufferReader &reader, DataLayout &layout);
friend void serialization::write(io::BufferWriter &writer, const DataLayout &layout);
// Fit aligned storage in buffer to 64 bytes to conform with AVX 512 types
inline void *align(void *&ptr) const noexcept
{
@ -239,13 +115,13 @@ class DataLayout
return ptr = reinterpret_cast<void *>(aligned);
}
inline void *GetAlignedBlockPtr(void *ptr, BlockID bid) const
inline void *GetAlignedBlockPtr(void *ptr, const std::string &name) const
{
for (auto i = 0; i < bid; i++)
for (auto iter = blocks.begin(); iter != blocks.end() && iter->first != name; ++iter)
{
ptr = static_cast<char *>(ptr) + sizeof(CANARY);
ptr = align(ptr);
ptr = static_cast<char *>(ptr) + GetBlockSize((BlockID)i);
ptr = static_cast<char *>(ptr) + GetBlockSize(name);
ptr = static_cast<char *>(ptr) + sizeof(CANARY);
}
@ -254,14 +130,14 @@ class DataLayout
return ptr;
}
template <typename T> inline T *GetBlockEnd(char *shared_memory, BlockID bid) const
template <typename T> inline T *GetBlockEnd(char *shared_memory, const std::string &name) const
{
auto begin = GetBlockPtr<T>(shared_memory, bid);
return begin + GetBlockEntries(bid);
auto begin = GetBlockPtr<T>(shared_memory, name);
return begin + GetBlockEntries(name);
}
static constexpr std::size_t BLOCK_ALIGNMENT = 64;
std::array<Block, NUM_BLOCKS> blocks;
std::map<std::string, Block> blocks;
};
enum SharedDataType
@ -298,9 +174,6 @@ inline std::string regionToString(const SharedDataType region)
return "INVALID_REGION";
}
}
static_assert(sizeof(block_id_to_name) / sizeof(*block_id_to_name) == DataLayout::NUM_BLOCKS,
"Number of blocks needs to match the number of Block names.");
}
}

9
src/engine/datafacade/process_memory_allocator.cpp
View File

@ -15,17 +15,16 @@ ProcessMemoryAllocator::ProcessMemoryAllocator(const storage::StorageConfig &con
storage::Storage storage(config);
// Calculate the layout/size of the memory block
internal_layout = std::make_unique<storage::DataLayout>();
storage.PopulateLayout(*internal_layout);
storage.PopulateLayout(internal_layout);
// Allocate the memory block, then load data from files into it
internal_memory = std::make_unique<char[]>(internal_layout->GetSizeOfLayout());
storage.PopulateData(*internal_layout, internal_memory.get());
internal_memory = std::make_unique<char[]>(internal_layout.GetSizeOfLayout());
storage.PopulateData(internal_layout, internal_memory.get());
}
ProcessMemoryAllocator::~ProcessMemoryAllocator() {}
storage::DataLayout &ProcessMemoryAllocator::GetLayout() { return *internal_layout.get(); }
const storage::DataLayout &ProcessMemoryAllocator::GetLayout() { return internal_layout; }
char *ProcessMemoryAllocator::GetMemory() { return internal_memory.get(); }
} // namespace datafacade

13
src/engine/datafacade/shared_memory_allocator.cpp
View File

@ -1,4 +1,7 @@
#include "engine/datafacade/shared_memory_allocator.hpp"
#include "storage/serialization.hpp"
#include "util/log.hpp"
#include "boost/assert.hpp"
@ -16,17 +19,21 @@ SharedMemoryAllocator::SharedMemoryAllocator(storage::SharedDataType data_region
BOOST_ASSERT(storage::SharedMemory::RegionExists(data_region));
m_large_memory = storage::makeSharedMemory(data_region);
storage::io::BufferReader reader(GetMemory());
storage::serialization::read(reader, data_layout);
layout_size = reader.GetPosition();
}
SharedMemoryAllocator::~SharedMemoryAllocator() {}
storage::DataLayout &SharedMemoryAllocator::GetLayout()
const storage::DataLayout &SharedMemoryAllocator::GetLayout()
{
return *reinterpret_cast<storage::DataLayout *>(m_large_memory->Ptr());
return data_layout;
}
char *SharedMemoryAllocator::GetMemory()
{
return reinterpret_cast<char *>(m_large_memory->Ptr()) + sizeof(storage::DataLayout);
return reinterpret_cast<char *>(m_large_memory->Ptr()) + layout_size;
}
} // namespace datafacade

2
src/osrm/osrm.cpp
View File

@ -35,7 +35,7 @@ OSRM::OSRM(engine::EngineConfig &config)
auto mem = storage::makeSharedMemory(barrier.data().region);
auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr());
if (layout->GetBlockSize(storage::DataLayout::NAME_VALUES) == 0)
if (layout->GetBlockSize("/common/names/values") == 0)
throw util::exception(
"No name data loaded, cannot continue. Have you run osrm-datastore to load data?");
}

357
src/storage/storage.cpp
View File

@ -70,7 +70,7 @@ namespace storage
{
namespace
{
template <typename OutIter> void readBlocks(const boost::filesystem::path &path, OutIter out)
inline void readBlocks(const boost::filesystem::path &path, DataLayout &layout)
{
tar::FileReader reader(path, tar::FileReader::VerifyFingerprint);
@ -83,7 +83,7 @@ template <typename OutIter> void readBlocks(const boost::filesystem::path &path,
if (name_end == std::string::npos)
{
auto number_of_elements = reader.ReadElementCount64(entry.name);
*out++ = NamedBlock{entry.name, Block{number_of_elements, entry.size}};
layout.SetBlock(entry.name, Block{number_of_elements, entry.size});
}
}
}
@ -159,15 +159,19 @@ int Storage::Run(int max_wait)
DataLayout layout;
PopulateLayout(layout);
io::BufferWriter writer;
serialization::write(writer, layout);
auto encoded_layout = writer.GetBuffer();
// Allocate shared memory block
auto regions_size = sizeof(layout) + layout.GetSizeOfLayout();
auto regions_size = encoded_layout.size() + layout.GetSizeOfLayout();
util::Log() << "Allocating shared memory of " << regions_size << " bytes";
auto data_memory = makeSharedMemory(next_region, regions_size);
// Copy memory layout to shared memory and populate data
char *shared_memory_ptr = static_cast<char *>(data_memory->Ptr());
memcpy(shared_memory_ptr, &layout, sizeof(layout));
PopulateData(layout, shared_memory_ptr + sizeof(layout));
memcpy(shared_memory_ptr, encoded_layout.data(), encoded_layout.size());
PopulateData(layout, shared_memory_ptr + encoded_layout.size());
{ // Lock for write access shared region mutex
boost::interprocess::scoped_lock<Monitor::mutex_type> lock(monitor.get_mutex(),
@ -236,91 +240,10 @@ void Storage::PopulateLayout(DataLayout &layout)
auto absolute_file_index_path =
boost::filesystem::absolute(config.GetPath(".osrm.fileIndex"));
layout.SetBlock(DataLayout::FILE_INDEX_PATH,
layout.SetBlock("/common/rtree/file_index_path",
make_block<char>(absolute_file_index_path.string().length() + 1));
}
std::unordered_map<std::string, DataLayout::BlockID> name_to_block_id = {
{"/mld/multilevelgraph/node_array", DataLayout::MLD_GRAPH_NODE_LIST},
{"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST},
{"/mld/multilevelgraph/node_to_edge_offset", DataLayout::MLD_GRAPH_NODE_TO_OFFSET},
{"/mld/multilevelgraph/connectivity_checksum", DataLayout::IGNORE_BLOCK},
{"/mld/multilevelpartition/level_data", DataLayout::MLD_LEVEL_DATA},
{"/mld/multilevelpartition/partition", DataLayout::MLD_PARTITION},
{"/mld/multilevelpartition/cell_to_children", DataLayout::MLD_CELL_TO_CHILDREN},
{"/mld/cellstorage/source_boundary", DataLayout::MLD_CELL_SOURCE_BOUNDARY},
{"/mld/cellstorage/destination_boundary", DataLayout::MLD_CELL_DESTINATION_BOUNDARY},
{"/mld/cellstorage/cells", DataLayout::MLD_CELLS},
{"/mld/cellstorage/level_to_cell_offset", DataLayout::MLD_CELL_LEVEL_OFFSETS},
{"/mld/metrics/0/weights", DataLayout::MLD_CELL_WEIGHTS_0},
{"/mld/metrics/1/weights", DataLayout::MLD_CELL_WEIGHTS_1},
{"/mld/metrics/2/weights", DataLayout::MLD_CELL_WEIGHTS_2},
{"/mld/metrics/3/weights", DataLayout::MLD_CELL_WEIGHTS_3},
{"/mld/metrics/4/weights", DataLayout::MLD_CELL_WEIGHTS_4},
{"/mld/metrics/5/weights", DataLayout::MLD_CELL_WEIGHTS_5},
{"/mld/metrics/6/weights", DataLayout::MLD_CELL_WEIGHTS_6},
{"/mld/metrics/7/weights", DataLayout::MLD_CELL_WEIGHTS_7},
{"/mld/metrics/0/durations", DataLayout::MLD_CELL_DURATIONS_0},
{"/mld/metrics/1/durations", DataLayout::MLD_CELL_DURATIONS_1},
{"/mld/metrics/2/durations", DataLayout::MLD_CELL_DURATIONS_2},
{"/mld/metrics/3/durations", DataLayout::MLD_CELL_DURATIONS_3},
{"/mld/metrics/4/durations", DataLayout::MLD_CELL_DURATIONS_4},
{"/mld/metrics/5/durations", DataLayout::MLD_CELL_DURATIONS_5},
{"/mld/metrics/6/durations", DataLayout::MLD_CELL_DURATIONS_6},
{"/mld/metrics/7/durations", DataLayout::MLD_CELL_DURATIONS_7},
{"/ch/checksum", DataLayout::HSGR_CHECKSUM},
{"/ch/contracted_graph/node_array", DataLayout::CH_GRAPH_NODE_LIST},
{"/ch/contracted_graph/edge_array", DataLayout::CH_GRAPH_EDGE_LIST},
{"/ch/connectivity_checksum", DataLayout::IGNORE_BLOCK},
{"/ch/edge_filter/0", DataLayout::CH_EDGE_FILTER_0},
{"/ch/edge_filter/1", DataLayout::CH_EDGE_FILTER_1},
{"/ch/edge_filter/2", DataLayout::CH_EDGE_FILTER_2},
{"/ch/edge_filter/3", DataLayout::CH_EDGE_FILTER_3},
{"/ch/edge_filter/4", DataLayout::CH_EDGE_FILTER_4},
{"/ch/edge_filter/5", DataLayout::CH_EDGE_FILTER_5},
{"/ch/edge_filter/6", DataLayout::CH_EDGE_FILTER_6},
{"/ch/edge_filter/7", DataLayout::CH_EDGE_FILTER_7},
{"/common/intersection_bearings/bearing_values", DataLayout::BEARING_VALUES},
{"/common/intersection_bearings/node_to_class_id", DataLayout::BEARING_CLASSID},
{"/common/intersection_bearings/class_id_to_ranges/block_offsets",
DataLayout::BEARING_OFFSETS},
{"/common/intersection_bearings/class_id_to_ranges/diff_blocks",
DataLayout::BEARING_BLOCKS},
{"/common/entry_classes", DataLayout::ENTRY_CLASS},
{"/common/properties", DataLayout::PROPERTIES},
{"/common/coordinates", DataLayout::COORDINATE_LIST},
{"/common/osm_node_ids/packed", DataLayout::OSM_NODE_ID_LIST},
{"/common/data_sources_names", DataLayout::DATASOURCES_NAMES},
{"/common/segment_data/index", DataLayout::GEOMETRIES_INDEX},
{"/common/segment_data/nodes", DataLayout::GEOMETRIES_NODE_LIST},
{"/common/segment_data/forward_weights/packed", DataLayout::GEOMETRIES_FWD_WEIGHT_LIST},
{"/common/segment_data/reverse_weights/packed", DataLayout::GEOMETRIES_REV_WEIGHT_LIST},
{"/common/segment_data/forward_durations/packed", DataLayout::GEOMETRIES_FWD_DURATION_LIST},
{"/common/segment_data/reverse_durations/packed", DataLayout::GEOMETRIES_REV_DURATION_LIST},
{"/common/segment_data/forward_data_sources", DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST},
{"/common/segment_data/reverse_data_sources", DataLayout::GEOMETRIES_REV_DATASOURCES_LIST},
{"/common/ebg_node_data/nodes", DataLayout::EDGE_BASED_NODE_DATA_LIST},
{"/common/ebg_node_data/annotations", DataLayout::ANNOTATION_DATA_LIST},
{"/common/turn_lanes/offsets", DataLayout::LANE_DESCRIPTION_OFFSETS},
{"/common/turn_lanes/masks", DataLayout::LANE_DESCRIPTION_MASKS},
{"/common/turn_lanes/data", DataLayout::TURN_LANE_DATA},
{"/common/maneuver_overrides/overrides", DataLayout::MANEUVER_OVERRIDES},
{"/common/maneuver_overrides/node_sequences", DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES},
{"/common/turn_penalty/weight", DataLayout::TURN_WEIGHT_PENALTIES},
{"/common/turn_penalty/duration", DataLayout::TURN_DURATION_PENALTIES},
{"/common/turn_data/pre_turn_bearings", DataLayout::PRE_TURN_BEARING},
{"/common/turn_data/post_turn_bearings", DataLayout::POST_TURN_BEARING},
{"/common/turn_data/turn_instructions", DataLayout::TURN_INSTRUCTION},
{"/common/turn_data/lane_data_ids", DataLayout::LANE_DATA_ID},
{"/common/turn_data/entry_class_ids", DataLayout::ENTRY_CLASSID},
{"/common/turn_data/connectivity_checksum", DataLayout::IGNORE_BLOCK},
{"/common/names/blocks", DataLayout::NAME_BLOCKS},
{"/common/names/values", DataLayout::NAME_VALUES},
{"/common/rtree/search_tree", DataLayout::R_SEARCH_TREE},
{"/common/rtree/search_tree_level_starts", DataLayout::R_SEARCH_TREE_LEVEL_STARTS},
};
std::vector<NamedBlock> blocks;
constexpr bool REQUIRED = true;
constexpr bool OPTIONAL = false;
std::vector<std::pair<bool, boost::filesystem::path>> tar_files = {
@ -349,7 +272,7 @@ void Storage::PopulateLayout(DataLayout &layout)
{
if (boost::filesystem::exists(file.second))
{
readBlocks(file.second, std::back_inserter(blocks));
readBlocks(file.second, layout);
}
else
{
@ -360,17 +283,6 @@ void Storage::PopulateLayout(DataLayout &layout)
}
}
}
for (const auto &block : blocks)
{
auto id_iter = name_to_block_id.find(std::get<0>(block));
if (id_iter == name_to_block_id.end())
{
throw util::exception("Could not map " + std::get<0>(block) +
" to a region in memory.");
}
layout.SetBlock(id_iter->second, std::get<1>(block));
}
}
void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
@ -385,14 +297,14 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// store the filename of the on-disk portion of the RTree
{
const auto file_index_path_ptr =
layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::FILE_INDEX_PATH);
layout.GetBlockPtr<char, true>(memory_ptr, "/common/rtree/file_index_path");
// make sure we have 0 ending
std::fill(file_index_path_ptr,
file_index_path_ptr + layout.GetBlockSize(DataLayout::FILE_INDEX_PATH),
file_index_path_ptr + layout.GetBlockSize("/common/rtree/file_index_path"),
0);
const auto absolute_file_index_path =
boost::filesystem::absolute(config.GetPath(".osrm.fileIndex")).string();
BOOST_ASSERT(static_cast<std::size_t>(layout.GetBlockSize(DataLayout::FILE_INDEX_PATH)) >=
BOOST_ASSERT(static_cast<std::size_t>(layout.GetBlockSize("/common/rtree/file_index_path")) >=
absolute_file_index_path.size());
std::copy(
absolute_file_index_path.begin(), absolute_file_index_path.end(), file_index_path_ptr);
@ -402,15 +314,15 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
{
const auto name_blocks_ptr =
layout.GetBlockPtr<extractor::NameTableView::IndexedData::BlockReference, true>(
memory_ptr, DataLayout::NAME_BLOCKS);
memory_ptr, "/common/names/blocks");
const auto name_values_ptr =
layout.GetBlockPtr<extractor::NameTableView::IndexedData::ValueType, true>(
memory_ptr, DataLayout::NAME_VALUES);
memory_ptr, "/common/names/values");
util::vector_view<extractor::NameTableView::IndexedData::BlockReference> blocks(
name_blocks_ptr, layout.GetBlockEntries(storage::DataLayout::NAME_BLOCKS));
name_blocks_ptr, layout.GetBlockEntries("/common/names/blocks"));
util::vector_view<extractor::NameTableView::IndexedData::ValueType> values(
name_values_ptr, layout.GetBlockEntries(storage::DataLayout::NAME_VALUES));
name_values_ptr, layout.GetBlockEntries("/common/names/values"));
extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
extractor::NameTableView name_table{index_data_view};
@ -420,24 +332,24 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Turn lane data
{
const auto turn_lane_data_ptr = layout.GetBlockPtr<util::guidance::LaneTupleIdPair, true>(
memory_ptr, DataLayout::TURN_LANE_DATA);
memory_ptr, "/common/turn_lanes/data");
util::vector_view<util::guidance::LaneTupleIdPair> turn_lane_data(
turn_lane_data_ptr, layout.GetBlockEntries(storage::DataLayout::TURN_LANE_DATA));
turn_lane_data_ptr, layout.GetBlockEntries("/common/turn_lanes/data"));
extractor::files::readTurnLaneData(config.GetPath(".osrm.tld"), turn_lane_data);
}
// Turn lane descriptions
{
auto offsets_ptr = layout.GetBlockPtr<std::uint32_t, true>(
memory_ptr, storage::DataLayout::LANE_DESCRIPTION_OFFSETS);
auto offsets_ptr =
layout.GetBlockPtr<std::uint32_t, true>(memory_ptr, "/common/turn_lanes/offsets");
util::vector_view<std::uint32_t> offsets(
offsets_ptr, layout.GetBlockEntries(storage::DataLayout::LANE_DESCRIPTION_OFFSETS));
offsets_ptr, layout.GetBlockEntries("/common/turn_lanes/offsets"));
auto masks_ptr = layout.GetBlockPtr<extractor::TurnLaneType::Mask, true>(
memory_ptr, storage::DataLayout::LANE_DESCRIPTION_MASKS);
memory_ptr, "/common/turn_lanes/masks");
util::vector_view<extractor::TurnLaneType::Mask> masks(
masks_ptr, layout.GetBlockEntries(storage::DataLayout::LANE_DESCRIPTION_MASKS));
masks_ptr, layout.GetBlockEntries("/common/turn_lanes/masks"));
extractor::files::readTurnLaneDescriptions(config.GetPath(".osrm.tls"), offsets, masks);
}
@ -445,17 +357,15 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Load edge-based nodes data
{
auto edge_based_node_data_list_ptr = layout.GetBlockPtr<extractor::EdgeBasedNode, true>(
memory_ptr, storage::DataLayout::EDGE_BASED_NODE_DATA_LIST);
memory_ptr, "/common/ebg_node_data/nodes");
util::vector_view<extractor::EdgeBasedNode> edge_based_node_data(
edge_based_node_data_list_ptr,
layout.GetBlockEntries(storage::DataLayout::EDGE_BASED_NODE_DATA_LIST));
edge_based_node_data_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/nodes"));
auto annotation_data_list_ptr =
layout.GetBlockPtr<extractor::NodeBasedEdgeAnnotation, true>(
memory_ptr, storage::DataLayout::ANNOTATION_DATA_LIST);
memory_ptr, "/common/ebg_node_data/annotations");
util::vector_view<extractor::NodeBasedEdgeAnnotation> annotation_data(
annotation_data_list_ptr,
layout.GetBlockEntries(storage::DataLayout::ANNOTATION_DATA_LIST));
annotation_data_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/annotations"));
extractor::EdgeBasedNodeDataView node_data(std::move(edge_based_node_data),
std::move(annotation_data));
@ -466,30 +376,30 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Load original edge data
{
const auto lane_data_id_ptr =
layout.GetBlockPtr<LaneDataID, true>(memory_ptr, storage::DataLayout::LANE_DATA_ID);
layout.GetBlockPtr<LaneDataID, true>(memory_ptr, "/common/turn_data/lane_data_ids");
util::vector_view<LaneDataID> lane_data_ids(
lane_data_id_ptr, layout.GetBlockEntries(storage::DataLayout::LANE_DATA_ID));
lane_data_id_ptr, layout.GetBlockEntries("/common/turn_data/lane_data_ids"));
const auto turn_instruction_list_ptr = layout.GetBlockPtr<guidance::TurnInstruction, true>(
memory_ptr, storage::DataLayout::TURN_INSTRUCTION);
memory_ptr, "/common/turn_data/turn_instructions");
util::vector_view<guidance::TurnInstruction> turn_instructions(
turn_instruction_list_ptr,
layout.GetBlockEntries(storage::DataLayout::TURN_INSTRUCTION));
layout.GetBlockEntries("/common/turn_data/turn_instructions"));
const auto entry_class_id_list_ptr =
layout.GetBlockPtr<EntryClassID, true>(memory_ptr, storage::DataLayout::ENTRY_CLASSID);
const auto entry_class_id_list_ptr = layout.GetBlockPtr<EntryClassID, true>(
memory_ptr, "/common/turn_data/entry_class_ids");
util::vector_view<EntryClassID> entry_class_ids(
entry_class_id_list_ptr, layout.GetBlockEntries(storage::DataLayout::ENTRY_CLASSID));
entry_class_id_list_ptr, layout.GetBlockEntries("/common/turn_data/entry_class_ids"));
const auto pre_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing, true>(
memory_ptr, storage::DataLayout::PRE_TURN_BEARING);
memory_ptr, "/common/turn_data/pre_turn_bearings");
util::vector_view<guidance::TurnBearing> pre_turn_bearings(
pre_turn_bearing_ptr, layout.GetBlockEntries(storage::DataLayout::PRE_TURN_BEARING));
pre_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/pre_turn_bearings"));
const auto post_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing, true>(
memory_ptr, storage::DataLayout::POST_TURN_BEARING);
memory_ptr, "/common/turn_data/post_turn_bearings");
util::vector_view<guidance::TurnBearing> post_turn_bearings(
post_turn_bearing_ptr, layout.GetBlockEntries(storage::DataLayout::POST_TURN_BEARING));
post_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/post_turn_bearings"));
guidance::TurnDataView turn_data(std::move(turn_instructions),
std::move(lane_data_ids),
@ -504,63 +414,63 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// load compressed geometry
{
auto geometries_index_ptr =
layout.GetBlockPtr<unsigned, true>(memory_ptr, storage::DataLayout::GEOMETRIES_INDEX);
layout.GetBlockPtr<unsigned, true>(memory_ptr, "/common/segment_data/index");
util::vector_view<unsigned> geometry_begin_indices(
geometries_index_ptr, layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_INDEX));
geometries_index_ptr, layout.GetBlockEntries("/common/segment_data/index"));
auto num_entries = layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_NODE_LIST);
auto num_entries = layout.GetBlockEntries("/common/segment_data/nodes");
auto geometries_node_list_ptr =
layout.GetBlockPtr<NodeID, true>(memory_ptr, storage::DataLayout::GEOMETRIES_NODE_LIST);
layout.GetBlockPtr<NodeID, true>(memory_ptr, "/common/segment_data/nodes");
util::vector_view<NodeID> geometry_node_list(geometries_node_list_ptr, num_entries);
auto geometries_fwd_weight_list_ptr =
layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type, true>(
memory_ptr, storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST);
memory_ptr, "/common/segment_data/forward_weights/packed");
extractor::SegmentDataView::SegmentWeightVector geometry_fwd_weight_list(
util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
geometries_fwd_weight_list_ptr,
layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST)),
layout.GetBlockEntries("/common/segment_data/forward_weights/packed")),
num_entries);
auto geometries_rev_weight_list_ptr =
layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type, true>(
memory_ptr, storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST);
memory_ptr, "/common/segment_data/reverse_weights/packed");
extractor::SegmentDataView::SegmentWeightVector geometry_rev_weight_list(
util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
geometries_rev_weight_list_ptr,
layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST)),
layout.GetBlockEntries("/common/segment_data/reverse_weights/packed")),
num_entries);
auto geometries_fwd_duration_list_ptr =
layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type, true>(
memory_ptr, storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST);
memory_ptr, "/common/segment_data/forward_durations/packed");
extractor::SegmentDataView::SegmentDurationVector geometry_fwd_duration_list(
util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
geometries_fwd_duration_list_ptr,
layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST)),
layout.GetBlockEntries("/common/segment_data/forward_durations/packed")),
num_entries);
auto geometries_rev_duration_list_ptr =
layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type, true>(
memory_ptr, storage::DataLayout::GEOMETRIES_REV_DURATION_LIST);
memory_ptr, "/common/segment_data/reverse_durations/packed");
extractor::SegmentDataView::SegmentDurationVector geometry_rev_duration_list(
util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
geometries_rev_duration_list_ptr,
layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_DURATION_LIST)),
layout.GetBlockEntries("/common/segment_data/reverse_durations/packed")),
num_entries);
auto geometries_fwd_datasources_list_ptr = layout.GetBlockPtr<DatasourceID, true>(
memory_ptr, storage::DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST);
memory_ptr, "/common/segment_data/forward_data_sources");
util::vector_view<DatasourceID> geometry_fwd_datasources_list(
geometries_fwd_datasources_list_ptr,
layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST));
layout.GetBlockEntries("/common/segment_data/forward_data_sources"));
auto geometries_rev_datasources_list_ptr = layout.GetBlockPtr<DatasourceID, true>(
memory_ptr, storage::DataLayout::GEOMETRIES_REV_DATASOURCES_LIST);
memory_ptr, "/common/segment_data/reverse_data_sources");
util::vector_view<DatasourceID> geometry_rev_datasources_list(
geometries_rev_datasources_list_ptr,
layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_DATASOURCES_LIST));
layout.GetBlockEntries("/common/segment_data/reverse_data_sources"));
extractor::SegmentDataView segment_data{std::move(geometry_begin_indices),
std::move(geometry_node_list),
@ -576,7 +486,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
{
const auto datasources_names_ptr = layout.GetBlockPtr<extractor::Datasources, true>(
memory_ptr, DataLayout::DATASOURCES_NAMES);
memory_ptr, "/common/data_sources_names");
extractor::files::readDatasources(config.GetPath(".osrm.datasource_names"),
*datasources_names_ptr);
}
@ -584,38 +494,36 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Loading list of coordinates
{
const auto coordinates_ptr =
layout.GetBlockPtr<util::Coordinate, true>(memory_ptr, DataLayout::COORDINATE_LIST);
layout.GetBlockPtr<util::Coordinate, true>(memory_ptr, "/common/coordinates");
const auto osmnodeid_ptr =
layout.GetBlockPtr<extractor::PackedOSMIDsView::block_type, true>(
memory_ptr, DataLayout::OSM_NODE_ID_LIST);
memory_ptr, "/common/osm_node_ids/packed");
util::vector_view<util::Coordinate> coordinates(
coordinates_ptr, layout.GetBlockEntries(DataLayout::COORDINATE_LIST));
coordinates_ptr, layout.GetBlockEntries("/common/coordinates"));
extractor::PackedOSMIDsView osm_node_ids(
util::vector_view<extractor::PackedOSMIDsView::block_type>(
osmnodeid_ptr, layout.GetBlockEntries(DataLayout::OSM_NODE_ID_LIST)),
layout.GetBlockEntries(DataLayout::COORDINATE_LIST));
osmnodeid_ptr, layout.GetBlockEntries("/common/osm_node_ids/packed")),
layout.GetBlockEntries("/common/coordinates"));
extractor::files::readNodes(config.GetPath(".osrm.nbg_nodes"), coordinates, osm_node_ids);
}
// load turn weight penalties
{
auto turn_duration_penalties_ptr = layout.GetBlockPtr<TurnPenalty, true>(
memory_ptr, storage::DataLayout::TURN_WEIGHT_PENALTIES);
auto turn_duration_penalties_ptr =
layout.GetBlockPtr<TurnPenalty, true>(memory_ptr, "/common/turn_penalty/weight");
util::vector_view<TurnPenalty> turn_duration_penalties(
turn_duration_penalties_ptr,
layout.GetBlockEntries(storage::DataLayout::TURN_WEIGHT_PENALTIES));
turn_duration_penalties_ptr, layout.GetBlockEntries("/common/turn_penalty/weight"));
extractor::files::readTurnWeightPenalty(config.GetPath(".osrm.turn_weight_penalties"),
turn_duration_penalties);
}
// load turn duration penalties
{
auto turn_duration_penalties_ptr = layout.GetBlockPtr<TurnPenalty, true>(
memory_ptr, storage::DataLayout::TURN_DURATION_PENALTIES);
auto turn_duration_penalties_ptr =
layout.GetBlockPtr<TurnPenalty, true>(memory_ptr, "/common/turn_penalty/duration");
util::vector_view<TurnPenalty> turn_duration_penalties(
turn_duration_penalties_ptr,
layout.GetBlockEntries(storage::DataLayout::TURN_DURATION_PENALTIES));
turn_duration_penalties_ptr, layout.GetBlockEntries("/common/turn_penalty/duration"));
extractor::files::readTurnDurationPenalty(config.GetPath(".osrm.turn_duration_penalties"),
turn_duration_penalties);
}
@ -624,15 +532,15 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
{
const auto rtree_ptr =
layout.GetBlockPtr<RTreeNode, true>(memory_ptr, DataLayout::R_SEARCH_TREE);
layout.GetBlockPtr<RTreeNode, true>(memory_ptr, "/common/rtree/search_tree");
util::vector_view<RTreeNode> search_tree(
rtree_ptr, layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE));
rtree_ptr, layout.GetBlockEntries("/common/rtree/search_tree"));
const auto rtree_levelstarts_ptr = layout.GetBlockPtr<std::uint64_t, true>(
memory_ptr, DataLayout::R_SEARCH_TREE_LEVEL_STARTS);
memory_ptr, "/common/rtree/search_tree_level_starts");
util::vector_view<std::uint64_t> rtree_level_starts(
rtree_levelstarts_ptr,
layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS));
layout.GetBlockEntries("/common/rtree/search_tree_level_starts"));
// we need this purely for the interface
util::vector_view<util::Coordinate> empty_coords;
@ -648,7 +556,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// load profile properties
{
const auto profile_properties_ptr = layout.GetBlockPtr<extractor::ProfileProperties, true>(
memory_ptr, DataLayout::PROPERTIES);
memory_ptr, "/common/properties");
extractor::files::readProfileProperties(config.GetPath(".osrm.properties"),
*profile_properties_ptr);
}
@ -656,24 +564,29 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Load intersection data
{
auto bearing_class_id_ptr = layout.GetBlockPtr<BearingClassID, true>(
memory_ptr, storage::DataLayout::BEARING_CLASSID);
memory_ptr, "/common/intersection_bearings/node_to_class_id");
util::vector_view<BearingClassID> bearing_class_id(
bearing_class_id_ptr, layout.GetBlockEntries(storage::DataLayout::BEARING_CLASSID));
bearing_class_id_ptr,
layout.GetBlockEntries("/common/intersection_bearings/node_to_class_id"));
auto bearing_values_ptr = layout.GetBlockPtr<DiscreteBearing, true>(
memory_ptr, storage::DataLayout::BEARING_VALUES);
memory_ptr, "/common/intersection_bearings/bearing_values");
util::vector_view<DiscreteBearing> bearing_values(
bearing_values_ptr, layout.GetBlockEntries(storage::DataLayout::BEARING_VALUES));
bearing_values_ptr,
layout.GetBlockEntries("/common/intersection_bearings/bearing_values"));
auto offsets_ptr =
layout.GetBlockPtr<unsigned, true>(memory_ptr, storage::DataLayout::BEARING_OFFSETS);
auto offsets_ptr = layout.GetBlockPtr<unsigned, true>(
memory_ptr, "/common/intersection_bearings/class_id_to_ranges/block_offsets");
auto blocks_ptr =
layout.GetBlockPtr<util::RangeTable<16, storage::Ownership::View>::BlockT, true>(
memory_ptr, storage::DataLayout::BEARING_BLOCKS);
memory_ptr, "/common/intersection_bearings/class_id_to_ranges/diff_blocks");
util::vector_view<unsigned> bearing_offsets(
offsets_ptr, layout.GetBlockEntries(storage::DataLayout::BEARING_OFFSETS));
offsets_ptr,
layout.GetBlockEntries(
"/common/intersection_bearings/class_id_to_ranges/block_offsets"));
util::vector_view<util::RangeTable<16, storage::Ownership::View>::BlockT> bearing_blocks(
blocks_ptr, layout.GetBlockEntries(storage::DataLayout::BEARING_BLOCKS));
blocks_ptr,
layout.GetBlockEntries("/common/intersection_bearings/class_id_to_ranges/diff_blocks"));
util::RangeTable<16, storage::Ownership::View> bearing_range_table(
bearing_offsets, bearing_blocks, static_cast<unsigned>(bearing_values.size()));
@ -682,9 +595,9 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)};
auto entry_class_ptr = layout.GetBlockPtr<util::guidance::EntryClass, true>(
memory_ptr, storage::DataLayout::ENTRY_CLASS);
memory_ptr, "/common/entry_classes");
util::vector_view<util::guidance::EntryClass> entry_classes(
entry_class_ptr, layout.GetBlockEntries(storage::DataLayout::ENTRY_CLASS));
entry_class_ptr, layout.GetBlockEntries("/common/entry_classes"));
extractor::files::readIntersections(
config.GetPath(".osrm.icd"), intersection_bearings_view, entry_classes);
@ -695,25 +608,22 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
{
auto graph_nodes_ptr =
layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
memory_ptr, storage::DataLayout::CH_GRAPH_NODE_LIST);
memory_ptr, "/ch/contracted_graph/node_array");
auto graph_edges_ptr =
layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
memory_ptr, storage::DataLayout::CH_GRAPH_EDGE_LIST);
auto checksum =
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
memory_ptr, "/ch/contracted_graph/edge_array");
auto checksum = layout.GetBlockPtr<unsigned, true>(memory_ptr, "/ch/checksum");
util::vector_view<contractor::QueryGraphView::NodeArrayEntry> node_list(
graph_nodes_ptr, layout.GetBlockEntries(storage::DataLayout::CH_GRAPH_NODE_LIST));
graph_nodes_ptr, layout.GetBlockEntries("/ch/contracted_graph/node_array"));
util::vector_view<contractor::QueryGraphView::EdgeArrayEntry> edge_list(
graph_edges_ptr, layout.GetBlockEntries(storage::DataLayout::CH_GRAPH_EDGE_LIST));
graph_edges_ptr, layout.GetBlockEntries("/ch/contracted_graph/edge_array"));
std::vector<util::vector_view<bool>> edge_filter;
for (auto index : util::irange<std::size_t>(0, NUM_METRICS))
{
auto block_id =
static_cast<DataLayout::BlockID>(storage::DataLayout::CH_EDGE_FILTER_0 + index);
auto data_ptr =
layout.GetBlockPtr<util::vector_view<bool>::Word, true>(memory_ptr, block_id);
auto block_id = "/ch/edge_filter/" + std::to_string(index);
auto data_ptr = layout.GetBlockPtr<util::vector_view<bool>::Word, true>(memory_ptr, block_id);
auto num_entries = layout.GetBlockEntries(block_id);
edge_filter.emplace_back(data_ptr, num_entries);
}
@ -736,35 +646,35 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
}
else
{
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
layout.GetBlockPtr<unsigned, true>(memory_ptr, "/ch/checksum");
layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
memory_ptr, DataLayout::CH_GRAPH_NODE_LIST);
memory_ptr, "/ch/contracted_graph/node_array");
layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
memory_ptr, DataLayout::CH_GRAPH_EDGE_LIST);
memory_ptr, "/ch/contracted_graph/edge_array");
}
}
{ // Loading MLD Data
if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
{
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_LEVEL_DATA) > 0);
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELL_TO_CHILDREN) > 0);
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_PARTITION) > 0);
BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/level_data") > 0);
BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0);
BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/partition") > 0);
auto level_data =
layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData, true>(
memory_ptr, storage::DataLayout::MLD_LEVEL_DATA);
memory_ptr, "/mld/multilevelpartition/level_data");
auto mld_partition_ptr = layout.GetBlockPtr<PartitionID, true>(
memory_ptr, storage::DataLayout::MLD_PARTITION);
memory_ptr, "/mld/multilevelpartition/partition");
auto partition_entries_count =
layout.GetBlockEntries(storage::DataLayout::MLD_PARTITION);
layout.GetBlockEntries("/mld/multilevelpartition/partition");
util::vector_view<PartitionID> partition(mld_partition_ptr, partition_entries_count);
auto mld_chilren_ptr = layout.GetBlockPtr<CellID, true>(
memory_ptr, storage::DataLayout::MLD_CELL_TO_CHILDREN);
memory_ptr, "/mld/multilevelpartition/cell_to_children");
auto children_entries_count =
layout.GetBlockEntries(storage::DataLayout::MLD_CELL_TO_CHILDREN);
layout.GetBlockEntries("/mld/multilevelpartition/cell_to_children");
util::vector_view<CellID> cell_to_children(mld_chilren_ptr, children_entries_count);
partitioner::MultiLevelPartitionView mlp{
@ -774,25 +684,25 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
if (boost::filesystem::exists(config.GetPath(".osrm.cells")))
{
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0);
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS) > 0);
BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/cells") > 0);
BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
auto mld_source_boundary_ptr = layout.GetBlockPtr<NodeID, true>(
memory_ptr, storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
auto mld_source_boundary_ptr =
layout.GetBlockPtr<NodeID, true>(memory_ptr, "/mld/cellstorage/source_boundary");
auto mld_destination_boundary_ptr = layout.GetBlockPtr<NodeID, true>(
memory_ptr, storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY);
memory_ptr, "/mld/cellstorage/destination_boundary");
auto mld_cells_ptr = layout.GetBlockPtr<partitioner::CellStorageView::CellData, true>(
memory_ptr, storage::DataLayout::MLD_CELLS);
memory_ptr, "/mld/cellstorage/cells");
auto mld_cell_level_offsets_ptr = layout.GetBlockPtr<std::uint64_t, true>(
memory_ptr, storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
memory_ptr, "/mld/cellstorage/level_to_cell_offset");
auto source_boundary_entries_count =
layout.GetBlockEntries(storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
layout.GetBlockEntries("/mld/cellstorage/source_boundary");
auto destination_boundary_entries_count =
layout.GetBlockEntries(storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY);
auto cells_entries_counts = layout.GetBlockEntries(storage::DataLayout::MLD_CELLS);
layout.GetBlockEntries("/mld/cellstorage/destination_boundary");
auto cells_entries_counts = layout.GetBlockEntries("/mld/cellstorage/cells");
auto cell_level_offsets_entries_count =
layout.GetBlockEntries(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
layout.GetBlockEntries("/mld/cellstorage/level_to_cell_offset");
util::vector_view<NodeID> source_boundary(mld_source_boundary_ptr,
source_boundary_entries_count);
@ -812,17 +722,15 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
{
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0);
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS) > 0);
BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/cells") > 0);
BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
std::vector<customizer::CellMetricView> metrics;
for (auto index : util::irange<std::size_t>(0, NUM_METRICS))
{
auto weights_block_id = static_cast<DataLayout::BlockID>(
storage::DataLayout::MLD_CELL_WEIGHTS_0 + index);
auto durations_block_id = static_cast<DataLayout::BlockID>(
storage::DataLayout::MLD_CELL_DURATIONS_0 + index);
auto weights_block_id = "/mld/metrics/" + std::to_string(index) + "/weights";
auto durations_block_id = "/mld/metrics/" + std::to_string(index) + "/durations";
auto weight_entries_count = layout.GetBlockEntries(weights_block_id);
auto duration_entries_count = layout.GetBlockEntries(durations_block_id);
@ -846,21 +754,21 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
auto graph_nodes_ptr =
layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry, true>(
memory_ptr, storage::DataLayout::MLD_GRAPH_NODE_LIST);
memory_ptr, "/mld/multilevelgraph/node_array");
auto graph_edges_ptr =
layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry, true>(
memory_ptr, storage::DataLayout::MLD_GRAPH_EDGE_LIST);
memory_ptr, "/mld/multilevelgraph/edge_array");
auto graph_node_to_offset_ptr =
layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset, true>(
memory_ptr, storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET);
memory_ptr, "/mld/multilevelgraph/node_to_edge_offset");
util::vector_view<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry> node_list(
graph_nodes_ptr, layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_NODE_LIST));
graph_nodes_ptr, layout.GetBlockEntries("/mld/multilevelgraph/node_array"));
util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry> edge_list(
graph_edges_ptr, layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_EDGE_LIST));
graph_edges_ptr, layout.GetBlockEntries("/mld/multilevelgraph/edge_array"));
util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset> node_to_offset(
graph_node_to_offset_ptr,
layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET));
layout.GetBlockEntries("/mld/multilevelgraph/node_to_edge_offset"));
std::uint32_t graph_connectivity_checksum = 0;
customizer::MultiLevelEdgeBasedGraphView graph_view(
@ -882,15 +790,16 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
{
const auto maneuver_overrides_ptr =
layout.GetBlockPtr<extractor::StorageManeuverOverride, true>(
memory_ptr, DataLayout::MANEUVER_OVERRIDES);
memory_ptr, "/common/maneuver_overrides/overrides");
const auto maneuver_override_node_sequences_ptr = layout.GetBlockPtr<NodeID, true>(
memory_ptr, DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES);
memory_ptr, "/common/maneuver_overrides/node_sequences");
util::vector_view<extractor::StorageManeuverOverride> maneuver_overrides(
maneuver_overrides_ptr, layout.GetBlockEntries(DataLayout::MANEUVER_OVERRIDES));
maneuver_overrides_ptr,
layout.GetBlockEntries("/common/maneuver_overrides/overrides"));
util::vector_view<NodeID> maneuver_override_node_sequences(
maneuver_override_node_sequences_ptr,
layout.GetBlockEntries(DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES));
layout.GetBlockEntries("/common/maneuver_overrides/node_sequences"));
extractor::files::readManeuverOverrides(config.GetPath(".osrm.maneuver_overrides"),
maneuver_overrides,

54
unit_tests/storage/data_layout.cpp Normal file
View File

@ -0,0 +1,54 @@
#include "storage/shared_datatype.hpp"
#include "../common/range_tools.hpp"
#include "../common/temporary_file.hpp"
#include <boost/filesystem.hpp>
#include <boost/test/unit_test.hpp>
#include <cmath>
BOOST_AUTO_TEST_SUITE(serialization)
using namespace osrm;
using namespace osrm::storage;
BOOST_AUTO_TEST_CASE(layout_write_test)
{
DataLayout layout;
Block block_1 {20, 8*20};
Block block_2 {1, 4*1};
Block block_3 {100, static_cast<std::uint64_t>(std::ceil(100/8.))};
layout.SetBlock("block1", block_1);
layout.SetBlock("block2", block_2);
layout.SetBlock("block3", block_3);
// Canary and alignment change layout size
BOOST_CHECK_GT(layout.GetSizeOfLayout(), block_1.byte_size + block_2.byte_size + block_3.byte_size);
BOOST_CHECK_EQUAL(layout.GetBlockSize("block1"), block_1.byte_size);
BOOST_CHECK_EQUAL(layout.GetBlockSize("block2"), block_2.byte_size);
BOOST_CHECK_EQUAL(layout.GetBlockSize("block3"), block_3.byte_size);
std::vector<char> buffer(layout.GetSizeOfLayout());
auto smallest_addr = buffer.data();
auto biggest_addr = buffer.data() + buffer.size();
{
auto block_1_ptr = layout.GetBlockPtr<std::uint64_t, true>(buffer.data(), "block1");
auto block_2_ptr = layout.GetBlockPtr<std::uint32_t, true>(buffer.data(), "block2");
auto block_3_ptr = layout.GetBlockPtr<unsigned char, true>(buffer.data(), "block3");
BOOST_CHECK_LT(smallest_addr, block_1_ptr);
BOOST_CHECK_GT(biggest_addr, block_1_ptr + layout.GetBlockEntries("block1"));
BOOST_CHECK_LT(smallest_addr, block_2_ptr);
BOOST_CHECK_GT(biggest_addr, block_2_ptr + layout.GetBlockEntries("block3"));
BOOST_CHECK_LT(smallest_addr, block_3_ptr);
BOOST_CHECK_GT(biggest_addr, block_3_ptr + layout.GetBlockEntries("block3"));
}
}
BOOST_AUTO_TEST_SUITE_END()