Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Remove fixed block identifiers

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-22 19:16:19 +00:00 committed by Patrick Niklaus
parent f3b7ab92ff
commit 4a9fdca7b2
14 changed files with 483 additions and 576 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/engine/datafacade/contiguous_block_allocator.hpp
View File

@ -16,7 +16,7 @@ class ContiguousBlockAllocator
virtual ~ContiguousBlockAllocator() = default; virtual ~ContiguousBlockAllocator() = default;
// interface to give access to the datafacades // interface to give access to the datafacades
virtual storage::DataLayout &GetLayout() = 0; virtual const storage::DataLayout &GetLayout() = 0;
virtual char *GetMemory() = 0; virtual char *GetMemory() = 0;
}; };

291
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
View File

@ -81,26 +81,25 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
// allocator that keeps the allocation data // allocator that keeps the allocation data
std::shared_ptr<ContiguousBlockAllocator> allocator; std::shared_ptr<ContiguousBlockAllocator> allocator;
void InitializeGraphPointer(storage::DataLayout &data_layout, void InitializeGraphPointer(const storage::DataLayout &data_layout,
char *memory_block, char *memory_block,
const std::size_t exclude_index) const std::size_t exclude_index)
{ {
auto graph_nodes_ptr = data_layout.GetBlockPtr<GraphNode>( auto graph_nodes_ptr =
memory_block, storage::DataLayout::CH_GRAPH_NODE_LIST); data_layout.GetBlockPtr<GraphNode>(memory_block, "/ch/contracted_graph/node_array");
auto graph_edges_ptr = data_layout.GetBlockPtr<GraphEdge>( auto graph_edges_ptr =
memory_block, storage::DataLayout::CH_GRAPH_EDGE_LIST); data_layout.GetBlockPtr<GraphEdge>(memory_block, "/ch/contracted_graph/edge_array");
auto filter_block_id = static_cast<storage::DataLayout::BlockID>( auto filter_block_id = "/ch/edge_filter/" + std::to_string(exclude_index);
storage::DataLayout::CH_EDGE_FILTER_0 + exclude_index);
auto edge_filter_ptr = auto edge_filter_ptr =
data_layout.GetBlockPtr<util::vector_view<bool>::Word>(memory_block, filter_block_id); data_layout.GetBlockPtr<util::vector_view<bool>::Word>(memory_block, filter_block_id);
util::vector_view<GraphNode> node_list( util::vector_view<GraphNode> node_list(
graph_nodes_ptr, data_layout.GetBlockEntries(storage::DataLayout::CH_GRAPH_NODE_LIST)); graph_nodes_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/node_array"));
util::vector_view<GraphEdge> edge_list( util::vector_view<GraphEdge> edge_list(
graph_edges_ptr, data_layout.GetBlockEntries(storage::DataLayout::CH_GRAPH_EDGE_LIST)); graph_edges_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/edge_array"));
util::vector_view<bool> edge_filter(edge_filter_ptr, util::vector_view<bool> edge_filter(edge_filter_ptr,
data_layout.GetBlockEntries(filter_block_id)); data_layout.GetBlockEntries(filter_block_id));
@ -115,7 +114,7 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index); InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
} }
void InitializeInternalPointers(storage::DataLayout &data_layout, void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block, char *memory_block,
const std::size_t exclude_index) const std::size_t exclude_index)
{ {
@ -224,29 +223,28 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
// allocator that keeps the allocation data // allocator that keeps the allocation data
std::shared_ptr<ContiguousBlockAllocator> allocator; std::shared_ptr<ContiguousBlockAllocator> allocator;
void InitializeProfilePropertiesPointer(storage::DataLayout &data_layout, void InitializeProfilePropertiesPointer(const storage::DataLayout &data_layout,
char *memory_block, char *memory_block,
const std::size_t exclude_index) const std::size_t exclude_index)
{ {
m_profile_properties = data_layout.GetBlockPtr<extractor::ProfileProperties>( m_profile_properties = data_layout.GetBlockPtr<extractor::ProfileProperties>(
memory_block, storage::DataLayout::PROPERTIES); memory_block, "/common/properties");
exclude_mask = m_profile_properties->excludable_classes[exclude_index]; exclude_mask = m_profile_properties->excludable_classes[exclude_index];
} }
void InitializeChecksumPointer(storage::DataLayout &data_layout, char *memory_block) void InitializeChecksumPointer(const storage::DataLayout &data_layout, char *memory_block)
{ {
m_check_sum = m_check_sum = *data_layout.GetBlockPtr<unsigned>(memory_block, "/ch/checksum");
*data_layout.GetBlockPtr<unsigned>(memory_block, storage::DataLayout::HSGR_CHECKSUM);
util::Log() << "set checksum: " << m_check_sum; util::Log() << "set checksum: " << m_check_sum;
} }
void InitializeRTreePointers(storage::DataLayout &data_layout, char *memory_block) void InitializeRTreePointers(const storage::DataLayout &data_layout, char *memory_block)
{ {
BOOST_ASSERT_MSG(!m_coordinate_list.empty(), "coordinates must be loaded before r-tree"); BOOST_ASSERT_MSG(!m_coordinate_list.empty(), "coordinates must be loaded before r-tree");
const auto file_index_ptr = const auto file_index_ptr =
data_layout.GetBlockPtr<char>(memory_block, storage::DataLayout::FILE_INDEX_PATH); data_layout.GetBlockPtr<char>(memory_block, "/common/rtree/file_index_path");
file_index_path = boost::filesystem::path(file_index_ptr); file_index_path = boost::filesystem::path(file_index_ptr);
if (!boost::filesystem::exists(file_index_path)) if (!boost::filesystem::exists(file_index_path))
{ {
@ -256,15 +254,15 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
} }
const auto rtree_ptr = const auto rtree_ptr =
data_layout.GetBlockPtr<RTreeNode>(memory_block, storage::DataLayout::R_SEARCH_TREE); data_layout.GetBlockPtr<RTreeNode>(memory_block, "/common/rtree/search_tree");
util::vector_view<RTreeNode> search_tree( util::vector_view<RTreeNode> search_tree(
rtree_ptr, data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE)); rtree_ptr, data_layout.GetBlockEntries("/common/rtree/search_tree"));
const auto rtree_levelstarts_ptr = data_layout.GetBlockPtr<std::uint64_t>( const auto rtree_levelstarts_ptr = data_layout.GetBlockPtr<std::uint64_t>(
memory_block, storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS); memory_block, "/common/rtree/search_tree_level_starts");
util::vector_view<std::uint64_t> rtree_level_starts( util::vector_view<std::uint64_t> rtree_level_starts(
rtree_levelstarts_ptr, rtree_levelstarts_ptr,
data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS)); data_layout.GetBlockEntries("/common/rtree/search_tree_level_starts"));
m_static_rtree.reset(new SharedRTree{std::move(search_tree), m_static_rtree.reset(new SharedRTree{std::move(search_tree),
std::move(rtree_level_starts), std::move(rtree_level_starts),
@ -274,70 +272,67 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
new SharedGeospatialQuery(*m_static_rtree, m_coordinate_list, *this)); new SharedGeospatialQuery(*m_static_rtree, m_coordinate_list, *this));
} }
void InitializeNodeInformationPointers(storage::DataLayout &layout, char *memory_ptr) void InitializeNodeInformationPointers(const storage::DataLayout &layout, char *memory_ptr)
{ {
const auto coordinate_list_ptr = const auto coordinate_list_ptr =
layout.GetBlockPtr<util::Coordinate>(memory_ptr, storage::DataLayout::COORDINATE_LIST); layout.GetBlockPtr<util::Coordinate>(memory_ptr, "/common/coordinates");
m_coordinate_list.reset(coordinate_list_ptr, m_coordinate_list.reset(coordinate_list_ptr, layout.GetBlockEntries("/common/coordinates"));
layout.GetBlockEntries(storage::DataLayout::COORDINATE_LIST));
const auto osmnodeid_ptr = layout.GetBlockPtr<extractor::PackedOSMIDsView::block_type>( const auto osmnodeid_ptr = layout.GetBlockPtr<extractor::PackedOSMIDsView::block_type>(
memory_ptr, storage::DataLayout::OSM_NODE_ID_LIST); memory_ptr, "/common/osm_node_ids/packed");
m_osmnodeid_list = extractor::PackedOSMIDsView( m_osmnodeid_list = extractor::PackedOSMIDsView(
util::vector_view<extractor::PackedOSMIDsView::block_type>( util::vector_view<extractor::PackedOSMIDsView::block_type>(
osmnodeid_ptr, layout.GetBlockEntries(storage::DataLayout::OSM_NODE_ID_LIST)), osmnodeid_ptr, layout.GetBlockEntries("/common/osm_node_ids/packed")),
// We (ab)use the number of coordinates here because we know we have the same amount of // We (ab)use the number of coordinates here because we know we have the same amount of
// ids // ids
layout.GetBlockEntries(storage::DataLayout::COORDINATE_LIST)); layout.GetBlockEntries("/common/coordinates"));
} }
void InitializeEdgeBasedNodeDataInformationPointers(storage::DataLayout &layout, void InitializeEdgeBasedNodeDataInformationPointers(const storage::DataLayout &layout,
char *memory_ptr) char *memory_ptr)
{ {
const auto edge_based_node_list_ptr = layout.GetBlockPtr<extractor::EdgeBasedNode>( const auto edge_based_node_list_ptr =
memory_ptr, storage::DataLayout::EDGE_BASED_NODE_DATA_LIST); layout.GetBlockPtr<extractor::EdgeBasedNode>(memory_ptr, "/common/ebg_node_data/nodes");
util::vector_view<extractor::EdgeBasedNode> edge_based_node_data_list( util::vector_view<extractor::EdgeBasedNode> edge_based_node_data_list(
edge_based_node_list_ptr, edge_based_node_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/nodes"));
layout.GetBlockEntries(storage::DataLayout::EDGE_BASED_NODE_DATA_LIST));
const auto annotation_data_list_ptr = const auto annotation_data_list_ptr =
layout.GetBlockPtr<extractor::NodeBasedEdgeAnnotation>( layout.GetBlockPtr<extractor::NodeBasedEdgeAnnotation>(
memory_ptr, storage::DataLayout::ANNOTATION_DATA_LIST); memory_ptr, "/common/ebg_node_data/annotations");
util::vector_view<extractor::NodeBasedEdgeAnnotation> annotation_data( util::vector_view<extractor::NodeBasedEdgeAnnotation> annotation_data(
annotation_data_list_ptr, annotation_data_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/annotations"));
layout.GetBlockEntries(storage::DataLayout::ANNOTATION_DATA_LIST));
edge_based_node_data = extractor::EdgeBasedNodeDataView( edge_based_node_data = extractor::EdgeBasedNodeDataView(
std::move(edge_based_node_data_list), std::move(annotation_data)); std::move(edge_based_node_data_list), std::move(annotation_data));
} }
void InitializeEdgeInformationPointers(storage::DataLayout &layout, char *memory_ptr) void InitializeEdgeInformationPointers(const storage::DataLayout &layout, char *memory_ptr)
{ {
const auto lane_data_id_ptr = const auto lane_data_id_ptr =
layout.GetBlockPtr<LaneDataID>(memory_ptr, storage::DataLayout::LANE_DATA_ID); layout.GetBlockPtr<LaneDataID>(memory_ptr, "/common/turn_data/lane_data_ids");
util::vector_view<LaneDataID> lane_data_ids( util::vector_view<LaneDataID> lane_data_ids(
lane_data_id_ptr, layout.GetBlockEntries(storage::DataLayout::LANE_DATA_ID)); lane_data_id_ptr, layout.GetBlockEntries("/common/turn_data/lane_data_ids"));
const auto turn_instruction_list_ptr = layout.GetBlockPtr<guidance::TurnInstruction>( const auto turn_instruction_list_ptr = layout.GetBlockPtr<guidance::TurnInstruction>(
memory_ptr, storage::DataLayout::TURN_INSTRUCTION); memory_ptr, "/common/turn_data/turn_instructions");
util::vector_view<guidance::TurnInstruction> turn_instructions( util::vector_view<guidance::TurnInstruction> turn_instructions(
turn_instruction_list_ptr, turn_instruction_list_ptr,
layout.GetBlockEntries(storage::DataLayout::TURN_INSTRUCTION)); layout.GetBlockEntries("/common/turn_data/turn_instructions"));
const auto entry_class_id_list_ptr = const auto entry_class_id_list_ptr =
layout.GetBlockPtr<EntryClassID>(memory_ptr, storage::DataLayout::ENTRY_CLASSID); layout.GetBlockPtr<EntryClassID>(memory_ptr, "/common/turn_data/entry_class_ids");
util::vector_view<EntryClassID> entry_class_ids( util::vector_view<EntryClassID> entry_class_ids(
entry_class_id_list_ptr, layout.GetBlockEntries(storage::DataLayout::ENTRY_CLASSID)); entry_class_id_list_ptr, layout.GetBlockEntries("/common/turn_data/entry_class_ids"));
const auto pre_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing>( const auto pre_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing>(
memory_ptr, storage::DataLayout::PRE_TURN_BEARING); memory_ptr, "/common/turn_data/pre_turn_bearings");
util::vector_view<guidance::TurnBearing> pre_turn_bearings( util::vector_view<guidance::TurnBearing> pre_turn_bearings(
pre_turn_bearing_ptr, layout.GetBlockEntries(storage::DataLayout::PRE_TURN_BEARING)); pre_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/pre_turn_bearings"));
const auto post_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing>( const auto post_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing>(
memory_ptr, storage::DataLayout::POST_TURN_BEARING); memory_ptr, "/common/turn_data/post_turn_bearings");
util::vector_view<guidance::TurnBearing> post_turn_bearings( util::vector_view<guidance::TurnBearing> post_turn_bearings(
post_turn_bearing_ptr, layout.GetBlockEntries(storage::DataLayout::POST_TURN_BEARING)); post_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/post_turn_bearings"));
turn_data = guidance::TurnDataView(std::move(turn_instructions), turn_data = guidance::TurnDataView(std::move(turn_instructions),
std::move(lane_data_ids), std::move(lane_data_ids),
@ -346,124 +341,120 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
std::move(post_turn_bearings)); std::move(post_turn_bearings));
} }
void InitializeNamePointers(storage::DataLayout &data_layout, char *memory_block) void InitializeNamePointers(const storage::DataLayout &data_layout, char *memory_block)
{ {
const auto name_blocks_ptr = const auto name_blocks_ptr =
data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::BlockReference>( data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::BlockReference>(
memory_block, storage::DataLayout::NAME_BLOCKS); memory_block, "/common/names/blocks");
const auto name_values_ptr = const auto name_values_ptr =
data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::ValueType>( data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::ValueType>(
memory_block, storage::DataLayout::NAME_VALUES); memory_block, "/common/names/values");
util::vector_view<extractor::NameTableView::IndexedData::BlockReference> blocks( util::vector_view<extractor::NameTableView::IndexedData::BlockReference> blocks(
name_blocks_ptr, data_layout.GetBlockEntries(storage::DataLayout::NAME_BLOCKS)); name_blocks_ptr, data_layout.GetBlockEntries("/common/names/blocks"));
util::vector_view<extractor::NameTableView::IndexedData::ValueType> values( util::vector_view<extractor::NameTableView::IndexedData::ValueType> values(
name_values_ptr, data_layout.GetBlockEntries(storage::DataLayout::NAME_VALUES)); name_values_ptr, data_layout.GetBlockEntries("/common/names/values"));
extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)}; extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
m_name_table = extractor::NameTableView{std::move(index_data_view)}; m_name_table = extractor::NameTableView{std::move(index_data_view)};
} }
void InitializeTurnLaneDescriptionsPointers(storage::DataLayout &data_layout, void InitializeTurnLaneDescriptionsPointers(const storage::DataLayout &data_layout,
char *memory_block) char *memory_block)
{ {
auto offsets_ptr = data_layout.GetBlockPtr<std::uint32_t>( auto offsets_ptr =
memory_block, storage::DataLayout::LANE_DESCRIPTION_OFFSETS); data_layout.GetBlockPtr<std::uint32_t>(memory_block, "/common/turn_lanes/offsets");
util::vector_view<std::uint32_t> offsets( util::vector_view<std::uint32_t> offsets(
offsets_ptr, offsets_ptr, data_layout.GetBlockEntries("/common/turn_lanes/offsets"));
data_layout.GetBlockEntries(storage::DataLayout::LANE_DESCRIPTION_OFFSETS));
m_lane_description_offsets = std::move(offsets); m_lane_description_offsets = std::move(offsets);
auto masks_ptr = data_layout.GetBlockPtr<extractor::TurnLaneType::Mask>( auto masks_ptr = data_layout.GetBlockPtr<extractor::TurnLaneType::Mask>(
memory_block, storage::DataLayout::LANE_DESCRIPTION_MASKS); memory_block, "/common/turn_lanes/masks");
util::vector_view<extractor::TurnLaneType::Mask> masks( util::vector_view<extractor::TurnLaneType::Mask> masks(
masks_ptr, data_layout.GetBlockEntries(storage::DataLayout::LANE_DESCRIPTION_MASKS)); masks_ptr, data_layout.GetBlockEntries("/common/turn_lanes/masks"));
m_lane_description_masks = std::move(masks); m_lane_description_masks = std::move(masks);
const auto lane_tupel_id_pair_ptr = const auto lane_tupel_id_pair_ptr =
data_layout.GetBlockPtr<util::guidance::LaneTupleIdPair>( data_layout.GetBlockPtr<util::guidance::LaneTupleIdPair>(memory_block,
memory_block, storage::DataLayout::TURN_LANE_DATA); "/common/turn_lanes/data");
util::vector_view<util::guidance::LaneTupleIdPair> lane_tupel_id_pair( util::vector_view<util::guidance::LaneTupleIdPair> lane_tupel_id_pair(
lane_tupel_id_pair_ptr, lane_tupel_id_pair_ptr, data_layout.GetBlockEntries("/common/turn_lanes/data"));
data_layout.GetBlockEntries(storage::DataLayout::TURN_LANE_DATA));
m_lane_tupel_id_pairs = std::move(lane_tupel_id_pair); m_lane_tupel_id_pairs = std::move(lane_tupel_id_pair);
} }
void InitializeTurnPenalties(storage::DataLayout &data_layout, char *memory_block) void InitializeTurnPenalties(const storage::DataLayout &data_layout, char *memory_block)
{ {
auto turn_weight_penalties_ptr = data_layout.GetBlockPtr<TurnPenalty>( auto turn_weight_penalties_ptr =
memory_block, storage::DataLayout::TURN_WEIGHT_PENALTIES); data_layout.GetBlockPtr<TurnPenalty>(memory_block, "/common/turn_penalty/weight");
m_turn_weight_penalties = util::vector_view<TurnPenalty>( m_turn_weight_penalties = util::vector_view<TurnPenalty>(
turn_weight_penalties_ptr, turn_weight_penalties_ptr, data_layout.GetBlockEntries("/common/turn_penalty/weight"));
data_layout.GetBlockEntries(storage::DataLayout::TURN_WEIGHT_PENALTIES)); auto turn_duration_penalties_ptr =
auto turn_duration_penalties_ptr = data_layout.GetBlockPtr<TurnPenalty>( data_layout.GetBlockPtr<TurnPenalty>(memory_block, "/common/turn_penalty/duration");
memory_block, storage::DataLayout::TURN_DURATION_PENALTIES);
m_turn_duration_penalties = util::vector_view<TurnPenalty>( m_turn_duration_penalties = util::vector_view<TurnPenalty>(
turn_duration_penalties_ptr, turn_duration_penalties_ptr,
data_layout.GetBlockEntries(storage::DataLayout::TURN_DURATION_PENALTIES)); data_layout.GetBlockEntries("/common/turn_penalty/duration"));
} }
void InitializeGeometryPointers(storage::DataLayout &data_layout, char *memory_block) void InitializeGeometryPointers(const storage::DataLayout &data_layout, char *memory_block)
{ {
auto geometries_index_ptr = auto geometries_index_ptr =
data_layout.GetBlockPtr<unsigned>(memory_block, storage::DataLayout::GEOMETRIES_INDEX); data_layout.GetBlockPtr<unsigned>(memory_block, "/common/segment_data/index");
util::vector_view<unsigned> geometry_begin_indices( util::vector_view<unsigned> geometry_begin_indices(
geometries_index_ptr, geometries_index_ptr, data_layout.GetBlockEntries("/common/segment_data/index"));
data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_INDEX));
auto num_entries = data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_NODE_LIST); auto num_entries = data_layout.GetBlockEntries("/common/segment_data/nodes");
auto geometries_node_list_ptr = data_layout.GetBlockPtr<NodeID>( auto geometries_node_list_ptr =
memory_block, storage::DataLayout::GEOMETRIES_NODE_LIST); data_layout.GetBlockPtr<NodeID>(memory_block, "/common/segment_data/nodes");
util::vector_view<NodeID> geometry_node_list(geometries_node_list_ptr, num_entries); util::vector_view<NodeID> geometry_node_list(geometries_node_list_ptr, num_entries);
auto geometries_fwd_weight_list_ptr = auto geometries_fwd_weight_list_ptr =
data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type>( data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type>(
memory_block, storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST); memory_block, "/common/segment_data/forward_weights/packed");
extractor::SegmentDataView::SegmentWeightVector geometry_fwd_weight_list( extractor::SegmentDataView::SegmentWeightVector geometry_fwd_weight_list(
util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>( util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
geometries_fwd_weight_list_ptr, geometries_fwd_weight_list_ptr,
data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST)), data_layout.GetBlockEntries("/common/segment_data/forward_weights/packed")),
num_entries); num_entries);
auto geometries_rev_weight_list_ptr = auto geometries_rev_weight_list_ptr =
data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type>( data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type>(
memory_block, storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST); memory_block, "/common/segment_data/reverse_weights/packed");
extractor::SegmentDataView::SegmentWeightVector geometry_rev_weight_list( extractor::SegmentDataView::SegmentWeightVector geometry_rev_weight_list(
util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>( util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
geometries_rev_weight_list_ptr, geometries_rev_weight_list_ptr,
data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST)), data_layout.GetBlockEntries("/common/segment_data/reverse_weights/packed")),
num_entries); num_entries);
auto geometries_fwd_duration_list_ptr = auto geometries_fwd_duration_list_ptr =
data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type>( data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type>(
memory_block, storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST); memory_block, "/common/segment_data/forward_durations/packed");
extractor::SegmentDataView::SegmentDurationVector geometry_fwd_duration_list( extractor::SegmentDataView::SegmentDurationVector geometry_fwd_duration_list(
util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>( util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
geometries_fwd_duration_list_ptr, geometries_fwd_duration_list_ptr,
data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST)), data_layout.GetBlockEntries("/common/segment_data/forward_durations/packed")),
num_entries); num_entries);
auto geometries_rev_duration_list_ptr = auto geometries_rev_duration_list_ptr =
data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type>( data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type>(
memory_block, storage::DataLayout::GEOMETRIES_REV_DURATION_LIST); memory_block, "/common/segment_data/reverse_durations/packed");
extractor::SegmentDataView::SegmentDurationVector geometry_rev_duration_list( extractor::SegmentDataView::SegmentDurationVector geometry_rev_duration_list(
util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>( util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
geometries_rev_duration_list_ptr, geometries_rev_duration_list_ptr,
data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_DURATION_LIST)), data_layout.GetBlockEntries("/common/segment_data/reverse_durations/packed")),
num_entries); num_entries);
auto geometries_fwd_datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>( auto geometries_fwd_datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>(
memory_block, storage::DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST); memory_block, "/common/segment_data/forward_data_sources");
util::vector_view<DatasourceID> geometry_fwd_datasources_list( util::vector_view<DatasourceID> geometry_fwd_datasources_list(
geometries_fwd_datasources_list_ptr, geometries_fwd_datasources_list_ptr,
data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST)); data_layout.GetBlockEntries("/common/segment_data/forward_data_sources"));
auto geometries_rev_datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>( auto geometries_rev_datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>(
memory_block, storage::DataLayout::GEOMETRIES_REV_DATASOURCES_LIST); memory_block, "/common/segment_data/reverse_data_sources");
util::vector_view<DatasourceID> geometry_rev_datasources_list( util::vector_view<DatasourceID> geometry_rev_datasources_list(
geometries_rev_datasources_list_ptr, geometries_rev_datasources_list_ptr,
data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_DATASOURCES_LIST)); data_layout.GetBlockEntries("/common/segment_data/reverse_data_sources"));
segment_data = extractor::SegmentDataView{std::move(geometry_begin_indices), segment_data = extractor::SegmentDataView{std::move(geometry_begin_indices),
std::move(geometry_node_list), std::move(geometry_node_list),
@ -475,30 +466,36 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
std::move(geometry_rev_datasources_list)}; std::move(geometry_rev_datasources_list)};
m_datasources = data_layout.GetBlockPtr<extractor::Datasources>( m_datasources = data_layout.GetBlockPtr<extractor::Datasources>(
memory_block, storage::DataLayout::DATASOURCES_NAMES); memory_block, "/common/data_sources_names");
} }
void InitializeIntersectionClassPointers(storage::DataLayout &data_layout, char *memory_block) void InitializeIntersectionClassPointers(const storage::DataLayout &data_layout,
char *memory_block)
{ {
auto bearing_class_id_ptr = data_layout.GetBlockPtr<BearingClassID>( auto bearing_class_id_ptr = data_layout.GetBlockPtr<BearingClassID>(
memory_block, storage::DataLayout::BEARING_CLASSID); memory_block, "/common/intersection_bearings/node_to_class_id");
util::vector_view<BearingClassID> bearing_class_id( util::vector_view<BearingClassID> bearing_class_id(
bearing_class_id_ptr, bearing_class_id_ptr,
data_layout.GetBlockEntries(storage::DataLayout::BEARING_CLASSID)); data_layout.GetBlockEntries("/common/intersection_bearings/node_to_class_id"));
auto bearing_values_ptr = data_layout.GetBlockPtr<DiscreteBearing>( auto bearing_values_ptr = data_layout.GetBlockPtr<DiscreteBearing>(
memory_block, storage::DataLayout::BEARING_VALUES); memory_block, "/common/intersection_bearings/bearing_values");
util::vector_view<DiscreteBearing> bearing_values( util::vector_view<DiscreteBearing> bearing_values(
bearing_values_ptr, data_layout.GetBlockEntries(storage::DataLayout::BEARING_VALUES)); bearing_values_ptr,
data_layout.GetBlockEntries("/common/intersection_bearings/bearing_values"));
auto offsets_ptr = auto offsets_ptr = data_layout.GetBlockPtr<unsigned>(
data_layout.GetBlockPtr<unsigned>(memory_block, storage::DataLayout::BEARING_OFFSETS); memory_block, "/common/intersection_bearings/class_id_to_ranges/block_offsets");
auto blocks_ptr = auto blocks_ptr = data_layout.GetBlockPtr<IndexBlock>(
data_layout.GetBlockPtr<IndexBlock>(memory_block, storage::DataLayout::BEARING_BLOCKS); memory_block, "/common/intersection_bearings/class_id_to_ranges/diff_blocks");
util::vector_view<unsigned> bearing_offsets( util::vector_view<unsigned> bearing_offsets(
offsets_ptr, data_layout.GetBlockEntries(storage::DataLayout::BEARING_OFFSETS)); offsets_ptr,
data_layout.GetBlockEntries(
"/common/intersection_bearings/class_id_to_ranges/block_offsets"));
util::vector_view<IndexBlock> bearing_blocks( util::vector_view<IndexBlock> bearing_blocks(
blocks_ptr, data_layout.GetBlockEntries(storage::DataLayout::BEARING_BLOCKS)); blocks_ptr,
data_layout.GetBlockEntries(
"/common/intersection_bearings/class_id_to_ranges/diff_blocks"));
util::RangeTable<16, storage::Ownership::View> bearing_range_table( util::RangeTable<16, storage::Ownership::View> bearing_range_table(
bearing_offsets, bearing_blocks, static_cast<unsigned>(bearing_values.size())); bearing_offsets, bearing_blocks, static_cast<unsigned>(bearing_values.size()));
@ -507,28 +504,29 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)}; std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)};
auto entry_class_ptr = data_layout.GetBlockPtr<util::guidance::EntryClass>( auto entry_class_ptr = data_layout.GetBlockPtr<util::guidance::EntryClass>(
memory_block, storage::DataLayout::ENTRY_CLASS); memory_block, "/common/entry_classes");
util::vector_view<util::guidance::EntryClass> entry_class_table( util::vector_view<util::guidance::EntryClass> entry_class_table(
entry_class_ptr, data_layout.GetBlockEntries(storage::DataLayout::ENTRY_CLASS)); entry_class_ptr, data_layout.GetBlockEntries("/common/entry_classes"));
m_entry_class_table = std::move(entry_class_table); m_entry_class_table = std::move(entry_class_table);
} }
void InitializeManeuverOverridePointers(storage::DataLayout &data_layout, char *memory_block) void InitializeManeuverOverridePointers(const storage::DataLayout &data_layout,
char *memory_block)
{ {
auto maneuver_overrides_ptr = data_layout.GetBlockPtr<extractor::StorageManeuverOverride>( auto maneuver_overrides_ptr = data_layout.GetBlockPtr<extractor::StorageManeuverOverride>(
memory_block, storage::DataLayout::MANEUVER_OVERRIDES); memory_block, "/common/maneuver_overrides/overrides");
m_maneuver_overrides = util::vector_view<extractor::StorageManeuverOverride>( m_maneuver_overrides = util::vector_view<extractor::StorageManeuverOverride>(
maneuver_overrides_ptr, maneuver_overrides_ptr,
data_layout.GetBlockEntries(storage::DataLayout::MANEUVER_OVERRIDES)); data_layout.GetBlockEntries("/common/maneuver_overrides/overrides"));
auto maneuver_override_node_sequences_ptr = data_layout.GetBlockPtr<NodeID>( auto maneuver_override_node_sequences_ptr = data_layout.GetBlockPtr<NodeID>(
memory_block, storage::DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES); memory_block, "/common/maneuver_overrides/node_sequences");
m_maneuver_override_node_sequences = util::vector_view<NodeID>( m_maneuver_override_node_sequences = util::vector_view<NodeID>(
maneuver_override_node_sequences_ptr, maneuver_override_node_sequences_ptr,
data_layout.GetBlockEntries(storage::DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES)); data_layout.GetBlockEntries("/common/maneuver_overrides/node_sequences"));
} }
void InitializeInternalPointers(storage::DataLayout &data_layout, void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block, char *memory_block,
const std::size_t exclude_index) const std::size_t exclude_index)
{ {
@ -939,7 +937,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
auto found_range = std::equal_range( auto found_range = std::equal_range(
m_maneuver_overrides.begin(), m_maneuver_overrides.end(), edge_based_node_id, Comp{}); m_maneuver_overrides.begin(), m_maneuver_overrides.end(), edge_based_node_id, Comp{});
std::for_each(found_range.first, found_range.second, [&](const auto & override) { std::for_each(found_range.first, found_range.second, [&](const auto &override) {
std::vector<NodeID> sequence( std::vector<NodeID> sequence(
m_maneuver_override_node_sequences.begin() + override.node_sequence_offset_begin, m_maneuver_override_node_sequences.begin() + override.node_sequence_offset_begin,
m_maneuver_override_node_sequences.begin() + override.node_sequence_offset_end); m_maneuver_override_node_sequences.begin() + override.node_sequence_offset_end);
@ -981,7 +979,7 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
QueryGraph query_graph; QueryGraph query_graph;
void InitializeInternalPointers(storage::DataLayout &data_layout, void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block, char *memory_block,
const std::size_t exclude_index) const std::size_t exclude_index)
{ {
@ -989,39 +987,38 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
InitializeGraphPointer(data_layout, memory_block); InitializeGraphPointer(data_layout, memory_block);
} }
void InitializeMLDDataPointers(storage::DataLayout &data_layout, void InitializeMLDDataPointers(const storage::DataLayout &data_layout,
char *memory_block, char *memory_block,
const std::size_t exclude_index) const std::size_t exclude_index)
{ {
if (data_layout.GetBlockSize(storage::DataLayout::MLD_PARTITION) > 0) if (data_layout.GetBlockSize("/mld/multilevelpartition/partition") > 0)
{ {
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_LEVEL_DATA) > 0); BOOST_ASSERT(data_layout.GetBlockSize("/mld/multilevelpartition/level_data") > 0);
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_TO_CHILDREN) > 0); BOOST_ASSERT(data_layout.GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0);
auto level_data = auto level_data =
data_layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData>( data_layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData>(
memory_block, storage::DataLayout::MLD_LEVEL_DATA); memory_block, "/mld/multilevelpartition/level_data");
auto mld_partition_ptr = data_layout.GetBlockPtr<PartitionID>( auto mld_partition_ptr = data_layout.GetBlockPtr<PartitionID>(
memory_block, storage::DataLayout::MLD_PARTITION); memory_block, "/mld/multilevelpartition/partition");
auto partition_entries_count = auto partition_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_PARTITION); data_layout.GetBlockEntries("/mld/multilevelpartition/partition");
util::vector_view<PartitionID> partition(mld_partition_ptr, partition_entries_count); util::vector_view<PartitionID> partition(mld_partition_ptr, partition_entries_count);
auto mld_chilren_ptr = data_layout.GetBlockPtr<CellID>( auto mld_chilren_ptr = data_layout.GetBlockPtr<CellID>(
memory_block, storage::DataLayout::MLD_CELL_TO_CHILDREN); memory_block, "/mld/multilevelpartition/cell_to_children");
auto children_entries_count = auto children_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_TO_CHILDREN); data_layout.GetBlockEntries("/mld/multilevelpartition/cell_to_children");
util::vector_view<CellID> cell_to_children(mld_chilren_ptr, children_entries_count); util::vector_view<CellID> cell_to_children(mld_chilren_ptr, children_entries_count);
mld_partition = mld_partition =
partitioner::MultiLevelPartitionView{level_data, partition, cell_to_children}; partitioner::MultiLevelPartitionView{level_data, partition, cell_to_children};
} }
const auto weights_block_id = static_cast<storage::DataLayout::BlockID>( const auto weights_block_id = "/mld/metrics/" + std::to_string(exclude_index) + "/weights";
storage::DataLayout::MLD_CELL_WEIGHTS_0 + exclude_index); const auto durations_block_id =
const auto durations_block_id = static_cast<storage::DataLayout::BlockID>( "/mld/metrics/" + std::to_string(exclude_index) + "/durations";
storage::DataLayout::MLD_CELL_DURATIONS_0 + exclude_index);
if (data_layout.GetBlockSize(weights_block_id) > 0) if (data_layout.GetBlockSize(weights_block_id) > 0)
{ {
@ -1029,10 +1026,8 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
data_layout.GetBlockPtr<EdgeWeight>(memory_block, weights_block_id); data_layout.GetBlockPtr<EdgeWeight>(memory_block, weights_block_id);
auto mld_cell_durations_ptr = auto mld_cell_durations_ptr =
data_layout.GetBlockPtr<EdgeDuration>(memory_block, durations_block_id); data_layout.GetBlockPtr<EdgeDuration>(memory_block, durations_block_id);
auto weight_entries_count = auto weight_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/weights");
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_WEIGHTS_0); auto duration_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/durations");
auto duration_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_DURATIONS_0);
BOOST_ASSERT(weight_entries_count == duration_entries_count); BOOST_ASSERT(weight_entries_count == duration_entries_count);
util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count); util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
util::vector_view<EdgeDuration> durations(mld_cell_durations_ptr, util::vector_view<EdgeDuration> durations(mld_cell_durations_ptr,
@ -1041,25 +1036,25 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
mld_cell_metric = customizer::CellMetricView{std::move(weights), std::move(durations)}; mld_cell_metric = customizer::CellMetricView{std::move(weights), std::move(durations)};
} }
if (data_layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0) if (data_layout.GetBlockSize("/mld/cellstorage/cells") > 0)
{ {
auto mld_source_boundary_ptr = data_layout.GetBlockPtr<NodeID>( auto mld_source_boundary_ptr =
memory_block, storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY); data_layout.GetBlockPtr<NodeID>(memory_block, "/mld/cellstorage/source_boundary");
auto mld_destination_boundary_ptr = data_layout.GetBlockPtr<NodeID>( auto mld_destination_boundary_ptr = data_layout.GetBlockPtr<NodeID>(
memory_block, storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY); memory_block, "/mld/cellstorage/destination_boundary");
auto mld_cells_ptr = data_layout.GetBlockPtr<partitioner::CellStorageView::CellData>( auto mld_cells_ptr = data_layout.GetBlockPtr<partitioner::CellStorageView::CellData>(
memory_block, storage::DataLayout::MLD_CELLS); memory_block, "/mld/cellstorage/cells");
auto mld_cell_level_offsets_ptr = data_layout.GetBlockPtr<std::uint64_t>( auto mld_cell_level_offsets_ptr = data_layout.GetBlockPtr<std::uint64_t>(
memory_block, storage::DataLayout::MLD_CELL_LEVEL_OFFSETS); memory_block, "/mld/cellstorage/level_to_cell_offset");
auto source_boundary_entries_count = auto source_boundary_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY); data_layout.GetBlockEntries("/mld/cellstorage/source_boundary");
auto destination_boundary_entries_count = auto destination_boundary_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY); data_layout.GetBlockEntries("/mld/cellstorage/destination_boundary");
auto cells_entries_counts = data_layout.GetBlockEntries(storage::DataLayout::MLD_CELLS); auto cells_entries_counts = data_layout.GetBlockEntries("/mld/cellstorage/cells");
auto cell_level_offsets_entries_count = auto cell_level_offsets_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS); data_layout.GetBlockEntries("/mld/cellstorage/level_to_cell_offset");
util::vector_view<NodeID> source_boundary(mld_source_boundary_ptr, util::vector_view<NodeID> source_boundary(mld_source_boundary_ptr,
source_boundary_entries_count); source_boundary_entries_count);
@ -1076,24 +1071,24 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
std::move(level_offsets)}; std::move(level_offsets)};
} }
} }
void InitializeGraphPointer(storage::DataLayout &data_layout, char *memory_block) void InitializeGraphPointer(const storage::DataLayout &data_layout, char *memory_block)
{ {
auto graph_nodes_ptr = data_layout.GetBlockPtr<GraphNode>( auto graph_nodes_ptr =
memory_block, storage::DataLayout::MLD_GRAPH_NODE_LIST); data_layout.GetBlockPtr<GraphNode>(memory_block, "/mld/multilevelgraph/node_array");
auto graph_edges_ptr = data_layout.GetBlockPtr<GraphEdge>( auto graph_edges_ptr =
memory_block, storage::DataLayout::MLD_GRAPH_EDGE_LIST); data_layout.GetBlockPtr<GraphEdge>(memory_block, "/mld/multilevelgraph/edge_array");
auto graph_node_to_offset_ptr = data_layout.GetBlockPtr<QueryGraph::EdgeOffset>( auto graph_node_to_offset_ptr = data_layout.GetBlockPtr<QueryGraph::EdgeOffset>(
memory_block, storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET); memory_block, "/mld/multilevelgraph/node_to_edge_offset");
util::vector_view<GraphNode> node_list( util::vector_view<GraphNode> node_list(
graph_nodes_ptr, data_layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_NODE_LIST)); graph_nodes_ptr, data_layout.GetBlockEntries("/mld/multilevelgraph/node_array"));
util::vector_view<GraphEdge> edge_list( util::vector_view<GraphEdge> edge_list(
graph_edges_ptr, data_layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_EDGE_LIST)); graph_edges_ptr, data_layout.GetBlockEntries("/mld/multilevelgraph/edge_array"));
util::vector_view<QueryGraph::EdgeOffset> node_to_offset( util::vector_view<QueryGraph::EdgeOffset> node_to_offset(
graph_node_to_offset_ptr, graph_node_to_offset_ptr,
data_layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET)); data_layout.GetBlockEntries("/mld/multilevelgraph/node_to_edge_offset"));
query_graph = query_graph =
QueryGraph(std::move(node_list), std::move(edge_list), std::move(node_to_offset)); QueryGraph(std::move(node_list), std::move(edge_list), std::move(node_to_offset));

4
include/engine/datafacade/process_memory_allocator.hpp
View File

@ -27,12 +27,12 @@ class ProcessMemoryAllocator : public ContiguousBlockAllocator
~ProcessMemoryAllocator() override final; ~ProcessMemoryAllocator() override final;
// interface to give access to the datafacades // interface to give access to the datafacades
storage::DataLayout &GetLayout() override final; const storage::DataLayout &GetLayout() override final;
char *GetMemory() override final; char *GetMemory() override final;
private: private:
storage::DataLayout internal_layout;
std::unique_ptr<char[]> internal_memory; std::unique_ptr<char[]> internal_memory;
std::unique_ptr<storage::DataLayout> internal_layout;
}; };
} // namespace datafacade } // namespace datafacade

4
include/engine/datafacade/shared_memory_allocator.hpp
View File

@ -27,10 +27,12 @@ class SharedMemoryAllocator : public ContiguousBlockAllocator
~SharedMemoryAllocator() override final; ~SharedMemoryAllocator() override final;
// interface to give access to the datafacades // interface to give access to the datafacades
storage::DataLayout &GetLayout() override final; const storage::DataLayout &GetLayout() override final;
char *GetMemory() override final; char *GetMemory() override final;
private: private:
std::size_t layout_size;
storage::DataLayout data_layout;
std::unique_ptr<storage::SharedMemory> m_large_memory; std::unique_ptr<storage::SharedMemory> m_large_memory;
}; };

2
include/engine/datafacade_factory.hpp
View File

@ -51,7 +51,7 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
} }
properties = allocator->GetLayout().template GetBlockPtr<extractor::ProfileProperties>( properties = allocator->GetLayout().template GetBlockPtr<extractor::ProfileProperties>(
allocator->GetMemory(), storage::DataLayout::PROPERTIES); allocator->GetMemory(), "/common/properties");
for (const auto index : util::irange<std::size_t>(0, properties->class_names.size())) for (const auto index : util::irange<std::size_t>(0, properties->class_names.size()))
{ {

26
include/engine/engine.hpp
View File

@ -152,8 +152,8 @@ bool Engine<routing_algorithms::ch::Algorithm>::CheckCompatibility(const EngineC
auto mem = storage::makeSharedMemory(barrier.data().region); auto mem = storage::makeSharedMemory(barrier.data().region);
auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr()); auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr());
return layout->GetBlockSize(storage::DataLayout::CH_GRAPH_NODE_LIST) > 4 && return layout->GetBlockSize("/ch/contracted_graph/node_array") > 4 &&
layout->GetBlockSize(storage::DataLayout::CH_GRAPH_EDGE_LIST) > 4; layout->GetBlockSize("/ch/contracted_graph/edge_array") > 4;
} }
else else
{ {
@ -173,19 +173,19 @@ bool Engine<routing_algorithms::mld::Algorithm>::CheckCompatibility(const Engine
auto mem = storage::makeSharedMemory(barrier.data().region); auto mem = storage::makeSharedMemory(barrier.data().region);
auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr()); auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr());
// checks that all the needed memory blocks are populated // checks that all the needed memory blocks are populated
// DataLayout::MLD_CELL_SOURCE_BOUNDARY and DataLayout::MLD_CELL_DESTINATION_BOUNDARY // "/mld/cellstorage/source_boundary" and "/mld/cellstorage/destination_boundary"
// are not checked, because in situations where there are so few nodes in the graph that // are not checked, because in situations where there are so few nodes in the graph that
// they all fit into one cell, they can be empty. // they all fit into one cell, they can be empty.
bool empty_data = layout->GetBlockSize(storage::DataLayout::MLD_LEVEL_DATA) > 0 && bool empty_data = layout->GetBlockSize("/mld/multilevelpartition/level_data") > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_PARTITION) > 0 && layout->GetBlockSize("/mld/multilevelpartition/partition") > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_CELL_TO_CHILDREN) > 0 && layout->GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_CELLS) > 0 && layout->GetBlockSize("/mld/cellstorage/cells") > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS) > 0 && layout->GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_GRAPH_NODE_LIST) > 0 && layout->GetBlockSize("/mld/multilevelgraph/node_array") > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_GRAPH_EDGE_LIST) > 0 && layout->GetBlockSize("/mld/multilevelgraph/edge_array") > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_CELL_WEIGHTS_0) > 0 && layout->GetBlockSize("/mld/metrics/0/weights") > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_CELL_DURATIONS_0) > 0 && layout->GetBlockSize("/mld/metrics/0/durations") > 0 &&
layout->GetBlockSize(storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET) > 0; layout->GetBlockSize("/mld/multilevelgraph/node_to_edge_offset") > 0;
return empty_data; return empty_data;
} }
else else

4
include/storage/io.hpp
View File

@ -303,6 +303,10 @@ class BufferReader
} }
} }
std::size_t GetPosition() {
return input_stream.tellg();
}
template <typename T> void ReadInto(T *dest, const std::size_t count) template <typename T> void ReadInto(T *dest, const std::size_t count)
{ {
#if !defined(__GNUC__) || (__GNUC__ > 4) #if !defined(__GNUC__) || (__GNUC__ > 4)

64
include/storage/serialization.hpp
View File

@ -7,12 +7,14 @@
#include "storage/io.hpp" #include "storage/io.hpp"
#include "storage/tar.hpp" #include "storage/tar.hpp"
#include "storage/shared_datatype.hpp"
#include <boost/function_output_iterator.hpp> #include <boost/function_output_iterator.hpp>
#include <boost/iterator/function_input_iterator.hpp> #include <boost/iterator/function_input_iterator.hpp>
#include <cmath> #include <cmath>
#include <cstdint> #include <cstdint>
#include <tuple>
#if USE_STXXL_LIBRARY #if USE_STXXL_LIBRARY
#include <stxxl/vector> #include <stxxl/vector>
@ -81,6 +83,32 @@ template <typename T> void write(io::BufferWriter &writer, const std::vector<T>
writer.WriteFrom(data.data(), count); writer.WriteFrom(data.data(), count);
} }
template<typename T>
inline void write(io::BufferWriter &writer, const T &data)
{
writer.WriteFrom(data);
}
template<typename T>
inline void read(io::BufferReader &reader, T &data)
{
reader.ReadInto(data);
}
inline void write(io::BufferWriter &writer, const std::string &data)
{
const auto count = data.size();
writer.WriteElementCount64(count);
writer.WriteFrom(data.data(), count);
}
inline void read(io::BufferReader &reader, std::string &data)
{
const auto count = reader.ReadElementCount64();
data.resize(count);
reader.ReadInto(const_cast<char *>(data.data()), count);
}
inline void write(tar::FileWriter &writer, const std::string &name, const std::string &data) inline void write(tar::FileWriter &writer, const std::string &name, const std::string &data)
{ {
const auto count = data.size(); const auto count = data.size();
@ -218,6 +246,42 @@ write<bool>(tar::FileWriter &writer, const std::string &name, const std::vector<
{ {
detail::writeBoolVector(writer, name, data); detail::writeBoolVector(writer, name, data);
} }
template <typename K, typename V> void read(io::BufferReader &reader, std::map<K, V> &data)
{
const auto count = reader.ReadElementCount64();
std::pair<K, V> pair;
for (auto index : util::irange<std::size_t>(0, count))
{
(void) index;
read(reader, pair.first);
read(reader, pair.second);
data.insert(pair);
}
}
template <typename K, typename V> void write(io::BufferWriter &writer, const std::map<K, V> &data)
{
const auto count = data.size();
writer.WriteElementCount64(count);
for (const auto &pair : data)
{
write(writer, pair.first);
write(writer, pair.second);
}
}
inline void read(io::BufferReader &reader, DataLayout &layout)
{
read(reader, layout.blocks);
}
inline void write(io::BufferWriter &writer, const DataLayout &layout)
{
write(writer, layout.blocks);
}
} }
} }
} }

227
include/storage/shared_datatype.hpp
View File

@ -2,6 +2,7 @@
#define SHARED_DATA_TYPE_HPP #define SHARED_DATA_TYPE_HPP
#include "storage/block.hpp" #include "storage/block.hpp"
#include "storage/io_fwd.hpp"
#include "util/exception.hpp" #include "util/exception.hpp"
#include "util/exception_utils.hpp" #include "util/exception_utils.hpp"
@ -11,219 +12,91 @@
#include <array> #include <array>
#include <cstdint> #include <cstdint>
#include <map>
namespace osrm namespace osrm
{ {
namespace storage namespace storage
{ {
class DataLayout;
namespace serialization
{
inline void read(io::BufferReader &reader, DataLayout &layout);
inline void write(io::BufferWriter &writer, const DataLayout &layout);
}
// Added at the start and end of each block as sanity check // Added at the start and end of each block as sanity check
const constexpr char CANARY[4] = {'O', 'S', 'R', 'M'}; const constexpr char CANARY[4] = {'O', 'S', 'R', 'M'};
const constexpr char *block_id_to_name[] = {"IGNORE_BLOCK",
"NAME_BLOCKS",
"NAME_VALUES",
"EDGE_BASED_NODE_DATA",
"ANNOTATION_DATA",
"CH_GRAPH_NODE_LIST",
"CH_GRAPH_EDGE_LIST",
"CH_EDGE_FILTER_0",
"CH_EDGE_FILTER_1",
"CH_EDGE_FILTER_2",
"CH_EDGE_FILTER_3",
"CH_EDGE_FILTER_4",
"CH_EDGE_FILTER_5",
"CH_EDGE_FILTER_6",
"CH_EDGE_FILTER_7",
"COORDINATE_LIST",
"OSM_NODE_ID_LIST",
"TURN_INSTRUCTION",
"ENTRY_CLASSID",
"R_SEARCH_TREE",
"R_SEARCH_TREE_LEVEL_STARTS",
"GEOMETRIES_INDEX",
"GEOMETRIES_NODE_LIST",
"GEOMETRIES_FWD_WEIGHT_LIST",
"GEOMETRIES_REV_WEIGHT_LIST",
"GEOMETRIES_FWD_DURATION_LIST",
"GEOMETRIES_REV_DURATION_LIST",
"GEOMETRIES_FWD_DATASOURCES_LIST",
"GEOMETRIES_REV_DATASOURCES_LIST",
"HSGR_CHECKSUM",
"FILE_INDEX_PATH",
"DATASOURCES_NAMES",
"PROPERTIES",
"BEARING_CLASSID",
"BEARING_OFFSETS",
"BEARING_BLOCKS",
"BEARING_VALUES",
"ENTRY_CLASS",
"LANE_DATA_ID",
"PRE_TURN_BEARING",
"POST_TURN_BEARING",
"TURN_LANE_DATA",
"LANE_DESCRIPTION_OFFSETS",
"LANE_DESCRIPTION_MASKS",
"TURN_WEIGHT_PENALTIES",
"TURN_DURATION_PENALTIES",
"MLD_LEVEL_DATA",
"MLD_PARTITION",
"MLD_CELL_TO_CHILDREN",
"MLD_CELL_WEIGHTS_0",
"MLD_CELL_WEIGHTS_1",
"MLD_CELL_WEIGHTS_2",
"MLD_CELL_WEIGHTS_3",
"MLD_CELL_WEIGHTS_4",
"MLD_CELL_WEIGHTS_5",
"MLD_CELL_WEIGHTS_6",
"MLD_CELL_WEIGHTS_7",
"MLD_CELL_DURATIONS_0",
"MLD_CELL_DURATIONS_1",
"MLD_CELL_DURATIONS_2",
"MLD_CELL_DURATIONS_3",
"MLD_CELL_DURATIONS_4",
"MLD_CELL_DURATIONS_5",
"MLD_CELL_DURATIONS_6",
"MLD_CELL_DURATIONS_7",
"MLD_CELL_SOURCE_BOUNDARY",
"MLD_CELL_DESTINATION_BOUNDARY",
"MLD_CELLS",
"MLD_CELL_LEVEL_OFFSETS",
"MLD_GRAPH_NODE_LIST",
"MLD_GRAPH_EDGE_LIST",
"MLD_GRAPH_NODE_TO_OFFSET",
"MANEUVER_OVERRIDES",
"MANEUVER_OVERRIDE_NODE_SEQUENCES"};
class DataLayout class DataLayout
{ {
public: public:
enum BlockID
{
IGNORE_BLOCK = 0,
NAME_BLOCKS,
NAME_VALUES,
EDGE_BASED_NODE_DATA_LIST,
ANNOTATION_DATA_LIST,
CH_GRAPH_NODE_LIST,
CH_GRAPH_EDGE_LIST,
CH_EDGE_FILTER_0,
CH_EDGE_FILTER_1,
CH_EDGE_FILTER_2,
CH_EDGE_FILTER_3,
CH_EDGE_FILTER_4,
CH_EDGE_FILTER_5,
CH_EDGE_FILTER_6,
CH_EDGE_FILTER_7,
COORDINATE_LIST,
OSM_NODE_ID_LIST,
TURN_INSTRUCTION,
ENTRY_CLASSID,
R_SEARCH_TREE,
R_SEARCH_TREE_LEVEL_STARTS,
GEOMETRIES_INDEX,
GEOMETRIES_NODE_LIST,
GEOMETRIES_FWD_WEIGHT_LIST,
GEOMETRIES_REV_WEIGHT_LIST,
GEOMETRIES_FWD_DURATION_LIST,
GEOMETRIES_REV_DURATION_LIST,
GEOMETRIES_FWD_DATASOURCES_LIST,
GEOMETRIES_REV_DATASOURCES_LIST,
HSGR_CHECKSUM,
FILE_INDEX_PATH,
DATASOURCES_NAMES,
PROPERTIES,
BEARING_CLASSID,
BEARING_OFFSETS,
BEARING_BLOCKS,
BEARING_VALUES,
ENTRY_CLASS,
LANE_DATA_ID,
PRE_TURN_BEARING,
POST_TURN_BEARING,
TURN_LANE_DATA,
LANE_DESCRIPTION_OFFSETS,
LANE_DESCRIPTION_MASKS,
TURN_WEIGHT_PENALTIES,
TURN_DURATION_PENALTIES,
MLD_LEVEL_DATA,
MLD_PARTITION,
MLD_CELL_TO_CHILDREN,
MLD_CELL_WEIGHTS_0,
MLD_CELL_WEIGHTS_1,
MLD_CELL_WEIGHTS_2,
MLD_CELL_WEIGHTS_3,
MLD_CELL_WEIGHTS_4,
MLD_CELL_WEIGHTS_5,
MLD_CELL_WEIGHTS_6,
MLD_CELL_WEIGHTS_7,
MLD_CELL_DURATIONS_0,
MLD_CELL_DURATIONS_1,
MLD_CELL_DURATIONS_2,
MLD_CELL_DURATIONS_3,
MLD_CELL_DURATIONS_4,
MLD_CELL_DURATIONS_5,
MLD_CELL_DURATIONS_6,
MLD_CELL_DURATIONS_7,
MLD_CELL_SOURCE_BOUNDARY,
MLD_CELL_DESTINATION_BOUNDARY,
MLD_CELLS,
MLD_CELL_LEVEL_OFFSETS,
MLD_GRAPH_NODE_LIST,
MLD_GRAPH_EDGE_LIST,
MLD_GRAPH_NODE_TO_OFFSET,
MANEUVER_OVERRIDES,
MANEUVER_OVERRIDE_NODE_SEQUENCES,
NUM_BLOCKS
};
DataLayout() : blocks{} {} DataLayout() : blocks{} {}
inline void SetBlock(BlockID bid, Block block) { blocks[bid] = std::move(block); } inline void SetBlock(const std::string &name, Block block) { blocks[name] = std::move(block); }
inline uint64_t GetBlockEntries(BlockID bid) const { return blocks[bid].num_entries; } inline uint64_t GetBlockEntries(const std::string &name) const
{
auto iter = blocks.find(name);
if (iter == blocks.end())
{
throw util::exception("Could not find block " + name);
}
inline uint64_t GetBlockSize(BlockID bid) const { return blocks[bid].byte_size; } return iter->second.num_entries;
}
inline uint64_t GetBlockSize(const std::string &name) const
{
auto iter = blocks.find(name);
if (iter == blocks.end())
{
throw util::exception("Could not find block " + name);
}
return iter->second.byte_size;
}
inline uint64_t GetSizeOfLayout() const inline uint64_t GetSizeOfLayout() const
{ {
uint64_t result = 0; uint64_t result = 0;
for (auto i = 0; i < NUM_BLOCKS; i++) for (const auto &name_and_block : blocks)
{ {
result += 2 * sizeof(CANARY) + GetBlockSize(static_cast<BlockID>(i)) + BLOCK_ALIGNMENT; result += 2 * sizeof(CANARY) + GetBlockSize(name_and_block.first) + BLOCK_ALIGNMENT;
} }
return result; return result;
} }
template <typename T, bool WRITE_CANARY = false> template <typename T, bool WRITE_CANARY = false>
inline T *GetBlockPtr(char *shared_memory, BlockID bid) const inline T *GetBlockPtr(char *shared_memory, const std::string &name) const
{ {
static_assert(BLOCK_ALIGNMENT % std::alignment_of<T>::value == 0, static_assert(BLOCK_ALIGNMENT % std::alignment_of<T>::value == 0,
"Datatype does not fit alignment constraints."); "Datatype does not fit alignment constraints.");
char *ptr = (char *)GetAlignedBlockPtr(shared_memory, bid); char *ptr = (char *)GetAlignedBlockPtr(shared_memory, name);
if (WRITE_CANARY) if (WRITE_CANARY)
{ {
char *start_canary_ptr = ptr - sizeof(CANARY); char *start_canary_ptr = ptr - sizeof(CANARY);
char *end_canary_ptr = ptr + GetBlockSize(bid); char *end_canary_ptr = ptr + GetBlockSize(name);
std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr); std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr); std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
} }
else else
{ {
char *start_canary_ptr = ptr - sizeof(CANARY); char *start_canary_ptr = ptr - sizeof(CANARY);
char *end_canary_ptr = ptr + GetBlockSize(bid); char *end_canary_ptr = ptr + GetBlockSize(name);
bool start_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), start_canary_ptr); bool start_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
bool end_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), end_canary_ptr); bool end_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
if (!start_canary_alive) if (!start_canary_alive)
{ {
throw util::exception("Start canary of block corrupted. (" + throw util::exception("Start canary of block corrupted. (" + name + ")" +
std::string(block_id_to_name[bid]) + ")" + SOURCE_REF); SOURCE_REF);
} }
if (!end_canary_alive) if (!end_canary_alive)
{ {
throw util::exception("End canary of block corrupted. (" + throw util::exception("End canary of block corrupted. (" + name + ")" + SOURCE_REF);
std::string(block_id_to_name[bid]) + ")" + SOURCE_REF);
} }
} }
@ -231,6 +104,9 @@ class DataLayout
} }
private: private:
friend void serialization::read(io::BufferReader &reader, DataLayout &layout);
friend void serialization::write(io::BufferWriter &writer, const DataLayout &layout);
// Fit aligned storage in buffer to 64 bytes to conform with AVX 512 types // Fit aligned storage in buffer to 64 bytes to conform with AVX 512 types
inline void *align(void *&ptr) const noexcept inline void *align(void *&ptr) const noexcept
{ {
@ -239,13 +115,13 @@ class DataLayout
return ptr = reinterpret_cast<void *>(aligned); return ptr = reinterpret_cast<void *>(aligned);
} }
inline void *GetAlignedBlockPtr(void *ptr, BlockID bid) const inline void *GetAlignedBlockPtr(void *ptr, const std::string &name) const
{ {
for (auto i = 0; i < bid; i++) for (auto iter = blocks.begin(); iter != blocks.end() && iter->first != name; ++iter)
{ {
ptr = static_cast<char *>(ptr) + sizeof(CANARY); ptr = static_cast<char *>(ptr) + sizeof(CANARY);
ptr = align(ptr); ptr = align(ptr);
ptr = static_cast<char *>(ptr) + GetBlockSize((BlockID)i); ptr = static_cast<char *>(ptr) + GetBlockSize(name);
ptr = static_cast<char *>(ptr) + sizeof(CANARY); ptr = static_cast<char *>(ptr) + sizeof(CANARY);
} }
@ -254,14 +130,14 @@ class DataLayout
return ptr; return ptr;
} }
template <typename T> inline T *GetBlockEnd(char *shared_memory, BlockID bid) const template <typename T> inline T *GetBlockEnd(char *shared_memory, const std::string &name) const
{ {
auto begin = GetBlockPtr<T>(shared_memory, bid); auto begin = GetBlockPtr<T>(shared_memory, name);
return begin + GetBlockEntries(bid); return begin + GetBlockEntries(name);
} }
static constexpr std::size_t BLOCK_ALIGNMENT = 64; static constexpr std::size_t BLOCK_ALIGNMENT = 64;
std::array<Block, NUM_BLOCKS> blocks; std::map<std::string, Block> blocks;
}; };
enum SharedDataType enum SharedDataType
@ -298,9 +174,6 @@ inline std::string regionToString(const SharedDataType region)
return "INVALID_REGION"; return "INVALID_REGION";
} }
} }
static_assert(sizeof(block_id_to_name) / sizeof(*block_id_to_name) == DataLayout::NUM_BLOCKS,
"Number of blocks needs to match the number of Block names.");
} }
} }

9
src/engine/datafacade/process_memory_allocator.cpp
View File

@ -15,17 +15,16 @@ ProcessMemoryAllocator::ProcessMemoryAllocator(const storage::StorageConfig &con
storage::Storage storage(config); storage::Storage storage(config);
// Calculate the layout/size of the memory block // Calculate the layout/size of the memory block
internal_layout = std::make_unique<storage::DataLayout>(); storage.PopulateLayout(internal_layout);
storage.PopulateLayout(*internal_layout);
// Allocate the memory block, then load data from files into it // Allocate the memory block, then load data from files into it
internal_memory = std::make_unique<char[]>(internal_layout->GetSizeOfLayout()); internal_memory = std::make_unique<char[]>(internal_layout.GetSizeOfLayout());
storage.PopulateData(*internal_layout, internal_memory.get()); storage.PopulateData(internal_layout, internal_memory.get());
} }
ProcessMemoryAllocator::~ProcessMemoryAllocator() {} ProcessMemoryAllocator::~ProcessMemoryAllocator() {}
storage::DataLayout &ProcessMemoryAllocator::GetLayout() { return *internal_layout.get(); } const storage::DataLayout &ProcessMemoryAllocator::GetLayout() { return internal_layout; }
char *ProcessMemoryAllocator::GetMemory() { return internal_memory.get(); } char *ProcessMemoryAllocator::GetMemory() { return internal_memory.get(); }
} // namespace datafacade } // namespace datafacade

13
src/engine/datafacade/shared_memory_allocator.cpp
View File

@ -1,4 +1,7 @@
#include "engine/datafacade/shared_memory_allocator.hpp" #include "engine/datafacade/shared_memory_allocator.hpp"
#include "storage/serialization.hpp"
#include "util/log.hpp" #include "util/log.hpp"
#include "boost/assert.hpp" #include "boost/assert.hpp"
@ -16,17 +19,21 @@ SharedMemoryAllocator::SharedMemoryAllocator(storage::SharedDataType data_region
BOOST_ASSERT(storage::SharedMemory::RegionExists(data_region)); BOOST_ASSERT(storage::SharedMemory::RegionExists(data_region));
m_large_memory = storage::makeSharedMemory(data_region); m_large_memory = storage::makeSharedMemory(data_region);
storage::io::BufferReader reader(GetMemory());
storage::serialization::read(reader, data_layout);
layout_size = reader.GetPosition();
} }
SharedMemoryAllocator::~SharedMemoryAllocator() {} SharedMemoryAllocator::~SharedMemoryAllocator() {}
storage::DataLayout &SharedMemoryAllocator::GetLayout() const storage::DataLayout &SharedMemoryAllocator::GetLayout()
{ {
return *reinterpret_cast<storage::DataLayout *>(m_large_memory->Ptr()); return data_layout;
} }
char *SharedMemoryAllocator::GetMemory() char *SharedMemoryAllocator::GetMemory()
{ {
return reinterpret_cast<char *>(m_large_memory->Ptr()) + sizeof(storage::DataLayout); return reinterpret_cast<char *>(m_large_memory->Ptr()) + layout_size;
} }
} // namespace datafacade } // namespace datafacade

2
src/osrm/osrm.cpp
View File

@ -35,7 +35,7 @@ OSRM::OSRM(engine::EngineConfig &config)
auto mem = storage::makeSharedMemory(barrier.data().region); auto mem = storage::makeSharedMemory(barrier.data().region);
auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr()); auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr());
if (layout->GetBlockSize(storage::DataLayout::NAME_VALUES) == 0) if (layout->GetBlockSize("/common/names/values") == 0)
throw util::exception( throw util::exception(
"No name data loaded, cannot continue. Have you run osrm-datastore to load data?"); "No name data loaded, cannot continue. Have you run osrm-datastore to load data?");
} }

357
src/storage/storage.cpp
View File

@ -70,7 +70,7 @@ namespace storage
{ {
namespace namespace
{ {
template <typename OutIter> void readBlocks(const boost::filesystem::path &path, OutIter out) inline void readBlocks(const boost::filesystem::path &path, DataLayout &layout)
{ {
tar::FileReader reader(path, tar::FileReader::VerifyFingerprint); tar::FileReader reader(path, tar::FileReader::VerifyFingerprint);
@ -83,7 +83,7 @@ template <typename OutIter> void readBlocks(const boost::filesystem::path &path,
if (name_end == std::string::npos) if (name_end == std::string::npos)
{ {
auto number_of_elements = reader.ReadElementCount64(entry.name); auto number_of_elements = reader.ReadElementCount64(entry.name);
*out++ = NamedBlock{entry.name, Block{number_of_elements, entry.size}}; layout.SetBlock(entry.name, Block{number_of_elements, entry.size});
} }
} }
} }
@ -159,15 +159,19 @@ int Storage::Run(int max_wait)
DataLayout layout; DataLayout layout;
PopulateLayout(layout); PopulateLayout(layout);
io::BufferWriter writer;
serialization::write(writer, layout);
auto encoded_layout = writer.GetBuffer();
// Allocate shared memory block // Allocate shared memory block
auto regions_size = sizeof(layout) + layout.GetSizeOfLayout(); auto regions_size = encoded_layout.size() + layout.GetSizeOfLayout();
util::Log() << "Allocating shared memory of " << regions_size << " bytes"; util::Log() << "Allocating shared memory of " << regions_size << " bytes";
auto data_memory = makeSharedMemory(next_region, regions_size); auto data_memory = makeSharedMemory(next_region, regions_size);
// Copy memory layout to shared memory and populate data // Copy memory layout to shared memory and populate data
char *shared_memory_ptr = static_cast<char *>(data_memory->Ptr()); char *shared_memory_ptr = static_cast<char *>(data_memory->Ptr());
memcpy(shared_memory_ptr, &layout, sizeof(layout)); memcpy(shared_memory_ptr, encoded_layout.data(), encoded_layout.size());
PopulateData(layout, shared_memory_ptr + sizeof(layout)); PopulateData(layout, shared_memory_ptr + encoded_layout.size());
{ // Lock for write access shared region mutex { // Lock for write access shared region mutex
boost::interprocess::scoped_lock<Monitor::mutex_type> lock(monitor.get_mutex(), boost::interprocess::scoped_lock<Monitor::mutex_type> lock(monitor.get_mutex(),
@ -236,91 +240,10 @@ void Storage::PopulateLayout(DataLayout &layout)
auto absolute_file_index_path = auto absolute_file_index_path =
boost::filesystem::absolute(config.GetPath(".osrm.fileIndex")); boost::filesystem::absolute(config.GetPath(".osrm.fileIndex"));
layout.SetBlock(DataLayout::FILE_INDEX_PATH, layout.SetBlock("/common/rtree/file_index_path",
make_block<char>(absolute_file_index_path.string().length() + 1)); make_block<char>(absolute_file_index_path.string().length() + 1));
} }
std::unordered_map<std::string, DataLayout::BlockID> name_to_block_id = {
{"/mld/multilevelgraph/node_array", DataLayout::MLD_GRAPH_NODE_LIST},
{"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST},
{"/mld/multilevelgraph/node_to_edge_offset", DataLayout::MLD_GRAPH_NODE_TO_OFFSET},
{"/mld/multilevelgraph/connectivity_checksum", DataLayout::IGNORE_BLOCK},
{"/mld/multilevelpartition/level_data", DataLayout::MLD_LEVEL_DATA},
{"/mld/multilevelpartition/partition", DataLayout::MLD_PARTITION},
{"/mld/multilevelpartition/cell_to_children", DataLayout::MLD_CELL_TO_CHILDREN},
{"/mld/cellstorage/source_boundary", DataLayout::MLD_CELL_SOURCE_BOUNDARY},
{"/mld/cellstorage/destination_boundary", DataLayout::MLD_CELL_DESTINATION_BOUNDARY},
{"/mld/cellstorage/cells", DataLayout::MLD_CELLS},
{"/mld/cellstorage/level_to_cell_offset", DataLayout::MLD_CELL_LEVEL_OFFSETS},
{"/mld/metrics/0/weights", DataLayout::MLD_CELL_WEIGHTS_0},
{"/mld/metrics/1/weights", DataLayout::MLD_CELL_WEIGHTS_1},
{"/mld/metrics/2/weights", DataLayout::MLD_CELL_WEIGHTS_2},
{"/mld/metrics/3/weights", DataLayout::MLD_CELL_WEIGHTS_3},
{"/mld/metrics/4/weights", DataLayout::MLD_CELL_WEIGHTS_4},
{"/mld/metrics/5/weights", DataLayout::MLD_CELL_WEIGHTS_5},
{"/mld/metrics/6/weights", DataLayout::MLD_CELL_WEIGHTS_6},
{"/mld/metrics/7/weights", DataLayout::MLD_CELL_WEIGHTS_7},
{"/mld/metrics/0/durations", DataLayout::MLD_CELL_DURATIONS_0},
{"/mld/metrics/1/durations", DataLayout::MLD_CELL_DURATIONS_1},
{"/mld/metrics/2/durations", DataLayout::MLD_CELL_DURATIONS_2},
{"/mld/metrics/3/durations", DataLayout::MLD_CELL_DURATIONS_3},
{"/mld/metrics/4/durations", DataLayout::MLD_CELL_DURATIONS_4},
{"/mld/metrics/5/durations", DataLayout::MLD_CELL_DURATIONS_5},
{"/mld/metrics/6/durations", DataLayout::MLD_CELL_DURATIONS_6},
{"/mld/metrics/7/durations", DataLayout::MLD_CELL_DURATIONS_7},
{"/ch/checksum", DataLayout::HSGR_CHECKSUM},
{"/ch/contracted_graph/node_array", DataLayout::CH_GRAPH_NODE_LIST},
{"/ch/contracted_graph/edge_array", DataLayout::CH_GRAPH_EDGE_LIST},
{"/ch/connectivity_checksum", DataLayout::IGNORE_BLOCK},
{"/ch/edge_filter/0", DataLayout::CH_EDGE_FILTER_0},
{"/ch/edge_filter/1", DataLayout::CH_EDGE_FILTER_1},
{"/ch/edge_filter/2", DataLayout::CH_EDGE_FILTER_2},
{"/ch/edge_filter/3", DataLayout::CH_EDGE_FILTER_3},
{"/ch/edge_filter/4", DataLayout::CH_EDGE_FILTER_4},
{"/ch/edge_filter/5", DataLayout::CH_EDGE_FILTER_5},
{"/ch/edge_filter/6", DataLayout::CH_EDGE_FILTER_6},
{"/ch/edge_filter/7", DataLayout::CH_EDGE_FILTER_7},
{"/common/intersection_bearings/bearing_values", DataLayout::BEARING_VALUES},
{"/common/intersection_bearings/node_to_class_id", DataLayout::BEARING_CLASSID},
{"/common/intersection_bearings/class_id_to_ranges/block_offsets",
DataLayout::BEARING_OFFSETS},
{"/common/intersection_bearings/class_id_to_ranges/diff_blocks",
DataLayout::BEARING_BLOCKS},
{"/common/entry_classes", DataLayout::ENTRY_CLASS},
{"/common/properties", DataLayout::PROPERTIES},
{"/common/coordinates", DataLayout::COORDINATE_LIST},
{"/common/osm_node_ids/packed", DataLayout::OSM_NODE_ID_LIST},
{"/common/data_sources_names", DataLayout::DATASOURCES_NAMES},
{"/common/segment_data/index", DataLayout::GEOMETRIES_INDEX},
{"/common/segment_data/nodes", DataLayout::GEOMETRIES_NODE_LIST},
{"/common/segment_data/forward_weights/packed", DataLayout::GEOMETRIES_FWD_WEIGHT_LIST},
{"/common/segment_data/reverse_weights/packed", DataLayout::GEOMETRIES_REV_WEIGHT_LIST},
{"/common/segment_data/forward_durations/packed", DataLayout::GEOMETRIES_FWD_DURATION_LIST},
{"/common/segment_data/reverse_durations/packed", DataLayout::GEOMETRIES_REV_DURATION_LIST},
{"/common/segment_data/forward_data_sources", DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST},
{"/common/segment_data/reverse_data_sources", DataLayout::GEOMETRIES_REV_DATASOURCES_LIST},
{"/common/ebg_node_data/nodes", DataLayout::EDGE_BASED_NODE_DATA_LIST},
{"/common/ebg_node_data/annotations", DataLayout::ANNOTATION_DATA_LIST},
{"/common/turn_lanes/offsets", DataLayout::LANE_DESCRIPTION_OFFSETS},
{"/common/turn_lanes/masks", DataLayout::LANE_DESCRIPTION_MASKS},
{"/common/turn_lanes/data", DataLayout::TURN_LANE_DATA},
{"/common/maneuver_overrides/overrides", DataLayout::MANEUVER_OVERRIDES},
{"/common/maneuver_overrides/node_sequences", DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES},
{"/common/turn_penalty/weight", DataLayout::TURN_WEIGHT_PENALTIES},
{"/common/turn_penalty/duration", DataLayout::TURN_DURATION_PENALTIES},
{"/common/turn_data/pre_turn_bearings", DataLayout::PRE_TURN_BEARING},
{"/common/turn_data/post_turn_bearings", DataLayout::POST_TURN_BEARING},
{"/common/turn_data/turn_instructions", DataLayout::TURN_INSTRUCTION},
{"/common/turn_data/lane_data_ids", DataLayout::LANE_DATA_ID},
{"/common/turn_data/entry_class_ids", DataLayout::ENTRY_CLASSID},
{"/common/turn_data/connectivity_checksum", DataLayout::IGNORE_BLOCK},
{"/common/names/blocks", DataLayout::NAME_BLOCKS},
{"/common/names/values", DataLayout::NAME_VALUES},
{"/common/rtree/search_tree", DataLayout::R_SEARCH_TREE},
{"/common/rtree/search_tree_level_starts", DataLayout::R_SEARCH_TREE_LEVEL_STARTS},
};
std::vector<NamedBlock> blocks;
constexpr bool REQUIRED = true; constexpr bool REQUIRED = true;
constexpr bool OPTIONAL = false; constexpr bool OPTIONAL = false;
std::vector<std::pair<bool, boost::filesystem::path>> tar_files = { std::vector<std::pair<bool, boost::filesystem::path>> tar_files = {
@ -349,7 +272,7 @@ void Storage::PopulateLayout(DataLayout &layout)
{ {
if (boost::filesystem::exists(file.second)) if (boost::filesystem::exists(file.second))
{ {
readBlocks(file.second, std::back_inserter(blocks)); readBlocks(file.second, layout);
} }
else else
{ {
@ -360,17 +283,6 @@ void Storage::PopulateLayout(DataLayout &layout)
} }
} }
} }
for (const auto &block : blocks)
{
auto id_iter = name_to_block_id.find(std::get<0>(block));
if (id_iter == name_to_block_id.end())
{
throw util::exception("Could not map " + std::get<0>(block) +
" to a region in memory.");
}
layout.SetBlock(id_iter->second, std::get<1>(block));
}
} }
void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
@ -385,14 +297,14 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// store the filename of the on-disk portion of the RTree // store the filename of the on-disk portion of the RTree
{ {
const auto file_index_path_ptr = const auto file_index_path_ptr =
layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::FILE_INDEX_PATH); layout.GetBlockPtr<char, true>(memory_ptr, "/common/rtree/file_index_path");
// make sure we have 0 ending // make sure we have 0 ending
std::fill(file_index_path_ptr, std::fill(file_index_path_ptr,
file_index_path_ptr + layout.GetBlockSize(DataLayout::FILE_INDEX_PATH), file_index_path_ptr + layout.GetBlockSize("/common/rtree/file_index_path"),
0); 0);
const auto absolute_file_index_path = const auto absolute_file_index_path =
boost::filesystem::absolute(config.GetPath(".osrm.fileIndex")).string(); boost::filesystem::absolute(config.GetPath(".osrm.fileIndex")).string();
BOOST_ASSERT(static_cast<std::size_t>(layout.GetBlockSize(DataLayout::FILE_INDEX_PATH)) >= BOOST_ASSERT(static_cast<std::size_t>(layout.GetBlockSize("/common/rtree/file_index_path")) >=
absolute_file_index_path.size()); absolute_file_index_path.size());
std::copy( std::copy(
absolute_file_index_path.begin(), absolute_file_index_path.end(), file_index_path_ptr); absolute_file_index_path.begin(), absolute_file_index_path.end(), file_index_path_ptr);
@ -402,15 +314,15 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
{ {
const auto name_blocks_ptr = const auto name_blocks_ptr =
layout.GetBlockPtr<extractor::NameTableView::IndexedData::BlockReference, true>( layout.GetBlockPtr<extractor::NameTableView::IndexedData::BlockReference, true>(
memory_ptr, DataLayout::NAME_BLOCKS); memory_ptr, "/common/names/blocks");
const auto name_values_ptr = const auto name_values_ptr =
layout.GetBlockPtr<extractor::NameTableView::IndexedData::ValueType, true>( layout.GetBlockPtr<extractor::NameTableView::IndexedData::ValueType, true>(
memory_ptr, DataLayout::NAME_VALUES); memory_ptr, "/common/names/values");
util::vector_view<extractor::NameTableView::IndexedData::BlockReference> blocks( util::vector_view<extractor::NameTableView::IndexedData::BlockReference> blocks(
name_blocks_ptr, layout.GetBlockEntries(storage::DataLayout::NAME_BLOCKS)); name_blocks_ptr, layout.GetBlockEntries("/common/names/blocks"));
util::vector_view<extractor::NameTableView::IndexedData::ValueType> values( util::vector_view<extractor::NameTableView::IndexedData::ValueType> values(
name_values_ptr, layout.GetBlockEntries(storage::DataLayout::NAME_VALUES)); name_values_ptr, layout.GetBlockEntries("/common/names/values"));
extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)}; extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
extractor::NameTableView name_table{index_data_view}; extractor::NameTableView name_table{index_data_view};
@ -420,24 +332,24 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Turn lane data // Turn lane data
{ {
const auto turn_lane_data_ptr = layout.GetBlockPtr<util::guidance::LaneTupleIdPair, true>( const auto turn_lane_data_ptr = layout.GetBlockPtr<util::guidance::LaneTupleIdPair, true>(
memory_ptr, DataLayout::TURN_LANE_DATA); memory_ptr, "/common/turn_lanes/data");
util::vector_view<util::guidance::LaneTupleIdPair> turn_lane_data( util::vector_view<util::guidance::LaneTupleIdPair> turn_lane_data(
turn_lane_data_ptr, layout.GetBlockEntries(storage::DataLayout::TURN_LANE_DATA)); turn_lane_data_ptr, layout.GetBlockEntries("/common/turn_lanes/data"));
extractor::files::readTurnLaneData(config.GetPath(".osrm.tld"), turn_lane_data); extractor::files::readTurnLaneData(config.GetPath(".osrm.tld"), turn_lane_data);
} }
// Turn lane descriptions // Turn lane descriptions
{ {
auto offsets_ptr = layout.GetBlockPtr<std::uint32_t, true>( auto offsets_ptr =
memory_ptr, storage::DataLayout::LANE_DESCRIPTION_OFFSETS); layout.GetBlockPtr<std::uint32_t, true>(memory_ptr, "/common/turn_lanes/offsets");
util::vector_view<std::uint32_t> offsets( util::vector_view<std::uint32_t> offsets(
offsets_ptr, layout.GetBlockEntries(storage::DataLayout::LANE_DESCRIPTION_OFFSETS)); offsets_ptr, layout.GetBlockEntries("/common/turn_lanes/offsets"));
auto masks_ptr = layout.GetBlockPtr<extractor::TurnLaneType::Mask, true>( auto masks_ptr = layout.GetBlockPtr<extractor::TurnLaneType::Mask, true>(
memory_ptr, storage::DataLayout::LANE_DESCRIPTION_MASKS); memory_ptr, "/common/turn_lanes/masks");
util::vector_view<extractor::TurnLaneType::Mask> masks( util::vector_view<extractor::TurnLaneType::Mask> masks(
masks_ptr, layout.GetBlockEntries(storage::DataLayout::LANE_DESCRIPTION_MASKS)); masks_ptr, layout.GetBlockEntries("/common/turn_lanes/masks"));
extractor::files::readTurnLaneDescriptions(config.GetPath(".osrm.tls"), offsets, masks); extractor::files::readTurnLaneDescriptions(config.GetPath(".osrm.tls"), offsets, masks);
} }
@ -445,17 +357,15 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Load edge-based nodes data // Load edge-based nodes data
{ {
auto edge_based_node_data_list_ptr = layout.GetBlockPtr<extractor::EdgeBasedNode, true>( auto edge_based_node_data_list_ptr = layout.GetBlockPtr<extractor::EdgeBasedNode, true>(
memory_ptr, storage::DataLayout::EDGE_BASED_NODE_DATA_LIST); memory_ptr, "/common/ebg_node_data/nodes");
util::vector_view<extractor::EdgeBasedNode> edge_based_node_data( util::vector_view<extractor::EdgeBasedNode> edge_based_node_data(
edge_based_node_data_list_ptr, edge_based_node_data_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/nodes"));
layout.GetBlockEntries(storage::DataLayout::EDGE_BASED_NODE_DATA_LIST));
auto annotation_data_list_ptr = auto annotation_data_list_ptr =
layout.GetBlockPtr<extractor::NodeBasedEdgeAnnotation, true>( layout.GetBlockPtr<extractor::NodeBasedEdgeAnnotation, true>(
memory_ptr, storage::DataLayout::ANNOTATION_DATA_LIST); memory_ptr, "/common/ebg_node_data/annotations");
util::vector_view<extractor::NodeBasedEdgeAnnotation> annotation_data( util::vector_view<extractor::NodeBasedEdgeAnnotation> annotation_data(
annotation_data_list_ptr, annotation_data_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/annotations"));
layout.GetBlockEntries(storage::DataLayout::ANNOTATION_DATA_LIST));
extractor::EdgeBasedNodeDataView node_data(std::move(edge_based_node_data), extractor::EdgeBasedNodeDataView node_data(std::move(edge_based_node_data),
std::move(annotation_data)); std::move(annotation_data));
@ -466,30 +376,30 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Load original edge data // Load original edge data
{ {
const auto lane_data_id_ptr = const auto lane_data_id_ptr =
layout.GetBlockPtr<LaneDataID, true>(memory_ptr, storage::DataLayout::LANE_DATA_ID); layout.GetBlockPtr<LaneDataID, true>(memory_ptr, "/common/turn_data/lane_data_ids");
util::vector_view<LaneDataID> lane_data_ids( util::vector_view<LaneDataID> lane_data_ids(
lane_data_id_ptr, layout.GetBlockEntries(storage::DataLayout::LANE_DATA_ID)); lane_data_id_ptr, layout.GetBlockEntries("/common/turn_data/lane_data_ids"));
const auto turn_instruction_list_ptr = layout.GetBlockPtr<guidance::TurnInstruction, true>( const auto turn_instruction_list_ptr = layout.GetBlockPtr<guidance::TurnInstruction, true>(
memory_ptr, storage::DataLayout::TURN_INSTRUCTION); memory_ptr, "/common/turn_data/turn_instructions");
util::vector_view<guidance::TurnInstruction> turn_instructions( util::vector_view<guidance::TurnInstruction> turn_instructions(
turn_instruction_list_ptr, turn_instruction_list_ptr,
layout.GetBlockEntries(storage::DataLayout::TURN_INSTRUCTION)); layout.GetBlockEntries("/common/turn_data/turn_instructions"));
const auto entry_class_id_list_ptr = const auto entry_class_id_list_ptr = layout.GetBlockPtr<EntryClassID, true>(
layout.GetBlockPtr<EntryClassID, true>(memory_ptr, storage::DataLayout::ENTRY_CLASSID); memory_ptr, "/common/turn_data/entry_class_ids");
util::vector_view<EntryClassID> entry_class_ids( util::vector_view<EntryClassID> entry_class_ids(
entry_class_id_list_ptr, layout.GetBlockEntries(storage::DataLayout::ENTRY_CLASSID)); entry_class_id_list_ptr, layout.GetBlockEntries("/common/turn_data/entry_class_ids"));
const auto pre_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing, true>( const auto pre_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing, true>(
memory_ptr, storage::DataLayout::PRE_TURN_BEARING); memory_ptr, "/common/turn_data/pre_turn_bearings");
util::vector_view<guidance::TurnBearing> pre_turn_bearings( util::vector_view<guidance::TurnBearing> pre_turn_bearings(
pre_turn_bearing_ptr, layout.GetBlockEntries(storage::DataLayout::PRE_TURN_BEARING)); pre_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/pre_turn_bearings"));
const auto post_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing, true>( const auto post_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing, true>(
memory_ptr, storage::DataLayout::POST_TURN_BEARING); memory_ptr, "/common/turn_data/post_turn_bearings");
util::vector_view<guidance::TurnBearing> post_turn_bearings( util::vector_view<guidance::TurnBearing> post_turn_bearings(
post_turn_bearing_ptr, layout.GetBlockEntries(storage::DataLayout::POST_TURN_BEARING)); post_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/post_turn_bearings"));
guidance::TurnDataView turn_data(std::move(turn_instructions), guidance::TurnDataView turn_data(std::move(turn_instructions),
std::move(lane_data_ids), std::move(lane_data_ids),
@ -504,63 +414,63 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// load compressed geometry // load compressed geometry
{ {
auto geometries_index_ptr = auto geometries_index_ptr =
layout.GetBlockPtr<unsigned, true>(memory_ptr, storage::DataLayout::GEOMETRIES_INDEX); layout.GetBlockPtr<unsigned, true>(memory_ptr, "/common/segment_data/index");
util::vector_view<unsigned> geometry_begin_indices( util::vector_view<unsigned> geometry_begin_indices(
geometries_index_ptr, layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_INDEX)); geometries_index_ptr, layout.GetBlockEntries("/common/segment_data/index"));
auto num_entries = layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_NODE_LIST); auto num_entries = layout.GetBlockEntries("/common/segment_data/nodes");
auto geometries_node_list_ptr = auto geometries_node_list_ptr =
layout.GetBlockPtr<NodeID, true>(memory_ptr, storage::DataLayout::GEOMETRIES_NODE_LIST); layout.GetBlockPtr<NodeID, true>(memory_ptr, "/common/segment_data/nodes");
util::vector_view<NodeID> geometry_node_list(geometries_node_list_ptr, num_entries); util::vector_view<NodeID> geometry_node_list(geometries_node_list_ptr, num_entries);
auto geometries_fwd_weight_list_ptr = auto geometries_fwd_weight_list_ptr =
layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type, true>( layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type, true>(
memory_ptr, storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST); memory_ptr, "/common/segment_data/forward_weights/packed");
extractor::SegmentDataView::SegmentWeightVector geometry_fwd_weight_list( extractor::SegmentDataView::SegmentWeightVector geometry_fwd_weight_list(
util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>( util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
geometries_fwd_weight_list_ptr, geometries_fwd_weight_list_ptr,
layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST)), layout.GetBlockEntries("/common/segment_data/forward_weights/packed")),
num_entries); num_entries);
auto geometries_rev_weight_list_ptr = auto geometries_rev_weight_list_ptr =
layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type, true>( layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type, true>(
memory_ptr, storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST); memory_ptr, "/common/segment_data/reverse_weights/packed");
extractor::SegmentDataView::SegmentWeightVector geometry_rev_weight_list( extractor::SegmentDataView::SegmentWeightVector geometry_rev_weight_list(
util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>( util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
geometries_rev_weight_list_ptr, geometries_rev_weight_list_ptr,
layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST)), layout.GetBlockEntries("/common/segment_data/reverse_weights/packed")),
num_entries); num_entries);
auto geometries_fwd_duration_list_ptr = auto geometries_fwd_duration_list_ptr =
layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type, true>( layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type, true>(
memory_ptr, storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST); memory_ptr, "/common/segment_data/forward_durations/packed");
extractor::SegmentDataView::SegmentDurationVector geometry_fwd_duration_list( extractor::SegmentDataView::SegmentDurationVector geometry_fwd_duration_list(
util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>( util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
geometries_fwd_duration_list_ptr, geometries_fwd_duration_list_ptr,
layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST)), layout.GetBlockEntries("/common/segment_data/forward_durations/packed")),
num_entries); num_entries);
auto geometries_rev_duration_list_ptr = auto geometries_rev_duration_list_ptr =
layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type, true>( layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type, true>(
memory_ptr, storage::DataLayout::GEOMETRIES_REV_DURATION_LIST); memory_ptr, "/common/segment_data/reverse_durations/packed");
extractor::SegmentDataView::SegmentDurationVector geometry_rev_duration_list( extractor::SegmentDataView::SegmentDurationVector geometry_rev_duration_list(
util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>( util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
geometries_rev_duration_list_ptr, geometries_rev_duration_list_ptr,
layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_DURATION_LIST)), layout.GetBlockEntries("/common/segment_data/reverse_durations/packed")),
num_entries); num_entries);
auto geometries_fwd_datasources_list_ptr = layout.GetBlockPtr<DatasourceID, true>( auto geometries_fwd_datasources_list_ptr = layout.GetBlockPtr<DatasourceID, true>(
memory_ptr, storage::DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST); memory_ptr, "/common/segment_data/forward_data_sources");
util::vector_view<DatasourceID> geometry_fwd_datasources_list( util::vector_view<DatasourceID> geometry_fwd_datasources_list(
geometries_fwd_datasources_list_ptr, geometries_fwd_datasources_list_ptr,
layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST)); layout.GetBlockEntries("/common/segment_data/forward_data_sources"));
auto geometries_rev_datasources_list_ptr = layout.GetBlockPtr<DatasourceID, true>( auto geometries_rev_datasources_list_ptr = layout.GetBlockPtr<DatasourceID, true>(
memory_ptr, storage::DataLayout::GEOMETRIES_REV_DATASOURCES_LIST); memory_ptr, "/common/segment_data/reverse_data_sources");
util::vector_view<DatasourceID> geometry_rev_datasources_list( util::vector_view<DatasourceID> geometry_rev_datasources_list(
geometries_rev_datasources_list_ptr, geometries_rev_datasources_list_ptr,
layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_DATASOURCES_LIST)); layout.GetBlockEntries("/common/segment_data/reverse_data_sources"));
extractor::SegmentDataView segment_data{std::move(geometry_begin_indices), extractor::SegmentDataView segment_data{std::move(geometry_begin_indices),
std::move(geometry_node_list), std::move(geometry_node_list),
@ -576,7 +486,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
{ {
const auto datasources_names_ptr = layout.GetBlockPtr<extractor::Datasources, true>( const auto datasources_names_ptr = layout.GetBlockPtr<extractor::Datasources, true>(
memory_ptr, DataLayout::DATASOURCES_NAMES); memory_ptr, "/common/data_sources_names");
extractor::files::readDatasources(config.GetPath(".osrm.datasource_names"), extractor::files::readDatasources(config.GetPath(".osrm.datasource_names"),
*datasources_names_ptr); *datasources_names_ptr);
} }
@ -584,38 +494,36 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Loading list of coordinates // Loading list of coordinates
{ {
const auto coordinates_ptr = const auto coordinates_ptr =
layout.GetBlockPtr<util::Coordinate, true>(memory_ptr, DataLayout::COORDINATE_LIST); layout.GetBlockPtr<util::Coordinate, true>(memory_ptr, "/common/coordinates");
const auto osmnodeid_ptr = const auto osmnodeid_ptr =
layout.GetBlockPtr<extractor::PackedOSMIDsView::block_type, true>( layout.GetBlockPtr<extractor::PackedOSMIDsView::block_type, true>(
memory_ptr, DataLayout::OSM_NODE_ID_LIST); memory_ptr, "/common/osm_node_ids/packed");
util::vector_view<util::Coordinate> coordinates( util::vector_view<util::Coordinate> coordinates(
coordinates_ptr, layout.GetBlockEntries(DataLayout::COORDINATE_LIST)); coordinates_ptr, layout.GetBlockEntries("/common/coordinates"));
extractor::PackedOSMIDsView osm_node_ids( extractor::PackedOSMIDsView osm_node_ids(
util::vector_view<extractor::PackedOSMIDsView::block_type>( util::vector_view<extractor::PackedOSMIDsView::block_type>(
osmnodeid_ptr, layout.GetBlockEntries(DataLayout::OSM_NODE_ID_LIST)), osmnodeid_ptr, layout.GetBlockEntries("/common/osm_node_ids/packed")),
layout.GetBlockEntries(DataLayout::COORDINATE_LIST)); layout.GetBlockEntries("/common/coordinates"));
extractor::files::readNodes(config.GetPath(".osrm.nbg_nodes"), coordinates, osm_node_ids); extractor::files::readNodes(config.GetPath(".osrm.nbg_nodes"), coordinates, osm_node_ids);
} }
// load turn weight penalties // load turn weight penalties
{ {
auto turn_duration_penalties_ptr = layout.GetBlockPtr<TurnPenalty, true>( auto turn_duration_penalties_ptr =
memory_ptr, storage::DataLayout::TURN_WEIGHT_PENALTIES); layout.GetBlockPtr<TurnPenalty, true>(memory_ptr, "/common/turn_penalty/weight");
util::vector_view<TurnPenalty> turn_duration_penalties( util::vector_view<TurnPenalty> turn_duration_penalties(
turn_duration_penalties_ptr, turn_duration_penalties_ptr, layout.GetBlockEntries("/common/turn_penalty/weight"));
layout.GetBlockEntries(storage::DataLayout::TURN_WEIGHT_PENALTIES));
extractor::files::readTurnWeightPenalty(config.GetPath(".osrm.turn_weight_penalties"), extractor::files::readTurnWeightPenalty(config.GetPath(".osrm.turn_weight_penalties"),
turn_duration_penalties); turn_duration_penalties);
} }
// load turn duration penalties // load turn duration penalties
{ {
auto turn_duration_penalties_ptr = layout.GetBlockPtr<TurnPenalty, true>( auto turn_duration_penalties_ptr =
memory_ptr, storage::DataLayout::TURN_DURATION_PENALTIES); layout.GetBlockPtr<TurnPenalty, true>(memory_ptr, "/common/turn_penalty/duration");
util::vector_view<TurnPenalty> turn_duration_penalties( util::vector_view<TurnPenalty> turn_duration_penalties(
turn_duration_penalties_ptr, turn_duration_penalties_ptr, layout.GetBlockEntries("/common/turn_penalty/duration"));
layout.GetBlockEntries(storage::DataLayout::TURN_DURATION_PENALTIES));
extractor::files::readTurnDurationPenalty(config.GetPath(".osrm.turn_duration_penalties"), extractor::files::readTurnDurationPenalty(config.GetPath(".osrm.turn_duration_penalties"),
turn_duration_penalties); turn_duration_penalties);
} }
@ -624,15 +532,15 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
{ {
const auto rtree_ptr = const auto rtree_ptr =
layout.GetBlockPtr<RTreeNode, true>(memory_ptr, DataLayout::R_SEARCH_TREE); layout.GetBlockPtr<RTreeNode, true>(memory_ptr, "/common/rtree/search_tree");
util::vector_view<RTreeNode> search_tree( util::vector_view<RTreeNode> search_tree(
rtree_ptr, layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE)); rtree_ptr, layout.GetBlockEntries("/common/rtree/search_tree"));
const auto rtree_levelstarts_ptr = layout.GetBlockPtr<std::uint64_t, true>( const auto rtree_levelstarts_ptr = layout.GetBlockPtr<std::uint64_t, true>(
memory_ptr, DataLayout::R_SEARCH_TREE_LEVEL_STARTS); memory_ptr, "/common/rtree/search_tree_level_starts");
util::vector_view<std::uint64_t> rtree_level_starts( util::vector_view<std::uint64_t> rtree_level_starts(
rtree_levelstarts_ptr, rtree_levelstarts_ptr,
layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS)); layout.GetBlockEntries("/common/rtree/search_tree_level_starts"));
// we need this purely for the interface // we need this purely for the interface
util::vector_view<util::Coordinate> empty_coords; util::vector_view<util::Coordinate> empty_coords;
@ -648,7 +556,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// load profile properties // load profile properties
{ {
const auto profile_properties_ptr = layout.GetBlockPtr<extractor::ProfileProperties, true>( const auto profile_properties_ptr = layout.GetBlockPtr<extractor::ProfileProperties, true>(
memory_ptr, DataLayout::PROPERTIES); memory_ptr, "/common/properties");
extractor::files::readProfileProperties(config.GetPath(".osrm.properties"), extractor::files::readProfileProperties(config.GetPath(".osrm.properties"),
*profile_properties_ptr); *profile_properties_ptr);
} }
@ -656,24 +564,29 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Load intersection data // Load intersection data
{ {
auto bearing_class_id_ptr = layout.GetBlockPtr<BearingClassID, true>( auto bearing_class_id_ptr = layout.GetBlockPtr<BearingClassID, true>(
memory_ptr, storage::DataLayout::BEARING_CLASSID); memory_ptr, "/common/intersection_bearings/node_to_class_id");
util::vector_view<BearingClassID> bearing_class_id( util::vector_view<BearingClassID> bearing_class_id(
bearing_class_id_ptr, layout.GetBlockEntries(storage::DataLayout::BEARING_CLASSID)); bearing_class_id_ptr,
layout.GetBlockEntries("/common/intersection_bearings/node_to_class_id"));
auto bearing_values_ptr = layout.GetBlockPtr<DiscreteBearing, true>( auto bearing_values_ptr = layout.GetBlockPtr<DiscreteBearing, true>(
memory_ptr, storage::DataLayout::BEARING_VALUES); memory_ptr, "/common/intersection_bearings/bearing_values");
util::vector_view<DiscreteBearing> bearing_values( util::vector_view<DiscreteBearing> bearing_values(
bearing_values_ptr, layout.GetBlockEntries(storage::DataLayout::BEARING_VALUES)); bearing_values_ptr,
layout.GetBlockEntries("/common/intersection_bearings/bearing_values"));
auto offsets_ptr = auto offsets_ptr = layout.GetBlockPtr<unsigned, true>(
layout.GetBlockPtr<unsigned, true>(memory_ptr, storage::DataLayout::BEARING_OFFSETS); memory_ptr, "/common/intersection_bearings/class_id_to_ranges/block_offsets");
auto blocks_ptr = auto blocks_ptr =
layout.GetBlockPtr<util::RangeTable<16, storage::Ownership::View>::BlockT, true>( layout.GetBlockPtr<util::RangeTable<16, storage::Ownership::View>::BlockT, true>(
memory_ptr, storage::DataLayout::BEARING_BLOCKS); memory_ptr, "/common/intersection_bearings/class_id_to_ranges/diff_blocks");
util::vector_view<unsigned> bearing_offsets( util::vector_view<unsigned> bearing_offsets(
offsets_ptr, layout.GetBlockEntries(storage::DataLayout::BEARING_OFFSETS)); offsets_ptr,
layout.GetBlockEntries(
"/common/intersection_bearings/class_id_to_ranges/block_offsets"));
util::vector_view<util::RangeTable<16, storage::Ownership::View>::BlockT> bearing_blocks( util::vector_view<util::RangeTable<16, storage::Ownership::View>::BlockT> bearing_blocks(
blocks_ptr, layout.GetBlockEntries(storage::DataLayout::BEARING_BLOCKS)); blocks_ptr,
layout.GetBlockEntries("/common/intersection_bearings/class_id_to_ranges/diff_blocks"));
util::RangeTable<16, storage::Ownership::View> bearing_range_table( util::RangeTable<16, storage::Ownership::View> bearing_range_table(
bearing_offsets, bearing_blocks, static_cast<unsigned>(bearing_values.size())); bearing_offsets, bearing_blocks, static_cast<unsigned>(bearing_values.size()));
@ -682,9 +595,9 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)}; std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)};
auto entry_class_ptr = layout.GetBlockPtr<util::guidance::EntryClass, true>( auto entry_class_ptr = layout.GetBlockPtr<util::guidance::EntryClass, true>(
memory_ptr, storage::DataLayout::ENTRY_CLASS); memory_ptr, "/common/entry_classes");
util::vector_view<util::guidance::EntryClass> entry_classes( util::vector_view<util::guidance::EntryClass> entry_classes(
entry_class_ptr, layout.GetBlockEntries(storage::DataLayout::ENTRY_CLASS)); entry_class_ptr, layout.GetBlockEntries("/common/entry_classes"));
extractor::files::readIntersections( extractor::files::readIntersections(
config.GetPath(".osrm.icd"), intersection_bearings_view, entry_classes); config.GetPath(".osrm.icd"), intersection_bearings_view, entry_classes);
@ -695,25 +608,22 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
{ {
auto graph_nodes_ptr = auto graph_nodes_ptr =
layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>( layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
memory_ptr, storage::DataLayout::CH_GRAPH_NODE_LIST); memory_ptr, "/ch/contracted_graph/node_array");
auto graph_edges_ptr = auto graph_edges_ptr =
layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>( layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
memory_ptr, storage::DataLayout::CH_GRAPH_EDGE_LIST); memory_ptr, "/ch/contracted_graph/edge_array");
auto checksum = auto checksum = layout.GetBlockPtr<unsigned, true>(memory_ptr, "/ch/checksum");
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
util::vector_view<contractor::QueryGraphView::NodeArrayEntry> node_list( util::vector_view<contractor::QueryGraphView::NodeArrayEntry> node_list(
graph_nodes_ptr, layout.GetBlockEntries(storage::DataLayout::CH_GRAPH_NODE_LIST)); graph_nodes_ptr, layout.GetBlockEntries("/ch/contracted_graph/node_array"));
util::vector_view<contractor::QueryGraphView::EdgeArrayEntry> edge_list( util::vector_view<contractor::QueryGraphView::EdgeArrayEntry> edge_list(
graph_edges_ptr, layout.GetBlockEntries(storage::DataLayout::CH_GRAPH_EDGE_LIST)); graph_edges_ptr, layout.GetBlockEntries("/ch/contracted_graph/edge_array"));
std::vector<util::vector_view<bool>> edge_filter; std::vector<util::vector_view<bool>> edge_filter;
for (auto index : util::irange<std::size_t>(0, NUM_METRICS)) for (auto index : util::irange<std::size_t>(0, NUM_METRICS))
{ {
auto block_id = auto block_id = "/ch/edge_filter/" + std::to_string(index);
static_cast<DataLayout::BlockID>(storage::DataLayout::CH_EDGE_FILTER_0 + index); auto data_ptr = layout.GetBlockPtr<util::vector_view<bool>::Word, true>(memory_ptr, block_id);
auto data_ptr =
layout.GetBlockPtr<util::vector_view<bool>::Word, true>(memory_ptr, block_id);
auto num_entries = layout.GetBlockEntries(block_id); auto num_entries = layout.GetBlockEntries(block_id);
edge_filter.emplace_back(data_ptr, num_entries); edge_filter.emplace_back(data_ptr, num_entries);
} }
@ -736,35 +646,35 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
} }
else else
{ {
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM); layout.GetBlockPtr<unsigned, true>(memory_ptr, "/ch/checksum");
layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>( layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
memory_ptr, DataLayout::CH_GRAPH_NODE_LIST); memory_ptr, "/ch/contracted_graph/node_array");
layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>( layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
memory_ptr, DataLayout::CH_GRAPH_EDGE_LIST); memory_ptr, "/ch/contracted_graph/edge_array");
} }
} }
{ // Loading MLD Data { // Loading MLD Data
if (boost::filesystem::exists(config.GetPath(".osrm.partition"))) if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
{ {
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_LEVEL_DATA) > 0); BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/level_data") > 0);
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELL_TO_CHILDREN) > 0); BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0);
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_PARTITION) > 0); BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/partition") > 0);
auto level_data = auto level_data =
layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData, true>( layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData, true>(
memory_ptr, storage::DataLayout::MLD_LEVEL_DATA); memory_ptr, "/mld/multilevelpartition/level_data");
auto mld_partition_ptr = layout.GetBlockPtr<PartitionID, true>( auto mld_partition_ptr = layout.GetBlockPtr<PartitionID, true>(
memory_ptr, storage::DataLayout::MLD_PARTITION); memory_ptr, "/mld/multilevelpartition/partition");
auto partition_entries_count = auto partition_entries_count =
layout.GetBlockEntries(storage::DataLayout::MLD_PARTITION); layout.GetBlockEntries("/mld/multilevelpartition/partition");
util::vector_view<PartitionID> partition(mld_partition_ptr, partition_entries_count); util::vector_view<PartitionID> partition(mld_partition_ptr, partition_entries_count);
auto mld_chilren_ptr = layout.GetBlockPtr<CellID, true>( auto mld_chilren_ptr = layout.GetBlockPtr<CellID, true>(
memory_ptr, storage::DataLayout::MLD_CELL_TO_CHILDREN); memory_ptr, "/mld/multilevelpartition/cell_to_children");
auto children_entries_count = auto children_entries_count =
layout.GetBlockEntries(storage::DataLayout::MLD_CELL_TO_CHILDREN); layout.GetBlockEntries("/mld/multilevelpartition/cell_to_children");
util::vector_view<CellID> cell_to_children(mld_chilren_ptr, children_entries_count); util::vector_view<CellID> cell_to_children(mld_chilren_ptr, children_entries_count);
partitioner::MultiLevelPartitionView mlp{ partitioner::MultiLevelPartitionView mlp{
@ -774,25 +684,25 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
if (boost::filesystem::exists(config.GetPath(".osrm.cells"))) if (boost::filesystem::exists(config.GetPath(".osrm.cells")))
{ {
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0); BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/cells") > 0);
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS) > 0); BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
auto mld_source_boundary_ptr = layout.GetBlockPtr<NodeID, true>( auto mld_source_boundary_ptr =
memory_ptr, storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY); layout.GetBlockPtr<NodeID, true>(memory_ptr, "/mld/cellstorage/source_boundary");
auto mld_destination_boundary_ptr = layout.GetBlockPtr<NodeID, true>( auto mld_destination_boundary_ptr = layout.GetBlockPtr<NodeID, true>(
memory_ptr, storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY); memory_ptr, "/mld/cellstorage/destination_boundary");
auto mld_cells_ptr = layout.GetBlockPtr<partitioner::CellStorageView::CellData, true>( auto mld_cells_ptr = layout.GetBlockPtr<partitioner::CellStorageView::CellData, true>(
memory_ptr, storage::DataLayout::MLD_CELLS); memory_ptr, "/mld/cellstorage/cells");
auto mld_cell_level_offsets_ptr = layout.GetBlockPtr<std::uint64_t, true>( auto mld_cell_level_offsets_ptr = layout.GetBlockPtr<std::uint64_t, true>(
memory_ptr, storage::DataLayout::MLD_CELL_LEVEL_OFFSETS); memory_ptr, "/mld/cellstorage/level_to_cell_offset");
auto source_boundary_entries_count = auto source_boundary_entries_count =
layout.GetBlockEntries(storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY); layout.GetBlockEntries("/mld/cellstorage/source_boundary");
auto destination_boundary_entries_count = auto destination_boundary_entries_count =
layout.GetBlockEntries(storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY); layout.GetBlockEntries("/mld/cellstorage/destination_boundary");
auto cells_entries_counts = layout.GetBlockEntries(storage::DataLayout::MLD_CELLS); auto cells_entries_counts = layout.GetBlockEntries("/mld/cellstorage/cells");
auto cell_level_offsets_entries_count = auto cell_level_offsets_entries_count =
layout.GetBlockEntries(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS); layout.GetBlockEntries("/mld/cellstorage/level_to_cell_offset");
util::vector_view<NodeID> source_boundary(mld_source_boundary_ptr, util::vector_view<NodeID> source_boundary(mld_source_boundary_ptr,
source_boundary_entries_count); source_boundary_entries_count);
@ -812,17 +722,15 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics"))) if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
{ {
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0); BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/cells") > 0);
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS) > 0); BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
std::vector<customizer::CellMetricView> metrics; std::vector<customizer::CellMetricView> metrics;
for (auto index : util::irange<std::size_t>(0, NUM_METRICS)) for (auto index : util::irange<std::size_t>(0, NUM_METRICS))
{ {
auto weights_block_id = static_cast<DataLayout::BlockID>( auto weights_block_id = "/mld/metrics/" + std::to_string(index) + "/weights";
storage::DataLayout::MLD_CELL_WEIGHTS_0 + index); auto durations_block_id = "/mld/metrics/" + std::to_string(index) + "/durations";
auto durations_block_id = static_cast<DataLayout::BlockID>(
storage::DataLayout::MLD_CELL_DURATIONS_0 + index);
auto weight_entries_count = layout.GetBlockEntries(weights_block_id); auto weight_entries_count = layout.GetBlockEntries(weights_block_id);
auto duration_entries_count = layout.GetBlockEntries(durations_block_id); auto duration_entries_count = layout.GetBlockEntries(durations_block_id);
@ -846,21 +754,21 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
auto graph_nodes_ptr = auto graph_nodes_ptr =
layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry, true>( layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry, true>(
memory_ptr, storage::DataLayout::MLD_GRAPH_NODE_LIST); memory_ptr, "/mld/multilevelgraph/node_array");
auto graph_edges_ptr = auto graph_edges_ptr =
layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry, true>( layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry, true>(
memory_ptr, storage::DataLayout::MLD_GRAPH_EDGE_LIST); memory_ptr, "/mld/multilevelgraph/edge_array");
auto graph_node_to_offset_ptr = auto graph_node_to_offset_ptr =
layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset, true>( layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset, true>(
memory_ptr, storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET); memory_ptr, "/mld/multilevelgraph/node_to_edge_offset");
util::vector_view<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry> node_list( util::vector_view<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry> node_list(
graph_nodes_ptr, layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_NODE_LIST)); graph_nodes_ptr, layout.GetBlockEntries("/mld/multilevelgraph/node_array"));
util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry> edge_list( util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry> edge_list(
graph_edges_ptr, layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_EDGE_LIST)); graph_edges_ptr, layout.GetBlockEntries("/mld/multilevelgraph/edge_array"));
util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset> node_to_offset( util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset> node_to_offset(
graph_node_to_offset_ptr, graph_node_to_offset_ptr,
layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET)); layout.GetBlockEntries("/mld/multilevelgraph/node_to_edge_offset"));
std::uint32_t graph_connectivity_checksum = 0; std::uint32_t graph_connectivity_checksum = 0;
customizer::MultiLevelEdgeBasedGraphView graph_view( customizer::MultiLevelEdgeBasedGraphView graph_view(
@ -882,15 +790,16 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
{ {
const auto maneuver_overrides_ptr = const auto maneuver_overrides_ptr =
layout.GetBlockPtr<extractor::StorageManeuverOverride, true>( layout.GetBlockPtr<extractor::StorageManeuverOverride, true>(
memory_ptr, DataLayout::MANEUVER_OVERRIDES); memory_ptr, "/common/maneuver_overrides/overrides");
const auto maneuver_override_node_sequences_ptr = layout.GetBlockPtr<NodeID, true>( const auto maneuver_override_node_sequences_ptr = layout.GetBlockPtr<NodeID, true>(
memory_ptr, DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES); memory_ptr, "/common/maneuver_overrides/node_sequences");
util::vector_view<extractor::StorageManeuverOverride> maneuver_overrides( util::vector_view<extractor::StorageManeuverOverride> maneuver_overrides(
maneuver_overrides_ptr, layout.GetBlockEntries(DataLayout::MANEUVER_OVERRIDES)); maneuver_overrides_ptr,
layout.GetBlockEntries("/common/maneuver_overrides/overrides"));
util::vector_view<NodeID> maneuver_override_node_sequences( util::vector_view<NodeID> maneuver_override_node_sequences(
maneuver_override_node_sequences_ptr, maneuver_override_node_sequences_ptr,
layout.GetBlockEntries(DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES)); layout.GetBlockEntries("/common/maneuver_overrides/node_sequences"));
extractor::files::readManeuverOverrides(config.GetPath(".osrm.maneuver_overrides"), extractor::files::readManeuverOverrides(config.GetPath(".osrm.maneuver_overrides"),
maneuver_overrides, maneuver_overrides,

54
unit_tests/storage/data_layout.cpp Normal file
View File

@ -0,0 +1,54 @@
#include "storage/shared_datatype.hpp"
#include "../common/range_tools.hpp"
#include "../common/temporary_file.hpp"
#include <boost/filesystem.hpp>
#include <boost/test/unit_test.hpp>
#include <cmath>
BOOST_AUTO_TEST_SUITE(serialization)
using namespace osrm;
using namespace osrm::storage;
BOOST_AUTO_TEST_CASE(layout_write_test)
{
DataLayout layout;
Block block_1 {20, 8*20};
Block block_2 {1, 4*1};
Block block_3 {100, static_cast<std::uint64_t>(std::ceil(100/8.))};
layout.SetBlock("block1", block_1);
layout.SetBlock("block2", block_2);
layout.SetBlock("block3", block_3);
// Canary and alignment change layout size
BOOST_CHECK_GT(layout.GetSizeOfLayout(), block_1.byte_size + block_2.byte_size + block_3.byte_size);
BOOST_CHECK_EQUAL(layout.GetBlockSize("block1"), block_1.byte_size);
BOOST_CHECK_EQUAL(layout.GetBlockSize("block2"), block_2.byte_size);
BOOST_CHECK_EQUAL(layout.GetBlockSize("block3"), block_3.byte_size);
std::vector<char> buffer(layout.GetSizeOfLayout());
auto smallest_addr = buffer.data();
auto biggest_addr = buffer.data() + buffer.size();
{
auto block_1_ptr = layout.GetBlockPtr<std::uint64_t, true>(buffer.data(), "block1");
auto block_2_ptr = layout.GetBlockPtr<std::uint32_t, true>(buffer.data(), "block2");
auto block_3_ptr = layout.GetBlockPtr<unsigned char, true>(buffer.data(), "block3");
BOOST_CHECK_LT(smallest_addr, block_1_ptr);
BOOST_CHECK_GT(biggest_addr, block_1_ptr + layout.GetBlockEntries("block1"));
BOOST_CHECK_LT(smallest_addr, block_2_ptr);
BOOST_CHECK_GT(biggest_addr, block_2_ptr + layout.GetBlockEntries("block3"));
BOOST_CHECK_LT(smallest_addr, block_3_ptr);
BOOST_CHECK_GT(biggest_addr, block_3_ptr + layout.GetBlockEntries("block3"));
}
}
BOOST_AUTO_TEST_SUITE_END()