From cb4586ebee81bd5ffcad02278115a3ffa041bc15 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 3 Apr 2018 15:16:53 +0000 Subject: [PATCH] Make use of the view factory in the data facade --- .../contiguous_internalmem_datafacade.hpp | 555 ++---------------- include/extractor/files.hpp | 10 +- include/storage/view_factory.hpp | 111 +++- include/util/static_rtree.hpp | 10 +- src/storage/storage.cpp | 16 +- 5 files changed, 154 insertions(+), 548 deletions(-) diff --git a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp index ec3a4491b..4281e295e 100644 --- a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp +++ b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp @@ -9,44 +9,13 @@ #include "engine/approach.hpp" #include "engine/geospatial_query.hpp" -#include "customizer/edge_based_graph.hpp" - -#include "extractor/datasources.hpp" -#include "extractor/edge_based_node.hpp" -#include "extractor/intersection_bearings_container.hpp" -#include "extractor/maneuver_override.hpp" -#include "extractor/name_table.hpp" -#include "extractor/node_data_container.hpp" -#include "extractor/packed_osm_ids.hpp" -#include "extractor/profile_properties.hpp" -#include "extractor/segment_data_container.hpp" -#include "extractor/turn_lane_types.hpp" - -#include "guidance/turn_bearing.hpp" -#include "guidance/turn_data_container.hpp" -#include "guidance/turn_instruction.hpp" - -#include "contractor/query_graph.hpp" - -#include "partitioner/cell_storage.hpp" -#include "partitioner/multi_level_partition.hpp" - #include "storage/shared_datatype.hpp" #include "storage/shared_memory_ownership.hpp" +#include "storage/view_factory.hpp" #include "util/exception.hpp" #include "util/exception_utils.hpp" -#include "util/filtered_graph.hpp" -#include "util/guidance/bearing_class.hpp" -#include "util/guidance/entry_class.hpp" -#include "util/guidance/turn_lanes.hpp" #include "util/log.hpp" -#include "util/packed_vector.hpp" -#include "util/range_table.hpp" -#include "util/rectangle.hpp" -#include "util/static_graph.hpp" -#include "util/static_rtree.hpp" -#include "util/typedefs.hpp" #include @@ -81,36 +50,6 @@ class ContiguousInternalMemoryAlgorithmDataFacade : public datafacade::Algor // allocator that keeps the allocation data std::shared_ptr allocator; - void InitializeGraphPointer(const storage::DataLayout &data_layout, - char *memory_block, - const std::string &metric_name, - const std::size_t exclude_index) - { - const std::string metric_prefix = "/ch/metrics/" + metric_name; - auto graph_nodes_ptr = data_layout.GetBlockPtr( - memory_block, metric_prefix + "/contracted_graph/node_array"); - - auto graph_edges_ptr = data_layout.GetBlockPtr( - memory_block, metric_prefix + "/contracted_graph/edge_array"); - - auto exclude_prefix = metric_prefix + "/exclude/" + std::to_string(exclude_index); - auto filter_block_id = exclude_prefix + "/edge_filter"; - - auto edge_filter_ptr = - data_layout.GetBlockPtr::Word>(memory_block, filter_block_id); - - util::vector_view node_list( - graph_nodes_ptr, - data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/node_array")); - util::vector_view edge_list( - graph_edges_ptr, - data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/edge_array")); - - util::vector_view edge_filter(edge_filter_ptr, - data_layout.GetBlockEntries(filter_block_id)); - m_query_graph = QueryGraph({node_list, edge_list}, edge_filter); - } - public: ContiguousInternalMemoryAlgorithmDataFacade( std::shared_ptr allocator_, @@ -127,7 +66,8 @@ class ContiguousInternalMemoryAlgorithmDataFacade : public datafacade::Algor const std::string &metric_name, const std::size_t exclude_index) { - InitializeGraphPointer(data_layout, memory_block, metric_name, exclude_index); + m_query_graph = make_filtered_graph_view( + memory_block, data_layout, "/ch/metrics/" + metric_name, exclude_index); } // search graph access @@ -218,7 +158,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade util::vector_view m_maneuver_overrides; util::vector_view m_maneuver_override_node_sequences; - std::unique_ptr m_static_rtree; + SharedRTree m_static_rtree; std::unique_ptr m_geospatial_query; boost::filesystem::path file_index_path; @@ -232,330 +172,55 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade // allocator that keeps the allocation data std::shared_ptr allocator; - void InitializeProfilePropertiesPointer(const storage::DataLayout &data_layout, - char *memory_block, - const std::string &metric_name, - const std::size_t exclude_index) - { - // TODO: For multi-metric support we need to have separate exclude classes per metric - (void)metric_name; - m_profile_properties = data_layout.GetBlockPtr( - memory_block, "/common/properties"); - - exclude_mask = m_profile_properties->excludable_classes[exclude_index]; - } - - void InitializeChecksumPointer(const storage::DataLayout &data_layout, char *memory_block) - { - m_check_sum = - *data_layout.GetBlockPtr(memory_block, "/common/connectivity_checksum"); - util::Log() << "set checksum: " << m_check_sum; - } - - void InitializeRTreePointers(const storage::DataLayout &data_layout, char *memory_block) - { - BOOST_ASSERT_MSG(!m_coordinate_list.empty(), "coordinates must be loaded before r-tree"); - - const auto file_index_ptr = - data_layout.GetBlockPtr(memory_block, "/common/rtree/file_index_path"); - file_index_path = boost::filesystem::path(file_index_ptr); - if (!boost::filesystem::exists(file_index_path)) - { - util::Log(logDEBUG) << "Leaf file name " << file_index_path.string(); - throw util::exception("Could not load " + file_index_path.string() + - "Is any data loaded into shared memory?" + SOURCE_REF); - } - - const auto rtree_ptr = - data_layout.GetBlockPtr(memory_block, "/common/rtree/search_tree"); - util::vector_view search_tree( - rtree_ptr, data_layout.GetBlockEntries("/common/rtree/search_tree")); - - const auto rtree_levelstarts_ptr = data_layout.GetBlockPtr( - memory_block, "/common/rtree/search_tree_level_starts"); - util::vector_view rtree_level_starts( - rtree_levelstarts_ptr, - data_layout.GetBlockEntries("/common/rtree/search_tree_level_starts")); - - m_static_rtree.reset(new SharedRTree{std::move(search_tree), - std::move(rtree_level_starts), - file_index_path, - m_coordinate_list}); - m_geospatial_query.reset( - new SharedGeospatialQuery(*m_static_rtree, m_coordinate_list, *this)); - } - - void InitializeNodeInformationPointers(const storage::DataLayout &layout, char *memory_ptr) - { - const auto coordinate_list_ptr = - layout.GetBlockPtr(memory_ptr, "/common/coordinates"); - m_coordinate_list.reset(coordinate_list_ptr, layout.GetBlockEntries("/common/coordinates")); - - const auto osmnodeid_ptr = layout.GetBlockPtr( - memory_ptr, "/common/osm_node_ids/packed"); - m_osmnodeid_list = extractor::PackedOSMIDsView( - util::vector_view( - osmnodeid_ptr, layout.GetBlockEntries("/common/osm_node_ids/packed")), - // We (ab)use the number of coordinates here because we know we have the same amount of - // ids - layout.GetBlockEntries("/common/coordinates")); - } - - void InitializeEdgeBasedNodeDataInformationPointers(const storage::DataLayout &layout, - char *memory_ptr) - { - const auto edge_based_node_list_ptr = - layout.GetBlockPtr(memory_ptr, "/common/ebg_node_data/nodes"); - util::vector_view edge_based_node_data_list( - edge_based_node_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/nodes")); - - const auto annotation_data_list_ptr = - layout.GetBlockPtr( - memory_ptr, "/common/ebg_node_data/annotations"); - util::vector_view annotation_data( - annotation_data_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/annotations")); - - edge_based_node_data = extractor::EdgeBasedNodeDataView( - std::move(edge_based_node_data_list), std::move(annotation_data)); - } - - void InitializeEdgeInformationPointers(const storage::DataLayout &layout, char *memory_ptr) - { - const auto lane_data_id_ptr = - layout.GetBlockPtr(memory_ptr, "/common/turn_data/lane_data_ids"); - util::vector_view lane_data_ids( - lane_data_id_ptr, layout.GetBlockEntries("/common/turn_data/lane_data_ids")); - - const auto turn_instruction_list_ptr = layout.GetBlockPtr( - memory_ptr, "/common/turn_data/turn_instructions"); - util::vector_view turn_instructions( - turn_instruction_list_ptr, - layout.GetBlockEntries("/common/turn_data/turn_instructions")); - - const auto entry_class_id_list_ptr = - layout.GetBlockPtr(memory_ptr, "/common/turn_data/entry_class_ids"); - util::vector_view entry_class_ids( - entry_class_id_list_ptr, layout.GetBlockEntries("/common/turn_data/entry_class_ids")); - - const auto pre_turn_bearing_ptr = layout.GetBlockPtr( - memory_ptr, "/common/turn_data/pre_turn_bearings"); - util::vector_view pre_turn_bearings( - pre_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/pre_turn_bearings")); - - const auto post_turn_bearing_ptr = layout.GetBlockPtr( - memory_ptr, "/common/turn_data/post_turn_bearings"); - util::vector_view post_turn_bearings( - post_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/post_turn_bearings")); - - turn_data = guidance::TurnDataView(std::move(turn_instructions), - std::move(lane_data_ids), - std::move(entry_class_ids), - std::move(pre_turn_bearings), - std::move(post_turn_bearings)); - } - - void InitializeNamePointers(const storage::DataLayout &data_layout, char *memory_block) - { - const auto name_blocks_ptr = - data_layout.GetBlockPtr( - memory_block, "/common/names/blocks"); - const auto name_values_ptr = - data_layout.GetBlockPtr( - memory_block, "/common/names/values"); - - util::vector_view blocks( - name_blocks_ptr, data_layout.GetBlockEntries("/common/names/blocks")); - util::vector_view values( - name_values_ptr, data_layout.GetBlockEntries("/common/names/values")); - - extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)}; - m_name_table = extractor::NameTableView{std::move(index_data_view)}; - } - - void InitializeTurnLaneDescriptionsPointers(const storage::DataLayout &data_layout, - char *memory_block) - { - auto offsets_ptr = - data_layout.GetBlockPtr(memory_block, "/common/turn_lanes/offsets"); - util::vector_view offsets( - offsets_ptr, data_layout.GetBlockEntries("/common/turn_lanes/offsets")); - m_lane_description_offsets = std::move(offsets); - - auto masks_ptr = data_layout.GetBlockPtr( - memory_block, "/common/turn_lanes/masks"); - - util::vector_view masks( - masks_ptr, data_layout.GetBlockEntries("/common/turn_lanes/masks")); - m_lane_description_masks = std::move(masks); - - const auto lane_tupel_id_pair_ptr = - data_layout.GetBlockPtr(memory_block, - "/common/turn_lanes/data"); - util::vector_view lane_tupel_id_pair( - lane_tupel_id_pair_ptr, data_layout.GetBlockEntries("/common/turn_lanes/data")); - m_lane_tupel_id_pairs = std::move(lane_tupel_id_pair); - } - - void InitializeTurnPenalties(const storage::DataLayout &data_layout, char *memory_block) - { - auto turn_weight_penalties_ptr = - data_layout.GetBlockPtr(memory_block, "/common/turn_penalty/weight"); - m_turn_weight_penalties = util::vector_view( - turn_weight_penalties_ptr, data_layout.GetBlockEntries("/common/turn_penalty/weight")); - auto turn_duration_penalties_ptr = - data_layout.GetBlockPtr(memory_block, "/common/turn_penalty/duration"); - m_turn_duration_penalties = util::vector_view( - turn_duration_penalties_ptr, - data_layout.GetBlockEntries("/common/turn_penalty/duration")); - } - - void InitializeGeometryPointers(const storage::DataLayout &data_layout, char *memory_block) - { - auto geometries_index_ptr = - data_layout.GetBlockPtr(memory_block, "/common/segment_data/index"); - util::vector_view geometry_begin_indices( - geometries_index_ptr, data_layout.GetBlockEntries("/common/segment_data/index")); - - auto num_entries = data_layout.GetBlockEntries("/common/segment_data/nodes"); - auto geometries_node_list_ptr = - data_layout.GetBlockPtr(memory_block, "/common/segment_data/nodes"); - util::vector_view geometry_node_list(geometries_node_list_ptr, num_entries); - - auto geometries_fwd_weight_list_ptr = - data_layout.GetBlockPtr( - memory_block, "/common/segment_data/forward_weights/packed"); - extractor::SegmentDataView::SegmentWeightVector geometry_fwd_weight_list( - util::vector_view( - geometries_fwd_weight_list_ptr, - data_layout.GetBlockEntries("/common/segment_data/forward_weights/packed")), - num_entries); - - auto geometries_rev_weight_list_ptr = - data_layout.GetBlockPtr( - memory_block, "/common/segment_data/reverse_weights/packed"); - extractor::SegmentDataView::SegmentWeightVector geometry_rev_weight_list( - util::vector_view( - geometries_rev_weight_list_ptr, - data_layout.GetBlockEntries("/common/segment_data/reverse_weights/packed")), - num_entries); - - auto geometries_fwd_duration_list_ptr = - data_layout.GetBlockPtr( - memory_block, "/common/segment_data/forward_durations/packed"); - extractor::SegmentDataView::SegmentDurationVector geometry_fwd_duration_list( - util::vector_view( - geometries_fwd_duration_list_ptr, - data_layout.GetBlockEntries("/common/segment_data/forward_durations/packed")), - num_entries); - - auto geometries_rev_duration_list_ptr = - data_layout.GetBlockPtr( - memory_block, "/common/segment_data/reverse_durations/packed"); - extractor::SegmentDataView::SegmentDurationVector geometry_rev_duration_list( - util::vector_view( - geometries_rev_duration_list_ptr, - data_layout.GetBlockEntries("/common/segment_data/reverse_durations/packed")), - num_entries); - - auto geometries_fwd_datasources_list_ptr = data_layout.GetBlockPtr( - memory_block, "/common/segment_data/forward_data_sources"); - util::vector_view geometry_fwd_datasources_list( - geometries_fwd_datasources_list_ptr, - data_layout.GetBlockEntries("/common/segment_data/forward_data_sources")); - - auto geometries_rev_datasources_list_ptr = data_layout.GetBlockPtr( - memory_block, "/common/segment_data/reverse_data_sources"); - util::vector_view geometry_rev_datasources_list( - geometries_rev_datasources_list_ptr, - data_layout.GetBlockEntries("/common/segment_data/reverse_data_sources")); - - segment_data = extractor::SegmentDataView{std::move(geometry_begin_indices), - std::move(geometry_node_list), - std::move(geometry_fwd_weight_list), - std::move(geometry_rev_weight_list), - std::move(geometry_fwd_duration_list), - std::move(geometry_rev_duration_list), - std::move(geometry_fwd_datasources_list), - std::move(geometry_rev_datasources_list)}; - - m_datasources = data_layout.GetBlockPtr( - memory_block, "/common/data_sources_names"); - } - - void InitializeIntersectionClassPointers(const storage::DataLayout &data_layout, - char *memory_block) - { - auto bearing_class_id_ptr = data_layout.GetBlockPtr( - memory_block, "/common/intersection_bearings/node_to_class_id"); - util::vector_view bearing_class_id( - bearing_class_id_ptr, - data_layout.GetBlockEntries("/common/intersection_bearings/node_to_class_id")); - - auto bearing_values_ptr = data_layout.GetBlockPtr( - memory_block, "/common/intersection_bearings/bearing_values"); - util::vector_view bearing_values( - bearing_values_ptr, - data_layout.GetBlockEntries("/common/intersection_bearings/bearing_values")); - - auto offsets_ptr = data_layout.GetBlockPtr( - memory_block, "/common/intersection_bearings/class_id_to_ranges/block_offsets"); - auto blocks_ptr = data_layout.GetBlockPtr( - memory_block, "/common/intersection_bearings/class_id_to_ranges/diff_blocks"); - util::vector_view bearing_offsets( - offsets_ptr, - data_layout.GetBlockEntries( - "/common/intersection_bearings/class_id_to_ranges/block_offsets")); - util::vector_view bearing_blocks( - blocks_ptr, - data_layout.GetBlockEntries( - "/common/intersection_bearings/class_id_to_ranges/diff_blocks")); - - util::RangeTable<16, storage::Ownership::View> bearing_range_table( - bearing_offsets, bearing_blocks, static_cast(bearing_values.size())); - - intersection_bearings_view = extractor::IntersectionBearingsView{ - std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)}; - - auto entry_class_ptr = data_layout.GetBlockPtr( - memory_block, "/common/entry_classes"); - util::vector_view entry_class_table( - entry_class_ptr, data_layout.GetBlockEntries("/common/entry_classes")); - m_entry_class_table = std::move(entry_class_table); - } - - void InitializeManeuverOverridePointers(const storage::DataLayout &data_layout, - char *memory_block) - { - auto maneuver_overrides_ptr = data_layout.GetBlockPtr( - memory_block, "/common/maneuver_overrides/overrides"); - m_maneuver_overrides = util::vector_view( - maneuver_overrides_ptr, - data_layout.GetBlockEntries("/common/maneuver_overrides/overrides")); - - auto maneuver_override_node_sequences_ptr = data_layout.GetBlockPtr( - memory_block, "/common/maneuver_overrides/node_sequences"); - m_maneuver_override_node_sequences = util::vector_view( - maneuver_override_node_sequences_ptr, - data_layout.GetBlockEntries("/common/maneuver_overrides/node_sequences")); - } - - void InitializeInternalPointers(const storage::DataLayout &data_layout, - char *memory_block, + void InitializeInternalPointers(const storage::DataLayout &layout, + char *memory_ptr, const std::string &metric_name, const std::size_t exclude_index) { - InitializeChecksumPointer(data_layout, memory_block); - InitializeNodeInformationPointers(data_layout, memory_block); - InitializeEdgeBasedNodeDataInformationPointers(data_layout, memory_block); - InitializeEdgeInformationPointers(data_layout, memory_block); - InitializeTurnPenalties(data_layout, memory_block); - InitializeGeometryPointers(data_layout, memory_block); - InitializeNamePointers(data_layout, memory_block); - InitializeTurnLaneDescriptionsPointers(data_layout, memory_block); - InitializeProfilePropertiesPointer(data_layout, memory_block, metric_name, exclude_index); - InitializeRTreePointers(data_layout, memory_block); - InitializeIntersectionClassPointers(data_layout, memory_block); - InitializeManeuverOverridePointers(data_layout, memory_block); + // TODO: For multi-metric support we need to have separate exclude classes per metric + (void)metric_name; + + m_profile_properties = + layout.GetBlockPtr(memory_ptr, "/common/properties"); + + exclude_mask = m_profile_properties->excludable_classes[exclude_index]; + + m_check_sum = + *layout.GetBlockPtr(memory_ptr, "/common/connectivity_checksum"); + + std::tie(m_coordinate_list, m_osmnodeid_list) = + make_nbn_data_view(memory_ptr, layout, "/common/nbn_data"); + + m_static_rtree = make_search_tree_view(memory_ptr, layout, "/common/rtree"); + m_geospatial_query.reset( + new SharedGeospatialQuery(m_static_rtree, m_coordinate_list, *this)); + + edge_based_node_data = make_ebn_data_view(memory_ptr, layout, "/common/ebg_node_data"); + + turn_data = make_turn_data_view(memory_ptr, layout, "/common/turn_data"); + + m_name_table = make_name_table_view(memory_ptr, layout, "/common/names"); + + std::tie(m_lane_description_offsets, m_lane_description_masks) = + make_turn_lane_description_views(memory_ptr, layout, "/common/turn_lanes"); + m_lane_tupel_id_pairs = make_lane_data_view(memory_ptr, layout, "/common/turn_lanes"); + + m_turn_weight_penalties = make_turn_weight_view(memory_ptr, layout, "/common/turn_penalty"); + m_turn_duration_penalties = + make_turn_duration_view(memory_ptr, layout, "/common/turn_penalty"); + + segment_data = make_segment_data_view(memory_ptr, layout, "/common/segment_data"); + + m_datasources = + layout.GetBlockPtr(memory_ptr, "/common/data_sources_names"); + + intersection_bearings_view = + make_intersection_bearings_view(memory_ptr, layout, "/common/intersection_bearings"); + + m_entry_class_table = make_entry_classes_view(memory_ptr, layout, "/common/entry_classes"); + + std::tie(m_maneuver_overrides, m_maneuver_override_node_sequences) = + make_maneuver_overrides_views(memory_ptr, layout, "/common/maneuver_overrides"); } public: @@ -996,122 +661,16 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade : public Algo QueryGraph query_graph; - void InitializeInternalPointers(const storage::DataLayout &data_layout, - char *memory_block, + void InitializeInternalPointers(const storage::DataLayout &layout, + char *memory_ptr, const std::string &metric_name, const std::size_t exclude_index) { - InitializeMLDDataPointers(data_layout, memory_block, metric_name, exclude_index); - InitializeGraphPointer(data_layout, memory_block); - } - - void InitializeMLDDataPointers(const storage::DataLayout &data_layout, - char *memory_block, - const std::string &metric_name, - const std::size_t exclude_index) - { - if (data_layout.GetBlockSize("/mld/multilevelpartition/partition") > 0) - { - BOOST_ASSERT(data_layout.GetBlockSize("/mld/multilevelpartition/level_data") > 0); - BOOST_ASSERT(data_layout.GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0); - - auto level_data = - data_layout.GetBlockPtr( - memory_block, "/mld/multilevelpartition/level_data"); - - auto mld_partition_ptr = data_layout.GetBlockPtr( - memory_block, "/mld/multilevelpartition/partition"); - auto partition_entries_count = - data_layout.GetBlockEntries("/mld/multilevelpartition/partition"); - util::vector_view partition(mld_partition_ptr, partition_entries_count); - - auto mld_chilren_ptr = data_layout.GetBlockPtr( - memory_block, "/mld/multilevelpartition/cell_to_children"); - auto children_entries_count = - data_layout.GetBlockEntries("/mld/multilevelpartition/cell_to_children"); - util::vector_view cell_to_children(mld_chilren_ptr, children_entries_count); - - mld_partition = - partitioner::MultiLevelPartitionView{level_data, partition, cell_to_children}; - } - - const auto exclude_prefix = - "/mld/metrics/" + metric_name + "/exclude/" + std::to_string(exclude_index); - const auto weights_block_id = exclude_prefix + "/weights"; - const auto durations_block_id = exclude_prefix + "/durations"; - - if (data_layout.GetBlockSize(weights_block_id) > 0) - { - auto mld_cell_weights_ptr = - data_layout.GetBlockPtr(memory_block, weights_block_id); - auto mld_cell_durations_ptr = - data_layout.GetBlockPtr(memory_block, durations_block_id); - auto weight_entries_count = data_layout.GetBlockEntries(weights_block_id); - auto duration_entries_count = data_layout.GetBlockEntries(durations_block_id); - BOOST_ASSERT(weight_entries_count == duration_entries_count); - util::vector_view weights(mld_cell_weights_ptr, weight_entries_count); - util::vector_view durations(mld_cell_durations_ptr, - duration_entries_count); - - mld_cell_metric = customizer::CellMetricView{std::move(weights), std::move(durations)}; - } - - if (data_layout.GetBlockSize("/mld/cellstorage/cells") > 0) - { - - auto mld_source_boundary_ptr = - data_layout.GetBlockPtr(memory_block, "/mld/cellstorage/source_boundary"); - auto mld_destination_boundary_ptr = data_layout.GetBlockPtr( - memory_block, "/mld/cellstorage/destination_boundary"); - auto mld_cells_ptr = data_layout.GetBlockPtr( - memory_block, "/mld/cellstorage/cells"); - auto mld_cell_level_offsets_ptr = data_layout.GetBlockPtr( - memory_block, "/mld/cellstorage/level_to_cell_offset"); - - auto source_boundary_entries_count = - data_layout.GetBlockEntries("/mld/cellstorage/source_boundary"); - auto destination_boundary_entries_count = - data_layout.GetBlockEntries("/mld/cellstorage/destination_boundary"); - auto cells_entries_counts = data_layout.GetBlockEntries("/mld/cellstorage/cells"); - auto cell_level_offsets_entries_count = - data_layout.GetBlockEntries("/mld/cellstorage/level_to_cell_offset"); - - util::vector_view source_boundary(mld_source_boundary_ptr, - source_boundary_entries_count); - util::vector_view destination_boundary(mld_destination_boundary_ptr, - destination_boundary_entries_count); - util::vector_view cells(mld_cells_ptr, - cells_entries_counts); - util::vector_view level_offsets(mld_cell_level_offsets_ptr, - cell_level_offsets_entries_count); - - mld_cell_storage = partitioner::CellStorageView{std::move(source_boundary), - std::move(destination_boundary), - std::move(cells), - std::move(level_offsets)}; - } - } - void InitializeGraphPointer(const storage::DataLayout &data_layout, char *memory_block) - { - auto graph_nodes_ptr = - data_layout.GetBlockPtr(memory_block, "/mld/multilevelgraph/node_array"); - - auto graph_edges_ptr = - data_layout.GetBlockPtr(memory_block, "/mld/multilevelgraph/edge_array"); - - auto graph_node_to_offset_ptr = data_layout.GetBlockPtr( - memory_block, "/mld/multilevelgraph/node_to_edge_offset"); - - util::vector_view node_list( - graph_nodes_ptr, data_layout.GetBlockEntries("/mld/multilevelgraph/node_array")); - util::vector_view edge_list( - graph_edges_ptr, data_layout.GetBlockEntries("/mld/multilevelgraph/edge_array")); - util::vector_view node_to_offset( - graph_node_to_offset_ptr, - data_layout.GetBlockEntries("/mld/multilevelgraph/node_to_edge_offset")); - - query_graph = - QueryGraph(std::move(node_list), std::move(edge_list), std::move(node_to_offset)); + mld_partition = make_partition_view(memory_ptr, layout, "/mld/multilevelpartition"); + mld_cell_metric = make_filtered_cell_metric_view( + memory_ptr, layout, "/mld/metrics/" + metric_name, exclude_index); + mld_cell_storage = make_cell_storage_view(memory_ptr, layout, "/mld/cellstorage"); + query_graph = make_multi_level_graph_view(memory_ptr, layout, "/mld/multilevelgraph"); } // allocator that keeps the allocation data diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index 9fa805f6b..34ac91c62 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -122,8 +122,8 @@ inline void readNodes(const boost::filesystem::path &path, const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; - storage::serialization::read(reader, "/common/coordinates", coordinates); - util::serialization::read(reader, "/common/osm_node_ids", osm_node_ids); + storage::serialization::read(reader, "/common/nbn_data/coordinates", coordinates); + util::serialization::read(reader, "/common/nbn_data/osm_node_ids", osm_node_ids); } // reads only coordinates from .osrm.nbg_nodes @@ -135,7 +135,7 @@ inline void readNodeCoordinates(const boost::filesystem::path &path, Coordinates const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; - storage::serialization::read(reader, "/common/coordinates", coordinates); + storage::serialization::read(reader, "/common/nbn_data/coordinates", coordinates); } // writes .osrm.nbg_nodes @@ -150,8 +150,8 @@ inline void writeNodes(const boost::filesystem::path &path, const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; storage::tar::FileWriter writer{path, fingerprint}; - storage::serialization::write(writer, "/common/coordinates", coordinates); - util::serialization::write(writer, "/common/osm_node_ids", osm_node_ids); + storage::serialization::write(writer, "/common/nbn_data/coordinates", coordinates); + util::serialization::write(writer, "/common/nbn_data/osm_node_ids", osm_node_ids); } // reads .osrm.cnbg_to_ebg diff --git a/include/storage/view_factory.hpp b/include/storage/view_factory.hpp index 91acdf9c7..d39dfbc62 100644 --- a/include/storage/view_factory.hpp +++ b/include/storage/view_factory.hpp @@ -35,6 +35,7 @@ #include "util/typedefs.hpp" #include "util/vector_view.hpp" +#include "util/filtered_graph.hpp" #include "util/packed_vector.hpp" #include "util/vector_view.hpp" @@ -72,7 +73,8 @@ make_packed_vector_view(char *memory_ptr, const DataLayout &layout, const std::s return V{packed_internal, std::numeric_limits::max()}; } -auto make_name_table_view(char *memory_ptr, const DataLayout &layout, const std::string &name) +inline auto +make_name_table_view(char *memory_ptr, const DataLayout &layout, const std::string &name) { auto blocks = make_vector_view( memory_ptr, layout, name + "/blocks"); @@ -83,14 +85,14 @@ auto make_name_table_view(char *memory_ptr, const DataLayout &layout, const std: return extractor::NameTableView{index_data_view}; } -auto make_lane_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name) +inline auto make_lane_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name) { return make_vector_view(memory_ptr, layout, name + "/data"); } -auto make_turn_lane_description_views(char *memory_ptr, - const DataLayout &layout, - const std::string &name) +inline auto make_turn_lane_description_views(char *memory_ptr, + const DataLayout &layout, + const std::string &name) { auto offsets = make_vector_view(memory_ptr, layout, name + "/offsets"); auto masks = @@ -99,7 +101,7 @@ auto make_turn_lane_description_views(char *memory_ptr, return std::make_tuple(offsets, masks); } -auto make_ebn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name) +inline auto make_ebn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name) { auto edge_based_node_data = make_vector_view(memory_ptr, layout, name + "/nodes"); @@ -110,7 +112,7 @@ auto make_ebn_data_view(char *memory_ptr, const DataLayout &layout, const std::s std::move(annotation_data)); } -auto make_turn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name) +inline auto make_turn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name) { auto lane_data_ids = make_vector_view(memory_ptr, layout, name + "/lane_data_ids"); @@ -133,7 +135,8 @@ auto make_turn_data_view(char *memory_ptr, const DataLayout &layout, const std:: std::move(post_turn_bearings)); } -auto make_segment_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name) +inline auto +make_segment_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name) { auto geometry_begin_indices = make_vector_view(memory_ptr, layout, name + "/index"); @@ -177,29 +180,39 @@ auto make_segment_data_view(char *memory_ptr, const DataLayout &layout, const st std::move(rev_datasources_list)}; } -auto make_coordinates_view(char *memory_ptr, const DataLayout &layout, const std::string &name) +inline auto +make_coordinates_view(char *memory_ptr, const DataLayout &layout, const std::string &name) { return make_vector_view(memory_ptr, layout, name); } -auto make_osm_ids_view(char *memory_ptr, const DataLayout &layout, const std::string &name) +inline auto make_osm_ids_view(char *memory_ptr, const DataLayout &layout, const std::string &name) { return make_packed_vector_view( memory_ptr, layout, name); } -auto make_turn_duration_view(char *memory_ptr, const DataLayout &layout, const std::string &name) +inline auto make_nbn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name) +{ + return std::make_tuple(make_coordinates_view(memory_ptr, layout, name + "/coordinates"), + make_osm_ids_view(memory_ptr, layout, name + "/osm_node_ids")); +} + +inline auto +make_turn_weight_view(char *memory_ptr, const DataLayout &layout, const std::string &name) { return make_vector_view(memory_ptr, layout, name + "/weight"); } -auto make_turn_penalties_view(char *memory_ptr, const DataLayout &layout, const std::string &name) +inline auto +make_turn_duration_view(char *memory_ptr, const DataLayout &layout, const std::string &name) { return make_vector_view(memory_ptr, layout, name + "/duration"); } -auto make_search_tree_view(char *memory_ptr, const DataLayout &layout, const std::string &name) +inline auto +make_search_tree_view(char *memory_ptr, const DataLayout &layout, const std::string &name) { using RTreeLeaf = extractor::EdgeBasedNodeSegment; using RTreeNode = util::StaticRTree::TreeNode; @@ -209,17 +222,23 @@ auto make_search_tree_view(char *memory_ptr, const DataLayout &layout, const std const auto rtree_level_starts = make_vector_view(memory_ptr, layout, name + "/search_tree_level_starts"); - const auto coordinates = make_coordinates_view(memory_ptr, layout, "/common/coordinates"); + const auto coordinates = + make_coordinates_view(memory_ptr, layout, "/common/nbn_data/coordinates"); const char *path = layout.GetBlockPtr(memory_ptr, name + "/file_index_path"); + if (!boost::filesystem::exists(boost::filesystem::path{path})) + { + throw util::exception("Could not load " + std::string(path) + "Does the leaf file exist?" + + SOURCE_REF); + } + return util::StaticRTree{ std::move(search_tree), std::move(rtree_level_starts), path, std::move(coordinates)}; } -auto make_intersection_bearings_view(char *memory_ptr, - const DataLayout &layout, - const std::string &name) +inline auto +make_intersection_bearings_view(char *memory_ptr, const DataLayout &layout, const std::string &name) { auto bearing_offsets = make_vector_view(memory_ptr, layout, name + "/class_id_to_ranges/block_offsets"); @@ -238,14 +257,14 @@ auto make_intersection_bearings_view(char *memory_ptr, std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)}; } -auto make_entry_classes_view(char *memory_ptr, const DataLayout &layout, const std::string &name) +inline auto +make_entry_classes_view(char *memory_ptr, const DataLayout &layout, const std::string &name) { return make_vector_view(memory_ptr, layout, name); } -auto make_contracted_metric_view(char *memory_ptr, - const DataLayout &layout, - const std::string &name) +inline auto +make_contracted_metric_view(char *memory_ptr, const DataLayout &layout, const std::string &name) { auto node_list = make_vector_view( memory_ptr, layout, name + "/contracted_graph/node_array"); @@ -262,7 +281,7 @@ auto make_contracted_metric_view(char *memory_ptr, std::move(edge_filter)}; } -auto make_partition_view(char *memory_ptr, const DataLayout &layout, const std::string &name) +inline auto make_partition_view(char *memory_ptr, const DataLayout &layout, const std::string &name) { auto level_data_ptr = layout.GetBlockPtr( memory_ptr, name + "/level_data"); @@ -274,7 +293,8 @@ auto make_partition_view(char *memory_ptr, const DataLayout &layout, const std:: level_data_ptr, std::move(partition), std::move(cell_to_children)}; } -auto make_cell_storage_view(char *memory_ptr, const DataLayout &layout, const std::string &name) +inline auto +make_cell_storage_view(char *memory_ptr, const DataLayout &layout, const std::string &name) { auto source_boundary = make_vector_view(memory_ptr, layout, name + "/source_boundary"); auto destination_boundary = @@ -290,7 +310,25 @@ auto make_cell_storage_view(char *memory_ptr, const DataLayout &layout, const st std::move(level_offsets)}; } -auto make_cell_metric_view(char *memory_ptr, const DataLayout &layout, const std::string &name) +inline auto make_filtered_cell_metric_view(char *memory_ptr, + const DataLayout &layout, + const std::string &name, + const std::size_t exclude_index) +{ + customizer::CellMetricView cell_metric; + + auto prefix = name + "/exclude/" + std::to_string(exclude_index); + auto weights_block_id = prefix + "/weights"; + auto durations_block_id = prefix + "/durations"; + + auto weights = make_vector_view(memory_ptr, layout, weights_block_id); + auto durations = make_vector_view(memory_ptr, layout, durations_block_id); + + return customizer::CellMetricView{std::move(weights), std::move(durations)}; +} + +inline auto +make_cell_metric_view(char *memory_ptr, const DataLayout &layout, const std::string &name) { std::vector cell_metric_excludes; @@ -311,9 +349,8 @@ auto make_cell_metric_view(char *memory_ptr, const DataLayout &layout, const std return cell_metric_excludes; } -auto make_multi_level_graph_view(char *memory_ptr, - const DataLayout &layout, - const std::string &name) +inline auto +make_multi_level_graph_view(char *memory_ptr, const DataLayout &layout, const std::string &name) { auto node_list = make_vector_view( memory_ptr, layout, name + "/node_array"); @@ -326,9 +363,8 @@ auto make_multi_level_graph_view(char *memory_ptr, std::move(node_list), std::move(edge_list), std::move(node_to_offset)); } -auto make_maneuver_overrides_views(char *memory_ptr, - const DataLayout &layout, - const std::string &name) +inline auto +make_maneuver_overrides_views(char *memory_ptr, const DataLayout &layout, const std::string &name) { auto maneuver_overrides = make_vector_view( memory_ptr, layout, name + "/overrides"); @@ -337,6 +373,21 @@ auto make_maneuver_overrides_views(char *memory_ptr, return std::make_tuple(maneuver_overrides, maneuver_override_node_sequences); } + +inline auto make_filtered_graph_view(char *memory_ptr, + const DataLayout &layout, + const std::string &name, + const std::size_t exclude_index) +{ + auto exclude_prefix = name + "/exclude/" + std::to_string(exclude_index); + auto edge_filter = make_vector_view(memory_ptr, layout, exclude_prefix + "/edge_filter"); + auto node_list = make_vector_view( + memory_ptr, layout, name + "/contracted_graph/node_array"); + auto edge_list = make_vector_view( + memory_ptr, layout, name + "/contracted_graph/edge_array"); + + return util::FilteredGraphView({node_list, edge_list}, edge_filter); +} } } diff --git a/include/util/static_rtree.hpp b/include/util/static_rtree.hpp index 7ac43e10f..b0a3c76e3 100644 --- a/include/util/static_rtree.hpp +++ b/include/util/static_rtree.hpp @@ -255,7 +255,7 @@ class StaticRTree // Representation of the in-memory search tree Vector m_search_tree; // Reference to the actual lon/lat data we need for doing math - const Vector &m_coordinate_list; + util::vector_view m_coordinate_list; // Holds the start indexes of each level in m_search_tree Vector m_tree_level_starts; // mmap'd .fileIndex file @@ -264,6 +264,7 @@ class StaticRTree util::vector_view m_objects; public: + StaticRTree() = default; StaticRTree(const StaticRTree &) = delete; StaticRTree &operator=(const StaticRTree &) = delete; StaticRTree(StaticRTree &&) = default; @@ -273,7 +274,7 @@ class StaticRTree explicit StaticRTree(const std::vector &input_data_vector, const Vector &coordinate_list, const boost::filesystem::path &on_disk_file_name) - : m_coordinate_list(coordinate_list) + : m_coordinate_list(coordinate_list.data(), coordinate_list.size()) { const auto element_count = input_data_vector.size(); std::vector input_wrapper_vector(element_count); @@ -460,7 +461,7 @@ class StaticRTree template > explicit StaticRTree(const boost::filesystem::path &on_disk_file_name, const Vector &coordinate_list) - : m_coordinate_list(coordinate_list) + : m_coordinate_list(coordinate_list.data(), coordinate_list.size()) { m_objects = mmapFile(on_disk_file_name, m_objects_region); } @@ -475,7 +476,8 @@ class StaticRTree Vector tree_level_starts, const boost::filesystem::path &on_disk_file_name, const Vector &coordinate_list) - : m_search_tree(std::move(search_tree_)), m_coordinate_list(coordinate_list), + : m_search_tree(std::move(search_tree_)), + m_coordinate_list(coordinate_list.data(), coordinate_list.size()), m_tree_level_starts(std::move(tree_level_starts)) { BOOST_ASSERT(m_tree_level_starts.size() >= 2); diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index eeb54292e..66aca9485 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -329,21 +329,15 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // Loading list of coordinates { - auto coordinates = - make_vector_view(memory_ptr, layout, "/common/coordinates"); - - auto osm_node_ids = - extractor::PackedOSMIDsView{make_vector_view( - memory_ptr, layout, "/common/osm_node_ids/packed"), - coordinates.size()}; - - extractor::files::readNodes(config.GetPath(".osrm.nbg_nodes"), coordinates, osm_node_ids); + auto views = make_nbn_data_view(memory_ptr, layout, "/common/nbn_data"); + extractor::files::readNodes( + config.GetPath(".osrm.nbg_nodes"), std::get<0>(views), std::get<1>(views)); } // load turn weight penalties { auto turn_duration_penalties = - make_vector_view(memory_ptr, layout, "/common/turn_penalty/weight"); + make_turn_weight_view(memory_ptr, layout, "/common/turn_penalty"); extractor::files::readTurnWeightPenalty(config.GetPath(".osrm.turn_weight_penalties"), turn_duration_penalties); } @@ -351,7 +345,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // load turn duration penalties { auto turn_duration_penalties = - make_vector_view(memory_ptr, layout, "/common/turn_penalty/duration"); + make_turn_duration_view(memory_ptr, layout, "/common/turn_penalty"); extractor::files::readTurnDurationPenalty(config.GetPath(".osrm.turn_duration_penalties"), turn_duration_penalties); }