Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Load data in two separate data regions

Browse Source
This commit is contained in:
Patrick Niklaus
2018-04-04 23:05:34 +00:00
committed by Patrick Niklaus
parent c7daa521ad
commit fea07f343b
14 changed files with 578 additions and 440 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/engine/datafacade/contiguous_block_allocator.hpp
+2 -3
View File
@@ -1,7 +1,7 @@
#ifndef OSRM_ENGINE_DATAFACADE_CONTIGUOUS_BLOCK_ALLOCATOR_HPP_
#define OSRM_ENGINE_DATAFACADE_CONTIGUOUS_BLOCK_ALLOCATOR_HPP_
#include "storage/shared_datatype.hpp"
#include "storage/shared_data_index.hpp"
namespace osrm
{
@@ -16,8 +16,7 @@ class ContiguousBlockAllocator
virtual ~ContiguousBlockAllocator() = default;
// interface to give access to the datafacades
virtual const storage::DataLayout &GetLayout() = 0;
virtual char *GetMemory() = 0;
virtual const storage::SharedDataIndex &GetIndex() = 0;
};
} // namespace datafacade
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
+29 -38
View File
@@ -57,17 +57,15 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
std::size_t exclude_index)
: allocator(std::move(allocator_))
{
InitializeInternalPointers(
allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
InitializeInternalPointers(allocator->GetIndex(), metric_name, exclude_index);
}
void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block,
void InitializeInternalPointers(const storage::SharedDataIndex &index,
const std::string &metric_name,
const std::size_t exclude_index)
{
m_query_graph = make_filtered_graph_view(
memory_block, data_layout, "/ch/metrics/" + metric_name, exclude_index);
m_query_graph =
make_filtered_graph_view(index, "/ch/metrics/" + metric_name, exclude_index);
}
// search graph access
@@ -172,8 +170,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
// allocator that keeps the allocation data
std::shared_ptr<ContiguousBlockAllocator> allocator;
void InitializeInternalPointers(const storage::DataLayout &layout,
char *memory_ptr,
void InitializeInternalPointers(const storage::SharedDataIndex &index,
const std::string &metric_name,
const std::size_t exclude_index)
{
@@ -181,46 +178,43 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
(void)metric_name;
m_profile_properties =
layout.GetBlockPtr<extractor::ProfileProperties>(memory_ptr, "/common/properties");
index.GetBlockPtr<extractor::ProfileProperties>("/common/properties");
exclude_mask = m_profile_properties->excludable_classes[exclude_index];
m_check_sum =
*layout.GetBlockPtr<std::uint32_t>(memory_ptr, "/common/connectivity_checksum");
m_check_sum = *index.GetBlockPtr<std::uint32_t>("/common/connectivity_checksum");
std::tie(m_coordinate_list, m_osmnodeid_list) =
make_nbn_data_view(memory_ptr, layout, "/common/nbn_data");
make_nbn_data_view(index, "/common/nbn_data");
m_static_rtree = make_search_tree_view(memory_ptr, layout, "/common/rtree");
m_static_rtree = make_search_tree_view(index, "/common/rtree");
m_geospatial_query.reset(
new SharedGeospatialQuery(m_static_rtree, m_coordinate_list, *this));
edge_based_node_data = make_ebn_data_view(memory_ptr, layout, "/common/ebg_node_data");
edge_based_node_data = make_ebn_data_view(index, "/common/ebg_node_data");
turn_data = make_turn_data_view(memory_ptr, layout, "/common/turn_data");
turn_data = make_turn_data_view(index, "/common/turn_data");
m_name_table = make_name_table_view(memory_ptr, layout, "/common/names");
m_name_table = make_name_table_view(index, "/common/names");
std::tie(m_lane_description_offsets, m_lane_description_masks) =
make_turn_lane_description_views(memory_ptr, layout, "/common/turn_lanes");
m_lane_tupel_id_pairs = make_lane_data_view(memory_ptr, layout, "/common/turn_lanes");
make_turn_lane_description_views(index, "/common/turn_lanes");
m_lane_tupel_id_pairs = make_lane_data_view(index, "/common/turn_lanes");
m_turn_weight_penalties = make_turn_weight_view(memory_ptr, layout, "/common/turn_penalty");
m_turn_duration_penalties =
make_turn_duration_view(memory_ptr, layout, "/common/turn_penalty");
m_turn_weight_penalties = make_turn_weight_view(index, "/common/turn_penalty");
m_turn_duration_penalties = make_turn_duration_view(index, "/common/turn_penalty");
segment_data = make_segment_data_view(memory_ptr, layout, "/common/segment_data");
segment_data = make_segment_data_view(index, "/common/segment_data");
m_datasources =
layout.GetBlockPtr<extractor::Datasources>(memory_ptr, "/common/data_sources_names");
m_datasources = index.GetBlockPtr<extractor::Datasources>("/common/data_sources_names");
intersection_bearings_view =
make_intersection_bearings_view(memory_ptr, layout, "/common/intersection_bearings");
make_intersection_bearings_view(index, "/common/intersection_bearings");
m_entry_class_table = make_entry_classes_view(memory_ptr, layout, "/common/entry_classes");
m_entry_class_table = make_entry_classes_view(index, "/common/entry_classes");
std::tie(m_maneuver_overrides, m_maneuver_override_node_sequences) =
make_maneuver_overrides_views(memory_ptr, layout, "/common/maneuver_overrides");
make_maneuver_overrides_views(index, "/common/maneuver_overrides");
}
public:
@@ -231,8 +225,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
const std::size_t exclude_index)
: allocator(std::move(allocator_))
{
InitializeInternalPointers(
allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
InitializeInternalPointers(allocator->GetIndex(), metric_name, exclude_index);
}
// node and edge information access
@@ -661,16 +654,15 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
QueryGraph query_graph;
void InitializeInternalPointers(const storage::DataLayout &layout,
char *memory_ptr,
void InitializeInternalPointers(const storage::SharedDataIndex &index,
const std::string &metric_name,
const std::size_t exclude_index)
{
mld_partition = make_partition_view(memory_ptr, layout, "/mld/multilevelpartition");
mld_cell_metric = make_filtered_cell_metric_view(
memory_ptr, layout, "/mld/metrics/" + metric_name, exclude_index);
mld_cell_storage = make_cell_storage_view(memory_ptr, layout, "/mld/cellstorage");
query_graph = make_multi_level_graph_view(memory_ptr, layout, "/mld/multilevelgraph");
mld_partition = make_partition_view(index, "/mld/multilevelpartition");
mld_cell_metric =
make_filtered_cell_metric_view(index, "/mld/metrics/" + metric_name, exclude_index);
mld_cell_storage = make_cell_storage_view(index, "/mld/cellstorage");
query_graph = make_multi_level_graph_view(index, "/mld/multilevelgraph");
}
// allocator that keeps the allocation data
@@ -683,8 +675,7 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
const std::size_t exclude_index)
: allocator(std::move(allocator_))
{
InitializeInternalPointers(
allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
InitializeInternalPointers(allocator->GetIndex(), metric_name, exclude_index);
}
const partitioner::MultiLevelPartitionView &GetMultiLevelPartition() const override
include/engine/datafacade/mmap_memory_allocator.hpp
+2 -3
View File
@@ -29,11 +29,10 @@ class MMapMemoryAllocator : public ContiguousBlockAllocator
~MMapMemoryAllocator() override final;
// interface to give access to the datafacades
storage::DataLayout &GetLayout() override final;
char *GetMemory() override final;
const storage::SharedDataIndex &GetIndex() override final;
private:
storage::DataLayout *data_layout;
storage::SharedDataIndex index;
util::vector_view<char> mapped_memory;
boost::iostreams::mapped_file mapped_memory_file;
};
include/engine/datafacade/process_memory_allocator.hpp
+2 -3
View File
@@ -27,11 +27,10 @@ class ProcessMemoryAllocator : public ContiguousBlockAllocator
~ProcessMemoryAllocator() override final;
// interface to give access to the datafacades
const storage::DataLayout &GetLayout() override final;
char *GetMemory() override final;
const storage::SharedDataIndex &GetIndex() override final;
private:
storage::DataLayout internal_layout;
storage::SharedDataIndex index;
std::unique_ptr<char[]> internal_memory;
};
include/engine/datafacade/shared_memory_allocator.hpp
+5 -7
View File
@@ -3,7 +3,7 @@
#include "engine/datafacade/contiguous_block_allocator.hpp"
#include "storage/shared_datatype.hpp"
#include "storage/shared_data_index.hpp"
#include "storage/shared_memory.hpp"
#include <memory>
@@ -23,17 +23,15 @@ namespace datafacade
class SharedMemoryAllocator : public ContiguousBlockAllocator
{
public:
explicit SharedMemoryAllocator(storage::SharedRegionRegister::ShmKey data_shm_key);
explicit SharedMemoryAllocator(const std::vector<storage::SharedRegionRegister::ShmKey> &shm_keys);
~SharedMemoryAllocator() override final;
// interface to give access to the datafacades
const storage::DataLayout &GetLayout() override final;
char *GetMemory() override final;
const storage::SharedDataIndex &GetIndex() override final;
private:
std::size_t layout_size;
storage::DataLayout data_layout;
std::unique_ptr<storage::SharedMemory> m_large_memory;
storage::SharedDataIndex index;
std::vector<std::unique_ptr<storage::SharedMemory>> memory_regions;
};
} // namespace datafacade