Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Load data in two separate data regions

Browse Source
This commit is contained in:
Patrick Niklaus 2018-04-04 23:05:34 +00:00 committed by Patrick Niklaus
parent c7daa521ad
commit fea07f343b
14 changed files with 578 additions and 440 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
include/engine/data_watchdog.hpp
View File

@ -43,18 +43,23 @@ class DataWatchdogImpl<AlgorithmT, datafacade::ContiguousInternalMemoryDataFacad
boost::interprocess::scoped_lock<mutex_type> current_region_lock(barrier.get_mutex()); boost::interprocess::scoped_lock<mutex_type> current_region_lock(barrier.get_mutex());
auto &shared_register = barrier.data(); auto &shared_register = barrier.data();
auto region_id = shared_register.Find(dataset_name + "/data"); auto static_region_id = shared_register.Find(dataset_name + "/static");
if (region_id == storage::SharedRegionRegister::INVALID_REGION_ID) auto updatable_region_id = shared_register.Find(dataset_name + "/updatable");
if (static_region_id == storage::SharedRegionRegister::INVALID_REGION_ID ||
updatable_region_id == storage::SharedRegionRegister::INVALID_REGION_ID)
{ {
throw util::exception("Could not find shared memory region for \"" + dataset_name + throw util::exception("Could not find shared memory region for \"" + dataset_name +
"/data\". Did you run osrm-datastore?"); "/data\". Did you run osrm-datastore?");
} }
shared_region = &shared_register.GetRegion(region_id); static_shared_region = &shared_register.GetRegion(static_region_id);
region = *shared_region; updatable_shared_region = &shared_register.GetRegion(updatable_region_id);
static_region = *static_shared_region;
updatable_region = *updatable_shared_region;
facade_factory = facade_factory =
DataFacadeFactory<datafacade::ContiguousInternalMemoryDataFacade, AlgorithmT>( DataFacadeFactory<datafacade::ContiguousInternalMemoryDataFacade, AlgorithmT>(
std::make_shared<datafacade::SharedMemoryAllocator>(region.shm_key)); std::make_shared<datafacade::SharedMemoryAllocator>(
std::vector<storage::SharedRegionRegister::ShmKey> {static_region.shm_key, updatable_region.shm_key}));
} }
watcher = std::thread(&DataWatchdogImpl::Run, this); watcher = std::thread(&DataWatchdogImpl::Run, this);
@ -83,20 +88,32 @@ class DataWatchdogImpl<AlgorithmT, datafacade::ContiguousInternalMemoryDataFacad
{ {
boost::interprocess::scoped_lock<mutex_type> current_region_lock(barrier.get_mutex()); boost::interprocess::scoped_lock<mutex_type> current_region_lock(barrier.get_mutex());
while (active && region.timestamp == shared_region->timestamp) while (active && static_region.timestamp == static_shared_region->timestamp &&
updatable_region.timestamp == updatable_shared_region->timestamp)
{ {
barrier.wait(current_region_lock); barrier.wait(current_region_lock);
} }
if (region.timestamp != shared_region->timestamp) if (!active)
break;
if (static_region.timestamp != static_shared_region->timestamp)
{ {
region = *shared_region; static_region = *static_shared_region;
}
if (updatable_region.timestamp != updatable_shared_region->timestamp)
{
updatable_region = *updatable_shared_region;
}
util::Log() << "updated facade to regions " << (int)static_region.shm_key << " and "
<< (int)updatable_region.shm_key << " with timestamps "
<< static_region.timestamp << " and " << updatable_region.timestamp;
facade_factory = facade_factory =
DataFacadeFactory<datafacade::ContiguousInternalMemoryDataFacade, AlgorithmT>( DataFacadeFactory<datafacade::ContiguousInternalMemoryDataFacade, AlgorithmT>(
std::make_shared<datafacade::SharedMemoryAllocator>(region.shm_key)); std::make_shared<datafacade::SharedMemoryAllocator>(
util::Log() << "updated facade to region " << (int)region.shm_key std::vector<storage::SharedRegionRegister::ShmKey>{static_region.shm_key, updatable_region.shm_key}));
<< " with timestamp " << region.timestamp;
}
} }
util::Log() << "DataWatchdog thread stopped"; util::Log() << "DataWatchdog thread stopped";
@ -106,8 +123,10 @@ class DataWatchdogImpl<AlgorithmT, datafacade::ContiguousInternalMemoryDataFacad
storage::SharedMonitor<storage::SharedRegionRegister> barrier; storage::SharedMonitor<storage::SharedRegionRegister> barrier;
std::thread watcher; std::thread watcher;
bool active; bool active;
storage::SharedRegion region; storage::SharedRegion static_region;
storage::SharedRegion *shared_region; storage::SharedRegion updatable_region;
storage::SharedRegion *static_shared_region;
storage::SharedRegion *updatable_shared_region;
DataFacadeFactory<datafacade::ContiguousInternalMemoryDataFacade, AlgorithmT> facade_factory; DataFacadeFactory<datafacade::ContiguousInternalMemoryDataFacade, AlgorithmT> facade_factory;
}; };
} }

5
include/engine/datafacade/contiguous_block_allocator.hpp
View File

@ -1,7 +1,7 @@
#ifndef OSRM_ENGINE_DATAFACADE_CONTIGUOUS_BLOCK_ALLOCATOR_HPP_ #ifndef OSRM_ENGINE_DATAFACADE_CONTIGUOUS_BLOCK_ALLOCATOR_HPP_
#define OSRM_ENGINE_DATAFACADE_CONTIGUOUS_BLOCK_ALLOCATOR_HPP_ #define OSRM_ENGINE_DATAFACADE_CONTIGUOUS_BLOCK_ALLOCATOR_HPP_
#include "storage/shared_datatype.hpp" #include "storage/shared_data_index.hpp"
namespace osrm namespace osrm
{ {
@ -16,8 +16,7 @@ class ContiguousBlockAllocator
virtual ~ContiguousBlockAllocator() = default; virtual ~ContiguousBlockAllocator() = default;
// interface to give access to the datafacades // interface to give access to the datafacades
virtual const storage::DataLayout &GetLayout() = 0; virtual const storage::SharedDataIndex &GetIndex() = 0;
virtual char *GetMemory() = 0;
}; };
} // namespace datafacade } // namespace datafacade

67
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
View File

@ -57,17 +57,15 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
std::size_t exclude_index) std::size_t exclude_index)
: allocator(std::move(allocator_)) : allocator(std::move(allocator_))
{ {
InitializeInternalPointers( InitializeInternalPointers(allocator->GetIndex(), metric_name, exclude_index);
allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
} }
void InitializeInternalPointers(const storage::DataLayout &data_layout, void InitializeInternalPointers(const storage::SharedDataIndex &index,
char *memory_block,
const std::string &metric_name, const std::string &metric_name,
const std::size_t exclude_index) const std::size_t exclude_index)
{ {
m_query_graph = make_filtered_graph_view( m_query_graph =
memory_block, data_layout, "/ch/metrics/" + metric_name, exclude_index); make_filtered_graph_view(index, "/ch/metrics/" + metric_name, exclude_index);
} }
// search graph access // search graph access
@ -172,8 +170,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
// allocator that keeps the allocation data // allocator that keeps the allocation data
std::shared_ptr<ContiguousBlockAllocator> allocator; std::shared_ptr<ContiguousBlockAllocator> allocator;
void InitializeInternalPointers(const storage::DataLayout &layout, void InitializeInternalPointers(const storage::SharedDataIndex &index,
char *memory_ptr,
const std::string &metric_name, const std::string &metric_name,
const std::size_t exclude_index) const std::size_t exclude_index)
{ {
@ -181,46 +178,43 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
(void)metric_name; (void)metric_name;
m_profile_properties = m_profile_properties =
layout.GetBlockPtr<extractor::ProfileProperties>(memory_ptr, "/common/properties"); index.GetBlockPtr<extractor::ProfileProperties>("/common/properties");
exclude_mask = m_profile_properties->excludable_classes[exclude_index]; exclude_mask = m_profile_properties->excludable_classes[exclude_index];
m_check_sum = m_check_sum = *index.GetBlockPtr<std::uint32_t>("/common/connectivity_checksum");
*layout.GetBlockPtr<std::uint32_t>(memory_ptr, "/common/connectivity_checksum");
std::tie(m_coordinate_list, m_osmnodeid_list) = std::tie(m_coordinate_list, m_osmnodeid_list) =
make_nbn_data_view(memory_ptr, layout, "/common/nbn_data"); make_nbn_data_view(index, "/common/nbn_data");
m_static_rtree = make_search_tree_view(memory_ptr, layout, "/common/rtree"); m_static_rtree = make_search_tree_view(index, "/common/rtree");
m_geospatial_query.reset( m_geospatial_query.reset(
new SharedGeospatialQuery(m_static_rtree, m_coordinate_list, *this)); new SharedGeospatialQuery(m_static_rtree, m_coordinate_list, *this));
edge_based_node_data = make_ebn_data_view(memory_ptr, layout, "/common/ebg_node_data"); edge_based_node_data = make_ebn_data_view(index, "/common/ebg_node_data");
turn_data = make_turn_data_view(memory_ptr, layout, "/common/turn_data"); turn_data = make_turn_data_view(index, "/common/turn_data");
m_name_table = make_name_table_view(memory_ptr, layout, "/common/names"); m_name_table = make_name_table_view(index, "/common/names");
std::tie(m_lane_description_offsets, m_lane_description_masks) = std::tie(m_lane_description_offsets, m_lane_description_masks) =
make_turn_lane_description_views(memory_ptr, layout, "/common/turn_lanes"); make_turn_lane_description_views(index, "/common/turn_lanes");
m_lane_tupel_id_pairs = make_lane_data_view(memory_ptr, layout, "/common/turn_lanes"); m_lane_tupel_id_pairs = make_lane_data_view(index, "/common/turn_lanes");
m_turn_weight_penalties = make_turn_weight_view(memory_ptr, layout, "/common/turn_penalty"); m_turn_weight_penalties = make_turn_weight_view(index, "/common/turn_penalty");
m_turn_duration_penalties = m_turn_duration_penalties = make_turn_duration_view(index, "/common/turn_penalty");
make_turn_duration_view(memory_ptr, layout, "/common/turn_penalty");
segment_data = make_segment_data_view(memory_ptr, layout, "/common/segment_data"); segment_data = make_segment_data_view(index, "/common/segment_data");
m_datasources = m_datasources = index.GetBlockPtr<extractor::Datasources>("/common/data_sources_names");
layout.GetBlockPtr<extractor::Datasources>(memory_ptr, "/common/data_sources_names");
intersection_bearings_view = intersection_bearings_view =
make_intersection_bearings_view(memory_ptr, layout, "/common/intersection_bearings"); make_intersection_bearings_view(index, "/common/intersection_bearings");
m_entry_class_table = make_entry_classes_view(memory_ptr, layout, "/common/entry_classes"); m_entry_class_table = make_entry_classes_view(index, "/common/entry_classes");
std::tie(m_maneuver_overrides, m_maneuver_override_node_sequences) = std::tie(m_maneuver_overrides, m_maneuver_override_node_sequences) =
make_maneuver_overrides_views(memory_ptr, layout, "/common/maneuver_overrides"); make_maneuver_overrides_views(index, "/common/maneuver_overrides");
} }
public: public:
@ -231,8 +225,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
const std::size_t exclude_index) const std::size_t exclude_index)
: allocator(std::move(allocator_)) : allocator(std::move(allocator_))
{ {
InitializeInternalPointers( InitializeInternalPointers(allocator->GetIndex(), metric_name, exclude_index);
allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
} }
// node and edge information access // node and edge information access
@ -661,16 +654,15 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
QueryGraph query_graph; QueryGraph query_graph;
void InitializeInternalPointers(const storage::DataLayout &layout, void InitializeInternalPointers(const storage::SharedDataIndex &index,
char *memory_ptr,
const std::string &metric_name, const std::string &metric_name,
const std::size_t exclude_index) const std::size_t exclude_index)
{ {
mld_partition = make_partition_view(memory_ptr, layout, "/mld/multilevelpartition"); mld_partition = make_partition_view(index, "/mld/multilevelpartition");
mld_cell_metric = make_filtered_cell_metric_view( mld_cell_metric =
memory_ptr, layout, "/mld/metrics/" + metric_name, exclude_index); make_filtered_cell_metric_view(index, "/mld/metrics/" + metric_name, exclude_index);
mld_cell_storage = make_cell_storage_view(memory_ptr, layout, "/mld/cellstorage"); mld_cell_storage = make_cell_storage_view(index, "/mld/cellstorage");
query_graph = make_multi_level_graph_view(memory_ptr, layout, "/mld/multilevelgraph"); query_graph = make_multi_level_graph_view(index, "/mld/multilevelgraph");
} }
// allocator that keeps the allocation data // allocator that keeps the allocation data
@ -683,8 +675,7 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
const std::size_t exclude_index) const std::size_t exclude_index)
: allocator(std::move(allocator_)) : allocator(std::move(allocator_))
{ {
InitializeInternalPointers( InitializeInternalPointers(allocator->GetIndex(), metric_name, exclude_index);
allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
} }
const partitioner::MultiLevelPartitionView &GetMultiLevelPartition() const override const partitioner::MultiLevelPartitionView &GetMultiLevelPartition() const override

5
include/engine/datafacade/mmap_memory_allocator.hpp
View File

@ -29,11 +29,10 @@ class MMapMemoryAllocator : public ContiguousBlockAllocator
~MMapMemoryAllocator() override final; ~MMapMemoryAllocator() override final;
// interface to give access to the datafacades // interface to give access to the datafacades
storage::DataLayout &GetLayout() override final; const storage::SharedDataIndex &GetIndex() override final;
char *GetMemory() override final;
private: private:
storage::DataLayout *data_layout; storage::SharedDataIndex index;
util::vector_view<char> mapped_memory; util::vector_view<char> mapped_memory;
boost::iostreams::mapped_file mapped_memory_file; boost::iostreams::mapped_file mapped_memory_file;
}; };

5
include/engine/datafacade/process_memory_allocator.hpp
View File

@ -27,11 +27,10 @@ class ProcessMemoryAllocator : public ContiguousBlockAllocator
~ProcessMemoryAllocator() override final; ~ProcessMemoryAllocator() override final;
// interface to give access to the datafacades // interface to give access to the datafacades
const storage::DataLayout &GetLayout() override final; const storage::SharedDataIndex &GetIndex() override final;
char *GetMemory() override final;
private: private:
storage::DataLayout internal_layout; storage::SharedDataIndex index;
std::unique_ptr<char[]> internal_memory; std::unique_ptr<char[]> internal_memory;
}; };

12
include/engine/datafacade/shared_memory_allocator.hpp
View File

@ -3,7 +3,7 @@
#include "engine/datafacade/contiguous_block_allocator.hpp" #include "engine/datafacade/contiguous_block_allocator.hpp"
#include "storage/shared_datatype.hpp" #include "storage/shared_data_index.hpp"
#include "storage/shared_memory.hpp" #include "storage/shared_memory.hpp"
#include <memory> #include <memory>
@ -23,17 +23,15 @@ namespace datafacade
class SharedMemoryAllocator : public ContiguousBlockAllocator class SharedMemoryAllocator : public ContiguousBlockAllocator
{ {
public: public:
explicit SharedMemoryAllocator(storage::SharedRegionRegister::ShmKey data_shm_key); explicit SharedMemoryAllocator(const std::vector<storage::SharedRegionRegister::ShmKey> &shm_keys);
~SharedMemoryAllocator() override final; ~SharedMemoryAllocator() override final;
// interface to give access to the datafacades // interface to give access to the datafacades
const storage::DataLayout &GetLayout() override final; const storage::SharedDataIndex &GetIndex() override final;
char *GetMemory() override final;
private: private:
std::size_t layout_size; storage::SharedDataIndex index;
storage::DataLayout data_layout; std::vector<std::unique_ptr<storage::SharedMemory>> memory_regions;
std::unique_ptr<storage::SharedMemory> m_large_memory;
}; };
} // namespace datafacade } // namespace datafacade

12
include/engine/datafacade_factory.hpp
View File

@ -46,15 +46,14 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
template <typename AllocatorT> template <typename AllocatorT>
DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::true_type) DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::true_type)
{ {
const auto &layout = allocator->GetLayout(); const auto &index = allocator->GetIndex();
properties = layout.template GetBlockPtr<extractor::ProfileProperties>( properties = index.template GetBlockPtr<extractor::ProfileProperties>("/common/properties");
allocator->GetMemory(), "/common/properties");
const auto &metric_name = properties->GetWeightName(); const auto &metric_name = properties->GetWeightName();
std::vector<std::string> exclude_prefixes; std::vector<std::string> exclude_prefixes;
auto exclude_path = std::string("/") + routing_algorithms::identifier<AlgorithmT>() + auto exclude_path = std::string("/") + routing_algorithms::identifier<AlgorithmT>() +
std::string("/metrics/") + metric_name + "/exclude/"; std::string("/metrics/") + metric_name + "/exclude/";
layout.List(exclude_path, std::back_inserter(exclude_prefixes)); index.List(exclude_path, std::back_inserter(exclude_prefixes));
facades.resize(exclude_prefixes.size()); facades.resize(exclude_prefixes.size());
if (facades.empty()) if (facades.empty())
@ -89,9 +88,8 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
template <typename AllocatorT> template <typename AllocatorT>
DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::false_type) DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::false_type)
{ {
const auto &layout = allocator->GetLayout(); const auto &index = allocator->GetIndex();
properties = layout.template GetBlockPtr<extractor::ProfileProperties>( properties = index.template GetBlockPtr<extractor::ProfileProperties>("/common/properties");
allocator->GetMemory(), "/common/properties");
const auto &metric_name = properties->GetWeightName(); const auto &metric_name = properties->GetWeightName();
facades.push_back(std::make_shared<const Facade>(allocator, metric_name, 0)); facades.push_back(std::make_shared<const Facade>(allocator, metric_name, 0));
} }

82
include/storage/shared_data_index.hpp Normal file
View File

@ -0,0 +1,82 @@
#ifndef OSRM_STORAGE_SHARED_DATA_INDEX_HPP
#define OSRM_STORAGE_SHARED_DATA_INDEX_HPP
#include "storage/shared_datatype.hpp"
#include <boost/function_output_iterator.hpp>
#include <unordered_map>
namespace osrm
{
namespace storage
{
// This class wraps one or more shared memory regions with the associated data layout
// to abstract away in which region a block of memory is stored.
class SharedDataIndex
{
public:
struct AllocatedRegion
{
char *memory_ptr;
DataLayout layout;
};
SharedDataIndex() = default;
SharedDataIndex(std::vector<AllocatedRegion> regions_) : regions(std::move(regions_))
{
// Build mapping from block name to region
for (auto index : util::irange<std::uint32_t>(0, regions.size()))
{
regions[index].layout.List("",
boost::make_function_output_iterator([&](const auto &name) {
block_to_region[name] = index;
}));
}
}
template <typename OutIter> void List(const std::string &name_prefix, OutIter out) const
{
for (const auto &region : regions)
{
region.layout.List(name_prefix, out);
}
}
template <typename T> auto GetBlockPtr(const std::string &name) const
{
const auto index_iter = block_to_region.find(name);
const auto &region = regions[index_iter->second];
return region.layout.GetBlockPtr<T>(region.memory_ptr, name);
}
template <typename T> auto GetBlockPtr(const std::string &name)
{
const auto index_iter = block_to_region.find(name);
const auto &region = regions[index_iter->second];
return region.layout.GetBlockPtr<T>(region.memory_ptr, name);
}
std::size_t GetBlockEntries(const std::string &name) const
{
const auto index_iter = block_to_region.find(name);
const auto &region = regions[index_iter->second];
return region.layout.GetBlockEntries(name);
}
std::size_t GetBlockSize(const std::string &name) const
{
const auto index_iter = block_to_region.find(name);
const auto &region = regions[index_iter->second];
return region.layout.GetBlockSize(name);
}
private:
std::vector<AllocatedRegion> regions;
std::unordered_map<std::string, std::uint32_t> block_to_region;
};
}
}
#endif

7
include/storage/storage.hpp
View File

@ -29,6 +29,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define STORAGE_HPP #define STORAGE_HPP
#include "storage/shared_datatype.hpp" #include "storage/shared_datatype.hpp"
#include "storage/shared_data_index.hpp"
#include "storage/storage_config.hpp" #include "storage/storage_config.hpp"
#include <boost/filesystem/path.hpp> #include <boost/filesystem/path.hpp>
@ -46,8 +47,10 @@ class Storage
int Run(int max_wait, const std::string &name); int Run(int max_wait, const std::string &name);
void PopulateLayout(DataLayout &layout); void PopulateStaticLayout(DataLayout &layout);
void PopulateData(const DataLayout &layout, char *memory_ptr); void PopulateUpdatableLayout(DataLayout &layout);
void PopulateStaticData(const SharedDataIndex &index);
void PopulateUpdatableData(const SharedDataIndex &index);
private: private:
StorageConfig config; StorageConfig config;

227
include/storage/view_factory.hpp
View File

@ -1,7 +1,7 @@
#ifndef OSRM_STOARGE_VIEW_FACTORY_HPP #ifndef OSRM_STOARGE_VIEW_FACTORY_HPP
#define OSRM_STOARGE_VIEW_FACTORY_HPP #define OSRM_STOARGE_VIEW_FACTORY_HPP
#include "storage/shared_datatype.hpp" #include "storage/shared_data_index.hpp"
#include "contractor/contracted_metric.hpp" #include "contractor/contracted_metric.hpp"
#include "contractor/query_graph.hpp" #include "contractor/query_graph.hpp"
@ -45,88 +45,78 @@ namespace storage
{ {
template <typename T> template <typename T>
util::vector_view<T> util::vector_view<T> make_vector_view(const SharedDataIndex &index, const std::string &name)
make_vector_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
{ {
return util::vector_view<T>(layout.GetBlockPtr<T>(memory_ptr, name), return util::vector_view<T>(index.GetBlockPtr<T>(name), index.GetBlockEntries(name));
layout.GetBlockEntries(name));
} }
template<> template<>
inline util::vector_view<bool> inline util::vector_view<bool> make_vector_view(const SharedDataIndex &index, const std::string &name)
make_vector_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
{ {
return util::vector_view<bool>( return util::vector_view<bool>(index.GetBlockPtr<util::vector_view<bool>::Word>(name),
layout.GetBlockPtr<util::vector_view<bool>::Word>(memory_ptr, name), index.GetBlockEntries(name));
layout.GetBlockEntries(name));
} }
template <typename T, std::size_t Bits> template <typename T, std::size_t Bits>
util::PackedVectorView<T, Bits> util::PackedVectorView<T, Bits> make_packed_vector_view(const SharedDataIndex &index,
make_packed_vector_view(char *memory_ptr, const DataLayout &layout, const std::string &name) const std::string &name)
{ {
using V = util::PackedVectorView<T, Bits>; using V = util::PackedVectorView<T, Bits>;
auto packed_internal = util::vector_view<typename V::block_type>( auto packed_internal = util::vector_view<typename V::block_type>(
layout.GetBlockPtr<typename V::block_type>(memory_ptr, name + "/packed"), index.GetBlockPtr<typename V::block_type>(name + "/packed"),
layout.GetBlockEntries(name + "/packed")); index.GetBlockEntries(name + "/packed"));
// the real size needs to come from an external source // the real size needs to come from an external source
return V{packed_internal, std::numeric_limits<std::size_t>::max()}; return V{packed_internal, std::numeric_limits<std::size_t>::max()};
} }
inline auto inline auto make_name_table_view(const SharedDataIndex &index, const std::string &name)
make_name_table_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
{ {
auto blocks = make_vector_view<extractor::NameTableView::IndexedData::BlockReference>( auto blocks = make_vector_view<extractor::NameTableView::IndexedData::BlockReference>(
memory_ptr, layout, name + "/blocks"); index, name + "/blocks");
auto values = make_vector_view<extractor::NameTableView::IndexedData::ValueType>( auto values =
memory_ptr, layout, name + "/values"); make_vector_view<extractor::NameTableView::IndexedData::ValueType>(index, name + "/values");
extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)}; extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
return extractor::NameTableView{index_data_view}; return extractor::NameTableView{index_data_view};
} }
inline auto make_lane_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name) inline auto make_lane_data_view(const SharedDataIndex &index, const std::string &name)
{ {
return make_vector_view<util::guidance::LaneTupleIdPair>(memory_ptr, layout, name + "/data"); return make_vector_view<util::guidance::LaneTupleIdPair>(index, name + "/data");
} }
inline auto make_turn_lane_description_views(char *memory_ptr, inline auto make_turn_lane_description_views(const SharedDataIndex &index, const std::string &name)
const DataLayout &layout,
const std::string &name)
{ {
auto offsets = make_vector_view<std::uint32_t>(memory_ptr, layout, name + "/offsets"); auto offsets = make_vector_view<std::uint32_t>(index, name + "/offsets");
auto masks = auto masks = make_vector_view<extractor::TurnLaneType::Mask>(index, name + "/masks");
make_vector_view<extractor::TurnLaneType::Mask>(memory_ptr, layout, name + "/masks");
return std::make_tuple(offsets, masks); return std::make_tuple(offsets, masks);
} }
inline auto make_ebn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name) inline auto make_ebn_data_view(const SharedDataIndex &index, const std::string &name)
{ {
auto edge_based_node_data = auto edge_based_node_data = make_vector_view<extractor::EdgeBasedNode>(index, name + "/nodes");
make_vector_view<extractor::EdgeBasedNode>(memory_ptr, layout, name + "/nodes"); auto annotation_data =
auto annotation_data = make_vector_view<extractor::NodeBasedEdgeAnnotation>( make_vector_view<extractor::NodeBasedEdgeAnnotation>(index, name + "/annotations");
memory_ptr, layout, name + "/annotations");
return extractor::EdgeBasedNodeDataView(std::move(edge_based_node_data), return extractor::EdgeBasedNodeDataView(std::move(edge_based_node_data),
std::move(annotation_data)); std::move(annotation_data));
} }
inline auto make_turn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name) inline auto make_turn_data_view(const SharedDataIndex &index, const std::string &name)
{ {
auto lane_data_ids = make_vector_view<LaneDataID>(memory_ptr, layout, name + "/lane_data_ids"); auto lane_data_ids = make_vector_view<LaneDataID>(index, name + "/lane_data_ids");
const auto turn_instructions = make_vector_view<guidance::TurnInstruction>( const auto turn_instructions =
memory_ptr, layout, name + "/turn_instructions"); make_vector_view<guidance::TurnInstruction>(index, name + "/turn_instructions");
const auto entry_class_ids = const auto entry_class_ids = make_vector_view<EntryClassID>(index, name + "/entry_class_ids");
make_vector_view<EntryClassID>(memory_ptr, layout, name + "/entry_class_ids");
const auto pre_turn_bearings = const auto pre_turn_bearings =
make_vector_view<guidance::TurnBearing>(memory_ptr, layout, name + "/pre_turn_bearings"); make_vector_view<guidance::TurnBearing>(index, name + "/pre_turn_bearings");
const auto post_turn_bearings = const auto post_turn_bearings =
make_vector_view<guidance::TurnBearing>(memory_ptr, layout, name + "/post_turn_bearings"); make_vector_view<guidance::TurnBearing>(index, name + "/post_turn_bearings");
return guidance::TurnDataView(std::move(turn_instructions), return guidance::TurnDataView(std::move(turn_instructions),
std::move(lane_data_ids), std::move(lane_data_ids),
@ -135,40 +125,39 @@ inline auto make_turn_data_view(char *memory_ptr, const DataLayout &layout, cons
std::move(post_turn_bearings)); std::move(post_turn_bearings));
} }
inline auto inline auto make_segment_data_view(const SharedDataIndex &index, const std::string &name)
make_segment_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
{ {
auto geometry_begin_indices = make_vector_view<unsigned>(memory_ptr, layout, name + "/index"); auto geometry_begin_indices = make_vector_view<unsigned>(index, name + "/index");
auto node_list = make_vector_view<NodeID>(memory_ptr, layout, name + "/nodes"); auto node_list = make_vector_view<NodeID>(index, name + "/nodes");
auto num_entries = layout.GetBlockEntries(name + "/nodes"); auto num_entries = index.GetBlockEntries(name + "/nodes");
extractor::SegmentDataView::SegmentWeightVector fwd_weight_list( extractor::SegmentDataView::SegmentWeightVector fwd_weight_list(
make_vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>( make_vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
memory_ptr, layout, name + "/forward_weights/packed"), index, name + "/forward_weights/packed"),
num_entries); num_entries);
extractor::SegmentDataView::SegmentWeightVector rev_weight_list( extractor::SegmentDataView::SegmentWeightVector rev_weight_list(
make_vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>( make_vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
memory_ptr, layout, name + "/reverse_weights/packed"), index, name + "/reverse_weights/packed"),
num_entries); num_entries);
extractor::SegmentDataView::SegmentDurationVector fwd_duration_list( extractor::SegmentDataView::SegmentDurationVector fwd_duration_list(
make_vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>( make_vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
memory_ptr, layout, name + "/forward_durations/packed"), index, name + "/forward_durations/packed"),
num_entries); num_entries);
extractor::SegmentDataView::SegmentDurationVector rev_duration_list( extractor::SegmentDataView::SegmentDurationVector rev_duration_list(
make_vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>( make_vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
memory_ptr, layout, name + "/reverse_durations/packed"), index, name + "/reverse_durations/packed"),
num_entries); num_entries);
auto fwd_datasources_list = auto fwd_datasources_list =
make_vector_view<DatasourceID>(memory_ptr, layout, name + "/forward_data_sources"); make_vector_view<DatasourceID>(index, name + "/forward_data_sources");
auto rev_datasources_list = auto rev_datasources_list =
make_vector_view<DatasourceID>(memory_ptr, layout, name + "/reverse_data_sources"); make_vector_view<DatasourceID>(index, name + "/reverse_data_sources");
return extractor::SegmentDataView{std::move(geometry_begin_indices), return extractor::SegmentDataView{std::move(geometry_begin_indices),
std::move(node_list), std::move(node_list),
@ -180,52 +169,46 @@ make_segment_data_view(char *memory_ptr, const DataLayout &layout, const std::st
std::move(rev_datasources_list)}; std::move(rev_datasources_list)};
} }
inline auto inline auto make_coordinates_view(const SharedDataIndex &index, const std::string &name)
make_coordinates_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
{ {
return make_vector_view<util::Coordinate>(memory_ptr, layout, name); return make_vector_view<util::Coordinate>(index, name);
} }
inline auto make_osm_ids_view(char *memory_ptr, const DataLayout &layout, const std::string &name) inline auto make_osm_ids_view(const SharedDataIndex &index, const std::string &name)
{ {
return make_packed_vector_view<extractor::PackedOSMIDsView::value_type, return make_packed_vector_view<extractor::PackedOSMIDsView::value_type,
extractor::PackedOSMIDsView::value_size>( extractor::PackedOSMIDsView::value_size>(index, name);
memory_ptr, layout, name);
} }
inline auto make_nbn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name) inline auto make_nbn_data_view(const SharedDataIndex &index, const std::string &name)
{ {
return std::make_tuple(make_coordinates_view(memory_ptr, layout, name + "/coordinates"), return std::make_tuple(make_coordinates_view(index, name + "/coordinates"),
make_osm_ids_view(memory_ptr, layout, name + "/osm_node_ids")); make_osm_ids_view(index, name + "/osm_node_ids"));
} }
inline auto inline auto make_turn_weight_view(const SharedDataIndex &index, const std::string &name)
make_turn_weight_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
{ {
return make_vector_view<TurnPenalty>(memory_ptr, layout, name + "/weight"); return make_vector_view<TurnPenalty>(index, name + "/weight");
} }
inline auto inline auto make_turn_duration_view(const SharedDataIndex &index, const std::string &name)
make_turn_duration_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
{ {
return make_vector_view<TurnPenalty>(memory_ptr, layout, name + "/duration"); return make_vector_view<TurnPenalty>(index, name + "/duration");
} }
inline auto inline auto make_search_tree_view(const SharedDataIndex &index, const std::string &name)
make_search_tree_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
{ {
using RTreeLeaf = extractor::EdgeBasedNodeSegment; using RTreeLeaf = extractor::EdgeBasedNodeSegment;
using RTreeNode = util::StaticRTree<RTreeLeaf, storage::Ownership::View>::TreeNode; using RTreeNode = util::StaticRTree<RTreeLeaf, storage::Ownership::View>::TreeNode;
const auto search_tree = make_vector_view<RTreeNode>(memory_ptr, layout, name + "/search_tree"); const auto search_tree = make_vector_view<RTreeNode>(index, name + "/search_tree");
const auto rtree_level_starts = const auto rtree_level_starts =
make_vector_view<std::uint64_t>(memory_ptr, layout, name + "/search_tree_level_starts"); make_vector_view<std::uint64_t>(index, name + "/search_tree_level_starts");
const auto coordinates = const auto coordinates = make_coordinates_view(index, "/common/nbn_data/coordinates");
make_coordinates_view(memory_ptr, layout, "/common/nbn_data/coordinates");
const char *path = layout.GetBlockPtr<char>(memory_ptr, name + "/file_index_path"); const char *path = index.template GetBlockPtr<char>(name + "/file_index_path");
if (!boost::filesystem::exists(boost::filesystem::path{path})) if (!boost::filesystem::exists(boost::filesystem::path{path}))
{ {
@ -237,72 +220,63 @@ make_search_tree_view(char *memory_ptr, const DataLayout &layout, const std::str
std::move(search_tree), std::move(rtree_level_starts), path, std::move(coordinates)}; std::move(search_tree), std::move(rtree_level_starts), path, std::move(coordinates)};
} }
inline auto inline auto make_intersection_bearings_view(const SharedDataIndex &index, const std::string &name)
make_intersection_bearings_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
{ {
auto bearing_offsets = auto bearing_offsets =
make_vector_view<unsigned>(memory_ptr, layout, name + "/class_id_to_ranges/block_offsets"); make_vector_view<unsigned>(index, name + "/class_id_to_ranges/block_offsets");
auto bearing_blocks = make_vector_view<util::RangeTable<16, storage::Ownership::View>::BlockT>( auto bearing_blocks = make_vector_view<util::RangeTable<16, storage::Ownership::View>::BlockT>(
memory_ptr, layout, name + "/class_id_to_ranges/diff_blocks"); index, name + "/class_id_to_ranges/diff_blocks");
auto bearing_values = auto bearing_values = make_vector_view<DiscreteBearing>(index, name + "/bearing_values");
make_vector_view<DiscreteBearing>(memory_ptr, layout, name + "/bearing_values");
util::RangeTable<16, storage::Ownership::View> bearing_range_table( util::RangeTable<16, storage::Ownership::View> bearing_range_table(
std::move(bearing_offsets), std::move(bearing_offsets),
std::move(bearing_blocks), std::move(bearing_blocks),
static_cast<unsigned>(bearing_values.size())); static_cast<unsigned>(bearing_values.size()));
auto bearing_class_id = auto bearing_class_id = make_vector_view<BearingClassID>(index, name + "/node_to_class_id");
make_vector_view<BearingClassID>(memory_ptr, layout, name + "/node_to_class_id");
return extractor::IntersectionBearingsView{ return extractor::IntersectionBearingsView{
std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)}; std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)};
} }
inline auto inline auto make_entry_classes_view(const SharedDataIndex &index, const std::string &name)
make_entry_classes_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
{ {
return make_vector_view<util::guidance::EntryClass>(memory_ptr, layout, name); return make_vector_view<util::guidance::EntryClass>(index, name);
} }
inline auto inline auto make_contracted_metric_view(const SharedDataIndex &index, const std::string &name)
make_contracted_metric_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
{ {
auto node_list = make_vector_view<contractor::QueryGraphView::NodeArrayEntry>( auto node_list = make_vector_view<contractor::QueryGraphView::NodeArrayEntry>(
memory_ptr, layout, name + "/contracted_graph/node_array"); index, name + "/contracted_graph/node_array");
auto edge_list = make_vector_view<contractor::QueryGraphView::EdgeArrayEntry>( auto edge_list = make_vector_view<contractor::QueryGraphView::EdgeArrayEntry>(
memory_ptr, layout, name + "/contracted_graph/edge_array"); index, name + "/contracted_graph/edge_array");
std::vector<util::vector_view<bool>> edge_filter; std::vector<util::vector_view<bool>> edge_filter;
layout.List(name + "/exclude", index.List(name + "/exclude",
boost::make_function_output_iterator([&](const auto &filter_name) { boost::make_function_output_iterator([&](const auto &filter_name) {
edge_filter.push_back(make_vector_view<bool>(memory_ptr, layout, filter_name)); edge_filter.push_back(make_vector_view<bool>(index, filter_name));
})); }));
return contractor::ContractedMetricView{{std::move(node_list), std::move(edge_list)}, return contractor::ContractedMetricView{{std::move(node_list), std::move(edge_list)},
std::move(edge_filter)}; std::move(edge_filter)};
} }
inline auto make_partition_view(char *memory_ptr, const DataLayout &layout, const std::string &name) inline auto make_partition_view(const SharedDataIndex &index, const std::string &name)
{ {
auto level_data_ptr = layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData>( auto level_data_ptr =
memory_ptr, name + "/level_data"); index.template GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData>(name +
auto partition = make_vector_view<PartitionID>(memory_ptr, layout, name + "/partition"); "/level_data");
auto cell_to_children = auto partition = make_vector_view<PartitionID>(index, name + "/partition");
make_vector_view<CellID>(memory_ptr, layout, name + "/cell_to_children"); auto cell_to_children = make_vector_view<CellID>(index, name + "/cell_to_children");
return partitioner::MultiLevelPartitionView{ return partitioner::MultiLevelPartitionView{
level_data_ptr, std::move(partition), std::move(cell_to_children)}; level_data_ptr, std::move(partition), std::move(cell_to_children)};
} }
inline auto inline auto make_cell_storage_view(const SharedDataIndex &index, const std::string &name)
make_cell_storage_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
{ {
auto source_boundary = make_vector_view<NodeID>(memory_ptr, layout, name + "/source_boundary"); auto source_boundary = make_vector_view<NodeID>(index, name + "/source_boundary");
auto destination_boundary = auto destination_boundary = make_vector_view<NodeID>(index, name + "/destination_boundary");
make_vector_view<NodeID>(memory_ptr, layout, name + "/destination_boundary"); auto cells = make_vector_view<partitioner::CellStorageView::CellData>(index, name + "/cells");
auto cells = make_vector_view<partitioner::CellStorageView::CellData>( auto level_offsets = make_vector_view<std::uint64_t>(index, name + "/level_to_cell_offset");
memory_ptr, layout, name + "/cells");
auto level_offsets =
make_vector_view<std::uint64_t>(memory_ptr, layout, name + "/level_to_cell_offset");
return partitioner::CellStorageView{std::move(source_boundary), return partitioner::CellStorageView{std::move(source_boundary),
std::move(destination_boundary), std::move(destination_boundary),
@ -310,8 +284,7 @@ make_cell_storage_view(char *memory_ptr, const DataLayout &layout, const std::st
std::move(level_offsets)}; std::move(level_offsets)};
} }
inline auto make_filtered_cell_metric_view(char *memory_ptr, inline auto make_filtered_cell_metric_view(const SharedDataIndex &index,
const DataLayout &layout,
const std::string &name, const std::string &name,
const std::size_t exclude_index) const std::size_t exclude_index)
{ {
@ -321,26 +294,25 @@ inline auto make_filtered_cell_metric_view(char *memory_ptr,
auto weights_block_id = prefix + "/weights"; auto weights_block_id = prefix + "/weights";
auto durations_block_id = prefix + "/durations"; auto durations_block_id = prefix + "/durations";
auto weights = make_vector_view<EdgeWeight>(memory_ptr, layout, weights_block_id); auto weights = make_vector_view<EdgeWeight>(index, weights_block_id);
auto durations = make_vector_view<EdgeDuration>(memory_ptr, layout, durations_block_id); auto durations = make_vector_view<EdgeDuration>(index, durations_block_id);
return customizer::CellMetricView{std::move(weights), std::move(durations)}; return customizer::CellMetricView{std::move(weights), std::move(durations)};
} }
inline auto inline auto make_cell_metric_view(const SharedDataIndex &index, const std::string &name)
make_cell_metric_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
{ {
std::vector<customizer::CellMetricView> cell_metric_excludes; std::vector<customizer::CellMetricView> cell_metric_excludes;
std::vector<std::string> metric_prefix_names; std::vector<std::string> metric_prefix_names;
layout.List(name + "/exclude/", std::back_inserter(metric_prefix_names)); index.List(name + "/exclude/", std::back_inserter(metric_prefix_names));
for (const auto &prefix : metric_prefix_names) for (const auto &prefix : metric_prefix_names)
{ {
auto weights_block_id = prefix + "/weights"; auto weights_block_id = prefix + "/weights";
auto durations_block_id = prefix + "/durations"; auto durations_block_id = prefix + "/durations";
auto weights = make_vector_view<EdgeWeight>(memory_ptr, layout, weights_block_id); auto weights = make_vector_view<EdgeWeight>(index, weights_block_id);
auto durations = make_vector_view<EdgeDuration>(memory_ptr, layout, durations_block_id); auto durations = make_vector_view<EdgeDuration>(index, durations_block_id);
cell_metric_excludes.push_back( cell_metric_excludes.push_back(
customizer::CellMetricView{std::move(weights), std::move(durations)}); customizer::CellMetricView{std::move(weights), std::move(durations)});
@ -349,42 +321,39 @@ make_cell_metric_view(char *memory_ptr, const DataLayout &layout, const std::str
return cell_metric_excludes; return cell_metric_excludes;
} }
inline auto inline auto make_multi_level_graph_view(const SharedDataIndex &index, const std::string &name)
make_multi_level_graph_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
{ {
auto node_list = make_vector_view<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry>( auto node_list = make_vector_view<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry>(
memory_ptr, layout, name + "/node_array"); index, name + "/node_array");
auto edge_list = make_vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry>( auto edge_list = make_vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry>(
memory_ptr, layout, name + "/edge_array"); index, name + "/edge_array");
auto node_to_offset = make_vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset>( auto node_to_offset = make_vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset>(
memory_ptr, layout, name + "/node_to_edge_offset"); index, name + "/node_to_edge_offset");
return customizer::MultiLevelEdgeBasedGraphView( return customizer::MultiLevelEdgeBasedGraphView(
std::move(node_list), std::move(edge_list), std::move(node_to_offset)); std::move(node_list), std::move(edge_list), std::move(node_to_offset));
} }
inline auto inline auto make_maneuver_overrides_views(const SharedDataIndex &index, const std::string &name)
make_maneuver_overrides_views(char *memory_ptr, const DataLayout &layout, const std::string &name)
{ {
auto maneuver_overrides = make_vector_view<extractor::StorageManeuverOverride>( auto maneuver_overrides =
memory_ptr, layout, name + "/overrides"); make_vector_view<extractor::StorageManeuverOverride>(index, name + "/overrides");
auto maneuver_override_node_sequences = auto maneuver_override_node_sequences =
make_vector_view<NodeID>(memory_ptr, layout, name + "/node_sequences"); make_vector_view<NodeID>(index, name + "/node_sequences");
return std::make_tuple(maneuver_overrides, maneuver_override_node_sequences); return std::make_tuple(maneuver_overrides, maneuver_override_node_sequences);
} }
inline auto make_filtered_graph_view(char *memory_ptr, inline auto make_filtered_graph_view(const SharedDataIndex &index,
const DataLayout &layout,
const std::string &name, const std::string &name,
const std::size_t exclude_index) const std::size_t exclude_index)
{ {
auto exclude_prefix = name + "/exclude/" + std::to_string(exclude_index); auto exclude_prefix = name + "/exclude/" + std::to_string(exclude_index);
auto edge_filter = make_vector_view<bool>(memory_ptr, layout, exclude_prefix + "/edge_filter"); auto edge_filter = make_vector_view<bool>(index, exclude_prefix + "/edge_filter");
auto node_list = make_vector_view<contractor::QueryGraphView::NodeArrayEntry>( auto node_list = make_vector_view<contractor::QueryGraphView::NodeArrayEntry>(
memory_ptr, layout, name + "/contracted_graph/node_array"); index, name + "/contracted_graph/node_array");
auto edge_list = make_vector_view<contractor::QueryGraphView::EdgeArrayEntry>( auto edge_list = make_vector_view<contractor::QueryGraphView::EdgeArrayEntry>(
memory_ptr, layout, name + "/contracted_graph/edge_array"); index, name + "/contracted_graph/edge_array");
return util::FilteredGraphView<contractor::QueryGraphView>({node_list, edge_list}, edge_filter); return util::FilteredGraphView<contractor::QueryGraphView>({node_list, edge_list}, edge_filter);
} }

36
src/engine/datafacade/mmap_memory_allocator.cpp
View File

@ -1,5 +1,7 @@
#include "engine/datafacade/mmap_memory_allocator.hpp" #include "engine/datafacade/mmap_memory_allocator.hpp"
#include "storage/io.hpp"
#include "storage/serialization.hpp"
#include "storage/storage.hpp" #include "storage/storage.hpp"
#include "util/log.hpp" #include "util/log.hpp"
@ -22,31 +24,43 @@ MMapMemoryAllocator::MMapMemoryAllocator(const storage::StorageConfig &config,
if (!boost::filesystem::exists(memory_file)) if (!boost::filesystem::exists(memory_file))
{ {
storage::DataLayout initial_layout; storage::DataLayout initial_layout;
storage.PopulateLayout(initial_layout); storage.PopulateStaticLayout(initial_layout);
storage.PopulateUpdatableLayout(initial_layout);
auto data_size = initial_layout.GetSizeOfLayout(); auto data_size = initial_layout.GetSizeOfLayout();
auto total_size = data_size + sizeof(storage::DataLayout);
storage::io::BufferWriter writer;
storage::serialization::write(writer, initial_layout);
auto encoded_layout = writer.GetBuffer();
auto total_size = data_size + encoded_layout.size();
mapped_memory = util::mmapFile<char>(memory_file, mapped_memory_file, total_size); mapped_memory = util::mmapFile<char>(memory_file, mapped_memory_file, total_size);
data_layout = reinterpret_cast<storage::DataLayout *>(mapped_memory.data()); std::copy(encoded_layout.begin(), encoded_layout.end(), mapped_memory.data());
*data_layout = initial_layout;
storage.PopulateData(*data_layout, GetMemory()); index = storage::SharedDataIndex(
{{mapped_memory.data() + encoded_layout.size(), std::move(initial_layout)}});
storage.PopulateStaticData(index);
storage.PopulateUpdatableData(index);
} }
else else
{ {
mapped_memory = util::mmapFile<char>(memory_file, mapped_memory_file); mapped_memory = util::mmapFile<char>(memory_file, mapped_memory_file);
data_layout = reinterpret_cast<storage::DataLayout *>(mapped_memory.data());
storage::DataLayout layout;
storage::io::BufferReader reader(mapped_memory.data());
storage::serialization::read(reader, layout);
auto layout_size = reader.GetPosition();
index = storage::SharedDataIndex({{mapped_memory.data() + layout_size, std::move(layout)}});
} }
} }
MMapMemoryAllocator::~MMapMemoryAllocator() {} MMapMemoryAllocator::~MMapMemoryAllocator() {}
storage::DataLayout &MMapMemoryAllocator::GetLayout() { return *data_layout; } const storage::SharedDataIndex &MMapMemoryAllocator::GetIndex() { return index; }
char *MMapMemoryAllocator::GetMemory()
{
return mapped_memory.data() + sizeof(storage::DataLayout);
}
} // namespace datafacade } // namespace datafacade
} // namespace engine } // namespace engine

15
src/engine/datafacade/process_memory_allocator.cpp
View File

@ -15,17 +15,22 @@ ProcessMemoryAllocator::ProcessMemoryAllocator(const storage::StorageConfig &con
storage::Storage storage(config); storage::Storage storage(config);
// Calculate the layout/size of the memory block // Calculate the layout/size of the memory block
storage.PopulateLayout(internal_layout); storage::DataLayout layout;
storage.PopulateStaticLayout(layout);
storage.PopulateUpdatableLayout(layout);
// Allocate the memory block, then load data from files into it // Allocate the memory block, then load data from files into it
internal_memory = std::make_unique<char[]>(internal_layout.GetSizeOfLayout()); internal_memory = std::make_unique<char[]>(layout.GetSizeOfLayout());
storage.PopulateData(internal_layout, internal_memory.get());
index = storage::SharedDataIndex({{internal_memory.get(), std::move(layout)}});
storage.PopulateStaticData(index);
storage.PopulateUpdatableData(index);
} }
ProcessMemoryAllocator::~ProcessMemoryAllocator() {} ProcessMemoryAllocator::~ProcessMemoryAllocator() {}
const storage::DataLayout &ProcessMemoryAllocator::GetLayout() { return internal_layout; } const storage::SharedDataIndex &ProcessMemoryAllocator::GetIndex() { return index; }
char *ProcessMemoryAllocator::GetMemory() { return internal_memory.get(); }
} // namespace datafacade } // namespace datafacade
} // namespace engine } // namespace engine

33
src/engine/datafacade/shared_memory_allocator.cpp
View File

@ -13,27 +13,32 @@ namespace engine
namespace datafacade namespace datafacade
{ {
SharedMemoryAllocator::SharedMemoryAllocator(storage::SharedRegionRegister::ShmKey data_shm_key) SharedMemoryAllocator::SharedMemoryAllocator(
const std::vector<storage::SharedRegionRegister::ShmKey> &shm_keys)
{ {
util::Log(logDEBUG) << "Loading new data for region " << (int)data_shm_key; std::vector<storage::SharedDataIndex::AllocatedRegion> regions;
BOOST_ASSERT(storage::SharedMemory::RegionExists(data_shm_key)); for (const auto shm_key : shm_keys)
m_large_memory = storage::makeSharedMemory(data_shm_key); {
util::Log(logDEBUG) << "Loading new data for region " << (int)shm_key;
BOOST_ASSERT(storage::SharedMemory::RegionExists(shm_key));
auto mem = storage::makeSharedMemory(shm_key);
storage::io::BufferReader reader(reinterpret_cast<char *>(m_large_memory->Ptr()), storage::io::BufferReader reader(reinterpret_cast<char *>(mem->Ptr()), mem->Size());
m_large_memory->Size()); storage::DataLayout layout;
storage::serialization::read(reader, data_layout); storage::serialization::read(reader, layout);
layout_size = reader.GetPosition(); auto layout_size = reader.GetPosition();
util::Log(logDEBUG) << "Data layout has size " << layout_size;
regions.push_back({reinterpret_cast<char *>(mem->Ptr()) + layout_size, std::move(layout)});
memory_regions.push_back(std::move(mem));
}
index = storage::SharedDataIndex{std::move(regions)};
} }
SharedMemoryAllocator::~SharedMemoryAllocator() {} SharedMemoryAllocator::~SharedMemoryAllocator() {}
const storage::DataLayout &SharedMemoryAllocator::GetLayout() { return data_layout; } const storage::SharedDataIndex &SharedMemoryAllocator::GetIndex() { return index; }
char *SharedMemoryAllocator::GetMemory()
{
return reinterpret_cast<char *>(m_large_memory->Ptr()) + layout_size;
}
} // namespace datafacade } // namespace datafacade
} // namespace engine } // namespace engine

455
src/storage/storage.cpp
View File

@ -42,7 +42,9 @@ namespace storage
{ {
namespace namespace
{ {
inline void readBlocks(const boost::filesystem::path &path, DataLayout &layout) using Monitor = SharedMonitor<SharedRegionRegister>;
void readBlocks(const boost::filesystem::path &path, DataLayout &layout)
{ {
tar::FileReader reader(path, tar::FileReader::VerifyFingerprint); tar::FileReader reader(path, tar::FileReader::VerifyFingerprint);
@ -59,9 +61,128 @@ inline void readBlocks(const boost::filesystem::path &path, DataLayout &layout)
} }
} }
} }
struct RegionHandle
{
std::unique_ptr<SharedMemory> memory;
char *data_ptr;
std::uint8_t shm_key;
};
auto setupRegion(SharedRegionRegister &shared_register, const DataLayout &layout)
{
// This is safe because we have an exclusive lock for all osrm-datastore processes.
auto shm_key = shared_register.ReserveKey();
// ensure that the shared memory region we want to write to is really removed
// this is only needef for failure recovery because we actually wait for all clients
// to detach at the end of the function
if (storage::SharedMemory::RegionExists(shm_key))
{
util::Log(logWARNING) << "Old shared memory region " << (int)shm_key << " still exists.";
util::UnbufferedLog() << "Retrying removal... ";
storage::SharedMemory::Remove(shm_key);
util::UnbufferedLog() << "ok.";
} }
using Monitor = SharedMonitor<SharedRegionRegister>; io::BufferWriter writer;
serialization::write(writer, layout);
auto encoded_static_layout = writer.GetBuffer();
// Allocate shared memory block
auto regions_size = encoded_static_layout.size() + layout.GetSizeOfLayout();
util::Log() << "Data layout has a size of " << encoded_static_layout.size() << " bytes";
util::Log() << "Allocating shared memory of " << regions_size << " bytes";
auto memory = makeSharedMemory(shm_key, regions_size);
// Copy memory static_layout to shared memory and populate data
char *shared_memory_ptr = static_cast<char *>(memory->Ptr());
auto data_ptr =
std::copy_n(encoded_static_layout.data(), encoded_static_layout.size(), shared_memory_ptr);
return RegionHandle{std::move(memory), data_ptr, shm_key};
}
bool swapData(Monitor &monitor,
SharedRegionRegister &shared_register,
const std::map<std::string, RegionHandle> &handles,
int max_wait)
{
std::vector<RegionHandle> old_handles;
{ // Lock for write access shared region mutex
boost::interprocess::scoped_lock<Monitor::mutex_type> lock(monitor.get_mutex(),
boost::interprocess::defer_lock);
if (max_wait >= 0)
{
if (!lock.timed_lock(boost::posix_time::microsec_clock::universal_time() +
boost::posix_time::seconds(max_wait)))
{
util::Log(logERROR) << "Could not aquire current region lock after " << max_wait
<< " seconds. Data update failed.";
for (auto &pair : handles)
{
SharedMemory::Remove(pair.second.shm_key);
}
return false;
}
}
else
{
lock.lock();
}
for (auto &pair : handles)
{
auto region_id = shared_register.Find(pair.first);
if (region_id == SharedRegionRegister::INVALID_REGION_ID)
{
region_id = shared_register.Register(pair.first, pair.second.shm_key);
}
else
{
auto &shared_region = shared_register.GetRegion(region_id);
old_handles.push_back(RegionHandle{
makeSharedMemory(shared_region.shm_key), nullptr, shared_region.shm_key});
shared_region.shm_key = pair.second.shm_key;
shared_region.timestamp++;
}
}
}
util::Log() << "All data loaded. Notify all client about new data in:";
for (const auto &pair : handles)
{
util::Log() << pair.first << "\t" << static_cast<int>(pair.second.shm_key);
}
monitor.notify_all();
for (auto &old_handle : old_handles)
{
util::UnbufferedLog() << "Marking old shared memory region "
<< static_cast<int>(old_handle.shm_key) << " for removal... ";
// SHMCTL(2): Mark the segment to be destroyed. The segment will actually be destroyed
// only after the last process detaches it.
storage::SharedMemory::Remove(old_handle.shm_key);
util::UnbufferedLog() << "ok.";
util::UnbufferedLog() << "Waiting for clients to detach... ";
old_handle.memory->WaitForDetach();
util::UnbufferedLog() << " ok.";
shared_register.ReleaseKey(old_handle.shm_key);
}
util::Log() << "All clients switched.";
return true;
}
}
Storage::Storage(StorageConfig config_) : config(std::move(config_)) {} Storage::Storage(StorageConfig config_) : config(std::move(config_)) {}
@ -103,106 +224,26 @@ int Storage::Run(int max_wait, const std::string &dataset_name)
Monitor monitor(SharedRegionRegister{}); Monitor monitor(SharedRegionRegister{});
auto &shared_register = monitor.data(); auto &shared_register = monitor.data();
// This is safe because we have an exclusive lock for all osrm-datastore processes.
auto shm_key = shared_register.ReserveKey();
// ensure that the shared memory region we want to write to is really removed
// this is only needef for failure recovery because we actually wait for all clients
// to detach at the end of the function
if (storage::SharedMemory::RegionExists(shm_key))
{
util::Log(logWARNING) << "Old shared memory region " << shm_key << " still exists.";
util::UnbufferedLog() << "Retrying removal... ";
storage::SharedMemory::Remove(shm_key);
util::UnbufferedLog() << "ok.";
}
util::Log() << "Loading data into " << static_cast<int>(shm_key);
// Populate a memory layout into stack memory // Populate a memory layout into stack memory
DataLayout layout; DataLayout static_layout;
PopulateLayout(layout); PopulateStaticLayout(static_layout);
DataLayout updatable_layout;
PopulateUpdatableLayout(updatable_layout);
io::BufferWriter writer; auto static_handle = setupRegion(shared_register, static_layout);
serialization::write(writer, layout); auto updatable_handle = setupRegion(shared_register, updatable_layout);
auto encoded_layout = writer.GetBuffer();
// Allocate shared memory block SharedDataIndex index{
auto regions_size = encoded_layout.size() + layout.GetSizeOfLayout(); {{static_handle.data_ptr, static_layout}, {updatable_handle.data_ptr, updatable_layout}}};
util::Log() << "Data layout has a size of " << encoded_layout.size() << " bytes";
util::Log() << "Allocating shared memory of " << regions_size << " bytes";
auto data_memory = makeSharedMemory(shm_key, regions_size);
// Copy memory layout to shared memory and populate data PopulateStaticData(index);
char *shared_memory_ptr = static_cast<char *>(data_memory->Ptr()); PopulateUpdatableData(index);
std::copy_n(encoded_layout.data(), encoded_layout.size(), shared_memory_ptr);
PopulateData(layout, shared_memory_ptr + encoded_layout.size());
std::uint32_t next_timestamp = 0; std::map<std::string, RegionHandle> handles;
std::uint8_t in_use_key = SharedRegionRegister::MAX_SHM_KEYS; handles[dataset_name + "/static"] = std::move(static_handle);
handles[dataset_name + "/updatable"] = std::move(updatable_handle);
{ // Lock for write access shared region mutex swapData(monitor, shared_register, handles, max_wait);
boost::interprocess::scoped_lock<Monitor::mutex_type> lock(monitor.get_mutex(),
boost::interprocess::defer_lock);
if (max_wait >= 0)
{
if (!lock.timed_lock(boost::posix_time::microsec_clock::universal_time() +
boost::posix_time::seconds(max_wait)))
{
util::Log(logERROR) << "Could not aquire current region lock after " << max_wait
<< " seconds. Data update failed.";
SharedMemory::Remove(shm_key);
return EXIT_FAILURE;
}
}
else
{
lock.lock();
}
auto region_id = shared_register.Find(dataset_name + "/data");
if (region_id == SharedRegionRegister::INVALID_REGION_ID)
{
region_id = shared_register.Register(dataset_name + "/data", shm_key);
}
else
{
auto &shared_region = shared_register.GetRegion(region_id);
next_timestamp = shared_region.timestamp + 1;
in_use_key = shared_region.shm_key;
shared_region.shm_key = shm_key;
shared_region.timestamp = next_timestamp;
}
}
util::Log() << "All data loaded. Notify all client about new data in "
<< static_cast<int>(shm_key) << " with timestamp " << next_timestamp;
monitor.notify_all();
// SHMCTL(2): Mark the segment to be destroyed. The segment will actually be destroyed
// only after the last process detaches it.
if (in_use_key != SharedRegionRegister::MAX_SHM_KEYS &&
storage::SharedMemory::RegionExists(in_use_key))
{
util::UnbufferedLog() << "Marking old shared memory region " << static_cast<int>(in_use_key)
<< " for removal... ";
// aquire a handle for the old shared memory region before we mark it for deletion
// we will need this to wait for all users to detach
auto in_use_shared_memory = makeSharedMemory(in_use_key);
storage::SharedMemory::Remove(in_use_key);
util::UnbufferedLog() << "ok.";
util::UnbufferedLog() << "Waiting for clients to detach... ";
in_use_shared_memory->WaitForDetach();
util::UnbufferedLog() << " ok.";
shared_register.ReleaseKey(in_use_key);
}
util::Log() << "All clients switched.";
return EXIT_SUCCESS; return EXIT_SUCCESS;
} }
@ -212,35 +253,28 @@ int Storage::Run(int max_wait, const std::string &dataset_name)
* memory needs to be allocated, and the position of each data structure * memory needs to be allocated, and the position of each data structure
* in that big block. It updates the fields in the DataLayout parameter. * in that big block. It updates the fields in the DataLayout parameter.
*/ */
void Storage::PopulateLayout(DataLayout &layout) void Storage::PopulateStaticLayout(DataLayout &static_layout)
{ {
{ {
auto absolute_file_index_path = auto absolute_file_index_path =
boost::filesystem::absolute(config.GetPath(".osrm.fileIndex")); boost::filesystem::absolute(config.GetPath(".osrm.fileIndex"));
layout.SetBlock("/common/rtree/file_index_path", static_layout.SetBlock("/common/rtree/file_index_path",
make_block<char>(absolute_file_index_path.string().length() + 1)); make_block<char>(absolute_file_index_path.string().length() + 1));
} }
constexpr bool REQUIRED = true; constexpr bool REQUIRED = true;
constexpr bool OPTIONAL = false; constexpr bool OPTIONAL = false;
std::vector<std::pair<bool, boost::filesystem::path>> tar_files = { std::vector<std::pair<bool, boost::filesystem::path>> tar_files = {
{OPTIONAL, config.GetPath(".osrm.mldgr")},
{OPTIONAL, config.GetPath(".osrm.cells")}, {OPTIONAL, config.GetPath(".osrm.cells")},
{OPTIONAL, config.GetPath(".osrm.partition")}, {OPTIONAL, config.GetPath(".osrm.partition")},
{OPTIONAL, config.GetPath(".osrm.cell_metrics")},
{OPTIONAL, config.GetPath(".osrm.hsgr")},
{REQUIRED, config.GetPath(".osrm.icd")}, {REQUIRED, config.GetPath(".osrm.icd")},
{REQUIRED, config.GetPath(".osrm.properties")}, {REQUIRED, config.GetPath(".osrm.properties")},
{REQUIRED, config.GetPath(".osrm.nbg_nodes")}, {REQUIRED, config.GetPath(".osrm.nbg_nodes")},
{REQUIRED, config.GetPath(".osrm.datasource_names")},
{REQUIRED, config.GetPath(".osrm.geometry")},
{REQUIRED, config.GetPath(".osrm.ebg_nodes")}, {REQUIRED, config.GetPath(".osrm.ebg_nodes")},
{REQUIRED, config.GetPath(".osrm.tls")}, {REQUIRED, config.GetPath(".osrm.tls")},
{REQUIRED, config.GetPath(".osrm.tld")}, {REQUIRED, config.GetPath(".osrm.tld")},
{REQUIRED, config.GetPath(".osrm.maneuver_overrides")}, {REQUIRED, config.GetPath(".osrm.maneuver_overrides")},
{REQUIRED, config.GetPath(".osrm.turn_weight_penalties")},
{REQUIRED, config.GetPath(".osrm.turn_duration_penalties")},
{REQUIRED, config.GetPath(".osrm.edges")}, {REQUIRED, config.GetPath(".osrm.edges")},
{REQUIRED, config.GetPath(".osrm.names")}, {REQUIRED, config.GetPath(".osrm.names")},
{REQUIRED, config.GetPath(".osrm.ramIndex")}, {REQUIRED, config.GetPath(".osrm.ramIndex")},
@ -250,7 +284,7 @@ void Storage::PopulateLayout(DataLayout &layout)
{ {
if (boost::filesystem::exists(file.second)) if (boost::filesystem::exists(file.second))
{ {
readBlocks(file.second, layout); readBlocks(file.second, static_layout);
} }
else else
{ {
@ -263,26 +297,51 @@ void Storage::PopulateLayout(DataLayout &layout)
} }
} }
void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) void Storage::PopulateUpdatableLayout(DataLayout &updatable_layout)
{ {
BOOST_ASSERT(memory_ptr != nullptr); constexpr bool REQUIRED = true;
constexpr bool OPTIONAL = false;
std::vector<std::pair<bool, boost::filesystem::path>> tar_files = {
{OPTIONAL, config.GetPath(".osrm.mldgr")},
{OPTIONAL, config.GetPath(".osrm.cell_metrics")},
{OPTIONAL, config.GetPath(".osrm.hsgr")},
{REQUIRED, config.GetPath(".osrm.datasource_names")},
{REQUIRED, config.GetPath(".osrm.geometry")},
{REQUIRED, config.GetPath(".osrm.turn_weight_penalties")},
{REQUIRED, config.GetPath(".osrm.turn_duration_penalties")},
};
// Connectivity matrix checksum for (const auto &file : tar_files)
std::uint32_t turns_connectivity_checksum = 0; {
if (boost::filesystem::exists(file.second))
{
readBlocks(file.second, updatable_layout);
}
else
{
if (file.first == REQUIRED)
{
throw util::exception("Could not find required filed: " +
std::get<1>(file).string());
}
}
}
}
void Storage::PopulateStaticData(const SharedDataIndex &index)
{
// read actual data into shared memory object // // read actual data into shared memory object //
// store the filename of the on-disk portion of the RTree // store the filename of the on-disk portion of the RTree
{ {
const auto file_index_path_ptr = const auto file_index_path_ptr = index.GetBlockPtr<char>("/common/rtree/file_index_path");
layout.GetBlockPtr<char>(memory_ptr, "/common/rtree/file_index_path");
// make sure we have 0 ending // make sure we have 0 ending
std::fill(file_index_path_ptr, std::fill(file_index_path_ptr,
file_index_path_ptr + layout.GetBlockSize("/common/rtree/file_index_path"), file_index_path_ptr + index.GetBlockSize("/common/rtree/file_index_path"),
0); 0);
const auto absolute_file_index_path = const auto absolute_file_index_path =
boost::filesystem::absolute(config.GetPath(".osrm.fileIndex")).string(); boost::filesystem::absolute(config.GetPath(".osrm.fileIndex")).string();
BOOST_ASSERT(static_cast<std::size_t>(layout.GetBlockSize( BOOST_ASSERT(static_cast<std::size_t>(index.GetBlockSize(
"/common/rtree/file_index_path")) >= absolute_file_index_path.size()); "/common/rtree/file_index_path")) >= absolute_file_index_path.size());
std::copy( std::copy(
absolute_file_index_path.begin(), absolute_file_index_path.end(), file_index_path_ptr); absolute_file_index_path.begin(), absolute_file_index_path.end(), file_index_path_ptr);
@ -290,81 +349,50 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Name data // Name data
{ {
auto name_table = make_name_table_view(memory_ptr, layout, "/common/names"); auto name_table = make_name_table_view(index, "/common/names");
extractor::files::readNames(config.GetPath(".osrm.names"), name_table); extractor::files::readNames(config.GetPath(".osrm.names"), name_table);
} }
// Turn lane data // Turn lane data
{ {
auto turn_lane_data = make_lane_data_view(memory_ptr, layout, "/common/turn_lanes"); auto turn_lane_data = make_lane_data_view(index, "/common/turn_lanes");
extractor::files::readTurnLaneData(config.GetPath(".osrm.tld"), turn_lane_data); extractor::files::readTurnLaneData(config.GetPath(".osrm.tld"), turn_lane_data);
} }
// Turn lane descriptions // Turn lane descriptions
{ {
auto views = make_turn_lane_description_views(memory_ptr, layout, "/common/turn_lanes"); auto views = make_turn_lane_description_views(index, "/common/turn_lanes");
extractor::files::readTurnLaneDescriptions( extractor::files::readTurnLaneDescriptions(
config.GetPath(".osrm.tls"), std::get<0>(views), std::get<1>(views)); config.GetPath(".osrm.tls"), std::get<0>(views), std::get<1>(views));
} }
// Load edge-based nodes data // Load edge-based nodes data
{ {
auto node_data = make_ebn_data_view(memory_ptr, layout, "/common/ebg_node_data"); auto node_data = make_ebn_data_view(index, "/common/ebg_node_data");
extractor::files::readNodeData(config.GetPath(".osrm.ebg_nodes"), node_data); extractor::files::readNodeData(config.GetPath(".osrm.ebg_nodes"), node_data);
} }
// Load original edge data // Load original edge data
{ {
auto turn_data = make_turn_data_view(memory_ptr, layout, "/common/turn_data"); auto turn_data = make_turn_data_view(index, "/common/turn_data");
auto connectivity_checksum_ptr = auto connectivity_checksum_ptr =
layout.GetBlockPtr<std::uint32_t>(memory_ptr, "/common/connectivity_checksum"); index.GetBlockPtr<std::uint32_t>("/common/connectivity_checksum");
guidance::files::readTurnData( guidance::files::readTurnData(
config.GetPath(".osrm.edges"), turn_data, *connectivity_checksum_ptr); config.GetPath(".osrm.edges"), turn_data, *connectivity_checksum_ptr);
turns_connectivity_checksum = *connectivity_checksum_ptr;
}
// load compressed geometry
{
auto segment_data = make_segment_data_view(memory_ptr, layout, "/common/segment_data");
extractor::files::readSegmentData(config.GetPath(".osrm.geometry"), segment_data);
}
{
const auto datasources_names_ptr =
layout.GetBlockPtr<extractor::Datasources>(memory_ptr, "/common/data_sources_names");
extractor::files::readDatasources(config.GetPath(".osrm.datasource_names"),
*datasources_names_ptr);
} }
// Loading list of coordinates // Loading list of coordinates
{ {
auto views = make_nbn_data_view(memory_ptr, layout, "/common/nbn_data"); auto views = make_nbn_data_view(index, "/common/nbn_data");
extractor::files::readNodes( extractor::files::readNodes(
config.GetPath(".osrm.nbg_nodes"), std::get<0>(views), std::get<1>(views)); config.GetPath(".osrm.nbg_nodes"), std::get<0>(views), std::get<1>(views));
} }
// load turn weight penalties
{
auto turn_duration_penalties =
make_turn_weight_view(memory_ptr, layout, "/common/turn_penalty");
extractor::files::readTurnWeightPenalty(config.GetPath(".osrm.turn_weight_penalties"),
turn_duration_penalties);
}
// load turn duration penalties
{
auto turn_duration_penalties =
make_turn_duration_view(memory_ptr, layout, "/common/turn_penalty");
extractor::files::readTurnDurationPenalty(config.GetPath(".osrm.turn_duration_penalties"),
turn_duration_penalties);
}
// store search tree portion of rtree // store search tree portion of rtree
{ {
auto rtree = make_search_tree_view(memory_ptr, layout, "/common/rtree"); auto rtree = make_search_tree_view(index, "/common/rtree");
extractor::files::readRamIndex(config.GetPath(".osrm.ramIndex"), rtree); extractor::files::readRamIndex(config.GetPath(".osrm.ramIndex"), rtree);
} }
@ -372,8 +400,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
std::string metric_name; std::string metric_name;
// load profile properties // load profile properties
{ {
const auto profile_properties_ptr = const auto profile_properties_ptr = index.GetBlockPtr<extractor::ProfileProperties>(
layout.GetBlockPtr<extractor::ProfileProperties>(memory_ptr, "/common/properties"); "/common/properties");
extractor::files::readProfileProperties(config.GetPath(".osrm.properties"), extractor::files::readProfileProperties(config.GetPath(".osrm.properties"),
*profile_properties_ptr); *profile_properties_ptr);
@ -383,49 +411,95 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Load intersection data // Load intersection data
{ {
auto intersection_bearings_view = auto intersection_bearings_view =
make_intersection_bearings_view(memory_ptr, layout, "/common/intersection_bearings"); make_intersection_bearings_view(index, "/common/intersection_bearings");
auto entry_classes = make_entry_classes_view(memory_ptr, layout, "/common/entry_classes"); auto entry_classes = make_entry_classes_view(index, "/common/entry_classes");
extractor::files::readIntersections( extractor::files::readIntersections(
config.GetPath(".osrm.icd"), intersection_bearings_view, entry_classes); config.GetPath(".osrm.icd"), intersection_bearings_view, entry_classes);
} }
if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
{
auto mlp = make_partition_view(index, "/mld/multilevelpartition");
partitioner::files::readPartition(config.GetPath(".osrm.partition"), mlp);
}
if (boost::filesystem::exists(config.GetPath(".osrm.cells")))
{
auto storage = make_cell_storage_view(index, "/mld/cellstorage");
partitioner::files::readCells(config.GetPath(".osrm.cells"), storage);
}
if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
{
auto exclude_metrics = make_cell_metric_view(index, "/mld/metrics/" + metric_name);
std::unordered_map<std::string, std::vector<customizer::CellMetricView>> metrics = {
{metric_name, std::move(exclude_metrics)},
};
customizer::files::readCellMetrics(config.GetPath(".osrm.cell_metrics"), metrics);
}
// load maneuver overrides
{
auto views = make_maneuver_overrides_views(index, "/common/maneuver_overrides");
extractor::files::readManeuverOverrides(
config.GetPath(".osrm.maneuver_overrides"), std::get<0>(views), std::get<1>(views));
}
}
void Storage::PopulateUpdatableData(const SharedDataIndex &index)
{
// load compressed geometry
{
auto segment_data = make_segment_data_view(index, "/common/segment_data");
extractor::files::readSegmentData(config.GetPath(".osrm.geometry"), segment_data);
}
{
const auto datasources_names_ptr = index.GetBlockPtr<extractor::Datasources>(
"/common/data_sources_names");
extractor::files::readDatasources(config.GetPath(".osrm.datasource_names"),
*datasources_names_ptr);
}
// load turn weight penalties
{
auto turn_duration_penalties = make_turn_weight_view(index, "/common/turn_penalty");
extractor::files::readTurnWeightPenalty(config.GetPath(".osrm.turn_weight_penalties"),
turn_duration_penalties);
}
// load turn duration penalties
{
auto turn_duration_penalties = make_turn_duration_view(index, "/common/turn_penalty");
extractor::files::readTurnDurationPenalty(config.GetPath(".osrm.turn_duration_penalties"),
turn_duration_penalties);
}
// FIXME we only need to get the weight name
std::string metric_name;
// load profile properties
{
extractor::ProfileProperties properties;
extractor::files::readProfileProperties(config.GetPath(".osrm.properties"), properties);
metric_name = properties.GetWeightName();
}
if (boost::filesystem::exists(config.GetPath(".osrm.hsgr"))) if (boost::filesystem::exists(config.GetPath(".osrm.hsgr")))
{ {
const std::string metric_prefix = "/ch/metrics/" + metric_name; const std::string metric_prefix = "/ch/metrics/" + metric_name;
auto contracted_metric = make_contracted_metric_view(memory_ptr, layout, metric_prefix); auto contracted_metric = make_contracted_metric_view(index, metric_prefix);
std::unordered_map<std::string, contractor::ContractedMetricView> metrics = { std::unordered_map<std::string, contractor::ContractedMetricView> metrics = {
{metric_name, std::move(contracted_metric)}}; {metric_name, std::move(contracted_metric)}};
std::uint32_t graph_connectivity_checksum = 0; std::uint32_t graph_connectivity_checksum = 0;
contractor::files::readGraph( contractor::files::readGraph(
config.GetPath(".osrm.hsgr"), metrics, graph_connectivity_checksum); config.GetPath(".osrm.hsgr"), metrics, graph_connectivity_checksum);
if (turns_connectivity_checksum != graph_connectivity_checksum)
{
throw util::exception(
"Connectivity checksum " + std::to_string(graph_connectivity_checksum) + " in " +
config.GetPath(".osrm.hsgr").string() + " does not equal to checksum " +
std::to_string(turns_connectivity_checksum) + " in " +
config.GetPath(".osrm.edges").string());
}
}
if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
{
auto mlp = make_partition_view(memory_ptr, layout, "/mld/multilevelpartition");
partitioner::files::readPartition(config.GetPath(".osrm.partition"), mlp);
}
if (boost::filesystem::exists(config.GetPath(".osrm.cells")))
{
auto storage = make_cell_storage_view(memory_ptr, layout, "/mld/cellstorage");
partitioner::files::readCells(config.GetPath(".osrm.cells"), storage);
} }
if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics"))) if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
{ {
auto exclude_metrics = auto exclude_metrics = make_cell_metric_view(index, "/mld/metrics/" + metric_name);
make_cell_metric_view(memory_ptr, layout, "/mld/metrics/" + metric_name);
std::unordered_map<std::string, std::vector<customizer::CellMetricView>> metrics = { std::unordered_map<std::string, std::vector<customizer::CellMetricView>> metrics = {
{metric_name, std::move(exclude_metrics)}, {metric_name, std::move(exclude_metrics)},
}; };
@ -434,27 +508,10 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
if (boost::filesystem::exists(config.GetPath(".osrm.mldgr"))) if (boost::filesystem::exists(config.GetPath(".osrm.mldgr")))
{ {
auto graph_view = make_multi_level_graph_view(memory_ptr, layout, "/mld/multilevelgraph"); auto graph_view = make_multi_level_graph_view(index, "/mld/multilevelgraph");
std::uint32_t graph_connectivity_checksum = 0; std::uint32_t graph_connectivity_checksum = 0;
partitioner::files::readGraph( partitioner::files::readGraph(
config.GetPath(".osrm.mldgr"), graph_view, graph_connectivity_checksum); config.GetPath(".osrm.mldgr"), graph_view, graph_connectivity_checksum);
if (turns_connectivity_checksum != graph_connectivity_checksum)
{
throw util::exception(
"Connectivity checksum " + std::to_string(graph_connectivity_checksum) + " in " +
config.GetPath(".osrm.mldgr").string() + " does not equal to checksum " +
std::to_string(turns_connectivity_checksum) + " in " +
config.GetPath(".osrm.edges").string());
}
}
// load maneuver overrides
{
auto views =
make_maneuver_overrides_views(memory_ptr, layout, "/common/maneuver_overrides");
extractor::files::readManeuverOverrides(
config.GetPath(".osrm.maneuver_overrides"), std::get<0>(views), std::get<1>(views));
} }
} }
} }