Load data in two separate data regions
This commit is contained in:
Patrick Niklaus
Patrick Niklaus
parent
c7daa521ad
commit
fea07f343b
@ -43,18 +43,23 @@ class DataWatchdogImpl<AlgorithmT, datafacade::ContiguousInternalMemoryDataFacad
|
||||
boost::interprocess::scoped_lock<mutex_type> current_region_lock(barrier.get_mutex());
|
||||
|
||||
auto &shared_register = barrier.data();
|
||||
auto region_id = shared_register.Find(dataset_name + "/data");
|
||||
if (region_id == storage::SharedRegionRegister::INVALID_REGION_ID)
|
||||
auto static_region_id = shared_register.Find(dataset_name + "/static");
|
||||
auto updatable_region_id = shared_register.Find(dataset_name + "/updatable");
|
||||
if (static_region_id == storage::SharedRegionRegister::INVALID_REGION_ID ||
|
||||
updatable_region_id == storage::SharedRegionRegister::INVALID_REGION_ID)
|
||||
{
|
||||
throw util::exception("Could not find shared memory region for \"" + dataset_name +
|
||||
"/data\". Did you run osrm-datastore?");
|
||||
}
|
||||
shared_region = &shared_register.GetRegion(region_id);
|
||||
region = *shared_region;
|
||||
static_shared_region = &shared_register.GetRegion(static_region_id);
|
||||
updatable_shared_region = &shared_register.GetRegion(updatable_region_id);
|
||||
static_region = *static_shared_region;
|
||||
updatable_region = *updatable_shared_region;
|
||||
|
||||
facade_factory =
|
||||
DataFacadeFactory<datafacade::ContiguousInternalMemoryDataFacade, AlgorithmT>(
|
||||
std::make_shared<datafacade::SharedMemoryAllocator>(region.shm_key));
|
||||
std::make_shared<datafacade::SharedMemoryAllocator>(
|
||||
std::vector<storage::SharedRegionRegister::ShmKey> {static_region.shm_key, updatable_region.shm_key}));
|
||||
}
|
||||
|
||||
watcher = std::thread(&DataWatchdogImpl::Run, this);
|
||||
@ -83,20 +88,32 @@ class DataWatchdogImpl<AlgorithmT, datafacade::ContiguousInternalMemoryDataFacad
|
||||
{
|
||||
boost::interprocess::scoped_lock<mutex_type> current_region_lock(barrier.get_mutex());
|
||||
|
||||
while (active && region.timestamp == shared_region->timestamp)
|
||||
while (active && static_region.timestamp == static_shared_region->timestamp &&
|
||||
updatable_region.timestamp == updatable_shared_region->timestamp)
|
||||
{
|
||||
barrier.wait(current_region_lock);
|
||||
}
|
||||
|
||||
if (region.timestamp != shared_region->timestamp)
|
||||
if (!active)
|
||||
break;
|
||||
|
||||
if (static_region.timestamp != static_shared_region->timestamp)
|
||||
{
|
||||
region = *shared_region;
|
||||
facade_factory =
|
||||
DataFacadeFactory<datafacade::ContiguousInternalMemoryDataFacade, AlgorithmT>(
|
||||
std::make_shared<datafacade::SharedMemoryAllocator>(region.shm_key));
|
||||
util::Log() << "updated facade to region " << (int)region.shm_key
|
||||
<< " with timestamp " << region.timestamp;
|
||||
static_region = *static_shared_region;
|
||||
}
|
||||
if (updatable_region.timestamp != updatable_shared_region->timestamp)
|
||||
{
|
||||
updatable_region = *updatable_shared_region;
|
||||
}
|
||||
|
||||
util::Log() << "updated facade to regions " << (int)static_region.shm_key << " and "
|
||||
<< (int)updatable_region.shm_key << " with timestamps "
|
||||
<< static_region.timestamp << " and " << updatable_region.timestamp;
|
||||
|
||||
facade_factory =
|
||||
DataFacadeFactory<datafacade::ContiguousInternalMemoryDataFacade, AlgorithmT>(
|
||||
std::make_shared<datafacade::SharedMemoryAllocator>(
|
||||
std::vector<storage::SharedRegionRegister::ShmKey>{static_region.shm_key, updatable_region.shm_key}));
|
||||
}
|
||||
|
||||
util::Log() << "DataWatchdog thread stopped";
|
||||
@ -106,8 +123,10 @@ class DataWatchdogImpl<AlgorithmT, datafacade::ContiguousInternalMemoryDataFacad
|
||||
storage::SharedMonitor<storage::SharedRegionRegister> barrier;
|
||||
std::thread watcher;
|
||||
bool active;
|
||||
storage::SharedRegion region;
|
||||
storage::SharedRegion *shared_region;
|
||||
storage::SharedRegion static_region;
|
||||
storage::SharedRegion updatable_region;
|
||||
storage::SharedRegion *static_shared_region;
|
||||
storage::SharedRegion *updatable_shared_region;
|
||||
DataFacadeFactory<datafacade::ContiguousInternalMemoryDataFacade, AlgorithmT> facade_factory;
|
||||
};
|
||||
}
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#ifndef OSRM_ENGINE_DATAFACADE_CONTIGUOUS_BLOCK_ALLOCATOR_HPP_
|
||||
#define OSRM_ENGINE_DATAFACADE_CONTIGUOUS_BLOCK_ALLOCATOR_HPP_
|
||||
|
||||
#include "storage/shared_datatype.hpp"
|
||||
#include "storage/shared_data_index.hpp"
|
||||
|
||||
namespace osrm
|
||||
{
|
||||
@ -16,8 +16,7 @@ class ContiguousBlockAllocator
|
||||
virtual ~ContiguousBlockAllocator() = default;
|
||||
|
||||
// interface to give access to the datafacades
|
||||
virtual const storage::DataLayout &GetLayout() = 0;
|
||||
virtual char *GetMemory() = 0;
|
||||
virtual const storage::SharedDataIndex &GetIndex() = 0;
|
||||
};
|
||||
|
||||
} // namespace datafacade
|
||||
|
||||
@ -57,17 +57,15 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
|
||||
std::size_t exclude_index)
|
||||
: allocator(std::move(allocator_))
|
||||
{
|
||||
InitializeInternalPointers(
|
||||
allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
|
||||
InitializeInternalPointers(allocator->GetIndex(), metric_name, exclude_index);
|
||||
}
|
||||
|
||||
void InitializeInternalPointers(const storage::DataLayout &data_layout,
|
||||
char *memory_block,
|
||||
void InitializeInternalPointers(const storage::SharedDataIndex &index,
|
||||
const std::string &metric_name,
|
||||
const std::size_t exclude_index)
|
||||
{
|
||||
m_query_graph = make_filtered_graph_view(
|
||||
memory_block, data_layout, "/ch/metrics/" + metric_name, exclude_index);
|
||||
m_query_graph =
|
||||
make_filtered_graph_view(index, "/ch/metrics/" + metric_name, exclude_index);
|
||||
}
|
||||
|
||||
// search graph access
|
||||
@ -172,8 +170,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
|
||||
// allocator that keeps the allocation data
|
||||
std::shared_ptr<ContiguousBlockAllocator> allocator;
|
||||
|
||||
void InitializeInternalPointers(const storage::DataLayout &layout,
|
||||
char *memory_ptr,
|
||||
void InitializeInternalPointers(const storage::SharedDataIndex &index,
|
||||
const std::string &metric_name,
|
||||
const std::size_t exclude_index)
|
||||
{
|
||||
@ -181,46 +178,43 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
|
||||
(void)metric_name;
|
||||
|
||||
m_profile_properties =
|
||||
layout.GetBlockPtr<extractor::ProfileProperties>(memory_ptr, "/common/properties");
|
||||
index.GetBlockPtr<extractor::ProfileProperties>("/common/properties");
|
||||
|
||||
exclude_mask = m_profile_properties->excludable_classes[exclude_index];
|
||||
|
||||
m_check_sum =
|
||||
*layout.GetBlockPtr<std::uint32_t>(memory_ptr, "/common/connectivity_checksum");
|
||||
m_check_sum = *index.GetBlockPtr<std::uint32_t>("/common/connectivity_checksum");
|
||||
|
||||
std::tie(m_coordinate_list, m_osmnodeid_list) =
|
||||
make_nbn_data_view(memory_ptr, layout, "/common/nbn_data");
|
||||
make_nbn_data_view(index, "/common/nbn_data");
|
||||
|
||||
m_static_rtree = make_search_tree_view(memory_ptr, layout, "/common/rtree");
|
||||
m_static_rtree = make_search_tree_view(index, "/common/rtree");
|
||||
m_geospatial_query.reset(
|
||||
new SharedGeospatialQuery(m_static_rtree, m_coordinate_list, *this));
|
||||
|
||||
edge_based_node_data = make_ebn_data_view(memory_ptr, layout, "/common/ebg_node_data");
|
||||
edge_based_node_data = make_ebn_data_view(index, "/common/ebg_node_data");
|
||||
|
||||
turn_data = make_turn_data_view(memory_ptr, layout, "/common/turn_data");
|
||||
turn_data = make_turn_data_view(index, "/common/turn_data");
|
||||
|
||||
m_name_table = make_name_table_view(memory_ptr, layout, "/common/names");
|
||||
m_name_table = make_name_table_view(index, "/common/names");
|
||||
|
||||
std::tie(m_lane_description_offsets, m_lane_description_masks) =
|
||||
make_turn_lane_description_views(memory_ptr, layout, "/common/turn_lanes");
|
||||
m_lane_tupel_id_pairs = make_lane_data_view(memory_ptr, layout, "/common/turn_lanes");
|
||||
make_turn_lane_description_views(index, "/common/turn_lanes");
|
||||
m_lane_tupel_id_pairs = make_lane_data_view(index, "/common/turn_lanes");
|
||||
|
||||
m_turn_weight_penalties = make_turn_weight_view(memory_ptr, layout, "/common/turn_penalty");
|
||||
m_turn_duration_penalties =
|
||||
make_turn_duration_view(memory_ptr, layout, "/common/turn_penalty");
|
||||
m_turn_weight_penalties = make_turn_weight_view(index, "/common/turn_penalty");
|
||||
m_turn_duration_penalties = make_turn_duration_view(index, "/common/turn_penalty");
|
||||
|
||||
segment_data = make_segment_data_view(memory_ptr, layout, "/common/segment_data");
|
||||
segment_data = make_segment_data_view(index, "/common/segment_data");
|
||||
|
||||
m_datasources =
|
||||
layout.GetBlockPtr<extractor::Datasources>(memory_ptr, "/common/data_sources_names");
|
||||
m_datasources = index.GetBlockPtr<extractor::Datasources>("/common/data_sources_names");
|
||||
|
||||
intersection_bearings_view =
|
||||
make_intersection_bearings_view(memory_ptr, layout, "/common/intersection_bearings");
|
||||
make_intersection_bearings_view(index, "/common/intersection_bearings");
|
||||
|
||||
m_entry_class_table = make_entry_classes_view(memory_ptr, layout, "/common/entry_classes");
|
||||
m_entry_class_table = make_entry_classes_view(index, "/common/entry_classes");
|
||||
|
||||
std::tie(m_maneuver_overrides, m_maneuver_override_node_sequences) =
|
||||
make_maneuver_overrides_views(memory_ptr, layout, "/common/maneuver_overrides");
|
||||
make_maneuver_overrides_views(index, "/common/maneuver_overrides");
|
||||
}
|
||||
|
||||
public:
|
||||
@ -231,8 +225,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
|
||||
const std::size_t exclude_index)
|
||||
: allocator(std::move(allocator_))
|
||||
{
|
||||
InitializeInternalPointers(
|
||||
allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
|
||||
InitializeInternalPointers(allocator->GetIndex(), metric_name, exclude_index);
|
||||
}
|
||||
|
||||
// node and edge information access
|
||||
@ -661,16 +654,15 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
|
||||
|
||||
QueryGraph query_graph;
|
||||
|
||||
void InitializeInternalPointers(const storage::DataLayout &layout,
|
||||
char *memory_ptr,
|
||||
void InitializeInternalPointers(const storage::SharedDataIndex &index,
|
||||
const std::string &metric_name,
|
||||
const std::size_t exclude_index)
|
||||
{
|
||||
mld_partition = make_partition_view(memory_ptr, layout, "/mld/multilevelpartition");
|
||||
mld_cell_metric = make_filtered_cell_metric_view(
|
||||
memory_ptr, layout, "/mld/metrics/" + metric_name, exclude_index);
|
||||
mld_cell_storage = make_cell_storage_view(memory_ptr, layout, "/mld/cellstorage");
|
||||
query_graph = make_multi_level_graph_view(memory_ptr, layout, "/mld/multilevelgraph");
|
||||
mld_partition = make_partition_view(index, "/mld/multilevelpartition");
|
||||
mld_cell_metric =
|
||||
make_filtered_cell_metric_view(index, "/mld/metrics/" + metric_name, exclude_index);
|
||||
mld_cell_storage = make_cell_storage_view(index, "/mld/cellstorage");
|
||||
query_graph = make_multi_level_graph_view(index, "/mld/multilevelgraph");
|
||||
}
|
||||
|
||||
// allocator that keeps the allocation data
|
||||
@ -683,8 +675,7 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
|
||||
const std::size_t exclude_index)
|
||||
: allocator(std::move(allocator_))
|
||||
{
|
||||
InitializeInternalPointers(
|
||||
allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
|
||||
InitializeInternalPointers(allocator->GetIndex(), metric_name, exclude_index);
|
||||
}
|
||||
|
||||
const partitioner::MultiLevelPartitionView &GetMultiLevelPartition() const override
|
||||
|
||||
@ -29,11 +29,10 @@ class MMapMemoryAllocator : public ContiguousBlockAllocator
|
||||
~MMapMemoryAllocator() override final;
|
||||
|
||||
// interface to give access to the datafacades
|
||||
storage::DataLayout &GetLayout() override final;
|
||||
char *GetMemory() override final;
|
||||
const storage::SharedDataIndex &GetIndex() override final;
|
||||
|
||||
private:
|
||||
storage::DataLayout *data_layout;
|
||||
storage::SharedDataIndex index;
|
||||
util::vector_view<char> mapped_memory;
|
||||
boost::iostreams::mapped_file mapped_memory_file;
|
||||
};
|
||||
|
||||
@ -27,11 +27,10 @@ class ProcessMemoryAllocator : public ContiguousBlockAllocator
|
||||
~ProcessMemoryAllocator() override final;
|
||||
|
||||
// interface to give access to the datafacades
|
||||
const storage::DataLayout &GetLayout() override final;
|
||||
char *GetMemory() override final;
|
||||
const storage::SharedDataIndex &GetIndex() override final;
|
||||
|
||||
private:
|
||||
storage::DataLayout internal_layout;
|
||||
storage::SharedDataIndex index;
|
||||
std::unique_ptr<char[]> internal_memory;
|
||||
};
|
||||
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
|
||||
#include "engine/datafacade/contiguous_block_allocator.hpp"
|
||||
|
||||
#include "storage/shared_datatype.hpp"
|
||||
#include "storage/shared_data_index.hpp"
|
||||
#include "storage/shared_memory.hpp"
|
||||
|
||||
#include <memory>
|
||||
@ -23,17 +23,15 @@ namespace datafacade
|
||||
class SharedMemoryAllocator : public ContiguousBlockAllocator
|
||||
{
|
||||
public:
|
||||
explicit SharedMemoryAllocator(storage::SharedRegionRegister::ShmKey data_shm_key);
|
||||
explicit SharedMemoryAllocator(const std::vector<storage::SharedRegionRegister::ShmKey> &shm_keys);
|
||||
~SharedMemoryAllocator() override final;
|
||||
|
||||
// interface to give access to the datafacades
|
||||
const storage::DataLayout &GetLayout() override final;
|
||||
char *GetMemory() override final;
|
||||
const storage::SharedDataIndex &GetIndex() override final;
|
||||
|
||||
private:
|
||||
std::size_t layout_size;
|
||||
storage::DataLayout data_layout;
|
||||
std::unique_ptr<storage::SharedMemory> m_large_memory;
|
||||
storage::SharedDataIndex index;
|
||||
std::vector<std::unique_ptr<storage::SharedMemory>> memory_regions;
|
||||
};
|
||||
|
||||
} // namespace datafacade
|
||||
|
||||
@ -46,15 +46,14 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
|
||||
template <typename AllocatorT>
|
||||
DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::true_type)
|
||||
{
|
||||
const auto &layout = allocator->GetLayout();
|
||||
properties = layout.template GetBlockPtr<extractor::ProfileProperties>(
|
||||
allocator->GetMemory(), "/common/properties");
|
||||
const auto &index = allocator->GetIndex();
|
||||
properties = index.template GetBlockPtr<extractor::ProfileProperties>("/common/properties");
|
||||
const auto &metric_name = properties->GetWeightName();
|
||||
|
||||
std::vector<std::string> exclude_prefixes;
|
||||
auto exclude_path = std::string("/") + routing_algorithms::identifier<AlgorithmT>() +
|
||||
std::string("/metrics/") + metric_name + "/exclude/";
|
||||
layout.List(exclude_path, std::back_inserter(exclude_prefixes));
|
||||
index.List(exclude_path, std::back_inserter(exclude_prefixes));
|
||||
facades.resize(exclude_prefixes.size());
|
||||
|
||||
if (facades.empty())
|
||||
@ -89,9 +88,8 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
|
||||
template <typename AllocatorT>
|
||||
DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::false_type)
|
||||
{
|
||||
const auto &layout = allocator->GetLayout();
|
||||
properties = layout.template GetBlockPtr<extractor::ProfileProperties>(
|
||||
allocator->GetMemory(), "/common/properties");
|
||||
const auto &index = allocator->GetIndex();
|
||||
properties = index.template GetBlockPtr<extractor::ProfileProperties>("/common/properties");
|
||||
const auto &metric_name = properties->GetWeightName();
|
||||
facades.push_back(std::make_shared<const Facade>(allocator, metric_name, 0));
|
||||
}
|
||||
|
||||
82
include/storage/shared_data_index.hpp
Normal file
82
include/storage/shared_data_index.hpp
Normal file
@ -0,0 +1,82 @@
|
||||
#ifndef OSRM_STORAGE_SHARED_DATA_INDEX_HPP
|
||||
#define OSRM_STORAGE_SHARED_DATA_INDEX_HPP
|
||||
|
||||
#include "storage/shared_datatype.hpp"
|
||||
|
||||
#include <boost/function_output_iterator.hpp>
|
||||
|
||||
#include <unordered_map>
|
||||
|
||||
namespace osrm
|
||||
{
|
||||
namespace storage
|
||||
{
|
||||
|
||||
// This class wraps one or more shared memory regions with the associated data layout
|
||||
// to abstract away in which region a block of memory is stored.
|
||||
class SharedDataIndex
|
||||
{
|
||||
public:
|
||||
struct AllocatedRegion
|
||||
{
|
||||
char *memory_ptr;
|
||||
DataLayout layout;
|
||||
};
|
||||
|
||||
SharedDataIndex() = default;
|
||||
SharedDataIndex(std::vector<AllocatedRegion> regions_) : regions(std::move(regions_))
|
||||
{
|
||||
// Build mapping from block name to region
|
||||
for (auto index : util::irange<std::uint32_t>(0, regions.size()))
|
||||
{
|
||||
regions[index].layout.List("",
|
||||
boost::make_function_output_iterator([&](const auto &name) {
|
||||
block_to_region[name] = index;
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
template <typename OutIter> void List(const std::string &name_prefix, OutIter out) const
|
||||
{
|
||||
for (const auto ®ion : regions)
|
||||
{
|
||||
region.layout.List(name_prefix, out);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T> auto GetBlockPtr(const std::string &name) const
|
||||
{
|
||||
const auto index_iter = block_to_region.find(name);
|
||||
const auto ®ion = regions[index_iter->second];
|
||||
return region.layout.GetBlockPtr<T>(region.memory_ptr, name);
|
||||
}
|
||||
|
||||
template <typename T> auto GetBlockPtr(const std::string &name)
|
||||
{
|
||||
const auto index_iter = block_to_region.find(name);
|
||||
const auto ®ion = regions[index_iter->second];
|
||||
return region.layout.GetBlockPtr<T>(region.memory_ptr, name);
|
||||
}
|
||||
|
||||
std::size_t GetBlockEntries(const std::string &name) const
|
||||
{
|
||||
const auto index_iter = block_to_region.find(name);
|
||||
const auto ®ion = regions[index_iter->second];
|
||||
return region.layout.GetBlockEntries(name);
|
||||
}
|
||||
|
||||
std::size_t GetBlockSize(const std::string &name) const
|
||||
{
|
||||
const auto index_iter = block_to_region.find(name);
|
||||
const auto ®ion = regions[index_iter->second];
|
||||
return region.layout.GetBlockSize(name);
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<AllocatedRegion> regions;
|
||||
std::unordered_map<std::string, std::uint32_t> block_to_region;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
@ -29,6 +29,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#define STORAGE_HPP
|
||||
|
||||
#include "storage/shared_datatype.hpp"
|
||||
#include "storage/shared_data_index.hpp"
|
||||
#include "storage/storage_config.hpp"
|
||||
|
||||
#include <boost/filesystem/path.hpp>
|
||||
@ -46,8 +47,10 @@ class Storage
|
||||
|
||||
int Run(int max_wait, const std::string &name);
|
||||
|
||||
void PopulateLayout(DataLayout &layout);
|
||||
void PopulateData(const DataLayout &layout, char *memory_ptr);
|
||||
void PopulateStaticLayout(DataLayout &layout);
|
||||
void PopulateUpdatableLayout(DataLayout &layout);
|
||||
void PopulateStaticData(const SharedDataIndex &index);
|
||||
void PopulateUpdatableData(const SharedDataIndex &index);
|
||||
|
||||
private:
|
||||
StorageConfig config;
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#ifndef OSRM_STOARGE_VIEW_FACTORY_HPP
|
||||
#define OSRM_STOARGE_VIEW_FACTORY_HPP
|
||||
|
||||
#include "storage/shared_datatype.hpp"
|
||||
#include "storage/shared_data_index.hpp"
|
||||
|
||||
#include "contractor/contracted_metric.hpp"
|
||||
#include "contractor/query_graph.hpp"
|
||||
@ -45,88 +45,78 @@ namespace storage
|
||||
{
|
||||
|
||||
template <typename T>
|
||||
util::vector_view<T>
|
||||
make_vector_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
util::vector_view<T> make_vector_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
return util::vector_view<T>(layout.GetBlockPtr<T>(memory_ptr, name),
|
||||
layout.GetBlockEntries(name));
|
||||
return util::vector_view<T>(index.GetBlockPtr<T>(name), index.GetBlockEntries(name));
|
||||
}
|
||||
|
||||
template <>
|
||||
inline util::vector_view<bool>
|
||||
make_vector_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
template<>
|
||||
inline util::vector_view<bool> make_vector_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
return util::vector_view<bool>(
|
||||
layout.GetBlockPtr<util::vector_view<bool>::Word>(memory_ptr, name),
|
||||
layout.GetBlockEntries(name));
|
||||
return util::vector_view<bool>(index.GetBlockPtr<util::vector_view<bool>::Word>(name),
|
||||
index.GetBlockEntries(name));
|
||||
}
|
||||
|
||||
template <typename T, std::size_t Bits>
|
||||
util::PackedVectorView<T, Bits>
|
||||
make_packed_vector_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
util::PackedVectorView<T, Bits> make_packed_vector_view(const SharedDataIndex &index,
|
||||
const std::string &name)
|
||||
{
|
||||
using V = util::PackedVectorView<T, Bits>;
|
||||
auto packed_internal = util::vector_view<typename V::block_type>(
|
||||
layout.GetBlockPtr<typename V::block_type>(memory_ptr, name + "/packed"),
|
||||
layout.GetBlockEntries(name + "/packed"));
|
||||
index.GetBlockPtr<typename V::block_type>(name + "/packed"),
|
||||
index.GetBlockEntries(name + "/packed"));
|
||||
// the real size needs to come from an external source
|
||||
return V{packed_internal, std::numeric_limits<std::size_t>::max()};
|
||||
}
|
||||
|
||||
inline auto
|
||||
make_name_table_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_name_table_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
auto blocks = make_vector_view<extractor::NameTableView::IndexedData::BlockReference>(
|
||||
memory_ptr, layout, name + "/blocks");
|
||||
auto values = make_vector_view<extractor::NameTableView::IndexedData::ValueType>(
|
||||
memory_ptr, layout, name + "/values");
|
||||
index, name + "/blocks");
|
||||
auto values =
|
||||
make_vector_view<extractor::NameTableView::IndexedData::ValueType>(index, name + "/values");
|
||||
|
||||
extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
|
||||
return extractor::NameTableView{index_data_view};
|
||||
}
|
||||
|
||||
inline auto make_lane_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_lane_data_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
return make_vector_view<util::guidance::LaneTupleIdPair>(memory_ptr, layout, name + "/data");
|
||||
return make_vector_view<util::guidance::LaneTupleIdPair>(index, name + "/data");
|
||||
}
|
||||
|
||||
inline auto make_turn_lane_description_views(char *memory_ptr,
|
||||
const DataLayout &layout,
|
||||
const std::string &name)
|
||||
inline auto make_turn_lane_description_views(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
auto offsets = make_vector_view<std::uint32_t>(memory_ptr, layout, name + "/offsets");
|
||||
auto masks =
|
||||
make_vector_view<extractor::TurnLaneType::Mask>(memory_ptr, layout, name + "/masks");
|
||||
auto offsets = make_vector_view<std::uint32_t>(index, name + "/offsets");
|
||||
auto masks = make_vector_view<extractor::TurnLaneType::Mask>(index, name + "/masks");
|
||||
|
||||
return std::make_tuple(offsets, masks);
|
||||
}
|
||||
|
||||
inline auto make_ebn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_ebn_data_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
auto edge_based_node_data =
|
||||
make_vector_view<extractor::EdgeBasedNode>(memory_ptr, layout, name + "/nodes");
|
||||
auto annotation_data = make_vector_view<extractor::NodeBasedEdgeAnnotation>(
|
||||
memory_ptr, layout, name + "/annotations");
|
||||
auto edge_based_node_data = make_vector_view<extractor::EdgeBasedNode>(index, name + "/nodes");
|
||||
auto annotation_data =
|
||||
make_vector_view<extractor::NodeBasedEdgeAnnotation>(index, name + "/annotations");
|
||||
|
||||
return extractor::EdgeBasedNodeDataView(std::move(edge_based_node_data),
|
||||
std::move(annotation_data));
|
||||
}
|
||||
|
||||
inline auto make_turn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_turn_data_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
auto lane_data_ids = make_vector_view<LaneDataID>(memory_ptr, layout, name + "/lane_data_ids");
|
||||
auto lane_data_ids = make_vector_view<LaneDataID>(index, name + "/lane_data_ids");
|
||||
|
||||
const auto turn_instructions = make_vector_view<guidance::TurnInstruction>(
|
||||
memory_ptr, layout, name + "/turn_instructions");
|
||||
const auto turn_instructions =
|
||||
make_vector_view<guidance::TurnInstruction>(index, name + "/turn_instructions");
|
||||
|
||||
const auto entry_class_ids =
|
||||
make_vector_view<EntryClassID>(memory_ptr, layout, name + "/entry_class_ids");
|
||||
const auto entry_class_ids = make_vector_view<EntryClassID>(index, name + "/entry_class_ids");
|
||||
|
||||
const auto pre_turn_bearings =
|
||||
make_vector_view<guidance::TurnBearing>(memory_ptr, layout, name + "/pre_turn_bearings");
|
||||
make_vector_view<guidance::TurnBearing>(index, name + "/pre_turn_bearings");
|
||||
|
||||
const auto post_turn_bearings =
|
||||
make_vector_view<guidance::TurnBearing>(memory_ptr, layout, name + "/post_turn_bearings");
|
||||
make_vector_view<guidance::TurnBearing>(index, name + "/post_turn_bearings");
|
||||
|
||||
return guidance::TurnDataView(std::move(turn_instructions),
|
||||
std::move(lane_data_ids),
|
||||
@ -135,40 +125,39 @@ inline auto make_turn_data_view(char *memory_ptr, const DataLayout &layout, cons
|
||||
std::move(post_turn_bearings));
|
||||
}
|
||||
|
||||
inline auto
|
||||
make_segment_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_segment_data_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
auto geometry_begin_indices = make_vector_view<unsigned>(memory_ptr, layout, name + "/index");
|
||||
auto geometry_begin_indices = make_vector_view<unsigned>(index, name + "/index");
|
||||
|
||||
auto node_list = make_vector_view<NodeID>(memory_ptr, layout, name + "/nodes");
|
||||
auto node_list = make_vector_view<NodeID>(index, name + "/nodes");
|
||||
|
||||
auto num_entries = layout.GetBlockEntries(name + "/nodes");
|
||||
auto num_entries = index.GetBlockEntries(name + "/nodes");
|
||||
|
||||
extractor::SegmentDataView::SegmentWeightVector fwd_weight_list(
|
||||
make_vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
|
||||
memory_ptr, layout, name + "/forward_weights/packed"),
|
||||
index, name + "/forward_weights/packed"),
|
||||
num_entries);
|
||||
|
||||
extractor::SegmentDataView::SegmentWeightVector rev_weight_list(
|
||||
make_vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
|
||||
memory_ptr, layout, name + "/reverse_weights/packed"),
|
||||
index, name + "/reverse_weights/packed"),
|
||||
num_entries);
|
||||
|
||||
extractor::SegmentDataView::SegmentDurationVector fwd_duration_list(
|
||||
make_vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
|
||||
memory_ptr, layout, name + "/forward_durations/packed"),
|
||||
index, name + "/forward_durations/packed"),
|
||||
num_entries);
|
||||
|
||||
extractor::SegmentDataView::SegmentDurationVector rev_duration_list(
|
||||
make_vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
|
||||
memory_ptr, layout, name + "/reverse_durations/packed"),
|
||||
index, name + "/reverse_durations/packed"),
|
||||
num_entries);
|
||||
|
||||
auto fwd_datasources_list =
|
||||
make_vector_view<DatasourceID>(memory_ptr, layout, name + "/forward_data_sources");
|
||||
make_vector_view<DatasourceID>(index, name + "/forward_data_sources");
|
||||
|
||||
auto rev_datasources_list =
|
||||
make_vector_view<DatasourceID>(memory_ptr, layout, name + "/reverse_data_sources");
|
||||
make_vector_view<DatasourceID>(index, name + "/reverse_data_sources");
|
||||
|
||||
return extractor::SegmentDataView{std::move(geometry_begin_indices),
|
||||
std::move(node_list),
|
||||
@ -180,52 +169,46 @@ make_segment_data_view(char *memory_ptr, const DataLayout &layout, const std::st
|
||||
std::move(rev_datasources_list)};
|
||||
}
|
||||
|
||||
inline auto
|
||||
make_coordinates_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_coordinates_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
return make_vector_view<util::Coordinate>(memory_ptr, layout, name);
|
||||
return make_vector_view<util::Coordinate>(index, name);
|
||||
}
|
||||
|
||||
inline auto make_osm_ids_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_osm_ids_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
return make_packed_vector_view<extractor::PackedOSMIDsView::value_type,
|
||||
extractor::PackedOSMIDsView::value_size>(
|
||||
memory_ptr, layout, name);
|
||||
extractor::PackedOSMIDsView::value_size>(index, name);
|
||||
}
|
||||
|
||||
inline auto make_nbn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_nbn_data_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
return std::make_tuple(make_coordinates_view(memory_ptr, layout, name + "/coordinates"),
|
||||
make_osm_ids_view(memory_ptr, layout, name + "/osm_node_ids"));
|
||||
return std::make_tuple(make_coordinates_view(index, name + "/coordinates"),
|
||||
make_osm_ids_view(index, name + "/osm_node_ids"));
|
||||
}
|
||||
|
||||
inline auto
|
||||
make_turn_weight_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_turn_weight_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
return make_vector_view<TurnPenalty>(memory_ptr, layout, name + "/weight");
|
||||
return make_vector_view<TurnPenalty>(index, name + "/weight");
|
||||
}
|
||||
|
||||
inline auto
|
||||
make_turn_duration_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_turn_duration_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
return make_vector_view<TurnPenalty>(memory_ptr, layout, name + "/duration");
|
||||
return make_vector_view<TurnPenalty>(index, name + "/duration");
|
||||
}
|
||||
|
||||
inline auto
|
||||
make_search_tree_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_search_tree_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
using RTreeLeaf = extractor::EdgeBasedNodeSegment;
|
||||
using RTreeNode = util::StaticRTree<RTreeLeaf, storage::Ownership::View>::TreeNode;
|
||||
|
||||
const auto search_tree = make_vector_view<RTreeNode>(memory_ptr, layout, name + "/search_tree");
|
||||
const auto search_tree = make_vector_view<RTreeNode>(index, name + "/search_tree");
|
||||
|
||||
const auto rtree_level_starts =
|
||||
make_vector_view<std::uint64_t>(memory_ptr, layout, name + "/search_tree_level_starts");
|
||||
make_vector_view<std::uint64_t>(index, name + "/search_tree_level_starts");
|
||||
|
||||
const auto coordinates =
|
||||
make_coordinates_view(memory_ptr, layout, "/common/nbn_data/coordinates");
|
||||
const auto coordinates = make_coordinates_view(index, "/common/nbn_data/coordinates");
|
||||
|
||||
const char *path = layout.GetBlockPtr<char>(memory_ptr, name + "/file_index_path");
|
||||
const char *path = index.template GetBlockPtr<char>(name + "/file_index_path");
|
||||
|
||||
if (!boost::filesystem::exists(boost::filesystem::path{path}))
|
||||
{
|
||||
@ -237,72 +220,63 @@ make_search_tree_view(char *memory_ptr, const DataLayout &layout, const std::str
|
||||
std::move(search_tree), std::move(rtree_level_starts), path, std::move(coordinates)};
|
||||
}
|
||||
|
||||
inline auto
|
||||
make_intersection_bearings_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_intersection_bearings_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
auto bearing_offsets =
|
||||
make_vector_view<unsigned>(memory_ptr, layout, name + "/class_id_to_ranges/block_offsets");
|
||||
make_vector_view<unsigned>(index, name + "/class_id_to_ranges/block_offsets");
|
||||
auto bearing_blocks = make_vector_view<util::RangeTable<16, storage::Ownership::View>::BlockT>(
|
||||
memory_ptr, layout, name + "/class_id_to_ranges/diff_blocks");
|
||||
auto bearing_values =
|
||||
make_vector_view<DiscreteBearing>(memory_ptr, layout, name + "/bearing_values");
|
||||
index, name + "/class_id_to_ranges/diff_blocks");
|
||||
auto bearing_values = make_vector_view<DiscreteBearing>(index, name + "/bearing_values");
|
||||
util::RangeTable<16, storage::Ownership::View> bearing_range_table(
|
||||
std::move(bearing_offsets),
|
||||
std::move(bearing_blocks),
|
||||
static_cast<unsigned>(bearing_values.size()));
|
||||
|
||||
auto bearing_class_id =
|
||||
make_vector_view<BearingClassID>(memory_ptr, layout, name + "/node_to_class_id");
|
||||
auto bearing_class_id = make_vector_view<BearingClassID>(index, name + "/node_to_class_id");
|
||||
return extractor::IntersectionBearingsView{
|
||||
std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)};
|
||||
}
|
||||
|
||||
inline auto
|
||||
make_entry_classes_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_entry_classes_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
return make_vector_view<util::guidance::EntryClass>(memory_ptr, layout, name);
|
||||
return make_vector_view<util::guidance::EntryClass>(index, name);
|
||||
}
|
||||
|
||||
inline auto
|
||||
make_contracted_metric_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_contracted_metric_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
auto node_list = make_vector_view<contractor::QueryGraphView::NodeArrayEntry>(
|
||||
memory_ptr, layout, name + "/contracted_graph/node_array");
|
||||
index, name + "/contracted_graph/node_array");
|
||||
auto edge_list = make_vector_view<contractor::QueryGraphView::EdgeArrayEntry>(
|
||||
memory_ptr, layout, name + "/contracted_graph/edge_array");
|
||||
index, name + "/contracted_graph/edge_array");
|
||||
|
||||
std::vector<util::vector_view<bool>> edge_filter;
|
||||
layout.List(name + "/exclude",
|
||||
boost::make_function_output_iterator([&](const auto &filter_name) {
|
||||
edge_filter.push_back(make_vector_view<bool>(memory_ptr, layout, filter_name));
|
||||
}));
|
||||
index.List(name + "/exclude",
|
||||
boost::make_function_output_iterator([&](const auto &filter_name) {
|
||||
edge_filter.push_back(make_vector_view<bool>(index, filter_name));
|
||||
}));
|
||||
|
||||
return contractor::ContractedMetricView{{std::move(node_list), std::move(edge_list)},
|
||||
std::move(edge_filter)};
|
||||
}
|
||||
|
||||
inline auto make_partition_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_partition_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
auto level_data_ptr = layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData>(
|
||||
memory_ptr, name + "/level_data");
|
||||
auto partition = make_vector_view<PartitionID>(memory_ptr, layout, name + "/partition");
|
||||
auto cell_to_children =
|
||||
make_vector_view<CellID>(memory_ptr, layout, name + "/cell_to_children");
|
||||
auto level_data_ptr =
|
||||
index.template GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData>(name +
|
||||
"/level_data");
|
||||
auto partition = make_vector_view<PartitionID>(index, name + "/partition");
|
||||
auto cell_to_children = make_vector_view<CellID>(index, name + "/cell_to_children");
|
||||
|
||||
return partitioner::MultiLevelPartitionView{
|
||||
level_data_ptr, std::move(partition), std::move(cell_to_children)};
|
||||
}
|
||||
|
||||
inline auto
|
||||
make_cell_storage_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_cell_storage_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
auto source_boundary = make_vector_view<NodeID>(memory_ptr, layout, name + "/source_boundary");
|
||||
auto destination_boundary =
|
||||
make_vector_view<NodeID>(memory_ptr, layout, name + "/destination_boundary");
|
||||
auto cells = make_vector_view<partitioner::CellStorageView::CellData>(
|
||||
memory_ptr, layout, name + "/cells");
|
||||
auto level_offsets =
|
||||
make_vector_view<std::uint64_t>(memory_ptr, layout, name + "/level_to_cell_offset");
|
||||
auto source_boundary = make_vector_view<NodeID>(index, name + "/source_boundary");
|
||||
auto destination_boundary = make_vector_view<NodeID>(index, name + "/destination_boundary");
|
||||
auto cells = make_vector_view<partitioner::CellStorageView::CellData>(index, name + "/cells");
|
||||
auto level_offsets = make_vector_view<std::uint64_t>(index, name + "/level_to_cell_offset");
|
||||
|
||||
return partitioner::CellStorageView{std::move(source_boundary),
|
||||
std::move(destination_boundary),
|
||||
@ -310,8 +284,7 @@ make_cell_storage_view(char *memory_ptr, const DataLayout &layout, const std::st
|
||||
std::move(level_offsets)};
|
||||
}
|
||||
|
||||
inline auto make_filtered_cell_metric_view(char *memory_ptr,
|
||||
const DataLayout &layout,
|
||||
inline auto make_filtered_cell_metric_view(const SharedDataIndex &index,
|
||||
const std::string &name,
|
||||
const std::size_t exclude_index)
|
||||
{
|
||||
@ -321,26 +294,25 @@ inline auto make_filtered_cell_metric_view(char *memory_ptr,
|
||||
auto weights_block_id = prefix + "/weights";
|
||||
auto durations_block_id = prefix + "/durations";
|
||||
|
||||
auto weights = make_vector_view<EdgeWeight>(memory_ptr, layout, weights_block_id);
|
||||
auto durations = make_vector_view<EdgeDuration>(memory_ptr, layout, durations_block_id);
|
||||
auto weights = make_vector_view<EdgeWeight>(index, weights_block_id);
|
||||
auto durations = make_vector_view<EdgeDuration>(index, durations_block_id);
|
||||
|
||||
return customizer::CellMetricView{std::move(weights), std::move(durations)};
|
||||
}
|
||||
|
||||
inline auto
|
||||
make_cell_metric_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_cell_metric_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
std::vector<customizer::CellMetricView> cell_metric_excludes;
|
||||
|
||||
std::vector<std::string> metric_prefix_names;
|
||||
layout.List(name + "/exclude/", std::back_inserter(metric_prefix_names));
|
||||
index.List(name + "/exclude/", std::back_inserter(metric_prefix_names));
|
||||
for (const auto &prefix : metric_prefix_names)
|
||||
{
|
||||
auto weights_block_id = prefix + "/weights";
|
||||
auto durations_block_id = prefix + "/durations";
|
||||
|
||||
auto weights = make_vector_view<EdgeWeight>(memory_ptr, layout, weights_block_id);
|
||||
auto durations = make_vector_view<EdgeDuration>(memory_ptr, layout, durations_block_id);
|
||||
auto weights = make_vector_view<EdgeWeight>(index, weights_block_id);
|
||||
auto durations = make_vector_view<EdgeDuration>(index, durations_block_id);
|
||||
|
||||
cell_metric_excludes.push_back(
|
||||
customizer::CellMetricView{std::move(weights), std::move(durations)});
|
||||
@ -349,42 +321,39 @@ make_cell_metric_view(char *memory_ptr, const DataLayout &layout, const std::str
|
||||
return cell_metric_excludes;
|
||||
}
|
||||
|
||||
inline auto
|
||||
make_multi_level_graph_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_multi_level_graph_view(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
auto node_list = make_vector_view<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry>(
|
||||
memory_ptr, layout, name + "/node_array");
|
||||
index, name + "/node_array");
|
||||
auto edge_list = make_vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry>(
|
||||
memory_ptr, layout, name + "/edge_array");
|
||||
index, name + "/edge_array");
|
||||
auto node_to_offset = make_vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset>(
|
||||
memory_ptr, layout, name + "/node_to_edge_offset");
|
||||
index, name + "/node_to_edge_offset");
|
||||
|
||||
return customizer::MultiLevelEdgeBasedGraphView(
|
||||
std::move(node_list), std::move(edge_list), std::move(node_to_offset));
|
||||
}
|
||||
|
||||
inline auto
|
||||
make_maneuver_overrides_views(char *memory_ptr, const DataLayout &layout, const std::string &name)
|
||||
inline auto make_maneuver_overrides_views(const SharedDataIndex &index, const std::string &name)
|
||||
{
|
||||
auto maneuver_overrides = make_vector_view<extractor::StorageManeuverOverride>(
|
||||
memory_ptr, layout, name + "/overrides");
|
||||
auto maneuver_overrides =
|
||||
make_vector_view<extractor::StorageManeuverOverride>(index, name + "/overrides");
|
||||
auto maneuver_override_node_sequences =
|
||||
make_vector_view<NodeID>(memory_ptr, layout, name + "/node_sequences");
|
||||
make_vector_view<NodeID>(index, name + "/node_sequences");
|
||||
|
||||
return std::make_tuple(maneuver_overrides, maneuver_override_node_sequences);
|
||||
}
|
||||
|
||||
inline auto make_filtered_graph_view(char *memory_ptr,
|
||||
const DataLayout &layout,
|
||||
inline auto make_filtered_graph_view(const SharedDataIndex &index,
|
||||
const std::string &name,
|
||||
const std::size_t exclude_index)
|
||||
{
|
||||
auto exclude_prefix = name + "/exclude/" + std::to_string(exclude_index);
|
||||
auto edge_filter = make_vector_view<bool>(memory_ptr, layout, exclude_prefix + "/edge_filter");
|
||||
auto edge_filter = make_vector_view<bool>(index, exclude_prefix + "/edge_filter");
|
||||
auto node_list = make_vector_view<contractor::QueryGraphView::NodeArrayEntry>(
|
||||
memory_ptr, layout, name + "/contracted_graph/node_array");
|
||||
index, name + "/contracted_graph/node_array");
|
||||
auto edge_list = make_vector_view<contractor::QueryGraphView::EdgeArrayEntry>(
|
||||
memory_ptr, layout, name + "/contracted_graph/edge_array");
|
||||
index, name + "/contracted_graph/edge_array");
|
||||
|
||||
return util::FilteredGraphView<contractor::QueryGraphView>({node_list, edge_list}, edge_filter);
|
||||
}
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
#include "engine/datafacade/mmap_memory_allocator.hpp"
|
||||
|
||||
#include "storage/io.hpp"
|
||||
#include "storage/serialization.hpp"
|
||||
#include "storage/storage.hpp"
|
||||
|
||||
#include "util/log.hpp"
|
||||
@ -22,31 +24,43 @@ MMapMemoryAllocator::MMapMemoryAllocator(const storage::StorageConfig &config,
|
||||
if (!boost::filesystem::exists(memory_file))
|
||||
{
|
||||
storage::DataLayout initial_layout;
|
||||
storage.PopulateLayout(initial_layout);
|
||||
storage.PopulateStaticLayout(initial_layout);
|
||||
storage.PopulateUpdatableLayout(initial_layout);
|
||||
|
||||
auto data_size = initial_layout.GetSizeOfLayout();
|
||||
auto total_size = data_size + sizeof(storage::DataLayout);
|
||||
|
||||
storage::io::BufferWriter writer;
|
||||
storage::serialization::write(writer, initial_layout);
|
||||
auto encoded_layout = writer.GetBuffer();
|
||||
|
||||
auto total_size = data_size + encoded_layout.size();
|
||||
|
||||
mapped_memory = util::mmapFile<char>(memory_file, mapped_memory_file, total_size);
|
||||
|
||||
data_layout = reinterpret_cast<storage::DataLayout *>(mapped_memory.data());
|
||||
*data_layout = initial_layout;
|
||||
storage.PopulateData(*data_layout, GetMemory());
|
||||
std::copy(encoded_layout.begin(), encoded_layout.end(), mapped_memory.data());
|
||||
|
||||
index = storage::SharedDataIndex(
|
||||
{{mapped_memory.data() + encoded_layout.size(), std::move(initial_layout)}});
|
||||
|
||||
storage.PopulateStaticData(index);
|
||||
storage.PopulateUpdatableData(index);
|
||||
}
|
||||
else
|
||||
{
|
||||
mapped_memory = util::mmapFile<char>(memory_file, mapped_memory_file);
|
||||
data_layout = reinterpret_cast<storage::DataLayout *>(mapped_memory.data());
|
||||
|
||||
storage::DataLayout layout;
|
||||
storage::io::BufferReader reader(mapped_memory.data());
|
||||
storage::serialization::read(reader, layout);
|
||||
auto layout_size = reader.GetPosition();
|
||||
|
||||
index = storage::SharedDataIndex({{mapped_memory.data() + layout_size, std::move(layout)}});
|
||||
}
|
||||
}
|
||||
|
||||
MMapMemoryAllocator::~MMapMemoryAllocator() {}
|
||||
|
||||
storage::DataLayout &MMapMemoryAllocator::GetLayout() { return *data_layout; }
|
||||
char *MMapMemoryAllocator::GetMemory()
|
||||
{
|
||||
return mapped_memory.data() + sizeof(storage::DataLayout);
|
||||
}
|
||||
const storage::SharedDataIndex &MMapMemoryAllocator::GetIndex() { return index; }
|
||||
|
||||
} // namespace datafacade
|
||||
} // namespace engine
|
||||
|
||||
@ -15,17 +15,22 @@ ProcessMemoryAllocator::ProcessMemoryAllocator(const storage::StorageConfig &con
|
||||
storage::Storage storage(config);
|
||||
|
||||
// Calculate the layout/size of the memory block
|
||||
storage.PopulateLayout(internal_layout);
|
||||
storage::DataLayout layout;
|
||||
storage.PopulateStaticLayout(layout);
|
||||
storage.PopulateUpdatableLayout(layout);
|
||||
|
||||
// Allocate the memory block, then load data from files into it
|
||||
internal_memory = std::make_unique<char[]>(internal_layout.GetSizeOfLayout());
|
||||
storage.PopulateData(internal_layout, internal_memory.get());
|
||||
internal_memory = std::make_unique<char[]>(layout.GetSizeOfLayout());
|
||||
|
||||
index = storage::SharedDataIndex({{internal_memory.get(), std::move(layout)}});
|
||||
|
||||
storage.PopulateStaticData(index);
|
||||
storage.PopulateUpdatableData(index);
|
||||
}
|
||||
|
||||
ProcessMemoryAllocator::~ProcessMemoryAllocator() {}
|
||||
|
||||
const storage::DataLayout &ProcessMemoryAllocator::GetLayout() { return internal_layout; }
|
||||
char *ProcessMemoryAllocator::GetMemory() { return internal_memory.get(); }
|
||||
const storage::SharedDataIndex &ProcessMemoryAllocator::GetIndex() { return index; }
|
||||
|
||||
} // namespace datafacade
|
||||
} // namespace engine
|
||||
|
||||
@ -13,27 +13,32 @@ namespace engine
|
||||
namespace datafacade
|
||||
{
|
||||
|
||||
SharedMemoryAllocator::SharedMemoryAllocator(storage::SharedRegionRegister::ShmKey data_shm_key)
|
||||
SharedMemoryAllocator::SharedMemoryAllocator(
|
||||
const std::vector<storage::SharedRegionRegister::ShmKey> &shm_keys)
|
||||
{
|
||||
util::Log(logDEBUG) << "Loading new data for region " << (int)data_shm_key;
|
||||
std::vector<storage::SharedDataIndex::AllocatedRegion> regions;
|
||||
|
||||
BOOST_ASSERT(storage::SharedMemory::RegionExists(data_shm_key));
|
||||
m_large_memory = storage::makeSharedMemory(data_shm_key);
|
||||
for (const auto shm_key : shm_keys)
|
||||
{
|
||||
util::Log(logDEBUG) << "Loading new data for region " << (int)shm_key;
|
||||
BOOST_ASSERT(storage::SharedMemory::RegionExists(shm_key));
|
||||
auto mem = storage::makeSharedMemory(shm_key);
|
||||
|
||||
storage::io::BufferReader reader(reinterpret_cast<char *>(m_large_memory->Ptr()),
|
||||
m_large_memory->Size());
|
||||
storage::serialization::read(reader, data_layout);
|
||||
layout_size = reader.GetPosition();
|
||||
util::Log(logDEBUG) << "Data layout has size " << layout_size;
|
||||
storage::io::BufferReader reader(reinterpret_cast<char *>(mem->Ptr()), mem->Size());
|
||||
storage::DataLayout layout;
|
||||
storage::serialization::read(reader, layout);
|
||||
auto layout_size = reader.GetPosition();
|
||||
|
||||
regions.push_back({reinterpret_cast<char *>(mem->Ptr()) + layout_size, std::move(layout)});
|
||||
memory_regions.push_back(std::move(mem));
|
||||
}
|
||||
|
||||
index = storage::SharedDataIndex{std::move(regions)};
|
||||
}
|
||||
|
||||
SharedMemoryAllocator::~SharedMemoryAllocator() {}
|
||||
|
||||
const storage::DataLayout &SharedMemoryAllocator::GetLayout() { return data_layout; }
|
||||
char *SharedMemoryAllocator::GetMemory()
|
||||
{
|
||||
return reinterpret_cast<char *>(m_large_memory->Ptr()) + layout_size;
|
||||
}
|
||||
const storage::SharedDataIndex &SharedMemoryAllocator::GetIndex() { return index; }
|
||||
|
||||
} // namespace datafacade
|
||||
} // namespace engine
|
||||
|
||||
@ -42,7 +42,9 @@ namespace storage
|
||||
{
|
||||
namespace
|
||||
{
|
||||
inline void readBlocks(const boost::filesystem::path &path, DataLayout &layout)
|
||||
using Monitor = SharedMonitor<SharedRegionRegister>;
|
||||
|
||||
void readBlocks(const boost::filesystem::path &path, DataLayout &layout)
|
||||
{
|
||||
tar::FileReader reader(path, tar::FileReader::VerifyFingerprint);
|
||||
|
||||
@ -59,9 +61,128 @@ inline void readBlocks(const boost::filesystem::path &path, DataLayout &layout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct RegionHandle
|
||||
{
|
||||
std::unique_ptr<SharedMemory> memory;
|
||||
char *data_ptr;
|
||||
std::uint8_t shm_key;
|
||||
};
|
||||
|
||||
auto setupRegion(SharedRegionRegister &shared_register, const DataLayout &layout)
|
||||
{
|
||||
// This is safe because we have an exclusive lock for all osrm-datastore processes.
|
||||
auto shm_key = shared_register.ReserveKey();
|
||||
|
||||
// ensure that the shared memory region we want to write to is really removed
|
||||
// this is only needef for failure recovery because we actually wait for all clients
|
||||
// to detach at the end of the function
|
||||
if (storage::SharedMemory::RegionExists(shm_key))
|
||||
{
|
||||
util::Log(logWARNING) << "Old shared memory region " << (int)shm_key << " still exists.";
|
||||
util::UnbufferedLog() << "Retrying removal... ";
|
||||
storage::SharedMemory::Remove(shm_key);
|
||||
util::UnbufferedLog() << "ok.";
|
||||
}
|
||||
|
||||
io::BufferWriter writer;
|
||||
serialization::write(writer, layout);
|
||||
auto encoded_static_layout = writer.GetBuffer();
|
||||
|
||||
// Allocate shared memory block
|
||||
auto regions_size = encoded_static_layout.size() + layout.GetSizeOfLayout();
|
||||
util::Log() << "Data layout has a size of " << encoded_static_layout.size() << " bytes";
|
||||
util::Log() << "Allocating shared memory of " << regions_size << " bytes";
|
||||
auto memory = makeSharedMemory(shm_key, regions_size);
|
||||
|
||||
// Copy memory static_layout to shared memory and populate data
|
||||
char *shared_memory_ptr = static_cast<char *>(memory->Ptr());
|
||||
auto data_ptr =
|
||||
std::copy_n(encoded_static_layout.data(), encoded_static_layout.size(), shared_memory_ptr);
|
||||
|
||||
return RegionHandle{std::move(memory), data_ptr, shm_key};
|
||||
}
|
||||
|
||||
using Monitor = SharedMonitor<SharedRegionRegister>;
|
||||
bool swapData(Monitor &monitor,
|
||||
SharedRegionRegister &shared_register,
|
||||
const std::map<std::string, RegionHandle> &handles,
|
||||
int max_wait)
|
||||
{
|
||||
std::vector<RegionHandle> old_handles;
|
||||
|
||||
{ // Lock for write access shared region mutex
|
||||
boost::interprocess::scoped_lock<Monitor::mutex_type> lock(monitor.get_mutex(),
|
||||
boost::interprocess::defer_lock);
|
||||
|
||||
if (max_wait >= 0)
|
||||
{
|
||||
if (!lock.timed_lock(boost::posix_time::microsec_clock::universal_time() +
|
||||
boost::posix_time::seconds(max_wait)))
|
||||
{
|
||||
util::Log(logERROR) << "Could not aquire current region lock after " << max_wait
|
||||
<< " seconds. Data update failed.";
|
||||
|
||||
for (auto &pair : handles)
|
||||
{
|
||||
SharedMemory::Remove(pair.second.shm_key);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
lock.lock();
|
||||
}
|
||||
|
||||
for (auto &pair : handles)
|
||||
{
|
||||
auto region_id = shared_register.Find(pair.first);
|
||||
if (region_id == SharedRegionRegister::INVALID_REGION_ID)
|
||||
{
|
||||
region_id = shared_register.Register(pair.first, pair.second.shm_key);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto &shared_region = shared_register.GetRegion(region_id);
|
||||
|
||||
old_handles.push_back(RegionHandle{
|
||||
makeSharedMemory(shared_region.shm_key), nullptr, shared_region.shm_key});
|
||||
|
||||
shared_region.shm_key = pair.second.shm_key;
|
||||
shared_region.timestamp++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
util::Log() << "All data loaded. Notify all client about new data in:";
|
||||
for (const auto &pair : handles)
|
||||
{
|
||||
util::Log() << pair.first << "\t" << static_cast<int>(pair.second.shm_key);
|
||||
}
|
||||
monitor.notify_all();
|
||||
|
||||
for (auto &old_handle : old_handles)
|
||||
{
|
||||
util::UnbufferedLog() << "Marking old shared memory region "
|
||||
<< static_cast<int>(old_handle.shm_key) << " for removal... ";
|
||||
|
||||
// SHMCTL(2): Mark the segment to be destroyed. The segment will actually be destroyed
|
||||
// only after the last process detaches it.
|
||||
storage::SharedMemory::Remove(old_handle.shm_key);
|
||||
util::UnbufferedLog() << "ok.";
|
||||
|
||||
util::UnbufferedLog() << "Waiting for clients to detach... ";
|
||||
old_handle.memory->WaitForDetach();
|
||||
util::UnbufferedLog() << " ok.";
|
||||
|
||||
shared_register.ReleaseKey(old_handle.shm_key);
|
||||
}
|
||||
|
||||
util::Log() << "All clients switched.";
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
Storage::Storage(StorageConfig config_) : config(std::move(config_)) {}
|
||||
|
||||
@ -103,106 +224,26 @@ int Storage::Run(int max_wait, const std::string &dataset_name)
|
||||
Monitor monitor(SharedRegionRegister{});
|
||||
auto &shared_register = monitor.data();
|
||||
|
||||
// This is safe because we have an exclusive lock for all osrm-datastore processes.
|
||||
auto shm_key = shared_register.ReserveKey();
|
||||
|
||||
// ensure that the shared memory region we want to write to is really removed
|
||||
// this is only needef for failure recovery because we actually wait for all clients
|
||||
// to detach at the end of the function
|
||||
if (storage::SharedMemory::RegionExists(shm_key))
|
||||
{
|
||||
util::Log(logWARNING) << "Old shared memory region " << shm_key << " still exists.";
|
||||
util::UnbufferedLog() << "Retrying removal... ";
|
||||
storage::SharedMemory::Remove(shm_key);
|
||||
util::UnbufferedLog() << "ok.";
|
||||
}
|
||||
|
||||
util::Log() << "Loading data into " << static_cast<int>(shm_key);
|
||||
|
||||
// Populate a memory layout into stack memory
|
||||
DataLayout layout;
|
||||
PopulateLayout(layout);
|
||||
DataLayout static_layout;
|
||||
PopulateStaticLayout(static_layout);
|
||||
DataLayout updatable_layout;
|
||||
PopulateUpdatableLayout(updatable_layout);
|
||||
|
||||
io::BufferWriter writer;
|
||||
serialization::write(writer, layout);
|
||||
auto encoded_layout = writer.GetBuffer();
|
||||
auto static_handle = setupRegion(shared_register, static_layout);
|
||||
auto updatable_handle = setupRegion(shared_register, updatable_layout);
|
||||
|
||||
// Allocate shared memory block
|
||||
auto regions_size = encoded_layout.size() + layout.GetSizeOfLayout();
|
||||
util::Log() << "Data layout has a size of " << encoded_layout.size() << " bytes";
|
||||
util::Log() << "Allocating shared memory of " << regions_size << " bytes";
|
||||
auto data_memory = makeSharedMemory(shm_key, regions_size);
|
||||
SharedDataIndex index{
|
||||
{{static_handle.data_ptr, static_layout}, {updatable_handle.data_ptr, updatable_layout}}};
|
||||
|
||||
// Copy memory layout to shared memory and populate data
|
||||
char *shared_memory_ptr = static_cast<char *>(data_memory->Ptr());
|
||||
std::copy_n(encoded_layout.data(), encoded_layout.size(), shared_memory_ptr);
|
||||
PopulateData(layout, shared_memory_ptr + encoded_layout.size());
|
||||
PopulateStaticData(index);
|
||||
PopulateUpdatableData(index);
|
||||
|
||||
std::uint32_t next_timestamp = 0;
|
||||
std::uint8_t in_use_key = SharedRegionRegister::MAX_SHM_KEYS;
|
||||
std::map<std::string, RegionHandle> handles;
|
||||
handles[dataset_name + "/static"] = std::move(static_handle);
|
||||
handles[dataset_name + "/updatable"] = std::move(updatable_handle);
|
||||
|
||||
{ // Lock for write access shared region mutex
|
||||
boost::interprocess::scoped_lock<Monitor::mutex_type> lock(monitor.get_mutex(),
|
||||
boost::interprocess::defer_lock);
|
||||
|
||||
if (max_wait >= 0)
|
||||
{
|
||||
if (!lock.timed_lock(boost::posix_time::microsec_clock::universal_time() +
|
||||
boost::posix_time::seconds(max_wait)))
|
||||
{
|
||||
util::Log(logERROR) << "Could not aquire current region lock after " << max_wait
|
||||
<< " seconds. Data update failed.";
|
||||
SharedMemory::Remove(shm_key);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
lock.lock();
|
||||
}
|
||||
|
||||
auto region_id = shared_register.Find(dataset_name + "/data");
|
||||
if (region_id == SharedRegionRegister::INVALID_REGION_ID)
|
||||
{
|
||||
region_id = shared_register.Register(dataset_name + "/data", shm_key);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto &shared_region = shared_register.GetRegion(region_id);
|
||||
next_timestamp = shared_region.timestamp + 1;
|
||||
in_use_key = shared_region.shm_key;
|
||||
shared_region.shm_key = shm_key;
|
||||
shared_region.timestamp = next_timestamp;
|
||||
}
|
||||
}
|
||||
|
||||
util::Log() << "All data loaded. Notify all client about new data in "
|
||||
<< static_cast<int>(shm_key) << " with timestamp " << next_timestamp;
|
||||
monitor.notify_all();
|
||||
|
||||
// SHMCTL(2): Mark the segment to be destroyed. The segment will actually be destroyed
|
||||
// only after the last process detaches it.
|
||||
if (in_use_key != SharedRegionRegister::MAX_SHM_KEYS &&
|
||||
storage::SharedMemory::RegionExists(in_use_key))
|
||||
{
|
||||
util::UnbufferedLog() << "Marking old shared memory region " << static_cast<int>(in_use_key)
|
||||
<< " for removal... ";
|
||||
|
||||
// aquire a handle for the old shared memory region before we mark it for deletion
|
||||
// we will need this to wait for all users to detach
|
||||
auto in_use_shared_memory = makeSharedMemory(in_use_key);
|
||||
|
||||
storage::SharedMemory::Remove(in_use_key);
|
||||
util::UnbufferedLog() << "ok.";
|
||||
|
||||
util::UnbufferedLog() << "Waiting for clients to detach... ";
|
||||
in_use_shared_memory->WaitForDetach();
|
||||
util::UnbufferedLog() << " ok.";
|
||||
|
||||
shared_register.ReleaseKey(in_use_key);
|
||||
}
|
||||
|
||||
util::Log() << "All clients switched.";
|
||||
swapData(monitor, shared_register, handles, max_wait);
|
||||
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
@ -212,35 +253,28 @@ int Storage::Run(int max_wait, const std::string &dataset_name)
|
||||
* memory needs to be allocated, and the position of each data structure
|
||||
* in that big block. It updates the fields in the DataLayout parameter.
|
||||
*/
|
||||
void Storage::PopulateLayout(DataLayout &layout)
|
||||
void Storage::PopulateStaticLayout(DataLayout &static_layout)
|
||||
{
|
||||
{
|
||||
auto absolute_file_index_path =
|
||||
boost::filesystem::absolute(config.GetPath(".osrm.fileIndex"));
|
||||
|
||||
layout.SetBlock("/common/rtree/file_index_path",
|
||||
make_block<char>(absolute_file_index_path.string().length() + 1));
|
||||
static_layout.SetBlock("/common/rtree/file_index_path",
|
||||
make_block<char>(absolute_file_index_path.string().length() + 1));
|
||||
}
|
||||
|
||||
constexpr bool REQUIRED = true;
|
||||
constexpr bool OPTIONAL = false;
|
||||
std::vector<std::pair<bool, boost::filesystem::path>> tar_files = {
|
||||
{OPTIONAL, config.GetPath(".osrm.mldgr")},
|
||||
{OPTIONAL, config.GetPath(".osrm.cells")},
|
||||
{OPTIONAL, config.GetPath(".osrm.partition")},
|
||||
{OPTIONAL, config.GetPath(".osrm.cell_metrics")},
|
||||
{OPTIONAL, config.GetPath(".osrm.hsgr")},
|
||||
{REQUIRED, config.GetPath(".osrm.icd")},
|
||||
{REQUIRED, config.GetPath(".osrm.properties")},
|
||||
{REQUIRED, config.GetPath(".osrm.nbg_nodes")},
|
||||
{REQUIRED, config.GetPath(".osrm.datasource_names")},
|
||||
{REQUIRED, config.GetPath(".osrm.geometry")},
|
||||
{REQUIRED, config.GetPath(".osrm.ebg_nodes")},
|
||||
{REQUIRED, config.GetPath(".osrm.tls")},
|
||||
{REQUIRED, config.GetPath(".osrm.tld")},
|
||||
{REQUIRED, config.GetPath(".osrm.maneuver_overrides")},
|
||||
{REQUIRED, config.GetPath(".osrm.turn_weight_penalties")},
|
||||
{REQUIRED, config.GetPath(".osrm.turn_duration_penalties")},
|
||||
{REQUIRED, config.GetPath(".osrm.edges")},
|
||||
{REQUIRED, config.GetPath(".osrm.names")},
|
||||
{REQUIRED, config.GetPath(".osrm.ramIndex")},
|
||||
@ -250,7 +284,7 @@ void Storage::PopulateLayout(DataLayout &layout)
|
||||
{
|
||||
if (boost::filesystem::exists(file.second))
|
||||
{
|
||||
readBlocks(file.second, layout);
|
||||
readBlocks(file.second, static_layout);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -263,26 +297,51 @@ void Storage::PopulateLayout(DataLayout &layout)
|
||||
}
|
||||
}
|
||||
|
||||
void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
|
||||
void Storage::PopulateUpdatableLayout(DataLayout &updatable_layout)
|
||||
{
|
||||
BOOST_ASSERT(memory_ptr != nullptr);
|
||||
constexpr bool REQUIRED = true;
|
||||
constexpr bool OPTIONAL = false;
|
||||
std::vector<std::pair<bool, boost::filesystem::path>> tar_files = {
|
||||
{OPTIONAL, config.GetPath(".osrm.mldgr")},
|
||||
{OPTIONAL, config.GetPath(".osrm.cell_metrics")},
|
||||
{OPTIONAL, config.GetPath(".osrm.hsgr")},
|
||||
{REQUIRED, config.GetPath(".osrm.datasource_names")},
|
||||
{REQUIRED, config.GetPath(".osrm.geometry")},
|
||||
{REQUIRED, config.GetPath(".osrm.turn_weight_penalties")},
|
||||
{REQUIRED, config.GetPath(".osrm.turn_duration_penalties")},
|
||||
};
|
||||
|
||||
// Connectivity matrix checksum
|
||||
std::uint32_t turns_connectivity_checksum = 0;
|
||||
for (const auto &file : tar_files)
|
||||
{
|
||||
if (boost::filesystem::exists(file.second))
|
||||
{
|
||||
readBlocks(file.second, updatable_layout);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (file.first == REQUIRED)
|
||||
{
|
||||
throw util::exception("Could not find required filed: " +
|
||||
std::get<1>(file).string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Storage::PopulateStaticData(const SharedDataIndex &index)
|
||||
{
|
||||
// read actual data into shared memory object //
|
||||
|
||||
// store the filename of the on-disk portion of the RTree
|
||||
{
|
||||
const auto file_index_path_ptr =
|
||||
layout.GetBlockPtr<char>(memory_ptr, "/common/rtree/file_index_path");
|
||||
const auto file_index_path_ptr = index.GetBlockPtr<char>("/common/rtree/file_index_path");
|
||||
// make sure we have 0 ending
|
||||
std::fill(file_index_path_ptr,
|
||||
file_index_path_ptr + layout.GetBlockSize("/common/rtree/file_index_path"),
|
||||
file_index_path_ptr + index.GetBlockSize("/common/rtree/file_index_path"),
|
||||
0);
|
||||
const auto absolute_file_index_path =
|
||||
boost::filesystem::absolute(config.GetPath(".osrm.fileIndex")).string();
|
||||
BOOST_ASSERT(static_cast<std::size_t>(layout.GetBlockSize(
|
||||
BOOST_ASSERT(static_cast<std::size_t>(index.GetBlockSize(
|
||||
"/common/rtree/file_index_path")) >= absolute_file_index_path.size());
|
||||
std::copy(
|
||||
absolute_file_index_path.begin(), absolute_file_index_path.end(), file_index_path_ptr);
|
||||
@ -290,81 +349,50 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
|
||||
|
||||
// Name data
|
||||
{
|
||||
auto name_table = make_name_table_view(memory_ptr, layout, "/common/names");
|
||||
auto name_table = make_name_table_view(index, "/common/names");
|
||||
extractor::files::readNames(config.GetPath(".osrm.names"), name_table);
|
||||
}
|
||||
|
||||
// Turn lane data
|
||||
{
|
||||
auto turn_lane_data = make_lane_data_view(memory_ptr, layout, "/common/turn_lanes");
|
||||
auto turn_lane_data = make_lane_data_view(index, "/common/turn_lanes");
|
||||
extractor::files::readTurnLaneData(config.GetPath(".osrm.tld"), turn_lane_data);
|
||||
}
|
||||
|
||||
// Turn lane descriptions
|
||||
{
|
||||
auto views = make_turn_lane_description_views(memory_ptr, layout, "/common/turn_lanes");
|
||||
auto views = make_turn_lane_description_views(index, "/common/turn_lanes");
|
||||
extractor::files::readTurnLaneDescriptions(
|
||||
config.GetPath(".osrm.tls"), std::get<0>(views), std::get<1>(views));
|
||||
}
|
||||
|
||||
// Load edge-based nodes data
|
||||
{
|
||||
auto node_data = make_ebn_data_view(memory_ptr, layout, "/common/ebg_node_data");
|
||||
auto node_data = make_ebn_data_view(index, "/common/ebg_node_data");
|
||||
extractor::files::readNodeData(config.GetPath(".osrm.ebg_nodes"), node_data);
|
||||
}
|
||||
|
||||
// Load original edge data
|
||||
{
|
||||
auto turn_data = make_turn_data_view(memory_ptr, layout, "/common/turn_data");
|
||||
auto turn_data = make_turn_data_view(index, "/common/turn_data");
|
||||
|
||||
auto connectivity_checksum_ptr =
|
||||
layout.GetBlockPtr<std::uint32_t>(memory_ptr, "/common/connectivity_checksum");
|
||||
index.GetBlockPtr<std::uint32_t>("/common/connectivity_checksum");
|
||||
|
||||
guidance::files::readTurnData(
|
||||
config.GetPath(".osrm.edges"), turn_data, *connectivity_checksum_ptr);
|
||||
|
||||
turns_connectivity_checksum = *connectivity_checksum_ptr;
|
||||
}
|
||||
|
||||
// load compressed geometry
|
||||
{
|
||||
auto segment_data = make_segment_data_view(memory_ptr, layout, "/common/segment_data");
|
||||
extractor::files::readSegmentData(config.GetPath(".osrm.geometry"), segment_data);
|
||||
}
|
||||
|
||||
{
|
||||
const auto datasources_names_ptr =
|
||||
layout.GetBlockPtr<extractor::Datasources>(memory_ptr, "/common/data_sources_names");
|
||||
extractor::files::readDatasources(config.GetPath(".osrm.datasource_names"),
|
||||
*datasources_names_ptr);
|
||||
}
|
||||
|
||||
// Loading list of coordinates
|
||||
{
|
||||
auto views = make_nbn_data_view(memory_ptr, layout, "/common/nbn_data");
|
||||
auto views = make_nbn_data_view(index, "/common/nbn_data");
|
||||
extractor::files::readNodes(
|
||||
config.GetPath(".osrm.nbg_nodes"), std::get<0>(views), std::get<1>(views));
|
||||
}
|
||||
|
||||
// load turn weight penalties
|
||||
{
|
||||
auto turn_duration_penalties =
|
||||
make_turn_weight_view(memory_ptr, layout, "/common/turn_penalty");
|
||||
extractor::files::readTurnWeightPenalty(config.GetPath(".osrm.turn_weight_penalties"),
|
||||
turn_duration_penalties);
|
||||
}
|
||||
|
||||
// load turn duration penalties
|
||||
{
|
||||
auto turn_duration_penalties =
|
||||
make_turn_duration_view(memory_ptr, layout, "/common/turn_penalty");
|
||||
extractor::files::readTurnDurationPenalty(config.GetPath(".osrm.turn_duration_penalties"),
|
||||
turn_duration_penalties);
|
||||
}
|
||||
|
||||
// store search tree portion of rtree
|
||||
{
|
||||
auto rtree = make_search_tree_view(memory_ptr, layout, "/common/rtree");
|
||||
auto rtree = make_search_tree_view(index, "/common/rtree");
|
||||
extractor::files::readRamIndex(config.GetPath(".osrm.ramIndex"), rtree);
|
||||
}
|
||||
|
||||
@ -372,8 +400,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
|
||||
std::string metric_name;
|
||||
// load profile properties
|
||||
{
|
||||
const auto profile_properties_ptr =
|
||||
layout.GetBlockPtr<extractor::ProfileProperties>(memory_ptr, "/common/properties");
|
||||
const auto profile_properties_ptr = index.GetBlockPtr<extractor::ProfileProperties>(
|
||||
"/common/properties");
|
||||
extractor::files::readProfileProperties(config.GetPath(".osrm.properties"),
|
||||
*profile_properties_ptr);
|
||||
|
||||
@ -383,49 +411,95 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
|
||||
// Load intersection data
|
||||
{
|
||||
auto intersection_bearings_view =
|
||||
make_intersection_bearings_view(memory_ptr, layout, "/common/intersection_bearings");
|
||||
auto entry_classes = make_entry_classes_view(memory_ptr, layout, "/common/entry_classes");
|
||||
make_intersection_bearings_view(index, "/common/intersection_bearings");
|
||||
auto entry_classes = make_entry_classes_view(index, "/common/entry_classes");
|
||||
extractor::files::readIntersections(
|
||||
config.GetPath(".osrm.icd"), intersection_bearings_view, entry_classes);
|
||||
}
|
||||
|
||||
if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
|
||||
{
|
||||
auto mlp = make_partition_view(index, "/mld/multilevelpartition");
|
||||
partitioner::files::readPartition(config.GetPath(".osrm.partition"), mlp);
|
||||
}
|
||||
|
||||
if (boost::filesystem::exists(config.GetPath(".osrm.cells")))
|
||||
{
|
||||
auto storage = make_cell_storage_view(index, "/mld/cellstorage");
|
||||
partitioner::files::readCells(config.GetPath(".osrm.cells"), storage);
|
||||
}
|
||||
|
||||
if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
|
||||
{
|
||||
auto exclude_metrics = make_cell_metric_view(index, "/mld/metrics/" + metric_name);
|
||||
std::unordered_map<std::string, std::vector<customizer::CellMetricView>> metrics = {
|
||||
{metric_name, std::move(exclude_metrics)},
|
||||
};
|
||||
customizer::files::readCellMetrics(config.GetPath(".osrm.cell_metrics"), metrics);
|
||||
}
|
||||
|
||||
// load maneuver overrides
|
||||
{
|
||||
auto views = make_maneuver_overrides_views(index, "/common/maneuver_overrides");
|
||||
extractor::files::readManeuverOverrides(
|
||||
config.GetPath(".osrm.maneuver_overrides"), std::get<0>(views), std::get<1>(views));
|
||||
}
|
||||
}
|
||||
|
||||
void Storage::PopulateUpdatableData(const SharedDataIndex &index)
|
||||
{
|
||||
// load compressed geometry
|
||||
{
|
||||
auto segment_data = make_segment_data_view(index, "/common/segment_data");
|
||||
extractor::files::readSegmentData(config.GetPath(".osrm.geometry"), segment_data);
|
||||
}
|
||||
|
||||
{
|
||||
const auto datasources_names_ptr = index.GetBlockPtr<extractor::Datasources>(
|
||||
"/common/data_sources_names");
|
||||
extractor::files::readDatasources(config.GetPath(".osrm.datasource_names"),
|
||||
*datasources_names_ptr);
|
||||
}
|
||||
|
||||
// load turn weight penalties
|
||||
{
|
||||
auto turn_duration_penalties = make_turn_weight_view(index, "/common/turn_penalty");
|
||||
extractor::files::readTurnWeightPenalty(config.GetPath(".osrm.turn_weight_penalties"),
|
||||
turn_duration_penalties);
|
||||
}
|
||||
|
||||
// load turn duration penalties
|
||||
{
|
||||
auto turn_duration_penalties = make_turn_duration_view(index, "/common/turn_penalty");
|
||||
extractor::files::readTurnDurationPenalty(config.GetPath(".osrm.turn_duration_penalties"),
|
||||
turn_duration_penalties);
|
||||
}
|
||||
|
||||
// FIXME we only need to get the weight name
|
||||
std::string metric_name;
|
||||
// load profile properties
|
||||
{
|
||||
extractor::ProfileProperties properties;
|
||||
extractor::files::readProfileProperties(config.GetPath(".osrm.properties"), properties);
|
||||
|
||||
metric_name = properties.GetWeightName();
|
||||
}
|
||||
|
||||
if (boost::filesystem::exists(config.GetPath(".osrm.hsgr")))
|
||||
{
|
||||
const std::string metric_prefix = "/ch/metrics/" + metric_name;
|
||||
auto contracted_metric = make_contracted_metric_view(memory_ptr, layout, metric_prefix);
|
||||
auto contracted_metric = make_contracted_metric_view(index, metric_prefix);
|
||||
std::unordered_map<std::string, contractor::ContractedMetricView> metrics = {
|
||||
{metric_name, std::move(contracted_metric)}};
|
||||
|
||||
std::uint32_t graph_connectivity_checksum = 0;
|
||||
contractor::files::readGraph(
|
||||
config.GetPath(".osrm.hsgr"), metrics, graph_connectivity_checksum);
|
||||
|
||||
if (turns_connectivity_checksum != graph_connectivity_checksum)
|
||||
{
|
||||
throw util::exception(
|
||||
"Connectivity checksum " + std::to_string(graph_connectivity_checksum) + " in " +
|
||||
config.GetPath(".osrm.hsgr").string() + " does not equal to checksum " +
|
||||
std::to_string(turns_connectivity_checksum) + " in " +
|
||||
config.GetPath(".osrm.edges").string());
|
||||
}
|
||||
}
|
||||
|
||||
if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
|
||||
{
|
||||
auto mlp = make_partition_view(memory_ptr, layout, "/mld/multilevelpartition");
|
||||
partitioner::files::readPartition(config.GetPath(".osrm.partition"), mlp);
|
||||
}
|
||||
|
||||
if (boost::filesystem::exists(config.GetPath(".osrm.cells")))
|
||||
{
|
||||
auto storage = make_cell_storage_view(memory_ptr, layout, "/mld/cellstorage");
|
||||
partitioner::files::readCells(config.GetPath(".osrm.cells"), storage);
|
||||
}
|
||||
|
||||
if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
|
||||
{
|
||||
auto exclude_metrics =
|
||||
make_cell_metric_view(memory_ptr, layout, "/mld/metrics/" + metric_name);
|
||||
auto exclude_metrics = make_cell_metric_view(index, "/mld/metrics/" + metric_name);
|
||||
std::unordered_map<std::string, std::vector<customizer::CellMetricView>> metrics = {
|
||||
{metric_name, std::move(exclude_metrics)},
|
||||
};
|
||||
@ -434,27 +508,10 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
|
||||
|
||||
if (boost::filesystem::exists(config.GetPath(".osrm.mldgr")))
|
||||
{
|
||||
auto graph_view = make_multi_level_graph_view(memory_ptr, layout, "/mld/multilevelgraph");
|
||||
auto graph_view = make_multi_level_graph_view(index, "/mld/multilevelgraph");
|
||||
std::uint32_t graph_connectivity_checksum = 0;
|
||||
partitioner::files::readGraph(
|
||||
config.GetPath(".osrm.mldgr"), graph_view, graph_connectivity_checksum);
|
||||
|
||||
if (turns_connectivity_checksum != graph_connectivity_checksum)
|
||||
{
|
||||
throw util::exception(
|
||||
"Connectivity checksum " + std::to_string(graph_connectivity_checksum) + " in " +
|
||||
config.GetPath(".osrm.mldgr").string() + " does not equal to checksum " +
|
||||
std::to_string(turns_connectivity_checksum) + " in " +
|
||||
config.GetPath(".osrm.edges").string());
|
||||
}
|
||||
}
|
||||
|
||||
// load maneuver overrides
|
||||
{
|
||||
auto views =
|
||||
make_maneuver_overrides_views(memory_ptr, layout, "/common/maneuver_overrides");
|
||||
extractor::files::readManeuverOverrides(
|
||||
config.GetPath(".osrm.maneuver_overrides"), std::get<0>(views), std::get<1>(views));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user