Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Read segment data directly into shm

Browse Source
This commit is contained in:
Patrick Niklaus 2017-04-02 01:49:46 +00:00 committed by Patrick Niklaus
parent 97d1de1beb
commit 3f5fc1e897
2 changed files with 47 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
include/extractor/files.hpp
View File

@ -50,7 +50,9 @@ inline void writeDatasources(const boost::filesystem::path &path, Datasources &s
} }
// reads .osrm.geometry // reads .osrm.geometry
inline void readSegmentData(const boost::filesystem::path &path, SegmentDataContainer &segment_data) template <bool UseShareMemory>
inline void readSegmentData(const boost::filesystem::path &path,
detail::SegmentDataContainerImpl<UseShareMemory> &segment_data)
{ {
const auto fingerprint = storage::io::FileReader::HasNoFingerprint; const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
storage::io::FileReader reader{path, fingerprint}; storage::io::FileReader reader{path, fingerprint};
@ -59,8 +61,9 @@ inline void readSegmentData(const boost::filesystem::path &path, SegmentDataCont
} }
// writes .osrm.geometry // writes .osrm.geometry
template <bool UseShareMemory>
inline void writeSegmentData(const boost::filesystem::path &path, inline void writeSegmentData(const boost::filesystem::path &path,
const SegmentDataContainer &segment_data) const detail::SegmentDataContainerImpl<UseShareMemory> &segment_data)
{ {
const auto fingerprint = storage::io::FileWriter::HasNoFingerprint; const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
storage::io::FileWriter writer{path, fingerprint}; storage::io::FileWriter writer{path, fingerprint};

79
src/storage/storage.cpp
View File

@ -650,50 +650,55 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// load compressed geometry // load compressed geometry
{ {
io::FileReader geometry_input_file(config.geometries_path, auto geometries_index_ptr =
io::FileReader::HasNoFingerprint); layout.GetBlockPtr<unsigned, true>(memory_ptr, storage::DataLayout::GEOMETRIES_INDEX);
util::vector_view<unsigned> geometry_begin_indices(
geometries_index_ptr, layout.num_entries[storage::DataLayout::GEOMETRIES_INDEX]);
const auto geometry_index_count = geometry_input_file.ReadElementCount32(); auto geometries_node_list_ptr = layout.GetBlockPtr<NodeID, true>(
const auto geometries_index_ptr = memory_ptr, storage::DataLayout::GEOMETRIES_NODE_LIST);
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::GEOMETRIES_INDEX); util::vector_view<NodeID> geometry_node_list(
BOOST_ASSERT(geometry_index_count == layout.num_entries[DataLayout::GEOMETRIES_INDEX]); geometries_node_list_ptr,
geometry_input_file.ReadInto(geometries_index_ptr, geometry_index_count); layout.num_entries[storage::DataLayout::GEOMETRIES_NODE_LIST]);
const auto geometries_node_id_list_ptr = auto geometries_fwd_weight_list_ptr = layout.GetBlockPtr<EdgeWeight, true>(
layout.GetBlockPtr<NodeID, true>(memory_ptr, DataLayout::GEOMETRIES_NODE_LIST); memory_ptr, storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST);
const auto geometry_node_lists_count = geometry_input_file.ReadElementCount32(); util::vector_view<EdgeWeight> geometry_fwd_weight_list(
BOOST_ASSERT(geometry_node_lists_count == geometries_fwd_weight_list_ptr,
layout.num_entries[DataLayout::GEOMETRIES_NODE_LIST]); layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST]);
geometry_input_file.ReadInto(geometries_node_id_list_ptr, geometry_node_lists_count);
const auto geometries_fwd_weight_list_ptr = layout.GetBlockPtr<EdgeWeight, true>( auto geometries_rev_weight_list_ptr = layout.GetBlockPtr<EdgeWeight, true>(
memory_ptr, DataLayout::GEOMETRIES_FWD_WEIGHT_LIST); memory_ptr, storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST);
BOOST_ASSERT(geometry_node_lists_count == util::vector_view<EdgeWeight> geometry_rev_weight_list(
layout.num_entries[DataLayout::GEOMETRIES_FWD_WEIGHT_LIST]); geometries_rev_weight_list_ptr,
geometry_input_file.ReadInto(geometries_fwd_weight_list_ptr, geometry_node_lists_count); layout.num_entries[storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST]);
const auto geometries_rev_weight_list_ptr = layout.GetBlockPtr<EdgeWeight, true>( auto geometries_fwd_duration_list_ptr = layout.GetBlockPtr<EdgeWeight, true>(
memory_ptr, DataLayout::GEOMETRIES_REV_WEIGHT_LIST); memory_ptr, storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST);
BOOST_ASSERT(geometry_node_lists_count == util::vector_view<EdgeWeight> geometry_fwd_duration_list(
layout.num_entries[DataLayout::GEOMETRIES_REV_WEIGHT_LIST]); geometries_fwd_duration_list_ptr,
geometry_input_file.ReadInto(geometries_rev_weight_list_ptr, geometry_node_lists_count); layout.num_entries[storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST]);
const auto geometries_fwd_duration_list_ptr = layout.GetBlockPtr<EdgeWeight, true>( auto geometries_rev_duration_list_ptr = layout.GetBlockPtr<EdgeWeight, true>(
memory_ptr, DataLayout::GEOMETRIES_FWD_DURATION_LIST); memory_ptr, storage::DataLayout::GEOMETRIES_REV_DURATION_LIST);
BOOST_ASSERT(geometry_node_lists_count == util::vector_view<EdgeWeight> geometry_rev_duration_list(
layout.num_entries[DataLayout::GEOMETRIES_FWD_DURATION_LIST]); geometries_rev_duration_list_ptr,
geometry_input_file.ReadInto(geometries_fwd_duration_list_ptr, geometry_node_lists_count); layout.num_entries[storage::DataLayout::GEOMETRIES_REV_DURATION_LIST]);
const auto geometries_rev_duration_list_ptr = layout.GetBlockPtr<EdgeWeight, true>( auto datasources_list_ptr = layout.GetBlockPtr<DatasourceID, true>(
memory_ptr, DataLayout::GEOMETRIES_REV_DURATION_LIST); memory_ptr, storage::DataLayout::DATASOURCES_LIST);
BOOST_ASSERT(geometry_node_lists_count == util::vector_view<DatasourceID> datasources_list(
layout.num_entries[DataLayout::GEOMETRIES_REV_DURATION_LIST]); datasources_list_ptr, layout.num_entries[storage::DataLayout::DATASOURCES_LIST]);
geometry_input_file.ReadInto(geometries_rev_duration_list_ptr, geometry_node_lists_count);
const auto datasource_list_ptr = extractor::SegmentDataView segment_data{std::move(geometry_begin_indices),
layout.GetBlockPtr<DatasourceID, true>(memory_ptr, DataLayout::DATASOURCES_LIST); std::move(geometry_node_list),
BOOST_ASSERT(geometry_node_lists_count == layout.num_entries[DataLayout::DATASOURCES_LIST]); std::move(geometry_fwd_weight_list),
geometry_input_file.ReadInto(datasource_list_ptr, geometry_node_lists_count); std::move(geometry_rev_weight_list),
std::move(geometry_fwd_duration_list),
std::move(geometry_rev_duration_list),
std::move(datasources_list)};
extractor::files::readSegmentData(config.geometries_path, segment_data);
} }
{ {