Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Implements Zero-Copy String Views onto Contiguous Memory, resolves #3265.

Browse Source 
- http://www.boost.org/doc/libs/1_61_0/libs/utility/doc/html/string_ref.html
- http://en.cppreference.com/w/cpp/string/basic_string_view
This commit is contained in:
Daniel J. Hofmann
2016-12-15 18:27:09 +01:00
parent b1f1c26703
commit c277b95f03
13 changed files with 219 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
+45 -42
View File
@@ -76,7 +76,7 @@ class ContiguousInternalMemoryDataFacade : public BaseDataFacade
util::ShM<util::Coordinate, true>::vector m_coordinate_list;
util::PackedVector<OSMNodeID, true> m_osmnodeid_list;
util::ShM<GeometryID, true>::vector m_via_geometry_list;
util::ShM<unsigned, true>::vector m_name_ID_list;
util::ShM<NameID, true>::vector m_name_ID_list;
util::ShM<LaneDataID, true>::vector m_lane_data_id;
util::ShM<extractor::guidance::TurnInstruction, true>::vector m_turn_instruction_list;
util::ShM<extractor::TravelMode, true>::vector m_travel_mode_list;
@@ -89,7 +89,7 @@ class ContiguousInternalMemoryDataFacade : public BaseDataFacade
util::ShM<EdgeWeight, true>::vector m_geometry_fwd_weight_list;
util::ShM<EdgeWeight, true>::vector m_geometry_rev_weight_list;
util::ShM<bool, true>::vector m_is_core_node;
util::ShM<uint8_t, true>::vector m_datasource_list;
util::ShM<DatasourceID, true>::vector m_datasource_list;
util::ShM<std::uint32_t, true>::vector m_lane_description_offsets;
util::ShM<extractor::guidance::TurnLaneType::Mask, true>::vector m_lane_description_masks;
@@ -231,8 +231,8 @@ class ContiguousInternalMemoryDataFacade : public BaseDataFacade
m_turn_instruction_list = std::move(turn_instruction_list);
const auto name_id_list_ptr =
data_layout.GetBlockPtr<unsigned>(memory_block, storage::DataLayout::NAME_ID_LIST);
util::ShM<unsigned, true>::vector name_id_list(
data_layout.GetBlockPtr<NameID>(memory_block, storage::DataLayout::NAME_ID_LIST);
util::ShM<NameID, true>::vector name_id_list(
name_id_list_ptr, data_layout.num_entries[storage::DataLayout::NAME_ID_LIST]);
m_name_ID_list = std::move(name_id_list);
@@ -340,9 +340,9 @@ class ContiguousInternalMemoryDataFacade : public BaseDataFacade
data_layout.num_entries[storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST]);
m_geometry_rev_weight_list = std::move(geometry_rev_weight_list);
auto datasources_list_ptr =
data_layout.GetBlockPtr<uint8_t>(memory_block, storage::DataLayout::DATASOURCES_LIST);
util::ShM<uint8_t, true>::vector datasources_list(
auto datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>(
memory_block, storage::DataLayout::DATASOURCES_LIST);
util::ShM<DatasourceID, true>::vector datasources_list(
datasources_list_ptr, data_layout.num_entries[storage::DataLayout::DATASOURCES_LIST]);
m_datasource_list = std::move(datasources_list);
@@ -482,7 +482,7 @@ class ContiguousInternalMemoryDataFacade : public BaseDataFacade
return m_coordinate_list[id];
}
OSMNodeID GetOSMNodeIDOfNode(const unsigned id) const override final
OSMNodeID GetOSMNodeIDOfNode(const NodeID id) const override final
{
return m_osmnodeid_list.at(id);
}
@@ -583,18 +583,18 @@ class ContiguousInternalMemoryDataFacade : public BaseDataFacade
return result_weights;
}
virtual GeometryID GetGeometryIndexForEdgeID(const unsigned id) const override final
virtual GeometryID GetGeometryIndexForEdgeID(const EdgeID id) const override final
{
return m_via_geometry_list.at(id);
}
extractor::guidance::TurnInstruction
GetTurnInstructionForEdgeID(const unsigned id) const override final
GetTurnInstructionForEdgeID(const EdgeID id) const override final
{
return m_turn_instruction_list.at(id);
}
extractor::TravelMode GetTravelModeForEdgeID(const unsigned id) const override final
extractor::TravelMode GetTravelModeForEdgeID(const EdgeID id) const override final
{
return m_travel_mode_list.at(id);
}
@@ -716,56 +716,59 @@ class ContiguousInternalMemoryDataFacade : public BaseDataFacade
unsigned GetCheckSum() const override final { return m_check_sum; }
unsigned GetNameIndexFromEdgeID(const unsigned id) const override final
NameID GetNameIndexFromEdgeID(const EdgeID id) const override final
{
return m_name_ID_list.at(id);
}
std::string GetNameForID(const unsigned name_id) const override final
StringView GetNameForID(const NameID id) const override final
{
if (std::numeric_limits<unsigned>::max() == name_id)
if (std::numeric_limits<NameID>::max() == id)
{
return "";
}
auto range = m_name_table->GetRange(name_id);
std::string result;
result.reserve(range.size());
if (range.begin() != range.end())
auto range = m_name_table->GetRange(id);
if (range.begin() == range.end())
{
result.resize(range.back() - range.front() + 1);
std::copy(m_names_char_list.begin() + range.front(),
m_names_char_list.begin() + range.back() + 1,
result.begin());
return "";
}
return result;
auto first = m_names_char_list.begin() + range.front();
auto last = m_names_char_list.begin() + range.back() + 1u;
// These iterators are useless: they're InputIterators onto a contiguous block of memory.
// Deref to get to the first element, then Addressof to get the memory address of the it.
const std::size_t len = &*last - &*first;
return StringView{&*first, len};
}
std::string GetRefForID(const unsigned name_id) const override final
StringView GetRefForID(const NameID id) const override final
{
// We store the ref after the name, destination and pronunciation of a street.
// We do this to get around the street length limit of 255 which would hit
// if we concatenate these. Order (see extractor_callbacks):
// name (0), destination (1), pronunciation (2), ref (3)
return GetNameForID(name_id + 3);
return GetNameForID(id + 3);
}
std::string GetPronunciationForID(const unsigned name_id) const override final
StringView GetPronunciationForID(const NameID id) const override final
{
// We store the pronunciation after the name and destination of a street.
// We do this to get around the street length limit of 255 which would hit
// if we concatenate these. Order (see extractor_callbacks):
// name (0), destination (1), pronunciation (2), ref (3)
return GetNameForID(name_id + 2);
return GetNameForID(id + 2);
}
std::string GetDestinationsForID(const unsigned name_id) const override final
StringView GetDestinationsForID(const NameID id) const override final
{
// We store the destination after the name of a street.
// We do this to get around the street length limit of 255 which would hit
// if we concatenate these. Order (see extractor_callbacks):
// name (0), destination (1), pronunciation (2), ref (3)
return GetNameForID(name_id + 1);
return GetNameForID(id + 1);
}
bool IsCoreNode(const NodeID id) const override final
@@ -782,7 +785,7 @@ class ContiguousInternalMemoryDataFacade : public BaseDataFacade
// Returns the data source ids that were used to supply the edge
// weights.
virtual std::vector<uint8_t>
virtual std::vector<DatasourceID>
GetUncompressedForwardDatasources(const EdgeID id) const override final
{
/*
@@ -797,7 +800,7 @@ class ContiguousInternalMemoryDataFacade : public BaseDataFacade
const unsigned begin = m_geometry_indices.at(id) + 1;
const unsigned end = m_geometry_indices.at(id + 1);
std::vector<uint8_t> result_datasources;
std::vector<DatasourceID> result_datasources;
result_datasources.resize(end - begin);
// If there was no datasource info, return an array of 0's.
@@ -820,7 +823,7 @@ class ContiguousInternalMemoryDataFacade : public BaseDataFacade
// Returns the data source ids that were used to supply the edge
// weights.
virtual std::vector<uint8_t>
virtual std::vector<DatasourceID>
GetUncompressedReverseDatasources(const EdgeID id) const override final
{
/*
@@ -835,7 +838,7 @@ class ContiguousInternalMemoryDataFacade : public BaseDataFacade
const unsigned begin = m_geometry_indices.at(id);
const unsigned end = m_geometry_indices.at(id + 1) - 1;
std::vector<uint8_t> result_datasources;
std::vector<DatasourceID> result_datasources;
result_datasources.resize(end - begin);
// If there was no datasource info, return an array of 0's.
@@ -856,19 +859,19 @@ class ContiguousInternalMemoryDataFacade : public BaseDataFacade
return result_datasources;
}
virtual std::string GetDatasourceName(const uint8_t datasource_name_id) const override final
StringView GetDatasourceName(const DatasourceID id) const override final
{
BOOST_ASSERT(m_datasource_name_offsets.size() >= 1);
BOOST_ASSERT(m_datasource_name_offsets.size() > datasource_name_id);
BOOST_ASSERT(m_datasource_name_offsets.size() > id);
std::string result;
result.reserve(m_datasource_name_lengths[datasource_name_id]);
std::copy(m_datasource_name_data.begin() + m_datasource_name_offsets[datasource_name_id],
m_datasource_name_data.begin() + m_datasource_name_offsets[datasource_name_id] +
m_datasource_name_lengths[datasource_name_id],
std::back_inserter(result));
auto first = m_datasource_name_data.begin() + m_datasource_name_offsets[id];
auto last = m_datasource_name_data.begin() + m_datasource_name_offsets[id] +
m_datasource_name_lengths[id];
// These iterators are useless: they're InputIterators onto a contiguous block of memory.
// Deref to get to the first element, then Addressof to get the memory address of the it.
const std::size_t len = &*last - &*first;
return result;
return StringView{&*first, len};
}
std::string GetTimestamp() const override final { return m_timestamp; }
include/engine/datafacade/datafacade_base.hpp
+15 -13
View File
@@ -17,6 +17,7 @@
#include "util/guidance/turn_lanes.hpp"
#include "util/integer_range.hpp"
#include "util/string_util.hpp"
#include "util/string_view.hpp"
#include "util/typedefs.hpp"
#include "osrm/coordinate.hpp"
@@ -34,6 +35,7 @@ namespace engine
namespace datafacade
{
using StringView = util::StringView;
using EdgeRange = util::range<EdgeID>;
class BaseDataFacade
@@ -74,10 +76,10 @@ class BaseDataFacade
const std::function<bool(EdgeData)> filter) const = 0;
// node and edge information access
virtual util::Coordinate GetCoordinateOfNode(const unsigned id) const = 0;
virtual OSMNodeID GetOSMNodeIDOfNode(const unsigned id) const = 0;
virtual util::Coordinate GetCoordinateOfNode(const NodeID id) const = 0;
virtual OSMNodeID GetOSMNodeIDOfNode(const NodeID id) const = 0;
virtual GeometryID GetGeometryIndexForEdgeID(const unsigned id) const = 0;
virtual GeometryID GetGeometryIndexForEdgeID(const EdgeID id) const = 0;
virtual std::vector<NodeID> GetUncompressedForwardGeometry(const EdgeID id) const = 0;
@@ -91,16 +93,16 @@ class BaseDataFacade
// Returns the data source ids that were used to supply the edge
// weights. Will return an empty array when only the base profile is used.
virtual std::vector<uint8_t> GetUncompressedForwardDatasources(const EdgeID id) const = 0;
virtual std::vector<uint8_t> GetUncompressedReverseDatasources(const EdgeID id) const = 0;
virtual std::vector<DatasourceID> GetUncompressedForwardDatasources(const EdgeID id) const = 0;
virtual std::vector<DatasourceID> GetUncompressedReverseDatasources(const EdgeID id) const = 0;
// Gets the name of a datasource
virtual std::string GetDatasourceName(const uint8_t datasource_name_id) const = 0;
virtual StringView GetDatasourceName(const DatasourceID id) const = 0;
virtual extractor::guidance::TurnInstruction
GetTurnInstructionForEdgeID(const unsigned id) const = 0;
GetTurnInstructionForEdgeID(const EdgeID id) const = 0;
virtual extractor::TravelMode GetTravelModeForEdgeID(const unsigned id) const = 0;
virtual extractor::TravelMode GetTravelModeForEdgeID(const EdgeID id) const = 0;
virtual std::vector<RTreeLeaf> GetEdgesInBox(const util::Coordinate south_west,
const util::Coordinate north_east) const = 0;
@@ -157,15 +159,15 @@ class BaseDataFacade
virtual bool IsCoreNode(const NodeID id) const = 0;
virtual unsigned GetNameIndexFromEdgeID(const unsigned id) const = 0;
virtual NameID GetNameIndexFromEdgeID(const EdgeID id) const = 0;
virtual std::string GetNameForID(const unsigned name_id) const = 0;
virtual StringView GetNameForID(const NameID id) const = 0;
virtual std::string GetRefForID(const unsigned name_id) const = 0;
virtual StringView GetRefForID(const NameID id) const = 0;
virtual std::string GetPronunciationForID(const unsigned name_id) const = 0;
virtual StringView GetPronunciationForID(const NameID id) const = 0;
virtual std::string GetDestinationsForID(const unsigned name_id) const = 0;
virtual StringView GetDestinationsForID(const NameID id) const = 0;
virtual std::size_t GetCoreSize() const = 0;