Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add support for disabling feature datasets (#6666)

Browse Source 
This change adds support for disabling datasets, such that specific
files are not loaded into memory when running OSRM. This enables users
to not pay the memory cost for features they do not intend to use.

Initially, there are two options:
- ROUTE_GEOMETRY, for disabling overview, steps, annotations and waypoints.
- ROUTE_STEPS, for disabling steps only.

Attempts to query features for which the datasets are disabled will
lead to a DisabledDatasetException being returned.
This commit is contained in:
Michael Bell
2023-08-04 18:43:37 +01:00
committed by GitHub
parent 522d0f066e
commit db7946d762
34 changed files with 1005 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
+103 -36
View File
@@ -31,6 +31,26 @@
namespace osrm::engine::datafacade
{
static const std::string DATASET_TURN_DATA = "TurnData";
static const std::string DATASET_TURN_LANE_DATA = "NameLaneData";
static const std::string DATASET_NAME_DATA = "NameData";
static const std::string DATASET_INTERSECTION_BEARINGS = "IntersectionBearings";
static const std::string DATASET_ENTRY_CLASS = "EntryClass";
/**
* Macro is not ideal. But without it we either have to:
* a) Write this boiler-plate for every usage of an optional dataset.
* b) Convert to a function and add lots of polluting NOLINT(bugprone-unchecked-optional-access)
* comments. This macro keeps the API code readable.
*/
#define CHECK_DATASET_DISABLED(val, dataset) \
{ \
if (!(val)) \
{ \
throw osrm::util::DisabledDatasetException((dataset)); \
} \
}
template <typename AlgorithmT> class ContiguousInternalMemoryAlgorithmDataFacade;
template <>
@@ -141,18 +161,15 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
std::string_view m_data_timestamp;
util::vector_view<util::Coordinate> m_coordinate_list;
extractor::PackedOSMIDsView m_osmnodeid_list;
util::vector_view<std::uint32_t> m_lane_description_offsets;
util::vector_view<extractor::TurnLaneType::Mask> m_lane_description_masks;
std::optional<util::vector_view<std::uint32_t>> m_lane_description_offsets;
std::optional<util::vector_view<extractor::TurnLaneType::Mask>> m_lane_description_masks;
util::vector_view<TurnPenalty> m_turn_weight_penalties;
util::vector_view<TurnPenalty> m_turn_duration_penalties;
extractor::SegmentDataView segment_data;
extractor::EdgeBasedNodeDataView edge_based_node_data;
guidance::TurnDataView turn_data;
std::optional<guidance::TurnDataView> turn_data;
util::vector_view<char> m_datasource_name_data;
util::vector_view<std::size_t> m_datasource_name_offsets;
util::vector_view<std::size_t> m_datasource_name_lengths;
util::vector_view<util::guidance::LaneTupleIdPair> m_lane_tupel_id_pairs;
std::optional<util::vector_view<util::guidance::LaneTupleIdPair>> m_lane_tuple_id_pairs;
util::vector_view<extractor::StorageManeuverOverride> m_maneuver_overrides;
util::vector_view<NodeID> m_maneuver_override_node_sequences;
@@ -161,16 +178,24 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
std::unique_ptr<SharedGeospatialQuery> m_geospatial_query;
boost::filesystem::path file_index_path;
extractor::IntersectionBearingsView intersection_bearings_view;
std::optional<extractor::IntersectionBearingsView> intersection_bearings_view;
extractor::NameTableView m_name_table;
std::optional<extractor::NameTableView> m_name_table;
// the look-up table for entry classes. An entry class lists the possibility of entry for all
// available turns. Such a class id is stored with every edge.
util::vector_view<util::guidance::EntryClass> m_entry_class_table;
std::optional<util::vector_view<util::guidance::EntryClass>> m_entry_class_table;
// allocator that keeps the allocation data
std::shared_ptr<ContiguousBlockAllocator> allocator;
bool isIndexed(const storage::SharedDataIndex &index, const std::string &name)
{
bool result = false;
index.List(name,
boost::make_function_output_iterator([&](const auto &) { result = true; }));
return result;
}
void InitializeInternalPointers(const storage::SharedDataIndex &index,
const std::string &metric_name,
const std::size_t exclude_index)
@@ -183,7 +208,17 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
exclude_mask = m_profile_properties->excludable_classes[exclude_index];
m_check_sum = *index.GetBlockPtr<std::uint32_t>("/common/connectivity_checksum");
// We no longer use "/common/connectivity_checksum", as osrm.edges is an optional dataset.
// Instead, we load the value from the MLD or CH graph, whichever is loaded.
if (isIndexed(index, "/mld/connectivity_checksum"))
{
m_check_sum = *index.GetBlockPtr<std::uint32_t>("/mld/connectivity_checksum");
}
else
{
BOOST_ASSERT(isIndexed(index, "/ch/connectivity_checksum"));
m_check_sum = *index.GetBlockPtr<std::uint32_t>("/ch/connectivity_checksum");
}
m_data_timestamp = make_timestamp_view(index, "/common/timestamp");
@@ -196,13 +231,23 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
edge_based_node_data = make_ebn_data_view(index, "/common/ebg_node_data");
turn_data = make_turn_data_view(index, "/common/turn_data");
if (isIndexed(index, "/common/turn_data"))
{
turn_data = make_turn_data_view(index, "/common/turn_data");
}
m_name_table = make_name_table_view(index, "/common/names");
if (isIndexed(index, "/common/names"))
{
m_name_table = make_name_table_view(index, "/common/names");
}
std::tie(m_lane_description_offsets, m_lane_description_masks) =
make_turn_lane_description_views(index, "/common/turn_lanes");
m_lane_tupel_id_pairs = make_lane_data_view(index, "/common/turn_lanes");
if (isIndexed(index, "/common/turn_lanes"))
{
std::tie(m_lane_description_offsets, m_lane_description_masks) =
make_turn_lane_description_views(index, "/common/turn_lanes");
m_lane_tuple_id_pairs = make_lane_data_view(index, "/common/turn_lanes");
}
m_turn_weight_penalties = make_turn_weight_view(index, "/common/turn_penalty");
m_turn_duration_penalties = make_turn_duration_view(index, "/common/turn_penalty");
@@ -211,10 +256,12 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
m_datasources = index.GetBlockPtr<extractor::Datasources>("/common/data_sources_names");
intersection_bearings_view =
make_intersection_bearings_view(index, "/common/intersection_bearings");
m_entry_class_table = make_entry_classes_view(index, "/common/entry_classes");
if (isIndexed(index, "/common/intersection_bearings"))
{
intersection_bearings_view =
make_intersection_bearings_view(index, "/common/intersection_bearings");
m_entry_class_table = make_entry_classes_view(index, "/common/entry_classes");
}
std::tie(m_maneuver_overrides, m_maneuver_override_node_sequences) =
make_maneuver_overrides_views(index, "/common/maneuver_overrides");
@@ -305,7 +352,8 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
osrm::guidance::TurnInstruction
GetTurnInstructionForEdgeID(const EdgeID edge_based_edge_id) const override final
{
return turn_data.GetTurnInstruction(edge_based_edge_id);
CHECK_DATASET_DISABLED(turn_data, DATASET_TURN_DATA);
return turn_data->GetTurnInstruction(edge_based_edge_id);
}
std::vector<RTreeLeaf> GetEdgesInBox(const util::Coordinate south_west,
@@ -406,27 +454,32 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
std::string_view GetNameForID(const NameID id) const override final
{
return m_name_table.GetNameForID(id);
CHECK_DATASET_DISABLED(m_name_table, DATASET_NAME_DATA);
return m_name_table->GetNameForID(id);
}
std::string_view GetRefForID(const NameID id) const override final
{
return m_name_table.GetRefForID(id);
CHECK_DATASET_DISABLED(m_name_table, DATASET_NAME_DATA);
return m_name_table->GetRefForID(id);
}
std::string_view GetPronunciationForID(const NameID id) const override final
{
return m_name_table.GetPronunciationForID(id);
CHECK_DATASET_DISABLED(m_name_table, DATASET_NAME_DATA);
return m_name_table->GetPronunciationForID(id);
}
std::string_view GetDestinationsForID(const NameID id) const override final
{
return m_name_table.GetDestinationsForID(id);
CHECK_DATASET_DISABLED(m_name_table, DATASET_NAME_DATA);
return m_name_table->GetDestinationsForID(id);
}
std::string_view GetExitsForID(const NameID id) const override final
{
return m_name_table.GetExitsForID(id);
CHECK_DATASET_DISABLED(m_name_table, DATASET_NAME_DATA);
return m_name_table->GetExitsForID(id);
}
std::string_view GetDatasourceName(const DatasourceID id) const override final
@@ -459,46 +512,60 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
util::guidance::BearingClass
GetBearingClass(const NodeID node_based_node_id) const override final
{
return intersection_bearings_view.GetBearingClass(node_based_node_id);
CHECK_DATASET_DISABLED(intersection_bearings_view, DATASET_INTERSECTION_BEARINGS);
return intersection_bearings_view->GetBearingClass(node_based_node_id);
}
guidance::TurnBearing PreTurnBearing(const EdgeID edge_based_edge_id) const override final
{
return turn_data.GetPreTurnBearing(edge_based_edge_id);
CHECK_DATASET_DISABLED(turn_data, DATASET_TURN_DATA);
return turn_data->GetPreTurnBearing(edge_based_edge_id);
}
guidance::TurnBearing PostTurnBearing(const EdgeID edge_based_edge_id) const override final
{
return turn_data.GetPostTurnBearing(edge_based_edge_id);
CHECK_DATASET_DISABLED(turn_data, DATASET_TURN_DATA);
return turn_data->GetPostTurnBearing(edge_based_edge_id);
}
util::guidance::EntryClass GetEntryClass(const EdgeID edge_based_edge_id) const override final
{
auto entry_class_id = turn_data.GetEntryClassID(edge_based_edge_id);
return m_entry_class_table.at(entry_class_id);
CHECK_DATASET_DISABLED(m_entry_class_table, DATASET_ENTRY_CLASS);
CHECK_DATASET_DISABLED(turn_data, DATASET_TURN_DATA);
auto entry_class_id = turn_data->GetEntryClassID(edge_based_edge_id);
return m_entry_class_table->at(entry_class_id);
}
bool HasLaneData(const EdgeID edge_based_edge_id) const override final
{
return turn_data.HasLaneData(edge_based_edge_id);
CHECK_DATASET_DISABLED(turn_data, DATASET_TURN_DATA);
return turn_data->HasLaneData(edge_based_edge_id);
}
util::guidance::LaneTupleIdPair
GetLaneData(const EdgeID edge_based_edge_id) const override final
{
CHECK_DATASET_DISABLED(turn_data, DATASET_TURN_DATA);
CHECK_DATASET_DISABLED(m_lane_tuple_id_pairs, DATASET_TURN_LANE_DATA);
BOOST_ASSERT(HasLaneData(edge_based_edge_id));
return m_lane_tupel_id_pairs.at(turn_data.GetLaneDataID(edge_based_edge_id));
return m_lane_tuple_id_pairs->at(turn_data->GetLaneDataID(edge_based_edge_id));
}
extractor::TurnLaneDescription
GetTurnDescription(const LaneDescriptionID lane_description_id) const override final
{
CHECK_DATASET_DISABLED(m_lane_description_offsets, DATASET_TURN_LANE_DATA);
CHECK_DATASET_DISABLED(m_lane_description_masks, DATASET_TURN_LANE_DATA);
if (lane_description_id == INVALID_LANE_DESCRIPTIONID)
return {};
else
return extractor::TurnLaneDescription(
m_lane_description_masks.begin() + m_lane_description_offsets[lane_description_id],
m_lane_description_masks.begin() +
m_lane_description_offsets[lane_description_id + 1]);
m_lane_description_masks->begin() +
m_lane_description_offsets->at(lane_description_id),
m_lane_description_masks->begin() +
m_lane_description_offsets->at(lane_description_id + 1));
}
bool IsLeftHandDriving(const NodeID edge_based_node_id) const override final
include/engine/engine_config.hpp
+3
View File
@@ -29,9 +29,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define ENGINE_CONFIG_HPP
#include "storage/storage_config.hpp"
#include "osrm/datasets.hpp"
#include <boost/filesystem/path.hpp>
#include <set>
#include <string>
namespace osrm::engine
@@ -89,6 +91,7 @@ struct EngineConfig final
boost::filesystem::path memory_file;
bool use_mmap = true;
Algorithm algorithm = Algorithm::CH;
std::vector<storage::FeatureDataset> disable_feature_dataset;
std::string verbosity;
std::string dataset_name;
};
include/nodejs/node_osrm_support.hpp
+42 -1
View File
@@ -205,9 +205,50 @@ inline engine_config_ptr argumentsToEngineConfig(const Napi::CallbackInfo &args)
}
}
auto disable_feature_dataset = params.Get("disable_feature_dataset");
if (disable_feature_dataset.IsArray())
{
Napi::Array datasets = disable_feature_dataset.As<Napi::Array>();
for (uint32_t i = 0; i < datasets.Length(); ++i)
{
Napi::Value dataset = datasets.Get(i);
if (!dataset.IsString())
{
ThrowError(args.Env(), "disable_feature_dataset list option must be a string");
return engine_config_ptr();
}
auto dataset_str = dataset.ToString().Utf8Value();
if (dataset_str == "ROUTE_GEOMETRY")
{
engine_config->disable_feature_dataset.push_back(
osrm::storage::FeatureDataset::ROUTE_GEOMETRY);
}
else if (dataset_str == "ROUTE_STEPS")
{
engine_config->disable_feature_dataset.push_back(
osrm::storage::FeatureDataset::ROUTE_STEPS);
}
else
{
ThrowError(
args.Env(),
"disable_feature_dataset array can include 'ROUTE_GEOMETRY', 'ROUTE_STEPS'.");
return engine_config_ptr();
}
}
}
else if (!disable_feature_dataset.IsUndefined())
{
ThrowError(args.Env(),
"disable_feature_dataset option must be an array and can include the string "
"values 'ROUTE_GEOMETRY', 'ROUTE_STEPS'.");
return engine_config_ptr();
}
if (!path.IsUndefined())
{
engine_config->storage_config = osrm::StorageConfig(path.ToString().Utf8Value());
engine_config->storage_config = osrm::StorageConfig(path.ToString().Utf8Value(),
engine_config->disable_feature_dataset);
engine_config->use_shared_memory = false;
}
include/osrm/datasets.hpp
+15
View File
@@ -0,0 +1,15 @@
#ifndef DATASETS_HPP
#define DATASETS_HPP
namespace osrm::storage
{
enum class FeatureDataset
{
ROUTE_STEPS,
ROUTE_GEOMETRY,
};
} // namespace osrm::storage
#endif
include/osrm/error_codes.hpp
+2 -1
View File
@@ -23,7 +23,8 @@ enum ErrorCode
FileIOError,
UnexpectedEndOfFile,
IncompatibleDataset,
UnknownAlgorithm
UnknownAlgorithm,
UnknownFeatureDataset
#ifndef NDEBUG
// Leave this at the end. In debug mode, we assert that the size of
// this enum matches the number of messages we have documented, and __ENDMARKER__
include/storage/io_config.hpp
+5
View File
@@ -35,6 +35,11 @@ struct IOConfig
return {base_path.string() + fileName};
}
bool IsRequiredConfiguredInput(const std::string &fileName) const
{
return IsConfigured(fileName, required_input_files);
}
boost::filesystem::path base_path;
protected:
include/storage/storage_config.hpp
+57 -18
View File
@@ -31,10 +31,61 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <boost/filesystem/path.hpp>
#include "storage/io_config.hpp"
#include "osrm/datasets.hpp"
#include <set>
namespace osrm::storage
{
std::istream &operator>>(std::istream &in, FeatureDataset &datasets);
static std::vector<boost::filesystem::path>
GetRequiredFiles(const std::vector<storage::FeatureDataset> &disabled_feature_dataset)
{
std::set<boost::filesystem::path> required{
".osrm.datasource_names",
".osrm.ebg_nodes",
".osrm.edges",
".osrm.fileIndex",
".osrm.geometry",
".osrm.icd",
".osrm.maneuver_overrides",
".osrm.names",
".osrm.nbg_nodes",
".osrm.properties",
".osrm.ramIndex",
".osrm.timestamp",
".osrm.tld",
".osrm.tls",
".osrm.turn_duration_penalties",
".osrm.turn_weight_penalties",
};
for (const auto &to_disable : disabled_feature_dataset)
{
switch (to_disable)
{
case FeatureDataset::ROUTE_STEPS:
for (const auto &dataset : {".osrm.icd", ".osrm.tld", ".osrm.tls"})
{
required.erase(dataset);
}
break;
case FeatureDataset::ROUTE_GEOMETRY:
for (const auto &dataset :
{".osrm.edges", ".osrm.icd", ".osrm.names", ".osrm.tld", ".osrm.tls"})
{
required.erase(dataset);
}
break;
}
}
return std::vector<boost::filesystem::path>(required.begin(), required.end());
;
}
/**
* Configures OSRM's file storage paths.
*
@@ -42,29 +93,17 @@ namespace osrm::storage
*/
struct StorageConfig final : IOConfig
{
StorageConfig(const boost::filesystem::path &base) : StorageConfig()
StorageConfig(const boost::filesystem::path &base,
const std::vector<storage::FeatureDataset> &disabled_feature_datasets_ = {})
: StorageConfig(disabled_feature_datasets_)
{
IOConfig::UseDefaultOutputNames(base);
}
StorageConfig()
StorageConfig(const std::vector<storage::FeatureDataset> &disabled_feature_datasets_ = {})
: IOConfig(
{".osrm.ramIndex",
".osrm.fileIndex",
".osrm.edges",
".osrm.geometry",
".osrm.turn_weight_penalties",
".osrm.turn_duration_penalties",
".osrm.datasource_names",
".osrm.ebg_nodes",
".osrm.names",
".osrm.nbg_nodes",
".osrm.timestamp",
".osrm.tls",
".osrm.tld",
".osrm.properties",
".osrm.icd",
".osrm.maneuver_overrides"},
GetRequiredFiles(disabled_feature_datasets_),
{".osrm.hsgr", ".osrm.cells", ".osrm.cell_metrics", ".osrm.mldgr", ".osrm.partition"},
{})
{
include/util/exception.hpp
+29 -2
View File
@@ -62,7 +62,7 @@ class exception : public std::exception
* user supplied bad data, etc).
*/
constexpr const std::array<const char *, 11> ErrorDescriptions = {{
constexpr const std::array<const char *, 12> ErrorDescriptions = {{
"", // Dummy - ErrorCode values start at 2
"", // Dummy - ErrorCode values start at 2
"Fingerprint did not match the expected value", // InvalidFingerprint
@@ -75,7 +75,8 @@ constexpr const std::array<const char *, 11> ErrorDescriptions = {{
// NOLINTNEXTLINE(bugprone-suspicious-missing-comma)
"The dataset you are trying to load is not " // IncompatibleDataset
"compatible with the routing algorithm you want to use.", // ...continued...
"Incompatible algorithm" // IncompatibleAlgorithm
"Incompatible algorithm", // IncompatibleAlgorithm
"Unknown feature dataset" // UnknownFeatureDataset
}};
#ifndef NDEBUG
@@ -84,6 +85,32 @@ static_assert(ErrorDescriptions.size() == ErrorCode::__ENDMARKER__,
"ErrorCode list and ErrorDescription lists are different sizes");
#endif
class DisabledDatasetException : public exception
{
public:
explicit DisabledDatasetException(const std::string &dataset_)
: exception(BuildMessage(dataset_)), dataset(dataset_)
{
}
const std::string &Dataset() const { return dataset; }
private:
// This function exists to 'anchor' the class, and stop the compiler from
// copying vtable and RTTI info into every object file that includes
// this header. (Caught by -Wweak-vtables under Clang.)
virtual void anchor() const override;
const std::string dataset;
static std::string BuildMessage(const std::string &dataset)
{
return "DisabledDatasetException: Your query tried to access the disabled dataset " +
dataset +
". Please check your configuration: "
"https://github.com/Project-OSRM/osrm-backend/wiki/Disabled-Datasets";
}
};
class RuntimeError : public exception
{
using Base = exception;