Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add support for disabling feature datasets (#6666)

Browse Source 
This change adds support for disabling datasets, such that specific
files are not loaded into memory when running OSRM. This enables users
to not pay the memory cost for features they do not intend to use.

Initially, there are two options:
- ROUTE_GEOMETRY, for disabling overview, steps, annotations and waypoints.
- ROUTE_STEPS, for disabling steps only.

Attempts to query features for which the datasets are disabled will
lead to a DisabledDatasetException being returned.
This commit is contained in:
Michael Bell
2023-08-04 18:43:37 +01:00
committed by GitHub
parent 522d0f066e
commit db7946d762
34 changed files with 1005 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/nodejs/node_osrm.cpp
+1
View File
@@ -74,6 +74,7 @@ Napi::Object Engine::Init(Napi::Env env, Napi::Object exports)
* Old behaviour: Path to a file on disk to store the memory using mmap. Current behaviour: setting this value is the same as setting `mmap_memory: true`.
* @param {Boolean} [options.mmap_memory] Map on-disk files to virtual memory addresses (mmap), rather than loading into RAM.
* @param {String} [options.path] The path to the `.osrm` files. This is mutually exclusive with setting {options.shared_memory} to true.
* @param {Array} [options.disable_feature_dataset] Disables a feature dataset from being loaded into memory if not needed. Options: `ROUTE_STEPS`, `ROUTE_GEOMETRY`.
* @param {Number} [options.max_locations_trip] Max. locations supported in trip query (default: unlimited).
* @param {Number} [options.max_locations_viaroute] Max. locations supported in viaroute query (default: unlimited).
* @param {Number} [options.max_locations_distance_table] Max. locations supported in distance table query (default: unlimited).
src/server/request_handler.cpp
+56 -42
View File
@@ -9,21 +9,16 @@
#include "util/log.hpp"
#include "util/string_util.hpp"
#include "util/timing_util.hpp"
#include "util/typedefs.hpp"
#include "engine/status.hpp"
#include "osrm/osrm.hpp"
#include "util/json_container.hpp"
#include <boost/iostreams/copy.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <boost/iostreams/filtering_streambuf.hpp>
#include <ctime>
#include <algorithm>
#include <iostream>
#include <iterator>
#include <string>
#include <thread>
@@ -36,6 +31,48 @@ void RequestHandler::RegisterServiceHandler(
service_handler = std::move(service_handler_);
}
void SendResponse(ServiceHandler::ResultT &result, http::reply &current_reply)
{
current_reply.headers.emplace_back("Access-Control-Allow-Origin", "*");
current_reply.headers.emplace_back("Access-Control-Allow-Methods", "GET");
current_reply.headers.emplace_back("Access-Control-Allow-Headers",
"X-Requested-With, Content-Type");
if (result.is<util::json::Object>())
{
current_reply.headers.emplace_back("Content-Type", "application/json; charset=UTF-8");
current_reply.headers.emplace_back("Content-Disposition",
"inline; filename=\"response.json\"");
util::json::render(current_reply.content, result.get<util::json::Object>());
}
else if (result.is<flatbuffers::FlatBufferBuilder>())
{
auto &buffer = result.get<flatbuffers::FlatBufferBuilder>();
current_reply.content.resize(buffer.GetSize());
std::copy(buffer.GetBufferPointer(),
buffer.GetBufferPointer() + buffer.GetSize(),
current_reply.content.begin());
current_reply.headers.emplace_back(
"Content-Type", "application/x-flatbuffers;schema=osrm.engine.api.fbresult");
}
else
{
BOOST_ASSERT(result.is<std::string>());
current_reply.content.resize(result.get<std::string>().size());
std::copy(result.get<std::string>().cbegin(),
result.get<std::string>().cend(),
current_reply.content.begin());
current_reply.headers.emplace_back("Content-Type", "application/x-protobuf");
}
// set headers
current_reply.headers.emplace_back("Content-Length",
std::to_string(current_reply.content.size()));
}
void RequestHandler::HandleRequest(const http::request &current_request, http::reply &current_reply)
{
if (!service_handler)
@@ -96,43 +133,7 @@ void RequestHandler::HandleRequest(const http::request &current_request, http::r
std::to_string(position) + ": \"" + context + "\"";
}
current_reply.headers.emplace_back("Access-Control-Allow-Origin", "*");
current_reply.headers.emplace_back("Access-Control-Allow-Methods", "GET");
current_reply.headers.emplace_back("Access-Control-Allow-Headers",
"X-Requested-With, Content-Type");
if (result.is<util::json::Object>())
{
current_reply.headers.emplace_back("Content-Type", "application/json; charset=UTF-8");
current_reply.headers.emplace_back("Content-Disposition",
"inline; filename=\"response.json\"");
util::json::render(current_reply.content, result.get<util::json::Object>());
}
else if (result.is<flatbuffers::FlatBufferBuilder>())
{
auto &buffer = result.get<flatbuffers::FlatBufferBuilder>();
current_reply.content.resize(buffer.GetSize());
std::copy(buffer.GetBufferPointer(),
buffer.GetBufferPointer() + buffer.GetSize(),
current_reply.content.begin());
current_reply.headers.emplace_back(
"Content-Type", "application/x-flatbuffers;schema=osrm.engine.api.fbresult");
}
else
{
BOOST_ASSERT(result.is<std::string>());
current_reply.content.resize(result.get<std::string>().size());
std::copy(result.get<std::string>().cbegin(),
result.get<std::string>().cend(),
current_reply.content.begin());
current_reply.headers.emplace_back("Content-Type", "application/x-protobuf");
}
// set headers
current_reply.headers.emplace_back("Content-Length",
std::to_string(current_reply.content.size()));
SendResponse(result, current_reply);
if (!std::getenv("DISABLE_ACCESS_LOGGING"))
{
@@ -168,6 +169,19 @@ void RequestHandler::HandleRequest(const http::request &current_request, http::r
<< request_string;
}
}
catch (const util::DisabledDatasetException &e)
{
current_reply.status = http::reply::bad_request;
ServiceHandler::ResultT result = util::json::Object();
auto &json_result = result.get<util::json::Object>();
json_result.values["code"] = "DisabledDataset";
json_result.values["message"] = e.what();
SendResponse(result, current_reply);
util::Log(logWARNING) << "[disabled dataset error][" << tid << "] code: DisabledDataset_"
<< e.Dataset() << ", uri: " << current_request.uri;
}
catch (const std::exception &e)
{
current_reply = http::reply::stock_reply(http::reply::internal_server_error);
src/storage/storage.cpp
+57 -43
View File
@@ -294,17 +294,20 @@ std::vector<std::pair<bool, boost::filesystem::path>> Storage::GetStaticFiles()
std::vector<std::pair<bool, boost::filesystem::path>> files = {
{IS_OPTIONAL, config.GetPath(".osrm.cells")},
{IS_OPTIONAL, config.GetPath(".osrm.partition")},
{IS_REQUIRED, config.GetPath(".osrm.icd")},
{IS_REQUIRED, config.GetPath(".osrm.properties")},
{IS_REQUIRED, config.GetPath(".osrm.nbg_nodes")},
{IS_REQUIRED, config.GetPath(".osrm.ebg_nodes")},
{IS_REQUIRED, config.GetPath(".osrm.tls")},
{IS_REQUIRED, config.GetPath(".osrm.tld")},
{IS_REQUIRED, config.GetPath(".osrm.timestamp")},
{IS_REQUIRED, config.GetPath(".osrm.maneuver_overrides")},
{IS_REQUIRED, config.GetPath(".osrm.edges")},
{IS_REQUIRED, config.GetPath(".osrm.names")},
{IS_REQUIRED, config.GetPath(".osrm.ramIndex")}};
{IS_REQUIRED, config.GetPath(".osrm.nbg_nodes")},
{IS_REQUIRED, config.GetPath(".osrm.ramIndex")},
{IS_REQUIRED, config.GetPath(".osrm.properties")},
{IS_REQUIRED, config.GetPath(".osrm.timestamp")}};
for (const auto &file : {".osrm.edges", ".osrm.names", ".osrm.icd", ".osrm.tls", ".osrm.tld"})
{
if (config.IsRequiredConfiguredInput(file))
{
files.push_back({IS_REQUIRED, config.GetPath(file)});
}
}
for (const auto &file : files)
{
@@ -394,12 +397,6 @@ void Storage::PopulateStaticData(const SharedDataIndex &index)
absolute_file_index_path.begin(), absolute_file_index_path.end(), file_index_path_ptr);
}
// Name data
{
auto name_table = make_name_table_view(index, "/common/names");
extractor::files::readNames(config.GetPath(".osrm.names"), name_table);
}
// Timestamp mark
{
auto timestamp_ref = make_timestamp_view(index, "/common/timestamp");
@@ -412,25 +409,39 @@ void Storage::PopulateStaticData(const SharedDataIndex &index)
}
// Turn lane data
if (config.IsRequiredConfiguredInput(".osrm.tld"))
{
auto turn_lane_data = make_lane_data_view(index, "/common/turn_lanes");
extractor::files::readTurnLaneData(config.GetPath(".osrm.tld"), turn_lane_data);
}
// Turn lane descriptions
if (config.IsRequiredConfiguredInput(".osrm.tls"))
{
auto views = make_turn_lane_description_views(index, "/common/turn_lanes");
extractor::files::readTurnLaneDescriptions(
config.GetPath(".osrm.tls"), std::get<0>(views), std::get<1>(views));
}
// Load edge-based nodes data
// Load intersection data
if (config.IsRequiredConfiguredInput(".osrm.icd"))
{
auto node_data = make_ebn_data_view(index, "/common/ebg_node_data");
extractor::files::readNodeData(config.GetPath(".osrm.ebg_nodes"), node_data);
auto intersection_bearings_view =
make_intersection_bearings_view(index, "/common/intersection_bearings");
auto entry_classes = make_entry_classes_view(index, "/common/entry_classes");
extractor::files::readIntersections(
config.GetPath(".osrm.icd"), intersection_bearings_view, entry_classes);
}
// Name data
if (config.IsRequiredConfiguredInput(".osrm.names"))
{
auto name_table = make_name_table_view(index, "/common/names");
extractor::files::readNames(config.GetPath(".osrm.names"), name_table);
}
// Load original edge data
if (config.IsRequiredConfiguredInput(".osrm.edges"))
{
auto turn_data = make_turn_data_view(index, "/common/turn_data");
@@ -441,6 +452,12 @@ void Storage::PopulateStaticData(const SharedDataIndex &index)
config.GetPath(".osrm.edges"), turn_data, *connectivity_checksum_ptr);
}
// Load edge-based nodes data
{
auto node_data = make_ebn_data_view(index, "/common/ebg_node_data");
extractor::files::readNodeData(config.GetPath(".osrm.ebg_nodes"), node_data);
}
// Loading list of coordinates
{
auto views = make_nbn_data_view(index, "/common/nbn_data");
@@ -466,15 +483,6 @@ void Storage::PopulateStaticData(const SharedDataIndex &index)
metric_name = profile_properties_ptr->GetWeightName();
}
// Load intersection data
{
auto intersection_bearings_view =
make_intersection_bearings_view(index, "/common/intersection_bearings");
auto entry_classes = make_entry_classes_view(index, "/common/entry_classes");
extractor::files::readIntersections(
config.GetPath(".osrm.icd"), intersection_bearings_view, entry_classes);
}
if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
{
auto mlp = make_partition_view(index, "/mld/multilevelpartition");
@@ -545,15 +553,18 @@ void Storage::PopulateUpdatableData(const SharedDataIndex &index)
contractor::files::readGraph(
config.GetPath(".osrm.hsgr"), metrics, graph_connectivity_checksum);
auto turns_connectivity_checksum =
*index.GetBlockPtr<std::uint32_t>("/common/connectivity_checksum");
if (turns_connectivity_checksum != graph_connectivity_checksum)
if (config.IsRequiredConfiguredInput("osrm.edges"))
{
throw util::exception(
"Connectivity checksum " + std::to_string(graph_connectivity_checksum) + " in " +
config.GetPath(".osrm.hsgr").string() + " does not equal to checksum " +
std::to_string(turns_connectivity_checksum) + " in " +
config.GetPath(".osrm.edges").string());
auto turns_connectivity_checksum =
*index.GetBlockPtr<std::uint32_t>("/common/connectivity_checksum");
if (turns_connectivity_checksum != graph_connectivity_checksum)
{
throw util::exception(
"Connectivity checksum " + std::to_string(graph_connectivity_checksum) +
" in " + config.GetPath(".osrm.hsgr").string() +
" does not equal to checksum " + std::to_string(turns_connectivity_checksum) +
" in " + config.GetPath(".osrm.edges").string());
}
}
}
@@ -573,15 +584,18 @@ void Storage::PopulateUpdatableData(const SharedDataIndex &index)
customizer::files::readGraph(
config.GetPath(".osrm.mldgr"), graph_view, graph_connectivity_checksum);
auto turns_connectivity_checksum =
*index.GetBlockPtr<std::uint32_t>("/common/connectivity_checksum");
if (turns_connectivity_checksum != graph_connectivity_checksum)
if (config.IsRequiredConfiguredInput("osrm.edges"))
{
throw util::exception(
"Connectivity checksum " + std::to_string(graph_connectivity_checksum) + " in " +
config.GetPath(".osrm.hsgr").string() + " does not equal to checksum " +
std::to_string(turns_connectivity_checksum) + " in " +
config.GetPath(".osrm.edges").string());
auto turns_connectivity_checksum =
*index.GetBlockPtr<std::uint32_t>("/common/connectivity_checksum");
if (turns_connectivity_checksum != graph_connectivity_checksum)
{
throw util::exception(
"Connectivity checksum " + std::to_string(graph_connectivity_checksum) +
" in " + config.GetPath(".osrm.mldgr").string() +
" does not equal to checksum " + std::to_string(turns_connectivity_checksum) +
" in " + config.GetPath(".osrm.edges").string());
}
}
}
}
src/storage/storage_config.cpp
+23
View File
@@ -0,0 +1,23 @@
#include "osrm/datasets.hpp"
#include "osrm/exception.hpp"
#include "util/exception_utils.hpp"
#include <boost/algorithm/string/case_conv.hpp>
namespace osrm::storage
{
std::istream &operator>>(std::istream &in, FeatureDataset &datasets)
{
std::string token;
in >> token;
boost::to_lower(token);
if (token == "route_steps")
datasets = FeatureDataset::ROUTE_STEPS;
else if (token == "route_geometry")
datasets = FeatureDataset::ROUTE_GEOMETRY;
else
throw util::RuntimeError(token, ErrorCode::UnknownFeatureDataset, SOURCE_REF);
return in;
}
} // namespace osrm::storage
src/tools/routed.cpp
+7 -2
View File
@@ -4,6 +4,7 @@
#include "util/meminfo.hpp"
#include "util/version.hpp"
#include "osrm/datasets.hpp"
#include "osrm/engine_config.hpp"
#include "osrm/exception.hpp"
#include "osrm/osrm.hpp"
@@ -23,7 +24,6 @@
#include <exception>
#include <future>
#include <iostream>
#include <memory>
#include <new>
#include <string>
#include <thread>
@@ -69,6 +69,7 @@ std::istream &operator>>(std::istream &in, EngineConfig::Algorithm &algorithm)
throw util::RuntimeError(token, ErrorCode::UnknownAlgorithm, SOURCE_REF);
return in;
}
} // namespace osrm::engine
// overload validate for the double type to allow "unlimited" as an input
@@ -155,6 +156,10 @@ inline unsigned generateServerProgramOptions(const int argc,
value<EngineConfig::Algorithm>(&config.algorithm)
->default_value(EngineConfig::Algorithm::CH, "CH"),
"Algorithm to use for the data. Can be CH, CoreCH, MLD.") //
("disable-feature-dataset",
value<std::vector<storage::FeatureDataset>>(&config.disable_feature_dataset)->multitoken(),
"Disables a feature dataset from being loaded into memory if not needed. Options: "
"ROUTE_STEPS, ROUTE_GEOMETRY") //
("max-viaroute-size",
value<int>(&config.max_locations_viaroute)->default_value(500),
"Max. locations supported in viaroute query") //
@@ -276,7 +281,7 @@ try
if (!base_path.empty())
{
config.storage_config = storage::StorageConfig(base_path);
config.storage_config = storage::StorageConfig(base_path, config.disable_feature_dataset);
}
if (!config.use_shared_memory && !config.storage_config.IsValid())
{
src/tools/store.cpp
+14 -3
View File
@@ -2,6 +2,7 @@
#include "storage/shared_memory.hpp"
#include "storage/shared_monitor.hpp"
#include "storage/storage.hpp"
#include "osrm/storage_config.hpp"
#include "osrm/exception.hpp"
#include "util/log.hpp"
@@ -9,6 +10,7 @@
#include "util/typedefs.hpp"
#include "util/version.hpp"
#include <boost/algorithm/string/case_conv.hpp>
#include <boost/filesystem.hpp>
#include <boost/program_options.hpp>
@@ -100,7 +102,8 @@ bool generateDataStoreOptions(const int argc,
std::string &dataset_name,
bool &list_datasets,
bool &list_blocks,
bool &only_metric)
bool &only_metric,
std::vector<storage::FeatureDataset> &disable_feature_dataset)
{
// declare a group of options that will be allowed only on command line
boost::program_options::options_description generic_options("Options");
@@ -125,6 +128,12 @@ bool generateDataStoreOptions(const int argc,
boost::program_options::value<std::string>(&dataset_name)->default_value(""),
"Name of the dataset to load into memory. This allows having multiple datasets in memory "
"at the same time.") //
("disable-feature-dataset",
boost::program_options::value<std::vector<storage::FeatureDataset>>(
&disable_feature_dataset)
->multitoken(),
"Disables a feature dataset from being loaded into memory if not needed. Options: "
"ROUTE_STEPS, ROUTE_GEOMETRY") //
("list",
boost::program_options::value<bool>(&list_datasets)
->default_value(false)
@@ -239,6 +248,7 @@ try
bool list_datasets = false;
bool list_blocks = false;
bool only_metric = false;
std::vector<storage::FeatureDataset> disable_feature_dataset;
if (!generateDataStoreOptions(argc,
argv,
verbosity,
@@ -247,7 +257,8 @@ try
dataset_name,
list_datasets,
list_blocks,
only_metric))
only_metric,
disable_feature_dataset))
{
return EXIT_SUCCESS;
}
@@ -260,7 +271,7 @@ try
return EXIT_SUCCESS;
}
storage::StorageConfig config(base_path);
storage::StorageConfig config(base_path, disable_feature_dataset);
if (!config.IsValid())
{
util::Log(logERROR) << "Config contains invalid file paths. Exiting!";
src/util/exception.cpp
+3
View File
@@ -1,5 +1,7 @@
#include "util/exception.hpp"
#include <utility>
// This function exists to 'anchor' the class, and stop the compiler from
// copying vtable and RTTI info into every object file that includes
// this header. (Caught by -Wweak-vtables under Clang.)
@@ -16,4 +18,5 @@ namespace osrm::util
void exception::anchor() const {}
void RuntimeError::anchor() const {}
void DisabledDatasetException::anchor() const {}
} // namespace osrm::util