Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add support for disabling feature datasets (#6666)

Browse Source 
This change adds support for disabling datasets, such that specific
files are not loaded into memory when running OSRM. This enables users
to not pay the memory cost for features they do not intend to use.

Initially, there are two options:
- ROUTE_GEOMETRY, for disabling overview, steps, annotations and waypoints.
- ROUTE_STEPS, for disabling steps only.

Attempts to query features for which the datasets are disabled will
lead to a DisabledDatasetException being returned.
This commit is contained in:
Michael Bell
2023-08-04 18:43:37 +01:00
committed by GitHub
parent 522d0f066e
commit db7946d762
34 changed files with 1005 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
features/lib/osrm_loader.js
+12 -9
View File
@@ -68,8 +68,9 @@ class OSRMDirectLoader extends OSRMBaseLoader {
super(scope);
}
load (inputFile, callback) {
this.inputFile = inputFile;
load (ctx, callback) {
this.inputFile = ctx.inputFile;
this.loaderArgs = ctx.loaderArgs;
this.shutdown(() => {
this.launch(callback);
});
@@ -78,7 +79,7 @@ class OSRMDirectLoader extends OSRMBaseLoader {
osrmUp (callback) {
if (this.osrmIsRunning()) return callback(new Error("osrm-routed already running!"));
const command_arguments = util.format('%s -p %d -i %s -a %s', this.inputFile, this.scope.OSRM_PORT, this.scope.OSRM_IP, this.scope.ROUTING_ALGORITHM);
const command_arguments = util.format('%s -p %d -i %s -a %s %s', this.inputFile, this.scope.OSRM_PORT, this.scope.OSRM_IP, this.scope.ROUTING_ALGORITHM, this.loaderArgs);
this.child = this.scope.runBin('osrm-routed', command_arguments, this.scope.environment, (err) => {
if (err && err.signal !== 'SIGINT') {
this.child = null;
@@ -101,8 +102,9 @@ class OSRMmmapLoader extends OSRMBaseLoader {
super(scope);
}
load (inputFile, callback) {
this.inputFile = inputFile;
load (ctx, callback) {
this.inputFile = ctx.inputFile;
this.loaderArgs = ctx.loaderArgs;
this.shutdown(() => {
this.launch(callback);
});
@@ -111,7 +113,7 @@ class OSRMmmapLoader extends OSRMBaseLoader {
osrmUp (callback) {
if (this.osrmIsRunning()) return callback(new Error("osrm-routed already running!"));
const command_arguments = util.format('%s -p %d -i %s -a %s --mmap', this.inputFile, this.scope.OSRM_PORT, this.scope.OSRM_IP, this.scope.ROUTING_ALGORITHM);
const command_arguments = util.format('%s -p %d -i %s -a %s --mmap %s', this.inputFile, this.scope.OSRM_PORT, this.scope.OSRM_IP, this.scope.ROUTING_ALGORITHM, this.loaderArgs);
this.child = this.scope.runBin('osrm-routed', command_arguments, this.scope.environment, (err) => {
if (err && err.signal !== 'SIGINT') {
this.child = null;
@@ -134,8 +136,9 @@ class OSRMDatastoreLoader extends OSRMBaseLoader {
super(scope);
}
load (inputFile, callback) {
this.inputFile = inputFile;
load (ctx, callback) {
this.inputFile = ctx.inputFile;
this.loaderArgs = ctx.loaderArgs;
this.loadData((err) => {
if (err) return callback(err);
@@ -148,7 +151,7 @@ class OSRMDatastoreLoader extends OSRMBaseLoader {
}
loadData (callback) {
const command_arguments = util.format('--dataset-name=%s %s', this.scope.DATASET_NAME, this.inputFile);
const command_arguments = util.format('--dataset-name=%s %s %s', this.scope.DATASET_NAME, this.inputFile, this.loaderArgs);
this.scope.runBin('osrm-datastore', command_arguments, this.scope.environment, (err) => {
if (err) return callback(new Error('*** osrm-datastore exited with ' + err.code + ': ' + err));
callback();
features/options/data/disabled_dataset.feature
+141
View File
@@ -0,0 +1,141 @@
@routing @disable-feature-dataset
Feature: disable-feature-dataset command line options
Background:
Given the profile "testbot"
And the node map
"""
0
a b c
"""
And the ways
| nodes |
| ab |
| bc |
Scenario: disable-feature-dataset - geometry disabled error
Given the data load extra arguments "--disable-feature-dataset ROUTE_GEOMETRY"
# The default values
And the query options
| overview | simplified |
| annotations | false |
| steps | false |
| skip_waypoints | false |
When I route I should get
| from | to | code |
| a | c | DisabledDataset |
When I plan a trip I should get
| waypoints | code |
| a,b,c | DisabledDataset |
When I match I should get
| trace | code |
| abc | DisabledDataset |
Scenario: disable-feature-dataset - geometry disabled error table
Given the data load extra arguments "--disable-feature-dataset ROUTE_GEOMETRY"
When I request nearest I should get
| in | code |
| 0 | DisabledDataset |
When I request a travel time matrix with these waypoints I should get the response code
| waypoints | code |
| a,b,c | DisabledDataset |
Scenario: disable-feature-dataset - geometry disabled success
Given the data load extra arguments "--disable-feature-dataset ROUTE_GEOMETRY"
# No geometry values returned
And the query options
| overview | false |
| annotations | false |
| steps | false |
| skip_waypoints | true |
When I route I should get
| from | to | code |
| a | c | Ok |
When I plan a trip I should get
| waypoints | code |
| a,b,c | Ok |
When I match I should get
| trace | code |
| abc | Ok |
Scenario: disable-feature-dataset - geometry disabled error table
Given the data load extra arguments "--disable-feature-dataset ROUTE_GEOMETRY"
And the query options
| skip_waypoints | true |
# You would never do this, but just to prove the point.
When I request nearest I should get
| in | code |
| 0 | Ok |
When I request a travel time matrix with these waypoints I should get the response code
| waypoints | code |
| a,b,c | Ok |
Scenario: disable-feature-dataset - steps disabled error
Given the data load extra arguments "--disable-feature-dataset ROUTE_STEPS"
# Default + annotations, steps
And the query options
| overview | simplified |
| annotations | true |
| steps | true |
When I route I should get
| from | to | code |
| a | c | DisabledDataset |
When I plan a trip I should get
| waypoints | code |
| a,b,c | DisabledDataset |
When I match I should get
| trace | code |
| abc | DisabledDataset |
Scenario: disable-feature-dataset - geometry disabled error table
Given the data load extra arguments "--disable-feature-dataset ROUTE_STEPS"
When I request nearest I should get
| in | code |
| 0 | Ok |
When I request a travel time matrix with these waypoints I should get the response code
| waypoints | code |
| a,b,c | Ok |
Scenario: disable-feature-dataset - steps disabled success
Given the data load extra arguments "--disable-feature-dataset ROUTE_STEPS"
# Default + steps
And the query options
| overview | simplified |
| annotations | true |
| steps | false |
When I route I should get
| from | to | code |
| a | c | Ok |
When I plan a trip I should get
| waypoints | code |
| a,b,c | Ok |
When I match I should get
| trace | code |
| abc | Ok |
features/step_definitions/data.js
+5
View File
@@ -33,6 +33,11 @@ module.exports = function () {
callback();
});
this.Given(/^the data load extra arguments "(.*?)"$/, (args, callback) => {
this.loaderArgs = this.expandOptions(args);
callback();
});
this.Given(/^a grid size of ([0-9.]+) meters$/, (meters, callback) => {
this.setGridSize(meters);
callback();
features/step_definitions/distance_matrix.js
+60 -3
View File
@@ -5,6 +5,7 @@ var FBResult = require('../support/fbresult_generated').osrm.engine.api.fbresult
module.exports = function () {
const durationsRegex = new RegExp(/^I request a travel time matrix I should get$/);
const durationsCodeOnlyRegex = new RegExp(/^I request a travel time matrix with these waypoints I should get the response code$/);
const distancesRegex = new RegExp(/^I request a travel distance matrix I should get$/);
const estimatesRegex = new RegExp(/^I request a travel time matrix I should get estimates for$/);
const durationsRegexFb = new RegExp(/^I request a travel time matrix with flatbuffers I should get$/);
@@ -17,6 +18,7 @@ module.exports = function () {
const FORMAT_FB = 'flatbuffers';
this.When(durationsRegex, function(table, callback) {tableParse.call(this, table, DURATIONS_NO_ROUTE, 'durations', FORMAT_JSON, callback);}.bind(this));
this.When(durationsCodeOnlyRegex, function(table, callback) {tableCodeOnlyParse.call(this, table, 'durations', FORMAT_JSON, callback);}.bind(this));
this.When(distancesRegex, function(table, callback) {tableParse.call(this, table, DISTANCES_NO_ROUTE, 'distances', FORMAT_JSON, callback);}.bind(this));
this.When(estimatesRegex, function(table, callback) {tableParse.call(this, table, DISTANCES_NO_ROUTE, 'fallback_speed_cells', FORMAT_JSON, callback);}.bind(this));
this.When(durationsRegexFb, function(table, callback) {tableParse.call(this, table, DURATIONS_NO_ROUTE, 'durations', FORMAT_FB, callback);}.bind(this));
@@ -27,6 +29,64 @@ const durationsParse = function(v) { return isNaN(parseInt(v)); };
const distancesParse = function(v) { return isNaN(parseFloat(v)); };
const estimatesParse = function(v) { return isNaN(parseFloat(v)); };
function tableCodeOnlyParse(table, annotation, format, callback) {
const params = this.queryParams;
params.annotations = ['durations','fallback_speed_cells'].indexOf(annotation) !== -1 ? 'duration' : 'distance';
params.output = format;
var got;
this.reprocessAndLoadData((e) => {
if (e) return callback(e);
var testRow = (row, ri, cb) => {
var afterRequest = (err, res) => {
if (err) return cb(err);
for (var k in row) {
var match = k.match(/param:(.*)/);
if (match) {
if (row[k] === '(nil)') {
params[match[1]] = null;
} else if (row[k]) {
params[match[1]] = [row[k]];
}
got[k] = row[k];
}
}
var json;
got.code = 'unknown';
if (res.body.length) {
json = JSON.parse(res.body);
got.code = json.code;
}
cb(null, got);
};
var params = this.queryParams,
waypoints = [];
if (row.waypoints) {
row.waypoints.split(',').forEach((n) => {
var node = this.findNodeByName(n);
if (!node) throw new Error(util.format('*** unknown waypoint node "%s"', n.trim()));
waypoints.push({ coord: node, type: 'loc' });
});
got = { waypoints: row.waypoints };
this.requestTable(waypoints, params, afterRequest);
} else {
throw new Error('*** no waypoints');
}
};
this.processRowsAndDiff(table, testRow, callback);
});
}
function tableParse(table, noRoute, annotation, format, callback) {
const parse = annotation == 'distances' ? distancesParse : (annotation == 'durations' ? durationsParse : estimatesParse);
@@ -62,9 +122,6 @@ function tableParse(table, noRoute, annotation, format, callback) {
});
}
var actual = [];
actual.push(table.headers);
this.reprocessAndLoadData((e) => {
if (e) return callback(e);
// compute matrix
features/step_definitions/nearest.js
+27 -19
View File
@@ -12,35 +12,43 @@ module.exports = function () {
var inNode = this.findNodeByName(row.in);
if (!inNode) throw new Error(util.format('*** unknown in-node "%s"', row.in));
var outNode = this.findNodeByName(row.out);
if (!outNode) throw new Error(util.format('*** unknown out-node "%s"', row.out));
this.requestNearest(inNode, this.queryParams, (err, response) => {
if (err) return cb(err);
var coord;
var headers = new Set(table.raw()[0]);
if (response.statusCode === 200 && response.body.length) {
var got = { in: row.in};
if (response.body.length) {
var json = JSON.parse(response.body);
got.code = json.code;
coord = json.waypoints[0].location;
if (response.statusCode === 200) {
var got = { in: row.in, out: row.out };
if (headers.has('data_version')) {
got.data_version = json.data_version || '';
}
Object.keys(row).forEach((key) => {
if (key === 'out') {
if (this.FuzzyMatch.matchLocation(coord, outNode)) {
got[key] = row[key];
} else {
row[key] = util.format('%s [%d,%d]', row[key], outNode.lat, outNode.lon);
}
if (headers.has('data_version')) {
got.data_version = json.data_version || '';
}
});
if (json.waypoints && json.waypoints.length && row.out) {
coord = json.waypoints[0].location;
got.out = row.out;
var outNode = this.findNodeByName(row.out);
if (!outNode) throw new Error(util.format('*** unknown out-node "%s"', row.out));
Object.keys(row).forEach((key) => {
if (key === 'out') {
if (this.FuzzyMatch.matchLocation(coord, outNode)) {
got[key] = row[key];
} else {
row[key] = util.format('%s [%d,%d]', row[key], outNode.lat, outNode.lon);
}
}
});
}
}
cb(null, got);
}
else {
features/step_definitions/trip.js
+1 -2
View File
@@ -91,7 +91,7 @@ module.exports = function () {
var encodedResult = '';
if (json.trips) row.trips.split(',').forEach((sub, si) => {
if (json.trips && row.trips) row.trips.split(',').forEach((sub, si) => {
if (si >= subTrips.length) {
ok = false;
} else {
@@ -134,7 +134,6 @@ module.exports = function () {
} else {
var params = this.queryParams,
waypoints = [];
params['steps'] = 'true';
if (row.from && row.to) {
var fromNode = this.findNodeByName(row.from);
if (!fromNode) throw new Error(util.format('*** unknown from-node "%s"', row.from));
features/support/data.js
+2 -1
View File
@@ -280,10 +280,11 @@ module.exports = function () {
};
this.reprocessAndLoadData = (callback) => {
let p = {loaderArgs: this.loaderArgs, inputFile: this.processedCacheFile};
let queue = d3.queue(1);
queue.defer(this.writeAndLinkOSM.bind(this));
queue.defer(this.extractContractPartitionAndCustomize.bind(this));
queue.defer(this.osrmLoader.load.bind(this.osrmLoader), this.processedCacheFile);
queue.defer(this.osrmLoader.load.bind(this.osrmLoader), p);
queue.awaitAll(callback);
};
features/support/hooks.js
+1
View File
@@ -37,6 +37,7 @@ module.exports = function () {
this.contractArgs = '';
this.partitionArgs = '';
this.customizeArgs = '';
this.loaderArgs = '';
this.environment = Object.assign(this.DEFAULT_ENVIRONMENT);
this.resetOSM();
features/support/route.js
+2 -1
View File
@@ -101,7 +101,8 @@ module.exports = function () {
this.requestTrip = (waypoints, userParams, callback) => {
var defaults = {
output: 'json'
output: 'json',
steps: 'true'
},
params = this.overwriteParams(defaults, userParams);