Add MLD testing to cucumber tests

cucumber.js

@@ -2,8 +2,6 @@ module.exports = {
     default: '--strict --tags ~@stress --tags ~@todo --require features/support --require features/step_definitions',
     verify: '--strict --tags ~@stress --tags ~@todo -f progress --require features/support --require features/step_definitions',
     todo: '--strict --tags @todo --require features/support --require features/step_definitions',
-    all: '--strict --require features/support --require features/step_definitions'
+    all: '--strict --require features/support --require features/step_definitions',
+    mld: '--strict --tags ~@stress --tags ~@todo --tags ~@match --tags ~@alternative --tags ~@traffic --tags ~@weight --tags ~@matrix --tags ~@trip --tags ~@via --require features/support --require features/step_definitions -f progress'
 }
-
-
-

features/car/summaries.feature

@@ -5,7 +5,7 @@ Feature: Basic Routing
         Given the profile "car"
         Given a grid size of 500 meters
 
-    @smallest
+    @smallest @via
     Scenario: Summaries when routing on a simple network
         Given the node map
             """

features/options/contract/version.feature

@@ -1,4 +1,4 @@
-@prepare @options @version
+@contract @options @version
 Feature: osrm-contract command line options: version
 # the regex will match these two formats:
 # v0.3.7.0          # this is the normal format when you build from a git clone

features/options/customize/files.feature

@@ -0,0 +1,23 @@
+@customize @options @files
+Feature: osrm-contract command line options: files
+    Background:
+        Given the profile "testbot"
+        And the node map
+            """
+            a b
+            """
+        And the ways
+            | nodes |
+            | ab    |
+        And the data has been extracted
+        And the data has been partitioned
+
+    Scenario: osrm-customize - Passing base file
+        When I run "osrm-customize {processed_file}"
+        Then it should exit successfully
+
+    Scenario: osrm-customize - Missing input file
+        When I try to run "osrm-customize over-the-rainbow.osrm"
+        And stderr should contain "over-the-rainbow.osrm"
+        And stderr should contain "not found"
+        And it should exit with an error

features/options/customize/help.feature

@@ -0,0 +1,35 @@
+@contract @options @help
+Feature: osrm-customize command line options: help
+
+    Scenario: osrm-customize - Help should be shown when no options are passed
+        When I try to run "osrm-customize"
+        Then stderr should be empty
+        And stdout should contain /osrm-customize(.exe)? <input.osrm> \[options\]:/
+        And stdout should contain "Options:"
+        And stdout should contain "--version"
+        And stdout should contain "--help"
+        And stdout should contain "Configuration:"
+        And stdout should contain "--threads"
+        And it should exit with an error
+
+    Scenario: osrm-customize - Help, short
+        When I run "osrm-customize -h"
+        Then stderr should be empty
+        And stdout should contain /osrm-customize(.exe)? <input.osrm> \[options\]:/
+        And stdout should contain "Options:"
+        And stdout should contain "--version"
+        And stdout should contain "--help"
+        And stdout should contain "Configuration:"
+        And stdout should contain "--threads"
+        And it should exit successfully
+
+    Scenario: osrm-customize - Help, long
+        When I run "osrm-customize --help"
+        Then stderr should be empty
+        And stdout should contain /osrm-customize(.exe)? <input.osrm> \[options\]:/
+        And stdout should contain "Options:"
+        And stdout should contain "--version"
+        And stdout should contain "--help"
+        And stdout should contain "Configuration:"
+        And stdout should contain "--threads"
+        And it should exit successfully

features/options/customize/invalid.feature

@@ -0,0 +1,20 @@
+@prepare @options @invalid
+Feature: osrm-customize command line options: invalid options
+
+    Background:
+        Given the profile "testbot"
+        And the node map
+            """
+            a b
+            """
+        And the ways
+            | nodes |
+            | ab    |
+        And the data has been extracted
+
+    Scenario: osrm-customize - Non-existing option
+        When I try to run "osrm-customize --fly-me-to-the-moon"
+        Then stdout should be empty
+        And stderr should contain "option"
+        And stderr should contain "fly-me-to-the-moon"
+        And it should exit with an error

features/options/customize/version.feature

@@ -0,0 +1,19 @@
+@prepare @options @version
+Feature: osrm-customize command line options: version
+
+    Background:
+        Given the profile "testbot"
+
+    Scenario: osrm-customize - Version, short
+        When I run "osrm-customize --v"
+        Then stderr should be empty
+        And stdout should contain 1 line
+        And stdout should contain /(v\d{1,2}\.\d{1,2}\.\d{1,2}|\w*-\d+-\w+)/
+        And it should exit successfully
+
+    Scenario: osrm-customize - Version, long
+        When I run "osrm-customize --version"
+        Then stderr should be empty
+        And stdout should contain 1 line
+        And stdout should contain /(v\d{1,2}\.\d{1,2}\.\d{1,2}|\w*-\d+-\w+)/
+        And it should exit successfully

features/options/partition/files.feature

@@ -0,0 +1,22 @@
+@partition @options @files
+Feature: osrm-partition command line options: files
+    Background:
+        Given the profile "testbot"
+        And the node map
+            """
+            a b
+            """
+        And the ways
+            | nodes |
+            | ab    |
+        And the data has been extracted
+
+    Scenario: osrm-partition - Passing base file
+        When I run "osrm-partition {processed_file}"
+        Then it should exit successfully
+
+    Scenario: osrm-partition - Missing input file
+        When I try to run "osrm-partition over-the-rainbow.osrm"
+        And stderr should contain "over-the-rainbow.osrm"
+        And stderr should contain "not found"
+        And it should exit with an error

features/options/partition/help.feature

@@ -0,0 +1,50 @@
+@partition @options @help
+Feature: osrm-partition command line options: help
+
+    Scenario: osrm-partition - Help should be shown when no options are passed
+        When I try to run "osrm-partition"
+        Then stderr should be empty
+        And stdout should contain /osrm-partition(.exe)? <input.osrm> \[options\]:/
+        And stdout should contain "Options:"
+        And stdout should contain "--version"
+        And stdout should contain "--help"
+        And stdout should contain "Configuration:"
+        And stdout should contain "--threads"
+        And stdout should contain "--min-cell-size"
+        And stdout should contain "--balance"
+        And stdout should contain "--boundary"
+        And stdout should contain "--optimizing-cuts"
+        And stdout should contain "--small-component-size"
+        And it should exit with an error
+
+    Scenario: osrm-partition - Help, short
+        When I run "osrm-partition -h"
+        Then stderr should be empty
+        And stdout should contain /osrm-partition(.exe)? <input.osrm> \[options\]:/
+        And stdout should contain "Options:"
+        And stdout should contain "--version"
+        And stdout should contain "--help"
+        And stdout should contain "Configuration:"
+        And stdout should contain "--threads"
+        And stdout should contain "--min-cell-size"
+        And stdout should contain "--balance"
+        And stdout should contain "--boundary"
+        And stdout should contain "--optimizing-cuts"
+        And stdout should contain "--small-component-size"
+        And it should exit successfully
+
+    Scenario: osrm-partition - Help, long
+        When I run "osrm-partition --help"
+        Then stderr should be empty
+        And stdout should contain /osrm-partition(.exe)? <input.osrm> \[options\]:/
+        And stdout should contain "Options:"
+        And stdout should contain "--version"
+        And stdout should contain "--help"
+        And stdout should contain "Configuration:"
+        And stdout should contain "--threads"
+        And stdout should contain "--min-cell-size"
+        And stdout should contain "--balance"
+        And stdout should contain "--boundary"
+        And stdout should contain "--optimizing-cuts"
+        And stdout should contain "--small-component-size"
+        And it should exit successfully

features/options/partition/invalid.feature

@@ -0,0 +1,20 @@
+@partition @options @invalid
+Feature: osrm-partition command line options: invalid options
+
+    Background:
+        Given the profile "testbot"
+        And the node map
+            """
+            a b
+            """
+        And the ways
+            | nodes |
+            | ab    |
+        And the data has been extracted
+
+    Scenario: osrm-partition - Non-existing option
+        When I try to run "osrm-partition --fly-me-to-the-moon"
+        Then stdout should be empty
+        And stderr should contain "option"
+        And stderr should contain "fly-me-to-the-moon"
+        And it should exit with an error

features/options/partition/version.feature

@@ -0,0 +1,19 @@
+@partition @options @version
+Feature: osrm-partition command line options: version
+
+    Background:
+        Given the profile "testbot"
+
+    Scenario: osrm-partition - Version, short
+        When I run "osrm-partition --v"
+        Then stderr should be empty
+        And stdout should contain 1 line
+        And stdout should contain /(v\d{1,2}\.\d{1,2}\.\d{1,2}|\w*-\d+-\w+)/
+        And it should exit successfully
+
+    Scenario: osrm-partition - Version, long
+        When I run "osrm-partition --version"
+        Then stderr should be empty
+        And stdout should contain 1 line
+        And stdout should contain /(v\d{1,2}\.\d{1,2}\.\d{1,2}|\w*-\d+-\w+)/
+        And it should exit successfully

features/step_definitions/data.js

@@ -23,6 +23,16 @@ module.exports = function () {
         callback();
     });
 
+    this.Given(/^the partition extra arguments "(.*?)"$/, (args, callback) => {
+        this.partitionArgs = this.expandOptions(args);
+        callback();
+    });
+
+    this.Given(/^the customize extra arguments "(.*?)"$/, (args, callback) => {
+        this.customizeArgs = this.expandOptions(args);
+        callback();
+    });
+
     this.Given(/^a grid size of ([0-9.]+) meters$/, (meters, callback) => {
         this.setGridSize(meters);
         callback();
@@ -259,12 +269,12 @@ module.exports = function () {
         this.writeAndLinkOSM(callback);
     });
 
-    this.Given(/^the data has been extracted$/, (callback) => {
+    this.Given(/^the data has been (extract|contract)ed$/, (step, callback) => {
         this.reprocess(callback);
     });
 
-    this.Given(/^the data has been contracted$/, (callback) => {
+    this.Given(/^the data has been (partition|customiz)ed$/, (step, callback) => {
-        this.reprocess(callback);
+        this.reprocessMLD(callback);
     });
 
     this.Given(/^osrm\-routed is stopped$/, (callback) => {

features/step_definitions/options.js

@@ -25,32 +25,16 @@ module.exports = function () {
         this.runAndSafeOutput('osrm-routed', options, callback);
     });
 
-    this.When(/^I run "osrm\-extract\s?(.*?)"$/, (options, callback) => {
+    this.When(/^I run "osrm\-(extract|contract|partition|customize)\s?(.*?)"$/, (binary, options, callback) => {
-        const stamp = this.processedCacheFile + '.extract';
+        const stamp = this.processedCacheFile + '.stamp_' + binary;
-        this.runAndSafeOutput('osrm-extract', options, (err) => {
+        this.runAndSafeOutput('osrm-' + binary, options, (err) => {
             if (err) return callback(err);
             fs.writeFile(stamp, 'ok', callback);
         });
     });
 
-    this.When(/^I run "osrm\-contract\s?(.*?)"$/, (options, callback) => {
+    this.When(/^I try to run "(osrm\-[a-z]+)\s?(.*?)"$/, (binary, options, callback) => {
-        const stamp = this.processedCacheFile + '.contract';
+        this.runAndSafeOutput(binary, options, () => { callback(); });
-        this.runAndSafeOutput('osrm-contract', options, (err) => {
-            if (err) return callback(err);
-            fs.writeFile(stamp, 'ok', callback);
-        });
-    });
-
-    this.When(/^I try to run "osrm\-routed\s?(.*?)"$/, (options, callback) => {
-        this.runAndSafeOutput('osrm-routed', options, () => { callback(); });
-    });
-
-    this.When(/^I try to run "osrm\-extract\s?(.*?)"$/, (options, callback) => {
-        this.runAndSafeOutput('osrm-extract', options, () => { callback(); });
-    });
-
-    this.When(/^I try to run "osrm\-contract\s?(.*?)"$/, (options, callback) => {
-        this.runAndSafeOutput('osrm-contract', options, () => { callback(); });
     });
 
     this.When(/^I run "osrm\-datastore\s?(.*?)"(?: with input "([^"]*)")?$/, (options, input, callback) => {

features/support/data.js

@@ -179,7 +179,7 @@ module.exports = function () {
     };
 
     this.extractData = (p, callback) => {
-        let stamp = p.processedCacheFile + '.extract';
+        let stamp = p.processedCacheFile + '.stamp_extract';
         fs.exists(stamp, (exists) => {
             if (exists) return callback();
 
@@ -193,7 +193,7 @@ module.exports = function () {
     };
 
     this.contractData = (p, callback) => {
-        let stamp = p.processedCacheFile + '.contract';
+        let stamp = p.processedCacheFile + '.stamp_contract';
         fs.exists(stamp, (exists) => {
             if (exists) return callback();
 
@@ -206,10 +206,39 @@ module.exports = function () {
         });
     };
 
+    this.partitionData = (p, callback) => {
+        let stamp = p.processedCacheFile + '.stamp_partition';
+        fs.exists(stamp, (exists) => {
+            if (exists) return callback();
+
+            this.runBin('osrm-partition', util.format('%s %s', p.partitionArgs, p.processedCacheFile), p.environment, (err) => {
+                if (err) {
+                    return callback(new Error(util.format('osrm-partition %s: %s', errorReason(err), err.cmd)));
+                }
+                fs.writeFile(stamp, 'ok', callback);
+            });
+        });
+    };
+
+    this.customizeData = (p, callback) => {
+        let stamp = p.processedCacheFile + '.stamp_customize';
+        fs.exists(stamp, (exists) => {
+            if (exists) return callback();
+
+            this.runBin('osrm-customize', util.format('%s %s', p.customizeArgs, p.processedCacheFile), p.environment, (err) => {
+                if (err) {
+                    return callback(new Error(util.format('osrm-customize %s: %s', errorReason(err), err)));
+                }
+                fs.writeFile(stamp, 'ok', callback);
+            });
+        });
+    };
+
     this.extractAndContract = (callback) => {
         // a shallow copy of scenario parameters to avoid data inconsistency
         // if a cucumber timeout occurs during deferred jobs
         let p = {extractArgs: this.extractArgs, contractArgs: this.contractArgs,
+                 partitionArgs: this.partitionArgs, customizeArgs: this.customizeArgs,
                  profileFile: this.profileFile, inputCacheFile: this.inputCacheFile,
                  processedCacheFile: this.processedCacheFile, environment: this.environment};
         let queue = d3.queue(1);
@@ -218,6 +247,19 @@ module.exports = function () {
         queue.awaitAll(callback);
     };
 
+    this.extractPartitionAndCustomize = (callback) => {
+        // a shallow copy of scenario parameters to avoid data inconsistency
+        // if a cucumber timeout occurs during deferred jobs
+        let p = {extractArgs: this.extractArgs, partitionArgs: this.partitionArgs, customizeArgs: this.customizeArgs,
+                 profileFile: this.profileFile, inputCacheFile: this.inputCacheFile,
+                 processedCacheFile: this.processedCacheFile, environment: this.environment};
+        let queue = d3.queue(1);
+        queue.defer(this.extractData.bind(this), p);
+        queue.defer(this.partitionData.bind(this), p);
+        queue.defer(this.customizeData.bind(this), p);
+        queue.awaitAll(callback);
+    };
+
     this.writeAndLinkOSM = (callback) => {
         let queue = d3.queue(1);
         queue.defer(this.writeOSM.bind(this));
@@ -232,10 +274,19 @@ module.exports = function () {
         queue.awaitAll(callback);
     };
 
+    this.reprocessMLD = (callback) => {
+        let queue = d3.queue(1);
+        queue.defer(this.writeAndLinkOSM.bind(this));
+        queue.defer(this.extractPartitionAndCustomize.bind(this));
+        queue.awaitAll(callback);
+    };
+
     this.reprocessAndLoadData = (callback) => {
         let queue = d3.queue(1);
         queue.defer(this.writeAndLinkOSM.bind(this));
-        queue.defer(this.extractAndContract.bind(this));
+        queue.defer((this.ROUTING_ALGORITHM === 'MLD' ?
+                     this.extractPartitionAndCustomize :
+                     this.extractAndContract).bind(this));
         queue.defer(this.osrmLoader.load.bind(this.osrmLoader), this.processedCacheFile);
         queue.awaitAll(callback);
     };

features/support/env.js

@@ -34,12 +34,12 @@ module.exports = function () {
         this.DEFAULT_LOAD_METHOD = 'datastore';
         this.DEFAULT_ORIGIN = [1,1];
         this.OSM_USER = 'osrm';
-        this.OSM_GENERATOR = 'osrm-test';
         this.OSM_UID = 1;
         this.OSM_TIMESTAMP = '2000-01-01T00:00:00Z';
         this.WAY_SPACING = 100;
         this.DEFAULT_GRID_SIZE = 100; // meters
-        this.ROUTING_ALGORITHM = 'CH';
+        // get algorithm name from the command line profile argument
+        this.ROUTING_ALGORITHM = process.argv[process.argv.indexOf('-p') + 1] === 'mld' ? 'MLD' : 'CH';
 
         this.OSRM_PORT = process.env.OSRM_PORT && parseInt(process.env.OSRM_PORT) || 5000;
         this.HOST = 'http://127.0.0.1:' + this.OSRM_PORT;

features/support/hooks.js

@@ -35,6 +35,8 @@ module.exports = function () {
         this.queryParams = {};
         this.extractArgs = '';
         this.contractArgs = '';
+        this.partitionArgs = '';
+        this.customizeArgs = '';
         this.environment = Object.assign(this.DEFAULT_ENVIRONMENT);
         this.resetOSM();
 

features/testbot/loop.feature

@@ -4,6 +4,7 @@ Feature: Avoid weird loops caused by rounding errors
     Background:
         Given the profile "testbot"
 
+    @via
     Scenario: Weird sidestreet loops
         Given the node map
             """
@@ -72,7 +73,7 @@ Feature: Avoid weird loops caused by rounding errors
             | from | to | route   |
             | x    | y  | abc,abc |
 
-    @412
+    @412 @via
     Scenario: Avoid weird loops 3
         And the node map
             """

features/testbot/weight.feature

@@ -240,6 +240,7 @@ Feature: Weight tests
             | e,d       | ,,    | 40m +-.1 | 4.009,1.11,0 | 189.9s,100s,0s |
             | d,e       | ,,    | 40m +-.1 | 2.21,1.11,0  | 10.1s,100s,0s  |
 
+    @traffic @speed
     Scenario: Step weights -- segment_function with speed and turn updates
         Given the profile file "testbot" extended with
         """
@@ -286,6 +287,7 @@ Feature: Weight tests
             | a,e       | ,,    | 60.1m    | 68.7,10,0 | 38.5s,11s,0s |
             | d,e       | ,,    | 39.9m    | 10,10,0   | 11s,11s,0s   |
 
+    @traffic @speed
     Scenario: Step weights -- segment_function with speed and turn updates with fallback to durations
         Given the profile file "testbot" extended with
         """

package.json

@@ -25,7 +25,7 @@
   },
   "scripts": {
     "lint": "eslint -c ./.eslintrc features/step_definitions/ features/support/",
-    "test": "npm run lint && node ./node_modules/cucumber/bin/cucumber.js features/ -p verify",
+    "test": "npm run lint && node ./node_modules/cucumber/bin/cucumber.js features/ -p verify && node ./node_modules/cucumber/bin/cucumber.js features/ -p mld",
     "clean-test": "rm -rf test/cache",
     "cucumber": "./node_modules/cucumber/bin/cucumber.js",
     "build-api-docs": "./scripts/build_api_docs.sh"

src/tools/customize.cpp

@@ -107,7 +107,7 @@ parseArguments(int argc, char *argv[], customizer::CustomizationConfig &customiz
     if (!option_variables.count("input"))
     {
         std::cout << visible_options;
-        return return_code::exit;
+        return return_code::fail;
     }
 
     return return_code::ok;

src/tools/partition.cpp

@@ -116,7 +116,7 @@ return_code parseArguments(int argc, char *argv[], partition::PartitionConfig &p
     if (!option_variables.count("input"))
     {
         std::cout << visible_options;
-        return return_code::exit;
+        return return_code::fail;
     }
 
     return return_code::ok;