From 7d124ce54d55ddba12cc8af05b85377ac7da8000 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 19 Sep 2016 14:13:44 -0700 Subject: [PATCH] Close GH-2795: Rewrite cucumber test caching (and support logic). Fixes #2745 --- cucumber.js | 10 +- features/car/traffic_speeds.feature | 9 +- features/car/traffic_turn_penalties.feature | 4 +- features/guidance/anticipate-lanes.feature | 4 +- features/guidance/turn-lanes.feature | 2 +- features/lib/hash.js | 31 ++ features/{support/build_osm.js => lib/osm.js} | 8 +- features/lib/osrm_loader.js | 169 +++++++++++ features/lib/table_diff.js | 54 ++++ features/lib/try_connect.js | 13 + features/lib/utils.js | 17 ++ features/options/contract/datasources.feature | 10 +- features/options/contract/files.feature | 13 +- features/options/contract/help.feature | 8 +- features/options/contract/invalid.feature | 4 +- features/options/contract/version.feature | 4 +- features/options/extract/files.feature | 14 +- features/options/extract/help.feature | 6 +- features/options/extract/invalid.feature | 4 +- features/options/extract/version.feature | 4 +- features/options/routed/files.feature | 2 +- features/options/routed/help.feature | 6 +- features/options/routed/invalid.feature | 8 +- features/options/routed/version.feature | 4 +- features/raster/extract.feature | 19 +- features/raster/weights.feature | 12 +- features/step_definitions/data.js | 60 ++-- features/step_definitions/distance_matrix.js | 8 - features/step_definitions/hooks.js | 18 -- features/step_definitions/matching.js | 1 - features/step_definitions/nearest.js | 8 - features/step_definitions/options.js | 56 ++-- features/step_definitions/requests.js | 2 +- features/step_definitions/routability.js | 8 +- features/step_definitions/trip.js | 9 - features/support/cache.js | 184 ++++++++++++ features/support/config.js | 127 --------- features/support/data.js | 266 +++++------------- features/support/data_classes.js | 40 +-- features/support/env.js | 110 ++++---- features/support/exception_classes.js | 132 --------- features/support/exceptions.js | 15 - features/support/hash.js | 43 --- features/support/hooks.js | 77 +++-- features/support/http.js | 12 +- features/support/launch.js | 5 - features/support/launch_classes.js | 164 ----------- features/support/log.js | 90 ------ features/support/route.js | 7 +- features/support/run.js | 72 +++-- features/support/shared_steps.js | 22 +- features/support/table_patch.js | 11 - features/testbot/bad.feature | 4 +- features/testbot/bugs.feature | 5 - features/testbot/matching.feature | 2 +- .../testbot/traffic_turn_penalties.feature | 2 +- features/testbot/via.feature | 2 +- package.json | 2 + profiles/rasterbot.lua | 6 +- profiles/rasterbotinterp.lua | 6 +- 60 files changed, 871 insertions(+), 1144 deletions(-) create mode 100644 features/lib/hash.js rename features/{support/build_osm.js => lib/osm.js} (96%) create mode 100644 features/lib/osrm_loader.js create mode 100644 features/lib/table_diff.js create mode 100644 features/lib/try_connect.js create mode 100644 features/lib/utils.js delete mode 100644 features/step_definitions/hooks.js create mode 100644 features/support/cache.js delete mode 100644 features/support/config.js delete mode 100644 features/support/exception_classes.js delete mode 100644 features/support/exceptions.js delete mode 100644 features/support/hash.js delete mode 100644 features/support/launch.js delete mode 100644 features/support/launch_classes.js delete mode 100644 features/support/log.js delete mode 100644 features/support/table_patch.js delete mode 100644 features/testbot/bugs.feature diff --git a/cucumber.js b/cucumber.js index 5a87dd7f1..0bf6f570f 100644 --- a/cucumber.js +++ b/cucumber.js @@ -1,10 +1,8 @@ module.exports = { - default: '--require features --tags ~@stress --tags ~@todo', - verify: '--require features --tags ~@todo --tags ~@bug --tags ~@stress -f progress', - jenkins: '--require features --tags ~@todo --tags ~@bug --tags ~@stress --tags ~@options -f progress', - bugs: '--require features --tags @bug', - todo: '--require features --tags @todo', - all: '--require features' + default: '--strict --tags ~@stress --tags ~@todo --require features/support --require features/step_definitions', + verify: '--strict --tags ~@stress --tags ~@todo -f progress --require features/support --require features/step_definitions', + todo: '--strict --tags @todo --require features/support --require features/step_definitions', + all: '--strict --require features/support --require features/step_definitions' } diff --git a/features/car/traffic_speeds.feature b/features/car/traffic_speeds.feature index c2ad09288..c26c69599 100644 --- a/features/car/traffic_speeds.feature +++ b/features/car/traffic_speeds.feature @@ -25,7 +25,7 @@ Feature: Traffic - speeds | fb | primary | Given the profile "testbot" Given the extract extra arguments "--generate-edge-lookup" - Given the contract extra arguments "--segment-speed-file speeds.csv" + Given the contract extra arguments "--segment-speed-file {speeds_file}" Given the speed file """ 1,2,0 @@ -69,7 +69,7 @@ Feature: Traffic - speeds | fb | primary | Given the profile "testbot" Given the extract extra arguments "--generate-edge-lookup" - Given the contract extra arguments "--segment-speed-file speeds.csv" + Given the contract extra arguments "--segment-speed-file {speeds_file}" Given the speed file """ 1,2,0 @@ -112,7 +112,6 @@ Feature: Traffic - speeds | fb | primary | Given the profile "testbot" Given the extract extra arguments "--generate-edge-lookup" - Given the contract extra arguments "--segment-speed-file speeds.csv" Given the speed file """ 1,2,-10 @@ -123,6 +122,6 @@ Feature: Traffic - speeds 4,1,-5 """ And the data has been extracted - When I run "osrm-contract --segment-speed-file speeds.csv {extracted_base}.osrm" + When I try to run "osrm-contract --segment-speed-file {speeds_file} {processed_file}" And stderr should contain "malformed" - And it should exit with code not 0 + And it should exit with an error diff --git a/features/car/traffic_turn_penalties.feature b/features/car/traffic_turn_penalties.feature index 09f5323b1..cb3907a5e 100644 --- a/features/car/traffic_turn_penalties.feature +++ b/features/car/traffic_turn_penalties.feature @@ -58,7 +58,7 @@ Feature: Traffic - turn penalties 8,11,12,23 1,4,5,-0.2 """ - And the contract extra arguments "--turn-penalty-file penalties.csv" + And the contract extra arguments "--turn-penalty-file {penalties_file}" When I route I should get | from | to | route | speed | time | | a | h | ad,dhk,dhk | 63 km/h | 11.5s +-1 | @@ -81,7 +81,7 @@ Feature: Traffic - turn penalties # double left - hdc penalty ever so slightly higher than imn; forces all the way around Scenario: Too-negative penalty clamps, but does not fail - Given the contract extra arguments "--turn-penalty-file penalties.csv" + Given the contract extra arguments "--turn-penalty-file {penalties_file}" And the profile "testbot" And the turn penalty file """ diff --git a/features/guidance/anticipate-lanes.feature b/features/guidance/anticipate-lanes.feature index f180b7a33..e235f2fba 100644 --- a/features/guidance/anticipate-lanes.feature +++ b/features/guidance/anticipate-lanes.feature @@ -422,7 +422,7 @@ Feature: Turn Lane Guidance | waypoints | route | turns | lanes | | a,e | main,main,main,main | depart,use lane straight,continue right,arrive | ,left:false straight:false straight:false straight:false straight:true straight:true right:false,straight:false straight:false right:false right:true right:true, | - @anticipate @todo @bug @2661 + @anticipate @todo @2661 Scenario: Anticipate with lanes in roundabout: roundabouts as the unit of anticipation Given the node map | | | e | | | @@ -667,7 +667,7 @@ Feature: Turn Lane Guidance | a,f | abc,bdeh,feg,feg | depart,turn right,turn right,arrive | ,none:false none:false right:false right:true,left:false none:false none:false right:true, | @anticipate - Scenario: Tripple Right keeping Left + Scenario: Triple Right keeping Left Given the node map | a | | | | b | | i | | | | | | | | | diff --git a/features/guidance/turn-lanes.feature b/features/guidance/turn-lanes.feature index e1d493b49..ff6c750c3 100644 --- a/features/guidance/turn-lanes.feature +++ b/features/guidance/turn-lanes.feature @@ -620,7 +620,7 @@ Feature: Turn Lane Guidance | a,d | hwy,hwy | depart,arrive | , | | a,e | hwy,ramp,ramp | depart,off ramp slight right,arrive | ,straight:false straight:false straight;slight right:true slight right:true, | - @bug @todo + @todo Scenario: Turning Off Ramp Given the node map | | a | | diff --git a/features/lib/hash.js b/features/lib/hash.js new file mode 100644 index 000000000..57563e3da --- /dev/null +++ b/features/lib/hash.js @@ -0,0 +1,31 @@ +'use strict'; + +const fs = require('fs'); +const crypto = require('crypto'); +const d3 = require('d3-queue'); + +module.exports = { + hashOfFiles: (paths, cb) => { + let queue = d3.queue(); + for (let i = 0; i < paths.length; ++i) { + queue.defer(fs.readFile, paths[i]); + } + queue.awaitAll((err, results) => { + if (err) return cb(err); + let checksum = crypto.createHash('md5'); + for (let i = 0; i < results.length; ++i) { + checksum.update(results[i]); + } + cb(null, checksum.digest('hex')); + }); + }, + + hashOfFile: (path, cb) => { + fs.readFile(path, (err, result) => { + if (err) return cb(err); + let checksum = crypto.createHash('md5'); + checksum.update(result); + cb(null, checksum.digest('hex')); + }); + } +}; diff --git a/features/support/build_osm.js b/features/lib/osm.js similarity index 96% rename from features/support/build_osm.js rename to features/lib/osm.js index 7fe6874ae..5387b7f01 100644 --- a/features/support/build_osm.js +++ b/features/lib/osm.js @@ -1,11 +1,7 @@ 'use strict'; -var builder = require('xmlbuilder'); - -var ensureDecimal = (i) => { - if (parseInt(i) === i) return i.toFixed(1); - else return i; -}; +const builder = require('xmlbuilder'); +const ensureDecimal = require('./utils').ensureDecimal; class DB { constructor () { diff --git a/features/lib/osrm_loader.js b/features/lib/osrm_loader.js new file mode 100644 index 000000000..28ec7ed2b --- /dev/null +++ b/features/lib/osrm_loader.js @@ -0,0 +1,169 @@ +'use strict'; + +const fs = require('fs'); +const util = require('util'); +const Timeout = require('node-timeout'); +const tryConnect = require('../lib/try_connect'); +const errorReason = require('./utils').errorReason; + +class OSRMBaseLoader{ + constructor (scope) { + this.scope = scope; + this.child = null; + } + + launch (callback) { + var limit = Timeout(this.scope.TIMEOUT, { err: new Error('*** Launching osrm-routed timed out.') }); + + var runLaunch = (cb) => { + this.osrmUp(() => { this.waitForConnection(cb); }); + }; + + runLaunch(limit((e) => { if (e) callback(e); else callback(); })); + } + + shutdown (callback) { + if (!this.osrmIsRunning()) return callback(); + + var limit = Timeout(this.scope.TIMEOUT, { err: new Error('*** Shutting down osrm-routed timed out.')}); + + this.osrmDown(limit(callback)); + } + + osrmIsRunning () { + return this.child && !this.child.killed; + } + + osrmDown (callback) { + if (this.osrmIsRunning()) { + this.child.on('exit', (code, signal) => {callback();}); + this.child.kill(); + } else callback(); + } + + waitForConnection (callback) { + var retryCount = 0; + let retry = (err) => { + if (err) { + if (retryCount < 10) { + retryCount++; + setTimeout(() => { tryConnect(this.scope.OSRM_PORT, retry); }, 10); + } else { + callback(new Error("Could not connect to osrm-routed after ten retries.")); + } + } + else + { + callback(); + } + }; + + tryConnect(this.scope.OSRM_PORT, retry); + } +}; + +class OSRMDirectLoader extends OSRMBaseLoader { + constructor (scope) { + super(scope); + } + + load (inputFile, callback) { + this.inputFile = inputFile; + this.shutdown(() => { + this.launch(callback); + }); + } + + osrmUp (callback) { + if (this.osrmIsRunning()) return callback(new Error("osrm-routed already running!")); + + this.child = this.scope.runBin('osrm-routed', util.format("%s -p %d", this.inputFile, this.scope.OSRM_PORT), this.scope.environment, (err) => { + if (err) { + throw new Error(util.format('osrm-routed %s: %s', errorReason(err), err.cmd)); + } + }); + callback(); + } +}; + +class OSRMDatastoreLoader extends OSRMBaseLoader { + constructor (scope) { + super(scope); + } + + load (inputFile, callback) { + this.inputFile = inputFile; + + this.loadData((err) => { + if (err) return callback(err); + if (!this.osrmIsRunning()) this.launch(callback); + else { + this.scope.setupOutputLog(this.child, fs.createWriteStream(this.scope.scenarioLogFile, {'flags': 'a'})); + callback(); + } + }); + } + + loadData (callback) { + this.scope.runBin('osrm-datastore', this.inputFile, this.scope.environment, (err) => { + if (err) return callback(new Error('*** osrm-datastore exited with ' + err.code + ': ' + err)); + callback(); + }); + } + + osrmUp (callback) { + if (this.osrmIsRunning()) return callback(); + + this.child = this.scope.runBin('osrm-routed', util.format('--shared-memory=1 -p %d', this.scope.OSRM_PORT), this.scope.environment, (err) => { + if (err) { + throw new Error(util.format('osrm-routed %s: %s', errorReason(err), err.cmd)); + } + }); + + // we call the callback here, becuase we don't want to wait for the child process to finish + callback(); + } +}; + +class OSRMLoader { + constructor (scope) { + this.scope = scope; + this.sharedLoader = new OSRMDatastoreLoader(this.scope); + this.directLoader = new OSRMDirectLoader(this.scope); + this.method = scope.DEFAULT_LOAD_METHOD; + } + + load (inputFile, callback) { + if (this.method === 'datastore') { + this.directLoader.shutdown((err) => { + if (err) return callback(err); + this.loader = this.sharedLoader; + this.sharedLoader.load(inputFile, callback); + }); + } else if (this.method === 'directly') { + this.sharedLoader.shutdown((err) => { + if (err) return callback(err); + this.loader = this.directLoader; + this.directLoader.load(inputFile, callback); + }); + } else { + callback(new Error('*** Unknown load method ' + method)); + } + } + + setLoadMethod (method) { + this.method = method; + } + + shutdown (callback) { + if (!this.loader) return callback(); + + this.loader.shutdown(callback); + } + + up () { + return this.loader ? this.loader.osrmIsRunning() : false; + } +}; + +module.exports = OSRMLoader; diff --git a/features/lib/table_diff.js b/features/lib/table_diff.js new file mode 100644 index 000000000..4acbd23c1 --- /dev/null +++ b/features/lib/table_diff.js @@ -0,0 +1,54 @@ +'use strict'; + +var util = require('util'); +var path = require('path'); +var fs = require('fs'); +var chalk = require('chalk'); + +var unescapeStr = (str) => str.replace(/\\\|/g, '\|').replace(/\\\\/g, '\\'); + +module.exports = function (expected, actual) { + let headers = expected.raw()[0]; + let expected_keys = expected.hashes(); + let diff = []; + let hasErrors = false; + + var good = 0, bad = 0; + + expected_keys.forEach((row, i) => { + var rowError = false; + + for (var j in row) { + if (unescapeStr(row[j]) != actual[i][j]) { + rowError = true; + hasErrors = true; + break; + } + } + + if (rowError) { + bad++; + diff.push(Object.assign({}, row, {c_status: 'undefined'})); + diff.push(Object.assign({}, actual[i], {c_status: 'comment'})); + } else { + good++; + diff.push(row); + } + }); + + if (!hasErrors) return null; + + var s = ['Tables were not identical:']; + s.push(headers.map(key => ' ' + key).join(' | ')); + diff.forEach((row) => { + var rowString = '| '; + headers.forEach((header) => { + if (!row.c_status) rowString += chalk.green(' ' + row[header] + ' | '); + else if (row.c_status === 'undefined') rowString += chalk.yellow('(-) ' + row[header] + ' | '); + else rowString += chalk.red('(+) ' + row[header] + ' | '); + }); + s.push(rowString); + }); + + return s.join('\n') + '\nTODO this is a temp workaround waiting for https://github.com/cucumber/cucumber-js/issues/534'; +}; diff --git a/features/lib/try_connect.js b/features/lib/try_connect.js new file mode 100644 index 000000000..0461dddb8 --- /dev/null +++ b/features/lib/try_connect.js @@ -0,0 +1,13 @@ +'use strict'; + +const net = require('net'); +const Timeout = require('node-timeout'); + +module.exports = function tryConnect(port, callback) { + net.connect({ port: port, host: '127.0.0.1' }) + .on('connect', () => { callback(); }) + .on('error', () => { + callback(new Error('Could not connect.')); + }); +} + diff --git a/features/lib/utils.js b/features/lib/utils.js new file mode 100644 index 000000000..27b63af6c --- /dev/null +++ b/features/lib/utils.js @@ -0,0 +1,17 @@ +'use strict'; + +const util = require('util'); + +module.exports = { + + ensureDecimal: (i) => { + if (parseInt(i) === i) return i.toFixed(1); + else return i; + }, + + errorReason: (err) => { + return err.signal ? + util.format('killed by signal %s', err.signal) : + util.format('exited with code %d', err.code); + } +}; diff --git a/features/options/contract/datasources.feature b/features/options/contract/datasources.feature index bf8eed5d5..19913885f 100644 --- a/features/options/contract/datasources.feature +++ b/features/options/contract/datasources.feature @@ -1,8 +1,7 @@ @prepare @options @files Feature: osrm-contract command line options: datasources # expansions: -# {extracted_base} => path to current extracted input file -# {profile} => path to current profile script +# {processed_file} => path to .osrm file Background: Given the profile "testbot" @@ -24,7 +23,6 @@ Feature: osrm-contract command line options: datasources And the data has been extracted Scenario: osrm-contract - Passing base file - When I run "osrm-contract --segment-speed-file speeds.csv {extracted_base}.osrm" - Then stderr should be empty - And datasource names should contain "lua profile,speeds" - And it should exit with code 0 + When I run "osrm-contract --segment-speed-file {speeds_file} {processed_file}" + Then datasource names should contain "lua profile,25_osrmcontract_passing_base_file_speeds" + And it should exit successfully diff --git a/features/options/contract/files.feature b/features/options/contract/files.feature index 5e50e14f2..46cf4fba0 100644 --- a/features/options/contract/files.feature +++ b/features/options/contract/files.feature @@ -1,9 +1,5 @@ @prepare @options @files Feature: osrm-contract command line options: files -# expansions: -# {extracted_base} => path to current extracted input file -# {profile} => path to current profile script - Background: Given the profile "testbot" And the node map @@ -14,12 +10,11 @@ Feature: osrm-contract command line options: files And the data has been extracted Scenario: osrm-contract - Passing base file - When I run "osrm-contract {extracted_base}.osrm" - Then stderr should be empty - And it should exit with code 0 + When I run "osrm-contract {processed_file}" + Then it should exit successfully Scenario: osrm-contract - Missing input file - When I run "osrm-contract over-the-rainbow.osrm" + When I try to run "osrm-contract over-the-rainbow.osrm" And stderr should contain "over-the-rainbow.osrm" And stderr should contain "not found" - And it should exit with code 1 + And it should exit with an error diff --git a/features/options/contract/help.feature b/features/options/contract/help.feature index 411bc55da..b4d81c557 100644 --- a/features/options/contract/help.feature +++ b/features/options/contract/help.feature @@ -2,7 +2,7 @@ Feature: osrm-contract command line options: help Scenario: osrm-contract - Help should be shown when no options are passed - When I run "osrm-contract" + When I try to run "osrm-contract" Then stderr should be empty And stdout should contain "osrm-contract [options]:" And stdout should contain "Options:" @@ -13,7 +13,7 @@ Feature: osrm-contract command line options: help And stdout should contain "--core" And stdout should contain "--level-cache" And stdout should contain "--segment-speed-file" - And it should exit with code 1 + And it should exit with an error Scenario: osrm-contract - Help, short When I run "osrm-contract -h" @@ -27,7 +27,7 @@ Feature: osrm-contract command line options: help And stdout should contain "--core" And stdout should contain "--level-cache" And stdout should contain "--segment-speed-file" - And it should exit with code 0 + And it should exit successfully Scenario: osrm-contract - Help, long When I run "osrm-contract --help" @@ -41,4 +41,4 @@ Feature: osrm-contract command line options: help And stdout should contain "--core" And stdout should contain "--level-cache" And stdout should contain "--segment-speed-file" - And it should exit with code 0 + And it should exit successfully diff --git a/features/options/contract/invalid.feature b/features/options/contract/invalid.feature index 38ee3ace9..127761ee3 100644 --- a/features/options/contract/invalid.feature +++ b/features/options/contract/invalid.feature @@ -5,8 +5,8 @@ Feature: osrm-contract command line options: invalid options Given the profile "testbot" Scenario: osrm-contract - Non-existing option - When I run "osrm-contract --fly-me-to-the-moon" + When I try to run "osrm-contract --fly-me-to-the-moon" Then stdout should be empty And stderr should contain "option" And stderr should contain "fly-me-to-the-moon" - And it should exit with code 1 + And it should exit with an error diff --git a/features/options/contract/version.feature b/features/options/contract/version.feature index be99bbed1..f361bb1e5 100644 --- a/features/options/contract/version.feature +++ b/features/options/contract/version.feature @@ -12,11 +12,11 @@ Feature: osrm-contract command line options: version Then stderr should be empty And stdout should contain 1 line And stdout should contain /(v\d{1,2}\.\d{1,2}\.\d{1,2}|\w*-\d+-\w+)/ - And it should exit with code 0 + And it should exit successfully Scenario: osrm-contract - Version, long When I run "osrm-contract --version" Then stderr should be empty And stdout should contain 1 line And stdout should contain /(v\d{1,2}\.\d{1,2}\.\d{1,2}|\w*-\d+-\w+)/ - And it should exit with code 0 + And it should exit successfully diff --git a/features/options/extract/files.feature b/features/options/extract/files.feature index aceab19f7..c4e14a278 100644 --- a/features/options/extract/files.feature +++ b/features/options/extract/files.feature @@ -14,17 +14,15 @@ Feature: osrm-extract command line options: files And the data has been saved to disk Scenario: osrm-extract - Passing base file - When I run "osrm-extract {osm_base}.osm --profile {profile}" - Then stderr should be empty - And it should exit with code 0 + When I run "osrm-extract {osm_file} --profile {profile_file}" + Then it should exit successfully Scenario: osrm-extract - Order of options should not matter - When I run "osrm-extract --profile {profile} {osm_base}.osm" - Then stderr should be empty - And it should exit with code 0 + When I run "osrm-extract --profile {profile_file} {osm_file}" + Then it should exit successfully Scenario: osrm-extract - Missing input file - When I run "osrm-extract over-the-rainbow.osrm --profile {profile}" + When I try to run "osrm-extract over-the-rainbow.osrm --profile {profile_file}" And stderr should contain "over-the-rainbow.osrm" And stderr should contain "not found" - And it should exit with code 1 + And it should exit with an error diff --git a/features/options/extract/help.feature b/features/options/extract/help.feature index cdf1eb9a3..0d400edba 100644 --- a/features/options/extract/help.feature +++ b/features/options/extract/help.feature @@ -16,7 +16,7 @@ Feature: osrm-extract command line options: help And stdout should contain "--threads" And stdout should contain "--generate-edge-lookup" And stdout should contain "--small-component-size" - And it should exit with code 0 + And it should exit successfully Scenario: osrm-extract - Help, short When I run "osrm-extract -h" @@ -30,7 +30,7 @@ Feature: osrm-extract command line options: help And stdout should contain "--threads" And stdout should contain "--generate-edge-lookup" And stdout should contain "--small-component-size" - And it should exit with code 0 + And it should exit successfully Scenario: osrm-extract - Help, long When I run "osrm-extract --help" @@ -44,4 +44,4 @@ Feature: osrm-extract command line options: help And stdout should contain "--threads" And stdout should contain "--generate-edge-lookup" And stdout should contain "--small-component-size" - And it should exit with code 0 + And it should exit successfully diff --git a/features/options/extract/invalid.feature b/features/options/extract/invalid.feature index 169e53caa..936f456fb 100644 --- a/features/options/extract/invalid.feature +++ b/features/options/extract/invalid.feature @@ -5,8 +5,8 @@ Feature: osrm-extract command line options: invalid options Given the profile "testbot" Scenario: osrm-extract - Non-existing option - When I run "osrm-extract --fly-me-to-the-moon" + When I try to run "osrm-extract --fly-me-to-the-moon" Then stdout should be empty And stderr should contain "option" And stderr should contain "fly-me-to-the-moon" - And it should exit with code 1 + And it should exit with an error diff --git a/features/options/extract/version.feature b/features/options/extract/version.feature index 0dd5f6588..77ee46cd8 100644 --- a/features/options/extract/version.feature +++ b/features/options/extract/version.feature @@ -12,11 +12,11 @@ Feature: osrm-extract command line options: version Then stderr should be empty And stdout should contain 1 line And stdout should contain /(v\d{1,2}\.\d{1,2}\.\d{1,2}|\w*-\d+-\w+)/ - And it should exit with code 0 + And it should exit successfully Scenario: osrm-extract - Version, long When I run "osrm-extract --version" Then stderr should be empty And stdout should contain 1 line And stdout should contain /(v\d{1,2}\.\d{1,2}\.\d{1,2}|\w*-\d+-\w+)/ - And it should exit with code 0 + And it should exit successfully diff --git a/features/options/routed/files.feature b/features/options/routed/files.feature index 59ce7c213..b28c8b11a 100644 --- a/features/options/routed/files.feature +++ b/features/options/routed/files.feature @@ -29,4 +29,4 @@ Feature: osrm-routed command line options: files And stdout should contain /^\[info\] loaded plugin: viaroute/ And stdout should contain /^\[info\] trial run/ And stdout should contain /^\[info\] shutdown completed/ - And it should exit with code 0 + And it should exit successfully diff --git a/features/options/routed/help.feature b/features/options/routed/help.feature index 8f64bd967..e8c6430f6 100644 --- a/features/options/routed/help.feature +++ b/features/options/routed/help.feature @@ -21,7 +21,7 @@ Feature: osrm-routed command line options: help And stdout should contain "--max-trip-size" And stdout should contain "--max-table-size" And stdout should contain "--max-matching-size" - And it should exit with code 0 + And it should exit successfully Scenario: osrm-routed - Help, short When I run "osrm-routed -h" @@ -40,7 +40,7 @@ Feature: osrm-routed command line options: help And stdout should contain "--max-trip-size" And stdout should contain "--max-table-size" And stdout should contain "--max-matching-size" - And it should exit with code 0 + And it should exit successfully Scenario: osrm-routed - Help, long When I run "osrm-routed --help" @@ -59,4 +59,4 @@ Feature: osrm-routed command line options: help And stdout should contain "--max-table-size" And stdout should contain "--max-table-size" And stdout should contain "--max-matching-size" - And it should exit with code 0 + And it should exit successfully diff --git a/features/options/routed/invalid.feature b/features/options/routed/invalid.feature index 9c8435784..78d28a064 100644 --- a/features/options/routed/invalid.feature +++ b/features/options/routed/invalid.feature @@ -5,14 +5,14 @@ Feature: osrm-routed command line options: invalid options Given the profile "testbot" Scenario: osrm-routed - Non-existing option - When I run "osrm-routed --fly-me-to-the-moon" + When I try to run "osrm-routed --fly-me-to-the-moon" Then stdout should be empty And stderr should contain "unrecognised" And stderr should contain "fly-me-to-the-moon" - And it should exit with code 1 + And it should exit with an error Scenario: osrm-routed - Missing file - When I run "osrm-routed over-the-rainbow.osrm" + When I try to run "osrm-routed over-the-rainbow.osrm" Then stderr should contain "over-the-rainbow.osrm" And stderr should contain "not found" - And it should exit with code 1 + And it should exit with an error diff --git a/features/options/routed/version.feature b/features/options/routed/version.feature index b544e36e6..0a3cad55e 100644 --- a/features/options/routed/version.feature +++ b/features/options/routed/version.feature @@ -12,11 +12,11 @@ Feature: osrm-routed command line options: version Then stderr should be empty And stdout should contain 1 line And stdout should contain /(v\d{1,2}\.\d{1,2}\.\d{1,2}|\w*-\d+-\w+)/ - And it should exit with code 0 + And it should exit successfully Scenario: osrm-routed - Version, long When I run "osrm-routed --version" Then stderr should be empty And stdout should contain 1 line And stdout should contain /(v\d{1,2}\.\d{1,2}\.\d{1,2}|\w*-\d+-\w+)/ - And it should exit with code 0 + And it should exit successfully diff --git a/features/raster/extract.feature b/features/raster/extract.feature index 9ca0635d8..86a753716 100644 --- a/features/raster/extract.feature +++ b/features/raster/extract.feature @@ -1,9 +1,5 @@ @raster @extract Feature: osrm-extract with a profile containing raster source -# expansions: -# {osm_base} => path to current input file -# {profile} => path to current profile script - Scenario: osrm-extract on a valid profile Given the profile "rasterbot" And the node map @@ -11,8 +7,15 @@ Feature: osrm-extract with a profile containing raster source And the ways | nodes | | ab | + And the raster source + """ + 0 0 0 0 + 0 0 0 250 + 0 0 250 500 + 0 0 0 250 + 0 0 0 0 + """ And the data has been saved to disk - When I run "osrm-extract {osm_base}.osm -p {profile}" - Then stderr should be empty - And stdout should contain "source loader" - And it should exit with code 0 + When I run "osrm-extract {osm_file} -p {profile_file}" + Then stdout should contain "source loader" + And it should exit successfully diff --git a/features/raster/weights.feature b/features/raster/weights.feature index ae782a722..1c03bdc1b 100644 --- a/features/raster/weights.feature +++ b/features/raster/weights.feature @@ -32,8 +32,8 @@ Feature: Raster - weights Scenario: Weighting not based on raster sources Given the profile "testbot" - When I run "osrm-extract {osm_base}.osm -p {profile}" - And I run "osrm-contract {osm_base}.osm" + When I run "osrm-extract {osm_file} -p {profile_file}" + And I run "osrm-contract {processed_file}" And I route I should get | from | to | route | speed | | a | b | ab,ab | 36 km/h | @@ -44,9 +44,9 @@ Feature: Raster - weights Scenario: Weighting based on raster sources Given the profile "rasterbot" - When I run "osrm-extract {osm_base}.osm -p {profile}" + When I run "osrm-extract {osm_file} -p {profile_file}" Then stdout should contain "evaluating segment" - And I run "osrm-contract {osm_base}.osm" + And I run "osrm-contract {processed_file}" And I route I should get | from | to | route | speed | | a | b | ab,ab | 8 km/h | @@ -62,9 +62,9 @@ Feature: Raster - weights Scenario: Weighting based on raster sources Given the profile "rasterbotinterp" - When I run "osrm-extract {osm_base}.osm -p {profile}" + When I run "osrm-extract {osm_file} -p {profile_file}" Then stdout should contain "evaluating segment" - And I run "osrm-contract {osm_base}.osm" + And I run "osrm-contract {processed_file}" And I route I should get | from | to | route | speed | | a | b | ab,ab | 8 km/h | diff --git a/features/step_definitions/data.js b/features/step_definitions/data.js index 58a3fc5ff..dbb3882d1 100644 --- a/features/step_definitions/data.js +++ b/features/step_definitions/data.js @@ -2,19 +2,23 @@ var util = require('util'); var path = require('path'); var fs = require('fs'); var d3 = require('d3-queue'); -var OSM = require('../support/build_osm'); +var OSM = require('../lib/osm'); module.exports = function () { this.Given(/^the profile "([^"]*)"$/, (profile, callback) => { - this.setProfile(profile, callback); + this.profile = profile; + this.profileFile = path.join(this.PROFILES_PATH, this.profile + '.lua'); + callback(); }); this.Given(/^the extract extra arguments "(.*?)"$/, (args, callback) => { - this.setExtractArgs(args, callback); + this.extractArgs = this.expandOptions(args); + callback(); }); this.Given(/^the contract extra arguments "(.*?)"$/, (args, callback) => { - this.setContractArgs(args, callback); + this.contractArgs = this.expandOptions(args); + callback(); }); this.Given(/^a grid size of ([0-9.]+) meters$/, (meters, callback) => { @@ -228,58 +232,46 @@ module.exports = function () { }); this.Given(/^the raster source$/, (data, callback) => { - this.updateFingerprintExtract(data); - fs.writeFile(path.resolve(this.TEST_FOLDER, 'rastersource.asc'), data, callback); + // TODO: Don't overwrite if it exists + fs.writeFile(this.rasterCacheFile, data, callback); + // we need this to pass it to the profiles + this.environment = Object.assign({OSRM_RASTER_SOURCE: this.rasterCacheFile}, this.environment); }); this.Given(/^the speed file$/, (data, callback) => { - this.updateFingerprintContract(data); - fs.writeFile(path.resolve(this.TEST_FOLDER, 'speeds.csv'), data, callback); + // TODO: Don't overwrite if it exists + fs.writeFile(this.speedsCacheFile, data, callback); }); this.Given(/^the turn penalty file$/, (data, callback) => { - this.updateFingerprintContract(data); - fs.writeFile(path.resolve(this.TEST_FOLDER, 'penalties.csv'), data, callback); + // TODO: Don't overwrite if it exists + fs.writeFile(this.penaltiesCacheFile, data, callback); }); this.Given(/^the data has been saved to disk$/, (callback) => { - try { - this.reprocess(callback); - } catch(e) { - this.processError = e; - callback(e); - } + this.reprocess(callback); }); this.Given(/^the data has been extracted$/, (callback) => { - this.osmData.populate(() => { - this.writeAndExtract((err) => { - if (err) this.processError = err; - callback(); - }); - }); + this.reprocess(callback); }); this.Given(/^the data has been contracted$/, (callback) => { - this.reprocess((err) => { - if (err) this.processError = err; - callback(); - }); + this.reprocess(callback); }); this.Given(/^osrm\-routed is stopped$/, (callback) => { - this.OSRMLoader.shutdown((err) => { - if (err) this.processError = err; - callback(); - }); + this.OSRMLoader.shutdown(callback); }); - this.Given(/^data is loaded directly/, () => { - this.loadMethod = 'directly'; + this.Given(/^data is loaded directly/, (callback) => { + this.osrmLoader.setLoadMethod('directly'); + callback(); }); - this.Given(/^data is loaded with datastore$/, () => { - this.loadMethod = 'datastore'; + this.Given(/^data is loaded with datastore$/, (callback) => { + this.osrmLoader.setLoadMethod('datastore'); + callback(); }); this.Given(/^the HTTP method "([^"]*)"$/, (method, callback) => { diff --git a/features/step_definitions/distance_matrix.js b/features/step_definitions/distance_matrix.js index f032f08e9..9c2bc0add 100644 --- a/features/step_definitions/distance_matrix.js +++ b/features/step_definitions/distance_matrix.js @@ -53,8 +53,6 @@ module.exports = function () { }); var testRow = (row, ri, cb) => { - var ok = true; - for (var k in result[ri]) { if (this.FuzzyMatch.match(result[ri][k], row[k])) { result[ri][k] = row[k]; @@ -62,15 +60,9 @@ module.exports = function () { result[ri][k] = ''; } else { result[ri][k] = result[ri][k].toString(); - ok = false; } } - if (!ok) { - var failed = { attempt: 'distance_matrix', query: this.query, response: response }; - this.logFail(row, result[ri], [failed]); - } - result[ri][''] = row['']; cb(null, result[ri]); }; diff --git a/features/step_definitions/hooks.js b/features/step_definitions/hooks.js deleted file mode 100644 index d6ed251b4..000000000 --- a/features/step_definitions/hooks.js +++ /dev/null @@ -1,18 +0,0 @@ -var util = require('util'); - -module.exports = function () { - this.Before((scenario, callback) => { - this.scenarioTitle = scenario.getName(); - - this.loadMethod = this.DEFAULT_LOAD_METHOD; - this.queryParams = {}; - var d = new Date(); - this.scenarioTime = util.format('%d-%d-%dT%s:%s:%sZ', d.getFullYear(), d.getMonth()+1, d.getDate(), d.getHours(), d.getMinutes(), d.getSeconds()); - this.resetData(); - this.hasLoggedPreprocessInfo = false; - this.hasLoggedScenarioInfo = false; - this.setGridSize(this.DEFAULT_GRID_SIZE); - this.setOrigin(this.DEFAULT_ORIGIN); - callback(); - }); -}; diff --git a/features/step_definitions/matching.js b/features/step_definitions/matching.js index e3dc4e6e3..41cd0ea37 100644 --- a/features/step_definitions/matching.js +++ b/features/step_definitions/matching.js @@ -157,7 +157,6 @@ module.exports = function () { } else { got.matchings = encodedResult; row.matchings = extendedTarget; - this.logFail(row, got, { matching: { query: this.query, response: res } }); } cb(null, got); diff --git a/features/step_definitions/nearest.js b/features/step_definitions/nearest.js index 919cb857d..450dce20a 100644 --- a/features/step_definitions/nearest.js +++ b/features/step_definitions/nearest.js @@ -22,24 +22,16 @@ module.exports = function () { var got = { in: row.in, out: row.out }; - var ok = true; - Object.keys(row).forEach((key) => { if (key === 'out') { if (this.FuzzyMatch.matchLocation(coord, outNode)) { got[key] = row[key]; } else { row[key] = util.format('%s [%d,%d]', row[key], outNode.lat, outNode.lon); - ok = false; } } }); - if (!ok) { - var failed = { attempt: 'nearest', query: this.query, response: response }; - this.logFail(row, got, [failed]); - } - cb(null, got); } else { diff --git a/features/step_definitions/options.js b/features/step_definitions/options.js index 5dc692992..d2177bffd 100644 --- a/features/step_definitions/options.js +++ b/features/step_definitions/options.js @@ -2,36 +2,58 @@ var assert = require('assert'); var fs = require('fs'); module.exports = function () { - this.When(/^I run "osrm\-routed\s?(.*?)"$/, { timeout: this.TIMEOUT }, (options, callback) => { - this.runBin('osrm-routed', options, () => { - callback(); + this.resetOptionsOutput = () => { + this.stdout = null; + this.stderr = null; + this.exitCode = null; + this.termSignal = null; + }; + + this.runAndSafeOutput = (binary, options, callback) => { + this.runBin(binary, this.expandOptions(options), this.environment, (err, stdout, stderr) => { + this.stdout = stdout; + this.stderr = stderr; + this.exitCode = err && err.code || 0; + this.termSignal = err && err.signal || ''; + callback(err); }); + }; + + this.When(/^I run "osrm\-routed\s?(.*?)"$/, { timeout: this.TIMEOUT }, (options, callback) => { + this.runAndSafeOutput('osrm-routed', options, callback); }); this.When(/^I run "osrm\-extract\s?(.*?)"$/, (options, callback) => { - this.runBin('osrm-extract', options, () => { - callback(); - }); + this.runAndSafeOutput('osrm-extract', options, callback); }); this.When(/^I run "osrm\-contract\s?(.*?)"$/, (options, callback) => { - this.runBin('osrm-contract', options, () => { - callback(); - }); + this.runAndSafeOutput('osrm-contract', options, callback); + }); + + this.When(/^I try to run "osrm\-routed\s?(.*?)"$/, (options, callback) => { + this.runAndSafeOutput('osrm-routed', options, () => { callback(); }); + }); + + this.When(/^I try to run "osrm\-extract\s?(.*?)"$/, (options, callback) => { + this.runAndSafeOutput('osrm-extract', options, () => { callback(); }); + }); + + this.When(/^I try to run "osrm\-contract\s?(.*?)"$/, (options, callback) => { + this.runAndSafeOutput('osrm-contract', options, () => { callback(); }); }); this.When(/^I run "osrm\-datastore\s?(.*?)"$/, (options, callback) => { - this.runBin('osrm-datastore', options, () => { - callback(); - }); + this.runAndSafeOutput('osrm-datastore', options, callback); }); - this.Then(/^it should exit with code (\d+)$/, (code) => { - assert.equal(this.exitCode, parseInt(code)); + this.Then(/^it should exit successfully$/, () => { + assert.equal(this.exitCode, 0); + assert.equal(this.termSignal, ''); }); - this.Then(/^it should exit with code not (\d+)$/, (code) => { - assert.notEqual(this.exitCode, parseInt(code)); + this.Then(/^it should exit with an error$/, () => { + assert.ok(this.exitCode !== 0 || this.termSignal); }); this.Then(/^stdout should contain "(.*?)"$/, (str) => { @@ -65,7 +87,7 @@ module.exports = function () { }); this.Then(/^datasource names should contain "(.+)"$/, (expectedData) => { - var actualData = fs.readFileSync(this.osmData.extractedFile + '.osrm.datasource_names', {encoding:'UTF-8'}).trim().split('\n').join(','); + var actualData = fs.readFileSync(this.processedCacheFile + '.datasource_names', {encoding:'UTF-8'}).trim().split('\n').join(','); assert.equal(actualData, expectedData); }); diff --git a/features/step_definitions/requests.js b/features/step_definitions/requests.js index cb1188457..36eef830b 100644 --- a/features/step_definitions/requests.js +++ b/features/step_definitions/requests.js @@ -51,7 +51,7 @@ module.exports = function () { }); this.Then(/^"([^"]*)" should return code (\d+)$/, (binary, code) => { - assert.ok(this.processError instanceof this.OSRMError); + assert.ok(this.processError instanceof Error); assert.equal(this.processError.process, binary); assert.equal(parseInt(this.processError.code), parseInt(code)); }); diff --git a/features/step_definitions/routability.js b/features/step_definitions/routability.js index c4190c547..ea2a45e65 100644 --- a/features/step_definitions/routability.js +++ b/features/step_definitions/routability.js @@ -13,7 +13,7 @@ module.exports = function () { } this.reprocessAndLoadData((e) => { - if (e) callback(e); + if (e) return callback(e); var testRow = (row, i, cb) => { var outputRow = row; @@ -41,10 +41,6 @@ module.exports = function () { } }); - if (outputRow != row) { - this.logFail(row, outputRow, result); - } - cb(null, outputRow); }); }; @@ -116,7 +112,7 @@ module.exports = function () { sq.defer(parseRes, key); }); - sq.awaitAll(() => { cb(null, result); }); + sq.awaitAll((err) => { cb(err, result); }); }); }; }; diff --git a/features/step_definitions/trip.js b/features/step_definitions/trip.js index fe4ff892b..57ce0795f 100644 --- a/features/step_definitions/trip.js +++ b/features/step_definitions/trip.js @@ -85,23 +85,14 @@ module.exports = function () { } else { got.trips = encodedResult; got.trips = extendedTarget; - this.logFail(row, got, { trip: { query: this.query, response: res }}); } - ok = true; - for (var key in row) { if (this.FuzzyMatch.match(got[key], row[key])) { got[key] = row[key]; - } else { - ok = false; } } - if (!ok) { - this.logFail(row, got, { trip: { query: this.query, response: res }}); - } - cb(null, got); }; diff --git a/features/support/cache.js b/features/support/cache.js new file mode 100644 index 000000000..e388f5976 --- /dev/null +++ b/features/support/cache.js @@ -0,0 +1,184 @@ +'use strict'; + +const d3 = require('d3-queue'); +const fs = require('fs'); +const util = require('util'); +const path = require('path'); +const mkdirp = require('mkdirp'); +const hash = require('../lib/hash'); +const rimraf = require('rimraf'); + +module.exports = function() { + this.initializeCache = (callback) => { + this.getOSRMHash((err, osrmHash) => { + if (err) return callback(err); + this.osrmHash = osrmHash; + callback(); + }); + }; + + // computes all paths for every feature + this.setupFeatures = (features, callback) => { + this.featureIDs = {}; + this.featureCacheDirectories = {}; + this.featureProcessedCacheDirectories = {}; + let queue = d3.queue(); + + function initializeFeature(feature, callback) { + let uri = feature.getUri(); + + // setup cache for feature data + hash.hashOfFile(uri, (err, hash) => { + if (err) return callback(err); + + // shorten uri to be realtive to 'features/' + let featurePath = path.relative(path.resolve('./features'), uri); + // bicycle/bollards/{HASH}/ + let featureID = path.join(featurePath, hash); + let featureCacheDirectory = this.getFeatureCacheDirectory(featureID); + let featureProcessedCacheDirectory = this.getFeatureProcessedCacheDirectory(featureCacheDirectory, this.osrmHash); + this.featureIDs[uri] = featureID; + this.featureCacheDirectories[uri] = featureCacheDirectory; + this.featureProcessedCacheDirectories[uri] = featureProcessedCacheDirectory; + + d3.queue(1) + .defer(mkdirp, featureProcessedCacheDirectory) + .defer(this.cleanupFeatureCache.bind(this), featureCacheDirectory, hash) + .defer(this.cleanupProcessedFeatureCache.bind(this), featureProcessedCacheDirectory, this.osrmHash) + .awaitAll(callback); + }); + } + + for (let i = 0; i < features.length; ++i) { + queue.defer(initializeFeature.bind(this), features[i]); + } + queue.awaitAll(callback); + }; + + this.cleanupProcessedFeatureCache = (directory, osrmHash, callback) => { + let parentPath = path.resolve(path.join(directory, '..')); + fs.readdir(parentPath, (err, files) => { + let q = d3.queue(); + function runStats(path, callback) { + fs.stat(path, (err, stat) => { + if (err) return callback(err); + callback(null, {file: path, stat: stat}); + }); + } + files.map(f => { q.defer(runStats, path.join(parentPath, f)); }); + q.awaitAll((err, results) => { + if (err) return callback(err); + let q = d3.queue(); + results.forEach(r => { + if (r.stat.isDirectory() && r.file.search(osrmHash) < 0) { + q.defer(rimraf, r.file); + } + }); + q.awaitAll(callback); + }); + }); + }; + + this.cleanupFeatureCache = (directory, featureHash, callback) => { + let parentPath = path.resolve(path.join(directory, '..')); + fs.readdir(parentPath, (err, files) => { + let q = d3.queue(); + files.filter(name => { return name !== featureHash;}) + .map((f) => { q.defer(rimraf, path.join(parentPath, f)); }); + q.awaitAll(callback); + }); + }; + + this.setupFeatureCache = (feature) => { + let uri = feature.getUri(); + this.featureID = this.featureIDs[uri]; + this.featureCacheDirectory = this.featureCacheDirectories[uri]; + this.featureProcessedCacheDirectory = this.featureProcessedCacheDirectories[uri]; + }; + + this.setupScenarioCache = (scenarioID) => { + this.scenarioCacheFile = this.getScenarioCacheFile(this.featureCacheDirectory, scenarioID); + this.processedCacheFile = this.getProcessedCacheFile(this.featureProcessedCacheDirectory, scenarioID); + this.inputCacheFile = this.getInputCacheFile(this.featureProcessedCacheDirectory, scenarioID); + this.rasterCacheFile = this.getRasterCacheFile(this.featureProcessedCacheDirectory, scenarioID); + this.speedsCacheFile = this.getSpeedsCacheFile(this.featureProcessedCacheDirectory, scenarioID); + this.penaltiesCacheFile = this.getPenaltiesCacheFile(this.featureProcessedCacheDirectory, scenarioID); + }; + + // returns a hash of all OSRM code side dependencies + this.getOSRMHash = (callback) => { + let dependencies = [ + this.OSRM_EXTRACT_PATH, + this.OSRM_CONTRACT_PATH, + this.LIB_OSRM_EXTRACT_PATH, + this.LIB_OSRM_CONTRACT_PATH + ]; + + var addLuaFiles = (directory, callback) => { + fs.readdir(path.normalize(directory), (err, files) => { + if (err) return callback(err); + + var luaFiles = files.filter(f => !!f.match(/\.lua$/)).map(f => path.normalize(directory + '/' + f)); + Array.prototype.push.apply(dependencies, luaFiles); + + callback(); + }); + }; + + // Note: we need a serialized queue here to ensure that the order of the files + // passed is stable. Otherwise the hash will not be stable + d3.queue(1) + .defer(addLuaFiles, this.PROFILES_PATH) + .defer(addLuaFiles, this.PROFILES_PATH + '/lib') + .awaitAll(hash.hashOfFiles.bind(hash, dependencies, callback)); + }; + + // test/cache/bicycle/bollards/{HASH}/ + this.getFeatureCacheDirectory = (featureID) => { + return path.join(this.CACHE_PATH, featureID); + }; + + // converts the scenario titles in file prefixes + this.getScenarioID = (scenario) => { + let name = scenario.getName().toLowerCase().replace(/[\/\-'=,\(\)]/g, '').replace(/\s/g, '_').replace(/__/g, '_').replace(/\.\./g, '.'); + return util.format('%d_%s', scenario.getLine(), name); + }; + + // test/cache/{feature_path}/{feature_hash}/{scenario}_raster.asc + this.getRasterCacheFile = (featureCacheDirectory, scenarioID) => { + return path.join(featureCacheDirectory, scenarioID) + '_raster.asc'; + }; + + // test/cache/{feature_path}/{feature_hash}/{scenario}_speeds.csv + this.getSpeedsCacheFile = (featureCacheDirectory, scenarioID) => { + return path.join(featureCacheDirectory, scenarioID) + '_speeds.csv'; + }; + + // test/cache/{feature_path}/{feature_hash}/{scenario}_penalties.csv + this.getPenaltiesCacheFile = (featureCacheDirectory, scenarioID) => { + return path.join(featureCacheDirectory, scenarioID) + '_penalties.csv'; + }; + + // test/cache/{feature_path}/{feature_hash}/{scenario}.osm + this.getScenarioCacheFile = (featureCacheDirectory, scenarioID) => { + return path.join(featureCacheDirectory, scenarioID) + '.osm'; + }; + + // test/cache/{feature_path}/{feature_hash}/{osrm_hash}/ + this.getFeatureProcessedCacheDirectory = (featureCacheDirectory, osrmHash) => { + return path.join(featureCacheDirectory, osrmHash); + }; + + // test/cache/{feature_path}/{feature_hash}/{osrm_hash}/{scenario}.osrm + this.getProcessedCacheFile = (featureProcessedCacheDirectory, scenarioID) => { + return path.join(featureProcessedCacheDirectory, scenarioID) + '.osrm'; + }; + + // test/cache/{feature_path}/{feature_hash}/{osrm_hash}/{scenario}.osm + this.getInputCacheFile = (featureProcessedCacheDirectory, scenarioID) => { + return path.join(featureProcessedCacheDirectory, scenarioID) + '.osm'; + }; + + + return this; +}; diff --git a/features/support/config.js b/features/support/config.js deleted file mode 100644 index 769755f4f..000000000 --- a/features/support/config.js +++ /dev/null @@ -1,127 +0,0 @@ -var fs = require('fs'); -var path = require('path'); -var util = require('util'); -var d3 = require('d3-queue'); -var OSM = require('./build_osm'); -var classes = require('./data_classes'); - -module.exports = function () { - this.initializeOptions = (callback) => { - this.profile = this.profile || this.DEFAULT_SPEEDPROFILE; - - this.OSMDB = this.OSMDB || new OSM.DB(); - - this.nameNodeHash = this.nameNodeHash || {}; - - this.locationHash = this.locationHash || {}; - - this.nameWayHash = this.nameWayHash || {}; - - this.osmData = new classes.osmData(this); - - this.OSRMLoader = this._OSRMLoader(); - - this.PREPROCESS_LOG_FILE = path.resolve(this.TEST_FOLDER, 'preprocessing.log'); - - this.LOG_FILE = path.resolve(this.TEST_FOLDER, 'fail.log'); - - this.HOST = 'http://127.0.0.1:' + this.OSRM_PORT; - - this.DESTINATION_REACHED = 15; // OSRM instruction code - - this.shortcutsHash = this.shortcutsHash || {}; - - var hashLuaLib = (cb) => { - fs.readdir(path.normalize(this.PROFILES_PATH + '/lib/'), (err, files) => { - if (err) cb(err); - var luaFiles = files.filter(f => !!f.match(/\.lua$/)).map(f => path.normalize(this.PROFILES_PATH + '/lib/' + f)); - this.hashOfFiles(luaFiles, hash => { - this.luaLibHash = hash; - cb(); - }); - }); - }; - - var hashProfile = (cb) => { - this.hashProfile((hash) => { - this.profileHash = hash; - cb(); - }); - }; - - var hashExtract = (cb) => { - var files = [ util.format('%s/osrm-extract%s', this.BIN_PATH, this.EXE), - util.format('%s/libosrm_extract%s', this.BIN_PATH, this.LIB) ]; - this.hashOfFiles(files, (hash) => { - this.binExtractHash = hash; - cb(); - }); - }; - - var hashContract = (cb) => { - var files = [ util.format('%s/osrm-contract%s', this.BIN_PATH, this.EXE), - util.format('%s/libosrm_contract%s', this.BIN_PATH, this.LIB) ]; - this.hashOfFiles(files, (hash) => { - this.binContractHash = hash; - cb(); - }); - }; - - var hashRouted = (cb) => { - var files = [ util.format('%s/osrm-routed%s', this.BIN_PATH, this.EXE), - util.format('%s/libosrm%s', this.BIN_PATH, this.LIB) ]; - this.hashOfFiles(files, (hash) => { - this.binRoutedHash = hash; - this.fingerprintRoute = this.hashString(this.binRoutedHash); - cb(); - }); - }; - - d3.queue() - .defer(hashLuaLib) - .defer(hashProfile) - .defer(hashExtract) - .defer(hashContract) - .defer(hashRouted) - .awaitAll(() => { - this.AfterConfiguration(() => { - callback(); - }); - }); - }; - - this.updateFingerprintExtract = (str) => { - this.fingerprintExtract = this.hashString([this.fingerprintExtract, str].join('-')); - }; - - this.updateFingerprintContract = (str) => { - this.fingerprintContract = this.hashString([this.fingerprintContract, str].join('-')); - }; - - this.setProfile = (profile, cb) => { - var lastProfile = this.profile; - if (profile !== lastProfile) { - this.profile = profile; - this.hashProfile((hash) => { - this.profileHash = hash; - this.updateFingerprintExtract(this.profileHash); - cb(); - }); - } else { - this.updateFingerprintExtract(this.profileHash); - cb(); - } - }; - - this.setExtractArgs = (args, callback) => { - this.extractArgs = args; - this.updateFingerprintExtract(args); - callback(); - }; - - this.setContractArgs = (args, callback) => { - this.contractArgs = args; - this.updateFingerprintContract(args); - callback(); - }; -}; diff --git a/features/support/data.js b/features/support/data.js index 48e5d612c..f0bf302d8 100644 --- a/features/support/data.js +++ b/features/support/data.js @@ -1,11 +1,14 @@ -var fs = require('fs'); -var path = require('path'); -var util = require('util'); -var exec = require('child_process').exec; -var d3 = require('d3-queue'); +'use strict'; -var OSM = require('./build_osm'); -var classes = require('./data_classes'); +const fs = require('fs'); +const util = require('util'); +const d3 = require('d3-queue'); + +const OSM = require('../lib/osm'); +const classes = require('./data_classes'); +const tableDiff = require('../lib/table_diff'); +const ensureDecimal = require('../lib/utils').ensureDecimal; +const errorReason = require('../lib/utils').errorReason; module.exports = function () { this.setGridSize = (meters) => { @@ -94,13 +97,8 @@ module.exports = function () { q.awaitAll(callback); }; - this.ensureDecimal = (i) => { - if (parseInt(i) === i) return i.toFixed(1); - else return i; - }; - this.tableCoordToLonLat = (ci, ri) => { - return [this.origin[0] + ci * this.zoom, this.origin[1] - ri * this.zoom].map(this.ensureDecimal); + return [this.origin[0] + ci * this.zoom, this.origin[1] - ri * this.zoom].map(ensureDecimal); }; this.addOSMNode = (name, lon, lat, id) => { @@ -132,10 +130,6 @@ module.exports = function () { return this.nameWayHash[s.toString()] || this.nameWayHash[s.toString().split('').reverse().join('')]; }; - this.resetData = () => { - this.resetOSM(); - }; - this.makeOSMId = () => { this.osmID = this.osmID + 1; return this.osmID; @@ -143,206 +137,88 @@ module.exports = function () { this.resetOSM = () => { this.OSMDB.clear(); - this.osmData.reset(); this.nameNodeHash = {}; this.locationHash = {}; + this.shortcutsHash = {}; this.nameWayHash = {}; this.osmID = 0; }; this.writeOSM = (callback) => { - fs.exists(this.DATA_FOLDER, (exists) => { - var mkDirFn = exists ? (cb) => { cb(); } : fs.mkdir.bind(fs.mkdir, this.DATA_FOLDER); - mkDirFn((err) => { - if (err) return callback(err); - var osmPath = path.resolve(this.DATA_FOLDER, util.format('%s.osm', this.osmData.osmFile)); - fs.exists(osmPath, (exists) => { - if (!exists) fs.writeFile(osmPath, this.osmData.str, callback); - else callback(); + fs.exists(this.scenarioCacheFile, (exists) => { + if (exists) callback(); + else { + this.OSMDB.toXML((xml) => { + fs.writeFile(this.scenarioCacheFile, xml, callback); }); - }); - }); - }; - - this.isExtracted = (callback) => { - fs.exists(util.format('%s.osrm', this.osmData.extractedFile), (core) => { - if (!core) return callback(false); - fs.exists(util.format('%s.osrm.names', this.osmData.extractedFile), (names) => { - if (!names) return callback(false); - fs.exists(util.format('%s.osrm.restrictions', this.osmData.extractedFile), (restrictions) => { - return callback(restrictions); - }); - }); - }); - }; - - this.isContracted = (callback) => { - fs.exists(util.format('%s.osrm.hsgr', this.osmData.contractedFile), callback); - }; - - this.writeTimestamp = (callback) => { - fs.writeFile(util.format('%s.osrm.timestamp', this.osmData.contractedFile), this.OSM_TIMESTAMP, callback); - }; - - this.writeInputData = (callback) => { - this.writeOSM((err) => { - if (err) return callback(err); - this.writeTimestamp(callback); - }); - }; - - this.extractData = (callback) => { - this.logPreprocessInfo(); - this.log(util.format('== Extracting %s.osm...', this.osmData.osmFile), 'preprocess'); - var cmd = util.format('%s/osrm-extract %s.osm %s --profile %s/%s.lua >>%s 2>&1', - this.BIN_PATH, this.osmData.osmFile, this.extractArgs || '', this.PROFILES_PATH, this.profile, this.PREPROCESS_LOG_FILE); - this.log(cmd); - process.chdir(this.TEST_FOLDER); - exec(cmd, (err) => { - if (err) { - this.log(util.format('*** Exited with code %d', err.code), 'preprocess'); - process.chdir('../'); - return callback(this.ExtractError(err.code, util.format('osrm-extract exited with code %d', err.code))); } - - var q = d3.queue(); - - var rename = (file, cb) => { - this.log(util.format('Renaming %s.%s to %s.%s', this.osmData.osmFile, file, this.osmData.extractedFile, file), 'preprocess'); - fs.rename([this.osmData.osmFile, file].join('.'), [this.osmData.extractedFile, file].join('.'), (err) => { - if (err) return cb(this.FileError(null, 'failed to rename data file after extracting')); - cb(); - }); - }; - - var renameIfExists = (file, cb) => { - fs.stat([this.osmData.osmFile, file].join('.'), (doesNotExistErr, exists) => { - if (exists) rename(file, cb); - else cb(); - }); - }; - - ['osrm', 'osrm.ebg', 'osrm.edges', 'osrm.enw', 'osrm.fileIndex', 'osrm.geometry', 'osrm.icd', - 'osrm.names', 'osrm.nodes', 'osrm.properties', 'osrm.ramIndex', 'osrm.restrictions', 'osrm.tld', 'osrm.tls'].forEach(file => { - q.defer(rename, file); - }); - - ['osrm.edge_penalties', 'osrm.edge_segment_lookup'].forEach(file => { - q.defer(renameIfExists, file); - }); - - q.awaitAll((err) => { - this.log('Finished extracting ' + this.osmData.extractedFile, 'preprocess'); - process.chdir('../'); - callback(err); - }); }); }; - this.contractData = (callback) => { - this.logPreprocessInfo(); - this.log(util.format('== Contracting %s.osm...', this.osmData.extractedFile), 'preprocess'); - var cmd = util.format('%s/osrm-contract %s %s.osrm >>%s 2>&1', - this.BIN_PATH, this.contractArgs || '', this.osmData.extractedFile, this.PREPROCESS_LOG_FILE); - this.log(cmd); - process.chdir(this.TEST_FOLDER); - exec(cmd, (err) => { - if (err) { - this.log(util.format('*** Exited with code %d', err.code), 'preprocess'); - process.chdir('../'); - return callback(this.ContractError(err.code, util.format('osrm-contract exited with code %d', err.code))); + this.linkOSM = (callback) => { + fs.exists(this.inputCacheFile, (exists) => { + if (exists) callback(); + else { + fs.link(this.scenarioCacheFile, this.inputCacheFile, callback); } + }); + }; - var rename = (file, cb) => { - this.log(util.format('Renaming %s.%s to %s.%s', this.osmData.extractedFile, file, this.osmData.contractedFile, file), 'preprocess'); - fs.rename([this.osmData.extractedFile, file].join('.'), [this.osmData.contractedFile, file].join('.'), (err) => { - if (err) return cb(this.FileError(null, 'failed to rename data file after contracting.')); - cb(); - }); - }; + this.extractData = (p, callback) => { + let stamp = p.processedCacheFile + '.extract'; + fs.exists(stamp, (exists) => { + if (exists) return callback(); - var renameIfExists = (file, cb) => { - fs.stat([this.osmData.extractedFile, file].join('.'), (doesNotExistErr, exists) => { - if (exists) rename(file, cb); - else cb(); - }); - }; - - var copy = (file, cb) => { - this.log(util.format('Copying %s.%s to %s.%s', this.osmData.extractedFile, file, this.osmData.contractedFile, file), 'preprocess'); - fs.createReadStream([this.osmData.extractedFile, file].join('.')) - .pipe(fs.createWriteStream([this.osmData.contractedFile, file].join('.')) - .on('finish', cb) - ) - .on('error', () => { - return cb(this.FileError(null, 'failed to copy data after contracting.')); - }); - }; - - var q = d3.queue(); - - ['osrm', 'osrm.core', 'osrm.datasource_indexes', 'osrm.datasource_names', 'osrm.ebg','osrm.edges', - 'osrm.enw', 'osrm.fileIndex', 'osrm.geometry', 'osrm.hsgr', 'osrm.icd','osrm.level', 'osrm.names', - 'osrm.nodes', 'osrm.properties', 'osrm.ramIndex', 'osrm.restrictions', 'osrm.tld', 'osrm.tls'].forEach((file) => { - q.defer(rename, file); - }); - - ['osrm.edge_penalties', 'osrm.edge_segment_lookup'].forEach(file => { - q.defer(renameIfExists, file); - }); - - [].forEach((file) => { - q.defer(copy, file); - }); - - q.awaitAll((err) => { - this.log('Finished contracting ' + this.osmData.contractedFile, 'preprocess'); - process.chdir('../'); - callback(err); + this.runBin('osrm-extract', util.format('%s --profile %s %s', p.extractArgs, p.profileFile, p.inputCacheFile), p.environment, (err) => { + if (err) { + return callback(new Error(util.format('osrm-extract %s: %s', errorReason(err), err.cmd))); + } + fs.writeFile(stamp, 'ok', callback); }); }); }; - var noop = (cb) => cb(); + this.contractData = (p, callback) => { + let stamp = p.processedCacheFile + '.contract'; + fs.exists(stamp, (exists) => { + if (exists) return callback(); + + this.runBin('osrm-contract', util.format('%s %s', p.contractArgs, p.processedCacheFile), p.environment, (err) => { + if (err) { + return callback(new Error(util.format('osrm-contract %s: %s', errorReason(err), err))); + } + fs.writeFile(stamp, 'ok', callback); + }); + }); + }; + + this.extractAndContract = (callback) => { + // a shallow copy of scenario parameters to avoid data inconsistency + // if a cucumber timeout occurs during deferred jobs + let p = {extractArgs: this.extractArgs, contractArgs: this.contractArgs, + profileFile: this.profileFile, inputCacheFile: this.inputCacheFile, + processedCacheFile: this.processedCacheFile, environment: this.environment}; + let queue = d3.queue(1); + queue.defer(this.extractData.bind(this), p); + queue.defer(this.contractData.bind(this), p); + queue.awaitAll(callback); + }; this.reprocess = (callback) => { - this.osmData.populate(() => { - this.isContracted((isContracted) => { - if (!isContracted) { - this.writeAndExtract((e) => { - if (e) return callback(e); - this.contractData((e) => { - if (e) return callback(e); - this.logPreprocessDone(); - callback(); - }); - }); - } else { - this.log('Already contracted ' + this.osmData.contractedFile, 'preprocess'); - callback(); - } - }); - }); - }; - - this.writeAndExtract = (callback) => { - this.writeInputData((e) => { - if (e) return callback(e); - this.isExtracted((isExtracted) => { - var extractFn = isExtracted ? noop : this.extractData; - if (isExtracted) this.log('Already extracted ' + this.osmData.extractedFile, 'preprocess'); - extractFn((e) => { - callback(e); - }); - }); - }); + let queue = d3.queue(1); + queue.defer(this.writeOSM.bind(this)); + queue.defer(this.linkOSM.bind(this)); + queue.defer(this.extractAndContract.bind(this)); + queue.awaitAll(callback); }; this.reprocessAndLoadData = (callback) => { - this.reprocess((e) => { - if (e) return callback(e); - this.OSRMLoader.load(util.format('%s.osrm', this.osmData.contractedFile), callback); - }); + let queue = d3.queue(1); + queue.defer(this.writeOSM.bind(this)); + queue.defer(this.linkOSM.bind(this)); + queue.defer(this.extractAndContract.bind(this)); + queue.defer(this.osrmLoader.load.bind(this.osrmLoader), this.processedCacheFile); + queue.awaitAll(callback); }; this.processRowsAndDiff = (table, fn, callback) => { @@ -352,7 +228,9 @@ module.exports = function () { q.awaitAll((err, actual) => { if (err) return callback(err); - this.diffTables(table, actual, {}, callback); + let diff = tableDiff(table, actual); + if (diff) callback(new Error(diff)); + else callback(); }); }; }; diff --git a/features/support/data_classes.js b/features/support/data_classes.js index 391bb1c6d..a17141e5e 100644 --- a/features/support/data_classes.js +++ b/features/support/data_classes.js @@ -1,7 +1,6 @@ 'use strict'; -var util = require('util'); -var path = require('path'); +const util = require('util'); module.exports = { Location: class { @@ -11,43 +10,6 @@ module.exports = { } }, - osmData: class { - constructor (scope) { - this.scope = scope; - this.str = null; - this.hash = null; - this.fingerprintOSM = null; - this.osmFile = null; - this.extractedFile = null; - this.contractedFile = null; - } - - populate (callback) { - this.scope.OSMDB.toXML((str) => { - this.str = str; - - this.hash = this.scope.hashString(str); - this.fingerprintOSM = this.scope.hashString(this.hash); - - this.osmFile = path.resolve(this.scope.DATA_FOLDER, this.fingerprintOSM); - - this.extractedFile = path.resolve([this.osmFile, this.scope.fingerprintExtract].join('_')); - this.contractedFile = path.resolve([this.osmFile, this.scope.fingerprintExtract, this.scope.fingerprintContract].join('_')); - - callback(); - }); - } - - reset () { - this.str = null; - this.hash = null; - this.fingerprintOSM = null; - this.osmFile = null; - this.extractedFile = null; - this.contractedFile = null; - } - }, - FuzzyMatch: class { match (got, want) { var matchPercent = want.match(/(.*)\s+~(.+)%$/), diff --git a/features/support/env.js b/features/support/env.js index c99d54403..564579ee9 100644 --- a/features/support/env.js +++ b/features/support/env.js @@ -1,32 +1,46 @@ -var path = require('path'); -var util = require('util'); -var fs = require('fs'); -var exec = require('child_process').exec; -var d3 = require('d3-queue'); +'use strict'; +const path = require('path'); +const util = require('util'); +const fs = require('fs'); +const d3 = require('d3-queue'); +const child_process = require('child_process'); +const tryConnect = require('../lib/try_connect'); + +// Sets up all constants that are valid for all features module.exports = function () { this.initializeEnv = (callback) => { - this.OSRM_PORT = process.env.OSRM_PORT && parseInt(process.env.OSRM_PORT) || 5000; this.TIMEOUT = process.env.CUCUMBER_TIMEOUT && parseInt(process.env.CUCUMBER_TIMEOUT) || 5000; + // set cucumber default timeout this.setDefaultTimeout(this.TIMEOUT); - this.ROOT_FOLDER = process.cwd(); + this.ROOT_PATH = process.cwd(); + + this.TEST_PATH = path.resolve(this.ROOT_PATH, 'test'); + this.CACHE_PATH = path.resolve(this.TEST_PATH, 'cache'); + this.LOGS_PATH = path.resolve(this.TEST_PATH, 'logs'); + + this.PROFILES_PATH = path.resolve(this.ROOT_PATH, 'profiles'); + this.FIXTURES_PATH = path.resolve(this.ROOT_PATH, 'unit_tests/fixtures'); + this.BIN_PATH = process.env.OSRM_BUILD_DIR && process.env.OSRM_BUILD_DIR || path.resolve(this.ROOT_PATH, 'build'); + var stxxl_config = path.resolve(this.ROOT_PATH, 'test/.stxxl'); + if (!fs.existsSync(stxxl_config)) { + return callback(new Error('*** '+stxxl_config+ 'does not exist')); + } + + this.DEFAULT_ENVIRONMENT = Object.assign({STXXLCFG: stxxl_config}, process.env); + this.DEFAULT_PROFILE = 'bicycle'; + this.DEFAULT_INPUT_FORMAT = 'osm'; + this.DEFAULT_LOAD_METHOD = 'datastore'; + this.DEFAULT_ORIGIN = [1,1]; this.OSM_USER = 'osrm'; this.OSM_GENERATOR = 'osrm-test'; this.OSM_UID = 1; - this.TEST_FOLDER = path.resolve(this.ROOT_FOLDER, 'test'); - this.DATA_FOLDER = path.resolve(this.TEST_FOLDER, 'cache'); this.OSM_TIMESTAMP = '2000-01-01T00:00:00Z'; - this.DEFAULT_SPEEDPROFILE = 'bicycle'; this.WAY_SPACING = 100; - this.DEFAULT_GRID_SIZE = 100; // meters - this.PROFILES_PATH = path.resolve(this.ROOT_FOLDER, 'profiles'); - this.FIXTURES_PATH = path.resolve(this.ROOT_FOLDER, 'unit_tests/fixtures'); - this.BIN_PATH = process.env.OSRM_BUILD_DIR && process.env.OSRM_BUILD_DIR || path.resolve(this.ROOT_FOLDER, 'build'); - this.DEFAULT_INPUT_FORMAT = 'osm'; - this.DEFAULT_ORIGIN = [1,1]; - this.DEFAULT_LOAD_METHOD = 'datastore'; - this.OSRM_ROUTED_LOG_FILE = path.resolve(this.TEST_FOLDER, 'osrm-routed.log'); - this.ERROR_LOG_FILE = path.resolve(this.TEST_FOLDER, 'error.log'); + this.DEFAULT_GRID_SIZE = 100; // meters + + this.OSRM_PORT = process.env.OSRM_PORT && parseInt(process.env.OSRM_PORT) || 5000; + this.HOST = 'http://127.0.0.1:' + this.OSRM_PORT; // TODO make sure this works on win if (process.platform.match(/indows.*/)) { @@ -37,36 +51,49 @@ module.exports = function () { } else { this.TERMSIGNAL = 'SIGTERM'; this.EXE = ''; - this.LIB = '.so'; + // TODO autodetect if this was build with shared or static libraries + this.LIB = process.env.BUILD_SHARED_LIBS && '.so' || '.a'; this.QQ = ''; } + this.OSRM_EXTRACT_PATH = path.resolve(util.format('%s/%s%s', this.BIN_PATH, 'osrm-extract', this.EXE)); + this.OSRM_CONTRACT_PATH = path.resolve(util.format('%s/%s%s', this.BIN_PATH, 'osrm-contract', this.EXE)); + this.OSRM_ROUTED_PATH = path.resolve(util.format('%s/%s%s', this.BIN_PATH, 'osrm-routed', this.EXE)); + this.LIB_OSRM_EXTRACT_PATH = util.format('%s/libosrm_extract%s', this.BIN_PATH, this.LIB), + this.LIB_OSRM_CONTRACT_PATH = util.format('%s/libosrm_contract%s', this.BIN_PATH, this.LIB), + this.LIB_OSRM_PATH = util.format('%s/libosrm%s', this.BIN_PATH, this.LIB); + // eslint-disable-next-line no-console console.info(util.format('Node Version', process.version)); if (parseInt(process.version.match(/v(\d)/)[1]) < 4) throw new Error('*** PLease upgrade to Node 4.+ to run OSRM cucumber tests'); - fs.exists(this.TEST_FOLDER, (exists) => { - if (!exists) throw new Error(util.format('*** Test folder %s doesn\'t exist.', this.TEST_FOLDER)); - callback(); + fs.exists(this.TEST_PATH, (exists) => { + if (exists) + return callback(); + else + return callback(new Error('*** Test folder doesn\'t exist.')); }); }; - this.verifyOSRMIsNotRunning = () => { - if (this.OSRMLoader.up()) { - throw new Error('*** osrm-routed is already running.'); - } + this.getProfilePath = (profile) => { + return path.resolve(this.PROFILES_PATH, profile + '.lua'); + }; + + this.verifyOSRMIsNotRunning = (callback) => { + tryConnect(this.OSRM_PORT, (err) => { + if (!err) return callback(new Error('*** osrm-routed is already running.')); + else callback(); + }); }; this.verifyExistenceOfBinaries = (callback) => { - var verify = (bin, cb) => { - var binPath = path.resolve(util.format('%s/%s%s', this.BIN_PATH, bin, this.EXE)); + var verify = (binPath, cb) => { fs.exists(binPath, (exists) => { - if (!exists) throw new Error(util.format('%s is missing. Build failed?', binPath)); + if (!exists) return cb(new Error(util.format('%s is missing. Build failed?', binPath))); var helpPath = util.format('%s --help > /dev/null 2>&1', binPath); - exec(helpPath, (err) => { + child_process.exec(helpPath, (err) => { if (err) { - this.log(util.format('*** Exited with code %d', err.code), 'preprocess'); - throw new Error(util.format('*** %s exited with code %d', helpPath, err.code)); + return cb(new Error(util.format('*** %s exited with code %d', helpPath, err.code))); } cb(); }); @@ -74,23 +101,12 @@ module.exports = function () { }; var q = d3.queue(); - ['osrm-extract', 'osrm-contract', 'osrm-routed'].forEach(bin => { q.defer(verify, bin); }); - q.awaitAll(() => { - callback(); - }); - }; - - this.AfterConfiguration = (callback) => { - this.clearLogFiles(() => { - this.verifyOSRMIsNotRunning(); - this.verifyExistenceOfBinaries(() => { - callback(); - }); - }); + [this.OSRM_EXTRACT_PATH, this.OSRM_CONTRACT_PATH, this.OSRM_ROUTED_PATH].forEach(bin => { q.defer(verify, bin); }); + q.awaitAll(callback); }; process.on('exit', () => { - if (this.OSRMLoader.loader) this.OSRMLoader.shutdown(() => {}); + this.osrmLoader.shutdown(() => {}); }); process.on('SIGINT', () => { diff --git a/features/support/exception_classes.js b/features/support/exception_classes.js deleted file mode 100644 index 36bdffe8b..000000000 --- a/features/support/exception_classes.js +++ /dev/null @@ -1,132 +0,0 @@ -'use strict'; - -var util = require('util'); -var path = require('path'); -var fs = require('fs'); -var chalk = require('chalk'); - -var OSRMError = class extends Error { - constructor (process, code, msg, log, lines) { - super(msg); - this.process = process; - this.code = code; - this.msg = msg; - this.lines = lines; - this.log = log; - } - - extract (callback) { - this.logTail(this.log, this.lines, callback); - } - - // toString (callback) { - // this.extract((tail) => { - // callback(util.format('*** %s\nLast %s from %s:\n%s\n', this.msg, this.lines, this.log, tail)); - // }); - // } - - logTail (logPath, n, callback) { - var expanded = path.resolve(this.TEST_FOLDER, logPath); - fs.exists(expanded, (exists) => { - if (exists) { - fs.readFile(expanded, (err, data) => { - var lines = data.toString().trim().split('\n'); - callback(lines - .slice(lines.length - n) - .map(line => util.format(' %s', line)) - .join('\n')); - }); - } else { - callback(util.format('File %s does not exist!', expanded)); - } - }); - } -}; - -var unescapeStr = (str) => str.replace(/\\\|/g, '\|').replace(/\\\\/g, '\\'); - -module.exports = { - OSRMError: OSRMError, - - FileError: class extends OSRMError { - constructor (logFile, code, msg) { - super ('fileutil', code, msg, logFile, 5); - } - }, - - LaunchError: class extends OSRMError { - constructor (logFile, launchProcess, code, msg) { - super (launchProcess, code, msg, logFile, 5); - } - }, - - ExtractError: class extends OSRMError { - constructor (logFile, code, msg) { - super('osrm-extract', code, msg, logFile, 3); - } - }, - - ContractError: class extends OSRMError { - constructor (logFile, code, msg) { - super('osrm-contract', code, msg, logFile, 3); - } - }, - - RoutedError: class extends OSRMError { - constructor (logFile, msg) { - super('osrm-routed', null, msg, logFile, 3); - } - }, - - TableDiffError: class extends Error { - constructor (expected, actual) { - super(); - this.headers = expected.raw()[0]; - this.expected = expected.hashes(); - this.actual = actual; - this.diff = []; - this.hasErrors = false; - - var good = 0, bad = 0; - - this.expected.forEach((row, i) => { - var rowError = false; - - for (var j in row) { - if (unescapeStr(row[j]) != actual[i][j]) { - rowError = true; - this.hasErrors = true; - break; - } - } - - if (rowError) { - bad++; - this.diff.push(Object.assign({}, row, {c_status: 'undefined'})); - this.diff.push(Object.assign({}, actual[i], {c_status: 'comment'})); - } else { - good++; - this.diff.push(row); - } - }); - } - - get string () { - if (!this.hasErrors) return null; - - var s = ['Tables were not identical:']; - s.push(this.headers.map(key => ' ' + key).join(' | ')); - this.diff.forEach((row) => { - var rowString = '| '; - this.headers.forEach((header) => { - if (!row.c_status) rowString += chalk.green(' ' + row[header] + ' | '); - else if (row.c_status === 'undefined') rowString += chalk.yellow('(-) ' + row[header] + ' | '); - else rowString += chalk.red('(+) ' + row[header] + ' | '); - }); - s.push(rowString); - }); - - return s.join('\n') + '\nTODO this is a temp workaround waiting for https://github.com/cucumber/cucumber-js/issues/534'; - } - } -}; diff --git a/features/support/exceptions.js b/features/support/exceptions.js deleted file mode 100644 index 6af1a93b0..000000000 --- a/features/support/exceptions.js +++ /dev/null @@ -1,15 +0,0 @@ -var exceptions = require('./exception_classes'); - -module.exports = function () { - this.OSRMError = exceptions.OSRMError, - - this.FileError = (code, msg) => new (exceptions.FileError.bind(exceptions.FileError, this.PREPROCESS_LOG_FILE))(code, msg); - - this.LaunchError = (code, launchProcess, msg) => new (exceptions.LaunchError.bind(exceptions.LaunchError, this.ERROR_LOG_FILE))(code, launchProcess, msg); - - this.ExtractError = (code, msg) => new (exceptions.ExtractError.bind(exceptions.ExtractError, this.PREPROCESS_LOG_FILE))(code, msg); - - this.ContractError = (code, msg) => new (exceptions.ContractError.bind(exceptions.ContractError, this.PREPROCESS_LOG_FILE))(code, msg); - - this.RoutedError = (msg) => new (exceptions.RoutedError.bind(exceptions.RoutedError, this.OSRM_ROUTED_LOG_FILE))(msg); -}; diff --git a/features/support/hash.js b/features/support/hash.js deleted file mode 100644 index 399dd51ca..000000000 --- a/features/support/hash.js +++ /dev/null @@ -1,43 +0,0 @@ -var fs = require('fs'); -var path = require('path'); -var crypto = require('crypto'); -var d3 = require('d3-queue'); - -module.exports = function () { - this.hashOfFiles = (paths, cb) => { - paths = Array.isArray(paths) ? paths : [paths]; - var shasum = crypto.createHash('sha1'), hashedFiles = false; - - var q = d3.queue(1); - - var addFile = (path, cb) => { - fs.readFile(path, (err, data) => { - if (err && err.code === 'ENOENT') cb(); // ignore non-existing files - else if (err) cb(err); - else { - shasum.update(data); - hashedFiles = true; - cb(); - } - }); - }; - - paths.forEach(path => { q.defer(addFile, path); }); - - q.awaitAll(err => { - if (err) throw new Error('*** Error reading files:', err); - if (!hashedFiles) throw new Error('*** No files found: [' + paths.join(', ') + ']'); - cb(shasum.digest('hex')); - }); - }; - - this.hashProfile = (cb) => { - this.hashOfFiles(path.resolve(this.PROFILES_PATH, this.profile + '.lua'), cb); - }; - - this.hashString = (str) => { - return crypto.createHash('sha1').update(str).digest('hex'); - }; - - return this; -}; diff --git a/features/support/hooks.js b/features/support/hooks.js index 1e265ea14..b0f6b0b77 100644 --- a/features/support/hooks.js +++ b/features/support/hooks.js @@ -1,36 +1,61 @@ -var util = require('util'); +'use strict'; + +var d3 = require('d3-queue'); +var path = require('path'); +var mkdirp = require('mkdirp'); +var rimraf = require('rimraf'); +var OSM = require('../lib/osm'); +var OSRMLoader = require('../lib/osrm_loader'); module.exports = function () { - this.BeforeFeatures((features, callback) => { - this.pid = null; - this.initializeEnv(() => { - this.initializeOptions(callback); - }); + this.registerHandler('BeforeFeatures', {timeout: 30000}, (features, callback) => { + this.osrmLoader = new OSRMLoader(this); + this.OSMDB = new OSM.DB(); + + let queue = d3.queue(1); + queue.defer(this.initializeEnv.bind(this)); + queue.defer(this.verifyOSRMIsNotRunning.bind(this)); + queue.defer(this.verifyExistenceOfBinaries.bind(this)); + queue.defer(this.initializeCache.bind(this)); + queue.defer(this.setupFeatures.bind(this, features)); + queue.awaitAll(callback); }); - this.Before((scenario, callback) => { - this.scenarioTitle = scenario.getName(); - - this.loadMethod = this.DEFAULT_LOAD_METHOD; - this.queryParams = {}; - var d = new Date(); - this.scenarioTime = util.format('%d-%d-%dT%s:%s:%sZ', d.getFullYear(), d.getMonth()+1, d.getDate(), d.getHours(), d.getMinutes(), d.getSeconds()); - this.resetData(); - this.hasLoggedPreprocessInfo = false; - this.hasLoggedScenarioInfo = false; - this.setGridSize(this.DEFAULT_GRID_SIZE); - this.setOrigin(this.DEFAULT_ORIGIN); - this.fingerprintExtract = this.hashString([this.luaLibHash, this.binExtractHash].join('-')); - this.fingerprintContract = this.hashString(this.binContractHash); + this.BeforeFeature((feature, callback) => { + this.profile = this.DEFAULT_PROFILE; + this.profileFile = path.join(this.PROFILES_PATH, this.profile + '.lua'); + this.setupFeatureCache(feature); callback(); }); + this.Before((scenario, callback) => { + this.osrmLoader.setLoadMethod(this.DEFAULT_LOAD_METHOD); + this.setGridSize(this.DEFAULT_GRID_SIZE); + this.setOrigin(this.DEFAULT_ORIGIN); + this.queryParams = {}; + this.extractArgs = ''; + this.contractArgs = ''; + this.environment = Object.assign(this.DEFAULT_ENVIRONMENT); + this.resetOSM(); + + this.scenarioID = this.getScenarioID(scenario); + this.setupScenarioCache(this.scenarioID); + + // setup output logging + let logDir = path.join(this.LOGS_PATH, this.featureID); + this.scenarioLogFile = path.join(logDir, this.scenarioID) + '.log'; + d3.queue(1) + .defer(mkdirp, logDir) + .defer(rimraf, this.scenarioLogFile) + .awaitAll(callback); + }); + this.After((scenario, callback) => { - this.setExtractArgs('', () => { - this.setContractArgs('', () => { - if (this.loadMethod === 'directly' && !!this.OSRMLoader.loader) this.OSRMLoader.shutdown(callback); - else callback(); - }); - }); + this.resetOptionsOutput(); + callback(); + }); + + this.AfterFeatures((features, callback) => { + callback(); }); }; diff --git a/features/support/http.js b/features/support/http.js index 3ae2edc93..71f61761a 100644 --- a/features/support/http.js +++ b/features/support/http.js @@ -19,6 +19,9 @@ module.exports = function () { return paramString; }; + // FIXME this needs to be simplified! + // - remove usage of node-timeout + // - replace with node's native timout mechanism this.sendRequest = (baseUri, parameters, callback) => { var limit = Timeout(this.TIMEOUT, { err: { statusCode: 408 } }); @@ -28,9 +31,9 @@ module.exports = function () { request(this.query, (err, res, body) => { if (err && err.code === 'ECONNREFUSED') { - throw new Error('*** osrm-routed is not running.'); + return cb(new Error('*** osrm-routed is not running.')); } else if (err && err.statusCode === 408) { - throw new Error(); + return cb(new Error()); } return cb(err, res, body); @@ -40,11 +43,10 @@ module.exports = function () { runRequest(limit((err, res, body) => { if (err) { if (err.statusCode === 408) - return callback(this.RoutedError('*** osrm-routed did not respond')); + return callback(new Error('*** osrm-routed did not respond')); else if (err.code === 'ECONNREFUSED') - return callback(this.RoutedError('*** osrm-routed is not running')); + return callback(new Error('*** osrm-routed is not running')); } - //console.log(body+"\n"); return callback(err, res, body); })); }; diff --git a/features/support/launch.js b/features/support/launch.js deleted file mode 100644 index ee335e301..000000000 --- a/features/support/launch.js +++ /dev/null @@ -1,5 +0,0 @@ -var launchClasses = require('./launch_classes'); - -module.exports = function () { - this._OSRMLoader = () => new (launchClasses._OSRMLoader.bind(launchClasses._OSRMLoader, this))(); -}; diff --git a/features/support/launch_classes.js b/features/support/launch_classes.js deleted file mode 100644 index 2aace8846..000000000 --- a/features/support/launch_classes.js +++ /dev/null @@ -1,164 +0,0 @@ -'use strict'; - -var fs = require('fs'); -var spawn = require('child_process').spawn; -var util = require('util'); -var net = require('net'); -var Timeout = require('node-timeout'); - -var OSRMBaseLoader = class { - constructor (scope) { - this.scope = scope; - } - - launch (callback) { - var limit = Timeout(this.scope.TIMEOUT, { err: this.scope.RoutedError('Launching osrm-routed timed out.') }); - - var runLaunch = (cb) => { - this.osrmUp(() => { this.waitForConnection(cb); }); - }; - - runLaunch(limit((e) => { if (e) callback(e); else callback(); })); - } - - shutdown (callback) { - var limit = Timeout(this.scope.TIMEOUT, { err: this.scope.RoutedError('Shutting down osrm-routed timed out.')}); - - var runShutdown = (cb) => { - this.osrmDown(cb); - }; - - runShutdown(limit((e) => { if (e) callback(e); else callback(); })); - } - - osrmIsRunning () { - return !!this.scope.pid && this.child && !this.child.killed; - } - - osrmDown (callback) { - if (this.scope.pid) { - process.kill(this.scope.pid, this.scope.TERMSIGNAL); - this.waitForShutdown(callback); - this.scope.pid = null; - } else callback(true); - } - - waitForConnection (callback) { - var retryCount = 0; - var connectWithRetry = () => { - net.connect({ port: this.scope.OSRM_PORT, host: '127.0.0.1' }) - .on('connect', () => { callback(); }) - .on('error', () => { - if (retryCount < 2) { - retryCount++; - setTimeout(connectWithRetry, 100); - } else { - callback(new Error('Could not connect to osrm-routed after three retires')); - } - }); - }; - - connectWithRetry(); - } - - waitForShutdown (callback) { - var check = () => { - if (!this.osrmIsRunning()) return callback(); - }; - setTimeout(check, 100); - } -}; - -var OSRMDirectLoader = class extends OSRMBaseLoader { - constructor (scope) { - super(scope); - } - - load (inputFile, callback) { - this.inputFile = inputFile; - this.shutdown(() => { - this.launch(callback); - }); - } - - osrmUp (callback) { - if (this.scope.pid) return callback(); - var writeToLog = (data) => { - fs.appendFile(this.scope.OSRM_ROUTED_LOG_FILE, data, (err) => { if (err) throw err; }); - }; - - var child = spawn(util.format('%s/osrm-routed', this.scope.BIN_PATH), [this.inputFile, util.format('-p%d', this.scope.OSRM_PORT)]); - this.scope.pid = child.pid; - child.stdout.on('data', writeToLog); - child.stderr.on('data', writeToLog); - - callback(); - } -}; - -var OSRMDatastoreLoader = class extends OSRMBaseLoader { - constructor (scope) { - super(scope); - } - - load (inputFile, callback) { - this.inputFile = inputFile; - this.loadData((err) => { - if (err) return callback(err); - if (!this.scope.pid) return this.launch(callback); - else callback(); - }); - } - - loadData (callback) { - this.scope.runBin('osrm-datastore', this.inputFile, (err) => { - if (err) return callback(this.scope.LaunchError(this.exitCode, 'datastore', err)); - callback(); - }); - } - - osrmUp (callback) { - if (this.scope.pid) return callback(); - var writeToLog = (data) => { - fs.appendFile(this.scope.OSRM_ROUTED_LOG_FILE, data, (err) => { if (err) throw err; }); - }; - - var child = spawn(util.format('%s/osrm-routed', this.scope.BIN_PATH), ['--shared-memory=1', util.format('-p%d', this.scope.OSRM_PORT)]); - this.child = child; - this.scope.pid = child.pid; - child.stdout.on('data', writeToLog); - child.stderr.on('data', writeToLog); - - callback(); - } -}; - -module.exports = { - _OSRMLoader: class { - constructor (scope) { - this.scope = scope; - this.loader = null; - } - - load (inputFile, callback) { - var method = this.scope.loadMethod; - if (method === 'datastore') { - this.loader = new OSRMDatastoreLoader(this.scope); - this.loader.load(inputFile, callback); - } else if (method === 'directly') { - this.loader = new OSRMDirectLoader(this.scope); - this.loader.load(inputFile, callback); - } else { - callback(new Error('*** Unknown load method ' + method)); - } - } - - shutdown (callback) { - this.loader.shutdown(callback); - } - - up () { - return this.loader ? this.loader.osrmIsRunning() : false; - } - } -}; diff --git a/features/support/log.js b/features/support/log.js deleted file mode 100644 index c428cb9d9..000000000 --- a/features/support/log.js +++ /dev/null @@ -1,90 +0,0 @@ -var fs = require('fs'); - -module.exports = function () { - this.clearLogFiles = (callback) => { - // emptying existing files, rather than deleting and writing new ones makes it - // easier to use tail -f from the command line - fs.writeFile(this.OSRM_ROUTED_LOG_FILE, '', err => { - if (err) throw err; - fs.writeFile(this.PREPROCESS_LOG_FILE, '', err => { - if (err) throw err; - fs.writeFile(this.LOG_FILE, '', err => { - if (err) throw err; - callback(); - }); - }); - }); - }; - - var log = this.log = (s, type) => { - s = s || ''; - type = type || null; - var file = type === 'preprocess' ? this.PREPROCESS_LOG_FILE : this.LOG_FILE; - fs.appendFile(file, s + '\n', err => { - if (err) throw err; - }); - }; - - this.logScenarioFailInfo = () => { - if (this.hasLoggedScenarioInfo) return; - - log('========================================='); - log('Failed scenario: ' + this.scenarioTitle); - log('Time: ' + this.scenarioTime); - log('Fingerprint osm stage: ' + this.osmData.fingerprintOSM); - log('Fingerprint extract stage: ' + this.fingerprintExtract); - log('Fingerprint contract stage: ' + this.fingerprintContract); - log('Fingerprint route stage: ' + this.fingerprintRoute); - log('Profile: ' + this.profile); - log(); - log('```xml'); // so output can be posted directly to github comment fields - log(this.osmData.str.trim()); - log('```'); - log(); - log(); - - this.hasLoggedScenarioInfo = true; - }; - - this.logFail = (expected, got, attempts) => { - this.logScenarioFailInfo(); - log('== '); - log('Expected: ' + JSON.stringify(expected)); - log('Got: ' + JSON.stringify(got)); - log(); - ['route','forw','backw'].forEach((direction) => { - if (attempts[direction]) { - log('Direction: ' + direction); - log('Query: ' + attempts[direction].query); - log('Response: ' + attempts[direction].response.body); - log(); - } - }); - }; - - this.logPreprocessInfo = () => { - if (this.hasLoggedPreprocessInfo) return; - log('=========================================', 'preprocess'); - log('Preprocessing data for scenario: ' + this.scenarioTitle, 'preprocess'); - log('Time: ' + this.scenarioTime, 'preprocess'); - log('', 'preprocess'); - log('== OSM data:', 'preprocess'); - log('```xml', 'preprocess'); // so output can be posted directly to github comment fields - log(this.osmData.str, 'preprocess'); - log('```', 'preprocess'); - log('', 'preprocess'); - log('== Profile:', 'preprocess'); - log(this.profile, 'preprocess'); - log('', 'preprocess'); - this.hasLoggedPreprocessInfo = true; - }; - - this.logPreprocess = (str) => { - this.logPreprocessInfo(); - log(str, 'preprocess'); - }; - - this.logPreprocessDone = () => { - log('Done with preprocessing at ' + new Date(), 'preprocess'); - }; -}; diff --git a/features/support/route.js b/features/support/route.js index 408a2a4e5..76ce70e43 100644 --- a/features/support/route.js +++ b/features/support/route.js @@ -1,7 +1,8 @@ 'use strict'; -var Timeout = require('node-timeout'); -var request = require('request'); +const Timeout = require('node-timeout'); +const request = require('request'); +const ensureDecimal = require('../lib/utils').ensureDecimal; module.exports = function () { this.requestPath = (service, params, callback) => { @@ -42,7 +43,7 @@ module.exports = function () { }; var encodeWaypoints = (waypoints) => { - return waypoints.map(w => [w.lon, w.lat].map(this.ensureDecimal).join(',')); + return waypoints.map(w => [w.lon, w.lat].map(ensureDecimal).join(',')); }; this.requestRoute = (waypoints, bearings, userParams, callback) => { diff --git a/features/support/run.js b/features/support/run.js index 35561d803..8c4d7fdc2 100644 --- a/features/support/run.js +++ b/features/support/run.js @@ -1,40 +1,52 @@ -var fs = require('fs'); -var util = require('util'); -var exec = require('child_process').exec; +'use strict'; + +const fs = require('fs'); +const util = require('util'); +const child_process = require('child_process'); module.exports = function () { - this.runBin = (bin, options, callback) => { - var opts = options.slice(); + // replaces placeholders for in user supplied commands + this.expandOptions = (options) => { + let opts = options.slice(); + let table = { + '{osm_file}': this.inputCacheFile, + '{processed_file}': this.processedCacheFile, + '{profile_file}': this.profileFile, + '{rastersource_file}': this.rasterCacheFile, + '{speeds_file}': this.speedsCacheFile, + '{penalties_file}': this.penaltiesCacheFile + }; - if (opts.match('{osm_base}')) { - if (!this.osmData.osmFile) throw new Error('*** {osm_base} is missing'); - opts = opts.replace('{osm_base}', this.osmData.osmFile); + for (let k in table) { + opts = opts.replace(k, table[k]); } - if (opts.match('{extracted_base}')) { - if (!this.osmData.extractedFile) throw new Error('*** {extracted_base} is missing'); - opts = opts.replace('{extracted_base}', this.osmData.extractedFile); + return opts; + }; + + this.setupOutputLog = (process, log) => { + if (process.logFunc) { + process.stdout.removeListener('data', process.logFunc); + process.stderr.removeListener('data', process.logFunc); } - if (opts.match('{contracted_base}')) { - if (!this.osmData.contractedFile) throw new Error('*** {contracted_base} is missing'); - opts = opts.replace('{contracted_base}', this.osmData.contractedFile); - } + process.logFunc = (message) => { log.write(message); }; + process.stdout.on('data', process.logFunc); + process.stderr.on('data', process.logFunc); + }; - if (opts.match('{profile}')) { - opts = opts.replace('{profile}', [this.PROFILES_PATH, this.profile + '.lua'].join('/')); - } - - var cmd = util.format('%s%s/%s%s%s %s 2>%s', this.QQ, this.BIN_PATH, bin, this.EXE, this.QQ, opts, this.ERROR_LOG_FILE); - process.chdir(this.TEST_FOLDER); - exec(cmd, (err, stdout, stderr) => { - this.stdout = stdout.toString(); - fs.readFile(this.ERROR_LOG_FILE, (e, data) => { - this.stderr = data ? data.toString() : ''; - this.exitCode = err && err.code || 0; - process.chdir('../'); - callback(err, stdout, stderr); - }); - }); + this.runBin = (bin, options, env, callback) => { + let cmd = util.format('%s%s/%s%s%s', this.QQ, this.BIN_PATH, bin, this.EXE, this.QQ); + let opts = options.split(' ').filter((x) => { return x && x.length > 0; }); + let log = fs.createWriteStream(this.scenarioLogFile, {'flags': 'a'}); + log.write(util.format('*** running %s %s\n', cmd, options)); + // we need to set a large maxbuffer here because we have long running processes like osrm-routed + // with lots of log output + let child = child_process.execFile(cmd, opts, {maxBuffer: 1024 * 1024 * 1000, env: env}, callback); + child.on('exit', function(code) { + log.end(util.format('*** %s exited with code %d\n', bin, code)); + }.bind(this)); + this.setupOutputLog(child, log); + return child; }; }; diff --git a/features/support/shared_steps.js b/features/support/shared_steps.js index 40c0a6530..0252cd47b 100644 --- a/features/support/shared_steps.js +++ b/features/support/shared_steps.js @@ -93,7 +93,7 @@ module.exports = function () { if (headers.has('distance')) { if (row.distance.length) { if (!row.distance.match(/\d+m/)) - throw new Error('*** Distance must be specified in meters. (ex: 250m)'); + return cb(new Error('*** Distance must be specified in meters. (ex: 250m)')); got.distance = instructions ? util.format('%dm', distance) : ''; } else { got.distance = ''; @@ -102,7 +102,7 @@ module.exports = function () { if (headers.has('time')) { if (!row.time.match(/\d+s/)) - throw new Error('*** Time must be specied in seconds. (ex: 60s)'); + return cb(new Error('*** Time must be specied in seconds. (ex: 60s)')); got.time = instructions ? util.format('%ds', time) : ''; } @@ -113,7 +113,7 @@ module.exports = function () { if (headers.has('speed')) { if (row.speed !== '' && instructions) { if (!row.speed.match(/\d+ km\/h/)) - throw new Error('*** Speed must be specied in km/h. (ex: 50 km/h)'); + cb(new Error('*** Speed must be specied in km/h. (ex: 50 km/h)')); var speed = time > 0 ? Math.round(3.6*distance/time) : null; got.speed = util.format('%d km/h', speed); } else { @@ -139,20 +139,12 @@ module.exports = function () { putValue('destinations', destinations); } - var ok = true; - for (var key in row) { if (this.FuzzyMatch.match(got[key], row[key])) { got[key] = row[key]; - } else { - ok = false; } } - if (!ok) { - this.logFail(row, got, { route: { query: this.query, response: res }}); - } - cb(null, got); } else { cb(new Error('request failed to return valid body')); @@ -189,11 +181,11 @@ module.exports = function () { if (row.from && row.to) { var fromNode = this.findNodeByName(row.from); - if (!fromNode) throw new Error(util.format('*** unknown from-node "%s"'), row.from); + if (!fromNode) return cb(new Error(util.format('*** unknown from-node "%s"'), row.from)); waypoints.push(fromNode); var toNode = this.findNodeByName(row.to); - if (!toNode) throw new Error(util.format('*** unknown to-node "%s"'), row.to); + if (!toNode) return cb(new Error(util.format('*** unknown to-node "%s"'), row.to)); waypoints.push(toNode); got.from = row.from; @@ -202,13 +194,13 @@ module.exports = function () { } else if (row.waypoints) { row.waypoints.split(',').forEach((n) => { var node = this.findNodeByName(n.trim()); - if (!node) throw new Error('*** unknown waypoint node "%s"', n.trim()); + if (!node) return cb(new Error('*** unknown waypoint node "%s"', n.trim())); waypoints.push(node); }); got.waypoints = row.waypoints; this.requestRoute(waypoints, bearings, params, afterRequest); } else { - throw new Error('*** no waypoints'); + return cb(new Error('*** no waypoints')); } } }; diff --git a/features/support/table_patch.js b/features/support/table_patch.js deleted file mode 100644 index 16ffebb8c..000000000 --- a/features/support/table_patch.js +++ /dev/null @@ -1,11 +0,0 @@ -var DifferentError = require('./exception_classes').TableDiffError; - -module.exports = function () { - this.diffTables = (expected, actual, options, callback) => { - // this is a temp workaround while waiting for https://github.com/cucumber/cucumber-js/issues/534 - - var error = new DifferentError(expected, actual); - - return callback(error.string); - }; -}; diff --git a/features/testbot/bad.feature b/features/testbot/bad.feature index 18ee5f862..aeddc95d1 100644 --- a/features/testbot/bad.feature +++ b/features/testbot/bad.feature @@ -11,8 +11,8 @@ Feature: Handle bad data in a graceful manner Given the ways | nodes | - When the data has been contracted - Then "osrm-extract" should return code 1 + When I try to run "osrm-extract {osm_file} --profile {profile_file}" + Then it should exit with an error Scenario: Only dead-end oneways Given the node map diff --git a/features/testbot/bugs.feature b/features/testbot/bugs.feature deleted file mode 100644 index 26be28aa6..000000000 --- a/features/testbot/bugs.feature +++ /dev/null @@ -1,5 +0,0 @@ -@routing @testbot @bug -Feature: Known bugs - - Background: - Given the profile "testbot" diff --git a/features/testbot/matching.feature b/features/testbot/matching.feature index 9c7faa01f..ead4b6a74 100644 --- a/features/testbot/matching.feature +++ b/features/testbot/matching.feature @@ -124,7 +124,7 @@ Feature: Basic Map Matching 1,2,36 """ - And the contract extra arguments "--segment-speed-file speeds.csv" + And the contract extra arguments "--segment-speed-file {speeds_file}" When I match I should get | trace | matchings | annotation | diff --git a/features/testbot/traffic_turn_penalties.feature b/features/testbot/traffic_turn_penalties.feature index a8f82e0dc..7ee9424ab 100644 --- a/features/testbot/traffic_turn_penalties.feature +++ b/features/testbot/traffic_turn_penalties.feature @@ -26,7 +26,7 @@ Feature: Traffic - turn penalties applied to turn onto which a phantom node snap 1,2,5,0,comment 3,4,7,-20 """ - And the contract extra arguments "--turn-penalty-file penalties.csv" + And the contract extra arguments "--turn-penalty-file {penalties_file}" When I route I should get | from | to | route | speed | time | | a | e | ab,be,be | 36 km/h | 40s +-1 | diff --git a/features/testbot/via.feature b/features/testbot/via.feature index fcd966902..62699b8f0 100644 --- a/features/testbot/via.feature +++ b/features/testbot/via.feature @@ -173,7 +173,7 @@ Feature: Via points | c,d,a | abc,bd,bd,bd,abc,abc | # See issue #2349 - @bug @todo + @todo Scenario: Via point at a dead end with oneway Given the node map | a | b | c | diff --git a/package.json b/package.json index 87d21ce87..c78842fa4 100644 --- a/package.json +++ b/package.json @@ -7,9 +7,11 @@ "chalk": "^1.1.3", "cucumber": "^1.2.1", "d3-queue": "^2.0.3", + "mkdirp": "^0.5.1", "node-timeout": "0.0.4", "polyline": "^0.2.0", "request": "^2.69.0", + "rimraf": "^2.5.4", "xmlbuilder": "^4.2.1" }, "bin": { diff --git a/profiles/rasterbot.lua b/profiles/rasterbot.lua index 03ff8f269..bc2b2b247 100644 --- a/profiles/rasterbot.lua +++ b/profiles/rasterbot.lua @@ -21,8 +21,12 @@ function way_function (way, result) end function source_function () + local path = os.getenv('OSRM_RASTER_SOURCE') + if not path then + path = "rastersource.asc" + end raster_source = sources:load( - "../test/rastersource.asc", + path, 0, -- lon_min 0.1, -- lon_max 0, -- lat_min diff --git a/profiles/rasterbotinterp.lua b/profiles/rasterbotinterp.lua index 8266b07c4..f81e6e2f5 100644 --- a/profiles/rasterbotinterp.lua +++ b/profiles/rasterbotinterp.lua @@ -21,8 +21,12 @@ function way_function (way, result) end function source_function () + local path = os.getenv('OSRM_RASTER_SOURCE') + if not path then + path = "rastersource.asc" + end raster_source = sources:load( - "../test/rastersource.asc", + path, 0, -- lon_min 0.1, -- lon_max 0, -- lat_min