diff --git a/package-lock.json b/package-lock.json index c5efc04e2..4731fa104 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,8 @@ "hasInstallScript": true, "license": "BSD-2-Clause", "dependencies": { - "@mapbox/node-pre-gyp": "^1.0.11" + "@mapbox/node-pre-gyp": "^1.0.11", + "seedrandom": "^3.0.5" }, "devDependencies": { "@babel/cli": "^7.18.10", @@ -14652,6 +14653,11 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "devOptional": true }, + "node_modules/seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" + }, "node_modules/semver": { "version": "5.7.2", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", @@ -30296,6 +30302,11 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "devOptional": true }, + "seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" + }, "semver": { "version": "5.7.2", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", diff --git a/package.json b/package.json index 02eee8430..f36d90da5 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "private": false, "description": "The Open Source Routing Machine is a high performance routing engine written in C++ designed to run on OpenStreetMap data.", "dependencies": { - "@mapbox/node-pre-gyp": "^1.0.11" + "@mapbox/node-pre-gyp": "^1.0.11", + "seedrandom": "^3.0.5" }, "browserify": { "transform": [ @@ -57,6 +58,7 @@ "jsonpath": "^1.1.1", "mkdirp": "^0.5.6", "node-addon-api": "^5.0.0", + "node-cmake": "^2.5.1", "node-timeout": "0.0.4", "polyline": "^0.2.0", "request": "^2.88.2", @@ -64,12 +66,13 @@ "tape": "^4.16.0", "turf": "^3.0.14", "uglify-js": "^3.17.0", - "xmlbuilder": "^4.2.1", - "node-cmake": "^2.5.1" + "xmlbuilder": "^4.2.1" }, "main": "lib/index.js", "binary": { - "napi_versions": [8], + "napi_versions": [ + 8 + ], "module_name": "node_osrm", "module_path": "./lib/binding_napi_v{napi_build_version}/", "host": "https://github.com", diff --git a/scripts/ci/bench.js b/scripts/ci/bench.js new file mode 100644 index 000000000..e882fab88 --- /dev/null +++ b/scripts/ci/bench.js @@ -0,0 +1,213 @@ +const fs = require('fs'); +const path = require('path'); +const readline = require('readline'); +const seedrandom = require('seedrandom'); + + +let RNG; + +class GPSData { + constructor(gpsTracesFilePath) { + this.tracks = {}; + this.coordinates = []; + this.trackIds = []; + this._loadGPSTraces(gpsTracesFilePath); + } + + _loadGPSTraces(gpsTracesFilePath) { + const expandedPath = path.resolve(gpsTracesFilePath); + const data = fs.readFileSync(expandedPath, 'utf-8'); + const lines = data.split('\n'); + const headers = lines[0].split(','); + + const latitudeIndex = headers.indexOf('Latitude'); + const longitudeIndex = headers.indexOf('Longitude'); + const trackIdIndex = headers.indexOf('TrackID'); + + for (let i = 1; i < lines.length; i++) { + if (lines[i].trim() === '') continue; // Skip empty lines + const row = lines[i].split(','); + + const latitude = parseFloat(row[latitudeIndex]); + const longitude = parseFloat(row[longitudeIndex]); + const trackId = row[trackIdIndex]; + + const coord = [longitude, latitude]; + this.coordinates.push(coord); + + if (!this.tracks[trackId]) { + this.tracks[trackId] = []; + } + this.tracks[trackId].push(coord); + } + + this.trackIds = Object.keys(this.tracks); + } + + getRandomCoordinate() { + const randomIndex = Math.floor(RNG() * this.coordinates.length); + return this.coordinates[randomIndex]; + } + + getRandomTrack() { + const randomIndex = Math.floor(RNG() * this.trackIds.length); + const trackId = this.trackIds[randomIndex]; + return this.tracks[trackId]; + } +}; + +async function runOSRMMethod(osrm, method, coordinates) { + const time = await new Promise((resolve, reject) => { + const startTime = process.hrtime(); + osrm[method]({coordinates}, (err, result) => { + if (err) { + if (['NoSegment', 'NoMatch', 'NoRoute', 'NoTrips'].includes(err.message)) { + resolve(null); + } else { + + reject(err); + } + } else { + const endTime = process.hrtime(startTime); + resolve(endTime[0] + endTime[1] / 1e9); + } + }); + }); + return time; +} + +async function nearest(osrm, numRequests, gpsData) { + const times = []; + for (let i = 0; i < numRequests; i++) { + const coord = gpsData.getRandomCoordinate(); + times.push(await runOSRMMethod(osrm, 'nearest', [coord])); + } + return times; +} + +async function route(osrm, numRequests, gpsData) { + const times = []; + for (let i = 0; i < numRequests; i++) { + const from = gpsData.getRandomCoordinate(); + const to = gpsData.getRandomCoordinate(); + + + times.push(await runOSRMMethod(osrm, 'route', [from, to])); + } + return times; +} + +async function table(osrm, numRequests, gpsData) { + const times = []; + for (let i = 0; i < numRequests; i++) { + const numPoints = Math.floor(RNG() * 50) + 50; + const coordinates = []; + for (let i = 0; i < numPoints; i++) { + coordinates.push(gpsData.getRandomCoordinate()); + } + + + times.push(await runOSRMMethod(osrm, 'table', coordinates)); + } +} + +async function match(osrm, numRequests, gpsData) { + const times = []; + for (let i = 0; i < numRequests; i++) { + const numPoints = Math.floor(RNG() * 50) + 50; + const coordinates = gpsData.getRandomTrack().slice(0, numPoints); + + + times.push(await runOSRMMethod(osrm, 'match', coordinates)); + + } +} + +async function trip(osrm, numRequests, gpsData) { + const times = []; + for (let i = 0; i < numRequests; i++) { + const numPoints = Math.floor(RNG() * 2) + 8; + const coordinates = []; + for (let i = 0; i < numPoints; i++) { + coordinates.push(gpsData.getRandomCoordinate()); + } + + + times.push(await runOSRMMethod(osrm, 'trip', coordinates)); + + } +} + +function bootstrapConfidenceInterval(data, numSamples = 1000, confidenceLevel = 0.95) { + let means = []; + let dataLength = data.length; + + for (let i = 0; i < numSamples; i++) { + let sample = []; + for (let j = 0; j < dataLength; j++) { + let randomIndex = Math.floor(RNG() * dataLength); + sample.push(data[randomIndex]); + } + let sampleMean = sample.reduce((a, b) => a + b, 0) / sample.length; + means.push(sampleMean); + } + + means.sort((a, b) => a - b); + let lowerBoundIndex = Math.floor((1 - confidenceLevel) / 2 * numSamples); + let upperBoundIndex = Math.floor((1 + confidenceLevel) / 2 * numSamples); + let mean = means.reduce((a, b) => a + b, 0) / means.length; + let lowerBound = means[lowerBoundIndex]; + let upperBound = means[upperBoundIndex]; + + return { mean: mean, lowerBound: lowerBound, upperBound: upperBound }; +} + +function calculateConfidenceInterval(data) { + let { mean, lowerBound, upperBound } = bootstrapConfidenceInterval(data); + let bestValue = Math.max(...data); + let errorMargin = (upperBound - lowerBound) / 2; + + return { mean, errorMargin, bestValue }; +} + + +async function main() { + const args = process.argv.slice(2); + + const {OSRM} = require(args[0]); + const path = args[1]; + const algorithm = args[2]; + const method = args[3]; + const numRequests = parseInt(args[4]); + const gpsTracesFilePath = args[5]; + const iterations = parseInt(args[6]); + + const gpsData = new GPSData(gpsTracesFilePath); + const osrm = new OSRM({path, algorithm}); + + + const functions = { + route: route, + table: table, + nearest: nearest, + match: match, + trip: trip + }; + const func = functions[method]; + if (!func) { + throw new Error('Unknown method'); + } + const allTimes = []; + for (let i = 0; i < iterations; i++) { + RNG = seedrandom(42); + allTimes.push(await func(osrm, numRequests, gpsData)); + } + + console.log(allTimes.length); + const opsPerSec = allTimes.map(times => times.length / times.reduce((a, b) => a + b, 0)); + const { mean, errorMargin, bestValue } = calculateConfidenceInterval(opsPerSec); + console.log(`Ops: ${mean} ± ${errorMargin} ops/s. Best: ${bestValue} ops/s`); + +} + +main(); diff --git a/scripts/ci/run_benchmarks.sh b/scripts/ci/run_benchmarks.sh index 0d0324d13..27f222345 100755 --- a/scripts/ci/run_benchmarks.sh +++ b/scripts/ci/run_benchmarks.sh @@ -53,22 +53,23 @@ function run_benchmarks_for_folder { mkdir -p $RESULTS_FOLDER BENCHMARKS_FOLDER="$BINARIES_FOLDER/src/benchmarks" - echo "Running match-bench MLD" - $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench" - echo "Running match-bench CH" - $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench" - echo "Running route-bench MLD" - $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench" - echo "Running route-bench CH" - $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench" - echo "Running alias" - $BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench" - echo "Running json-render-bench" - $BENCHMARKS_FOLDER/json-render-bench "$FOLDER/test/data/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench" - echo "Running packedvector-bench" - $BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench" - echo "Running rtree-bench" - $BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench" + + # echo "Running match-bench MLD" + # $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench" + # echo "Running match-bench CH" + # $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench" + # echo "Running route-bench MLD" + # $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench" + # echo "Running route-bench CH" + # $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench" + # echo "Running alias" + # $BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench" + # echo "Running json-render-bench" + # $BENCHMARKS_FOLDER/json-render-bench "$FOLDER/test/data/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench" + # echo "Running packedvector-bench" + # $BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench" + # echo "Running rtree-bench" + # $BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench" cp -rf $OSM_PBF $FOLDER/data.osm.pbf @@ -81,40 +82,51 @@ function run_benchmarks_for_folder { echo "Running osrm-contract" measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-contract $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_contract.bench" + for ALGORITHM in ch mld; do for BENCH in nearest table trip route match; do echo "Running random $BENCH $ALGORITHM" START=$(date +%s.%N) - $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" $ALGORITHM $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_${ALGORITHM}.bench" 5 || true + node $BENCHMARKS_FOLDER/bench.js $FOLDER/lib/binding/node_osrm.node $FOLDER/data.osrm $ALGORITHM $BENCH 1000 $GPS_TRACES > "$RESULTS_FOLDER/node_${BENCH}_${ALGORITHM}.bench" 5 || true END=$(date +%s.%N) DIFF=$(echo "$END - $START" | bc) echo "Took: ${DIFF}s" done done + # for ALGORITHM in ch mld; do + # for BENCH in nearest table trip route match; do + # echo "Running random $BENCH $ALGORITHM" + # START=$(date +%s.%N) + # $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" $ALGORITHM $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_${ALGORITHM}.bench" 5 || true + # END=$(date +%s.%N) + # DIFF=$(echo "$END - $START" | bc) + # echo "Took: ${DIFF}s" + # done + # done - for ALGORITHM in ch mld; do - $BINARIES_FOLDER/osrm-routed --algorithm $ALGORITHM $FOLDER/data.osrm > /dev/null 2>&1 & - OSRM_ROUTED_PID=$! + # for ALGORITHM in ch mld; do + # $BINARIES_FOLDER/osrm-routed --algorithm $ALGORITHM $FOLDER/data.osrm > /dev/null 2>&1 & + # OSRM_ROUTED_PID=$! - # wait for osrm-routed to start - if ! curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true" > /dev/null 2>&1; then - echo "osrm-routed failed to start for algorithm $ALGORITHM" - kill -9 $OSRM_ROUTED_PID - continue - fi + # # wait for osrm-routed to start + # if ! curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true" > /dev/null 2>&1; then + # echo "osrm-routed failed to start for algorithm $ALGORITHM" + # kill -9 $OSRM_ROUTED_PID + # continue + # fi - for METHOD in route nearest trip table match; do - echo "Running e2e benchmark for $METHOD $ALGORITHM" - START=$(date +%s.%N) - python3 $SCRIPTS_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --iterations 5 --num_requests 1000 --gps_traces_file_path $GPS_TRACES > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench - END=$(date +%s.%N) - DIFF=$(echo "$END - $START" | bc) - echo "Took: ${DIFF}s" - done + # for METHOD in route nearest trip table match; do + # echo "Running e2e benchmark for $METHOD $ALGORITHM" + # START=$(date +%s.%N) + # python3 $SCRIPTS_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --iterations 5 --num_requests 1000 --gps_traces_file_path $GPS_TRACES > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench + # END=$(date +%s.%N) + # DIFF=$(echo "$END - $START" | bc) + # echo "Took: ${DIFF}s" + # done - kill -9 $OSRM_ROUTED_PID - done + # kill -9 $OSRM_ROUTED_PID + # done } run_benchmarks_for_folder