Try to add NodeJs based benchmarks
This commit is contained in:
parent
8ae9abaa63
commit
5e3af8e601
13
package-lock.json
generated
13
package-lock.json
generated
@ -10,7 +10,8 @@
|
||||
"hasInstallScript": true,
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"@mapbox/node-pre-gyp": "^1.0.11"
|
||||
"@mapbox/node-pre-gyp": "^1.0.11",
|
||||
"seedrandom": "^3.0.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/cli": "^7.18.10",
|
||||
@ -14652,6 +14653,11 @@
|
||||
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
|
||||
"devOptional": true
|
||||
},
|
||||
"node_modules/seedrandom": {
|
||||
"version": "3.0.5",
|
||||
"resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz",
|
||||
"integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg=="
|
||||
},
|
||||
"node_modules/semver": {
|
||||
"version": "5.7.2",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
|
||||
@ -30296,6 +30302,11 @@
|
||||
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
|
||||
"devOptional": true
|
||||
},
|
||||
"seedrandom": {
|
||||
"version": "3.0.5",
|
||||
"resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz",
|
||||
"integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg=="
|
||||
},
|
||||
"semver": {
|
||||
"version": "5.7.2",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
|
||||
|
11
package.json
11
package.json
@ -4,7 +4,8 @@
|
||||
"private": false,
|
||||
"description": "The Open Source Routing Machine is a high performance routing engine written in C++ designed to run on OpenStreetMap data.",
|
||||
"dependencies": {
|
||||
"@mapbox/node-pre-gyp": "^1.0.11"
|
||||
"@mapbox/node-pre-gyp": "^1.0.11",
|
||||
"seedrandom": "^3.0.5"
|
||||
},
|
||||
"browserify": {
|
||||
"transform": [
|
||||
@ -57,6 +58,7 @@
|
||||
"jsonpath": "^1.1.1",
|
||||
"mkdirp": "^0.5.6",
|
||||
"node-addon-api": "^5.0.0",
|
||||
"node-cmake": "^2.5.1",
|
||||
"node-timeout": "0.0.4",
|
||||
"polyline": "^0.2.0",
|
||||
"request": "^2.88.2",
|
||||
@ -64,12 +66,13 @@
|
||||
"tape": "^4.16.0",
|
||||
"turf": "^3.0.14",
|
||||
"uglify-js": "^3.17.0",
|
||||
"xmlbuilder": "^4.2.1",
|
||||
"node-cmake": "^2.5.1"
|
||||
"xmlbuilder": "^4.2.1"
|
||||
},
|
||||
"main": "lib/index.js",
|
||||
"binary": {
|
||||
"napi_versions": [8],
|
||||
"napi_versions": [
|
||||
8
|
||||
],
|
||||
"module_name": "node_osrm",
|
||||
"module_path": "./lib/binding_napi_v{napi_build_version}/",
|
||||
"host": "https://github.com",
|
||||
|
213
scripts/ci/bench.js
Normal file
213
scripts/ci/bench.js
Normal file
@ -0,0 +1,213 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const readline = require('readline');
|
||||
const seedrandom = require('seedrandom');
|
||||
|
||||
|
||||
let RNG;
|
||||
|
||||
class GPSData {
|
||||
constructor(gpsTracesFilePath) {
|
||||
this.tracks = {};
|
||||
this.coordinates = [];
|
||||
this.trackIds = [];
|
||||
this._loadGPSTraces(gpsTracesFilePath);
|
||||
}
|
||||
|
||||
_loadGPSTraces(gpsTracesFilePath) {
|
||||
const expandedPath = path.resolve(gpsTracesFilePath);
|
||||
const data = fs.readFileSync(expandedPath, 'utf-8');
|
||||
const lines = data.split('\n');
|
||||
const headers = lines[0].split(',');
|
||||
|
||||
const latitudeIndex = headers.indexOf('Latitude');
|
||||
const longitudeIndex = headers.indexOf('Longitude');
|
||||
const trackIdIndex = headers.indexOf('TrackID');
|
||||
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
if (lines[i].trim() === '') continue; // Skip empty lines
|
||||
const row = lines[i].split(',');
|
||||
|
||||
const latitude = parseFloat(row[latitudeIndex]);
|
||||
const longitude = parseFloat(row[longitudeIndex]);
|
||||
const trackId = row[trackIdIndex];
|
||||
|
||||
const coord = [longitude, latitude];
|
||||
this.coordinates.push(coord);
|
||||
|
||||
if (!this.tracks[trackId]) {
|
||||
this.tracks[trackId] = [];
|
||||
}
|
||||
this.tracks[trackId].push(coord);
|
||||
}
|
||||
|
||||
this.trackIds = Object.keys(this.tracks);
|
||||
}
|
||||
|
||||
getRandomCoordinate() {
|
||||
const randomIndex = Math.floor(RNG() * this.coordinates.length);
|
||||
return this.coordinates[randomIndex];
|
||||
}
|
||||
|
||||
getRandomTrack() {
|
||||
const randomIndex = Math.floor(RNG() * this.trackIds.length);
|
||||
const trackId = this.trackIds[randomIndex];
|
||||
return this.tracks[trackId];
|
||||
}
|
||||
};
|
||||
|
||||
async function runOSRMMethod(osrm, method, coordinates) {
|
||||
const time = await new Promise((resolve, reject) => {
|
||||
const startTime = process.hrtime();
|
||||
osrm[method]({coordinates}, (err, result) => {
|
||||
if (err) {
|
||||
if (['NoSegment', 'NoMatch', 'NoRoute', 'NoTrips'].includes(err.message)) {
|
||||
resolve(null);
|
||||
} else {
|
||||
|
||||
reject(err);
|
||||
}
|
||||
} else {
|
||||
const endTime = process.hrtime(startTime);
|
||||
resolve(endTime[0] + endTime[1] / 1e9);
|
||||
}
|
||||
});
|
||||
});
|
||||
return time;
|
||||
}
|
||||
|
||||
async function nearest(osrm, numRequests, gpsData) {
|
||||
const times = [];
|
||||
for (let i = 0; i < numRequests; i++) {
|
||||
const coord = gpsData.getRandomCoordinate();
|
||||
times.push(await runOSRMMethod(osrm, 'nearest', [coord]));
|
||||
}
|
||||
return times;
|
||||
}
|
||||
|
||||
async function route(osrm, numRequests, gpsData) {
|
||||
const times = [];
|
||||
for (let i = 0; i < numRequests; i++) {
|
||||
const from = gpsData.getRandomCoordinate();
|
||||
const to = gpsData.getRandomCoordinate();
|
||||
|
||||
|
||||
times.push(await runOSRMMethod(osrm, 'route', [from, to]));
|
||||
}
|
||||
return times;
|
||||
}
|
||||
|
||||
async function table(osrm, numRequests, gpsData) {
|
||||
const times = [];
|
||||
for (let i = 0; i < numRequests; i++) {
|
||||
const numPoints = Math.floor(RNG() * 50) + 50;
|
||||
const coordinates = [];
|
||||
for (let i = 0; i < numPoints; i++) {
|
||||
coordinates.push(gpsData.getRandomCoordinate());
|
||||
}
|
||||
|
||||
|
||||
times.push(await runOSRMMethod(osrm, 'table', coordinates));
|
||||
}
|
||||
}
|
||||
|
||||
async function match(osrm, numRequests, gpsData) {
|
||||
const times = [];
|
||||
for (let i = 0; i < numRequests; i++) {
|
||||
const numPoints = Math.floor(RNG() * 50) + 50;
|
||||
const coordinates = gpsData.getRandomTrack().slice(0, numPoints);
|
||||
|
||||
|
||||
times.push(await runOSRMMethod(osrm, 'match', coordinates));
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
async function trip(osrm, numRequests, gpsData) {
|
||||
const times = [];
|
||||
for (let i = 0; i < numRequests; i++) {
|
||||
const numPoints = Math.floor(RNG() * 2) + 8;
|
||||
const coordinates = [];
|
||||
for (let i = 0; i < numPoints; i++) {
|
||||
coordinates.push(gpsData.getRandomCoordinate());
|
||||
}
|
||||
|
||||
|
||||
times.push(await runOSRMMethod(osrm, 'trip', coordinates));
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
function bootstrapConfidenceInterval(data, numSamples = 1000, confidenceLevel = 0.95) {
|
||||
let means = [];
|
||||
let dataLength = data.length;
|
||||
|
||||
for (let i = 0; i < numSamples; i++) {
|
||||
let sample = [];
|
||||
for (let j = 0; j < dataLength; j++) {
|
||||
let randomIndex = Math.floor(RNG() * dataLength);
|
||||
sample.push(data[randomIndex]);
|
||||
}
|
||||
let sampleMean = sample.reduce((a, b) => a + b, 0) / sample.length;
|
||||
means.push(sampleMean);
|
||||
}
|
||||
|
||||
means.sort((a, b) => a - b);
|
||||
let lowerBoundIndex = Math.floor((1 - confidenceLevel) / 2 * numSamples);
|
||||
let upperBoundIndex = Math.floor((1 + confidenceLevel) / 2 * numSamples);
|
||||
let mean = means.reduce((a, b) => a + b, 0) / means.length;
|
||||
let lowerBound = means[lowerBoundIndex];
|
||||
let upperBound = means[upperBoundIndex];
|
||||
|
||||
return { mean: mean, lowerBound: lowerBound, upperBound: upperBound };
|
||||
}
|
||||
|
||||
function calculateConfidenceInterval(data) {
|
||||
let { mean, lowerBound, upperBound } = bootstrapConfidenceInterval(data);
|
||||
let bestValue = Math.max(...data);
|
||||
let errorMargin = (upperBound - lowerBound) / 2;
|
||||
|
||||
return { mean, errorMargin, bestValue };
|
||||
}
|
||||
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
const {OSRM} = require(args[0]);
|
||||
const path = args[1];
|
||||
const algorithm = args[2];
|
||||
const method = args[3];
|
||||
const numRequests = parseInt(args[4]);
|
||||
const gpsTracesFilePath = args[5];
|
||||
const iterations = parseInt(args[6]);
|
||||
|
||||
const gpsData = new GPSData(gpsTracesFilePath);
|
||||
const osrm = new OSRM({path, algorithm});
|
||||
|
||||
|
||||
const functions = {
|
||||
route: route,
|
||||
table: table,
|
||||
nearest: nearest,
|
||||
match: match,
|
||||
trip: trip
|
||||
};
|
||||
const func = functions[method];
|
||||
if (!func) {
|
||||
throw new Error('Unknown method');
|
||||
}
|
||||
const allTimes = [];
|
||||
for (let i = 0; i < iterations; i++) {
|
||||
RNG = seedrandom(42);
|
||||
allTimes.push(await func(osrm, numRequests, gpsData));
|
||||
}
|
||||
|
||||
console.log(allTimes.length);
|
||||
const opsPerSec = allTimes.map(times => times.length / times.reduce((a, b) => a + b, 0));
|
||||
const { mean, errorMargin, bestValue } = calculateConfidenceInterval(opsPerSec);
|
||||
console.log(`Ops: ${mean} ± ${errorMargin} ops/s. Best: ${bestValue} ops/s`);
|
||||
|
||||
}
|
||||
|
||||
main();
|
@ -53,22 +53,23 @@ function run_benchmarks_for_folder {
|
||||
mkdir -p $RESULTS_FOLDER
|
||||
|
||||
BENCHMARKS_FOLDER="$BINARIES_FOLDER/src/benchmarks"
|
||||
echo "Running match-bench MLD"
|
||||
$BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench"
|
||||
echo "Running match-bench CH"
|
||||
$BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench"
|
||||
echo "Running route-bench MLD"
|
||||
$BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench"
|
||||
echo "Running route-bench CH"
|
||||
$BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench"
|
||||
echo "Running alias"
|
||||
$BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench"
|
||||
echo "Running json-render-bench"
|
||||
$BENCHMARKS_FOLDER/json-render-bench "$FOLDER/test/data/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench"
|
||||
echo "Running packedvector-bench"
|
||||
$BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench"
|
||||
echo "Running rtree-bench"
|
||||
$BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench"
|
||||
|
||||
# echo "Running match-bench MLD"
|
||||
# $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench"
|
||||
# echo "Running match-bench CH"
|
||||
# $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench"
|
||||
# echo "Running route-bench MLD"
|
||||
# $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench"
|
||||
# echo "Running route-bench CH"
|
||||
# $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench"
|
||||
# echo "Running alias"
|
||||
# $BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench"
|
||||
# echo "Running json-render-bench"
|
||||
# $BENCHMARKS_FOLDER/json-render-bench "$FOLDER/test/data/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench"
|
||||
# echo "Running packedvector-bench"
|
||||
# $BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench"
|
||||
# echo "Running rtree-bench"
|
||||
# $BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench"
|
||||
|
||||
cp -rf $OSM_PBF $FOLDER/data.osm.pbf
|
||||
|
||||
@ -81,40 +82,51 @@ function run_benchmarks_for_folder {
|
||||
echo "Running osrm-contract"
|
||||
measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-contract $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_contract.bench"
|
||||
|
||||
|
||||
for ALGORITHM in ch mld; do
|
||||
for BENCH in nearest table trip route match; do
|
||||
echo "Running random $BENCH $ALGORITHM"
|
||||
START=$(date +%s.%N)
|
||||
$BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" $ALGORITHM $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_${ALGORITHM}.bench" 5 || true
|
||||
node $BENCHMARKS_FOLDER/bench.js $FOLDER/lib/binding/node_osrm.node $FOLDER/data.osrm $ALGORITHM $BENCH 1000 $GPS_TRACES > "$RESULTS_FOLDER/node_${BENCH}_${ALGORITHM}.bench" 5 || true
|
||||
END=$(date +%s.%N)
|
||||
DIFF=$(echo "$END - $START" | bc)
|
||||
echo "Took: ${DIFF}s"
|
||||
done
|
||||
done
|
||||
# for ALGORITHM in ch mld; do
|
||||
# for BENCH in nearest table trip route match; do
|
||||
# echo "Running random $BENCH $ALGORITHM"
|
||||
# START=$(date +%s.%N)
|
||||
# $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" $ALGORITHM $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_${ALGORITHM}.bench" 5 || true
|
||||
# END=$(date +%s.%N)
|
||||
# DIFF=$(echo "$END - $START" | bc)
|
||||
# echo "Took: ${DIFF}s"
|
||||
# done
|
||||
# done
|
||||
|
||||
|
||||
for ALGORITHM in ch mld; do
|
||||
$BINARIES_FOLDER/osrm-routed --algorithm $ALGORITHM $FOLDER/data.osrm > /dev/null 2>&1 &
|
||||
OSRM_ROUTED_PID=$!
|
||||
# for ALGORITHM in ch mld; do
|
||||
# $BINARIES_FOLDER/osrm-routed --algorithm $ALGORITHM $FOLDER/data.osrm > /dev/null 2>&1 &
|
||||
# OSRM_ROUTED_PID=$!
|
||||
|
||||
# wait for osrm-routed to start
|
||||
if ! curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true" > /dev/null 2>&1; then
|
||||
echo "osrm-routed failed to start for algorithm $ALGORITHM"
|
||||
kill -9 $OSRM_ROUTED_PID
|
||||
continue
|
||||
fi
|
||||
# # wait for osrm-routed to start
|
||||
# if ! curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true" > /dev/null 2>&1; then
|
||||
# echo "osrm-routed failed to start for algorithm $ALGORITHM"
|
||||
# kill -9 $OSRM_ROUTED_PID
|
||||
# continue
|
||||
# fi
|
||||
|
||||
for METHOD in route nearest trip table match; do
|
||||
echo "Running e2e benchmark for $METHOD $ALGORITHM"
|
||||
START=$(date +%s.%N)
|
||||
python3 $SCRIPTS_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --iterations 5 --num_requests 1000 --gps_traces_file_path $GPS_TRACES > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench
|
||||
END=$(date +%s.%N)
|
||||
DIFF=$(echo "$END - $START" | bc)
|
||||
echo "Took: ${DIFF}s"
|
||||
done
|
||||
# for METHOD in route nearest trip table match; do
|
||||
# echo "Running e2e benchmark for $METHOD $ALGORITHM"
|
||||
# START=$(date +%s.%N)
|
||||
# python3 $SCRIPTS_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --iterations 5 --num_requests 1000 --gps_traces_file_path $GPS_TRACES > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench
|
||||
# END=$(date +%s.%N)
|
||||
# DIFF=$(echo "$END - $START" | bc)
|
||||
# echo "Took: ${DIFF}s"
|
||||
# done
|
||||
|
||||
kill -9 $OSRM_ROUTED_PID
|
||||
done
|
||||
# kill -9 $OSRM_ROUTED_PID
|
||||
# done
|
||||
}
|
||||
|
||||
run_benchmarks_for_folder
|
||||
|
Loading…
Reference in New Issue
Block a user