Try to add NodeJs based benchmarks

package-lock.json

@@ -10,7 +10,8 @@
       "hasInstallScript": true,
       "license": "BSD-2-Clause",
       "dependencies": {
-        "@mapbox/node-pre-gyp": "^1.0.11"
+        "@mapbox/node-pre-gyp": "^1.0.11",
+        "seedrandom": "^3.0.5"
       },
       "devDependencies": {
         "@babel/cli": "^7.18.10",
@@ -14652,6 +14653,11 @@
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
       "devOptional": true
     },
+    "node_modules/seedrandom": {
+      "version": "3.0.5",
+      "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz",
+      "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg=="
+    },
     "node_modules/semver": {
       "version": "5.7.2",
       "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
@@ -30296,6 +30302,11 @@
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
       "devOptional": true
     },
+    "seedrandom": {
+      "version": "3.0.5",
+      "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz",
+      "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg=="
+    },
     "semver": {
       "version": "5.7.2",
       "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",

package.json

@@ -4,7 +4,8 @@
   "private": false,
   "description": "The Open Source Routing Machine is a high performance routing engine written in C++ designed to run on OpenStreetMap data.",
   "dependencies": {
-    "@mapbox/node-pre-gyp": "^1.0.11"
+    "@mapbox/node-pre-gyp": "^1.0.11",
+    "seedrandom": "^3.0.5"
   },
   "browserify": {
     "transform": [
@@ -57,6 +58,7 @@
     "jsonpath": "^1.1.1",
     "mkdirp": "^0.5.6",
     "node-addon-api": "^5.0.0",
+    "node-cmake": "^2.5.1",
     "node-timeout": "0.0.4",
     "polyline": "^0.2.0",
     "request": "^2.88.2",
@@ -64,12 +66,13 @@
     "tape": "^4.16.0",
     "turf": "^3.0.14",
     "uglify-js": "^3.17.0",
-    "xmlbuilder": "^4.2.1",
+    "xmlbuilder": "^4.2.1"
-    "node-cmake": "^2.5.1"
   },
   "main": "lib/index.js",
   "binary": {
-    "napi_versions": [8],
+    "napi_versions": [
+      8
+    ],
     "module_name": "node_osrm",
     "module_path": "./lib/binding_napi_v{napi_build_version}/",
     "host": "https://github.com",

scripts/ci/bench.js

@@ -0,0 +1,213 @@
+const fs = require('fs');
+const path = require('path');
+const readline = require('readline');
+const seedrandom = require('seedrandom');
+
+
+let RNG;
+
+class GPSData {
+    constructor(gpsTracesFilePath) {
+        this.tracks = {};
+        this.coordinates = [];
+        this.trackIds = [];
+        this._loadGPSTraces(gpsTracesFilePath);
+    }
+
+    _loadGPSTraces(gpsTracesFilePath) {
+        const expandedPath = path.resolve(gpsTracesFilePath);
+        const data = fs.readFileSync(expandedPath, 'utf-8');
+        const lines = data.split('\n');
+        const headers = lines[0].split(',');
+
+        const latitudeIndex = headers.indexOf('Latitude');
+        const longitudeIndex = headers.indexOf('Longitude');
+        const trackIdIndex = headers.indexOf('TrackID');
+
+        for (let i = 1; i < lines.length; i++) {
+            if (lines[i].trim() === '') continue; // Skip empty lines
+            const row = lines[i].split(',');
+
+            const latitude = parseFloat(row[latitudeIndex]);
+            const longitude = parseFloat(row[longitudeIndex]);
+            const trackId = row[trackIdIndex];
+
+            const coord = [longitude, latitude];
+            this.coordinates.push(coord);
+
+            if (!this.tracks[trackId]) {
+                this.tracks[trackId] = [];
+            }
+            this.tracks[trackId].push(coord);
+        }
+
+        this.trackIds = Object.keys(this.tracks);
+    }
+
+    getRandomCoordinate() {
+        const randomIndex = Math.floor(RNG() * this.coordinates.length);
+        return this.coordinates[randomIndex];
+    }
+
+    getRandomTrack() {
+        const randomIndex = Math.floor(RNG() * this.trackIds.length);
+        const trackId = this.trackIds[randomIndex];
+        return this.tracks[trackId];
+    }
+};
+
+async function runOSRMMethod(osrm, method, coordinates) {
+    const time = await new Promise((resolve, reject) => {
+        const startTime = process.hrtime();
+        osrm[method]({coordinates}, (err, result) => {
+            if (err) {
+                if (['NoSegment', 'NoMatch', 'NoRoute', 'NoTrips'].includes(err.message)) {
+                    resolve(null);
+                } else {
+
+                reject(err);
+                }
+            } else {
+                const endTime = process.hrtime(startTime);
+                resolve(endTime[0] + endTime[1] / 1e9);
+            }
+        });
+    });
+    return time;
+}
+
+async function nearest(osrm, numRequests, gpsData) {
+    const times = [];
+    for (let i = 0; i < numRequests; i++) {
+        const coord = gpsData.getRandomCoordinate();
+        times.push(await runOSRMMethod(osrm, 'nearest', [coord]));
+    }
+    return times;
+}
+
+async function route(osrm, numRequests, gpsData) {
+    const times = [];
+    for (let i = 0; i < numRequests; i++) {
+        const from = gpsData.getRandomCoordinate();
+        const to = gpsData.getRandomCoordinate();
+
+        
+        times.push(await runOSRMMethod(osrm, 'route', [from, to]));
+    }
+    return times;
+}
+
+async function table(osrm, numRequests, gpsData) {
+    const times = [];
+    for (let i = 0; i < numRequests; i++) {
+        const numPoints = Math.floor(RNG() * 50) + 50;
+        const coordinates = [];
+        for (let i = 0; i < numPoints; i++) {
+            coordinates.push(gpsData.getRandomCoordinate());
+        }
+
+        
+        times.push(await runOSRMMethod(osrm, 'table', coordinates));
+    }
+}
+
+async function match(osrm, numRequests, gpsData) {
+    const times = [];
+    for (let i = 0; i < numRequests; i++) {
+        const numPoints = Math.floor(RNG() * 50) + 50;
+        const coordinates = gpsData.getRandomTrack().slice(0, numPoints);
+
+        
+        times.push(await runOSRMMethod(osrm, 'match', coordinates));
+
+    }
+}
+
+async function trip(osrm, numRequests, gpsData) {
+    const times = [];
+    for (let i = 0; i < numRequests; i++) {
+        const numPoints = Math.floor(RNG() * 2) + 8;
+        const coordinates = [];
+        for (let i = 0; i < numPoints; i++) {
+            coordinates.push(gpsData.getRandomCoordinate());
+        }
+
+        
+        times.push(await runOSRMMethod(osrm, 'trip', coordinates));
+
+    }
+}
+
+function bootstrapConfidenceInterval(data, numSamples = 1000, confidenceLevel = 0.95) {
+    let means = [];
+    let dataLength = data.length;
+
+    for (let i = 0; i < numSamples; i++) {
+        let sample = [];
+        for (let j = 0; j < dataLength; j++) {
+            let randomIndex = Math.floor(RNG() * dataLength);
+            sample.push(data[randomIndex]);
+        }
+        let sampleMean = sample.reduce((a, b) => a + b, 0) / sample.length;
+        means.push(sampleMean);
+    }
+
+    means.sort((a, b) => a - b);
+    let lowerBoundIndex = Math.floor((1 - confidenceLevel) / 2 * numSamples);
+    let upperBoundIndex = Math.floor((1 + confidenceLevel) / 2 * numSamples);
+    let mean = means.reduce((a, b) => a + b, 0) / means.length;
+    let lowerBound = means[lowerBoundIndex];
+    let upperBound = means[upperBoundIndex];
+
+    return { mean: mean, lowerBound: lowerBound, upperBound: upperBound };
+}
+
+function calculateConfidenceInterval(data) {
+    let { mean, lowerBound, upperBound } = bootstrapConfidenceInterval(data);
+    let bestValue = Math.max(...data);
+    let errorMargin = (upperBound - lowerBound) / 2;
+
+    return { mean, errorMargin, bestValue };
+}
+
+
+async function main() {
+    const args = process.argv.slice(2);
+
+    const {OSRM} = require(args[0]);
+    const path = args[1];
+    const algorithm = args[2];
+    const method = args[3];
+    const numRequests = parseInt(args[4]);
+    const gpsTracesFilePath = args[5];
+    const iterations = parseInt(args[6]);
+
+    const gpsData = new GPSData(gpsTracesFilePath);
+    const osrm = new OSRM({path, algorithm});
+
+
+    const functions = {
+        route: route,
+        table: table,
+        nearest: nearest,
+        match: match,
+        trip: trip
+    };
+    const func = functions[method];
+    if (!func) {
+        throw new Error('Unknown method');
+    }
+    const allTimes = [];
+    for (let i = 0; i < iterations; i++) {
+        RNG = seedrandom(42);
+        allTimes.push(await func(osrm, numRequests, gpsData));
+    }
+
+    console.log(allTimes.length);
+    const opsPerSec = allTimes.map(times => times.length / times.reduce((a, b) => a + b, 0));
+    const { mean, errorMargin, bestValue } = calculateConfidenceInterval(opsPerSec);
+    console.log(`Ops: ${mean} ± ${errorMargin} ops/s. Best: ${bestValue} ops/s`);
+
+}
+
+main();

scripts/ci/run_benchmarks.sh

@@ -53,22 +53,23 @@ function run_benchmarks_for_folder {
     mkdir -p $RESULTS_FOLDER
 
     BENCHMARKS_FOLDER="$BINARIES_FOLDER/src/benchmarks"
-    echo "Running match-bench MLD"
+
-    $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench"
+    # echo "Running match-bench MLD"
-    echo "Running match-bench CH"
+    # $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench"
-    $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench"
+    # echo "Running match-bench CH"
-    echo "Running route-bench MLD"
+    # $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench"
-    $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench"
+    # echo "Running route-bench MLD"
-    echo "Running route-bench CH"
+    # $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench"
-    $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench"
+    # echo "Running route-bench CH"
-    echo "Running alias"
+    # $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench"
-    $BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench"
+    # echo "Running alias"
-    echo "Running json-render-bench"
+    # $BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench"
-    $BENCHMARKS_FOLDER/json-render-bench  "$FOLDER/test/data/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench"
+    # echo "Running json-render-bench"
-    echo "Running packedvector-bench"
+    # $BENCHMARKS_FOLDER/json-render-bench  "$FOLDER/test/data/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench"
-    $BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench"
+    # echo "Running packedvector-bench"
-    echo "Running rtree-bench"
+    # $BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench"
-    $BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench"
+    # echo "Running rtree-bench"
+    # $BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench"
 
     cp -rf $OSM_PBF $FOLDER/data.osm.pbf
 
@@ -81,40 +82,51 @@ function run_benchmarks_for_folder {
     echo "Running osrm-contract"
     measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-contract $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_contract.bench"
 
+
     for ALGORITHM in ch mld; do
         for BENCH in nearest table trip route match; do
             echo "Running random $BENCH $ALGORITHM"
             START=$(date +%s.%N)
-            $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" $ALGORITHM $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_${ALGORITHM}.bench" 5 || true
+            node $BENCHMARKS_FOLDER/bench.js $FOLDER/lib/binding/node_osrm.node $FOLDER/data.osrm $ALGORITHM $BENCH 1000 $GPS_TRACES > "$RESULTS_FOLDER/node_${BENCH}_${ALGORITHM}.bench" 5 || true
             END=$(date +%s.%N)
             DIFF=$(echo "$END - $START" | bc)
             echo "Took: ${DIFF}s"
         done
     done
+    # for ALGORITHM in ch mld; do
+    #     for BENCH in nearest table trip route match; do
+    #         echo "Running random $BENCH $ALGORITHM"
+    #         START=$(date +%s.%N)
+    #         $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" $ALGORITHM $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_${ALGORITHM}.bench" 5 || true
+    #         END=$(date +%s.%N)
+    #         DIFF=$(echo "$END - $START" | bc)
+    #         echo "Took: ${DIFF}s"
+    #     done
+    # done
 
 
-    for ALGORITHM in ch mld; do
+    # for ALGORITHM in ch mld; do
-        $BINARIES_FOLDER/osrm-routed --algorithm $ALGORITHM $FOLDER/data.osrm > /dev/null 2>&1 &
+    #     $BINARIES_FOLDER/osrm-routed --algorithm $ALGORITHM $FOLDER/data.osrm > /dev/null 2>&1 &
-        OSRM_ROUTED_PID=$!
+    #     OSRM_ROUTED_PID=$!
 
-        # wait for osrm-routed to start
+    #     # wait for osrm-routed to start
-        if ! curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true" > /dev/null 2>&1; then
+    #     if ! curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true" > /dev/null 2>&1; then
-            echo "osrm-routed failed to start for algorithm $ALGORITHM"
+    #         echo "osrm-routed failed to start for algorithm $ALGORITHM"
-            kill -9 $OSRM_ROUTED_PID
+    #         kill -9 $OSRM_ROUTED_PID
-            continue
+    #         continue
-        fi
+    #     fi
 
-        for METHOD in route nearest trip table match; do
+    #     for METHOD in route nearest trip table match; do
-            echo "Running e2e benchmark for $METHOD $ALGORITHM"
+    #         echo "Running e2e benchmark for $METHOD $ALGORITHM"
-            START=$(date +%s.%N)
+    #         START=$(date +%s.%N)
-            python3 $SCRIPTS_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --iterations 5 --num_requests 1000 --gps_traces_file_path $GPS_TRACES > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench
+    #         python3 $SCRIPTS_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --iterations 5 --num_requests 1000 --gps_traces_file_path $GPS_TRACES > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench
-            END=$(date +%s.%N)
+    #         END=$(date +%s.%N)
-            DIFF=$(echo "$END - $START" | bc)
+    #         DIFF=$(echo "$END - $START" | bc)
-            echo "Took: ${DIFF}s"
+    #         echo "Took: ${DIFF}s"
-        done
+    #     done
 
-        kill -9 $OSRM_ROUTED_PID
+    #     kill -9 $OSRM_ROUTED_PID
-    done
+    # done
 }
 
 run_benchmarks_for_folder