Add benchmarks comparison job

.github/workflows/osrm-backend.yml

@@ -642,7 +642,7 @@ jobs:
           make -C test/data 
       - name: Run Benchmark on Master Branch
         run: |
-          ./pr/build/src/benchmarks/match-bench ./pr/test/data/mld/monaco.osrm mld > pr_bench.txt
+          ./pr/build/src/benchmarks/match-bench ./pr/test/data/mld/monaco.osrm mld > pr/match.bench
       - name: Checkout Master Branch
         uses: actions/checkout@v3
         with:
@@ -657,14 +657,13 @@ jobs:
           make -j$(nproc) benchmarks
           cd ..
           make -C test/data 
-      - name: Run Benchmark on Master Branch
+      - name: Run Benchmarks
         run: |
-          ./master/build/src/benchmarks/match-bench ./master/test/data/mld/monaco.osrm mld > master_bench.txt
+          ./scripts/ci/run_benchmarks.sh master pr
       - name: Compare Benchmarks
         run: |
-          cat pr_bench.txt
+          cat pr_results/match.bench
-          cat master_bench.txt
+          cat master_results/match.bench
-            - name: Run benchmark and update PR
       - name: Post Benchmark Results
         run: |
-          python3 scripts/ci/post_benchmark_results.py
+          python3 pr/scripts/ci/post_benchmark_results.py master_results pr_results

scripts/ci/post_benchmark_results.py

@@ -1,6 +1,7 @@
 import requests
 import os
 import re
+import sys
 
 GITHUB_TOKEN = os.getenv('GITHUB_TOKEN')
 REPO = os.getenv('GITHUB_REPOSITORY')  
@@ -8,15 +9,9 @@ PR_NUMBER = os.getenv('PR_NUMBER')
 
 REPO_OWNER, REPO_NAME = REPO.split('/')
 
-benchmark_results = [
-  {'master': '0.002s', 'pr': '0.0018s'},
-  {'master': '0.003s', 'pr': '0.0025s'},
-  {'master': '0.004s', 'pr': '0.0038s'}
-]
-
 def create_markdown_table(results):
-    header = "| Master | PR |\n|--------|----|"
+    header = "| Benchmark | Master | PR |\n|--------|----|"
-    rows = [f"| {result['master']} | {result['pr']} |" for result in results]
+    rows = [f"| {result['name']} | {result['master']} | {result['pr']} |" for result in results]
     return f"{header}\n" + "\n".join(rows)
 
 def get_pr_comments(repo_owner, repo_name, pr_number):
@@ -34,7 +29,37 @@ def update_comment(comment_id, repo_owner, repo_name, body):
     response.raise_for_status()
     return response.json()
 
+
+def collect_benchmark_results(master_folder, pr_folder):
+    results = []
+    results_index = {}
+
+    for file in os.listdir(master_folder):
+        if not file.endswith('.bench'): continue
+        with open(f"{master_folder}/{file}") as f:
+            result = f.read().strip()
+            results.append({'master': result, 'pr': None, 'name': os.path.splitext(file)[0]})
+            results_index[file] = len(results) - 1
+
+    for file in os.listdir(pr_folder):
+        if not file.endswith('.bench'): continue
+        with open(f"{pr_folder}/{file}") as f:
+            result = f.read().strip()
+            if file in results_index:
+                results[results_index[file]]['pr'] = result
+            else:
+                results.append({'master': None, 'pr': result, 'name': os.path.splitext(file)[0]})
+
 def main():
+    if len(sys.argv) != 3:
+        print("Usage: python post_benchmark_results.py <master_folder> <pr_folder>")
+        exit(1)
+
+    master_folder = sys.argv[1]
+    pr_folder = sys.argv[2]
+
+    benchmark_results = collect_benchmark_results(master_folder, pr_folder)
+
     comments = get_pr_comments(REPO_OWNER, REPO_NAME, PR_NUMBER)
     if comments and len(comments) > 0:
         first_comment = comments[0]

scripts/ci/run_benchmarks.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+set -e pipefail
+
+function run_benchmarks_for_folder {
+    echo "Running benchmarks for $1"
+
+    FOLDER=$1
+    RESULTS_FOLDER=$2
+
+    mkdir -p $RESULTS_FOLDER
+
+    ./$FOLDER/build/src/benchmarks/match-bench "./$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench"
+    ./$FOLDER/build/src/benchmarks/match-bench "./$FOLDER/test/data/ch/monaco.osrm" > "$RESULTS_FOLDER/match_ch.bench"
+    ./$FOLDER/build/src/benchmarks/alias-bench > "$RESULTS_FOLDER/alias.bench"
+    ./$FOLDER/build/src/benchmarks/json-render-bench  "./$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench"
+    ./$FOLDER/build/src/benchmarks/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench"
+    ./$FOLDER/build/src/benchmarks/rtree-bench "./$FOLDER/test/data/monaco.osrm.ramIndex" "./$FOLDER/test/data/monaco.osrm.fileIndex" "./$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench"
+}
+
+run_benchmarks_for_folder $1 "${1}_results"
+run_benchmarks_for_folder $2 "${2}_results"
+