Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Calculate confidence interval for benchmark measurements

Browse Source
This commit is contained in:
Siarhei Fedartsou 2024-06-15 23:00:30 +02:00
parent 19662450b1
commit 2eb620bb33
2 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/osrm-backend.yml vendored
View File

@ -676,7 +676,7 @@ jobs:
path: pr path: pr
- name: Install dependencies - name: Install dependencies
run: | run: |
python3 -m pip install "conan<2.0.0" "requests==2.31.0" "numpy==1.26.4" python3 -m pip install "conan<2.0.0" "requests==2.31.0" "numpy==1.26.4" "scipy==1.13.1"
sudo apt-get update -y && sudo apt-get install ccache sudo apt-get update -y && sudo apt-get install ccache
- name: Prepare data - name: Prepare data
run: | run: |

3
scripts/ci/e2e_benchmark.py
View File

@ -7,6 +7,7 @@ import csv
import numpy as np import numpy as np
import time import time
import argparse import argparse
from scipy import stats
class BenchmarkRunner: class BenchmarkRunner:
def __init__(self, gps_traces_file_path): def __init__(self, gps_traces_file_path):
@ -82,7 +83,7 @@ def calculate_confidence_interval(data):
assert len(data) == 5, f"Shape: {data.shape}" assert len(data) == 5, f"Shape: {data.shape}"
mean = np.mean(data) mean = np.mean(data)
std_err = np.std(data, ddof=1) / np.sqrt(len(data)) std_err = np.std(data, ddof=1) / np.sqrt(len(data))
h = std_err * 1.96 # 95% confidence interval h = std_err * stats.t.ppf((1 + 0.95) / 2., len(data) - 1) # 95% confidence interval using t-distribution
return mean, h return mean, h
def main(): def main():