Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Calculate confidence interval for benchmark measurements

Browse Source
This commit is contained in:
Siarhei Fedartsou 2024-06-16 11:03:36 +02:00
parent 685f4cca3c
commit 91459df87e
2 changed files with 16 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
scripts/ci/e2e_benchmark.py
View File

@ -91,12 +91,21 @@ class BenchmarkRunner:
else: else:
raise Exception(f"Unknown benchmark: {benchmark_name}") raise Exception(f"Unknown benchmark: {benchmark_name}")
def bootstrap_confidence_interval(data, num_samples=1000, confidence_level=0.95):
means = []
for _ in range(num_samples):
sample = np.random.choice(data, size=len(data), replace=True)
means.append(np.mean(sample))
lower_bound = np.percentile(means, (1 - confidence_level) / 2 * 100)
upper_bound = np.percentile(means, (1 + confidence_level) / 2 * 100)
mean = np.mean(means)
return mean, lower_bound, upper_bound
def calculate_confidence_interval(data): def calculate_confidence_interval(data):
#assert len(data) == 5, f"Shape: {data.shape}" mean, lower, upper = bootstrap_confidence_interval(data)
mean = np.mean(data) min_value = np.min(data)
std_err = np.std(data, ddof=1) / np.sqrt(len(data)) return mean, (upper - lower) / 2, min_value
h = std_err * stats.t.ppf((1 + 0.95) / 2., len(data) - 1) # 95% confidence interval using t-distribution
return mean, h, np.min(data)
def main(): def main():
parser = argparse.ArgumentParser(description='Run GPS benchmark tests.') parser = argparse.ArgumentParser(description='Run GPS benchmark tests.')
@ -108,6 +117,7 @@ def main():
args = parser.parse_args() args = parser.parse_args()
np.random.seed(42)
runner = BenchmarkRunner(args.gps_traces_file_path) runner = BenchmarkRunner(args.gps_traces_file_path)

2
scripts/ci/run_benchmarks.sh
View File

@ -101,7 +101,7 @@ function run_benchmarks_for_folder {
fi fi
for METHOD in route nearest trip table match; do for METHOD in route nearest trip table match; do
python3 $SCRIPTS_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --iterations 15 --num_requests 1000 --gps_traces_file_path $GPS_TRACES > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench python3 $SCRIPTS_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --iterations 5 --num_requests 1000 --gps_traces_file_path $GPS_TRACES > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench
done done
kill -9 $OSRM_ROUTED_PID kill -9 $OSRM_ROUTED_PID