Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Calculate confidence interval for benchmark measurements

Browse Source
This commit is contained in:
Siarhei Fedartsou 2024-06-16 09:26:41 +02:00
parent 02fbdd65df
commit 685f4cca3c
1 changed files with 14 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
scripts/ci/e2e_benchmark.py
View File

@ -8,6 +8,11 @@ import numpy as np
import time import time
import argparse import argparse
from scipy import stats from scipy import stats
import logging
logging.getLogger("urllib3").setLevel(logging.CRITICAL)
logging.getLogger("requests").setLevel(logging.CRITICAL)
class BenchmarkRunner: class BenchmarkRunner:
def __init__(self, gps_traces_file_path): def __init__(self, gps_traces_file_path):
@ -91,7 +96,7 @@ def calculate_confidence_interval(data):
mean = np.mean(data) mean = np.mean(data)
std_err = np.std(data, ddof=1) / np.sqrt(len(data)) std_err = np.std(data, ddof=1) / np.sqrt(len(data))
h = std_err * stats.t.ppf((1 + 0.95) / 2., len(data) - 1) # 95% confidence interval using t-distribution h = std_err * stats.t.ppf((1 + 0.95) / 2., len(data) - 1) # 95% confidence interval using t-distribution
return mean, h return mean, h, np.min(data)
def main(): def main():
parser = argparse.ArgumentParser(description='Run GPS benchmark tests.') parser = argparse.ArgumentParser(description='Run GPS benchmark tests.')
@ -115,15 +120,15 @@ def main():
print('Shape: ', all_times.shape) print('Shape: ', all_times.shape)
total_time, total_ci = calculate_confidence_interval(np.sum(all_times, axis=1)) total_time, total_ci, total_min = calculate_confidence_interval(np.sum(all_times, axis=1))
min_time, min_ci = calculate_confidence_interval(np.min(all_times, axis=1)) min_time, min_ci, _ = calculate_confidence_interval(np.min(all_times, axis=1))
mean_time, mean_ci = calculate_confidence_interval(np.mean(all_times, axis=1)) mean_time, mean_ci, _ = calculate_confidence_interval(np.mean(all_times, axis=1))
median_time, median_ci = calculate_confidence_interval(np.median(all_times, axis=1)) median_time, median_ci, _ = calculate_confidence_interval(np.median(all_times, axis=1))
perc_95_time, perc_95_ci = calculate_confidence_interval(np.percentile(all_times, 95, axis=1)) perc_95_time, perc_95_ci, _ = calculate_confidence_interval(np.percentile(all_times, 95, axis=1))
perc_99_time, perc_99_ci = calculate_confidence_interval(np.percentile(all_times, 99, axis=1)) perc_99_time, perc_99_ci, _ = calculate_confidence_interval(np.percentile(all_times, 99, axis=1))
max_time, max_ci = calculate_confidence_interval(np.max(all_times, axis=1)) max_time, max_ci, _ = calculate_confidence_interval(np.max(all_times, axis=1))
print(f'Total: {total_time:.2f}ms ± {total_ci:.2f}ms') print(f'Total: {total_time:.2f}ms ± {total_ci:.2f}ms. Min: {total_min:.2f}ms')
print(f"Min time: {min_time:.2f}ms ± {min_ci:.2f}ms") print(f"Min time: {min_time:.2f}ms ± {min_ci:.2f}ms")
print(f"Mean time: {mean_time:.2f}ms ± {mean_ci:.2f}ms") print(f"Mean time: {mean_time:.2f}ms ± {mean_ci:.2f}ms")
print(f"Median time: {median_time:.2f}ms ± {median_ci:.2f}ms") print(f"Median time: {median_time:.2f}ms ± {median_ci:.2f}ms")