Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Calculate confidence interval for benchmark measurements

Browse Source
This commit is contained in:
Siarhei Fedartsou 2024-06-15 21:20:40 +02:00
parent 5585ccc224
commit 2d0baa739c
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scripts/ci/e2e_benchmark.py
View File

@ -78,6 +78,7 @@ class BenchmarkRunner:
raise Exception(f"Unknown benchmark: {benchmark_name}")
def calculate_confidence_interval(data):
print('Data: ', data.shape)
assert len(data) == 5
mean = np.mean(data)
std_err = np.std(data, ddof=1) / np.sqrt(len(data))
@ -89,7 +90,7 @@ def main():
parser.add_argument('--host', type=str, required=True, help='Host URL')
parser.add_argument('--method', type=str, required=True, choices=['route', 'table', 'match', 'nearest', 'trip'], help='Benchmark method')
parser.add_argument('--num_requests', type=int, required=True, help='Number of requests to perform')
parser.add_argument('--iterations', type=int, default=5, required=True, help='Number of iterations to run the benchmark')
parser.add_argument('--iterations', type=int, required=True, help='Number of iterations to run the benchmark')
parser.add_argument('--gps_traces_file_path', type=str, required=True, help='Path to the GPS traces file')
args = parser.parse_args()
@ -104,6 +105,8 @@ def main():
all_times.append(times)
all_times = np.asarray(all_times)
print('Shape: ', all_times.shape)
total_time, total_ci = calculate_confidence_interval(np.sum(all_times, axis=0))
min_time, min_ci = calculate_confidence_interval(np.min(all_times, axis=0))
mean_time, mean_ci = calculate_confidence_interval(np.mean(all_times, axis=0))