From 685f4cca3c0d8d323f4e17fd47e894c968fc34b4 Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Sun, 16 Jun 2024 09:26:41 +0200 Subject: [PATCH] Calculate confidence interval for benchmark measurements --- scripts/ci/e2e_benchmark.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/scripts/ci/e2e_benchmark.py b/scripts/ci/e2e_benchmark.py index 74ddf02c3..7b835e1ea 100644 --- a/scripts/ci/e2e_benchmark.py +++ b/scripts/ci/e2e_benchmark.py @@ -8,6 +8,11 @@ import numpy as np import time import argparse from scipy import stats +import logging + +logging.getLogger("urllib3").setLevel(logging.CRITICAL) +logging.getLogger("requests").setLevel(logging.CRITICAL) + class BenchmarkRunner: def __init__(self, gps_traces_file_path): @@ -91,7 +96,7 @@ def calculate_confidence_interval(data): mean = np.mean(data) std_err = np.std(data, ddof=1) / np.sqrt(len(data)) h = std_err * stats.t.ppf((1 + 0.95) / 2., len(data) - 1) # 95% confidence interval using t-distribution - return mean, h + return mean, h, np.min(data) def main(): parser = argparse.ArgumentParser(description='Run GPS benchmark tests.') @@ -115,15 +120,15 @@ def main(): print('Shape: ', all_times.shape) - total_time, total_ci = calculate_confidence_interval(np.sum(all_times, axis=1)) - min_time, min_ci = calculate_confidence_interval(np.min(all_times, axis=1)) - mean_time, mean_ci = calculate_confidence_interval(np.mean(all_times, axis=1)) - median_time, median_ci = calculate_confidence_interval(np.median(all_times, axis=1)) - perc_95_time, perc_95_ci = calculate_confidence_interval(np.percentile(all_times, 95, axis=1)) - perc_99_time, perc_99_ci = calculate_confidence_interval(np.percentile(all_times, 99, axis=1)) - max_time, max_ci = calculate_confidence_interval(np.max(all_times, axis=1)) + total_time, total_ci, total_min = calculate_confidence_interval(np.sum(all_times, axis=1)) + min_time, min_ci, _ = calculate_confidence_interval(np.min(all_times, axis=1)) + mean_time, mean_ci, _ = calculate_confidence_interval(np.mean(all_times, axis=1)) + median_time, median_ci, _ = calculate_confidence_interval(np.median(all_times, axis=1)) + perc_95_time, perc_95_ci, _ = calculate_confidence_interval(np.percentile(all_times, 95, axis=1)) + perc_99_time, perc_99_ci, _ = calculate_confidence_interval(np.percentile(all_times, 99, axis=1)) + max_time, max_ci, _ = calculate_confidence_interval(np.max(all_times, axis=1)) - print(f'Total: {total_time:.2f}ms ± {total_ci:.2f}ms') + print(f'Total: {total_time:.2f}ms ± {total_ci:.2f}ms. Min: {total_min:.2f}ms') print(f"Min time: {min_time:.2f}ms ± {min_ci:.2f}ms") print(f"Mean time: {mean_time:.2f}ms ± {mean_ci:.2f}ms") print(f"Median time: {median_time:.2f}ms ± {median_ci:.2f}ms")