From a11928787861a1e80adcd658124807542e66931a Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Sat, 15 Jun 2024 18:56:10 +0200 Subject: [PATCH] Calculate confidence interval for benchmark measurements --- scripts/ci/e2e_benchmark.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/ci/e2e_benchmark.py b/scripts/ci/e2e_benchmark.py index 089524936..ebe0f0827 100644 --- a/scripts/ci/e2e_benchmark.py +++ b/scripts/ci/e2e_benchmark.py @@ -102,13 +102,13 @@ def main(): times = runner.run(args.method, args.host, args.num_requests) all_times.extend(times) - total_time, total_ci = calculate_confidence_interval(np.sum(all_times, axis=1)) - min_time, min_ci = calculate_confidence_interval(np.min(all_times, axis=1)) - mean_time, mean_ci = calculate_confidence_interval(np.mean(all_times, axis=1)) - median_time, median_ci = calculate_confidence_interval(np.median(all_times, axis=1)) - perc_95_time, perc_95_ci = calculate_confidence_interval(np.percentile(all_times, 95, axis=1)) - perc_99_time, perc_99_ci = calculate_confidence_interval(np.percentile(all_times, 99, axis=1)) - max_time, max_ci = calculate_confidence_interval(np.max(all_times, axis=1)) + total_time, total_ci = calculate_confidence_interval(np.sum(all_times, axis=0)) + min_time, min_ci = calculate_confidence_interval(np.min(all_times, axis=0)) + mean_time, mean_ci = calculate_confidence_interval(np.mean(all_times, axis=0)) + median_time, median_ci = calculate_confidence_interval(np.median(all_times, axis=0)) + perc_95_time, perc_95_ci = calculate_confidence_interval(np.percentile(all_times, 95, axis=0)) + perc_99_time, perc_99_ci = calculate_confidence_interval(np.percentile(all_times, 99, axis=0)) + max_time, max_ci = calculate_confidence_interval(np.max(all_times, axis=0)) print(f'Total: {total_time}ms ± {total_ci:.2f}ms') print(f"Min time: {min_time}ms ± {min_ci:.2f}ms")