From 2d0baa739c51bbc63415ec1ccdfb3ed028454f55 Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Sat, 15 Jun 2024 21:20:40 +0200 Subject: [PATCH] Calculate confidence interval for benchmark measurements --- scripts/ci/e2e_benchmark.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/ci/e2e_benchmark.py b/scripts/ci/e2e_benchmark.py index d1620fc8f..f7763769d 100644 --- a/scripts/ci/e2e_benchmark.py +++ b/scripts/ci/e2e_benchmark.py @@ -78,6 +78,7 @@ class BenchmarkRunner: raise Exception(f"Unknown benchmark: {benchmark_name}") def calculate_confidence_interval(data): + print('Data: ', data.shape) assert len(data) == 5 mean = np.mean(data) std_err = np.std(data, ddof=1) / np.sqrt(len(data)) @@ -89,7 +90,7 @@ def main(): parser.add_argument('--host', type=str, required=True, help='Host URL') parser.add_argument('--method', type=str, required=True, choices=['route', 'table', 'match', 'nearest', 'trip'], help='Benchmark method') parser.add_argument('--num_requests', type=int, required=True, help='Number of requests to perform') - parser.add_argument('--iterations', type=int, default=5, required=True, help='Number of iterations to run the benchmark') + parser.add_argument('--iterations', type=int, required=True, help='Number of iterations to run the benchmark') parser.add_argument('--gps_traces_file_path', type=str, required=True, help='Path to the GPS traces file') args = parser.parse_args() @@ -104,6 +105,8 @@ def main(): all_times.append(times) all_times = np.asarray(all_times) + print('Shape: ', all_times.shape) + total_time, total_ci = calculate_confidence_interval(np.sum(all_times, axis=0)) min_time, min_ci = calculate_confidence_interval(np.min(all_times, axis=0)) mean_time, mean_ci = calculate_confidence_interval(np.mean(all_times, axis=0))