From 9b7ae757c419c426fd58bd449ef383c2eb4d43b3 Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Sat, 8 Jun 2024 09:16:03 +0200 Subject: [PATCH] wip --- .github/workflows/osrm-backend.yml | 2 +- scripts/ci/e2e_benchmark.py | 16 ++-- scripts/ci/locustfile.py | 73 ------------------- .../ci/process_locust_benchmark_results.py | 31 -------- scripts/ci/run_benchmarks.sh | 15 +--- 5 files changed, 11 insertions(+), 126 deletions(-) delete mode 100644 scripts/ci/locustfile.py delete mode 100644 scripts/ci/process_locust_benchmark_results.py diff --git a/.github/workflows/osrm-backend.yml b/.github/workflows/osrm-backend.yml index b3ae2f1cb..61e570394 100644 --- a/.github/workflows/osrm-backend.yml +++ b/.github/workflows/osrm-backend.yml @@ -656,7 +656,7 @@ jobs: path: pr - name: Install dependencies run: | - python3 -m pip install "conan<2.0.0" "requests==2.31.0" "locust==2.28.0" numpy + python3 -m pip install "conan<2.0.0" "requests==2.31.0" "numpy==1.26.4" sudo apt-get update -y && sudo apt-get install ccache - name: Prepare data run: | diff --git a/scripts/ci/e2e_benchmark.py b/scripts/ci/e2e_benchmark.py index 23c343ba9..f8514b957 100644 --- a/scripts/ci/e2e_benchmark.py +++ b/scripts/ci/e2e_benchmark.py @@ -37,7 +37,7 @@ class BenchmarkRunner: if code == 'NoSegment' or code == 'NoMatch': continue raise Exception(f"Error: {response.status_code} {response.text}") - times.append(end_time - start_time) + times.append((end_time - start_time) * 1000) # convert to ms return times @@ -86,13 +86,13 @@ def main(): runner = BenchmarkRunner() times = runner.run(args.method, args.host, args.num_requests) - print(f'Total: {np.sum(times)}') - print(f"Min time: {np.min(times)}") - print(f"Mean time: {np.mean(times)}") - print(f"Median time: {np.median(times)}") - print(f"95th percentile: {np.percentile(times, 95)}") - print(f"99th percentile: {np.percentile(times, 99)}") - print(f"Max time: {np.max(times)}") + print(f'Total: {np.sum(times)}ms') + print(f"Min time: {np.min(times)}ms") + print(f"Mean time: {np.mean(times)}ms") + print(f"Median time: {np.median(times)}ms") + print(f"95th percentile: {np.percentile(times, 95)}ms") + print(f"99th percentile: {np.percentile(times, 99)}ms") + print(f"Max time: {np.max(times)}ms") if __name__ == '__main__': main() diff --git a/scripts/ci/locustfile.py b/scripts/ci/locustfile.py deleted file mode 100644 index 431fd9938..000000000 --- a/scripts/ci/locustfile.py +++ /dev/null @@ -1,73 +0,0 @@ -from locust import HttpUser, TaskSet, task, constant -import csv -import random -from collections import defaultdict -import os - -class OSRMTasks(TaskSet): - def on_start(self): - random.seed(42) - - self.coordinates = [] - self.tracks = defaultdict(list) - - gps_traces_file_path = os.path.expanduser('~/gps_traces.csv') - with open(gps_traces_file_path, 'r') as file: - reader = csv.DictReader(file) - for row in reader: - coord = (float(row['Latitude']), float(row['Longitude'])) - self.coordinates.append(coord) - self.tracks[row['TrackID']].append(coord) - self.track_ids = list(self.tracks.keys()) - - @task - def get_route(self): - start = (52.1887017,12.9249433) # self.coordinates[0] #random.choice(self.coordinates) - end = (52.7606980,14.0521751) #self.coordinates[-1] #random.choice(self.coordinates) - - start_coord = f"{start[1]:.6f},{start[0]:.6f}" - end_coord = f"{end[1]:.6f},{end[0]:.6f}" - - self.client.get(f"/route/v1/driving/{start_coord};{end_coord}?overview=full&steps=true", name="route") - - # @task - # def get_table(self): - # num_coords = random.randint(3, 100) - # selected_coords = random.sample(self.coordinates, num_coords) - # coords_str = ";".join([f"{coord[1]:.6f},{coord[0]:.6f}" for coord in selected_coords]) - - # self.client.get(f"/table/v1/driving/{coords_str}", name="table") - - # @task - # def get_match(self): - # num_coords = random.randint(50, 100) - # track_id = random.choice(self.track_ids) - # track_coords = self.tracks[track_id][:num_coords] - # coords_str = ";".join([f"{coord[1]:.6f},{coord[0]:.6f}" for coord in track_coords]) - # radiues_str = ";".join([f"{random.randint(5, 20)}" for _ in range(len(track_coords))]) - - # with self.client.get(f"/match/v1/driving/{coords_str}?steps=true&radiuses={radiues_str}", name="match", catch_response=True) as response: - # if response.status_code == 400: - # j = response.json() - # # it is expected that some of requests will fail with such error: map matching fails sometimes - # if j['code'] == 'NoSegment' or j['code'] == 'NoMatch': - # response.success() - - # @task - # def get_nearest(self): - # coord = random.choice(self.coordinates) - # coord_str = f"{coord[1]:.6f},{coord[0]:.6f}" - - # self.client.get(f"/nearest/v1/driving/{coord_str}", name="nearest") - - # @task - # def get_trip(self): - # num_coords = random.randint(2, 10) - # selected_coords = random.sample(self.coordinates, num_coords) - # coords_str = ";".join([f"{coord[1]:.6f},{coord[0]:.6f}" for coord in selected_coords]) - - # self.client.get(f"/trip/v1/driving/{coords_str}?steps=true", name="trip") - -class OSRMUser(HttpUser): - tasks = [OSRMTasks] - wait_time = constant(0.05) diff --git a/scripts/ci/process_locust_benchmark_results.py b/scripts/ci/process_locust_benchmark_results.py deleted file mode 100644 index ad71daf7f..000000000 --- a/scripts/ci/process_locust_benchmark_results.py +++ /dev/null @@ -1,31 +0,0 @@ -import sys -import csv - -def main(locust_csv_base_name, suffix, output_folder): - with open(f"{locust_csv_base_name}_stats.csv", 'r') as file: - reader = csv.DictReader(file) - for row in reader: - name = row['Name'] - if name == 'Aggregated': continue - - statistics = f''' -requests: {row['Request Count']} -failures: {row['Failure Count']} -req/s: {float(row['Requests/s']):.3f}req/s -avg: {float(row['Average Response Time']):.3f}ms -50%: {row['50%']}ms -75%: {row['75%']}ms -95%: {row['95%']}ms -98%: {row['98%']}ms -99%: {row['99%']}ms -min: {float(row['Min Response Time']):.3f}ms -max: {float(row['Max Response Time']):.3f}ms -''' - with open(f"{output_folder}/e2e_{name}_{suffix}.bench", 'w') as f: - f.write(statistics) - -if __name__ == '__main__': - if len(sys.argv) != 4: - print(f"Usage: {sys.argv[0]} ") - sys.exit(1) - main(sys.argv[1], sys.argv[2], sys.argv[3]) \ No newline at end of file diff --git a/scripts/ci/run_benchmarks.sh b/scripts/ci/run_benchmarks.sh index 19bee9dba..b6f08c4d7 100755 --- a/scripts/ci/run_benchmarks.sh +++ b/scripts/ci/run_benchmarks.sh @@ -18,7 +18,7 @@ function run_benchmarks_for_folder { FOLDER=$1 RESULTS_FOLDER=$2 - LOCUSTFILE_FOLDER=$3 + SCRIPTS_FOLDER=$3 mkdir -p $RESULTS_FOLDER @@ -48,20 +48,9 @@ function run_benchmarks_for_folder { # wait for osrm-routed to start curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true" - # locust -f $LOCUSTFILE_FOLDER/scripts/ci/locustfile.py \ - # --headless \ - # --processes -1 \ - # --users 1 \ - # --spawn-rate 1 \ - # --host http://localhost:5000 \ - # --run-time 1m \ - # --csv=locust_results_$ALGORITHM \ - # --loglevel ERROR - - # python3 $FOLDER/scripts/ci/process_locust_benchmark_results.py locust_results_$ALGORITHM $ALGORITHM $RESULTS_FOLDER for METHOD in route table nearest trip match; do - python3 $LOCUSTFILE_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --num_requests 10000 > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench + python3 $SCRIPTS_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --num_requests 1000 > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench done kill -0 $OSRM_ROUTED_PID