From 5ee5b898541f0c52352499ae864c90898d675036 Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Sun, 26 May 2024 19:07:32 +0200 Subject: [PATCH] wip --- scripts/ci/locustfile.py | 26 +++++++++--------- scripts/ci/process_locust_benchmark.py | 38 ++++++++++++++++++++++++++ scripts/ci/run_benchmarks.sh | 17 +++--------- 3 files changed, 55 insertions(+), 26 deletions(-) create mode 100644 scripts/ci/process_locust_benchmark.py diff --git a/scripts/ci/locustfile.py b/scripts/ci/locustfile.py index 00c09f3ce..60d18ce75 100644 --- a/scripts/ci/locustfile.py +++ b/scripts/ci/locustfile.py @@ -30,22 +30,22 @@ class OSRMTasks(TaskSet): self.client.get(f"/route/v1/driving/{start_coord};{end_coord}?overview=full&steps=true", name="driving") - @task - def get_table(self): - num_coords = random.randint(3, 250) - selected_coords = random.sample(self.coordinates, num_coords) - coords_str = ";".join([f"{coord[1]:.6f},{coord[0]:.6f}" for coord in selected_coords]) + # @task + # def get_table(self): + # num_coords = random.randint(3, 50) + # selected_coords = random.sample(self.coordinates, num_coords) + # coords_str = ";".join([f"{coord[1]:.6f},{coord[0]:.6f}" for coord in selected_coords]) - self.client.get(f"/table/v1/driving/{coords_str}", name="table") + # self.client.get(f"/table/v1/driving/{coords_str}", name="table") - @task - def get_match(self): - num_coords = random.randint(3, 250) - track_id = random.choice(self.track_ids) - track_coords = self.tracks[track_id][:num_coords] - coords_str = ";".join([f"{coord[1]:.6f},{coord[0]:.6f}" for coord in track_coords]) + # @task + # def get_match(self): + # num_coords = random.randint(3, 50) + # track_id = random.choice(self.track_ids) + # track_coords = self.tracks[track_id][:num_coords] + # coords_str = ";".join([f"{coord[1]:.6f},{coord[0]:.6f}" for coord in track_coords]) - self.client.get(f"/match/v1/driving/{coords_str}?steps=true", name="match") + # self.client.get(f"/match/v1/driving/{coords_str}?steps=true", name="match") @task def get_nearest(self): diff --git a/scripts/ci/process_locust_benchmark.py b/scripts/ci/process_locust_benchmark.py new file mode 100644 index 000000000..b495455be --- /dev/null +++ b/scripts/ci/process_locust_benchmark.py @@ -0,0 +1,38 @@ +import sys +import csv + +def check_csv_is_empty(file_path): + with open(file_path, 'r') as file: + lines = file.readlines() + return len(lines) <= 1 + +def main(locust_csv_base_name, prefix, output_folder): + if not check_csv_is_empty(f"{locust_csv_base_name}_exceptions.csv") or not check_csv_is_empty(f"{locust_csv_base_name}_failures"): + raise Exception("There are exceptions or failures in the locust benchmark") + + with open(f"{locust_csv_base_name}_stats.csv", 'r') as file: + reader = csv.DictReader(file) + for row in reader: + name = row['Name'] + if name == 'Aggregated': continue + + statistics = f''' +Request Count: {row['Request Count']} +Requests/s: {row['Requests/s']}req/s +Failures/s: {row['Failures/s']}fail/s +avg: {row['Average Response Time']}ms +50%: {row['50%']} +75%: {row['75%']}ms +95%: {row['95%']}ms +98%: {row['98%']}ms +99%: {row['99%']}ms +max: {row['Max Response Time']}ms +''' + with open(f"{prefix}_{name}_{output_folder}.bench", 'w') as f: + f.write(statistics) + +if __name__ == '__main__': + if len(sys.argv) != 4: + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + main(sys.argv[1], sys.argv[2], sys.argv[3]) \ No newline at end of file diff --git a/scripts/ci/run_benchmarks.sh b/scripts/ci/run_benchmarks.sh index ae7ae827d..022173a10 100755 --- a/scripts/ci/run_benchmarks.sh +++ b/scripts/ci/run_benchmarks.sh @@ -11,7 +11,7 @@ function run_benchmarks_for_folder { BENCHMARKS_FOLDER="$FOLDER/build/src/benchmarks" - # ./$BENCHMARKS_FOLDER/match-bench "./$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench" + ./$BENCHMARKS_FOLDER/match-bench "./$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench" # ./$BENCHMARKS_FOLDER/match-bench "./$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench" # ./$BENCHMARKS_FOLDER/route-bench "./$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench" # ./$BENCHMARKS_FOLDER/route-bench "./$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench" @@ -42,20 +42,11 @@ function run_benchmarks_for_folder { --spawn-rate 1 \ --host http://localhost:5000 \ --run-time 1m \ - --csv=results \ + --csv=locust_results \ --loglevel ERROR - echo "STATS: " - cat results_stats.csv - # echo "FAILURES: " - # cat results_failures.csv - # echo "EXCEPTIONS: " - # cat results_exceptions.csv - # echo "STATS HISTORY: " - # cat results_stats_history.csv -# 2024-05-26T16:48:29.2826130Z Type,Name,Request Count,Failure Count,Median Response Time,Average Response Time,Min Response Time,Max Response Time,Average Content Size,Requests/s,Failures/s,50%,66%,75%,80%,90%,95%,98%,99%,99.9%,99.99%,100% -# 2024-05-26T16:48:29.2827761Z GET,/route/v1/driving,7668,0,4,15.894057099894761,1.168323000001692,137.2044539999706,26387.300208659362,127.83539601258259,0.0,4,6,44,45,47,48,49,50,100,140,140 -# 2024-05-26T16:48:29.2829001Z ,Aggregated,7668,0,4,15.894057099894761,1.168323000001692,137.2044539999706,26387.300208659362,127.83539601258259,0.0,4,6,44,45,47,48,49,50,100,140,140 + python3 $FOLDER/scripts/ci/process_locust_results.py locust_results mld $RESULTS_FOLDER + kill -0 $OSRM_ROUTED_PID fi