wip
This commit is contained in:
Siarhei Fedartsou
parent
862307e31e
commit
9b7ae757c4
2
.github/workflows/osrm-backend.yml
vendored
2
.github/workflows/osrm-backend.yml
vendored
@ -656,7 +656,7 @@ jobs:
|
|||||||
path: pr
|
path: pr
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: |
|
run: |
|
||||||
python3 -m pip install "conan<2.0.0" "requests==2.31.0" "locust==2.28.0" numpy
|
python3 -m pip install "conan<2.0.0" "requests==2.31.0" "numpy==1.26.4"
|
||||||
sudo apt-get update -y && sudo apt-get install ccache
|
sudo apt-get update -y && sudo apt-get install ccache
|
||||||
- name: Prepare data
|
- name: Prepare data
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
@ -37,7 +37,7 @@ class BenchmarkRunner:
|
|||||||
if code == 'NoSegment' or code == 'NoMatch':
|
if code == 'NoSegment' or code == 'NoMatch':
|
||||||
continue
|
continue
|
||||||
raise Exception(f"Error: {response.status_code} {response.text}")
|
raise Exception(f"Error: {response.status_code} {response.text}")
|
||||||
times.append(end_time - start_time)
|
times.append((end_time - start_time) * 1000) # convert to ms
|
||||||
|
|
||||||
return times
|
return times
|
||||||
|
|
||||||
@ -86,13 +86,13 @@ def main():
|
|||||||
runner = BenchmarkRunner()
|
runner = BenchmarkRunner()
|
||||||
times = runner.run(args.method, args.host, args.num_requests)
|
times = runner.run(args.method, args.host, args.num_requests)
|
||||||
|
|
||||||
print(f'Total: {np.sum(times)}')
|
print(f'Total: {np.sum(times)}ms')
|
||||||
print(f"Min time: {np.min(times)}")
|
print(f"Min time: {np.min(times)}ms")
|
||||||
print(f"Mean time: {np.mean(times)}")
|
print(f"Mean time: {np.mean(times)}ms")
|
||||||
print(f"Median time: {np.median(times)}")
|
print(f"Median time: {np.median(times)}ms")
|
||||||
print(f"95th percentile: {np.percentile(times, 95)}")
|
print(f"95th percentile: {np.percentile(times, 95)}ms")
|
||||||
print(f"99th percentile: {np.percentile(times, 99)}")
|
print(f"99th percentile: {np.percentile(times, 99)}ms")
|
||||||
print(f"Max time: {np.max(times)}")
|
print(f"Max time: {np.max(times)}ms")
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
main()
|
main()
|
||||||
|
|||||||
@ -1,73 +0,0 @@
|
|||||||
from locust import HttpUser, TaskSet, task, constant
|
|
||||||
import csv
|
|
||||||
import random
|
|
||||||
from collections import defaultdict
|
|
||||||
import os
|
|
||||||
|
|
||||||
class OSRMTasks(TaskSet):
|
|
||||||
def on_start(self):
|
|
||||||
random.seed(42)
|
|
||||||
|
|
||||||
self.coordinates = []
|
|
||||||
self.tracks = defaultdict(list)
|
|
||||||
|
|
||||||
gps_traces_file_path = os.path.expanduser('~/gps_traces.csv')
|
|
||||||
with open(gps_traces_file_path, 'r') as file:
|
|
||||||
reader = csv.DictReader(file)
|
|
||||||
for row in reader:
|
|
||||||
coord = (float(row['Latitude']), float(row['Longitude']))
|
|
||||||
self.coordinates.append(coord)
|
|
||||||
self.tracks[row['TrackID']].append(coord)
|
|
||||||
self.track_ids = list(self.tracks.keys())
|
|
||||||
|
|
||||||
@task
|
|
||||||
def get_route(self):
|
|
||||||
start = (52.1887017,12.9249433) # self.coordinates[0] #random.choice(self.coordinates)
|
|
||||||
end = (52.7606980,14.0521751) #self.coordinates[-1] #random.choice(self.coordinates)
|
|
||||||
|
|
||||||
start_coord = f"{start[1]:.6f},{start[0]:.6f}"
|
|
||||||
end_coord = f"{end[1]:.6f},{end[0]:.6f}"
|
|
||||||
|
|
||||||
self.client.get(f"/route/v1/driving/{start_coord};{end_coord}?overview=full&steps=true", name="route")
|
|
||||||
|
|
||||||
# @task
|
|
||||||
# def get_table(self):
|
|
||||||
# num_coords = random.randint(3, 100)
|
|
||||||
# selected_coords = random.sample(self.coordinates, num_coords)
|
|
||||||
# coords_str = ";".join([f"{coord[1]:.6f},{coord[0]:.6f}" for coord in selected_coords])
|
|
||||||
|
|
||||||
# self.client.get(f"/table/v1/driving/{coords_str}", name="table")
|
|
||||||
|
|
||||||
# @task
|
|
||||||
# def get_match(self):
|
|
||||||
# num_coords = random.randint(50, 100)
|
|
||||||
# track_id = random.choice(self.track_ids)
|
|
||||||
# track_coords = self.tracks[track_id][:num_coords]
|
|
||||||
# coords_str = ";".join([f"{coord[1]:.6f},{coord[0]:.6f}" for coord in track_coords])
|
|
||||||
# radiues_str = ";".join([f"{random.randint(5, 20)}" for _ in range(len(track_coords))])
|
|
||||||
|
|
||||||
# with self.client.get(f"/match/v1/driving/{coords_str}?steps=true&radiuses={radiues_str}", name="match", catch_response=True) as response:
|
|
||||||
# if response.status_code == 400:
|
|
||||||
# j = response.json()
|
|
||||||
# # it is expected that some of requests will fail with such error: map matching fails sometimes
|
|
||||||
# if j['code'] == 'NoSegment' or j['code'] == 'NoMatch':
|
|
||||||
# response.success()
|
|
||||||
|
|
||||||
# @task
|
|
||||||
# def get_nearest(self):
|
|
||||||
# coord = random.choice(self.coordinates)
|
|
||||||
# coord_str = f"{coord[1]:.6f},{coord[0]:.6f}"
|
|
||||||
|
|
||||||
# self.client.get(f"/nearest/v1/driving/{coord_str}", name="nearest")
|
|
||||||
|
|
||||||
# @task
|
|
||||||
# def get_trip(self):
|
|
||||||
# num_coords = random.randint(2, 10)
|
|
||||||
# selected_coords = random.sample(self.coordinates, num_coords)
|
|
||||||
# coords_str = ";".join([f"{coord[1]:.6f},{coord[0]:.6f}" for coord in selected_coords])
|
|
||||||
|
|
||||||
# self.client.get(f"/trip/v1/driving/{coords_str}?steps=true", name="trip")
|
|
||||||
|
|
||||||
class OSRMUser(HttpUser):
|
|
||||||
tasks = [OSRMTasks]
|
|
||||||
wait_time = constant(0.05)
|
|
||||||
@ -1,31 +0,0 @@
|
|||||||
import sys
|
|
||||||
import csv
|
|
||||||
|
|
||||||
def main(locust_csv_base_name, suffix, output_folder):
|
|
||||||
with open(f"{locust_csv_base_name}_stats.csv", 'r') as file:
|
|
||||||
reader = csv.DictReader(file)
|
|
||||||
for row in reader:
|
|
||||||
name = row['Name']
|
|
||||||
if name == 'Aggregated': continue
|
|
||||||
|
|
||||||
statistics = f'''
|
|
||||||
requests: {row['Request Count']}
|
|
||||||
failures: {row['Failure Count']}
|
|
||||||
req/s: {float(row['Requests/s']):.3f}req/s
|
|
||||||
avg: {float(row['Average Response Time']):.3f}ms
|
|
||||||
50%: {row['50%']}ms
|
|
||||||
75%: {row['75%']}ms
|
|
||||||
95%: {row['95%']}ms
|
|
||||||
98%: {row['98%']}ms
|
|
||||||
99%: {row['99%']}ms
|
|
||||||
min: {float(row['Min Response Time']):.3f}ms
|
|
||||||
max: {float(row['Max Response Time']):.3f}ms
|
|
||||||
'''
|
|
||||||
with open(f"{output_folder}/e2e_{name}_{suffix}.bench", 'w') as f:
|
|
||||||
f.write(statistics)
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
if len(sys.argv) != 4:
|
|
||||||
print(f"Usage: {sys.argv[0]} <locust csv base name> <suffix> <output folder>")
|
|
||||||
sys.exit(1)
|
|
||||||
main(sys.argv[1], sys.argv[2], sys.argv[3])
|
|
||||||
@ -18,7 +18,7 @@ function run_benchmarks_for_folder {
|
|||||||
|
|
||||||
FOLDER=$1
|
FOLDER=$1
|
||||||
RESULTS_FOLDER=$2
|
RESULTS_FOLDER=$2
|
||||||
LOCUSTFILE_FOLDER=$3
|
SCRIPTS_FOLDER=$3
|
||||||
|
|
||||||
mkdir -p $RESULTS_FOLDER
|
mkdir -p $RESULTS_FOLDER
|
||||||
|
|
||||||
@ -48,20 +48,9 @@ function run_benchmarks_for_folder {
|
|||||||
|
|
||||||
# wait for osrm-routed to start
|
# wait for osrm-routed to start
|
||||||
curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true"
|
curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true"
|
||||||
# locust -f $LOCUSTFILE_FOLDER/scripts/ci/locustfile.py \
|
|
||||||
# --headless \
|
|
||||||
# --processes -1 \
|
|
||||||
# --users 1 \
|
|
||||||
# --spawn-rate 1 \
|
|
||||||
# --host http://localhost:5000 \
|
|
||||||
# --run-time 1m \
|
|
||||||
# --csv=locust_results_$ALGORITHM \
|
|
||||||
# --loglevel ERROR
|
|
||||||
|
|
||||||
# python3 $FOLDER/scripts/ci/process_locust_benchmark_results.py locust_results_$ALGORITHM $ALGORITHM $RESULTS_FOLDER
|
|
||||||
|
|
||||||
for METHOD in route table nearest trip match; do
|
for METHOD in route table nearest trip match; do
|
||||||
python3 $LOCUSTFILE_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --num_requests 10000 > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench
|
python3 $SCRIPTS_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --num_requests 1000 > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench
|
||||||
done
|
done
|
||||||
|
|
||||||
kill -0 $OSRM_ROUTED_PID
|
kill -0 $OSRM_ROUTED_PID
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user