Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

wip

Browse Source
This commit is contained in:
Siarhei Fedartsou 2024-05-26 13:20:57 +02:00
parent 8c907a218e
commit aee8f0febe
2 changed files with 127 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

207
.github/workflows/osrm-backend.yml vendored
View File

@ -100,50 +100,50 @@ jobs:
# npm run docs && ./scripts/error_on_dirty.sh # npm run docs && ./scripts/error_on_dirty.sh
# npm audit --production # npm audit --production
docker-image: # docker-image:
# needs: format-taginfo-docs # # needs: format-taginfo-docs
runs-on: ubuntu-22.04 # runs-on: ubuntu-22.04
continue-on-error: false # continue-on-error: false
steps: # steps:
- name: Check out the repo # - name: Check out the repo
uses: actions/checkout@v4 # uses: actions/checkout@v4
- run: pip install locust # - run: pip install locust
- name: Enable osm.pbf cache # - name: Enable osm.pbf cache
uses: actions/cache@v4 # uses: actions/cache@v4
with: # with:
path: berlin-latest.osm.pbf # path: berlin-latest.osm.pbf
key: v1-berlin-osm-pbf # key: v1-berlin-osm-pbf
restore-keys: | # restore-keys: |
v1-berlin-osm-pbf # v1-berlin-osm-pbf
- name: Docker build # - name: Docker build
run: | # run: |
docker build -t osrm-backend-local -f docker/Dockerfile . # docker build -t osrm-backend-local -f docker/Dockerfile .
- name: Test Docker image # - name: Test Docker image
run: | # run: |
if [ ! -f "${PWD}/berlin-latest.osm.pbf" ]; then # if [ ! -f "${PWD}/berlin-latest.osm.pbf" ]; then
wget http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf # wget http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf
fi # fi
TAG=osrm-backend-local # TAG=osrm-backend-local
# when `--memory-swap` value equals `--memory` it means container won't use swap # # when `--memory-swap` value equals `--memory` it means container won't use swap
# see https://docs.docker.com/config/containers/resource_constraints/#--memory-swap-details # # see https://docs.docker.com/config/containers/resource_constraints/#--memory-swap-details
MEMORY_ARGS="--memory=1g --memory-swap=1g" # MEMORY_ARGS="--memory=1g --memory-swap=1g"
docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-extract --dump-nbg-graph -p /opt/car.lua /data/berlin-latest.osm.pbf # docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-extract --dump-nbg-graph -p /opt/car.lua /data/berlin-latest.osm.pbf
docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-components /data/berlin-latest.osrm.nbg /data/berlin-latest.geojson # docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-components /data/berlin-latest.osrm.nbg /data/berlin-latest.geojson
if [ ! -s "${PWD}/berlin-latest.geojson" ] # if [ ! -s "${PWD}/berlin-latest.geojson" ]
then # then
>&2 echo "No berlin-latest.geojson found" # >&2 echo "No berlin-latest.geojson found"
exit 1 # exit 1
fi # fi
# removing `.osrm.nbg` to check that whole pipeline works without it # # removing `.osrm.nbg` to check that whole pipeline works without it
rm -rf "${PWD}/berlin-latest.osrm.nbg" # rm -rf "${PWD}/berlin-latest.osrm.nbg"
docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-partition /data/berlin-latest.osrm # docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-partition /data/berlin-latest.osrm
docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-customize /data/berlin-latest.osrm # docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-customize /data/berlin-latest.osrm
docker run $MEMORY_ARGS --name=osrm-container -t -p 5000:5000 -v "${PWD}:/data" "${TAG}" osrm-routed --algorithm mld /data/berlin-latest.osrm & # docker run $MEMORY_ARGS --name=osrm-container -t -p 5000:5000 -v "${PWD}:/data" "${TAG}" osrm-routed --algorithm mld /data/berlin-latest.osrm &
curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true" # curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true"
locust -f locustfile.py --headless --users 10 --spawn-rate 1 --host http://localhost:5000 --run-time 1m # locust -f locustfile.py --headless --users 10 --spawn-rate 1 --host http://localhost:5000 --run-time 1m
docker stop osrm-container # docker stop osrm-container
# build-test-publish: # build-test-publish:
@ -613,68 +613,69 @@ jobs:
# token: ${{ secrets.GITHUB_TOKEN }} # token: ${{ secrets.GITHUB_TOKEN }}
# benchmarks: benchmarks:
# if: github.event_name == 'pull_request' if: github.event_name == 'pull_request'
#needs: [format-taginfo-docs] #needs: [format-taginfo-docs]
# runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
# env: env:
# CCOMPILER: clang-13 CCOMPILER: clang-13
# CXXCOMPILER: clang++-13 CXXCOMPILER: clang++-13
# CC: clang-13 CC: clang-13
# CXX: clang++-13 CXX: clang++-13
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# PR_NUMBER: ${{ github.event.pull_request.number }} PR_NUMBER: ${{ github.event.pull_request.number }}
# GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_REPOSITORY: ${{ github.repository }}
# steps: steps:
# - name: Enable compiler cache - name: Enable compiler cache
# uses: actions/cache@v4 uses: actions/cache@v4
# with: with:
# path: ~/.ccache path: ~/.ccache
# key: v1-ccache-benchmarks-${{ github.sha }} key: v1-ccache-benchmarks-${{ github.sha }}
# restore-keys: | restore-keys: |
# v1-ccache-benchmarks- v1-ccache-benchmarks-
# - name: Enable Conan cache - name: Enable Conan cache
# uses: actions/cache@v4 uses: actions/cache@v4
# with: with:
# path: ~/.conan path: ~/.conan
# key: v1-conan-benchmarks-${{ github.sha }} key: v1-conan-benchmarks-${{ github.sha }}
# restore-keys: | restore-keys: |
# v1-conan-benchmarks- v1-conan-benchmarks-
# - name: Checkout PR Branch - run: pip install locust
# uses: actions/checkout@v4 - name: Checkout PR Branch
# with: uses: actions/checkout@v4
# ref: ${{ github.head_ref }} with:
# path: pr ref: ${{ github.head_ref }}
# - run: python3 -m pip install "conan<2.0.0" "requests==2.31.0" path: pr
# - name: Build PR Branch - run: python3 -m pip install "conan<2.0.0" "requests==2.31.0"
# run: | - name: Build PR Branch
# mkdir -p pr/build run: |
# cd pr/build mkdir -p pr/build
# cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release .. cd pr/build
# make -j$(nproc) cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release ..
# make -j$(nproc) benchmarks make -j$(nproc)
# cd .. make -j$(nproc) benchmarks
# make -C test/data cd ..
# - name: Checkout Base Branch make -C test/data
# uses: actions/checkout@v4 - name: Checkout Base Branch
# with: uses: actions/checkout@v4
# ref: ${{ github.event.pull_request.base.ref }} with:
# path: base ref: ${{ github.event.pull_request.base.ref }}
# - name: Build Base Branch path: base
# run: | - name: Build Base Branch
# mkdir base/build run: |
# cd base/build mkdir base/build
# cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release .. cd base/build
# make -j$(nproc) cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release ..
# make -j$(nproc) benchmarks make -j$(nproc)
# cd .. make -j$(nproc) benchmarks
# make -C test/data cd ..
# - name: Run Benchmarks make -C test/data
# run: | - name: Run Benchmarks
# ./pr/scripts/ci/run_benchmarks.sh base pr run: |
# - name: Post Benchmark Results ./pr/scripts/ci/run_benchmarks.sh base pr
# run: | - name: Post Benchmark Results
# python3 pr/scripts/ci/post_benchmark_results.py base_results pr_results run: |
python3 pr/scripts/ci/post_benchmark_results.py base_results pr_results
# ci-complete: # ci-complete:
# runs-on: ubuntu-22.04 # runs-on: ubuntu-22.04

30
scripts/ci/run_benchmarks.sh
View File

@ -11,14 +11,28 @@ function run_benchmarks_for_folder {
BENCHMARKS_FOLDER="$FOLDER/build/src/benchmarks" BENCHMARKS_FOLDER="$FOLDER/build/src/benchmarks"
./$BENCHMARKS_FOLDER/match-bench "./$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench" # ./$BENCHMARKS_FOLDER/match-bench "./$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench"
./$BENCHMARKS_FOLDER/match-bench "./$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench" # ./$BENCHMARKS_FOLDER/match-bench "./$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench"
./$BENCHMARKS_FOLDER/route-bench "./$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench" || true # TODO: remove `true` when this benchmark will be merged to master # ./$BENCHMARKS_FOLDER/route-bench "./$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench" || true # TODO: remove `true` when this benchmark will be merged to master
./$BENCHMARKS_FOLDER/route-bench "./$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench" || true # TODO: remove `true` when this benchmark will be merged to master # ./$BENCHMARKS_FOLDER/route-bench "./$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench" || true # TODO: remove `true` when this benchmark will be merged to master
./$BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench" # ./$BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench"
./$BENCHMARKS_FOLDER/json-render-bench "./$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench" # ./$BENCHMARKS_FOLDER/json-render-bench "./$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench"
./$BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench" # ./$BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench"
./$BENCHMARKS_FOLDER/rtree-bench "./$FOLDER/test/data/monaco.osrm.ramIndex" "./$FOLDER/test/data/monaco.osrm.fileIndex" "./$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench" # ./$BENCHMARKS_FOLDER/rtree-bench "./$FOLDER/test/data/monaco.osrm.ramIndex" "./$FOLDER/test/data/monaco.osrm.fileIndex" "./$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench"
BINARIES_FOLDER="$FOLDER/build"
wget http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf
$BINARIES_FOLDER/osrm-extract -p ./profiles/car.lua ./berlin-latest.osm.pbf
$BINARIES_FOLDER/osrm-partition ./berlin-latest.osrm
$BINARIES_FOLDER/osrm-customize ./berlin-latest.osrm
$BINARIES_FOLDER/osrm-routed --algorithm mld /data/berlin-latest.osrm &
OSRM_ROUTED_PID=$!
# TODO: save results
curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true"
locust -f locustfile.py --headless --users 10 --spawn-rate 1 --host http://localhost:5000 --run-time 1m
kill -0 $OSRM_ROUTED_PID
} }
run_benchmarks_for_folder $1 "${1}_results" run_benchmarks_for_folder $1 "${1}_results"