diff --git a/.github/workflows/osrm-backend.yml b/.github/workflows/osrm-backend.yml index da4b68e38..3dceefd32 100644 --- a/.github/workflows/osrm-backend.yml +++ b/.github/workflows/osrm-backend.yml @@ -100,50 +100,50 @@ jobs: # npm run docs && ./scripts/error_on_dirty.sh # npm audit --production - docker-image: - # needs: format-taginfo-docs - runs-on: ubuntu-22.04 - continue-on-error: false - steps: - - name: Check out the repo - uses: actions/checkout@v4 - - run: pip install locust - - name: Enable osm.pbf cache - uses: actions/cache@v4 - with: - path: berlin-latest.osm.pbf - key: v1-berlin-osm-pbf - restore-keys: | - v1-berlin-osm-pbf - - name: Docker build - run: | - docker build -t osrm-backend-local -f docker/Dockerfile . - - name: Test Docker image - run: | - if [ ! -f "${PWD}/berlin-latest.osm.pbf" ]; then - wget http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf - fi - TAG=osrm-backend-local - # when `--memory-swap` value equals `--memory` it means container won't use swap - # see https://docs.docker.com/config/containers/resource_constraints/#--memory-swap-details - MEMORY_ARGS="--memory=1g --memory-swap=1g" - docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-extract --dump-nbg-graph -p /opt/car.lua /data/berlin-latest.osm.pbf - docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-components /data/berlin-latest.osrm.nbg /data/berlin-latest.geojson - if [ ! -s "${PWD}/berlin-latest.geojson" ] - then - >&2 echo "No berlin-latest.geojson found" - exit 1 - fi + # docker-image: + # # needs: format-taginfo-docs + # runs-on: ubuntu-22.04 + # continue-on-error: false + # steps: + # - name: Check out the repo + # uses: actions/checkout@v4 + # - run: pip install locust + # - name: Enable osm.pbf cache + # uses: actions/cache@v4 + # with: + # path: berlin-latest.osm.pbf + # key: v1-berlin-osm-pbf + # restore-keys: | + # v1-berlin-osm-pbf + # - name: Docker build + # run: | + # docker build -t osrm-backend-local -f docker/Dockerfile . + # - name: Test Docker image + # run: | + # if [ ! -f "${PWD}/berlin-latest.osm.pbf" ]; then + # wget http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf + # fi + # TAG=osrm-backend-local + # # when `--memory-swap` value equals `--memory` it means container won't use swap + # # see https://docs.docker.com/config/containers/resource_constraints/#--memory-swap-details + # MEMORY_ARGS="--memory=1g --memory-swap=1g" + # docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-extract --dump-nbg-graph -p /opt/car.lua /data/berlin-latest.osm.pbf + # docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-components /data/berlin-latest.osrm.nbg /data/berlin-latest.geojson + # if [ ! -s "${PWD}/berlin-latest.geojson" ] + # then + # >&2 echo "No berlin-latest.geojson found" + # exit 1 + # fi - # removing `.osrm.nbg` to check that whole pipeline works without it - rm -rf "${PWD}/berlin-latest.osrm.nbg" + # # removing `.osrm.nbg` to check that whole pipeline works without it + # rm -rf "${PWD}/berlin-latest.osrm.nbg" - docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-partition /data/berlin-latest.osrm - docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-customize /data/berlin-latest.osrm - docker run $MEMORY_ARGS --name=osrm-container -t -p 5000:5000 -v "${PWD}:/data" "${TAG}" osrm-routed --algorithm mld /data/berlin-latest.osrm & - curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true" - locust -f locustfile.py --headless --users 10 --spawn-rate 1 --host http://localhost:5000 --run-time 1m - docker stop osrm-container + # docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-partition /data/berlin-latest.osrm + # docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-customize /data/berlin-latest.osrm + # docker run $MEMORY_ARGS --name=osrm-container -t -p 5000:5000 -v "${PWD}:/data" "${TAG}" osrm-routed --algorithm mld /data/berlin-latest.osrm & + # curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true" + # locust -f locustfile.py --headless --users 10 --spawn-rate 1 --host http://localhost:5000 --run-time 1m + # docker stop osrm-container # build-test-publish: @@ -613,68 +613,69 @@ jobs: # token: ${{ secrets.GITHUB_TOKEN }} - # benchmarks: - # if: github.event_name == 'pull_request' - # needs: [format-taginfo-docs] - # runs-on: ubuntu-22.04 - # env: - # CCOMPILER: clang-13 - # CXXCOMPILER: clang++-13 - # CC: clang-13 - # CXX: clang++-13 - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # PR_NUMBER: ${{ github.event.pull_request.number }} - # GITHUB_REPOSITORY: ${{ github.repository }} - # steps: - # - name: Enable compiler cache - # uses: actions/cache@v4 - # with: - # path: ~/.ccache - # key: v1-ccache-benchmarks-${{ github.sha }} - # restore-keys: | - # v1-ccache-benchmarks- - # - name: Enable Conan cache - # uses: actions/cache@v4 - # with: - # path: ~/.conan - # key: v1-conan-benchmarks-${{ github.sha }} - # restore-keys: | - # v1-conan-benchmarks- - # - name: Checkout PR Branch - # uses: actions/checkout@v4 - # with: - # ref: ${{ github.head_ref }} - # path: pr - # - run: python3 -m pip install "conan<2.0.0" "requests==2.31.0" - # - name: Build PR Branch - # run: | - # mkdir -p pr/build - # cd pr/build - # cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release .. - # make -j$(nproc) - # make -j$(nproc) benchmarks - # cd .. - # make -C test/data - # - name: Checkout Base Branch - # uses: actions/checkout@v4 - # with: - # ref: ${{ github.event.pull_request.base.ref }} - # path: base - # - name: Build Base Branch - # run: | - # mkdir base/build - # cd base/build - # cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release .. - # make -j$(nproc) - # make -j$(nproc) benchmarks - # cd .. - # make -C test/data - # - name: Run Benchmarks - # run: | - # ./pr/scripts/ci/run_benchmarks.sh base pr - # - name: Post Benchmark Results - # run: | - # python3 pr/scripts/ci/post_benchmark_results.py base_results pr_results + benchmarks: + if: github.event_name == 'pull_request' + #needs: [format-taginfo-docs] + runs-on: ubuntu-22.04 + env: + CCOMPILER: clang-13 + CXXCOMPILER: clang++-13 + CC: clang-13 + CXX: clang++-13 + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + GITHUB_REPOSITORY: ${{ github.repository }} + steps: + - name: Enable compiler cache + uses: actions/cache@v4 + with: + path: ~/.ccache + key: v1-ccache-benchmarks-${{ github.sha }} + restore-keys: | + v1-ccache-benchmarks- + - name: Enable Conan cache + uses: actions/cache@v4 + with: + path: ~/.conan + key: v1-conan-benchmarks-${{ github.sha }} + restore-keys: | + v1-conan-benchmarks- + - run: pip install locust + - name: Checkout PR Branch + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + path: pr + - run: python3 -m pip install "conan<2.0.0" "requests==2.31.0" + - name: Build PR Branch + run: | + mkdir -p pr/build + cd pr/build + cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release .. + make -j$(nproc) + make -j$(nproc) benchmarks + cd .. + make -C test/data + - name: Checkout Base Branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.ref }} + path: base + - name: Build Base Branch + run: | + mkdir base/build + cd base/build + cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release .. + make -j$(nproc) + make -j$(nproc) benchmarks + cd .. + make -C test/data + - name: Run Benchmarks + run: | + ./pr/scripts/ci/run_benchmarks.sh base pr + - name: Post Benchmark Results + run: | + python3 pr/scripts/ci/post_benchmark_results.py base_results pr_results # ci-complete: # runs-on: ubuntu-22.04 diff --git a/scripts/ci/run_benchmarks.sh b/scripts/ci/run_benchmarks.sh index 6aea4e089..ea55f23df 100755 --- a/scripts/ci/run_benchmarks.sh +++ b/scripts/ci/run_benchmarks.sh @@ -11,14 +11,28 @@ function run_benchmarks_for_folder { BENCHMARKS_FOLDER="$FOLDER/build/src/benchmarks" - ./$BENCHMARKS_FOLDER/match-bench "./$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench" - ./$BENCHMARKS_FOLDER/match-bench "./$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench" - ./$BENCHMARKS_FOLDER/route-bench "./$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench" || true # TODO: remove `true` when this benchmark will be merged to master - ./$BENCHMARKS_FOLDER/route-bench "./$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench" || true # TODO: remove `true` when this benchmark will be merged to master - ./$BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench" - ./$BENCHMARKS_FOLDER/json-render-bench "./$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench" - ./$BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench" - ./$BENCHMARKS_FOLDER/rtree-bench "./$FOLDER/test/data/monaco.osrm.ramIndex" "./$FOLDER/test/data/monaco.osrm.fileIndex" "./$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench" + # ./$BENCHMARKS_FOLDER/match-bench "./$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench" + # ./$BENCHMARKS_FOLDER/match-bench "./$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench" + # ./$BENCHMARKS_FOLDER/route-bench "./$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench" || true # TODO: remove `true` when this benchmark will be merged to master + # ./$BENCHMARKS_FOLDER/route-bench "./$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench" || true # TODO: remove `true` when this benchmark will be merged to master + # ./$BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench" + # ./$BENCHMARKS_FOLDER/json-render-bench "./$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench" + # ./$BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench" + # ./$BENCHMARKS_FOLDER/rtree-bench "./$FOLDER/test/data/monaco.osrm.ramIndex" "./$FOLDER/test/data/monaco.osrm.fileIndex" "./$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench" + + BINARIES_FOLDER="$FOLDER/build" + wget http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf + $BINARIES_FOLDER/osrm-extract -p ./profiles/car.lua ./berlin-latest.osm.pbf + $BINARIES_FOLDER/osrm-partition ./berlin-latest.osrm + $BINARIES_FOLDER/osrm-customize ./berlin-latest.osrm + $BINARIES_FOLDER/osrm-routed --algorithm mld /data/berlin-latest.osrm & + OSRM_ROUTED_PID=$! + + # TODO: save results + curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true" + locust -f locustfile.py --headless --users 10 --spawn-rate 1 --host http://localhost:5000 --run-time 1m + + kill -0 $OSRM_ROUTED_PID } run_benchmarks_for_folder $1 "${1}_results"