Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

wip

Browse Source
This commit is contained in:
Siarhei Fedartsou 2024-10-29 18:06:43 +01:00
parent 7c1f0bf62f
commit 12163b0be9
4 changed files with 127 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

251
.github/workflows/osrm-backend.yml vendored
View File

@ -24,7 +24,7 @@ concurrency:
jobs:
windows-release-node:
#needs: format-taginfo-docs
needs: format-taginfo-docs
runs-on: windows-2022
continue-on-error: false
env:
@ -51,11 +51,8 @@ jobs:
cd build
python3 -m venv .venv
source .venv/Scripts/Activate
python3 -m pip install conan==2.7.1
conan profile detect --force
cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_CONAN=ON -DENABLE_NODE_BINDINGS=ON ..
@ -657,130 +654,130 @@ jobs:
ccache -p
ccache -s
# benchmarks:
# if: github.event_name == 'pull_request'
# needs: [format-taginfo-docs]
# runs-on: self-hosted
# env:
# CCOMPILER: clang-16
# CXXCOMPILER: clang++-16
# CC: clang-16
# CXX: clang++-16
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# PR_NUMBER: ${{ github.event.pull_request.number }}
# GITHUB_REPOSITORY: ${{ github.repository }}
# RUN_BIG_BENCHMARK: ${{ contains(github.event.pull_request.labels.*.name, 'Performance') }}
# steps:
# - name: Checkout PR Branch
# uses: actions/checkout@v4
# with:
# ref: ${{ github.head_ref }}
# path: pr
# - name: Activate virtualenv
# run: |
# python3 -m venv .venv
# source .venv/bin/activate
# echo PATH=$PATH >> $GITHUB_ENV
# pip install "conan==2.7.1" "requests==2.31.0" "numpy==1.26.4"
# - name: Prepare data
# run: |
# if [ "$RUN_BIG_BENCHMARK" = "true" ]; then
# rm -rf ~/data.osm.pbf
# wget http://download.geofabrik.de/europe/poland-latest.osm.pbf -O ~/data.osm.pbf --quiet
# gunzip -c ./pr/test/data/poland_gps_traces.csv.gz > ~/gps_traces.csv
# else
# if [ ! -f "~/data.osm.pbf" ]; then
# wget http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf -O ~/data.osm.pbf
# else
# echo "Using cached data.osm.pbf"
# fi
# gunzip -c ./pr/test/data/berlin_gps_traces.csv.gz > ~/gps_traces.csv
# fi
# - name: Prepare environment
# run: |
# echo "CCACHE_DIR=$HOME/.ccache" >> $GITHUB_ENV
# mkdir -p $HOME/.ccache
# ccache --zero-stats
# ccache --max-size=256M
# - name: Checkout Base Branch
# uses: actions/checkout@v4
# with:
# ref: ${{ github.event.pull_request.base.ref }}
# path: base
# - name: Build Base Branch
# run: |
# cd base
# npm ci --ignore-scripts
# cd ..
# mkdir base/build
# cd base/build
# cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release -DENABLE_NODE_BINDINGS=ON ..
# make -j$(nproc)
# make -j$(nproc) benchmarks
# cd ..
# make -C test/data
# - name: Build PR Branch
# run: |
# cd pr
# npm ci --ignore-scripts
# cd ..
# mkdir -p pr/build
# cd pr/build
# cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release -DENABLE_NODE_BINDINGS=ON ..
# make -j$(nproc)
# make -j$(nproc) benchmarks
# cd ..
# make -C test/data
# # we run benchmarks in tmpfs to avoid impact of disk IO
# - name: Create folder for tmpfs
# run: |
# # if by any chance it was mounted before(e.g. due to previous job failed), unmount it
# sudo umount ~/benchmarks | true
# rm -rf ~/benchmarks
# mkdir -p ~/benchmarks
# # see https://llvm.org/docs/Benchmarking.html
# - name: Run PR Benchmarks
# run: |
# sudo cset shield -c 2-3 -k on
# sudo mount -t tmpfs -o size=4g none ~/benchmarks
# cp -rf pr/build ~/benchmarks/build
# cp -rf pr/lib ~/benchmarks/lib
# mkdir -p ~/benchmarks/test
# cp -rf pr/test/data ~/benchmarks/test/data
# cp -rf pr/profiles ~/benchmarks/profiles
benchmarks:
if: github.event_name == 'pull_request'
needs: [format-taginfo-docs]
runs-on: self-hosted
env:
CCOMPILER: clang-16
CXXCOMPILER: clang++-16
CC: clang-16
CXX: clang++-16
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
GITHUB_REPOSITORY: ${{ github.repository }}
RUN_BIG_BENCHMARK: ${{ contains(github.event.pull_request.labels.*.name, 'Performance') }}
steps:
- name: Checkout PR Branch
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
path: pr
- name: Activate virtualenv
run: |
python3 -m venv .venv
source .venv/bin/activate
echo PATH=$PATH >> $GITHUB_ENV
pip install "conan==2.7.1" "requests==2.31.0" "numpy==1.26.4"
- name: Prepare data
run: |
if [ "$RUN_BIG_BENCHMARK" = "true" ]; then
rm -rf ~/data.osm.pbf
wget http://download.geofabrik.de/europe/poland-latest.osm.pbf -O ~/data.osm.pbf --quiet
gunzip -c ./pr/test/data/poland_gps_traces.csv.gz > ~/gps_traces.csv
else
if [ ! -f "~/data.osm.pbf" ]; then
wget http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf -O ~/data.osm.pbf
else
echo "Using cached data.osm.pbf"
fi
gunzip -c ./pr/test/data/berlin_gps_traces.csv.gz > ~/gps_traces.csv
fi
- name: Prepare environment
run: |
echo "CCACHE_DIR=$HOME/.ccache" >> $GITHUB_ENV
mkdir -p $HOME/.ccache
ccache --zero-stats
ccache --max-size=256M
- name: Checkout Base Branch
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.base.ref }}
path: base
- name: Build Base Branch
run: |
cd base
npm ci --ignore-scripts
cd ..
mkdir base/build
cd base/build
cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release -DENABLE_NODE_BINDINGS=ON ..
make -j$(nproc)
make -j$(nproc) benchmarks
cd ..
make -C test/data
- name: Build PR Branch
run: |
cd pr
npm ci --ignore-scripts
cd ..
mkdir -p pr/build
cd pr/build
cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release -DENABLE_NODE_BINDINGS=ON ..
make -j$(nproc)
make -j$(nproc) benchmarks
cd ..
make -C test/data
# we run benchmarks in tmpfs to avoid impact of disk IO
- name: Create folder for tmpfs
run: |
# if by any chance it was mounted before(e.g. due to previous job failed), unmount it
sudo umount ~/benchmarks | true
rm -rf ~/benchmarks
mkdir -p ~/benchmarks
# see https://llvm.org/docs/Benchmarking.html
- name: Run PR Benchmarks
run: |
sudo cset shield -c 2-3 -k on
sudo mount -t tmpfs -o size=4g none ~/benchmarks
cp -rf pr/build ~/benchmarks/build
cp -rf pr/lib ~/benchmarks/lib
mkdir -p ~/benchmarks/test
cp -rf pr/test/data ~/benchmarks/test/data
cp -rf pr/profiles ~/benchmarks/profiles
# sudo cset shield --exec -- ./pr/scripts/ci/run_benchmarks.sh -f ~/benchmarks -r $(pwd)/pr_results -s $(pwd)/pr -b ~/benchmarks/build -o ~/data.osm.pbf -g ~/gps_traces.csv
# sudo umount ~/benchmarks
# sudo cset shield --reset
# - name: Run Base Benchmarks
# run: |
# sudo cset shield -c 2-3 -k on
# sudo mount -t tmpfs -o size=4g none ~/benchmarks
# cp -rf base/build ~/benchmarks/build
# cp -rf base/lib ~/benchmarks/lib
# mkdir -p ~/benchmarks/test
# cp -rf base/test/data ~/benchmarks/test/data
# cp -rf base/profiles ~/benchmarks/profiles
sudo cset shield --exec -- ./pr/scripts/ci/run_benchmarks.sh -f ~/benchmarks -r $(pwd)/pr_results -s $(pwd)/pr -b ~/benchmarks/build -o ~/data.osm.pbf -g ~/gps_traces.csv
sudo umount ~/benchmarks
sudo cset shield --reset
- name: Run Base Benchmarks
run: |
sudo cset shield -c 2-3 -k on
sudo mount -t tmpfs -o size=4g none ~/benchmarks
cp -rf base/build ~/benchmarks/build
cp -rf base/lib ~/benchmarks/lib
mkdir -p ~/benchmarks/test
cp -rf base/test/data ~/benchmarks/test/data
cp -rf base/profiles ~/benchmarks/profiles
# # TODO: remove it when base branch will have this file at needed location
# if [ ! -f ~/benchmarks/test/data/portugal_to_korea.json ]; then
# cp base/src/benchmarks/portugal_to_korea.json ~/benchmarks/test/data/portugal_to_korea.json
# fi
# # we intentionally use scripts from PR branch to be able to update them and see results in the same PR
# sudo cset shield --exec -- cset shield --exec -- ./pr/scripts/ci/run_benchmarks.sh -f ~/benchmarks -r $(pwd)/base_results -s $(pwd)/pr -b ~/benchmarks/build -o ~/data.osm.pbf -g ~/gps_traces.csv
# sudo umount ~/benchmarks
# sudo cset shield --reset
# - name: Post Benchmark Results
# run: |
# python3 pr/scripts/ci/post_benchmark_results.py base_results pr_results
# - name: Show CCache statistics
# run: |
# ccache -p
# ccache -s
# TODO: remove it when base branch will have this file at needed location
if [ ! -f ~/benchmarks/test/data/portugal_to_korea.json ]; then
cp base/src/benchmarks/portugal_to_korea.json ~/benchmarks/test/data/portugal_to_korea.json
fi
# we intentionally use scripts from PR branch to be able to update them and see results in the same PR
sudo cset shield --exec -- cset shield --exec -- ./pr/scripts/ci/run_benchmarks.sh -f ~/benchmarks -r $(pwd)/base_results -s $(pwd)/pr -b ~/benchmarks/build -o ~/data.osm.pbf -g ~/gps_traces.csv
sudo umount ~/benchmarks
sudo cset shield --reset
- name: Post Benchmark Results
run: |
python3 pr/scripts/ci/post_benchmark_results.py base_results pr_results
- name: Show CCache statistics
run: |
ccache -p
ccache -s
# ci-complete:
# runs-on: ubuntu-22.04
# needs: [build-test-publish, docker-image-matrix, windows-release-node, benchmarks]
# steps:
# - run: echo "CI complete"
ci-complete:
runs-on: ubuntu-22.04
needs: [build-test-publish, docker-image-matrix, windows-release-node, benchmarks]
steps:
- run: echo "CI complete"

6
CMakeLists.txt
View File

@ -22,6 +22,7 @@ if (NOT WIN32 AND NOT DEFINED ENV{OSRM_BUILD_DIR})
set(ENV{OSRM_BUILD_DIR} ${CMAKE_CURRENT_BINARY_DIR})
endif()
option(ENABLE_CONAN "Use conan for dependencies" OFF)
option(ENABLE_CCACHE "Speed up incremental rebuilds via ccache" ON)
option(BUILD_TOOLS "Build OSRM tools" OFF)
option(BUILD_PACKAGE "Build OSRM package" OFF)
@ -45,7 +46,6 @@ if (ENABLE_CONAN)
else()
set(CMAKE_TOOLCHAIN_FILE "./conan_toolchain.cmake")
# message(STATUS "Conan install stdout: ${CONAN_EXECUTABLE} install .. --output-folder=. --build=missing --settings compiler.cppstd=20 --settings build_type=${CMAKE_BUILD_TYPE}")
execute_process(
COMMAND ${CONAN_EXECUTABLE} install .. --output-folder=. --build=missing --settings compiler.cppstd=20 --settings build_type=${CMAKE_BUILD_TYPE}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
@ -53,11 +53,9 @@ if (ENABLE_CONAN)
ERROR_VARIABLE conan_stderr
RESULT_VARIABLE conan_install_result
)
message(STATUS "Conan install stdout: ${conan_stdout} ${CMAKE_BUILD_TYPE}")
message(STATUS "Conan install stderr: ${conan_stderr}")
if (NOT conan_install_result EQUAL 0)
message(FATAL_ERROR "Conan install failed.")
message(FATAL_ERROR "Conan install failed: ${conan_install_result}. Stderr: ${conan_stderr}. Stdout: ${conan_stdout}")
endif()
endif()
endif()

42
benchmark.js
View File

@ -1,42 +0,0 @@
var OSRM = require('.');
var monaco_mld_path = require('./test/nodejs/constants').mld_data_path;
var two_test_coordinates = require('./test/nodejs/constants').two_test_coordinates;
const { performance } = require('perf_hooks');
const osrm = new OSRM({path: monaco_mld_path, algorithm: 'MLD'});
const numberOfRoutes = 10;
let completedRoutes = 0;
let totalTime = 0;
function benchmarkRoutes() {
const startTime = performance.now();
for (let i = 0; i < numberOfRoutes; i++) {
const options = {
coordinates: [two_test_coordinates[0], two_test_coordinates[1]],
annotations: ['distance']
};
for (let i = 0; i < 1000; ++i) {
options.coordinates.push(two_test_coordinates[i % 2], two_test_coordinates[(i + 1) % 2]);
}
osrm.table(options, function(err, route) {
if (err) {
console.error(err);
return;
}
completedRoutes++;
if (completedRoutes === numberOfRoutes) {
const endTime = performance.now();
totalTime = endTime - startTime;
console.log(`Total time for ${numberOfRoutes} routes: ${totalTime}ms`);
console.log(`Average time per route: ${totalTime / numberOfRoutes}ms`);
}
});
}
}
benchmarkRoutes();

2
unit_tests/CMakeLists.txt
View File

@ -126,7 +126,7 @@ add_executable(util-tests
${UtilTestsSources}
$<TARGET_OBJECTS:UTIL> $<TARGET_OBJECTS:MICROTAR>)
if(NOT WIN32 AND NOT Boost_USE_STATIC_LIBS AND NOT ENABLE_CONAN)
if(NOT WIN32 AND NOT Boost_USE_STATIC_LIBS)
add_definitions(-DBOOST_TEST_DYN_LINK)
endif()