Compare commits

...

53 Commits

Author SHA1 Message Date
Siarhei Fedartsou 8b878a841e Merge branch 'master' into sf-ankerl 2024-06-26 18:39:02 +02:00
Siarhei Fedartsou 8aade382bb wip 2024-06-26 18:10:53 +02:00
Siarhei Fedartsou a522016953 wip 2024-06-26 18:10:19 +02:00
Siarhei Fedartsou 0e17869e21 Re-use priority queue in StaticRTree (#6952) 2024-06-25 18:58:48 +02:00
Siarhei Fedartsou 6f444be1de ankerl 2024-06-25 18:27:53 +02:00
Siarhei Fedartsou 72527d63a0 Merge branch 'master' into sf-ankerl 2024-06-25 17:53:09 +02:00
Siarhei Fedartsou 5a48ce85b3 Use tmpfs for running benchmarks (#6966) 2024-06-25 17:49:41 +02:00
Siarhei Fedartsou 7d72dfebf7 Optimise encodePolyline function (#6940) 2024-06-22 20:04:32 +02:00
Siarhei Fedartsou 3d01d96036 Avoid reallocations in base64 encoding (#6951) 2024-06-22 08:50:18 +02:00
Siarhei Fedartsou e8da3d9231 Calculate confidence interval for benchmark measurements (#6950) 2024-06-21 21:43:27 +02:00
Siarhei Fedartsou d9ce9cf780 Get rid of unused Boost dependencies (#6960) 2024-06-21 19:04:34 +02:00
Siarhei Fedartsou 3285ce3b2f Configure stale bot (#6963) 2024-06-21 08:51:56 +02:00
Dennis Luxen f1eabc3ecc Replace boost::filesystem with std (#6432) 2024-06-20 21:44:28 +02:00
knowname c3f2a6cdb9 Add CI job for building Alpine-based Docker image (#6959) 2024-06-20 09:08:57 +02:00
knowname 97f676d5a3 add alpine dockerfile (#6958) 2024-06-19 14:15:23 +02:00
Siarhei Fedartsou 7ffc08be28 Bump versions of TBB, Expat and Lua installed via Conan (#6957) 2024-06-19 07:46:56 +02:00
Siarhei Fedartsou feb9389436 Apply micro-optimisation for Table & Trip APIs (#6949) 2024-06-15 18:57:26 +02:00
Siarhei Fedartsou a0eda3e7d7 Apply micro-optimisation for Route API (#6948) 2024-06-15 18:56:40 +02:00
Siarhei Fedartsou 7652f6ca6b Apply micro-optimisation for Match API (#6945) 2024-06-14 16:04:19 +02:00
Siarhei Fedartsou 3ff2819922 wip 2024-06-13 18:31:19 +02:00
Siarhei Fedartsou aa4e6b1cf3 Apply micro-optimisation for Nearest API (#6944) 2024-06-13 18:18:36 +02:00
Siarhei Fedartsou 559768c351 wip 2024-06-12 21:47:58 +02:00
Siarhei Fedartsou 316b8dbec2 wip 2024-06-12 21:05:39 +02:00
Siarhei Fedartsou 1724c3a7fa wip 2024-06-12 21:03:13 +02:00
Siarhei Fedartsou 112e089166 wip 2024-06-12 20:01:37 +02:00
Siarhei Fedartsou 250c406098 wip 2024-06-12 17:18:47 +02:00
Siarhei Fedartsou 11c3369cf4 wip 2024-06-12 13:33:18 +02:00
Siarhei Fedartsou f4ce1dc8f6 wip 2024-06-12 09:21:20 +02:00
Siarhei Fedartsou 49f07cbb8e wip 2024-06-11 22:41:49 +02:00
Siarhei Fedartsou 68d5dbc919 wip 2024-06-11 16:27:21 +02:00
Siarhei Fedartsou 068a1da098 wip 2024-06-11 09:14:57 +02:00
Siarhei Fedartsou de2f392960 Avoid copy of intersection in totalTurnAngle (#6938) 2024-06-10 22:40:37 +02:00
Siarhei Fedartsou ff02ae92f4 wip 2024-06-10 21:43:49 +02:00
Siarhei Fedartsou 0188d2bccd Use std::unordered_map::emplace instead of operator[] when producing JSONs (#6936) 2024-06-10 21:29:31 +02:00
Siarhei Fedartsou 054161fc7e Avoid copy of vectors in MakeRoute function (#6939) 2024-06-10 20:06:43 +02:00
Siarhei Fedartsou a65c9cbcb1 wip 2024-06-10 12:33:53 +02:00
Siarhei Fedartsou 13d8eebbca wip 2024-06-09 22:57:31 +02:00
Siarhei Fedartsou 93820be50e Avoid copy of intersection in totalTurnAngle 2024-06-09 19:37:44 +02:00
Siarhei Fedartsou 47d1630e7f wip 2024-06-09 18:17:27 +02:00
Siarhei Fedartsou 79da3793c1 wip 2024-06-09 17:07:35 +02:00
Siarhei Fedartsou c5cf8c31ac Merge branch 'master' into sf-ankerl 2024-06-09 17:03:53 +02:00
Siarhei Fedartsou 67558b796f wip 2024-06-09 13:11:09 +02:00
Siarhei Fedartsou 2a3f539bb2 wip 2024-06-09 11:13:31 +02:00
Siarhei Fedartsou ada954cd8d wip 2024-06-09 10:43:28 +02:00
Siarhei Fedartsou 9398bbc382 wip 2024-06-09 10:39:20 +02:00
Siarhei Fedartsou 2e3f3e90ef Merge branch 'master' into sf-ankerl 2024-06-08 22:29:49 +02:00
Siarhei Fedartsou 2e54842ce8 Merge branch 'master' into sf-ankerl 2024-06-05 21:48:14 +02:00
Siarhei Fedartsou cf4141dffd Try to use ankerl::unordered_dense::map instead of std::unordered_map in UnorderedMapStorage 2024-05-30 19:49:06 +02:00
Siarhei Fedartsou c3683201e6 Try to use ankerl::unordered_dense::map instead of std::unordered_map in UnorderedMapStorage 2024-05-30 19:21:38 +02:00
Siarhei Fedartsou 0f5ffc2a84 Try to use ankerl::unordered_dense::map instead of std::unordered_map in UnorderedMapStorage 2024-05-30 19:19:30 +02:00
Siarhei Fedartsou fafe1d4f81 Merge commit 'f1087e81ecdca5a59ba5ffca684c955c5b38f7c2' as 'third_party/unordered_dense' 2024-05-30 19:06:16 +02:00
Siarhei Fedartsou f1087e81ec Squashed 'third_party/unordered_dense/' content from commit 231e48c94
git-subtree-dir: third_party/unordered_dense
git-subtree-split: 231e48c9426bd21c273669e5fdcd042c146975cf
2024-05-30 19:06:16 +02:00
Siarhei Fedartsou 178bcb974e Add ankerl 2024-05-30 19:06:09 +02:00
221 changed files with 17552 additions and 1096 deletions
+10 -10
View File
@@ -6,6 +6,9 @@ on:
jobs:
publish:
strategy:
matrix:
docker-base-image: ["debian", "alpine"]
runs-on: ubuntu-latest
steps:
- name: Check out the repo
@@ -53,10 +56,10 @@ jobs:
with:
push: true
platforms: linux/amd64,linux/arm64
file: ./docker/Dockerfile
file: ./docker/Dockerfile-${{ matrix.docker-base-image }}
tags: ${{ steps.metadebug.outputs.tags }}
build-args: |
DOCKER_TAG=${{ join(steps.metadebug.outputs.tags ) }}
DOCKER_TAG=${{ join(steps.metadebug.outputs.tags ) }}-${{ matrix.docker-base-image }}
- name: Build container image - assertions
@@ -64,10 +67,10 @@ jobs:
with:
push: true
platforms: linux/amd64,linux/arm64
file: ./docker/Dockerfile
file: ./docker/Dockerfile-${{ matrix.docker-base-image }}
tags: ${{ steps.metaassertions.outputs.tags }}
build-args: |
DOCKER_TAG=${{ join(steps.metaassertions.outputs.tags ) }}
DOCKER_TAG=${{ join(steps.metaassertions.outputs.tags ) }}-${{ matrix.docker-base-image }}
# build and publish "normal" image as last to get it listed on top
- name: Build container image - normal
@@ -75,10 +78,7 @@ jobs:
with:
push: true
platforms: linux/amd64,linux/arm64
file: ./docker/Dockerfile
tags: ${{ steps.meta.outputs.tags }}
file: ./docker/Dockerfile-${{ matrix.docker-base-image }}
tags: ${{ steps.meta.outputs.tags }}
build-args: |
DOCKER_TAG=${{ join(steps.meta.outputs.tags ) }}
DOCKER_TAG=${{ join(steps.meta.outputs.tags ) }}-${{ matrix.docker-base-image }}
+44 -16
View File
@@ -116,7 +116,10 @@ jobs:
npm run docs && ./scripts/error_on_dirty.sh
npm audit --production
docker-image:
docker-image-matrix:
strategy:
matrix:
docker-base-image: ["debian", "alpine"]
needs: format-taginfo-docs
runs-on: ubuntu-22.04
continue-on-error: false
@@ -132,7 +135,7 @@ jobs:
v1-berlin-osm-pbf
- name: Docker build
run: |
docker build -t osrm-backend-local -f docker/Dockerfile .
docker build -t osrm-backend-local -f docker/Dockerfile-${{ matrix.docker-base-image }} .
- name: Test Docker image
run: |
if [ ! -f "${PWD}/berlin-latest.osm.pbf" ]; then
@@ -149,7 +152,6 @@ jobs:
>&2 echo "No berlin-latest.geojson found"
exit 1
fi
# removing `.osrm.nbg` to check that whole pipeline works without it
rm -rf "${PWD}/berlin-latest.osrm.nbg"
@@ -686,7 +688,7 @@ jobs:
gunzip -c ./pr/test/data/poland_gps_traces.csv.gz > ~/gps_traces.csv
else
if [ ! -f "~/data.osm.pbf" ]; then
wget http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf -O ~/data.osm.pbf
wget http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf -O ~/data.osm.pbf --quiet
else
echo "Using cached data.osm.pbf"
fi
@@ -698,15 +700,6 @@ jobs:
mkdir -p $HOME/.ccache
ccache --zero-stats
ccache --max-size=256M
- name: Build PR Branch
run: |
mkdir -p pr/build
cd pr/build
cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release ..
make -j$(nproc)
make -j$(nproc) benchmarks
cd ..
make -C test/data
- name: Checkout Base Branch
uses: actions/checkout@v4
with:
@@ -721,9 +714,43 @@ jobs:
make -j$(nproc) benchmarks
cd ..
make -C test/data
- name: Run Benchmarks
- name: Build PR Branch
run: |
./pr/scripts/ci/run_benchmarks.sh base pr
mkdir -p pr/build
cd pr/build
cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release ..
make -j$(nproc)
make -j$(nproc) benchmarks
cd ..
make -C test/data
# we run benchmarks in tmpfs to avoid impact of disk IO
- name: Create folder for tmpfs
run: mkdir -p /opt/benchmarks
- name: Run PR Benchmarks
run: |
sudo mount -t tmpfs -o size=4g none /opt/benchmarks
cp -rf pr/build /opt/benchmarks/build
mkdir -p /opt/benchmarks/test
cp -rf pr/test/data /opt/benchmarks/test/data
cp -rf pr/profiles /opt/benchmarks/profiles
./pr/scripts/ci/run_benchmarks.sh -f /opt/benchmarks -r $(pwd)/pr_results -s $(pwd)/pr -b /opt/benchmarks/build -o ~/data.osm.pbf -g ~/gps_traces.csv
sudo umount /opt/benchmarks
- name: Run Base Benchmarks
run: |
sudo mount -t tmpfs -o size=4g none /opt/benchmarks
cp -rf base/build /opt/benchmarks/build
mkdir -p /opt/benchmarks/test
cp -rf base/test/data /opt/benchmarks/test/data
cp -rf base/profiles /opt/benchmarks/profiles
# TODO: remove it when base branch will have this file at needed location
if [ ! -f /opt/benchmarks/test/data/portugal_to_korea.json ]; then
cp base/src/benchmarks/portugal_to_korea.json /opt/benchmarks/test/data/portugal_to_korea.json
fi
# we intentionally use scripts from PR branch to be able to update them and see results in the same PR
./pr/scripts/ci/run_benchmarks.sh -f /opt/benchmarks -r $(pwd)/base_results -s $(pwd)/pr -b /opt/benchmarks/build -o ~/data.osm.pbf -g ~/gps_traces.csv
sudo umount /opt/benchmarks
- name: Post Benchmark Results
run: |
python3 pr/scripts/ci/post_benchmark_results.py base_results pr_results
@@ -734,6 +761,7 @@ jobs:
ci-complete:
runs-on: ubuntu-22.04
needs: [build-test-publish, docker-image, windows-release-node, benchmarks]
needs: [build-test-publish, docker-image-matrix, windows-release-node, benchmarks]
steps:
- run: echo "CI complete"
+24
View File
@@ -0,0 +1,24 @@
name: 'Close stale issues'
on:
schedule:
- cron: '30 1 * * *' # every day at 1:30am
permissions:
issues: write
pull-requests: write
jobs:
stale:
runs-on: ubuntu-24.04
steps:
- uses: actions/stale@v9
with:
stale-issue-message: 'This issue seems to be stale. It will be closed in 30 days if no further activity occurs.'
stale-pr-message: 'This PR seems to be stale. Is it still relevant?'
days-before-issue-stale: 180 # 6 months
days-before-issue-close: 30 # 1 month
days-before-pr-stale: 180 # 6 months
days-before-pr-close: -1 # never close PRs
exempt-issue-labels: 'Do Not Stale,Feature Request,Performance,Bug Report,CI,Starter Task,Refactor,Guidance'
+12
View File
@@ -24,6 +24,17 @@
- NodeJS:
- CHANGED: Use node-api instead of NAN. [#6452](https://github.com/Project-OSRM/osrm-backend/pull/6452)
- Misc:
- CHANGED: Re-use priority queue in StaticRTree. [#6952](https://github.com/Project-OSRM/osrm-backend/pull/6952)
- CHANGED: Optimise encodePolyline function. [#6940](https://github.com/Project-OSRM/osrm-backend/pull/6940)
- CHANGED: Avoid reallocations in base64 encoding. [#6951](https://github.com/Project-OSRM/osrm-backend/pull/6951)
- CHANGED: Get rid of unused Boost dependencies. [#6960](https://github.com/Project-OSRM/osrm-backend/pull/6960)
- CHANGED: Apply micro-optimisation for Table & Trip APIs. [#6949](https://github.com/Project-OSRM/osrm-backend/pull/6949)
- CHANGED: Apply micro-optimisation for Route API. [#6948](https://github.com/Project-OSRM/osrm-backend/pull/6948)
- CHANGED: Apply micro-optimisation for Match API. [#6945](https://github.com/Project-OSRM/osrm-backend/pull/6945)
- CHANGED: Apply micro-optimisation for Nearest API. [#6944](https://github.com/Project-OSRM/osrm-backend/pull/6944)
- CHANGED: Avoid copy of intersection in totalTurnAngle. [#6938](https://github.com/Project-OSRM/osrm-backend/pull/6938)
- CHANGED: Use std::unordered_map::emplace instead of operator[] when producing JSONs. [#6936](https://github.com/Project-OSRM/osrm-backend/pull/6936)
- CHANGED: Avoid copy of vectors in MakeRoute function. [#6939](https://github.com/Project-OSRM/osrm-backend/pull/6939)
- FIXED: Fix bugprone-unused-return-value clang-tidy warning. [#6934](https://github.com/Project-OSRM/osrm-backend/pull/6934)
- FIXED: Fix performance-noexcept-move-constructor clang-tidy warning. [#6931](https://github.com/Project-OSRM/osrm-backend/pull/6933)
- FIXED: Fix performance-noexcept-swap clang-tidy warning. [#6931](https://github.com/Project-OSRM/osrm-backend/pull/6931)
@@ -37,6 +48,7 @@
- CHANGED: Avoid copy of std::function-based callback in path unpacking [#6895](https://github.com/Project-OSRM/osrm-backend/pull/6895)
- CHANGED: Replace boost::hash by std::hash [#6892](https://github.com/Project-OSRM/osrm-backend/pull/6892)
- CHANGED: Partial fix migration from boost::optional to std::optional [#6551](https://github.com/Project-OSRM/osrm-backend/issues/6551)
- CHANGED: Replace boost::filesystem with std::filesystem [#6432](https://github.com/Project-OSRM/osrm-backend/pull/6432)
- CHANGED: Update Conan Boost version to 1.85.0. [#6868](https://github.com/Project-OSRM/osrm-backend/pull/6868)
- FIXED: Fix an error in a RouteParameters AnnotationsType operator overload. [#6646](https://github.com/Project-OSRM/osrm-backend/pull/6646)
- ADDED: Add support for "unlimited" to be passed as a value for the default-radius and max-matching-radius flags. [#6599](https://github.com/Project-OSRM/osrm-backend/pull/6599)
+10 -25
View File
@@ -122,7 +122,7 @@ include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/include/)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/include/)
include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/sol2/include)
set(BOOST_COMPONENTS date_time chrono filesystem iostreams program_options regex system thread unit_test_framework)
set(BOOST_COMPONENTS date_time iostreams program_options thread unit_test_framework)
configure_file(
${CMAKE_CURRENT_SOURCE_DIR}/include/util/version.hpp.in
@@ -299,6 +299,10 @@ include_directories(SYSTEM ${PROTOZERO_INCLUDE_DIR})
set(VTZERO_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/vtzero/include")
include_directories(SYSTEM ${VTZERO_INCLUDE_DIR})
set(ANKERL_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/unordered_dense/include")
include_directories(SYSTEM ${ANKERL_INCLUDE_DIR})
set(FLATBUFFERS_BUILD_TESTS OFF CACHE BOOL "Disable the build of Flatbuffers tests and samples.")
set(FLATBUFFERS_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/flatbuffers")
set(FLATBUFFERS_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/flatbuffers/include")
@@ -330,20 +334,12 @@ if(ENABLE_CONAN)
set(CONAN_BOOST_VERSION "1.85.0@#14265ec82b25d91305bbb3b30d3357f8")
set(CONAN_BZIP2_VERSION "1.0.8@#d1b2d5816f25865acf978501dff1f897")
set(CONAN_EXPAT_VERSION "2.2.10@#916908d4a570ad839edd25322c3268cd")
set(CONAN_LUA_VERSION "5.4.4@#3ec62efc37cd0a5d80b9e5cb35277360")
set(CONAN_TBB_VERSION "2021.3.0@#507ec17cbd51a84167e143b20d170eea")
set(CONAN_EXPAT_VERSION "2.6.2@#2d385d0d50eb5561006a7ff9e356656b")
set(CONAN_LUA_VERSION "5.4.6@#658d6089093cf01992c2737ab2e96763")
set(CONAN_TBB_VERSION "2021.12.0@#e56e5b44be8d690530585dd3634c0106")
set(CONAN_SYSTEM_INCLUDES ON)
# TODO:
# if we link TBB dynamically osrm-extract.exe finishes on the first access to any TBB symbol
# with exit code = -1073741515, which means that program cannot load required DLL.
if (MSVC)
set(TBB_SHARED False)
else()
set(TBB_SHARED True)
endif()
set(CONAN_ARGS
REQUIRES
@@ -356,9 +352,7 @@ if(ENABLE_CONAN)
GENERATORS cmake_find_package json # json generator generates a conanbuildinfo.json in the build folder so (non-CMake) projects can easily parse OSRM's dependencies
KEEP_RPATHS
NO_OUTPUT_DIRS
OPTIONS boost:filesystem_version=3 # https://stackoverflow.com/questions/73392648/error-with-boost-filesystem-version-in-cmake
onetbb:shared=${TBB_SHARED}
boost:without_stacktrace=True # Apple Silicon cross-compilation fails without it
OPTIONS boost:without_stacktrace=True # Apple Silicon cross-compilation fails without it
BUILD missing
)
@@ -387,14 +381,10 @@ if(ENABLE_CONAN)
set(Boost_USE_STATIC_LIBS ON)
find_package(Boost REQUIRED COMPONENTS ${BOOST_COMPONENTS})
set(Boost_DATE_TIME_LIBRARY "${Boost_date_time_LIB_TARGETS}")
set(Boost_CHRONO_LIBRARY "${Boost_chrono_LIB_TARGETS}")
set(Boost_PROGRAM_OPTIONS_LIBRARY "${Boost_program_options_LIB_TARGETS}")
set(Boost_FILESYSTEM_LIBRARY "${Boost_filesystem_LIB_TARGETS}")
set(Boost_IOSTREAMS_LIBRARY "${Boost_iostreams_LIB_TARGETS}")
set(Boost_THREAD_LIBRARY "${Boost_thread_LIB_TARGETS}")
set(Boost_SYSTEM_LIBRARY "${Boost_system_LIB_TARGETS}")
set(Boost_ZLIB_LIBRARY "${Boost_zlib_LIB_TARGETS}")
set(Boost_REGEX_LIBRARY "${Boost_regex_LIB_TARGETS}")
set(Boost_UNIT_TEST_FRAMEWORK_LIBRARY "${Boost_unit_test_framework_LIB_TARGETS}")
@@ -467,7 +457,6 @@ add_dependency_includes(${ZLIB_INCLUDE_DIRS})
add_dependency_defines(-DBOOST_SPIRIT_USE_PHOENIX_V3)
add_dependency_defines(-DBOOST_RESULT_OF_USE_DECLTYPE)
add_dependency_defines(-DBOOST_FILESYSTEM_NO_DEPRECATED)
# Workaround for https://github.com/boostorg/phoenix/issues/111
add_dependency_defines(-DBOOST_PHOENIX_STL_TUPLE_H_)
@@ -477,11 +466,8 @@ include_directories(SYSTEM ${DEPENDENCIES_INCLUDE_DIRS})
set(BOOST_BASE_LIBRARIES
${Boost_DATE_TIME_LIBRARY}
${Boost_CHRONO_LIBRARY}
${Boost_FILESYSTEM_LIBRARY}
${Boost_IOSTREAMS_LIBRARY}
${Boost_THREAD_LIBRARY}
${Boost_SYSTEM_LIBRARY})
${Boost_THREAD_LIBRARY})
set(BOOST_ENGINE_LIBRARIES
${Boost_ZLIB_LIBRARY}
@@ -500,7 +486,6 @@ endif()
set(EXTRACTOR_LIBRARIES
${BZIP2_LIBRARIES}
${Boost_REGEX_LIBRARY}
${BOOST_BASE_LIBRARIES}
${CMAKE_THREAD_LIBS_INIT}
${EXPAT_LIBRARIES}
-68
View File
@@ -1,68 +0,0 @@
FROM debian:bookworm-slim as builder
ARG DOCKER_TAG
ARG BUILD_CONCURRENCY
RUN mkdir -p /src && mkdir -p /opt
RUN apt-get update && \
apt-get -y --no-install-recommends install ca-certificates cmake make git gcc g++ libbz2-dev libxml2-dev wget \
libzip-dev libboost1.81-all-dev lua5.4 liblua5.4-dev pkg-config -o APT::Install-Suggests=0 -o APT::Install-Recommends=0
RUN NPROC=${BUILD_CONCURRENCY:-$(nproc)} && \
ldconfig /usr/local/lib && \
git clone --branch v2021.12.0 --single-branch https://github.com/oneapi-src/oneTBB.git && \
cd oneTBB && \
mkdir build && \
cd build && \
cmake -DTBB_TEST=OFF -DCMAKE_BUILD_TYPE=Release .. && \
cmake --build . && \
cmake --install .
COPY . /src
WORKDIR /src
RUN NPROC=${BUILD_CONCURRENCY:-$(nproc)} && \
export CXXFLAGS="-Wno-array-bounds -Wno-uninitialized" && \
echo "Building OSRM ${DOCKER_TAG}" && \
git show --format="%H" | head -n1 > /opt/OSRM_GITSHA && \
echo "Building OSRM gitsha $(cat /opt/OSRM_GITSHA)" && \
mkdir -p build && \
cd build && \
BUILD_TYPE="Release" && \
ENABLE_ASSERTIONS="Off" && \
BUILD_TOOLS="Off" && \
case ${DOCKER_TAG} in *"-debug"*) BUILD_TYPE="Debug";; esac && \
case ${DOCKER_TAG} in *"-assertions"*) BUILD_TYPE="RelWithDebInfo" && ENABLE_ASSERTIONS="On" && BUILD_TOOLS="On";; esac && \
echo "Building ${BUILD_TYPE} with ENABLE_ASSERTIONS=${ENABLE_ASSERTIONS} BUILD_TOOLS=${BUILD_TOOLS}" && \
cmake .. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DENABLE_ASSERTIONS=${ENABLE_ASSERTIONS} -DBUILD_TOOLS=${BUILD_TOOLS} -DENABLE_LTO=On && \
make -j${NPROC} install && \
cd ../profiles && \
cp -r * /opt && \
strip /usr/local/bin/* && \
rm -rf /src
# Multistage build to reduce image size - https://docs.docker.com/engine/userguide/eng-image/multistage-build/#use-multi-stage-builds
# Only the content below ends up in the image, this helps remove /src from the image (which is large)
FROM debian:bookworm-slim as runstage
COPY --from=builder /usr/local /usr/local
COPY --from=builder /opt /opt
RUN apt-get update && \
apt-get install -y --no-install-recommends libboost-program-options1.81.0 libboost-regex1.81.0 \
libboost-date-time1.81.0 libboost-chrono1.81.0 libboost-filesystem1.81.0 \
libboost-iostreams1.81.0 libboost-system1.81.0 libboost-thread1.81.0 \
expat liblua5.4-0 && \
rm -rf /var/lib/apt/lists/* && \
# add /usr/local/lib to ldconfig to allow loading libraries from there
ldconfig /usr/local/lib
RUN /usr/local/bin/osrm-extract --help && \
/usr/local/bin/osrm-routed --help && \
/usr/local/bin/osrm-contract --help && \
/usr/local/bin/osrm-partition --help && \
/usr/local/bin/osrm-customize --help
WORKDIR /opt
EXPOSE 5000
+1
View File
@@ -0,0 +1 @@
Dockerfile-debian
+62
View File
@@ -0,0 +1,62 @@
FROM alpine:3.20.0 as alpine-mimalloc
RUN apk add --no-cache mimalloc
ENV LD_PRELOAD=/usr/lib/libmimalloc.so.2
ENV MIMALLOC_LARGE_OS_PAGES=1
FROM alpine-mimalloc as builder
ARG DOCKER_TAG
ARG BUILD_CONCURRENCY
RUN mkdir -p /src && mkdir -p /opt
RUN apk add --no-cache \
cmake make git clang libbz2 libxml2 \
boost-dev boost-program_options boost-filesystem boost-iostreams boost-thread \
lua5.4-dev onetbb-dev expat-dev
COPY . /src
WORKDIR /src
RUN NPROC=${BUILD_CONCURRENCY:-$(nproc)} && \
echo "Building OSRM ${DOCKER_TAG}" && \
git show --format="%H" | head -n1 > /opt/OSRM_GITSHA && \
echo "Building OSRM gitsha $(cat /opt/OSRM_GITSHA)" && \
mkdir -p build && \
cd build && \
BUILD_TYPE="Release" && \
ENABLE_ASSERTIONS="Off" && \
BUILD_TOOLS="Off" && \
case ${DOCKER_TAG} in *"-debug"*) BUILD_TYPE="Debug";; esac && \
case ${DOCKER_TAG} in *"-assertions"*) BUILD_TYPE="RelWithDebInfo" && ENABLE_ASSERTIONS="On" && BUILD_TOOLS="On";; esac && \
echo "Building ${BUILD_TYPE} with ENABLE_ASSERTIONS=${ENABLE_ASSERTIONS} BUILD_TOOLS=${BUILD_TOOLS}" && \
cmake .. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DENABLE_ASSERTIONS=${ENABLE_ASSERTIONS} -DBUILD_TOOLS=${BUILD_TOOLS} -DENABLE_LTO=On && \
make -j${NPROC} install && \
cd ../profiles && \
cp -r * /opt && \
strip /usr/local/bin/* && \
rm -rf /src
# Multistage build to reduce image size - https://docs.docker.com/engine/userguide/eng-image/multistage-build/#use-multi-stage-builds
# Only the content below ends up in the image, this helps remove /src from the image (which is large)
FROM alpine-mimalloc as runstage
COPY --from=builder /usr/local /usr/local
COPY --from=builder /opt /opt
RUN apk add --no-cache \
boost-program_options boost-date_time boost-iostreams boost-thread \
expat lua5.4 onetbb && \
ldconfig /usr/local/lib
RUN /usr/local/bin/osrm-extract --help && \
/usr/local/bin/osrm-routed --help && \
/usr/local/bin/osrm-contract --help && \
/usr/local/bin/osrm-partition --help && \
/usr/local/bin/osrm-customize --help
WORKDIR /opt
EXPOSE 5000
+67
View File
@@ -0,0 +1,67 @@
FROM debian:bookworm-slim as builder
ARG DOCKER_TAG
ARG BUILD_CONCURRENCY
RUN mkdir -p /src && mkdir -p /opt
RUN apt-get update && \
apt-get -y --no-install-recommends install ca-certificates cmake make git gcc g++ libbz2-dev libxml2-dev wget \
libzip-dev libboost1.81-all-dev lua5.4 liblua5.4-dev pkg-config -o APT::Install-Suggests=0 -o APT::Install-Recommends=0
RUN NPROC=${BUILD_CONCURRENCY:-$(nproc)} && \
ldconfig /usr/local/lib && \
git clone --branch v2021.12.0 --single-branch https://github.com/oneapi-src/oneTBB.git && \
cd oneTBB && \
mkdir build && \
cd build && \
cmake -DTBB_TEST=OFF -DCMAKE_BUILD_TYPE=Release .. && \
cmake --build . && \
cmake --install .
COPY . /src
WORKDIR /src
RUN NPROC=${BUILD_CONCURRENCY:-$(nproc)} && \
export CXXFLAGS="-Wno-array-bounds -Wno-uninitialized" && \
echo "Building OSRM ${DOCKER_TAG}" && \
git show --format="%H" | head -n1 > /opt/OSRM_GITSHA && \
echo "Building OSRM gitsha $(cat /opt/OSRM_GITSHA)" && \
mkdir -p build && \
cd build && \
BUILD_TYPE="Release" && \
ENABLE_ASSERTIONS="Off" && \
BUILD_TOOLS="Off" && \
case ${DOCKER_TAG} in *"-debug"*) BUILD_TYPE="Debug";; esac && \
case ${DOCKER_TAG} in *"-assertions"*) BUILD_TYPE="RelWithDebInfo" && ENABLE_ASSERTIONS="On" && BUILD_TOOLS="On";; esac && \
echo "Building ${BUILD_TYPE} with ENABLE_ASSERTIONS=${ENABLE_ASSERTIONS} BUILD_TOOLS=${BUILD_TOOLS}" && \
cmake .. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DENABLE_ASSERTIONS=${ENABLE_ASSERTIONS} -DBUILD_TOOLS=${BUILD_TOOLS} -DENABLE_LTO=On && \
make -j${NPROC} install && \
cd ../profiles && \
cp -r * /opt && \
strip /usr/local/bin/* && \
rm -rf /src
# Multistage build to reduce image size - https://docs.docker.com/engine/userguide/eng-image/multistage-build/#use-multi-stage-builds
# Only the content below ends up in the image, this helps remove /src from the image (which is large)
FROM debian:bookworm-slim as runstage
COPY --from=builder /usr/local /usr/local
COPY --from=builder /opt /opt
RUN apt-get update && \
apt-get install -y --no-install-recommends \
libboost-program-options1.81.0 libboost-date-time1.81.0 libboost-iostreams1.81.0 libboost-thread1.81.0 \
expat liblua5.4-0 && \
rm -rf /var/lib/apt/lists/* && \
# add /usr/local/lib to ldconfig to allow loading libraries from there
ldconfig /usr/local/lib
RUN /usr/local/bin/osrm-extract --help && \
/usr/local/bin/osrm-routed --help && \
/usr/local/bin/osrm-contract --help && \
/usr/local/bin/osrm-partition --help && \
/usr/local/bin/osrm-customize --help
WORKDIR /opt
EXPOSE 5000
Regular → Executable
+5 -1
View File
@@ -6,4 +6,8 @@
# ensure that "COPY . /src" is referring to the repo root, not the directory
# that contains the Dockerfile.
# This script gets executed with a pwd of wherever the Dockerfile is.
docker build --build-arg BUILD_CONCURRENCY=${CONCURRENCY:-1} --build-arg DOCKER_TAG=${DOCKER_TAG} -t $IMAGE_NAME -f Dockerfile ..
DOCKER_BUILD="docker build --build-arg BUILD_CONCURRENCY=${CONCURRENCY} --build-arg DOCKER_TAG=${DOCKER_TAG:?unset} -t ${IMAGE_NAME:?unset} -f"
$DOCKER_BUILD Dockerfile ..
$DOCKER_BUILD Dockerfile-alpine ..
+1 -1
View File
@@ -42,7 +42,7 @@ module.exports = function () {
this.OSRM_PORT = process.env.OSRM_PORT && parseInt(process.env.OSRM_PORT) || 5000;
this.OSRM_IP = process.env.OSRM_IP || '127.0.0.1';
this.OSRM_CONNECTION_RETRIES = process.env.OSRM_CONNECTION_RETRIES && parseInt(process.env.OSRM_CONNECTION_RETRIES) || 10;
this.OSRM_CONNECTION_EXP_BACKOFF_COEF = process.env.OSRM_CONNECTION_EXP_BACKOFF_COEF && parseFloat(process.env.OSRM_CONNECTION_EXP_BACKOFF_COEF) || 1.0;
this.OSRM_CONNECTION_EXP_BACKOFF_COEF = process.env.OSRM_CONNECTION_EXP_BACKOFF_COEF && parseFloat(process.env.OSRM_CONNECTION_EXP_BACKOFF_COEF) || 1.1;
this.HOST = `http://${this.OSRM_IP}:${this.OSRM_PORT}`;
+2 -3
View File
@@ -31,8 +31,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "storage/io_config.hpp"
#include "updater/updater_config.hpp"
#include <boost/filesystem/path.hpp>
#include <filesystem>
#include <string>
namespace osrm::contractor
@@ -47,7 +46,7 @@ struct ContractorConfig final : storage::IOConfig
}
// Infer the output names from the path of the .osrm file
void UseDefaultOutputNames(const boost::filesystem::path &base)
void UseDefaultOutputNames(const std::filesystem::path &base)
{
IOConfig::UseDefaultOutputNames(base);
updater_config.UseDefaultOutputNames(base);
+2 -2
View File
@@ -9,7 +9,7 @@ namespace osrm::contractor::files
{
// reads .osrm.hsgr file
template <typename ContractedMetricT>
inline void readGraph(const boost::filesystem::path &path,
inline void readGraph(const std::filesystem::path &path,
std::unordered_map<std::string, ContractedMetricT> &metrics,
std::uint32_t &connectivity_checksum)
{
@@ -30,7 +30,7 @@ inline void readGraph(const boost::filesystem::path &path,
// writes .osrm.hsgr file
template <typename ContractedMetricT>
inline void writeGraph(const boost::filesystem::path &path,
inline void writeGraph(const std::filesystem::path &path,
const std::unordered_map<std::string, ContractedMetricT> &metrics,
const std::uint32_t connectivity_checksum)
{
+2 -3
View File
@@ -1,9 +1,8 @@
#ifndef OSRM_CUSTOMIZE_CUSTOMIZER_CONFIG_HPP
#define OSRM_CUSTOMIZE_CUSTOMIZER_CONFIG_HPP
#include <boost/filesystem/path.hpp>
#include <array>
#include <filesystem>
#include <string>
#include "storage/io_config.hpp"
@@ -27,7 +26,7 @@ struct CustomizationConfig final : storage::IOConfig
{
}
void UseDefaultOutputNames(const boost::filesystem::path &base)
void UseDefaultOutputNames(const std::filesystem::path &base)
{
IOConfig::UseDefaultOutputNames(base);
updater_config.UseDefaultOutputNames(base);
+1 -1
View File
@@ -9,7 +9,7 @@
#include "storage/shared_memory_ownership.hpp"
#include <boost/filesystem/path.hpp>
#include <filesystem>
namespace osrm::customizer
{
+4 -4
View File
@@ -14,7 +14,7 @@ namespace osrm::customizer::files
// reads .osrm.cell_metrics file
template <typename CellMetricT>
inline void readCellMetrics(const boost::filesystem::path &path,
inline void readCellMetrics(const std::filesystem::path &path,
std::unordered_map<std::string, std::vector<CellMetricT>> &metrics)
{
static_assert(std::is_same<CellMetricView, CellMetricT>::value ||
@@ -44,7 +44,7 @@ inline void readCellMetrics(const boost::filesystem::path &path,
// writes .osrm.cell_metrics file
template <typename CellMetricT>
inline void
writeCellMetrics(const boost::filesystem::path &path,
writeCellMetrics(const std::filesystem::path &path,
const std::unordered_map<std::string, std::vector<CellMetricT>> &metrics)
{
static_assert(std::is_same<CellMetricView, CellMetricT>::value ||
@@ -72,7 +72,7 @@ writeCellMetrics(const boost::filesystem::path &path,
// reads .osrm.mldgr file
template <typename MultiLevelGraphT>
inline void readGraph(const boost::filesystem::path &path,
inline void readGraph(const std::filesystem::path &path,
MultiLevelGraphT &graph,
std::uint32_t &connectivity_checksum)
{
@@ -88,7 +88,7 @@ inline void readGraph(const boost::filesystem::path &path,
// writes .osrm.mldgr file
template <typename MultiLevelGraphT>
inline void writeGraph(const boost::filesystem::path &path,
inline void writeGraph(const std::filesystem::path &path,
const MultiLevelGraphT &graph,
const std::uint32_t connectivity_checksum)
{
+17 -17
View File
@@ -77,19 +77,19 @@ class MatchAPI final : public RouteAPI
sub_routes[index].unpacked_path_segments,
sub_routes[index].source_traversed_in_reverse,
sub_routes[index].target_traversed_in_reverse);
route.values["confidence"] = sub_matchings[index].confidence;
routes.values.push_back(std::move(route));
route.values.emplace("confidence", sub_matchings[index].confidence);
routes.values.emplace_back(std::move(route));
}
if (!parameters.skip_waypoints)
{
response.values["tracepoints"] = MakeTracepoints(sub_matchings);
response.values.emplace("tracepoints", MakeTracepoints(sub_matchings));
}
response.values["matchings"] = std::move(routes);
response.values["code"] = "Ok";
response.values.emplace("matchings", std::move(routes));
response.values.emplace("code", "Ok");
auto data_timestamp = facade.GetTimestamp();
if (!data_timestamp.empty())
{
response.values["data_version"] = data_timestamp;
response.values.emplace("data_version", data_timestamp);
}
}
@@ -132,13 +132,13 @@ class MatchAPI final : public RouteAPI
if (tidy_result.can_be_removed[trace_index])
{
waypoints.push_back(fbresult::WaypointBuilder(fb_result).Finish());
waypoints.emplace_back(fbresult::WaypointBuilder(fb_result).Finish());
continue;
}
auto matching_index = trace_idx_to_matching_idx[trace_index];
if (matching_index.NotMatched())
{
waypoints.push_back(fbresult::WaypointBuilder(fb_result).Finish());
waypoints.emplace_back(fbresult::WaypointBuilder(fb_result).Finish());
continue;
}
const auto &phantom =
@@ -165,7 +165,7 @@ class MatchAPI final : public RouteAPI
{
waypoint->add_waypoint_index(matching_index.point_index);
}
waypoints.push_back(waypoint->Finish());
waypoints.emplace_back(waypoint->Finish());
}
return fb_result.CreateVector(waypoints);
@@ -186,23 +186,23 @@ class MatchAPI final : public RouteAPI
{
if (tidy_result.can_be_removed[trace_index])
{
waypoints.values.push_back(util::json::Null());
waypoints.values.emplace_back(util::json::Null());
continue;
}
auto matching_index = trace_idx_to_matching_idx[trace_index];
if (matching_index.NotMatched())
{
waypoints.values.push_back(util::json::Null());
waypoints.values.emplace_back(util::json::Null());
continue;
}
const auto &phantom =
sub_matchings[matching_index.sub_matching_index].nodes[matching_index.point_index];
auto waypoint = BaseAPI::MakeWaypoint({phantom});
waypoint.values["matchings_index"] = matching_index.sub_matching_index;
waypoint.values["waypoint_index"] = matching_index.point_index;
waypoint.values["alternatives_count"] =
sub_matchings[matching_index.sub_matching_index]
.alternatives_count[matching_index.point_index];
waypoint.values.emplace("matchings_index", matching_index.sub_matching_index);
waypoint.values.emplace("waypoint_index", matching_index.point_index);
waypoint.values.emplace("alternatives_count",
sub_matchings[matching_index.sub_matching_index]
.alternatives_count[matching_index.point_index]);
// waypoint indices need to be adjusted if route legs were collapsed
// waypoint parameter assumes there is only one match object
if (!parameters.waypoints.empty())
@@ -217,7 +217,7 @@ class MatchAPI final : public RouteAPI
waypoint.values["waypoint_index"] = util::json::Null();
}
}
waypoints.values.push_back(std::move(waypoint));
waypoints.values.emplace_back(std::move(waypoint));
}
return waypoints;
+7 -7
View File
@@ -100,23 +100,23 @@ class NearestAPI final : public BaseAPI
auto waypoint = MakeWaypoint({phantom_node});
util::json::Array nodes;
nodes.values.reserve(2);
auto node_values = MakeNodes(phantom_node);
nodes.values.push_back(node_values.first);
nodes.values.push_back(node_values.second);
waypoint.values["nodes"] = std::move(nodes);
nodes.values.emplace_back(node_values.first);
nodes.values.emplace_back(node_values.second);
waypoint.values.emplace("nodes", std::move(nodes));
return waypoint;
});
response.values["waypoints"] = std::move(waypoints);
response.values.emplace("waypoints", std::move(waypoints));
}
response.values["code"] = "Ok";
response.values.emplace("code", "Ok");
auto data_timestamp = facade.GetTimestamp();
if (!data_timestamp.empty())
{
response.values["data_version"] = data_timestamp;
response.values.emplace("data_version", data_timestamp);
}
}
+43 -35
View File
@@ -110,14 +110,14 @@ class RouteAPI : public BaseAPI
if (!parameters.skip_waypoints)
{
response.values["waypoints"] = BaseAPI::MakeWaypoints(waypoint_candidates);
response.values.emplace("waypoints", BaseAPI::MakeWaypoints(waypoint_candidates));
}
response.values["routes"] = std::move(jsRoutes);
response.values["code"] = "Ok";
response.values.emplace("routes", std::move(jsRoutes));
response.values.emplace("code", "Ok");
auto data_timestamp = facade.GetTimestamp();
if (!data_timestamp.empty())
{
response.values["data_version"] = data_timestamp;
response.values.emplace("data_version", data_timestamp);
}
}
@@ -340,8 +340,8 @@ class RouteAPI : public BaseAPI
unpacked_path_segments,
source_traversed_in_reverse,
target_traversed_in_reverse);
std::vector<guidance::RouteLeg> legs = legs_info.first;
std::vector<guidance::LegGeometry> leg_geometries = legs_info.second;
std::vector<guidance::RouteLeg> &legs = legs_info.first;
std::vector<guidance::LegGeometry> &leg_geometries = legs_info.second;
auto route = guidance::assembleRoute(legs);
// Fill legs
@@ -716,8 +716,8 @@ class RouteAPI : public BaseAPI
unpacked_path_segments,
source_traversed_in_reverse,
target_traversed_in_reverse);
std::vector<guidance::RouteLeg> legs = legs_info.first;
std::vector<guidance::LegGeometry> leg_geometries = legs_info.second;
std::vector<guidance::RouteLeg> &legs = legs_info.first;
std::vector<guidance::LegGeometry> &leg_geometries = legs_info.second;
auto route = guidance::assembleRoute(legs);
boost::optional<util::json::Value> json_overview =
@@ -784,49 +784,57 @@ class RouteAPI : public BaseAPI
if (requested_annotations & RouteParameters::AnnotationsType::Speed)
{
double prev_speed = 0;
annotation.values["speed"] = GetAnnotations(
leg_geometry,
[&prev_speed](const guidance::LegGeometry::Annotation &anno)
{
if (anno.duration < std::numeric_limits<double>::min())
{
return prev_speed;
}
else
{
auto speed = std::round(anno.distance / anno.duration * 10.) / 10.;
prev_speed = speed;
return util::json::clamp_float(speed);
}
});
annotation.values.emplace(
"speed",
GetAnnotations(leg_geometry,
[&prev_speed](const guidance::LegGeometry::Annotation &anno)
{
if (anno.duration < std::numeric_limits<double>::min())
{
return prev_speed;
}
else
{
auto speed =
std::round(anno.distance / anno.duration * 10.) /
10.;
prev_speed = speed;
return util::json::clamp_float(speed);
}
}));
}
if (requested_annotations & RouteParameters::AnnotationsType::Duration)
{
annotation.values["duration"] =
annotation.values.emplace(
"duration",
GetAnnotations(leg_geometry,
[](const guidance::LegGeometry::Annotation &anno)
{ return anno.duration; });
{ return anno.duration; }));
}
if (requested_annotations & RouteParameters::AnnotationsType::Distance)
{
annotation.values["distance"] =
annotation.values.emplace(
"distance",
GetAnnotations(leg_geometry,
[](const guidance::LegGeometry::Annotation &anno)
{ return anno.distance; });
{ return anno.distance; }));
}
if (requested_annotations & RouteParameters::AnnotationsType::Weight)
{
annotation.values["weight"] = GetAnnotations(
leg_geometry,
[](const guidance::LegGeometry::Annotation &anno) { return anno.weight; });
annotation.values.emplace(
"weight",
GetAnnotations(leg_geometry,
[](const guidance::LegGeometry::Annotation &anno)
{ return anno.weight; }));
}
if (requested_annotations & RouteParameters::AnnotationsType::Datasources)
{
annotation.values["datasources"] =
annotation.values.emplace(
"datasources",
GetAnnotations(leg_geometry,
[](const guidance::LegGeometry::Annotation &anno)
{ return anno.datasource; });
{ return anno.datasource; }));
}
if (requested_annotations & RouteParameters::AnnotationsType::Nodes)
{
@@ -837,7 +845,7 @@ class RouteAPI : public BaseAPI
nodes.values.push_back(
static_cast<std::uint64_t>(facade.GetOSMNodeIDOfNode(node_id)));
}
annotation.values["nodes"] = std::move(nodes);
annotation.values.emplace("nodes", std::move(nodes));
}
// Add any supporting metadata, if needed
if (requested_annotations & RouteParameters::AnnotationsType::Datasources)
@@ -853,8 +861,8 @@ class RouteAPI : public BaseAPI
break;
datasource_names.values.push_back(std::string(facade.GetDatasourceName(i)));
}
metadata.values["datasource_names"] = datasource_names;
annotation.values["metadata"] = metadata;
metadata.values.emplace("datasource_names", datasource_names);
annotation.values.emplace("metadata", metadata);
}
annotations.push_back(std::move(annotation));
+15 -12
View File
@@ -179,7 +179,7 @@ class TableAPI final : public BaseAPI
{
if (!parameters.skip_waypoints)
{
response.values["sources"] = MakeWaypoints(candidates);
response.values.emplace("sources", MakeWaypoints(candidates));
}
number_of_sources = candidates.size();
}
@@ -187,7 +187,7 @@ class TableAPI final : public BaseAPI
{
if (!parameters.skip_waypoints)
{
response.values["sources"] = MakeWaypoints(candidates, parameters.sources);
response.values.emplace("sources", MakeWaypoints(candidates, parameters.sources));
}
}
@@ -195,7 +195,7 @@ class TableAPI final : public BaseAPI
{
if (!parameters.skip_waypoints)
{
response.values["destinations"] = MakeWaypoints(candidates);
response.values.emplace("destinations", MakeWaypoints(candidates));
}
number_of_destinations = candidates.size();
}
@@ -203,34 +203,37 @@ class TableAPI final : public BaseAPI
{
if (!parameters.skip_waypoints)
{
response.values["destinations"] =
MakeWaypoints(candidates, parameters.destinations);
response.values.emplace("destinations",
MakeWaypoints(candidates, parameters.destinations));
}
}
if (parameters.annotations & TableParameters::AnnotationsType::Duration)
{
response.values["durations"] =
MakeDurationTable(tables.first, number_of_sources, number_of_destinations);
response.values.emplace(
"durations",
MakeDurationTable(tables.first, number_of_sources, number_of_destinations));
}
if (parameters.annotations & TableParameters::AnnotationsType::Distance)
{
response.values["distances"] =
MakeDistanceTable(tables.second, number_of_sources, number_of_destinations);
response.values.emplace(
"distances",
MakeDistanceTable(tables.second, number_of_sources, number_of_destinations));
}
if (parameters.fallback_speed != from_alias<double>(INVALID_FALLBACK_SPEED) &&
parameters.fallback_speed > 0)
{
response.values["fallback_speed_cells"] = MakeEstimatesTable(fallback_speed_cells);
response.values.emplace("fallback_speed_cells",
MakeEstimatesTable(fallback_speed_cells));
}
response.values["code"] = "Ok";
response.values.emplace("code", "Ok");
auto data_timestamp = facade.GetTimestamp();
if (!data_timestamp.empty())
{
response.values["data_version"] = data_timestamp;
response.values.emplace("data_version", data_timestamp);
}
}
+6 -6
View File
@@ -79,14 +79,14 @@ class TripAPI final : public RouteAPI
}
if (!parameters.skip_waypoints)
{
response.values["waypoints"] = MakeWaypoints(sub_trips, candidates);
response.values.emplace("waypoints", MakeWaypoints(sub_trips, candidates));
}
response.values["trips"] = std::move(routes);
response.values["code"] = "Ok";
response.values.emplace("trips", std::move(routes));
response.values.emplace("code", "Ok");
auto data_timestamp = facade.GetTimestamp();
if (!data_timestamp.empty())
{
response.values["data_version"] = data_timestamp;
response.values.emplace("data_version", data_timestamp);
}
}
@@ -151,8 +151,8 @@ class TripAPI final : public RouteAPI
BOOST_ASSERT(!trip_index.NotUsed());
auto waypoint = BaseAPI::MakeWaypoint(candidates[input_index]);
waypoint.values["trips_index"] = trip_index.sub_trip_index;
waypoint.values["waypoint_index"] = trip_index.point_index;
waypoint.values.emplace("trips_index", trip_index.sub_trip_index);
waypoint.values.emplace("waypoint_index", trip_index.point_index);
waypoints.values.push_back(std::move(waypoint));
}
+16 -11
View File
@@ -47,24 +47,29 @@ namespace engine
// Encodes a chunk of memory to Base64.
inline std::string encodeBase64(const unsigned char *first, std::size_t size)
{
std::vector<unsigned char> bytes{first, first + size};
BOOST_ASSERT(!bytes.empty());
BOOST_ASSERT(size > 0);
std::size_t bytes_to_pad{0};
std::string encoded;
encoded.reserve(((size + 2) / 3) * 4);
while (bytes.size() % 3 != 0)
auto padding = (3 - size % 3) % 3;
BOOST_ASSERT(padding == 0 || padding == 1 || padding == 2);
for (auto itr = detail::Base64FromBinary(first); itr != detail::Base64FromBinary(first + size);
++itr)
{
bytes_to_pad += 1;
bytes.push_back(0);
encoded.push_back(*itr);
}
BOOST_ASSERT(bytes_to_pad == 0 || bytes_to_pad == 1 || bytes_to_pad == 2);
BOOST_ASSERT_MSG(0 == bytes.size() % 3, "base64 input data size is not a multiple of 3");
for (size_t index = 0; index < padding; ++index)
{
encoded.push_back('=');
}
std::string encoded{detail::Base64FromBinary{bytes.data()},
detail::Base64FromBinary{bytes.data() + (bytes.size() - bytes_to_pad)}};
BOOST_ASSERT(encoded.size() == (size + 2) / 3 * 4);
return encoded.append(bytes_to_pad, '=');
return encoded;
}
// C++11 standard 3.9.1/1: Plain char, signed char, and unsigned char are three distinct types
@@ -177,7 +177,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
SharedRTree m_static_rtree;
std::unique_ptr<SharedGeospatialQuery> m_geospatial_query;
boost::filesystem::path file_index_path;
std::filesystem::path file_index_path;
std::optional<extractor::IntersectionBearingsView> intersection_bearings_view;
+2 -3
View File
@@ -31,8 +31,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "storage/storage_config.hpp"
#include "osrm/datasets.hpp"
#include <boost/filesystem/path.hpp>
#include <filesystem>
#include <set>
#include <string>
@@ -83,7 +82,7 @@ struct EngineConfig final
boost::optional<double> default_radius = -1.0;
int max_alternatives = 3; // set an arbitrary upper bound; can be adjusted by user
bool use_shared_memory = true;
boost::filesystem::path memory_file;
std::filesystem::path memory_file;
bool use_mmap = true;
Algorithm algorithm = Algorithm::CH;
std::vector<storage::FeatureDataset> disable_feature_dataset;
@@ -202,8 +202,8 @@ inline double totalTurnAngle(const RouteStep &entry_step, const RouteStep &exit_
if (entry_step.geometry_begin > exit_step.geometry_begin)
return totalTurnAngle(exit_step, entry_step);
const auto exit_intersection = exit_step.intersections.front();
const auto entry_intersection = entry_step.intersections.front();
const auto &exit_intersection = exit_step.intersections.front();
const auto &entry_intersection = entry_step.intersections.front();
if ((exit_intersection.out >= exit_intersection.bearings.size()) ||
(entry_intersection.in >= entry_intersection.bearings.size()))
return entry_intersection.bearings[entry_intersection.out];
+17 -20
View File
@@ -12,7 +12,7 @@ namespace osrm::engine
{
namespace detail
{
std::string encode(std::vector<int> &numbers);
void encode(int number_to_encode, std::string &output);
std::int32_t decode_polyline_integer(std::string::const_iterator &first,
std::string::const_iterator last);
} // namespace detail
@@ -30,27 +30,24 @@ std::string encodePolyline(CoordVectorForwardIter begin, CoordVectorForwardIter
return {};
}
std::vector<int> delta_numbers;
BOOST_ASSERT(size > 0);
delta_numbers.reserve((size - 1) * 2);
std::string output;
// just a guess that we will need ~4 bytes per coordinate to avoid reallocations
output.reserve(size * 4);
int current_lat = 0;
int current_lon = 0;
std::for_each(
begin,
end,
[&delta_numbers, &current_lat, &current_lon, coordinate_to_polyline](
const util::Coordinate loc)
{
const int lat_diff =
std::round(static_cast<int>(loc.lat) * coordinate_to_polyline) - current_lat;
const int lon_diff =
std::round(static_cast<int>(loc.lon) * coordinate_to_polyline) - current_lon;
delta_numbers.emplace_back(lat_diff);
delta_numbers.emplace_back(lon_diff);
current_lat += lat_diff;
current_lon += lon_diff;
});
return detail::encode(delta_numbers);
for (auto it = begin; it != end; ++it)
{
const int lat_diff =
std::round(static_cast<int>(it->lat) * coordinate_to_polyline) - current_lat;
const int lon_diff =
std::round(static_cast<int>(it->lon) * coordinate_to_polyline) - current_lon;
detail::encode(lat_diff, output);
detail::encode(lon_diff, output);
current_lat += lat_diff;
current_lon += lon_diff;
}
return output;
}
// Decodes geometry from polyline format
+6 -7
View File
@@ -28,13 +28,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef EXTRACTOR_CONFIG_HPP
#define EXTRACTOR_CONFIG_HPP
#include <boost/filesystem/path.hpp>
#include "storage/io_config.hpp"
#include <array>
#include <filesystem>
#include <string>
#include "storage/io_config.hpp"
namespace osrm::extractor
{
@@ -71,14 +70,14 @@ struct ExtractorConfig final : storage::IOConfig
{
}
void UseDefaultOutputNames(const boost::filesystem::path &base)
void UseDefaultOutputNames(const std::filesystem::path &base)
{
IOConfig::UseDefaultOutputNames(base);
}
boost::filesystem::path input_path;
boost::filesystem::path profile_path;
std::vector<boost::filesystem::path> location_dependent_data_paths;
std::filesystem::path input_path;
std::filesystem::path profile_path;
std::vector<std::filesystem::path> location_dependent_data_paths;
std::string data_version;
unsigned requested_num_threads = 0;
+53 -57
View File
@@ -23,9 +23,9 @@ namespace osrm::extractor::files
// writes the .osrm.icd file
template <typename IntersectionBearingsT, typename EntryClassVectorT>
inline void writeIntersections(const boost::filesystem::path &path,
const IntersectionBearingsT &intersection_bearings,
const EntryClassVectorT &entry_classes)
void writeIntersections(const std::filesystem::path &path,
const IntersectionBearingsT &intersection_bearings,
const EntryClassVectorT &entry_classes)
{
static_assert(std::is_same<IntersectionBearingsContainer, IntersectionBearingsT>::value ||
std::is_same<IntersectionBearingsView, IntersectionBearingsT>::value,
@@ -39,9 +39,9 @@ inline void writeIntersections(const boost::filesystem::path &path,
// read the .osrm.icd file
template <typename IntersectionBearingsT, typename EntryClassVectorT>
inline void readIntersections(const boost::filesystem::path &path,
IntersectionBearingsT &intersection_bearings,
EntryClassVectorT &entry_classes)
void readIntersections(const std::filesystem::path &path,
IntersectionBearingsT &intersection_bearings,
EntryClassVectorT &entry_classes)
{
static_assert(std::is_same<IntersectionBearingsContainer, IntersectionBearingsT>::value ||
std::is_same<IntersectionBearingsView, IntersectionBearingsT>::value,
@@ -54,8 +54,7 @@ inline void readIntersections(const boost::filesystem::path &path,
}
// reads .osrm.properties
inline void readProfileProperties(const boost::filesystem::path &path,
ProfileProperties &properties)
inline void readProfileProperties(const std::filesystem::path &path, ProfileProperties &properties)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
@@ -64,7 +63,7 @@ inline void readProfileProperties(const boost::filesystem::path &path,
}
// writes .osrm.properties
inline void writeProfileProperties(const boost::filesystem::path &path,
inline void writeProfileProperties(const std::filesystem::path &path,
const ProfileProperties &properties)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
@@ -74,7 +73,7 @@ inline void writeProfileProperties(const boost::filesystem::path &path,
}
template <typename EdgeBasedEdgeVector>
void writeEdgeBasedGraph(const boost::filesystem::path &path,
void writeEdgeBasedGraph(const std::filesystem::path &path,
EdgeID const number_of_edge_based_nodes,
const EdgeBasedEdgeVector &edge_based_edge_list,
const std::uint32_t connectivity_checksum)
@@ -92,7 +91,7 @@ void writeEdgeBasedGraph(const boost::filesystem::path &path,
// reads .osrm.ebg file
template <typename EdgeBasedEdgeVector>
void readEdgeBasedGraph(const boost::filesystem::path &path,
void readEdgeBasedGraph(const std::filesystem::path &path,
EdgeID &number_of_edge_based_nodes,
EdgeBasedEdgeVector &edge_based_edge_list,
std::uint32_t &connectivity_checksum)
@@ -108,9 +107,9 @@ void readEdgeBasedGraph(const boost::filesystem::path &path,
// reads .osrm.nbg_nodes
template <typename CoordinatesT, typename PackedOSMIDsT>
inline void readNodes(const boost::filesystem::path &path,
CoordinatesT &coordinates,
PackedOSMIDsT &osm_node_ids)
void readNodes(const std::filesystem::path &path,
CoordinatesT &coordinates,
PackedOSMIDsT &osm_node_ids)
{
static_assert(std::is_same<typename CoordinatesT::value_type, util::Coordinate>::value, "");
static_assert(std::is_same<typename PackedOSMIDsT::value_type, OSMNodeID>::value, "");
@@ -124,7 +123,7 @@ inline void readNodes(const boost::filesystem::path &path,
// reads only coordinates from .osrm.nbg_nodes
template <typename CoordinatesT>
inline void readNodeCoordinates(const boost::filesystem::path &path, CoordinatesT &coordinates)
void readNodeCoordinates(const std::filesystem::path &path, CoordinatesT &coordinates)
{
static_assert(std::is_same<typename CoordinatesT::value_type, util::Coordinate>::value, "");
@@ -136,9 +135,9 @@ inline void readNodeCoordinates(const boost::filesystem::path &path, Coordinates
// writes .osrm.nbg_nodes
template <typename CoordinatesT, typename PackedOSMIDsT>
inline void writeNodes(const boost::filesystem::path &path,
const CoordinatesT &coordinates,
const PackedOSMIDsT &osm_node_ids)
void writeNodes(const std::filesystem::path &path,
const CoordinatesT &coordinates,
const PackedOSMIDsT &osm_node_ids)
{
static_assert(std::is_same<typename CoordinatesT::value_type, util::Coordinate>::value, "");
static_assert(std::is_same<typename PackedOSMIDsT::value_type, OSMNodeID>::value, "");
@@ -151,7 +150,7 @@ inline void writeNodes(const boost::filesystem::path &path,
}
// reads .osrm.cnbg_to_ebg
inline void readNBGMapping(const boost::filesystem::path &path, std::vector<NBGToEBG> &mapping)
inline void readNBGMapping(const std::filesystem::path &path, std::vector<NBGToEBG> &mapping)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
@@ -160,8 +159,7 @@ inline void readNBGMapping(const boost::filesystem::path &path, std::vector<NBGT
}
// writes .osrm.cnbg_to_ebg
inline void writeNBGMapping(const boost::filesystem::path &path,
const std::vector<NBGToEBG> &mapping)
inline void writeNBGMapping(const std::filesystem::path &path, const std::vector<NBGToEBG> &mapping)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
@@ -170,7 +168,7 @@ inline void writeNBGMapping(const boost::filesystem::path &path,
}
// reads .osrm.datasource_names
inline void readDatasources(const boost::filesystem::path &path, Datasources &sources)
inline void readDatasources(const std::filesystem::path &path, Datasources &sources)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
@@ -179,7 +177,7 @@ inline void readDatasources(const boost::filesystem::path &path, Datasources &so
}
// writes .osrm.datasource_names
inline void writeDatasources(const boost::filesystem::path &path, Datasources &sources)
inline void writeDatasources(const std::filesystem::path &path, Datasources &sources)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
@@ -189,7 +187,7 @@ inline void writeDatasources(const boost::filesystem::path &path, Datasources &s
// reads .osrm.geometry
template <typename SegmentDataT>
inline void readSegmentData(const boost::filesystem::path &path, SegmentDataT &segment_data)
void readSegmentData(const std::filesystem::path &path, SegmentDataT &segment_data)
{
static_assert(std::is_same<SegmentDataContainer, SegmentDataT>::value ||
std::is_same<SegmentDataView, SegmentDataT>::value,
@@ -202,7 +200,7 @@ inline void readSegmentData(const boost::filesystem::path &path, SegmentDataT &s
// writes .osrm.geometry
template <typename SegmentDataT>
inline void writeSegmentData(const boost::filesystem::path &path, const SegmentDataT &segment_data)
void writeSegmentData(const std::filesystem::path &path, const SegmentDataT &segment_data)
{
static_assert(std::is_same<SegmentDataContainer, SegmentDataT>::value ||
std::is_same<SegmentDataView, SegmentDataT>::value,
@@ -215,7 +213,7 @@ inline void writeSegmentData(const boost::filesystem::path &path, const SegmentD
// reads .osrm.ebg_nodes
template <typename NodeDataT>
inline void readNodeData(const boost::filesystem::path &path, NodeDataT &node_data)
inline void readNodeData(const std::filesystem::path &path, NodeDataT &node_data)
{
static_assert(std::is_same<EdgeBasedNodeDataContainer, NodeDataT>::value ||
std::is_same<EdgeBasedNodeDataView, NodeDataT>::value ||
@@ -229,7 +227,7 @@ inline void readNodeData(const boost::filesystem::path &path, NodeDataT &node_da
// writes .osrm.ebg_nodes
template <typename NodeDataT>
inline void writeNodeData(const boost::filesystem::path &path, const NodeDataT &node_data)
inline void writeNodeData(const std::filesystem::path &path, const NodeDataT &node_data)
{
static_assert(std::is_same<EdgeBasedNodeDataContainer, NodeDataT>::value ||
std::is_same<EdgeBasedNodeDataView, NodeDataT>::value ||
@@ -243,7 +241,7 @@ inline void writeNodeData(const boost::filesystem::path &path, const NodeDataT &
// reads .osrm.tls
template <typename OffsetsT, typename MaskT>
inline void readTurnLaneDescriptions(const boost::filesystem::path &path,
inline void readTurnLaneDescriptions(const std::filesystem::path &path,
OffsetsT &turn_offsets,
MaskT &turn_masks)
{
@@ -260,7 +258,7 @@ inline void readTurnLaneDescriptions(const boost::filesystem::path &path,
// writes .osrm.tls
template <typename OffsetsT, typename MaskT>
inline void writeTurnLaneDescriptions(const boost::filesystem::path &path,
inline void writeTurnLaneDescriptions(const std::filesystem::path &path,
const OffsetsT &turn_offsets,
const MaskT &turn_masks)
{
@@ -277,7 +275,7 @@ inline void writeTurnLaneDescriptions(const boost::filesystem::path &path,
// reads .osrm.tld
template <typename TurnLaneDataT>
inline void readTurnLaneData(const boost::filesystem::path &path, TurnLaneDataT &turn_lane_data)
inline void readTurnLaneData(const std::filesystem::path &path, TurnLaneDataT &turn_lane_data)
{
static_assert(
std::is_same<typename TurnLaneDataT::value_type, util::guidance::LaneTupleIdPair>::value,
@@ -291,7 +289,7 @@ inline void readTurnLaneData(const boost::filesystem::path &path, TurnLaneDataT
// writes .osrm.tld
template <typename TurnLaneDataT>
inline void writeTurnLaneData(const boost::filesystem::path &path,
inline void writeTurnLaneData(const std::filesystem::path &path,
const TurnLaneDataT &turn_lane_data)
{
static_assert(
@@ -306,7 +304,7 @@ inline void writeTurnLaneData(const boost::filesystem::path &path,
// reads .osrm.timestamp
template <typename TimestampDataT>
inline void readTimestamp(const boost::filesystem::path &path, TimestampDataT &timestamp)
inline void readTimestamp(const std::filesystem::path &path, TimestampDataT &timestamp)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
@@ -316,7 +314,7 @@ inline void readTimestamp(const boost::filesystem::path &path, TimestampDataT &t
// writes .osrm.timestamp
template <typename TimestampDataT>
inline void writeTimestamp(const boost::filesystem::path &path, const TimestampDataT &timestamp)
inline void writeTimestamp(const std::filesystem::path &path, const TimestampDataT &timestamp)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
@@ -326,7 +324,7 @@ inline void writeTimestamp(const boost::filesystem::path &path, const TimestampD
// reads .osrm.maneuver_overrides
template <typename StorageManeuverOverrideT, typename NodeSequencesT>
inline void readManeuverOverrides(const boost::filesystem::path &path,
inline void readManeuverOverrides(const std::filesystem::path &path,
StorageManeuverOverrideT &maneuver_overrides,
NodeSequencesT &node_sequences)
{
@@ -340,7 +338,7 @@ inline void readManeuverOverrides(const boost::filesystem::path &path,
}
// writes .osrm.maneuver_overrides
inline void writeManeuverOverrides(const boost::filesystem::path &path,
inline void writeManeuverOverrides(const std::filesystem::path &path,
const std::vector<StorageManeuverOverride> &maneuver_overrides,
const std::vector<NodeID> &node_sequences)
{
@@ -355,7 +353,7 @@ inline void writeManeuverOverrides(const boost::filesystem::path &path,
// writes .osrm.turn_weight_penalties
template <typename TurnPenaltyT>
inline void writeTurnWeightPenalty(const boost::filesystem::path &path,
inline void writeTurnWeightPenalty(const std::filesystem::path &path,
const TurnPenaltyT &turn_penalty)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
@@ -366,7 +364,7 @@ inline void writeTurnWeightPenalty(const boost::filesystem::path &path,
// read .osrm.turn_weight_penalties
template <typename TurnPenaltyT>
inline void readTurnWeightPenalty(const boost::filesystem::path &path, TurnPenaltyT &turn_penalty)
inline void readTurnWeightPenalty(const std::filesystem::path &path, TurnPenaltyT &turn_penalty)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
@@ -376,7 +374,7 @@ inline void readTurnWeightPenalty(const boost::filesystem::path &path, TurnPenal
// writes .osrm.turn_duration_penalties
template <typename TurnPenaltyT>
inline void writeTurnDurationPenalty(const boost::filesystem::path &path,
inline void writeTurnDurationPenalty(const std::filesystem::path &path,
const TurnPenaltyT &turn_penalty)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
@@ -387,7 +385,7 @@ inline void writeTurnDurationPenalty(const boost::filesystem::path &path,
// read .osrm.turn_weight_penalties
template <typename TurnPenaltyT>
inline void readTurnDurationPenalty(const boost::filesystem::path &path, TurnPenaltyT &turn_penalty)
inline void readTurnDurationPenalty(const std::filesystem::path &path, TurnPenaltyT &turn_penalty)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
@@ -397,7 +395,7 @@ inline void readTurnDurationPenalty(const boost::filesystem::path &path, TurnPen
// writes .osrm.turn_penalties_index
template <typename TurnIndexT>
inline void writeTurnPenaltiesIndex(const boost::filesystem::path &path,
inline void writeTurnPenaltiesIndex(const std::filesystem::path &path,
const TurnIndexT &turn_penalties_index)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
@@ -408,7 +406,7 @@ inline void writeTurnPenaltiesIndex(const boost::filesystem::path &path,
// read .osrm.turn_penalties_index
template <typename TurnIndexT>
inline void readTurnPenaltiesIndex(const boost::filesystem::path &path,
inline void readTurnPenaltiesIndex(const std::filesystem::path &path,
TurnIndexT &turn_penalties_index)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
@@ -419,7 +417,7 @@ inline void readTurnPenaltiesIndex(const boost::filesystem::path &path,
// writes .osrm.restrictions
template <typename ConditionalRestrictionsT>
inline void writeConditionalRestrictions(const boost::filesystem::path &path,
inline void writeConditionalRestrictions(const std::filesystem::path &path,
const ConditionalRestrictionsT &conditional_restrictions)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
@@ -430,7 +428,7 @@ inline void writeConditionalRestrictions(const boost::filesystem::path &path,
// read .osrm.restrictions
template <typename ConditionalRestrictionsT>
inline void readConditionalRestrictions(const boost::filesystem::path &path,
inline void readConditionalRestrictions(const std::filesystem::path &path,
ConditionalRestrictionsT &conditional_restrictions)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
@@ -441,7 +439,7 @@ inline void readConditionalRestrictions(const boost::filesystem::path &path,
// reads .osrm file which is a temporary file of osrm-extract
template <typename PackedOSMIDsT>
void readRawNBGraph(const boost::filesystem::path &path,
void readRawNBGraph(const std::filesystem::path &path,
std::vector<util::Coordinate> &coordinates,
PackedOSMIDsT &osm_node_ids,
std::vector<extractor::NodeBasedEdge> &edge_list)
@@ -466,8 +464,7 @@ void readRawNBGraph(const boost::filesystem::path &path,
storage::serialization::read(reader, "/extractor/edges", edge_list);
}
template <typename NameTableT>
void readNames(const boost::filesystem::path &path, NameTableT &table)
template <typename NameTableT> void readNames(const std::filesystem::path &path, NameTableT &table)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
@@ -476,7 +473,7 @@ void readNames(const boost::filesystem::path &path, NameTableT &table)
}
template <typename NameTableT>
void writeNames(const boost::filesystem::path &path, const NameTableT &table)
void writeNames(const std::filesystem::path &path, const NameTableT &table)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
@@ -485,7 +482,7 @@ void writeNames(const boost::filesystem::path &path, const NameTableT &table)
}
template <typename NodeWeightsVectorT>
void readEdgeBasedNodeWeights(const boost::filesystem::path &path, NodeWeightsVectorT &weights)
void readEdgeBasedNodeWeights(const std::filesystem::path &path, NodeWeightsVectorT &weights)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
@@ -494,8 +491,7 @@ void readEdgeBasedNodeWeights(const boost::filesystem::path &path, NodeWeightsVe
}
template <typename NodeDistancesVectorT>
void readEdgeBasedNodeDistances(const boost::filesystem::path &path,
NodeDistancesVectorT &distances)
void readEdgeBasedNodeDistances(const std::filesystem::path &path, NodeDistancesVectorT &distances)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
@@ -504,7 +500,7 @@ void readEdgeBasedNodeDistances(const boost::filesystem::path &path,
}
template <typename NodeWeightsVectorT, typename NodeDurationsVectorT, typename NodeDistancesVectorT>
void writeEdgeBasedNodeWeightsDurationsDistances(const boost::filesystem::path &path,
void writeEdgeBasedNodeWeightsDurationsDistances(const std::filesystem::path &path,
const NodeWeightsVectorT &weights,
const NodeDurationsVectorT &durations,
const NodeDistancesVectorT &distances)
@@ -518,7 +514,7 @@ void writeEdgeBasedNodeWeightsDurationsDistances(const boost::filesystem::path &
}
template <typename NodeWeightsVectorT, typename NodeDurationsVectorT>
void readEdgeBasedNodeWeightsDurations(const boost::filesystem::path &path,
void readEdgeBasedNodeWeightsDurations(const std::filesystem::path &path,
NodeWeightsVectorT &weights,
NodeDurationsVectorT &durations)
{
@@ -530,7 +526,7 @@ void readEdgeBasedNodeWeightsDurations(const boost::filesystem::path &path,
}
template <typename NodeWeightsVectorT, typename NodeDurationsVectorT>
void writeEdgeBasedNodeWeightsDurations(const boost::filesystem::path &path,
void writeEdgeBasedNodeWeightsDurations(const std::filesystem::path &path,
const NodeWeightsVectorT &weights,
const NodeDurationsVectorT &durations)
{
@@ -542,7 +538,7 @@ void writeEdgeBasedNodeWeightsDurations(const boost::filesystem::path &path,
}
template <typename RTreeT>
void writeRamIndex(const boost::filesystem::path &path, const RTreeT &rtree)
void writeRamIndex(const std::filesystem::path &path, const RTreeT &rtree)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
@@ -550,7 +546,7 @@ void writeRamIndex(const boost::filesystem::path &path, const RTreeT &rtree)
util::serialization::write(writer, "/common/rtree", rtree);
}
template <typename RTreeT> void readRamIndex(const boost::filesystem::path &path, RTreeT &rtree)
template <typename RTreeT> void readRamIndex(const std::filesystem::path &path, RTreeT &rtree)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
@@ -559,7 +555,7 @@ template <typename RTreeT> void readRamIndex(const boost::filesystem::path &path
}
template <typename EdgeListT>
void writeCompressedNodeBasedGraph(const boost::filesystem::path &path, const EdgeListT &edge_list)
void writeCompressedNodeBasedGraph(const std::filesystem::path &path, const EdgeListT &edge_list)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
@@ -568,7 +564,7 @@ void writeCompressedNodeBasedGraph(const boost::filesystem::path &path, const Ed
}
template <typename EdgeListT>
void readCompressedNodeBasedGraph(const boost::filesystem::path &path, EdgeListT &edge_list)
void readCompressedNodeBasedGraph(const std::filesystem::path &path, EdgeListT &edge_list)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
@@ -1,13 +1,13 @@
#ifndef OSRM_LOCATION_DEPENDENT_DATA_HPP
#define OSRM_LOCATION_DEPENDENT_DATA_HPP
#include <boost/filesystem/path.hpp>
#include <boost/geometry.hpp>
#include <boost/geometry/geometries/point_xy.hpp>
#include <boost/geometry/index/rtree.hpp>
#include <osmium/osm/way.hpp>
#include <filesystem>
#include <string>
#include <unordered_map>
@@ -30,7 +30,7 @@ struct LocationDependentData
using property_t = boost::variant<boost::blank, double, std::string, bool>;
using properties_t = std::unordered_map<std::string, property_t>;
LocationDependentData(const std::vector<boost::filesystem::path> &file_paths);
LocationDependentData(const std::vector<std::filesystem::path> &file_paths);
bool empty() const { return rtree.empty(); }
@@ -39,7 +39,7 @@ struct LocationDependentData
property_t FindByKey(const std::vector<std::size_t> &property_indexes, const char *key) const;
private:
void loadLocationDependentData(const boost::filesystem::path &file_path,
void loadLocationDependentData(const std::filesystem::path &file_path,
std::vector<rtree_t::value_type> &bounding_boxes);
rtree_t rtree;
@@ -12,8 +12,7 @@
#include "util/coordinate.hpp"
#include "util/node_based_graph.hpp"
#include <boost/filesystem/path.hpp>
#include <filesystem>
#include <memory>
#include <string>
#include <unordered_set>
+2 -3
View File
@@ -7,14 +7,13 @@
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <boost/assert.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/foreach.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/qi_int.hpp>
#include <storage/io.hpp>
#include <filesystem>
#include <iterator>
#include <string>
#include <unordered_map>
@@ -40,7 +39,7 @@ struct RasterDatum
class RasterGrid
{
public:
RasterGrid(const boost::filesystem::path &filepath, std::size_t _xdim, std::size_t _ydim)
RasterGrid(const std::filesystem::path &filepath, std::size_t _xdim, std::size_t _ydim)
{
xdim = _xdim;
ydim = _ydim;
@@ -70,7 +70,7 @@ class Sol2ScriptingEnvironment final : public ScriptingEnvironment
explicit Sol2ScriptingEnvironment(
const std::string &file_name,
const std::vector<boost::filesystem::path> &location_dependent_data_paths);
const std::vector<std::filesystem::path> &location_dependent_data_paths);
~Sol2ScriptingEnvironment() override = default;
const ProfileProperties &GetProfileProperties() override;
+2 -3
View File
@@ -8,13 +8,12 @@
#include "storage/shared_memory_ownership.hpp"
#include "storage/tar_fwd.hpp"
#include <boost/filesystem/path.hpp>
#include <boost/range/adaptor/reversed.hpp>
#include <boost/range/iterator_range.hpp>
#include <unordered_map>
#include <filesystem>
#include <string>
#include <unordered_map>
#include <vector>
namespace osrm::extractor
+6 -2
View File
@@ -10,12 +10,16 @@
#include <boost/assert.hpp>
#include <cstdint>
#include <filesystem>
#include <type_traits>
namespace osrm::guidance::files
{
// reads .osrm.edges
template <typename TurnDataT>
inline void readTurnData(const boost::filesystem::path &path,
inline void readTurnData(const std::filesystem::path &path,
TurnDataT &turn_data,
std::uint32_t &connectivity_checksum)
{
@@ -32,7 +36,7 @@ inline void readTurnData(const boost::filesystem::path &path,
// writes .osrm.edges
template <typename TurnDataT>
inline void writeTurnData(const boost::filesystem::path &path,
inline void writeTurnData(const std::filesystem::path &path,
const TurnDataT &turn_data,
const std::uint32_t connectivity_checksum)
{
@@ -173,8 +173,8 @@ graphToEdges(const DynamicEdgeBasedGraph &edge_based_graph)
for (auto edge : edge_based_graph.GetAdjacentEdgeRange(node))
{
const auto &data = edge_based_graph.GetEdgeData(edge);
// we only need to save the forward edges, since the read method will
// convert from forward to bi-directional edges again
// we only need to save the forward edges, since the read method
// will convert from forward to bi-directional edges again
if (data.forward)
{
auto target = edge_based_graph.GetTarget(edge);
@@ -191,7 +191,7 @@ graphToEdges(const DynamicEdgeBasedGraph &edge_based_graph)
return edges;
}
inline DynamicEdgeBasedGraph LoadEdgeBasedGraph(const boost::filesystem::path &path)
inline DynamicEdgeBasedGraph LoadEdgeBasedGraph(const std::filesystem::path &path)
{
EdgeID number_of_edge_based_nodes;
std::vector<extractor::EdgeBasedEdge> edges;
+6 -6
View File
@@ -10,7 +10,7 @@ namespace osrm::partitioner::files
// read .osrm.partition file
template <typename MultiLevelPartitionT>
inline void readPartition(const boost::filesystem::path &path, MultiLevelPartitionT &mlp)
inline void readPartition(const std::filesystem::path &path, MultiLevelPartitionT &mlp)
{
static_assert(std::is_same<MultiLevelPartitionView, MultiLevelPartitionT>::value ||
std::is_same<MultiLevelPartition, MultiLevelPartitionT>::value,
@@ -24,7 +24,7 @@ inline void readPartition(const boost::filesystem::path &path, MultiLevelPartiti
// writes .osrm.partition file
template <typename MultiLevelPartitionT>
inline void writePartition(const boost::filesystem::path &path, const MultiLevelPartitionT &mlp)
inline void writePartition(const std::filesystem::path &path, const MultiLevelPartitionT &mlp)
{
static_assert(std::is_same<MultiLevelPartitionView, MultiLevelPartitionT>::value ||
std::is_same<MultiLevelPartition, MultiLevelPartitionT>::value,
@@ -38,7 +38,7 @@ inline void writePartition(const boost::filesystem::path &path, const MultiLevel
// reads .osrm.cells file
template <typename CellStorageT>
inline void readCells(const boost::filesystem::path &path, CellStorageT &storage)
inline void readCells(const std::filesystem::path &path, CellStorageT &storage)
{
static_assert(std::is_same<CellStorageView, CellStorageT>::value ||
std::is_same<CellStorage, CellStorageT>::value,
@@ -52,7 +52,7 @@ inline void readCells(const boost::filesystem::path &path, CellStorageT &storage
// writes .osrm.cells file
template <typename CellStorageT>
inline void writeCells(const boost::filesystem::path &path, CellStorageT &storage)
inline void writeCells(const std::filesystem::path &path, CellStorageT &storage)
{
static_assert(std::is_same<CellStorageView, CellStorageT>::value ||
std::is_same<CellStorage, CellStorageT>::value,
@@ -66,7 +66,7 @@ inline void writeCells(const boost::filesystem::path &path, CellStorageT &storag
// reads .osrm.mldgr file
template <typename MultiLevelGraphT>
inline void readGraph(const boost::filesystem::path &path,
inline void readGraph(const std::filesystem::path &path,
MultiLevelGraphT &graph,
std::uint32_t &connectivity_checksum)
{
@@ -80,7 +80,7 @@ inline void readGraph(const boost::filesystem::path &path,
// writes .osrm.mldgr file
template <typename MultiLevelGraphT>
inline void writeGraph(const boost::filesystem::path &path,
inline void writeGraph(const std::filesystem::path &path,
const MultiLevelGraphT &graph,
const std::uint32_t connectivity_checksum)
{
+2 -3
View File
@@ -1,9 +1,8 @@
#ifndef OSRM_PARTITIONER_CONFIG_HPP
#define OSRM_PARTITIONER_CONFIG_HPP
#include <boost/filesystem/path.hpp>
#include <array>
#include <filesystem>
#include <string>
#include "storage/io_config.hpp"
@@ -29,7 +28,7 @@ struct PartitionerConfig final : storage::IOConfig
{
}
void UseDefaultOutputNames(const boost::filesystem::path &base)
void UseDefaultOutputNames(const std::filesystem::path &base)
{
IOConfig::UseDefaultOutputNames(base);
}
+13 -14
View File
@@ -10,14 +10,13 @@
#include "util/log.hpp"
#include "util/version.hpp"
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/iostreams/device/array.hpp>
#include <boost/iostreams/seek.hpp>
#include <boost/iostreams/stream.hpp>
#include <cerrno>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <tuple>
#include <type_traits>
@@ -35,11 +34,11 @@ class FileReader
};
FileReader(const std::string &filename, const FingerprintFlag flag)
: FileReader(boost::filesystem::path(filename), flag)
: FileReader(std::filesystem::path(filename), flag)
{
}
FileReader(const boost::filesystem::path &filepath_, const FingerprintFlag flag)
FileReader(const std::filesystem::path &filepath_, const FingerprintFlag flag)
: filepath(filepath_), fingerprint(flag)
{
input_stream.open(filepath, std::ios::binary);
@@ -58,14 +57,14 @@ class FileReader
std::size_t GetSize()
{
const boost::filesystem::path path(filepath);
const std::filesystem::path path(filepath);
try
{
return std::size_t(boost::filesystem::file_size(path)) -
return std::size_t(std::filesystem::file_size(path)) -
((fingerprint == FingerprintFlag::VerifyFingerprint) ? sizeof(util::FingerPrint)
: 0);
}
catch (const boost::filesystem::filesystem_error &ex)
catch (const std::filesystem::filesystem_error &ex)
{
std::cout << ex.what() << std::endl;
throw;
@@ -196,8 +195,8 @@ class FileReader
}
private:
const boost::filesystem::path filepath;
boost::filesystem::ifstream input_stream;
const std::filesystem::path filepath;
std::ifstream input_stream;
FingerprintFlag fingerprint;
};
@@ -211,11 +210,11 @@ class FileWriter
};
FileWriter(const std::string &filename, const FingerprintFlag flag)
: FileWriter(boost::filesystem::path(filename), flag)
: FileWriter(std::filesystem::path(filename), flag)
{
}
FileWriter(const boost::filesystem::path &filepath_, const FingerprintFlag flag)
FileWriter(const std::filesystem::path &filepath_, const FingerprintFlag flag)
: filepath(filepath_), fingerprint(flag)
{
output_stream.open(filepath, std::ios::binary);
@@ -284,8 +283,8 @@ class FileWriter
}
private:
const boost::filesystem::path filepath;
boost::filesystem::ofstream output_stream;
const std::filesystem::path filepath;
std::ofstream output_stream;
FingerprintFlag fingerprint;
};
+13 -13
View File
@@ -3,19 +3,19 @@
#include "util/exception.hpp"
#include <array>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/path.hpp>
#include <array>
#include <filesystem>
#include <string>
namespace osrm::storage
{
struct IOConfig
{
IOConfig(std::vector<boost::filesystem::path> required_input_files_,
std::vector<boost::filesystem::path> optional_input_files_,
std::vector<boost::filesystem::path> output_files_)
IOConfig(std::vector<std::filesystem::path> required_input_files_,
std::vector<std::filesystem::path> optional_input_files_,
std::vector<std::filesystem::path> output_files_)
: required_input_files(std::move(required_input_files_)),
optional_input_files(std::move(optional_input_files_)),
output_files(std::move(output_files_))
@@ -24,7 +24,7 @@ struct IOConfig
bool IsValid() const;
std::vector<std::string> GetMissingFiles() const;
boost::filesystem::path GetPath(const std::string &fileName) const
std::filesystem::path GetPath(const std::string &fileName) const
{
if (!IsConfigured(fileName, required_input_files) &&
!IsConfigured(fileName, optional_input_files) && !IsConfigured(fileName, output_files))
@@ -40,11 +40,11 @@ struct IOConfig
return IsConfigured(fileName, required_input_files);
}
boost::filesystem::path base_path;
std::filesystem::path base_path;
protected:
// Infer the base path from the path of the .osrm file
void UseDefaultOutputNames(const boost::filesystem::path &base)
void UseDefaultOutputNames(const std::filesystem::path &base)
{
// potentially strip off the .osrm (or other) extensions for
// determining the base path=
@@ -67,7 +67,7 @@ struct IOConfig
private:
static bool IsConfigured(const std::string &fileName,
const std::vector<boost::filesystem::path> &paths)
const std::vector<std::filesystem::path> &paths)
{
for (auto &path : paths)
{
@@ -80,9 +80,9 @@ struct IOConfig
return false;
}
std::vector<boost::filesystem::path> required_input_files;
std::vector<boost::filesystem::path> optional_input_files;
std::vector<boost::filesystem::path> output_files;
std::vector<std::filesystem::path> required_input_files;
std::vector<std::filesystem::path> optional_input_files;
std::vector<std::filesystem::path> output_files;
};
} // namespace osrm::storage
+9 -9
View File
@@ -5,8 +5,6 @@
#include "util/exception_utils.hpp"
#include "util/log.hpp"
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/interprocess/mapped_region.hpp>
#ifndef _WIN32
#include <boost/interprocess/xsi_shared_memory.hpp>
@@ -23,6 +21,8 @@
#include <algorithm>
#include <exception>
#include <filesystem>
#include <fstream>
#include <thread>
#include "storage/shared_memory_ownership.hpp"
@@ -32,10 +32,10 @@ namespace osrm::storage
struct OSRMLockFile
{
template <typename IdentifierT> boost::filesystem::path operator()(const IdentifierT &id)
template <typename IdentifierT> std::filesystem::path operator()(const IdentifierT &id)
{
boost::filesystem::path temp_dir = boost::filesystem::temp_directory_path();
boost::filesystem::path lock_file = temp_dir / ("osrm-" + std::to_string(id) + ".lock");
std::filesystem::path temp_dir = std::filesystem::temp_directory_path();
std::filesystem::path lock_file = temp_dir / ("osrm-" + std::to_string(id) + ".lock");
return lock_file;
}
};
@@ -51,7 +51,7 @@ class SharedMemory
SharedMemory &operator=(const SharedMemory &) = delete;
template <typename IdentifierT>
SharedMemory(const boost::filesystem::path &lock_file,
SharedMemory(const std::filesystem::path &lock_file,
const IdentifierT id,
const uint64_t size = 0)
: key(lock_file.string().c_str(), id)
@@ -202,7 +202,7 @@ class SharedMemory
void *Ptr() const { return region.get_address(); }
std::size_t Size() const { return region.get_size(); }
SharedMemory(const boost::filesystem::path &lock_file, const int id, const uint64_t size = 0)
SharedMemory(const std::filesystem::path &lock_file, const int id, const uint64_t size = 0)
{
sprintf(key, "%s.%d", "osrm.lock", id);
if (0 == size)
@@ -290,7 +290,7 @@ std::unique_ptr<SharedMemory> makeSharedMemory(const IdentifierT &id, const uint
try
{
LockFileT lock_file;
if (!boost::filesystem::exists(lock_file(id)))
if (!std::filesystem::exists(lock_file(id)))
{
if (0 == size)
{
@@ -298,7 +298,7 @@ std::unique_ptr<SharedMemory> makeSharedMemory(const IdentifierT &id, const uint
}
else
{
boost::filesystem::ofstream ofs(lock_file(id));
std::ofstream ofs(lock_file(id));
}
}
return std::make_unique<SharedMemory>(lock_file(id), id, size);
+5 -6
View File
@@ -32,15 +32,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "storage/shared_datatype.hpp"
#include "storage/storage_config.hpp"
#include <boost/filesystem/path.hpp>
#include <filesystem>
#include <string>
#include <vector>
namespace osrm::storage
{
void populateLayoutFromFile(const boost::filesystem::path &path, storage::BaseDataLayout &layout);
void populateLayoutFromFile(const std::filesystem::path &path, storage::BaseDataLayout &layout);
class Storage
{
@@ -51,10 +50,10 @@ class Storage
void PopulateStaticData(const SharedDataIndex &index);
void PopulateUpdatableData(const SharedDataIndex &index);
void PopulateLayout(storage::BaseDataLayout &layout,
const std::vector<std::pair<bool, boost::filesystem::path>> &files);
const std::vector<std::pair<bool, std::filesystem::path>> &files);
std::string PopulateLayoutWithRTree(storage::BaseDataLayout &layout);
std::vector<std::pair<bool, boost::filesystem::path>> GetUpdatableFiles();
std::vector<std::pair<bool, boost::filesystem::path>> GetStaticFiles();
std::vector<std::pair<bool, std::filesystem::path>> GetUpdatableFiles();
std::vector<std::pair<bool, std::filesystem::path>> GetStaticFiles();
private:
StorageConfig config;
+7 -6
View File
@@ -28,22 +28,23 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef STORAGE_CONFIG_HPP
#define STORAGE_CONFIG_HPP
#include <boost/filesystem/path.hpp>
#include "storage/io_config.hpp"
#include "osrm/datasets.hpp"
#include <filesystem>
#include <istream>
#include <set>
#include <vector>
namespace osrm::storage
{
std::istream &operator>>(std::istream &in, FeatureDataset &datasets);
static std::vector<boost::filesystem::path>
static std::vector<std::filesystem::path>
GetRequiredFiles(const std::vector<storage::FeatureDataset> &disabled_feature_dataset)
{
std::set<boost::filesystem::path> required{
std::set<std::filesystem::path> required{
".osrm.datasource_names",
".osrm.ebg_nodes",
".osrm.edges",
@@ -82,7 +83,7 @@ GetRequiredFiles(const std::vector<storage::FeatureDataset> &disabled_feature_da
}
}
return std::vector<boost::filesystem::path>(required.begin(), required.end());
return std::vector<std::filesystem::path>(required.begin(), required.end());
;
}
@@ -94,7 +95,7 @@ GetRequiredFiles(const std::vector<storage::FeatureDataset> &disabled_feature_da
struct StorageConfig final : IOConfig
{
StorageConfig(const boost::filesystem::path &base,
StorageConfig(const std::filesystem::path &base,
const std::vector<storage::FeatureDataset> &disabled_feature_datasets_ = {})
: StorageConfig(disabled_feature_datasets_)
{
+6 -6
View File
@@ -7,7 +7,7 @@
#include "util/integer_range.hpp"
#include "util/version.hpp"
#include <boost/filesystem/path.hpp>
#include <filesystem>
extern "C"
{
@@ -19,7 +19,7 @@ namespace osrm::storage::tar
namespace detail
{
inline void
checkMTarError(int error_code, const boost::filesystem::path &filepath, const std::string &name)
checkMTarError(int error_code, const std::filesystem::path &filepath, const std::string &name)
{
switch (error_code)
{
@@ -78,7 +78,7 @@ class FileReader
HasNoFingerprint
};
FileReader(const boost::filesystem::path &path, FingerprintFlag flag) : path(path)
FileReader(const std::filesystem::path &path, FingerprintFlag flag) : path(path)
{
auto ret = mtar_open(&handle, path.string().c_str(), "r");
detail::checkMTarError(ret, path, "");
@@ -204,7 +204,7 @@ class FileReader
return true;
}
boost::filesystem::path path;
std::filesystem::path path;
mtar_t handle;
};
@@ -217,7 +217,7 @@ class FileWriter
HasNoFingerprint
};
FileWriter(const boost::filesystem::path &path, FingerprintFlag flag) : path(path)
FileWriter(const std::filesystem::path &path, FingerprintFlag flag) : path(path)
{
auto ret = mtar_open(&handle, path.string().c_str(), "w");
detail::checkMTarError(ret, path, "");
@@ -305,7 +305,7 @@ class FileWriter
WriteFrom("osrm_fingerprint.meta", fingerprint);
}
boost::filesystem::path path;
std::filesystem::path path;
mtar_t handle;
};
} // namespace osrm::storage::tar
+1 -1
View File
@@ -202,7 +202,7 @@ inline auto make_search_tree_view(const SharedDataIndex &index, const std::strin
const char *path = index.template GetBlockPtr<char>(name + "/file_index_path");
if (!boost::filesystem::exists(boost::filesystem::path{path}))
if (!std::filesystem::exists(std::filesystem::path{path}))
{
throw util::exception("Could not load " + std::string(path) + "Does the leaf file exist?" +
SOURCE_REF);
+2 -2
View File
@@ -12,12 +12,12 @@
#include <tbb/spin_mutex.h>
#include <boost/exception/diagnostic_information.hpp>
#include <boost/filesystem.hpp>
#include <boost/iostreams/device/mapped_file.hpp>
#include <boost/phoenix.hpp>
#include <boost/spirit/include/qi.hpp>
#include <exception>
#include <filesystem>
#include <stdexcept>
#include <vector>
@@ -100,7 +100,7 @@ template <typename Key, typename Value> struct CSVFilesParser
std::vector<std::pair<Key, Value>> result;
try
{
if (boost::filesystem::file_size(filename) == 0)
if (std::filesystem::file_size(filename) == 0)
return result;
boost::iostreams::mapped_file_source mmap(filename);
+5 -6
View File
@@ -28,14 +28,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef OSRM_UPDATER_UPDATER_CONFIG_HPP
#define OSRM_UPDATER_UPDATER_CONFIG_HPP
#include <boost/filesystem/path.hpp>
#include <chrono>
#include <string>
#include "storage/io_config.hpp"
#include "storage/storage_config.hpp"
#include <chrono>
#include <filesystem>
#include <string>
namespace osrm::updater
{
@@ -59,7 +58,7 @@ struct UpdaterConfig final : storage::IOConfig
{
}
void UseDefaultOutputNames(const boost::filesystem::path &base)
void UseDefaultOutputNames(const std::filesystem::path &base)
{
IOConfig::UseDefaultOutputNames(base);
}
+40
View File
@@ -0,0 +1,40 @@
#pragma once
#include <algorithm>
#include <boost/assert.hpp>
#include <vector>
namespace osrm::util
{
// in its essence it is std::priority_queue, but with `clear` method
template <typename T> class BinaryHeap
{
public:
bool empty() const { return heap_.empty(); }
const T &top() const
{
BOOST_ASSERT(!heap_.empty());
return heap_.front();
}
void pop()
{
BOOST_ASSERT(!heap_.empty());
std::pop_heap(heap_.begin(), heap_.end());
heap_.pop_back();
}
template <typename... Args> void emplace(Args &&...args)
{
heap_.emplace_back(std::forward<Args>(args)...);
std::push_heap(heap_.begin(), heap_.end());
}
void clear() { heap_.clear(); }
private:
std::vector<T> heap_;
};
} // namespace osrm::util
+2 -2
View File
@@ -8,7 +8,7 @@ extern "C"
#include <lualib.h>
}
#include <boost/filesystem.hpp>
#include <filesystem>
#include <string>
@@ -20,7 +20,7 @@ namespace osrm::util
// See http://lua-users.org/wiki/PackagePath for details on the package.path syntax.
inline void luaAddScriptFolderToLoadPath(lua_State *lua_state, const char *file_name)
{
boost::filesystem::path profile_path = boost::filesystem::canonical(file_name);
std::filesystem::path profile_path = std::filesystem::canonical(file_name);
std::string folder = profile_path.parent_path().generic_string();
const std::string lua_code = "package.path = \"" + folder + "/?.lua;\" .. package.path";
luaL_dostring(lua_state, lua_code.c_str());
+10 -9
View File
@@ -5,20 +5,22 @@
#include "util/exception_utils.hpp"
#include "util/vector_view.hpp"
#include <boost/filesystem/path.hpp>
#include <boost/iostreams/device/mapped_file.hpp>
#include <filesystem>
namespace osrm::util
{
namespace detail
{
template <typename T, typename MmapContainerT>
util::vector_view<T> mmapFile(const boost::filesystem::path &file, MmapContainerT &mmap_container)
util::vector_view<T> mmapFile(const std::filesystem::path &file, MmapContainerT &mmap_container)
{
try
{
mmap_container.open(file);
auto path_string = file.string();
mmap_container.open(path_string);
std::size_t num_objects = mmap_container.size() / sizeof(T);
auto data_ptr = mmap_container.data();
BOOST_ASSERT(reinterpret_cast<uintptr_t>(data_ptr) % alignof(T) == 0);
@@ -33,9 +35,8 @@ util::vector_view<T> mmapFile(const boost::filesystem::path &file, MmapContainer
}
template <typename T, typename MmapContainerT>
util::vector_view<T> mmapFile(const boost::filesystem::path &file,
MmapContainerT &mmap_container,
const std::size_t size)
util::vector_view<T>
mmapFile(const std::filesystem::path &file, MmapContainerT &mmap_container, const std::size_t size)
{
try
{
@@ -61,21 +62,21 @@ util::vector_view<T> mmapFile(const boost::filesystem::path &file,
} // namespace detail
template <typename T>
util::vector_view<const T> mmapFile(const boost::filesystem::path &file,
util::vector_view<const T> mmapFile(const std::filesystem::path &file,
boost::iostreams::mapped_file_source &mmap_container)
{
return detail::mmapFile<const T>(file, mmap_container);
}
template <typename T>
util::vector_view<T> mmapFile(const boost::filesystem::path &file,
util::vector_view<T> mmapFile(const std::filesystem::path &file,
boost::iostreams::mapped_file &mmap_container)
{
return detail::mmapFile<T>(file, mmap_container);
}
template <typename T>
util::vector_view<T> mmapFile(const boost::filesystem::path &file,
util::vector_view<T> mmapFile(const std::filesystem::path &file,
boost::iostreams::mapped_file &mmap_container,
std::size_t size)
{
+1 -1
View File
@@ -15,7 +15,7 @@ namespace osrm::util
using DataRange = std::pair<const char *, const char *>;
using DataMap = std::unordered_map<std::string, DataRange>;
inline DataMap mmapTarFile(const boost::filesystem::path &path,
inline DataMap mmapTarFile(const std::filesystem::path &path,
boost::iostreams::mapped_file_source &region)
{
DataMap map;
+5 -67
View File
@@ -5,58 +5,19 @@
#include <boost/heap/d_ary_heap.hpp>
#include <algorithm>
#include <ankerl/unordered_dense.h>
#include <boost/unordered_map.hpp>
#include <cstdint>
#include <limits>
#include <map>
#include <optional>
#include <unordered_map>
#include <unordered_set>
#include <vector>
namespace osrm::util
{
template <typename NodeID, typename Key> class GenerationArrayStorage
{
using GenerationCounter = std::uint16_t;
public:
explicit GenerationArrayStorage(std::size_t size)
: positions(size, 0), generation(1), generations(size, 0)
{
}
Key &operator[](NodeID node)
{
generation[node] = generation;
return positions[node];
}
Key peek_index(const NodeID node) const
{
if (generations[node] < generation)
{
return std::numeric_limits<Key>::max();
}
return positions[node];
}
void Clear()
{
generation++;
// if generation overflows we end up at 0 again and need to clear the vector
if (generation == 0)
{
generation = 1;
std::fill(generations.begin(), generations.end(), 0);
}
}
private:
GenerationCounter generation;
std::vector<GenerationCounter> generations;
std::vector<Key> positions;
};
template <typename NodeID, typename Key> class ArrayStorage
{
public:
@@ -72,33 +33,10 @@ template <typename NodeID, typename Key> class ArrayStorage
std::vector<Key> positions;
};
template <typename NodeID, typename Key> class MapStorage
{
public:
explicit MapStorage(std::size_t) {}
Key &operator[](NodeID node) { return nodes[node]; }
void Clear() { nodes.clear(); }
Key peek_index(const NodeID node) const
{
const auto iter = nodes.find(node);
if (nodes.end() != iter)
{
return iter->second;
}
return std::numeric_limits<Key>::max();
}
private:
std::map<NodeID, Key> nodes;
};
template <typename NodeID, typename Key> class UnorderedMapStorage
{
public:
explicit UnorderedMapStorage(std::size_t) { nodes.rehash(1000); }
explicit UnorderedMapStorage(std::size_t) {}
Key &operator[](const NodeID node) { return nodes[node]; }
@@ -121,7 +59,7 @@ template <typename NodeID, typename Key> class UnorderedMapStorage
void Clear() { nodes.clear(); }
private:
std::unordered_map<NodeID, Key> nodes;
ankerl::unordered_dense::segmented_map<NodeID, Key> nodes;
};
template <typename NodeID,
+17 -13
View File
@@ -2,7 +2,9 @@
#define STATIC_RTREE_HPP
#include "storage/tar_fwd.hpp"
#include "osrm/coordinate.hpp"
#include "util/bearing.hpp"
#include "util/binary_heap.hpp"
#include "util/coordinate_calculation.hpp"
#include "util/deallocating_vector.hpp"
#include "util/exception.hpp"
@@ -15,12 +17,9 @@
#include "util/vector_view.hpp"
#include "util/web_mercator.hpp"
#include "osrm/coordinate.hpp"
#include "storage/shared_memory_ownership.hpp"
#include <boost/assert.hpp>
#include <boost/filesystem.hpp>
#include <boost/format.hpp>
#include <boost/iostreams/device/mapped_file.hpp>
@@ -30,6 +29,7 @@
#include <algorithm>
#include <array>
#include <filesystem>
#include <limits>
#include <memory>
#include <queue>
@@ -271,7 +271,7 @@ class StaticRTree
// Construct a packed Hilbert-R-Tree with Kamel-Faloutsos algorithm [1]
explicit StaticRTree(const std::vector<EdgeDataT> &input_data_vector,
const Vector<Coordinate> &coordinate_list,
const boost::filesystem::path &on_disk_file_name)
const std::filesystem::path &on_disk_file_name)
: m_coordinate_list(coordinate_list.data(), coordinate_list.size())
{
const auto element_count = input_data_vector.size();
@@ -458,7 +458,7 @@ class StaticRTree
* Constructs an empty RTree for de-serialization.
*/
template <typename = std::enable_if<Ownership == storage::Ownership::Container>>
explicit StaticRTree(const boost::filesystem::path &on_disk_file_name,
explicit StaticRTree(const std::filesystem::path &on_disk_file_name,
const Vector<Coordinate> &coordinate_list)
: m_coordinate_list(coordinate_list.data(), coordinate_list.size()),
m_objects(mmapFile<EdgeDataT>(on_disk_file_name, m_objects_region))
@@ -473,7 +473,7 @@ class StaticRTree
*/
explicit StaticRTree(Vector<TreeNode> search_tree_,
Vector<std::uint64_t> tree_level_starts,
const boost::filesystem::path &on_disk_file_name,
const std::filesystem::path &on_disk_file_name,
const Vector<Coordinate> &coordinate_list)
: m_search_tree(std::move(search_tree_)),
m_coordinate_list(coordinate_list.data(), coordinate_list.size()),
@@ -554,9 +554,12 @@ class StaticRTree
auto projected_coordinate = web_mercator::fromWGS84(input_coordinate);
Coordinate fixed_projected_coordinate{projected_coordinate};
// we re-use queue for each query to avoid re-allocating memory
static thread_local util::BinaryHeap<QueryCandidate> traversal_queue;
traversal_queue.clear();
// initialize queue with root element
std::priority_queue<QueryCandidate> traversal_queue;
traversal_queue.push(QueryCandidate{0, TreeIndex{}});
traversal_queue.emplace(QueryCandidate{0, TreeIndex{}});
while (!traversal_queue.empty())
{
@@ -710,10 +713,11 @@ class StaticRTree
// distance must be non-negative
BOOST_ASSERT(0. <= squared_distance);
BOOST_ASSERT(i < std::numeric_limits<std::uint32_t>::max());
traversal_queue.push(QueryCandidate{squared_distance,
leaf_id,
static_cast<std::uint32_t>(i),
Coordinate{projected_nearest}});
traversal_queue.emplace(QueryCandidate{squared_distance,
leaf_id,
static_cast<std::uint32_t>(i),
Coordinate{projected_nearest}});
}
}
@@ -742,7 +746,7 @@ class StaticRTree
child.minimum_bounding_rectangle.GetMinSquaredDist(
fixed_projected_input_coordinate);
traversal_queue.push(QueryCandidate{
traversal_queue.emplace(QueryCandidate{
squared_lower_bound_to_element,
TreeIndex(parent.level + 1, child_index - m_tree_level_starts[parent.level + 1])});
}
+2 -2
View File
@@ -3,13 +3,13 @@
#include "util/log.hpp"
#include <boost/filesystem/path.hpp>
#include <boost/geometry.hpp>
#include <boost/geometry/index/rtree.hpp>
#include <rapidjson/document.h>
#include <chrono>
#include <filesystem>
#include <optional>
namespace osrm::updater
@@ -32,7 +32,7 @@ class Timezoner
Timezoner() = default;
Timezoner(const char geojson[], std::time_t utc_time_now);
Timezoner(const boost::filesystem::path &tz_shapes_filename, std::time_t utc_time_now);
Timezoner(const std::filesystem::path &tz_shapes_filename, std::time_t utc_time_now);
std::optional<struct tm> operator()(const point_t &point) const;
+54 -18
View File
@@ -1,5 +1,4 @@
import requests
import sys
import random
from collections import defaultdict
import os
@@ -8,12 +7,13 @@ import numpy as np
import time
import argparse
class BenchmarkRunner:
def __init__(self):
def __init__(self, gps_traces_file_path):
self.coordinates = []
self.tracks = defaultdict(list)
gps_traces_file_path = os.path.expanduser('~/gps_traces.csv')
gps_traces_file_path = os.path.expanduser(gps_traces_file_path)
with open(gps_traces_file_path, 'r') as file:
reader = csv.DictReader(file)
for row in reader:
@@ -36,10 +36,9 @@ class BenchmarkRunner:
response = requests.get(url)
end_time = time.time()
if response.status_code != 200:
if benchmark_name == 'match':
code = response.json()['code']
if code == 'NoSegment' or code == 'NoMatch':
continue
code = response.json()['code']
if code in ['NoSegment', 'NoMatch', 'NoRoute', 'NoTrips']:
continue
raise Exception(f"Error: {response.status_code} {response.text}")
times.append((end_time - start_time) * 1000) # convert to ms
@@ -54,7 +53,7 @@ class BenchmarkRunner:
end_coord = f"{end[1]:.6f},{end[0]:.6f}"
return f"{host}/route/v1/driving/{start_coord};{end_coord}?overview=full&steps=true"
elif benchmark_name == 'table':
num_coords = random.randint(3, 100)
num_coords = random.randint(3, 12)
selected_coords = random.sample(self.coordinates, num_coords)
coords_str = ";".join([f"{coord[1]:.6f},{coord[0]:.6f}" for coord in selected_coords])
return f"{host}/table/v1/driving/{coords_str}"
@@ -77,26 +76,63 @@ class BenchmarkRunner:
else:
raise Exception(f"Unknown benchmark: {benchmark_name}")
def bootstrap_confidence_interval(data, num_samples=1000, confidence_level=0.95):
means = []
for _ in range(num_samples):
sample = np.random.choice(data, size=len(data), replace=True)
means.append(np.mean(sample))
lower_bound = np.percentile(means, (1 - confidence_level) / 2 * 100)
upper_bound = np.percentile(means, (1 + confidence_level) / 2 * 100)
mean = np.mean(means)
return mean, lower_bound, upper_bound
def calculate_confidence_interval(data):
mean, lower, upper = bootstrap_confidence_interval(data)
min_value = np.min(data)
return mean, (upper - lower) / 2, min_value
def main():
parser = argparse.ArgumentParser(description='Run GPS benchmark tests.')
parser.add_argument('--host', type=str, required=True, help='Host URL')
parser.add_argument('--method', type=str, required=True, choices=['route', 'table', 'match', 'nearest', 'trip'], help='Benchmark method')
parser.add_argument('--num_requests', type=int, required=True, help='Number of requests to perform')
parser.add_argument('--iterations', type=int, required=True, help='Number of iterations to run the benchmark')
parser.add_argument('--gps_traces_file_path', type=str, required=True, help='Path to the GPS traces file')
args = parser.parse_args()
random.seed(42)
np.random.seed(42)
runner = BenchmarkRunner()
times = runner.run(args.method, args.host, args.num_requests)
runner = BenchmarkRunner(args.gps_traces_file_path)
all_times = []
for _ in range(args.iterations):
random.seed(42)
times = runner.run(args.method, args.host, args.num_requests)
all_times.append(times)
all_times = np.asarray(all_times)
print(f'Total: {np.sum(times)}ms')
print(f"Min time: {np.min(times)}ms")
print(f"Mean time: {np.mean(times)}ms")
print(f"Median time: {np.median(times)}ms")
print(f"95th percentile: {np.percentile(times, 95)}ms")
print(f"99th percentile: {np.percentile(times, 99)}ms")
print(f"Max time: {np.max(times)}ms")
assert all_times.shape == (args.iterations, all_times.shape[1])
total_time, total_ci, total_best = calculate_confidence_interval(np.sum(all_times, axis=1))
ops_per_sec, ops_per_sec_ci, ops_per_sec_best = calculate_confidence_interval(float(all_times.shape[1]) / np.sum(all_times / 1000, axis=1))
min_time, min_ci, _ = calculate_confidence_interval(np.min(all_times, axis=1))
mean_time, mean_ci, _ = calculate_confidence_interval(np.mean(all_times, axis=1))
median_time, median_ci, _ = calculate_confidence_interval(np.median(all_times, axis=1))
perc_95_time, perc_95_ci, _ = calculate_confidence_interval(np.percentile(all_times, 95, axis=1))
perc_99_time, perc_99_ci, _ = calculate_confidence_interval(np.percentile(all_times, 99, axis=1))
max_time, max_ci, _ = calculate_confidence_interval(np.max(all_times, axis=1))
print(f'Ops: {ops_per_sec:.2f} ± {ops_per_sec_ci:.2f} ops/s. Best: {ops_per_sec_best:.2f} ops/s')
print(f'Total: {total_time:.2f}ms ± {total_ci:.2f}ms. Best: {total_best:.2f}ms')
print(f"Min time: {min_time:.2f}ms ± {min_ci:.2f}ms")
print(f"Mean time: {mean_time:.2f}ms ± {mean_ci:.2f}ms")
print(f"Median time: {median_time:.2f}ms ± {median_ci:.2f}ms")
print(f"95th percentile: {perc_95_time:.2f}ms ± {perc_95_ci:.2f}ms")
print(f"99th percentile: {perc_99_time:.2f}ms ± {perc_99_ci:.2f}ms")
print(f"Max time: {max_time:.2f}ms ± {max_ci:.2f}ms")
if __name__ == '__main__':
main()
+1 -1
View File
@@ -102,4 +102,4 @@ def main():
if __name__ == "__main__":
main()
main()
+82 -33
View File
@@ -1,72 +1,121 @@
#!/bin/bash
set -eou pipefail
function usage {
echo "Usage: $0 -f <folder> -r <results_folder> -s <scripts_folder> -b <binaries_folder> -o <osm_pbf> -g <gps_traces>"
exit 1
}
while getopts ":f:r:s:b:o:g:" opt; do
case $opt in
f) FOLDER="$OPTARG"
;;
r) RESULTS_FOLDER="$OPTARG"
;;
s) SCRIPTS_FOLDER="$OPTARG"
;;
b) BINARIES_FOLDER="$OPTARG"
;;
o) OSM_PBF="$OPTARG"
;;
g) GPS_TRACES="$OPTARG"
;;
\?) echo "Invalid option -$OPTARG" >&2
usage
;;
:) echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
if [ -z "${FOLDER:-}" ] || [ -z "${RESULTS_FOLDER:-}" ] || [ -z "${SCRIPTS_FOLDER:-}" ] || [ -z "${BINARIES_FOLDER:-}" ] || [ -z "${OSM_PBF:-}" ] || [ -z "${GPS_TRACES:-}" ]; then
usage
fi
function measure_peak_ram_and_time {
COMMAND=$1
OUTPUT_FILE=$2
if [ "$(uname)" == "Darwin" ]; then
# on macOS time has different parameters, so simply run command on macOS
$COMMAND > /dev/null 2>&1
else
OUTPUT=$(/usr/bin/time -f "%e %M" $COMMAND 2>&1 | tail -n 1)
OUTPUT=$(/usr/bin/time -f "%e %M" $COMMAND 2>&1 | tail -n 1)
TIME=$(echo $OUTPUT | awk '{print $1}')
PEAK_RAM_KB=$(echo $OUTPUT | awk '{print $2}')
PEAK_RAM_MB=$(echo "scale=2; $PEAK_RAM_KB / 1024" | bc)
echo "Time: ${TIME}s Peak RAM: ${PEAK_RAM_MB}MB" > $OUTPUT_FILE
TIME=$(echo $OUTPUT | awk '{print $1}')
PEAK_RAM_KB=$(echo $OUTPUT | awk '{print $2}')
PEAK_RAM_MB=$(echo "scale=2; $PEAK_RAM_KB / 1024" | bc)
echo "Time: ${TIME}s Peak RAM: ${PEAK_RAM_MB}MB" > $OUTPUT_FILE
fi
}
function run_benchmarks_for_folder {
echo "Running benchmarks for $1"
FOLDER=$1
RESULTS_FOLDER=$2
SCRIPTS_FOLDER=$3
mkdir -p $RESULTS_FOLDER
BENCHMARKS_FOLDER="$FOLDER/build/src/benchmarks"
BENCHMARKS_FOLDER="$BINARIES_FOLDER/src/benchmarks"
echo "Running match-bench MLD"
$BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench"
echo "Running match-bench CH"
$BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench"
echo "Running route-bench MLD"
$BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench"
echo "Running route-bench CH"
$BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench"
echo "Running alias"
$BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench"
echo "Running json-render-bench"
$BENCHMARKS_FOLDER/json-render-bench "$FOLDER/test/data/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench"
echo "Running packedvector-bench"
$BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench"
echo "Running rtree-bench"
$BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench"
./$BENCHMARKS_FOLDER/match-bench "./$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench"
./$BENCHMARKS_FOLDER/match-bench "./$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench"
./$BENCHMARKS_FOLDER/route-bench "./$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench"
./$BENCHMARKS_FOLDER/route-bench "./$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench"
./$BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench"
./$BENCHMARKS_FOLDER/json-render-bench "./$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench"
./$BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench"
./$BENCHMARKS_FOLDER/rtree-bench "./$FOLDER/test/data/monaco.osrm.ramIndex" "./$FOLDER/test/data/monaco.osrm.fileIndex" "./$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench"
BINARIES_FOLDER="$FOLDER/build"
cp ~/data.osm.pbf $FOLDER
cp -rf $OSM_PBF $FOLDER/data.osm.pbf
echo "Running osrm-extract"
measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-extract -p $FOLDER/profiles/car.lua $FOLDER/data.osm.pbf" "$RESULTS_FOLDER/osrm_extract.bench"
echo "Running osrm-partition"
measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-partition $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_partition.bench"
echo "Running osrm-customize"
measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-customize $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_customize.bench"
echo "Running osrm-contract"
measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-contract $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_contract.bench"
for BENCH in nearest table trip route match; do
./$BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" mld ~/gps_traces.csv ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_mld.bench" || true
./$BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" ch ~/gps_traces.csv ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_ch.bench" || true
for ALGORITHM in ch mld; do
for BENCH in nearest table trip route match; do
echo "Running random $BENCH $ALGORITHM"
START=$(date +%s.%N)
$BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" $ALGORITHM $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_${ALGORITHM}.bench" 5 || true
END=$(date +%s.%N)
DIFF=$(echo "$END - $START" | bc)
echo "Took: ${DIFF}s"
done
done
for ALGORITHM in ch mld; do
$BINARIES_FOLDER/osrm-routed --algorithm $ALGORITHM $FOLDER/data.osrm &
$BINARIES_FOLDER/osrm-routed --algorithm $ALGORITHM $FOLDER/data.osrm > /dev/null 2>&1 &
OSRM_ROUTED_PID=$!
# wait for osrm-routed to start
if ! curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true"; then
if ! curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true" > /dev/null 2>&1; then
echo "osrm-routed failed to start for algorithm $ALGORITHM"
kill -9 $OSRM_ROUTED_PID
continue
fi
for METHOD in route nearest trip table match; do
python3 $SCRIPTS_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --num_requests 1000 > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench
echo "Running e2e benchmark for $METHOD $ALGORITHM"
START=$(date +%s.%N)
python3 $SCRIPTS_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --iterations 5 --num_requests 1000 --gps_traces_file_path $GPS_TRACES > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench
END=$(date +%s.%N)
DIFF=$(echo "$END - $START" | bc)
echo "Took: ${DIFF}s"
done
kill -9 $OSRM_ROUTED_PID
done
}
run_benchmarks_for_folder $1 "${1}_results" $2
run_benchmarks_for_folder $2 "${2}_results" $2
run_benchmarks_for_folder
+96
View File
@@ -0,0 +1,96 @@
@ECHO OFF
SETLOCAL
SET EL=0
ECHO NUMBER_OF_PROCESSORS^: %NUMBER_OF_PROCESSORS%
SET PROJECT_DIR=%CD%
SET CONFIGURATION=Release
mkdir build
IF %ERRORLEVEL% NEQ 0 GOTO ERROR
cd build
IF %ERRORLEVEL% NEQ 0 GOTO ERROR
cmake -DENABLE_CONAN=ON -DENABLE_NODE_BINDINGS=ON -DCMAKE_BUILD_TYPE=%CONFIGURATION% -G "Visual Studio 17 2022" ..
IF %ERRORLEVEL% NEQ 0 GOTO ERROR
msbuild OSRM.sln ^
/p:Configuration=%CONFIGURATION% ^
/p:Platform=x64 ^
/p:nowarn="4244;4267;4365;4456;4514;4625;4626;4710;4711;4820;5026;5027" ^
/t:rebuild ^
/p:BuildInParallel=true ^
/m:%NUMBER_OF_PROCESSORS% ^
/toolsversion:Current ^
/clp:Verbosity=quiet ^
/nologo
IF %ERRORLEVEL% NEQ 0 GOTO ERROR
CD %PROJECT_DIR%\build
IF %ERRORLEVEL% NEQ 0 GOTO ERROR
ECHO running extractor-tests.exe ...
unit_tests\%CONFIGURATION%\extractor-tests.exe
IF %ERRORLEVEL% NEQ 0 GOTO ERROR
ECHO running contractor-tests.exe ...
unit_tests\%CONFIGURATION%\contractor-tests.exe
IF %ERRORLEVEL% NEQ 0 GOTO ERROR
ECHO running engine-tests.exe ...
unit_tests\%CONFIGURATION%\engine-tests.exe
IF %ERRORLEVEL% NEQ 0 GOTO ERROR
ECHO running util-tests.exe ...
unit_tests\%CONFIGURATION%\util-tests.exe
IF %ERRORLEVEL% NEQ 0 GOTO ERROR
ECHO running server-tests.exe ...
unit_tests\%CONFIGURATION%\server-tests.exe
IF %ERRORLEVEL% NEQ 0 GOTO ERROR
ECHO running partitioner-tests.exe ...
unit_tests\%CONFIGURATION%\partitioner-tests.exe
IF %ERRORLEVEL% NEQ 0 GOTO ERROR
ECHO running customizer-tests.exe ...
unit_tests\%CONFIGURATION%\customizer-tests.exe
IF %ERRORLEVEL% NEQ 0 GOTO ERROR
SET test_region=monaco
SET test_region_ch=ch\monaco
SET test_region_corech=corech\monaco
SET test_region_mld=mld\monaco
SET test_osm=%test_region%.osm.pbf
COPY %PROJECT_DIR%\test\data\%test_region%.osm.pbf %test_osm%
%CONFIGURATION%\osrm-extract.exe -p %PROJECT_DIR%\profiles\car.lua %test_osm%
IF %ERRORLEVEL% NEQ 0 GOTO ERROR
MKDIR ch
XCOPY %test_region%.osrm.* ch\
XCOPY %test_region%.osrm ch\
MKDIR corech
XCOPY %test_region%.osrm.* corech\
XCOPY %test_region%.osrm corech\
MKDIR mld
XCOPY %test_region%.osrm.* mld\
XCOPY %test_region%.osrm mld\
%CONFIGURATION%\osrm-contract.exe %test_region_ch%.osrm
%CONFIGURATION%\osrm-contract.exe --core 0.8 %test_region_corech%.osrm
%CONFIGURATION%\osrm-partition.exe %test_region_mld%.osrm
%CONFIGURATION%\osrm-customize.exe %test_region_mld%.osrm
XCOPY /Y ch\*.* ..\test\data\ch\
XCOPY /Y corech\*.* ..\test\data\corech\
XCOPY /Y mld\*.* ..\test\data\mld\
unit_tests\%CONFIGURATION%\library-tests.exe
IF %ERRORLEVEL% NEQ 0 GOTO ERROR
:ERROR
ECHO ~~~~~~~~~~~~~~~~~~~~~~ ERROR %~f0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
ECHO ERRORLEVEL^: %ERRORLEVEL%
SET EL=%ERRORLEVEL%
:DONE
ECHO ~~~~~~~~~~~~~~~~~~~~~~ DONE %~f0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
EXIT /b %EL%
+4 -1
View File
@@ -30,6 +30,9 @@ VTZERO_TAG=v1.1.0
FMT_PATH="fmtlib/fmt"
FMT_TAG=v10.2.1
ANKERL_PATH="martinus/unordered_dense"
ANKERL_TAG=v4.4.0
function update_subtree () {
name=$(echo "$1" | tr '[:lower:]' '[:upper:]')
path=$(tmpvar=${name}_PATH && echo ${!tmpvar})
@@ -53,6 +56,6 @@ function update_subtree () {
}
## Update dependencies
for dep in osmium sol rapidjson microtar protozero vtzero fmt; do
for dep in ankerl osmium sol rapidjson microtar protozero vtzero fmt; do
update_subtree $dep
done
+400 -312
View File
@@ -45,8 +45,12 @@ class GPSTraces
std::vector<osrm::util::Coordinate> coordinates;
mutable std::mt19937 gen;
int seed;
public:
GPSTraces(int seed) : gen(std::random_device{}()) { gen.seed(seed); }
GPSTraces(int seed) : gen(std::random_device{}()), seed(seed) { gen.seed(seed); }
void resetSeed() const { gen.seed(seed); }
bool readCSV(const std::string &filename)
{
@@ -101,75 +105,200 @@ class GPSTraces
return coordinates[dis(gen)];
}
const std::vector<osrm::util::Coordinate> &getRandomTrace() const
std::vector<osrm::util::Coordinate> getRandomTrace() const
{
std::uniform_int_distribution<> dis(0, trackIDs.size() - 1);
auto it = trackIDs.begin();
std::advance(it, dis(gen));
return traces.at(*it);
const auto &trace = traces.at(*it);
std::uniform_int_distribution<> length_dis(50, 100);
size_t length = length_dis(gen);
if (trace.size() <= length + 1)
{
return trace;
}
std::uniform_int_distribution<> start_dis(0, trace.size() - length - 1);
size_t start_index = start_dis(gen);
return std::vector<osrm::util::Coordinate>(trace.begin() + start_index,
trace.begin() + start_index + length);
}
};
// Struct to hold confidence interval data
struct ConfidenceInterval
{
double mean;
double confidence;
double min;
};
// Helper function to calculate the bootstrap confidence interval
ConfidenceInterval confidenceInterval(const std::vector<double> &data,
int num_samples = 1000,
double confidence_level = 0.95)
{
std::vector<double> means;
std::default_random_engine generator;
std::uniform_int_distribution<int> distribution(0, data.size() - 1);
for (int i = 0; i < num_samples; ++i)
{
std::vector<double> sample;
for (size_t j = 0; j < data.size(); ++j)
{
sample.push_back(data[distribution(generator)]);
}
double sample_mean = std::accumulate(sample.begin(), sample.end(), 0.0) / sample.size();
means.push_back(sample_mean);
}
std::sort(means.begin(), means.end());
double lower_bound = means[(int)((1 - confidence_level) / 2 * num_samples)];
double upper_bound = means[(int)((1 + confidence_level) / 2 * num_samples)];
double mean = std::accumulate(means.begin(), means.end(), 0.0) / means.size();
ConfidenceInterval ci = {
mean, (upper_bound - lower_bound) / 2, *std::min_element(data.begin(), data.end())};
return ci;
}
class Statistics
{
public:
void push(double timeMs)
{
times.push_back(timeMs);
sorted = false;
}
explicit Statistics(int iterations) : times(iterations) {}
double mean() { return sum() / times.size(); }
void push(double timeMs, int iteration) { times[iteration].push_back(timeMs); }
double sum()
ConfidenceInterval mean()
{
double sum = 0;
for (auto time : times)
std::vector<double> means;
means.reserve(times.size());
for (const auto &iter_times : times)
{
sum += time;
means.push_back(std::accumulate(iter_times.begin(), iter_times.end(), 0.0) /
iter_times.size());
}
return sum;
return confidenceInterval(means);
}
double min() { return *std::min_element(times.begin(), times.end()); }
double max() { return *std::max_element(times.begin(), times.end()); }
double percentile(double p)
ConfidenceInterval total()
{
const auto &times = getTimes();
return times[static_cast<size_t>(p * times.size())];
std::vector<double> sums;
sums.reserve(times.size());
for (const auto &iter_times : times)
{
sums.push_back(std::accumulate(iter_times.begin(), iter_times.end(), 0.0));
}
return confidenceInterval(sums);
}
ConfidenceInterval min()
{
std::vector<double> mins;
mins.reserve(times.size());
for (const auto &iter_times : times)
{
mins.push_back(*std::min_element(iter_times.begin(), iter_times.end()));
}
return confidenceInterval(mins);
}
ConfidenceInterval max()
{
std::vector<double> maxs;
maxs.reserve(times.size());
for (const auto &iter_times : times)
{
maxs.push_back(*std::max_element(iter_times.begin(), iter_times.end()));
}
return confidenceInterval(maxs);
}
ConfidenceInterval percentile(double p)
{
std::vector<double> percentiles;
percentiles.reserve(times.size());
for (const auto &iter_times : times)
{
auto sorted_times = iter_times;
std::sort(sorted_times.begin(), sorted_times.end());
percentiles.push_back(sorted_times[static_cast<size_t>(p * sorted_times.size())]);
}
return confidenceInterval(percentiles);
}
ConfidenceInterval ops_per_sec()
{
std::vector<double> ops;
ops.reserve(times.size());
for (const auto &iter_times : times)
{
double total_time = std::accumulate(iter_times.begin(), iter_times.end(), 0.0) / 1000.0;
ops.push_back(iter_times.size() / total_time);
}
return confidenceInterval(ops);
}
private:
std::vector<double> getTimes()
{
if (!sorted)
{
std::sort(times.begin(), times.end());
sorted = true;
}
return times;
}
std::vector<double> times;
bool sorted = false;
// vector of times for each iteration
std::vector<std::vector<double>> times;
};
std::ostream &operator<<(std::ostream &os, Statistics &statistics)
{
os << std::fixed << std::setprecision(2);
os << "total: " << statistics.sum() << "ms" << std::endl;
os << "avg: " << statistics.mean() << "ms" << std::endl;
os << "min: " << statistics.min() << "ms" << std::endl;
os << "max: " << statistics.max() << "ms" << std::endl;
os << "p99: " << statistics.percentile(0.99) << "ms" << std::endl;
ConfidenceInterval mean_ci = statistics.mean();
ConfidenceInterval total_ci = statistics.total();
ConfidenceInterval min_ci = statistics.min();
ConfidenceInterval max_ci = statistics.max();
ConfidenceInterval p99_ci = statistics.percentile(0.99);
ConfidenceInterval ops_ci = statistics.ops_per_sec();
os << "ops: " << ops_ci.mean << " ± " << ops_ci.confidence << " ops/s. "
<< "best: " << ops_ci.min << "ops/s." << std::endl;
os << "total: " << total_ci.mean << " ± " << total_ci.confidence << "ms. "
<< "best: " << total_ci.min << "ms." << std::endl;
os << "avg: " << mean_ci.mean << " ± " << mean_ci.confidence << "ms" << std::endl;
os << "min: " << min_ci.mean << " ± " << min_ci.confidence << "ms" << std::endl;
os << "max: " << max_ci.mean << " ± " << max_ci.confidence << "ms" << std::endl;
os << "p99: " << p99_ci.mean << " ± " << p99_ci.confidence << "ms" << std::endl;
return os;
}
void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
template <typename Benchmark, typename BenchmarkBody>
void runBenchmarks(const std::vector<Benchmark> &benchmarks,
int iterations,
int opsPerIteration,
const OSRM &osrm,
const GPSTraces &gpsTraces,
const BenchmarkBody &benchmarkBody)
{
for (const auto &benchmark : benchmarks)
{
Statistics statistics{iterations};
for (int iteration = 0; iteration < iterations; ++iteration)
{
gpsTraces.resetSeed();
for (int i = 0; i < opsPerIteration; ++i)
{
benchmarkBody(iteration, benchmark, osrm, gpsTraces, statistics);
}
}
std::cout << benchmark.name << std::endl;
std::cout << statistics << std::endl;
}
}
void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations)
{
struct Benchmark
{
std::string name;
@@ -179,114 +308,83 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
std::optional<size_t> alternatives = std::nullopt;
std::optional<double> radius = std::nullopt;
};
auto run_benchmark = [&](const Benchmark &benchmark)
{
Statistics statistics;
auto NUM = 10000;
for (int i = 0; i < NUM; ++i)
{
RouteParameters params;
params.overview = benchmark.overview;
params.steps = benchmark.steps;
for (size_t i = 0; i < benchmark.coordinates; ++i)
{
params.coordinates.push_back(gpsTraces.getRandomCoordinate());
}
if (benchmark.alternatives)
{
params.alternatives = *benchmark.alternatives;
}
if (benchmark.radius)
{
params.radiuses = std::vector<boost::optional<double>>(
params.coordinates.size(), boost::make_optional(*benchmark.radius));
}
engine::api::ResultT result = json::Object();
TIMER_START(routes);
const auto rc = osrm.Route(params, result);
TIMER_STOP(routes);
statistics.push(TIMER_MSEC(routes));
auto &json_result = std::get<json::Object>(result);
if (rc != Status::Ok || json_result.values.find("routes") == json_result.values.end())
{
auto code = std::get<json::String>(json_result.values["code"]).value;
if (code != "NoSegment" && code != "NoRoute")
{
throw std::runtime_error{"Couldn't route: " + code};
}
}
}
std::cout << benchmark.name << std::endl;
std::cout << statistics << std::endl;
};
std::vector<Benchmark> benchmarks = {
{"10000 routes, 3 coordinates, no alternatives, overview=full, steps=true",
{"1000 routes, 3 coordinates, no alternatives, overview=full, steps=true",
3,
RouteParameters::OverviewType::Full,
true,
std::nullopt},
{"10000 routes, 2 coordinates, no alternatives, overview=full, steps=true",
2,
RouteParameters::OverviewType::Full,
true,
std::nullopt},
{"10000 routes, 2 coordinates, 3 alternatives, overview=full, steps=true",
{"1000 routes, 2 coordinates, 3 alternatives, overview=full, steps=true",
2,
RouteParameters::OverviewType::Full,
true,
3},
{"10000 routes, 3 coordinates, no alternatives, overview=false, steps=false",
{"1000 routes, 3 coordinates, no alternatives, overview=false, steps=false",
3,
RouteParameters::OverviewType::False,
false,
std::nullopt},
{"10000 routes, 2 coordinates, no alternatives, overview=false, steps=false",
{"1000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false",
2,
RouteParameters::OverviewType::False,
false,
std::nullopt},
{"10000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false",
2,
RouteParameters::OverviewType::False,
false,
3},
{"10000 routes, 3 coordinates, no alternatives, overview=false, steps=false, radius=750",
3,
RouteParameters::OverviewType::False,
false,
std::nullopt,
750},
{"10000 routes, 2 coordinates, no alternatives, overview=false, steps=false, radius=750",
2,
RouteParameters::OverviewType::False,
false,
std::nullopt,
750},
{"10000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false, radius=750",
2,
RouteParameters::OverviewType::False,
false,
3,
750}
3}};
};
runBenchmarks(benchmarks,
iterations,
1000,
osrm,
gpsTraces,
[](int iteration,
const Benchmark &benchmark,
const OSRM &osrm,
const GPSTraces &gpsTraces,
Statistics &statistics)
{
RouteParameters params;
params.overview = benchmark.overview;
params.steps = benchmark.steps;
for (const auto &benchmark : benchmarks)
{
run_benchmark(benchmark);
}
for (size_t i = 0; i < benchmark.coordinates; ++i)
{
params.coordinates.push_back(gpsTraces.getRandomCoordinate());
}
if (benchmark.alternatives)
{
params.alternatives = *benchmark.alternatives;
}
if (benchmark.radius)
{
params.radiuses = std::vector<boost::optional<double>>(
params.coordinates.size(), boost::make_optional(*benchmark.radius));
}
engine::api::ResultT result = json::Object();
TIMER_START(routes);
const auto rc = osrm.Route(params, result);
TIMER_STOP(routes);
auto &json_result = std::get<json::Object>(result);
if (rc != Status::Ok ||
json_result.values.find("routes") == json_result.values.end())
{
auto code = std::get<json::String>(json_result.values["code"]).value;
if (code != "NoSegment" && code != "NoRoute")
{
throw std::runtime_error{"Couldn't route: " + code};
}
}
else
{
statistics.push(TIMER_MSEC(routes), iteration);
}
});
}
void runMatchBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
void runMatchBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations)
{
struct Benchmark
{
@@ -294,59 +392,56 @@ void runMatchBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
std::optional<size_t> radius = std::nullopt;
};
auto run_benchmark = [&](const Benchmark &benchmark)
{
Statistics statistics;
std::vector<Benchmark> benchmarks = {{"500 matches, default radius"},
{"500 matches, radius=10", 10},
{"500 matches, radius=20", 20}};
auto NUM = 1000;
for (int i = 0; i < NUM; ++i)
{
engine::api::ResultT result = json::Object();
runBenchmarks(benchmarks,
iterations,
500,
osrm,
gpsTraces,
[](int iteration,
const Benchmark &benchmark,
const OSRM &osrm,
const GPSTraces &gpsTraces,
Statistics &statistics)
{
engine::api::ResultT result = json::Object();
engine::api::MatchParameters params;
params.coordinates = gpsTraces.getRandomTrace();
params.radiuses = {};
if (benchmark.radius)
{
for (size_t index = 0; index < params.coordinates.size(); ++index)
{
params.radiuses.emplace_back(*benchmark.radius);
}
}
engine::api::MatchParameters params;
params.coordinates = gpsTraces.getRandomTrace();
params.radiuses = {};
if (benchmark.radius)
{
for (size_t index = 0; index < params.coordinates.size(); ++index)
{
params.radiuses.emplace_back(*benchmark.radius);
}
}
TIMER_START(match);
const auto rc = osrm.Match(params, result);
TIMER_STOP(match);
TIMER_START(match);
const auto rc = osrm.Match(params, result);
TIMER_STOP(match);
statistics.push(TIMER_MSEC(match));
auto &json_result = std::get<json::Object>(result);
if (rc != Status::Ok ||
json_result.values.find("matchings") == json_result.values.end())
{
auto code = std::get<json::String>(json_result.values["code"]).value;
if (code != "NoSegment" && code != "NoMatch")
{
throw std::runtime_error{"Couldn't route: " + code};
}
}
}
std::cout << benchmark.name << std::endl;
std::cout << statistics << std::endl;
};
std::vector<Benchmark> benchmarks = {{"1000 matches, default radius"},
{"1000 matches, radius=10", 10},
{"1000 matches, radius=20", 20}};
for (const auto &benchmark : benchmarks)
{
run_benchmark(benchmark);
}
auto &json_result = std::get<json::Object>(result);
if (rc != Status::Ok ||
json_result.values.find("matchings") == json_result.values.end())
{
auto code = std::get<json::String>(json_result.values["code"]).value;
if (code != "NoSegment" && code != "NoMatch")
{
throw std::runtime_error{"Couldn't route: " + code};
}
}
else
{
statistics.push(TIMER_MSEC(match), iteration);
}
});
}
void runNearestBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
void runNearestBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations)
{
struct Benchmark
{
@@ -354,54 +449,52 @@ void runNearestBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
std::optional<size_t> number_of_results = std::nullopt;
};
auto run_benchmark = [&](const Benchmark &benchmark)
{
Statistics statistics;
auto NUM = 10000;
for (int i = 0; i < NUM; ++i)
{
engine::api::ResultT result = json::Object();
NearestParameters params;
params.coordinates.push_back(gpsTraces.getRandomCoordinate());
if (benchmark.number_of_results)
{
params.number_of_results = *benchmark.number_of_results;
}
TIMER_START(nearest);
const auto rc = osrm.Nearest(params, result);
TIMER_STOP(nearest);
statistics.push(TIMER_MSEC(nearest));
auto &json_result = std::get<json::Object>(result);
if (rc != Status::Ok ||
json_result.values.find("waypoints") == json_result.values.end())
{
auto code = std::get<json::String>(json_result.values["code"]).value;
if (code != "NoSegment")
{
throw std::runtime_error{"Couldn't find nearest point"};
}
}
}
std::cout << benchmark.name << std::endl;
std::cout << statistics << std::endl;
};
std::vector<Benchmark> benchmarks = {{"10000 nearest, number_of_results=1", 1},
{"10000 nearest, number_of_results=5", 5},
{"10000 nearest, number_of_results=10", 10}};
for (const auto &benchmark : benchmarks)
{
run_benchmark(benchmark);
}
runBenchmarks(benchmarks,
iterations,
10000,
osrm,
gpsTraces,
[](int iteration,
const Benchmark &benchmark,
const OSRM &osrm,
const GPSTraces &gpsTraces,
Statistics &statistics)
{
engine::api::ResultT result = json::Object();
NearestParameters params;
params.coordinates.push_back(gpsTraces.getRandomCoordinate());
if (benchmark.number_of_results)
{
params.number_of_results = *benchmark.number_of_results;
}
TIMER_START(nearest);
const auto rc = osrm.Nearest(params, result);
TIMER_STOP(nearest);
auto &json_result = std::get<json::Object>(result);
if (rc != Status::Ok ||
json_result.values.find("waypoints") == json_result.values.end())
{
auto code = std::get<json::String>(json_result.values["code"]).value;
if (code != "NoSegment")
{
throw std::runtime_error{"Couldn't find nearest point"};
}
}
else
{
statistics.push(TIMER_MSEC(nearest), iteration);
}
});
}
void runTripBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
void runTripBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations)
{
struct Benchmark
{
@@ -409,54 +502,52 @@ void runTripBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
size_t coordinates;
};
auto run_benchmark = [&](const Benchmark &benchmark)
{
Statistics statistics;
auto NUM = 1000;
for (int i = 0; i < NUM; ++i)
{
engine::api::ResultT result = json::Object();
TripParameters params;
params.roundtrip = true;
for (size_t i = 0; i < benchmark.coordinates; ++i)
{
params.coordinates.push_back(gpsTraces.getRandomCoordinate());
}
TIMER_START(trip);
const auto rc = osrm.Trip(params, result);
TIMER_STOP(trip);
statistics.push(TIMER_MSEC(trip));
auto &json_result = std::get<json::Object>(result);
if (rc != Status::Ok || json_result.values.find("trips") == json_result.values.end())
{
auto code = std::get<json::String>(json_result.values["code"]).value;
if (code != "NoSegment")
{
throw std::runtime_error{"Couldn't find trip"};
}
}
}
std::cout << benchmark.name << std::endl;
std::cout << statistics << std::endl;
};
std::vector<Benchmark> benchmarks = {
{"1000 trips, 3 coordinates", 3},
{"1000 trips, 4 coordinates", 4},
{"1000 trips, 5 coordinates", 5},
{"250 trips, 3 coordinates", 3},
{"250 trips, 5 coordinates", 5},
};
for (const auto &benchmark : benchmarks)
{
run_benchmark(benchmark);
}
runBenchmarks(benchmarks,
iterations,
250,
osrm,
gpsTraces,
[](int iteration,
const Benchmark &benchmark,
const OSRM &osrm,
const GPSTraces &gpsTraces,
Statistics &statistics)
{
engine::api::ResultT result = json::Object();
TripParameters params;
params.roundtrip = true;
for (size_t i = 0; i < benchmark.coordinates; ++i)
{
params.coordinates.push_back(gpsTraces.getRandomCoordinate());
}
TIMER_START(trip);
const auto rc = osrm.Trip(params, result);
TIMER_STOP(trip);
auto &json_result = std::get<json::Object>(result);
if (rc != Status::Ok ||
json_result.values.find("trips") == json_result.values.end())
{
auto code = std::get<json::String>(json_result.values["code"]).value;
if (code != "NoSegment")
{
throw std::runtime_error{"Couldn't find trip"};
}
}
else
{
statistics.push(TIMER_MSEC(trip), iteration);
}
});
}
void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations)
{
struct Benchmark
{
@@ -464,51 +555,46 @@ void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
size_t coordinates;
};
auto run_benchmark = [&](const Benchmark &benchmark)
{
Statistics statistics;
auto NUM = 250;
for (int i = 0; i < NUM; ++i)
{
engine::api::ResultT result = json::Object();
TableParameters params;
for (size_t i = 0; i < benchmark.coordinates; ++i)
{
params.coordinates.push_back(gpsTraces.getRandomCoordinate());
}
TIMER_START(table);
const auto rc = osrm.Table(params, result);
TIMER_STOP(table);
statistics.push(TIMER_MSEC(table));
auto &json_result = std::get<json::Object>(result);
if (rc != Status::Ok ||
json_result.values.find("durations") == json_result.values.end())
{
auto code = std::get<json::String>(json_result.values["code"]).value;
if (code != "NoSegment")
{
throw std::runtime_error{"Couldn't compute table"};
}
}
}
std::cout << benchmark.name << std::endl;
std::cout << statistics << std::endl;
};
std::vector<Benchmark> benchmarks = {{"250 tables, 3 coordinates", 3},
{"250 tables, 25 coordinates", 25},
{"250 tables, 50 coordinates", 50},
{"250 tables, 100 coordinates", 100}};
{"250 tables, 50 coordinates", 50}};
for (const auto &benchmark : benchmarks)
{
run_benchmark(benchmark);
}
runBenchmarks(benchmarks,
iterations,
250,
osrm,
gpsTraces,
[](int iteration,
const Benchmark &benchmark,
const OSRM &osrm,
const GPSTraces &gpsTraces,
Statistics &statistics)
{
engine::api::ResultT result = json::Object();
TableParameters params;
for (size_t i = 0; i < benchmark.coordinates; ++i)
{
params.coordinates.push_back(gpsTraces.getRandomCoordinate());
}
TIMER_START(table);
const auto rc = osrm.Table(params, result);
TIMER_STOP(table);
statistics.push(TIMER_MSEC(table), iteration);
auto &json_result = std::get<json::Object>(result);
if (rc != Status::Ok ||
json_result.values.find("durations") == json_result.values.end())
{
auto code = std::get<json::String>(json_result.values["code"]).value;
if (code != "NoSegment")
{
throw std::runtime_error{"Couldn't compute table"};
}
}
});
}
} // namespace
@@ -516,11 +602,11 @@ void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
int main(int argc, const char *argv[])
try
{
if (argc < 5)
if (argc < 6)
{
std::cerr
<< "Usage: " << argv[0]
<< " data.osrm <mld|ch> <path to GPS traces.csv> <route|match|trip|table|nearest>\n";
std::cerr << "Usage: " << argv[0]
<< " data.osrm <mld|ch> <path to GPS traces.csv> "
"<route|match|trip|table|nearest> <number_of_iterations>\n";
return EXIT_FAILURE;
}
@@ -537,27 +623,29 @@ try
GPSTraces gpsTraces{42};
gpsTraces.readCSV(argv[3]);
int iterations = std::stoi(argv[5]);
const auto benchmarkToRun = std::string{argv[4]};
if (benchmarkToRun == "route")
{
runRouteBenchmark(osrm, gpsTraces);
runRouteBenchmark(osrm, gpsTraces, iterations);
}
else if (benchmarkToRun == "match")
{
runMatchBenchmark(osrm, gpsTraces);
runMatchBenchmark(osrm, gpsTraces, iterations);
}
else if (benchmarkToRun == "nearest")
{
runNearestBenchmark(osrm, gpsTraces);
runNearestBenchmark(osrm, gpsTraces, iterations);
}
else if (benchmarkToRun == "trip")
{
runTripBenchmark(osrm, gpsTraces);
runTripBenchmark(osrm, gpsTraces, iterations);
}
else if (benchmarkToRun == "table")
{
runTableBenchmark(osrm, gpsTraces);
runTableBenchmark(osrm, gpsTraces, iterations);
}
else
{
+1 -1
View File
@@ -254,7 +254,7 @@ try
<< std::endl;
};
for (auto radius : std::vector<std::optional<double>>{std::nullopt, 5.0, 10.0, 15.0, 30.0})
for (auto radius : std::vector<std::optional<double>>{std::nullopt, 10.0})
{
run_benchmark(radius);
}
-34
View File
@@ -96,12 +96,6 @@ try
RouteParameters::OverviewType::Full,
true,
std::nullopt},
{"1000 routes, 2 coordinates, no alternatives, overview=full, steps=true",
{{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}},
{FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}},
RouteParameters::OverviewType::Full,
true,
std::nullopt},
{"1000 routes, 2 coordinates, 3 alternatives, overview=full, steps=true",
{{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}},
{FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}},
@@ -115,40 +109,12 @@ try
RouteParameters::OverviewType::False,
false,
std::nullopt},
{"1000 routes, 2 coordinates, no alternatives, overview=false, steps=false",
{{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}},
{FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}},
RouteParameters::OverviewType::False,
false,
std::nullopt},
{"1000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false",
{{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}},
{FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}},
RouteParameters::OverviewType::False,
false,
3},
{"1000 routes, 3 coordinates, no alternatives, overview=false, steps=false, radius=750",
{{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}},
{FloatLongitude{7.421844922513342}, FloatLatitude{43.73690777888953}},
{FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}},
RouteParameters::OverviewType::False,
false,
std::nullopt,
750},
{"1000 routes, 2 coordinates, no alternatives, overview=false, steps=false, radius=750",
{{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}},
{FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}},
RouteParameters::OverviewType::False,
false,
std::nullopt,
750},
{"1000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false, radius=750",
{{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}},
{FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}},
RouteParameters::OverviewType::False,
false,
3,
750}
};
+1 -2
View File
@@ -11,11 +11,10 @@
#include "util/serialization.hpp"
#include "util/timing_util.hpp"
#include <fstream>
#include <iostream>
#include <random>
#include <boost/filesystem/fstream.hpp>
namespace osrm::benchmarks
{
+67 -50
View File
@@ -44,21 +44,22 @@ util::json::Array lanesFromIntersection(const guidance::IntermediateIntersection
{
BOOST_ASSERT(intersection.lanes.lanes_in_turn >= 1);
util::json::Array result;
result.values.reserve(intersection.lane_description.size());
LaneID lane_id = intersection.lane_description.size();
for (const auto &lane_desc : intersection.lane_description)
{
--lane_id;
util::json::Object lane;
lane.values["indications"] = toJSON(lane_desc);
lane.values.emplace("indications", toJSON(lane_desc));
if (lane_id >= intersection.lanes.first_lane_from_the_right &&
lane_id <
intersection.lanes.first_lane_from_the_right + intersection.lanes.lanes_in_turn)
lane.values["valid"] = util::json::True();
lane.values.emplace("valid", util::json::True());
else
lane.values["valid"] = util::json::False();
lane.values.emplace("valid", util::json::False());
result.values.push_back(lane);
result.values.emplace_back(std::move(lane));
}
return result;
@@ -77,6 +78,7 @@ std::string waypointTypeToString(const guidance::WaypointType waypoint_type)
util::json::Value coordinateToLonLat(const util::Coordinate &coordinate)
{
util::json::Array array;
array.values.reserve(2);
array.values.push_back(static_cast<double>(util::toFloating(coordinate.lon)));
array.values.push_back(static_cast<double>(util::toFloating(coordinate.lat)));
return util::json::Value{std::move(array)};
@@ -98,17 +100,20 @@ util::json::Object makeStepManeuver(const guidance::StepManeuver &maneuver)
// These invalid responses should never happen: log if they do happen
BOOST_ASSERT_MSG(maneuver_type != "invalid", "unexpected invalid maneuver type");
step_maneuver.values["type"] = std::move(maneuver_type);
step_maneuver.values.emplace("type", std::move(maneuver_type));
if (detail::isValidModifier(maneuver))
step_maneuver.values["modifier"] =
osrm::guidance::instructionModifierToString(maneuver.instruction.direction_modifier);
step_maneuver.values.emplace(
"modifier",
osrm::guidance::instructionModifierToString(maneuver.instruction.direction_modifier));
step_maneuver.values["location"] = detail::coordinateToLonLat(maneuver.location);
step_maneuver.values["bearing_before"] = detail::roundAndClampBearing(maneuver.bearing_before);
step_maneuver.values["bearing_after"] = detail::roundAndClampBearing(maneuver.bearing_after);
step_maneuver.values.emplace("location", detail::coordinateToLonLat(maneuver.location));
step_maneuver.values.emplace("bearing_before",
detail::roundAndClampBearing(maneuver.bearing_before));
step_maneuver.values.emplace("bearing_after",
detail::roundAndClampBearing(maneuver.bearing_after));
if (maneuver.exit != 0)
step_maneuver.values["exit"] = maneuver.exit;
step_maneuver.values.emplace("exit", maneuver.exit);
return step_maneuver;
}
@@ -137,16 +142,16 @@ util::json::Object makeIntersection(const guidance::IntermediateIntersection &in
return util::json::False();
});
result.values["location"] = detail::coordinateToLonLat(intersection.location);
result.values["bearings"] = bearings;
result.values["entry"] = entry;
result.values.emplace("location", detail::coordinateToLonLat(intersection.location));
result.values.emplace("bearings", bearings);
result.values.emplace("entry", entry);
if (intersection.in != guidance::IntermediateIntersection::NO_INDEX)
result.values["in"] = intersection.in;
result.values.emplace("in", intersection.in);
if (intersection.out != guidance::IntermediateIntersection::NO_INDEX)
result.values["out"] = intersection.out;
result.values.emplace("out", intersection.out);
if (detail::hasValidLanes(intersection))
result.values["lanes"] = detail::lanesFromIntersection(intersection);
result.values.emplace("lanes", detail::lanesFromIntersection(intersection));
if (!intersection.classes.empty())
{
@@ -157,7 +162,7 @@ util::json::Object makeIntersection(const guidance::IntermediateIntersection &in
std::back_inserter(classes.values),
[](const std::string &class_name)
{ return util::json::String{class_name}; });
result.values["classes"] = std::move(classes);
result.values.emplace("classes", std::move(classes));
}
return result;
@@ -166,39 +171,44 @@ util::json::Object makeIntersection(const guidance::IntermediateIntersection &in
util::json::Object makeRouteStep(guidance::RouteStep step, util::json::Value geometry)
{
util::json::Object route_step;
route_step.values["distance"] = std::round(step.distance * 10) / 10.;
route_step.values["duration"] = step.duration;
route_step.values["weight"] = step.weight;
route_step.values["name"] = std::move(step.name);
route_step.values.reserve(15);
route_step.values.emplace("distance", std::round(step.distance * 10) / 10.);
route_step.values.emplace("duration", step.duration);
route_step.values.emplace("weight", step.weight);
route_step.values.emplace("name", step.name);
if (!step.ref.empty())
route_step.values["ref"] = std::move(step.ref);
route_step.values.emplace("ref", step.ref);
if (!step.pronunciation.empty())
route_step.values["pronunciation"] = std::move(step.pronunciation);
route_step.values.emplace("pronunciation", step.pronunciation);
if (!step.destinations.empty())
route_step.values["destinations"] = std::move(step.destinations);
route_step.values.emplace("destinations", step.destinations);
if (!step.exits.empty())
route_step.values["exits"] = std::move(step.exits);
route_step.values.emplace("exits", step.exits);
if (!step.rotary_name.empty())
{
route_step.values["rotary_name"] = std::move(step.rotary_name);
route_step.values.emplace("rotary_name", step.rotary_name);
if (!step.rotary_pronunciation.empty())
{
route_step.values["rotary_pronunciation"] = std::move(step.rotary_pronunciation);
route_step.values.emplace("rotary_pronunciation", step.rotary_pronunciation);
}
}
route_step.values["mode"] = extractor::travelModeToString(step.mode);
route_step.values["maneuver"] = makeStepManeuver(step.maneuver);
route_step.values["geometry"] = std::move(geometry);
route_step.values["driving_side"] = step.is_left_hand_driving ? "left" : "right";
route_step.values.emplace("mode", extractor::travelModeToString(step.mode));
route_step.values.emplace("maneuver", makeStepManeuver(step.maneuver));
route_step.values.emplace("geometry", std::move(geometry));
route_step.values.emplace("driving_side", step.is_left_hand_driving ? "left" : "right");
util::json::Array intersections;
intersections.values.reserve(step.intersections.size());
std::transform(step.intersections.begin(),
step.intersections.end(),
std::back_inserter(intersections.values),
makeIntersection);
route_step.values["intersections"] = std::move(intersections);
route_step.values.emplace("intersections", std::move(intersections));
return route_step;
}
@@ -209,14 +219,16 @@ util::json::Object makeRoute(const guidance::Route &route,
const char *weight_name)
{
util::json::Object json_route;
json_route.values["distance"] = route.distance;
json_route.values["duration"] = route.duration;
json_route.values["weight"] = route.weight;
json_route.values["weight_name"] = weight_name;
json_route.values["legs"] = std::move(legs);
json_route.values.reserve(6);
json_route.values.emplace("distance", route.distance);
json_route.values.emplace("duration", route.duration);
json_route.values.emplace("weight", route.weight);
json_route.values.emplace("weight_name", weight_name);
json_route.values.emplace("legs", std::move(legs));
if (geometry)
{
json_route.values["geometry"] = *std::move(geometry);
json_route.values.emplace("geometry", *std::move(geometry));
}
return json_route;
}
@@ -225,9 +237,11 @@ util::json::Object
makeWaypoint(const util::Coordinate &location, const double &distance, std::string name)
{
util::json::Object waypoint;
waypoint.values["location"] = detail::coordinateToLonLat(location);
waypoint.values["name"] = std::move(name);
waypoint.values["distance"] = distance;
waypoint.values.reserve(3);
waypoint.values.emplace("location", detail::coordinateToLonLat(location));
waypoint.values.emplace("name", std::move(name));
waypoint.values.emplace("distance", distance);
return waypoint;
}
@@ -237,26 +251,29 @@ util::json::Object makeWaypoint(const util::Coordinate &location,
const Hint &location_hints)
{
auto waypoint = makeWaypoint(location, distance, std::move(name));
waypoint.values["hint"] = location_hints.ToBase64();
waypoint.values.reserve(1);
waypoint.values.emplace("hint", location_hints.ToBase64());
return waypoint;
}
util::json::Object makeRouteLeg(guidance::RouteLeg leg, util::json::Array steps)
{
util::json::Object route_leg;
route_leg.values["distance"] = leg.distance;
route_leg.values["duration"] = leg.duration;
route_leg.values["weight"] = leg.weight;
route_leg.values["summary"] = std::move(leg.summary);
route_leg.values["steps"] = std::move(steps);
route_leg.values.reserve(5);
route_leg.values.emplace("distance", leg.distance);
route_leg.values.emplace("duration", leg.duration);
route_leg.values.emplace("weight", leg.weight);
route_leg.values.emplace("summary", std::move(leg.summary));
route_leg.values.emplace("steps", std::move(steps));
return route_leg;
}
util::json::Object
makeRouteLeg(guidance::RouteLeg leg, util::json::Array steps, util::json::Object annotation)
{
util::json::Object route_leg = makeRouteLeg(std::move(leg), std::move(steps));
route_leg.values["annotation"] = std::move(annotation);
route_leg.values.reserve(1);
route_leg.values.emplace("annotation", std::move(annotation));
return route_leg;
}
@@ -44,7 +44,7 @@ MMapMemoryAllocator::MMapMemoryAllocator(const storage::StorageConfig &config)
for (const auto &file : files)
{
if (boost::filesystem::exists(file.second))
if (std::filesystem::exists(file.second))
{
std::unique_ptr<storage::BaseDataLayout> layout =
std::make_unique<storage::TarDataLayout>();
+12 -27
View File
@@ -10,9 +10,19 @@
namespace osrm::engine::detail // anonymous to keep TU local
{
std::string encode(int number_to_encode)
void encode(int number_to_encode, std::string &output)
{
std::string output;
if (number_to_encode < 0)
{
const unsigned binary = std::llabs(number_to_encode);
const unsigned twos = (~binary) + 1u;
const unsigned shl = twos << 1u;
number_to_encode = static_cast<int>(~shl);
}
else
{
number_to_encode <<= 1u;
}
while (number_to_encode >= 0x20)
{
const int next_value = (0x20 | (number_to_encode & 0x1f)) + 63;
@@ -22,31 +32,6 @@ std::string encode(int number_to_encode)
number_to_encode += 63;
output += static_cast<char>(number_to_encode);
return output;
}
std::string encode(std::vector<int> &numbers)
{
std::string output;
for (auto &number : numbers)
{
if (number < 0)
{
const unsigned binary = std::llabs(number);
const unsigned twos = (~binary) + 1u;
const unsigned shl = twos << 1u;
number = static_cast<int>(~shl);
}
else
{
number <<= 1u;
}
}
for (const int number : numbers)
{
output += encode(number);
}
return output;
}
// https://developers.google.com/maps/documentation/utilities/polylinealgorithm
+3 -2
View File
@@ -2,10 +2,11 @@
#include "util/log.hpp"
#include <boost/assert.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/numeric/conversion/cast.hpp>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <limits>
#include <string>
+4 -4
View File
@@ -7,17 +7,17 @@
#include <rapidjson/error/en.h>
#include <rapidjson/istreamwrapper.h>
#include <boost/filesystem.hpp>
#include <boost/geometry/algorithms/equals.hpp>
#include <boost/iterator/function_output_iterator.hpp>
#include <filesystem>
#include <fstream>
#include <string>
namespace osrm::extractor
{
LocationDependentData::LocationDependentData(const std::vector<boost::filesystem::path> &file_paths)
LocationDependentData::LocationDependentData(const std::vector<std::filesystem::path> &file_paths)
{
std::vector<rtree_t::value_type> bounding_boxes;
for (const auto &path : file_paths)
@@ -32,12 +32,12 @@ LocationDependentData::LocationDependentData(const std::vector<boost::filesystem
}
void LocationDependentData::loadLocationDependentData(
const boost::filesystem::path &file_path, std::vector<rtree_t::value_type> &bounding_boxes)
const std::filesystem::path &file_path, std::vector<rtree_t::value_type> &bounding_boxes)
{
if (file_path.empty())
return;
if (!boost::filesystem::exists(file_path) || !boost::filesystem::is_regular_file(file_path))
if (!std::filesystem::exists(file_path) || !std::filesystem::is_regular_file(file_path))
{
throw osrm::util::exception(std::string("File with location-dependent data ") +
file_path.string() + " does not exists");
+2 -2
View File
@@ -103,8 +103,8 @@ int RasterContainer::LoadRasterSource(const std::string &path_string,
util::Log() << "[source loader] Loading from " << path_string << " ... ";
TIMER_START(loading_source);
boost::filesystem::path filepath(path_string);
if (!boost::filesystem::exists(filepath))
std::filesystem::path filepath(path_string);
if (!std::filesystem::exists(filepath))
{
throw util::RuntimeError(
path_string, ErrorCode::FileOpenError, SOURCE_REF, "File not found");
+1 -1
View File
@@ -113,7 +113,7 @@ void handle_lua_error(const sol::protected_function_result &luares)
Sol2ScriptingEnvironment::Sol2ScriptingEnvironment(
const std::string &file_name,
const std::vector<boost::filesystem::path> &location_dependent_data_paths)
const std::vector<std::filesystem::path> &location_dependent_data_paths)
: file_name(file_name), location_dependent_data(location_dependent_data_paths)
{
util::Log() << "Using script " << file_name;
+7 -8
View File
@@ -19,16 +19,15 @@
#include "util/json_container.hpp"
#include "util/log.hpp"
#include "util/mmap_file.hpp"
#include <algorithm>
#include <iterator>
#include <vector>
#include "util/timing_util.hpp"
#include <boost/assert.hpp>
#include <boost/filesystem/operations.hpp>
#include <tbb/global_control.h>
#include "util/timing_util.hpp"
#include <algorithm>
#include <filesystem>
#include <iterator>
#include <vector>
namespace osrm::partitioner
{
@@ -171,11 +170,11 @@ int Partitioner::Run(const PartitionerConfig &config)
extractor::files::writeManeuverOverrides(filename, maneuver_overrides, node_sequences);
}
if (boost::filesystem::exists(config.GetPath(".osrm.hsgr")))
if (std::filesystem::exists(config.GetPath(".osrm.hsgr")))
{
util::Log(logWARNING) << "Found existing .osrm.hsgr file, removing. You need to re-run "
"osrm-contract after osrm-partition.";
boost::filesystem::remove(config.GetPath(".osrm.hsgr"));
std::filesystem::remove(config.GetPath(".osrm.hsgr"));
}
TIMER_STOP(renumber);
util::Log() << "Renumbered data in " << TIMER_SEC(renumber) << " seconds";
+2 -4
View File
@@ -2,14 +2,12 @@
#include "util/log.hpp"
#include <boost/filesystem.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <filesystem>
namespace osrm::storage
{
namespace fs = boost::filesystem;
namespace fs = std::filesystem;
bool IOConfig::IsValid() const
{
+24 -26
View File
@@ -23,13 +23,11 @@
#endif
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/interprocess/sync/file_lock.hpp>
#include <boost/interprocess/sync/scoped_lock.hpp>
#include <cstdint>
#include <filesystem>
#include <fstream>
#include <iterator>
#include <new>
@@ -164,7 +162,7 @@ bool swapData(Monitor &monitor,
}
} // namespace
void populateLayoutFromFile(const boost::filesystem::path &path, storage::BaseDataLayout &layout)
void populateLayoutFromFile(const std::filesystem::path &path, storage::BaseDataLayout &layout)
{
tar::FileReader reader(path, tar::FileReader::VerifyFingerprint);
@@ -190,11 +188,11 @@ int Storage::Run(int max_wait, const std::string &dataset_name, bool only_metric
util::LogPolicy::GetInstance().Unmute();
boost::filesystem::path lock_path =
boost::filesystem::temp_directory_path() / "osrm-datastore.lock";
if (!boost::filesystem::exists(lock_path))
std::filesystem::path lock_path =
std::filesystem::temp_directory_path() / "osrm-datastore.lock";
if (!std::filesystem::exists(lock_path))
{
boost::filesystem::ofstream ofs(lock_path);
std::ofstream ofs(lock_path);
}
boost::interprocess::file_lock file_lock(lock_path.string().c_str());
@@ -257,7 +255,7 @@ int Storage::Run(int max_wait, const std::string &dataset_name, bool only_metric
std::unique_ptr<storage::BaseDataLayout> static_layout =
std::make_unique<storage::ContiguousDataLayout>();
Storage::PopulateLayoutWithRTree(*static_layout);
std::vector<std::pair<bool, boost::filesystem::path>> files = Storage::GetStaticFiles();
std::vector<std::pair<bool, std::filesystem::path>> files = Storage::GetStaticFiles();
Storage::PopulateLayout(*static_layout, files);
auto static_handle = setupRegion(shared_register, *static_layout);
regions.push_back({static_handle.data_ptr, std::move(static_layout)});
@@ -266,7 +264,7 @@ int Storage::Run(int max_wait, const std::string &dataset_name, bool only_metric
std::unique_ptr<storage::BaseDataLayout> updatable_layout =
std::make_unique<storage::ContiguousDataLayout>();
std::vector<std::pair<bool, boost::filesystem::path>> files = Storage::GetUpdatableFiles();
std::vector<std::pair<bool, std::filesystem::path>> files = Storage::GetUpdatableFiles();
Storage::PopulateLayout(*updatable_layout, files);
auto updatable_handle = setupRegion(shared_register, *updatable_layout);
regions.push_back({updatable_handle.data_ptr, std::move(updatable_layout)});
@@ -285,12 +283,12 @@ int Storage::Run(int max_wait, const std::string &dataset_name, bool only_metric
return EXIT_SUCCESS;
}
std::vector<std::pair<bool, boost::filesystem::path>> Storage::GetStaticFiles()
std::vector<std::pair<bool, std::filesystem::path>> Storage::GetStaticFiles()
{
constexpr bool IS_REQUIRED = true;
constexpr bool IS_OPTIONAL = false;
std::vector<std::pair<bool, boost::filesystem::path>> files = {
std::vector<std::pair<bool, std::filesystem::path>> files = {
{IS_OPTIONAL, config.GetPath(".osrm.cells")},
{IS_OPTIONAL, config.GetPath(".osrm.partition")},
{IS_REQUIRED, config.GetPath(".osrm.ebg_nodes")},
@@ -310,7 +308,7 @@ std::vector<std::pair<bool, boost::filesystem::path>> Storage::GetStaticFiles()
for (const auto &file : files)
{
if (file.first == IS_REQUIRED && !boost::filesystem::exists(file.second))
if (file.first == IS_REQUIRED && !std::filesystem::exists(file.second))
{
throw util::exception("Could not find required file(s): " + std::get<1>(file).string());
}
@@ -319,12 +317,12 @@ std::vector<std::pair<bool, boost::filesystem::path>> Storage::GetStaticFiles()
return files;
}
std::vector<std::pair<bool, boost::filesystem::path>> Storage::GetUpdatableFiles()
std::vector<std::pair<bool, std::filesystem::path>> Storage::GetUpdatableFiles()
{
constexpr bool IS_REQUIRED = true;
constexpr bool IS_OPTIONAL = false;
std::vector<std::pair<bool, boost::filesystem::path>> files = {
std::vector<std::pair<bool, std::filesystem::path>> files = {
{IS_OPTIONAL, config.GetPath(".osrm.mldgr")},
{IS_OPTIONAL, config.GetPath(".osrm.cell_metrics")},
{IS_OPTIONAL, config.GetPath(".osrm.hsgr")},
@@ -335,7 +333,7 @@ std::vector<std::pair<bool, boost::filesystem::path>> Storage::GetUpdatableFiles
for (const auto &file : files)
{
if (file.first == IS_REQUIRED && !boost::filesystem::exists(file.second))
if (file.first == IS_REQUIRED && !std::filesystem::exists(file.second))
{
throw util::exception("Could not find required file(s): " + std::get<1>(file).string());
}
@@ -347,9 +345,9 @@ std::vector<std::pair<bool, boost::filesystem::path>> Storage::GetUpdatableFiles
std::string Storage::PopulateLayoutWithRTree(storage::BaseDataLayout &layout)
{
// Figure out the path to the rtree file (it's not a tar file)
auto absolute_file_index_path = boost::filesystem::absolute(config.GetPath(".osrm.fileIndex"));
auto absolute_file_index_path = std::filesystem::absolute(config.GetPath(".osrm.fileIndex"));
// Convert the boost::filesystem::path object into a plain string
// Convert the std::filesystem::path object into a plain string
// that can then be stored as a member of an allocator object
auto rtree_filename = absolute_file_index_path.string();
@@ -366,11 +364,11 @@ std::string Storage::PopulateLayoutWithRTree(storage::BaseDataLayout &layout)
* in that big block. It updates the fields in the layout parameter.
*/
void Storage::PopulateLayout(storage::BaseDataLayout &layout,
const std::vector<std::pair<bool, boost::filesystem::path>> &files)
const std::vector<std::pair<bool, std::filesystem::path>> &files)
{
for (const auto &file : files)
{
if (boost::filesystem::exists(file.second))
if (std::filesystem::exists(file.second))
{
populateLayoutFromFile(file.second, layout);
}
@@ -389,7 +387,7 @@ void Storage::PopulateStaticData(const SharedDataIndex &index)
file_index_path_ptr + index.GetBlockSize("/common/rtree/file_index_path"),
0);
const auto absolute_file_index_path =
boost::filesystem::absolute(config.GetPath(".osrm.fileIndex")).string();
std::filesystem::absolute(config.GetPath(".osrm.fileIndex")).string();
BOOST_ASSERT(static_cast<std::size_t>(index.GetBlockSize(
"/common/rtree/file_index_path")) >= absolute_file_index_path.size());
std::copy(
@@ -482,13 +480,13 @@ void Storage::PopulateStaticData(const SharedDataIndex &index)
metric_name = profile_properties_ptr->GetWeightName();
}
if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
if (std::filesystem::exists(config.GetPath(".osrm.partition")))
{
auto mlp = make_partition_view(index, "/mld/multilevelpartition");
partitioner::files::readPartition(config.GetPath(".osrm.partition"), mlp);
}
if (boost::filesystem::exists(config.GetPath(".osrm.cells")))
if (std::filesystem::exists(config.GetPath(".osrm.cells")))
{
auto storage = make_cell_storage_view(index, "/mld/cellstorage");
partitioner::files::readCells(config.GetPath(".osrm.cells"), storage);
@@ -541,7 +539,7 @@ void Storage::PopulateUpdatableData(const SharedDataIndex &index)
metric_name = properties.GetWeightName();
}
if (boost::filesystem::exists(config.GetPath(".osrm.hsgr")))
if (std::filesystem::exists(config.GetPath(".osrm.hsgr")))
{
const std::string metric_prefix = "/ch/metrics/" + metric_name;
auto contracted_metric = make_contracted_metric_view(index, metric_prefix);
@@ -567,7 +565,7 @@ void Storage::PopulateUpdatableData(const SharedDataIndex &index)
}
}
if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
if (std::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
{
auto exclude_metrics = make_cell_metric_view(index, "/mld/metrics/" + metric_name);
std::unordered_map<std::string, std::vector<customizer::CellMetricView>> metrics = {
@@ -576,7 +574,7 @@ void Storage::PopulateUpdatableData(const SharedDataIndex &index)
customizer::files::readCellMetrics(config.GetPath(".osrm.cell_metrics"), metrics);
}
if (boost::filesystem::exists(config.GetPath(".osrm.mldgr")))
if (std::filesystem::exists(config.GetPath(".osrm.mldgr")))
{
auto graph_view = make_multi_level_graph_view(index, "/mld/multilevelgraph");
std::uint32_t graph_connectivity_checksum = 0;
+2 -2
View File
@@ -10,7 +10,6 @@
#include "util/tarjan_scc.hpp"
#include "util/typedefs.hpp"
#include <boost/filesystem.hpp>
#include <boost/iterator/function_output_iterator.hpp>
#include <tbb/parallel_sort.h>
@@ -19,6 +18,7 @@
#include <cstdlib>
#include <algorithm>
#include <filesystem>
#include <fstream>
#include <memory>
#include <ostream>
@@ -118,7 +118,7 @@ int main(int argc, char *argv[])
const std::string inpath{argv[1]};
const std::string outpath{argv[2]};
if (boost::filesystem::exists(outpath))
if (std::filesystem::exists(outpath))
{
util::Log(logWARNING) << "Components file " << outpath << " already exists";
return EXIT_FAILURE;
+3 -3
View File
@@ -6,12 +6,12 @@
#include "util/timezones.hpp"
#include "util/version.hpp"
#include <boost/filesystem.hpp>
#include <boost/program_options.hpp>
#include <boost/program_options/errors.hpp>
#include <cstdlib>
#include <exception>
#include <filesystem>
#include <new>
#include <ostream>
#include <thread>
@@ -86,7 +86,7 @@ return_code parseArguments(int argc,
boost::program_options::options_description hidden_options("Hidden options");
hidden_options.add_options()(
"input,i",
boost::program_options::value<boost::filesystem::path>(&contractor_config.base_path),
boost::program_options::value<std::filesystem::path>(&contractor_config.base_path),
"Input file in .osm, .osm.bz2 or .osm.pbf format");
// positional option
@@ -99,7 +99,7 @@ return_code parseArguments(int argc,
const auto *executable = argv[0];
boost::program_options::options_description visible_options(
"Usage: " + boost::filesystem::path(executable).filename().string() +
"Usage: " + std::filesystem::path(executable).filename().string() +
" <input.osrm> [options]");
visible_options.add(generic_options).add(config_options);
+3 -3
View File
@@ -5,9 +5,9 @@
#include "util/meminfo.hpp"
#include "util/version.hpp"
#include <boost/filesystem.hpp>
#include <boost/program_options.hpp>
#include <filesystem>
#include <iostream>
#include <thread>
@@ -76,7 +76,7 @@ return_code parseArguments(int argc,
boost::program_options::options_description hidden_options("Hidden options");
hidden_options.add_options()(
"input,i",
boost::program_options::value<boost::filesystem::path>(&customization_config.base_path),
boost::program_options::value<std::filesystem::path>(&customization_config.base_path),
"Input base file path");
// positional option
@@ -89,7 +89,7 @@ return_code parseArguments(int argc,
const auto *executable = argv[0];
boost::program_options::options_description visible_options(
boost::filesystem::path(executable).filename().string() + " <input.osrm> [options]");
std::filesystem::path(executable).filename().string() + " <input.osrm> [options]");
visible_options.add(generic_options).add(config_options);
// parse command line options
+8 -10
View File
@@ -2,19 +2,17 @@
#include "osrm/extractor.hpp"
#include "osrm/extractor_config.hpp"
#include "util/log.hpp"
#include "util/meminfo.hpp"
#include "util/version.hpp"
#include <boost/filesystem.hpp>
#include <boost/program_options.hpp>
#include <cstdlib>
#include <exception>
#include <filesystem>
#include <iostream>
#include <new>
#include <thread>
#include "util/meminfo.hpp"
using namespace osrm;
enum class return_code : unsigned
@@ -40,7 +38,7 @@ return_code parseArguments(int argc,
boost::program_options::options_description config_options("Configuration");
config_options.add_options()(
"profile,p",
boost::program_options::value<boost::filesystem::path>(&extractor_config.profile_path)
boost::program_options::value<std::filesystem::path>(&extractor_config.profile_path)
->default_value("profiles/car.lua"),
"Path to LUA routing profile")(
"data_version,d",
@@ -67,7 +65,7 @@ return_code parseArguments(int argc,
->default_value(false),
"Save conditional restrictions found during extraction to disk for use "
"during contraction")("location-dependent-data",
boost::program_options::value<std::vector<boost::filesystem::path>>(
boost::program_options::value<std::vector<std::filesystem::path>>(
&extractor_config.location_dependent_data_paths)
->composing(),
"GeoJSON files with location-dependent data")(
@@ -88,7 +86,7 @@ return_code parseArguments(int argc,
boost::program_options::options_description hidden_options("Hidden options");
hidden_options.add_options()(
"input,i",
boost::program_options::value<boost::filesystem::path>(&extractor_config.input_path),
boost::program_options::value<std::filesystem::path>(&extractor_config.input_path),
"Input file in .osm, .osm.bz2 or .osm.pbf format")(
"generate-edge-lookup",
boost::program_options::bool_switch(&dummy)->implicit_value(true)->default_value(false),
@@ -104,7 +102,7 @@ return_code parseArguments(int argc,
const auto *executable = argv[0];
boost::program_options::options_description visible_options(
boost::filesystem::path(executable).filename().string() +
std::filesystem::path(executable).filename().string() +
" <input.osm/.osm.bz2/.osm.pbf> [options]");
visible_options.add(generic_options).add(config_options);
@@ -176,14 +174,14 @@ try
return EXIT_FAILURE;
}
if (!boost::filesystem::is_regular_file(extractor_config.input_path))
if (!std::filesystem::is_regular_file(extractor_config.input_path))
{
util::Log(logERROR) << "Input file " << extractor_config.input_path.string()
<< " not found!";
return EXIT_FAILURE;
}
if (!boost::filesystem::is_regular_file(extractor_config.profile_path))
if (!std::filesystem::is_regular_file(extractor_config.profile_path))
{
util::Log(logERROR) << "Profile " << extractor_config.profile_path.string()
<< " not found!";
+8 -9
View File
@@ -3,9 +3,6 @@
#include "util/log.hpp"
#include "util/timing_util.hpp"
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <cmath>
#include <cstdio>
#include <fcntl.h>
@@ -15,6 +12,8 @@
#include <algorithm>
#include <chrono>
#include <filesystem>
#include <fstream>
#include <iomanip>
#include <numeric>
#include <random>
@@ -45,7 +44,7 @@ void runStatistics(std::vector<double> &timings_vector, Statistics &stats)
}
} // namespace osrm::tools
boost::filesystem::path test_path;
std::filesystem::path test_path;
int main(int argc, char *argv[])
{
@@ -66,7 +65,7 @@ int main(int argc, char *argv[])
return -1;
}
test_path = boost::filesystem::path(argv[1]);
test_path = std::filesystem::path(argv[1]);
test_path /= "osrm.tst";
osrm::util::Log(logDEBUG) << "temporary file: " << test_path.string();
@@ -74,7 +73,7 @@ int main(int argc, char *argv[])
if (2 == argc)
{
// create file to test
if (boost::filesystem::exists(test_path))
if (std::filesystem::exists(test_path))
{
throw osrm::util::exception("Data file already exists: " + test_path.string() +
SOURCE_REF);
@@ -120,7 +119,7 @@ int main(int argc, char *argv[])
else
{
// Run Non-Cached I/O benchmarks
if (!boost::filesystem::exists(test_path))
if (!std::filesystem::exists(test_path))
{
throw osrm::util::exception("data file does not exist" + SOURCE_REF);
}
@@ -298,9 +297,9 @@ int main(int argc, char *argv[])
<< "max: " << stats.max << "ms, "
<< "dev: " << stats.dev << "ms";
if (boost::filesystem::exists(test_path))
if (std::filesystem::exists(test_path))
{
boost::filesystem::remove(test_path);
std::filesystem::remove(test_path);
osrm::util::Log(logDEBUG) << "removing temporary files";
}
}
+5 -5
View File
@@ -8,10 +8,10 @@
#include "util/version.hpp"
#include <boost/algorithm/string/join.hpp>
#include <boost/filesystem.hpp>
#include <boost/program_options.hpp>
#include <boost/range/adaptor/transformed.hpp>
#include <filesystem>
#include <iostream>
#include <iterator>
#include <regex>
@@ -119,7 +119,7 @@ return_code parseArguments(int argc,
boost::program_options::options_description hidden_options("Hidden options");
hidden_options.add_options()(
"input,i",
boost::program_options::value<boost::filesystem::path>(&config.base_path),
boost::program_options::value<std::filesystem::path>(&config.base_path),
"Input base file path");
// positional option
@@ -132,7 +132,7 @@ return_code parseArguments(int argc,
const auto *executable = argv[0];
boost::program_options::options_description visible_options(
boost::filesystem::path(executable).filename().string() + " <input.osrm> [options]");
std::filesystem::path(executable).filename().string() + " <input.osrm> [options]");
visible_options.add(generic_options).add(config_options);
// parse command line options
@@ -216,9 +216,9 @@ try
return EXIT_FAILURE;
}
auto check_file = [](const boost::filesystem::path &path)
auto check_file = [](const std::filesystem::path &path)
{
if (!boost::filesystem::is_regular_file(path))
if (!std::filesystem::is_regular_file(path))
{
util::Log(logERROR) << "Input file " << path << " not found!";
return false;
+7 -7
View File
@@ -12,7 +12,6 @@
#include <boost/algorithm/string/case_conv.hpp>
#include <boost/any.hpp>
#include <boost/filesystem.hpp>
#include <boost/optional/optional_io.hpp>
#include <boost/program_options.hpp>
@@ -22,6 +21,7 @@
#include <chrono>
#include <exception>
#include <filesystem>
#include <future>
#include <iostream>
#include <new>
@@ -102,7 +102,7 @@ void validate(boost::any &v, const std::vector<std::string> &values, double *, d
// generate boost::program_options object for the routing part
inline unsigned generateServerProgramOptions(const int argc,
const char *argv[],
boost::filesystem::path &base_path,
std::filesystem::path &base_path,
std::string &ip_address,
int &ip_port,
bool &trial,
@@ -110,8 +110,8 @@ inline unsigned generateServerProgramOptions(const int argc,
int &requested_thread_num,
short &keepalive_timeout)
{
using boost::filesystem::path;
using boost::program_options::value;
using std::filesystem::path;
const auto hardware_threads = std::max<int>(1, std::thread::hardware_concurrency());
@@ -148,7 +148,7 @@ inline unsigned generateServerProgramOptions(const int argc,
value<bool>(&config.use_shared_memory)->implicit_value(true)->default_value(false),
"Load data from shared memory") //
("memory_file",
value<boost::filesystem::path>(&config.memory_file),
value<std::filesystem::path>(&config.memory_file),
"DEPRECATED: Will behave the same as --mmap.")(
"mmap,m",
value<bool>(&config.use_mmap)->implicit_value(true)->default_value(false),
@@ -192,7 +192,7 @@ inline unsigned generateServerProgramOptions(const int argc,
// hidden options, will be allowed on command line, but will not be shown to the user
boost::program_options::options_description hidden_options("Hidden options");
hidden_options.add_options()(
"base,b", value<boost::filesystem::path>(&base_path), "base path to .osrm file");
"base,b", value<std::filesystem::path>(&base_path), "base path to .osrm file");
// positional option
boost::program_options::positional_options_description positional_options;
@@ -204,7 +204,7 @@ inline unsigned generateServerProgramOptions(const int argc,
const auto *executable = argv[0];
boost::program_options::options_description visible_options(
boost::filesystem::path(executable).filename().string() + " <base.osrm> [<options>]");
std::filesystem::path(executable).filename().string() + " <base.osrm> [<options>]");
visible_options.add(generic_options).add(config_options);
// parse command line options
@@ -267,7 +267,7 @@ try
int ip_port;
EngineConfig config;
boost::filesystem::path base_path;
std::filesystem::path base_path;
int requested_thread_num = 1;
short keepalive_timeout = 5;
+5 -5
View File
@@ -11,11 +11,11 @@
#include "util/version.hpp"
#include <boost/algorithm/string/case_conv.hpp>
#include <boost/filesystem.hpp>
#include <boost/program_options.hpp>
#include <csignal>
#include <cstdlib>
#include <filesystem>
using namespace osrm;
@@ -97,7 +97,7 @@ void springClean()
bool generateDataStoreOptions(const int argc,
const char *argv[],
std::string &verbosity,
boost::filesystem::path &base_path,
std::filesystem::path &base_path,
int &max_wait,
std::string &dataset_name,
bool &list_datasets,
@@ -155,7 +155,7 @@ bool generateDataStoreOptions(const int argc,
// hidden options, will be allowed on command line but will not be shown to the user
boost::program_options::options_description hidden_options("Hidden options");
hidden_options.add_options()("base,b",
boost::program_options::value<boost::filesystem::path>(&base_path),
boost::program_options::value<std::filesystem::path>(&base_path),
"base path to .osrm file");
// positional option
@@ -168,7 +168,7 @@ bool generateDataStoreOptions(const int argc,
const auto *executable = argv[0];
boost::program_options::options_description visible_options(
boost::filesystem::path(executable).filename().string() + " [<options>] <configuration>");
std::filesystem::path(executable).filename().string() + " [<options>] <configuration>");
visible_options.add(generic_options).add(config_options);
// print help options if no infile is specified
@@ -242,7 +242,7 @@ try
util::LogPolicy::GetInstance().Unmute();
std::string verbosity;
boost::filesystem::path base_path;
std::filesystem::path base_path;
int max_wait = -1;
std::string dataset_name;
bool list_datasets = false;
+2 -2
View File
@@ -25,7 +25,6 @@
#include "util/typedefs.hpp"
#include <boost/assert.hpp>
#include <boost/filesystem/fstream.hpp>
#include <tbb/blocked_range.h>
#include <tbb/concurrent_vector.h>
@@ -38,6 +37,7 @@
#include <atomic>
#include <bitset>
#include <cstdint>
#include <fstream>
#include <iterator>
#include <memory>
#include <tuple>
@@ -408,7 +408,7 @@ void saveDatasourcesNames(const UpdaterConfig &config)
// for rendering in the debug tiles.
for (auto const &name : config.segment_speed_lookup_paths)
{
sources.SetSourceName(source, boost::filesystem::path(name).stem().string());
sources.SetSourceName(source, std::filesystem::path(name).stem().string());
source++;
}
+2 -4
View File
@@ -3,13 +3,11 @@
#include "util/geojson_validation.hpp"
#include "util/log.hpp"
#include <boost/filesystem/fstream.hpp>
#include <boost/filesystem/path.hpp>
#include <rapidjson/document.h>
#include <rapidjson/error/en.h>
#include <rapidjson/istreamwrapper.h>
#include <filesystem>
#include <fstream>
#include <optional>
#include <regex>
@@ -40,7 +38,7 @@ Timezoner::Timezoner(const char geojson[], std::time_t utc_time_now)
LoadLocalTimesRTree(doc, utc_time_now);
}
Timezoner::Timezoner(const boost::filesystem::path &tz_shapes_filename, std::time_t utc_time_now)
Timezoner::Timezoner(const std::filesystem::path &tz_shapes_filename, std::time_t utc_time_now)
{
util::Log() << "Time zone validation based on UTC time : " << utc_time_now;
+20
View File
@@ -0,0 +1,20 @@
# see https://clang.llvm.org/docs/ClangFormatStyleOptions.html
---
BasedOnStyle: LLVM
Language: Cpp
Standard: c++17
ColumnLimit: 127
AccessModifierOffset: -4
AlignEscapedNewlines: Left
AllowShortFunctionsOnASingleLine: Empty
AllowShortLambdasOnASingleLine: false
AlwaysBreakTemplateDeclarations: true
BinPackArguments: false
BinPackParameters: false
BreakConstructorInitializers: BeforeComma
BreakStringLiterals: false
IndentPPDirectives: AfterHash
IndentWidth: 4
PointerAlignment: Left
UseTab: Never
+55
View File
@@ -0,0 +1,55 @@
---
Checks: '*
-abseil-string-find-str-contains
-altera*
-bugprone-easily-swappable-parameters
-cert-err58-cpp
-cppcoreguidelines-avoid-magic-numbers
-cppcoreguidelines-pro-bounds-constant-array-index
-cppcoreguidelines-pro-bounds-pointer-arithmetic
-fuchsia*
-llvm-header-guard
-llvmlibc*
-readability-function-cognitive-complexity
-readability-identifier-length
-readability-magic-numbers
'
HeaderFilterRegex: ''
CheckOptions:
cppcoreguidelines-avoid-do-while.IgnoreMacros: 'true'
readability-identifier-naming.MacroDefinitionCase: 'UPPER_CASE'
readability-identifier-naming.TemplateParameterCase: 'CamelCase'
readability-identifier-naming.TypeTemplateParameterCase: 'CamelCase'
readability-identifier-naming.ValueTemplateParameterCase: 'CamelCase'
readability-identifier-naming.ParameterPackCase: 'lower_case'
readability-identifier-naming.AbstractClassCase: 'lower_case'
readability-identifier-naming.ClassCase: 'lower_case'
readability-identifier-naming.ClassMemberCase: 'lower_case'
readability-identifier-naming.ConstantCase: 'lower_case'
readability-identifier-naming.ConstexprVariableCase: 'lower_case'
readability-identifier-naming.EnumCase: 'lower_case'
readability-identifier-naming.EnumConstantCase: 'lower_case'
readability-identifier-naming.FunctionCase: 'lower_case'
readability-identifier-naming.GlobalConstantCase: 'lower_case'
readability-identifier-naming.LocalVariableCase: 'lower_case'
readability-identifier-naming.MemberCase: 'lower_case'
readability-identifier-naming.NamespaceCase: 'lower_case'
readability-identifier-naming.ParameterCase: 'lower_case'
readability-identifier-naming.StructCase: 'lower_case'
readability-identifier-naming.TypeAliasCase: 'lower_case'
readability-identifier-naming.TypedefCase: 'lower_case'
readability-identifier-naming.StaticConstantCase: 'lower_case'
readability-identifier-naming.StaticVariableCase: 'lower_case'
readability-identifier-naming.UnionCase: 'lower_case'
readability-identifier-naming.VariableCase: 'lower_case'
readability-identifier-naming.GlobalConstantPrefix: 'global_'
readability-identifier-naming.GlobalVariablePrefix: 'global_'
readability-identifier-naming.MemberPrefix: 'm_'
readability-identifier-naming.PrivateMemberPrefix: 'm_'
readability-identifier-naming.PrivateMemberPrefix: 'm_'
readability-identifier-naming.ProtectedMemberPrefix: 'm_'
readability-identifier-naming.PublicMemberPrefix: ''
readability-identifier-naming.StaticConstantPrefix: 'static_'
readability-identifier-naming.StaticVariablePrefix: 'static_'
readability-identifier-naming.ClassMemberPrefix: 'static_'
...
+1
View File
@@ -0,0 +1 @@
data/fuzz
+13
View File
@@ -0,0 +1,13 @@
# These are supported funding model platforms
github: [martinus] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
patreon: # Replace with a single Patreon username
open_collective: # Replace with a single Open Collective username
ko_fi: # Replace with a single Ko-fi username
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
liberapay: # Replace with a single Liberapay username
issuehunt: # Replace with a single IssueHunt username
otechie: # Replace with a single Otechie username
lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
@@ -0,0 +1,28 @@
---
name: Bug report
about: Create a report to help us improve
title: "[BUG] "
labels: bug
assignees: martinus
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1.
2.
3.
**Expected behavior**
A clear and concise description of what you expected to happen.
**System (please complete the following information):**
- OS: [e.g. Linux]
- Compiler: [e.g. clang++, g++]
- Version [e.g. 13.0.1]
**Additional context**
Add any other context about the problem here.
@@ -0,0 +1,20 @@
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: enhancement
assignees: martinus
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.
+74
View File
@@ -0,0 +1,74 @@
name: Build, Test, Lint
on: [push, pull_request]
# see https://github.com/mesonbuild/meson/blob/master/docs/markdown/Continuous-Integration.md
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- run: ./scripts/lint/lint-version.py
linux:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- run: sudo apt-get install -yq libboost-dev
- uses: hendrikmuhs/ccache-action@v1.2
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- run: pip install meson ninja
- run: meson setup builddir/
env:
CXX: ccache c++
- run: meson test -C builddir/ -v
- uses: actions/upload-artifact@v3
if: failure()
with:
name: Linux_Meson_Testlog
path: builddir/meson-logs/testlog.txt
macos:
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- run: brew install gcc ccache meson ninja
- run: meson setup builddir/
env:
CXX: ccache c++
- run: meson test -C builddir/ -v
- uses: actions/upload-artifact@v3
if: failure()
with:
name: MacOS_Meson_Testlog
path: builddir/meson-logs/testlog.txt
windows:
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- uses: BSFishy/pip-action@v1
with:
packages: ninja meson
- uses: ilammy/msvc-dev-cmd@v1
- run: meson setup builddir
- run: meson test -C builddir -v
- uses: actions/upload-artifact@v3
if: failure()
with:
name: Windows_Meson_Testlog
path: |
builddir/meson-logs/testlog.txt
builddir/test/udm-test.exe
+14
View File
@@ -0,0 +1,14 @@
build
builddir
.cache
.vscode
compile_commands.json
# ignore all in subprojects except the .wrap files
/subprojects/*
!/subprojects/*.wrap
# c++ modules
*.pcm
a.out
*.o
+61
View File
@@ -0,0 +1,61 @@
cmake_minimum_required(VERSION 3.12)
project("unordered_dense"
VERSION 4.4.0
DESCRIPTION "A fast & densely stored hashmap and hashset based on robin-hood backward shift deletion"
HOMEPAGE_URL "https://github.com/martinus/unordered_dense")
include(GNUInstallDirs)
# determine whether this is a standalone project or included by other projects
if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
set(_unordered_dense_is_toplevel_project TRUE)
else()
set(_unordered_dense_is_toplevel_project FALSE)
endif()
add_library(unordered_dense INTERFACE)
add_library(unordered_dense::unordered_dense ALIAS unordered_dense)
target_include_directories(
unordered_dense
INTERFACE
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
target_compile_features(unordered_dense INTERFACE cxx_std_17)
if(_unordered_dense_is_toplevel_project)
# locations are provided by GNUInstallDirs
install(
TARGETS unordered_dense
EXPORT unordered_dense_Targets
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
include(CMakePackageConfigHelpers)
write_basic_package_version_file(
"unordered_denseConfigVersion.cmake"
VERSION ${PROJECT_VERSION}
COMPATIBILITY SameMajorVersion)
configure_package_config_file(
"${PROJECT_SOURCE_DIR}/cmake/unordered_denseConfig.cmake.in"
"${PROJECT_BINARY_DIR}/unordered_denseConfig.cmake"
INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
install(
EXPORT unordered_dense_Targets
FILE unordered_denseTargets.cmake
NAMESPACE unordered_dense::
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
install(
FILES "${PROJECT_BINARY_DIR}/unordered_denseConfig.cmake"
"${PROJECT_BINARY_DIR}/unordered_denseConfigVersion.cmake"
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
install(
DIRECTORY ${PROJECT_SOURCE_DIR}/include/ankerl
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
endif()
+76
View File
@@ -0,0 +1,76 @@
# Contributor Covenant Code of Conduct
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to making participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, sex characteristics, gender identity and expression,
level of experience, education, socio-economic status, nationality, personal
appearance, race, religion, or sexual identity and orientation.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic
address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
## Scope
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at martin.ankerl@gmail.com. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
[homepage]: https://www.contributor-covenant.org
For answers to common questions about this code of conduct, see
https://www.contributor-covenant.org/faq
+3
View File
@@ -0,0 +1,3 @@
* Coding style should be consistent with the code around you.
* Use automatic formatting with clang-format.
* One feature per pull request
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2022 Martin Leitner-Ankerl
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+374
View File
@@ -0,0 +1,374 @@
<a id="top"></a>
[![Release](https://img.shields.io/github/release/martinus/unordered_dense.svg)](https://github.com/martinus/unordered_dense/releases)
[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/martinus/unordered_dense/main/LICENSE)
[![meson_build_test](https://github.com/martinus/unordered_dense/actions/workflows/main.yml/badge.svg)](https://github.com/martinus/unordered_dense/actions)
[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/6220/badge)](https://bestpractices.coreinfrastructure.org/projects/6220)
[![Sponsors](https://img.shields.io/github/sponsors/martinus?style=social)](https://github.com/sponsors/martinus)
# 🚀 ankerl::unordered_dense::{map, set} <!-- omit in toc -->
A fast & densely stored hashmap and hashset based on robin-hood backward shift deletion for C++17 and later.
The classes `ankerl::unordered_dense::map` and `ankerl::unordered_dense::set` are (almost) drop-in replacements of `std::unordered_map` and `std::unordered_set`. While they don't have as strong iterator / reference stability guaranties, they are typically *much* faster.
Additionally, there are `ankerl::unordered_dense::segmented_map` and `ankerl::unordered_dense::segmented_set` with lower peak memory usage. and stable iterator/references on insert.
- [1. Overview](#1-overview)
- [2. Installation](#2-installation)
- [2.1. Installing using cmake](#21-installing-using-cmake)
- [3. Usage](#3-usage)
- [3.1. Modules](#31-modules)
- [3.2. Hash](#32-hash)
- [3.2.1. Simple Hash](#321-simple-hash)
- [3.2.2. High Quality Hash](#322-high-quality-hash)
- [3.2.3. Specialize `ankerl::unordered_dense::hash`](#323-specialize-ankerlunordered_densehash)
- [3.2.4. Heterogeneous Overloads using `is_transparent`](#324-heterogeneous-overloads-using-is_transparent)
- [3.2.5. Automatic Fallback to `std::hash`](#325-automatic-fallback-to-stdhash)
- [3.2.6. Hash the Whole Memory](#326-hash-the-whole-memory)
- [3.3. Container API](#33-container-api)
- [3.3.1. `auto extract() && -> value_container_type`](#331-auto-extract----value_container_type)
- [3.3.2. `extract()` single Elements](#332-extract-single-elements)
- [3.3.3. `[[nodiscard]] auto values() const noexcept -> value_container_type const&`](#333-nodiscard-auto-values-const-noexcept---value_container_type-const)
- [3.3.4. `auto replace(value_container_type&& container)`](#334-auto-replacevalue_container_type-container)
- [3.4. Custom Container Types](#34-custom-container-types)
- [3.5. Custom Bucket Types](#35-custom-bucket-types)
- [3.5.1. `ankerl::unordered_dense::bucket_type::standard`](#351-ankerlunordered_densebucket_typestandard)
- [3.5.2. `ankerl::unordered_dense::bucket_type::big`](#352-ankerlunordered_densebucket_typebig)
- [4. `segmented_map` and `segmented_set`](#4-segmented_map-and-segmented_set)
- [5. Design](#5-design)
- [5.1. Inserts](#51-inserts)
- [5.2. Lookups](#52-lookups)
- [5.3. Removals](#53-removals)
- [6. Real World Usage](#6-real-world-usage)
## 1. Overview
The chosen design has a few advantages over `std::unordered_map`:
* Perfect iteration speed - Data is stored in a `std::vector`, all data is contiguous!
* Very fast insertion & lookup speed, in the same ballpark as [`absl::flat_hash_map`](https://abseil.io/docs/cpp/guides/container`)
* Low memory usage
* Full support for `std::allocators`, and [polymorphic allocators](https://en.cppreference.com/w/cpp/memory/polymorphic_allocator). There are `ankerl::unordered_dense::pmr` typedefs available
* Customizeable storage type: with a template parameter you can e.g. switch from `std::vector` to `boost::interprocess::vector` or any other compatible random-access container.
* Better debugging: the underlying data can be easily seen in any debugger that can show an `std::vector`.
There's no free lunch, so there are a few disadvantages:
* Deletion speed is relatively slow. This needs two lookups: one for the element to delete, and one for the element that is moved onto the newly empty spot.
* no `const Key` in `std::pair<Key, Value>`
* Iterators and references are not stable on insert or erase.
## 2. Installation
<!-- See https://github.com/bernedom/SI/blob/main/doc/installation-guide.md -->
The default installation location is `/usr/local`.
### 2.1. Installing using cmake
Clone the repository and run these commands in the cloned folder:
```sh
mkdir build && cd build
cmake ..
cmake --build . --target install
```
Consider setting an install prefix if you do not want to install `unordered_dense` system wide, like so:
```sh
mkdir build && cd build
cmake -DCMAKE_INSTALL_PREFIX:PATH=${HOME}/unordered_dense_install ..
cmake --build . --target install
```
To make use of the installed library, add this to your project:
```cmake
find_package(unordered_dense CONFIG REQUIRED)
target_link_libraries(your_project_name unordered_dense::unordered_dense)
```
## 3. Usage
### 3.1. Modules
`ankerl::unordered_dense` supports c++20 modules. Simply compile `src/ankerl.unordered_dense.cpp` and use the resulting module, e.g. like so:
```sh
clang++ -std=c++20 -I include --precompile -x c++-module src/ankerl.unordered_dense.cpp
clang++ -std=c++20 -c ankerl.unordered_dense.pcm
```
To use the module with e.g. in `module_test.cpp`, use
```cpp
import ankerl.unordered_dense;
```
and compile with e.g.
```sh
clang++ -std=c++20 -fprebuilt-module-path=. ankerl.unordered_dense.o module_test.cpp -o main
```
A simple demo script can be found in `test/modules`.
### 3.2. Hash
`ankerl::unordered_dense::hash` is a fast and high quality hash, based on [wyhash](https://github.com/wangyi-fudan/wyhash). The `ankerl::unordered_dense` map/set differentiates between hashes of high quality (good [avalanching effect](https://en.wikipedia.org/wiki/Avalanche_effect)) and bad quality. Hashes with good quality contain a special marker:
```cpp
using is_avalanching = void;
```
This is the cases for the specializations `bool`, `char`, `signed char`, `unsigned char`, `char8_t`, `char16_t`, `char32_t`, `wchar_t`, `short`, `unsigned short`, `int`, `unsigned int`, `long`, `long long`, `unsigned long`, `unsigned long long`, `T*`, `std::unique_ptr<T>`, `std::shared_ptr<T>`, `enum`, `std::basic_string<C>`, and `std::basic_string_view<C>`.
Hashes that do not contain such a marker are assumed to be of bad quality and receive an additional mixing step inside the map/set implementation.
#### 3.2.1. Simple Hash
Consider a simple custom key type:
```cpp
struct id {
uint64_t value{};
auto operator==(id const& other) const -> bool {
return value == other.value;
}
};
```
The simplest implementation of a hash is this:
```cpp
struct custom_hash_simple {
auto operator()(id const& x) const noexcept -> uint64_t {
return x.value;
}
};
```
This can be used e.g. with
```cpp
auto ids = ankerl::unordered_dense::set<id, custom_hash_simple>();
```
Since `custom_hash_simple` doesn't have a `using is_avalanching = void;` marker it is considered to be of bad quality and additional mixing of `x.value` is automatically provided inside the set.
#### 3.2.2. High Quality Hash
Back to the `id` example, we can easily implement a higher quality hash:
```cpp
struct custom_hash_avalanching {
using is_avalanching = void;
auto operator()(id const& x) const noexcept -> uint64_t {
return ankerl::unordered_dense::detail::wyhash::hash(x.value);
}
};
```
We know `wyhash::hash` is of high quality, so we can add `using is_avalanching = void;` which makes the map/set directly use the returned value.
#### 3.2.3. Specialize `ankerl::unordered_dense::hash`
Instead of creating a new class you can also specialize `ankerl::unordered_dense::hash`:
```cpp
template <>
struct ankerl::unordered_dense::hash<id> {
using is_avalanching = void;
[[nodiscard]] auto operator()(id const& x) const noexcept -> uint64_t {
return detail::wyhash::hash(x.value);
}
};
```
#### 3.2.4. Heterogeneous Overloads using `is_transparent`
This map/set supports heterogeneous overloads as described in [P2363 Extending associative containers with the remaining heterogeneous overloads](https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p2363r3.html) which is [targeted for C++26](https://wg21.link/p2077r2). This has overloads for `find`, `count`, `contains`, `equal_range` (see [P0919R3](https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2018/p0919r3.html)), `erase` (see [P2077R2](https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2077r2.html)), and `try_emplace`, `insert_or_assign`, `operator[]`, `at`, and `insert` & `emplace` for sets (see [P2363R3](https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p2363r3.html)).
For heterogeneous overloads to take affect, both `hasher` and `key_equal` need to have the attribute `is_transparent` set.
Here is an example implementation that's usable with any string types that is convertible to `std::string_view` (e.g. `char const*` and `std::string`):
```cpp
struct string_hash {
using is_transparent = void; // enable heterogeneous overloads
using is_avalanching = void; // mark class as high quality avalanching hash
[[nodiscard]] auto operator()(std::string_view str) const noexcept -> uint64_t {
return ankerl::unordered_dense::hash<std::string_view>{}(str);
}
};
```
To make use of this hash you'll need to specify it as a type, and also a `key_equal` with `is_transparent` like [std::equal_to<>](https://en.cppreference.com/w/cpp/utility/functional/equal_to_void):
```cpp
auto map = ankerl::unordered_dense::map<std::string, size_t, string_hash, std::equal_to<>>();
```
For more information see the examples in `test/unit/transparent.cpp`.
#### 3.2.5. Automatic Fallback to `std::hash`
When an implementation for `std::hash` of a custom type is available, this is automatically used and assumed to be of bad quality (thus `std::hash` is used, but an additional mixing step is performed).
#### 3.2.6. Hash the Whole Memory
When the type [has a unique object representation](https://en.cppreference.com/w/cpp/types/has_unique_object_representations) (no padding, trivially copyable), one can just hash the object's memory. Consider a simple class
```cpp
struct point {
int x{};
int y{};
auto operator==(point const& other) const -> bool {
return x == other.x && y == other.y;
}
};
```
A fast and high quality hash can be easily provided like so:
```cpp
struct custom_hash_unique_object_representation {
using is_avalanching = void;
[[nodiscard]] auto operator()(point const& f) const noexcept -> uint64_t {
static_assert(std::has_unique_object_representations_v<point>);
return ankerl::unordered_dense::detail::wyhash::hash(&f, sizeof(f));
}
};
```
### 3.3. Container API
In addition to the standard `std::unordered_map` API (see https://en.cppreference.com/w/cpp/container/unordered_map) we have additional API that is somewhat similar to the node API, but leverages the fact that we're using a random access container internally:
#### 3.3.1. `auto extract() && -> value_container_type`
Extracts the internally used container. `*this` is emptied.
#### 3.3.2. `extract()` single Elements
Similar to `erase()` I have an API call `extract()`. It behaves exactly the same as `erase`, except that the return value is the moved element that is removed from the container:
* `auto extract(const_iterator it) -> value_type`
* `auto extract(Key const& key) -> std::optional<value_type>`
* `template <class K> auto extract(K&& key) -> std::optional<value_type>`
Note that the `extract(key)` API returns an `std::optional<value_type>` that is empty when the key is not found.
#### 3.3.3. `[[nodiscard]] auto values() const noexcept -> value_container_type const&`
Exposes the underlying values container.
#### 3.3.4. `auto replace(value_container_type&& container)`
Discards the internally held container and replaces it with the one passed. Non-unique elements are
removed, and the container will be partly reordered when non-unique elements are found.
### 3.4. Custom Container Types
`unordered_dense` accepts a custom allocator, but you can also specify a custom container for that template argument. That way it is possible to replace the internally used `std::vector` with e.g. `std::deque` or any other container like `boost::interprocess::vector`. This supports fancy pointers (e.g. [offset_ptr](https://www.boost.org/doc/libs/1_80_0/doc/html/interprocess/offset_ptr.html)), so the container can be used with e.g. shared memory provided by `boost::interprocess`.
### 3.5. Custom Bucket Types
The map/set supports two different bucket types. The default should be good for pretty much everyone.
#### 3.5.1. `ankerl::unordered_dense::bucket_type::standard`
* Up to 2^32 = 4.29 billion elements.
* 8 bytes overhead per bucket.
#### 3.5.2. `ankerl::unordered_dense::bucket_type::big`
* up to 2^63 = 9223372036854775808 elements.
* 12 bytes overhead per bucket.
## 4. `segmented_map` and `segmented_set`
`ankerl::unordered_dense` provides a custom container implementation that has lower memory requirements than the default `std::vector`. Memory is not contiguous, but it can allocate segments without having to reallocate and move all the elements. In summary, this leads to
* Much smoother memory usage, memory usage increases continuously.
* No high peak memory usage.
* Faster insertion because elements never need to be moved to new allocated blocks
* Slightly slower indexing compared to `std::vector` because an additional indirection is needed.
Here is a comparison against `absl::flat_hash_map` and the `ankerl::unordered_dense::map` when inserting 10 million entries
![allocated memory](doc/allocated_memory.png)
Abseil is fastest for this simple inserting test, taking a bit over 0.8 seconds. It's peak memory usage is about 430 MB. Note how the memory usage goes down after the last peak; when it goes down to ~290MB it has finished rehashing and could free the previously used memory block.
`ankerl::unordered_dense::segmented_map` doesn't have these peaks, and instead has a smooth increase of memory usage. Note there are still sudden drops & increases in memory because the indexing data structure needs still needs to increase by a fixed factor. But due to holding the data in a separate container we are able to first free the old data structure, and then allocate a new, bigger indexing structure; thus we do not have peaks.
## 5. Design
The map/set has two data structures:
* `std::vector<value_type>` which holds all data. map/set iterators are just `std::vector<value_type>::iterator`!
* An indexing structure (bucket array), which is a flat array with 8-byte buckets.
### 5.1. Inserts
Whenever an element is added it is `emplace_back` to the vector. The key is hashed, and an entry (bucket) is added at the
corresponding location in the bucket array. The bucket has this structure:
```cpp
struct Bucket {
uint32_t dist_and_fingerprint;
uint32_t value_idx;
};
```
Each bucket stores 3 things:
* The distance of that value from the original hashed location (3 most significant bytes in `dist_and_fingerprint`)
* A fingerprint; 1 byte of the hash (lowest significant byte in `dist_and_fingerprint`)
* An index where in the vector the actual data is stored.
This structure is especially designed for the collision resolution strategy robin-hood hashing with backward shift
deletion.
### 5.2. Lookups
The key is hashed and the bucket array is searched if it has an entry at that location with that fingerprint. When found,
the key in the data vector is compared, and when equal the value is returned.
### 5.3. Removals
Since all data is stored in a vector, removals are a bit more complicated:
1. First, lookup the element to delete in the index array.
2. When found, replace that element in the vector with the last element in the vector.
3. Update *two* locations in the bucket array: First remove the bucket for the removed element
4. Then, update the `value_idx` of the moved element. This requires another lookup.
## 6. Real World Usage
On 2023-09-10 I did a quick search on github to see if this map is used in any popular open source projects. Here are some of the projects
I found. Please send me a note if you want on that list!
* [PruaSlicer](https://github.com/prusa3d/PrusaSlicer) - G-code generator for 3D printers (RepRap, Makerbot, Ultimaker etc.)
* [Kismet](https://github.com/kismetwireless/kismet): Wi-Fi, Bluetooth, RF, and more. Kismet is a sniffer, WIDS, and wardriving tool for Wi-Fi, Bluetooth, Zigbee, RF, and more, which runs on Linux and macOS
* [Rspamd](https://github.com/rspamd/rspamd) - Fast, free and open-source spam filtering system.
* [kallisto](https://github.com/pachterlab/kallisto) - Near-optimal RNA-Seq quantification
* [Slang](https://github.com/shader-slang/slang) - Slang is a shading language that makes it easier to build and maintain large shader codebases in a modular and extensible fashion.
* [CyberFSR2](https://github.com/PotatoOfDoom/CyberFSR2) - Drop-in DLSS replacement with FSR 2.0 for various games such as Cyberpunk 2077.
* [ossia score](https://github.com/ossia/score) - A free, open-source, cross-platform intermedia sequencer for precise and flexible scripting of interactive scenarios.
* [HiveWE](https://github.com/stijnherfst/HiveWE) - A Warcraft III World Editor (WE) that focusses on speed and ease of use.
* [opentxs](https://github.com/Open-Transactions/opentxs) - The Open-Transactions project is a collaborative effort to develop a robust, commercial-grade, fully-featured, free-software toolkit implementing the OTX protocol as well as a full-strength financial cryptography library, API, GUI, command-line interface, and prototype notary server.
* [LuisaCompute](https://github.com/LuisaGroup/LuisaCompute) - High-Performance Rendering Framework on Stream Architectures
* [Lethe](https://github.com/lethe-cfd/lethe) - Lethe (pronounced /ˈliːθiː/) is open-source computational fluid dynamics (CFD) software which uses high-order continuous Galerkin formulations to solve the incompressible NavierStokes equations (among others).
* [PECOS](https://github.com/amzn/pecos) - PECOS is a versatile and modular machine learning (ML) framework for fast learning and inference on problems with large output spaces, such as extreme multi-label ranking (XMR) and large-scale retrieval.
* [Operon](https://github.com/heal-research/operon) - A modern C++ framework for symbolic regression that uses genetic programming to explore a hypothesis space of possible mathematical expressions in order to find the best-fitting model for a given regression target.
* [MashMap](https://github.com/marbl/MashMap) - A fast approximate aligner for long DNA sequences
* [minigpt4.cpp](https://github.com/Maknee/minigpt4.cpp) - Port of MiniGPT4 in C++ (4bit, 5bit, 6bit, 8bit, 16bit CPU inference with GGML)
@@ -0,0 +1,4 @@
@PACKAGE_INIT@
include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
check_required_components("@PROJECT_NAME@")
+44
View File
@@ -0,0 +1,44 @@
#!/usr/bin/gnuplot
#set terminal pngcairo
#set terminal pngcairo size 730,510 enhanced font 'Verdana,10'
set terminal pngcairo size 800,600 enhanced font 'Verdana,10'
# define axis
# remove border on top and right and set color to gray
set style line 11 lc rgb '#808080' lt 1
set border 3 back ls 11
set tics nomirror
# define grid
set style line 12 lc rgb '#808080' lt 0 lw 1
set grid back ls 12
# line styles
set style line 1 lt 1 lc rgb '#1B9E77' # dark teal
set style line 2 lt 1 lc rgb '#D95F02' # dark orange
set style line 3 lt 1 lc rgb '#7570B3' # dark lilac
set style line 4 lt 1 lc rgb '#E7298A' # dark magenta
set style line 5 lt 1 lc rgb '#66A61E' # dark lime green
set style line 6 lt 1 lc rgb '#E6AB02' # dark banana
set style line 7 lt 1 lc rgb '#A6761D' # dark tan
set style line 8 lt 1 lc rgb '#666666' # dark gray
set style line 101 lc rgb '#808080' lt 1 lw 1
set border 3 front ls 101
set tics nomirror out scale 0.75
set key left top
set output 'allocated_memory.png'
set xlabel "Runtime [s]"
set ylabel "Allocated memory [MB]"
set title "Inserting 10 Million uint64\\\_t -> uint64\\\_t pairs"
# allocated_memory_segmented_vector.txt allocated_memory_std_unordered_map.txt allocated_memory_std_vector.txt
plot \
'allocated_memory_segmented_vector.txt' using ($1):($2/1e6) w steps ls 1 lw 2 title "ankerl::unordered\\\_dense::segmented\\\_map" , \
'allocated_memory_std_vector.txt' using ($1):($2/1e6) w steps ls 2 lw 2 title "ankerl::unordered\\\_dense::map" , \
'allocated_memory_absl_flat_hash_map.txt' using ($1):($2/1e6) w steps ls 3 lw 2 title "absl::flat\\\_hash\\\_map"
Binary file not shown.

After

Width:  |  Height:  |  Size: 32 KiB

+6
View File
@@ -0,0 +1,6 @@
cmake_minimum_required(VERSION 3.12)
project("UnorderedDenseExample")
add_executable(UnorderedDenseExample main.cpp)
find_package(unordered_dense CONFIG REQUIRED)
target_link_libraries(UnorderedDenseExample unordered_dense::unordered_dense)

Some files were not shown because too many files have changed in this diff Show More