diff --git a/third_party/libosmium/.travis.yml b/third_party/libosmium/.travis.yml index 6ebdd7167..ac0d270e2 100644 --- a/third_party/libosmium/.travis.yml +++ b/third_party/libosmium/.travis.yml @@ -9,46 +9,151 @@ language: cpp sudo: false matrix: - include: - - os: linux - compiler: clang - env: BUILD_TYPE=Dev - - os: linux - compiler: clang - env: BUILD_TYPE=Release - - os: linux - compiler: gcc - env: BUILD_TYPE=Dev - - os: linux - compiler: gcc - env: BUILD_TYPE=Release - - os: osx - compiler: clang - env: BUILD_TYPE=Dev - - os: osx - compiler: clang - env: BUILD_TYPE=Release + include: + + # 1/ Linux Clang Builds + - os: linux + compiler: clang + addons: + apt: + sources: ['llvm-toolchain-precise-3.5', 'ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['clang-3.5', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='clang++-3.5' BUILD_TYPE='Release' + + - os: linux + compiler: clang + addons: + apt: + sources: ['llvm-toolchain-precise-3.5', 'ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['clang-3.5', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='clang++-3.5' BUILD_TYPE='Dev' + + + - os: linux + compiler: clang + addons: + apt: + sources: ['llvm-toolchain-precise-3.6', 'ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['clang-3.6', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='clang++-3.6' BUILD_TYPE='Release' + + - os: linux + compiler: clang + addons: + apt: + sources: ['llvm-toolchain-precise-3.6', 'ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['clang-3.6', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='clang++-3.6' BUILD_TYPE='Dev' + + + - os: linux + compiler: clang + addons: + apt: + sources: ['llvm-toolchain-precise-3.7', 'ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['clang-3.7', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='clang++-3.7' BUILD_TYPE='Release' + + - os: linux + compiler: clang + addons: + apt: + sources: ['llvm-toolchain-precise-3.7', 'ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['clang-3.7', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='clang++-3.7' BUILD_TYPE='Dev' + + + # 2/ Linux GCC Builds + - os: linux + compiler: gcc + addons: + apt: + sources: ['ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['g++-4.8', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='g++-4.8' COMPILER_FLAGS='-Wno-return-type' BUILD_TYPE='Release' + + - os: linux + compiler: gcc + addons: + apt: + sources: ['ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['g++-4.8', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='g++-4.8' COMPILER_FLAGS='-Wno-return-type' BUILD_TYPE='Dev' + + + - os: linux + compiler: gcc + addons: + apt: + sources: ['ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['g++-4.9', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='g++-4.9' BUILD_TYPE='Release' + + - os: linux + compiler: gcc + addons: + apt: + sources: ['ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['g++-4.9', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='g++-4.9' BUILD_TYPE='Dev' + + + - os: linux + compiler: gcc + addons: + apt: + sources: ['ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['g++-5', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='g++-5' BUILD_TYPE='Release' + + - os: linux + compiler: gcc + addons: + apt: + sources: ['ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['g++-5', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='g++-5' BUILD_TYPE='Dev' + + + # 3/ OSX Clang Builds + - os: osx + osx_image: xcode6.4 + compiler: clang + env: COMPILER='clang++' BUILD_TYPE='Dev' + + - os: osx + osx_image: xcode6.4 + compiler: clang + env: COMPILER='clang++' BUILD_TYPE='Release' + + + - os: osx + osx_image: xcode7 + compiler: clang + env: COMPILER='clang++' BUILD_TYPE='Dev' + + - os: osx + osx_image: xcode7 + compiler: clang + env: COMPILER='clang++' BUILD_TYPE='Release' -# http://docs.travis-ci.com/user/apt/ -addons: - apt: - sources: - - boost-latest - - ubuntu-toolchain-r-test - packages: - - g++-4.8 - - gcc-4.8 - - libboost1.55-dev - - libboost-program-options1.55-dev - - libgdal-dev - - libgeos++-dev - - libproj-dev - - libsparsehash-dev - - spatialite-bin install: - - scripts/travis_install.sh + - DEPS_DIR="${TRAVIS_BUILD_DIR}/deps" + - mkdir -p ${DEPS_DIR} && cd ${DEPS_DIR} + - git clone --quiet --depth 1 https://github.com/osmcode/osm-testdata.git + - | + if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then + brew remove gdal + brew install cmake boost google-sparsehash gdal + fi + +before_script: + - cd ${TRAVIS_BUILD_DIR} + - mkdir build && cd build + - CXX=${COMPILER} CXXFLAGS=${COMPILER_FLAGS} cmake -LA .. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DOSM_TESTDATA="${TRAVIS_BUILD_DIR}/deps/osm-testdata" script: - - scripts/travis_script.sh + - make VERBOSE=1 + - ctest --output-on-failure diff --git a/third_party/libosmium/CHANGELOG.md b/third_party/libosmium/CHANGELOG.md index 22eb06aac..e9377b656 100644 --- a/third_party/libosmium/CHANGELOG.md +++ b/third_party/libosmium/CHANGELOG.md @@ -12,6 +12,132 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Fixed + +## [2.5.4] - 2015-12-03 + +### Changed + +- Included gdalcpp.hpp header was updated to version 1.1.1. +- Included protozero library was updated to version 1.2.3. +- Workarounds for missing constexpr support in Visual Studio removed. All + constexpr features we need are supported now. +- Some code cleanup after running clang-tidy on the code. +- Re-added `Buffer::value_type` typedef. Turns out it is needed when using + `std::back_inserter` on the Buffer. + +### Fixed + +- Bugs with Timestamp code on 32 bit platforms. This necessitated + some changes in Timestamp which might lead to changes in user + code. +- Bug in segment intersection code (which appeared on i686 platform). + + +## [2.5.3] - 2015-11-17 + +### Added + +- `osmium::make_diff_iterator()` helper function. + +### Changed + +- Deprecated `osmium::Buffer::set_full_callback()`. +- Removed DataFile class which was never used anywhere. +- Removed unused and obscure `Buffer::value_type` typedef. + +### Fixed + +- Possible overrun in Buffer when using the full-callback. +- Incorrect swapping of Buffer. + + +## [2.5.2] - 2015-11-06 + +# Fixed + +- Writing data through an OutputIterator was extremly slow due to + lock contention. + + +## [2.5.1] - 2015-11-05 + +### Added + +- Header `osmium/fwd.hpp` with forward declarations of the most commonly + used Osmium classes. + +### Changed + +- Moved `osmium/io/overwrite.hpp` to `osmium/io/writer_options.hpp` + If you still include the old file, you'll get a warning. + + +## [2.5.0] - 2015-11-04 + +### Added + +- Helper functions to make input iterator ranges and output iterators. +- Add support for reading o5m and o5c files. +- Option for osmium::io::Writer to fsync file after writing. +- Lots of internal asserts() and other robustness checks. + +### Changed + +- Updated included protozero library to version 1.2.0. +- Complete overhaul of the I/O system making it much more robust against + wrong data and failures during I/O operations. +- Speed up PBF writing by running parts of it in parallel. +- OutputIterator doesn't hold an internal buffer any more, but it uses + one in Writer. Calling flush() on the OutputIterator isn't needed any + more. +- Reader now throws when trying to read after eof or an error. +- I/O functions that used to throw std::runtime_error now throw + osmium::io_error or derived. +- Optional parameters on osmium::io::Writer now work in any order. + +### Fixed + +- PBF reader now decodes locations of invisible nodes properly. +- Invalid Delta encode iterator dereference. +- Lots of includes fixed to include (only) what's used. +- Dangling reference in area assembly code. + + +## [2.4.1] - 2015-08-29 + +### Fixed + +- CRC calculation of tags and changesets. + + +## [2.4.0] - 2015-08-29 + +### Added + +- Checks that user names, member roles and tag keys and values are not longer + than 256 * 4 bytes. That is the maximum length 256 Unicode characters + can have in UTF-8 encoding. +- Support for GDAL 2. GDAL 1 still works. + +### Changed + +- Improved CMake build scripts. +- Updated internal version of Protozero to 1.1.0. +- Removed `toogr*` examples. They are in their own repository now. + See https://github.com/osmcode/osm-gis-export. +- Files about to be memory-mapped (for instance index files) are now set + to binary mode on Windows so the application doesn't have to do this. + +### Fixed + +- Hanging program when trying to open file with an unknown file format. +- Building problems with old boost versions. +- Initialization errors in PBF writer. +- Bug in byte swap code. +- Output on Windows now always uses binary mode, even when writing to + stdout, so OSM xml and opl files always use LF line endings. + + ## [2.3.0] - 2015-08-18 ### Added @@ -108,8 +234,15 @@ This project adheres to [Semantic Versioning](http://semver.org/). Doxygen (up to version 1.8.8). This version contains a workaround to fix this. -[unreleased]: https://github.com/osmcode/libosmium/compare/v2.3.0...HEAD -[2.3.0]: https://github.com/osmcode/libosmium/compare/v2.3.0...v2.3.0 +[unreleased]: https://github.com/osmcode/libosmium/compare/v2.5.4...HEAD +[2.5.4]: https://github.com/osmcode/libosmium/compare/v2.5.3...v2.5.4 +[2.5.3]: https://github.com/osmcode/libosmium/compare/v2.5.2...v2.5.3 +[2.5.2]: https://github.com/osmcode/libosmium/compare/v2.5.1...v2.5.2 +[2.5.1]: https://github.com/osmcode/libosmium/compare/v2.5.0...v2.5.1 +[2.5.0]: https://github.com/osmcode/libosmium/compare/v2.4.1...v2.5.0 +[2.4.1]: https://github.com/osmcode/libosmium/compare/v2.4.0...v2.4.1 +[2.4.0]: https://github.com/osmcode/libosmium/compare/v2.3.0...v2.4.0 +[2.3.0]: https://github.com/osmcode/libosmium/compare/v2.2.0...v2.3.0 [2.2.0]: https://github.com/osmcode/libosmium/compare/v2.1.0...v2.2.0 [2.1.0]: https://github.com/osmcode/libosmium/compare/v2.0.0...v2.1.0 diff --git a/third_party/libosmium/CMakeLists.txt b/third_party/libosmium/CMakeLists.txt index fba967a4e..0764915ea 100644 --- a/third_party/libosmium/CMakeLists.txt +++ b/third_party/libosmium/CMakeLists.txt @@ -9,8 +9,6 @@ cmake_minimum_required(VERSION 2.8 FATAL_ERROR) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - #----------------------------------------------------------------------------- # @@ -26,13 +24,13 @@ set(CMAKE_CONFIGURATION_TYPES "Debug;Release;RelWithDebInfo;MinSizeRel;Dev;Cover project(libosmium) set(LIBOSMIUM_VERSION_MAJOR 2) -set(LIBOSMIUM_VERSION_MINOR 3) -set(LIBOSMIUM_VERSION_PATCH 0) +set(LIBOSMIUM_VERSION_MINOR 5) +set(LIBOSMIUM_VERSION_PATCH 4) set(LIBOSMIUM_VERSION - "${LIBOSMIUM_VERSION_MAJOR}.${LIBOSMIUM_VERSION_MINOR}.${LIBOSMIUM_VERSION_PATCH}" - CACHE STRING - "Libosmium version") + "${LIBOSMIUM_VERSION_MAJOR}.${LIBOSMIUM_VERSION_MINOR}.${LIBOSMIUM_VERSION_PATCH}") + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) #----------------------------------------------------------------------------- @@ -56,6 +54,10 @@ option(BUILD_HEADERS "compile every header file on its own" ${dev_build}) option(BUILD_BENCHMARKS "compile benchmark programs" ${dev_build}) option(BUILD_DATA_TESTS "compile data tests, please run them with ctest" ${dev_build}) +option(INSTALL_GDALCPP "also install gdalcpp headers" OFF) +option(INSTALL_PROTOZERO "also install protozero headers" OFF) +option(INSTALL_UTFCPP "also install utfcpp headers" OFF) + #----------------------------------------------------------------------------- # @@ -118,29 +120,39 @@ find_package(Boost 1.38) mark_as_advanced(CLEAR BOOST_ROOT) if(Boost_FOUND) - include_directories(${Boost_INCLUDE_DIRS}) + include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) else() set(BOOST_ROOT "NOT FOUND: please choose" CACHE PATH "") message(FATAL_ERROR "PLEASE, specify the directory where the Boost library is installed in BOOST_ROOT") endif() -set(OSMIUM_INCLUDE_DIR include) +# set OSMIUM_INCLUDE_DIR so FindOsmium will not set anything different +set(OSMIUM_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include") + +include_directories(${OSMIUM_INCLUDE_DIR}) + find_package(Osmium COMPONENTS io gdal geos proj sparsehash) -include_directories(${OSMIUM_INCLUDE_DIRS}) + +# The find_package put the directory where it found the libosmium includes +# into OSMIUM_INCLUDE_DIRS. We remove it again, because we want to make +# sure to use our own include directory already set up above. +list(FIND OSMIUM_INCLUDE_DIRS "${OSMIUM_INCLUDE_DIR}" _own_index) +list(REMOVE_AT OSMIUM_INCLUDE_DIRS ${_own_index}) +set(_own_index) + +include_directories(SYSTEM ${OSMIUM_INCLUDE_DIRS}) if(MSVC) find_path(GETOPT_INCLUDE_DIR getopt.h) find_library(GETOPT_LIBRARY NAMES wingetopt) if(GETOPT_INCLUDE_DIR AND GETOPT_LIBRARY) - include_directories(${GETOPT_INCLUDE_DIR}) + include_directories(SYSTEM ${GETOPT_INCLUDE_DIR}) list(APPEND OSMIUM_LIBRARIES ${GETOPT_LIBRARY}) else() set(GETOPT_MISSING 1) endif() endif() -include_directories(include) - #----------------------------------------------------------------------------- # @@ -205,6 +217,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Dev") add_definitions(-Werror) endif() add_definitions(${OSMIUM_WARNING_OPTIONS}) +# add_definitions(${OSMIUM_WARNING_OPTIONS} ${OSMIUM_DRACONIC_CLANG_OPTIONS} -Wno-documentation -Wno-format-nonliteral -Wno-deprecated -Wno-covered-switch-default -Wno-shadow) endif() # Force RelWithDebInfo build type if none was given @@ -256,19 +269,21 @@ find_program(CPPCHECK cppcheck) if(CPPCHECK) message(STATUS "Looking for cppcheck - found") set(CPPCHECK_OPTIONS - --enable=warning,style,performance,portability,information,missingInclude) + --enable=warning,style,performance,portability,information,missingInclude --force -Uassert) # cpp doesn't find system includes for some reason, suppress that report set(CPPCHECK_OPTIONS ${CPPCHECK_OPTIONS} --suppress=missingIncludeSystem) file(GLOB_RECURSE ALL_INCLUDES include/osmium/*.hpp) file(GLOB ALL_EXAMPLES examples/*.cpp) + file(GLOB ALL_BENCHMARKS benchmarks/*.cpp) file(GLOB ALL_UNIT_TESTS test/t/*/test_*.cpp) file(GLOB ALL_DATA_TESTS test/data-tests/*.cpp) if(Osmium_DEBUG) message(STATUS "Checking includes : ${ALL_INCLUDES}") message(STATUS "Checking example code : ${ALL_EXAMPLES}") + message(STATUS "Checking benchmarks : ${ALL_BENCHMARKS}") message(STATUS "Checking unit test code: ${ALL_UNIT_TESTS}") message(STATUS "Checking data test code: ${ALL_DATA_TESTS}") endif() @@ -276,6 +291,7 @@ if(CPPCHECK) set(CPPCHECK_FILES ${ALL_INCLUDES} ${ALL_EXAMPLES} + ${ALL_BENCHMARKS} ${ALL_UNIT_TESTS} ${ALL_DATA_TESTS}) @@ -288,7 +304,7 @@ if(CPPCHECK) else() message(STATUS "Looking for cppcheck - not found") message(STATUS " Build target 'cppcheck' will not be available.") -endif(CPPCHECK) +endif() #----------------------------------------------------------------------------- @@ -345,11 +361,115 @@ if(BUILD_HEADERS) endforeach() endif() + +#----------------------------------------------------------------------------- +# +# Optional "clang-tidy" target +# +#----------------------------------------------------------------------------- +message(STATUS "Looking for clang-tidy") +find_program(CLANG_TIDY NAMES clang-tidy clang-tidy-3.9 clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy-3.5) + +if(CLANG_TIDY) + message(STATUS "Looking for clang-tidy - found") + + if(BUILD_EXAMPLES) + file(GLOB CT_ALL_EXAMPLES examples/*.cpp) + endif() + + if(BUILD_TESTING) + file(GLOB CT_ALL_UNIT_TESTS test/t/*/test_*.cpp) + endif() + + if(BUILD_HEADERS) + file(GLOB_RECURSE CT_ALL_INCLUDES ${CMAKE_BINARY_DIR}/header_check/osmium__*.cpp) + endif() + + if(BUILD_BENCHMARKS) + file(GLOB CT_ALL_BENCHMARKS benchmarks/*.cpp) + endif() + + if(BUILD_DATA_TESTS) + file(GLOB CT_ALL_DATA_TESTS test/data-tests/*.cpp) + endif() + + if(Osmium_DEBUG) + message(STATUS "Checking example code : ${CT_ALL_EXAMPLES}") + message(STATUS "Checking unit test code: ${CT_ALL_UNIT_TESTS}") + message(STATUS "Checking includes : ${CT_ALL_INCLUDES}") + message(STATUS "Checking benchmarks : ${CT_ALL_BENCHMARKS}") + message(STATUS "Checking data test code: ${CT_ALL_DATA_TESTS}") + endif() + + set(CT_CHECK_FILES + ${CT_ALL_EXAMPLES} + ${CT_ALL_UNIT_TESTS} + ${CT_ALL_INCLUDES} + ${CT_ALL_BENCHMARKS} + ${CT_ALL_DATA_TESTS}) + + # For a list of check options, see: + # http://clang.llvm.org/extra/clang-tidy/checks/list.html + + list(APPEND CT_CHECKS "cert-*" + "-cert-err60-cpp") # even the std lib doesn't do this + + # disabled, because it is slow +# list(APPEND CT_CHECKS "clang-analyzer-*") + + list(APPEND CT_CHECKS "google-*" + "-google-explicit-constructor" + "-google-readability-casting" + "-google-readability-function") + + list(APPEND CT_CHECKS "llvm-*" + "-llvm-include-order") + + list(APPEND CT_CHECKS "misc-*" + "-misc-argument-comment") + + list(APPEND CT_CHECKS "modernize-*") + + list(APPEND CT_CHECKS "readability-*" + "-readability-identifier-naming" + "-readability-named-parameter") + + string(REPLACE ";" "," ALL_CHECKS "${CT_CHECKS}") + + add_custom_target(clang-tidy + ${CLANG_TIDY} + -p ${CMAKE_BINARY_DIR} + -header-filter='include/osmium/.*' + -checks="${ALL_CHECKS}" + ${CT_CHECK_FILES} + ) +else() + message(STATUS "Looking for clang-tidy - not found") + message(STATUS " Build target 'clang-tidy' will not be available.") +endif() + +#----------------------------------------------------------------------------- +# +# Installation +# +# External libraries are only installed if the options are set in case they +# are installed from somewhere else. +# +#----------------------------------------------------------------------------- install(DIRECTORY include/osmium DESTINATION include) -# We only have a copy of this file so we can use older boost versions which -# don't have it. We probably don't want to install it. -#install(FILES include/boost_unicode_iterator.hpp DESTINATION include) +if(INSTALL_GDALCPP) + install(include/gdalcpp.hpp DESTINATION include) +endif() + +if(INSTALL_PROTOZERO) + install(DIRECTORY include/protozero DESTINATION include) +endif() + +if(INSTALL_UTFCPP) + install(include/utf8.hpp DESTINATION include) + install(DIRECTORY include/utf8 DESTINATION include) +endif() #----------------------------------------------------------------------------- diff --git a/third_party/libosmium/CONTRIBUTING.md b/third_party/libosmium/CONTRIBUTING.md index 323c84744..1064b94de 100644 --- a/third_party/libosmium/CONTRIBUTING.md +++ b/third_party/libosmium/CONTRIBUTING.md @@ -36,22 +36,21 @@ different. * Class names begin with uppercase chars and use CamelCase. Smaller helper classes are usually defined as struct and have lowercase names. * Macros (and only macros) are all uppercase. Use macros sparingly, usually - a constexpr is better. + a simple (maybe constexpr) inline function is better. Undef macros after use + if possible. +* Macros should only be used for controlling which parts of the code should be + included when compiling or to avoid major code repetitions. * Variables, attributes, and function names are lowercase with `underscores_between_words`. * Class attribute names start with `m_` (member). -* Template parameters are single uppercase letters or start with uppercase `T` - and use CamelCase. -* Typedefs have `names_like_this_type` which end in `_type`. -* Macros should only be used for controlling which parts of the code should be - included when compiling. * Use `descriptive_variable_names`, exceptions are well-established conventions like `i` for a loop variable. Iterators are usually called `it`. * Declare variables where they are first used (C++ style), not at the beginning of a function (old C style). * Names from external namespaces (even `std`) are always mentioned explicitly. Do not use `using` (except for `std::swap`). This way we can't even by - accident pollute the namespace of the code including Osmium. + accident pollute the namespace of the code using Osmium. +* Always use the standard swap idiom: `using std::swap; swap(foo, bar);`. * `#include` directives appear in three "blocks" after the copyright notice. The blocks are separated by blank lines. First block contains `#include`s for standard C/C++ includes, second block for any external libs used, third @@ -64,8 +63,20 @@ different. * All files have suffix `.hpp`. * Closing } of all classes and namespaces should have a trailing comment with the name of the class/namespace. -* All constructors with one or more arguments should be declared "explicit" - unless there is a reason for them not to be. Document that reason. +* All constructors with one (or more arguments if they have a default) should + be declared "explicit" unless there is a reason for them not to be. Document + that reason. +* If a class has any of the special methods (copy/move constructor/assigment, + destructor) it should have all of them, possibly marking them as default or + deleted. +* Typedefs have `names_like_this_type` which end in `_type`. Typedefs should + use the new `using foo_type = bar` syntax instead of the old + `typedef bar foo_type`. +* Template parameters are single uppercase letters or start with uppercase `T` + and use CamelCase. +* Always use `typename` in templates, not `class`: `template `. +* The ellipsis in variadic template never has a space to the left of it and + always has a space to the right: `template ` etc. Keep to the indentation and other styles used in the code. Use `make indent` in the toplevel directory to fix indentation and styling. It calls `astyle` @@ -81,15 +92,15 @@ about which compilers support which feature and what operating system versions or distributions have which versions of these compilers installed. GCC 4.6 - too old, not supported (Ubuntu 12.04 LTS) -GCC 4.7.2 - can probably not be supported (Debian wheezy/stable) -GCC 4.7.3 - works -GCC 4.8 - works -clang 3.0 - too old, not supported (Debian wheezy/stable, Ubuntu 12.04 LTS) -clang 3.2 - works +GCC 4.7.2 - can probably not be supported (Debian wheezy) +GCC 4.7.3 - probably works +GCC 4.8 - works and is supported from here on +clang 3.0 - too old, not supported (Debian wheezy, Ubuntu 12.04 LTS) +clang 3.2 - probably works +clang 3.5 - works and is supported from here on -C++11 features you should not use: -* Inherited Constructors (works only in GCC 4.8+ and clang 3.3+, not in Visual - Studio) +Use `include/osmium/util/compatibility.hpp` if there are compatibility problems +between compilers due to different C++11 support. ## Checking your code diff --git a/third_party/libosmium/README.md b/third_party/libosmium/README.md index 9676d80b7..68fc2f61b 100644 --- a/third_party/libosmium/README.md +++ b/third_party/libosmium/README.md @@ -4,7 +4,7 @@ http://osmcode.org/libosmium A fast and flexible C++ library for working with OpenStreetMap data. -[![Build Status](https://secure.travis-ci.org/osmcode/libosmium.png)](http://travis-ci.org/osmcode/libosmium) +[![Build Status](https://secure.travis-ci.org/osmcode/libosmium.png)](https://travis-ci.org/osmcode/libosmium) [![Build status](https://ci.appveyor.com/api/projects/status/mkbg6e6stdgq7c1b?svg=true)](https://ci.appveyor.com/project/Mapbox/libosmium) Libosmium is developed on Linux, but also works on OSX and Windows (with some @@ -27,9 +27,15 @@ you need for your programs. For details see the [list of dependencies](https://github.com/osmcode/libosmium/wiki/Libosmium-dependencies). -The [protozero](https://github.com/mapbox/protozero) and -[utf8-cpp](http://utfcpp.sourceforge.net/) header-only libraries are included -in the libosmium repository. +The following external (header-only) libraries are included in the libosmium +repository: +* [gdalcpp](https://github.com/joto/gdalcpp) +* [protozero](https://github.com/mapbox/protozero) +* [utfcpp](http://utfcpp.sourceforge.net/) + +If you want (some of) those libraries to be installed along with libosmium +itself when calling `make install`, you have to use the CMake options +`INSTALL_GDALCPP`, `INSTALL_PROTOZERO`, and/or `INSTALL_UTFCPP`. ## Directories diff --git a/third_party/libosmium/appveyor.yml b/third_party/libosmium/appveyor.yml index a05c396cc..8244d98e7 100644 --- a/third_party/libosmium/appveyor.yml +++ b/third_party/libosmium/appveyor.yml @@ -9,16 +9,10 @@ environment: - config: Dev - config: RelWithDebInfo -# branches to build -branches: - # whitelist - only: - - master - shallow_clone: true # Operating system (build VM template) -os: Visual Studio 2014 CTP4 +os: Visual Studio 2015 # scripts that are called at very beginning, before repo cloning init: @@ -46,6 +40,8 @@ install: - set PATH=%LODEPSDIR%\expat\lib;%PATH% #libtiff.dll - set PATH=%LODEPSDIR%\libtiff\lib;%PATH% + #jpeg.dll + - set PATH=%LODEPSDIR%\jpeg\lib;%PATH% #zlibwapi.dll - set PATH=%LODEPSDIR%\zlib\lib;%PATH% #convert backslashes in bzip2 path to forward slashes @@ -71,27 +67,16 @@ build_script: # This will produce lots of LNK4099 warnings which can be ignored. # Unfortunately they can't be disabled, see # http://stackoverflow.com/questions/661606/visual-c-how-to-disable-specific-linker-warnings - - cmake .. -LA -G "Visual Studio 14 Win64" + - cmake -LA -G "Visual Studio 14 Win64" -DOsmium_DEBUG=TRUE -DCMAKE_BUILD_TYPE=%config% -DBUILD_HEADERS=OFF -DBOOST_ROOT=%LODEPSDIR%\boost - -DBoost_PROGRAM_OPTIONS_LIBRARY=%LODEPSDIR%\boost\lib\libboost_program_options-vc140-mt-1_57.lib + -DBoost_PROGRAM_OPTIONS_LIBRARY=%LODEPSDIR%\boost\lib\libboost_program_options-vc140-mt-1_58.lib -DZLIB_LIBRARY=%LODEPSDIR%\zlib\lib\zlibwapi.lib - -DZLIB_INCLUDE_DIR=%LODEPSDIR%\zlib\include - -DEXPAT_LIBRARY=%LODEPSDIR%\expat\lib\libexpat.lib - -DEXPAT_INCLUDE_DIR=%LODEPSDIR%\expat\include - -DBZIP2_LIBRARIES=%LIBBZIP2% - -DBZIP2_INCLUDE_DIR=%LODEPSDIR%\bzip2\include - -DGDAL_LIBRARY=%LODEPSDIR%\gdal\lib\gdal_i.lib - -DGDAL_INCLUDE_DIR=%LODEPSDIR%\gdal\include - -DGEOS_LIBRARY=%LODEPSDIR%\geos\lib\geos.lib - -DGEOS_INCLUDE_DIR=%LODEPSDIR%\geos\include - -DPROJ_LIBRARY=%LODEPSDIR%\proj\lib\proj.lib - -DPROJ_INCLUDE_DIR=%LODEPSDIR%\proj\include - -DSPARSEHASH_INCLUDE_DIR=%LODEPSDIR%\sparsehash\include - -DGETOPT_LIBRARY=%LODEPSDIR%\wingetopt\lib\wingetopt.lib - -DGETOPT_INCLUDE_DIR=%LODEPSDIR%\wingetopt\include + -DBZIP2_LIBRARY_RELEASE=%LIBBZIP2% + -DCMAKE_PREFIX_PATH=%LODEPSDIR%\zlib;%LODEPSDIR%\expat;%LODEPSDIR%\bzip2;%LODEPSDIR%\geos;%LODEPSDIR%\gdal;%LODEPSDIR%\proj;%LODEPSDIR%\sparsehash;%LODEPSDIR%\wingetopt + .. - msbuild libosmium.sln /p:Configuration=%config% /toolsversion:14.0 /p:Platform=x64 /p:PlatformToolset=v140 #- cmake .. -LA -G "NMake Makefiles" # -DOsmium_DEBUG=TRUE diff --git a/third_party/libosmium/cmake/FindOsmium.cmake b/third_party/libosmium/cmake/FindOsmium.cmake index bb140718b..fba8ffb92 100644 --- a/third_party/libosmium/cmake/FindOsmium.cmake +++ b/third_party/libosmium/cmake/FindOsmium.cmake @@ -19,7 +19,7 @@ # Then add the following in your CMakeLists.txt: # # find_package(Osmium REQUIRED COMPONENTS ) -# include_directories(${OSMIUM_INCLUDE_DIRS}) +# include_directories(SYSTEM ${OSMIUM_INCLUDE_DIRS}) # # For the substitute a space separated list of one or more of the # following components: @@ -56,31 +56,13 @@ find_path(OSMIUM_INCLUDE_DIR osmium/osm.hpp PATH_SUFFIXES include PATHS ../libosmium - ../../libosmium - libosmium ~/Library/Frameworks /Library/Frameworks - /usr/local - /usr/ /opt/local # DarwinPorts /opt ) -# Handle the QUIETLY and REQUIRED arguments and set OSMIUM_FOUND to TRUE if -# all listed variables are TRUE. -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(OSMIUM REQUIRED_VARS OSMIUM_INCLUDE_DIR) - -# Copy the results to the output variables. -if(OSMIUM_FOUND) - set(OSMIUM_INCLUDE_DIRS ${OSMIUM_INCLUDE_DIR}) -else() - set(OSMIUM_INCLUDE_DIRS "") -endif() - -if(Osmium_FIND_REQUIRED AND NOT OSMIUM_FOUND) - message(FATAL_ERROR "Can not find libosmium headers, please install them or configure the paths") -endif() +set(OSMIUM_INCLUDE_DIRS "${OSMIUM_INCLUDE_DIR}") #---------------------------------------------------------------------- # @@ -113,6 +95,7 @@ if(Osmium_USE_PBF) find_package(ZLIB) find_package(Threads) + list(APPEND OSMIUM_EXTRA_FIND_VARS ZLIB_FOUND Threads_FOUND) if(ZLIB_FOUND AND Threads_FOUND) list(APPEND OSMIUM_PBF_LIBRARIES ${ZLIB_LIBRARIES} @@ -125,7 +108,6 @@ if(Osmium_USE_PBF) ${ZLIB_INCLUDE_DIR} ) else() - set(_missing_libraries 1) message(WARNING "Osmium: Can not find some libraries for PBF input/output, please install them or configure the paths.") endif() endif() @@ -138,6 +120,7 @@ if(Osmium_USE_XML) find_package(ZLIB) find_package(Threads) + list(APPEND OSMIUM_EXTRA_FIND_VARS EXPAT_FOUND BZIP2_FOUND ZLIB_FOUND Threads_FOUND) if(EXPAT_FOUND AND BZIP2_FOUND AND ZLIB_FOUND AND Threads_FOUND) list(APPEND OSMIUM_XML_LIBRARIES ${EXPAT_LIBRARIES} @@ -151,7 +134,6 @@ if(Osmium_USE_XML) ${ZLIB_INCLUDE_DIR} ) else() - set(_missing_libraries 1) message(WARNING "Osmium: Can not find some libraries for XML input/output, please install them or configure the paths.") endif() endif() @@ -172,12 +154,12 @@ if(Osmium_USE_GEOS) find_path(GEOS_INCLUDE_DIR geos/geom.h) find_library(GEOS_LIBRARY NAMES geos) + list(APPEND OSMIUM_EXTRA_FIND_VARS GEOS_INCLUDE_DIR GEOS_LIBRARY) if(GEOS_INCLUDE_DIR AND GEOS_LIBRARY) SET(GEOS_FOUND 1) list(APPEND OSMIUM_LIBRARIES ${GEOS_LIBRARY}) list(APPEND OSMIUM_INCLUDE_DIRS ${GEOS_INCLUDE_DIR}) else() - set(_missing_libraries 1) message(WARNING "Osmium: GEOS library is required but not found, please install it or configure the paths.") endif() endif() @@ -187,11 +169,11 @@ endif() if(Osmium_USE_GDAL) find_package(GDAL) + list(APPEND OSMIUM_EXTRA_FIND_VARS GDAL_FOUND) if(GDAL_FOUND) list(APPEND OSMIUM_LIBRARIES ${GDAL_LIBRARIES}) list(APPEND OSMIUM_INCLUDE_DIRS ${GDAL_INCLUDE_DIRS}) else() - set(_missing_libraries 1) message(WARNING "Osmium: GDAL library is required but not found, please install it or configure the paths.") endif() endif() @@ -202,12 +184,12 @@ if(Osmium_USE_PROJ) find_path(PROJ_INCLUDE_DIR proj_api.h) find_library(PROJ_LIBRARY NAMES proj) + list(APPEND OSMIUM_EXTRA_FIND_VARS PROJ_INCLUDE_DIR PROJ_LIBRARY) if(PROJ_INCLUDE_DIR AND PROJ_LIBRARY) set(PROJ_FOUND 1) list(APPEND OSMIUM_LIBRARIES ${PROJ_LIBRARY}) list(APPEND OSMIUM_INCLUDE_DIRS ${PROJ_INCLUDE_DIR}) else() - set(_missing_libraries 1) message(WARNING "Osmium: PROJ.4 library is required but not found, please install it or configure the paths.") endif() endif() @@ -217,21 +199,19 @@ endif() if(Osmium_USE_SPARSEHASH) find_path(SPARSEHASH_INCLUDE_DIR google/sparsetable) + list(APPEND OSMIUM_EXTRA_FIND_VARS SPARSEHASH_INCLUDE_DIR) if(SPARSEHASH_INCLUDE_DIR) # Find size of sparsetable::size_type. This does not work on older # CMake versions because they can do this check only in C, not in C++. - include(CheckTypeSize) - set(CMAKE_REQUIRED_INCLUDES ${SPARSEHASH_INCLUDE_DIR}) - set(CMAKE_EXTRA_INCLUDE_FILES "google/sparsetable") - check_type_size("google::sparsetable::size_type" SPARSETABLE_SIZE_TYPE LANGUAGE CXX) - set(CMAKE_EXTRA_INCLUDE_FILES) - set(CMAKE_REQUIRED_INCLUDES) - - # Falling back to checking size_t if google::sparsetable::size_type - # could not be checked. - if(SPARSETABLE_SIZE_TYPE STREQUAL "") - check_type_size("void*" VOID_PTR_SIZE) - set(SPARSETABLE_SIZE_TYPE ${VOID_PTR_SIZE}) + if (NOT CMAKE_VERSION VERSION_LESS 3.0) + include(CheckTypeSize) + set(CMAKE_REQUIRED_INCLUDES ${SPARSEHASH_INCLUDE_DIR}) + set(CMAKE_EXTRA_INCLUDE_FILES "google/sparsetable") + check_type_size("google::sparsetable::size_type" SPARSETABLE_SIZE_TYPE LANGUAGE CXX) + set(CMAKE_EXTRA_INCLUDE_FILES) + set(CMAKE_REQUIRED_INCLUDES) + else() + set(SPARSETABLE_SIZE_TYPE ${CMAKE_SIZEOF_VOID_P}) endif() # Sparsetable::size_type must be at least 8 bytes (64bit), otherwise @@ -244,7 +224,6 @@ if(Osmium_USE_SPARSEHASH) message(WARNING "Osmium: Disabled Google SparseHash library on 32bit system (size_type=${SPARSETABLE_SIZE_TYPE}).") endif() else() - set(_missing_libraries 1) message(WARNING "Osmium: Google SparseHash library is required but not found, please install it or configure the paths.") endif() endif() @@ -274,9 +253,14 @@ endif() # Check that all required libraries are available # #---------------------------------------------------------------------- -if(Osmium_FIND_REQUIRED AND _missing_libraries) - message(FATAL_ERROR "Required library or libraries missing. Aborting.") +if (OSMIUM_EXTRA_FIND_VARS) + list(REMOVE_DUPLICATES OSMIUM_EXTRA_FIND_VARS) endif() +# Handle the QUIETLY and REQUIRED arguments and set OSMIUM_FOUND to TRUE if +# all listed variables are TRUE. +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Osmium REQUIRED_VARS OSMIUM_INCLUDE_DIR ${OSMIUM_EXTRA_FIND_VARS}) +unset(OSMIUM_EXTRA_FIND_VARS) #---------------------------------------------------------------------- # diff --git a/third_party/libosmium/cmake/iwyu.sh b/third_party/libosmium/cmake/iwyu.sh index f7d8a15e8..ceea106c3 100755 --- a/third_party/libosmium/cmake/iwyu.sh +++ b/third_party/libosmium/cmake/iwyu.sh @@ -16,7 +16,7 @@ echo "INCLUDE WHAT YOU USE REPORT:" >$log allok=yes -for file in `find include/osmium -name \*.hpp`; do +for file in `find include/osmium -name \*.hpp | sort`; do mkdir -p `dirname build/check_reports/$file` ifile="build/check_reports/${file%.hpp}.iwyu" $cmdline $file >$ifile 2>&1 diff --git a/third_party/libosmium/examples/CMakeLists.txt b/third_party/libosmium/examples/CMakeLists.txt index c9f59603d..a04a843f2 100644 --- a/third_party/libosmium/examples/CMakeLists.txt +++ b/third_party/libosmium/examples/CMakeLists.txt @@ -14,12 +14,10 @@ set(EXAMPLES count create_node_cache debug + filter_discussions index read serdump - toogr - toogr2 - toogr2_exp use_node_cache CACHE STRING "Example programs" ) @@ -30,7 +28,7 @@ set(EXAMPLES # Examples depending on wingetopt # #----------------------------------------------------------------------------- -set(GETOPT_EXAMPLES area_test convert serdump toogr toogr2 toogr2_exp) +set(GETOPT_EXAMPLES area_test convert serdump) if(NOT GETOPT_MISSING) foreach(example ${GETOPT_EXAMPLES}) list(APPEND EXAMPLE_LIBS_${example} ${GETOPT_LIBRARY}) @@ -74,27 +72,6 @@ else() endif() -#----------------------------------------------------------------------------- -# -# Examples depending on GDAL/PROJ.4/SparseHash -# -#----------------------------------------------------------------------------- -set(OGR_EXAMPLES toogr toogr2 toogr2_exp) - -if(GDAL_FOUND AND PROJ_FOUND AND SPARSEHASH_FOUND) - foreach(example ${OGR_EXAMPLES}) - list(APPEND EXAMPLE_LIBS_${example} ${GDAL_LIBRARIES}) - list(APPEND EXAMPLE_LIBS_${example} ${PROJ_LIBRARIES}) - endforeach() -else() - message(STATUS "Configuring examples - Skipping examples because GDAL and/or Proj.4 and/or SparseHash not found:") - foreach(example ${OGR_EXAMPLES}) - message(STATUS " - osmium_${example}") - list(REMOVE_ITEM EXAMPLES ${example}) - endforeach() -endif() - - #----------------------------------------------------------------------------- # # Configure examples diff --git a/third_party/libosmium/examples/osmium_filter_discussions.cpp b/third_party/libosmium/examples/osmium_filter_discussions.cpp new file mode 100644 index 000000000..bba25b752 --- /dev/null +++ b/third_party/libosmium/examples/osmium_filter_discussions.cpp @@ -0,0 +1,72 @@ +/* + + Read OSM changesets with discussions from a changeset dump like the one + you get from http://planet.osm.org/planet/discussions-latest.osm.bz2 + and write out only those changesets which have discussions (ie comments). + + The code in this example file is released into the Public Domain. + +*/ + +#include // for std::copy_if +#include // for std::cout, std::cerr + +// we want to read OSM files in XML format +// (other formats don't support full changesets, so only XML is needed here) +#include +#include + +// we want to write OSM files in XML format +#include +#include + +// we want to support any compressioon (.gz2 and .bz2) +#include + +int main(int argc, char* argv[]) { + if (argc != 3) { + std::cout << "Usage: " << argv[0] << " INFILE OUTFILE\n"; + exit(1); + } + + // The input file, deduce file format from file suffix + osmium::io::File infile(argv[1]); + + // The output file, force class XML OSM file format + osmium::io::File outfile(argv[2], "osm"); + + // Initialize Reader for the input file. + // Read only changesets (will ignore nodes, ways, and + // relations if there are any). + osmium::io::Reader reader(infile, osmium::osm_entity_bits::changeset); + + // Get the header from the input file + osmium::io::Header header = reader.header(); + + // Initialize writer for the output file. Use the header from the input + // file for the output file. This will copy over some header information. + // The last parameter will tell the writer that it is allowed to overwrite + // an existing file. Without it, it will refuse to do so. + osmium::io::Writer writer(outfile, header, osmium::io::overwrite::allow); + + // Create range of input iterators that will iterator over all changesets + // delivered from input file through the "reader". + auto input_range = osmium::io::make_input_iterator_range(reader); + + // Create an output iterator writing through the "writer" object to the + // output file. + auto output_iterator = osmium::io::make_output_iterator(writer); + + // Copy all changesets from input to output that have at least one comment. + std::copy_if(input_range.begin(), input_range.end(), output_iterator, [](const osmium::Changeset& changeset) { + return changeset.num_comments() > 0; + }); + + // Explicitly close the writer and reader. Will throw an exception if + // there is a problem. If you wait for the destructor to close the writer + // and reader, you will not notice the problem, because destructors must + // not throw. + writer.close(); + reader.close(); +} + diff --git a/third_party/libosmium/examples/osmium_toogr.cpp b/third_party/libosmium/examples/osmium_toogr.cpp deleted file mode 100644 index 7c5a965c5..000000000 --- a/third_party/libosmium/examples/osmium_toogr.cpp +++ /dev/null @@ -1,244 +0,0 @@ -/* - - This is an example tool that converts OSM data to some output format - like Spatialite or Shapefiles using the OGR library. - - The code in this example file is released into the Public Domain. - -*/ - -#include -#include - -#include -#include -#include - -#include -#include -#include - -typedef osmium::index::map::Dummy index_neg_type; -typedef osmium::index::map::Map index_pos_type; - -typedef osmium::handler::NodeLocationsForWays location_handler_type; - -class MyOGRHandler : public osmium::handler::Handler { - - OGRDataSource* m_data_source; - OGRLayer* m_layer_point; - OGRLayer* m_layer_linestring; - - osmium::geom::OGRFactory<> m_factory; - -public: - - MyOGRHandler(const std::string& driver_name, const std::string& filename) { - - OGRRegisterAll(); - - OGRSFDriver* driver = OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(driver_name.c_str()); - if (!driver) { - std::cerr << driver_name << " driver not available.\n"; - exit(1); - } - - CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "FALSE"); - const char* options[] = { "SPATIALITE=TRUE", nullptr }; - m_data_source = driver->CreateDataSource(filename.c_str(), const_cast(options)); - if (!m_data_source) { - std::cerr << "Creation of output file failed.\n"; - exit(1); - } - - OGRSpatialReference sparef; - sparef.SetWellKnownGeogCS("WGS84"); - m_layer_point = m_data_source->CreateLayer("postboxes", &sparef, wkbPoint, nullptr); - if (!m_layer_point) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_point_field_id("id", OFTReal); - layer_point_field_id.SetWidth(10); - - if (m_layer_point->CreateField(&layer_point_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_point_field_operator("operator", OFTString); - layer_point_field_operator.SetWidth(30); - - if (m_layer_point->CreateField(&layer_point_field_operator) != OGRERR_NONE) { - std::cerr << "Creating operator field failed.\n"; - exit(1); - } - - /* Transactions might make things faster, then again they might not. - Feel free to experiment and benchmark and report back. */ - m_layer_point->StartTransaction(); - - m_layer_linestring = m_data_source->CreateLayer("roads", &sparef, wkbLineString, nullptr); - if (!m_layer_linestring) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_linestring_field_id("id", OFTReal); - layer_linestring_field_id.SetWidth(10); - - if (m_layer_linestring->CreateField(&layer_linestring_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_linestring_field_type("type", OFTString); - layer_linestring_field_type.SetWidth(30); - - if (m_layer_linestring->CreateField(&layer_linestring_field_type) != OGRERR_NONE) { - std::cerr << "Creating type field failed.\n"; - exit(1); - } - - m_layer_linestring->StartTransaction(); - } - - ~MyOGRHandler() { - m_layer_linestring->CommitTransaction(); - m_layer_point->CommitTransaction(); - OGRDataSource::DestroyDataSource(m_data_source); - OGRCleanupAll(); - } - - void node(const osmium::Node& node) { - const char* amenity = node.tags().get_value_by_key("amenity"); - if (amenity && !strcmp(amenity, "post_box")) { - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_point->GetLayerDefn()); - std::unique_ptr ogr_point = m_factory.create_point(node); - feature->SetGeometry(ogr_point.get()); - feature->SetField("id", static_cast(node.id())); - feature->SetField("operator", node.tags().get_value_by_key("operator")); - - if (m_layer_point->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); - } - } - - void way(const osmium::Way& way) { - const char* highway = way.tags().get_value_by_key("highway"); - if (highway) { - try { - std::unique_ptr ogr_linestring = m_factory.create_linestring(way); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_linestring->GetLayerDefn()); - feature->SetGeometry(ogr_linestring.get()); - feature->SetField("id", static_cast(way.id())); - feature->SetField("type", highway); - - if (m_layer_linestring->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); - } catch (osmium::geometry_error&) { - std::cerr << "Ignoring illegal geometry for way " << way.id() << ".\n"; - } - } - } - -}; - -/* ================================================== */ - -void print_help() { - std::cout << "osmium_toogr [OPTIONS] [INFILE [OUTFILE]]\n\n" \ - << "If INFILE is not given stdin is assumed.\n" \ - << "If OUTFILE is not given 'ogr_out' is used.\n" \ - << "\nOptions:\n" \ - << " -h, --help This help message\n" \ - << " -l, --location_store=TYPE Set location store\n" \ - << " -f, --format=FORMAT Output OGR format (Default: 'SQLite')\n" \ - << " -L See available location stores\n"; -} - -int main(int argc, char* argv[]) { - const auto& map_factory = osmium::index::MapFactory::instance(); - - static struct option long_options[] = { - {"help", no_argument, 0, 'h'}, - {"format", required_argument, 0, 'f'}, - {"location_store", required_argument, 0, 'l'}, - {"list_location_stores", no_argument, 0, 'L'}, - {0, 0, 0, 0} - }; - - std::string output_format { "SQLite" }; - std::string location_store { "sparse_mem_array" }; - - while (true) { - int c = getopt_long(argc, argv, "hf:l:L", long_options, 0); - if (c == -1) { - break; - } - - switch (c) { - case 'h': - print_help(); - exit(0); - case 'f': - output_format = optarg; - break; - case 'l': - location_store = optarg; - break; - case 'L': - std::cout << "Available map types:\n"; - for (const auto& map_type : map_factory.map_types()) { - std::cout << " " << map_type << "\n"; - } - exit(0); - default: - exit(1); - } - } - - std::string input_filename; - std::string output_filename("ogr_out"); - int remaining_args = argc - optind; - if (remaining_args > 2) { - std::cerr << "Usage: " << argv[0] << " [OPTIONS] [INFILE [OUTFILE]]" << std::endl; - exit(1); - } else if (remaining_args == 2) { - input_filename = argv[optind]; - output_filename = argv[optind+1]; - } else if (remaining_args == 1) { - input_filename = argv[optind]; - } else { - input_filename = "-"; - } - - osmium::io::Reader reader(input_filename); - - std::unique_ptr index_pos = map_factory.create_map(location_store); - index_neg_type index_neg; - location_handler_type location_handler(*index_pos, index_neg); - location_handler.ignore_errors(); - - MyOGRHandler ogr_handler(output_format, output_filename); - - osmium::apply(reader, location_handler, ogr_handler); - reader.close(); - - int locations_fd = open("locations.dump", O_WRONLY | O_CREAT, 0644); - if (locations_fd < 0) { - throw std::system_error(errno, std::system_category(), "Open failed"); - } - index_pos->dump_as_list(locations_fd); - close(locations_fd); -} - diff --git a/third_party/libosmium/examples/osmium_toogr2.cpp b/third_party/libosmium/examples/osmium_toogr2.cpp deleted file mode 100644 index e1b505688..000000000 --- a/third_party/libosmium/examples/osmium_toogr2.cpp +++ /dev/null @@ -1,331 +0,0 @@ -/* - - This is an example tool that converts OSM data to some output format - like Spatialite or Shapefiles using the OGR library. - - This version does multipolygon handling (in contrast to the osmium_toogr - example which doesn't). - - The code in this example file is released into the Public Domain. - -*/ - -#include -#include - -// usually you only need one or two of these -#include -#include - -#include -#include -#include -#include - -#include -//#include -#include -#include -#include - -typedef osmium::index::map::Dummy index_neg_type; - -typedef osmium::index::map::SparseMemArray index_pos_type; - -typedef osmium::handler::NodeLocationsForWays location_handler_type; - -class MyOGRHandler : public osmium::handler::Handler { - - OGRDataSource* m_data_source; - OGRLayer* m_layer_point; - OGRLayer* m_layer_linestring; - OGRLayer* m_layer_polygon; - - // Choose one of the following: - - // 1. Use WGS84, do not project coordinates. - //osmium::geom::OGRFactory<> m_factory {}; - - // 2. Project coordinates into "Web Mercator". - osmium::geom::OGRFactory m_factory; - - // 3. Use any projection that the proj library can handle. - // (Initialize projection with EPSG code or proj string). - // In addition you need to link with "-lproj" and add - // #include . - //osmium::geom::OGRFactory m_factory {osmium::geom::Projection(3857)}; - -public: - - MyOGRHandler(const std::string& driver_name, const std::string& filename) { - - OGRRegisterAll(); - - OGRSFDriver* driver = OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(driver_name.c_str()); - if (!driver) { - std::cerr << driver_name << " driver not available.\n"; - exit(1); - } - - CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "FALSE"); - const char* options[] = { "SPATIALITE=TRUE", nullptr }; - m_data_source = driver->CreateDataSource(filename.c_str(), const_cast(options)); - if (!m_data_source) { - std::cerr << "Creation of output file failed.\n"; - exit(1); - } - - OGRSpatialReference sparef; - sparef.importFromProj4(m_factory.proj_string().c_str()); - - m_layer_point = m_data_source->CreateLayer("postboxes", &sparef, wkbPoint, nullptr); - if (!m_layer_point) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_point_field_id("id", OFTReal); - layer_point_field_id.SetWidth(10); - - if (m_layer_point->CreateField(&layer_point_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_point_field_operator("operator", OFTString); - layer_point_field_operator.SetWidth(30); - - if (m_layer_point->CreateField(&layer_point_field_operator) != OGRERR_NONE) { - std::cerr << "Creating operator field failed.\n"; - exit(1); - } - - /* Transactions might make things faster, then again they might not. - Feel free to experiment and benchmark and report back. */ - m_layer_point->StartTransaction(); - - m_layer_linestring = m_data_source->CreateLayer("roads", &sparef, wkbLineString, nullptr); - if (!m_layer_linestring) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_linestring_field_id("id", OFTReal); - layer_linestring_field_id.SetWidth(10); - - if (m_layer_linestring->CreateField(&layer_linestring_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_linestring_field_type("type", OFTString); - layer_linestring_field_type.SetWidth(30); - - if (m_layer_linestring->CreateField(&layer_linestring_field_type) != OGRERR_NONE) { - std::cerr << "Creating type field failed.\n"; - exit(1); - } - - m_layer_linestring->StartTransaction(); - - m_layer_polygon = m_data_source->CreateLayer("buildings", &sparef, wkbMultiPolygon, nullptr); - if (!m_layer_polygon) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_polygon_field_id("id", OFTInteger); - layer_polygon_field_id.SetWidth(10); - - if (m_layer_polygon->CreateField(&layer_polygon_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_polygon_field_type("type", OFTString); - layer_polygon_field_type.SetWidth(30); - - if (m_layer_polygon->CreateField(&layer_polygon_field_type) != OGRERR_NONE) { - std::cerr << "Creating type field failed.\n"; - exit(1); - } - - m_layer_polygon->StartTransaction(); - } - - ~MyOGRHandler() { - m_layer_polygon->CommitTransaction(); - m_layer_linestring->CommitTransaction(); - m_layer_point->CommitTransaction(); - OGRDataSource::DestroyDataSource(m_data_source); - OGRCleanupAll(); - } - - void node(const osmium::Node& node) { - const char* amenity = node.tags()["amenity"]; - if (amenity && !strcmp(amenity, "post_box")) { - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_point->GetLayerDefn()); - std::unique_ptr ogr_point = m_factory.create_point(node); - feature->SetGeometry(ogr_point.get()); - feature->SetField("id", static_cast(node.id())); - feature->SetField("operator", node.tags()["operator"]); - - if (m_layer_point->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); - } - } - - void way(const osmium::Way& way) { - const char* highway = way.tags()["highway"]; - if (highway) { - try { - std::unique_ptr ogr_linestring = m_factory.create_linestring(way); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_linestring->GetLayerDefn()); - feature->SetGeometry(ogr_linestring.get()); - feature->SetField("id", static_cast(way.id())); - feature->SetField("type", highway); - - if (m_layer_linestring->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); - } catch (osmium::geometry_error&) { - std::cerr << "Ignoring illegal geometry for way " << way.id() << ".\n"; - } - } - } - - void area(const osmium::Area& area) { - const char* building = area.tags()["building"]; - if (building) { - try { - std::unique_ptr ogr_polygon = m_factory.create_multipolygon(area); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_polygon->GetLayerDefn()); - feature->SetGeometry(ogr_polygon.get()); - feature->SetField("id", static_cast(area.id())); - feature->SetField("type", building); - - std::string type = ""; - if (area.from_way()) { - type += "w"; - } else { - type += "r"; - } - feature->SetField("type", type.c_str()); - - if (m_layer_polygon->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); - } catch (osmium::geometry_error&) { - std::cerr << "Ignoring illegal geometry for area " << area.id() << " created from " << (area.from_way() ? "way" : "relation") << " with id=" << area.orig_id() << ".\n"; - } - } - } - -}; - -/* ================================================== */ - -void print_help() { - std::cout << "osmium_toogr [OPTIONS] [INFILE [OUTFILE]]\n\n" \ - << "If INFILE is not given stdin is assumed.\n" \ - << "If OUTFILE is not given 'ogr_out' is used.\n" \ - << "\nOptions:\n" \ - << " -h, --help This help message\n" \ - << " -d, --debug Enable debug output\n" \ - << " -f, --format=FORMAT Output OGR format (Default: 'SQLite')\n"; -} - -int main(int argc, char* argv[]) { - static struct option long_options[] = { - {"help", no_argument, 0, 'h'}, - {"debug", no_argument, 0, 'd'}, - {"format", required_argument, 0, 'f'}, - {0, 0, 0, 0} - }; - - std::string output_format("SQLite"); - bool debug = false; - - while (true) { - int c = getopt_long(argc, argv, "hdf:", long_options, 0); - if (c == -1) { - break; - } - - switch (c) { - case 'h': - print_help(); - exit(0); - case 'd': - debug = true; - break; - case 'f': - output_format = optarg; - break; - default: - exit(1); - } - } - - std::string input_filename; - std::string output_filename("ogr_out"); - int remaining_args = argc - optind; - if (remaining_args > 2) { - std::cerr << "Usage: " << argv[0] << " [OPTIONS] [INFILE [OUTFILE]]" << std::endl; - exit(1); - } else if (remaining_args == 2) { - input_filename = argv[optind]; - output_filename = argv[optind+1]; - } else if (remaining_args == 1) { - input_filename = argv[optind]; - } else { - input_filename = "-"; - } - - osmium::area::Assembler::config_type assembler_config; - assembler_config.enable_debug_output(debug); - osmium::area::MultipolygonCollector collector(assembler_config); - - std::cerr << "Pass 1...\n"; - osmium::io::Reader reader1(input_filename); - collector.read_relations(reader1); - reader1.close(); - std::cerr << "Pass 1 done\n"; - - index_pos_type index_pos; - index_neg_type index_neg; - location_handler_type location_handler(index_pos, index_neg); - location_handler.ignore_errors(); - - MyOGRHandler ogr_handler(output_format, output_filename); - - std::cerr << "Pass 2...\n"; - osmium::io::Reader reader2(input_filename); - - osmium::apply(reader2, location_handler, ogr_handler, collector.handler([&ogr_handler](const osmium::memory::Buffer& area_buffer) { - osmium::apply(area_buffer, ogr_handler); - })); - - reader2.close(); - std::cerr << "Pass 2 done\n"; - - std::vector incomplete_relations = collector.get_incomplete_relations(); - if (!incomplete_relations.empty()) { - std::cerr << "Warning! Some member ways missing for these multipolygon relations:"; - for (const auto* relation : incomplete_relations) { - std::cerr << " " << relation->id(); - } - std::cerr << "\n"; - } -} - diff --git a/third_party/libosmium/examples/osmium_toogr2_exp.cpp b/third_party/libosmium/examples/osmium_toogr2_exp.cpp deleted file mode 100644 index db8d5cf4e..000000000 --- a/third_party/libosmium/examples/osmium_toogr2_exp.cpp +++ /dev/null @@ -1,305 +0,0 @@ -/* - - This is an example tool that converts OSM data to some output format - like Spatialite or Shapefiles using the OGR library. - - This version does multipolygon handling (in contrast to the osmium_toogr - example which doesn't). - - This version (..._exp) uses a new experimental unsupported interface. - - The code in this example file is released into the Public Domain. - -*/ - -#include -#include - -#include - -#include - -#include -//#include -#include -#include -#include -#include - -typedef osmium::index::map::SparseMemArray index_type; -typedef osmium::handler::NodeLocationsForWays location_handler_type; - -class MyOGRHandler : public osmium::handler::Handler { - - OGRDataSource* m_data_source; - OGRLayer* m_layer_point; - OGRLayer* m_layer_linestring; - OGRLayer* m_layer_polygon; - - // Choose one of the following: - - // 1. Use WGS84, do not project coordinates. - //osmium::geom::OGRFactory<> m_factory {}; - - // 2. Project coordinates into "Web Mercator". - osmium::geom::OGRFactory m_factory; - - // 3. Use any projection that the proj library can handle. - // (Initialize projection with EPSG code or proj string). - // In addition you need to link with "-lproj" and add - // #include . - //osmium::geom::OGRFactory m_factory {osmium::geom::Projection(3857)}; - -public: - - MyOGRHandler(const std::string& driver_name, const std::string& filename) { - - OGRRegisterAll(); - - OGRSFDriver* driver = OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(driver_name.c_str()); - if (!driver) { - std::cerr << driver_name << " driver not available.\n"; - exit(1); - } - - CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "FALSE"); - const char* options[] = { "SPATIALITE=TRUE", nullptr }; - m_data_source = driver->CreateDataSource(filename.c_str(), const_cast(options)); - if (!m_data_source) { - std::cerr << "Creation of output file failed.\n"; - exit(1); - } - - OGRSpatialReference sparef; - sparef.importFromProj4(m_factory.proj_string().c_str()); - - m_layer_point = m_data_source->CreateLayer("postboxes", &sparef, wkbPoint, nullptr); - if (!m_layer_point) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_point_field_id("id", OFTReal); - layer_point_field_id.SetWidth(10); - - if (m_layer_point->CreateField(&layer_point_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_point_field_operator("operator", OFTString); - layer_point_field_operator.SetWidth(30); - - if (m_layer_point->CreateField(&layer_point_field_operator) != OGRERR_NONE) { - std::cerr << "Creating operator field failed.\n"; - exit(1); - } - - /* Transactions might make things faster, then again they might not. - Feel free to experiment and benchmark and report back. */ - m_layer_point->StartTransaction(); - - m_layer_linestring = m_data_source->CreateLayer("roads", &sparef, wkbLineString, nullptr); - if (!m_layer_linestring) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_linestring_field_id("id", OFTReal); - layer_linestring_field_id.SetWidth(10); - - if (m_layer_linestring->CreateField(&layer_linestring_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_linestring_field_type("type", OFTString); - layer_linestring_field_type.SetWidth(30); - - if (m_layer_linestring->CreateField(&layer_linestring_field_type) != OGRERR_NONE) { - std::cerr << "Creating type field failed.\n"; - exit(1); - } - - m_layer_linestring->StartTransaction(); - - m_layer_polygon = m_data_source->CreateLayer("buildings", &sparef, wkbMultiPolygon, nullptr); - if (!m_layer_polygon) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_polygon_field_id("id", OFTInteger); - layer_polygon_field_id.SetWidth(10); - - if (m_layer_polygon->CreateField(&layer_polygon_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_polygon_field_type("type", OFTString); - layer_polygon_field_type.SetWidth(30); - - if (m_layer_polygon->CreateField(&layer_polygon_field_type) != OGRERR_NONE) { - std::cerr << "Creating type field failed.\n"; - exit(1); - } - - m_layer_polygon->StartTransaction(); - } - - ~MyOGRHandler() { - m_layer_polygon->CommitTransaction(); - m_layer_linestring->CommitTransaction(); - m_layer_point->CommitTransaction(); - OGRDataSource::DestroyDataSource(m_data_source); - OGRCleanupAll(); - } - - void node(const osmium::Node& node) { - const char* amenity = node.tags()["amenity"]; - if (amenity && !strcmp(amenity, "post_box")) { - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_point->GetLayerDefn()); - std::unique_ptr ogr_point = m_factory.create_point(node); - feature->SetGeometry(ogr_point.get()); - feature->SetField("id", static_cast(node.id())); - feature->SetField("operator", node.tags()["operator"]); - - if (m_layer_point->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); - } - } - - void way(const osmium::Way& way) { - const char* highway = way.tags()["highway"]; - if (highway) { - try { - std::unique_ptr ogr_linestring = m_factory.create_linestring(way); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_linestring->GetLayerDefn()); - feature->SetGeometry(ogr_linestring.get()); - feature->SetField("id", static_cast(way.id())); - feature->SetField("type", highway); - - if (m_layer_linestring->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); - } catch (osmium::geometry_error&) { - std::cerr << "Ignoring illegal geometry for way " << way.id() << ".\n"; - } - } - } - - void area(const osmium::Area& area) { - const char* building = area.tags()["building"]; - if (building) { - try { - std::unique_ptr ogr_polygon = m_factory.create_multipolygon(area); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_polygon->GetLayerDefn()); - feature->SetGeometry(ogr_polygon.get()); - feature->SetField("id", static_cast(area.id())); - feature->SetField("type", building); - - std::string type = ""; - if (area.from_way()) { - type += "w"; - } else { - type += "r"; - } - feature->SetField("type", type.c_str()); - - if (m_layer_polygon->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); - } catch (osmium::geometry_error&) { - std::cerr << "Ignoring illegal geometry for area " << area.id() << " created from " << (area.from_way() ? "way" : "relation") << " with id=" << area.orig_id() << ".\n"; - } - } - } - -}; - -/* ================================================== */ - -void print_help() { - std::cout << "osmium_toogr [OPTIONS] [INFILE [OUTFILE]]\n\n" \ - << "If INFILE is not given stdin is assumed.\n" \ - << "If OUTFILE is not given 'ogr_out' is used.\n" \ - << "\nOptions:\n" \ - << " -h, --help This help message\n" \ - << " -f, --format=FORMAT Output OGR format (Default: 'SQLite')\n"; -} - -int main(int argc, char* argv[]) { - static struct option long_options[] = { - {"help", no_argument, 0, 'h'}, - {"format", required_argument, 0, 'f'}, - {0, 0, 0, 0} - }; - - std::string output_format("SQLite"); - - while (true) { - int c = getopt_long(argc, argv, "hf:", long_options, 0); - if (c == -1) { - break; - } - - switch (c) { - case 'h': - print_help(); - exit(0); - case 'f': - output_format = optarg; - break; - default: - exit(1); - } - } - - std::string input_filename; - std::string output_filename("ogr_out"); - int remaining_args = argc - optind; - if (remaining_args > 2) { - std::cerr << "Usage: " << argv[0] << " [OPTIONS] [INFILE [OUTFILE]]" << std::endl; - exit(1); - } else if (remaining_args == 2) { - input_filename = argv[optind]; - output_filename = argv[optind+1]; - } else if (remaining_args == 1) { - input_filename = argv[optind]; - } else { - input_filename = "-"; - } - - index_type index_pos; - location_handler_type location_handler(index_pos); - osmium::experimental::FlexReader exr(input_filename, location_handler, osmium::osm_entity_bits::object); - - MyOGRHandler ogr_handler(output_format, output_filename); - - while (auto buffer = exr.read()) { - osmium::apply(buffer, ogr_handler); - } - - exr.close(); - - std::vector incomplete_relations = exr.collector().get_incomplete_relations(); - if (!incomplete_relations.empty()) { - std::cerr << "Warning! Some member ways missing for these multipolygon relations:"; - for (const auto* relation : incomplete_relations) { - std::cerr << " " << relation->id(); - } - std::cerr << "\n"; - } -} - diff --git a/third_party/libosmium/include/gdalcpp.hpp b/third_party/libosmium/include/gdalcpp.hpp new file mode 100644 index 000000000..1502f2f0c --- /dev/null +++ b/third_party/libosmium/include/gdalcpp.hpp @@ -0,0 +1,406 @@ +#ifndef GDALCPP_HPP +#define GDALCPP_HPP + +/* + +C++11 wrapper classes for GDAL/OGR. + +Version 1.1.1 + +https://github.com/joto/gdalcpp + +Copyright 2015 Jochen Topf + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace gdalcpp { + +#if GDAL_VERSION_MAJOR >= 2 + using gdal_driver_type = GDALDriver; + using gdal_dataset_type = GDALDataset; +#else + using gdal_driver_type = OGRSFDriver; + using gdal_dataset_type = OGRDataSource; +#endif + + /** + * Exception thrown for all errors in this class. + */ + class gdal_error : public std::runtime_error { + + std::string m_driver; + std::string m_dataset; + std::string m_layer; + std::string m_field; + OGRErr m_error; + + public: + + gdal_error(const std::string& message, + OGRErr error, + const std::string& driver = "", + const std::string& dataset = "", + const std::string& layer = "", + const std::string& field = "") : + std::runtime_error(message), + m_driver(driver), + m_dataset(dataset), + m_layer(layer), + m_field(field), + m_error(error) { + } + + const std::string& driver() const { + return m_driver; + } + + const std::string& dataset() const { + return m_dataset; + } + + const std::string& layer() const { + return m_layer; + } + + const std::string& field() const { + return m_field; + } + + OGRErr error() const { + return m_error; + } + + }; // class gdal_error + + namespace detail { + + struct init_wrapper { + init_wrapper() { OGRRegisterAll(); } + ~init_wrapper() { OGRCleanupAll(); } + }; + + struct init_library { + init_library() { + static init_wrapper iw; + } + }; + + class Driver : private init_library { + + gdal_driver_type* m_driver; + + public: + + Driver(const std::string& driver_name) : + init_library(), +#if GDAL_VERSION_MAJOR >= 2 + m_driver(GetGDALDriverManager()->GetDriverByName(driver_name.c_str())) { +#else + m_driver(OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(driver_name.c_str())) { +#endif + if (!m_driver) { + throw gdal_error(std::string("unknown driver: '") + driver_name + "'", OGRERR_NONE, driver_name); + } + } + + gdal_driver_type& get() const { + return *m_driver; + } + + }; // struct Driver + + struct Options { + + std::vector m_options; + std::unique_ptr m_ptrs; + + Options(const std::vector& options) : + m_options(options), + m_ptrs(new const char*[options.size()+1]) { + std::transform(m_options.begin(), m_options.end(), m_ptrs.get(), [&](const std::string& s) { + return s.data(); + }); + m_ptrs[options.size()] = nullptr; + } + + char** get() const { + return const_cast(m_ptrs.get()); + } + + }; // struct Options + + } // namespace detail + + class SRS { + + OGRSpatialReference m_spatial_reference; + + public: + + SRS() : + m_spatial_reference() { + auto result = m_spatial_reference.SetWellKnownGeogCS("WGS84"); + if (result != OGRERR_NONE) { + throw gdal_error(std::string("can not initialize spatial reference system WGS84"), result); + } + } + + explicit SRS(int epsg) : + m_spatial_reference() { + auto result = m_spatial_reference.importFromEPSG(epsg); + if (result != OGRERR_NONE) { + throw gdal_error(std::string("can not initialize spatial reference system for EPSG:") + std::to_string(epsg), result); + } + } + + explicit SRS(const char* name) : + m_spatial_reference() { + auto result = m_spatial_reference.importFromProj4(name); + if (result != OGRERR_NONE) { + throw gdal_error(std::string("can not initialize spatial reference system '") + name + "'", result); + } + } + + explicit SRS(const std::string& name) : + m_spatial_reference() { + auto result = m_spatial_reference.importFromProj4(name.c_str()); + if (result != OGRERR_NONE) { + throw gdal_error(std::string("can not initialize spatial reference system '") + name + "'", result); + } + } + + explicit SRS(const OGRSpatialReference& spatial_reference) : + m_spatial_reference(spatial_reference) { + } + + OGRSpatialReference& get() { + return m_spatial_reference; + } + + const OGRSpatialReference& get() const { + return m_spatial_reference; + } + + }; // class SRS + + class Dataset { + + struct gdal_dataset_deleter { + + void operator()(gdal_dataset_type* ds) { +#if GDAL_VERSION_MAJOR >= 2 + GDALClose(ds); +#else + OGRDataSource::DestroyDataSource(ds); +#endif + } + + }; // struct gdal_dataset_deleter + + std::string m_driver_name; + std::string m_dataset_name; + detail::Options m_options; + SRS m_srs; + std::unique_ptr m_dataset; + + public: + + Dataset(const std::string& driver_name, const std::string& dataset_name, const SRS& srs = SRS{}, const std::vector& options = {}) : + m_driver_name(driver_name), + m_dataset_name(dataset_name), + m_options(options), + m_srs(srs), +#if GDAL_VERSION_MAJOR >= 2 + m_dataset(detail::Driver(driver_name).get().Create(dataset_name.c_str(), 0, 0, 0, GDT_Unknown, m_options.get())) { +#else + m_dataset(detail::Driver(driver_name).get().CreateDataSource(dataset_name.c_str(), m_options.get())) { +#endif + if (!m_dataset) { + throw gdal_error(std::string("failed to create dataset '") + dataset_name + "'", OGRERR_NONE, driver_name, dataset_name); + } + } + + const std::string& driver_name() const { + return m_driver_name; + } + + const std::string& dataset_name() const { + return m_dataset_name; + } + + gdal_dataset_type& get() const { + return *m_dataset; + } + + SRS& srs() { + return m_srs; + } + + void exec(const char* sql) { + auto result = m_dataset->ExecuteSQL(sql, nullptr, nullptr); + if (result) { + m_dataset->ReleaseResultSet(result); + } + } + + void exec(const std::string& sql) { + exec(sql.c_str()); + } + + + Dataset& start_transaction() { +#if GDAL_VERSION_MAJOR >= 2 + m_dataset->StartTransaction(); +#endif + return *this; + } + + Dataset& commit_transaction() { +#if GDAL_VERSION_MAJOR >= 2 + m_dataset->CommitTransaction(); +#endif + return *this; + } + + }; // class Dataset + + class Layer { + + detail::Options m_options; + Dataset& m_dataset; + OGRLayer* m_layer; + + public: + + Layer(Dataset& dataset, const std::string& layer_name, OGRwkbGeometryType type, const std::vector& options = {}) : + m_options(options), + m_dataset(dataset), + m_layer(dataset.get().CreateLayer(layer_name.c_str(), &dataset.srs().get(), type, m_options.get())) { + if (!m_layer) { + throw gdal_error(std::string("failed to create layer '") + layer_name + "'", OGRERR_NONE, + dataset.driver_name(), dataset.dataset_name(), layer_name); + } + } + + OGRLayer& get() { + return *m_layer; + } + + const OGRLayer& get() const { + return *m_layer; + } + + Dataset& dataset() const { + return m_dataset; + } + + const char* name() const { + return m_layer->GetName(); + } + + Layer& add_field(const std::string& field_name, OGRFieldType type, int width, int precision=0) { + OGRFieldDefn field(field_name.c_str(), type); + field.SetWidth(width); + field.SetPrecision(precision); + + if (m_layer->CreateField(&field) != OGRERR_NONE) { + throw gdal_error(std::string("failed to create field '") + field_name + "' in layer '" + name() + "'", OGRERR_NONE, + m_dataset.driver_name(), m_dataset.dataset_name(), name(), field_name); + } + + return *this; + } + + Layer& start_transaction() { + OGRErr result = m_layer->StartTransaction(); + if (result != OGRERR_NONE) { + throw gdal_error(std::string("starting transaction on layer '") + name() + "' failed", result, m_dataset.driver_name(), m_dataset.dataset_name(), name()); + } + return *this; + } + + Layer& commit_transaction() { + OGRErr result = m_layer->CommitTransaction(); + if (result != OGRERR_NONE) { + throw gdal_error(std::string("committing transaction on layer '") + name() + "' failed", result, m_dataset.driver_name(), m_dataset.dataset_name(), name()); + } + return *this; + } + + }; // class Layer + + class Feature { + + Layer& m_layer; + OGRFeature m_feature; + + public: + + Feature(Layer& layer, std::unique_ptr&& geometry) : + m_layer(layer), + m_feature(m_layer.get().GetLayerDefn()) { + OGRErr result = m_feature.SetGeometryDirectly(geometry.release()); + if (result != OGRERR_NONE) { + throw gdal_error(std::string("setting feature geometry in layer '") + m_layer.name() + "' failed", result, m_layer.dataset().driver_name(), m_layer.dataset().dataset_name()); + } + } + + void add_to_layer() { + OGRErr result = m_layer.get().CreateFeature(&m_feature); + if (result != OGRERR_NONE) { + throw gdal_error(std::string("creating feature in layer '") + m_layer.name() + "' failed", result, m_layer.dataset().driver_name(), m_layer.dataset().dataset_name()); + } + } + + template + Feature& set_field(int n, T&& arg) { + m_feature.SetField(n, std::forward(arg)); + return *this; + } + + template + Feature& set_field(const char* name, T&& arg) { + m_feature.SetField(name, std::forward(arg)); + return *this; + } + + }; // class Feature + +} // namespace gdalcpp + +#endif // GDALCPP_HPP diff --git a/third_party/libosmium/include/osmium/area/assembler.hpp b/third_party/libosmium/include/osmium/area/assembler.hpp index f17299109..87feea29e 100644 --- a/third_party/libosmium/include/osmium/area/assembler.hpp +++ b/third_party/libosmium/include/osmium/area/assembler.hpp @@ -34,9 +34,13 @@ DEALINGS IN THE SOFTWARE. */ #include +#include +#include #include #include #include +#include +#include #include #include diff --git a/third_party/libosmium/include/osmium/area/detail/node_ref_segment.hpp b/third_party/libosmium/include/osmium/area/detail/node_ref_segment.hpp index 43569a865..aab5b8fa3 100644 --- a/third_party/libosmium/include/osmium/area/detail/node_ref_segment.hpp +++ b/third_party/libosmium/include/osmium/area/detail/node_ref_segment.hpp @@ -113,7 +113,7 @@ namespace osmium { return m_second; } - bool to_left_of(const osmium::Location location) const { + bool to_left_of(const osmium::Location& location) const { // std::cerr << "segment " << first() << "--" << second() << " to_left_of(" << location << "\n"; if (first().location() == location || second().location() == location) { @@ -195,8 +195,8 @@ namespace osmium { } inline bool y_range_overlap(const NodeRefSegment& s1, const NodeRefSegment& s2) { - auto m1 = std::minmax(s1.first().location().y(), s1.second().location().y()); - auto m2 = std::minmax(s2.first().location().y(), s2.second().location().y()); + const std::pair m1 = std::minmax(s1.first().location().y(), s1.second().location().y()); + const std::pair m2 = std::minmax(s2.first().location().y(), s2.second().location().y()); if (m1.first > m2.second || m2.first > m1.second) { return false; } @@ -204,19 +204,25 @@ namespace osmium { } /** - * Calculate the intersection between to NodeRefSegments. The result is returned - * as a Location. Note that because the Location uses integers with limited - * precision internally, the result might be slightly different than the - * numerically correct location. + * Calculate the intersection between two NodeRefSegments. The + * result is returned as a Location. Note that because the Location + * uses integers with limited precision internally, the result + * might be slightly different than the numerically correct + * location. * - * If the segments touch in one of their endpoints, it doesn't count as an - * intersection. + * This function uses integer arithmentic as much as possible and + * will not work if the segments are longer than about half the + * planet. This shouldn't happen with real data, so it isn't a big + * problem. * - * If the segments intersect not in a single point but in multiple points, ie - * if they overlap, this is NOT detected. + * If the segments touch in one of their endpoints, it doesn't + * count as an intersection. * - * @returns Undefined osmium::Location if there is no intersection or a defined - * Location if the segments intersect. + * If the segments intersect not in a single point but in multiple + * points, ie if they overlap, this is NOT detected. + * + * @returns Undefined osmium::Location if there is no intersection + * or a defined Location if the segments intersect. */ inline osmium::Location calculate_intersection(const NodeRefSegment& s1, const NodeRefSegment& s2) { if (s1.first().location() == s2.first().location() || @@ -226,26 +232,32 @@ namespace osmium { return osmium::Location(); } - auto d = (static_cast(s2.second().y()) - static_cast(s2.first().y())) * - (static_cast(s1.second().x()) - static_cast(s1.first().x())) - - (static_cast(s2.second().x()) - static_cast(s2.first().x())) * - (static_cast(s1.second().y()) - static_cast(s1.first().y())); + int64_t s1ax = s1.first().x(); + int64_t s1ay = s1.first().y(); + int64_t s1bx = s1.second().x(); + int64_t s1by = s1.second().y(); + int64_t s2ax = s2.first().x(); + int64_t s2ay = s2.first().y(); + int64_t s2bx = s2.second().x(); + int64_t s2by = s2.second().y(); + + int64_t d = (s2by - s2ay) * (s1bx - s1ax) - + (s2bx - s2ax) * (s1by - s1ay); if (d != 0) { - double denom = ((s2.second().lat() - s2.first().lat())*(s1.second().lon() - s1.first().lon())) - - ((s2.second().lon() - s2.first().lon())*(s1.second().lat() - s1.first().lat())); + int64_t na = (s2bx - s2ax) * (s1ay - s2ay) - + (s2by - s2ay) * (s1ax - s2ax); - double nume_a = ((s2.second().lon() - s2.first().lon())*(s1.first().lat() - s2.first().lat())) - - ((s2.second().lat() - s2.first().lat())*(s1.first().lon() - s2.first().lon())); + int64_t nb = (s1bx - s1ax) * (s1ay - s2ay) - + (s1by - s1ay) * (s1ax - s2ax); - double nume_b = ((s1.second().lon() - s1.first().lon())*(s1.first().lat() - s2.first().lat())) - - ((s1.second().lat() - s1.first().lat())*(s1.first().lon() - s2.first().lon())); + if ((d > 0 && na >= 0 && na <= d && nb >= 0 && nb <= d) || + (d < 0 && na <= 0 && na >= d && nb <= 0 && nb >= d)) { + + double ua = double(na) / d; + int32_t ix = int32_t(s1ax + ua*(s1bx - s1ax)); + int32_t iy = int32_t(s1ay + ua*(s1by - s1ay)); - if ((denom > 0 && nume_a >= 0 && nume_a <= denom && nume_b >= 0 && nume_b <= denom) || - (denom < 0 && nume_a <= 0 && nume_a >= denom && nume_b <= 0 && nume_b >= denom)) { - double ua = nume_a / denom; - double ix = s1.first().lon() + ua*(s1.second().lon() - s1.first().lon()); - double iy = s1.first().lat() + ua*(s1.second().lat() - s1.first().lat()); return osmium::Location(ix, iy); } } diff --git a/third_party/libosmium/include/osmium/area/detail/proto_ring.hpp b/third_party/libosmium/include/osmium/area/detail/proto_ring.hpp index c0f545c98..162e2896d 100644 --- a/third_party/libosmium/include/osmium/area/detail/proto_ring.hpp +++ b/third_party/libosmium/include/osmium/area/detail/proto_ring.hpp @@ -34,12 +34,14 @@ DEALINGS IN THE SOFTWARE. */ #include -#include +#include +#include #include -#include +#include #include #include +#include #include #include @@ -148,7 +150,8 @@ namespace osmium { } void swap_segments(ProtoRing& other) { - std::swap(m_segments, other.m_segments); + using std::swap; + swap(m_segments, other.m_segments); } void add_inner_ring(ProtoRing* ring) { diff --git a/third_party/libosmium/include/osmium/area/detail/segment_list.hpp b/third_party/libosmium/include/osmium/area/detail/segment_list.hpp index ca6071ede..289ecf0c9 100644 --- a/third_party/libosmium/include/osmium/area/detail/segment_list.hpp +++ b/third_party/libosmium/include/osmium/area/detail/segment_list.hpp @@ -41,6 +41,8 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include +#include #include #include diff --git a/third_party/libosmium/include/osmium/area/multipolygon_collector.hpp b/third_party/libosmium/include/osmium/area/multipolygon_collector.hpp index bf2a4ce86..2881597b0 100644 --- a/third_party/libosmium/include/osmium/area/multipolygon_collector.hpp +++ b/third_party/libosmium/include/osmium/area/multipolygon_collector.hpp @@ -53,7 +53,7 @@ namespace osmium { namespace relations { class RelationMeta; - } + } // namespace relations /** * @brief Code related to the building of areas (multipolygons) from relations. @@ -71,7 +71,7 @@ namespace osmium { * * @tparam TAssembler Multipolygon Assembler class. */ - template + template class MultipolygonCollector : public osmium::relations::Collector, false, true, false> { typedef typename osmium::relations::Collector, false, true, false> collector_type; @@ -87,7 +87,8 @@ namespace osmium { void flush_output_buffer() { if (this->callback()) { osmium::memory::Buffer buffer(initial_output_buffer_size); - std::swap(buffer, m_output_buffer); + using std::swap; + swap(buffer, m_output_buffer); this->callback()(std::move(buffer)); } } @@ -176,28 +177,6 @@ namespace osmium { } catch (osmium::invalid_location&) { // XXX ignore } - - // clear member metas - for (const auto& member : relation.members()) { - if (member.ref() != 0) { - auto& mmv = this->member_meta(member.type()); - auto range = std::equal_range(mmv.begin(), mmv.end(), osmium::relations::MemberMeta(member.ref())); - assert(range.first != range.second); - - // if this is the last time this object was needed - // then mark it as removed - if (osmium::relations::count_not_removed(range.first, range.second) == 1) { - this->get_member(range.first->buffer_offset()).set_removed(true); - } - - for (auto it = range.first; it != range.second; ++it) { - if (!it->removed() && relation.id() == this->get_relation(it->relation_pos()).id()) { - it->remove(); - break; - } - } - } - } } void flush() { @@ -206,7 +185,10 @@ namespace osmium { osmium::memory::Buffer read() { osmium::memory::Buffer buffer(initial_output_buffer_size, osmium::memory::Buffer::auto_grow::yes); - std::swap(buffer, m_output_buffer); + + using std::swap; + swap(buffer, m_output_buffer); + return buffer; } diff --git a/third_party/libosmium/include/osmium/area/problem_reporter_exception.hpp b/third_party/libosmium/include/osmium/area/problem_reporter_exception.hpp index 5e743c6d8..7c9a5e393 100644 --- a/third_party/libosmium/include/osmium/area/problem_reporter_exception.hpp +++ b/third_party/libosmium/include/osmium/area/problem_reporter_exception.hpp @@ -54,7 +54,7 @@ namespace osmium { ProblemReporterStream(m_sstream) { } - virtual ~ProblemReporterException() = default; + ~ProblemReporterException() override = default; void report_duplicate_node(osmium::object_id_type node_id1, osmium::object_id_type node_id2, osmium::Location location) override { m_sstream.str(); diff --git a/third_party/libosmium/include/osmium/area/problem_reporter_ogr.hpp b/third_party/libosmium/include/osmium/area/problem_reporter_ogr.hpp index c437a3f57..68fae3bb8 100644 --- a/third_party/libosmium/include/osmium/area/problem_reporter_ogr.hpp +++ b/third_party/libosmium/include/osmium/area/problem_reporter_ogr.hpp @@ -42,34 +42,12 @@ DEALINGS IN THE SOFTWARE. * @attention If you include this file, you'll need to link with `libgdal`. */ -#ifdef _MSC_VER -# pragma warning(push) -# pragma warning(disable : 4458) -#else -# pragma GCC diagnostic push -# ifdef __clang__ -# pragma GCC diagnostic ignored "-Wdocumentation-unknown-command" -# endif -# pragma GCC diagnostic ignored "-Wfloat-equal" -# pragma GCC diagnostic ignored "-Wold-style-cast" -# pragma GCC diagnostic ignored "-Wpadded" -# pragma GCC diagnostic ignored "-Wredundant-decls" -# pragma GCC diagnostic ignored "-Wshadow" -#endif - -#include -#include - -#ifdef _MSC_VER -# pragma warning(pop) -#else -# pragma GCC diagnostic pop -#endif - #include -#include + +#include #include +#include #include #include #include @@ -86,24 +64,15 @@ namespace osmium { osmium::geom::OGRFactory<> m_ogr_factory; - OGRDataSource* m_data_source; - - OGRLayer* m_layer_perror; - OGRLayer* m_layer_lerror; + gdalcpp::Layer m_layer_perror; + gdalcpp::Layer m_layer_lerror; void write_point(const char* problem_type, osmium::object_id_type id1, osmium::object_id_type id2, osmium::Location location) { - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_perror->GetLayerDefn()); - std::unique_ptr ogr_point = m_ogr_factory.create_point(location); - feature->SetGeometry(ogr_point.get()); - feature->SetField("id1", static_cast(id1)); - feature->SetField("id2", static_cast(id2)); - feature->SetField("problem_type", problem_type); - - if (m_layer_perror->CreateFeature(feature) != OGRERR_NONE) { - std::runtime_error("Failed to create feature on layer 'perrors'"); - } - - OGRFeature::DestroyFeature(feature); + gdalcpp::Feature feature(m_layer_perror, m_ogr_factory.create_point(location)); + feature.set_field("id1", static_cast(id1)); + feature.set_field("id2", static_cast(id2)); + feature.set_field("problem_type", problem_type); + feature.add_to_layer(); } void write_line(const char* problem_type, osmium::object_id_type id1, osmium::object_id_type id2, osmium::Location loc1, osmium::Location loc2) { @@ -112,83 +81,30 @@ namespace osmium { std::unique_ptr ogr_linestring = std::unique_ptr(new OGRLineString()); ogr_linestring->addPoint(ogr_point1.get()); ogr_linestring->addPoint(ogr_point2.get()); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_lerror->GetLayerDefn()); - feature->SetGeometry(ogr_linestring.get()); - feature->SetField("id1", static_cast(id1)); - feature->SetField("id2", static_cast(id2)); - feature->SetField("problem_type", problem_type); - if (m_layer_lerror->CreateFeature(feature) != OGRERR_NONE) { - std::runtime_error("Failed to create feature on layer 'lerrors'"); - } - - OGRFeature::DestroyFeature(feature); + gdalcpp::Feature feature(m_layer_lerror, std::move(ogr_linestring)); + feature.set_field("id1", static_cast(id1)); + feature.set_field("id2", static_cast(id2)); + feature.set_field("problem_type", problem_type); + feature.add_to_layer(); } public: - explicit ProblemReporterOGR(OGRDataSource* data_source) : - m_data_source(data_source) { + explicit ProblemReporterOGR(gdalcpp::Dataset& dataset) : + m_layer_perror(dataset, "perrors", wkbPoint), + m_layer_lerror(dataset, "lerrors", wkbLineString) { - OGRSpatialReference sparef; - sparef.SetWellKnownGeogCS("WGS84"); + m_layer_perror.add_field("id1", OFTReal, 10); + m_layer_perror.add_field("id2", OFTReal, 10); + m_layer_perror.add_field("problem_type", OFTString, 30); - m_layer_perror = m_data_source->CreateLayer("perrors", &sparef, wkbPoint, nullptr); - if (!m_layer_perror) { - std::runtime_error("Layer creation failed for layer 'perrors'"); - } - - OGRFieldDefn layer_perror_field_id1("id1", OFTReal); - layer_perror_field_id1.SetWidth(10); - - if (m_layer_perror->CreateField(&layer_perror_field_id1) != OGRERR_NONE) { - std::runtime_error("Creating field 'id1' failed for layer 'perrors'"); - } - - OGRFieldDefn layer_perror_field_id2("id2", OFTReal); - layer_perror_field_id2.SetWidth(10); - - if (m_layer_perror->CreateField(&layer_perror_field_id2) != OGRERR_NONE) { - std::runtime_error("Creating field 'id2' failed for layer 'perrors'"); - } - - OGRFieldDefn layer_perror_field_problem_type("problem_type", OFTString); - layer_perror_field_problem_type.SetWidth(30); - - if (m_layer_perror->CreateField(&layer_perror_field_problem_type) != OGRERR_NONE) { - std::runtime_error("Creating field 'problem_type' failed for layer 'perrors'"); - } - - /**************/ - - m_layer_lerror = m_data_source->CreateLayer("lerrors", &sparef, wkbLineString, nullptr); - if (!m_layer_lerror) { - std::runtime_error("Layer creation failed for layer 'lerrors'"); - } - - OGRFieldDefn layer_lerror_field_id1("id1", OFTReal); - layer_lerror_field_id1.SetWidth(10); - - if (m_layer_lerror->CreateField(&layer_lerror_field_id1) != OGRERR_NONE) { - std::runtime_error("Creating field 'id1' failed for layer 'lerrors'"); - } - - OGRFieldDefn layer_lerror_field_id2("id2", OFTReal); - layer_lerror_field_id2.SetWidth(10); - - if (m_layer_lerror->CreateField(&layer_lerror_field_id2) != OGRERR_NONE) { - std::runtime_error("Creating field 'id2' failed for layer 'lerrors'"); - } - - OGRFieldDefn layer_lerror_field_problem_type("problem_type", OFTString); - layer_lerror_field_problem_type.SetWidth(30); - - if (m_layer_lerror->CreateField(&layer_lerror_field_problem_type) != OGRERR_NONE) { - std::runtime_error("Creating field 'problem_type' failed for layer 'lerrors'"); - } + m_layer_lerror.add_field("id1", OFTReal, 10); + m_layer_lerror.add_field("id2", OFTReal, 10); + m_layer_lerror.add_field("problem_type", OFTString, 30); } - virtual ~ProblemReporterOGR() = default; + ~ProblemReporterOGR() override = default; void report_duplicate_node(osmium::object_id_type node_id1, osmium::object_id_type node_id2, osmium::Location location) override { write_point("duplicate_node", node_id1, node_id2, location); diff --git a/third_party/libosmium/include/osmium/area/problem_reporter_stream.hpp b/third_party/libosmium/include/osmium/area/problem_reporter_stream.hpp index ddcb343e8..b6a004cdf 100644 --- a/third_party/libosmium/include/osmium/area/problem_reporter_stream.hpp +++ b/third_party/libosmium/include/osmium/area/problem_reporter_stream.hpp @@ -54,7 +54,7 @@ namespace osmium { m_out(&out) { } - virtual ~ProblemReporterStream() = default; + ~ProblemReporterStream() override = default; void header(const char* msg) { *m_out << "DATA PROBLEM: " << msg << " on " << item_type_to_char(m_object_type) << m_object_id << ": "; diff --git a/third_party/libosmium/include/osmium/builder/builder.hpp b/third_party/libosmium/include/osmium/builder/builder.hpp index 4424d88e1..63eb4bb2d 100644 --- a/third_party/libosmium/include/osmium/builder/builder.hpp +++ b/third_party/libosmium/include/osmium/builder/builder.hpp @@ -134,7 +134,7 @@ namespace osmium { * Reserve space for an object of class T in buffer and return * pointer to it. */ - template + template T* reserve_space_for() { assert(m_buffer.is_aligned()); return reinterpret_cast(m_buffer.reserve_space(sizeof(T))); @@ -182,7 +182,7 @@ namespace osmium { }; // class Builder - template + template class ObjectBuilder : public Builder { static_assert(std::is_base_of::value, "ObjectBuilder can only build objects derived from osmium::memory::Item"); diff --git a/third_party/libosmium/include/osmium/builder/osm_object_builder.hpp b/third_party/libosmium/include/osmium/builder/osm_object_builder.hpp index 074076ce6..82b4b23ce 100644 --- a/third_party/libosmium/include/osmium/builder/osm_object_builder.hpp +++ b/third_party/libosmium/include/osmium/builder/osm_object_builder.hpp @@ -33,9 +33,11 @@ DEALINGS IN THE SOFTWARE. */ +#include #include #include #include +#include #include #include @@ -53,7 +55,7 @@ namespace osmium { namespace memory { class Buffer; - } + } // namespace memory namespace builder { @@ -76,6 +78,12 @@ namespace osmium { * @param value Tag value (0-terminated string). */ void add_tag(const char* key, const char* value) { + if (std::strlen(key) > osmium::max_osm_string_length) { + throw std::length_error("OSM tag key is too long"); + } + if (std::strlen(value) > osmium::max_osm_string_length) { + throw std::length_error("OSM tag value is too long"); + } add_size(append(key) + append(value)); } @@ -87,8 +95,15 @@ namespace osmium { * @param value Pointer to tag value. * @param value_length Length of value (not including the \0 byte). */ - void add_tag(const char* key, const string_size_type key_length, const char* value, const string_size_type value_length) { - add_size(append(key, key_length) + append_zero() + append(value, value_length) + append_zero()); + void add_tag(const char* key, const size_t key_length, const char* value, const size_t value_length) { + if (key_length > osmium::max_osm_string_length) { + throw std::length_error("OSM tag key is too long"); + } + if (value_length > osmium::max_osm_string_length) { + throw std::length_error("OSM tag value is too long"); + } + add_size(append(key, osmium::memory::item_size_type(key_length)) + append_zero() + + append(value, osmium::memory::item_size_type(value_length)) + append_zero()); } /** @@ -98,13 +113,46 @@ namespace osmium { * @param value Tag value. */ void add_tag(const std::string& key, const std::string& value) { - add_size(append(key.data(), static_cast_with_assert(key.size() + 1)) + - append(value.data(), static_cast_with_assert(value.size() + 1))); + if (key.size() > osmium::max_osm_string_length) { + throw std::length_error("OSM tag key is too long"); + } + if (value.size() > osmium::max_osm_string_length) { + throw std::length_error("OSM tag value is too long"); + } + add_size(append(key.data(), osmium::memory::item_size_type(key.size()) + 1) + + append(value.data(), osmium::memory::item_size_type(value.size()) + 1)); + } + + /** + * Add tag to buffer. + * + * @param tag Tag. + */ + void add_tag(const osmium::Tag& tag) { + add_size(append(tag.key()) + append(tag.value())); + } + + /** + * Add tag to buffer. + * + * @param tag Pair of key/value 0-terminated strings. + */ + void add_tag(const std::pair& tag) { + add_tag(tag.first, tag.second); + } + + /** + * Add tag to buffer. + * + * @param tag Pair of std::string references. + */ + void add_tag(const std::pair& tag) { + add_tag(tag.first, tag.second); } }; // class TagListBuilder - template + template class NodeRefListBuilder : public ObjectBuilder { public: @@ -122,7 +170,7 @@ namespace osmium { static_cast(this)->add_size(sizeof(osmium::NodeRef)); } - void add_node_ref(const object_id_type ref, const osmium::Location location = Location()) { + void add_node_ref(const object_id_type ref, const osmium::Location& location = Location{}) { add_node_ref(NodeRef(ref, location)); } @@ -141,35 +189,17 @@ namespace osmium { * will be set. * @param role The role. * @param length Length of role (without \0 termination). + * @throws std:length_error If role is longer than osmium::max_osm_string_length */ - void add_role(osmium::RelationMember& member, const char* role, const string_size_type length) { - member.set_role_size(length + 1); - add_size(append(role, length) + append_zero()); + void add_role(osmium::RelationMember& member, const char* role, const size_t length) { + if (length > osmium::max_osm_string_length) { + throw std::length_error("OSM relation member role is too long"); + } + member.set_role_size(osmium::string_size_type(length) + 1); + add_size(append(role, osmium::memory::item_size_type(length)) + append_zero()); add_padding(true); } - /** - * Add role to buffer. - * - * @param member Relation member object where the length of the role - * will be set. - * @param role \0-terminated role. - */ - void add_role(osmium::RelationMember& member, const char* role) { - add_role(member, role, static_cast_with_assert(std::strlen(role))); - } - - /** - * Add role to buffer. - * - * @param member Relation member object where the length of the role - * will be set. - * @param role Role. - */ - void add_role(osmium::RelationMember& member, const std::string& role) { - add_role(member, role.data(), static_cast_with_assert(role.size())); - } - public: explicit RelationMemberListBuilder(osmium::memory::Buffer& buffer, Builder* parent = nullptr) : @@ -190,8 +220,10 @@ namespace osmium { * @param full_member Optional pointer to the member object. If it * is available a copy will be added to the * relation. + * @throws std:length_error If role_length is greater than + * osmium::max_osm_string_length */ - void add_member(osmium::item_type type, object_id_type ref, const char* role, const string_size_type role_length, const osmium::OSMObject* full_member = nullptr) { + void add_member(osmium::item_type type, object_id_type ref, const char* role, const size_t role_length, const osmium::OSMObject* full_member = nullptr) { osmium::RelationMember* member = reserve_space_for(); new (member) osmium::RelationMember(ref, type, full_member != nullptr); add_size(sizeof(RelationMember)); @@ -210,9 +242,10 @@ namespace osmium { * @param full_member Optional pointer to the member object. If it * is available a copy will be added to the * relation. + * @throws std:length_error If role is longer than osmium::max_osm_string_length */ void add_member(osmium::item_type type, object_id_type ref, const char* role, const osmium::OSMObject* full_member = nullptr) { - add_member(type, ref, role, strlen(role), full_member); + add_member(type, ref, role, std::strlen(role), full_member); } /** @@ -224,6 +257,7 @@ namespace osmium { * @param full_member Optional pointer to the member object. If it * is available a copy will be added to the * relation. + * @throws std:length_error If role is longer than osmium::max_osm_string_length */ void add_member(osmium::item_type type, object_id_type ref, const std::string& role, const osmium::OSMObject* full_member = nullptr) { add_member(type, ref, role.data(), role.size(), full_member); @@ -231,7 +265,65 @@ namespace osmium { }; // class RelationMemberListBuilder - template + class ChangesetDiscussionBuilder : public ObjectBuilder { + + osmium::ChangesetComment* m_comment = nullptr; + + void add_user(osmium::ChangesetComment& comment, const char* user, const size_t length) { + if (length > osmium::max_osm_string_length) { + throw std::length_error("OSM user name is too long"); + } + comment.set_user_size(osmium::string_size_type(length) + 1); + add_size(append(user, osmium::memory::item_size_type(length)) + append_zero()); + } + + void add_text(osmium::ChangesetComment& comment, const char* text, const size_t length) { + // XXX There is no limit on the length of a comment text. We + // limit it here to 2^16-2 characters, because that's all that + // will fit into our internal data structure. This is not ideal, + // and will have to be discussed and cleared up. + if (length > std::numeric_limits::max() - 1) { + throw std::length_error("OSM changeset comment is too long"); + } + comment.set_text_size(osmium::string_size_type(length) + 1); + add_size(append(text, osmium::memory::item_size_type(length)) + append_zero()); + add_padding(true); + } + + public: + + explicit ChangesetDiscussionBuilder(osmium::memory::Buffer& buffer, Builder* parent = nullptr) : + ObjectBuilder(buffer, parent) { + } + + ~ChangesetDiscussionBuilder() { + assert(!m_comment && "You have to always call both add_comment() and then add_comment_text() in that order for each comment!"); + add_padding(); + } + + void add_comment(osmium::Timestamp date, osmium::user_id_type uid, const char* user) { + assert(!m_comment && "You have to always call both add_comment() and then add_comment_text() in that order for each comment!"); + m_comment = reserve_space_for(); + new (m_comment) osmium::ChangesetComment(date, uid); + add_size(sizeof(ChangesetComment)); + add_user(*m_comment, user, std::strlen(user)); + } + + void add_comment_text(const char* text) { + assert(m_comment && "You have to always call both add_comment() and then add_comment_text() in that order for each comment!"); + add_text(*m_comment, text, std::strlen(text)); + m_comment = nullptr; + } + + void add_comment_text(const std::string& text) { + assert(m_comment && "You have to always call both add_comment() and then add_comment_text() in that order for each comment!"); + add_text(*m_comment, text.c_str(), text.size()); + m_comment = nullptr; + } + + }; // class ChangesetDiscussionBuilder + + template class OSMObjectBuilder : public ObjectBuilder { public: diff --git a/third_party/libosmium/include/osmium/diff_handler.hpp b/third_party/libosmium/include/osmium/diff_handler.hpp index 4f9b3a1a0..06d8a93ed 100644 --- a/third_party/libosmium/include/osmium/diff_handler.hpp +++ b/third_party/libosmium/include/osmium/diff_handler.hpp @@ -46,8 +46,7 @@ namespace osmium { public: - DiffHandler() { - } + DiffHandler() = default; void node(const osmium::DiffNode&) const { } diff --git a/third_party/libosmium/include/osmium/diff_iterator.hpp b/third_party/libosmium/include/osmium/diff_iterator.hpp index 0ddf7ff7a..6725fecc8 100644 --- a/third_party/libosmium/include/osmium/diff_iterator.hpp +++ b/third_party/libosmium/include/osmium/diff_iterator.hpp @@ -43,7 +43,12 @@ namespace osmium { class OSMObject; - template + /** + * An input iterator wrapping any iterator over OSMObjects. When + * dereferenced it will yield DiffObject objects pointing to the + * underlying OSMObjects. + */ + template class DiffIterator : public std::iterator { static_assert(std::is_base_of::value, "TBasicIterator::value_type must derive from osmium::OSMObject"); @@ -56,37 +61,29 @@ namespace osmium { mutable osmium::DiffObject m_diff; - void set_diff() const { + void set_diff() const noexcept { assert(m_curr != m_end); - TBasicIterator prev = m_prev; - if (prev->type() != m_curr->type() || prev->id() != m_curr->id()) { - prev = m_curr; - } + bool use_curr_for_prev = m_prev->type() != m_curr->type() || m_prev->id() != m_curr->id(); + bool use_curr_for_next = m_next == m_end || m_next->type() != m_curr->type() || m_next->id() != m_curr->id(); - TBasicIterator next = m_next; - if (next == m_end || next->type() != m_curr->type() || next->id() != m_curr->id()) { - next = m_curr; - } - - m_diff = osmium::DiffObject(*prev, *m_curr, *next); + m_diff = std::move(osmium::DiffObject{ + *(use_curr_for_prev ? m_curr : m_prev), + *m_curr, + *(use_curr_for_next ? m_curr : m_next) + }); } public: - explicit DiffIterator(TBasicIterator begin, TBasicIterator end) : + DiffIterator(TBasicIterator begin, TBasicIterator end) : m_prev(begin), m_curr(begin), m_next(begin == end ? begin : ++begin), - m_end(end) { + m_end(std::move(end)), + m_diff() { } - DiffIterator(const DiffIterator&) = default; - DiffIterator& operator=(const DiffIterator&) = default; - - DiffIterator(DiffIterator&&) = default; - DiffIterator& operator=(DiffIterator&&) = default; - DiffIterator& operator++() { m_prev = std::move(m_curr); m_curr = m_next; @@ -104,26 +101,35 @@ namespace osmium { return tmp; } - bool operator==(const DiffIterator& rhs) const { + bool operator==(const DiffIterator& rhs) const noexcept { return m_curr == rhs.m_curr && m_end == rhs.m_end; } - bool operator!=(const DiffIterator& rhs) const { + bool operator!=(const DiffIterator& rhs) const noexcept { return !(*this == rhs); } - reference operator*() const { + reference operator*() const noexcept { set_diff(); return m_diff; } - pointer operator->() const { + pointer operator->() const noexcept { set_diff(); return &m_diff; } }; // class DiffIterator + /** + * Create a DiffIterator based on the given iterators. + */ + template + inline DiffIterator make_diff_iterator(TBasicIterator begin, + TBasicIterator end) { + return DiffIterator{begin, end}; + } + } // namespace osmium #endif // OSMIUM_DIFF_ITERATOR_HPP diff --git a/third_party/libosmium/include/osmium/diff_visitor.hpp b/third_party/libosmium/include/osmium/diff_visitor.hpp index 5e72a7bb7..ac16a8ed4 100644 --- a/third_party/libosmium/include/osmium/diff_visitor.hpp +++ b/third_party/libosmium/include/osmium/diff_visitor.hpp @@ -43,7 +43,7 @@ namespace osmium { namespace detail { - template + template inline void apply_diff_iterator_recurse(const osmium::DiffObject& diff, THandler& handler) { switch (diff.type()) { case osmium::item_type::node: @@ -60,7 +60,7 @@ namespace osmium { } } - template + template inline void apply_diff_iterator_recurse(const osmium::DiffObject& diff, THandler& handler, TRest&... more) { apply_diff_iterator_recurse(diff, handler); apply_diff_iterator_recurse(diff, more...); @@ -68,9 +68,9 @@ namespace osmium { } // namespace detail - template + template inline void apply_diff(TIterator it, TIterator end, THandlers&... handlers) { - typedef osmium::DiffIterator diff_iterator; + using diff_iterator = osmium::DiffIterator; diff_iterator dit(it, end); diff_iterator dend(end, end); @@ -82,19 +82,19 @@ namespace osmium { class OSMObject; - template + template inline void apply_diff(TSource& source, THandlers&... handlers) { apply_diff(osmium::io::InputIterator {source}, osmium::io::InputIterator {}, handlers...); } - template + template inline void apply_diff(osmium::memory::Buffer& buffer, THandlers&... handlers) { apply_diff(buffer.begin(), buffer.end(), handlers...); } - template + template inline void apply_diff(const osmium::memory::Buffer& buffer, THandlers&... handlers) { apply_diff(buffer.cbegin(), buffer.cend(), handlers...); } diff --git a/third_party/libosmium/include/osmium/dynamic_handler.hpp b/third_party/libosmium/include/osmium/dynamic_handler.hpp index 9d0bd66f5..7077554ff 100644 --- a/third_party/libosmium/include/osmium/dynamic_handler.hpp +++ b/third_party/libosmium/include/osmium/dynamic_handler.hpp @@ -36,16 +36,11 @@ DEALINGS IN THE SOFTWARE. #include #include +#include #include namespace osmium { - class Node; - class Way; - class Relation; - class Area; - class Changeset; - namespace handler { namespace detail { @@ -83,11 +78,11 @@ namespace osmium { // to either call handler style functions or visitor style operator(). #define OSMIUM_DYNAMIC_HANDLER_DISPATCH(_name_, _type_) \ -template \ +template \ auto _name_##_dispatch(THandler& handler, const osmium::_type_& object, int) -> decltype(handler._name_(object), void()) { \ handler._name_(object); \ } \ -template \ +template \ auto _name_##_dispatch(THandler& handler, const osmium::_type_& object, long) -> decltype(handler(object), void()) { \ handler(object); \ } @@ -98,47 +93,47 @@ auto _name_##_dispatch(THandler& handler, const osmium::_type_& object, long) -> OSMIUM_DYNAMIC_HANDLER_DISPATCH(changeset, Changeset) OSMIUM_DYNAMIC_HANDLER_DISPATCH(area, Area) - template + template auto flush_dispatch(THandler& handler, int) -> decltype(handler.flush(), void()) { handler.flush(); } - template + template void flush_dispatch(THandler&, long) {} - template + template class HandlerWrapper : public HandlerWrapperBase { THandler m_handler; public: - template + template HandlerWrapper(TArgs&&... args) : m_handler(std::forward(args)...) { } - void node(const osmium::Node& node) override final { + void node(const osmium::Node& node) final { node_dispatch(m_handler, node, 0); } - void way(const osmium::Way& way) override final { + void way(const osmium::Way& way) final { way_dispatch(m_handler, way, 0); } - void relation(const osmium::Relation& relation) override final { + void relation(const osmium::Relation& relation) final { relation_dispatch(m_handler, relation, 0); } - void area(const osmium::Area& area) override final { + void area(const osmium::Area& area) final { area_dispatch(m_handler, area, 0); } - void changeset(const osmium::Changeset& changeset) override final { + void changeset(const osmium::Changeset& changeset) final { changeset_dispatch(m_handler, changeset, 0); } - void flush() override final { + void flush() final { flush_dispatch(m_handler, 0); } @@ -157,7 +152,7 @@ auto _name_##_dispatch(THandler& handler, const osmium::_type_& object, long) -> m_impl(impl_ptr(new osmium::handler::detail::HandlerWrapperBase)) { } - template + template void set(TArgs&&... args) { m_impl = impl_ptr(new osmium::handler::detail::HandlerWrapper(std::forward(args)...)); } @@ -188,7 +183,7 @@ auto _name_##_dispatch(THandler& handler, const osmium::_type_& object, long) -> }; // class DynamicHandler - } // namspace handler + } // namespace handler } // namespace osmium diff --git a/third_party/libosmium/include/osmium/experimental/flex_reader.hpp b/third_party/libosmium/include/osmium/experimental/flex_reader.hpp index f00a5ecaf..c1d235746 100644 --- a/third_party/libosmium/include/osmium/experimental/flex_reader.hpp +++ b/third_party/libosmium/include/osmium/experimental/flex_reader.hpp @@ -33,10 +33,18 @@ DEALINGS IN THE SOFTWARE. */ -#include -#include -#include +#include +#include + #include +#include +#include +#include +#include +#include +#include +#include +#include namespace osmium { @@ -45,7 +53,7 @@ namespace osmium { */ namespace experimental { - template + template class FlexReader { bool m_with_areas; @@ -104,7 +112,7 @@ namespace osmium { return buffer; } - osmium::io::Header header() const { + osmium::io::Header header() { return m_reader.header(); } diff --git a/third_party/libosmium/include/osmium/fwd.hpp b/third_party/libosmium/include/osmium/fwd.hpp new file mode 100644 index 000000000..bfcb5f57f --- /dev/null +++ b/third_party/libosmium/include/osmium/fwd.hpp @@ -0,0 +1,70 @@ +#ifndef OSMIUM_FWD_HPP +#define OSMIUM_FWD_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +/** + * + * @file + * + * This file contains forward declarations for commonly used Osmium classes. + * + */ + +namespace osmium { + + class Area; + class Box; + class Changeset; + class ChangesetComment; + class ChangesetDiscussion; + class InnerRing; + class Location; + class Node; + class NodeRef; + class NodeRefList; + class OSMEntity; + class OSMObject; + class OuterRing; + class Relation; + class RelationMemberList; + class Segment; + class Tag; + class TagList; + class Timestamp; + class Way; + class WayNodeList; + +} // namespace osmium + +#endif // OSMIUM_FWD_HPP diff --git a/third_party/libosmium/include/osmium/geom/coordinates.hpp b/third_party/libosmium/include/osmium/geom/coordinates.hpp index 2bad57e0d..6544e68aa 100644 --- a/third_party/libosmium/include/osmium/geom/coordinates.hpp +++ b/third_party/libosmium/include/osmium/geom/coordinates.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include diff --git a/third_party/libosmium/include/osmium/geom/factory.hpp b/third_party/libosmium/include/osmium/geom/factory.hpp index 9be050d25..19375806b 100644 --- a/third_party/libosmium/include/osmium/geom/factory.hpp +++ b/third_party/libosmium/include/osmium/geom/factory.hpp @@ -61,7 +61,7 @@ namespace osmium { public: - geometry_error(const std::string& message, const char* object_type = "", osmium::object_id_type id = 0) : + explicit geometry_error(const std::string& message, const char* object_type = "", osmium::object_id_type id = 0) : std::runtime_error(message), m_message(message), m_id(id) { @@ -89,7 +89,7 @@ namespace osmium { return m_id; } - virtual const char* what() const noexcept override { + const char* what() const noexcept override { return m_message.c_str(); } @@ -142,7 +142,7 @@ namespace osmium { /** * Geometry factory. */ - template + template class GeometryFactory { /** @@ -166,8 +166,8 @@ namespace osmium { /** * Constructor for default initialized projection. */ - template - GeometryFactory(TArgs&&... args) : + template + explicit GeometryFactory(TArgs&&... args) : m_projection(), m_impl(std::forward(args)...) { } @@ -176,12 +176,13 @@ namespace osmium { * Constructor for explicitly initialized projection. Note that the * projection is moved into the GeometryFactory. */ - template - GeometryFactory(TProjection&& projection, TArgs&&... args) : + template + explicit GeometryFactory(TProjection&& projection, TArgs&&... args) : m_projection(std::move(projection)), m_impl(std::forward(args)...) { } + typedef TProjection projection_type; typedef typename TGeomImpl::point_type point_type; typedef typename TGeomImpl::linestring_type linestring_type; typedef typename TGeomImpl::polygon_type polygon_type; @@ -198,7 +199,7 @@ namespace osmium { /* Point */ - point_type create_point(const osmium::Location location) const { + point_type create_point(const osmium::Location& location) const { return m_impl.make_point(m_projection(location)); } @@ -226,7 +227,7 @@ namespace osmium { m_impl.linestring_start(); } - template + template size_t fill_linestring(TIter it, TIter end) { size_t num_points = 0; for (; it != end; ++it, ++num_points) { @@ -235,7 +236,7 @@ namespace osmium { return num_points; } - template + template size_t fill_linestring_unique(TIter it, TIter end) { size_t num_points = 0; osmium::Location last_location; @@ -300,7 +301,7 @@ namespace osmium { m_impl.polygon_start(); } - template + template size_t fill_polygon(TIter it, TIter end) { size_t num_points = 0; for (; it != end; ++it, ++num_points) { @@ -309,7 +310,7 @@ namespace osmium { return num_points; } - template + template size_t fill_polygon_unique(TIter it, TIter end) { size_t num_points = 0; osmium::Location last_location; diff --git a/third_party/libosmium/include/osmium/geom/geojson.hpp b/third_party/libosmium/include/osmium/geom/geojson.hpp index 7d5953535..e5b5a9cfc 100644 --- a/third_party/libosmium/include/osmium/geom/geojson.hpp +++ b/third_party/libosmium/include/osmium/geom/geojson.hpp @@ -88,7 +88,10 @@ namespace osmium { linestring_type linestring_finish(size_t /* num_points */) { assert(!m_str.empty()); std::string str; - std::swap(str, m_str); + + using std::swap; + swap(str, m_str); + str.back() = ']'; str += "}"; return str; @@ -134,7 +137,10 @@ namespace osmium { multipolygon_type multipolygon_finish() { assert(!m_str.empty()); std::string str; - std::swap(str, m_str); + + using std::swap; + swap(str, m_str); + str.back() = ']'; str += "}"; return str; @@ -144,7 +150,7 @@ namespace osmium { } // namespace detail - template + template using GeoJSONFactory = GeometryFactory; } // namespace geom diff --git a/third_party/libosmium/include/osmium/geom/geos.hpp b/third_party/libosmium/include/osmium/geom/geos.hpp index 771b08736..49b1fd7c0 100644 --- a/third_party/libosmium/include/osmium/geom/geos.hpp +++ b/third_party/libosmium/include/osmium/geom/geos.hpp @@ -228,7 +228,7 @@ namespace osmium { } // namespace detail - template + template using GEOSFactory = GeometryFactory; } // namespace geom diff --git a/third_party/libosmium/include/osmium/geom/ogr.hpp b/third_party/libosmium/include/osmium/geom/ogr.hpp index f33971c7d..4d5995cf7 100644 --- a/third_party/libosmium/include/osmium/geom/ogr.hpp +++ b/third_party/libosmium/include/osmium/geom/ogr.hpp @@ -47,35 +47,7 @@ DEALINGS IN THE SOFTWARE. #include #include -#ifdef _MSC_VER -# pragma warning(push) -# pragma warning(disable : 4458) -# pragma warning(disable : 4251) -#else -# pragma GCC diagnostic push -# ifdef __clang__ -# pragma GCC diagnostic ignored "-Wdocumentation-unknown-command" -# endif -# pragma GCC diagnostic ignored "-Wfloat-equal" -# pragma GCC diagnostic ignored "-Wold-style-cast" -# pragma GCC diagnostic ignored "-Wpadded" -# pragma GCC diagnostic ignored "-Wredundant-decls" -# pragma GCC diagnostic ignored "-Wshadow" -#endif - -/* Strictly speaking the following include would be enough here, - but everybody using this file will very likely need the other includes, - so we are adding them here, so that not everybody will need all those - pragmas to disable warnings. */ -//#include -#include -#include - -#ifdef _MSC_VER -# pragma warning(pop) -#else -# pragma GCC diagnostic pop -#endif +#include #include #include @@ -196,7 +168,7 @@ namespace osmium { } // namespace detail - template + template using OGRFactory = GeometryFactory; } // namespace geom diff --git a/third_party/libosmium/include/osmium/geom/rapid_geojson.hpp b/third_party/libosmium/include/osmium/geom/rapid_geojson.hpp index a3d46870c..87e479bf1 100644 --- a/third_party/libosmium/include/osmium/geom/rapid_geojson.hpp +++ b/third_party/libosmium/include/osmium/geom/rapid_geojson.hpp @@ -46,7 +46,7 @@ namespace osmium { * A geometry factory implementation that can be used with the * RapidJSON (https://github.com/miloyip/rapidjson) JSON writer. */ - template + template class RapidGeoJSONFactoryImpl { TWriter* m_writer; @@ -180,7 +180,7 @@ namespace osmium { } // namespace detail - template + template using RapidGeoJSONFactory = GeometryFactory, TProjection>; } // namespace geom diff --git a/third_party/libosmium/include/osmium/geom/tile.hpp b/third_party/libosmium/include/osmium/geom/tile.hpp index 9cd0b282b..6ca068262 100644 --- a/third_party/libosmium/include/osmium/geom/tile.hpp +++ b/third_party/libosmium/include/osmium/geom/tile.hpp @@ -67,10 +67,10 @@ namespace osmium { explicit Tile(uint32_t zoom, const osmium::Location& location) : z(zoom) { osmium::geom::Coordinates c = lonlat_to_mercator(location); - const int32_t n = 1LL << zoom; + const int32_t n = 1 << zoom; const double scale = detail::max_coordinate_epsg3857 * 2 / n; - x = detail::restrict_to_range((c.x + detail::max_coordinate_epsg3857) / scale, 0, n-1); - y = detail::restrict_to_range((detail::max_coordinate_epsg3857 - c.y) / scale, 0, n-1); + x = uint32_t(detail::restrict_to_range(int32_t((c.x + detail::max_coordinate_epsg3857) / scale), 0, n-1)); + y = uint32_t(detail::restrict_to_range(int32_t((detail::max_coordinate_epsg3857 - c.y) / scale), 0, n-1)); } }; // struct Tile diff --git a/third_party/libosmium/include/osmium/geom/wkb.hpp b/third_party/libosmium/include/osmium/geom/wkb.hpp index a290c02d0..49833e66b 100644 --- a/third_party/libosmium/include/osmium/geom/wkb.hpp +++ b/third_party/libosmium/include/osmium/geom/wkb.hpp @@ -188,7 +188,9 @@ namespace osmium { linestring_type linestring_finish(size_t num_points) { set_size(m_linestring_size_offset, num_points); std::string data; - std::swap(data, m_data); + + using std::swap; + swap(data, m_data); if (m_out_type == out_type::hex) { return convert_to_hex(data); @@ -246,7 +248,9 @@ namespace osmium { multipolygon_type multipolygon_finish() { set_size(m_multipolygon_size_offset, m_polygons); std::string data; - std::swap(data, m_data); + + using std::swap; + swap(data, m_data); if (m_out_type == out_type::hex) { return convert_to_hex(data); @@ -259,7 +263,7 @@ namespace osmium { } // namespace detail - template + template using WKBFactory = GeometryFactory; } // namespace geom diff --git a/third_party/libosmium/include/osmium/geom/wkt.hpp b/third_party/libosmium/include/osmium/geom/wkt.hpp index 4fea96baa..9cf5371a9 100644 --- a/third_party/libosmium/include/osmium/geom/wkt.hpp +++ b/third_party/libosmium/include/osmium/geom/wkt.hpp @@ -86,7 +86,10 @@ namespace osmium { linestring_type linestring_finish(size_t /* num_points */) { assert(!m_str.empty()); std::string str; - std::swap(str, m_str); + + using std::swap; + swap(str, m_str); + str.back() = ')'; return str; } @@ -131,7 +134,10 @@ namespace osmium { multipolygon_type multipolygon_finish() { assert(!m_str.empty()); std::string str; - std::swap(str, m_str); + + using std::swap; + swap(str, m_str); + str.back() = ')'; return str; } @@ -140,7 +146,7 @@ namespace osmium { } // namespace detail - template + template using WKTFactory = GeometryFactory; } // namespace geom diff --git a/third_party/libosmium/include/osmium/handler.hpp b/third_party/libosmium/include/osmium/handler.hpp index 34d878511..47e997efc 100644 --- a/third_party/libosmium/include/osmium/handler.hpp +++ b/third_party/libosmium/include/osmium/handler.hpp @@ -33,19 +33,9 @@ DEALINGS IN THE SOFTWARE. */ -namespace osmium { +#include - class OSMObject; - class Node; - class Way; - class Relation; - class Area; - class Changeset; - class TagList; - class WayNodeList; - class RelationMemberList; - class OuterRing; - class InnerRing; +namespace osmium { /** * @brief Osmium handlers provide callbacks for OSM objects @@ -89,12 +79,15 @@ namespace osmium { void inner_ring(const osmium::InnerRing&) const { } + void changeset_discussion(const osmium::ChangesetDiscussion&) const { + } + void flush() const { } }; // class Handler - } // namspace handler + } // namespace handler } // namespace osmium diff --git a/third_party/libosmium/include/osmium/handler/chain.hpp b/third_party/libosmium/include/osmium/handler/chain.hpp index 1af3962fd..4f3291ca6 100644 --- a/third_party/libosmium/include/osmium/handler/chain.hpp +++ b/third_party/libosmium/include/osmium/handler/chain.hpp @@ -38,14 +38,14 @@ DEALINGS IN THE SOFTWARE. #include #define OSMIUM_CHAIN_HANDLER_CALL(_func_, _type_) \ - template \ + template \ struct call_ ## _func_ { \ void operator()(THandlers& handlers, osmium::_type_& object) { \ std::get(handlers)._func_(object); \ call_ ## _func_()(handlers, object); \ } \ }; \ - template \ + template \ struct call_ ## _func_ { \ void operator()(THandlers&, osmium::_type_&) {} \ }; @@ -64,13 +64,13 @@ namespace osmium { * This handler allows chaining of any number of handlers into a single * handler. */ - template + template class ChainHandler : public osmium::handler::Handler { typedef std::tuple handlers_type; handlers_type m_handlers; - template + template struct call_flush { void operator()(THandlers& handlers) { std::get(handlers).flush(); @@ -78,7 +78,7 @@ namespace osmium { } }; // struct call_flush - template + template struct call_flush { void operator()(THandlers&) {} }; // struct call_flush diff --git a/third_party/libosmium/include/osmium/handler/node_locations_for_ways.hpp b/third_party/libosmium/include/osmium/handler/node_locations_for_ways.hpp index 9b9fcbf3f..8d31310c3 100644 --- a/third_party/libosmium/include/osmium/handler/node_locations_for_ways.hpp +++ b/third_party/libosmium/include/osmium/handler/node_locations_for_ways.hpp @@ -60,7 +60,7 @@ namespace osmium { * get(id) methods. * @tparam TStorageNegIDs Same but for negative IDs. */ - template + template class NodeLocationsForWays : public osmium::handler::Handler { static_assert(std::is_base_of, TStoragePosIDs>::value, "Index class must be derived from osmium::index::map::Map"); diff --git a/third_party/libosmium/include/osmium/index/bool_vector.hpp b/third_party/libosmium/include/osmium/index/bool_vector.hpp index 94e1f7268..04850a5fc 100644 --- a/third_party/libosmium/include/osmium/index/bool_vector.hpp +++ b/third_party/libosmium/include/osmium/index/bool_vector.hpp @@ -56,11 +56,13 @@ namespace osmium { public: BoolVector() = default; + BoolVector(const BoolVector&) = default; BoolVector(BoolVector&&) = default; BoolVector& operator=(const BoolVector&) = default; BoolVector& operator=(BoolVector&&) = default; - ~BoolVector() = default; + + ~BoolVector() noexcept = default; void set(T id, bool value = true) { if (m_bits.size() <= id) { diff --git a/third_party/libosmium/include/osmium/index/detail/create_map_with_fd.hpp b/third_party/libosmium/include/osmium/index/detail/create_map_with_fd.hpp index 5ccbfc809..a2e6b768a 100644 --- a/third_party/libosmium/include/osmium/index/detail/create_map_with_fd.hpp +++ b/third_party/libosmium/include/osmium/index/detail/create_map_with_fd.hpp @@ -47,19 +47,18 @@ namespace osmium { namespace detail { - template + template inline T* create_map_with_fd(const std::vector& config) { if (config.size() == 1) { return new T(); - } else { - assert(config.size() > 1); - const std::string& filename = config[1]; - int fd = ::open(filename.c_str(), O_CREAT | O_RDWR, 0644); - if (fd == -1) { - throw std::runtime_error(std::string("can't open file '") + filename + "': " + strerror(errno)); - } - return new T(fd); } + assert(config.size() > 1); + const std::string& filename = config[1]; + int fd = ::open(filename.c_str(), O_CREAT | O_RDWR, 0644); + if (fd == -1) { + throw std::runtime_error(std::string("can't open file '") + filename + "': " + strerror(errno)); + } + return new T(fd); } } // namespace detail diff --git a/third_party/libosmium/include/osmium/index/detail/mmap_vector_anon.hpp b/third_party/libosmium/include/osmium/index/detail/mmap_vector_anon.hpp index fc016261e..12a18036b 100644 --- a/third_party/libosmium/include/osmium/index/detail/mmap_vector_anon.hpp +++ b/third_party/libosmium/include/osmium/index/detail/mmap_vector_anon.hpp @@ -54,6 +54,8 @@ namespace osmium { mmap_vector_base() { } + ~mmap_vector_anon() noexcept = default; + }; // class mmap_vector_anon } // namespace detail diff --git a/third_party/libosmium/include/osmium/index/detail/mmap_vector_base.hpp b/third_party/libosmium/include/osmium/index/detail/mmap_vector_base.hpp index 9b6476812..e5f28e9b9 100644 --- a/third_party/libosmium/include/osmium/index/detail/mmap_vector_base.hpp +++ b/third_party/libosmium/include/osmium/index/detail/mmap_vector_base.hpp @@ -60,7 +60,7 @@ namespace osmium { public: - explicit mmap_vector_base(int fd, size_t capacity, size_t size = 0) : + mmap_vector_base(int fd, size_t capacity, size_t size = 0) : m_size(size), m_mapping(capacity, osmium::util::MemoryMapping::mapping_mode::write_shared, fd) { } @@ -70,6 +70,8 @@ namespace osmium { m_mapping(capacity) { } + ~mmap_vector_base() noexcept = default; + typedef T value_type; typedef T& reference; typedef const T& const_reference; @@ -78,8 +80,6 @@ namespace osmium { typedef T* iterator; typedef const T* const_iterator; - ~mmap_vector_base() = default; - void close() { m_mapping.unmap(); } diff --git a/third_party/libosmium/include/osmium/index/detail/mmap_vector_file.hpp b/third_party/libosmium/include/osmium/index/detail/mmap_vector_file.hpp index 1dadbcb45..54ef5137e 100644 --- a/third_party/libosmium/include/osmium/index/detail/mmap_vector_file.hpp +++ b/third_party/libosmium/include/osmium/index/detail/mmap_vector_file.hpp @@ -50,17 +50,21 @@ namespace osmium { public: - explicit mmap_vector_file() : mmap_vector_base( + mmap_vector_file() : + mmap_vector_base( osmium::detail::create_tmp_file(), osmium::detail::mmap_vector_size_increment) { } - explicit mmap_vector_file(int fd) : mmap_vector_base( + explicit mmap_vector_file(int fd) : + mmap_vector_base( fd, osmium::util::file_size(fd) / sizeof(T), osmium::util::file_size(fd) / sizeof(T)) { } + ~mmap_vector_file() noexcept = default; + }; // class mmap_vector_file } // namespace detail diff --git a/third_party/libosmium/include/osmium/index/detail/vector_map.hpp b/third_party/libosmium/include/osmium/index/detail/vector_map.hpp index 9c1cf4ed8..2a13061e2 100644 --- a/third_party/libosmium/include/osmium/index/detail/vector_map.hpp +++ b/third_party/libosmium/include/osmium/index/detail/vector_map.hpp @@ -48,7 +48,7 @@ namespace osmium { namespace map { - template + template class VectorBasedDenseMap : public Map { TVector m_vector; @@ -68,20 +68,20 @@ namespace osmium { m_vector(fd) { } - ~VectorBasedDenseMap() = default; + ~VectorBasedDenseMap() noexcept final = default; - void reserve(const size_t size) override final { + void reserve(const size_t size) final { m_vector.reserve(size); } - void set(const TId id, const TValue value) override final { + void set(const TId id, const TValue value) final { if (size() <= id) { m_vector.resize(id+1); } m_vector[id] = value; } - const TValue get(const TId id) const override final { + const TValue get(const TId id) const final { try { const TValue& value = m_vector.at(id); if (value == osmium::index::empty_value()) { @@ -93,7 +93,7 @@ namespace osmium { } } - size_t size() const override final { + size_t size() const final { return m_vector.size(); } @@ -101,16 +101,16 @@ namespace osmium { return m_vector.size() * sizeof(element_type); } - size_t used_memory() const override final { + size_t used_memory() const final { return sizeof(TValue) * size(); } - void clear() override final { + void clear() final { m_vector.clear(); m_vector.shrink_to_fit(); } - void dump_as_array(const int fd) override final { + void dump_as_array(const int fd) final { osmium::io::detail::reliable_write(fd, reinterpret_cast(m_vector.data()), byte_size()); } @@ -161,17 +161,17 @@ namespace osmium { m_vector() { } - VectorBasedSparseMap(int fd) : + explicit VectorBasedSparseMap(int fd) : m_vector(fd) { } - ~VectorBasedSparseMap() override final = default; + ~VectorBasedSparseMap() final = default; - void set(const TId id, const TValue value) override final { + void set(const TId id, const TValue value) final { m_vector.push_back(element_type(id, value)); } - const TValue get(const TId id) const override final { + const TValue get(const TId id) const final { const element_type element { id, osmium::index::empty_value() @@ -186,7 +186,7 @@ namespace osmium { } } - size_t size() const override final { + size_t size() const final { return m_vector.size(); } @@ -194,20 +194,20 @@ namespace osmium { return m_vector.size() * sizeof(element_type); } - size_t used_memory() const override final { + size_t used_memory() const final { return sizeof(element_type) * size(); } - void clear() override final { + void clear() final { m_vector.clear(); m_vector.shrink_to_fit(); } - void sort() override final { + void sort() final { std::sort(m_vector.begin(), m_vector.end()); } - void dump_as_list(const int fd) override final { + void dump_as_list(const int fd) final { osmium::io::detail::reliable_write(fd, reinterpret_cast(m_vector.data()), byte_size()); } diff --git a/third_party/libosmium/include/osmium/index/detail/vector_multimap.hpp b/third_party/libosmium/include/osmium/index/detail/vector_multimap.hpp index 1d875fcba..789fc9499 100644 --- a/third_party/libosmium/include/osmium/index/detail/vector_multimap.hpp +++ b/third_party/libosmium/include/osmium/index/detail/vector_multimap.hpp @@ -75,9 +75,9 @@ namespace osmium { m_vector(fd) { } - ~VectorBasedSparseMultimap() = default; + ~VectorBasedSparseMultimap() noexcept final = default; - void set(const TId id, const TValue value) override final { + void set(const TId id, const TValue value) final { m_vector.push_back(element_type(id, value)); } @@ -105,7 +105,7 @@ namespace osmium { }); } - size_t size() const override final { + size_t size() const final { return m_vector.size(); } @@ -113,16 +113,16 @@ namespace osmium { return m_vector.size() * sizeof(element_type); } - size_t used_memory() const override final { + size_t used_memory() const final { return sizeof(element_type) * size(); } - void clear() override final { + void clear() final { m_vector.clear(); m_vector.shrink_to_fit(); } - void sort() override final { + void sort() final { std::sort(m_vector.begin(), m_vector.end()); } @@ -147,7 +147,7 @@ namespace osmium { ); } - void dump_as_list(const int fd) override final { + void dump_as_list(const int fd) final { osmium::io::detail::reliable_write(fd, reinterpret_cast(m_vector.data()), byte_size()); } diff --git a/third_party/libosmium/include/osmium/index/index.hpp b/third_party/libosmium/include/osmium/index/index.hpp index f415192d2..78c142d82 100644 --- a/third_party/libosmium/include/osmium/index/index.hpp +++ b/third_party/libosmium/include/osmium/index/index.hpp @@ -89,7 +89,7 @@ namespace osmium { * the full range, so the max value is a good "empty" value. */ template <> - inline OSMIUM_CONSTEXPR size_t empty_value() { + inline constexpr size_t empty_value() { return std::numeric_limits::max(); } diff --git a/third_party/libosmium/include/osmium/index/map.hpp b/third_party/libosmium/include/osmium/index/map.hpp index 61af672cd..68a9c201c 100644 --- a/third_party/libosmium/include/osmium/index/map.hpp +++ b/third_party/libosmium/include/osmium/index/map.hpp @@ -84,7 +84,8 @@ namespace osmium { template class Map { - static_assert(std::is_integral::value && std::is_unsigned::value, "TId template parameter for class Map must be unsigned integral type"); + static_assert(std::is_integral::value && std::is_unsigned::value, + "TId template parameter for class Map must be unsigned integral type"); Map(const Map&) = delete; Map& operator=(const Map&) = delete; @@ -104,7 +105,7 @@ namespace osmium { Map() = default; - virtual ~Map() = default; + virtual ~Map() noexcept = default; virtual void reserve(const size_t) { // default implementation is empty @@ -147,10 +148,16 @@ namespace osmium { // default implementation is empty } + // This function could usually be const in derived classes, + // but not always. It could, for instance, sort internal data. + // This is why it is not declared const here. virtual void dump_as_list(const int /*fd*/) { throw std::runtime_error("can't dump as list"); } + // This function could usually be const in derived classes, + // but not always. It could, for instance, sort internal data. + // This is why it is not declared const here. virtual void dump_as_array(const int /*fd*/) { throw std::runtime_error("can't dump as array"); } @@ -252,12 +259,14 @@ namespace osmium { #define OSMIUM_CONCATENATE_DETAIL_(x, y) x##y #define OSMIUM_CONCATENATE_(x, y) OSMIUM_CONCATENATE_DETAIL_(x, y) -#define OSMIUM_MAKE_UNIQUE_(x) OSMIUM_CONCATENATE_(x, __COUNTER__) #define REGISTER_MAP(id, value, klass, name) \ -namespace { \ - const bool OSMIUM_MAKE_UNIQUE_(registered_index_map_##name) = osmium::index::register_map(#name); \ -} +namespace osmium { namespace index { namespace detail { \ + const bool OSMIUM_CONCATENATE_(registered_, name) = osmium::index::register_map(#name); \ + inline bool OSMIUM_CONCATENATE_(get_registered_, name)() noexcept { \ + return OSMIUM_CONCATENATE_(registered_, name); \ + } \ +} } } } // namespace index diff --git a/third_party/libosmium/include/osmium/index/map/dummy.hpp b/third_party/libosmium/include/osmium/index/map/dummy.hpp index de05d1d69..d6a360e56 100644 --- a/third_party/libosmium/include/osmium/index/map/dummy.hpp +++ b/third_party/libosmium/include/osmium/index/map/dummy.hpp @@ -56,25 +56,25 @@ namespace osmium { Dummy() = default; - ~Dummy() override final = default; + ~Dummy() noexcept final = default; - void set(const TId, const TValue) override final { + void set(const TId, const TValue) final { // intentionally left blank } - const TValue get(const TId id) const override final { + const TValue get(const TId id) const final { not_found_error(id); } - size_t size() const override final { + size_t size() const final { return 0; } - size_t used_memory() const override final { + size_t used_memory() const final { return 0; } - void clear() override final { + void clear() final { } }; // class Dummy diff --git a/third_party/libosmium/include/osmium/index/map/sparse_mem_map.hpp b/third_party/libosmium/include/osmium/index/map/sparse_mem_map.hpp index 2b9048b3b..d001b6791 100644 --- a/third_party/libosmium/include/osmium/index/map/sparse_mem_map.hpp +++ b/third_party/libosmium/include/osmium/index/map/sparse_mem_map.hpp @@ -71,36 +71,37 @@ namespace osmium { SparseMemMap() = default; - ~SparseMemMap() override final = default; + ~SparseMemMap() noexcept final = default; - void set(const TId id, const TValue value) override final { + void set(const TId id, const TValue value) final { m_elements[id] = value; } - const TValue get(const TId id) const override final { - try { - return m_elements.at(id); - } catch (std::out_of_range&) { + const TValue get(const TId id) const final { + auto it = m_elements.find(id); + if (it == m_elements.end()) { not_found_error(id); } + return it->second; } - size_t size() const override final { + size_t size() const noexcept final { return m_elements.size(); } - size_t used_memory() const override final { + size_t used_memory() const noexcept final { return element_size * m_elements.size(); } - void clear() override final { + void clear() final { m_elements.clear(); } - void dump_as_list(const int fd) override final { + void dump_as_list(const int fd) final { typedef typename std::map::value_type t; std::vector v; - std::copy(m_elements.begin(), m_elements.end(), std::back_inserter(v)); + v.reserve(m_elements.size()); + std::copy(m_elements.cbegin(), m_elements.cend(), std::back_inserter(v)); osmium::io::detail::reliable_write(fd, reinterpret_cast(v.data()), sizeof(t) * v.size()); } diff --git a/third_party/libosmium/include/osmium/index/map/sparse_mem_table.hpp b/third_party/libosmium/include/osmium/index/map/sparse_mem_table.hpp index 09ee81bbc..797a9265e 100644 --- a/third_party/libosmium/include/osmium/index/map/sparse_mem_table.hpp +++ b/third_party/libosmium/include/osmium/index/map/sparse_mem_table.hpp @@ -88,16 +88,16 @@ namespace osmium { m_elements(grow_size) { } - ~SparseMemTable() override final = default; + ~SparseMemTable() noexcept final = default; - void set(const TId id, const TValue value) override final { + void set(const TId id, const TValue value) final { if (id >= m_elements.size()) { m_elements.resize(id + m_grow_size); } m_elements[id] = value; } - const TValue get(const TId id) const override final { + const TValue get(const TId id) const final { if (id >= m_elements.size()) { not_found_error(id); } @@ -107,22 +107,23 @@ namespace osmium { return m_elements[id]; } - size_t size() const override final { + size_t size() const final { return m_elements.size(); } - size_t used_memory() const override final { + size_t used_memory() const final { // unused elements use 1 bit, used elements sizeof(TValue) bytes // http://google-sparsehash.googlecode.com/svn/trunk/doc/sparsetable.html return (m_elements.size() / 8) + (m_elements.num_nonempty() * sizeof(TValue)); } - void clear() override final { + void clear() final { m_elements.clear(); } - void dump_as_list(const int fd) override final { + void dump_as_list(const int fd) final { std::vector> v; + v.reserve(m_elements.size()); int n = 0; for (const TValue value : m_elements) { if (value != osmium::index::empty_value()) { diff --git a/third_party/libosmium/include/osmium/index/multimap.hpp b/third_party/libosmium/include/osmium/index/multimap.hpp index c817b6fb6..a7e1aadb3 100644 --- a/third_party/libosmium/include/osmium/index/multimap.hpp +++ b/third_party/libosmium/include/osmium/index/multimap.hpp @@ -118,7 +118,7 @@ namespace osmium { }; // class Multimap - } // namespace map + } // namespace multimap } // namespace index diff --git a/third_party/libosmium/include/osmium/index/multimap/hybrid.hpp b/third_party/libosmium/include/osmium/index/multimap/hybrid.hpp index ac2d96452..e13ee680f 100644 --- a/third_party/libosmium/include/osmium/index/multimap/hybrid.hpp +++ b/third_party/libosmium/include/osmium/index/multimap/hybrid.hpp @@ -62,16 +62,18 @@ namespace osmium { public: - explicit HybridIterator(typename main_map_type::iterator begin_main, - typename main_map_type::iterator end_main, - typename extra_map_type::iterator begin_extra, - typename extra_map_type::iterator end_extra) : + HybridIterator(typename main_map_type::iterator begin_main, + typename main_map_type::iterator end_main, + typename extra_map_type::iterator begin_extra, + typename extra_map_type::iterator end_extra) : m_begin_main(begin_main), m_end_main(end_main), m_begin_extra(begin_extra), m_end_extra(end_extra) { } + ~HybridIterator() noexcept = default; + HybridIterator& operator++() { if (m_begin_main == m_end_main) { ++m_begin_extra; @@ -134,11 +136,13 @@ namespace osmium { m_extra() { } - size_t size() const override final { + ~Hybrid() noexcept = default; + + size_t size() const final { return m_main.size() + m_extra.size(); } - size_t used_memory() const override final { + size_t used_memory() const final { return m_main.used_memory() + m_extra.used_memory(); } @@ -150,7 +154,7 @@ namespace osmium { m_main.set(id, value); } - void set(const TId id, const TValue value) override final { + void set(const TId id, const TValue value) final { m_extra.set(id, value); } @@ -175,17 +179,17 @@ namespace osmium { m_main.sort(); } - void dump_as_list(const int fd) override final { + void dump_as_list(const int fd) final { consolidate(); m_main.dump_as_list(fd); } - void clear() override final { + void clear() final { m_main.clear(); m_extra.clear(); } - void sort() override final { + void sort() final { m_main.sort(); } diff --git a/third_party/libosmium/include/osmium/index/multimap/sparse_mem_multimap.hpp b/third_party/libosmium/include/osmium/index/multimap/sparse_mem_multimap.hpp index 5b4715279..84cb640d9 100644 --- a/third_party/libosmium/include/osmium/index/multimap/sparse_mem_multimap.hpp +++ b/third_party/libosmium/include/osmium/index/multimap/sparse_mem_multimap.hpp @@ -78,13 +78,13 @@ namespace osmium { SparseMemMultimap() = default; - ~SparseMemMultimap() noexcept override final = default; + ~SparseMemMultimap() noexcept final = default; void unsorted_set(const TId id, const TValue value) { m_elements.emplace(id, value); } - void set(const TId id, const TValue value) override final { + void set(const TId id, const TValue value) final { m_elements.emplace(id, value); } @@ -114,15 +114,15 @@ namespace osmium { return m_elements.end(); } - size_t size() const override final { + size_t size() const final { return m_elements.size(); } - size_t used_memory() const override final { + size_t used_memory() const final { return element_size * m_elements.size(); } - void clear() override final { + void clear() final { m_elements.clear(); } @@ -130,12 +130,12 @@ namespace osmium { // intentionally left blank } - void dump_as_list(const int fd) override final { + void dump_as_list(const int fd) final { std::vector v; + v.reserve(m_elements.size()); for (const auto& element : m_elements) { v.emplace_back(element.first, element.second); } -// std::copy(m_elements.cbegin(), m_elements.cend(), std::back_inserter(v)); std::sort(v.begin(), v.end()); osmium::io::detail::reliable_write(fd, reinterpret_cast(v.data()), sizeof(element_type) * v.size()); } diff --git a/third_party/libosmium/include/osmium/io/any_input.hpp b/third_party/libosmium/include/osmium/io/any_input.hpp index d16d069a5..36f43b785 100644 --- a/third_party/libosmium/include/osmium/io/any_input.hpp +++ b/third_party/libosmium/include/osmium/io/any_input.hpp @@ -47,5 +47,6 @@ DEALINGS IN THE SOFTWARE. #include // IWYU pragma: export #include // IWYU pragma: export +#include // IWYU pragma: export #endif // OSMIUM_IO_ANY_INPUT_HPP diff --git a/third_party/libosmium/include/osmium/io/bzip2_compression.hpp b/third_party/libosmium/include/osmium/io/bzip2_compression.hpp index e961a87ab..fc0e33c53 100644 --- a/third_party/libosmium/include/osmium/io/bzip2_compression.hpp +++ b/third_party/libosmium/include/osmium/io/bzip2_compression.hpp @@ -55,7 +55,9 @@ DEALINGS IN THE SOFTWARE. #endif #include +#include #include +#include #include #include @@ -65,13 +67,13 @@ namespace osmium { * Exception thrown when there are problems compressing or * decompressing bzip2 files. */ - struct bzip2_error : public std::runtime_error { + struct bzip2_error : public io_error { int bzip2_error_code; int system_errno; bzip2_error(const std::string& what, int error_code) : - std::runtime_error(what), + io_error(what), bzip2_error_code(error_code), system_errno(error_code == BZ_IO_ERROR ? errno : 0) { } @@ -105,8 +107,8 @@ namespace osmium { public: - explicit Bzip2Compressor(int fd) : - Compressor(), + explicit Bzip2Compressor(int fd, fsync sync) : + Compressor(sync), m_file(fdopen(dup(fd), "wb")), m_bzerror(BZ_OK), m_bzfile(::BZ2_bzWriteOpen(&m_bzerror, m_file, 6, 0, 0)) { @@ -115,11 +117,15 @@ namespace osmium { } } - ~Bzip2Compressor() override final { - close(); + ~Bzip2Compressor() noexcept final { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } } - void write(const std::string& data) override final { + void write(const std::string& data) final { int error; ::BZ2_bzWrite(&error, m_bzfile, const_cast(data.data()), static_cast_with_assert(data.size())); if (error != BZ_OK && error != BZ_STREAM_END) { @@ -127,13 +133,18 @@ namespace osmium { } } - void close() override final { + void close() final { if (m_bzfile) { int error; ::BZ2_bzWriteClose(&error, m_bzfile, 0, nullptr, nullptr); m_bzfile = nullptr; if (m_file) { - fclose(m_file); + if (do_fsync()) { + osmium::io::detail::reliable_fsync(::fileno(m_file)); + } + if (fclose(m_file) != 0) { + throw std::system_error(errno, std::system_category(), "Close failed"); + } } if (error != BZ_OK) { detail::throw_bzip2_error(m_bzfile, "write close failed", error); @@ -152,7 +163,7 @@ namespace osmium { public: - Bzip2Decompressor(int fd) : + explicit Bzip2Decompressor(int fd) : Decompressor(), m_file(fdopen(dup(fd), "rb")), m_bzerror(BZ_OK), @@ -162,11 +173,15 @@ namespace osmium { } } - ~Bzip2Decompressor() override final { - close(); + ~Bzip2Decompressor() noexcept final { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } } - std::string read() override final { + std::string read() final { std::string buffer; if (!m_stream_end) { @@ -203,13 +218,15 @@ namespace osmium { return buffer; } - void close() override final { + void close() final { if (m_bzfile) { int error; ::BZ2_bzReadClose(&error, m_bzfile); m_bzfile = nullptr; if (m_file) { - fclose(m_file); + if (fclose(m_file) != 0) { + throw std::system_error(errno, std::system_category(), "Close failed"); + } } if (error != BZ_OK) { detail::throw_bzip2_error(m_bzfile, "read close failed", error); @@ -240,11 +257,15 @@ namespace osmium { } } - ~Bzip2BufferDecompressor() override final { - BZ2_bzDecompressEnd(&m_bzstream); + ~Bzip2BufferDecompressor() noexcept final { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } } - std::string read() override final { + std::string read() final { std::string output; if (m_buffer) { @@ -270,22 +291,28 @@ namespace osmium { return output; } + void close() final { + BZ2_bzDecompressEnd(&m_bzstream); + } + }; // class Bzip2BufferDecompressor - namespace { + namespace detail { -// we want the register_compression() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" + // we want the register_compression() function to run, setting + // the variable is only a side-effect, it will never be used const bool registered_bzip2_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::bzip2, - [](int fd) { return new osmium::io::Bzip2Compressor(fd); }, + [](int fd, fsync sync) { return new osmium::io::Bzip2Compressor(fd, sync); }, [](int fd) { return new osmium::io::Bzip2Decompressor(fd); }, [](const char* buffer, size_t size) { return new osmium::io::Bzip2BufferDecompressor(buffer, size); } ); -#pragma GCC diagnostic pop - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_bzip2_compression() noexcept { + return registered_bzip2_compression; + } + + } // namespace detail } // namespace io diff --git a/third_party/libosmium/include/osmium/io/compression.hpp b/third_party/libosmium/include/osmium/io/compression.hpp index 252976181..b337503ca 100644 --- a/third_party/libosmium/include/osmium/io/compression.hpp +++ b/third_party/libosmium/include/osmium/io/compression.hpp @@ -40,6 +40,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include #include #ifndef _MSC_VER @@ -49,7 +50,9 @@ DEALINGS IN THE SOFTWARE. #endif #include +#include #include +#include #include namespace osmium { @@ -58,11 +61,21 @@ namespace osmium { class Compressor { + fsync m_fsync; + + protected: + + bool do_fsync() const { + return m_fsync == fsync::yes; + } + public: - Compressor() = default; + explicit Compressor(fsync sync) : + m_fsync(sync) { + } - virtual ~Compressor() { + virtual ~Compressor() noexcept { } virtual void write(const std::string& data) = 0; @@ -85,13 +98,12 @@ namespace osmium { Decompressor(Decompressor&&) = delete; Decompressor& operator=(Decompressor&&) = delete; - virtual ~Decompressor() { + virtual ~Decompressor() noexcept { } virtual std::string read() = 0; - virtual void close() { - } + virtual void close() = 0; }; // class Decompressor @@ -106,13 +118,16 @@ namespace osmium { public: - typedef std::function create_compressor_type; + typedef std::function create_compressor_type; typedef std::function create_decompressor_type_fd; typedef std::function create_decompressor_type_buffer; private: - typedef std::map> compression_map_type; + typedef std::map> compression_map_type; compression_map_type m_callbacks; @@ -128,7 +143,7 @@ namespace osmium { std::string error_message {"Support for compression '"}; error_message += as_string(compression); error_message += "' not compiled into this binary."; - throw std::runtime_error(error_message); + throw unsupported_file_format_error(error_message); } public: @@ -144,15 +159,20 @@ namespace osmium { create_decompressor_type_fd create_decompressor_fd, create_decompressor_type_buffer create_decompressor_buffer) { - compression_map_type::value_type cc(compression, std::make_tuple(create_compressor, create_decompressor_fd, create_decompressor_buffer)); + compression_map_type::value_type cc(compression, + std::make_tuple(create_compressor, + create_decompressor_fd, + create_decompressor_buffer)); + return m_callbacks.insert(cc).second; } - std::unique_ptr create_compressor(osmium::io::file_compression compression, int fd) { + template + std::unique_ptr create_compressor(osmium::io::file_compression compression, TArgs&&... args) { auto it = m_callbacks.find(compression); if (it != m_callbacks.end()) { - return std::unique_ptr(std::get<0>(it->second)(fd)); + return std::unique_ptr(std::get<0>(it->second)(std::forward(args)...)); } error(compression); @@ -186,23 +206,31 @@ namespace osmium { public: - NoCompressor(int fd) : - Compressor(), + NoCompressor(int fd, fsync sync) : + Compressor(sync), m_fd(fd) { } - ~NoCompressor() override final { - close(); + ~NoCompressor() noexcept final { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } } - void write(const std::string& data) override final { + void write(const std::string& data) final { osmium::io::detail::reliable_write(m_fd, data.data(), data.size()); } - void close() override final { + void close() final { if (m_fd >= 0) { - ::close(m_fd); + int fd = m_fd; m_fd = -1; + if (do_fsync()) { + osmium::io::detail::reliable_fsync(fd); + } + osmium::io::detail::reliable_close(fd); } } @@ -216,7 +244,7 @@ namespace osmium { public: - NoDecompressor(int fd) : + explicit NoDecompressor(int fd) : Decompressor(), m_fd(fd), m_buffer(nullptr), @@ -230,11 +258,15 @@ namespace osmium { m_buffer_size(size) { } - ~NoDecompressor() override final { - close(); + ~NoDecompressor() noexcept final { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } } - std::string read() override final { + std::string read() final { std::string buffer; if (m_buffer) { @@ -249,35 +281,38 @@ namespace osmium { if (nread < 0) { throw std::system_error(errno, std::system_category(), "Read failed"); } - buffer.resize(nread); + buffer.resize(std::string::size_type(nread)); } return buffer; } - void close() override final { + void close() final { if (m_fd >= 0) { - ::close(m_fd); + int fd = m_fd; m_fd = -1; + osmium::io::detail::reliable_close(fd); } } }; // class NoDecompressor - namespace { + namespace detail { -// we want the register_compression() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" + // we want the register_compression() function to run, setting + // the variable is only a side-effect, it will never be used const bool registered_no_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::none, - [](int fd) { return new osmium::io::NoCompressor(fd); }, + [](int fd, fsync sync) { return new osmium::io::NoCompressor(fd, sync); }, [](int fd) { return new osmium::io::NoDecompressor(fd); }, [](const char* buffer, size_t size) { return new osmium::io::NoDecompressor(buffer, size); } ); -#pragma GCC diagnostic pop - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_no_compression() noexcept { + return registered_no_compression; + } + + } // namespace detail } // namespace io diff --git a/third_party/libosmium/include/osmium/io/detail/debug_output_format.hpp b/third_party/libosmium/include/osmium/io/detail/debug_output_format.hpp index efecc5833..f1766dea6 100644 --- a/third_party/libosmium/include/osmium/io/detail/debug_output_format.hpp +++ b/third_party/libosmium/include/osmium/io/detail/debug_output_format.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include #include @@ -41,14 +40,10 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include #include #include #include -#include - -#include #include #include #include @@ -87,65 +82,32 @@ namespace osmium { constexpr const char* color_white = "\x1b[37m"; constexpr const char* color_reset = "\x1b[0m"; + struct debug_output_options { + + /// Should metadata of objects be added? + bool add_metadata; + + /// Output with ANSI colors? + bool use_color; + + }; + /** * Writes out one buffer with OSM data in Debug format. */ - class DebugOutputBlock : public osmium::handler::Handler { + class DebugOutputBlock : public OutputBlock { - static constexpr size_t tmp_buffer_size = 50; + debug_output_options m_options; - std::shared_ptr m_input_buffer; - - std::shared_ptr m_out; - - char m_tmp_buffer[tmp_buffer_size+1]; - - bool m_add_metadata; - bool m_use_color; - - template - void output_formatted(const char* format, TArgs&&... args) { -#ifndef NDEBUG - int len = -#endif -#ifndef _MSC_VER - snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward(args)...); -#else - _snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward(args)...); -#endif - assert(len > 0 && static_cast(len) < tmp_buffer_size); - *m_out += m_tmp_buffer; - } + const char* m_utf8_prefix = ""; + const char* m_utf8_suffix = ""; void append_encoded_string(const char* data) { - const char* end = data + std::strlen(data); - - while (data != end) { - const char* last = data; - uint32_t c = utf8::next(data, end); - - // This is a list of Unicode code points that we let - // through instead of escaping them. It is incomplete - // and can be extended later. - // Generally we don't want to let through any - // non-printing characters. - if ((0x0020 <= c && c <= 0x0021) || - (0x0023 <= c && c <= 0x003b) || - (0x003d == c) || - (0x003f <= c && c <= 0x007e) || - (0x00a1 <= c && c <= 0x00ac) || - (0x00ae <= c && c <= 0x05ff)) { - m_out->append(last, data); - } else { - write_color(color_red); - output_formatted("", c); - write_color(color_blue); - } - } + append_debug_encoded_string(*m_out, data, m_utf8_prefix, m_utf8_suffix); } void write_color(const char* color) { - if (m_use_color) { + if (m_options.use_color) { *m_out += color; } } @@ -177,15 +139,38 @@ namespace osmium { *m_out += ": "; } + void write_comment_field(const char* name) { + write_color(color_cyan); + *m_out += name; + write_color(color_reset); + *m_out += ": "; + } + + void write_counter(int width, int n) { + write_color(color_white); + output_formatted(" %0*d: ", width, n++); + write_color(color_reset); + } + void write_error(const char* msg) { write_color(color_red); *m_out += msg; write_color(color_reset); } + void write_timestamp(const osmium::Timestamp& timestamp) { + if (timestamp.valid()) { + *m_out += timestamp.to_iso(); + output_formatted(" (%d)", timestamp.seconds_since_epoch()); + } else { + write_error("NOT SET"); + } + *m_out += '\n'; + } + void write_meta(const osmium::OSMObject& object) { output_formatted("%" PRId64 "\n", object.id()); - if (m_add_metadata) { + if (m_options.add_metadata) { write_fieldname("version"); output_formatted(" %d", object.version()); if (object.visible()) { @@ -196,8 +181,7 @@ namespace osmium { write_fieldname("changeset"); output_formatted("%d\n", object.changeset()); write_fieldname("timestamp"); - *m_out += object.timestamp().to_iso(); - output_formatted(" (%d)\n", object.timestamp()); + write_timestamp(object.timestamp()); write_fieldname("user"); output_formatted(" %d ", object.uid()); write_string(object.user()); @@ -211,14 +195,14 @@ namespace osmium { *m_out += padding; output_formatted(" %d\n", tags.size()); - osmium::max_op max; + osmium::max_op max; for (const auto& tag : tags) { max.update(std::strlen(tag.key())); } for (const auto& tag : tags) { *m_out += " "; write_string(tag.key()); - int spacing = max() - std::strlen(tag.key()); + auto spacing = max() - std::strlen(tag.key()); while (spacing--) { *m_out += " "; } @@ -255,12 +239,11 @@ namespace osmium { public: - explicit DebugOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata, bool use_color) : - m_input_buffer(std::make_shared(std::move(buffer))), - m_out(std::make_shared()), - m_tmp_buffer(), - m_add_metadata(add_metadata), - m_use_color(use_color) { + DebugOutputBlock(osmium::memory::Buffer&& buffer, const debug_output_options& options) : + OutputBlock(std::move(buffer)), + m_options(options), + m_utf8_prefix(options.use_color ? color_red : ""), + m_utf8_suffix(options.use_color ? color_blue : "") { } DebugOutputBlock(const DebugOutputBlock&) = default; @@ -269,13 +252,15 @@ namespace osmium { DebugOutputBlock(DebugOutputBlock&&) = default; DebugOutputBlock& operator=(DebugOutputBlock&&) = default; - ~DebugOutputBlock() = default; + ~DebugOutputBlock() noexcept = default; std::string operator()() { osmium::apply(m_input_buffer->cbegin(), m_input_buffer->cend(), *this); std::string out; - std::swap(out, *m_out); + using std::swap; + swap(out, *m_out); + return out; } @@ -313,7 +298,8 @@ namespace osmium { int width = int(log10(way.nodes().size())) + 1; int n = 0; for (const auto& node_ref : way.nodes()) { - output_formatted(" %0*d: %10" PRId64, width, n++, node_ref.ref()); + write_counter(width, n++); + output_formatted("%10" PRId64, node_ref.ref()); if (node_ref.location().valid()) { output_formatted(" (%.7f,%.7f)", node_ref.location().lon_without_check(), node_ref.location().lat_without_check()); } @@ -335,7 +321,7 @@ namespace osmium { int width = int(log10(relation.members().size())) + 1; int n = 0; for (const auto& member : relation.members()) { - output_formatted(" %0*d: ", width, n++); + write_counter(width, n++); *m_out += short_typename[item_type_to_nwr_index(member.type())]; output_formatted(" %10" PRId64 " ", member.ref()); write_string(member.role()); @@ -348,24 +334,26 @@ namespace osmium { void changeset(const osmium::Changeset& changeset) { write_object_type("changeset"); output_formatted("%d\n", changeset.id()); + write_fieldname("num changes"); output_formatted("%d", changeset.num_changes()); if (changeset.num_changes() == 0) { write_error(" NO CHANGES!"); } *m_out += '\n'; + write_fieldname("created at"); *m_out += ' '; - *m_out += changeset.created_at().to_iso(); - output_formatted(" (%d)\n", changeset.created_at()); + write_timestamp(changeset.created_at()); + write_fieldname("closed at"); *m_out += " "; if (changeset.closed()) { - *m_out += changeset.closed_at().to_iso(); - output_formatted(" (%d)\n", changeset.closed_at()); + write_timestamp(changeset.closed_at()); } else { write_error("OPEN!\n"); } + write_fieldname("user"); output_formatted(" %d ", changeset.uid()); write_string(changeset.user()); @@ -374,51 +362,73 @@ namespace osmium { write_box(changeset.bounds()); write_tags(changeset.tags(), " "); + if (changeset.num_comments() > 0) { + write_fieldname("comments"); + output_formatted(" %d\n", changeset.num_comments()); + + int width = int(log10(changeset.num_comments())) + 1; + int n = 0; + for (const auto& comment : changeset.discussion()) { + write_counter(width, n++); + + write_comment_field("date"); + write_timestamp(comment.date()); + output_formatted(" %*s", width, ""); + + write_comment_field("user"); + output_formatted("%d ", comment.uid()); + write_string(comment.user()); + output_formatted("\n %*s", width, ""); + + write_comment_field("text"); + write_string(comment.text()); + *m_out += '\n'; + } + } + *m_out += '\n'; } - }; // DebugOutputBlock + }; // class DebugOutputBlock class DebugOutputFormat : public osmium::io::detail::OutputFormat { - bool m_add_metadata; - bool m_use_color; - - public: - - DebugOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : - OutputFormat(file, output_queue), - m_add_metadata(file.get("add_metadata") != "false"), - m_use_color(file.get("color") == "true") { - } - - DebugOutputFormat(const DebugOutputFormat&) = delete; - DebugOutputFormat& operator=(const DebugOutputFormat&) = delete; - - void write_buffer(osmium::memory::Buffer&& buffer) override final { - m_output_queue.push(osmium::thread::Pool::instance().submit(DebugOutputBlock{std::move(buffer), m_add_metadata, m_use_color})); - } + debug_output_options m_options; void write_fieldname(std::string& out, const char* name) { out += " "; - if (m_use_color) { + if (m_options.use_color) { out += color_cyan; } out += name; - if (m_use_color) { + if (m_options.use_color) { out += color_reset; } out += ": "; } - void write_header(const osmium::io::Header& header) override final { + public: + + DebugOutputFormat(const osmium::io::File& file, future_string_queue_type& output_queue) : + OutputFormat(output_queue), + m_options() { + m_options.add_metadata = file.is_not_false("add_metadata"); + m_options.use_color = file.is_true("color"); + } + + DebugOutputFormat(const DebugOutputFormat&) = delete; + DebugOutputFormat& operator=(const DebugOutputFormat&) = delete; + + ~DebugOutputFormat() noexcept final = default; + + void write_header(const osmium::io::Header& header) final { std::string out; - if (m_use_color) { + if (m_options.use_color) { out += color_bold; } out += "header\n"; - if (m_use_color) { + if (m_options.use_color) { out += color_reset; } @@ -445,33 +455,26 @@ namespace osmium { } out += "\n=============================================\n\n"; - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(std::move(out)); + send_to_output_queue(std::move(out)); } - void close() override final { - std::string out; - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(out); + void write_buffer(osmium::memory::Buffer&& buffer) final { + m_output_queue.push(osmium::thread::Pool::instance().submit(DebugOutputBlock{std::move(buffer), m_options})); } }; // class DebugOutputFormat - namespace { + // we want the register_output_format() function to run, setting + // the variable is only a side-effect, it will never be used + const bool registered_debug_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::debug, + [](const osmium::io::File& file, future_string_queue_type& output_queue) { + return new osmium::io::detail::DebugOutputFormat(file, output_queue); + }); -// we want the register_output_format() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" - const bool registered_debug_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::debug, - [](const osmium::io::File& file, data_queue_type& output_queue) { - return new osmium::io::detail::DebugOutputFormat(file, output_queue); - }); -#pragma GCC diagnostic pop - - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_debug_output() noexcept { + return registered_debug_output; + } } // namespace detail diff --git a/third_party/libosmium/include/osmium/io/detail/input_format.hpp b/third_party/libosmium/include/osmium/io/detail/input_format.hpp index 03e1190c3..d26b1ee92 100644 --- a/third_party/libosmium/include/osmium/io/detail/input_format.hpp +++ b/third_party/libosmium/include/osmium/io/detail/input_format.hpp @@ -33,13 +33,16 @@ DEALINGS IN THE SOFTWARE. */ +#include #include +#include #include #include #include #include #include +#include #include #include #include @@ -48,106 +51,156 @@ DEALINGS IN THE SOFTWARE. namespace osmium { - namespace thread { - template class Queue; - } // namespace thread - namespace io { namespace detail { - /** - * Virtual base class for all classes reading OSM files in different - * formats. - * - * Do not use this class or derived classes directly. Use the - * osmium::io::Reader class instead. - */ - class InputFormat { + class Parser { + + future_buffer_queue_type& m_output_queue; + std::promise& m_header_promise; + queue_wrapper m_input_queue; + osmium::osm_entity_bits::type m_read_types; + bool m_header_is_done; protected: - osmium::io::File m_file; - osmium::osm_entity_bits::type m_read_which_entities; - osmium::io::Header m_header; - - explicit InputFormat(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities) : - m_file(file), - m_read_which_entities(read_which_entities) { - m_header.set_has_multiple_object_versions(m_file.has_multiple_object_versions()); + std::string get_input() { + return m_input_queue.pop(); } - InputFormat(const InputFormat&) = delete; - InputFormat(InputFormat&&) = delete; + bool input_done() const { + return m_input_queue.has_reached_end_of_data(); + } - InputFormat& operator=(const InputFormat&) = delete; - InputFormat& operator=(InputFormat&&) = delete; + osmium::osm_entity_bits::type read_types() const { + return m_read_types; + } + + bool header_is_done() const { + return m_header_is_done; + } + + void set_header_value(const osmium::io::Header& header) { + if (!m_header_is_done) { + m_header_is_done = true; + m_header_promise.set_value(header); + } + } + + void set_header_exception(const std::exception_ptr& exception) { + if (!m_header_is_done) { + m_header_is_done = true; + m_header_promise.set_exception(exception); + } + } + + /** + * Wrap the buffer into a future and add it to the output queue. + */ + void send_to_output_queue(osmium::memory::Buffer&& buffer) { + add_to_queue(m_output_queue, std::move(buffer)); + } + + void send_to_output_queue(std::future&& future) { + m_output_queue.push(std::move(future)); + } public: - virtual ~InputFormat() { + Parser(future_string_queue_type& input_queue, + future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_types) : + m_output_queue(output_queue), + m_header_promise(header_promise), + m_input_queue(input_queue), + m_read_types(read_types), + m_header_is_done(false) { } - virtual osmium::memory::Buffer read() = 0; + Parser(const Parser&) = delete; + Parser& operator=(const Parser&) = delete; - virtual void close() { + Parser(Parser&&) = delete; + Parser& operator=(Parser&&) = delete; + + virtual ~Parser() noexcept = default; + + virtual void run() = 0; + + void parse() { + try { + run(); + } catch (...) { + std::exception_ptr exception = std::current_exception(); + set_header_exception(exception); + add_to_queue(m_output_queue, std::move(exception)); + } + + add_end_of_data_to_queue(m_output_queue); } - virtual osmium::io::Header header() { - return m_header; - } - - }; // class InputFormat + }; // class Parser /** - * This factory class is used to create objects that read OSM data - * written in a specified format. + * This factory class is used to create objects that decode OSM + * data written in a specified format. * - * Do not use this class directly. Instead use the osmium::io::Reader - * class. + * Do not use this class directly. Use the osmium::io::Reader + * class instead. */ - class InputFormatFactory { + class ParserFactory { public: - typedef std::function&)> create_input_type; + typedef std::function< + std::unique_ptr( + future_string_queue_type&, + future_buffer_queue_type&, + std::promise& header_promise, + osmium::osm_entity_bits::type read_which_entities + ) + > create_parser_type; private: - typedef std::map map_type; + typedef std::map map_type; map_type m_callbacks; - InputFormatFactory() : + ParserFactory() : m_callbacks() { } public: - static InputFormatFactory& instance() { - static InputFormatFactory factory; + static ParserFactory& instance() { + static ParserFactory factory; return factory; } - bool register_input_format(osmium::io::file_format format, create_input_type create_function) { + bool register_parser(osmium::io::file_format format, create_parser_type create_function) { if (! m_callbacks.insert(map_type::value_type(format, create_function)).second) { return false; } return true; } - std::unique_ptr create_input(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue& input_queue) { - file.check(); - + create_parser_type get_creator_function(const osmium::io::File& file) { auto it = m_callbacks.find(file.format()); - if (it != m_callbacks.end()) { - return std::unique_ptr((it->second)(file, read_which_entities, input_queue)); + if (it == m_callbacks.end()) { + throw unsupported_file_format_error( + std::string("Can not open file '") + + file.filename() + + "' with type '" + + as_string(file.format()) + + "'. No support for reading this format in this program."); } - - throw std::runtime_error(std::string("Support for input format '") + as_string(file.format()) + "' not compiled into this binary."); + return it->second; } - }; // class InputFormatFactory + }; // class ParserFactory } // namespace detail diff --git a/third_party/libosmium/include/osmium/io/detail/o5m_input_format.hpp b/third_party/libosmium/include/osmium/io/detail/o5m_input_format.hpp new file mode 100644 index 000000000..0318432e8 --- /dev/null +++ b/third_party/libosmium/include/osmium/io/detail/o5m_input_format.hpp @@ -0,0 +1,636 @@ +#ifndef OSMIUM_IO_DETAIL_O5M_INPUT_FORMAT_HPP +#define OSMIUM_IO_DETAIL_O5M_INPUT_FORMAT_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace osmium { + + /** + * Exception thrown when the o5m deocder failed. The exception contains + * (if available) information about the place where the error happened + * and the type of error. + */ + struct o5m_error : public io_error { + + explicit o5m_error(const char* what) : + io_error(std::string("o5m format error: ") + what) { + } + + }; // struct o5m_error + + namespace io { + + namespace detail { + + // Implementation of the o5m/o5c file formats according to the + // description at http://wiki.openstreetmap.org/wiki/O5m . + + class ReferenceTable { + + // The following settings are from the o5m description: + + // The maximum number of entries in this table. + const uint64_t number_of_entries = 15000; + + // The size of one entry in the table. + const unsigned int entry_size = 256; + + // The maximum length of a string in the table including + // two \0 bytes. + const unsigned int max_length = 250 + 2; + + // The data is stored in this string. It is default constructed + // and then resized on demand the first time something is added. + // This is done because the ReferenceTable is in a O5mParser + // object which will be copied from one thread to another. This + // way the string is still small when it is copied. + std::string m_table; + + unsigned int current_entry = 0; + + public: + + void clear() { + current_entry = 0; + } + + void add(const char* string, size_t size) { + if (m_table.empty()) { + m_table.resize(entry_size * number_of_entries); + } + if (size <= max_length) { + std::copy_n(string, size, &m_table[current_entry * entry_size]); + if (++current_entry == number_of_entries) { + current_entry = 0; + } + } + } + + const char* get(uint64_t index) const { + if (m_table.empty() || index == 0 || index > number_of_entries) { + throw o5m_error("reference to non-existing string in table"); + } + auto entry = (current_entry + number_of_entries - index) % number_of_entries; + return &m_table[entry * entry_size]; + } + + }; // class ReferenceTable + + class O5mParser : public Parser { + + static constexpr int buffer_size = 2 * 1000 * 1000; + + osmium::io::Header m_header; + + osmium::memory::Buffer m_buffer; + + std::string m_input; + + const char* m_data; + const char* m_end; + + ReferenceTable m_reference_table; + + static int64_t zvarint(const char** data, const char* end) { + return protozero::decode_zigzag64(protozero::decode_varint(data, end)); + } + + bool ensure_bytes_available(size_t need_bytes) { + if ((m_end - m_data) >= long(need_bytes)) { + return true; + } + + if (input_done() && (m_input.size() < need_bytes)) { + return false; + } + + m_input.erase(0, m_data - m_input.data()); + + while (m_input.size() < need_bytes) { + std::string data = get_input(); + if (input_done()) { + return false; + } + m_input.append(data); + } + + m_data = m_input.data(); + m_end = m_input.data() + m_input.size(); + + return true; + } + + void check_header_magic() { + static const unsigned char header_magic[] = { 0xff, 0xe0, 0x04, 'o', '5' }; + + if (std::strncmp(reinterpret_cast(header_magic), m_data, sizeof(header_magic))) { + throw o5m_error("wrong header magic"); + } + + m_data += sizeof(header_magic); + } + + void check_file_type() { + if (*m_data == 'm') { // o5m data file + m_header.set_has_multiple_object_versions(false); + } else if (*m_data == 'c') { // o5c change file + m_header.set_has_multiple_object_versions(true); + } else { + throw o5m_error("wrong header magic"); + } + + m_data++; + } + + void check_file_format_version() { + if (*m_data != '2') { + throw o5m_error("wrong header magic"); + } + + m_data++; + } + + void decode_header() { + if (! ensure_bytes_available(7)) { // overall length of header + throw o5m_error("file too short (incomplete header info)"); + } + + check_header_magic(); + check_file_type(); + check_file_format_version(); + } + + void mark_header_as_done() { + set_header_value(m_header); + } + + osmium::util::DeltaDecode m_delta_id; + + osmium::util::DeltaDecode m_delta_timestamp; + osmium::util::DeltaDecode m_delta_changeset; + osmium::util::DeltaDecode m_delta_lon; + osmium::util::DeltaDecode m_delta_lat; + + osmium::util::DeltaDecode m_delta_way_node_id; + osmium::util::DeltaDecode m_delta_member_ids[3]; + + void reset() { + m_reference_table.clear(); + + m_delta_id.clear(); + m_delta_timestamp.clear(); + m_delta_changeset.clear(); + m_delta_lon.clear(); + m_delta_lat.clear(); + + m_delta_way_node_id.clear(); + m_delta_member_ids[0].clear(); + m_delta_member_ids[1].clear(); + m_delta_member_ids[2].clear(); + } + + const char* decode_string(const char** dataptr, const char* const end) { + if (**dataptr == 0x00) { // get inline string + (*dataptr)++; + if (*dataptr == end) { + throw o5m_error("string format error"); + } + return *dataptr; + } else { // get from reference table + auto index = protozero::decode_varint(dataptr, end); + return m_reference_table.get(index); + } + } + + std::pair decode_user(const char** dataptr, const char* const end) { + bool update_pointer = (**dataptr == 0x00); + const char* data = decode_string(dataptr, end); + const char* start = data; + + auto uid = protozero::decode_varint(&data, end); + + if (data == end) { + throw o5m_error("missing user name"); + } + + const char* user = ++data; + + if (uid == 0 && update_pointer) { + m_reference_table.add("\0\0", 2); + *dataptr = data; + return std::make_pair(0, ""); + } + + while (*data++) { + if (data == end) { + throw o5m_error("no null byte in user name"); + } + } + + if (update_pointer) { + m_reference_table.add(start, data - start); + *dataptr = data; + } + + return std::make_pair(static_cast_with_assert(uid), user); + } + + void decode_tags(osmium::builder::Builder* builder, const char** dataptr, const char* const end) { + osmium::builder::TagListBuilder tl_builder(m_buffer, builder); + + while(*dataptr != end) { + bool update_pointer = (**dataptr == 0x00); + const char* data = decode_string(dataptr, end); + const char* start = data; + + while (*data++) { + if (data == end) { + throw o5m_error("no null byte in tag key"); + } + } + + const char* value = data; + while (*data++) { + if (data == end) { + throw o5m_error("no null byte in tag value"); + } + } + + if (update_pointer) { + m_reference_table.add(start, data - start); + *dataptr = data; + } + + tl_builder.add_tag(start, value); + } + } + + const char* decode_info(osmium::OSMObject& object, const char** dataptr, const char* const end) { + const char* user = ""; + + if (**dataptr == 0x00) { // no info section + ++*dataptr; + } else { // has info section + object.set_version(static_cast_with_assert(protozero::decode_varint(dataptr, end))); + auto timestamp = m_delta_timestamp.update(zvarint(dataptr, end)); + if (timestamp != 0) { // has timestamp + object.set_timestamp(timestamp); + object.set_changeset(m_delta_changeset.update(zvarint(dataptr, end))); + if (*dataptr != end) { + auto uid_user = decode_user(dataptr, end); + object.set_uid(uid_user.first); + user = uid_user.second; + } else { + object.set_uid(user_id_type(0)); + } + } + } + + return user; + } + + void decode_node(const char* data, const char* const end) { + osmium::builder::NodeBuilder builder(m_buffer); + osmium::Node& node = builder.object(); + + node.set_id(m_delta_id.update(zvarint(&data, end))); + + builder.add_user(decode_info(node, &data, end)); + + if (data == end) { + // no location, object is deleted + builder.object().set_visible(false); + builder.object().set_location(osmium::Location{}); + } else { + auto lon = m_delta_lon.update(zvarint(&data, end)); + auto lat = m_delta_lat.update(zvarint(&data, end)); + builder.object().set_location(osmium::Location{lon, lat}); + + if (data != end) { + decode_tags(&builder, &data, end); + } + } + + m_buffer.commit(); + } + + void decode_way(const char* data, const char* const end) { + osmium::builder::WayBuilder builder(m_buffer); + osmium::Way& way = builder.object(); + + way.set_id(m_delta_id.update(zvarint(&data, end))); + + builder.add_user(decode_info(way, &data, end)); + + if (data == end) { + // no reference section, object is deleted + builder.object().set_visible(false); + } else { + auto reference_section_length = protozero::decode_varint(&data, end); + if (reference_section_length > 0) { + const char* const end_refs = data + reference_section_length; + if (end_refs > end) { + throw o5m_error("way nodes ref section too long"); + } + + osmium::builder::WayNodeListBuilder wn_builder(m_buffer, &builder); + + while (data < end_refs) { + wn_builder.add_node_ref(m_delta_way_node_id.update(zvarint(&data, end))); + } + } + + if (data != end) { + decode_tags(&builder, &data, end); + } + } + + m_buffer.commit(); + } + + osmium::item_type decode_member_type(char c) { + if (c < '0' || c > '2') { + throw o5m_error("unknown member type"); + } + return osmium::nwr_index_to_item_type(c - '0'); + } + + std::pair decode_role(const char** dataptr, const char* const end) { + bool update_pointer = (**dataptr == 0x00); + const char* data = decode_string(dataptr, end); + const char* start = data; + + auto member_type = decode_member_type(*data++); + if (data == end) { + throw o5m_error("missing role"); + } + const char* role = data; + + while (*data++) { + if (data == end) { + throw o5m_error("no null byte in role"); + } + } + + if (update_pointer) { + m_reference_table.add(start, data - start); + *dataptr = data; + } + + return std::make_pair(member_type, role); + } + + void decode_relation(const char* data, const char* const end) { + osmium::builder::RelationBuilder builder(m_buffer); + osmium::Relation& relation = builder.object(); + + relation.set_id(m_delta_id.update(zvarint(&data, end))); + + builder.add_user(decode_info(relation, &data, end)); + + if (data == end) { + // no reference section, object is deleted + builder.object().set_visible(false); + } else { + auto reference_section_length = protozero::decode_varint(&data, end); + if (reference_section_length > 0) { + const char* const end_refs = data + reference_section_length; + if (end_refs > end) { + throw o5m_error("relation format error"); + } + + osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder); + + while (data < end_refs) { + auto delta_id = zvarint(&data, end); + if (data == end) { + throw o5m_error("relation member format error"); + } + auto type_role = decode_role(&data, end); + auto i = osmium::item_type_to_nwr_index(type_role.first); + auto ref = m_delta_member_ids[i].update(delta_id); + rml_builder.add_member(type_role.first, ref, type_role.second); + } + } + + if (data != end) { + decode_tags(&builder, &data, end); + } + } + + m_buffer.commit(); + } + + void decode_bbox(const char* data, const char* const end) { + auto sw_lon = zvarint(&data, end); + auto sw_lat = zvarint(&data, end); + auto ne_lon = zvarint(&data, end); + auto ne_lat = zvarint(&data, end); + + m_header.add_box(osmium::Box{osmium::Location{sw_lon, sw_lat}, + osmium::Location{ne_lon, ne_lat}}); + } + + void decode_timestamp(const char* data, const char* const end) { + auto timestamp = osmium::Timestamp(zvarint(&data, end)).to_iso(); + m_header.set("o5m_timestamp", timestamp); + m_header.set("timestamp", timestamp); + } + + void flush() { + osmium::memory::Buffer buffer(buffer_size); + using std::swap; + swap(m_buffer, buffer); + send_to_output_queue(std::move(buffer)); + } + + enum class dataset_type : unsigned char { + node = 0x10, + way = 0x11, + relation = 0x12, + bounding_box = 0xdb, + timestamp = 0xdc, + header = 0xe0, + sync = 0xee, + jump = 0xef, + reset = 0xff + }; + + void decode_data() { + while (ensure_bytes_available(1)) { + dataset_type ds_type = dataset_type(*m_data++); + if (ds_type > dataset_type::jump) { + if (ds_type == dataset_type::reset) { + reset(); + } + } else { + ensure_bytes_available(protozero::max_varint_length); + + uint64_t length = 0; + try { + length = protozero::decode_varint(&m_data, m_end); + } catch (protozero::end_of_buffer_exception&) { + throw o5m_error("premature end of file"); + } + + if (! ensure_bytes_available(length)) { + throw o5m_error("premature end of file"); + } + + switch (ds_type) { + case dataset_type::node: + mark_header_as_done(); + if (read_types() & osmium::osm_entity_bits::node) { + decode_node(m_data, m_data + length); + } + break; + case dataset_type::way: + mark_header_as_done(); + if (read_types() & osmium::osm_entity_bits::way) { + decode_way(m_data, m_data + length); + } + break; + case dataset_type::relation: + mark_header_as_done(); + if (read_types() & osmium::osm_entity_bits::relation) { + decode_relation(m_data, m_data + length); + } + break; + case dataset_type::bounding_box: + decode_bbox(m_data, m_data + length); + break; + case dataset_type::timestamp: + decode_timestamp(m_data, m_data + length); + break; + default: + // ignore unknown datasets + break; + } + + if (read_types() == osmium::osm_entity_bits::nothing && header_is_done()) { + break; + } + + m_data += length; + + if (m_buffer.committed() > buffer_size / 10 * 9) { + flush(); + } + } + } + + if (m_buffer.committed()) { + flush(); + } + + mark_header_as_done(); + } + + public: + + O5mParser(future_string_queue_type& input_queue, + future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_types) : + Parser(input_queue, output_queue, header_promise, read_types), + m_header(), + m_buffer(buffer_size), + m_input(), + m_data(m_input.data()), + m_end(m_data) { + } + + ~O5mParser() noexcept final = default; + + void run() final { + osmium::thread::set_thread_name("_osmium_o5m_in"); + + decode_header(); + decode_data(); + } + + }; // class O5mParser + + // we want the register_parser() function to run, setting + // the variable is only a side-effect, it will never be used + const bool registered_o5m_parser = ParserFactory::instance().register_parser( + file_format::o5m, + [](future_string_queue_type& input_queue, + future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_which_entities) { + return std::unique_ptr(new O5mParser(input_queue, output_queue, header_promise, read_which_entities)); + }); + + // dummy function to silence the unused variable warning from above + inline bool get_registered_o5m_parser() noexcept { + return registered_o5m_parser; + } + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_O5M_INPUT_FORMAT_HPP diff --git a/third_party/libosmium/include/osmium/io/detail/opl_output_format.hpp b/third_party/libosmium/include/osmium/io/detail/opl_output_format.hpp index a3103d9bf..2d863ea82 100644 --- a/third_party/libosmium/include/osmium/io/detail/opl_output_format.hpp +++ b/third_party/libosmium/include/osmium/io/detail/opl_output_format.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include #include @@ -41,14 +40,10 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include #include #include #include -#include - -#include #include #include #include @@ -74,71 +69,27 @@ namespace osmium { namespace detail { + struct opl_output_options { + + /// Should metadata of objects be added? + bool add_metadata; + + }; + /** * Writes out one buffer with OSM data in OPL format. */ - class OPLOutputBlock : public osmium::handler::Handler { + class OPLOutputBlock : public OutputBlock { - static constexpr size_t tmp_buffer_size = 100; - - std::shared_ptr m_input_buffer; - - std::shared_ptr m_out; - - char m_tmp_buffer[tmp_buffer_size+1]; - - bool m_add_metadata; - - template - void output_formatted(const char* format, TArgs&&... args) { -#ifndef NDEBUG - int len = -#endif -#ifndef _MSC_VER - snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward(args)...); -#else - _snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward(args)...); -#endif - assert(len > 0 && static_cast(len) < tmp_buffer_size); - *m_out += m_tmp_buffer; - } + opl_output_options m_options; void append_encoded_string(const char* data) { - const char* end = data + std::strlen(data); - - while (data != end) { - const char* last = data; - uint32_t c = utf8::next(data, end); - - // This is a list of Unicode code points that we let - // through instead of escaping them. It is incomplete - // and can be extended later. - // Generally we don't want to let through any character - // that has special meaning in the OPL format such as - // space, comma, @, etc. and any non-printing characters. - if ((0x0021 <= c && c <= 0x0024) || - (0x0026 <= c && c <= 0x002b) || - (0x002d <= c && c <= 0x003c) || - (0x003e <= c && c <= 0x003f) || - (0x0041 <= c && c <= 0x007e) || - (0x00a1 <= c && c <= 0x00ac) || - (0x00ae <= c && c <= 0x05ff)) { - m_out->append(last, data); - } else { - *m_out += '%'; - if (c <= 0xff) { - output_formatted("%02x", c); - } else { - output_formatted("%04x", c); - } - *m_out += '%'; - } - } + osmium::io::detail::append_utf8_encoded_string(*m_out, data); } void write_meta(const osmium::OSMObject& object) { output_formatted("%" PRId64, object.id()); - if (m_add_metadata) { + if (m_options.add_metadata) { output_formatted(" v%d d", object.version()); *m_out += (object.visible() ? 'V' : 'D'); output_formatted(" c%d t", object.changeset()); @@ -160,7 +111,7 @@ namespace osmium { } } - void write_location(const osmium::Location location, const char x, const char y) { + void write_location(const osmium::Location& location, const char x, const char y) { if (location) { output_formatted(" %c%.7f %c%.7f", x, location.lon_without_check(), y, location.lat_without_check()); } else { @@ -173,11 +124,9 @@ namespace osmium { public: - explicit OPLOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata) : - m_input_buffer(std::make_shared(std::move(buffer))), - m_out(std::make_shared()), - m_tmp_buffer(), - m_add_metadata(add_metadata) { + OPLOutputBlock(osmium::memory::Buffer&& buffer, const opl_output_options& options) : + OutputBlock(std::move(buffer)), + m_options(options) { } OPLOutputBlock(const OPLOutputBlock&) = default; @@ -186,13 +135,15 @@ namespace osmium { OPLOutputBlock(OPLOutputBlock&&) = default; OPLOutputBlock& operator=(OPLOutputBlock&&) = default; - ~OPLOutputBlock() = default; + ~OPLOutputBlock() noexcept = default; std::string operator()() { osmium::apply(m_input_buffer->cbegin(), m_input_buffer->cend(), *this); std::string out; - std::swap(out, *m_out); + using std::swap; + swap(out, *m_out); + return out; } @@ -244,7 +195,7 @@ namespace osmium { *m_out += changeset.created_at().to_iso(); *m_out += " e"; *m_out += changeset.closed_at().to_iso(); - output_formatted(" i%d u", changeset.uid()); + output_formatted(" d%d i%d u", changeset.num_comments(), changeset.uid()); append_encoded_string(changeset.user()); write_location(changeset.bounds().bottom_left(), 'x', 'y'); write_location(changeset.bounds().top_right(), 'X', 'Y'); @@ -264,48 +215,42 @@ namespace osmium { *m_out += '\n'; } - }; // OPLOutputBlock + }; // class OPLOutputBlock class OPLOutputFormat : public osmium::io::detail::OutputFormat { - bool m_add_metadata; + opl_output_options m_options; public: - OPLOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : - OutputFormat(file, output_queue), - m_add_metadata(file.get("add_metadata") != "false") { + OPLOutputFormat(const osmium::io::File& file, future_string_queue_type& output_queue) : + OutputFormat(output_queue), + m_options() { + m_options.add_metadata = file.is_not_false("add_metadata"); } OPLOutputFormat(const OPLOutputFormat&) = delete; OPLOutputFormat& operator=(const OPLOutputFormat&) = delete; - void write_buffer(osmium::memory::Buffer&& buffer) override final { - m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer), m_add_metadata})); - } + ~OPLOutputFormat() noexcept final = default; - void close() override final { - std::string out; - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(out); + void write_buffer(osmium::memory::Buffer&& buffer) final { + m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer), m_options})); } }; // class OPLOutputFormat - namespace { + // we want the register_output_format() function to run, setting + // the variable is only a side-effect, it will never be used + const bool registered_opl_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::opl, + [](const osmium::io::File& file, future_string_queue_type& output_queue) { + return new osmium::io::detail::OPLOutputFormat(file, output_queue); + }); -// we want the register_output_format() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" - const bool registered_opl_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::opl, - [](const osmium::io::File& file, data_queue_type& output_queue) { - return new osmium::io::detail::OPLOutputFormat(file, output_queue); - }); -#pragma GCC diagnostic pop - - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_opl_output() noexcept { + return registered_opl_output; + } } // namespace detail diff --git a/third_party/libosmium/include/osmium/io/detail/output_format.hpp b/third_party/libosmium/include/osmium/io/detail/output_format.hpp index 529a1890f..4c38c4d1e 100644 --- a/third_party/libosmium/include/osmium/io/detail/output_format.hpp +++ b/third_party/libosmium/include/osmium/io/detail/output_format.hpp @@ -34,29 +34,48 @@ DEALINGS IN THE SOFTWARE. */ #include -#include #include #include #include #include #include +#include +#include +#include #include #include -#include -#include +#include namespace osmium { - namespace memory { - class Buffer; - } + namespace io { + class Header; + } // namespace io namespace io { namespace detail { - typedef osmium::thread::Queue> data_queue_type; + class OutputBlock : public osmium::handler::Handler { + + protected: + + std::shared_ptr m_input_buffer; + + std::shared_ptr m_out; + + explicit OutputBlock(osmium::memory::Buffer&& buffer) : + m_input_buffer(std::make_shared(std::move(buffer))), + m_out(std::make_shared()) { + } + + template + void output_formatted(const char* format, TArgs&&... args) { + append_printf_formatted_string(*m_out, format, std::forward(args)...); + } + + }; // class OutputBlock; /** * Virtual base class for all classes writing OSM files in different @@ -69,13 +88,19 @@ namespace osmium { protected: - osmium::io::File m_file; - data_queue_type& m_output_queue; + future_string_queue_type& m_output_queue; + + /** + * Wrap the string into a future and add it to the output + * queue. + */ + void send_to_output_queue(std::string&& data) { + add_to_queue(m_output_queue, std::move(data)); + } public: - explicit OutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : - m_file(file), + explicit OutputFormat(future_string_queue_type& output_queue) : m_output_queue(output_queue) { } @@ -85,15 +110,15 @@ namespace osmium { OutputFormat& operator=(const OutputFormat&) = delete; OutputFormat& operator=(OutputFormat&&) = delete; - virtual ~OutputFormat() { - } + virtual ~OutputFormat() noexcept = default; virtual void write_header(const osmium::io::Header&) { } virtual void write_buffer(osmium::memory::Buffer&&) = 0; - virtual void close() = 0; + virtual void write_end() { + } }; // class OutputFormat @@ -108,7 +133,7 @@ namespace osmium { public: - typedef std::function create_output_type; + typedef std::function create_output_type; private: @@ -134,15 +159,18 @@ namespace osmium { return true; } - std::unique_ptr create_output(const osmium::io::File& file, data_queue_type& output_queue) { - file.check(); - + std::unique_ptr create_output(const osmium::io::File& file, future_string_queue_type& output_queue) { auto it = m_callbacks.find(file.format()); if (it != m_callbacks.end()) { return std::unique_ptr((it->second)(file, output_queue)); } - throw std::runtime_error(std::string("Support for output format '") + as_string(file.format()) + "' not compiled into this binary."); + throw unsupported_file_format_error( + std::string("Can not open file '") + + file.filename() + + "' with type '" + + as_string(file.format()) + + "'. No support for writing this format in this program."); } }; // class OutputFormatFactory diff --git a/third_party/libosmium/include/osmium/io/detail/pbf.hpp b/third_party/libosmium/include/osmium/io/detail/pbf.hpp index 15e457a12..13d552910 100644 --- a/third_party/libosmium/include/osmium/io/detail/pbf.hpp +++ b/third_party/libosmium/include/osmium/io/detail/pbf.hpp @@ -33,6 +33,7 @@ DEALINGS IN THE SOFTWARE. */ +#include #include // needed for htonl and ntohl @@ -53,11 +54,11 @@ namespace osmium { */ struct pbf_error : public io_error { - pbf_error(const std::string& what) : + explicit pbf_error(const std::string& what) : io_error(std::string("PBF error: ") + what) { } - pbf_error(const char* what) : + explicit pbf_error(const char* what) : io_error(std::string("PBF error: ") + what) { } @@ -79,9 +80,9 @@ namespace osmium { const int64_t resolution_convert = lonlat_resolution / osmium::Location::coordinate_precision; - } + } // namespace detail - } + } // namespace io } // namespace osmium diff --git a/third_party/libosmium/include/osmium/io/detail/pbf_decoder.hpp b/third_party/libosmium/include/osmium/io/detail/pbf_decoder.hpp index 79e899ff8..09e09bf02 100644 --- a/third_party/libosmium/include/osmium/io/detail/pbf_decoder.hpp +++ b/third_party/libosmium/include/osmium/io/detail/pbf_decoder.hpp @@ -37,8 +37,12 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include #include +#include +#include +#include +#include +#include #include @@ -62,13 +66,14 @@ namespace osmium { namespace detail { using ptr_len_type = std::pair; + using osm_string_len_type = std::pair; class PBFPrimitiveBlockDecoder { static constexpr size_t initial_buffer_size = 2 * 1024 * 1024; ptr_len_type m_data; - std::vector m_stringtable; + std::vector m_stringtable; int64_t m_lon_offset = 0; int64_t m_lat_offset = 0; @@ -86,7 +91,11 @@ namespace osmium { protozero::pbf_message pbf_string_table(data); while (pbf_string_table.next(OSMFormat::StringTable::repeated_bytes_s)) { - m_stringtable.push_back(pbf_string_table.get_data()); + auto str_len = pbf_string_table.get_data(); + if (str_len.second > osmium::max_osm_string_length) { + throw osmium::pbf_error("overlong string in string table"); + } + m_stringtable.emplace_back(str_len.first, osmium::string_size_type(str_len.second)); } } @@ -156,8 +165,8 @@ namespace osmium { } } - ptr_len_type decode_info(const ptr_len_type& data, osmium::OSMObject& object) { - ptr_len_type user = std::make_pair("", 0); + osm_string_len_type decode_info(const ptr_len_type& data, osmium::OSMObject& object) { + osm_string_len_type user = std::make_pair("", 0); protozero::pbf_message pbf_info(data); while (pbf_info.next()) { @@ -220,7 +229,7 @@ namespace osmium { } int32_t convert_pbf_coordinate(int64_t c) const { - return (c * m_granularity + m_lon_offset) / resolution_convert; + return int32_t((c * m_granularity + m_lon_offset) / resolution_convert); } void decode_node(const ptr_len_type& data) { @@ -232,7 +241,7 @@ namespace osmium { int64_t lon = std::numeric_limits::max(); int64_t lat = std::numeric_limits::max(); - ptr_len_type user = { "", 0 }; + osm_string_len_type user = { "", 0 }; protozero::pbf_message pbf_node(data); while (pbf_node.next()) { @@ -285,7 +294,7 @@ namespace osmium { kv_type vals; std::pair refs; - ptr_len_type user = { "", 0 }; + osm_string_len_type user = { "", 0 }; protozero::pbf_message pbf_way(data); while (pbf_way.next()) { @@ -334,7 +343,7 @@ namespace osmium { std::pair refs; std::pair types; - ptr_len_type user = { "", 0 }; + osm_string_len_type user = { "", 0 }; protozero::pbf_message pbf_relation(data); while (pbf_relation.next()) { @@ -512,7 +521,7 @@ namespace osmium { // this is against the spec, must have same number of elements throw osmium::pbf_error("PBF format error"); } - visible = *visibles.first++; + visible = (*visibles.first++) != 0; } node.set_visible(visible); @@ -522,10 +531,14 @@ namespace osmium { builder.add_user(""); } + // even if the node isn't visible, there's still a record + // of its lat/lon in the dense arrays. + const auto lon = dense_longitude.update(*lons.first++); + const auto lat = dense_latitude.update(*lats.first++); if (visible) { builder.object().set_location(osmium::Location( - convert_pbf_coordinate(dense_longitude.update(*lons.first++)), - convert_pbf_coordinate(dense_latitude.update(*lats.first++)) + convert_pbf_coordinate(lon), + convert_pbf_coordinate(lat) )); } @@ -552,7 +565,7 @@ namespace osmium { public: - explicit PBFPrimitiveBlockDecoder(const ptr_len_type& data, osmium::osm_entity_bits::type read_types) : + PBFPrimitiveBlockDecoder(const ptr_len_type& data, osmium::osm_entity_bits::type read_types) : m_data(data), m_read_types(read_types) { } @@ -563,7 +576,7 @@ namespace osmium { PBFPrimitiveBlockDecoder(PBFPrimitiveBlockDecoder&&) = delete; PBFPrimitiveBlockDecoder& operator=(PBFPrimitiveBlockDecoder&&) = delete; - ~PBFPrimitiveBlockDecoder() = default; + ~PBFPrimitiveBlockDecoder() noexcept = default; osmium::memory::Buffer operator()() { try { @@ -579,8 +592,8 @@ namespace osmium { }; // class PBFPrimitiveBlockDecoder inline ptr_len_type decode_blob(const std::string& blob_data, std::string& output) { - int32_t raw_size; - std::pair zlib_data; + int32_t raw_size = 0; + std::pair zlib_data = {nullptr, 0}; protozero::pbf_message pbf_blob(blob_data); while (pbf_blob.next()) { @@ -609,7 +622,7 @@ namespace osmium { } } - if (zlib_data.second != 0) { + if (zlib_data.second != 0 && raw_size != 0) { return osmium::io::detail::zlib_uncompress_string( zlib_data.first, static_cast(zlib_data.second), @@ -694,7 +707,11 @@ namespace osmium { header.set("generator", pbf_header_block.get_string()); break; case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp: - header.set("osmosis_replication_timestamp", osmium::Timestamp(pbf_header_block.get_int64()).to_iso()); + { + auto timestamp = osmium::Timestamp(pbf_header_block.get_int64()).to_iso(); + header.set("osmosis_replication_timestamp", timestamp); + header.set("timestamp", timestamp); + } break; case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_sequence_number: header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.get_int64())); @@ -741,7 +758,7 @@ namespace osmium { PBFDataBlobDecoder(PBFDataBlobDecoder&&) = default; PBFDataBlobDecoder& operator=(PBFDataBlobDecoder&&) = default; - ~PBFDataBlobDecoder() = default; + ~PBFDataBlobDecoder() noexcept = default; osmium::memory::Buffer operator()() { std::string output; diff --git a/third_party/libosmium/include/osmium/io/detail/pbf_input_format.hpp b/third_party/libosmium/include/osmium/io/detail/pbf_input_format.hpp index 7817d27d1..e9d0e71d1 100644 --- a/third_party/libosmium/include/osmium/io/detail/pbf_input_format.hpp +++ b/third_party/libosmium/include/osmium/io/detail/pbf_input_format.hpp @@ -34,17 +34,12 @@ DEALINGS IN THE SOFTWARE. */ #include -#include #include -#include #include #include #include -#include #include -#include #include -#include #include #include #include @@ -58,40 +53,22 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include #include -#include #include -#include #include #include #include -#include #include -#include #include namespace osmium { namespace io { - class File; - namespace detail { - /** - * Class for parsing PBF files. - */ - class PBFInputFormat : public osmium::io::detail::InputFormat { + class PBFParser : public Parser { - typedef osmium::thread::Queue> queue_type; - - bool m_use_thread_pool; - bool m_eof { false }; - queue_type m_queue; - std::atomic m_quit_input_thread; - std::thread m_reader; - osmium::thread::Queue& m_input_queue; std::string m_input_buffer; /** @@ -103,9 +80,8 @@ namespace osmium { */ std::string read_from_input_queue(size_t size) { while (m_input_buffer.size() < size) { - std::string new_data; - m_input_queue.wait_and_pop(new_data); - if (new_data.empty()) { + std::string new_data = get_input(); + if (input_done()) { throw osmium::pbf_error("truncated data (EOF encountered)"); } m_input_buffer += new_data; @@ -113,7 +89,10 @@ namespace osmium { std::string output { m_input_buffer.substr(size) }; m_input_buffer.resize(size); - std::swap(output, m_input_buffer); + + using std::swap; + swap(output, m_input_buffer); + return output; } @@ -125,7 +104,7 @@ namespace osmium { uint32_t size_in_network_byte_order; try { - std::string input_data = read_from_input_queue(sizeof(size_in_network_byte_order)); + const std::string input_data = read_from_input_queue(sizeof(size_in_network_byte_order)); size_in_network_byte_order = *reinterpret_cast(input_data.data()); } catch (osmium::pbf_error&) { return 0; // EOF @@ -174,125 +153,85 @@ namespace osmium { size_t check_type_and_get_blob_size(const char* expected_type) { assert(expected_type); - auto size = read_blob_header_size_from_file(); + const auto size = read_blob_header_size_from_file(); if (size == 0) { // EOF return 0; } - std::string blob_header = read_from_input_queue(size); + const std::string blob_header = read_from_input_queue(size); return decode_blob_header(protozero::pbf_message(blob_header), expected_type); } - void parse_osm_data(osmium::osm_entity_bits::type read_types) { - osmium::thread::set_thread_name("_osmium_pbf_in"); - - while (auto size = check_type_and_get_blob_size("OSMData")) { - std::string input_buffer = read_from_input_queue(size); - if (input_buffer.size() > max_uncompressed_blob_size) { - throw osmium::pbf_error(std::string("invalid blob size: " + std::to_string(input_buffer.size()))); - } - - if (m_use_thread_pool) { - m_queue.push(osmium::thread::Pool::instance().submit(PBFDataBlobDecoder{ std::move(input_buffer), read_types })); - } else { - std::promise promise; - m_queue.push(promise.get_future()); - PBFDataBlobDecoder data_blob_parser{ std::move(input_buffer), read_types }; - promise.set_value(data_blob_parser()); - } - - if (m_quit_input_thread) { - return; - } + std::string read_from_input_queue_with_check(size_t size) { + if (size > max_uncompressed_blob_size) { + throw osmium::pbf_error(std::string("invalid blob size: " + + std::to_string(size))); } - - // Send an empty buffer to signal the reader that we are - // done. - std::promise promise; - m_queue.push(promise.get_future()); - promise.set_value(osmium::memory::Buffer{}); + return read_from_input_queue(size); } - void signal_input_thread_to_quit() { - m_quit_input_thread = true; + // Parse the header in the PBF OSMHeader blob. + void parse_header_blob() { + osmium::io::Header header; + const auto size = check_type_and_get_blob_size("OSMHeader"); + header = decode_header(read_from_input_queue_with_check(size)); + set_header_value(header); + } + + void parse_data_blobs() { + while (const auto size = check_type_and_get_blob_size("OSMData")) { + std::string input_buffer = read_from_input_queue_with_check(size); + + PBFDataBlobDecoder data_blob_parser{ std::move(input_buffer), read_types() }; + + if (osmium::config::use_pool_threads_for_pbf_parsing()) { + send_to_output_queue(osmium::thread::Pool::instance().submit(std::move(data_blob_parser))); + } else { + send_to_output_queue(data_blob_parser()); + } + } } public: - /** - * Instantiate PBF Parser - * - * @param file osmium::io::File instance describing file to be read from. - * @param read_which_entities Which types of OSM entities (nodes, ways, relations, changesets) should be parsed? - * @param input_queue String queue where data is read from. - */ - PBFInputFormat(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue& input_queue) : - osmium::io::detail::InputFormat(file, read_which_entities), - m_use_thread_pool(osmium::config::use_pool_threads_for_pbf_parsing()), - m_queue(20, "pbf_parser_results"), // XXX - m_quit_input_thread(false), - m_input_queue(input_queue), + PBFParser(future_string_queue_type& input_queue, + future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_types) : + Parser(input_queue, output_queue, header_promise, read_types), m_input_buffer() { + } - // handle OSMHeader - const auto size = check_type_and_get_blob_size("OSMHeader"); - m_header = decode_header(read_from_input_queue(size)); + ~PBFParser() noexcept final = default; - if (m_read_which_entities != osmium::osm_entity_bits::nothing) { - m_reader = std::thread(&PBFInputFormat::parse_osm_data, this, m_read_which_entities); + void run() final { + osmium::thread::set_thread_name("_osmium_pbf_in"); + + parse_header_blob(); + + if (read_types() != osmium::osm_entity_bits::nothing) { + parse_data_blobs(); } } - ~PBFInputFormat() { - signal_input_thread_to_quit(); - if (m_reader.joinable()) { - m_reader.join(); - } - } + }; // class PBFParser - /** - * Returns the next buffer with OSM data read from the PBF - * file. Blocks if data is not available yet. - * Returns an empty buffer at end of input. - */ - osmium::memory::Buffer read() override { - osmium::memory::Buffer buffer; - if (m_eof) { - return buffer; - } + // we want the register_parser() function to run, setting + // the variable is only a side-effect, it will never be used + const bool registered_pbf_parser = ParserFactory::instance().register_parser( + file_format::pbf, + [](future_string_queue_type& input_queue, + future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_which_entities) { + return std::unique_ptr(new PBFParser(input_queue, output_queue, header_promise, read_which_entities)); + }); - std::future buffer_future; - m_queue.wait_and_pop(buffer_future); - - try { - buffer = std::move(buffer_future.get()); - if (!buffer) { - m_eof = true; - } - return buffer; - } catch (...) { - m_eof = true; - signal_input_thread_to_quit(); - throw; - } - } - - }; // class PBFInputFormat - - namespace { - -// we want the register_input_format() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" - const bool registered_pbf_input = osmium::io::detail::InputFormatFactory::instance().register_input_format(osmium::io::file_format::pbf, - [](const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue& input_queue) { - return new osmium::io::detail::PBFInputFormat(file, read_which_entities, input_queue); - }); -#pragma GCC diagnostic pop - - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_pbf_parser() noexcept { + return registered_pbf_parser; + } } // namespace detail diff --git a/third_party/libosmium/include/osmium/io/detail/pbf_output_format.hpp b/third_party/libosmium/include/osmium/io/detail/pbf_output_format.hpp index 8d8a079b4..88c1cf489 100644 --- a/third_party/libosmium/include/osmium/io/detail/pbf_output_format.hpp +++ b/third_party/libosmium/include/osmium/io/detail/pbf_output_format.hpp @@ -34,19 +34,18 @@ DEALINGS IN THE SOFTWARE. */ #include -#include +#include #include #include #include -#include #include #include -#include #include -#include #include #include +// needed for older boost libraries +#define BOOST_RESULT_OF_USE_DECLTYPE #include #include @@ -71,6 +70,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include #include #include #include @@ -81,6 +81,32 @@ namespace osmium { namespace detail { + struct pbf_output_options { + + /// Should nodes be encoded in DenseNodes? + bool use_dense_nodes; + + /** + * Should the PBF blobs contain zlib compressed data? + * + * The zlib compression is optional, it's possible to store the + * blobs in raw format. Disabling the compression can improve + * the writing speed a little but the output will be 2x to 3x + * bigger. + */ + bool use_compression; + + /// Should metadata of objects be written? + bool add_metadata; + + /// Add the "HistoricalInformation" header flag. + bool add_historical_information_flag; + + /// Should the visible flag be added to all OSM objects? + bool add_visible_flag; + + }; + /** * Maximum number of items in a primitive block. * @@ -104,43 +130,81 @@ namespace osmium { return static_cast(std::round(lonlat * lonlat_resolution / location_granularity)); } - /** - * Serialize a protobuf message into a Blob, optionally apply compression - * and return it together with a BlobHeader ready to be written to a file. - * - * @param type Type-string used in the BlobHeader. - * @param msg Protobuf-message. - * @param use_compression Should the output be compressed using zlib? - */ - inline std::string serialize_blob(const std::string& type, const std::string& msg, bool use_compression) { - std::string blob_data; - protozero::pbf_builder pbf_blob(blob_data); + enum class pbf_blob_type { + header = 0, + data = 1 + }; - if (use_compression) { - pbf_blob.add_int32(FileFormat::Blob::optional_int32_raw_size, msg.size()); - pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_zlib_data, osmium::io::detail::zlib_compress(msg)); - } else { - pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_raw, msg); + class SerializeBlob { + + std::string m_msg; + + pbf_blob_type m_blob_type; + + bool m_use_compression; + + public: + + /** + * Initialize a blob serializer. + * + * @param msg Protobuf-message containing the blob data + * @param type Type of blob. + * @param use_compression Should the output be compressed using + * zlib? + */ + SerializeBlob(std::string&& msg, pbf_blob_type type, bool use_compression) : + m_msg(std::move(msg)), + m_blob_type(type), + m_use_compression(use_compression) { } - std::string blob_header_data; - protozero::pbf_builder pbf_blob_header(blob_header_data); + /** + * Serialize a protobuf message into a Blob, optionally apply + * compression and return it together with a BlobHeader ready + * to be written to a file. + */ + std::string operator()() { + assert(m_msg.size() <= max_uncompressed_blob_size); - pbf_blob_header.add_string(FileFormat::BlobHeader::required_string_type, type); - pbf_blob_header.add_int32(FileFormat::BlobHeader::required_int32_datasize, blob_data.size()); + std::string blob_data; + protozero::pbf_builder pbf_blob(blob_data); - uint32_t sz = htonl(static_cast_with_assert(blob_header_data.size())); + if (m_use_compression) { + pbf_blob.add_int32(FileFormat::Blob::optional_int32_raw_size, int32_t(m_msg.size())); + pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_zlib_data, osmium::io::detail::zlib_compress(m_msg)); + } else { + pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_raw, m_msg); + } - // write to output: the 4-byte BlobHeader-Size followed by the BlobHeader followed by the Blob - std::string output; - output.reserve(sizeof(sz) + blob_header_data.size() + blob_data.size()); - output.append(reinterpret_cast(&sz), sizeof(sz)); - output.append(blob_header_data); - output.append(blob_data); + std::string blob_header_data; + protozero::pbf_builder pbf_blob_header(blob_header_data); - return output; - } + pbf_blob_header.add_string(FileFormat::BlobHeader::required_string_type, m_blob_type == pbf_blob_type::data ? "OSMData" : "OSMHeader"); + pbf_blob_header.add_int32(FileFormat::BlobHeader::required_int32_datasize, static_cast_with_assert(blob_data.size())); + uint32_t sz = htonl(static_cast_with_assert(blob_header_data.size())); + + // write to output: the 4-byte BlobHeader-Size followed by the BlobHeader followed by the Blob + std::string output; + output.reserve(sizeof(sz) + blob_header_data.size() + blob_data.size()); + output.append(reinterpret_cast(&sz), sizeof(sz)); + output.append(blob_header_data); + output.append(blob_data); + + return output; + } + + }; // class SerializeBlob + + /** + * Contains the code to pack any number of nodes into a DenseNode + * structure. + * + * Because this needs to allocate a lot of memory on the heap, + * only one object of this class will be created and then re-used + * after calling clear() on it. + */ class DenseNodes { StringTable& m_stringtable; @@ -158,27 +222,26 @@ namespace osmium { std::vector m_lons; std::vector m_tags; - osmium::util::DeltaEncode m_delta_id; + osmium::util::DeltaEncode m_delta_id; - osmium::util::DeltaEncode m_delta_timestamp; - osmium::util::DeltaEncode m_delta_changeset; - osmium::util::DeltaEncode m_delta_uid; - osmium::util::DeltaEncode m_delta_user_sid; + osmium::util::DeltaEncode m_delta_timestamp; + osmium::util::DeltaEncode m_delta_changeset; + osmium::util::DeltaEncode m_delta_uid; + osmium::util::DeltaEncode m_delta_user_sid; - osmium::util::DeltaEncode m_delta_lat; - osmium::util::DeltaEncode m_delta_lon; + osmium::util::DeltaEncode m_delta_lat; + osmium::util::DeltaEncode m_delta_lon; - bool m_add_metadata; - bool m_add_visible; + const pbf_output_options& m_options; public: - DenseNodes(StringTable& stringtable, bool add_metadata, bool add_visible) : + DenseNodes(StringTable& stringtable, const pbf_output_options& options) : m_stringtable(stringtable), - m_add_metadata(add_metadata), - m_add_visible(add_visible) { + m_options(options) { } + /// Clear object for re-use. Keep the allocated memory. void clear() { m_ids.clear(); @@ -211,13 +274,13 @@ namespace osmium { void add_node(const osmium::Node& node) { m_ids.push_back(m_delta_id.update(node.id())); - if (m_add_metadata) { - m_versions.push_back(node.version()); - m_timestamps.push_back(m_delta_timestamp.update(node.timestamp())); + if (m_options.add_metadata) { + m_versions.push_back(static_cast_with_assert(node.version())); + m_timestamps.push_back(m_delta_timestamp.update(uint32_t(node.timestamp()))); m_changesets.push_back(m_delta_changeset.update(node.changeset())); m_uids.push_back(m_delta_uid.update(node.uid())); m_user_sids.push_back(m_delta_user_sid.update(m_stringtable.add(node.user()))); - if (m_add_visible) { + if (m_options.add_visible_flag) { m_visibles.push_back(node.visible()); } } @@ -226,8 +289,8 @@ namespace osmium { m_lons.push_back(m_delta_lon.update(lonlat2int(node.location().lon_without_check()))); for (const auto& tag : node.tags()) { - m_tags.push_back(m_stringtable.add(tag.key())); - m_tags.push_back(m_stringtable.add(tag.value())); + m_tags.push_back(static_cast_with_assert(m_stringtable.add(tag.key()))); + m_tags.push_back(static_cast_with_assert(m_stringtable.add(tag.value()))); } m_tags.push_back(0); } @@ -238,7 +301,7 @@ namespace osmium { pbf_dense_nodes.add_packed_sint64(OSMFormat::DenseNodes::packed_sint64_id, m_ids.cbegin(), m_ids.cend()); - if (m_add_metadata) { + if (m_options.add_metadata) { protozero::pbf_builder pbf_dense_info(pbf_dense_nodes, OSMFormat::DenseNodes::optional_DenseInfo_denseinfo); pbf_dense_info.add_packed_int32(OSMFormat::DenseInfo::packed_int32_version, m_versions.cbegin(), m_versions.cend()); pbf_dense_info.add_packed_sint64(OSMFormat::DenseInfo::packed_sint64_timestamp, m_timestamps.cbegin(), m_timestamps.cend()); @@ -246,7 +309,7 @@ namespace osmium { pbf_dense_info.add_packed_sint32(OSMFormat::DenseInfo::packed_sint32_uid, m_uids.cbegin(), m_uids.cend()); pbf_dense_info.add_packed_sint32(OSMFormat::DenseInfo::packed_sint32_user_sid, m_user_sids.cbegin(), m_user_sids.cend()); - if (m_add_visible) { + if (m_options.add_visible_flag) { pbf_dense_info.add_packed_bool(OSMFormat::DenseInfo::packed_bool_visible, m_visibles.cbegin(), m_visibles.cend()); } } @@ -272,11 +335,11 @@ namespace osmium { public: - PrimitiveBlock(bool add_metadata, bool add_visible) : + explicit PrimitiveBlock(const pbf_output_options& options) : m_pbf_primitive_group_data(), m_pbf_primitive_group(m_pbf_primitive_group_data), m_stringtable(), - m_dense_nodes(m_stringtable, add_metadata, add_visible), + m_dense_nodes(m_stringtable, options), m_type(OSMFormat::PrimitiveGroup::unknown), m_count(0) { } @@ -312,7 +375,7 @@ namespace osmium { ++m_count; } - size_t add_string(const char* s) { + uint32_t store_in_stringtable(const char* s) { return m_stringtable.add(s); } @@ -350,24 +413,7 @@ namespace osmium { class PBFOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler { - /// Should nodes be encoded in DenseNodes? - bool m_use_dense_nodes; - - /** - * Should the PBF blobs contain zlib compressed data? - * - * The zlib compression is optional, it's possible to store the - * blobs in raw format. Disabling the compression can improve - * the writing speed a little but the output will be 2x to 3x - * bigger. - */ - bool m_use_compression; - - /// Should metadata of objects be written? - bool m_add_metadata; - - /// Should the visible flag be added to objects? - bool m_add_visible; + pbf_output_options m_options; PrimitiveBlock m_primitive_block; @@ -386,20 +432,22 @@ namespace osmium { primitive_block.add_message(OSMFormat::PrimitiveBlock::repeated_PrimitiveGroup_primitivegroup, m_primitive_block.group_data()); - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(serialize_blob("OSMData", primitive_block_data, m_use_compression)); + m_output_queue.push(osmium::thread::Pool::instance().submit( + SerializeBlob{std::move(primitive_block_data), + pbf_blob_type::data, + m_options.use_compression} + )); } template void add_meta(const osmium::OSMObject& object, T& pbf_object) { const osmium::TagList& tags = object.tags(); - auto map_tag_key = [this](const osmium::Tag& tag) -> size_t { - return m_primitive_block.add_string(tag.key()); + auto map_tag_key = [this](const osmium::Tag& tag) -> uint32_t { + return m_primitive_block.store_in_stringtable(tag.key()); }; - auto map_tag_value = [this](const osmium::Tag& tag) -> size_t { - return m_primitive_block.add_string(tag.value()); + auto map_tag_value = [this](const osmium::Tag& tag) -> uint32_t { + return m_primitive_block.store_in_stringtable(tag.value()); }; pbf_object.add_packed_uint32(T::enum_type::packed_uint32_keys, @@ -410,39 +458,46 @@ namespace osmium { boost::make_transform_iterator(tags.begin(), map_tag_value), boost::make_transform_iterator(tags.end(), map_tag_value)); - if (m_add_metadata) { + if (m_options.add_metadata) { protozero::pbf_builder pbf_info(pbf_object, T::enum_type::optional_Info_info); - pbf_info.add_int32(OSMFormat::Info::optional_int32_version, object.version()); - pbf_info.add_int64(OSMFormat::Info::optional_int64_timestamp, object.timestamp()); + pbf_info.add_int32(OSMFormat::Info::optional_int32_version, static_cast_with_assert(object.version())); + pbf_info.add_int64(OSMFormat::Info::optional_int64_timestamp, uint32_t(object.timestamp())); pbf_info.add_int64(OSMFormat::Info::optional_int64_changeset, object.changeset()); - pbf_info.add_int32(OSMFormat::Info::optional_int32_uid, object.uid()); - pbf_info.add_uint32(OSMFormat::Info::optional_uint32_user_sid, m_primitive_block.add_string(object.user())); - if (m_add_visible) { + pbf_info.add_int32(OSMFormat::Info::optional_int32_uid, static_cast_with_assert(object.uid())); + pbf_info.add_uint32(OSMFormat::Info::optional_uint32_user_sid, m_primitive_block.store_in_stringtable(object.user())); + if (m_options.add_visible_flag) { pbf_info.add_bool(OSMFormat::Info::optional_bool_visible, object.visible()); } } } - PBFOutputFormat(const PBFOutputFormat&) = delete; - PBFOutputFormat& operator=(const PBFOutputFormat&) = delete; + void switch_primitive_block_type(OSMFormat::PrimitiveGroup type) { + if (!m_primitive_block.can_add(type)) { + store_primitive_block(); + m_primitive_block.reset(type); + } + } public: - explicit PBFOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : - OutputFormat(file, output_queue), - m_use_dense_nodes(file.get("pbf_dense_nodes") != "false"), - m_use_compression(file.get("pbf_compression") != "none" && file.get("pbf_compression") != "false"), - m_add_metadata(file.get("pbf_add_metadata") != "false" && file.get("add_metadata") != "false"), - m_add_visible(file.has_multiple_object_versions()), - m_primitive_block(m_add_metadata, m_add_visible) { + PBFOutputFormat(const osmium::io::File& file, future_string_queue_type& output_queue) : + OutputFormat(output_queue), + m_options(), + m_primitive_block(m_options) { + m_options.use_dense_nodes = file.is_not_false("pbf_dense_nodes"); + m_options.use_compression = file.get("pbf_compression") != "none" && file.is_not_false("pbf_compression"); + m_options.add_metadata = file.is_not_false("pbf_add_metadata") && file.is_not_false("add_metadata"); + m_options.add_historical_information_flag = file.has_multiple_object_versions(); + m_options.add_visible_flag = file.has_multiple_object_versions(); } - void write_buffer(osmium::memory::Buffer&& buffer) override final { - osmium::apply(buffer.cbegin(), buffer.cend(), *this); - } + PBFOutputFormat(const PBFOutputFormat&) = delete; + PBFOutputFormat& operator=(const PBFOutputFormat&) = delete; - void write_header(const osmium::io::Header& header) override final { + ~PBFOutputFormat() noexcept final = default; + + void write_header(const osmium::io::Header& header) final { std::string data; protozero::pbf_builder pbf_header_block(data); @@ -450,19 +505,19 @@ namespace osmium { protozero::pbf_builder pbf_header_bbox(pbf_header_block, OSMFormat::HeaderBlock::optional_HeaderBBox_bbox); osmium::Box box = header.joined_boxes(); - pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_left, box.bottom_left().lon() * lonlat_resolution); - pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_right, box.top_right().lon() * lonlat_resolution); - pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_top, box.top_right().lat() * lonlat_resolution); - pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_bottom, box.bottom_left().lat() * lonlat_resolution); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_left, int64_t(box.bottom_left().lon() * lonlat_resolution)); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_right, int64_t(box.top_right().lon() * lonlat_resolution)); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_top, int64_t(box.top_right().lat() * lonlat_resolution)); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_bottom, int64_t(box.bottom_left().lat() * lonlat_resolution)); } pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "OsmSchema-V0.6"); - if (m_use_dense_nodes) { + if (m_options.use_dense_nodes) { pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "DenseNodes"); } - if (m_file.has_multiple_object_versions()) { + if (m_options.add_historical_information_flag) { pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "HistoricalInformation"); } @@ -471,7 +526,7 @@ namespace osmium { std::string osmosis_replication_timestamp = header.get("osmosis_replication_timestamp"); if (!osmosis_replication_timestamp.empty()) { osmium::Timestamp ts(osmosis_replication_timestamp.c_str()); - pbf_header_block.add_int64(OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp, ts); + pbf_header_block.add_int64(OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp, uint32_t(ts)); } std::string osmosis_replication_sequence_number = header.get("osmosis_replication_sequence_number"); @@ -484,20 +539,23 @@ namespace osmium { pbf_header_block.add_string(OSMFormat::HeaderBlock::optional_string_osmosis_replication_base_url, osmosis_replication_base_url); } - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(serialize_blob("OSMHeader", data, m_use_compression)); + m_output_queue.push(osmium::thread::Pool::instance().submit( + SerializeBlob{std::move(data), + pbf_blob_type::header, + m_options.use_compression} + )); } - void switch_primitive_block_type(OSMFormat::PrimitiveGroup type) { - if (!m_primitive_block.can_add(type)) { - store_primitive_block(); - m_primitive_block.reset(type); - } + void write_buffer(osmium::memory::Buffer&& buffer) final { + osmium::apply(buffer.cbegin(), buffer.cend(), *this); + } + + void write_end() final { + store_primitive_block(); } void node(const osmium::Node& node) { - if (m_use_dense_nodes) { + if (m_options.use_dense_nodes) { switch_primitive_block_type(OSMFormat::PrimitiveGroup::optional_DenseNodes_dense); m_primitive_block.add_dense_node(node); return; @@ -538,8 +596,8 @@ namespace osmium { pbf_relation.add_int64(OSMFormat::Relation::required_int64_id, relation.id()); add_meta(relation, pbf_relation); - auto map_member_role = [this](const osmium::RelationMember& member) -> size_t { - return m_primitive_block.add_string(member.role()); + auto map_member_role = [this](const osmium::RelationMember& member) -> uint32_t { + return m_primitive_block.store_in_stringtable(member.role()); }; pbf_relation.add_packed_int32(OSMFormat::Relation::packed_int32_roles_sid, boost::make_transform_iterator(relation.members().begin(), map_member_role), @@ -554,41 +612,27 @@ namespace osmium { it_type last { members.cend(), members.cend(), map_member_ref }; pbf_relation.add_packed_sint64(OSMFormat::Relation::packed_sint64_memids, first, last); - static auto map_member_type = [](const osmium::RelationMember& member) noexcept -> int { - return osmium::item_type_to_nwr_index(member.type()); + static auto map_member_type = [](const osmium::RelationMember& member) noexcept -> int32_t { + return int32_t(osmium::item_type_to_nwr_index(member.type())); }; pbf_relation.add_packed_int32(OSMFormat::Relation::packed_MemberType_types, boost::make_transform_iterator(relation.members().begin(), map_member_type), boost::make_transform_iterator(relation.members().end(), map_member_type)); } - /** - * Finalize the writing process, flush any open primitive - * blocks to the file and close the file. - */ - void close() override final { - store_primitive_block(); - - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(std::string()); - } - }; // class PBFOutputFormat - namespace { + // we want the register_output_format() function to run, setting + // the variable is only a side-effect, it will never be used + const bool registered_pbf_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::pbf, + [](const osmium::io::File& file, future_string_queue_type& output_queue) { + return new osmium::io::detail::PBFOutputFormat(file, output_queue); + }); -// we want the register_output_format() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" - const bool registered_pbf_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::pbf, - [](const osmium::io::File& file, data_queue_type& output_queue) { - return new osmium::io::detail::PBFOutputFormat(file, output_queue); - }); -#pragma GCC diagnostic pop - - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_pbf_output() noexcept { + return registered_pbf_output; + } } // namespace detail diff --git a/third_party/libosmium/include/osmium/io/detail/queue_util.hpp b/third_party/libosmium/include/osmium/io/detail/queue_util.hpp new file mode 100644 index 000000000..6c9f071bc --- /dev/null +++ b/third_party/libosmium/include/osmium/io/detail/queue_util.hpp @@ -0,0 +1,157 @@ +#ifndef OSMIUM_IO_DETAIL_QUEUE_UTIL_HPP +#define OSMIUM_IO_DETAIL_QUEUE_UTIL_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include + +#include +#include + +namespace osmium { + + namespace io { + + namespace detail { + + /** + * This type of queue contains buffers with OSM data in them. + * The "end of file" is marked by an invalid Buffer. + * The buffers are wrapped in a std::future so that they can also + * transport exceptions. The future also helps with keeping the + * data in order. + */ + using future_buffer_queue_type = osmium::thread::Queue>; + + /** + * This type of queue contains OSM file data in the form it is + * stored on disk, ie encoded as XML, PBF, etc. + * The "end of file" is marked by an empty string. + */ + using string_queue_type = osmium::thread::Queue; + + /** + * This type of queue contains OSM file data in the form it is + * stored on disk, ie encoded as XML, PBF, etc. + * The "end of file" is marked by an empty string. + * The strings are wrapped in a std::future so that they can also + * transport exceptions. The future also helps with keeping the + * data in order. + */ + using future_string_queue_type = osmium::thread::Queue>; + + template + inline void add_to_queue(osmium::thread::Queue>& queue, T&& data) { + std::promise promise; + queue.push(promise.get_future()); + promise.set_value(std::forward(data)); + } + + template + inline void add_to_queue(osmium::thread::Queue>& queue, std::exception_ptr&& exception) { + std::promise promise; + queue.push(promise.get_future()); + promise.set_exception(std::move(exception)); + } + + template + inline void add_end_of_data_to_queue(osmium::thread::Queue>& queue) { + add_to_queue(queue, T{}); + } + + inline bool at_end_of_data(const std::string& data) { + return data.empty(); + } + + inline bool at_end_of_data(osmium::memory::Buffer& buffer) { + return !buffer; + } + + template + class queue_wrapper { + + using queue_type = osmium::thread::Queue>; + + queue_type& m_queue; + bool m_has_reached_end_of_data; + + public: + + explicit queue_wrapper(queue_type& queue) : + m_queue(queue), + m_has_reached_end_of_data(false) { + } + + ~queue_wrapper() noexcept { + drain(); + } + + void drain() { + while (!m_has_reached_end_of_data) { + try { + pop(); + } catch (...) { + // Ignore any exceptions. + } + } + } + + bool has_reached_end_of_data() const noexcept { + return m_has_reached_end_of_data; + } + + T pop() { + T data; + if (!m_has_reached_end_of_data) { + std::future data_future; + m_queue.wait_and_pop(data_future); + data = std::move(data_future.get()); + if (at_end_of_data(data)) { + m_has_reached_end_of_data = true; + } + } + return data; + } + + }; // class queue_wrapper + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_QUEUE_UTIL_HPP diff --git a/third_party/libosmium/include/osmium/io/detail/read_thread.hpp b/third_party/libosmium/include/osmium/io/detail/read_thread.hpp index bce4f5507..6f96c0b65 100644 --- a/third_party/libosmium/include/osmium/io/detail/read_thread.hpp +++ b/third_party/libosmium/include/osmium/io/detail/read_thread.hpp @@ -34,14 +34,13 @@ DEALINGS IN THE SOFTWARE. */ #include -#include -#include +#include #include #include #include #include -#include +#include #include namespace osmium { @@ -50,52 +49,80 @@ namespace osmium { namespace detail { - class ReadThread { + /** + * This code uses an internally managed thread to read data from + * the input file and (optionally) decompress it. The result is + * sent to the given queue. Any exceptions will also be send to + * the queue. + */ + class ReadThreadManager { - osmium::thread::Queue& m_queue; - osmium::io::Decompressor* m_decompressor; + // only used in the sub-thread + osmium::io::Decompressor& m_decompressor; + future_string_queue_type& m_queue; - // If this is set in the main thread, we have to wrap up at the - // next possible moment. - std::atomic& m_done; + // used in both threads + std::atomic m_done; - public: + // only used in the main thread + std::thread m_thread; - explicit ReadThread(osmium::thread::Queue& queue, osmium::io::Decompressor* decompressor, std::atomic& done) : - m_queue(queue), - m_decompressor(decompressor), - m_done(done) { - } - - bool operator()() { - osmium::thread::set_thread_name("_osmium_input"); + void run_in_thread() { + osmium::thread::set_thread_name("_osmium_read"); try { while (!m_done) { - std::string data {m_decompressor->read()}; - if (data.empty()) { - m_queue.push(std::move(data)); + std::string data {m_decompressor.read()}; + if (at_end_of_data(data)) { break; } - m_queue.push(std::move(data)); - while (m_queue.size() > 10 && !m_done) { - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } + add_to_queue(m_queue, std::move(data)); } - m_decompressor->close(); + m_decompressor.close(); } catch (...) { - // If there is an exception in this thread, we make sure - // to push an empty string onto the queue to signal the - // end-of-data to the reading thread so that it will not - // hang. Then we re-throw the exception. - m_queue.push(std::string()); - throw; + add_to_queue(m_queue, std::current_exception()); } - return true; + + add_end_of_data_to_queue(m_queue); } - }; // class ReadThread + public: + + ReadThreadManager(osmium::io::Decompressor& decompressor, + future_string_queue_type& queue) : + m_decompressor(decompressor), + m_queue(queue), + m_done(false), + m_thread(std::thread(&ReadThreadManager::run_in_thread, this)) { + } + + ReadThreadManager(const ReadThreadManager&) = delete; + ReadThreadManager& operator=(const ReadThreadManager&) = delete; + + ReadThreadManager(ReadThreadManager&&) = delete; + ReadThreadManager& operator=(ReadThreadManager&&) = delete; + + ~ReadThreadManager() noexcept { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } + } + + void stop() noexcept { + m_done = true; + } + + void close() { + stop(); + if (m_thread.joinable()) { + m_thread.join(); + } + } + + }; // class ReadThreadManager } // namespace detail diff --git a/third_party/libosmium/include/osmium/io/detail/read_write.hpp b/third_party/libosmium/include/osmium/io/detail/read_write.hpp index 9863bd719..6db6d8aa8 100644 --- a/third_party/libosmium/include/osmium/io/detail/read_write.hpp +++ b/third_party/libosmium/include/osmium/io/detail/read_write.hpp @@ -35,6 +35,7 @@ DEALINGS IN THE SOFTWARE. #include #include +#include #include #include #include @@ -45,7 +46,7 @@ DEALINGS IN THE SOFTWARE. # include #endif -#include +#include namespace osmium { @@ -68,23 +69,26 @@ namespace osmium { */ inline int open_for_writing(const std::string& filename, osmium::io::overwrite allow_overwrite = osmium::io::overwrite::no) { if (filename == "" || filename == "-") { - return 1; // stdout - } else { - int flags = O_WRONLY | O_CREAT; - if (allow_overwrite == osmium::io::overwrite::allow) { - flags |= O_TRUNC; - } else { - flags |= O_EXCL; - } #ifdef _WIN32 - flags |= O_BINARY; + _setmode(1, _O_BINARY); #endif - int fd = ::open(filename.c_str(), flags, 0666); - if (fd < 0) { - throw std::system_error(errno, std::system_category(), std::string("Open failed for '") + filename + "'"); - } - return fd; + return 1; // stdout } + + int flags = O_WRONLY | O_CREAT; + if (allow_overwrite == osmium::io::overwrite::allow) { + flags |= O_TRUNC; + } else { + flags |= O_EXCL; + } +#ifdef _WIN32 + flags |= O_BINARY; +#endif + int fd = ::open(filename.c_str(), flags, 0666); + if (fd < 0) { + throw std::system_error(errno, std::system_category(), std::string("Open failed for '") + filename + "'"); + } + return fd; } /** @@ -98,17 +102,17 @@ namespace osmium { inline int open_for_reading(const std::string& filename) { if (filename == "" || filename == "-") { return 0; // stdin - } else { - int flags = O_RDONLY; -#ifdef _WIN32 - flags |= O_BINARY; -#endif - int fd = ::open(filename.c_str(), flags); - if (fd < 0) { - throw std::system_error(errno, std::system_category(), std::string("Open failed for '") + filename + "'"); - } - return fd; } + + int flags = O_RDONLY; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + int fd = ::open(filename.c_str(), flags); + if (fd < 0) { + throw std::system_error(errno, std::system_category(), std::string("Open failed for '") + filename + "'"); + } + return fd; } /** @@ -151,6 +155,22 @@ namespace osmium { reliable_write(fd, reinterpret_cast(output_buffer), size); } + inline void reliable_fsync(const int fd) { +#ifdef _WIN32 + if (_commit(fd) != 0) { +#else + if (::fsync(fd) != 0) { +#endif + throw std::system_error(errno, std::system_category(), "Fsync failed"); + } + } + + inline void reliable_close(const int fd) { + if (::close(fd) != 0) { + throw std::system_error(errno, std::system_category(), "Close failed"); + } + } + } // namespace detail } // namespace io diff --git a/third_party/libosmium/include/osmium/io/detail/string_table.hpp b/third_party/libosmium/include/osmium/io/detail/string_table.hpp index ae9d5f0ce..1cb142d00 100644 --- a/third_party/libosmium/include/osmium/io/detail/string_table.hpp +++ b/third_party/libosmium/include/osmium/io/detail/string_table.hpp @@ -34,6 +34,7 @@ DEALINGS IN THE SOFTWARE. */ #include +#include #include #include #include @@ -41,6 +42,8 @@ DEALINGS IN THE SOFTWARE. #include #include +#include + namespace osmium { namespace io { @@ -72,7 +75,7 @@ namespace osmium { public: - StringStore(size_t chunk_size) : + explicit StringStore(size_t chunk_size) : m_chunk_size(chunk_size), m_chunks() { add_chunk(); @@ -172,15 +175,15 @@ namespace osmium { // These functions get you some idea how much memory was // used. - int get_chunk_size() const noexcept { + size_t get_chunk_size() const noexcept { return m_chunk_size; } - int get_chunk_count() const noexcept { + size_t get_chunk_count() const noexcept { return m_chunks.size(); } - int get_used_bytes_in_last_chunk() const noexcept { + size_t get_used_bytes_in_last_chunk() const noexcept { return m_chunks.front().size(); } @@ -196,9 +199,16 @@ namespace osmium { class StringTable { + // This is the maximum number of entries in a string table. + // This should never be reached in practice but we better + // make sure it doesn't. If we had max_uncompressed_blob_size + // many entries, we are sure they would never fit into a PBF + // Blob. + static constexpr const uint32_t max_entries = max_uncompressed_blob_size; + StringStore m_strings; std::map m_index; - size_t m_size; + uint32_t m_size; public: @@ -216,18 +226,23 @@ namespace osmium { m_strings.add(""); } - size_t size() const noexcept { + uint32_t size() const noexcept { return m_size + 1; } - size_t add(const char* s) { + uint32_t add(const char* s) { auto f = m_index.find(s); if (f != m_index.end()) { - return f->second; + return uint32_t(f->second); } const char* cs = m_strings.add(s); m_index[cs] = ++m_size; + + if (m_size > max_entries) { + throw osmium::pbf_error("string table has too many entries"); + } + return m_size; } diff --git a/third_party/libosmium/include/osmium/io/detail/string_util.hpp b/third_party/libosmium/include/osmium/io/detail/string_util.hpp new file mode 100644 index 000000000..672266a92 --- /dev/null +++ b/third_party/libosmium/include/osmium/io/detail/string_util.hpp @@ -0,0 +1,206 @@ +#ifndef OSMIUM_IO_DETAIL_STRING_UTIL_HPP +#define OSMIUM_IO_DETAIL_STRING_UTIL_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include + +#include + +namespace osmium { + + namespace io { + + namespace detail { + +#ifndef _MSC_VER +# define SNPRINTF std::snprintf +#else +# define SNPRINTF _snprintf +#endif + + template + inline int string_snprintf(std::string& out, + size_t old_size, + size_t max_size, + const char* format, + TArgs&&... args) { + out.resize(old_size + max_size); + + return SNPRINTF(max_size ? const_cast(out.c_str()) + old_size : nullptr, + max_size, + format, + std::forward(args)...); + } + +#undef SNPRINTF + + /** + * This is a helper function for writing printf-like formatted + * data into a std::string. + * + * @param out The data will be appended to this string. + * @param format A string with formatting instructions a la printf. + * @param args Any further arguments like in printf. + * @throws std::bad_alloc If the string needed to grow and there + * wasn't enough memory. + */ + template + inline void append_printf_formatted_string(std::string& out, + const char* format, + TArgs&&... args) { + + // First try to write string with the max_size, if that doesn't + // work snprintf will tell us how much space it needs. We + // reserve that much space and try again. So this will always + // work, even if the output is larger than the given max_size. + // + // Unfortunately this trick doesn't work on Windows, because + // the _snprintf() function there only returns the length it + // needs if max_size==0 and the buffer pointer is the null + // pointer. So we have to take this into account. + +#ifndef _MSC_VER + static const size_t max_size = 100; +#else + static const size_t max_size = 0; +#endif + + size_t old_size = out.size(); + + int len = string_snprintf(out, + old_size, + max_size, + format, + std::forward(args)...); + assert(len > 0); + + if (size_t(len) >= max_size) { + int len2 = string_snprintf(out, + old_size, + size_t(len) + 1, + format, + std::forward(args)...); + assert(len2 == len); + } + + out.resize(old_size + size_t(len)); + } + + inline void append_utf8_encoded_string(std::string& out, const char* data) { + const char* end = data + std::strlen(data); + + while (data != end) { + const char* last = data; + uint32_t c = utf8::next(data, end); + + // This is a list of Unicode code points that we let + // through instead of escaping them. It is incomplete + // and can be extended later. + // Generally we don't want to let through any character + // that has special meaning in the OPL format such as + // space, comma, @, etc. and any non-printing characters. + if ((0x0021 <= c && c <= 0x0024) || + (0x0026 <= c && c <= 0x002b) || + (0x002d <= c && c <= 0x003c) || + (0x003e <= c && c <= 0x003f) || + (0x0041 <= c && c <= 0x007e) || + (0x00a1 <= c && c <= 0x00ac) || + (0x00ae <= c && c <= 0x05ff)) { + out.append(last, data); + } else { + out += '%'; + if (c <= 0xff) { + append_printf_formatted_string(out, "%02x", c); + } else { + append_printf_formatted_string(out, "%04x", c); + } + out += '%'; + } + } + } + + inline void append_xml_encoded_string(std::string& out, const char* data) { + for (; *data != '\0'; ++data) { + switch(*data) { + case '&': out += "&"; break; + case '\"': out += """; break; + case '\'': out += "'"; break; + case '<': out += "<"; break; + case '>': out += ">"; break; + case '\n': out += " "; break; + case '\r': out += " "; break; + case '\t': out += " "; break; + default: out += *data; break; + } + } + } + + inline void append_debug_encoded_string(std::string& out, const char* data, const char* prefix, const char* suffix) { + const char* end = data + std::strlen(data); + + while (data != end) { + const char* last = data; + uint32_t c = utf8::next(data, end); + + // This is a list of Unicode code points that we let + // through instead of escaping them. It is incomplete + // and can be extended later. + // Generally we don't want to let through any + // non-printing characters. + if ((0x0020 <= c && c <= 0x0021) || + (0x0023 <= c && c <= 0x003b) || + (0x003d == c) || + (0x003f <= c && c <= 0x007e) || + (0x00a1 <= c && c <= 0x00ac) || + (0x00ae <= c && c <= 0x05ff)) { + out.append(last, data); + } else { + out.append(prefix); + append_printf_formatted_string(out, "", c); + out.append(suffix); + } + } + } + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_STRING_UTIL_HPP diff --git a/third_party/libosmium/include/osmium/io/detail/write_thread.hpp b/third_party/libosmium/include/osmium/io/detail/write_thread.hpp index fad22ed69..81ab1947f 100644 --- a/third_party/libosmium/include/osmium/io/detail/write_thread.hpp +++ b/third_party/libosmium/include/osmium/io/detail/write_thread.hpp @@ -33,11 +33,13 @@ DEALINGS IN THE SOFTWARE. */ +#include +#include #include #include #include -#include +#include #include namespace osmium { @@ -46,33 +48,52 @@ namespace osmium { namespace detail { + /** + * This codes runs in its own thread, getting data from the given + * queue, (optionally) compressing it, and writing it to the output + * file. + */ class WriteThread { - typedef osmium::io::detail::data_queue_type data_queue_type; - - data_queue_type& m_input_queue; - osmium::io::Compressor* m_compressor; + queue_wrapper m_queue; + std::unique_ptr m_compressor; + std::promise m_promise; public: - explicit WriteThread(data_queue_type& input_queue, osmium::io::Compressor* compressor) : - m_input_queue(input_queue), - m_compressor(compressor) { + WriteThread(future_string_queue_type& input_queue, + std::unique_ptr&& compressor, + std::promise&& promise) : + m_queue(input_queue), + m_compressor(std::move(compressor)), + m_promise(std::move(promise)) { } - bool operator()() { - osmium::thread::set_thread_name("_osmium_output"); + WriteThread(const WriteThread&) = delete; + WriteThread& operator=(const WriteThread&) = delete; - std::future data_future; - std::string data; - do { - m_input_queue.wait_and_pop(data_future); - data = data_future.get(); - m_compressor->write(data); - } while (!data.empty()); + WriteThread(WriteThread&&) = delete; + WriteThread& operator=(WriteThread&&) = delete; - m_compressor->close(); - return true; + ~WriteThread() noexcept = default; + + void operator()() { + osmium::thread::set_thread_name("_osmium_write"); + + try { + while (true) { + std::string data = m_queue.pop(); + if (at_end_of_data(data)) { + break; + } + m_compressor->write(data); + } + m_compressor->close(); + m_promise.set_value(true); + } catch (...) { + m_promise.set_exception(std::current_exception()); + m_queue.drain(); + } } }; // class WriteThread diff --git a/third_party/libosmium/include/osmium/io/detail/xml_input_format.hpp b/third_party/libosmium/include/osmium/io/detail/xml_input_format.hpp index b0f3da339..11d3cba08 100644 --- a/third_party/libosmium/include/osmium/io/detail/xml_input_format.hpp +++ b/third_party/libosmium/include/osmium/io/detail/xml_input_format.hpp @@ -33,21 +33,14 @@ DEALINGS IN THE SOFTWARE. */ -#include #include -#include #include #include #include #include #include -#include #include -#include -#include -#include #include -#include #include #include @@ -55,6 +48,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include #include #include #include @@ -130,23 +124,11 @@ namespace osmium { namespace io { - class File; - namespace detail { - /** - * Once the header is fully parsed this exception will be thrown if - * the caller is not interested in anything else except the header. - * It will break off the parsing at this point. - * - * This exception is never seen by user code, it is caught internally. - */ - class ParserIsDone : std::exception { - }; + class XMLParser : public Parser { - class XMLParser { - - static constexpr int buffer_size = 10 * 1000 * 1000; + static constexpr int buffer_size = 2 * 1000 * 1000; enum class context { root, @@ -155,6 +137,9 @@ namespace osmium { way, relation, changeset, + discussion, + comment, + comment_text, ignored_node, ignored_way, ignored_relation, @@ -175,29 +160,22 @@ namespace osmium { osmium::memory::Buffer m_buffer; - std::unique_ptr m_node_builder; - std::unique_ptr m_way_builder; - std::unique_ptr m_relation_builder; - std::unique_ptr m_changeset_builder; + std::unique_ptr m_node_builder; + std::unique_ptr m_way_builder; + std::unique_ptr m_relation_builder; + std::unique_ptr m_changeset_builder; + std::unique_ptr m_changeset_discussion_builder; - std::unique_ptr m_tl_builder; - std::unique_ptr m_wnl_builder; - std::unique_ptr m_rml_builder; + std::unique_ptr m_tl_builder; + std::unique_ptr m_wnl_builder; + std::unique_ptr m_rml_builder; - osmium::thread::Queue& m_input_queue; - osmium::thread::Queue& m_queue; - std::promise& m_header_promise; - - osmium::osm_entity_bits::type m_read_types; - - std::atomic& m_done; - - bool m_header_is_done; + std::string m_comment_text; /** * A C++ wrapper for the Expat parser that makes sure no memory is leaked. */ - template + template class ExpatXMLParser { XML_Parser m_parser; @@ -210,15 +188,20 @@ namespace osmium { static_cast(data)->end_element(element); } + static void XMLCALL character_data_wrapper(void* data, const XML_Char* text, int len) { + static_cast(data)->characters(text, len); + } + public: - ExpatXMLParser(T* callback_object) : + explicit ExpatXMLParser(T* callback_object) : m_parser(XML_ParserCreate(nullptr)) { if (!m_parser) { throw osmium::io_error("Internal error: Can not create parser"); } XML_SetUserData(m_parser, callback_object); XML_SetElementHandler(m_parser, start_element_wrapper, end_element_wrapper); + XML_SetCharacterDataHandler(m_parser, character_data_wrapper); } ExpatXMLParser(const ExpatXMLParser&) = delete; @@ -227,7 +210,7 @@ namespace osmium { ExpatXMLParser& operator=(const ExpatXMLParser&) = delete; ExpatXMLParser& operator=(ExpatXMLParser&&) = delete; - ~ExpatXMLParser() { + ~ExpatXMLParser() noexcept { XML_ParserFree(m_parser); } @@ -239,126 +222,14 @@ namespace osmium { }; // class ExpatXMLParser - /** - * A helper class that makes sure a promise is kept. It stores - * a reference to some piece of data and to a promise and, on - * destruction, sets the value of the promise from the data. - */ - template - class PromiseKeeper { - - T& m_data; - std::promise& m_promise; - bool m_done; - - public: - - PromiseKeeper(T& data, std::promise& promise) : - m_data(data), - m_promise(promise), - m_done(false) { + template + static void check_attributes(const XML_Char** attrs, T check) { + while (*attrs) { + check(attrs[0], attrs[1]); + attrs += 2; } - - void fullfill_promise() { - if (!m_done) { - m_promise.set_value(m_data); - m_done = true; - } - } - - ~PromiseKeeper() { - fullfill_promise(); - } - - }; // class PromiseKeeper - - public: - - explicit XMLParser(osmium::thread::Queue& input_queue, osmium::thread::Queue& queue, std::promise& header_promise, osmium::osm_entity_bits::type read_types, std::atomic& done) : - m_context(context::root), - m_last_context(context::root), - m_in_delete_section(false), - m_header(), - m_buffer(buffer_size), - m_node_builder(), - m_way_builder(), - m_relation_builder(), - m_changeset_builder(), - m_tl_builder(), - m_wnl_builder(), - m_rml_builder(), - m_input_queue(input_queue), - m_queue(queue), - m_header_promise(header_promise), - m_read_types(read_types), - m_done(done), - m_header_is_done(false) { } - /** - * The copy constructor is needed for storing XMLParser in a std::function. - * The copy will look the same as if it has been initialized with the - * same parameters as the original. Any state changes in the original will - * not be reflected in the copy. - */ - XMLParser(const XMLParser& other) : - m_context(context::root), - m_last_context(context::root), - m_in_delete_section(false), - m_header(), - m_buffer(buffer_size), - m_node_builder(), - m_way_builder(), - m_relation_builder(), - m_changeset_builder(), - m_tl_builder(), - m_wnl_builder(), - m_rml_builder(), - m_input_queue(other.m_input_queue), - m_queue(other.m_queue), - m_header_promise(other.m_header_promise), - m_read_types(other.m_read_types), - m_done(other.m_done), - m_header_is_done(other.m_header_is_done) { - } - - XMLParser(XMLParser&&) = default; - - XMLParser& operator=(const XMLParser&) = delete; - - XMLParser& operator=(XMLParser&&) = default; - - ~XMLParser() = default; - - bool operator()() { - ExpatXMLParser parser(this); - PromiseKeeper promise_keeper(m_header, m_header_promise); - bool last; - do { - std::string data; - m_input_queue.wait_and_pop(data); - last = data.empty(); - try { - parser(data, last); - if (m_header_is_done) { - promise_keeper.fullfill_promise(); - } - } catch (ParserIsDone&) { - return true; - } catch (...) { - m_queue.push(osmium::memory::Buffer()); // empty buffer to signify eof - throw; - } - } while (!last && !m_done); - if (m_buffer.committed() > 0) { - m_queue.push(std::move(m_buffer)); - } - m_queue.push(osmium::memory::Buffer()); // empty buffer to signify eof - return true; - } - - private: - const char* init_object(osmium::OSMObject& object, const XML_Char** attrs) { const char* user = ""; @@ -367,17 +238,18 @@ namespace osmium { } osmium::Location location; - for (int count = 0; attrs[count]; count += 2) { - if (!strcmp(attrs[count], "lon")) { - location.set_lon(std::atof(attrs[count+1])); // XXX doesn't detect garbage after the number - } else if (!strcmp(attrs[count], "lat")) { - location.set_lat(std::atof(attrs[count+1])); // XXX doesn't detect garbage after the number - } else if (!strcmp(attrs[count], "user")) { - user = attrs[count+1]; + + check_attributes(attrs, [&location, &user, &object](const XML_Char* name, const XML_Char* value) { + if (!strcmp(name, "lon")) { + location.set_lon(std::atof(value)); // XXX doesn't detect garbage after the number + } else if (!strcmp(name, "lat")) { + location.set_lat(std::atof(value)); // XXX doesn't detect garbage after the number + } else if (!strcmp(name, "user")) { + user = value; } else { - object.set_attribute(attrs[count], attrs[count+1]); + object.set_attribute(name, value); } - } + }); if (location && object.type() == osmium::item_type::node) { static_cast(object).set_location(location); @@ -392,21 +264,21 @@ namespace osmium { osmium::Location min; osmium::Location max; - for (int count = 0; attrs[count]; count += 2) { - if (!strcmp(attrs[count], "min_lon")) { - min.set_lon(atof(attrs[count+1])); - } else if (!strcmp(attrs[count], "min_lat")) { - min.set_lat(atof(attrs[count+1])); - } else if (!strcmp(attrs[count], "max_lon")) { - max.set_lon(atof(attrs[count+1])); - } else if (!strcmp(attrs[count], "max_lat")) { - max.set_lat(atof(attrs[count+1])); - } else if (!strcmp(attrs[count], "user")) { - user = attrs[count+1]; + check_attributes(attrs, [&min, &max, &user, &new_changeset](const XML_Char* name, const XML_Char* value) { + if (!strcmp(name, "min_lon")) { + min.set_lon(atof(value)); + } else if (!strcmp(name, "min_lat")) { + min.set_lat(atof(value)); + } else if (!strcmp(name, "max_lon")) { + max.set_lon(atof(value)); + } else if (!strcmp(name, "max_lat")) { + max.set_lat(atof(value)); + } else if (!strcmp(name, "user")) { + user = value; } else { - new_changeset.set_attribute(attrs[count], attrs[count+1]); + new_changeset.set_attribute(name, value); } - } + }); new_changeset.bounds().extend(min); new_changeset.bounds().extend(max); @@ -414,32 +286,24 @@ namespace osmium { builder->add_user(user); } - void check_tag(osmium::builder::Builder* builder, const XML_Char* element, const XML_Char** attrs) { - if (!strcmp(element, "tag")) { - m_wnl_builder.reset(); - m_rml_builder.reset(); - - const char* key = ""; - const char* value = ""; - for (int count = 0; attrs[count]; count += 2) { - if (attrs[count][0] == 'k' && attrs[count][1] == 0) { - key = attrs[count+1]; - } else if (attrs[count][0] == 'v' && attrs[count][1] == 0) { - value = attrs[count+1]; - } + void get_tag(osmium::builder::Builder* builder, const XML_Char** attrs) { + const char* k = ""; + const char* v = ""; + check_attributes(attrs, [&k, &v](const XML_Char* name, const XML_Char* value) { + if (name[0] == 'k' && name[1] == 0) { + k = value; + } else if (name[0] == 'v' && name[1] == 0) { + v = value; } - if (!m_tl_builder) { - m_tl_builder = std::unique_ptr(new osmium::builder::TagListBuilder(m_buffer, builder)); - } - m_tl_builder->add_tag(key, value); + }); + if (!m_tl_builder) { + m_tl_builder = std::unique_ptr(new osmium::builder::TagListBuilder(m_buffer, builder)); } + m_tl_builder->add_tag(k, v); } - void header_is_done() { - m_header_is_done = true; - if (m_read_types == osmium::osm_entity_bits::nothing) { - throw ParserIsDone(); - } + void mark_header_as_done() { + set_header_value(m_header); } void start_element(const XML_Char* element, const XML_Char** attrs) { @@ -449,16 +313,16 @@ namespace osmium { if (!strcmp(element, "osmChange")) { m_header.set_has_multiple_object_versions(true); } - for (int count = 0; attrs[count]; count += 2) { - if (!strcmp(attrs[count], "version")) { - m_header.set("version", attrs[count+1]); - if (strcmp(attrs[count+1], "0.6")) { - throw osmium::format_version_error(attrs[count+1]); + check_attributes(attrs, [this](const XML_Char* name, const XML_Char* value) { + if (!strcmp(name, "version")) { + m_header.set("version", value); + if (strcmp(value, "0.6")) { + throw osmium::format_version_error(value); } - } else if (!strcmp(attrs[count], "generator")) { - m_header.set("generator", attrs[count+1]); + } else if (!strcmp(name, "generator")) { + m_header.set("generator", value); } - } + }); if (m_header.get("version") == "") { throw osmium::format_version_error(); } @@ -470,8 +334,8 @@ namespace osmium { case context::top: assert(!m_tl_builder); if (!strcmp(element, "node")) { - header_is_done(); - if (m_read_types & osmium::osm_entity_bits::node) { + mark_header_as_done(); + if (read_types() & osmium::osm_entity_bits::node) { m_node_builder = std::unique_ptr(new osmium::builder::NodeBuilder(m_buffer)); m_node_builder->add_user(init_object(m_node_builder->object(), attrs)); m_context = context::node; @@ -479,8 +343,8 @@ namespace osmium { m_context = context::ignored_node; } } else if (!strcmp(element, "way")) { - header_is_done(); - if (m_read_types & osmium::osm_entity_bits::way) { + mark_header_as_done(); + if (read_types() & osmium::osm_entity_bits::way) { m_way_builder = std::unique_ptr(new osmium::builder::WayBuilder(m_buffer)); m_way_builder->add_user(init_object(m_way_builder->object(), attrs)); m_context = context::way; @@ -488,8 +352,8 @@ namespace osmium { m_context = context::ignored_way; } } else if (!strcmp(element, "relation")) { - header_is_done(); - if (m_read_types & osmium::osm_entity_bits::relation) { + mark_header_as_done(); + if (read_types() & osmium::osm_entity_bits::relation) { m_relation_builder = std::unique_ptr(new osmium::builder::RelationBuilder(m_buffer)); m_relation_builder->add_user(init_object(m_relation_builder->object(), attrs)); m_context = context::relation; @@ -497,8 +361,8 @@ namespace osmium { m_context = context::ignored_relation; } } else if (!strcmp(element, "changeset")) { - header_is_done(); - if (m_read_types & osmium::osm_entity_bits::changeset) { + mark_header_as_done(); + if (read_types() & osmium::osm_entity_bits::changeset) { m_changeset_builder = std::unique_ptr(new osmium::builder::ChangesetBuilder(m_buffer)); init_changeset(m_changeset_builder.get(), attrs); m_context = context::changeset; @@ -508,17 +372,17 @@ namespace osmium { } else if (!strcmp(element, "bounds")) { osmium::Location min; osmium::Location max; - for (int count = 0; attrs[count]; count += 2) { - if (!strcmp(attrs[count], "minlon")) { - min.set_lon(atof(attrs[count+1])); - } else if (!strcmp(attrs[count], "minlat")) { - min.set_lat(atof(attrs[count+1])); - } else if (!strcmp(attrs[count], "maxlon")) { - max.set_lon(atof(attrs[count+1])); - } else if (!strcmp(attrs[count], "maxlat")) { - max.set_lat(atof(attrs[count+1])); + check_attributes(attrs, [&min, &max](const XML_Char* name, const XML_Char* value) { + if (!strcmp(name, "minlon")) { + min.set_lon(atof(value)); + } else if (!strcmp(name, "minlat")) { + min.set_lat(atof(value)); + } else if (!strcmp(name, "maxlon")) { + max.set_lon(atof(value)); + } else if (!strcmp(name, "maxlat")) { + max.set_lat(atof(value)); } - } + }); osmium::Box box; box.extend(min).extend(max); m_header.add_box(box); @@ -529,7 +393,9 @@ namespace osmium { case context::node: m_last_context = context::node; m_context = context::in_object; - check_tag(m_node_builder.get(), element, attrs); + if (!strcmp(element, "tag")) { + get_tag(m_node_builder.get(), attrs); + } break; case context::way: m_last_context = context::way; @@ -541,13 +407,14 @@ namespace osmium { m_wnl_builder = std::unique_ptr(new osmium::builder::WayNodeListBuilder(m_buffer, m_way_builder.get())); } - for (int count = 0; attrs[count]; count += 2) { - if (!strcmp(attrs[count], "ref")) { - m_wnl_builder->add_node_ref(osmium::string_to_object_id(attrs[count+1])); + check_attributes(attrs, [this](const XML_Char* name, const XML_Char* value) { + if (!strcmp(name, "ref")) { + m_wnl_builder->add_node_ref(osmium::string_to_object_id(value)); } - } - } else { - check_tag(m_way_builder.get(), element, attrs); + }); + } else if (!strcmp(element, "tag")) { + m_wnl_builder.reset(); + get_tag(m_way_builder.get(), attrs); } break; case context::relation: @@ -560,28 +427,68 @@ namespace osmium { m_rml_builder = std::unique_ptr(new osmium::builder::RelationMemberListBuilder(m_buffer, m_relation_builder.get())); } - char type = 'x'; - object_id_type ref = 0; + item_type type = item_type::undefined; + object_id_type ref = 0; const char* role = ""; - for (int count = 0; attrs[count]; count += 2) { - if (!strcmp(attrs[count], "type")) { - type = static_cast(attrs[count+1][0]); - } else if (!strcmp(attrs[count], "ref")) { - ref = osmium::string_to_object_id(attrs[count+1]); - } else if (!strcmp(attrs[count], "role")) { - role = static_cast(attrs[count+1]); + check_attributes(attrs, [&type, &ref, &role](const XML_Char* name, const XML_Char* value) { + if (!strcmp(name, "type")) { + type = char_to_item_type(value[0]); + } else if (!strcmp(name, "ref")) { + ref = osmium::string_to_object_id(value); + } else if (!strcmp(name, "role")) { + role = static_cast(value); } + }); + if (type != item_type::node && type != item_type::way && type != item_type::relation) { + throw osmium::xml_error("Unknown type on relation member"); } - // XXX assert type, ref, role are set - m_rml_builder->add_member(char_to_item_type(type), ref, role); - } else { - check_tag(m_relation_builder.get(), element, attrs); + if (ref == 0) { + throw osmium::xml_error("Missing ref on relation member"); + } + m_rml_builder->add_member(type, ref, role); + } else if (!strcmp(element, "tag")) { + m_rml_builder.reset(); + get_tag(m_relation_builder.get(), attrs); } break; case context::changeset: m_last_context = context::changeset; - m_context = context::in_object; - check_tag(m_changeset_builder.get(), element, attrs); + if (!strcmp(element, "discussion")) { + m_context = context::discussion; + m_tl_builder.reset(); + if (!m_changeset_discussion_builder) { + m_changeset_discussion_builder = std::unique_ptr(new osmium::builder::ChangesetDiscussionBuilder(m_buffer, m_changeset_builder.get())); + } + } else if (!strcmp(element, "tag")) { + m_context = context::in_object; + m_changeset_discussion_builder.reset(); + get_tag(m_changeset_builder.get(), attrs); + } + break; + case context::discussion: + if (!strcmp(element, "comment")) { + m_context = context::comment; + osmium::Timestamp date; + osmium::user_id_type uid = 0; + const char* user = ""; + check_attributes(attrs, [&date, &uid, &user](const XML_Char* name, const XML_Char* value) { + if (!strcmp(name, "date")) { + date = osmium::Timestamp(value); + } else if (!strcmp(name, "uid")) { + uid = osmium::string_to_user_id(value); + } else if (!strcmp(name, "user")) { + user = static_cast(value); + } + }); + m_changeset_discussion_builder->add_comment(date, uid, user); + } + break; + case context::comment: + if (!strcmp(element, "text")) { + m_context = context::comment_text; + } + break; + case context::comment_text: break; case context::ignored_node: break; @@ -604,7 +511,7 @@ namespace osmium { break; case context::top: if (!strcmp(element, "osm") || !strcmp(element, "osmChange")) { - header_is_done(); + mark_header_as_done(); m_context = context::root; } else if (!strcmp(element, "delete")) { m_in_delete_section = false; @@ -639,11 +546,25 @@ namespace osmium { case context::changeset: assert(!strcmp(element, "changeset")); m_tl_builder.reset(); + m_changeset_discussion_builder.reset(); m_changeset_builder.reset(); m_buffer.commit(); m_context = context::top; flush_buffer(); break; + case context::discussion: + assert(!strcmp(element, "discussion")); + m_context = context::changeset; + break; + case context::comment: + assert(!strcmp(element, "comment")); + m_context = context::discussion; + break; + case context::comment_text: + assert(!strcmp(element, "text")); + m_context = context::comment; + m_changeset_discussion_builder->add_comment_text(m_comment_text); + break; case context::in_object: m_context = m_last_context; break; @@ -670,85 +591,84 @@ namespace osmium { } } + void characters(const XML_Char* text, int len) { + if (m_context == context::comment_text) { + m_comment_text.append(text, len); + } else { + m_comment_text.resize(0); + } + } + void flush_buffer() { - if (m_buffer.capacity() - m_buffer.committed() < 1000 * 1000) { - m_queue.push(std::move(m_buffer)); + if (m_buffer.committed() > buffer_size / 10 * 9) { + send_to_output_queue(std::move(m_buffer)); osmium::memory::Buffer buffer(buffer_size); - std::swap(m_buffer, buffer); + using std::swap; + swap(m_buffer, buffer); + } + } + + public: + + XMLParser(future_string_queue_type& input_queue, + future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_types) : + Parser(input_queue, output_queue, header_promise, read_types), + m_context(context::root), + m_last_context(context::root), + m_in_delete_section(false), + m_header(), + m_buffer(buffer_size), + m_node_builder(), + m_way_builder(), + m_relation_builder(), + m_changeset_builder(), + m_changeset_discussion_builder(), + m_tl_builder(), + m_wnl_builder(), + m_rml_builder() { + } + + ~XMLParser() noexcept final = default; + + void run() final { + osmium::thread::set_thread_name("_osmium_xml_in"); + + ExpatXMLParser parser(this); + + while (!input_done()) { + std::string data = get_input(); + parser(data, input_done()); + if (read_types() == osmium::osm_entity_bits::nothing && header_is_done()) { + break; + } + } + + mark_header_as_done(); + + if (m_buffer.committed() > 0) { + send_to_output_queue(std::move(m_buffer)); } } }; // class XMLParser - class XMLInputFormat : public osmium::io::detail::InputFormat { + // we want the register_parser() function to run, setting + // the variable is only a side-effect, it will never be used + const bool registered_xml_parser = ParserFactory::instance().register_parser( + file_format::xml, + [](future_string_queue_type& input_queue, + future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_which_entities) { + return std::unique_ptr(new XMLParser(input_queue, output_queue, header_promise, read_which_entities)); + }); - static constexpr size_t max_queue_size = 100; - - osmium::thread::Queue m_queue; - std::atomic m_done; - std::promise m_header_promise; - std::future m_parser_future; - - public: - - /** - * Instantiate XML Parser - * - * @param file osmium::io::File instance describing file to be read from. - * @param read_which_entities Which types of OSM entities (nodes, ways, relations, changesets) should be parsed? - * @param input_queue String queue where data is read from. - */ - explicit XMLInputFormat(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue& input_queue) : - osmium::io::detail::InputFormat(file, read_which_entities), - m_queue(max_queue_size, "xml_parser_results"), - m_done(false), - m_header_promise(), - m_parser_future(std::async(std::launch::async, XMLParser(input_queue, m_queue, m_header_promise, read_which_entities, m_done))) { - } - - ~XMLInputFormat() { - try { - close(); - } catch (...) { - // ignore any exceptions at this point because destructor should not throw - } - } - - virtual osmium::io::Header header() override final { - osmium::thread::check_for_exception(m_parser_future); - return m_header_promise.get_future().get(); - } - - osmium::memory::Buffer read() override { - osmium::memory::Buffer buffer; - if (!m_done || !m_queue.empty()) { - m_queue.wait_and_pop(buffer); - } - - osmium::thread::check_for_exception(m_parser_future); - return buffer; - } - - void close() override { - m_done = true; - osmium::thread::wait_until_done(m_parser_future); - } - - }; // class XMLInputFormat - - namespace { - -// we want the register_input_format() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" - const bool registered_xml_input = osmium::io::detail::InputFormatFactory::instance().register_input_format(osmium::io::file_format::xml, - [](const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue& input_queue) { - return new osmium::io::detail::XMLInputFormat(file, read_which_entities, input_queue); - }); -#pragma GCC diagnostic pop - - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_xml_parser() noexcept { + return registered_xml_parser; + } } // namespace detail diff --git a/third_party/libosmium/include/osmium/io/detail/xml_output_format.hpp b/third_party/libosmium/include/osmium/io/detail/xml_output_format.hpp index 2a381d5ab..3d7878c8e 100644 --- a/third_party/libosmium/include/osmium/io/detail/xml_output_format.hpp +++ b/third_party/libosmium/include/osmium/io/detail/xml_output_format.hpp @@ -33,20 +33,17 @@ DEALINGS IN THE SOFTWARE. */ -#include -#include +#include #include #include #include #include #include #include -#include #include #include #include -#include #include #include #include @@ -75,45 +72,23 @@ namespace osmium { struct XMLWriteError {}; - namespace { + struct xml_output_options { - void xml_string(std::string& out, const char* in) { - for (; *in != '\0'; ++in) { - switch(*in) { - case '&': out += "&"; break; - case '\"': out += """; break; - case '\'': out += "'"; break; - case '<': out += "<"; break; - case '>': out += ">"; break; - case '\n': out += " "; break; - case '\r': out += " "; break; - case '\t': out += " "; break; - default: out += *in; break; - } - } - } + /// Should metadata of objects be added? + bool add_metadata; - const size_t tmp_buffer_size = 100; + /// Should the visible flag be added to all OSM objects? + bool add_visible_flag; - template - void oprintf(std::string& out, const char* format, T value) { - char buffer[tmp_buffer_size+1]; - size_t max_size = sizeof(buffer)/sizeof(char); -#ifndef NDEBUG - int len = -#endif -#ifndef _MSC_VER - snprintf(buffer, max_size, format, value); -#else - _snprintf(buffer, max_size, format, value); -#endif - assert(len > 0 && static_cast(len) < max_size); - out += buffer; - } + /** + * Should , , "operations" be added? + * (This is used for .osc files.) + */ + bool use_change_ops; - } // anonymous namespace + }; - class XMLOutputBlock : public osmium::handler::Handler { + class XMLOutputBlock : public OutputBlock { // operation (create, modify, delete) for osc files enum class operation { @@ -123,15 +98,9 @@ namespace osmium { op_delete = 3 }; // enum class operation - std::shared_ptr m_input_buffer; - - std::shared_ptr m_out; - operation m_last_op {operation::op_none}; - const bool m_add_metadata; - const bool m_write_visible_flag; - const bool m_write_change_ops; + xml_output_options m_options; void write_spaces(int num) { for (; num != 0; --num) { @@ -139,20 +108,20 @@ namespace osmium { } } + int prefix_spaces() { + return m_options.use_change_ops ? 4 : 2; + } + void write_prefix() { - if (m_write_change_ops) { - write_spaces(4); - } else { - write_spaces(2); - } + write_spaces(prefix_spaces()); } void write_meta(const osmium::OSMObject& object) { - oprintf(*m_out, " id=\"%" PRId64 "\"", object.id()); + output_formatted(" id=\"%" PRId64 "\"", object.id()); - if (m_add_metadata) { + if (m_options.add_metadata) { if (object.version()) { - oprintf(*m_out, " version=\"%d\"", object.version()); + output_formatted(" version=\"%d\"", object.version()); } if (object.timestamp()) { @@ -162,16 +131,16 @@ namespace osmium { } if (!object.user_is_anonymous()) { - oprintf(*m_out, " uid=\"%d\" user=\"", object.uid()); - xml_string(*m_out, object.user()); + output_formatted(" uid=\"%d\" user=\"", object.uid()); + append_xml_encoded_string(*m_out, object.user()); *m_out += "\""; } if (object.changeset()) { - oprintf(*m_out, " changeset=\"%d\"", object.changeset()); + output_formatted(" changeset=\"%d\"", object.changeset()); } - if (m_write_visible_flag) { + if (m_options.add_visible_flag) { if (object.visible()) { *m_out += " visible=\"true\""; } else { @@ -181,17 +150,31 @@ namespace osmium { } } - void write_tags(const osmium::TagList& tags) { + void write_tags(const osmium::TagList& tags, int spaces) { for (const auto& tag : tags) { - write_prefix(); + write_spaces(spaces); *m_out += " \n"; } } + void write_discussion(const osmium::ChangesetDiscussion& comments) { + for (const auto& comment : comments) { + output_formatted(" \n"; + *m_out += " "; + append_xml_encoded_string(*m_out, comment.text()); + *m_out += "\n \n"; + } + *m_out += " \n"; + } + void open_close_op_tag(const operation op = operation::op_none) { if (op == m_last_op) { return; @@ -230,12 +213,9 @@ namespace osmium { public: - explicit XMLOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata, bool write_visible_flag, bool write_change_ops) : - m_input_buffer(std::make_shared(std::move(buffer))), - m_out(std::make_shared()), - m_add_metadata(add_metadata), - m_write_visible_flag(write_visible_flag && !write_change_ops), - m_write_change_ops(write_change_ops) { + XMLOutputBlock(osmium::memory::Buffer&& buffer, const xml_output_options& options) : + OutputBlock(std::move(buffer)), + m_options(options) { } XMLOutputBlock(const XMLOutputBlock&) = default; @@ -244,22 +224,24 @@ namespace osmium { XMLOutputBlock(XMLOutputBlock&&) = default; XMLOutputBlock& operator=(XMLOutputBlock&&) = default; - ~XMLOutputBlock() = default; + ~XMLOutputBlock() noexcept = default; std::string operator()() { osmium::apply(m_input_buffer->cbegin(), m_input_buffer->cend(), *this); - if (m_write_change_ops) { + if (m_options.use_change_ops) { open_close_op_tag(); } std::string out; - std::swap(out, *m_out); + using std::swap; + swap(out, *m_out); + return out; } void node(const osmium::Node& node) { - if (m_write_change_ops) { + if (m_options.use_change_ops) { open_close_op_tag(node.visible() ? (node.version() == 1 ? operation::op_create : operation::op_modify) : operation::op_delete); } @@ -283,14 +265,14 @@ namespace osmium { *m_out += ">\n"; - write_tags(node.tags()); + write_tags(node.tags(), prefix_spaces()); write_prefix(); *m_out += "\n"; } void way(const osmium::Way& way) { - if (m_write_change_ops) { + if (m_options.use_change_ops) { open_close_op_tag(way.visible() ? (way.version() == 1 ? operation::op_create : operation::op_modify) : operation::op_delete); } @@ -307,17 +289,17 @@ namespace osmium { for (const auto& node_ref : way.nodes()) { write_prefix(); - oprintf(*m_out, " \n", node_ref.ref()); + output_formatted(" \n", node_ref.ref()); } - write_tags(way.tags()); + write_tags(way.tags(), prefix_spaces()); write_prefix(); *m_out += "\n"; } void relation(const osmium::Relation& relation) { - if (m_write_change_ops) { + if (m_options.use_change_ops) { open_close_op_tag(relation.visible() ? (relation.version() == 1 ? operation::op_create : operation::op_modify) : operation::op_delete); } @@ -336,22 +318,21 @@ namespace osmium { write_prefix(); *m_out += " \n"; } - write_tags(relation.tags()); + write_tags(relation.tags(), prefix_spaces()); write_prefix(); *m_out += "\n"; } void changeset(const osmium::Changeset& changeset) { - write_prefix(); - *m_out += " 0) { + *m_out += " \n"; + write_discussion(changeset.discussion()); + } + + *m_out += " \n"; } }; // class XMLOutputBlock class XMLOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler { - bool m_add_metadata; - bool m_write_visible_flag; + xml_output_options m_options; public: - XMLOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : - OutputFormat(file, output_queue), - m_add_metadata(file.get("add_metadata") != "false"), - m_write_visible_flag(file.has_multiple_object_versions() || m_file.is_true("force_visible_flag")) { + XMLOutputFormat(const osmium::io::File& file, future_string_queue_type& output_queue) : + OutputFormat(output_queue), + m_options() { + m_options.add_metadata = file.is_not_false("add_metadata"); + m_options.use_change_ops = file.is_true("xml_change_format"); + m_options.add_visible_flag = (file.has_multiple_object_versions() || file.is_true("force_visible_flag")) && !m_options.use_change_ops; } XMLOutputFormat(const XMLOutputFormat&) = delete; XMLOutputFormat& operator=(const XMLOutputFormat&) = delete; - ~XMLOutputFormat() override final { - } + ~XMLOutputFormat() noexcept final = default; - void write_buffer(osmium::memory::Buffer&& buffer) override final { - m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_add_metadata, m_write_visible_flag, m_file.is_true("xml_change_format")})); - } - - void write_header(const osmium::io::Header& header) override final { + void write_header(const osmium::io::Header& header) final { std::string out = "\n"; - if (m_file.is_true("xml_change_format")) { + if (m_options.use_change_ops) { out += "\n"; } else { out += "\n"; } + append_xml_encoded_string(out, header.get("generator").c_str()); + out += "\">\n"; for (const auto& box : header.boxes()) { out += " \n", box.top_right().lat()); + append_printf_formatted_string(out, " minlon=\"%.7f\"", box.bottom_left().lon()); + append_printf_formatted_string(out, " minlat=\"%.7f\"", box.bottom_left().lat()); + append_printf_formatted_string(out, " maxlon=\"%.7f\"", box.top_right().lon()); + append_printf_formatted_string(out, " maxlat=\"%.7f\"/>\n", box.top_right().lat()); } - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(std::move(out)); + send_to_output_queue(std::move(out)); } - void close() override final { - { - std::string out; - if (m_file.is_true("xml_change_format")) { - out += "\n"; - } else { - out += "\n"; - } + void write_buffer(osmium::memory::Buffer&& buffer) final { + m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_options})); + } - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(std::move(out)); + void write_end() final { + std::string out; + + if (m_options.use_change_ops) { + out += "\n"; + } else { + out += "\n"; } - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(std::string()); + send_to_output_queue(std::move(out)); } }; // class XMLOutputFormat - namespace { + // we want the register_output_format() function to run, setting + // the variable is only a side-effect, it will never be used + const bool registered_xml_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::xml, + [](const osmium::io::File& file, future_string_queue_type& output_queue) { + return new osmium::io::detail::XMLOutputFormat(file, output_queue); + }); -// we want the register_output_format() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" - const bool registered_xml_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::xml, - [](const osmium::io::File& file, data_queue_type& output_queue) { - return new osmium::io::detail::XMLOutputFormat(file, output_queue); - }); -#pragma GCC diagnostic pop - - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_xml_output() noexcept { + return registered_xml_output; + } } // namespace detail - } // namespace output + } // namespace io } // namespace osmium diff --git a/third_party/libosmium/include/osmium/io/detail/zlib.hpp b/third_party/libosmium/include/osmium/io/detail/zlib.hpp index fc0baf344..4d86168d2 100644 --- a/third_party/libosmium/include/osmium/io/detail/zlib.hpp +++ b/third_party/libosmium/include/osmium/io/detail/zlib.hpp @@ -39,6 +39,7 @@ DEALINGS IN THE SOFTWARE. #include +#include #include namespace osmium { @@ -69,7 +70,7 @@ namespace osmium { ); if (result != Z_OK) { - throw std::runtime_error(std::string("failed to compress data: ") + zError(result)); + throw io_error(std::string("failed to compress data: ") + zError(result)); } output.resize(output_size); @@ -99,7 +100,7 @@ namespace osmium { ); if (result != Z_OK) { - throw std::runtime_error(std::string("failed to uncompress data: ") + zError(result)); + throw io_error(std::string("failed to uncompress data: ") + zError(result)); } return std::make_pair(output.data(), output.size()); diff --git a/third_party/libosmium/include/osmium/io/error.hpp b/third_party/libosmium/include/osmium/io/error.hpp index 07652bc59..6b5c677c0 100644 --- a/third_party/libosmium/include/osmium/io/error.hpp +++ b/third_party/libosmium/include/osmium/io/error.hpp @@ -43,16 +43,28 @@ namespace osmium { */ struct io_error : public std::runtime_error { - io_error(const std::string& what) : + explicit io_error(const std::string& what) : std::runtime_error(what) { } - io_error(const char* what) : + explicit io_error(const char* what) : std::runtime_error(what) { } }; // struct io_error + struct unsupported_file_format_error : public io_error { + + explicit unsupported_file_format_error(const std::string& what) : + io_error(what) { + } + + explicit unsupported_file_format_error(const char* what) : + io_error(what) { + } + + }; // struct unsupported_file_format_error + } // namespace osmium #endif // OSMIUM_IO_ERROR_HPP diff --git a/third_party/libosmium/include/osmium/io/file.hpp b/third_party/libosmium/include/osmium/io/file.hpp index 3bbfacc6d..87fcc3707 100644 --- a/third_party/libosmium/include/osmium/io/file.hpp +++ b/third_party/libosmium/include/osmium/io/file.hpp @@ -39,6 +39,7 @@ DEALINGS IN THE SOFTWARE. #include #include +#include #include #include #include @@ -255,9 +256,9 @@ namespace osmium { * Check file format etc. for consistency and throw exception if * there is a problem. * - * @throws std::runtime_error + * @throws osmium::io_error */ - void check() const { + const File& check() const { if (m_file_format == file_format::unknown) { std::string msg = "Could not detect file format"; if (!m_format_string.empty()) { @@ -273,8 +274,9 @@ namespace osmium { msg += "'"; } msg += "."; - throw std::runtime_error(msg); + throw io_error(msg); } + return *this; } file_format format() const noexcept { diff --git a/third_party/libosmium/include/osmium/io/gzip_compression.hpp b/third_party/libosmium/include/osmium/io/gzip_compression.hpp index 204bfd521..184787539 100644 --- a/third_party/libosmium/include/osmium/io/gzip_compression.hpp +++ b/third_party/libosmium/include/osmium/io/gzip_compression.hpp @@ -49,7 +49,9 @@ DEALINGS IN THE SOFTWARE. #include #include +#include #include +#include #include #include @@ -59,13 +61,13 @@ namespace osmium { * Exception thrown when there are problems compressing or * decompressing gzip files. */ - struct gzip_error : public std::runtime_error { + struct gzip_error : public io_error { int gzip_error_code; int system_errno; gzip_error(const std::string& what, int error_code) : - std::runtime_error(what), + io_error(what), gzip_error_code(error_code), system_errno(error_code == Z_ERRNO ? errno : 0) { } @@ -93,23 +95,29 @@ namespace osmium { class GzipCompressor : public Compressor { + int m_fd; gzFile m_gzfile; public: - explicit GzipCompressor(int fd) : - Compressor(), + explicit GzipCompressor(int fd, fsync sync) : + Compressor(sync), + m_fd(dup(fd)), m_gzfile(::gzdopen(fd, "w")) { if (!m_gzfile) { detail::throw_gzip_error(m_gzfile, "write initialization failed"); } } - ~GzipCompressor() override final { - close(); + ~GzipCompressor() noexcept final { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } } - void write(const std::string& data) override final { + void write(const std::string& data) final { if (!data.empty()) { int nwrite = ::gzwrite(m_gzfile, data.data(), static_cast_with_assert(data.size())); if (nwrite == 0) { @@ -118,13 +126,17 @@ namespace osmium { } } - void close() override final { + void close() final { if (m_gzfile) { int result = ::gzclose(m_gzfile); m_gzfile = nullptr; if (result != Z_OK) { detail::throw_gzip_error(m_gzfile, "write close failed", result); } + if (do_fsync()) { + osmium::io::detail::reliable_fsync(m_fd); + } + osmium::io::detail::reliable_close(m_fd); } } @@ -144,11 +156,15 @@ namespace osmium { } } - ~GzipDecompressor() override final { - close(); + ~GzipDecompressor() noexcept final { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } } - std::string read() override final { + std::string read() final { std::string buffer(osmium::io::Decompressor::input_buffer_size, '\0'); int nread = ::gzread(m_gzfile, const_cast(buffer.data()), static_cast_with_assert(buffer.size())); if (nread < 0) { @@ -158,7 +174,7 @@ namespace osmium { return buffer; } - void close() override final { + void close() final { if (m_gzfile) { int result = ::gzclose(m_gzfile); m_gzfile = nullptr; @@ -194,11 +210,15 @@ namespace osmium { } } - ~GzipBufferDecompressor() override final { - inflateEnd(&m_zstream); + ~GzipBufferDecompressor() noexcept final { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } } - std::string read() override final { + std::string read() final { std::string output; if (m_buffer) { @@ -227,22 +247,28 @@ namespace osmium { return output; } + void close() final { + inflateEnd(&m_zstream); + } + }; // class GzipBufferDecompressor - namespace { + namespace detail { -// we want the register_compression() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" + // we want the register_compression() function to run, setting + // the variable is only a side-effect, it will never be used const bool registered_gzip_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::gzip, - [](int fd) { return new osmium::io::GzipCompressor(fd); }, + [](int fd, fsync sync) { return new osmium::io::GzipCompressor(fd, sync); }, [](int fd) { return new osmium::io::GzipDecompressor(fd); }, [](const char* buffer, size_t size) { return new osmium::io::GzipBufferDecompressor(buffer, size); } ); -#pragma GCC diagnostic pop - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_gzip_compression() noexcept { + return registered_gzip_compression; + } + + } // namespace detail } // namespace io diff --git a/third_party/libosmium/include/osmium/io/input_iterator.hpp b/third_party/libosmium/include/osmium/io/input_iterator.hpp index f6197299d..864776303 100644 --- a/third_party/libosmium/include/osmium/io/input_iterator.hpp +++ b/third_party/libosmium/include/osmium/io/input_iterator.hpp @@ -52,7 +52,7 @@ namespace osmium { * source. It hides all the buffer handling and makes the contents of a * source accessible as a normal STL input iterator. */ - template + template class InputIterator { static_assert(std::is_base_of::value, "TItem must derive from osmium::buffer::Item"); @@ -133,6 +133,44 @@ namespace osmium { }; // class InputIterator + template + class InputIteratorRange { + + InputIterator m_begin; + InputIterator m_end; + + public: + + InputIteratorRange(InputIterator&& begin, + InputIterator&& end) : + m_begin(std::move(begin)), + m_end(std::move(end)) { + } + + InputIterator begin() const noexcept { + return m_begin; + } + + InputIterator end() const noexcept { + return m_end; + } + + const InputIterator cbegin() const noexcept { + return m_begin; + } + + const InputIterator cend() const noexcept { + return m_end; + } + + }; // class InputIteratorRange + + template + InputIteratorRange make_input_iterator_range(TSource& source) { + using it_type = InputIterator; + return InputIteratorRange(it_type{source}, it_type{}); + } + } // namespace io } // namespace osmium diff --git a/third_party/libosmium/include/osmium/io/o5m_input.hpp b/third_party/libosmium/include/osmium/io/o5m_input.hpp new file mode 100644 index 000000000..b63f72853 --- /dev/null +++ b/third_party/libosmium/include/osmium/io/o5m_input.hpp @@ -0,0 +1,45 @@ +#ifndef OSMIUM_IO_O5M_INPUT_HPP +#define OSMIUM_IO_O5M_INPUT_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +/** + * @file + * + * Include this file if you want to read OSM o5m and o5c files. + */ + +#include // IWYU pragma: export +#include // IWYU pragma: export + +#endif // OSMIUM_IO_O5M_INPUT_HPP diff --git a/third_party/libosmium/include/osmium/io/output_iterator.hpp b/third_party/libosmium/include/osmium/io/output_iterator.hpp index 608852fa9..1cf1d1dcb 100644 --- a/third_party/libosmium/include/osmium/io/output_iterator.hpp +++ b/third_party/libosmium/include/osmium/io/output_iterator.hpp @@ -40,6 +40,7 @@ DEALINGS IN THE SOFTWARE. #include #include +#include namespace osmium { @@ -49,30 +50,26 @@ namespace osmium { namespace io { - template + template class OutputIterator : public std::iterator { - struct buffer_wrapper { - - osmium::memory::Buffer buffer; - - buffer_wrapper(size_t buffer_size) : - buffer(buffer_size, osmium::memory::Buffer::auto_grow::no) { - } - - }; // struct buffer_wrapper - - static constexpr size_t default_buffer_size = 10 * 1024 * 1024; - TDest* m_destination; - std::shared_ptr m_buffer_wrapper; - public: - explicit OutputIterator(TDest& destination, const size_t buffer_size = default_buffer_size) : - m_destination(&destination), - m_buffer_wrapper(std::make_shared(buffer_size)) { + explicit OutputIterator(TDest& destination) : + m_destination(&destination) { + } + + /** + * @deprecated + * Use of buffer size argument on OutputIterator + * constructor is deprecated. Call Writer::set_buffer_size() + * instead if you want to change the default. + */ + OSMIUM_DEPRECATED OutputIterator(TDest& destination, const size_t buffer_size) : + m_destination(&destination) { + destination.set_buffer_size(buffer_size); } OutputIterator(const OutputIterator&) = default; @@ -83,19 +80,17 @@ namespace osmium { ~OutputIterator() = default; - void flush() { - osmium::memory::Buffer buffer(m_buffer_wrapper->buffer.capacity(), osmium::memory::Buffer::auto_grow::no); - std::swap(m_buffer_wrapper->buffer, buffer); - (*m_destination)(std::move(buffer)); + /** + * @deprecated + * Calling OutputIterator::flush() is usually not + * needed any more. Call flush() on the Writer instead if needed. + */ + OSMIUM_DEPRECATED void flush() { + m_destination->flush(); } OutputIterator& operator=(const osmium::memory::Item& item) { - try { - m_buffer_wrapper->buffer.push_back(item); - } catch (osmium::buffer_is_full&) { - flush(); - m_buffer_wrapper->buffer.push_back(item); - } + (*m_destination)(item); return *this; } @@ -117,6 +112,23 @@ namespace osmium { }; // class OutputIterator + template + OutputIterator make_output_iterator(TDest& destination) { + return OutputIterator{destination}; + } + + /** + * @deprecated + * Use of buffer size argument on make_output_iterator is deprecated. + * Call Writer::set_buffer_size() instead if you want to change the + * default. + */ + template + OSMIUM_DEPRECATED OutputIterator make_output_iterator(TDest& destination, const size_t buffer_size) { + destination.set_buffer_size(buffer_size); + return OutputIterator{destination}; + } + } // namespace io } // namespace osmium diff --git a/third_party/libosmium/include/osmium/io/overwrite.hpp b/third_party/libosmium/include/osmium/io/overwrite.hpp index e33894bd8..f9fbc71af 100644 --- a/third_party/libosmium/include/osmium/io/overwrite.hpp +++ b/third_party/libosmium/include/osmium/io/overwrite.hpp @@ -33,20 +33,7 @@ DEALINGS IN THE SOFTWARE. */ -namespace osmium { - - namespace io { - - /** - * Allow overwriting of existing file. - */ - enum class overwrite : bool { - no = false, - allow = true - }; - - } // namespace io - -} // namespace osmium +#pragma message("Including overwrite.hpp is deprecated, #include instead.") +#include #endif // OSMIUM_IO_OVERWRITE_HPP diff --git a/third_party/libosmium/include/osmium/io/reader.hpp b/third_party/libosmium/include/osmium/io/reader.hpp index c68a8e180..fd2a6d240 100644 --- a/third_party/libosmium/include/osmium/io/reader.hpp +++ b/third_party/libosmium/include/osmium/io/reader.hpp @@ -33,10 +33,10 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include #include +#include #include #include #include @@ -55,12 +55,13 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include +#include #include #include #include #include #include -#include namespace osmium { @@ -74,17 +75,46 @@ namespace osmium { */ class Reader { + static constexpr size_t max_input_queue_size = 20; // XXX + static constexpr size_t max_osmdata_queue_size = 20; // XXX + osmium::io::File m_file; osmium::osm_entity_bits::type m_read_which_entities; - std::atomic m_input_done; + + enum class status { + okay = 0, // normal reading + error = 1, // some error occurred while reading + closed = 2, // close() called successfully after eof + eof = 3 // eof of file was reached without error + } m_status; + int m_childpid; - osmium::thread::Queue m_input_queue; + detail::future_string_queue_type m_input_queue; std::unique_ptr m_decompressor; - std::future m_read_future; - std::unique_ptr m_input; + osmium::io::detail::ReadThreadManager m_read_thread_manager; + + detail::future_buffer_queue_type m_osmdata_queue; + detail::queue_wrapper m_osmdata_queue_wrapper; + + std::future m_header_future; + osmium::io::Header m_header; + + osmium::thread::thread_handler m_thread; + + // This function will run in a separate thread. + static void parser_thread(const osmium::io::File& file, + detail::future_string_queue_type& input_queue, + detail::future_buffer_queue_type& osmdata_queue, + std::promise&& header_promise, + osmium::osm_entity_bits::type read_which_entities) { + std::promise promise = std::move(header_promise); + auto creator = detail::ParserFactory::instance().get_creator_function(file); + auto parser = creator(input_queue, osmdata_queue, promise, read_which_entities); + parser->parse(); + } #ifndef _WIN32 /** @@ -149,7 +179,7 @@ namespace osmium { #ifndef _WIN32 return execute("curl", filename, childpid); #else - throw std::runtime_error("Reading OSM files from the network currently not supported on Windows."); + throw io_error("Reading OSM files from the network currently not supported on Windows."); #endif } else { return osmium::io::detail::open_for_reading(filename); @@ -168,16 +198,23 @@ namespace osmium { * parsed. */ explicit Reader(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities = osmium::osm_entity_bits::all) : - m_file(file), + m_file(file.check()), m_read_which_entities(read_which_entities), - m_input_done(false), + m_status(status::okay), m_childpid(0), - m_input_queue(20, "raw_input"), // XXX + m_input_queue(max_input_queue_size, "raw_input"), m_decompressor(m_file.buffer() ? osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), m_file.buffer(), m_file.buffer_size()) : osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), open_input_file_or_url(m_file.filename(), &m_childpid))), - m_read_future(std::async(std::launch::async, detail::ReadThread(m_input_queue, m_decompressor.get(), m_input_done))), - m_input(osmium::io::detail::InputFormatFactory::instance().create_input(m_file, m_read_which_entities, m_input_queue)) { + m_read_thread_manager(*m_decompressor, m_input_queue), + m_osmdata_queue(max_osmdata_queue_size, "parser_results"), + m_osmdata_queue_wrapper(m_osmdata_queue), + m_header_future(), + m_header(), + m_thread() { + std::promise header_promise; + m_header_future = header_promise.get_future(); + m_thread = osmium::thread::thread_handler{parser_thread, std::ref(m_file), std::ref(m_input_queue), std::ref(m_osmdata_queue), std::move(header_promise), read_which_entities}; } explicit Reader(const std::string& filename, osmium::osm_entity_bits::type read_types = osmium::osm_entity_bits::all) : @@ -191,27 +228,37 @@ namespace osmium { Reader(const Reader&) = delete; Reader& operator=(const Reader&) = delete; - ~Reader() { + Reader(Reader&&) = default; + Reader& operator=(Reader&&) = default; + + ~Reader() noexcept { try { close(); - } - catch (...) { + } catch (...) { + // Ignore any exceptions because destructor must not throw. } } /** * Close down the Reader. A call to this is optional, because the * destructor of Reader will also call this. But if you don't call - * this function first, the destructor might throw an exception - * which is not good. + * this function first, you might miss an exception, because the + * destructor is not allowed to throw. * - * @throws Some form of std::runtime_error when there is a problem. + * @throws Some form of osmium::io_error when there is a problem. */ void close() { - // Signal to input child process that it should wrap up. - m_input_done = true; + m_status = status::closed; - m_input->close(); + m_read_thread_manager.stop(); + + m_osmdata_queue_wrapper.drain(); + + try { + m_read_thread_manager.close(); + } catch (...) { + // Ignore any exceptions. + } #ifndef _WIN32 if (m_childpid) { @@ -226,15 +273,32 @@ namespace osmium { m_childpid = 0; } #endif - - osmium::thread::wait_until_done(m_read_future); } /** * Get the header data from the file. + * + * @returns Header. + * @throws Some form of osmium::io_error if there is an error. */ - osmium::io::Header header() const { - return m_input->header(); + osmium::io::Header header() { + if (m_status == status::error) { + throw io_error("Can not get header from reader when in status 'error'"); + } + + try { + if (m_header_future.valid()) { + m_header = m_header_future.get(); + if (m_read_which_entities == osmium::osm_entity_bits::nothing) { + m_status = status::eof; + } + } + } catch (...) { + close(); + m_status = status::error; + throw; + } + return m_header; } /** @@ -245,32 +309,36 @@ namespace osmium { * constructed. * * @returns Buffer. - * @throws Some form of std::runtime_error if there is an error. + * @throws Some form of osmium::io_error if there is an error. */ osmium::memory::Buffer read() { - // If an exception happened in the input thread, re-throw - // it in this (the main) thread. - osmium::thread::check_for_exception(m_read_future); + osmium::memory::Buffer buffer; - if (m_read_which_entities == osmium::osm_entity_bits::nothing || m_input_done) { - // If the caller didn't want anything but the header, it will - // always get an empty buffer here. - return osmium::memory::Buffer(); + if (m_status != status::okay || + m_read_which_entities == osmium::osm_entity_bits::nothing) { + throw io_error("Can not read from reader when in status 'closed', 'eof', or 'error'"); } - // m_input->read() can return an invalid buffer to signal EOF, - // or a valid buffer with or without data. A valid buffer - // without data is not an error, it just means we have to - // keep getting the next buffer until there is one with data. - while (true) { - osmium::memory::Buffer buffer = m_input->read(); - if (!buffer) { - m_input_done = true; - return buffer; - } - if (buffer.committed() > 0) { - return buffer; + try { + // m_input_format.read() can return an invalid buffer to signal EOF, + // or a valid buffer with or without data. A valid buffer + // without data is not an error, it just means we have to + // keep getting the next buffer until there is one with data. + while (true) { + buffer = m_osmdata_queue_wrapper.pop(); + if (detail::at_end_of_data(buffer)) { + m_status = status::eof; + m_read_thread_manager.close(); + return buffer; + } + if (buffer.committed() > 0) { + return buffer; + } } + } catch (...) { + close(); + m_status = status::error; + throw; } } @@ -279,7 +347,7 @@ namespace osmium { * data has been read. It is also set by calling close(). */ bool eof() const { - return m_input_done; + return m_status == status::eof || m_status == status::closed; } }; // class Reader @@ -292,7 +360,7 @@ namespace osmium { * unless you are working with small OSM files and/or have lots of * RAM. */ - template + template osmium::memory::Buffer read_file(TArgs&&... args) { osmium::memory::Buffer buffer(1024*1024, osmium::memory::Buffer::auto_grow::yes); diff --git a/third_party/libosmium/include/osmium/io/writer.hpp b/third_party/libosmium/include/osmium/io/writer.hpp index 64afe2066..7cd133d0e 100644 --- a/third_party/libosmium/include/osmium/io/writer.hpp +++ b/third_party/libosmium/include/osmium/io/writer.hpp @@ -33,17 +33,23 @@ DEALINGS IN THE SOFTWARE. */ +#include +#include #include +#include #include +#include #include #include #include +#include #include #include +#include #include #include -#include +#include #include #include @@ -54,21 +60,112 @@ namespace osmium { /** * This is the user-facing interface for writing OSM files. Instantiate * an object of this class with a file name or osmium::io::File object - * and optionally the data for the header and then call operator() on it - * to write Buffers. Call close() to finish up. + * and optionally the data for the header and then call operator() on + * it to write Buffers or Items. + * + * The writer uses multithreading internally to do the actual encoding + * of the data into the intended format, possible compress the data and + * then write it out. But this is intentionally hidden from the user + * of this class who can use it without knowing those details. + * + * If you are done call the close() method to finish up. Only if you + * don't get an exception from the close() method, you can be sure + * the data is written correctly (modulo operating system buffering). + * The destructor of this class will also do the right thing if you + * forget to call close(), but because the destructor can't throw you + * will not get informed about any problems. + * + * The writer is usually used to write complete blocks of data stored + * in osmium::memory::Buffers. But you can also write single + * osmium::memory::Items. In this case the Writer uses an internal + * Buffer. */ class Writer { + static constexpr size_t default_buffer_size = 10 * 1024 * 1024; + osmium::io::File m_file; - osmium::io::detail::data_queue_type m_output_queue; + detail::future_string_queue_type m_output_queue; std::unique_ptr m_output; - std::unique_ptr m_compressor; + osmium::memory::Buffer m_buffer; + + size_t m_buffer_size; std::future m_write_future; + osmium::thread::thread_handler m_thread; + + enum class status { + okay = 0, // normal writing + error = 1, // some error occurred while writing + closed = 2 // close() called successfully + } m_status; + + // This function will run in a separate thread. + static void write_thread(detail::future_string_queue_type& output_queue, + std::unique_ptr&& compressor, + std::promise&& write_promise) { + detail::WriteThread write_thread{output_queue, + std::move(compressor), + std::move(write_promise)}; + write_thread(); + } + + void do_write(osmium::memory::Buffer&& buffer) { + if (buffer && buffer.committed() > 0) { + m_output->write_buffer(std::move(buffer)); + } + } + + void do_flush() { + osmium::thread::check_for_exception(m_write_future); + if (m_buffer && m_buffer.committed() > 0) { + osmium::memory::Buffer buffer{m_buffer_size, + osmium::memory::Buffer::auto_grow::no}; + using std::swap; + swap(m_buffer, buffer); + + m_output->write_buffer(std::move(buffer)); + } + } + + template + void ensure_cleanup(TFunction func, TArgs&&... args) { + if (m_status != status::okay) { + throw io_error("Can not write to writer when in status 'closed' or 'error'"); + } + + try { + func(std::forward(args)...); + } catch (...) { + m_status = status::error; + detail::add_to_queue(m_output_queue, std::current_exception()); + detail::add_end_of_data_to_queue(m_output_queue); + throw; + } + } + + struct options_type { + osmium::io::Header header; + overwrite allow_overwrite = overwrite::no; + fsync sync = fsync::no; + }; + + static void set_option(options_type& options, const osmium::io::Header& header) { + options.header = header; + } + + static void set_option(options_type& options, overwrite value) { + options.allow_overwrite = value; + } + + static void set_option(options_type& options, fsync value) { + options.sync = value; + } + public: /** @@ -76,64 +173,166 @@ namespace osmium { * header to it. * * @param file File (contains name and format info) to open. - * @param header Optional header data. If this is not given sensible - * defaults will be used. See the default constructor - * of osmium::io::Header for details. - * @param allow_overwrite Allow overwriting of existing file? Can be - * osmium::io::overwrite::allow or osmium::io::overwrite::no - * (default). + * @param args All further arguments are optional and can appear + * in any order: * - * @throws std::runtime_error If the file could not be opened. + * * osmium::io::Header: Optional header data. If this is + * not given, a default constructed osmium::io::Header + * object will be used. + * + * * osmium::io::overwrite: Allow overwriting of existing file? + * Can be osmium::io::overwrite::allow or + * osmium::io::overwrite::no (default). + * + * * osmium::io::fsync: Should fsync be called on the file + * before closing it? Can be osmium::io::fsync::yes or + * osmium::io::fsync::no (default). + * + * @throws osmium::io_error If there was an error. * @throws std::system_error If the file could not be opened. */ - explicit Writer(const osmium::io::File& file, const osmium::io::Header& header = osmium::io::Header(), overwrite allow_overwrite = overwrite::no) : - m_file(file), + template + explicit Writer(const osmium::io::File& file, TArgs&&... args) : + m_file(file.check()), m_output_queue(20, "raw_output"), // XXX m_output(osmium::io::detail::OutputFormatFactory::instance().create_output(m_file, m_output_queue)), - m_compressor(osmium::io::CompressionFactory::instance().create_compressor(file.compression(), osmium::io::detail::open_for_writing(m_file.filename(), allow_overwrite))), - m_write_future(std::async(std::launch::async, detail::WriteThread(m_output_queue, m_compressor.get()))) { - assert(!m_file.buffer()); - m_output->write_header(header); + m_buffer(), + m_buffer_size(default_buffer_size), + m_write_future(), + m_thread(), + m_status(status::okay) { + assert(!m_file.buffer()); // XXX can't handle pseudo-files + + options_type options; + (void)std::initializer_list{ + (set_option(options, args), 0)... + }; + + std::unique_ptr compressor = + CompressionFactory::instance().create_compressor(file.compression(), + osmium::io::detail::open_for_writing(m_file.filename(), options.allow_overwrite), + options.sync); + + std::promise write_promise; + m_write_future = write_promise.get_future(); + m_thread = osmium::thread::thread_handler{write_thread, std::ref(m_output_queue), std::move(compressor), std::move(write_promise)}; + + ensure_cleanup([&](){ + m_output->write_header(options.header); + }); } - explicit Writer(const std::string& filename, const osmium::io::Header& header = osmium::io::Header(), overwrite allow_overwrite = overwrite::no) : - Writer(osmium::io::File(filename), header, allow_overwrite) { + template + explicit Writer(const std::string& filename, TArgs&&... args) : + Writer(osmium::io::File(filename), std::forward(args)...) { } - explicit Writer(const char* filename, const osmium::io::Header& header = osmium::io::Header(), overwrite allow_overwrite = overwrite::no) : - Writer(osmium::io::File(filename), header, allow_overwrite) { + template + explicit Writer(const char* filename, TArgs&&... args) : + Writer(osmium::io::File(filename), std::forward(args)...) { } Writer(const Writer&) = delete; Writer& operator=(const Writer&) = delete; - ~Writer() { - close(); - } + Writer(Writer&&) = default; + Writer& operator=(Writer&&) = default; - /** - * Write contents of a buffer to the output file. - * - * @throws Some form of std::runtime_error when there is a problem. - */ - void operator()(osmium::memory::Buffer&& buffer) { - osmium::thread::check_for_exception(m_write_future); - if (buffer.committed() > 0) { - m_output->write_buffer(std::move(buffer)); + ~Writer() noexcept { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. } } /** - * Flush writes to output file and closes it. If you do not - * call this, the destructor of Writer will also do the same - * thing. But because this call might thrown an exception, - * it is better to call close() explicitly. + * Get the currently configured size of the internal buffer. + */ + size_t buffer_size() const noexcept { + return m_buffer_size; + } + + /** + * Set the size of the internal buffer. This will only take effect + * if you have not yet written anything or after the next flush(). + */ + void set_buffer_size(size_t size) noexcept { + m_buffer_size = size; + } + + /** + * Flush the internal buffer if it contains any data. This is + * usually not needed as the buffer gets flushed on close() + * automatically. * - * @throws Some form of std::runtime_error when there is a problem. + * @throws Some form of osmium::io_error when there is a problem. + */ + void flush() { + ensure_cleanup([&](){ + do_flush(); + }); + } + + /** + * Write contents of a buffer to the output file. The buffer is + * moved into this function and will be in an undefined moved-from + * state afterwards. + * + * @param buffer Buffer that is being written out. + * @throws Some form of osmium::io_error when there is a problem. + */ + void operator()(osmium::memory::Buffer&& buffer) { + ensure_cleanup([&](){ + do_flush(); + do_write(std::move(buffer)); + }); + } + + /** + * Add item to the internal buffer for eventual writing to the + * output file. + * + * @param item Item to write (usually an OSM object). + * @throws Some form of osmium::io_error when there is a problem. + */ + void operator()(const osmium::memory::Item& item) { + ensure_cleanup([&](){ + if (!m_buffer) { + m_buffer = osmium::memory::Buffer{m_buffer_size, + osmium::memory::Buffer::auto_grow::no}; + } + try { + m_buffer.push_back(item); + } catch (osmium::buffer_is_full&) { + do_flush(); + m_buffer.push_back(item); + } + }); + } + + /** + * Flushes internal buffer and closes output file. If you do not + * call this, the destructor of Writer will also do the same + * thing. But because this call might throw an exception, which + * the destructor will ignore, it is better to call close() + * explicitly. + * + * @throws Some form of osmium::io_error when there is a problem. */ void close() { - m_output->close(); - osmium::thread::wait_until_done(m_write_future); + if (m_status == status::okay) { + ensure_cleanup([&](){ + do_write(std::move(m_buffer)); + m_output->write_end(); + m_status = status::closed; + detail::add_end_of_data_to_queue(m_output_queue); + }); + } + + if (m_write_future.valid()) { + m_write_future.get(); + } } }; // class Writer diff --git a/third_party/libosmium/include/osmium/io/writer_options.hpp b/third_party/libosmium/include/osmium/io/writer_options.hpp new file mode 100644 index 000000000..ef1955312 --- /dev/null +++ b/third_party/libosmium/include/osmium/io/writer_options.hpp @@ -0,0 +1,60 @@ +#ifndef OSMIUM_IO_WRITER_OPTIONS_HPP +#define OSMIUM_IO_WRITER_OPTIONS_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +namespace osmium { + + namespace io { + + /** + * Allow overwriting of existing file? + */ + enum class overwrite : bool { + no = false, + allow = true + }; + + /** + * Should writer do an fsync before closing the file? + */ + enum class fsync : bool { + no = false, + yes = true + }; + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_WRITER_OPTIONS_HPP diff --git a/third_party/libosmium/include/osmium/memory/buffer.hpp b/third_party/libosmium/include/osmium/memory/buffer.hpp index d800c685c..bc6e9f8b9 100644 --- a/third_party/libosmium/include/osmium/memory/buffer.hpp +++ b/third_party/libosmium/include/osmium/memory/buffer.hpp @@ -46,6 +46,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include namespace osmium { @@ -89,12 +90,17 @@ namespace osmium { * * By default, if a buffer gets full it will throw a buffer_is_full exception. * You can use the set_full_callback() method to set a callback functor - * which will be called instead of throwing an exception. + * which will be called instead of throwing an exception. The full + * callback functionality is deprecated and will be removed in the + * future. See the documentation for set_full_callback() for alternatives. */ class Buffer { public: + // This is needed so we can call std::back_inserter() on a Buffer. + using value_type = Item; + enum class auto_grow : bool { yes = true, no = false @@ -112,12 +118,13 @@ namespace osmium { public: - typedef Item value_type; - /** - * The constructor without any parameters creates a non-initialized + * The constructor without any parameters creates an invalid, * buffer, ie an empty hull of a buffer that has no actual memory - * associated with it. It can be used to signify end-of-input. + * associated with it. It can be used to signify end-of-data. + * + * Most methods of the Buffer class will not work with an invalid + * buffer. */ Buffer() noexcept : m_memory(), @@ -128,12 +135,14 @@ namespace osmium { } /** - * Constructs an externally memory-managed buffer using the given - * memory and size. + * Constructs a valid externally memory-managed buffer using the + * given memory and size. * * @param data A pointer to some already initialized data. * @param size The size of the initialized data. - * @throws std::invalid_argument When the size isn't a multiple of the alignment. + * + * @throws std::invalid_argument if the size isn't a multiple of + * the alignment. */ explicit Buffer(unsigned char* data, size_t size) : m_memory(), @@ -147,13 +156,15 @@ namespace osmium { } /** - * Constructs an externally memory-managed buffer with the given - * capacity that already contains 'committed' bytes of data. + * Constructs a valid externally memory-managed buffer with the + * given capacity that already contains 'committed' bytes of data. * * @param data A pointer to some (possibly initialized) data. * @param capacity The size of the memory for this buffer. * @param committed The size of the initialized data. If this is 0, the buffer startes out empty. - * @throws std::invalid_argument When the capacity or committed isn't a multiple of the alignment. + * + * @throws std::invalid_argument if the capacity or committed isn't + * a multiple of the alignment. */ explicit Buffer(unsigned char* data, size_t capacity, size_t committed) : m_memory(), @@ -170,10 +181,18 @@ namespace osmium { } /** - * Create an internally memory-managed buffer with the given capacity. - * different in that it internally gets dynamic memory of the - * required size. The dynamic memory will be automatically - * freed when the Buffer is destroyed. + * Constructs a valid internally memory-managed buffer with the + * given capacity. + * Will internally get dynamic memory of the required size. + * The dynamic memory will be automatically freed when the Buffer + * is destroyed. + * + * @param capacity The (initial) size of the memory for this buffer. + * @param auto_grow Should this buffer automatically grow when it + * becomes to small? + * + * @throws std::invalid_argument if the capacity isn't a multiple + * of the alignment. */ explicit Buffer(size_t capacity, auto_grow auto_grow = auto_grow::yes) : m_memory(capacity), @@ -199,13 +218,17 @@ namespace osmium { /** * Return a pointer to data inside the buffer. + * + * @pre The buffer must be valid. */ unsigned char* data() const noexcept { + assert(m_data); return m_data; } /** - * Returns the capacity of the buffer, ie how many bytes it can contain. + * Returns the capacity of the buffer, ie how many bytes it can + * contain. Always returns 0 on invalid buffers. */ size_t capacity() const noexcept { return m_capacity; @@ -213,6 +236,7 @@ namespace osmium { /** * Returns the number of bytes already filled in this buffer. + * Always returns 0 on invalid buffers. */ size_t committed() const noexcept { return m_committed; @@ -221,6 +245,7 @@ namespace osmium { /** * Returns the number of bytes currently filled in this buffer that * are not yet committed. + * Always returns 0 on invalid buffers. */ size_t written() const noexcept { return m_written; @@ -229,28 +254,57 @@ namespace osmium { /** * This tests if the current state of the buffer is aligned * properly. Can be used for asserts. + * + * @pre The buffer must be valid. */ bool is_aligned() const noexcept { + assert(m_data); return (m_written % align_bytes == 0) && (m_committed % align_bytes == 0); } /** * Set functor to be called whenever the buffer is full * instead of throwing buffer_is_full. + * + * The behaviour is undefined if you call this on an invalid + * buffer. + * + * @pre The buffer must be valid. + * + * @deprecated + * Callback functionality will be removed in the future. Either + * detect the buffer_is_full exception or use a buffer with + * auto_grow::yes. If you want to avoid growing buffers, check + * that the used size of the buffer (committed()) is small enough + * compared to the capacity (for instance small than 90% of the + * capacity) before adding anything to the Buffer. If the buffer + * is initialized with auto_grow::yes, it will still grow in the + * rare case that a very large object will be added taking more + * than the difference between committed() and capacity(). */ - void set_full_callback(std::function full) { + OSMIUM_DEPRECATED void set_full_callback(std::function full) { + assert(m_data); m_full = full; } /** * Grow capacity of this buffer to the given size. * This works only with internally memory-managed buffers. - * If the given size is not larger than the current capacity, nothing is done. + * If the given size is not larger than the current capacity, + * nothing is done. * Already written but not committed data is discarded. * + * @pre The buffer must be valid. + * * @param size New capacity. + * + * @throws std::logic_error if the buffer doesn't use internal + * memory management. + * @throws std::invalid_argument if the size isn't a multiple + * of the alignment. */ void grow(size_t size) { + assert(m_data); if (m_memory.empty()) { throw std::logic_error("Can't grow Buffer if it doesn't use internal memory management."); } @@ -267,9 +321,15 @@ namespace osmium { /** * Mark currently written bytes in the buffer as committed. * - * @returns Last number of committed bytes before this commit. + * @pre The buffer must be valid and aligned properly (as indicated + * by is_aligned(). + * + * @returns Number of committed bytes before this commit. Can be + * used as an offset into the buffer to get to the + * object being committed by this call. */ size_t commit() { + assert(m_data); assert(is_aligned()); const size_t offset = m_committed; @@ -279,14 +339,19 @@ namespace osmium { /** * Roll back changes in buffer to last committed state. + * + * @pre The buffer must be valid. */ void rollback() { + assert(m_data); m_written = m_committed; } /** * Clear the buffer. * + * No-op on an invalid buffer. + * * @returns Number of bytes in the buffer before it was cleared. */ size_t clear() { @@ -299,11 +364,16 @@ namespace osmium { /** * Get the data in the buffer at the given offset. * + * @pre The buffer must be valid. + * * @tparam T Type we want to the data to be interpreted as. - * @returns Reference of given type pointing to the data in the buffer. + * + * @returns Reference of given type pointing to the data in the + * buffer. */ - template + template T& get(const size_t offset) const { + assert(m_data); return *reinterpret_cast(&m_data[offset]); } @@ -320,23 +390,35 @@ namespace osmium { * * * If you have set a callback with set_full_callback(), it is * called. After the call returns, you must have either grown - * the buffer or cleared it by calling buffer.clear(). + * the buffer or cleared it by calling buffer.clear(). (Usage + * of the full callback is deprecated and this functionality + * will be removed in the future. See the documentation for + * set_full_callback() for alternatives. * * If no callback is defined and this buffer uses internal * memory management, the buffers capacity is grown, so that * the new data will fit. * * Else the buffer_is_full exception is thrown. * + * @pre The buffer must be valid. + * * @param size Number of bytes to reserve. + * * @returns Pointer to reserved space. Note that this pointer is - * only guaranteed to be valid until the next call to - * reserve_space(). - * @throws osmium::buffer_is_full Might be thrown if the buffer is full. + * only guaranteed to be valid until the next call to + * reserve_space(). + * + * @throws osmium::buffer_is_full if the buffer is full there is + * no callback defined and the buffer isn't auto-growing. */ unsigned char* reserve_space(const size_t size) { + assert(m_data); + // try to flush the buffer empty first. + if (m_written + size > m_capacity && m_full) { + m_full(*this); + } + // if there's still not enough space, then try growing the buffer. if (m_written + size > m_capacity) { - if (m_full) { - m_full(*this); - } else if (!m_memory.empty() && (m_auto_grow == auto_grow::yes)) { + if (!m_memory.empty() && (m_auto_grow == auto_grow::yes)) { // double buffer size until there is enough space size_t new_capacity = m_capacity * 2; while (m_written + size > new_capacity) { @@ -359,12 +441,17 @@ namespace osmium { * Note that you have to eventually call commit() to actually * commit this data. * + * @pre The buffer must be valid. + * * @tparam T Class of the item to be copied. + * * @param item Reference to the item to be copied. + * * @returns Reference to newly copied data in the buffer. */ - template + template T& add_item(const T& item) { + assert(m_data); unsigned char* target = reserve_space(item.padded_size()); std::copy_n(reinterpret_cast(&item), item.padded_size(), target); return *reinterpret_cast(target); @@ -373,91 +460,176 @@ namespace osmium { /** * Add committed contents of the given buffer to this buffer. * + * @pre The buffer must be valid. + * * Note that you have to eventually call commit() to actually * commit this data. + * + * @param buffer The source of the copy. Must be valid. */ void add_buffer(const Buffer& buffer) { + assert(m_data && buffer); unsigned char* target = reserve_space(buffer.committed()); - std::copy_n(reinterpret_cast(buffer.data()), buffer.committed(), target); + std::copy_n(buffer.data(), buffer.committed(), target); } /** * Add an item to the buffer. This function is provided so that * you can use std::back_inserter. + * + * @pre The buffer must be valid. + * + * @param item The item to be added. */ void push_back(const osmium::memory::Item& item) { + assert(m_data); add_item(item); commit(); } /** - * These iterators can be used to iterate over all items in - * a buffer. + * An iterator that can be used to iterate over all items of + * type T in a buffer. */ - template + template using t_iterator = osmium::memory::ItemIterator; - template + /** + * A const iterator that can be used to iterate over all items of + * type T in a buffer. + */ + template using t_const_iterator = osmium::memory::ItemIterator; - typedef t_iterator iterator; - typedef t_const_iterator const_iterator; + /** + * An iterator that can be used to iterate over all OSMEntity + * objects in a buffer. + */ + using iterator = t_iterator; - template + /** + * A const iterator that can be used to iterate over all OSMEntity + * objects in a buffer. + */ + using const_iterator = t_const_iterator; + + /** + * Get iterator for iterating over all items of type T in the + * buffer. + * + * @pre The buffer must be valid. + * + * @returns Iterator to first item of type T in the buffer. + */ + template t_iterator begin() { + assert(m_data); return t_iterator(m_data, m_data + m_committed); } + /** + * Get iterator for iterating over all objects of class OSMEntity + * in the buffer. + * + * @pre The buffer must be valid. + * + * @returns Iterator to first OSMEntity in the buffer. + */ iterator begin() { + assert(m_data); return iterator(m_data, m_data + m_committed); } - template + /** + * Get iterator for iterating over all items of type T in the + * buffer. + * + * @pre The buffer must be valid. + * + * @returns Iterator to first item of type T after given offset + * in the buffer. + */ + template t_iterator get_iterator(size_t offset) { + assert(m_data); return t_iterator(m_data + offset, m_data + m_committed); } + /** + * Get iterator for iterating over all objects of class OSMEntity + * in the buffer. + * + * @pre The buffer must be valid. + * + * @returns Iterator to first OSMEntity after given offset in the + * buffer. + */ iterator get_iterator(size_t offset) { + assert(m_data); return iterator(m_data + offset, m_data + m_committed); } - template + /** + * Get iterator for iterating over all items of type T in the + * buffer. + * + * @pre The buffer must be valid. + * + * @returns End iterator. + */ + template t_iterator end() { + assert(m_data); return t_iterator(m_data + m_committed, m_data + m_committed); } + /** + * Get iterator for iterating over all objects of class OSMEntity + * in the buffer. + * + * @pre The buffer must be valid. + * + * @returns End iterator. + */ iterator end() { + assert(m_data); return iterator(m_data + m_committed, m_data + m_committed); } - template + template t_const_iterator cbegin() const { + assert(m_data); return t_const_iterator(m_data, m_data + m_committed); } const_iterator cbegin() const { + assert(m_data); return const_iterator(m_data, m_data + m_committed); } - template + template t_const_iterator get_iterator(size_t offset) const { + assert(m_data); return t_const_iterator(m_data + offset, m_data + m_committed); } const_iterator get_iterator(size_t offset) const { + assert(m_data); return const_iterator(m_data + offset, m_data + m_committed); } - template + template t_const_iterator cend() const { + assert(m_data); return t_const_iterator(m_data + m_committed, m_data + m_committed); } const_iterator cend() const { + assert(m_data); return const_iterator(m_data + m_committed, m_data + m_committed); } - template + template t_const_iterator begin() const { return cbegin(); } @@ -466,7 +638,7 @@ namespace osmium { return cbegin(); } - template + template t_const_iterator end() const { return cend(); } @@ -476,9 +648,9 @@ namespace osmium { } /** - * In a bool context any initialized buffer is true. + * In a bool context any valid buffer is true. */ - explicit operator bool() const { + explicit operator bool() const noexcept { return m_data != nullptr; } @@ -490,6 +662,8 @@ namespace osmium { swap(lhs.m_capacity, rhs.m_capacity); swap(lhs.m_written, rhs.m_written); swap(lhs.m_committed, rhs.m_committed); + swap(lhs.m_auto_grow, rhs.m_auto_grow); + swap(lhs.m_full, rhs.m_full); } /** @@ -497,17 +671,20 @@ namespace osmium { * non-removed items forward in the buffer overwriting removed * items and then correcting the m_written and m_committed numbers. * - * Note that calling this function invalidates all iterators on this - * buffer and all offsets in this buffer. + * Note that calling this function invalidates all iterators on + * this buffer and all offsets in this buffer. * * For every non-removed item that moves its position, the function * 'moving_in_buffer' is called on the given callback object with * the old and new offsets in the buffer where the object used to * be and is now, respectively. This call can be used to update any * indexes. + * + * @pre The buffer must be valid. */ - template + template void purge_removed(TCallbackClass* callback) { + assert(m_data); if (begin() == end()) { return; } @@ -537,7 +714,17 @@ namespace osmium { }; // class Buffer + /** + * Compare two buffers for equality. + * + * Buffers are equal if they are both invalid or if they are both + * valid and have the same data pointer, capacity and committed + * data. + */ inline bool operator==(const Buffer& lhs, const Buffer& rhs) noexcept { + if (!lhs || !rhs) { + return !lhs && !rhs; + } return lhs.data() == rhs.data() && lhs.capacity() == rhs.capacity() && lhs.committed() == rhs.committed(); } diff --git a/third_party/libosmium/include/osmium/memory/collection.hpp b/third_party/libosmium/include/osmium/memory/collection.hpp index 5cf3cc6e1..3878b9a3a 100644 --- a/third_party/libosmium/include/osmium/memory/collection.hpp +++ b/third_party/libosmium/include/osmium/memory/collection.hpp @@ -43,7 +43,7 @@ namespace osmium { namespace memory { - template + template class CollectionIterator : public std::iterator { // This data_type is either 'unsigned char*' or 'const unsigned char*' depending @@ -59,7 +59,7 @@ namespace osmium { m_data(nullptr) { } - CollectionIterator(data_type data) noexcept : + explicit CollectionIterator(data_type data) noexcept : m_data(data) { } @@ -101,7 +101,7 @@ namespace osmium { }; // class CollectionIterator - template + template class Collection : public Item { public: diff --git a/third_party/libosmium/include/osmium/memory/item.hpp b/third_party/libosmium/include/osmium/memory/item.hpp index dc544049a..f95ce8807 100644 --- a/third_party/libosmium/include/osmium/memory/item.hpp +++ b/third_party/libosmium/include/osmium/memory/item.hpp @@ -43,7 +43,7 @@ namespace osmium { namespace builder { class Builder; - } + } // namespace builder namespace memory { @@ -102,10 +102,10 @@ namespace osmium { uint16_t m_removed : 1; uint16_t m_padding : 15; - template + template friend class CollectionIterator; - template + template friend class ItemIterator; friend class osmium::builder::Builder; diff --git a/third_party/libosmium/include/osmium/memory/item_iterator.hpp b/third_party/libosmium/include/osmium/memory/item_iterator.hpp index 3e5b5fa8b..87af56830 100644 --- a/third_party/libosmium/include/osmium/memory/item_iterator.hpp +++ b/third_party/libosmium/include/osmium/memory/item_iterator.hpp @@ -38,29 +38,17 @@ DEALINGS IN THE SOFTWARE. #include #include +#include #include #include namespace osmium { - class Node; - class Way; - class Relation; - class Area; - class Changeset; - class OSMObject; - class OSMEntity; - class TagList; - class WayNodeList; - class RelationMemberList; - class InnerRing; - class OuterRing; - namespace memory { namespace detail { - template + template inline bool type_is_compatible(osmium::item_type) noexcept { return true; } @@ -127,7 +115,7 @@ namespace osmium { } // namespace detail - template + template class ItemIterator : public std::iterator { static_assert(std::is_base_of::value, "TMember must derive from osmium::memory::Item"); @@ -160,7 +148,7 @@ namespace osmium { advance_to_next_item_of_right_type(); } - template + template ItemIterator cast() const { return ItemIterator(m_data, m_end); } @@ -217,7 +205,7 @@ namespace osmium { } explicit operator bool() const { - return m_data != nullptr; + return (m_data != nullptr) && (m_data != m_end); } template diff --git a/third_party/libosmium/include/osmium/object_pointer_collection.hpp b/third_party/libosmium/include/osmium/object_pointer_collection.hpp index 752470305..85566b6a4 100644 --- a/third_party/libosmium/include/osmium/object_pointer_collection.hpp +++ b/third_party/libosmium/include/osmium/object_pointer_collection.hpp @@ -84,7 +84,7 @@ namespace osmium { /** * Sort objects according to the given order functor. */ - template + template void sort(TCompare&& compare) { std::sort(m_objects.begin(), m_objects.end(), std::forward(compare)); } diff --git a/third_party/libosmium/include/osmium/osm/area.hpp b/third_party/libosmium/include/osmium/osm/area.hpp index 3e129d0fb..7fb2a798b 100644 --- a/third_party/libosmium/include/osmium/osm/area.hpp +++ b/third_party/libosmium/include/osmium/osm/area.hpp @@ -48,7 +48,7 @@ namespace osmium { namespace builder { template class ObjectBuilder; - } + } // namespace builder /** * An outer ring of an Area. @@ -167,6 +167,7 @@ namespace osmium { case osmium::item_type::way_node_list: case osmium::item_type::relation_member_list: case osmium::item_type::relation_member_list_with_full_members: + case osmium::item_type::changeset_discussion: assert(false && "Children of Area can only be outer/inner_ring and tag_list."); break; } diff --git a/third_party/libosmium/include/osmium/osm/box.hpp b/third_party/libosmium/include/osmium/osm/box.hpp index 631f91911..155f5e902 100644 --- a/third_party/libosmium/include/osmium/osm/box.hpp +++ b/third_party/libosmium/include/osmium/osm/box.hpp @@ -154,14 +154,14 @@ namespace osmium { * Box is valid, ie. defined and inside usual bounds * (-180<=lon<=180, -90<=lat<=90). */ - OSMIUM_CONSTEXPR bool valid() const noexcept { + constexpr bool valid() const noexcept { return bottom_left().valid() && top_right().valid(); } /** * Access bottom-left location. */ - OSMIUM_CONSTEXPR Location bottom_left() const noexcept { + constexpr Location bottom_left() const noexcept { return m_bottom_left; } @@ -175,7 +175,7 @@ namespace osmium { /** * Access top-right location. */ - OSMIUM_CONSTEXPR Location top_right() const noexcept { + constexpr Location top_right() const noexcept { return m_top_right; } @@ -216,7 +216,7 @@ namespace osmium { * Boxes are equal if both locations are equal. Undefined boxes will * compare equal. */ - inline OSMIUM_CONSTEXPR bool operator==(const Box& lhs, const Box& rhs) noexcept { + inline constexpr bool operator==(const Box& lhs, const Box& rhs) noexcept { return lhs.bottom_left() == rhs.bottom_left() && lhs.top_right() == rhs.top_right(); } diff --git a/third_party/libosmium/include/osmium/osm/changeset.hpp b/third_party/libosmium/include/osmium/osm/changeset.hpp index 07bc0dd95..db3f717da 100644 --- a/third_party/libosmium/include/osmium/osm/changeset.hpp +++ b/third_party/libosmium/include/osmium/osm/changeset.hpp @@ -48,8 +48,102 @@ DEALINGS IN THE SOFTWARE. namespace osmium { namespace builder { - template class ObjectBuilder; - } + class ChangesetDiscussionBuilder; + template class ObjectBuilder; + } // namespace builder + + class Changeset; + + class ChangesetComment : public osmium::memory::detail::ItemHelper { + + friend class osmium::builder::ChangesetDiscussionBuilder; + + osmium::Timestamp m_date; + osmium::user_id_type m_uid {0}; + string_size_type m_user_size; + string_size_type m_text_size; + + ChangesetComment(const ChangesetComment&) = delete; + ChangesetComment(ChangesetComment&&) = delete; + + ChangesetComment& operator=(const ChangesetComment&) = delete; + ChangesetComment& operator=(ChangesetComment&&) = delete; + + unsigned char* endpos() { + return data() + osmium::memory::padded_length(sizeof(ChangesetComment) + m_user_size + m_text_size); + } + + const unsigned char* endpos() const { + return data() + osmium::memory::padded_length(sizeof(ChangesetComment) + m_user_size + m_text_size); + } + + template + friend class osmium::memory::CollectionIterator; + + unsigned char* next() { + return endpos(); + } + + unsigned const char* next() const { + return endpos(); + } + + void set_user_size(string_size_type size) noexcept { + m_user_size = size; + } + + void set_text_size(string_size_type size) noexcept { + m_text_size = size; + } + + public: + + static constexpr item_type collection_type = item_type::changeset_discussion; + + ChangesetComment(osmium::Timestamp date, osmium::user_id_type uid) noexcept : + m_date(date), + m_uid(uid), + m_user_size(0), + m_text_size(0) { + } + + osmium::Timestamp date() const noexcept { + return m_date; + } + + osmium::user_id_type uid() const noexcept { + return m_uid; + } + + const char* user() const noexcept { + return reinterpret_cast(data() + sizeof(ChangesetComment)); + } + + const char* text() const noexcept { + return reinterpret_cast(data() + sizeof(ChangesetComment) + m_user_size); + } + + }; // class ChangesetComment + + class ChangesetDiscussion : public osmium::memory::Collection { + + friend class osmium::builder::ObjectBuilder; + + public: + + typedef size_t size_type; + + ChangesetDiscussion() : + osmium::memory::Collection() { + } + + size_type size() const noexcept { + return static_cast(std::distance(begin(), end())); + } + + }; // class ChangesetDiscussion + + static_assert(sizeof(ChangesetDiscussion) % osmium::memory::align_bytes == 0, "Class osmium::ChangesetDiscussion has wrong size to be aligned properly!"); /** * \brief An OSM Changeset, a group of changes made by a single user over @@ -62,13 +156,16 @@ namespace osmium { friend class osmium::builder::ObjectBuilder; + osmium::Box m_bounds; osmium::Timestamp m_created_at; osmium::Timestamp m_closed_at; - osmium::Box m_bounds; changeset_id_type m_id {0}; num_changes_type m_num_changes {0}; + num_comments_type m_num_comments {0}; user_id_type m_uid {0}; string_size_type m_user_size; + int16_t m_padding1 {0}; + int32_t m_padding2 {0}; Changeset() : OSMEntity(sizeof(Changeset), osmium::item_type::changeset) { @@ -188,7 +285,7 @@ namespace osmium { * @param timestamp Timestamp * @returns Reference to changeset to make calls chainable. */ - Changeset& set_created_at(const osmium::Timestamp timestamp) { + Changeset& set_created_at(const osmium::Timestamp& timestamp) { m_created_at = timestamp; return *this; } @@ -199,7 +296,7 @@ namespace osmium { * @param timestamp Timestamp * @returns Reference to changeset to make calls chainable. */ - Changeset& set_closed_at(const osmium::Timestamp timestamp) { + Changeset& set_closed_at(const osmium::Timestamp& timestamp) { m_closed_at = timestamp; return *this; } @@ -216,10 +313,26 @@ namespace osmium { } /// Set the number of changes in this changeset - Changeset& set_num_changes(const char* num_changes) noexcept { + Changeset& set_num_changes(const char* num_changes) { return set_num_changes(osmium::string_to_num_changes(num_changes)); } + /// Get the number of comments in this changeset + num_comments_type num_comments() const noexcept { + return m_num_comments; + } + + /// Set the number of comments in this changeset + Changeset& set_num_comments(num_comments_type num_comments) noexcept { + m_num_comments = num_comments; + return *this; + } + + /// Set the number of comments in this changeset + Changeset& set_num_comments(const char* num_comments) { + return set_num_comments(osmium::string_to_num_comments(num_comments)); + } + /** * Get the bounding box of this changeset. * @@ -260,6 +373,8 @@ namespace osmium { set_id(value); } else if (!strcmp(attr, "num_changes")) { set_num_changes(value); + } else if (!strcmp(attr, "comments_count")) { + set_num_comments(value); } else if (!strcmp(attr, "created_at")) { set_created_at(osmium::Timestamp(value)); } else if (!strcmp(attr, "closed_at")) { @@ -296,6 +411,14 @@ namespace osmium { return cend(); } + ChangesetDiscussion& discussion() { + return osmium::detail::subitem_of_type(begin(), end()); + } + + const ChangesetDiscussion& discussion() const { + return osmium::detail::subitem_of_type(cbegin(), cend()); + } + }; // class Changeset static_assert(sizeof(Changeset) % osmium::memory::align_bytes == 0, "Class osmium::Changeset has wrong size to be aligned properly!"); diff --git a/third_party/libosmium/include/osmium/osm/crc.hpp b/third_party/libosmium/include/osmium/osm/crc.hpp index eefa4a13e..f5e01e47b 100644 --- a/third_party/libosmium/include/osmium/osm/crc.hpp +++ b/third_party/libosmium/include/osmium/osm/crc.hpp @@ -46,10 +46,9 @@ DEALINGS IN THE SOFTWARE. namespace osmium { - template - class CRC { + namespace util { - static inline uint16_t byte_swap_16(uint16_t value) noexcept { + inline uint16_t byte_swap_16(uint16_t value) noexcept { # if defined(__GNUC__) || defined(__clang__) return __builtin_bswap16(value); # else @@ -57,27 +56,32 @@ namespace osmium { # endif } - static inline uint32_t byte_swap_32(uint32_t value) noexcept { + inline uint32_t byte_swap_32(uint32_t value) noexcept { # if defined(__GNUC__) || defined(__clang__) return __builtin_bswap32(value); # else return (value >> 24) | - ((value >> 8) & 0x0000FF00) | - ((value << 8) & 0x00FF0000) | + ((value >> 8) & 0x0000FF00) | + ((value << 8) & 0x00FF0000) | (value << 24); # endif } - static inline uint64_t byte_swap_64(uint64_t value) noexcept { + inline uint64_t byte_swap_64(uint64_t value) noexcept { # if defined(__GNUC__) || defined(__clang__) return __builtin_bswap64(value); # else uint64_t val1 = byte_swap_32(value & 0xFFFFFFFF); uint64_t val2 = byte_swap_32(value >> 32); - return (val1 << 32) & val2; + return (val1 << 32) | val2; # endif } + } // namespace util + + template + class CRC { + TCRC m_crc; public: @@ -90,37 +94,37 @@ namespace osmium { return m_crc; } - void update_bool(bool value) { + void update_bool(const bool value) { m_crc.process_byte(value); } - void update_int8(uint8_t value) { + void update_int8(const uint8_t value) { m_crc.process_byte(value); } - void update_int16(uint16_t value) { + void update_int16(const uint16_t value) { #if __BYTE_ORDER == __LITTLE_ENDIAN m_crc.process_bytes(&value, sizeof(uint16_t)); #else - uint16_t v = byte_swap_16(value); + uint16_t v = osmium::util::byte_swap_16(value); m_crc.process_bytes(&v, sizeof(uint16_t)); #endif } - void update_int32(uint32_t value) { + void update_int32(const uint32_t value) { #if __BYTE_ORDER == __LITTLE_ENDIAN m_crc.process_bytes(&value, sizeof(uint32_t)); #else - uint32_t v = byte_swap_32(value); + uint32_t v = osmium::util::byte_swap_32(value); m_crc.process_bytes(&v, sizeof(uint32_t)); #endif } - void update_int64(uint64_t value) { + void update_int64(const uint64_t value) { #if __BYTE_ORDER == __LITTLE_ENDIAN m_crc.process_bytes(&value, sizeof(uint64_t)); #else - uint64_t v = byte_swap_64(value); + uint64_t v = osmium::util::byte_swap_64(value); m_crc.process_bytes(&v, sizeof(uint64_t)); #endif } @@ -156,7 +160,10 @@ namespace osmium { } void update(const TagList& tags) { - m_crc.process_bytes(tags.data(), tags.byte_size()); + for (const Tag& tag : tags) { + update_string(tag.key()); + update_string(tag.value()); + } } void update(const osmium::RelationMember& member) { @@ -206,14 +213,26 @@ namespace osmium { } } + void update(const osmium::ChangesetDiscussion& discussion) { + for (const auto& comment : discussion) { + update(comment.date()); + update_int32(comment.uid()); + update_string(comment.user()); + update_string(comment.text()); + } + } + void update(const osmium::Changeset& changeset) { update_int64(changeset.id()); update(changeset.created_at()); update(changeset.closed_at()); update(changeset.bounds()); update_int32(changeset.num_changes()); + update_int32(changeset.num_comments()); update_int32(changeset.uid()); update_string(changeset.user()); + update(changeset.tags()); + update(changeset.discussion()); } }; // class CRC diff --git a/third_party/libosmium/include/osmium/osm/diff_object.hpp b/third_party/libosmium/include/osmium/osm/diff_object.hpp index 1e053fdda..d9872eaea 100644 --- a/third_party/libosmium/include/osmium/osm/diff_object.hpp +++ b/third_party/libosmium/include/osmium/osm/diff_object.hpp @@ -33,6 +33,9 @@ DEALINGS IN THE SOFTWARE. */ +#include + +#include #include #include #include @@ -40,75 +43,158 @@ DEALINGS IN THE SOFTWARE. namespace osmium { - class Node; - class Way; - class Relation; - + /** + * A DiffObject holds pointers to three OSMObjects, the current object, + * the previous, and the next. They always have the same type (Node, Way, + * or Relation) and the same ID, but may have different versions. + * + * It is used when iterating over OSM files with history data to make + * working with versioned OSM objects easier. Because you have access to + * the previous and next objects as well as the current one, comparisons + * between object versions is easy. + * + * If the current object is the first version available, the previous + * pointer must be the same as the current one. If the current object is + * the last version available, the next pointer must be the same as the + * current one. + * + * DiffObjects are immutable. + */ class DiffObject { - protected: - - osmium::OSMObject* m_prev; - osmium::OSMObject* m_curr; - osmium::OSMObject* m_next; + const osmium::OSMObject* m_prev; + const osmium::OSMObject* m_curr; + const osmium::OSMObject* m_next; public: + /** + * Default construct an empty DiffObject. Most methods of this class + * can not be called on empty DiffObjects. + */ DiffObject() noexcept : m_prev(nullptr), m_curr(nullptr), m_next(nullptr) { } - explicit DiffObject(osmium::OSMObject& prev, osmium::OSMObject& curr, osmium::OSMObject& next) noexcept : + /** + * Construct a non-empty DiffObject from the given OSMObjects. All + * OSMObjects must be of the same type (Node, Way, or Relation) and + * have the same ID. + */ + DiffObject(const osmium::OSMObject& prev, const osmium::OSMObject& curr, const osmium::OSMObject& next) noexcept : m_prev(&prev), m_curr(&curr), m_next(&next) { + assert(prev.type() == curr.type() && curr.type() == next.type()); + assert(prev.id() == curr.id() && curr.id() == next.id()); } - DiffObject(const DiffObject&) = default; - DiffObject& operator=(const DiffObject&) = default; - - DiffObject(DiffObject&&) = default; - DiffObject& operator=(DiffObject&&) = default; + /** + * Check whether the DiffObject was created empty. + */ + bool empty() const noexcept { + return m_prev == nullptr; + } + /** + * Get the previous object stored. + * + * @pre DiffObject must not be empty. + */ const osmium::OSMObject& prev() const noexcept { + assert(m_prev && m_curr && m_next); return *m_prev; } + /** + * Get the current object stored. + * + * @pre DiffObject must not be empty. + */ const osmium::OSMObject& curr() const noexcept { + assert(m_prev && m_curr && m_next); return *m_curr; } + /** + * Get the next object stored. + * + * @pre DiffObject must not be empty. + */ const osmium::OSMObject& next() const noexcept { + assert(m_prev && m_curr && m_next); return *m_next; } + /** + * Is the current object version the first (with this type and ID)? + * + * @pre DiffObject must not be empty. + */ bool first() const noexcept { + assert(m_prev && m_curr && m_next); return m_prev == m_curr; } + /** + * Is the current object version the last (with this type and ID)? + * + * @pre DiffObject must not be empty. + */ bool last() const noexcept { + assert(m_prev && m_curr && m_next); return m_curr == m_next; } + /** + * Return the type of the current object. + * + * @pre DiffObject must not be empty. + */ osmium::item_type type() const noexcept { + assert(m_prev && m_curr && m_next); return m_curr->type(); } + /** + * Return the ID of the current object. + * + * @pre DiffObject must not be empty. + */ osmium::object_id_type id() const noexcept { + assert(m_prev && m_curr && m_next); return m_curr->id(); } + /** + * Return the version of the current object. + * + * @pre DiffObject must not be empty. + */ osmium::object_version_type version() const noexcept { + assert(m_prev && m_curr && m_next); return m_curr->version(); } + /** + * Return the changeset ID of the current object. + * + * @pre DiffObject must not be empty. + */ osmium::changeset_id_type changeset() const noexcept { + assert(m_prev && m_curr && m_next); return m_curr->changeset(); } + /** + * Return the timestamp when the current object version was created. + * + * @pre DiffObject must not be empty. + */ const osmium::Timestamp start_time() const noexcept { + assert(m_prev && m_curr && m_next); return m_curr->timestamp(); } @@ -118,8 +204,11 @@ namespace osmium { * is valid. If this is the last version of the object, this will * return a special "end of time" timestamp that is guaranteed to * be larger than any normal timestamp. + * + * @pre DiffObject must not be empty. */ const osmium::Timestamp end_time() const noexcept { + assert(m_prev && m_curr && m_next); return last() ? osmium::end_of_time() : m_next->timestamp(); } @@ -129,8 +218,11 @@ namespace osmium { * * This is a bit more complex than you'd think, because we have to * handle the case properly where the start_time() == end_time(). + * + * @pre DiffObject must not be empty. */ bool is_between(const osmium::Timestamp& from, const osmium::Timestamp& to) const noexcept { + assert(m_prev && m_curr && m_next); return start_time() < to && ((start_time() != end_time() && end_time() > from) || (start_time() == end_time() && end_time() >= from)); @@ -138,45 +230,42 @@ namespace osmium { /** * Current object version is visible at the given timestamp. + * + * @pre DiffObject must not be empty. */ bool is_visible_at(const osmium::Timestamp& timestamp) const noexcept { + assert(m_prev && m_curr && m_next); return start_time() <= timestamp && end_time() > timestamp && m_curr->visible(); } }; // class DiffObject - template + template class DiffObjectDerived : public DiffObject { public: - DiffObjectDerived(T& prev, T& curr, T& next) noexcept : + DiffObjectDerived(const T& prev, const T& curr, const T& next) noexcept : DiffObject(prev, curr, next) { } - DiffObjectDerived(const DiffObjectDerived&) = default; - DiffObjectDerived& operator=(const DiffObjectDerived&) = default; - - DiffObjectDerived(DiffObjectDerived&&) = default; - DiffObjectDerived& operator=(DiffObjectDerived&&) = default; - const T& prev() const noexcept { - return *static_cast(m_prev); + return static_cast(DiffObject::prev()); } const T& curr() const noexcept { - return *static_cast(m_curr); + return static_cast(DiffObject::curr()); } const T& next() const noexcept { - return *static_cast(m_next); + return static_cast(DiffObject::next()); } }; // class DiffObjectDerived - typedef DiffObjectDerived DiffNode; - typedef DiffObjectDerived DiffWay; - typedef DiffObjectDerived DiffRelation; + using DiffNode = DiffObjectDerived; + using DiffWay = DiffObjectDerived; + using DiffRelation = DiffObjectDerived; } // namespace osmium diff --git a/third_party/libosmium/include/osmium/osm/entity.hpp b/third_party/libosmium/include/osmium/osm/entity.hpp index ce292c8d6..c7f70553c 100644 --- a/third_party/libosmium/include/osmium/osm/entity.hpp +++ b/third_party/libosmium/include/osmium/osm/entity.hpp @@ -41,7 +41,7 @@ namespace osmium { namespace detail { - template + template inline TSubitem& subitem_of_type(TIter it, TIter end) { for (; it != end; ++it) { if (it->type() == TSubitem::itemtype) { diff --git a/third_party/libosmium/include/osmium/osm/item_type.hpp b/third_party/libosmium/include/osmium/osm/item_type.hpp index 54975e326..95826bc98 100644 --- a/third_party/libosmium/include/osmium/osm/item_type.hpp +++ b/third_party/libosmium/include/osmium/osm/item_type.hpp @@ -53,13 +53,18 @@ namespace osmium { relation_member_list = 0x13, relation_member_list_with_full_members = 0x23, outer_ring = 0x40, - inner_ring = 0x41 + inner_ring = 0x41, + changeset_discussion = 0x80 }; // enum class item_type /** * Return item_type for index: * 0 -> node, 1 -> way, 2 -> relation + * + * @param i Index. Must be between 0 and 2. + * + * @returns Item type. */ inline item_type nwr_index_to_item_type(unsigned int i) noexcept { assert(i <= 2); @@ -69,6 +74,10 @@ namespace osmium { /** * Return index for item_type: * node -> 0, way -> 1, relation -> 2 + * + * @param type Item type. Must be node, way, or relation. + * + * @returns Index. */ inline unsigned int item_type_to_nwr_index(item_type type) noexcept { unsigned int i = static_cast(type); @@ -102,6 +111,8 @@ namespace osmium { return item_type::outer_ring; case 'I': return item_type::inner_ring; + case 'D': + return item_type::changeset_discussion; default: return item_type::undefined; } @@ -136,6 +147,8 @@ namespace osmium { return 'O'; case item_type::inner_ring: return 'I'; + case item_type::changeset_discussion: + return 'D'; } } @@ -165,6 +178,8 @@ namespace osmium { return "outer_ring"; case item_type::inner_ring: return "inner_ring"; + case item_type::changeset_discussion: + return "changeset_discussion"; } } #pragma GCC diagnostic pop diff --git a/third_party/libosmium/include/osmium/osm/location.hpp b/third_party/libosmium/include/osmium/osm/location.hpp index 0d4fdc13d..f79117e0a 100644 --- a/third_party/libosmium/include/osmium/osm/location.hpp +++ b/third_party/libosmium/include/osmium/osm/location.hpp @@ -52,11 +52,11 @@ namespace osmium { */ struct invalid_location : public std::range_error { - invalid_location(const std::string& what) : + explicit invalid_location(const std::string& what) : std::range_error(what) { } - invalid_location(const char* what) : + explicit invalid_location(const char* what) : std::range_error(what) { } @@ -95,7 +95,7 @@ namespace osmium { return static_cast(std::round(c * coordinate_precision)); } - static OSMIUM_CONSTEXPR double fix_to_double(const int32_t c) noexcept { + static constexpr double fix_to_double(const int32_t c) noexcept { return static_cast(c) / coordinate_precision; } @@ -238,11 +238,11 @@ namespace osmium { /** * Locations are equal if both coordinates are equal. */ - inline OSMIUM_CONSTEXPR bool operator==(const Location& lhs, const Location& rhs) noexcept { + inline constexpr bool operator==(const Location& lhs, const Location& rhs) noexcept { return lhs.x() == rhs.x() && lhs.y() == rhs.y(); } - inline OSMIUM_CONSTEXPR bool operator!=(const Location& lhs, const Location& rhs) noexcept { + inline constexpr bool operator!=(const Location& lhs, const Location& rhs) noexcept { return ! (lhs == rhs); } @@ -251,19 +251,19 @@ namespace osmium { * the y coordinate. If either of the locations is * undefined the result is undefined. */ - inline OSMIUM_CONSTEXPR bool operator<(const Location& lhs, const Location& rhs) noexcept { + inline constexpr bool operator<(const Location& lhs, const Location& rhs) noexcept { return (lhs.x() == rhs.x() && lhs.y() < rhs.y()) || lhs.x() < rhs.x(); } - inline OSMIUM_CONSTEXPR bool operator>(const Location& lhs, const Location& rhs) noexcept { + inline constexpr bool operator>(const Location& lhs, const Location& rhs) noexcept { return rhs < lhs; } - inline OSMIUM_CONSTEXPR bool operator<=(const Location& lhs, const Location& rhs) noexcept { + inline constexpr bool operator<=(const Location& lhs, const Location& rhs) noexcept { return ! (rhs < lhs); } - inline OSMIUM_CONSTEXPR bool operator>=(const Location& lhs, const Location& rhs) noexcept { + inline constexpr bool operator>=(const Location& lhs, const Location& rhs) noexcept { return ! (lhs < rhs); } diff --git a/third_party/libosmium/include/osmium/osm/node.hpp b/third_party/libosmium/include/osmium/osm/node.hpp index 123bfc4fa..1ff7d1c28 100644 --- a/third_party/libosmium/include/osmium/osm/node.hpp +++ b/third_party/libosmium/include/osmium/osm/node.hpp @@ -41,8 +41,8 @@ DEALINGS IN THE SOFTWARE. namespace osmium { namespace builder { - template class ObjectBuilder; - } + template class ObjectBuilder; + } // namespace builder class Node : public OSMObject { diff --git a/third_party/libosmium/include/osmium/osm/node_ref.hpp b/third_party/libosmium/include/osmium/osm/node_ref.hpp index 72359cd0f..e1c9c12e1 100644 --- a/third_party/libosmium/include/osmium/osm/node_ref.hpp +++ b/third_party/libosmium/include/osmium/osm/node_ref.hpp @@ -54,15 +54,21 @@ namespace osmium { public: - NodeRef(const osmium::object_id_type ref = 0, const osmium::Location& location = Location()) noexcept : + constexpr NodeRef(const osmium::object_id_type ref = 0, const osmium::Location& location = Location()) noexcept : m_ref(ref), m_location(location) { } - osmium::object_id_type ref() const noexcept { + /** + * Get reference ID of this NodeRef. + */ + constexpr osmium::object_id_type ref() const noexcept { return m_ref; } + /** + * Get absolute value of the reference ID of this NodeRef. + */ osmium::unsigned_object_id_type positive_ref() const noexcept { return static_cast(std::abs(m_ref)); } @@ -74,31 +80,60 @@ namespace osmium { return m_location; } - osmium::Location location() const noexcept { + /** + * Get location of this NodeRef. + */ + constexpr osmium::Location location() const noexcept { return m_location; } + /** + * Get longitude of the location in this NodeRef. + * + * @throws osmium::invalid_location if the location is not set. + */ double lon() const { return m_location.lon(); } + /** + * Get latitude of the location in this NodeRef. + * + * @throws osmium::invalid_location if the location is not set. + */ double lat() const { return m_location.lat(); } - int32_t x() const noexcept { + /** + * Get internal x value of the location in this NodeRef. + */ + constexpr int32_t x() const noexcept { return m_location.x(); } - int32_t y() const noexcept { + /** + * Get internal y value of the location in this NodeRef. + */ + constexpr int32_t y() const noexcept { return m_location.y(); } + /** + * Set the referenced ID. + * + * @returns Reference to this NodeRef for chaining calls. + */ NodeRef& set_ref(const osmium::object_id_type ref) noexcept { m_ref = ref; return *this; } + /** + * Set the location. + * + * @returns Reference to this NodeRef for chaining calls. + */ NodeRef& set_location(const osmium::Location& location) noexcept { m_location = location; return *this; @@ -106,27 +141,50 @@ namespace osmium { }; // class NodeRef - inline bool operator==(const NodeRef& lhs, const NodeRef& rhs) noexcept { + /** + * Compare two NodeRefs. They are equal if they reference the same Node ID. + */ + inline constexpr bool operator==(const NodeRef& lhs, const NodeRef& rhs) noexcept { return lhs.ref() == rhs.ref(); } - inline bool operator!=(const NodeRef& lhs, const NodeRef& rhs) noexcept { + /** + * Compare two NodeRefs. They are not equal if they reference different + * Node IDs. + */ + inline constexpr bool operator!=(const NodeRef& lhs, const NodeRef& rhs) noexcept { return ! (lhs == rhs); } - inline bool operator<(const NodeRef& lhs, const NodeRef& rhs) noexcept { + /** + * Compare two NodeRefs. NodeRefs are ordered according to the Node ID + * they reference. + */ + inline constexpr bool operator<(const NodeRef& lhs, const NodeRef& rhs) noexcept { return lhs.ref() < rhs.ref(); } - inline bool operator>(const NodeRef& lhs, const NodeRef& rhs) noexcept { + /** + * Compare two NodeRefs. NodeRefs are ordered according to the Node ID + * they reference. + */ + inline constexpr bool operator>(const NodeRef& lhs, const NodeRef& rhs) noexcept { return rhs < lhs; } - inline bool operator<=(const NodeRef& lhs, const NodeRef& rhs) noexcept { + /** + * Compare two NodeRefs. NodeRefs are ordered according to the Node ID + * they reference. + */ + inline constexpr bool operator<=(const NodeRef& lhs, const NodeRef& rhs) noexcept { return ! (rhs < lhs); } - inline bool operator>=(const NodeRef& lhs, const NodeRef& rhs) noexcept { + /** + * Compare two NodeRefs. NodeRefs are ordered according to the Node ID + * they reference. + */ + inline constexpr bool operator>=(const NodeRef& lhs, const NodeRef& rhs) noexcept { return ! (lhs < rhs); } @@ -139,32 +197,32 @@ namespace osmium { } /** - * Functor to compare NodeRefs by Location instead of id. + * Functor to compare NodeRefs by Location instead of ID. */ struct location_equal { - bool operator()(const NodeRef& lhs, const NodeRef& rhs) const noexcept { + constexpr bool operator()(const NodeRef& lhs, const NodeRef& rhs) const noexcept { return lhs.location() == rhs.location(); } - typedef NodeRef first_argument_type; - typedef NodeRef second_argument_type; - typedef bool result_type; + using first_argument_type = NodeRef; + using second_argument_type = NodeRef; + using result_type = bool; }; // struct location_equal /** - * Functor to compare NodeRefs by Location instead of id. + * Functor to compare NodeRefs by Location instead of ID. */ struct location_less { - bool operator()(const NodeRef& lhs, const NodeRef& rhs) const noexcept { + constexpr bool operator()(const NodeRef& lhs, const NodeRef& rhs) const noexcept { return lhs.location() < rhs.location(); } - typedef NodeRef first_argument_type; - typedef NodeRef second_argument_type; - typedef bool result_type; + using first_argument_type = NodeRef; + using second_argument_type = NodeRef; + using result_type = bool; }; // struct location_less diff --git a/third_party/libosmium/include/osmium/osm/node_ref_list.hpp b/third_party/libosmium/include/osmium/osm/node_ref_list.hpp index f0dfedbc1..f990b88c0 100644 --- a/third_party/libosmium/include/osmium/osm/node_ref_list.hpp +++ b/third_party/libosmium/include/osmium/osm/node_ref_list.hpp @@ -44,29 +44,29 @@ DEALINGS IN THE SOFTWARE. namespace osmium { /** - * A vector of NodeRef objects. Usually this is not instantiated directly, - * but one of its subclasses are used. + * An ordered collection of NodeRef objects. Usually this is not + * instantiated directly, but one of its subclasses are used. */ class NodeRefList : public osmium::memory::Item { public: - NodeRefList(osmium::item_type itemtype) noexcept : + explicit NodeRefList(osmium::item_type itemtype) noexcept : osmium::memory::Item(sizeof(NodeRefList), itemtype) { } /** - * Checks whether the node list is empty. + * Checks whether the collection is empty. */ bool empty() const noexcept { return sizeof(NodeRefList) == byte_size(); } /** - * Returns the number of nodes in the list. + * Returns the number of NodeRefs in the collection. */ size_t size() const noexcept { - auto size_node_refs = osmium::memory::Item::byte_size() - sizeof(NodeRefList); + auto size_node_refs = byte_size() - sizeof(NodeRefList); assert(size_node_refs % sizeof(NodeRef) == 0); return size_node_refs / sizeof(NodeRef); } @@ -74,8 +74,9 @@ namespace osmium { /** * Access specified element. * - * @param n Get this element of the list. * @pre @code n < size() @endcode + * + * @param n Get the n-th element of the collection. */ const NodeRef& operator[](size_t n) const noexcept { assert(n < size()); @@ -104,16 +105,18 @@ namespace osmium { } /** - * Checks whether the first and last node in the list have the same ID. + * Checks whether the first and last node in the collection have the + * same ID. The locations are not checked. * * @pre @code !empty() @endcode */ bool is_closed() const noexcept { - return front().ref() == back().ref(); + return ends_have_same_id(); } /** - * Checks whether the first and last node in the list have the same ID. + * Checks whether the first and last node in the collection have the + * same ID. The locations are not checked. * * @pre @code !empty() @endcode */ @@ -122,8 +125,8 @@ namespace osmium { } /** - * Checks whether the first and last node in the list have the same - * location. The ID is not checked. + * Checks whether the first and last node in the collection have the + * same location. The IDs are not checked. * * @pre @code !empty() @endcode * @pre @code front().location() && back().location() @endcode @@ -133,9 +136,9 @@ namespace osmium { return front().location() == back().location(); } - typedef NodeRef* iterator; - typedef const NodeRef* const_iterator; - typedef std::reverse_iterator const_reverse_iterator; + using iterator = NodeRef*; + using const_iterator = const NodeRef*; + using const_reverse_iterator = std::reverse_iterator; /// Returns an iterator to the beginning. iterator begin() noexcept { diff --git a/third_party/libosmium/include/osmium/osm/object.hpp b/third_party/libosmium/include/osmium/osm/object.hpp index 8c745ce9b..c0f46adbd 100644 --- a/third_party/libosmium/include/osmium/osm/object.hpp +++ b/third_party/libosmium/include/osmium/osm/object.hpp @@ -281,7 +281,7 @@ namespace osmium { * @param timestamp Timestamp * @returns Reference to object to make calls chainable. */ - OSMObject& set_timestamp(const osmium::Timestamp timestamp) noexcept { + OSMObject& set_timestamp(const osmium::Timestamp& timestamp) noexcept { m_timestamp = timestamp; return *this; } @@ -355,38 +355,38 @@ namespace osmium { return cend(); } - template + template using t_iterator = osmium::memory::ItemIterator; - template + template using t_const_iterator = osmium::memory::ItemIterator; - template + template t_iterator begin() { return t_iterator(subitems_position(), next()); } - template + template t_iterator end() { return t_iterator(next(), next()); } - template + template t_const_iterator cbegin() const { return t_const_iterator(subitems_position(), next()); } - template + template t_const_iterator cend() const { return t_const_iterator(next(), next()); } - template + template t_const_iterator begin() const { return cbegin(); } - template + template t_const_iterator end() const { return cend(); } diff --git a/third_party/libosmium/include/osmium/osm/relation.hpp b/third_party/libosmium/include/osmium/osm/relation.hpp index 99a4f4cd1..eec51193d 100644 --- a/third_party/libosmium/include/osmium/osm/relation.hpp +++ b/third_party/libosmium/include/osmium/osm/relation.hpp @@ -47,9 +47,9 @@ DEALINGS IN THE SOFTWARE. namespace osmium { namespace builder { - template class ObjectBuilder; + template class ObjectBuilder; class RelationMemberListBuilder; - } + } // namespace builder class RelationMember : public osmium::memory::detail::ItemHelper { @@ -74,23 +74,21 @@ namespace osmium { return data() + osmium::memory::padded_length(sizeof(RelationMember) + m_role_size); } - template + template friend class osmium::memory::CollectionIterator; unsigned char* next() { if (full_member()) { return endpos() + reinterpret_cast(endpos())->byte_size(); - } else { - return endpos(); } + return endpos(); } unsigned const char* next() const { if (full_member()) { return endpos() + reinterpret_cast(endpos())->byte_size(); - } else { - return endpos(); } + return endpos(); } void set_role_size(string_size_type size) noexcept { diff --git a/third_party/libosmium/include/osmium/osm/segment.hpp b/third_party/libosmium/include/osmium/osm/segment.hpp index f3a82c97f..fe43102fc 100644 --- a/third_party/libosmium/include/osmium/osm/segment.hpp +++ b/third_party/libosmium/include/osmium/osm/segment.hpp @@ -65,12 +65,12 @@ namespace osmium { ~Segment() = default; /// Return first Location of Segment. - OSMIUM_CONSTEXPR osmium::Location first() const noexcept { + constexpr osmium::Location first() const noexcept { return m_first; } /// Return second Location of Segment. - OSMIUM_CONSTEXPR osmium::Location second() const noexcept { + constexpr osmium::Location second() const noexcept { return m_second; } @@ -84,11 +84,11 @@ namespace osmium { }; // class Segment /// Segments are equal if both their locations are equal - inline OSMIUM_CONSTEXPR bool operator==(const Segment& lhs, const Segment& rhs) noexcept { + inline constexpr bool operator==(const Segment& lhs, const Segment& rhs) noexcept { return lhs.first() == rhs.first() && lhs.second() == rhs.second(); } - inline OSMIUM_CONSTEXPR bool operator!=(const Segment& lhs, const Segment& rhs) noexcept { + inline constexpr bool operator!=(const Segment& lhs, const Segment& rhs) noexcept { return ! (lhs == rhs); } diff --git a/third_party/libosmium/include/osmium/osm/tag.hpp b/third_party/libosmium/include/osmium/osm/tag.hpp index 2e93ede24..4eba2210e 100644 --- a/third_party/libosmium/include/osmium/osm/tag.hpp +++ b/third_party/libosmium/include/osmium/osm/tag.hpp @@ -53,7 +53,7 @@ namespace osmium { Tag& operator=(const Tag&) = delete; Tag& operator=(Tag&&) = delete; - template + template friend class osmium::memory::CollectionIterator; static unsigned char* after_null(unsigned char* ptr) { @@ -122,9 +122,8 @@ namespace osmium { }); if (result == cend()) { return default_value; - } else { - return result->value(); } + return result->value(); } const char* operator[](const char* key) const noexcept { diff --git a/third_party/libosmium/include/osmium/osm/timestamp.hpp b/third_party/libosmium/include/osmium/osm/timestamp.hpp index 390f0e745..651b43f68 100644 --- a/third_party/libosmium/include/osmium/osm/timestamp.hpp +++ b/third_party/libosmium/include/osmium/osm/timestamp.hpp @@ -47,17 +47,18 @@ namespace osmium { /** * A timestamp. Internal representation is an unsigned 32bit integer - * holding seconds since epoch, so this will overflow in 2038. + * holding seconds since epoch (1970-01-01T00:00:00Z), so this will + * overflow in 2106. We can use an unsigned integer here, because the + * OpenStreetMap project was started long after 1970, so there will + * never be dates before that. */ class Timestamp { // length of ISO timestamp string yyyy-mm-ddThh:mm:ssZ\0 static constexpr int timestamp_length = 20 + 1; - /** - * The timestamp format for OSM timestamps in strftime(3) format. - * This is the ISO-Format yyyy-mm-ddThh:mm:ssZ - */ + // The timestamp format for OSM timestamps in strftime(3) format. + // This is the ISO-Format "yyyy-mm-ddThh:mm:ssZ". static const char* timestamp_format() { static const char f[timestamp_length] = "%Y-%m-%dT%H:%M:%SZ"; return f; @@ -67,19 +68,32 @@ namespace osmium { public: + /** + * Default construct an invalid Timestamp. + */ constexpr Timestamp() noexcept : m_timestamp(0) { } - // Not "explicit" so that conversions from time_t work - // like in node.timestamp(123); - constexpr Timestamp(time_t timestamp) noexcept : - m_timestamp(static_cast(timestamp)) { + /** + * Construct a Timestamp from any integer type containing the seconds + * since the epoch. This will not check for overruns, you have to + * make sure the value fits into a uint32_t which is used internally + * in the Timestamp. + * + * The constructor is not declared "explicit" so that conversions + * like @code node.set_timestamp(123); @endcode work. + */ + template ::value, int>::type = 0> + constexpr Timestamp(T timestamp) noexcept : + m_timestamp(uint32_t(timestamp)) { } /** - * Construct timestamp from ISO date/time string. - * Throws std::invalid_argument, if the timestamp can not be parsed. + * Construct timestamp from ISO date/time string in the format + * "yyyy-mm-ddThh:mm:ssZ". + * + * @throws std::invalid_argument if the timestamp can not be parsed. */ explicit Timestamp(const char* timestamp) { #ifndef _WIN32 @@ -105,16 +119,51 @@ namespace osmium { #endif } + /** + * Construct timestamp from ISO date/time string in the format + * "yyyy-mm-ddThh:mm:ssZ". + * + * @throws std::invalid_argument if the timestamp can not be parsed. + */ + explicit Timestamp(const std::string& timestamp) : + Timestamp(timestamp.c_str()) { + } + + /** + * Returns true if this timestamp is valid (ie set to something other + * than 0). + */ + bool valid() const noexcept { + return m_timestamp != 0; + } + + /// Explicit conversion into bool. + explicit constexpr operator bool() const noexcept { + return m_timestamp != 0; + } + + /// Explicit conversion into time_t. constexpr time_t seconds_since_epoch() const noexcept { - return static_cast(m_timestamp); - } - - constexpr operator time_t() const noexcept { - return static_cast(m_timestamp); + return time_t(m_timestamp); } + /// Explicit conversion into uint32_t. explicit constexpr operator uint32_t() const noexcept { - return m_timestamp; + return uint32_t(m_timestamp); + } + + /// Explicit conversion into uint64_t. + explicit constexpr operator uint64_t() const noexcept { + return uint64_t(m_timestamp); + } + + /** + * Implicit conversion into time_t. + * + * @deprecated You should call seconds_since_epoch() explicitly instead. + */ + OSMIUM_DEPRECATED constexpr operator time_t() const noexcept { + return static_cast(m_timestamp); } template @@ -128,7 +177,8 @@ namespace osmium { } /** - * Return UTC Unix time as string in ISO date/time format. + * Return UTC Unix time as string in ISO date/time + * ("yyyy-mm-ddThh:mm:ssZ") format. */ std::string to_iso() const { std::string s; @@ -156,12 +206,20 @@ namespace osmium { }; // class Timestamp - inline OSMIUM_CONSTEXPR Timestamp start_of_time() noexcept { + /** + * A special Timestamp guaranteed to be ordered before any other valid + * Timestamp. + */ + inline constexpr Timestamp start_of_time() noexcept { return Timestamp(1); } - inline OSMIUM_CONSTEXPR Timestamp end_of_time() noexcept { - return Timestamp(std::numeric_limits::max()); + /** + * A special Timestamp guaranteed to be ordered after any other valid + * Timestamp. + */ + inline constexpr Timestamp end_of_time() noexcept { + return Timestamp(std::numeric_limits::max()); } template @@ -170,6 +228,30 @@ namespace osmium { return out; } + inline bool operator==(const Timestamp& lhs, const Timestamp& rhs) noexcept { + return uint32_t(lhs) == uint32_t(rhs); + } + + inline bool operator!=(const Timestamp& lhs, const Timestamp& rhs) noexcept { + return !(lhs == rhs); + } + + inline bool operator<(const Timestamp& lhs, const Timestamp& rhs) noexcept { + return uint32_t(lhs) < uint32_t(rhs); + } + + inline bool operator>(const Timestamp& lhs, const Timestamp& rhs) noexcept { + return rhs < lhs; + } + + inline bool operator<=(const Timestamp& lhs, const Timestamp& rhs) noexcept { + return ! (rhs < lhs); + } + + inline bool operator>=(const Timestamp& lhs, const Timestamp& rhs) noexcept { + return ! (lhs < rhs); + } + template <> inline osmium::Timestamp min_op_start_value() { return end_of_time(); diff --git a/third_party/libosmium/include/osmium/osm/types.hpp b/third_party/libosmium/include/osmium/osm/types.hpp index b3414e594..e4250d966 100644 --- a/third_party/libosmium/include/osmium/osm/types.hpp +++ b/third_party/libosmium/include/osmium/osm/types.hpp @@ -49,6 +49,7 @@ namespace osmium { typedef uint32_t user_id_type; ///< Type for OSM user IDs. typedef int32_t signed_user_id_type; ///< Type for signed OSM user IDs. typedef uint32_t num_changes_type; ///< Type for changeset num_changes. + typedef uint32_t num_comments_type; ///< Type for changeset num_comments. /** * Size for strings in OSM data such as user names, tag keys, roles, etc. @@ -57,6 +58,9 @@ namespace osmium { */ typedef uint16_t string_size_type; + // maximum of 256 characters of max 4 bytes each (in UTF-8 encoding) + constexpr const int max_osm_string_length = 256 * 4; + } // namespace osmium #endif // OSMIUM_OSM_TYPES_HPP diff --git a/third_party/libosmium/include/osmium/osm/types_from_string.hpp b/third_party/libosmium/include/osmium/osm/types_from_string.hpp index b8de14c90..67ab2c157 100644 --- a/third_party/libosmium/include/osmium/osm/types_from_string.hpp +++ b/third_party/libosmium/include/osmium/osm/types_from_string.hpp @@ -43,9 +43,19 @@ DEALINGS IN THE SOFTWARE. #include #include +#include namespace osmium { + /** + * Convert string with object id to object_id_type. + * + * @pre input must not be nullptr. + * + * @param input Input string. + * + * @throws std::range_error if the value is out of range. + */ inline object_id_type string_to_object_id(const char* input) { assert(input); if (*input != '\0' && !std::isspace(*input)) { @@ -58,6 +68,19 @@ namespace osmium { throw std::range_error(std::string("illegal id: '") + input + "'"); } + /** + * Parse string with object type identifier followed by object id. This + * reads strings like "n1234" and "w10". + * + * @pre input must not be nullptr. + * + * @param input Input string. + * @param types Allowed types. Must not be osmium::osm_entity_bits::nothing. + * + * @returns std::pair of type and id. + * + * @throws std::range_error if the value is out of range. + */ inline std::pair string_to_object_id(const char* input, osmium::osm_entity_bits::type types) { assert(input); assert(types != osmium::osm_entity_bits::nothing); @@ -75,7 +98,7 @@ namespace osmium { namespace detail { - inline long string_to_ulong(const char* input, const char *name) { + inline unsigned long string_to_ulong(const char* input, const char *name) { if (*input != '\0' && *input != '-' && !std::isspace(*input)) { char* end; auto value = std::strtoul(input, &end, 10); @@ -88,27 +111,77 @@ namespace osmium { } // namespace detail + /** + * Convert string with object version to object_version_type. + * + * @pre input must not be nullptr. + * + * @param input Input string. + * + * @throws std::range_error if the value is out of range. + */ inline object_version_type string_to_object_version(const char* input) { assert(input); - return static_cast(detail::string_to_ulong(input, "version")); + return static_cast_with_assert(detail::string_to_ulong(input, "version")); } + /** + * Convert string with object version to object_version_type. + * + * @pre input must not be nullptr. + * + * @param input Input string. + * + * @throws std::range_error if the value is out of range. + */ inline changeset_id_type string_to_changeset_id(const char* input) { assert(input); - return static_cast(detail::string_to_ulong(input, "changeset")); + return static_cast_with_assert(detail::string_to_ulong(input, "changeset")); } + /** + * Convert string with user id to signed_user_id_type. + * + * @pre input must not be nullptr. + * + * @param input Input string. + * + * @throws std::range_error if the value is out of range. + */ inline signed_user_id_type string_to_user_id(const char* input) { assert(input); if (input[0] == '-' && input[1] == '1' && input[2] == '\0') { return -1; } - return static_cast(detail::string_to_ulong(input, "user id")); + return static_cast_with_assert(detail::string_to_ulong(input, "user id")); } + /** + * Convert string with number of changes to num_changes_type. + * + * @pre input must not be nullptr. + * + * @param input Input string. + * + * @throws std::range_error if the value is out of range. + */ inline num_changes_type string_to_num_changes(const char* input) { assert(input); - return static_cast(detail::string_to_ulong(input, "value for num changes")); + return static_cast_with_assert(detail::string_to_ulong(input, "value for num changes")); + } + + /** + * Convert string with number of comments to num_comments_type. + * + * @pre input must not be nullptr. + * + * @param input Input string. + * + * @throws std::range_error if the value is out of range. + */ + inline num_comments_type string_to_num_comments(const char* input) { + assert(input); + return static_cast_with_assert(detail::string_to_ulong(input, "value for num comments")); } } // namespace osmium diff --git a/third_party/libosmium/include/osmium/osm/way.hpp b/third_party/libosmium/include/osmium/osm/way.hpp index 3c5f1f6dc..90cde8c3c 100644 --- a/third_party/libosmium/include/osmium/osm/way.hpp +++ b/third_party/libosmium/include/osmium/osm/way.hpp @@ -43,8 +43,8 @@ DEALINGS IN THE SOFTWARE. namespace osmium { namespace builder { - template class ObjectBuilder; - } + template class ObjectBuilder; + } // namespace builder /** * List of node references (id and location) in a way. diff --git a/third_party/libosmium/include/osmium/relations/collector.hpp b/third_party/libosmium/include/osmium/relations/collector.hpp index 40e377393..7d27b3398 100644 --- a/third_party/libosmium/include/osmium/relations/collector.hpp +++ b/third_party/libosmium/include/osmium/relations/collector.hpp @@ -39,9 +39,10 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include +//#include #include +#include #include #include #include // IWYU pragma: keep @@ -55,9 +56,6 @@ DEALINGS IN THE SOFTWARE. namespace osmium { - class Node; - class Way; - /** * @brief Code related to the assembly of OSM relations */ @@ -91,7 +89,7 @@ namespace osmium { * * @tparam TRelations Are we interested in member relations? */ - template + template class Collector { /** @@ -124,82 +122,15 @@ namespace osmium { TCollector& m_collector; - /** - * This variable is initialized with the number of different - * kinds of OSM objects we are interested in. If we only need - * way members (for instance for the multipolygon collector) - * it is intialized with 1 for instance. If node and way - * members are needed, it is initialized with 2. - * - * In the after_* methods of this handler, it is decremented - * and once it reaches 0, we know we have all members available - * that we are ever going to get. - */ - int m_want_types; - - /** - * Find this object in the member vectors and add it to all - * relations that need it. - * - * @returns true if the member was added to at least one - * relation and false otherwise - */ - bool find_and_add_object(const osmium::OSMObject& object) { - auto& mmv = m_collector.member_meta(object.type()); - auto range = std::equal_range(mmv.begin(), mmv.end(), MemberMeta(object.id())); - - if (osmium::relations::count_not_removed(range.first, range.second) == 0) { - // nothing found - return false; - } - - { - m_collector.members_buffer().add_item(object); - const size_t member_offset = m_collector.members_buffer().commit(); - - for (auto it = range.first; it != range.second; ++it) { - it->set_buffer_offset(member_offset); - } - } - - for (auto it = range.first; it != range.second; ++it) { - MemberMeta& member_meta = *it; - if (member_meta.removed()) { - break; - } - assert(member_meta.member_id() == object.id()); - assert(member_meta.relation_pos() < m_collector.m_relations.size()); - RelationMeta& relation_meta = m_collector.m_relations[member_meta.relation_pos()]; -// std::cerr << " => " << member_meta.member_pos() << " < " << m_collector.get_relation(relation_meta).members().size() << " (id=" << m_collector.get_relation(relation_meta).id() << ")\n"; - assert(member_meta.member_pos() < m_collector.get_relation(relation_meta).members().size()); -// std::cerr << " add way " << member_meta.member_id() << " to rel " << m_collector.get_relation(relation_meta).id() << " at pos " << member_meta.member_pos() << "\n"; - relation_meta.got_one_member(); - if (relation_meta.has_all_members()) { - const size_t relation_offset = member_meta.relation_pos(); - m_collector.complete_relation(relation_meta); - m_collector.m_relations[relation_offset] = RelationMeta(); - m_collector.possibly_purge_removed_members(); - } - } - - // Remove MemberMetas that were marked as removed. - mmv.erase(std::remove_if(mmv.begin(), mmv.end(), [](MemberMeta& mm) { - return mm.removed(); - }), mmv.end()); - - return true; - } - public: HandlerPass2(TCollector& collector) noexcept : - m_collector(collector), - m_want_types((TNodes?1:0) + (TWays?1:0) + (TRelations?1:0)) { + m_collector(collector) { } void node(const osmium::Node& node) { if (TNodes) { - if (! find_and_add_object(node)) { + if (! m_collector.find_and_add_object(node)) { m_collector.node_not_in_any_relation(node); } } @@ -207,7 +138,7 @@ namespace osmium { void way(const osmium::Way& way) { if (TWays) { - if (! find_and_add_object(way)) { + if (! m_collector.find_and_add_object(way)) { m_collector.way_not_in_any_relation(way); } } @@ -215,7 +146,7 @@ namespace osmium { void relation(const osmium::Relation& relation) { if (TRelations) { - if (! find_and_add_object(relation)) { + if (! m_collector.find_and_add_object(relation)) { m_collector.relation_not_in_any_relation(relation); } } @@ -227,6 +158,8 @@ namespace osmium { }; // class HandlerPass2 + private: + HandlerPass2 m_handler_pass2; // All relations we are interested in will be kept in this buffer @@ -375,6 +308,8 @@ namespace osmium { return m_members_buffer.get(offset); } + private: + /** * Tell the Collector that you are interested in this relation * and want it kept until all members have been assembled and @@ -424,6 +359,84 @@ namespace osmium { std::sort(m_member_meta[2].begin(), m_member_meta[2].end()); } + /** + * Find this object in the member vectors and add it to all + * relations that need it. + * + * @returns true if the member was added to at least one + * relation and false otherwise + */ + bool find_and_add_object(const osmium::OSMObject& object) { + auto& mmv = member_meta(object.type()); + auto range = std::equal_range(mmv.begin(), mmv.end(), MemberMeta(object.id())); + + if (osmium::relations::count_not_removed(range.first, range.second) == 0) { + // nothing found + return false; + } + + { + members_buffer().add_item(object); + const size_t member_offset = members_buffer().commit(); + + for (auto it = range.first; it != range.second; ++it) { + it->set_buffer_offset(member_offset); + } + } + + for (auto it = range.first; it != range.second; ++it) { + MemberMeta& member_meta = *it; + if (member_meta.removed()) { + break; + } + assert(member_meta.member_id() == object.id()); + assert(member_meta.relation_pos() < m_relations.size()); + RelationMeta& relation_meta = m_relations[member_meta.relation_pos()]; +// std::cerr << " => " << member_meta.member_pos() << " < " << get_relation(relation_meta).members().size() << " (id=" << get_relation(relation_meta).id() << ")\n"; + assert(member_meta.member_pos() < get_relation(relation_meta).members().size()); +// std::cerr << " add way " << member_meta.member_id() << " to rel " << get_relation(relation_meta).id() << " at pos " << member_meta.member_pos() << "\n"; + relation_meta.got_one_member(); + if (relation_meta.has_all_members()) { + const size_t relation_offset = member_meta.relation_pos(); + static_cast(this)->complete_relation(relation_meta); + clear_member_metas(relation_meta); + m_relations[relation_offset] = RelationMeta(); + possibly_purge_removed_members(); + } + } + + // Remove MemberMetas that were marked as removed. + mmv.erase(std::remove_if(mmv.begin(), mmv.end(), [](MemberMeta& mm) { + return mm.removed(); + }), mmv.end()); + + return true; + } + + void clear_member_metas(const osmium::relations::RelationMeta& relation_meta) { + const osmium::Relation& relation = get_relation(relation_meta); + for (const auto& member : relation.members()) { + if (member.ref() != 0) { + auto& mmv = member_meta(member.type()); + auto range = std::equal_range(mmv.begin(), mmv.end(), MemberMeta(member.ref())); + assert(range.first != range.second); + + // if this is the last time this object was needed + // then mark it as removed + if (osmium::relations::count_not_removed(range.first, range.second) == 1) { + get_member(range.first->buffer_offset()).set_removed(true); + } + + for (auto it = range.first; it != range.second; ++it) { + if (!it->removed() && relation.id() == get_relation(it->relation_pos()).id()) { + it->remove(); + break; + } + } + } + } + } + public: uint64_t used_memory() const { @@ -474,14 +487,14 @@ namespace osmium { return range.first->buffer_offset(); } - template + template void read_relations(TIter begin, TIter end) { HandlerPass1 handler(*static_cast(this)); osmium::apply(begin, end, handler); sort_member_meta(); } - template + template void read_relations(TSource& source) { read_relations(std::begin(source), std::end(source)); source.close(); @@ -490,7 +503,7 @@ namespace osmium { void moving_in_buffer(size_t old_offset, size_t new_offset) { const osmium::OSMObject& object = m_members_buffer.get(old_offset); auto& mmv = member_meta(object.type()); - auto range = std::equal_range(mmv.begin(), mmv.end(), osmium::relations::MemberMeta(object.id())); + auto range = std::equal_range(mmv.begin(), mmv.end(), MemberMeta(object.id())); for (auto it = range.first; it != range.second; ++it) { assert(it->buffer_offset() == old_offset); it->set_buffer_offset(new_offset); @@ -506,13 +519,15 @@ namespace osmium { void possibly_purge_removed_members() { ++m_count_complete; if (m_count_complete > 10000) { // XXX - const size_t size_before = m_members_buffer.committed(); +// const size_t size_before = m_members_buffer.committed(); m_members_buffer.purge_removed(this); +/* const size_t size_after = m_members_buffer.committed(); double percent = static_cast(size_before - size_after); percent /= size_before; percent *= 100; -// std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast(percent) << "%)\n"; + std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast(percent) << "%)\n"; +*/ m_count_complete = 0; } } diff --git a/third_party/libosmium/include/osmium/relations/detail/member_meta.hpp b/third_party/libosmium/include/osmium/relations/detail/member_meta.hpp index a45088eab..ea86734ea 100644 --- a/third_party/libosmium/include/osmium/relations/detail/member_meta.hpp +++ b/third_party/libosmium/include/osmium/relations/detail/member_meta.hpp @@ -144,7 +144,7 @@ namespace osmium { * @param begin Begin of iterator range * @param end End of iterator range */ - template + template inline typename std::iterator_traits::difference_type count_not_removed(TIter begin, TIter end) { return std::count_if(begin, end, [](MemberMeta& mm) { return !mm.removed(); diff --git a/third_party/libosmium/include/osmium/tags/filter.hpp b/third_party/libosmium/include/osmium/tags/filter.hpp index 3c1946c54..0d3fc4ee2 100644 --- a/third_party/libosmium/include/osmium/tags/filter.hpp +++ b/third_party/libosmium/include/osmium/tags/filter.hpp @@ -46,7 +46,7 @@ namespace osmium { namespace tags { - template + template struct match_key { bool operator()(const TKey& rule_key, const char* tag_key) { return rule_key == tag_key; @@ -59,7 +59,7 @@ namespace osmium { } }; // struct match_key_prefix - template + template struct match_value { bool operator()(const TValue& rule_value, const char* tag_value) { return rule_value == tag_value; @@ -73,7 +73,7 @@ namespace osmium { } }; // struct match_value - template , class TValueComp=match_value> + template , typename TValueComp=match_value> class Filter { typedef TKey key_type; @@ -115,7 +115,7 @@ namespace osmium { m_default_result(default_result) { } - template ::value, int>::type = 0> + template ::value, int>::type = 0> Filter& add(bool result, const key_type& key, const value_type& value) { m_rules.emplace_back(result, false, key, value); return *this; diff --git a/third_party/libosmium/include/osmium/tags/taglist.hpp b/third_party/libosmium/include/osmium/tags/taglist.hpp index d7c78dc79..8fc9c68cd 100644 --- a/third_party/libosmium/include/osmium/tags/taglist.hpp +++ b/third_party/libosmium/include/osmium/tags/taglist.hpp @@ -45,17 +45,17 @@ namespace osmium { */ namespace tags { - template + template inline bool match_any_of(const osmium::TagList& tag_list, TFilter&& filter) { return std::any_of(tag_list.cbegin(), tag_list.cend(), std::forward(filter)); } - template + template inline bool match_all_of(const osmium::TagList& tag_list, TFilter&& filter) { return std::all_of(tag_list.cbegin(), tag_list.cend(), std::forward(filter)); } - template + template inline bool match_none_of(const osmium::TagList& tag_list, TFilter&& filter) { return std::none_of(tag_list.cbegin(), tag_list.cend(), std::forward(filter)); } diff --git a/third_party/libosmium/include/osmium/thread/function_wrapper.hpp b/third_party/libosmium/include/osmium/thread/function_wrapper.hpp index fe0a49257..2fc0b7e22 100644 --- a/third_party/libosmium/include/osmium/thread/function_wrapper.hpp +++ b/third_party/libosmium/include/osmium/thread/function_wrapper.hpp @@ -50,7 +50,9 @@ namespace osmium { struct impl_base { virtual ~impl_base() = default; - virtual void call() = 0; + virtual bool call() { + return true; + } }; // struct impl_base @@ -58,28 +60,38 @@ namespace osmium { template struct impl_type : impl_base { + F m_functor; - impl_type(F&& functor) : - m_functor(std::move(functor)) { + explicit impl_type(F&& functor) : + m_functor(std::forward(functor)) { } - void call() override { + bool call() override { m_functor(); + return false; } + }; // struct impl_type public: // Constructor must not be "explicit" for wrapper // to work seemlessly. - template - function_wrapper(F&& f) : - impl(new impl_type(std::move(f))) { + template + function_wrapper(TFunction&& f) : + impl(new impl_type(std::forward(f))) { } - void operator()() { - impl->call(); + // The integer parameter is only used to signal that we want + // the special function wrapper that makes the worker thread + // shut down. + function_wrapper(int) : + impl(new impl_base()) { + } + + bool operator()() { + return impl->call(); } function_wrapper() = default; diff --git a/third_party/libosmium/include/osmium/thread/pool.hpp b/third_party/libosmium/include/osmium/thread/pool.hpp index 391603108..f7b4f4038 100644 --- a/third_party/libosmium/include/osmium/thread/pool.hpp +++ b/third_party/libosmium/include/osmium/thread/pool.hpp @@ -54,6 +54,32 @@ namespace osmium { */ namespace thread { + namespace detail { + + // Maximum number of allowed pool threads (just to keep the user + // from setting something silly). + constexpr const int max_pool_threads = 256; + + inline int get_pool_size(int num_threads, int user_setting, unsigned hardware_concurrency) { + if (num_threads == 0) { + num_threads = user_setting ? user_setting : -2; + } + + if (num_threads < 0) { + num_threads += hardware_concurrency; + } + + if (num_threads < 1) { + num_threads = 1; + } else if (num_threads > max_pool_threads) { + num_threads = max_pool_threads; + } + + return num_threads; + } + + } // namespace detail + /** * Thread pool. */ @@ -83,7 +109,6 @@ namespace osmium { }; // class thread_joiner - std::atomic m_done; osmium::thread::Queue m_work_queue; std::vector m_threads; thread_joiner m_joiner; @@ -91,11 +116,15 @@ namespace osmium { void worker_thread() { osmium::thread::set_thread_name("_osmium_worker"); - while (!m_done) { + while (true) { function_wrapper task; m_work_queue.wait_and_pop_with_timeout(task); if (task) { - task(); + if (task()) { + // The called tasks returns true only when the + // worker thread should shut down. + return; + } } } } @@ -113,26 +142,17 @@ namespace osmium { * In all cases the minimum number of threads in the pool is 1. */ explicit Pool(int num_threads, size_t max_queue_size) : - m_done(false), m_work_queue(max_queue_size, "work"), m_threads(), m_joiner(m_threads), - m_num_threads(num_threads) { - - if (m_num_threads == 0) { - m_num_threads = osmium::config::get_pool_threads(); - } - - if (m_num_threads <= 0) { - m_num_threads = std::max(1, static_cast(std::thread::hardware_concurrency()) + m_num_threads); - } + m_num_threads(detail::get_pool_size(num_threads, osmium::config::get_pool_threads(), std::thread::hardware_concurrency())) { try { for (int i = 0; i < m_num_threads; ++i) { m_threads.push_back(std::thread(&Pool::worker_thread, this)); } } catch (...) { - m_done = true; + shutdown_all_workers(); throw; } } @@ -147,8 +167,15 @@ namespace osmium { return pool; } + void shutdown_all_workers() { + for (int i = 0; i < m_num_threads; ++i) { + // The special function wrapper makes a worker shut down. + m_work_queue.push(function_wrapper{0}); + } + } + ~Pool() { - m_done = true; + shutdown_all_workers(); m_work_queue.shutdown(); } diff --git a/third_party/libosmium/include/osmium/thread/queue.hpp b/third_party/libosmium/include/osmium/thread/queue.hpp index 76ad9a020..65b18475c 100644 --- a/third_party/libosmium/include/osmium/thread/queue.hpp +++ b/third_party/libosmium/include/osmium/thread/queue.hpp @@ -75,6 +75,9 @@ namespace osmium { /// The largest size the queue has been so far. size_t m_largest_size; + /// The number of times push() was called on the queue. + std::atomic m_push_counter; + /// The number of times the queue was full and a thread pushing /// to the queue was blocked. std::atomic m_full_counter; @@ -89,7 +92,7 @@ namespace osmium { * 0 for an unlimited size. * @param name Optional name for this queue. (Used for debugging.) */ - Queue(size_t max_size = 0, const std::string& name = "") : + explicit Queue(size_t max_size = 0, const std::string& name = "") : m_max_size(max_size), m_name(name), m_mutex(), @@ -99,6 +102,7 @@ namespace osmium { #ifdef OSMIUM_DEBUG_QUEUE_SIZE , m_largest_size(0), + m_push_counter(0), m_full_counter(0) #endif { @@ -107,7 +111,7 @@ namespace osmium { ~Queue() { shutdown(); #ifdef OSMIUM_DEBUG_QUEUE_SIZE - std::cerr << "queue '" << m_name << "' with max_size=" << m_max_size << " had largest size " << m_largest_size << " and was full " << m_full_counter << " times\n"; + std::cerr << "queue '" << m_name << "' with max_size=" << m_max_size << " had largest size " << m_largest_size << " and was full " << m_full_counter << " times in " << m_push_counter << " push() calls\n"; #endif } @@ -116,6 +120,9 @@ namespace osmium { * call will block if the queue is full. */ void push(T value) { +#ifdef OSMIUM_DEBUG_QUEUE_SIZE + ++m_push_counter; +#endif if (m_max_size) { while (size() >= m_max_size) { std::this_thread::sleep_for(full_queue_sleep_duration); diff --git a/third_party/libosmium/include/osmium/thread/util.hpp b/third_party/libosmium/include/osmium/thread/util.hpp index ca4f6dd53..a20e618a0 100644 --- a/third_party/libosmium/include/osmium/thread/util.hpp +++ b/third_party/libosmium/include/osmium/thread/util.hpp @@ -49,7 +49,7 @@ namespace osmium { * the exception stored in the future if there was one. Otherwise it * will just return. */ - template + template inline void check_for_exception(std::future& future) { if (future.valid() && future.wait_for(std::chrono::seconds(0)) == std::future_status::ready) { future.get(); @@ -60,7 +60,7 @@ namespace osmium { * Wait until the given future becomes ready. Will block if the future * is not ready. Can be called more than once unlike future.get(). */ - template + template inline void wait_until_done(std::future& future) { if (future.valid()) { future.get(); @@ -71,15 +71,44 @@ namespace osmium { * Set name of current thread for debugging. This only works on Linux. */ #ifdef __linux__ - inline void set_thread_name(const char* name) { + inline void set_thread_name(const char* name) noexcept { prctl(PR_SET_NAME, name, 0, 0, 0); } #else - inline void set_thread_name(const char*) { + inline void set_thread_name(const char*) noexcept { // intentionally left blank } #endif + class thread_handler { + + std::thread m_thread; + + public: + + thread_handler() : + m_thread() { + } + + template + explicit thread_handler(TFunction&& f, TArgs&&... args) : + m_thread(std::forward(f), std::forward(args)...) { + } + + thread_handler(const thread_handler&) = delete; + thread_handler& operator=(const thread_handler&) = delete; + + thread_handler(thread_handler&&) = default; + thread_handler& operator=(thread_handler&&) = default; + + ~thread_handler() { + if (m_thread.joinable()) { + m_thread.join(); + } + } + + }; // class thread_handler + } // namespace thread } // namespace osmium diff --git a/third_party/libosmium/include/osmium/util/compatibility.hpp b/third_party/libosmium/include/osmium/util/compatibility.hpp index 90d85c502..27adca7b7 100644 --- a/third_party/libosmium/include/osmium/util/compatibility.hpp +++ b/third_party/libosmium/include/osmium/util/compatibility.hpp @@ -34,14 +34,20 @@ DEALINGS IN THE SOFTWARE. */ // Workarounds for MSVC which doesn't support -// * constexpr in all cases yet // * [[noreturn]] #ifdef _MSC_VER -# define OSMIUM_CONSTEXPR # define OSMIUM_NORETURN __declspec(noreturn) #else -# define OSMIUM_CONSTEXPR constexpr # define OSMIUM_NORETURN [[noreturn]] #endif +// [[deprecated]] is only available in C++14, use this for the time being +#ifdef __GNUC__ +# define OSMIUM_DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) +# define OSMIUM_DEPRECATED __declspec(deprecated) +#else +# define OSMIUM_DEPRECATED +#endif + #endif // OSMIUM_UTIL_COMPATIBILITY_HPP diff --git a/third_party/libosmium/include/osmium/util/config.hpp b/third_party/libosmium/include/osmium/util/config.hpp index 3285eedbb..e31cd6a9b 100644 --- a/third_party/libosmium/include/osmium/util/config.hpp +++ b/third_party/libosmium/include/osmium/util/config.hpp @@ -49,7 +49,7 @@ namespace osmium { if (env) { return std::atoi(env); } - return -2; + return 0; } inline bool use_pool_threads_for_pbf_parsing() { diff --git a/third_party/libosmium/include/osmium/util/data_file.hpp b/third_party/libosmium/include/osmium/util/data_file.hpp deleted file mode 100644 index 22bf1910a..000000000 --- a/third_party/libosmium/include/osmium/util/data_file.hpp +++ /dev/null @@ -1,194 +0,0 @@ -#ifndef OSMIUM_UTIL_DATA_FILE_HPP -#define OSMIUM_UTIL_DATA_FILE_HPP - -/* - -This file is part of Osmium (http://osmcode.org/libosmium). - -Copyright 2013-2015 Jochen Topf and others (see README). - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -*/ - -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 -# include -# include -#endif - -#include - -namespace osmium { - - namespace util { - - /** - * Class wrapper for convenient access to some low-level file - * functions. - */ - class DataFile { - - FILE* m_file; - - public: - - /** - * Create and open a temporary file. It is removed after opening. - * - * @throws std::system_error if something went wrong. - */ - DataFile() : - m_file(::tmpfile()) { - if (!m_file) { - throw std::system_error(errno, std::system_category(), "tmpfile failed"); - } - } - - /** - * Create and open a temporary file with the specified size. It - * is removed after opening. - * - * @throws std::system_error if something went wrong. - */ - explicit DataFile(size_t size) : - DataFile() { - grow(size); - } - - /** - * Create and open a named file. - * - * @param filename the name of the file - * @param writable should the file be writable? - * @throws std::system_error if something went wrong. - */ - DataFile(const char* filename, bool writable) : - m_file(::fopen(filename, writable ? "wb+" : "rb" )) { - if (!m_file) { - throw std::system_error(errno, std::system_category(), "fopen failed"); - } - } - - /** - * Create and open a named file. - * - * @param filename the name of the file - * @param writable should the file be writable? - * @throws std::system_error if something went wrong. - */ - DataFile(const std::string& filename, bool writable) : - DataFile(filename.c_str(), writable) { - } - - /** - * In boolean context the DataFile class returns true if the file - * is open. - */ - operator bool() const noexcept { - return m_file != nullptr; - } - - /** - * Close the file. - * - * Does nothing if the file is already closed. - * - * @throws std::system_error if file could not be closed - */ - void close() { - if (m_file) { - if (::fclose(m_file) != 0) { - throw std::system_error(errno, std::system_category(), "fclose failed"); - } - m_file = nullptr; - } - } - - ~DataFile() noexcept { - try { - close(); - } catch (std::system_error&) { - // ignore - } - } - - /** - * Get file descriptor of underlying file. - * - * @throws std::runtime_errro if file is not open - * @throws std::system_error if fileno(3) call failed - */ - int fd() const { - if (!m_file) { - throw std::runtime_error("no open file"); - } - - int fd = ::fileno(m_file); - - if (fd == -1) { - throw std::system_error(errno, std::system_category(), "fileno failed"); - } - - return fd; - } - - /** - * Ask the operating system for the size of this file. - * - * @throws std::system_error if fstat(2) call failed - */ - size_t size() const { - return osmium::util::file_size(fd()); - } - - /** - * Grow file to given size. - * - * If the file is large enough already, nothing is done. - * The file is never shrunk. - * - * @throws std::system_error if ftruncate(2) call failed - */ - void grow(size_t new_size) const { - if (size() < new_size) { - osmium::util::resize_file(fd(), new_size); - } - } - - }; // class DataFile - - } // namespace util - -} // namespace osmium - - -#endif // OSMIUM_UTIL_DATA_FILE_HPP diff --git a/third_party/libosmium/include/osmium/util/delta.hpp b/third_party/libosmium/include/osmium/util/delta.hpp index 0c77e5242..558a1d4f9 100644 --- a/third_party/libosmium/include/osmium/util/delta.hpp +++ b/third_party/libosmium/include/osmium/util/delta.hpp @@ -37,6 +37,8 @@ DEALINGS IN THE SOFTWARE. #include #include +#include + namespace osmium { namespace util { @@ -44,25 +46,39 @@ namespace osmium { /** * Helper class for delta encoding. */ - template + template class DeltaEncode { - T m_value; + static_assert(std::is_integral::value, + "DeltaEncode value type must be some integer"); + + static_assert(std::is_integral::value && std::is_signed::value, + "DeltaEncode delta type must be some signed integer"); + + TValue m_value; public: - DeltaEncode(T value = 0) : + using value_type = TValue; + using delta_type = TDelta; + + explicit DeltaEncode(TValue value = 0) : m_value(value) { } - void clear() { + void clear() noexcept { m_value = 0; } - T update(T new_value) { + TValue value() const noexcept { + return m_value; + } + + TDelta update(TValue new_value) noexcept { using std::swap; swap(m_value, new_value); - return m_value - new_value; + return static_cast_with_assert(m_value) - + static_cast_with_assert(new_value); } }; // class DeltaEncode @@ -70,52 +86,63 @@ namespace osmium { /** * Helper class for delta decoding. */ - template + template class DeltaDecode { - T m_value; + static_assert(std::is_integral::value, + "DeltaDecode value type must be some integer"); + + static_assert(std::is_integral::value && std::is_signed::value, + "DeltaDecode delta type must be some signed integer"); + + TValue m_value; public: + using value_type = TValue; + using delta_type = TDelta; + DeltaDecode() : m_value(0) { } - void clear() { + void clear() noexcept { m_value = 0; } - T update(T delta) { - m_value += delta; + TValue update(TDelta delta) noexcept { + m_value = static_cast_with_assert( + static_cast_with_assert(m_value) + delta); return m_value; } }; // class DeltaDecode - template + template class DeltaEncodeIterator : public std::iterator { - typedef TValue value_type; - TBaseIterator m_it; TBaseIterator m_end; - value_type m_delta; - DeltaEncode m_value; TTransform m_trans; + DeltaEncode m_value; + TDelta m_delta; public: + using value_type = TValue; + using delta_type = TDelta; + DeltaEncodeIterator(TBaseIterator first, TBaseIterator last, TTransform& trans) : m_it(first), m_end(last), - m_delta(m_trans(m_it)), - m_value(m_delta), - m_trans(trans) { + m_trans(trans), + m_value(m_it != m_end ? m_trans(m_it) : 0), + m_delta(static_cast_with_assert(m_value.value())) { } DeltaEncodeIterator& operator++() { - if (m_it != m_end) { - m_delta = m_value.update(m_trans(++m_it)); + if (++m_it != m_end) { + m_delta = m_value.update(m_trans(m_it)); } return *this; } @@ -126,7 +153,7 @@ namespace osmium { return tmp; } - value_type operator*() { + TDelta operator*() { return m_delta; } diff --git a/third_party/libosmium/include/osmium/util/double.hpp b/third_party/libosmium/include/osmium/util/double.hpp index 85a250807..e0701394c 100644 --- a/third_party/libosmium/include/osmium/util/double.hpp +++ b/third_party/libosmium/include/osmium/util/double.hpp @@ -68,8 +68,12 @@ namespace osmium { #endif assert(len > 0 && len < max_double_length); - while (buffer[len-1] == '0') --len; - if (buffer[len-1] == '.') --len; + while (buffer[len-1] == '0') { + --len; + } + if (buffer[len-1] == '.') { + --len; + } return std::copy_n(buffer, len, iterator); } diff --git a/third_party/libosmium/include/osmium/util/file.hpp b/third_party/libosmium/include/osmium/util/file.hpp index 461f4e642..39e01af83 100644 --- a/third_party/libosmium/include/osmium/util/file.hpp +++ b/third_party/libosmium/include/osmium/util/file.hpp @@ -52,6 +52,8 @@ DEALINGS IN THE SOFTWARE. # define ftruncate _chsize_s #endif +#include + namespace osmium { namespace util { @@ -92,7 +94,7 @@ namespace osmium { * @throws std::system_error If ftruncate(2) call failed */ inline void resize_file(int fd, size_t new_size) { - if (::ftruncate(fd, new_size) != 0) { + if (::ftruncate(fd, static_cast_with_assert(new_size)) != 0) { throw std::system_error(errno, std::system_category(), "ftruncate failed"); } } @@ -108,7 +110,7 @@ namespace osmium { return si.dwPageSize; #else // Unix implementation - return ::sysconf(_SC_PAGESIZE); + return size_t(::sysconf(_SC_PAGESIZE)); #endif } diff --git a/third_party/libosmium/include/osmium/util/memory_mapping.hpp b/third_party/libosmium/include/osmium/util/memory_mapping.hpp index e48aff282..b187c3c80 100644 --- a/third_party/libosmium/include/osmium/util/memory_mapping.hpp +++ b/third_party/libosmium/include/osmium/util/memory_mapping.hpp @@ -38,11 +38,13 @@ DEALINGS IN THE SOFTWARE. #include #include +#include #include #ifndef _WIN32 # include #else +# include # include # include # include @@ -85,6 +87,9 @@ namespace osmium { * On Unix systems this wraps the mmap(), munmap(), and the mremap() * system calls. On Windows it wraps the CreateFileMapping(), * CloseHandle(), MapViewOfFile(), and UnmapViewOfFile() functions. + * + * On Windows the file will be set to binary mode before the memory + * mapping. */ class MemoryMapping { @@ -169,7 +174,8 @@ private: * created, otherwise a mapping based on the file descriptor will * be created. * - * @pre size > 0 or mode == write_shared oder write_private + * @pre @code size > 0 @endcode or + * @code mode == write_shared || mode == write_private @endcode * * @param size Size of the mapping in bytes * @param mode Mapping mode: readonly, or writable (shared or private) @@ -179,8 +185,12 @@ private: */ MemoryMapping(size_t size, mapping_mode mode, int fd=-1, off_t offset=0); - /// DEPRECATED: For backwards compatibility - MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0) : + /** + * @deprecated + * For backwards compatibility only. Use the constructor taking + * a mapping_mode as second argument instead. + */ + OSMIUM_DEPRECATED MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0) : MemoryMapping(size, writable ? mapping_mode::write_shared : mapping_mode::readonly, fd, offset) { } @@ -209,7 +219,7 @@ private: try { unmap(); } catch (std::system_error&) { - // ignore + // Ignore any exceptions because destructor must not throw. } } @@ -228,8 +238,9 @@ private: * systems it will unmap and remap the memory. This can only be * done for file-based mappings, not anonymous mappings! * - * @param new_size Number of bytes to resize to - * @throws std::system_error if the remapping fails + * @param new_size Number of bytes to resize to (must be > 0). + * + * @throws std::system_error if the remapping fails. */ void resize(size_t new_size); @@ -237,7 +248,7 @@ private: * In a boolean context a MemoryMapping is true when it is a valid * existing mapping. */ - operator bool() const noexcept { + explicit operator bool() const noexcept { return is_valid(); } @@ -349,8 +360,12 @@ private: m_mapping(sizeof(T) * size, mode, fd, sizeof(T) * offset) { } - /// DEPRECATED: For backwards compatibility - TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset = 0) : + /** + * @deprecated + * For backwards compatibility only. Use the constructor taking + * a mapping_mode as second argument instead. + */ + OSMIUM_DEPRECATED TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset = 0) : m_mapping(sizeof(T) * size, writable ? MemoryMapping::mapping_mode::write_shared : MemoryMapping::mapping_mode::readonly, fd, sizeof(T) * offset) { } @@ -375,7 +390,7 @@ private: * Releases the mapping by calling unmap(). Will never throw. * Call unmap() instead if you want to be notified of any error. */ - ~TypedMemoryMapping() = default; + ~TypedMemoryMapping() noexcept = default; /** * Unmap a mapping. If the mapping is not valid, it will do @@ -405,7 +420,7 @@ private: * In a boolean context a TypedMemoryMapping is true when it is * a valid existing mapping. */ - operator bool() const noexcept { + explicit operator bool() const noexcept { return !!m_mapping; } @@ -655,6 +670,9 @@ inline HANDLE osmium::util::MemoryMapping::get_handle() const noexcept { } inline HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept { + if (m_fd != -1) { + _setmode(m_fd, _O_BINARY); + } return CreateFileMapping(get_handle(), nullptr, get_protection(), osmium::util::dword_hi(static_cast(m_size) + m_offset), osmium::util::dword_lo(static_cast(m_size) + m_offset), nullptr); } diff --git a/third_party/libosmium/include/osmium/util/options.hpp b/third_party/libosmium/include/osmium/util/options.hpp index fea075230..1019c8bbf 100644 --- a/third_party/libosmium/include/osmium/util/options.hpp +++ b/third_party/libosmium/include/osmium/util/options.hpp @@ -47,46 +47,65 @@ namespace osmium { * as a base class. Options are stored and retrieved by key using the * different set() and get() methods. * + * Both keys and values are stored as strings. The values "true", + * "yes", "false", and "no" are interpreted as boolean values in some + * functions. + * * You can iterate over all set options. Dereferencing an iterator * yields a std::pair of the key and value strings. */ class Options { - typedef std::map option_map; + using option_map = std::map; option_map m_options; public: - typedef option_map::iterator iterator; - typedef option_map::const_iterator const_iterator; - typedef option_map::value_type value_type; + using iterator = option_map::iterator; + using const_iterator = option_map::const_iterator; + using value_type = option_map::value_type; + /** + * Construct empty option set. + */ Options() = default; + /** + * Construct option set from initializer list: + * @code + * Options options{ { "foo", "true" }, { "bar", "17" } }; + * @endcode + */ explicit Options(const std::initializer_list& values) : m_options(values) { } - Options(const Options&) = default; - Options& operator=(const Options&) = default; - - Options(Options&&) = default; - Options& operator=(Options&&) = default; - - ~Options() = default; - + /** + * Set option 'key' to 'value'. + */ void set(const std::string& key, const std::string& value) { m_options[key] = value; } + /** + * Set option 'key' to 'value'. + */ void set(const std::string& key, const char* value) { m_options[key] = value; } + /** + * Set option 'key' to 'value'. + */ void set(const std::string& key, bool value) { m_options[key] = value ? "true" : "false"; } + /** + * Set option from string in the form 'key=value'. If the string + * contains no equal sign, the whole string is the key and it will + * be set to "true". + */ void set(std::string data) { size_t pos = data.find_first_of('='); if (pos == std::string::npos) { @@ -99,7 +118,7 @@ namespace osmium { } /** - * Get value of "key" option. If not set the default_value (or + * Get value of "key" option. If not set, the default_value (or * empty string) is returned. */ std::string get(const std::string& key, const std::string& default_value="") const noexcept { @@ -112,36 +131,67 @@ namespace osmium { /** * Is this option set to a true value ("true" or "yes")? + * Will return false if the value is unset. */ bool is_true(const std::string& key) const noexcept { std::string value = get(key); return (value == "true" || value == "yes"); } + /** + * Is this option not set to a false value ("false" or "no")? + * Will return true if the value is unset. + */ + bool is_not_false(const std::string& key) const noexcept { + std::string value = get(key); + return !(value == "false" || value == "no"); + } + + /** + * The number of options set. + */ size_t size() const noexcept { return m_options.size(); } + /** + * Returns an iterator to the beginning. + */ iterator begin() noexcept { return m_options.begin(); } + /** + * Returns an iterator to the end. + */ iterator end() noexcept { return m_options.end(); } + /** + * Returns an iterator to the beginning. + */ const_iterator begin() const noexcept { return m_options.cbegin(); } + /** + * Returns an iterator to the end. + */ const_iterator end() const noexcept { return m_options.cend(); } + /** + * Returns an iterator to the beginning. + */ const_iterator cbegin() const noexcept { return m_options.cbegin(); } + /** + * Returns a iterator to the end. + */ const_iterator cend() const noexcept { return m_options.cend(); } diff --git a/third_party/libosmium/include/osmium/visitor.hpp b/third_party/libosmium/include/osmium/visitor.hpp index 0250f11d4..c76eb17df 100644 --- a/third_party/libosmium/include/osmium/visitor.hpp +++ b/third_party/libosmium/include/osmium/visitor.hpp @@ -35,6 +35,7 @@ DEALINGS IN THE SOFTWARE. #include +#include #include // IWYU pragma: keep #include #include @@ -43,22 +44,16 @@ DEALINGS IN THE SOFTWARE. namespace osmium { - class TagList; - class WayNodeList; - class RelationMemberList; - class OuterRing; - class InnerRing; - namespace memory { class Item; - } + } // namespace memory namespace detail { template using ConstIfConst = typename std::conditional::value, typename std::add_const::type, U>::type; - template + template inline void apply_item_recurse(TItem& item, THandler& handler) { switch (item.type()) { case osmium::item_type::undefined: @@ -98,10 +93,13 @@ namespace osmium { case osmium::item_type::inner_ring: handler.inner_ring(static_cast&>(item)); break; + case osmium::item_type::changeset_discussion: + handler.changeset_discussion(static_cast&>(item)); + break; } } - template + template inline void apply_item_recurse(const osmium::OSMEntity& item, THandler& handler) { switch (item.type()) { case osmium::item_type::node: @@ -128,7 +126,7 @@ namespace osmium { } } - template + template inline void apply_item_recurse(osmium::OSMEntity& item, THandler& handler) { switch (item.type()) { case osmium::item_type::node: @@ -155,7 +153,7 @@ namespace osmium { } } - template + template inline void apply_item_recurse(const osmium::OSMObject& item, THandler& handler) { switch (item.type()) { case osmium::item_type::node: @@ -179,7 +177,7 @@ namespace osmium { } } - template + template inline void apply_item_recurse(osmium::OSMObject& item, THandler& handler) { switch (item.type()) { case osmium::item_type::node: @@ -203,18 +201,18 @@ namespace osmium { } } - template + template inline void apply_item_recurse(TItem& item, THandler& handler, TRest&... more) { apply_item_recurse(item, handler); apply_item_recurse(item, more...); } - template + template inline void flush_recurse(THandler& handler) { handler.flush(); } - template + template inline void flush_recurse(THandler& handler, TRest&... more) { flush_recurse(handler); flush_recurse(more...); @@ -222,17 +220,17 @@ namespace osmium { } // namespace detail - template + template inline void apply_item(const osmium::memory::Item& item, THandlers&... handlers) { detail::apply_item_recurse(item, handlers...); } - template + template inline void apply_item(osmium::memory::Item& item, THandlers&... handlers) { detail::apply_item_recurse(item, handlers...); } - template + template inline void apply(TIterator it, TIterator end, THandlers&... handlers) { for (; it != end; ++it) { detail::apply_item_recurse(*it, handlers...); @@ -240,12 +238,12 @@ namespace osmium { detail::flush_recurse(handlers...); } - template + template inline void apply(TContainer& c, THandlers&... handlers) { apply(std::begin(c), std::end(c), handlers...); } - template + template inline void apply(const osmium::memory::Buffer& buffer, THandlers&... handlers) { apply(buffer.cbegin(), buffer.cend(), handlers...); } diff --git a/third_party/libosmium/include/protozero/byteswap.hpp b/third_party/libosmium/include/protozero/byteswap.hpp index d019c28c5..a018c1c17 100644 --- a/third_party/libosmium/include/protozero/byteswap.hpp +++ b/third_party/libosmium/include/protozero/byteswap.hpp @@ -10,30 +10,51 @@ documentation. *****************************************************************************/ +/** + * @file byteswap.hpp + * + * @brief Contains functions to swap bytes in values (for different endianness). + */ + +#include #include +#include + namespace protozero { +/** + * Swap N byte value between endianness formats. This template function must + * be specialized to actually work. + */ template inline void byteswap(const char* /*data*/, char* /*result*/) { - assert(false); -} - -template <> -inline void byteswap<1>(const char* data, char* result) { - result[0] = data[0]; + static_assert(N == 1, "Can only swap 4 or 8 byte values"); } +/** + * Swap 4 byte value (int32_t, uint32_t, float) between endianness formats. + */ template <> inline void byteswap<4>(const char* data, char* result) { +#ifdef PROTOZERO_USE_BUILTIN_BSWAP + *reinterpret_cast(result) = __builtin_bswap32(*reinterpret_cast(data)); +#else result[3] = data[0]; result[2] = data[1]; result[1] = data[2]; result[0] = data[3]; +#endif } +/** + * Swap 8 byte value (int64_t, uint64_t, double) between endianness formats. + */ template <> inline void byteswap<8>(const char* data, char* result) { +#ifdef PROTOZERO_USE_BUILTIN_BSWAP + *reinterpret_cast(result) = __builtin_bswap64(*reinterpret_cast(data)); +#else result[7] = data[0]; result[6] = data[1]; result[5] = data[2]; @@ -42,6 +63,7 @@ inline void byteswap<8>(const char* data, char* result) { result[2] = data[5]; result[1] = data[6]; result[0] = data[7]; +#endif } } // end namespace protozero diff --git a/third_party/libosmium/include/protozero/config.hpp b/third_party/libosmium/include/protozero/config.hpp new file mode 100644 index 000000000..4086994fb --- /dev/null +++ b/third_party/libosmium/include/protozero/config.hpp @@ -0,0 +1,57 @@ +#ifndef PROTOZERO_CONFIG_HPP +#define PROTOZERO_CONFIG_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +#include + +/** + * @file config.hpp + * + * @brief Contains macro checks for different configurations. + */ + +#define PROTOZERO_LITTLE_ENDIAN 1234 +#define PROTOZERO_BIG_ENDIAN 4321 + +// Find out which byte order the machine has. +#if defined(__BYTE_ORDER) +# if (__BYTE_ORDER == __LITTLE_ENDIAN) +# define PROTOZERO_BYTE_ORDER PROTOZERO_LITTLE_ENDIAN +# endif +# if (__BYTE_ORDER == __BIG_ENDIAN) +# define PROTOZERO_BYTE_ORDER PROTOZERO_BIG_ENDIAN +# endif +#else +// This probably isn't a very good default, but might do until we figure +// out something better. +# define PROTOZERO_BYTE_ORDER PROTOZERO_LITTLE_ENDIAN +#endif + +// On some ARM machines and depending on compiler settings access to unaligned +// floating point values will result in a SIGBUS. Do not use the bare pointers +// in this case. +#if PROTOZERO_BYTE_ORDER == PROTOZERO_LITTLE_ENDIAN +# if !defined(__arm__) && !defined(_M_ARM) +# define PROTOZERO_USE_BARE_POINTER_FOR_PACKED_FIXED +# endif +#endif + +// Check whether __builtin_bswap is available +#if defined(__GNUC__) || defined(__clang__) +# define PROTOZERO_USE_BUILTIN_BSWAP +#endif + +// Wrapper for assert() used for testing +#ifndef protozero_assert +# define protozero_assert(x) assert(x) +#endif + +#endif // PROTOZERO_CONFIG_HPP diff --git a/third_party/libosmium/include/protozero/pbf_builder.hpp b/third_party/libosmium/include/protozero/pbf_builder.hpp index d49a7ba59..063fa9cb9 100644 --- a/third_party/libosmium/include/protozero/pbf_builder.hpp +++ b/third_party/libosmium/include/protozero/pbf_builder.hpp @@ -10,6 +10,12 @@ documentation. *****************************************************************************/ +/** + * @file pbf_builder.hpp + * + * @brief Contains the pbf_builder template class. + */ + #include #include @@ -17,10 +23,22 @@ documentation. namespace protozero { +/** + * The pbf_builder is used to write PBF formatted messages into a buffer. It + * is based on the pbf_writer class and has all the same methods. The + * difference is that whereever the pbf_writer class takes an integer tag, + * this template class takes a tag of the template type T. + * + * Almost all methods in this class can throw an std::bad_alloc exception if + * the std::string used as a buffer wants to resize. + * + * Read the tutorial to understand how this class is used. + */ template class pbf_builder : public pbf_writer { - static_assert(std::is_same::type>::value, "T must be enum with underlying type protozero::pbf_tag_type"); + static_assert(std::is_same::type>::value, + "T must be enum with underlying type protozero::pbf_tag_type"); public: @@ -35,6 +53,7 @@ public: pbf_writer(parent_writer, pbf_tag_type(tag)) { } +/// @cond INTERNAL #define PROTOZERO_WRITER_WRAP_ADD_SCALAR(name, type) \ inline void add_##name(T tag, type value) { \ pbf_writer::add_##name(pbf_tag_type(tag), value); \ @@ -55,6 +74,9 @@ public: PROTOZERO_WRITER_WRAP_ADD_SCALAR(float, float) PROTOZERO_WRITER_WRAP_ADD_SCALAR(double, double) +#undef PROTOZERO_WRITER_WRAP_ADD_SCALAR +/// @endcond + inline void add_bytes(T tag, const char* value, size_t size) { pbf_writer::add_bytes(pbf_tag_type(tag), value, size); } @@ -83,6 +105,7 @@ public: pbf_writer::add_message(pbf_tag_type(tag), value); } +/// @cond INTERNAL #define PROTOZERO_WRITER_WRAP_ADD_PACKED(name) \ template \ inline void add_packed_##name(T tag, InputIterator first, InputIterator last) { \ @@ -104,6 +127,9 @@ public: PROTOZERO_WRITER_WRAP_ADD_PACKED(float) PROTOZERO_WRITER_WRAP_ADD_PACKED(double) +#undef PROTOZERO_WRITER_WRAP_ADD_PACKED +/// @endcond + }; } // end namespace protozero diff --git a/third_party/libosmium/include/protozero/pbf_message.hpp b/third_party/libosmium/include/protozero/pbf_message.hpp index af29a00f4..7fef06f81 100644 --- a/third_party/libosmium/include/protozero/pbf_message.hpp +++ b/third_party/libosmium/include/protozero/pbf_message.hpp @@ -10,6 +10,12 @@ documentation. *****************************************************************************/ +/** + * @file pbf_message.hpp + * + * @brief Contains the pbf_message class. + */ + #include #include @@ -17,6 +23,44 @@ documentation. namespace protozero { +/** + * This class represents a protobuf message. Either a top-level message or + * a nested sub-message. Top-level messages can be created from any buffer + * with a pointer and length: + * + * @code + * enum class Message : protozero::pbf_tag_type { + * ... + * }; + * + * std::string buffer; + * // fill buffer... + * pbf_message message(buffer.data(), buffer.size()); + * @endcode + * + * Sub-messages are created using get_message(): + * + * @code + * enum class SubMessage : protozero::pbf_tag_type { + * ... + * }; + * + * pbf_message message(...); + * message.next(); + * pbf_message submessage = message.get_message(); + * @endcode + * + * All methods of the pbf_message class except get_bytes() and get_string() + * provide the strong exception guarantee, ie they either succeed or do not + * change the pbf_message object they are called on. Use the get_data() method + * instead of get_bytes() or get_string(), if you need this guarantee. + * + * This template class is based on the pbf_reader class and has all the same + * methods. The difference is that whereever the pbf_reader class takes an + * integer tag, this template class takes a tag of the template type T. + * + * Read the tutorial to understand how this class is used. + */ template class pbf_message : public pbf_reader { diff --git a/third_party/libosmium/include/protozero/pbf_reader.hpp b/third_party/libosmium/include/protozero/pbf_reader.hpp index 1c5ed0d70..aced901c6 100644 --- a/third_party/libosmium/include/protozero/pbf_reader.hpp +++ b/third_party/libosmium/include/protozero/pbf_reader.hpp @@ -16,7 +16,6 @@ documentation. * @brief Contains the pbf_reader class. */ -#include #include #include #include @@ -24,19 +23,15 @@ documentation. #include #include -#include +#include #include +#include #include -#if __BYTE_ORDER != __LITTLE_ENDIAN +#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN # include #endif -/// Wrapper for assert() used for testing -#ifndef protozero_assert -# define protozero_assert(x) assert(x) -#endif - namespace protozero { /** @@ -77,19 +72,27 @@ class pbf_reader { // The tag of the current field. pbf_tag_type m_tag = 0; + // Copy N bytes from src to dest on little endian machines, on big endian + // swap the bytes in the process. + template + static void copy_or_byteswap(const char* src, void* dest) noexcept { +#if PROTOZERO_BYTE_ORDER == PROTOZERO_LITTLE_ENDIAN + memcpy(dest, src, N); +#else + byteswap(src, reinterpret_cast(dest)); +#endif + } + template inline T get_fixed() { T result; skip_bytes(sizeof(T)); -#if __BYTE_ORDER == __LITTLE_ENDIAN - memcpy(&result, m_data - sizeof(T), sizeof(T)); -#else - byteswap(m_data - sizeof(T), reinterpret_cast(&result)); -#endif + copy_or_byteswap(m_data - sizeof(T), &result); return result; } -#if __BYTE_ORDER == __LITTLE_ENDIAN +#ifdef PROTOZERO_USE_BARE_POINTER_FOR_PACKED_FIXED + template inline std::pair packed_fixed() { protozero_assert(tag() != 0 && "call next() before accessing field value"); @@ -128,7 +131,7 @@ class pbf_reader { T operator*() { T result; - byteswap(m_data, reinterpret_cast(&result)); + copy_or_byteswap(m_data , &result); return result; } @@ -161,6 +164,7 @@ class pbf_reader { return std::make_pair(const_fixed_iterator(m_data-len, m_data), const_fixed_iterator(m_data, m_data)); } + #endif template inline T get_varint(); @@ -866,8 +870,16 @@ bool pbf_reader::next() { protozero_assert(((m_tag > 0 && m_tag < 19000) || (m_tag > 19999 && m_tag <= ((1 << 29) - 1))) && "tag out of range"); m_wire_type = pbf_wire_type(value & 0x07); -// XXX do we want this check? or should it throw an exception? -// protozero_assert((m_wire_type <=2 || m_wire_type == 5) && "illegal wire type"); + switch (m_wire_type) { + case pbf_wire_type::varint: + case pbf_wire_type::fixed64: + case pbf_wire_type::length_delimited: + case pbf_wire_type::fixed32: + break; + default: + throw unknown_pbf_wire_type_exception(); + } + return true; } diff --git a/third_party/libosmium/include/protozero/pbf_writer.hpp b/third_party/libosmium/include/protozero/pbf_writer.hpp index 53cbfdf1c..2b78cb889 100644 --- a/third_party/libosmium/include/protozero/pbf_writer.hpp +++ b/third_party/libosmium/include/protozero/pbf_writer.hpp @@ -16,7 +16,6 @@ documentation. * @brief Contains the pbf_writer class. */ -#include #include #include #include @@ -24,18 +23,14 @@ documentation. #include #include +#include #include #include -#if __BYTE_ORDER != __LITTLE_ENDIAN +#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN # include #endif -/// Wrapper for assert() used for testing -#ifndef protozero_assert -# define protozero_assert(x) assert(x) -#endif - namespace protozero { /** @@ -71,7 +66,7 @@ class pbf_writer { inline void add_fixed(T value) { protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); protozero_assert(m_data); -#if __BYTE_ORDER == __LITTLE_ENDIAN +#if PROTOZERO_BYTE_ORDER == PROTOZERO_LITTLE_ENDIAN m_data->append(reinterpret_cast(&value), sizeof(T)); #else auto size = m_data->size(); @@ -229,7 +224,9 @@ public: */ inline void add_bool(pbf_tag_type tag, bool value) { add_field(tag, pbf_wire_type::varint); - add_fixed(value); + protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); + protozero_assert(m_data); + m_data->append(1, value); } /** @@ -378,7 +375,7 @@ public: inline void add_bytes(pbf_tag_type tag, const char* value, size_t size) { protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); protozero_assert(m_data); - assert(size <= std::numeric_limits::max()); + protozero_assert(size <= std::numeric_limits::max()); add_length_varint(tag, pbf_length_type(size)); m_data->append(value, size); } diff --git a/third_party/libosmium/include/protozero/varint.hpp b/third_party/libosmium/include/protozero/varint.hpp index bc9c3296d..27536fd39 100644 --- a/third_party/libosmium/include/protozero/varint.hpp +++ b/third_party/libosmium/include/protozero/varint.hpp @@ -16,10 +16,6 @@ documentation. * @brief Contains low-level varint and zigzag encoding and decoding functions. */ -#if __BYTE_ORDER != __LITTLE_ENDIAN -# error "This code only works on little endian machines." -#endif - #include #include diff --git a/third_party/libosmium/include/protozero/version.hpp b/third_party/libosmium/include/protozero/version.hpp new file mode 100644 index 000000000..4f129acc6 --- /dev/null +++ b/third_party/libosmium/include/protozero/version.hpp @@ -0,0 +1,22 @@ +#ifndef PROTOZERO_VERSION_HPP +#define PROTOZERO_VERSION_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +#define PROTOZERO_VERSION_MAJOR 1 +#define PROTOZERO_VERSION_MINOR 2 +#define PROTOZERO_VERSION_PATCH 3 + +#define PROTOZERO_VERSION_CODE (PROTOZERO_VERSION_MAJOR * 10000 + PROTOZERO_VERSION_MINOR * 100 + PROTOZERO_VERSION_PATCH) + +#define PROTOZERO_VERSION_STRING "1.2.3" + + +#endif // PROTOZERO_VERSION_HPP diff --git a/third_party/libosmium/scripts/travis_install.sh b/third_party/libosmium/scripts/travis_install.sh deleted file mode 100755 index 119e9fd15..000000000 --- a/third_party/libosmium/scripts/travis_install.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh -# -# travis_install.sh -# - -if [ "$TRAVIS_OS_NAME" = "osx" ]; then - - brew install google-sparsehash || true - - brew install --without-python boost || true - - # workaround for gdal homebrew problem - brew remove gdal - brew install gdal - -fi - -cd .. -git clone --quiet --depth 1 https://github.com/osmcode/osm-testdata.git - diff --git a/third_party/libosmium/scripts/travis_script.sh b/third_party/libosmium/scripts/travis_script.sh deleted file mode 100755 index 75b3b3657..000000000 --- a/third_party/libosmium/scripts/travis_script.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -# -# travis_script.sh -# - -mkdir build -cd build - -# GCC ignores the pragmas in the code that disable the "return-type" warning -# selectively, so use this workaround. -if [ "${CXX}" = "g++" ]; then - WORKAROUND="-DCMAKE_CXX_FLAGS=-Wno-return-type" -else - WORKAROUND="" -fi - -if [ "${CXX}" = "g++" ]; then - CXX=g++-4.8 - CC=gcc-4.8 -fi - -cmake -LA \ - -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ - ${WORKAROUND} \ - .. - -make VERBOSE=1 -ctest --output-on-failure - diff --git a/third_party/libosmium/test/CMakeLists.txt b/third_party/libosmium/test/CMakeLists.txt index 00474577a..f57416120 100644 --- a/third_party/libosmium/test/CMakeLists.txt +++ b/third_party/libosmium/test/CMakeLists.txt @@ -106,6 +106,7 @@ add_unit_test(basic test_timestamp) add_unit_test(basic test_types_from_string) add_unit_test(basic test_way) +add_unit_test(buffer test_buffer_basics) add_unit_test(buffer test_buffer_node) add_unit_test(buffer test_buffer_purge) @@ -133,9 +134,15 @@ add_unit_test(index test_id_to_location ENABLE_IF ${SPARSEHASH_FOUND}) add_unit_test(io test_bzip2 ENABLE_IF ${BZIP2_FOUND} LIBS ${BZIP2_LIBRARIES}) add_unit_test(io test_file_formats) -add_unit_test(io test_reader LIBS "${OSMIUM_XML_LIBRARIES}") +add_unit_test(io test_reader LIBS "${OSMIUM_XML_LIBRARIES};${OSMIUM_PBF_LIBRARIES}") +add_unit_test(io test_reader_with_mock_decompression ENABLE_IF ${Threads_FOUND} LIBS ${OSMIUM_XML_LIBRARIES}) +add_unit_test(io test_reader_with_mock_parser ENABLE_IF ${Threads_FOUND} LIBS ${CMAKE_THREAD_LIBS_INIT}) +add_unit_test(io test_output_utils) add_unit_test(io test_output_iterator ENABLE_IF ${Threads_FOUND} LIBS ${CMAKE_THREAD_LIBS_INIT}) add_unit_test(io test_string_table) +add_unit_test(io test_writer ENABLE_IF ${Threads_FOUND} LIBS ${OSMIUM_XML_LIBRARIES}) +add_unit_test(io test_writer_with_mock_compression ENABLE_IF ${Threads_FOUND} LIBS ${OSMIUM_XML_LIBRARIES}) +add_unit_test(io test_writer_with_mock_encoder ENABLE_IF ${Threads_FOUND} LIBS ${OSMIUM_XML_LIBRARIES}) add_unit_test(tags test_filter) add_unit_test(tags test_operators) @@ -144,7 +151,6 @@ add_unit_test(tags test_tag_list) add_unit_test(thread test_pool ENABLE_IF ${Threads_FOUND} LIBS ${CMAKE_THREAD_LIBS_INIT}) add_unit_test(util test_cast_with_assert) -add_unit_test(util test_data_file) add_unit_test(util test_delta) add_unit_test(util test_double) add_unit_test(util test_file) diff --git a/third_party/libosmium/test/data-tests/CMakeLists.txt b/third_party/libosmium/test/data-tests/CMakeLists.txt index 89aead98e..a36c31cfd 100644 --- a/third_party/libosmium/test/data-tests/CMakeLists.txt +++ b/third_party/libosmium/test/data-tests/CMakeLists.txt @@ -88,11 +88,11 @@ set_tests_properties(testdata-overview PROPERTIES # #----------------------------------------------------------------------------- -find_package(Ruby 1.9) +find_program(RUBY ruby) find_package(Gem COMPONENTS json) find_program(SPATIALITE spatialite) -if(RUBY_FOUND AND GEM_json_FOUND AND SPATIALITE) +if(RUBY AND GEM_json_FOUND AND SPATIALITE) add_executable(testdata-multipolygon testdata-multipolygon.cpp) target_link_libraries(testdata-multipolygon ${OSMIUM_XML_LIBRARIES} @@ -102,7 +102,7 @@ if(RUBY_FOUND AND GEM_json_FOUND AND SPATIALITE) add_test(NAME testdata-multipolygon COMMAND ${CMAKE_COMMAND} -D OSM_TESTDATA=${OSM_TESTDATA} - -D RUBY=${RUBY_EXECUTABLE} + -D RUBY=${RUBY} -P ${CMAKE_CURRENT_SOURCE_DIR}/run-testdata-multipolygon.cmake) set_tests_properties(testdata-multipolygon PROPERTIES LABELS "data;slow") diff --git a/third_party/libosmium/test/data-tests/testdata-multipolygon.cpp b/third_party/libosmium/test/data-tests/testdata-multipolygon.cpp index 0fd0d9849..cf4fc521a 100644 --- a/third_party/libosmium/test/data-tests/testdata-multipolygon.cpp +++ b/third_party/libosmium/test/data-tests/testdata-multipolygon.cpp @@ -3,6 +3,8 @@ #include #include +#include + #include #include @@ -41,10 +43,9 @@ inline tagmap_type create_map(const osmium::TagList& taglist) { class TestHandler : public osmium::handler::Handler { - OGRDataSource* m_data_source; - OGRLayer* m_layer_point; - OGRLayer* m_layer_linestring; - OGRLayer* m_layer_polygon; + gdalcpp::Layer m_layer_point; + gdalcpp::Layer m_layer_lines; + gdalcpp::Layer m_layer_mpoly; osmium::geom::OGRFactory<> m_ogr_factory; osmium::geom::WKTFactory<> m_wkt_factory; @@ -55,84 +56,20 @@ class TestHandler : public osmium::handler::Handler { public: - TestHandler(OGRDataSource* data_source) : - m_data_source(data_source), + explicit TestHandler(gdalcpp::Dataset& dataset) : + m_layer_point(dataset, "points", wkbPoint), + m_layer_lines(dataset, "lines", wkbLineString), + m_layer_mpoly(dataset, "multipolygons", wkbMultiPolygon), m_out("multipolygon-tests.json") { - OGRSpatialReference sparef; - sparef.SetWellKnownGeogCS("WGS84"); + m_layer_point.add_field("id", OFTReal, 10); + m_layer_point.add_field("type", OFTString, 30); - /**************/ + m_layer_lines.add_field("id", OFTReal, 10); + m_layer_lines.add_field("type", OFTString, 30); - m_layer_point = m_data_source->CreateLayer("points", &sparef, wkbPoint, nullptr); - if (!m_layer_point) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_point_field_id("id", OFTReal); - layer_point_field_id.SetWidth(10); - - if (m_layer_point->CreateField(&layer_point_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_point_field_type("type", OFTString); - layer_point_field_type.SetWidth(30); - - if (m_layer_point->CreateField(&layer_point_field_type) != OGRERR_NONE) { - std::cerr << "Creating type field failed.\n"; - exit(1); - } - - /**************/ - - m_layer_linestring = m_data_source->CreateLayer("lines", &sparef, wkbLineString, nullptr); - if (!m_layer_linestring) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_linestring_field_id("id", OFTReal); - layer_linestring_field_id.SetWidth(10); - - if (m_layer_linestring->CreateField(&layer_linestring_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_linestring_field_type("type", OFTString); - layer_linestring_field_type.SetWidth(30); - - if (m_layer_linestring->CreateField(&layer_linestring_field_type) != OGRERR_NONE) { - std::cerr << "Creating type field failed.\n"; - exit(1); - } - - /**************/ - - m_layer_polygon = m_data_source->CreateLayer("multipolygons", &sparef, wkbMultiPolygon, nullptr); - if (!m_layer_polygon) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_polygon_field_id("id", OFTInteger); - layer_polygon_field_id.SetWidth(10); - - if (m_layer_polygon->CreateField(&layer_polygon_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_polygon_field_from_type("from_type", OFTString); - layer_polygon_field_from_type.SetWidth(1); - - if (m_layer_polygon->CreateField(&layer_polygon_field_from_type) != OGRERR_NONE) { - std::cerr << "Creating from_type field failed.\n"; - exit(1); - } + m_layer_mpoly.add_field("id", OFTReal, 10); + m_layer_mpoly.add_field("from_type", OFTString, 1); } ~TestHandler() { @@ -140,34 +77,18 @@ public: } void node(const osmium::Node& node) { - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_point->GetLayerDefn()); - std::unique_ptr ogr_point = m_ogr_factory.create_point(node); - feature->SetGeometry(ogr_point.get()); - feature->SetField("id", static_cast(node.id())); - feature->SetField("type", node.tags().get_value_by_key("type")); - - if (m_layer_point->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); + gdalcpp::Feature feature(m_layer_point, m_ogr_factory.create_point(node)); + feature.set_field("id", static_cast(node.id())); + feature.set_field("type", node.tags().get_value_by_key("type")); + feature.add_to_layer(); } void way(const osmium::Way& way) { try { - std::unique_ptr ogr_linestring = m_ogr_factory.create_linestring(way); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_linestring->GetLayerDefn()); - feature->SetGeometry(ogr_linestring.get()); - feature->SetField("id", static_cast(way.id())); - feature->SetField("type", way.tags().get_value_by_key("type")); - - if (m_layer_linestring->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); + gdalcpp::Feature feature(m_layer_lines, m_ogr_factory.create_linestring(way)); + feature.set_field("id", static_cast(way.id())); + feature.set_field("type", way.tags().get_value_by_key("type")); + feature.add_to_layer(); } catch (osmium::geometry_error&) { std::cerr << "Ignoring illegal geometry for way " << way.id() << ".\n"; } @@ -200,10 +121,8 @@ public: m_out << "INVALID\"\n}"; } try { - std::unique_ptr ogr_polygon = m_ogr_factory.create_multipolygon(area); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_polygon->GetLayerDefn()); - feature->SetGeometry(ogr_polygon.get()); - feature->SetField("id", static_cast(area.orig_id())); + gdalcpp::Feature feature(m_layer_mpoly, m_ogr_factory.create_multipolygon(area)); + feature.set_field("id", static_cast(area.orig_id())); std::string from_type; if (area.from_way()) { @@ -211,14 +130,8 @@ public: } else { from_type = "r"; } - feature->SetField("from_type", from_type.c_str()); - - if (m_layer_polygon->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); + feature.set_field("from_type", from_type.c_str()); + feature.add_to_layer(); } catch (osmium::geometry_error&) { std::cerr << "Ignoring illegal geometry for area " << area.id() << " created from " << (area.from_way() ? "way" : "relation") << " with id=" << area.orig_id() << ".\n"; } @@ -228,26 +141,6 @@ public: /* ================================================== */ -OGRDataSource* initialize_database(const std::string& output_format, const std::string& output_filename) { - OGRRegisterAll(); - - OGRSFDriver* driver = OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(output_format.c_str()); - if (!driver) { - std::cerr << output_format << " driver not available.\n"; - exit(1); - } - - CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "FALSE"); - const char* options[] = { "SPATIALITE=TRUE", nullptr }; - OGRDataSource* data_source = driver->CreateDataSource(output_filename.c_str(), const_cast(options)); - if (!data_source) { - std::cerr << "Creation of output file failed.\n"; - exit(1); - } - - return data_source; -} - int main(int argc, char* argv[]) { if (argc != 2) { std::cerr << "Usage: " << argv[0] << " INFILE\n"; @@ -258,9 +151,10 @@ int main(int argc, char* argv[]) { std::string input_filename(argv[1]); std::string output_filename("multipolygon.db"); - OGRDataSource* data_source = initialize_database(output_format, output_filename); + CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "FALSE"); + gdalcpp::Dataset dataset{output_format, output_filename, gdalcpp::SRS{}, { "SPATIALITE=TRUE" }}; - osmium::area::ProblemReporterOGR problem_reporter(data_source); + osmium::area::ProblemReporterOGR problem_reporter(dataset); osmium::area::Assembler::config_type assembler_config(&problem_reporter); assembler_config.enable_debug_output(); osmium::area::MultipolygonCollector collector(assembler_config); @@ -275,7 +169,7 @@ int main(int argc, char* argv[]) { location_handler_type location_handler(index); location_handler.ignore_errors(); - TestHandler test_handler(data_source); + TestHandler test_handler(dataset); std::cerr << "Pass 2...\n"; osmium::io::Reader reader2(input_filename); @@ -284,8 +178,5 @@ int main(int argc, char* argv[]) { })); reader2.close(); std::cerr << "Pass 2 done\n"; - - OGRDataSource::DestroyDataSource(data_source); - OGRCleanupAll(); } diff --git a/third_party/libosmium/test/data-tests/testdata-overview.cpp b/third_party/libosmium/test/data-tests/testdata-overview.cpp index 2d63dc684..43d672dd1 100644 --- a/third_party/libosmium/test/data-tests/testdata-overview.cpp +++ b/third_party/libosmium/test/data-tests/testdata-overview.cpp @@ -2,6 +2,8 @@ #include +#include + #include #include @@ -15,154 +17,53 @@ typedef osmium::handler::NodeLocationsForWays location_handler_type; class TestOverviewHandler : public osmium::handler::Handler { - OGRDataSource* m_data_source; - - OGRLayer* m_layer_nodes; - OGRLayer* m_layer_labels; - OGRLayer* m_layer_ways; + gdalcpp::Layer m_layer_nodes; + gdalcpp::Layer m_layer_labels; + gdalcpp::Layer m_layer_ways; osmium::geom::OGRFactory<> m_factory; public: - TestOverviewHandler(const std::string& driver_name, const std::string& filename) { + explicit TestOverviewHandler(gdalcpp::Dataset& dataset) : + m_layer_nodes(dataset, "nodes", wkbPoint), + m_layer_labels(dataset, "labels", wkbPoint), + m_layer_ways(dataset, "ways", wkbLineString) { - OGRRegisterAll(); + m_layer_nodes.add_field("id", OFTReal, 10); - OGRSFDriver* driver = OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(driver_name.c_str()); - if (!driver) { - std::cerr << driver_name << " driver not available.\n"; - exit(1); - } + m_layer_labels.add_field("id", OFTReal, 10); + m_layer_labels.add_field("label", OFTString, 30); - CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "FALSE"); - const char* options[] = { "SPATIALITE=TRUE", nullptr }; - m_data_source = driver->CreateDataSource(filename.c_str(), const_cast(options)); - if (!m_data_source) { - std::cerr << "Creation of output file failed.\n"; - exit(1); - } - - OGRSpatialReference sparef; - sparef.SetWellKnownGeogCS("WGS84"); - - // nodes layer - - m_layer_nodes = m_data_source->CreateLayer("nodes", &sparef, wkbPoint, nullptr); - if (!m_layer_nodes) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_nodes_field_id("id", OFTReal); - layer_nodes_field_id.SetWidth(10); - - if (m_layer_nodes->CreateField(&layer_nodes_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - // labels layer - - m_layer_labels = m_data_source->CreateLayer("labels", &sparef, wkbPoint, nullptr); - if (!m_layer_labels) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_labels_field_id("id", OFTReal); - layer_labels_field_id.SetWidth(10); - - if (m_layer_labels->CreateField(&layer_labels_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_labels_field_label("label", OFTString); - layer_labels_field_label.SetWidth(30); - - if (m_layer_labels->CreateField(&layer_labels_field_label) != OGRERR_NONE) { - std::cerr << "Creating label field failed.\n"; - exit(1); - } - - // ways layer - - m_layer_ways = m_data_source->CreateLayer("ways", &sparef, wkbLineString, nullptr); - if (!m_layer_ways) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_way_field_id("id", OFTReal); - layer_way_field_id.SetWidth(10); - - if (m_layer_ways->CreateField(&layer_way_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_way_field_test("test", OFTInteger); - layer_way_field_test.SetWidth(3); - - if (m_layer_ways->CreateField(&layer_way_field_test) != OGRERR_NONE) { - std::cerr << "Creating test field failed.\n"; - exit(1); - } - } - - ~TestOverviewHandler() { - OGRDataSource::DestroyDataSource(m_data_source); - OGRCleanupAll(); + m_layer_ways.add_field("id", OFTReal, 10); + m_layer_ways.add_field("test", OFTInteger, 3); } void node(const osmium::Node& node) { const char* label = node.tags().get_value_by_key("label"); if (label) { - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_labels->GetLayerDefn()); - std::unique_ptr ogr_point = m_factory.create_point(node); - feature->SetGeometry(ogr_point.get()); - feature->SetField("id", static_cast(node.id())); - feature->SetField("label", label); - - if (m_layer_labels->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); + gdalcpp::Feature feature(m_layer_labels, m_factory.create_point(node)); + feature.set_field("id", static_cast(node.id())); + feature.set_field("label", label); + feature.add_to_layer(); } else { - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_nodes->GetLayerDefn()); - std::unique_ptr ogr_point = m_factory.create_point(node); - feature->SetGeometry(ogr_point.get()); - feature->SetField("id", static_cast(node.id())); - - if (m_layer_nodes->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - OGRFeature::DestroyFeature(feature); + gdalcpp::Feature feature(m_layer_nodes, m_factory.create_point(node)); + feature.set_field("id", static_cast(node.id())); + feature.add_to_layer(); } } void way(const osmium::Way& way) { try { - std::unique_ptr ogr_linestring = m_factory.create_linestring(way); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_ways->GetLayerDefn()); - feature->SetGeometry(ogr_linestring.get()); - feature->SetField("id", static_cast(way.id())); + gdalcpp::Feature feature(m_layer_ways, m_factory.create_linestring(way)); + feature.set_field("id", static_cast(way.id())); const char* test = way.tags().get_value_by_key("test"); if (test) { - feature->SetField("test", test); + feature.set_field("test", test); } - if (m_layer_ways->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); + feature.add_to_layer(); } catch (osmium::geometry_error&) { std::cerr << "Ignoring illegal geometry for way " << way.id() << ".\n"; } @@ -183,13 +84,16 @@ int main(int argc, char* argv[]) { std::string output_filename("testdata-overview.db"); ::unlink(output_filename.c_str()); + CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "FALSE"); + gdalcpp::Dataset dataset(output_format, output_filename, gdalcpp::SRS{}, { "SPATIALITE=TRUE" }); + osmium::io::Reader reader(input_filename); index_type index; location_handler_type location_handler(index); location_handler.ignore_errors(); - TestOverviewHandler handler(output_format, output_filename); + TestOverviewHandler handler(dataset); osmium::apply(reader, location_handler, handler); reader.close(); diff --git a/third_party/libosmium/test/data-tests/testdata-xml.cpp b/third_party/libosmium/test/data-tests/testdata-xml.cpp index 8102759d1..b5a0e9028 100644 --- a/third_party/libosmium/test/data-tests/testdata-xml.cpp +++ b/third_party/libosmium/test/data-tests/testdata-xml.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -72,24 +73,27 @@ std::string read_gz_file(const char* test_id, const char* suffix) { header_buffer_type parse_xml(std::string input) { - osmium::thread::Queue input_queue; - osmium::thread::Queue output_queue; + osmium::io::detail::future_string_queue_type input_queue; + osmium::io::detail::future_buffer_queue_type output_queue; std::promise header_promise; - std::atomic done {false}; - input_queue.push(input); - input_queue.push(std::string()); // EOF marker + std::future header_future = header_promise.get_future(); - osmium::io::detail::XMLParser parser(input_queue, output_queue, header_promise, osmium::osm_entity_bits::all, done); - parser(); + osmium::io::detail::add_to_queue(input_queue, std::move(input)); + osmium::io::detail::add_to_queue(input_queue, std::string{}); + + osmium::io::detail::XMLParser parser(input_queue, output_queue, header_promise, osmium::osm_entity_bits::all); + parser.parse(); header_buffer_type result; - result.header = header_promise.get_future().get(); - output_queue.wait_and_pop(result.buffer); + result.header = header_future.get(); + std::future future_buffer; + output_queue.wait_and_pop(future_buffer); + result.buffer = future_buffer.get(); if (result.buffer) { - osmium::memory::Buffer buffer; - output_queue.wait_and_pop(buffer); - assert(!buffer); + std::future future_buffer2; + output_queue.wait_and_pop(future_buffer2); + assert(!future_buffer2.get()); } return result; @@ -534,9 +538,10 @@ TEST_CASE("Reading OSM XML 200") { osmium::io::Header header = reader.header(); REQUIRE(header.get("generator") == "testdata"); - osmium::memory::Buffer buffer = reader.read(); - REQUIRE(0 == buffer.committed()); - REQUIRE(! buffer); + REQUIRE_THROWS({ + reader.read(); + }); + reader.close(); } diff --git a/third_party/libosmium/test/t/area/test_node_ref_segment.cpp b/third_party/libosmium/test/t/area/test_node_ref_segment.cpp index 309768744..3261c24e1 100644 --- a/third_party/libosmium/test/t/area/test_node_ref_segment.cpp +++ b/third_party/libosmium/test/t/area/test_node_ref_segment.cpp @@ -52,6 +52,20 @@ TEST_CASE("NodeRefSegmentClass") { REQUIRE(calculate_intersection(s1, s7) == osmium::Location()); } + SECTION("intersection of very long segments") { + NodeRefSegment s1({ 1, {90.0, 90.0}}, { 2, {-90.0, -90.0}}, nullptr, nullptr); + NodeRefSegment s2({ 1, {-90.0, 90.0}}, { 2, {90.0, -90.0}}, nullptr, nullptr); + REQUIRE(calculate_intersection(s1, s2) == osmium::Location(0.0, 0.0)); + + NodeRefSegment s3({ 1, {-90.0, -90.0}}, { 2, {90.0, 90.0}}, nullptr, nullptr); + NodeRefSegment s4({ 1, {-90.0, 90.0}}, { 2, {90.0, -90.0}}, nullptr, nullptr); + REQUIRE(calculate_intersection(s3, s4) == osmium::Location(0.0, 0.0)); + + NodeRefSegment s5({ 1, {-90.0000001, -90.0}}, { 2, {90.0, 90.0}}, nullptr, nullptr); + NodeRefSegment s6({ 1, {-90.0, 90.0}}, { 2, {90.0, -90.0}}, nullptr, nullptr); + REQUIRE(calculate_intersection(s5, s6) == osmium::Location(0.0, 0.0)); + } + SECTION("to_left_of") { osmium::Location loc { 2.0, 2.0 }; diff --git a/third_party/libosmium/test/t/basic/test_changeset.cpp b/third_party/libosmium/test/t/basic/test_changeset.cpp index fc9f1bdee..d1f3fde87 100644 --- a/third_party/libosmium/test/t/basic/test_changeset.cpp +++ b/third_party/libosmium/test/t/basic/test_changeset.cpp @@ -21,11 +21,13 @@ TEST_CASE("Basic Changeset") { .set_created_at(100) .set_closed_at(200) .set_num_changes(7) + .set_num_comments(3) .set_uid(9); REQUIRE(42 == cs1.id()); REQUIRE(9 == cs1.uid()); REQUIRE(7 == cs1.num_changes()); + REQUIRE(3 == cs1.num_comments()); REQUIRE(true == cs1.closed()); REQUIRE(osmium::Timestamp(100) == cs1.created_at()); REQUIRE(osmium::Timestamp(200) == cs1.closed_at()); @@ -33,7 +35,7 @@ TEST_CASE("Basic Changeset") { REQUIRE(std::string("user") == cs1.user()); crc32.update(cs1); - REQUIRE(crc32().checksum() == 0xf44aff25); + REQUIRE(crc32().checksum() == 0x502e8c0e); osmium::Changeset& cs2 = buffer_add_changeset(buffer, "user", @@ -42,11 +44,13 @@ TEST_CASE("Basic Changeset") { cs2.set_id(43) .set_created_at(120) .set_num_changes(21) + .set_num_comments(osmium::num_comments_type(0)) .set_uid(9); REQUIRE(43 == cs2.id()); REQUIRE(9 == cs2.uid()); REQUIRE(21 == cs2.num_changes()); + REQUIRE(0 == cs2.num_comments()); REQUIRE(false == cs2.closed()); REQUIRE(osmium::Timestamp(120) == cs2.created_at()); REQUIRE(osmium::Timestamp() == cs2.closed_at()); @@ -61,3 +65,59 @@ TEST_CASE("Basic Changeset") { REQUIRE(false == (cs1 >= cs2)); } + +TEST_CASE("Create changeset without helper") { + osmium::memory::Buffer buffer(10 * 1000); + osmium::builder::ChangesetBuilder builder(buffer); + + osmium::Changeset& cs1 = builder.object(); + cs1.set_id(42) + .set_created_at(100) + .set_closed_at(200) + .set_num_changes(7) + .set_num_comments(2) + .set_uid(9); + + builder.add_user("user"); + add_tags(buffer, builder, { + {"key1", "val1"}, + {"key2", "val2"} + }); + + { + osmium::builder::ChangesetDiscussionBuilder disc_builder(buffer, &builder); + disc_builder.add_comment(osmium::Timestamp(300), 10, "user2"); + disc_builder.add_comment_text("foo"); + disc_builder.add_comment(osmium::Timestamp(400), 9, "user"); + disc_builder.add_comment_text("bar"); + } + + buffer.commit(); + + REQUIRE(42 == cs1.id()); + REQUIRE(9 == cs1.uid()); + REQUIRE(7 == cs1.num_changes()); + REQUIRE(2 == cs1.num_comments()); + REQUIRE(true == cs1.closed()); + REQUIRE(osmium::Timestamp(100) == cs1.created_at()); + REQUIRE(osmium::Timestamp(200) == cs1.closed_at()); + REQUIRE(2 == cs1.tags().size()); + REQUIRE(std::string("user") == cs1.user()); + + auto cit = cs1.discussion().begin(); + + REQUIRE(cit != cs1.discussion().end()); + REQUIRE(cit->date() == osmium::Timestamp(300)); + REQUIRE(cit->uid() == 10); + REQUIRE(std::string("user2") == cit->user()); + REQUIRE(std::string("foo") == cit->text()); + + REQUIRE(++cit != cs1.discussion().end()); + REQUIRE(cit->date() == osmium::Timestamp(400)); + REQUIRE(cit->uid() == 9); + REQUIRE(std::string("user") == cit->user()); + REQUIRE(std::string("bar") == cit->text()); + + REQUIRE(++cit == cs1.discussion().end()); +} + diff --git a/third_party/libosmium/test/t/basic/test_crc.cpp b/third_party/libosmium/test/t/basic/test_crc.cpp index aab1013f4..fcd50a13c 100644 --- a/third_party/libosmium/test/t/basic/test_crc.cpp +++ b/third_party/libosmium/test/t/basic/test_crc.cpp @@ -24,6 +24,27 @@ TEST_CASE("CRC of basic datatypes") { REQUIRE(crc32().checksum() == 0x8fe62899); } + SECTION("Int16") { + crc32.update_int16(0x0123U); + crc32.update_int16(0x1234U); + + REQUIRE(crc32().checksum() == 0xda923744); + } + + SECTION("Int32") { + crc32.update_int32(0x01234567UL); + crc32.update_int32(0x12345678UL); + + REQUIRE(crc32().checksum() == 0x9b4e2af3); + } + + SECTION("Int64") { + crc32.update_int64(0x0123456789abcdefULL); + crc32.update_int64(0x123456789abcdef0ULL); + + REQUIRE(crc32().checksum() == 0x6d8b7267); + } + SECTION("String") { const char* str = "foobar"; crc32.update_string(str); diff --git a/third_party/libosmium/test/t/basic/test_node.cpp b/third_party/libosmium/test/t/basic/test_node.cpp index db5b4cd53..9f8b181fe 100644 --- a/third_party/libosmium/test/t/basic/test_node.cpp +++ b/third_party/libosmium/test/t/basic/test_node.cpp @@ -37,12 +37,12 @@ SECTION("node_builder") { REQUIRE(333 == node.changeset()); REQUIRE(21 == node.uid()); REQUIRE(std::string("foo") == node.user()); - REQUIRE(123 == node.timestamp()); + REQUIRE(123 == uint32_t(node.timestamp())); REQUIRE(osmium::Location(3.5, 4.7) == node.location()); REQUIRE(2 == node.tags().size()); crc32.update(node); - REQUIRE(crc32().checksum() == 0xc696802f); + REQUIRE(crc32().checksum() == 0x7dc553f9); node.set_visible(false); REQUIRE(false == node.visible()); @@ -61,7 +61,7 @@ SECTION("node_default_attributes") { REQUIRE(0 == node.changeset()); REQUIRE(0 == node.uid()); REQUIRE(std::string("") == node.user()); - REQUIRE(0 == node.timestamp()); + REQUIRE(0 == uint32_t(node.timestamp())); REQUIRE(osmium::Location() == node.location()); REQUIRE(0 == node.tags().size()); } diff --git a/third_party/libosmium/test/t/basic/test_object_comparisons.cpp b/third_party/libosmium/test/t/basic/test_object_comparisons.cpp index 2bfdcad90..ec9e6fae8 100644 --- a/third_party/libosmium/test/t/basic/test_object_comparisons.cpp +++ b/third_party/libosmium/test/t/basic/test_object_comparisons.cpp @@ -31,20 +31,20 @@ TEST_CASE("Object_Comparisons") { node1.set_version(1); node2.set_id(15); node2.set_version(2); - REQUIRE(true == (node1 < node2)); - REQUIRE(false == (node1 > node2)); + REQUIRE(node1 < node2); + REQUIRE_FALSE(node1 > node2); node1.set_id(20); node1.set_version(1); node2.set_id(20); node2.set_version(2); - REQUIRE(true == (node1 < node2)); - REQUIRE(false == (node1 > node2)); + REQUIRE(node1 < node2); + REQUIRE_FALSE(node1 > node2); node1.set_id(-10); node1.set_version(2); node2.set_id(-15); node2.set_version(1); - REQUIRE(true == (node1 < node2)); - REQUIRE(false == (node1 > node2)); + REQUIRE(node1 < node2); + REQUIRE_FALSE(node1 > node2); } SECTION("order_types") { @@ -122,26 +122,26 @@ TEST_CASE("Object_Comparisons") { const osmium::Way& way = static_cast(*(++it)); const osmium::Relation& relation = static_cast(*(++it)); - REQUIRE(true == (node1 < node2)); - REQUIRE(true == (node2 < way)); - REQUIRE(false == (node2 > way)); - REQUIRE(true == (way < relation)); - REQUIRE(true == (node1 < relation)); + REQUIRE(node1 < node2); + REQUIRE(node2 < way); + REQUIRE_FALSE(node2 > way); + REQUIRE(way < relation); + REQUIRE(node1 < relation); - REQUIRE(true == osmium::object_order_type_id_version()(node1, node2)); - REQUIRE(true == osmium::object_order_type_id_reverse_version()(node2, node1)); - REQUIRE(true == osmium::object_order_type_id_version()(node1, way)); - REQUIRE(true == osmium::object_order_type_id_reverse_version()(node1, way)); + REQUIRE(osmium::object_order_type_id_version()(node1, node2)); + REQUIRE(osmium::object_order_type_id_reverse_version()(node2, node1)); + REQUIRE(osmium::object_order_type_id_version()(node1, way)); + REQUIRE(osmium::object_order_type_id_reverse_version()(node1, way)); - REQUIRE(false == osmium::object_equal_type_id_version()(node1, node2)); - REQUIRE(true == osmium::object_equal_type_id_version()(node2, node3)); + REQUIRE_FALSE(osmium::object_equal_type_id_version()(node1, node2)); + REQUIRE(osmium::object_equal_type_id_version()(node2, node3)); - REQUIRE(true == osmium::object_equal_type_id()(node1, node2)); - REQUIRE(true == osmium::object_equal_type_id()(node2, node3)); + REQUIRE(osmium::object_equal_type_id()(node1, node2)); + REQUIRE(osmium::object_equal_type_id()(node2, node3)); - REQUIRE(false == osmium::object_equal_type_id_version()(node1, way)); - REQUIRE(false == osmium::object_equal_type_id_version()(node1, relation)); - REQUIRE(false == osmium::object_equal_type_id()(node1, relation)); + REQUIRE_FALSE(osmium::object_equal_type_id_version()(node1, way)); + REQUIRE_FALSE(osmium::object_equal_type_id_version()(node1, relation)); + REQUIRE_FALSE(osmium::object_equal_type_id()(node1, relation)); } } diff --git a/third_party/libosmium/test/t/basic/test_relation.cpp b/third_party/libosmium/test/t/basic/test_relation.cpp index fd5c7b4ad..66b201c90 100644 --- a/third_party/libosmium/test/t/basic/test_relation.cpp +++ b/third_party/libosmium/test/t/basic/test_relation.cpp @@ -36,7 +36,7 @@ TEST_CASE("Build relation") { REQUIRE(333 == relation.changeset()); REQUIRE(21 == relation.uid()); REQUIRE(std::string("foo") == relation.user()); - REQUIRE(123 == relation.timestamp()); + REQUIRE(123 == uint32_t(relation.timestamp())); REQUIRE(2 == relation.tags().size()); REQUIRE(3 == relation.members().size()); @@ -61,5 +61,16 @@ TEST_CASE("Build relation") { } crc32.update(relation); - REQUIRE(crc32().checksum() == 0xebcd836d); + REQUIRE(crc32().checksum() == 0x2c2352e); } + +TEST_CASE("Member role too long") { + osmium::memory::Buffer buffer(10000); + + osmium::builder::RelationMemberListBuilder builder(buffer); + + const char role[2000] = ""; + builder.add_member(osmium::item_type::node, 1, role, 1024); + REQUIRE_THROWS(builder.add_member(osmium::item_type::node, 1, role, 1025)); +} + diff --git a/third_party/libosmium/test/t/basic/test_timestamp.cpp b/third_party/libosmium/test/t/basic/test_timestamp.cpp index f015730ea..f80ffcf5f 100644 --- a/third_party/libosmium/test/t/basic/test_timestamp.cpp +++ b/third_party/libosmium/test/t/basic/test_timestamp.cpp @@ -8,34 +8,45 @@ TEST_CASE("Timestamp") { SECTION("can be default initialized to invalid value") { osmium::Timestamp t; - REQUIRE(0 == t); + REQUIRE(0 == uint32_t(t)); REQUIRE("" == t.to_iso()); + REQUIRE_FALSE(t.valid()); } SECTION("invalid value is zero") { osmium::Timestamp t(static_cast(0)); - REQUIRE(0 == t); + REQUIRE(0 == uint32_t(t)); REQUIRE("" == t.to_iso()); + REQUIRE_FALSE(t.valid()); } SECTION("can be initialized from time_t") { osmium::Timestamp t(static_cast(1)); - REQUIRE(1 == t); + REQUIRE(1 == uint32_t(t)); REQUIRE("1970-01-01T00:00:01Z" == t.to_iso()); + REQUIRE(t.valid()); + } + + SECTION("can be initialized from const char*") { + osmium::Timestamp t("2000-01-01T00:00:00Z"); + REQUIRE("2000-01-01T00:00:00Z" == t.to_iso()); + REQUIRE(t.valid()); } SECTION("can be initialized from string") { - osmium::Timestamp t("2000-01-01T00:00:00Z"); + std::string s = "2000-01-01T00:00:00Z"; + osmium::Timestamp t(s); REQUIRE("2000-01-01T00:00:00Z" == t.to_iso()); + REQUIRE(t.valid()); } SECTION("throws if initialized from bad string") { REQUIRE_THROWS_AS(osmium::Timestamp("x"), std::invalid_argument); } - SECTION("can be implicitly cast to time_t") { + SECTION("can be explicitly cast to time_t") { osmium::Timestamp t(4242); - time_t x = t; + time_t x = t.seconds_since_epoch(); REQUIRE(x == 4242); } @@ -50,6 +61,10 @@ TEST_CASE("Timestamp") { osmium::Timestamp t1(10); osmium::Timestamp t2(50); REQUIRE(t1 < t2); + REQUIRE(t1 > osmium::start_of_time()); + REQUIRE(t2 > osmium::start_of_time()); + REQUIRE(t1 < osmium::end_of_time()); + REQUIRE(t2 < osmium::end_of_time()); } SECTION("can be written to stream") { diff --git a/third_party/libosmium/test/t/basic/test_way.cpp b/third_party/libosmium/test/t/basic/test_way.cpp index 7c7bc2148..370cd0105 100644 --- a/third_party/libosmium/test/t/basic/test_way.cpp +++ b/third_party/libosmium/test/t/basic/test_way.cpp @@ -36,7 +36,7 @@ SECTION("way_builder") { REQUIRE(333 == way.changeset()); REQUIRE(21 == way.uid()); REQUIRE(std::string("foo") == way.user()); - REQUIRE(123 == way.timestamp()); + REQUIRE(123 == uint32_t(way.timestamp())); REQUIRE(2 == way.tags().size()); REQUIRE(3 == way.nodes().size()); REQUIRE(1 == way.nodes()[0].ref()); @@ -45,7 +45,7 @@ SECTION("way_builder") { REQUIRE(! way.is_closed()); crc32.update(way); - REQUIRE(crc32().checksum() == 0x20fe7a30); + REQUIRE(crc32().checksum() == 0x7676d0c2); } SECTION("closed_way") { diff --git a/third_party/libosmium/test/t/buffer/test_buffer_basics.cpp b/third_party/libosmium/test/t/buffer/test_buffer_basics.cpp new file mode 100644 index 000000000..ffe725168 --- /dev/null +++ b/third_party/libosmium/test/t/buffer/test_buffer_basics.cpp @@ -0,0 +1,34 @@ +#include "catch.hpp" + +#include + +TEST_CASE("Buffer basics") { + + osmium::memory::Buffer invalid_buffer1; + osmium::memory::Buffer invalid_buffer2; + osmium::memory::Buffer empty_buffer1(1024); + osmium::memory::Buffer empty_buffer2(2048); + + REQUIRE(!invalid_buffer1); + REQUIRE(!invalid_buffer2); + REQUIRE(empty_buffer1); + REQUIRE(empty_buffer2); + + REQUIRE(invalid_buffer1 == invalid_buffer2); + REQUIRE(invalid_buffer1 != empty_buffer1); + REQUIRE(empty_buffer1 != empty_buffer2); + + REQUIRE(invalid_buffer1.capacity() == 0); + REQUIRE(invalid_buffer1.written() == 0); + REQUIRE(invalid_buffer1.committed() == 0); + + REQUIRE(empty_buffer1.capacity() == 1024); + REQUIRE(empty_buffer1.written() == 0); + REQUIRE(empty_buffer1.committed() == 0); + + REQUIRE(empty_buffer2.capacity() == 2048); + REQUIRE(empty_buffer2.written() == 0); + REQUIRE(empty_buffer2.committed() == 0); + +} + diff --git a/third_party/libosmium/test/t/buffer/test_buffer_node.cpp b/third_party/libosmium/test/t/buffer/test_buffer_node.cpp index 9bc8f701d..ba2431b80 100644 --- a/third_party/libosmium/test/t/buffer/test_buffer_node.cpp +++ b/third_party/libosmium/test/t/buffer/test_buffer_node.cpp @@ -9,7 +9,7 @@ void check_node_1(osmium::Node& node) { REQUIRE(true == node.visible()); REQUIRE(333 == node.changeset()); REQUIRE(21 == node.uid()); - REQUIRE(123 == node.timestamp()); + REQUIRE(123 == uint32_t(node.timestamp())); REQUIRE(osmium::Location(3.5, 4.7) == node.location()); REQUIRE(std::string("testuser") == node.user()); @@ -28,7 +28,7 @@ void check_node_2(osmium::Node& node) { REQUIRE(true == node.visible()); REQUIRE(333 == node.changeset()); REQUIRE(21 == node.uid()); - REQUIRE(123 == node.timestamp()); + REQUIRE(123 == uint32_t(node.timestamp())); REQUIRE(osmium::Location(3.5, 4.7) == node.location()); REQUIRE(std::string("testuser") == node.user()); @@ -56,13 +56,14 @@ void check_node_2(osmium::Node& node) { REQUIRE(2 == n); } -TEST_CASE("Buffer_Node") { +TEST_CASE("Node in Buffer") { - SECTION("buffer_node") { - constexpr size_t buffer_size = 10000; - unsigned char data[buffer_size]; + constexpr size_t buffer_size = 10000; + unsigned char data[buffer_size]; - osmium::memory::Buffer buffer(data, buffer_size, 0); + osmium::memory::Buffer buffer(data, buffer_size, 0); + + SECTION("Add node to buffer") { { // add node 1 @@ -132,4 +133,67 @@ TEST_CASE("Buffer_Node") { } + SECTION("Add buffer to another one") { + + { + // add node 1 + osmium::builder::NodeBuilder node_builder(buffer); + osmium::Node& node = node_builder.object(); + REQUIRE(osmium::item_type::node == node.type()); + + node.set_id(1); + node.set_version(3); + node.set_visible(true); + node.set_changeset(333); + node.set_uid(21); + node.set_timestamp(123); + node.set_location(osmium::Location(3.5, 4.7)); + + node_builder.add_user("testuser"); + + buffer.commit(); + } + + osmium::memory::Buffer buffer2(buffer_size, osmium::memory::Buffer::auto_grow::yes); + + buffer2.add_buffer(buffer); + buffer2.commit(); + + REQUIRE(buffer.committed() == buffer2.committed()); + const osmium::Node& node = buffer2.get(0); + REQUIRE(node.id() == 1); + REQUIRE(123 == uint32_t(node.timestamp())); + } + + SECTION("Use back_inserter on buffer") { + + { + // add node 1 + osmium::builder::NodeBuilder node_builder(buffer); + osmium::Node& node = node_builder.object(); + REQUIRE(osmium::item_type::node == node.type()); + + node.set_id(1); + node.set_version(3); + node.set_visible(true); + node.set_changeset(333); + node.set_uid(21); + node.set_timestamp(123); + node.set_location(osmium::Location(3.5, 4.7)); + + node_builder.add_user("testuser"); + + buffer.commit(); + } + + osmium::memory::Buffer buffer2(buffer_size, osmium::memory::Buffer::auto_grow::yes); + + std::copy(buffer.begin(), buffer.end(), std::back_inserter(buffer2)); + + REQUIRE(buffer.committed() == buffer2.committed()); + const osmium::Node& node = buffer2.get(0); + REQUIRE(node.id() == 1); + REQUIRE(123 == uint32_t(node.timestamp())); + } } + diff --git a/third_party/libosmium/test/t/geom/test_tile_data.hpp b/third_party/libosmium/test/t/geom/test_tile_data.hpp index e5c0953c0..8a22cfa51 100644 --- a/third_party/libosmium/test/t/geom/test_tile_data.hpp +++ b/third_party/libosmium/test/t/geom/test_tile_data.hpp @@ -1,5 +1,7 @@ -std::string s = R"(127.4864358 16.8380041 223904 118630 18 +#include + +static std::string s = R"(127.4864358 16.8380041 223904 118630 18 163.1103174 39.4760232 121 48 7 -4.1372725 -22.5105386 31 36 6 98.7193066 -36.2312406 1 1 1 diff --git a/third_party/libosmium/test/t/io/deleted_nodes.osh b/third_party/libosmium/test/t/io/deleted_nodes.osh new file mode 100644 index 000000000..639e05149 --- /dev/null +++ b/third_party/libosmium/test/t/io/deleted_nodes.osh @@ -0,0 +1,5 @@ + + + + + diff --git a/third_party/libosmium/test/t/io/deleted_nodes.osh.pbf b/third_party/libosmium/test/t/io/deleted_nodes.osh.pbf new file mode 100644 index 000000000..8a94870d1 Binary files /dev/null and b/third_party/libosmium/test/t/io/deleted_nodes.osh.pbf differ diff --git a/third_party/libosmium/test/t/io/test_output_utils.cpp b/third_party/libosmium/test/t/io/test_output_utils.cpp new file mode 100644 index 000000000..a76068348 --- /dev/null +++ b/third_party/libosmium/test/t/io/test_output_utils.cpp @@ -0,0 +1,153 @@ + +#include "catch.hpp" + +#include + +#include + +TEST_CASE("output formatted") { + + std::string out; + + SECTION("small results") { + osmium::io::detail::append_printf_formatted_string(out, "%d", 17); + REQUIRE(out == "17"); + } + + SECTION("several parameters") { + osmium::io::detail::append_printf_formatted_string(out, "%d %s", 17, "foo"); + REQUIRE(out == "17 foo"); + + } + + SECTION("string already containing something") { + out += "foo"; + osmium::io::detail::append_printf_formatted_string(out, " %d", 23); + REQUIRE(out == "foo 23"); + } + + SECTION("large results") { + const char* str = + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"; + + osmium::io::detail::append_printf_formatted_string(out, "%s", str); + + REQUIRE(out == str); + } + +} + +TEST_CASE("UTF8 encoding") { + + std::string out; + + SECTION("append to string") { + out += "1234"; + osmium::io::detail::append_utf8_encoded_string(out, "abc"); + REQUIRE(out == "1234abc"); + } + + SECTION("don't encode alphabetic characters") { + const char* s = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + osmium::io::detail::append_utf8_encoded_string(out, s); + REQUIRE(out == s); + } + + SECTION("don't encode numeric characters") { + const char* s = "0123456789"; + osmium::io::detail::append_utf8_encoded_string(out, s); + REQUIRE(out == s); + } + + SECTION("don't encode lots of often used characters characters") { + const char* s = ".-;:_#+"; + osmium::io::detail::append_utf8_encoded_string(out, s); + REQUIRE(out == s); + } + + SECTION("encode characters that are special in OPL") { + osmium::io::detail::append_utf8_encoded_string(out, " \n,=@"); + REQUIRE(out == "%20%%0a%%2c%%3d%%40%"); + } + +// workaround for missing support for u8 string literals on Windows +#if !defined(_MSC_VER) + + SECTION("encode multibyte character") { + osmium::io::detail::append_utf8_encoded_string(out, u8"\u30dc_\U0001d11e_\U0001f6eb"); + REQUIRE(out == "%30dc%_%1d11e%_%1f6eb%"); + } + +#endif + +} + +TEST_CASE("html encoding") { + + std::string out; + + SECTION("do not encode normal characters") { + const char* s = "abc123,.-"; + osmium::io::detail::append_xml_encoded_string(out, s); + REQUIRE(out == s); + } + + SECTION("encode special XML characters") { + const char* s = "& \" \' < > \n \r \t"; + osmium::io::detail::append_xml_encoded_string(out, s); + REQUIRE(out == "& " ' < > "); + } + +} + +TEST_CASE("debug encoding") { + + std::string out; + + SECTION("do not encode normal characters") { + const char* s = "abc123,.-"; + osmium::io::detail::append_debug_encoded_string(out, s, "[", "]"); + REQUIRE(out == s); + } + + SECTION("encode some unicode characters") { + const char* s = u8"\n_\u30dc_\U0001d11e_\U0001f6eb"; + osmium::io::detail::append_debug_encoded_string(out, s, "[", "]"); + REQUIRE(out == "[]_[]_[]_[]"); + } + +} + +TEST_CASE("encoding of non-printable characters in the first 127 characters") { + + std::locale cloc("C"); + char s[] = "a\0"; + + for (char c = 1; c < 0x7f; ++c) { + std::string out; + s[0] = c; + + SECTION("utf8 encode") { + osmium::io::detail::append_utf8_encoded_string(out, s); + + if (!std::isprint(c, cloc)) { + REQUIRE(out[0] == '%'); + } + } + + SECTION("debug encode") { + osmium::io::detail::append_debug_encoded_string(out, s, "", ""); + + if (!std::isprint(c, cloc)) { + REQUIRE(out[0] == '<'); + } + } + + } + +} + diff --git a/third_party/libosmium/test/t/io/test_reader.cpp b/third_party/libosmium/test/t/io/test_reader.cpp index 9a06d8408..a83af52ab 100644 --- a/third_party/libosmium/test/t/io/test_reader.cpp +++ b/third_party/libosmium/test/t/io/test_reader.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -17,6 +18,27 @@ struct CountHandler : public osmium::handler::Handler { }; // class CountHandler +struct ZeroPositionNodeCountHandler : public osmium::handler::Handler { + + // number of nodes seen at zero position, or visible with undefined + // location. + int count = 0; + int total_count = 0; // total number of nodes seen + const osmium::Location zero = osmium::Location(int32_t(0), int32_t(0)); + + void node(osmium::Node &n) { + // no nodes in the history file have a zero location, and + // no visible nodes should have an undefined location. + if ((n.location() == zero) || + (n.visible() && !n.location())) { + ++count; + } + ++total_count; + } + +}; // class ZeroPositionNodeCountHandler + + TEST_CASE("Reader") { SECTION("reader can be initialized with file") { @@ -34,7 +56,7 @@ TEST_CASE("Reader") { osmium::apply(reader, handler); } - SECTION("should return invalid buffer after eof") { + SECTION("should throw after eof") { osmium::io::File file(with_data_dir("t/io/data.osm")); osmium::io::Reader reader(file); @@ -45,9 +67,9 @@ TEST_CASE("Reader") { REQUIRE(reader.eof()); - // extra read always returns invalid buffer - osmium::memory::Buffer buffer = reader.read(); - REQUIRE(!buffer); + REQUIRE_THROWS_AS({ + reader.read(); + }, osmium::io_error); } SECTION("should not hang when apply() is called twice on reader") { @@ -56,7 +78,9 @@ TEST_CASE("Reader") { osmium::handler::Handler handler; osmium::apply(reader, handler); - osmium::apply(reader, handler); + REQUIRE_THROWS_AS({ + osmium::apply(reader, handler); + }, osmium::io_error); } SECTION("should work with a buffer with uncompressed data") { @@ -113,5 +137,76 @@ TEST_CASE("Reader") { REQUIRE(handler.count == 1); } + SECTION("should decode zero node positions in history (XML)") { + osmium::io::Reader reader(with_data_dir("t/io/deleted_nodes.osh"), + osmium::osm_entity_bits::node); + ZeroPositionNodeCountHandler handler; + + REQUIRE(handler.count == 0); + REQUIRE(handler.total_count == 0); + + osmium::apply(reader, handler); + + REQUIRE(handler.count == 0); + REQUIRE(handler.total_count == 2); + } + + SECTION("should decode zero node positions in history (PBF)") { + osmium::io::Reader reader(with_data_dir("t/io/deleted_nodes.osh.pbf"), + osmium::osm_entity_bits::node); + ZeroPositionNodeCountHandler handler; + + REQUIRE(handler.count == 0); + REQUIRE(handler.total_count == 0); + + osmium::apply(reader, handler); + + REQUIRE(handler.count == 0); + REQUIRE(handler.total_count == 2); + } + +} + +TEST_CASE("Reader failure modes") { + + SECTION("should fail with nonexistent file") { + REQUIRE_THROWS({ + osmium::io::Reader reader(with_data_dir("t/io/nonexistent-file.osm")); + }); + } + + SECTION("should fail with nonexistent file (gz)") { + REQUIRE_THROWS({ + osmium::io::Reader reader(with_data_dir("t/io/nonexistent-file.osm.gz")); + }); + } + + SECTION("should fail with nonexistent file (pbf)") { + REQUIRE_THROWS({ + osmium::io::Reader reader(with_data_dir("t/io/nonexistent-file.osm.pbf")); + }); + } + + SECTION("should work when there is an exception in main thread before getting header") { + try { + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + REQUIRE(!reader.eof()); + throw std::runtime_error("foo"); + } catch (...) { + } + + } + + SECTION("should work when there is an exception in main thread while reading") { + try { + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + REQUIRE(!reader.eof()); + auto header = reader.header(); + throw std::runtime_error("foo"); + } catch (...) { + } + + } + } diff --git a/third_party/libosmium/test/t/io/test_reader_with_mock_decompression.cpp b/third_party/libosmium/test/t/io/test_reader_with_mock_decompression.cpp new file mode 100644 index 000000000..566295aff --- /dev/null +++ b/third_party/libosmium/test/t/io/test_reader_with_mock_decompression.cpp @@ -0,0 +1,145 @@ + +#include "catch.hpp" +#include "utils.hpp" + +#include + +#include +#include + +// The MockDecompressor behaves like other Decompressor classes, but "invents" +// OSM data in XML format that can be read. Through a parameter to the +// constructor it can be instructed to throw an exception in specific parts +// of its code. This is then used to test the internals of the Reader. + +class MockDecompressor : public osmium::io::Decompressor { + + std::string m_fail_in; + int m_read_count = 0; + +public: + + MockDecompressor(const std::string& fail_in) : + Decompressor(), + m_fail_in(fail_in) { + if (m_fail_in == "constructor") { + throw std::runtime_error("error constructor"); + } + } + + ~MockDecompressor() noexcept final = default; + + void add_node(std::string& s, int i) { + s += "\n"; + } + + std::string read() final { + std::string buffer; + ++m_read_count; + + if (m_read_count == 1) { + if (m_fail_in == "first read") { + throw std::runtime_error("error first read"); + } else { + buffer += "\n\n"; + for (int i = 0; i < 1000; ++i) { + add_node(buffer, i); + } + } + } else if (m_read_count == 2) { + if (m_fail_in == "second read") { + throw std::runtime_error("error second read"); + } else { + for (int i = 1000; i < 2000; ++i) { + add_node(buffer, i); + } + } + } else if (m_read_count == 3) { + buffer += ""; + } + + return buffer; + } + + void close() final { + if (m_fail_in == "close") { + throw std::runtime_error("error close"); + } + } + +}; // class MockDecompressor + +TEST_CASE("Test Reader using MockDecompressor") { + + std::string fail_in; + + osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::gzip, + [](int, osmium::io::fsync) { return nullptr; }, + [&](int) { return new MockDecompressor(fail_in); }, + [](const char*, size_t) { return nullptr; } + ); + + SECTION("fail in constructor") { + fail_in = "constructor"; + + try { + osmium::io::Reader reader(with_data_dir("t/io/data.osm.gz")); + REQUIRE(false); + } catch (std::runtime_error& e) { + REQUIRE(std::string{e.what()} == "error constructor"); + } + } + + SECTION("fail in first read") { + fail_in = "first read"; + + try { + osmium::io::Reader reader(with_data_dir("t/io/data.osm.gz")); + reader.read(); + REQUIRE(false); + } catch (std::runtime_error& e) { + REQUIRE(std::string{e.what()} == "error first read"); + } + } + + SECTION("fail in second read") { + fail_in = "second read"; + + try { + osmium::io::Reader reader(with_data_dir("t/io/data.osm.gz")); + reader.read(); + reader.read(); + REQUIRE(false); + } catch (std::runtime_error& e) { + REQUIRE(std::string{e.what()} == "error second read"); + } + } + + SECTION("fail in close") { + fail_in = "close"; + + try { + osmium::io::Reader reader(with_data_dir("t/io/data.osm.gz")); + reader.read(); + reader.read(); + reader.read(); + reader.close(); + REQUIRE(false); + } catch (std::runtime_error& e) { + REQUIRE(std::string{e.what()} == "error close"); + } + } + + SECTION("not failing") { + fail_in = "not"; + + osmium::io::Reader reader(with_data_dir("t/io/data.osm.gz")); + reader.read(); + reader.close(); + REQUIRE(true); + } + +} + diff --git a/third_party/libosmium/test/t/io/test_reader_with_mock_parser.cpp b/third_party/libosmium/test/t/io/test_reader_with_mock_parser.cpp new file mode 100644 index 000000000..7c7dcddba --- /dev/null +++ b/third_party/libosmium/test/t/io/test_reader_with_mock_parser.cpp @@ -0,0 +1,123 @@ + +#include "catch.hpp" +#include "utils.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +class MockParser : public osmium::io::detail::Parser { + + std::string m_fail_in; + +public: + + MockParser(osmium::io::detail::future_string_queue_type& input_queue, + osmium::io::detail::future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_types, + const std::string& fail_in) : + Parser(input_queue, output_queue, header_promise, read_types), + m_fail_in(fail_in) { + } + + osmium::memory::Buffer create_testdata() { + osmium::memory::Buffer buffer(1000); + + { + osmium::builder::NodeBuilder nb(buffer); + nb.add_user("foo"); + } + buffer.commit(); + + return buffer; + } + + void run() final { + osmium::thread::set_thread_name("_osmium_mock_in"); + + if (m_fail_in == "header") { + throw std::runtime_error("error in header"); + } + + set_header_value(osmium::io::Header{}); + + send_to_output_queue(create_testdata()); + + if (m_fail_in == "read") { + throw std::runtime_error("error in read"); + } + } + +}; // class MockParser + +TEST_CASE("Test Reader using MockParser") { + + std::string fail_in; + + osmium::io::detail::ParserFactory::instance().register_parser( + osmium::io::file_format::xml, + [&](osmium::io::detail::future_string_queue_type& input_queue, + osmium::io::detail::future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_which_entities) { + return std::unique_ptr(new MockParser(input_queue, output_queue, header_promise, read_which_entities, fail_in)); + }); + + SECTION("no failure") { + fail_in = ""; + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + auto header = reader.header(); + REQUIRE(reader.read()); + REQUIRE(!reader.read()); + REQUIRE(reader.eof()); + reader.close(); + } + + SECTION("throw in header") { + fail_in = "header"; + try { + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + reader.header(); + } catch (std::runtime_error& e) { + REQUIRE(std::string{e.what()} == "error in header"); + } + } + + SECTION("throw in read") { + fail_in = "read"; + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + reader.header(); + try { + reader.read(); + } catch (std::runtime_error& e) { + REQUIRE(std::string{e.what()} == "error in read"); + } + reader.close(); + } + + SECTION("throw in user code") { + fail_in = ""; + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + reader.header(); + try { + throw std::runtime_error("error in user code"); + } catch (std::runtime_error& e) { + REQUIRE(std::string{e.what()} == "error in user code"); + } + REQUIRE(reader.read()); + REQUIRE(!reader.read()); + REQUIRE(reader.eof()); + reader.close(); + } + +} + diff --git a/third_party/libosmium/test/t/io/test_string_table.cpp b/third_party/libosmium/test/t/io/test_string_table.cpp index 7fedfcfca..ab977e8bc 100644 --- a/third_party/libosmium/test/t/io/test_string_table.cpp +++ b/third_party/libosmium/test/t/io/test_string_table.cpp @@ -33,9 +33,9 @@ TEST_CASE("String store") { } SECTION("add zero-length string and longer strings") { - const char* s1 = ss.add(""); - const char* s2 = ss.add("xxx"); - const char* s3 = ss.add("yyyyy"); + ss.add(""); + ss.add("xxx"); + ss.add("yyyyy"); auto it = ss.begin(); REQUIRE(std::string(*it++) == ""); diff --git a/third_party/libosmium/test/t/io/test_writer.cpp b/third_party/libosmium/test/t/io/test_writer.cpp new file mode 100644 index 000000000..45593cf60 --- /dev/null +++ b/third_party/libosmium/test/t/io/test_writer.cpp @@ -0,0 +1,117 @@ +#include "catch.hpp" +#include "utils.hpp" + +#include + +#include +#include +#include +#include +#include + +TEST_CASE("Writer") { + + osmium::io::Header header; + header.set("generator", "test_writer.cpp"); + + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + osmium::memory::Buffer buffer = reader.read(); + REQUIRE(buffer); + REQUIRE(buffer.committed() > 0); + auto num = std::distance(buffer.cbegin(), buffer.cend()); + REQUIRE(num > 0); + REQUIRE(buffer.cbegin()->id() == 1); + + std::string filename; + + SECTION("Empty writes") { + + SECTION("Empty buffer") { + filename = "test-writer-out-empty-buffer.osm"; + osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow); + osmium::memory::Buffer empty_buffer(1024); + writer(std::move(empty_buffer)); + writer.close(); + } + + SECTION("Invalid buffer") { + filename = "test-writer-out-invalid-buffer.osm"; + osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow); + osmium::memory::Buffer invalid_buffer; + writer(std::move(invalid_buffer)); + writer.close(); + } + + osmium::io::Reader reader_check(filename); + osmium::memory::Buffer buffer_check = reader_check.read(); + REQUIRE(!buffer_check); + } + + SECTION("Successfull writes") { + + SECTION("Writer buffer") { + filename = "test-writer-out-buffer.osm"; + osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow); + writer(std::move(buffer)); + writer.close(); + + REQUIRE_THROWS_AS({ + writer(osmium::memory::Buffer{}); + }, osmium::io_error); + } + + SECTION("Writer item") { + filename = "test-writer-out-item.osm"; + osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow); + for (const auto& item : buffer) { + writer(item); + } + writer.close(); + } + + SECTION("Writer output iterator") { + filename = "test-writer-out-iterator.osm"; + osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow); + auto it = osmium::io::make_output_iterator(writer); + std::copy(buffer.cbegin(), buffer.cend(), it); + writer.close(); + } + + osmium::io::Reader reader_check(filename); + osmium::memory::Buffer buffer_check = reader_check.read(); + REQUIRE(buffer_check); + REQUIRE(buffer_check.committed() > 0); + REQUIRE(std::distance(buffer_check.cbegin(), buffer_check.cend()) == num); + REQUIRE(buffer_check.cbegin()->id() == 1); + + } + + SECTION("Interrupted write") { + + int error = 0; + try { + + SECTION("fail after open") { + filename = "test-writer-out-fail1.osm"; + osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow); + throw 1; + } + + SECTION("fail after write") { + filename = "test-writer-out-fail2.osm"; + osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow); + writer(std::move(buffer)); + throw 2; + } + + } catch (int e) { + error = e; + } + + REQUIRE(error > 0); + + } + +} + + diff --git a/third_party/libosmium/test/t/io/test_writer_with_mock_compression.cpp b/third_party/libosmium/test/t/io/test_writer_with_mock_compression.cpp new file mode 100644 index 000000000..c2d3bbd49 --- /dev/null +++ b/third_party/libosmium/test/t/io/test_writer_with_mock_compression.cpp @@ -0,0 +1,99 @@ + +#include "catch.hpp" +#include "utils.hpp" + +#include +#include + +#include +#include +#include + +class MockCompressor : public osmium::io::Compressor { + + std::string m_fail_in; + +public: + + MockCompressor(const std::string& fail_in) : + Compressor(osmium::io::fsync::no), + m_fail_in(fail_in) { + if (m_fail_in == "constructor") { + throw std::logic_error("constructor"); + } + } + + ~MockCompressor() noexcept final = default; + + void write(const std::string&) final { + if (m_fail_in == "write") { + throw std::logic_error("write"); + } + } + + void close() final { + if (m_fail_in == "close") { + throw std::logic_error("close"); + } + } + +}; // class MockCompressor + +TEST_CASE("Write with mock compressor") { + + std::string fail_in; + + osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::gzip, + [&](int, osmium::io::fsync) { return new MockCompressor(fail_in); }, + [](int) { return nullptr; }, + [](const char*, size_t) { return nullptr; } + ); + + osmium::io::Header header; + header.set("generator", "test_writer_with_mock_compression.cpp"); + + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + osmium::memory::Buffer buffer = reader.read(); + REQUIRE(buffer); + REQUIRE(buffer.committed() > 0); + auto num = std::distance(buffer.cbegin(), buffer.cend()); + REQUIRE(num > 0); + + SECTION("fail on construction") { + + fail_in = "constructor"; + + REQUIRE_THROWS_AS({ + osmium::io::Writer writer("test-writer-mock-fail-on-construction.osm.gz", header, osmium::io::overwrite::allow); + writer(std::move(buffer)); + writer.close(); + }, std::logic_error); + + } + + SECTION("fail on write") { + + fail_in = "write"; + + REQUIRE_THROWS_AS({ + osmium::io::Writer writer("test-writer-mock-fail-on-write.osm.gz", header, osmium::io::overwrite::allow); + writer(std::move(buffer)); + writer.close(); + }, std::logic_error); + + } + + SECTION("fail on close") { + + fail_in = "close"; + + REQUIRE_THROWS_AS({ + osmium::io::Writer writer("test-writer-mock-fail-on-close.osm.gz", header, osmium::io::overwrite::allow); + writer(std::move(buffer)); + writer.close(); + }, std::logic_error); + + } + +} + diff --git a/third_party/libosmium/test/t/io/test_writer_with_mock_encoder.cpp b/third_party/libosmium/test/t/io/test_writer_with_mock_encoder.cpp new file mode 100644 index 000000000..a43d59183 --- /dev/null +++ b/third_party/libosmium/test/t/io/test_writer_with_mock_encoder.cpp @@ -0,0 +1,105 @@ + +#include "catch.hpp" +#include "utils.hpp" + +#include +#include + +#include +#include +#include +#include +#include + +class MockOutputFormat : public osmium::io::detail::OutputFormat { + + std::string m_fail_in; + +public: + + MockOutputFormat(const osmium::io::File&, osmium::io::detail::future_string_queue_type& output_queue, const std::string& fail_in) : + OutputFormat(output_queue), + m_fail_in(fail_in) { + } + + void write_header(const osmium::io::Header&) final { + if (m_fail_in == "header") { + throw std::logic_error("header"); + } + send_to_output_queue(std::string{"header"}); + } + + void write_buffer(osmium::memory::Buffer&&) final { + if (m_fail_in == "write") { + throw std::logic_error("write"); + } + send_to_output_queue(std::string{"write"}); + } + + void write_end() final { + if (m_fail_in == "write_end") { + throw std::logic_error("write_end"); + } + send_to_output_queue(std::string{"end"}); + } + +}; // class MockOutputFormat + +TEST_CASE("Test Writer with MockOutputFormat") { + + std::string fail_in; + + osmium::io::detail::OutputFormatFactory::instance().register_output_format( + osmium::io::file_format::xml, + [&](const osmium::io::File& file, osmium::io::detail::future_string_queue_type& output_queue) { + return new MockOutputFormat(file, output_queue, fail_in); + }); + + osmium::io::Header header; + header.set("generator", "test_writer_with_mock_encoder.cpp"); + + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + osmium::memory::Buffer buffer = reader.read(); + REQUIRE(buffer); + REQUIRE(buffer.committed() > 0); + auto num = std::distance(buffer.cbegin(), buffer.cend()); + REQUIRE(num > 0); + + SECTION("error in header") { + + fail_in = "header"; + + REQUIRE_THROWS_AS({ + osmium::io::Writer writer("test-writer-mock-fail-on-construction.osm", header, osmium::io::overwrite::allow); + writer(std::move(buffer)); + writer.close(); + }, std::logic_error); + + } + + SECTION("error in write") { + + fail_in = "write"; + + REQUIRE_THROWS_AS({ + osmium::io::Writer writer("test-writer-mock-fail-on-construction.osm", header, osmium::io::overwrite::allow); + writer(std::move(buffer)); + writer.close(); + }, std::logic_error); + + } + + SECTION("error in write_end") { + + fail_in = "write_end"; + + REQUIRE_THROWS_AS({ + osmium::io::Writer writer("test-writer-mock-fail-on-construction.osm", header, osmium::io::overwrite::allow); + writer(std::move(buffer)); + writer.close(); + }, std::logic_error); + + } + +} + diff --git a/third_party/libosmium/test/t/tags/test_tag_list.cpp b/third_party/libosmium/test/t/tags/test_tag_list.cpp index 77523e796..295f51af8 100644 --- a/third_party/libosmium/test/t/tags/test_tag_list.cpp +++ b/third_party/libosmium/test/t/tags/test_tag_list.cpp @@ -100,3 +100,14 @@ TEST_CASE("empty keys and values are okay") { REQUIRE(std::string("") == tl.get_value_by_key("empty value")); REQUIRE(std::string("empty key") == tl.get_value_by_key("")); } + +TEST_CASE("tag key or value is too long") { + osmium::memory::Buffer buffer(10240); + osmium::builder::TagListBuilder builder(buffer); + + const char kv[2000] = ""; + builder.add_tag(kv, 1, kv, 1000); + REQUIRE_THROWS(builder.add_tag(kv, 1500, kv, 1)); + REQUIRE_THROWS(builder.add_tag(kv, 1, kv, 1500)); +} + diff --git a/third_party/libosmium/test/t/thread/test_pool.cpp b/third_party/libosmium/test/t/thread/test_pool.cpp index 5fa6bbab5..c1047db49 100644 --- a/third_party/libosmium/test/t/thread/test_pool.cpp +++ b/third_party/libosmium/test/t/thread/test_pool.cpp @@ -5,14 +5,7 @@ #include #include - -static std::atomic result; - -struct test_job_ok { - void operator()() const { - result = 1; - } -}; +#include struct test_job_with_result { int operator()() const { @@ -21,44 +14,55 @@ struct test_job_with_result { }; struct test_job_throw { - void operator()() const { + OSMIUM_NORETURN void operator()() const { throw std::runtime_error("exception in pool thread"); } }; +TEST_CASE("number of threads in pool") { + + // hardcoded setting + REQUIRE(osmium::thread::detail::get_pool_size( 1, 0, 2) == 1); + REQUIRE(osmium::thread::detail::get_pool_size( 4, 0, 2) == 4); + REQUIRE(osmium::thread::detail::get_pool_size( 4, 0, 4) == 4); + REQUIRE(osmium::thread::detail::get_pool_size(16, 0, 4) == 16); + REQUIRE(osmium::thread::detail::get_pool_size(16, 0, 16) == 16); + REQUIRE(osmium::thread::detail::get_pool_size( 8, 4, 2) == 8); + REQUIRE(osmium::thread::detail::get_pool_size( 8, 16, 2) == 8); + REQUIRE(osmium::thread::detail::get_pool_size(-2, 16, 2) == 1); + REQUIRE(osmium::thread::detail::get_pool_size(-2, 16, 8) == 6); + + // user decides through OSMIUM_POOL_THREADS env variable + REQUIRE(osmium::thread::detail::get_pool_size( 0, 0, 2) == 1); + REQUIRE(osmium::thread::detail::get_pool_size( 0, -2, 4) == 2); + REQUIRE(osmium::thread::detail::get_pool_size( 0, -1, 8) == 7); + REQUIRE(osmium::thread::detail::get_pool_size( 0, 0, 16) == 14); + REQUIRE(osmium::thread::detail::get_pool_size( 0, 1, 16) == 1); + REQUIRE(osmium::thread::detail::get_pool_size( 0, 2, 16) == 2); + REQUIRE(osmium::thread::detail::get_pool_size( 0, 4, 16) == 4); + REQUIRE(osmium::thread::detail::get_pool_size( 0, 8, 16) == 8); + + // outliers + REQUIRE(osmium::thread::detail::get_pool_size(-100, 0, 16) == 1); + REQUIRE(osmium::thread::detail::get_pool_size(1000, 0, 16) == 256); + +} + TEST_CASE("thread") { + auto& pool = osmium::thread::Pool::instance(); + SECTION("can get access to thread pool") { - auto& pool = osmium::thread::Pool::instance(); REQUIRE(pool.queue_empty()); } SECTION("can send job to thread pool") { - auto& pool = osmium::thread::Pool::instance(); - result = 0; - auto future = pool.submit(test_job_ok {}); - - // wait a bit for the other thread to get a chance to run - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - - REQUIRE(result == 1); - - future.get(); - - REQUIRE(true); - } - - SECTION("can send job to thread pool") { - auto& pool = osmium::thread::Pool::instance(); auto future = pool.submit(test_job_with_result {}); REQUIRE(future.get() == 42); } SECTION("can throw from job in thread pool") { - auto& pool = osmium::thread::Pool::instance(); - result = 0; - auto future = pool.submit(test_job_throw {}); REQUIRE_THROWS_AS(future.get(), std::runtime_error); diff --git a/third_party/libosmium/test/t/util/test_data_file.cpp b/third_party/libosmium/test/t/util/test_data_file.cpp deleted file mode 100644 index 3f432f943..000000000 --- a/third_party/libosmium/test/t/util/test_data_file.cpp +++ /dev/null @@ -1,81 +0,0 @@ -#include "catch.hpp" - -#include - -#include - -TEST_CASE("temporary file") { - - SECTION("create/open") { - osmium::util::DataFile file; - - REQUIRE(!!file); - int fd = file.fd(); - - REQUIRE(fd > 0); - - const char buf[] = "foobar"; - REQUIRE(::write(fd, buf, sizeof(buf)) == sizeof(buf)); - - file.close(); - - REQUIRE(!file); - } - -} - -TEST_CASE("named file") { - - SECTION("create/open") { - { - osmium::util::DataFile file("test.data", true); - - REQUIRE(!!file); - int fd = file.fd(); - - REQUIRE(fd > 0); - - REQUIRE(file.size() == 0); - - const char buf[] = "foobar"; - REQUIRE(::write(fd, buf, sizeof(buf) - 1) == sizeof(buf) - 1); - - file.close(); - - REQUIRE(!file); - } - { - osmium::util::DataFile file("test.data", false); - - REQUIRE(!!file); - int fd = file.fd(); - - REQUIRE(fd > 0); - - REQUIRE(file.size() == 6); - - char buf[10]; - int len = ::read(fd, buf, sizeof(buf)); - - REQUIRE(len == 6); - REQUIRE(!strncmp(buf, "foobar", 6)); - - file.close(); - - REQUIRE(!file); - REQUIRE(unlink("test.data") == 0); - } - } - - SECTION("grow file") { - osmium::util::DataFile file("test.data", true); - - REQUIRE(!!file); - - REQUIRE(file.size() == 0); - file.grow(10); - REQUIRE(file.size() == 10); - } - -} - diff --git a/third_party/libosmium/test/t/util/test_delta.cpp b/third_party/libosmium/test/t/util/test_delta.cpp index cebcca8ed..667c9b443 100644 --- a/third_party/libosmium/test/t/util/test_delta.cpp +++ b/third_party/libosmium/test/t/util/test_delta.cpp @@ -4,24 +4,50 @@ #include -TEST_CASE("delta encode") { +TEST_CASE("delta encode int") { osmium::util::DeltaEncode x; SECTION("int") { REQUIRE(x.update(17) == 17); REQUIRE(x.update(10) == -7); + REQUIRE(x.update(-10) == -20); } } -TEST_CASE("delta decode") { +TEST_CASE("delta decode int") { osmium::util::DeltaDecode x; SECTION("int") { REQUIRE(x.update(17) == 17); REQUIRE(x.update(10) == 27); + REQUIRE(x.update(-40) == -13); + } + +} + +TEST_CASE("delta encode unsigned int") { + + osmium::util::DeltaEncode x; + + SECTION("int") { + REQUIRE(x.update(17) == 17); + REQUIRE(x.update(10) == -7); + REQUIRE(x.update(0) == -10); + } + +} + +TEST_CASE("delta decode unsigned int") { + + osmium::util::DeltaDecode x; + + SECTION("int") { + REQUIRE(x.update(17) == 17); + REQUIRE(x.update(10) == 27); + REQUIRE(x.update(-15) == 12); } } @@ -30,13 +56,13 @@ TEST_CASE("delta encode and decode") { std::vector a = { 5, -9, 22, 13, 0, 23 }; - osmium::util::DeltaEncode de; + osmium::util::DeltaEncode de; std::vector b; for (int x : a) { b.push_back(de.update(x)); } - osmium::util::DeltaDecode dd; + osmium::util::DeltaDecode dd; std::vector c; for (int x : b) { c.push_back(dd.update(x)); diff --git a/third_party/libosmium/test/t/util/test_file.cpp b/third_party/libosmium/test/t/util/test_file.cpp index 2787261a5..475f28596 100644 --- a/third_party/libosmium/test/t/util/test_file.cpp +++ b/third_party/libosmium/test/t/util/test_file.cpp @@ -3,6 +3,7 @@ #include #ifdef _WIN32 +#include // https://msdn.microsoft.com/en-us/library/ksazx244.aspx // https://msdn.microsoft.com/en-us/library/a9yf33zb.aspx class DoNothingInvalidParameterHandler { @@ -23,6 +24,7 @@ public: DoNothingInvalidParameterHandler() : old_handler(_set_invalid_parameter_handler(invalid_parameter_handler)) { + _CrtSetReportMode(_CRT_ASSERT, 0); } ~DoNothingInvalidParameterHandler() { diff --git a/third_party/libosmium/test/t/util/test_options.cpp b/third_party/libosmium/test/t/util/test_options.cpp index 969f20103..8cba0950b 100644 --- a/third_party/libosmium/test/t/util/test_options.cpp +++ b/third_party/libosmium/test/t/util/test_options.cpp @@ -6,43 +6,76 @@ TEST_CASE("Options") { - SECTION("set_simple") { - osmium::util::Options o; + osmium::util::Options o; + + SECTION("set a single value from string") { o.set("foo", "bar"); REQUIRE("bar" == o.get("foo")); REQUIRE("" == o.get("empty")); REQUIRE("default" == o.get("empty", "default")); + REQUIRE(!o.is_true("foo")); REQUIRE(!o.is_true("empty")); + + REQUIRE(o.is_not_false("foo")); + REQUIRE(o.is_not_false("empty")); + REQUIRE(1 == o.size()); } - SECTION("set_from_bool") { - osmium::util::Options o; + SECTION("set values from booleans") { o.set("t", true); o.set("f", false); REQUIRE("true" == o.get("t")); REQUIRE("false" == o.get("f")); REQUIRE("" == o.get("empty")); + REQUIRE(o.is_true("t")); REQUIRE(!o.is_true("f")); + + REQUIRE(o.is_not_false("t")); + REQUIRE(!o.is_not_false("f")); + REQUIRE(2 == o.size()); } - SECTION("set_from_single_string_with_equals") { - osmium::util::Options o; + SECTION("set value from string with equal sign") { o.set("foo=bar"); REQUIRE("bar" == o.get("foo")); REQUIRE(1 == o.size()); } - SECTION("set_from_single_string_without_equals") { - osmium::util::Options o; + SECTION("set value from string without equal sign") { o.set("foo"); REQUIRE("true" == o.get("foo")); + REQUIRE(o.is_true("foo")); + REQUIRE(o.is_not_false("foo")); + REQUIRE(1 == o.size()); } } +TEST_CASE("Options with initializer list") { + + osmium::util::Options o{ { "foo", "true" }, { "bar", "17" } }; + + REQUIRE(o.get("foo") == "true"); + REQUIRE(o.get("bar") == "17"); + REQUIRE(o.is_true("foo")); + REQUIRE_FALSE(o.is_true("bar")); + REQUIRE(o.size() == 2); + + SECTION("Change existing value") { + o.set("foo", "false"); + REQUIRE_FALSE(o.is_true("foo")); + } + + SECTION("Add new value") { + o.set("new", "something"); + REQUIRE_FALSE(o.is_true("new")); + REQUIRE(o.get("new") == "something"); + } +} +