Merge commit '788bc67faa7738cf7c6b2a192ecf3e3567d1c20e' into develop

third_party/libosmium/.travis.yml

@@ -6,46 +6,49 @@
 
 language: cpp
 
-compiler:
+sudo: false
- - gcc
- - clang
 
-env:
+matrix:
- - CONFIGURATION=Dev
+    include:
- - CONFIGURATION=Release
+        - os: linux
+          compiler: clang
+          env: BUILD_TYPE=Dev
+        - os: linux
+          compiler: clang
+          env: BUILD_TYPE=Release
+        - os: linux
+          compiler: gcc
+          env: BUILD_TYPE=Dev
+        - os: linux
+          compiler: gcc
+          env: BUILD_TYPE=Release
+        - os: osx
+          compiler: clang
+          env: BUILD_TYPE=Dev
+        - os: osx
+          compiler: clang
+          env: BUILD_TYPE=Release
 
-before_install:
+# http://docs.travis-ci.com/user/apt/
- # we need at least g++-4.8 for c++11 features
+addons:
- - sudo add-apt-repository --yes ppa:ubuntu-toolchain-r/test
+    apt:
- - sudo apt-get update --yes --quiet
+        sources:
+            - boost-latest
+            - ubuntu-toolchain-r-test
+        packages:
+            - g++-4.8
+            - gcc-4.8
+            - libboost1.55-dev
+            - libboost-program-options1.55-dev
+            - libgdal-dev
+            - libgeos++-dev
+            - libproj-dev
+            - libsparsehash-dev
+            - spatialite-bin
 
 install:
- - cd ..
+    - scripts/travis_install.sh
- # upgrade compilers
- - sudo apt-get install --yes gcc-4.8 g++-4.8
- # make sure 'cpp' is the just installed current one
- - sudo rm /usr/bin/cpp
- - sudo ln -s /usr/bin/cpp-4.8 /usr/bin/cpp
- # upgrade libosmium dependencies
- - sudo apt-get install --yes make libboost-dev libboost-program-options-dev libsparsehash-dev libprotobuf-dev protobuf-compiler libgeos++-dev libproj-dev libgdal1h libgdal-dev
- - git clone https://github.com/osmcode/osm-testdata.git
- # OSMPBF is too old, install from git
- #- sudo apt-get install --yes libosmpbf-dev
- - git clone https://github.com/scrosby/OSM-binary.git
- - cd OSM-binary/src
- - make
- - sudo make install
- - cd ../..
- - cd libosmium
-
-before_script:
- - true
 
 script:
- - if [ "${CXX}" = 'g++' ]; then export CXX=g++-4.8; fi;
+    - scripts/travis_script.sh
- - mkdir build
- - cd build
- - cmake -LA -DCMAKE_BUILD_TYPE=${CONFIGURATION} ..
- - make VERBOSE=1
- - ctest --output-on-failure
 

third_party/libosmium/CHANGELOG.md

@@ -6,6 +6,88 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [unreleased] -
 
+### Added
+
+### Changed
+
+### Fixed
+
+## [2.3.0] - 2015-08-18
+
+### Added
+
+- Allow instantiating osmium::geom::GEOSFactory with existing GEOS factory.
+- Low-level functions to support generating a architecture- and endian-
+  independant CRC from OSM data. This is intended to be uses with boost::crc.
+- Add new debug output format. This format is not intended to be read
+  automatically, but for human consumption. It formats the data nicely.
+- Make writing of metadata configurable for XML and OPL output (use
+  `add_metadata=false` as file option).
+
+### Changed
+
+- Changed `add_user()` and `add_role()` in builders to use string length
+  without the 0-termination.
+- Improved code setting file format from suffix/format argument.
+- Memory mapping utility class now supports readonly, private writable or
+  shared writable operation.
+- Allow empty version (0) in PBF files.
+- Use utf8cpp header-only lib instead of boost for utf8 decoding. The library
+  is included in the libosmium distribution.
+- New PBF reader and writer based on the protozero. A complete rewrite of the
+  code for reading and writing OSM PBF files. It doesn't use the Google
+  protobuf library and it doesn't use the OSMPBF/OSM-Binary library any more.
+  Instead is uses the protozero lightweight protobuf header library which is
+  included in the code. Not only does the new code have less dependencies, it
+  is faster and more robust. https://github.com/mapbox/protozero
+
+### Fixed
+
+- Various smaller bug fixes.
+- Add encoding for relation member roles in OPL format.
+- Change character encoding to new format in OPL: variable length hex code
+  between % characters instead of a % followed by 4-digit hex code. This is
+  necessary because unicode characters can be longer than the 4-digit hex
+  code.
+- XML writer: The linefeed, carriage return, and tab characters are now
+  escaped properly.
+- Reading large XML files could block.
+
+## [2.2.0] - 2015-07-04
+
+### Added
+
+- Conversion functions for some low-level types.
+- BoolVector index class.
+- `min_op`/`max_op` utility functions.
+- More tests here and there.
+- Helper methods `is_between()` and `is_visible_at()` to DiffObject.
+- GeoJSON factory using the RapidJSON library.
+- Support for tile calculations.
+- Create simple polygons from ways in geom factories.
+- `MemoryMapping` and `TypedMemoryMapping` helper classes.
+- `close()` function to `mmap_vector_base` class.
+- Function on `Buffer` class to get iterator to specific offset.
+- Explicit cast operator from `osmium::Timestamp` to `uint32_t`.
+
+### Changed
+
+- Throw exception on illegal values in functions parsing strings to get ids,
+  versions, etc.
+- Improved error message for geometry exceptions.
+
+### Fixed
+
+- Throw exception from `dump_as_array()` and `dump_as_list()` functions if not
+  implemented in an index.
+- After writing OSM files, program could stall up to a second.
+- Dense location store was written out only partially.
+- Use `uint64_t` as counter in benchmarks, so there can be no overflows.
+- Example programs now read packed XML files, too.
+- Refactoring of memory mapping code. Removes leak on Windows.
+- Better check for invalid locations.
+- Mark `cbegin()` and `cend()` of `mmap_vector_base` as const functions.
+
 ## [2.1.0] - 2015-03-31
 
 ### Added
@@ -26,6 +108,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
   Doxygen (up to version 1.8.8). This version contains a workaround to fix
   this.
 
-[unreleased]: https://github.com/osmcode/libosmium/compare/v2.1.0...HEAD
+[unreleased]: https://github.com/osmcode/libosmium/compare/v2.3.0...HEAD
+[2.3.0]: https://github.com/osmcode/libosmium/compare/v2.3.0...v2.3.0
+[2.2.0]: https://github.com/osmcode/libosmium/compare/v2.1.0...v2.2.0
 [2.1.0]: https://github.com/osmcode/libosmium/compare/v2.0.0...v2.1.0
 

third_party/libosmium/CMakeLists.txt

@@ -18,7 +18,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 #
 #-----------------------------------------------------------------------------
 
-set(CMAKE_CONFIGURATION_TYPES "Debug;Release;RelWithDebInfo;MinSizeRel;Dev"
+set(CMAKE_CONFIGURATION_TYPES "Debug;Release;RelWithDebInfo;MinSizeRel;Dev;Coverage"
     CACHE STRING
     "List of available configuration types"
     FORCE)
@@ -26,11 +26,13 @@ set(CMAKE_CONFIGURATION_TYPES "Debug;Release;RelWithDebInfo;MinSizeRel;Dev"
 project(libosmium)
 
 set(LIBOSMIUM_VERSION_MAJOR 2)
-set(LIBOSMIUM_VERSION_MINOR 1)
+set(LIBOSMIUM_VERSION_MINOR 3)
 set(LIBOSMIUM_VERSION_PATCH 0)
 
 set(LIBOSMIUM_VERSION
-    ${LIBOSMIUM_VERSION_MAJOR}.${LIBOSMIUM_VERSION_MINOR}.${LIBOSMIUM_VERSION_PATCH})
+    "${LIBOSMIUM_VERSION_MAJOR}.${LIBOSMIUM_VERSION_MINOR}.${LIBOSMIUM_VERSION_PATCH}"
+    CACHE STRING
+    "Libosmium version")
 
 
 #-----------------------------------------------------------------------------
@@ -55,6 +57,57 @@ option(BUILD_BENCHMARKS "compile benchmark programs" ${dev_build})
 option(BUILD_DATA_TESTS "compile data tests, please run them with ctest" ${dev_build})
 
 
+#-----------------------------------------------------------------------------
+#
+#  Coverage support
+#
+#-----------------------------------------------------------------------------
+
+include(CheckCXXCompilerFlag)
+check_cxx_compiler_flag("-fkeep-inline-functions" HAS_KEEP_INLINE_FUNCTIONS)
+if(HAS_KEEP_INLINE_FUNCTIONS)
+    set(extra_coverage_flags_ "-fkeep-inline-functions")
+endif()
+
+set(CMAKE_CXX_FLAGS_COVERAGE
+    "-g -O0 -fno-inline-functions -fno-inline --coverage ${extra_coverage_flags_}"
+    CACHE STRING "Flags used by the compiler during coverage builds.")
+
+set(CMAKE_EXE_LINKER_FLAGS_COVERAGE
+    "--coverage"
+    CACHE STRING "Flags used by the linker during coverage builds.")
+
+if(CMAKE_BUILD_TYPE STREQUAL "Coverage")
+    if(BUILD_EXAMPLES OR BUILD_HEADERS OR BUILD_BENCHMARKS OR BUILD_DATA_TESTS)
+        message(WARNING "Coverage builds don't work for anything but the unit tests")
+    endif()
+
+    if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+        string(REGEX REPLACE "^([0-9]+)\\.([0-9]+).*$" "llvm-cov-\\1.\\2"
+               gcov_ ${CMAKE_CXX_COMPILER_VERSION})
+    else()
+        set(gcov_ "gcov")
+    endif()
+
+    find_program(GCOV ${gcov_} DOC "Coverage tool")
+    find_program(GCOVR "gcovr" DOC "Coverage report tool")
+
+    set(coverage_report_dir "${CMAKE_BINARY_DIR}/coverage")
+    file(MAKE_DIRECTORY ${coverage_report_dir})
+    add_custom_target(coverage
+        ${GCOVR}
+        ${CMAKE_BINARY_DIR}
+        --root=${CMAKE_SOURCE_DIR}
+        --html --html-details
+        #--verbose
+        #--keep
+        '--filter=.*include/osmium.*'
+        --sort-percentage
+        --gcov-executable=${GCOV}
+        --output=${coverage_report_dir}/index.html)
+endif()
+
+
 #-----------------------------------------------------------------------------
 #
 #  Find external dependencies
@@ -113,8 +166,10 @@ endif()
 #-----------------------------------------------------------------------------
 if(MSVC)
     set(USUAL_COMPILE_OPTIONS "/Ox")
+    set(USUAL_LINK_OPTIONS "/debug")
 else()
     set(USUAL_COMPILE_OPTIONS "-O3 -g")
+    set(USUAL_LINK_OPTIONS "")
 endif()
 
 if(WIN32)
@@ -126,7 +181,7 @@ set(CMAKE_CXX_FLAGS_DEV "${USUAL_COMPILE_OPTIONS}"
     CACHE STRING "Flags used by the compiler during developer builds."
     FORCE)
 
-set(CMAKE_EXE_LINKER_FLAGS_DEV ""
+set(CMAKE_EXE_LINKER_FLAGS_DEV "${USUAL_LINK_OPTIONS}"
     CACHE STRING "Flags used by the linker during developer builds."
     FORCE)
 mark_as_advanced(

third_party/libosmium/EXTERNAL_LICENSES.txt

@@ -0,0 +1,233 @@
+
+==== For protozero from https://github.com/mapbox/protozero
+
+protozero copyright (c) Mapbox.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+==== For protozero from https://github.com/mapbox/protozero
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+==== For utf8.h
+
+Copyright 2006 Nemanja Trifunovic
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+

third_party/libosmium/README.md

@@ -27,6 +27,10 @@ you need for your programs.
 For details see the
 [list of dependencies](https://github.com/osmcode/libosmium/wiki/Libosmium-dependencies).
 
+The [protozero](https://github.com/mapbox/protozero) and
+[utf8-cpp](http://utfcpp.sourceforge.net/) header-only libraries are included
+in the libosmium repository.
+
 
 ## Directories
 

third_party/libosmium/appveyor.yml

@@ -15,6 +15,8 @@ branches:
   only:
     - master
 
+shallow_clone: true
+
 # Operating system (build VM template)
 os: Visual Studio 2014 CTP4
 
@@ -27,7 +29,7 @@ clone_folder: c:\projects\libosmium
 platform: x64
 
 install:
-  # show all availble env vars
+  # show all available env vars
   - set
   - echo cmake on AppVeyor
   - cmake -version
@@ -50,8 +52,8 @@ install:
   #cmake cannot find it otherwise
   - set LIBBZIP2=%LODEPSDIR%\bzip2\lib\libbz2.lib
   - set LIBBZIP2=%LIBBZIP2:\=/%
-  - ps: Start-FileDownload https://mapnik.s3.amazonaws.com/deps/cmake-3.1.0-win32-x86.7z -FileName cm.7z
+  - ps: Start-FileDownload https://mapbox.s3.amazonaws.com/windows-builds/windows-build-deps/cmake-3.1.0-win32-x86.7z -FileName cm.7z
-  - ps: Start-FileDownload https://mapnik.s3.amazonaws.com/dist/dev/libosmium-deps-win-14.0-x64.7z -FileName lodeps.7z
+  - ps: Start-FileDownload https://mapbox.s3.amazonaws.com/windows-builds/windows-build-deps/libosmium-deps-win-14.0-x64.7z -FileName lodeps.7z
   - 7z x cm.7z | %windir%\system32\find "ing archive"
   - 7z x lodeps.7z | %windir%\system32\find "ing archive"
   - echo %LODEPSDIR%
@@ -59,19 +61,64 @@ install:
   - echo our own cmake
   - cmake -version
   - cd c:\projects
-  - git clone https://github.com/osmcode/osm-testdata.git
+  - git clone --depth 1 https://github.com/osmcode/osm-testdata.git
 
 build_script:
   - cd c:\projects\libosmium
   - mkdir build
   - cd build
   - echo %config%
-  - cmake .. -LA -G "Visual Studio 14 Win64" -DOsmium_DEBUG=TRUE -DCMAKE_BUILD_TYPE=%config% -DBUILD_BENCHMARKS=OFF -DBOOST_ROOT=%LODEPSDIR%\boost -DBoost_PROGRAM_OPTIONS_LIBRARY=%LODEPSDIR%\boost\lib\libboost_program_options-vc140-mt-1_57.lib -DOSMPBF_LIBRARY=%LODEPSDIR%\osmpbf\lib\osmpbf.lib -DOSMPBF_INCLUDE_DIR=%LODEPSDIR%\osmpbf\include -DPROTOBUF_LIBRARY=%LODEPSDIR%\protobuf\lib\libprotobuf.lib -DPROTOBUF_LITE_LIBRARY=%LODEPSDIR%\protobuf\lib\libprotobuf-lite.lib -DPROTOBUF_INCLUDE_DIR=%LODEPSDIR%\protobuf\include -DZLIB_LIBRARY=%LODEPSDIR%\zlib\lib\zlibwapi.lib -DZLIB_INCLUDE_DIR=%LODEPSDIR%\zlib\include -DEXPAT_LIBRARY=%LODEPSDIR%\expat\lib\libexpat.lib -DEXPAT_INCLUDE_DIR=%LODEPSDIR%\expat\include -DBZIP2_LIBRARIES=%LIBBZIP2% -DBZIP2_INCLUDE_DIR=%LODEPSDIR%\bzip2\include -DGDAL_LIBRARY=%LODEPSDIR%\gdal\lib\gdal_i.lib -DGDAL_INCLUDE_DIR=%LODEPSDIR%\gdal\include -DGEOS_LIBRARY=%LODEPSDIR%\geos\lib\geos.lib -DGEOS_INCLUDE_DIR=%LODEPSDIR%\geos\include -DPROJ_LIBRARY=%LODEPSDIR%\proj\lib\proj.lib -DPROJ_INCLUDE_DIR=%LODEPSDIR%\proj\include -DSPARSEHASH_INCLUDE_DIR=%LODEPSDIR%\sparsehash\include -DGETOPT_LIBRARY=%LODEPSDIR%\wingetopt\lib\wingetopt.lib -DGETOPT_INCLUDE_DIR=%LODEPSDIR%\wingetopt\include
+  # This will produce lots of LNK4099 warnings which can be ignored.
+  # Unfortunately they can't be disabled, see
+  # http://stackoverflow.com/questions/661606/visual-c-how-to-disable-specific-linker-warnings
+  - cmake .. -LA -G "Visual Studio 14 Win64"
+    -DOsmium_DEBUG=TRUE
+    -DCMAKE_BUILD_TYPE=%config%
+    -DBUILD_HEADERS=OFF
+    -DBOOST_ROOT=%LODEPSDIR%\boost
+    -DBoost_PROGRAM_OPTIONS_LIBRARY=%LODEPSDIR%\boost\lib\libboost_program_options-vc140-mt-1_57.lib
+    -DZLIB_LIBRARY=%LODEPSDIR%\zlib\lib\zlibwapi.lib
+    -DZLIB_INCLUDE_DIR=%LODEPSDIR%\zlib\include
+    -DEXPAT_LIBRARY=%LODEPSDIR%\expat\lib\libexpat.lib
+    -DEXPAT_INCLUDE_DIR=%LODEPSDIR%\expat\include
+    -DBZIP2_LIBRARIES=%LIBBZIP2%
+    -DBZIP2_INCLUDE_DIR=%LODEPSDIR%\bzip2\include
+    -DGDAL_LIBRARY=%LODEPSDIR%\gdal\lib\gdal_i.lib
+    -DGDAL_INCLUDE_DIR=%LODEPSDIR%\gdal\include
+    -DGEOS_LIBRARY=%LODEPSDIR%\geos\lib\geos.lib
+    -DGEOS_INCLUDE_DIR=%LODEPSDIR%\geos\include
+    -DPROJ_LIBRARY=%LODEPSDIR%\proj\lib\proj.lib
+    -DPROJ_INCLUDE_DIR=%LODEPSDIR%\proj\include
+    -DSPARSEHASH_INCLUDE_DIR=%LODEPSDIR%\sparsehash\include
+    -DGETOPT_LIBRARY=%LODEPSDIR%\wingetopt\lib\wingetopt.lib
+    -DGETOPT_INCLUDE_DIR=%LODEPSDIR%\wingetopt\include
   - msbuild libosmium.sln /p:Configuration=%config% /toolsversion:14.0 /p:Platform=x64 /p:PlatformToolset=v140
-  #- cmake .. -LA -G "NMake Makefiles" -DOsmium_DEBUG=TRUE -DCMAKE_BUILD_TYPE=%config% -DBOOST_ROOT=%LODEPSDIR%\boost -DBoost_PROGRAM_OPTIONS_LIBRARY=%LODEPSDIR%\boost\lib\libboost_program_options-vc140-mt-1_57.lib -DOSMPBF_LIBRARY=%LODEPSDIR%\osmpbf\lib\osmpbf.lib -DOSMPBF_INCLUDE_DIR=%LODEPSDIR%\osmpbf\include -DPROTOBUF_LIBRARY=%LODEPSDIR%\protobuf\lib\libprotobuf.lib -DPROTOBUF_LITE_LIBRARY=%LODEPSDIR%\protobuf\lib\libprotobuf-lite.lib -DPROTOBUF_INCLUDE_DIR=%LODEPSDIR%\protobuf\include -DZLIB_LIBRARY=%LODEPSDIR%\zlib\lib\zlibwapi.lib -DZLIB_INCLUDE_DIR=%LODEPSDIR%\zlib\include -DEXPAT_LIBRARY=%LODEPSDIR%\expat\lib\libexpat.lib -DEXPAT_INCLUDE_DIR=%LODEPSDIR%\expat\include -DBZIP2_LIBRARIES=%LIBBZIP2% -DBZIP2_INCLUDE_DIR=%LODEPSDIR%\bzip2\include -DGDAL_LIBRARY=%LODEPSDIR%\gdal\lib\gdal_i.lib -DGDAL_INCLUDE_DIR=%LODEPSDIR%\gdal\include -DGEOS_LIBRARY=%LODEPSDIR%\geos\lib\geos.lib -DGEOS_INCLUDE_DIR=%LODEPSDIR%\geos\include -DPROJ_LIBRARY=%LODEPSDIR%\proj\lib\proj.lib -DPROJ_INCLUDE_DIR=%LODEPSDIR%\proj\include -DSPARSEHASH_INCLUDE_DIR=%LODEPSDIR%\sparsehash\include -DGETOPT_LIBRARY=%LODEPSDIR%\wingetopt\lib\wingetopt.lib -DGETOPT_INCLUDE_DIR=%LODEPSDIR%\wingetopt\include
+  #- cmake .. -LA -G "NMake Makefiles"
+  #  -DOsmium_DEBUG=TRUE
+  #  -DCMAKE_BUILD_TYPE=%config%
+  #  -DBOOST_ROOT=%LODEPSDIR%\boost
+  #  -DBoost_PROGRAM_OPTIONS_LIBRARY=%LODEPSDIR%\boost\lib\libboost_program_options-vc140-mt-1_57.lib
+  #  -DZLIB_LIBRARY=%LODEPSDIR%\zlib\lib\zlibwapi.lib
+  #  -DZLIB_INCLUDE_DIR=%LODEPSDIR%\zlib\include
+  #  -DEXPAT_LIBRARY=%LODEPSDIR%\expat\lib\libexpat.lib
+  #  -DEXPAT_INCLUDE_DIR=%LODEPSDIR%\expat\include
+  #  -DBZIP2_LIBRARIES=%LIBBZIP2%
+  #  -DBZIP2_INCLUDE_DIR=%LODEPSDIR%\bzip2\include
+  #  -DGDAL_LIBRARY=%LODEPSDIR%\gdal\lib\gdal_i.lib
+  #  -DGDAL_INCLUDE_DIR=%LODEPSDIR%\gdal\include
+  #  -DGEOS_LIBRARY=%LODEPSDIR%\geos\lib\geos.lib
+  #  -DGEOS_INCLUDE_DIR=%LODEPSDIR%\geos\include
+  #  -DPROJ_LIBRARY=%LODEPSDIR%\proj\lib\proj.lib
+  #  -DPROJ_INCLUDE_DIR=%LODEPSDIR%\proj\include
+  #  -DSPARSEHASH_INCLUDE_DIR=%LODEPSDIR%\sparsehash\include
+  #  -DGETOPT_LIBRARY=%LODEPSDIR%\wingetopt\lib\wingetopt.lib
+  #  -DGETOPT_INCLUDE_DIR=%LODEPSDIR%\wingetopt\include
   #- nmake
 
 test_script:
-  # -LE fails_on_windows exempts tests we know will fail
+  # "-E testdata-overview" exempts one test we know fails on Appveyor
-  - ctest --output-on-failure -C %config% -LE fails_on_windows
+  #    because we currently don't have spatialite support.
+  - ctest --output-on-failure
+    -C %config%
+    -E testdata-overview
 

third_party/libosmium/benchmarks/CMakeLists.txt

@@ -13,6 +13,7 @@ set(BENCHMARKS
     count_tag
     index_map
     static_vs_dynamic_index
+    write_pbf
     CACHE STRING "Benchmark programs"
 )
 

third_party/libosmium/benchmarks/osmium_benchmark_count.cpp

@@ -4,6 +4,7 @@
 
 */
 
+#include <cstdint>
 #include <iostream>
 
 #include <osmium/io/any_input.hpp>
@@ -12,9 +13,9 @@
 
 struct CountHandler : public osmium::handler::Handler {
 
-    int nodes = 0;
+    uint64_t nodes = 0;
-    int ways = 0;
+    uint64_t ways = 0;
-    int relations = 0;
+    uint64_t relations = 0;
 
     void node(osmium::Node&) {
         ++nodes;
@@ -48,7 +49,5 @@ int main(int argc, char* argv[]) {
     std::cout << "Nodes: "     << handler.nodes << "\n";
     std::cout << "Ways: "      << handler.ways << "\n";
     std::cout << "Relations: " << handler.relations << "\n";
-
-    google::protobuf::ShutdownProtobufLibrary();
 }
 

third_party/libosmium/benchmarks/osmium_benchmark_count_tag.cpp

@@ -4,6 +4,7 @@
 
 */
 
+#include <cstdint>
 #include <iostream>
 
 #include <osmium/io/any_input.hpp>
@@ -12,8 +13,8 @@
 
 struct CountHandler : public osmium::handler::Handler {
 
-    int counter = 0;
+    uint64_t counter = 0;
-    int all = 0;
+    uint64_t all = 0;
 
     void node(osmium::Node& node) {
         ++all;
@@ -49,7 +50,5 @@ int main(int argc, char* argv[]) {
     reader.close();
 
     std::cout << "r_all=" << handler.all << " r_counter="  << handler.counter << "\n";
-
-    google::protobuf::ShutdownProtobufLibrary();
 }
 

third_party/libosmium/benchmarks/osmium_benchmark_index_map.cpp

@@ -35,7 +35,5 @@ int main(int argc, char* argv[]) {
 
     osmium::apply(reader, location_handler);
     reader.close();
-
-    google::protobuf::ShutdownProtobufLibrary();
 }
 

third_party/libosmium/benchmarks/osmium_benchmark_static_vs_dynamic_index.cpp

@@ -46,7 +46,6 @@ int main(int argc, char* argv[]) {
     std::string input_filename = argv[1];
 
     osmium::memory::Buffer buffer = osmium::io::read_file(input_filename);
-    google::protobuf::ShutdownProtobufLibrary();
 
     const auto& map_factory = osmium::index::MapFactory<osmium::unsigned_object_id_type, osmium::Location>::instance();
 

third_party/libosmium/benchmarks/osmium_benchmark_write_pbf.cpp

@@ -0,0 +1,34 @@
+/*
+
+  The code in this file is released into the Public Domain.
+
+*/
+
+#include <cstdint>
+#include <vector>
+
+#include <osmium/io/any_input.hpp>
+#include <osmium/io/any_output.hpp>
+
+int main(int argc, char* argv[]) {
+    if (argc != 3) {
+        std::cerr << "Usage: " << argv[0] << " INPUT-FILE OUTPUT-FILE\n";
+        exit(1);
+    }
+
+    std::string input_filename = argv[1];
+    std::string output_filename = argv[2];
+
+    osmium::io::Reader reader(input_filename);
+    osmium::io::File output_file(output_filename, "pbf");
+    osmium::io::Header header;
+    osmium::io::Writer writer(output_file, header, osmium::io::overwrite::allow);
+
+    while (osmium::memory::Buffer buffer = reader.read()) {
+        writer(std::move(buffer));
+    }
+
+    writer.close();
+    reader.close();
+}
+

third_party/libosmium/benchmarks/run_benchmark_write_pbf.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+#
+#  run_benchmark_write_pbf.sh
+#
+#  Will read the input file and after reading it into memory completely,
+#  write it to /dev/null. Because this will need the time to read *and* write
+#  the file, it will report the times for reading and writing. You can
+#  subtract the times needed for the "count" benchmark to (roughly) get the
+#  write times.
+#
+
+set -e
+
+BENCHMARK_NAME=write_pbf
+
+. @CMAKE_BINARY_DIR@/benchmarks/setup.sh
+
+CMD=$OB_DIR/osmium_benchmark_$BENCHMARK_NAME
+
+echo "# file size num mem time cpu_kernel cpu_user cpu_percent cmd options"
+for data in $OB_DATA_FILES; do
+    filename=`basename $data`
+    filesize=`stat --format="%s" --dereference $data`
+    for n in $OB_SEQ; do
+        $OB_TIME_CMD -f "$filename $filesize $n $OB_TIME_FORMAT" $CMD $data /dev/null 2>&1 >/dev/null | sed -e "s%$DATA_DIR/%%" | sed -e "s%$OB_DIR/%%"
+    done
+done
+

third_party/libosmium/cmake/FindOSMPBF.cmake

@@ -1,50 +0,0 @@
-#
-# Locate OSMPBF library
-#
-# This module defines
-#  OSMPBF_FOUND        - if false, do not try to link to OSMPBF
-#  OSMPBF_LIBRARIES    - full library path name
-#  OSMPBF_INCLUDE_DIRS - where to find OSMPBF.hpp
-#
-# Note that the expected include convention is
-#  #include <osmpbf/osmpbf.h>
-# and not
-#  #include <osmpbf.h>
-#
-
-find_path(OSMPBF_INCLUDE_DIR osmpbf/osmpbf.h
-    HINTS $ENV{OSMPBF_DIR}
-    PATH_SUFFIXES include
-    PATHS
-        ~/Library/Frameworks
-        /Library/Frameworks
-        /usr/local
-        /usr
-        /opt/local # DarwinPorts
-        /opt
-)
-
-find_library(OSMPBF_LIBRARY
-    NAMES osmpbf
-    HINTS $ENV{OSMPBF_DIR}
-    PATH_SUFFIXES lib64 lib
-    PATHS
-        ~/Library/Frameworks
-        /Library/Frameworks
-        /usr/local
-        /usr
-        /opt/local
-        /opt
-)
-
-# Handle the QUIETLY and REQUIRED arguments and set OSMPBF_FOUND to TRUE if
-# all listed variables are TRUE.
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(OSMPBF DEFAULT_MSG OSMPBF_LIBRARY OSMPBF_INCLUDE_DIR)
-
-# Copy the results to the output variables.
-if(OSMPBF_FOUND)
-    set(OSMPBF_INCLUDE_DIRS ${OSMPBF_INCLUDE_DIR})
-    set(OSMPBF_LIBRARIES ${OSMPBF_LIBRARY})
-endif()
-

third_party/libosmium/cmake/FindOsmium.cmake

@@ -110,15 +110,11 @@ endif()
 #----------------------------------------------------------------------
 # Component 'pbf'
 if(Osmium_USE_PBF)
-    find_package(OSMPBF)
-    find_package(Protobuf)
     find_package(ZLIB)
     find_package(Threads)
 
-    if(OSMPBF_FOUND AND PROTOBUF_FOUND AND ZLIB_FOUND AND Threads_FOUND)
+    if(ZLIB_FOUND AND Threads_FOUND)
         list(APPEND OSMIUM_PBF_LIBRARIES
-            ${OSMPBF_LIBRARIES}
-            ${PROTOBUF_LITE_LIBRARY}
             ${ZLIB_LIBRARIES}
             ${CMAKE_THREAD_LIBS_INIT}
         )
@@ -126,8 +122,6 @@ if(Osmium_USE_PBF)
             list(APPEND OSMIUM_PBF_LIBRARIES ws2_32)
         endif()
         list(APPEND OSMIUM_INCLUDE_DIRS
-            ${OSMPBF_INCLUDE_DIRS}
-            ${PROTOBUF_INCLUDE_DIR}
             ${ZLIB_INCLUDE_DIR}
         )
     else()
@@ -325,7 +319,7 @@ endif()
 if(MSVC)
     set(OSMIUM_WARNING_OPTIONS "/W3 /wd4514" CACHE STRING "Recommended warning options for libosmium")
 else()
-    set(OSMIUM_WARNING_OPTIONS "-Wall -Wextra -pedantic -Wredundant-decls -Wdisabled-optimization -Wctor-dtor-privacy -Wnon-virtual-dtor -Woverloaded-virtual -Wsign-promo -Wold-style-cast -Wno-return-type" CACHE STRING "Recommended warning options for libosmium")
+    set(OSMIUM_WARNING_OPTIONS "-Wall -Wextra -pedantic -Wredundant-decls -Wdisabled-optimization -Wctor-dtor-privacy -Wnon-virtual-dtor -Woverloaded-virtual -Wsign-promo -Wold-style-cast" CACHE STRING "Recommended warning options for libosmium")
 endif()
 
 set(OSMIUM_DRACONIC_CLANG_OPTIONS "-Wdocumentation -Wunused-exception-parameter -Wmissing-declarations -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-unused-macros -Wno-exit-time-destructors -Wno-global-constructors -Wno-padded -Wno-switch-enum -Wno-missing-prototypes -Wno-weak-vtables -Wno-cast-align -Wno-float-equal")

third_party/libosmium/cmake/iwyu.sh

@@ -10,12 +10,12 @@ cmdline="iwyu -Xiwyu --mapping_file=osmium.imp -std=c++11 -I include"
 
 log=build/iwyu.log
 
+mkdir -p build/check_reports
+
 echo "INCLUDE WHAT YOU USE REPORT:" >$log
 
 allok=yes
 
-mkdir -p build/check_reports
-
 for file in `find include/osmium -name \*.hpp`; do
     mkdir -p `dirname build/check_reports/$file`
     ifile="build/check_reports/${file%.hpp}.iwyu"

third_party/libosmium/doc/CMakeLists.txt

@@ -21,8 +21,6 @@ if(DOXYGEN_FOUND)
         WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
         COMMENT "Generating API documentation with Doxygen" VERBATIM
     )
-#    install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/html"
-#            DESTINATION "share/doc/libosmium-dev")
 else()
     message(STATUS "Looking for doxygen - not found")
     message(STATUS "  Disabled making of documentation.")

third_party/libosmium/doc/Doxyfile.in

@@ -1128,7 +1128,7 @@ HTML_COLORSTYLE_GAMMA  = 80
 # The default value is: YES.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_TIMESTAMP         = YES
+HTML_TIMESTAMP         = NO
 
 # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
 # documentation will contain sections that can be hidden and shown after the

third_party/libosmium/doc/README.md

@@ -3,6 +3,6 @@ The `header.html` is created with:
 
 `doxygen -w html header.html footer.html stylesheet.css`
 
-This might have to be rn again for newer Doxygen versions. After that add
+This might have to be run again for newer Doxygen versions. After that add
 changes back in.
 

third_party/libosmium/examples/osmium_area_test.cpp

@@ -132,7 +132,5 @@ int main(int argc, char* argv[]) {
         }
         std::cerr << "\n";
     }
-
-    google::protobuf::ShutdownProtobufLibrary();
 }
 

third_party/libosmium/examples/osmium_convert.cpp

@@ -106,7 +106,6 @@ int main(int argc, char* argv[]) {
         exit_code = 1;
     }
 
-    google::protobuf::ShutdownProtobufLibrary();
     return exit_code;
 }
 

third_party/libosmium/examples/osmium_count.cpp

@@ -7,6 +7,7 @@
 
 */
 
+#include <cstdint>
 #include <iostream>
 
 #include <osmium/io/any_input.hpp>
@@ -15,9 +16,9 @@
 
 struct CountHandler : public osmium::handler::Handler {
 
-    int nodes = 0;
+    uint64_t nodes = 0;
-    int ways = 0;
+    uint64_t ways = 0;
-    int relations = 0;
+    uint64_t relations = 0;
 
     void node(osmium::Node&) {
         ++nodes;
@@ -51,7 +52,5 @@ int main(int argc, char* argv[]) {
     std::cout << "Nodes: "     << handler.nodes << "\n";
     std::cout << "Ways: "      << handler.ways << "\n";
     std::cout << "Relations: " << handler.relations << "\n";
-
-    google::protobuf::ShutdownProtobufLibrary();
 }
 

third_party/libosmium/examples/osmium_create_node_cache.cpp

@@ -12,8 +12,7 @@
 #include <fcntl.h>
 #include <iostream>
 
-#include <osmium/io/pbf_input.hpp>
+#include <osmium/io/any_input.hpp>
-#include <osmium/io/xml_input.hpp>
 
 #include <osmium/index/map/dummy.hpp>
 #include <osmium/index/map/dense_mmap_array.hpp>
@@ -51,8 +50,6 @@ int main(int argc, char* argv[]) {
     osmium::apply(reader, location_handler);
     reader.close();
 
-    google::protobuf::ShutdownProtobufLibrary();
-
     return 0;
 }
 

third_party/libosmium/examples/osmium_debug.cpp

@@ -46,7 +46,5 @@ int main(int argc, char* argv[]) {
     }
 
     reader.close();
-
-    google::protobuf::ShutdownProtobufLibrary();
 }
 

third_party/libosmium/examples/osmium_read.cpp

@@ -26,7 +26,5 @@ int main(int argc, char* argv[]) {
     }
 
     reader.close();
-
-    google::protobuf::ShutdownProtobufLibrary();
 }
 

third_party/libosmium/examples/osmium_serdump.cpp

@@ -19,8 +19,7 @@
 # include <direct.h>
 #endif
 
-#include <osmium/io/pbf_input.hpp>
+#include <osmium/io/any_input.hpp>
-#include <osmium/io/xml_input.hpp>
 #include <osmium/handler/disk_store.hpp>
 #include <osmium/handler/object_relations.hpp>
 
@@ -203,7 +202,5 @@ int main(int argc, char* argv[]) {
         map_relation2relation.dump_as_list(fd);
         close(fd);
     }
-
-    google::protobuf::ShutdownProtobufLibrary();
 }
 

third_party/libosmium/examples/osmium_toogr.cpp

@@ -234,8 +234,6 @@ int main(int argc, char* argv[]) {
     osmium::apply(reader, location_handler, ogr_handler);
     reader.close();
 
-    google::protobuf::ShutdownProtobufLibrary();
-
     int locations_fd = open("locations.dump", O_WRONLY | O_CREAT, 0644);
     if (locations_fd < 0) {
         throw std::system_error(errno, std::system_category(), "Open failed");

third_party/libosmium/examples/osmium_toogr2.cpp

@@ -327,7 +327,5 @@ int main(int argc, char* argv[]) {
         }
         std::cerr << "\n";
     }
-
-    google::protobuf::ShutdownProtobufLibrary();
 }
 

third_party/libosmium/examples/osmium_toogr2_exp.cpp

@@ -301,7 +301,5 @@ int main(int argc, char* argv[]) {
         }
         std::cerr << "\n";
     }
-
-    google::protobuf::ShutdownProtobufLibrary();
 }
 

third_party/libosmium/examples/osmium_use_node_cache.cpp

@@ -12,8 +12,7 @@
 #include <fcntl.h>
 #include <iostream>
 
-#include <osmium/io/pbf_input.hpp>
+#include <osmium/io/any_input.hpp>
-#include <osmium/io/xml_input.hpp>
 
 #include <osmium/index/map/dummy.hpp>
 #include <osmium/index/map/dense_file_array.hpp>
@@ -64,8 +63,6 @@ int main(int argc, char* argv[]) {
     osmium::apply(reader, location_handler, handler);
     reader.close();
 
-    google::protobuf::ShutdownProtobufLibrary();
-
     return 0;
 }
 

third_party/libosmium/include/boost_unicode_iterator.hpp

@@ -1,776 +0,0 @@
-/*
- *
- * Copyright (c) 2004
- * John Maddock
- *
- * Use, modification and distribution are subject to the 
- * Boost Software License, Version 1.0. (See accompanying file 
- * LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
- *
- */
- 
- /*
-  *   LOCATION:    see http://www.boost.org for most recent version.
-  *   FILE         unicode_iterator.hpp
-  *   VERSION      see <boost/version.hpp>
-  *   DESCRIPTION: Iterator adapters for converting between different Unicode encodings.
-  */
-
-/****************************************************************************
-
-Contents:
-~~~~~~~~~
-
-1) Read Only, Input Adapters:
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-template <class BaseIterator, class U8Type = ::boost::uint8_t>
-class u32_to_u8_iterator;
-
-Adapts sequence of UTF-32 code points to "look like" a sequence of UTF-8.
-
-template <class BaseIterator, class U32Type = ::boost::uint32_t>
-class u8_to_u32_iterator;
-
-Adapts sequence of UTF-8 code points to "look like" a sequence of UTF-32.
-
-template <class BaseIterator, class U16Type = ::boost::uint16_t>
-class u32_to_u16_iterator;
-
-Adapts sequence of UTF-32 code points to "look like" a sequence of UTF-16.
-
-template <class BaseIterator, class U32Type = ::boost::uint32_t>
-class u16_to_u32_iterator;
-
-Adapts sequence of UTF-16 code points to "look like" a sequence of UTF-32.
-
-2) Single pass output iterator adapters:
-
-template <class BaseIterator>
-class utf8_output_iterator;
-
-Accepts UTF-32 code points and forwards them on as UTF-8 code points.
-
-template <class BaseIterator>
-class utf16_output_iterator;
-
-Accepts UTF-32 code points and forwards them on as UTF-16 code points.
-
-****************************************************************************/
-
-#ifndef BOOST_REGEX_UNICODE_ITERATOR_HPP
-#define BOOST_REGEX_UNICODE_ITERATOR_HPP
-#include <boost/cstdint.hpp>
-#include <boost/assert.hpp>
-#include <boost/iterator/iterator_facade.hpp>
-#include <boost/static_assert.hpp>
-#include <boost/throw_exception.hpp>
-#include <stdexcept>
-#ifndef BOOST_NO_STD_LOCALE
-#include <sstream>
-#include <ios>
-#endif
-#include <limits.h> // CHAR_BIT
-
-namespace boost{
-
-namespace detail{
-
-static const ::boost::uint16_t high_surrogate_base = 0xD7C0u;
-static const ::boost::uint16_t low_surrogate_base = 0xDC00u;
-static const ::boost::uint32_t ten_bit_mask = 0x3FFu;
-
-inline bool is_high_surrogate(::boost::uint16_t v)
-{
-   return (v & 0xFFFFFC00u) == 0xd800u;
-}
-inline bool is_low_surrogate(::boost::uint16_t v)
-{
-   return (v & 0xFFFFFC00u) == 0xdc00u;
-}
-template <class T>
-inline bool is_surrogate(T v)
-{
-   return (v & 0xFFFFF800u) == 0xd800;
-}
-
-inline unsigned utf8_byte_count(boost::uint8_t c)
-{
-   // if the most significant bit with a zero in it is in position
-   // 8-N then there are N bytes in this UTF-8 sequence:
-   boost::uint8_t mask = 0x80u;
-   unsigned result = 0;
-   while(c & mask)
-   {
-      ++result;
-      mask >>= 1;
-   }
-   return (result == 0) ? 1 : ((result > 4) ? 4 : result);
-}
-
-inline unsigned utf8_trailing_byte_count(boost::uint8_t c)
-{
-   return utf8_byte_count(c) - 1;
-}
-
-#ifdef BOOST_MSVC
-#pragma warning(push)
-#pragma warning(disable:4100)
-#endif
-inline void invalid_utf32_code_point(::boost::uint32_t val)
-{
-#ifndef BOOST_NO_STD_LOCALE
-   std::stringstream ss;
-   ss << "Invalid UTF-32 code point U+" << std::showbase << std::hex << val << " encountered while trying to encode UTF-16 sequence";
-   std::out_of_range e(ss.str());
-#else
-   std::out_of_range e("Invalid UTF-32 code point encountered while trying to encode UTF-16 sequence");
-#endif
-   boost::throw_exception(e);
-}
-#ifdef BOOST_MSVC
-#pragma warning(pop)
-#endif
-
-
-} // namespace detail
-
-template <class BaseIterator, class U16Type = ::boost::uint16_t>
-class u32_to_u16_iterator
-   : public boost::iterator_facade<u32_to_u16_iterator<BaseIterator, U16Type>, U16Type, std::bidirectional_iterator_tag, const U16Type>
-{
-   typedef boost::iterator_facade<u32_to_u16_iterator<BaseIterator, U16Type>, U16Type, std::bidirectional_iterator_tag, const U16Type> base_type;
-
-#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION)
-   typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
-
-   BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32);
-   BOOST_STATIC_ASSERT(sizeof(U16Type)*CHAR_BIT == 16);
-#endif
-
-public:
-   typename base_type::reference
-      dereference()const
-   {
-      if(m_current == 2)
-         extract_current();
-      return m_values[m_current];
-   }
-   bool equal(const u32_to_u16_iterator& that)const
-   {
-      if(m_position == that.m_position)
-      {
-         // Both m_currents must be equal, or both even
-         // this is the same as saying their sum must be even:
-         return (m_current + that.m_current) & 1u ? false : true;
-      }
-      return false;
-   }
-   void increment()
-   {
-      // if we have a pending read then read now, so that we know whether
-      // to skip a position, or move to a low-surrogate:
-      if(m_current == 2)
-      {
-         // pending read:
-         extract_current();
-      }
-      // move to the next surrogate position:
-      ++m_current;
-      // if we've reached the end skip a position:
-      if(m_values[m_current] == 0)
-      {
-         m_current = 2;
-         ++m_position;
-      }
-   }
-   void decrement()
-   {
-      if(m_current != 1)
-      {
-         // decrementing an iterator always leads to a valid position:
-         --m_position;
-         extract_current();
-         m_current = m_values[1] ? 1 : 0;
-      }
-      else
-      {
-         m_current = 0;
-      }
-   }
-   BaseIterator base()const
-   {
-      return m_position;
-   }
-   // construct:
-   u32_to_u16_iterator() : m_position(), m_current(0)
-   {
-      m_values[0] = 0;
-      m_values[1] = 0;
-      m_values[2] = 0;
-   }
-   u32_to_u16_iterator(BaseIterator b) : m_position(b), m_current(2)
-   {
-      m_values[0] = 0;
-      m_values[1] = 0;
-      m_values[2] = 0;
-   }
-private:
-
-   void extract_current()const
-   {
-      // begin by checking for a code point out of range:
-      ::boost::uint32_t v = *m_position;
-      if(v >= 0x10000u)
-      {
-         if(v > 0x10FFFFu)
-            detail::invalid_utf32_code_point(*m_position);
-         // split into two surrogates:
-         m_values[0] = static_cast<U16Type>(v >> 10) + detail::high_surrogate_base;
-         m_values[1] = static_cast<U16Type>(v & detail::ten_bit_mask) + detail::low_surrogate_base;
-         m_current = 0;
-         BOOST_ASSERT(detail::is_high_surrogate(m_values[0]));
-         BOOST_ASSERT(detail::is_low_surrogate(m_values[1]));
-      }
-      else
-      {
-         // 16-bit code point:
-         m_values[0] = static_cast<U16Type>(*m_position);
-         m_values[1] = 0;
-         m_current = 0;
-         // value must not be a surrogate:
-         if(detail::is_surrogate(m_values[0]))
-            detail::invalid_utf32_code_point(*m_position);
-      }
-   }
-   BaseIterator m_position;
-   mutable U16Type m_values[3];
-   mutable unsigned m_current;
-};
-
-template <class BaseIterator, class U32Type = ::boost::uint32_t>
-class u16_to_u32_iterator
-   : public boost::iterator_facade<u16_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type>
-{
-   typedef boost::iterator_facade<u16_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type;
-   // special values for pending iterator reads:
-   BOOST_STATIC_CONSTANT(U32Type, pending_read = 0xffffffffu);
-
-#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION)
-   typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
-
-   BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 16);
-   BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32);
-#endif
-
-public:
-   typename base_type::reference
-      dereference()const
-   {
-      if(m_value == pending_read)
-         extract_current();
-      return m_value;
-   }
-   bool equal(const u16_to_u32_iterator& that)const
-   {
-      return m_position == that.m_position;
-   }
-   void increment()
-   {
-      // skip high surrogate first if there is one:
-      if(detail::is_high_surrogate(*m_position)) ++m_position;
-      ++m_position;
-      m_value = pending_read;
-   }
-   void decrement()
-   {
-      --m_position;
-      // if we have a low surrogate then go back one more:
-      if(detail::is_low_surrogate(*m_position)) 
-         --m_position;
-      m_value = pending_read;
-   }
-   BaseIterator base()const
-   {
-      return m_position;
-   }
-   // construct:
-   u16_to_u32_iterator() : m_position()
-   {
-      m_value = pending_read;
-   }
-   u16_to_u32_iterator(BaseIterator b) : m_position(b)
-   {
-      m_value = pending_read;
-   }
-   //
-   // Range checked version:
-   //
-   u16_to_u32_iterator(BaseIterator b, BaseIterator start, BaseIterator end) : m_position(b)
-   {
-      m_value = pending_read;
-      //
-      // The range must not start with a low surrogate, or end in a high surrogate,
-      // otherwise we run the risk of running outside the underlying input range.
-      // Likewise b must not be located at a low surrogate.
-      //
-      boost::uint16_t val;
-      if(start != end)
-      {
-         if((b != start) && (b != end))
-         {
-            val = *b;
-            if(detail::is_surrogate(val) && ((val & 0xFC00u) == 0xDC00u))
-               invalid_code_point(val);
-         }
-         val = *start;
-         if(detail::is_surrogate(val) && ((val & 0xFC00u) == 0xDC00u))
-            invalid_code_point(val);
-         val = *--end;
-         if(detail::is_high_surrogate(val))
-            invalid_code_point(val);
-      }
-   }
-private:
-   static void invalid_code_point(::boost::uint16_t val)
-   {
-#ifndef BOOST_NO_STD_LOCALE
-      std::stringstream ss;
-      ss << "Misplaced UTF-16 surrogate U+" << std::showbase << std::hex << val << " encountered while trying to encode UTF-32 sequence";
-      std::out_of_range e(ss.str());
-#else
-      std::out_of_range e("Misplaced UTF-16 surrogate encountered while trying to encode UTF-32 sequence");
-#endif
-      boost::throw_exception(e);
-   }
-   void extract_current()const
-   {
-      m_value = static_cast<U32Type>(static_cast< ::boost::uint16_t>(*m_position));
-      // if the last value is a high surrogate then adjust m_position and m_value as needed:
-      if(detail::is_high_surrogate(*m_position))
-      {
-         // precondition; next value must have be a low-surrogate:
-         BaseIterator next(m_position);
-         ::boost::uint16_t t = *++next;
-         if((t & 0xFC00u) != 0xDC00u)
-            invalid_code_point(t);
-         m_value = (m_value - detail::high_surrogate_base) << 10;
-         m_value |= (static_cast<U32Type>(static_cast< ::boost::uint16_t>(t)) & detail::ten_bit_mask);
-      }
-      // postcondition; result must not be a surrogate:
-      if(detail::is_surrogate(m_value))
-         invalid_code_point(static_cast< ::boost::uint16_t>(m_value));
-   }
-   BaseIterator m_position;
-   mutable U32Type m_value;
-};
-
-template <class BaseIterator, class U8Type = ::boost::uint8_t>
-class u32_to_u8_iterator
-   : public boost::iterator_facade<u32_to_u8_iterator<BaseIterator, U8Type>, U8Type, std::bidirectional_iterator_tag, const U8Type>
-{
-   typedef boost::iterator_facade<u32_to_u8_iterator<BaseIterator, U8Type>, U8Type, std::bidirectional_iterator_tag, const U8Type> base_type;
-   
-#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION)
-   typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
-
-   BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32);
-   BOOST_STATIC_ASSERT(sizeof(U8Type)*CHAR_BIT == 8);
-#endif
-
-public:
-   typename base_type::reference
-      dereference()const
-   {
-      if(m_current == 4)
-         extract_current();
-      return m_values[m_current];
-   }
-   bool equal(const u32_to_u8_iterator& that)const
-   {
-      if(m_position == that.m_position)
-      {
-         // either the m_current's must be equal, or one must be 0 and 
-         // the other 4: which means neither must have bits 1 or 2 set:
-         return (m_current == that.m_current)
-            || (((m_current | that.m_current) & 3) == 0);
-      }
-      return false;
-   }
-   void increment()
-   {
-      // if we have a pending read then read now, so that we know whether
-      // to skip a position, or move to a low-surrogate:
-      if(m_current == 4)
-      {
-         // pending read:
-         extract_current();
-      }
-      // move to the next surrogate position:
-      ++m_current;
-      // if we've reached the end skip a position:
-      if(m_values[m_current] == 0)
-      {
-         m_current = 4;
-         ++m_position;
-      }
-   }
-   void decrement()
-   {
-      if((m_current & 3) == 0)
-      {
-         --m_position;
-         extract_current();
-         m_current = 3;
-         while(m_current && (m_values[m_current] == 0))
-            --m_current;
-      }
-      else
-         --m_current;
-   }
-   BaseIterator base()const
-   {
-      return m_position;
-   }
-   // construct:
-   u32_to_u8_iterator() : m_position(), m_current(0)
-   {
-      m_values[0] = 0;
-      m_values[1] = 0;
-      m_values[2] = 0;
-      m_values[3] = 0;
-      m_values[4] = 0;
-   }
-   u32_to_u8_iterator(BaseIterator b) : m_position(b), m_current(4)
-   {
-      m_values[0] = 0;
-      m_values[1] = 0;
-      m_values[2] = 0;
-      m_values[3] = 0;
-      m_values[4] = 0;
-   }
-private:
-
-   void extract_current()const
-   {
-      boost::uint32_t c = *m_position;
-      if(c > 0x10FFFFu)
-         detail::invalid_utf32_code_point(c);
-      if(c < 0x80u)
-      {
-         m_values[0] = static_cast<unsigned char>(c);
-         m_values[1] = static_cast<unsigned char>(0u);
-         m_values[2] = static_cast<unsigned char>(0u);
-         m_values[3] = static_cast<unsigned char>(0u);
-      }
-      else if(c < 0x800u)
-      {
-         m_values[0] = static_cast<unsigned char>(0xC0u + (c >> 6));
-         m_values[1] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
-         m_values[2] = static_cast<unsigned char>(0u);
-         m_values[3] = static_cast<unsigned char>(0u);
-      }
-      else if(c < 0x10000u)
-      {
-         m_values[0] = static_cast<unsigned char>(0xE0u + (c >> 12));
-         m_values[1] = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
-         m_values[2] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
-         m_values[3] = static_cast<unsigned char>(0u);
-      }
-      else
-      {
-         m_values[0] = static_cast<unsigned char>(0xF0u + (c >> 18));
-         m_values[1] = static_cast<unsigned char>(0x80u + ((c >> 12) & 0x3Fu));
-         m_values[2] = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
-         m_values[3] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
-      }
-      m_current= 0;
-   }
-   BaseIterator m_position;
-   mutable U8Type m_values[5];
-   mutable unsigned m_current;
-};
-
-template <class BaseIterator, class U32Type = ::boost::uint32_t>
-class u8_to_u32_iterator
-   : public boost::iterator_facade<u8_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type>
-{
-   typedef boost::iterator_facade<u8_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type;
-   // special values for pending iterator reads:
-   BOOST_STATIC_CONSTANT(U32Type, pending_read = 0xffffffffu);
-
-#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION)
-   typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
-
-   BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 8);
-   BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32);
-#endif
-
-public:
-   typename base_type::reference
-      dereference()const
-   {
-      if(m_value == pending_read)
-         extract_current();
-      return m_value;
-   }
-   bool equal(const u8_to_u32_iterator& that)const
-   {
-      return m_position == that.m_position;
-   }
-   void increment()
-   {
-      // We must not start with a continuation character:
-      if((static_cast<boost::uint8_t>(*m_position) & 0xC0) == 0x80)
-         invalid_sequence();
-      // skip high surrogate first if there is one:
-      unsigned c = detail::utf8_byte_count(*m_position);
-      if(m_value == pending_read)
-      {
-         // Since we haven't read in a value, we need to validate the code points:
-         for(unsigned i = 0; i < c; ++i)
-         {
-            ++m_position;
-            // We must have a continuation byte:
-            if((i != c - 1) && ((static_cast<boost::uint8_t>(*m_position) & 0xC0) != 0x80))
-               invalid_sequence();
-         }
-      }
-      else
-      {
-         std::advance(m_position, c);
-      }
-      m_value = pending_read;
-   }
-   void decrement()
-   {
-      // Keep backtracking until we don't have a trailing character:
-      unsigned count = 0;
-      while((*--m_position & 0xC0u) == 0x80u) ++count;
-      // now check that the sequence was valid:
-      if(count != detail::utf8_trailing_byte_count(*m_position))
-         invalid_sequence();
-      m_value = pending_read;
-   }
-   BaseIterator base()const
-   {
-      return m_position;
-   }
-   // construct:
-   u8_to_u32_iterator() : m_position()
-   {
-      m_value = pending_read;
-   }
-   u8_to_u32_iterator(BaseIterator b) : m_position(b)
-   {
-      m_value = pending_read;
-   }
-   //
-   // Checked constructor:
-   //
-   u8_to_u32_iterator(BaseIterator b, BaseIterator start, BaseIterator end) : m_position(b)
-   {
-      m_value = pending_read;
-      //
-      // We must not start with a continuation character, or end with a 
-      // truncated UTF-8 sequence otherwise we run the risk of going past
-      // the start/end of the underlying sequence:
-      //
-      if(start != end)
-      {
-         unsigned char v = *start;
-         if((v & 0xC0u) == 0x80u)
-            invalid_sequence();
-         if((b != start) && (b != end) && ((*b & 0xC0u) == 0x80u))
-            invalid_sequence();
-         BaseIterator pos = end;
-         do
-         {
-            v = *--pos;
-         }
-         while((start != pos) && ((v & 0xC0u) == 0x80u));
-         std::ptrdiff_t extra = detail::utf8_byte_count(v);
-         if(std::distance(pos, end) < extra)
-            invalid_sequence();
-      }
-   }
-private:
-   static void invalid_sequence()
-   {
-      std::out_of_range e("Invalid UTF-8 sequence encountered while trying to encode UTF-32 character");
-      boost::throw_exception(e);
-   }
-   void extract_current()const
-   {
-      m_value = static_cast<U32Type>(static_cast< ::boost::uint8_t>(*m_position));
-      // we must not have a continuation character:
-      if((m_value & 0xC0u) == 0x80u)
-         invalid_sequence();
-      // see how many extra bytes we have:
-      unsigned extra = detail::utf8_trailing_byte_count(*m_position);
-      // extract the extra bits, 6 from each extra byte:
-      BaseIterator next(m_position);
-      for(unsigned c = 0; c < extra; ++c)
-      {
-         ++next;
-         m_value <<= 6;
-         // We must have a continuation byte:
-         if((static_cast<boost::uint8_t>(*next) & 0xC0) != 0x80)
-            invalid_sequence();
-         m_value += static_cast<boost::uint8_t>(*next) & 0x3Fu;
-      }
-      // we now need to remove a few of the leftmost bits, but how many depends
-      // upon how many extra bytes we've extracted:
-      static const boost::uint32_t masks[4] = 
-      {
-         0x7Fu,
-         0x7FFu,
-         0xFFFFu,
-         0x1FFFFFu,
-      };
-      m_value &= masks[extra];
-      // check the result:
-      if(m_value > static_cast<U32Type>(0x10FFFFu))
-         invalid_sequence();
-   }
-   BaseIterator m_position;
-   mutable U32Type m_value;
-};
-
-template <class BaseIterator>
-class utf16_output_iterator
-{
-public:
-   typedef void                                   difference_type;
-   typedef void                                   value_type;
-   typedef boost::uint32_t*                       pointer;
-   typedef boost::uint32_t&                       reference;
-   typedef std::output_iterator_tag               iterator_category;
-
-   utf16_output_iterator(const BaseIterator& b)
-      : m_position(b){}
-   utf16_output_iterator(const utf16_output_iterator& that)
-      : m_position(that.m_position){}
-   utf16_output_iterator& operator=(const utf16_output_iterator& that)
-   {
-      m_position = that.m_position;
-      return *this;
-   }
-   const utf16_output_iterator& operator*()const
-   {
-      return *this;
-   }
-   void operator=(boost::uint32_t val)const
-   {
-      push(val);
-   }
-   utf16_output_iterator& operator++()
-   {
-      return *this;
-   }
-   utf16_output_iterator& operator++(int)
-   {
-      return *this;
-   }
-   BaseIterator base()const
-   {
-      return m_position;
-   }
-private:
-   void push(boost::uint32_t v)const
-   {
-      if(v >= 0x10000u)
-      {
-         // begin by checking for a code point out of range:
-         if(v > 0x10FFFFu)
-            detail::invalid_utf32_code_point(v);
-         // split into two surrogates:
-         *m_position++ = static_cast<boost::uint16_t>(v >> 10) + detail::high_surrogate_base;
-         *m_position++ = static_cast<boost::uint16_t>(v & detail::ten_bit_mask) + detail::low_surrogate_base;
-      }
-      else
-      {
-         // 16-bit code point:
-         // value must not be a surrogate:
-         if(detail::is_surrogate(v))
-            detail::invalid_utf32_code_point(v);
-         *m_position++ = static_cast<boost::uint16_t>(v);
-      }
-   }
-   mutable BaseIterator m_position;
-};
-
-template <class BaseIterator>
-class utf8_output_iterator
-{
-public:
-   typedef void                                   difference_type;
-   typedef void                                   value_type;
-   typedef boost::uint32_t*                       pointer;
-   typedef boost::uint32_t&                       reference;
-   typedef std::output_iterator_tag               iterator_category;
-
-   utf8_output_iterator(const BaseIterator& b)
-      : m_position(b){}
-   utf8_output_iterator(const utf8_output_iterator& that)
-      : m_position(that.m_position){}
-   utf8_output_iterator& operator=(const utf8_output_iterator& that)
-   {
-      m_position = that.m_position;
-      return *this;
-   }
-   const utf8_output_iterator& operator*()const
-   {
-      return *this;
-   }
-   void operator=(boost::uint32_t val)const
-   {
-      push(val);
-   }
-   utf8_output_iterator& operator++()
-   {
-      return *this;
-   }
-   utf8_output_iterator& operator++(int)
-   {
-      return *this;
-   }
-   BaseIterator base()const
-   {
-      return m_position;
-   }
-private:
-   void push(boost::uint32_t c)const
-   {
-      if(c > 0x10FFFFu)
-         detail::invalid_utf32_code_point(c);
-      if(c < 0x80u)
-      {
-         *m_position++ = static_cast<unsigned char>(c);
-      }
-      else if(c < 0x800u)
-      {
-         *m_position++ = static_cast<unsigned char>(0xC0u + (c >> 6));
-         *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
-      }
-      else if(c < 0x10000u)
-      {
-         *m_position++ = static_cast<unsigned char>(0xE0u + (c >> 12));
-         *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
-         *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
-      }
-      else
-      {
-         *m_position++ = static_cast<unsigned char>(0xF0u + (c >> 18));
-         *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 12) & 0x3Fu));
-         *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
-         *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
-      }
-   }
-   mutable BaseIterator m_position;
-};
-
-} // namespace boost
-
-#endif // BOOST_REGEX_UNICODE_ITERATOR_HPP
-

third_party/libosmium/include/mmap_for_windows.hpp

@@ -1,103 +0,0 @@
-#ifndef MMAP_FOR_WINDOWS_HPP
-#define MMAP_FOR_WINDOWS_HPP
-
-/* mmap() replacement for Windows
- *
- * Author: Mike Frysinger <vapier@gentoo.org>
- * Placed into the public domain
- */
-
-/* References:
- * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx
- * CloseHandle:       http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx
- * MapViewOfFile:     http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx
- * UnmapViewOfFile:   http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx
- */
-
-#include <io.h>
-#include <windows.h>
-#include <sys/types.h>
-
-#define PROT_READ     0x1
-#define PROT_WRITE    0x2
-/* This flag is only available in WinXP+ */
-#ifdef FILE_MAP_EXECUTE
-#define PROT_EXEC     0x4
-#else
-#define PROT_EXEC        0x0
-#define FILE_MAP_EXECUTE 0
-#endif
-
-#define MAP_SHARED    0x01
-#define MAP_PRIVATE   0x02
-#define MAP_ANONYMOUS 0x20
-#define MAP_ANON      MAP_ANONYMOUS
-#define MAP_FAILED    ((void *) -1)
-
-static DWORD dword_hi(uint64_t x) {
-    return static_cast<DWORD>(x >> 32);
-}
-
-static DWORD dword_lo(uint64_t x) {
-    return static_cast<DWORD>(x & 0xffffffff);
-}
-
-static void *mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset)
-{
-    if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC))
-        return MAP_FAILED;
-    if (fd == -1) {
-        if (!(flags & MAP_ANON) || offset)
-            return MAP_FAILED;
-    } else if (flags & MAP_ANON)
-        return MAP_FAILED;
-
-    DWORD flProtect;
-    if (prot & PROT_WRITE) {
-        if (prot & PROT_EXEC)
-            flProtect = PAGE_EXECUTE_READWRITE;
-        else
-            flProtect = PAGE_READWRITE;
-    } else if (prot & PROT_EXEC) {
-        if (prot & PROT_READ)
-            flProtect = PAGE_EXECUTE_READ;
-        else if (prot & PROT_EXEC)
-            flProtect = PAGE_EXECUTE;
-    } else
-        flProtect = PAGE_READONLY;
-
-    uint64_t end = static_cast<uint64_t>(length) + offset;
-    HANDLE mmap_fd;
-    if (fd == -1)
-        mmap_fd = INVALID_HANDLE_VALUE;
-    else
-        mmap_fd = (HANDLE)_get_osfhandle(fd);
-
-    HANDLE h = CreateFileMapping(mmap_fd, NULL, flProtect, dword_hi(end), dword_lo(end), NULL);
-    if (h == NULL)
-        return MAP_FAILED;
-
-    DWORD dwDesiredAccess;
-    if (prot & PROT_WRITE)
-        dwDesiredAccess = FILE_MAP_WRITE;
-    else
-        dwDesiredAccess = FILE_MAP_READ;
-    if (prot & PROT_EXEC)
-        dwDesiredAccess |= FILE_MAP_EXECUTE;
-    if (flags & MAP_PRIVATE)
-        dwDesiredAccess |= FILE_MAP_COPY;
-    void *ret = MapViewOfFile(h, dwDesiredAccess, dword_hi(offset), dword_lo(offset), length);
-    if (ret == NULL) {
-        CloseHandle(h);
-        ret = MAP_FAILED;
-    }
-    return ret;
-}
-
-static int munmap(void *addr, size_t length)
-{
-    return UnmapViewOfFile(addr) ? 0 : -1;
-    /* ruh-ro, we leaked handle from CreateFileMapping() ... */
-}
-
-#endif

third_party/libosmium/include/osmium/area/assembler.hpp

@@ -171,7 +171,7 @@ namespace osmium {
             }
 
             void add_tags_to_area(osmium::builder::AreaBuilder& builder, const osmium::Relation& relation) const {
-                auto count = std::count_if(relation.tags().begin(), relation.tags().end(), filter());
+                const auto count = std::count_if(relation.tags().begin(), relation.tags().end(), filter());
 
                 if (debug()) {
                     std::cerr << "  found " << count << " tags on relation (without ignored ones)\n";
@@ -331,7 +331,7 @@ namespace osmium {
                 if (debug()) {
                     std::cerr << "      has_closed_subring_back()\n";
                 }
-                auto end = ring.segments().end();
+                const auto end = ring.segments().end();
                 for (auto it = ring.segments().begin() + 1; it != end - 1; ++it) {
                     if (has_same_location(nr, it->first())) {
                         split_off_subring(ring, it, it, end);
@@ -348,7 +348,7 @@ namespace osmium {
                 if (debug()) {
                     std::cerr << "      has_closed_subring_front()\n";
                 }
-                auto end = ring.segments().end();
+                const auto end = ring.segments().end();
                 for (auto it = ring.segments().begin() + 1; it != end - 1; ++it) {
                     if (has_same_location(nr, it->second())) {
                         split_off_subring(ring, it, ring.segments().begin(), it+1);
@@ -366,22 +366,22 @@ namespace osmium {
                 osmium::area::detail::ProtoRing::segments_type segments(ring.segments().size());
                 std::copy(ring.segments().begin(), ring.segments().end(), segments.begin());
                 std::sort(segments.begin(), segments.end());
-                auto it = std::adjacent_find(segments.begin(), segments.end(), [this](const osmium::area::detail::NodeRefSegment& s1, const osmium::area::detail::NodeRefSegment& s2) {
+                const auto it = std::adjacent_find(segments.begin(), segments.end(), [this](const osmium::area::detail::NodeRefSegment& s1, const osmium::area::detail::NodeRefSegment& s2) {
                     return has_same_location(s1.first(), s2.first());
                 });
                 if (it == segments.end()) {
                     return false;
                 }
-                auto r1 = std::find_first_of(ring.segments().begin(), ring.segments().end(), it, it+1);
+                const auto r1 = std::find_first_of(ring.segments().begin(), ring.segments().end(), it, it+1);
                 assert(r1 != ring.segments().end());
-                auto r2 = std::find_first_of(ring.segments().begin(), ring.segments().end(), it+1, it+2);
+                const auto r2 = std::find_first_of(ring.segments().begin(), ring.segments().end(), it+1, it+2);
                 assert(r2 != ring.segments().end());
 
                 if (debug()) {
                     std::cerr << "      found subring in ring " << ring << " at " << it->first() << "\n";
                 }
 
-                auto m = std::minmax(r1, r2);
+                const auto m = std::minmax(r1, r2);
 
                 ProtoRing new_ring(m.first, m.second);
                 ring.remove_segments(m.first, m.second);
@@ -537,7 +537,7 @@ namespace osmium {
                 }
 
                 for (const auto ringptr : m_outer_rings) {
-                    for (const auto segment : ringptr->segments()) {
+                    for (const auto& segment : ringptr->segments()) {
                         if (!segment.role_outer()) {
                             ++m_inner_outer_mismatches;
                             if (debug()) {
@@ -550,7 +550,7 @@ namespace osmium {
                     }
                 }
                 for (const auto ringptr : m_inner_rings) {
-                    for (const auto segment : ringptr->segments()) {
+                    for (const auto& segment : ringptr->segments()) {
                         if (!segment.role_inner()) {
                             ++m_inner_outer_mismatches;
                             if (debug()) {

third_party/libosmium/include/osmium/area/multipolygon_collector.hpp

@@ -41,6 +41,8 @@ DEALINGS IN THE SOFTWARE.
 
 #include <osmium/memory/buffer.hpp>
 #include <osmium/osm/item_type.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/node_ref.hpp>
 #include <osmium/osm/relation.hpp>
 #include <osmium/osm/tag.hpp>
 #include <osmium/osm/way.hpp>
@@ -49,8 +51,6 @@ DEALINGS IN THE SOFTWARE.
 
 namespace osmium {
 
-    struct invalid_location;
-
     namespace relations {
         class RelationMeta;
     }
@@ -107,8 +107,8 @@ namespace osmium {
             }
 
             /**
-             * We are interested in all relations tagged with type=multipolygon or
+             * We are interested in all relations tagged with type=multipolygon
-             * type=boundary.
+             * or type=boundary.
              *
              * Overwritten from the base class.
              */
@@ -142,15 +142,22 @@ namespace osmium {
              * Overwritten from the base class.
              */
             void way_not_in_any_relation(const osmium::Way& way) {
-                if (way.nodes().size() > 3 && way.ends_have_same_location()) {
+                // you need at least 4 nodes to make up a polygon
-                    // way is closed and has enough nodes, build simple multipolygon
+                if (way.nodes().size() <= 3) {
-                    try {
+                    return;
+                }
+                try {
+                    if (!way.nodes().front().location() || !way.nodes().back().location()) {
+                        throw osmium::invalid_location("invalid location");
+                    }
+                    if (way.ends_have_same_location()) {
+                        // way is closed and has enough nodes, build simple multipolygon
                         TAssembler assembler(m_assembler_config);
                         assembler(way, m_output_buffer);
                         possibly_flush_output_buffer();
-                    } catch (osmium::invalid_location&) {
-                        // XXX ignore
                     }
+                } catch (osmium::invalid_location&) {
+                    // XXX ignore
                 }
             }
 

third_party/libosmium/include/osmium/builder/builder.hpp

@@ -147,6 +147,7 @@ namespace osmium {
              * @param length Length of data in bytes. If data is a
              *               \0-terminated string, length must contain the
              *               \0 byte.
+             * @returns The number of bytes appended (length).
              */
             osmium::memory::item_size_type append(const char* data, const osmium::memory::item_size_type length) {
                 unsigned char* target = m_buffer.reserve_space(length);
@@ -156,11 +157,24 @@ namespace osmium {
 
             /**
              * Append \0-terminated string to buffer.
+             *
+             * @param str \0-terminated string.
+             * @returns The number of bytes appended (strlen(str) + 1).
              */
             osmium::memory::item_size_type append(const char* str) {
                 return append(str, static_cast<osmium::memory::item_size_type>(std::strlen(str) + 1));
             }
 
+            /**
+             * Append '\0' to the buffer.
+             *
+             * @returns The number of bytes appended (always 1).
+             */
+            osmium::memory::item_size_type append_zero() {
+                *m_buffer.reserve_space(1) = '\0';
+                return 1;
+            }
+
             /// Return the buffer this builder is using.
             osmium::memory::Buffer& buffer() noexcept {
                 return m_buffer;
@@ -188,11 +202,11 @@ namespace osmium {
              * Add user name to buffer.
              *
              * @param user Pointer to user name.
-             * @param length Length of user name including \0 byte.
+             * @param length Length of user name (without \0 termination).
              */
             void add_user(const char* user, const string_size_type length) {
-                object().set_user_size(length);
+                object().set_user_size(length + 1);
-                add_size(append(user, length));
+                add_size(append(user, length) + append_zero());
                 add_padding(true);
             }
 
@@ -202,7 +216,7 @@ namespace osmium {
              * @param user Pointer to \0-terminated user name.
              */
             void add_user(const char* user) {
-                add_user(user, static_cast_with_assert<string_size_type>(std::strlen(user) + 1));
+                add_user(user, static_cast_with_assert<string_size_type>(std::strlen(user)));
             }
 
             /**
@@ -211,7 +225,7 @@ namespace osmium {
              * @param user User name.
              */
             void add_user(const std::string& user) {
-                add_user(user.data(), static_cast_with_assert<string_size_type>(user.size() + 1));
+                add_user(user.data(), static_cast_with_assert<string_size_type>(user.size()));
             }
 
         }; // class ObjectBuilder

third_party/libosmium/include/osmium/builder/osm_object_builder.hpp

@@ -72,13 +72,25 @@ namespace osmium {
             /**
              * Add tag to buffer.
              *
-             * @param key Tag key.
+             * @param key Tag key (0-terminated string).
-             * @param value Tag value.
+             * @param value Tag value (0-terminated string).
              */
             void add_tag(const char* key, const char* value) {
                 add_size(append(key) + append(value));
             }
 
+            /**
+             * Add tag to buffer.
+             *
+             * @param key Pointer to tag key.
+             * @param key_length Length of key (not including the \0 byte).
+             * @param value Pointer to tag value.
+             * @param value_length Length of value (not including the \0 byte).
+             */
+            void add_tag(const char* key, const string_size_type key_length, const char* value, const string_size_type value_length) {
+                add_size(append(key, key_length) + append_zero() + append(value, value_length) + append_zero());
+            }
+
             /**
              * Add tag to buffer.
              *
@@ -128,11 +140,11 @@ namespace osmium {
              * @param member Relation member object where the length of the role
              *               will be set.
              * @param role The role.
-             * @param length Length of role string including \0 termination.
+             * @param length Length of role (without \0 termination).
              */
             void add_role(osmium::RelationMember& member, const char* role, const string_size_type length) {
-                member.set_role_size(length);
+                member.set_role_size(length + 1);
-                add_size(append(role, length));
+                add_size(append(role, length) + append_zero());
                 add_padding(true);
             }
 
@@ -144,7 +156,7 @@ namespace osmium {
              * @param role \0-terminated role.
              */
             void add_role(osmium::RelationMember& member, const char* role) {
-                add_role(member, role, static_cast_with_assert<string_size_type>(std::strlen(role) + 1));
+                add_role(member, role, static_cast_with_assert<string_size_type>(std::strlen(role)));
             }
 
             /**
@@ -155,7 +167,7 @@ namespace osmium {
              * @param role Role.
              */
             void add_role(osmium::RelationMember& member, const std::string& role) {
-                add_role(member, role.data(), static_cast_with_assert<string_size_type>(role.size() + 1));
+                add_role(member, role.data(), static_cast_with_assert<string_size_type>(role.size()));
             }
 
         public:
@@ -174,18 +186,33 @@ namespace osmium {
              * @param type The type (node, way, or relation).
              * @param ref The ID of the member.
              * @param role The role of the member.
+             * @param role_length Length of the role (without \0 termination).
+             * @param full_member Optional pointer to the member object. If it
+             *                    is available a copy will be added to the
+             *                    relation.
+             */
+            void add_member(osmium::item_type type, object_id_type ref, const char* role, const string_size_type role_length, const osmium::OSMObject* full_member = nullptr) {
+                osmium::RelationMember* member = reserve_space_for<osmium::RelationMember>();
+                new (member) osmium::RelationMember(ref, type, full_member != nullptr);
+                add_size(sizeof(RelationMember));
+                add_role(*member, role, role_length);
+                if (full_member) {
+                    add_item(full_member);
+                }
+            }
+
+            /**
+             * Add a member to the relation.
+             *
+             * @param type The type (node, way, or relation).
+             * @param ref The ID of the member.
+             * @param role The role of the member (\0 terminated string).
              * @param full_member Optional pointer to the member object. If it
              *                    is available a copy will be added to the
              *                    relation.
              */
             void add_member(osmium::item_type type, object_id_type ref, const char* role, const osmium::OSMObject* full_member = nullptr) {
-                osmium::RelationMember* member = reserve_space_for<osmium::RelationMember>();
+                add_member(type, ref, role, strlen(role), full_member);
-                new (member) osmium::RelationMember(ref, type, full_member != nullptr);
-                add_size(sizeof(RelationMember));
-                add_role(*member, role);
-                if (full_member) {
-                    add_item(full_member);
-                }
             }
 
             /**
@@ -199,13 +226,7 @@ namespace osmium {
              *                    relation.
              */
             void add_member(osmium::item_type type, object_id_type ref, const std::string& role, const osmium::OSMObject* full_member = nullptr) {
-                osmium::RelationMember* member = reserve_space_for<osmium::RelationMember>();
+                add_member(type, ref, role.data(), role.size(), full_member);
-                new (member) osmium::RelationMember(ref, type, full_member != nullptr);
-                add_size(sizeof(RelationMember));
-                add_role(*member, role);
-                if (full_member) {
-                    add_item(full_member);
-                }
             }
 
         }; // class RelationMemberListBuilder

third_party/libosmium/include/osmium/geom/factory.hpp

@@ -54,14 +54,43 @@ namespace osmium {
      * Exception thrown when an invalid geometry is encountered. An example
      * would be a linestring with less than two points.
      */
-    struct geometry_error : public std::runtime_error {
+    class geometry_error : public std::runtime_error {
 
-        geometry_error(const std::string& what) :
+        std::string m_message;
-            std::runtime_error(what) {
+        osmium::object_id_type m_id;
+
+    public:
+
+        geometry_error(const std::string& message, const char* object_type = "", osmium::object_id_type id = 0) :
+            std::runtime_error(message),
+            m_message(message),
+            m_id(id) {
+            if (m_id != 0) {
+                m_message += " (";
+                m_message += object_type;
+                m_message += "_id=";
+                m_message += std::to_string(m_id);
+                m_message += ")";
+            }
         }
 
-        geometry_error(const char* what) :
+        void set_id(const char* object_type, osmium::object_id_type id) {
-            std::runtime_error(what) {
+            if (m_id == 0 && id != 0) {
+                m_message += " (";
+                m_message += object_type;
+                m_message += "_id=";
+                m_message += std::to_string(id);
+                m_message += ")";
+            }
+            m_id = id;
+        }
+
+        osmium::object_id_type id() const noexcept {
+            return m_id;
+        }
+
+        virtual const char* what() const noexcept override {
+            return m_message.c_str();
         }
 
     }; // struct geometry_error
@@ -174,11 +203,21 @@ namespace osmium {
             }
 
             point_type create_point(const osmium::Node& node) {
-                return create_point(node.location());
+                try {
+                    return create_point(node.location());
+                } catch (osmium::geometry_error& e) {
+                    e.set_id("node", node.id());
+                    throw;
+                }
             }
 
             point_type create_point(const osmium::NodeRef& node_ref) {
-                return create_point(node_ref.location());
+                try {
+                    return create_point(node_ref.location());
+                } catch (osmium::geometry_error& e) {
+                    e.set_id("node", node_ref.ref());
+                    throw;
+                }
             }
 
             /* LineString */
@@ -240,14 +279,19 @@ namespace osmium {
                 }
 
                 if (num_points < 2) {
-                    throw osmium::geometry_error("not enough points for linestring");
+                    throw osmium::geometry_error("need at least two points for linestring");
                 }
 
                 return linestring_finish(num_points);
             }
 
             linestring_type create_linestring(const osmium::Way& way, use_nodes un=use_nodes::unique, direction dir=direction::forward) {
-                return create_linestring(way.nodes(), un, dir);
+                try {
+                    return create_linestring(way.nodes(), un, dir);
+                } catch (osmium::geometry_error& e) {
+                    e.set_id("way", way.id());
+                    throw;
+                }
             }
 
             /* Polygon */
@@ -283,40 +327,86 @@ namespace osmium {
                 return m_impl.polygon_finish(num_points);
             }
 
-            /* MultiPolygon */
+            polygon_type create_polygon(const osmium::WayNodeList& wnl, use_nodes un = use_nodes::unique, direction dir = direction::forward) {
+                polygon_start();
+                size_t num_points = 0;
 
-            multipolygon_type create_multipolygon(const osmium::Area& area) {
+                if (un == use_nodes::unique) {
-                size_t num_polygons = 0;
+                    osmium::Location last_location;
-                size_t num_rings = 0;
+                    switch (dir) {
-                m_impl.multipolygon_start();
+                        case direction::forward:
-
+                            num_points = fill_polygon_unique(wnl.cbegin(), wnl.cend());
-                for (auto it = area.cbegin(); it != area.cend(); ++it) {
+                            break;
-                    const osmium::OuterRing& ring = static_cast<const osmium::OuterRing&>(*it);
+                        case direction::backward:
-                    if (it->type() == osmium::item_type::outer_ring) {
+                            num_points = fill_polygon_unique(wnl.crbegin(), wnl.crend());
-                        if (num_polygons > 0) {
+                            break;
-                            m_impl.multipolygon_polygon_finish();
+                    }
-                        }
+                } else {
-                        m_impl.multipolygon_polygon_start();
+                    switch (dir) {
-                        m_impl.multipolygon_outer_ring_start();
+                        case direction::forward:
-                        add_points(ring);
+                            num_points = fill_polygon(wnl.cbegin(), wnl.cend());
-                        m_impl.multipolygon_outer_ring_finish();
+                            break;
-                        ++num_rings;
+                        case direction::backward:
-                        ++num_polygons;
+                            num_points = fill_polygon(wnl.crbegin(), wnl.crend());
-                    } else if (it->type() == osmium::item_type::inner_ring) {
+                            break;
-                        m_impl.multipolygon_inner_ring_start();
-                        add_points(ring);
-                        m_impl.multipolygon_inner_ring_finish();
-                        ++num_rings;
                     }
                 }
 
-                // if there are no rings, this area is invalid
+                if (num_points < 4) {
-                if (num_rings == 0) {
+                    throw osmium::geometry_error("need at least four points for polygon");
-                    throw osmium::geometry_error("invalid area");
                 }
 
-                m_impl.multipolygon_polygon_finish();
+                return polygon_finish(num_points);
-                return m_impl.multipolygon_finish();
+            }
+
+            polygon_type create_polygon(const osmium::Way& way, use_nodes un=use_nodes::unique, direction dir=direction::forward) {
+                try {
+                    return create_polygon(way.nodes(), un, dir);
+                } catch (osmium::geometry_error& e) {
+                    e.set_id("way", way.id());
+                    throw;
+                }
+            }
+
+            /* MultiPolygon */
+
+            multipolygon_type create_multipolygon(const osmium::Area& area) {
+                try {
+                    size_t num_polygons = 0;
+                    size_t num_rings = 0;
+                    m_impl.multipolygon_start();
+
+                    for (auto it = area.cbegin(); it != area.cend(); ++it) {
+                        const osmium::OuterRing& ring = static_cast<const osmium::OuterRing&>(*it);
+                        if (it->type() == osmium::item_type::outer_ring) {
+                            if (num_polygons > 0) {
+                                m_impl.multipolygon_polygon_finish();
+                            }
+                            m_impl.multipolygon_polygon_start();
+                            m_impl.multipolygon_outer_ring_start();
+                            add_points(ring);
+                            m_impl.multipolygon_outer_ring_finish();
+                            ++num_rings;
+                            ++num_polygons;
+                        } else if (it->type() == osmium::item_type::inner_ring) {
+                            m_impl.multipolygon_inner_ring_start();
+                            add_points(ring);
+                            m_impl.multipolygon_inner_ring_finish();
+                            ++num_rings;
+                        }
+                    }
+
+                    // if there are no rings, this area is invalid
+                    if (num_rings == 0) {
+                        throw osmium::geometry_error("area contains no rings");
+                    }
+
+                    m_impl.multipolygon_polygon_finish();
+                    return m_impl.multipolygon_finish();
+                } catch (osmium::geometry_error& e) {
+                    e.set_id("area", area.id());
+                    throw;
+                }
             }
 
         }; // class GeometryFactory

third_party/libosmium/include/osmium/geom/geos.hpp

@@ -42,6 +42,8 @@ DEALINGS IN THE SOFTWARE.
  * @attention If you include this file, you'll need to link with `libgeos`.
  */
 
+#include <memory>
+#include <string>
 #include <utility>
 
 #include <geos/geom/Coordinate.h>
@@ -69,8 +71,8 @@ namespace osmium {
 
     struct geos_geometry_error : public geometry_error {
 
-        geos_geometry_error() :
+        geos_geometry_error(const char* message) :
-            geometry_error("geometry creation failed in GEOS library, see nested exception for details") {
+            geometry_error(std::string("geometry creation failed in GEOS library: ") + message) {
         }
 
     }; // struct geos_geometry_error
@@ -81,8 +83,9 @@ namespace osmium {
 
             class GEOSFactoryImpl {
 
-                geos::geom::PrecisionModel m_precision_model;
+                std::unique_ptr<const geos::geom::PrecisionModel> m_precision_model;
-                geos::geom::GeometryFactory m_geos_factory;
+                std::unique_ptr<geos::geom::GeometryFactory> m_our_geos_factory;
+                geos::geom::GeometryFactory* m_geos_factory;
 
                 std::unique_ptr<geos::geom::CoordinateSequence> m_coordinate_sequence;
                 std::vector<std::unique_ptr<geos::geom::LinearRing>> m_rings;
@@ -96,18 +99,25 @@ namespace osmium {
                 typedef std::unique_ptr<geos::geom::MultiPolygon> multipolygon_type;
                 typedef std::unique_ptr<geos::geom::LinearRing>   ring_type;
 
+                explicit GEOSFactoryImpl(geos::geom::GeometryFactory& geos_factory) :
+                    m_precision_model(nullptr),
+                    m_our_geos_factory(nullptr),
+                    m_geos_factory(&geos_factory) {
+                }
+
                 explicit GEOSFactoryImpl(int srid = -1) :
-                    m_precision_model(),
+                    m_precision_model(new geos::geom::PrecisionModel),
-                    m_geos_factory(&m_precision_model, srid) {
+                    m_our_geos_factory(new geos::geom::GeometryFactory(m_precision_model.get(), srid)),
+                    m_geos_factory(m_our_geos_factory.get()) {
                 }
 
                 /* Point */
 
                 point_type make_point(const osmium::geom::Coordinates& xy) const {
                     try {
-                        return point_type(m_geos_factory.createPoint(geos::geom::Coordinate(xy.x, xy.y)));
+                        return point_type(m_geos_factory->createPoint(geos::geom::Coordinate(xy.x, xy.y)));
-                    } catch (geos::util::GEOSException&) {
+                    } catch (geos::util::GEOSException& e) {
-                        THROW(osmium::geos_geometry_error());
+                        THROW(osmium::geos_geometry_error(e.what()));
                     }
                 }
 
@@ -115,25 +125,25 @@ namespace osmium {
 
                 void linestring_start() {
                     try {
-                        m_coordinate_sequence.reset(m_geos_factory.getCoordinateSequenceFactory()->create(static_cast<size_t>(0), 2));
+                        m_coordinate_sequence.reset(m_geos_factory->getCoordinateSequenceFactory()->create(static_cast<size_t>(0), 2));
-                    } catch (geos::util::GEOSException&) {
+                    } catch (geos::util::GEOSException& e) {
-                        THROW(osmium::geos_geometry_error());
+                        THROW(osmium::geos_geometry_error(e.what()));
                     }
                 }
 
                 void linestring_add_location(const osmium::geom::Coordinates& xy) {
                     try {
                         m_coordinate_sequence->add(geos::geom::Coordinate(xy.x, xy.y));
-                    } catch (geos::util::GEOSException&) {
+                    } catch (geos::util::GEOSException& e) {
-                        THROW(osmium::geos_geometry_error());
+                        THROW(osmium::geos_geometry_error(e.what()));
                     }
                 }
 
                 linestring_type linestring_finish(size_t /* num_points */) {
                     try {
-                        return linestring_type(m_geos_factory.createLineString(m_coordinate_sequence.release()));
+                        return linestring_type(m_geos_factory->createLineString(m_coordinate_sequence.release()));
-                    } catch (geos::util::GEOSException&) {
+                    } catch (geos::util::GEOSException& e) {
-                        THROW(osmium::geos_geometry_error());
+                        THROW(osmium::geos_geometry_error(e.what()));
                     }
                 }
 
@@ -154,50 +164,50 @@ namespace osmium {
                         std::transform(std::next(m_rings.begin(), 1), m_rings.end(), std::back_inserter(*inner_rings), [](std::unique_ptr<geos::geom::LinearRing>& r) {
                             return r.release();
                         });
-                        m_polygons.emplace_back(m_geos_factory.createPolygon(m_rings[0].release(), inner_rings));
+                        m_polygons.emplace_back(m_geos_factory->createPolygon(m_rings[0].release(), inner_rings));
                         m_rings.clear();
-                    } catch (geos::util::GEOSException&) {
+                    } catch (geos::util::GEOSException& e) {
-                        THROW(osmium::geos_geometry_error());
+                        THROW(osmium::geos_geometry_error(e.what()));
                     }
                 }
 
                 void multipolygon_outer_ring_start() {
                     try {
-                        m_coordinate_sequence.reset(m_geos_factory.getCoordinateSequenceFactory()->create(static_cast<size_t>(0), 2));
+                        m_coordinate_sequence.reset(m_geos_factory->getCoordinateSequenceFactory()->create(static_cast<size_t>(0), 2));
-                    } catch (geos::util::GEOSException&) {
+                    } catch (geos::util::GEOSException& e) {
-                        THROW(osmium::geos_geometry_error());
+                        THROW(osmium::geos_geometry_error(e.what()));
                     }
                 }
 
                 void multipolygon_outer_ring_finish() {
                     try {
-                        m_rings.emplace_back(m_geos_factory.createLinearRing(m_coordinate_sequence.release()));
+                        m_rings.emplace_back(m_geos_factory->createLinearRing(m_coordinate_sequence.release()));
-                    } catch (geos::util::GEOSException&) {
+                    } catch (geos::util::GEOSException& e) {
-                        THROW(osmium::geos_geometry_error());
+                        THROW(osmium::geos_geometry_error(e.what()));
                     }
                 }
 
                 void multipolygon_inner_ring_start() {
                     try {
-                        m_coordinate_sequence.reset(m_geos_factory.getCoordinateSequenceFactory()->create(static_cast<size_t>(0), 2));
+                        m_coordinate_sequence.reset(m_geos_factory->getCoordinateSequenceFactory()->create(static_cast<size_t>(0), 2));
-                    } catch (geos::util::GEOSException&) {
+                    } catch (geos::util::GEOSException& e) {
-                        THROW(osmium::geos_geometry_error());
+                        THROW(osmium::geos_geometry_error(e.what()));
                     }
                 }
 
                 void multipolygon_inner_ring_finish() {
                     try {
-                        m_rings.emplace_back(m_geos_factory.createLinearRing(m_coordinate_sequence.release()));
+                        m_rings.emplace_back(m_geos_factory->createLinearRing(m_coordinate_sequence.release()));
-                    } catch (geos::util::GEOSException&) {
+                    } catch (geos::util::GEOSException& e) {
-                        THROW(osmium::geos_geometry_error());
+                        THROW(osmium::geos_geometry_error(e.what()));
                     }
                 }
 
                 void multipolygon_add_location(const osmium::geom::Coordinates& xy) {
                     try {
                         m_coordinate_sequence->add(geos::geom::Coordinate(xy.x, xy.y));
-                    } catch (geos::util::GEOSException&) {
+                    } catch (geos::util::GEOSException& e) {
-                        THROW(osmium::geos_geometry_error());
+                        THROW(osmium::geos_geometry_error(e.what()));
                     }
                 }
 
@@ -208,9 +218,9 @@ namespace osmium {
                             return p.release();
                         });
                         m_polygons.clear();
-                        return multipolygon_type(m_geos_factory.createMultiPolygon(polygons));
+                        return multipolygon_type(m_geos_factory->createMultiPolygon(polygons));
-                    } catch (geos::util::GEOSException&) {
+                    } catch (geos::util::GEOSException& e) {
-                        THROW(osmium::geos_geometry_error());
+                        THROW(osmium::geos_geometry_error(e.what()));
                     }
                 }
 

third_party/libosmium/include/osmium/geom/mercator_projection.hpp

@@ -47,6 +47,7 @@ namespace osmium {
         namespace detail {
 
             constexpr double earth_radius_for_epsg3857 = 6378137.0;
+            constexpr double max_coordinate_epsg3857 = 20037508.34;
 
             constexpr inline double lon_to_x(double lon) {
                 return earth_radius_for_epsg3857 * deg_to_rad(lon);

third_party/libosmium/include/osmium/geom/rapid_geojson.hpp

@@ -0,0 +1,190 @@
+#ifndef OSMIUM_GEOM_RAPID_GEOJSON_HPP
+#define OSMIUM_GEOM_RAPID_GEOJSON_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <osmium/geom/coordinates.hpp>
+#include <osmium/geom/factory.hpp>
+
+namespace osmium {
+
+    namespace geom {
+
+        namespace detail {
+
+            /**
+             * A geometry factory implementation that can be used with the
+             * RapidJSON (https://github.com/miloyip/rapidjson) JSON writer.
+             */
+            template <class TWriter>
+            class RapidGeoJSONFactoryImpl {
+
+                TWriter* m_writer;
+
+            public:
+
+                typedef void point_type;
+                typedef void linestring_type;
+                typedef void polygon_type;
+                typedef void multipolygon_type;
+                typedef void ring_type;
+
+                RapidGeoJSONFactoryImpl(TWriter& writer) :
+                    m_writer(&writer) {
+                }
+
+                /* Point */
+
+                // { "type": "Point", "coordinates": [100.0, 0.0] }
+                point_type make_point(const osmium::geom::Coordinates& xy) const {
+                    m_writer->String("geometry");
+                    m_writer->StartObject();
+                    m_writer->String("type");
+                    m_writer->String("Point");
+                    m_writer->String("coordinates");
+                    m_writer->StartArray();
+                    m_writer->Double(xy.x);
+                    m_writer->Double(xy.y);
+                    m_writer->EndArray();
+                    m_writer->EndObject();
+                }
+
+                /* LineString */
+
+                // { "type": "LineString", "coordinates": [ [100.0, 0.0], [101.0, 1.0] ] }
+                void linestring_start() {
+                    m_writer->String("geometry");
+                    m_writer->StartObject();
+                    m_writer->String("type");
+                    m_writer->String("LineString");
+                    m_writer->String("coordinates");
+                    m_writer->StartArray();
+                }
+
+                void linestring_add_location(const osmium::geom::Coordinates& xy) {
+                    m_writer->StartArray();
+                    m_writer->Double(xy.x);
+                    m_writer->Double(xy.y);
+                    m_writer->EndArray();
+                }
+
+                linestring_type linestring_finish(size_t /* num_points */) {
+                    m_writer->EndArray();
+                    m_writer->EndObject();
+                }
+
+                /* Polygon */
+
+                // { "type": "Polygon", "coordinates": [[[100.0, 0.0], [101.0, 1.0]]] }
+                void polygon_start() {
+                    m_writer->String("geometry");
+                    m_writer->StartObject();
+                    m_writer->String("type");
+                    m_writer->String("Polygon");
+                    m_writer->String("coordinates");
+                    m_writer->StartArray();
+                    m_writer->StartArray();
+                }
+
+                void polygon_add_location(const osmium::geom::Coordinates& xy) {
+                    m_writer->StartArray();
+                    m_writer->Double(xy.x);
+                    m_writer->Double(xy.y);
+                    m_writer->EndArray();
+                }
+
+                polygon_type polygon_finish(size_t /* num_points */) {
+                    m_writer->EndArray();
+                    m_writer->EndArray();
+                    m_writer->EndObject();
+                }
+
+                /* MultiPolygon */
+
+                void multipolygon_start() {
+                    m_writer->String("geometry");
+                    m_writer->StartObject();
+                    m_writer->String("type");
+                    m_writer->String("MultiPolygon");
+                    m_writer->String("coordinates");
+                    m_writer->StartArray();
+                }
+
+                void multipolygon_polygon_start() {
+                    m_writer->StartArray();
+                }
+
+                void multipolygon_polygon_finish() {
+                    m_writer->EndArray();
+                }
+
+                void multipolygon_outer_ring_start() {
+                    m_writer->StartArray();
+                }
+
+                void multipolygon_outer_ring_finish() {
+                    m_writer->EndArray();
+                }
+
+                void multipolygon_inner_ring_start() {
+                    m_writer->StartArray();
+                }
+
+                void multipolygon_inner_ring_finish() {
+                    m_writer->EndArray();
+                }
+
+                void multipolygon_add_location(const osmium::geom::Coordinates& xy) {
+                    m_writer->StartArray();
+                    m_writer->Double(xy.x);
+                    m_writer->Double(xy.y);
+                    m_writer->EndArray();
+                }
+
+                multipolygon_type multipolygon_finish() {
+                    m_writer->EndArray();
+                    m_writer->EndObject();
+                }
+
+            }; // class RapidGeoJSONFactoryImpl
+
+        } // namespace detail
+
+        template <class TWriter, class TProjection = IdentityProjection>
+        using RapidGeoJSONFactory = GeometryFactory<detail::RapidGeoJSONFactoryImpl<TWriter>, TProjection>;
+
+    } // namespace geom
+
+} // namespace osmium
+
+#endif // OSMIUM_GEOM_RAPID_GEOJSON_HPP

third_party/libosmium/include/osmium/geom/tile.hpp

@@ -0,0 +1,101 @@
+#ifndef OSMIUM_GEOM_TILE_HPP
+#define OSMIUM_GEOM_TILE_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstdint>
+
+#include <osmium/geom/mercator_projection.hpp>
+
+namespace osmium {
+
+    namespace geom {
+
+        namespace detail {
+
+            template <typename T>
+            inline T restrict_to_range(T value, T min, T max) {
+                if (value < min) return min;
+                if (value > max) return max;
+                return value;
+            }
+
+        } // namespace detail
+
+        /**
+         * A tile in the usual Mercator projection.
+         */
+        struct Tile {
+
+            uint32_t x;
+            uint32_t y;
+            uint32_t z;
+
+            explicit Tile(uint32_t zoom, uint32_t tx, uint32_t ty) noexcept : x(tx), y(ty), z(zoom) {
+            }
+
+            explicit Tile(uint32_t zoom, const osmium::Location& location) :
+                z(zoom) {
+                osmium::geom::Coordinates c = lonlat_to_mercator(location);
+                const int32_t n = 1LL << zoom;
+                const double scale = detail::max_coordinate_epsg3857 * 2 / n;
+                x = detail::restrict_to_range<int32_t>((c.x + detail::max_coordinate_epsg3857) / scale, 0, n-1);
+                y = detail::restrict_to_range<int32_t>((detail::max_coordinate_epsg3857 - c.y) / scale, 0, n-1);
+            }
+
+        }; // struct Tile
+
+        inline bool operator==(const Tile& a, const Tile& b) {
+            return a.z == b.z && a.x == b.x && a.y == b.y;
+        }
+
+        inline bool operator!=(const Tile& a, const Tile& b) {
+            return ! (a == b);
+        }
+
+        /**
+         * This defines an arbitrary order on tiles for use in std::map etc.
+         */
+        inline bool operator<(const Tile& a, const Tile& b) {
+            if (a.z < b.z) return true;
+            if (a.z > b.z) return false;
+            if (a.x < b.x) return true;
+            if (a.x > b.x) return false;
+            return a.y < b.y;
+        }
+
+    } // namespace geom
+
+} // namespace osmium
+
+#endif // OSMIUM_GEOM_TILE_HPP

third_party/libosmium/include/osmium/geom/wkb.hpp

@@ -37,18 +37,10 @@ DEALINGS IN THE SOFTWARE.
 #include <cstdint>
 #include <string>
 
-// Windows is only available for little endian architectures
-// http://stackoverflow.com/questions/6449468/can-i-safely-assume-that-windows-installations-will-always-be-little-endian
-#if !defined(_WIN32) && !defined(__APPLE__)
-# include <endian.h>
-#else
-# define __LITTLE_ENDIAN 1234
-# define __BYTE_ORDER __LITTLE_ENDIAN
-#endif
-
 #include <osmium/geom/coordinates.hpp>
 #include <osmium/geom/factory.hpp>
 #include <osmium/util/cast.hpp>
+#include <osmium/util/endian.hpp>
 
 namespace osmium {
 

third_party/libosmium/include/osmium/index/bool_vector.hpp

@@ -0,0 +1,83 @@
+#ifndef OSMIUM_INDEX_BOOL_VECTOR_HPP
+#define OSMIUM_INDEX_BOOL_VECTOR_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <type_traits>
+#include <vector>
+
+namespace osmium {
+
+    namespace index {
+
+        /**
+         * Index storing one bit for each Id. The index automatically scales
+         * with the Ids stored. Default value is 'false'. Storage uses
+         * std::vector<bool> and needs a minimum of memory if the Ids are
+         * dense.
+         */
+        template <typename T>
+        class BoolVector {
+
+            static_assert(std::is_unsigned<T>::value, "Needs unsigned type");
+
+            std::vector<bool> m_bits;
+
+        public:
+
+            BoolVector() = default;
+            BoolVector(const BoolVector&) = default;
+            BoolVector(BoolVector&&) = default;
+            BoolVector& operator=(const BoolVector&) = default;
+            BoolVector& operator=(BoolVector&&) = default;
+            ~BoolVector() = default;
+
+            void set(T id, bool value = true) {
+                if (m_bits.size() <= id) {
+                    m_bits.resize(id + 1024 * 1024);
+                }
+
+                m_bits[id] = value;
+            }
+
+            bool get(T id) const {
+                return id < m_bits.size() && m_bits[id];
+            }
+
+        }; // class BoolVector
+
+    } // namespace index
+
+} // namespace osmium
+
+#endif // OSMIUM_INDEX_BOOL_VECTOR_HPP

third_party/libosmium/include/osmium/index/detail/create_map_with_fd.hpp

@@ -39,8 +39,6 @@ DEALINGS IN THE SOFTWARE.
 #include <fcntl.h>
 #include <stdexcept>
 #include <string>
-#include <sys/stat.h>
-#include <sys/types.h>
 #include <vector>
 
 namespace osmium {

third_party/libosmium/include/osmium/index/detail/mmap_vector_anon.hpp

@@ -35,9 +35,6 @@ DEALINGS IN THE SOFTWARE.
 
 #ifdef __linux__
 
-#include <cstddef>
-
-#include <osmium/index/detail/typed_mmap.hpp>
 #include <osmium/index/detail/mmap_vector_base.hpp>
 
 namespace osmium {
@@ -45,26 +42,16 @@ namespace osmium {
     namespace detail {
 
         /**
-         * This class looks and behaves like STL vector, but uses mmap internally.
+         * This class looks and behaves like STL vector, but uses mmap
+         * internally.
          */
         template <typename T>
-        class mmap_vector_anon : public mmap_vector_base<T, mmap_vector_anon> {
+        class mmap_vector_anon : public mmap_vector_base<T> {
 
         public:
 
             mmap_vector_anon() :
-                mmap_vector_base<T, osmium::detail::mmap_vector_anon>(
+                mmap_vector_base<T>() {
-                    -1,
-                    osmium::detail::mmap_vector_size_increment,
-                    0,
-                    osmium::detail::typed_mmap<T>::map(osmium::detail::mmap_vector_size_increment)) {
-            }
-
-            void reserve(size_t new_capacity) {
-                if (new_capacity > this->capacity()) {
-                    this->data(osmium::detail::typed_mmap<T>::remap(this->data(), this->capacity(), new_capacity));
-                    this->m_capacity = new_capacity;
-                }
             }
 
         }; // class mmap_vector_anon

third_party/libosmium/include/osmium/index/detail/mmap_vector_base.hpp

@@ -34,11 +34,10 @@ DEALINGS IN THE SOFTWARE.
 */
 
 #include <cstddef>
-#include <new>
+#include <new> // IWYU pragma: keep
 #include <stdexcept>
 
-#include <osmium/index/detail/typed_mmap.hpp>
+#include <osmium/util/memory_mapping.hpp>
-#include <osmium/util/compatibility.hpp>
 
 namespace osmium {
 
@@ -48,40 +47,29 @@ namespace osmium {
 
         /**
          * This is a base class for implementing classes that look like
-         * STL vector but use mmap internally. This class can not be used
+         * STL vector but use mmap internally. Do not use this class itself,
-         * on it's own. Use the derived classes mmap_vector_anon or
+         * use the derived classes mmap_vector_anon or mmap_vector_file.
-         * mmap_vector_file.
          */
-        template <typename T, template <typename> class TDerived>
+        template <typename T>
         class mmap_vector_base {
 
         protected:
 
-            int m_fd;
-            size_t m_capacity;
             size_t m_size;
-            T* m_data;
+            osmium::util::TypedMemoryMapping<T> m_mapping;
-
-            explicit mmap_vector_base(int fd, size_t capacity, size_t size, T* data) noexcept :
-                m_fd(fd),
-                m_capacity(capacity),
-                m_size(size),
-                m_data(data) {
-            }
-
-            explicit mmap_vector_base(int fd, size_t capacity, size_t size) :
-                m_fd(fd),
-                m_capacity(capacity),
-                m_size(size),
-                m_data(osmium::detail::typed_mmap<T>::grow_and_map(capacity, m_fd)) {
-            }
-
-            void data(T* data) {
-                m_data = data;
-            }
 
         public:
 
+            explicit mmap_vector_base(int fd, size_t capacity, size_t size = 0) :
+                m_size(size),
+                m_mapping(capacity, osmium::util::MemoryMapping::mapping_mode::write_shared, fd) {
+            }
+
+            explicit mmap_vector_base(size_t capacity = mmap_vector_size_increment) :
+                m_size(0),
+                m_mapping(capacity) {
+            }
+
             typedef T value_type;
             typedef T& reference;
             typedef const T& const_reference;
@@ -90,12 +78,14 @@ namespace osmium {
             typedef T* iterator;
             typedef const T* const_iterator;
 
-            ~mmap_vector_base() {
+            ~mmap_vector_base() = default;
-                osmium::detail::typed_mmap<T>::unmap(m_data, m_capacity);
+
+            void close() {
+                m_mapping.unmap();
             }
 
             size_t capacity() const noexcept {
-                return m_capacity;
+                return m_mapping.size();
             }
 
             size_t size() const noexcept {
@@ -106,23 +96,23 @@ namespace osmium {
                 return m_size == 0;
             }
 
-            const T* data() const noexcept {
+            const T* data() const {
-                return m_data;
+                return m_mapping.begin();
             }
 
-            T* data() noexcept {
+            T* data() {
-                return m_data;
+                return m_mapping.begin();
             }
 
             T& operator[](size_t n) {
-                return m_data[n];
+                return data()[n];
             }
 
             T at(size_t n) const {
                 if (n >= m_size) {
                     throw std::out_of_range("out of range");
                 }
-                return m_data[n];
+                return data()[n];
             }
 
             void clear() noexcept {
@@ -134,16 +124,22 @@ namespace osmium {
             }
 
             void push_back(const T& value) {
-                if (m_size >= m_capacity) {
+                if (m_size >= capacity()) {
                     resize(m_size+1);
                 }
-                m_data[m_size] = value;
+                data()[m_size] = value;
                 ++m_size;
             }
 
+            void reserve(size_t new_capacity) {
+                if (new_capacity > capacity()) {
+                    m_mapping.resize(new_capacity);
+                }
+            }
+
             void resize(size_t new_size) {
                 if (new_size > capacity()) {
-                    static_cast<TDerived<T>*>(this)->reserve(new_size + osmium::detail::mmap_vector_size_increment);
+                    reserve(new_size + osmium::detail::mmap_vector_size_increment);
                 }
                 if (new_size > size()) {
                     new (data() + size()) T[new_size - size()];
@@ -152,27 +148,27 @@ namespace osmium {
             }
 
             iterator begin() noexcept {
-                return m_data;
+                return data();
             }
 
             iterator end() noexcept {
-                return m_data + m_size;
+                return data() + m_size;
             }
 
             const_iterator begin() const noexcept {
-                return m_data;
+                return data();
             }
 
             const_iterator end() const noexcept {
-                return m_data + m_size;
+                return data() + m_size;
             }
 
-            const_iterator cbegin() noexcept {
+            const_iterator cbegin() const noexcept {
-                return m_data;
+                return data();
             }
 
-            const_iterator cend() noexcept {
+            const_iterator cend() const noexcept {
-                return m_data + m_size;
+                return data() + m_size;
             }
 
         }; // class mmap_vector_base

third_party/libosmium/include/osmium/index/detail/mmap_vector_file.hpp

@@ -33,11 +33,9 @@ DEALINGS IN THE SOFTWARE.
 
 */
 
-#include <cstddef>
-
-#include <osmium/index/detail/typed_mmap.hpp>
 #include <osmium/index/detail/mmap_vector_base.hpp>
 #include <osmium/index/detail/tmpfile.hpp>
+#include <osmium/util/file.hpp>
 
 namespace osmium {
 
@@ -48,32 +46,19 @@ namespace osmium {
          * internally.
          */
         template <typename T>
-        class mmap_vector_file : public mmap_vector_base<T, mmap_vector_file> {
+        class mmap_vector_file : public mmap_vector_base<T> {
 
         public:
 
-            explicit mmap_vector_file() :
+            explicit mmap_vector_file() : mmap_vector_base<T>(
-                mmap_vector_base<T, osmium::detail::mmap_vector_file>(
                     osmium::detail::create_tmp_file(),
-                    osmium::detail::mmap_vector_size_increment,
+                    osmium::detail::mmap_vector_size_increment) {
-                    0) {
             }
 
-            explicit mmap_vector_file(int fd) :
+            explicit mmap_vector_file(int fd) : mmap_vector_base<T>(
-                mmap_vector_base<T, osmium::detail::mmap_vector_file>(
                     fd,
-                    osmium::detail::typed_mmap<T>::file_size(fd) == 0 ?
+                    osmium::util::file_size(fd) / sizeof(T),
-                        osmium::detail::mmap_vector_size_increment :
+                    osmium::util::file_size(fd) / sizeof(T)) {
-                        osmium::detail::typed_mmap<T>::file_size(fd),
-                    osmium::detail::typed_mmap<T>::file_size(fd)) {
-            }
-
-            void reserve(size_t new_capacity) {
-                if (new_capacity > this->capacity()) {
-                    typed_mmap<T>::unmap(this->data(), this->capacity());
-                    this->data(typed_mmap<T>::grow_and_map(new_capacity, this->m_fd));
-                    this->m_capacity = new_capacity;
-                }
             }
 
         }; // class mmap_vector_file

third_party/libosmium/include/osmium/index/detail/typed_mmap.hpp

@@ -1,229 +0,0 @@
-#ifndef OSMIUM_INDEX_DETAIL_TYPED_MMAP_HPP
-#define OSMIUM_INDEX_DETAIL_TYPED_MMAP_HPP
-
-/*
-
-This file is part of Osmium (http://osmcode.org/libosmium).
-
-Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
-
-Boost Software License - Version 1.0 - August 17th, 2003
-
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-
-*/
-
-#include <cerrno>
-#include <cstddef>
-#include <stdexcept>
-#include <system_error>
-
-#include <sys/stat.h>
-
-#ifndef _WIN32
-# include <sys/mman.h>
-#else
-# include <mmap_for_windows.hpp>
-#endif
-
-#ifndef _MSC_VER
-# include <unistd.h>
-#else
-# define ftruncate _chsize
-#endif
-
-// for bsd systems
-#ifndef MAP_ANONYMOUS
-# define MAP_ANONYMOUS MAP_ANON
-#endif
-
-#include <osmium/util/cast.hpp>
-
-namespace osmium {
-
-    /**
-     * @brief Namespace for Osmium internal use
-     */
-    namespace detail {
-
-        /**
-         * This is a helper class for working with memory mapped files and
-         * anonymous shared memory. It wraps the necessary system calls
-         * adding:
-         * - error checking: all functions throw exceptions where needed
-         * - internal casts and size calculations allow use with user defined
-         *   type T instead of void*
-         *
-         * This class only contains static functions. It should never be
-         * instantiated.
-         *
-         * @tparam T Type of objects we want to store.
-         */
-        template <typename T>
-        class typed_mmap {
-
-        public:
-
-            /**
-             * Create anonymous private memory mapping with enough space for size
-             * objects of type T.
-             *
-             * Note that no constructor is called for any of the objects in this memory!
-             *
-             * @param size Number of objects of type T that should fit into this memory
-             * @returns Pointer to mapped memory
-             * @throws std::system_error If mmap(2) failed
-             */
-            static T* map(size_t size) {
-                void* addr = ::mmap(nullptr, sizeof(T) * size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wold-style-cast"
-                if (addr == MAP_FAILED) {
-                    throw std::system_error(errno, std::system_category(), "mmap failed");
-                }
-#pragma GCC diagnostic pop
-                return reinterpret_cast<T*>(addr);
-            }
-
-            /**
-             * Create shared memory mapping of a file with enough space for size
-             * objects of type T. The file must already have at least the
-             * required size.
-             *
-             * Note that no constructor is called for any of the objects in this memory!
-             *
-             * @param size Number of objects of type T that should fit into this memory
-             * @param fd File descriptor
-             * @param write True if data should be writable
-             * @returns Pointer to mapped memory
-             * @throws std::system_error If mmap(2) failed
-             */
-            static T* map(size_t size, int fd, bool write = false) {
-                int prot = PROT_READ;
-                if (write) {
-                    prot |= PROT_WRITE;
-                }
-                void* addr = ::mmap(nullptr, sizeof(T) * size, prot, MAP_SHARED, fd, 0);
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wold-style-cast"
-                if (addr == MAP_FAILED) {
-                    throw std::system_error(errno, std::system_category(), "mmap failed");
-                }
-#pragma GCC diagnostic pop
-                return reinterpret_cast<T*>(addr);
-            }
-
-// mremap(2) is only available on linux systems
-#ifdef __linux__
-            /**
-             * Grow memory mapping created with map().
-             *
-             * Note that no constructor is called for any of the objects in this memory!
-             *
-             * @param data Pointer to current mapping (as returned by typed_mmap())
-             * @param old_size Number of objects currently stored in this memory
-             * @param new_size Number of objects we want to have space for
-             * @throws std::system_error If mremap(2) call failed
-             */
-            static T* remap(T* data, size_t old_size, size_t new_size) {
-                void* addr = ::mremap(reinterpret_cast<void*>(data), sizeof(T) * old_size, sizeof(T) * new_size, MREMAP_MAYMOVE);
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wold-style-cast"
-                if (addr == MAP_FAILED) {
-                    throw std::system_error(errno, std::system_category(), "mremap failed");
-                }
-#pragma GCC diagnostic pop
-                return reinterpret_cast<T*>(addr);
-            }
-#endif
-
-            /**
-             * Release memory from map() call.
-             *
-             * Note that no destructor is called for the objects in this memory!
-             *
-             * @param data Pointer to the data
-             * @param size Number of objects of type T stored
-             * @throws std::system_error If munmap(2) call failed
-             */
-            static void unmap(T* data, size_t size) {
-                if (::munmap(reinterpret_cast<void*>(data), sizeof(T) * size) != 0) {
-                    throw std::system_error(errno, std::system_category(), "munmap failed");
-                }
-            }
-
-            /**
-             * Get number of objects of type T that would fit into a file.
-             *
-             * @param fd File descriptor
-             * @returns Number of objects of type T in this file
-             * @throws std::system_error If fstat(2) call failed
-             * @throws std::length_error If size of the file isn't a multiple of sizeof(T)
-             */
-            static size_t file_size(int fd) {
-                struct stat s;
-                if (fstat(fd, &s) < 0) {
-                    throw std::system_error(errno, std::system_category(), "fstat failed");
-                }
-                if (static_cast<size_t>(s.st_size) % sizeof(T) != 0) {
-                    throw std::length_error("file size has to be multiple of object size");
-                }
-                return static_cast<size_t>(s.st_size) / sizeof(T);
-            }
-
-            /**
-             * Grow file so there is enough space for at least new_size objects
-             * of type T. If the file is large enough already, nothing is done.
-             * The file is never shrunk.
-             *
-             * @param new_size Number of objects of type T that should fit into this file
-             * @param fd File descriptor
-             * @throws std::system_error If ftruncate(2) call failed
-             */
-            static void grow_file(size_t new_size, int fd) {
-                if (file_size(fd) < new_size) {
-                    if (::ftruncate(fd, static_cast_with_assert<off_t>(sizeof(T) * new_size)) < 0) {
-                        throw std::system_error(errno, std::system_category(), "ftruncate failed");
-                    }
-                }
-            }
-
-            /**
-             * Grow file to given size (if it is smaller) and mmap it.
-             *
-             * @param size Number of objects of type T that should fit into this file
-             * @param fd File descriptor
-             * @throws Errors thrown by grow_file() or map()
-             */
-            static T* grow_and_map(size_t size, int fd) {
-                grow_file(size, fd);
-                return map(size, fd, true);
-            }
-
-        }; // class typed_mmap
-
-    } // namespace detail
-
-} // namespace osmium
-
-#endif // OSMIUM_INDEX_DETAIL_TYPED_MMAP_HPP

third_party/libosmium/include/osmium/index/detail/vector_map.hpp

@@ -68,7 +68,7 @@ namespace osmium {
                     m_vector(fd) {
                 }
 
-                ~VectorBasedDenseMap() {}
+                ~VectorBasedDenseMap() = default;
 
                 void reserve(const size_t size) override final {
                     m_vector.reserve(size);
@@ -97,6 +97,10 @@ namespace osmium {
                     return m_vector.size();
                 }
 
+                size_t byte_size() const {
+                    return m_vector.size() * sizeof(element_type);
+                }
+
                 size_t used_memory() const override final {
                     return sizeof(TValue) * size();
                 }
@@ -106,6 +110,10 @@ namespace osmium {
                     m_vector.shrink_to_fit();
                 }
 
+                void dump_as_array(const int fd) override final {
+                    osmium::io::detail::reliable_write(fd, reinterpret_cast<const char*>(m_vector.data()), byte_size());
+                }
+
                 iterator begin() {
                     return m_vector.begin();
                 }

third_party/libosmium/include/osmium/index/detail/vector_multimap.hpp

@@ -67,6 +67,16 @@ namespace osmium {
 
             public:
 
+                VectorBasedSparseMultimap() :
+                    m_vector() {
+                }
+
+                explicit VectorBasedSparseMultimap(int fd) :
+                    m_vector(fd) {
+                }
+
+                ~VectorBasedSparseMultimap() = default;
+
                 void set(const TId id, const TValue value) override final {
                     m_vector.push_back(element_type(id, value));
                 }
@@ -141,6 +151,30 @@ namespace osmium {
                     osmium::io::detail::reliable_write(fd, reinterpret_cast<const char*>(m_vector.data()), byte_size());
                 }
 
+                iterator begin() {
+                    return m_vector.begin();
+                }
+
+                iterator end() {
+                    return m_vector.end();
+                }
+
+                const_iterator cbegin() const {
+                    return m_vector.cbegin();
+                }
+
+                const_iterator cend() const {
+                    return m_vector.cend();
+                }
+
+                const_iterator begin() const {
+                    return m_vector.cbegin();
+                }
+
+                const_iterator end() const {
+                    return m_vector.cend();
+                }
+
             }; // class VectorBasedSparseMultimap
 
         } // namespace multimap

third_party/libosmium/include/osmium/index/index.hpp

@@ -67,7 +67,7 @@ namespace osmium {
         template <typename TKey>
         OSMIUM_NORETURN void not_found_error(TKey key) {
             std::stringstream s;
-            s << "id " << key << " no found";
+            s << "id " << key << " not found";
             throw not_found(s.str());
         }
 

third_party/libosmium/include/osmium/index/map.hpp

@@ -148,7 +148,11 @@ namespace osmium {
                 }
 
                 virtual void dump_as_list(const int /*fd*/) {
-                    std::runtime_error("can't dump as list");
+                    throw std::runtime_error("can't dump as list");
+                }
+
+                virtual void dump_as_array(const int /*fd*/) {
+                    throw std::runtime_error("can't dump as array");
                 }
 
             }; // class Map
@@ -195,6 +199,10 @@ namespace osmium {
                 return m_callbacks.emplace(map_type_name, func).second;
             }
 
+            bool has_map_type(const std::string& map_type_name) const {
+                return m_callbacks.count(map_type_name);
+            }
+
             std::vector<std::string> map_types() const {
                 std::vector<std::string> result;
 
@@ -242,9 +250,13 @@ namespace osmium {
             });
         }
 
+#define OSMIUM_CONCATENATE_DETAIL_(x, y) x##y
+#define OSMIUM_CONCATENATE_(x, y) OSMIUM_CONCATENATE_DETAIL_(x, y)
+#define OSMIUM_MAKE_UNIQUE_(x) OSMIUM_CONCATENATE_(x, __COUNTER__)
+
 #define REGISTER_MAP(id, value, klass, name) \
 namespace { \
-    const bool registered_index_map_##name = osmium::index::register_map<id, value, klass>(#name); \
+    const bool OSMIUM_MAKE_UNIQUE_(registered_index_map_##name) = osmium::index::register_map<id, value, klass>(#name); \
 }
 
     } // namespace index

third_party/libosmium/include/osmium/index/map/dense_mmap_array.hpp

@@ -35,7 +35,7 @@ DEALINGS IN THE SOFTWARE.
 
 #ifdef __linux__
 
-#include <osmium/index/detail/mmap_vector_anon.hpp>
+#include <osmium/index/detail/mmap_vector_anon.hpp> // IWYU pragma: keep
 #include <osmium/index/detail/vector_map.hpp>
 
 #define OSMIUM_HAS_INDEX_MAP_DENSE_MMAP_ARRAY

third_party/libosmium/include/osmium/index/map/sparse_mem_map.hpp

@@ -33,7 +33,7 @@ DEALINGS IN THE SOFTWARE.
 
 */
 
-#include <algorithm>
+#include <algorithm> // IWYU pragma: keep (for std::copy)
 #include <cstddef>
 #include <iterator>
 #include <map>

third_party/libosmium/include/osmium/io/any_input.hpp

@@ -39,8 +39,8 @@ DEALINGS IN THE SOFTWARE.
  * Include this file if you want to read all kinds of OSM files.
  *
  * @attention If you include this file, you'll need to link with
- *            `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only),
+ *            `ws2_32` (Windows only), `libexpat`, `libz`, `libbz2`,
- *            `libexpat`, `libz`, `libbz2`, and enable multithreading.
+ *            and enable multithreading.
  */
 
 #include <osmium/io/any_compression.hpp> // IWYU pragma: export

third_party/libosmium/include/osmium/io/any_output.hpp

@@ -39,12 +39,13 @@ DEALINGS IN THE SOFTWARE.
  * Include this file if you want to write all kinds of OSM files.
  *
  * @attention If you include this file, you'll need to link with
- *            `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only),
+ *            `ws2_32` (Windows only), `libz`, `libbz2`, and enable
- *            `libz`, `libbz2`, and enable multithreading.
+ *            multithreading.
  */
 
 #include <osmium/io/any_compression.hpp> // IWYU pragma: export
 
+#include <osmium/io/debug_output.hpp> // IWYU pragma: export
 #include <osmium/io/opl_output.hpp> // IWYU pragma: export
 #include <osmium/io/pbf_output.hpp> // IWYU pragma: export
 #include <osmium/io/xml_output.hpp> // IWYU pragma: export

third_party/libosmium/include/osmium/io/bzip2_compression.hpp

@@ -274,11 +274,16 @@ namespace osmium {
 
         namespace {
 
+// we want the register_compression() function to run, setting the variable
+// is only a side-effect, it will never be used
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
             const bool registered_bzip2_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::bzip2,
                 [](int fd) { return new osmium::io::Bzip2Compressor(fd); },
                 [](int fd) { return new osmium::io::Bzip2Decompressor(fd); },
                 [](const char* buffer, size_t size) { return new osmium::io::Bzip2BufferDecompressor(buffer, size); }
             );
+#pragma GCC diagnostic pop
 
         } // anonymous namespace
 

third_party/libosmium/include/osmium/io/compression.hpp

@@ -266,11 +266,16 @@ namespace osmium {
 
         namespace {
 
+// we want the register_compression() function to run, setting the variable
+// is only a side-effect, it will never be used
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
             const bool registered_no_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::none,
                 [](int fd) { return new osmium::io::NoCompressor(fd); },
                 [](int fd) { return new osmium::io::NoDecompressor(fd); },
                 [](const char* buffer, size_t size) { return new osmium::io::NoDecompressor(buffer, size); }
             );
+#pragma GCC diagnostic pop
 
         } // anonymous namespace
 

third_party/libosmium/include/osmium/io/debug_output.hpp

@@ -0,0 +1,39 @@
+#ifndef OSMIUM_IO_DEBUG_OUTPUT_HPP
+#define OSMIUM_IO_DEBUG_OUTPUT_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <osmium/io/writer.hpp> // IWYU pragma: export
+#include <osmium/io/detail/debug_output_format.hpp> // IWYU pragma: export
+
+#endif // OSMIUM_IO_DEBUG_OUTPUT_HPP

third_party/libosmium/include/osmium/io/detail/debug_output_format.hpp

@@ -0,0 +1,482 @@
+#ifndef OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP
+#define OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <chrono>
+#include <cinttypes>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <future>
+#include <iterator>
+#include <memory>
+#include <ratio>
+#include <string>
+#include <thread>
+#include <utility>
+
+#include <utf8.h>
+
+#include <osmium/handler.hpp>
+#include <osmium/io/detail/output_format.hpp>
+#include <osmium/io/file_format.hpp>
+#include <osmium/memory/buffer.hpp>
+#include <osmium/memory/collection.hpp>
+#include <osmium/osm/box.hpp>
+#include <osmium/osm/changeset.hpp>
+#include <osmium/osm/item_type.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/node.hpp>
+#include <osmium/osm/object.hpp>
+#include <osmium/osm/relation.hpp>
+#include <osmium/osm/tag.hpp>
+#include <osmium/osm/timestamp.hpp>
+#include <osmium/osm/way.hpp>
+#include <osmium/thread/pool.hpp>
+#include <osmium/util/minmax.hpp>
+#include <osmium/visitor.hpp>
+
+namespace osmium {
+
+    namespace io {
+
+        class File;
+
+        namespace detail {
+
+            constexpr const char* color_bold    = "\x1b[1m";
+            constexpr const char* color_black   = "\x1b[30m";
+            constexpr const char* color_gray    = "\x1b[30;1m";
+            constexpr const char* color_red     = "\x1b[31m";
+            constexpr const char* color_green   = "\x1b[32m";
+            constexpr const char* color_yellow  = "\x1b[33m";
+            constexpr const char* color_blue    = "\x1b[34m";
+            constexpr const char* color_magenta = "\x1b[35m";
+            constexpr const char* color_cyan    = "\x1b[36m";
+            constexpr const char* color_white   = "\x1b[37m";
+            constexpr const char* color_reset   = "\x1b[0m";
+
+            /**
+             * Writes out one buffer with OSM data in Debug format.
+             */
+            class DebugOutputBlock : public osmium::handler::Handler {
+
+                static constexpr size_t tmp_buffer_size = 50;
+
+                std::shared_ptr<osmium::memory::Buffer> m_input_buffer;
+
+                std::shared_ptr<std::string> m_out;
+
+                char m_tmp_buffer[tmp_buffer_size+1];
+
+                bool m_add_metadata;
+                bool m_use_color;
+
+                template <typename... TArgs>
+                void output_formatted(const char* format, TArgs&&... args) {
+#ifndef NDEBUG
+                    int len =
+#endif
+#ifndef _MSC_VER
+                    snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward<TArgs>(args)...);
+#else
+                    _snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward<TArgs>(args)...);
+#endif
+                    assert(len > 0 && static_cast<size_t>(len) < tmp_buffer_size);
+                    *m_out += m_tmp_buffer;
+                }
+
+                void append_encoded_string(const char* data) {
+                    const char* end = data + std::strlen(data);
+
+                    while (data != end) {
+                        const char* last = data;
+                        uint32_t c = utf8::next(data, end);
+
+                        // This is a list of Unicode code points that we let
+                        // through instead of escaping them. It is incomplete
+                        // and can be extended later.
+                        // Generally we don't want to let through any
+                        // non-printing characters.
+                        if ((0x0020 <= c && c <= 0x0021) ||
+                            (0x0023 <= c && c <= 0x003b) ||
+                            (0x003d == c) ||
+                            (0x003f <= c && c <= 0x007e) ||
+                            (0x00a1 <= c && c <= 0x00ac) ||
+                            (0x00ae <= c && c <= 0x05ff)) {
+                            m_out->append(last, data);
+                        } else {
+                            write_color(color_red);
+                            output_formatted("<U+%04X>", c);
+                            write_color(color_blue);
+                        }
+                    }
+                }
+
+                void write_color(const char* color) {
+                    if (m_use_color) {
+                        *m_out += color;
+                    }
+                }
+
+                void write_string(const char* string) {
+                    *m_out += '"';
+                    write_color(color_blue);
+                    append_encoded_string(string);
+                    write_color(color_reset);
+                    *m_out += '"';
+                }
+
+                void write_object_type(const char* object_type, bool visible = true) {
+                    if (visible) {
+                        write_color(color_bold);
+                    } else {
+                        write_color(color_white);
+                    }
+                    *m_out += object_type;
+                    write_color(color_reset);
+                    *m_out += ' ';
+                }
+
+                void write_fieldname(const char* name) {
+                    *m_out += "  ";
+                    write_color(color_cyan);
+                    *m_out += name;
+                    write_color(color_reset);
+                    *m_out += ": ";
+                }
+
+                void write_error(const char* msg) {
+                    write_color(color_red);
+                    *m_out += msg;
+                    write_color(color_reset);
+                }
+
+                void write_meta(const osmium::OSMObject& object) {
+                    output_formatted("%" PRId64 "\n", object.id());
+                    if (m_add_metadata) {
+                        write_fieldname("version");
+                        output_formatted("  %d", object.version());
+                        if (object.visible()) {
+                            *m_out += " visible\n";
+                        } else {
+                            write_error(" deleted\n");
+                        }
+                        write_fieldname("changeset");
+                        output_formatted("%d\n", object.changeset());
+                        write_fieldname("timestamp");
+                        *m_out += object.timestamp().to_iso();
+                        output_formatted(" (%d)\n", object.timestamp());
+                        write_fieldname("user");
+                        output_formatted("     %d ", object.uid());
+                        write_string(object.user());
+                        *m_out += '\n';
+                    }
+                }
+
+                void write_tags(const osmium::TagList& tags, const char* padding="") {
+                    if (!tags.empty()) {
+                        write_fieldname("tags");
+                        *m_out += padding;
+                        output_formatted("     %d\n", tags.size());
+
+                        osmium::max_op<int> max;
+                        for (const auto& tag : tags) {
+                            max.update(std::strlen(tag.key()));
+                        }
+                        for (const auto& tag : tags) {
+                            *m_out += "    ";
+                            write_string(tag.key());
+                            int spacing = max() - std::strlen(tag.key());
+                            while (spacing--) {
+                                *m_out += " ";
+                            }
+                            *m_out += " = ";
+                            write_string(tag.value());
+                            *m_out += '\n';
+                        }
+                    }
+                }
+
+                void write_location(const osmium::Location& location) {
+                    write_fieldname("lon/lat");
+                    output_formatted("  %.7f,%.7f", location.lon_without_check(), location.lat_without_check());
+                    if (!location.valid()) {
+                        write_error(" INVALID LOCATION!");
+                    }
+                    *m_out += '\n';
+                }
+
+                void write_box(const osmium::Box& box) {
+                    write_fieldname("box l/b/r/t");
+                    if (!box) {
+                        write_error("BOX NOT SET!\n");
+                        return;
+                    }
+                    const auto& bl = box.bottom_left();
+                    const auto& tr = box.top_right();
+                    output_formatted("%.7f,%.7f %.7f,%.7f", bl.lon_without_check(), bl.lat_without_check(), tr.lon_without_check(), tr.lat_without_check());
+                    if (!box.valid()) {
+                        write_error(" INVALID BOX!");
+                    }
+                    *m_out += '\n';
+                }
+
+            public:
+
+                explicit DebugOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata, bool use_color) :
+                    m_input_buffer(std::make_shared<osmium::memory::Buffer>(std::move(buffer))),
+                    m_out(std::make_shared<std::string>()),
+                    m_tmp_buffer(),
+                    m_add_metadata(add_metadata),
+                    m_use_color(use_color) {
+                }
+
+                DebugOutputBlock(const DebugOutputBlock&) = default;
+                DebugOutputBlock& operator=(const DebugOutputBlock&) = default;
+
+                DebugOutputBlock(DebugOutputBlock&&) = default;
+                DebugOutputBlock& operator=(DebugOutputBlock&&) = default;
+
+                ~DebugOutputBlock() = default;
+
+                std::string operator()() {
+                    osmium::apply(m_input_buffer->cbegin(), m_input_buffer->cend(), *this);
+
+                    std::string out;
+                    std::swap(out, *m_out);
+                    return out;
+                }
+
+                void node(const osmium::Node& node) {
+                    write_object_type("node", node.visible());
+                    write_meta(node);
+
+                    if (node.visible()) {
+                        write_location(node.location());
+                    }
+
+                    write_tags(node.tags());
+
+                    *m_out += '\n';
+                }
+
+                void way(const osmium::Way& way) {
+                    write_object_type("way", way.visible());
+                    write_meta(way);
+                    write_tags(way.tags());
+
+                    write_fieldname("nodes");
+
+                    output_formatted("    %d", way.nodes().size());
+                    if (way.nodes().size() < 2) {
+                        write_error(" LESS THAN 2 NODES!\n");
+                    } else if (way.nodes().size() > 2000) {
+                        write_error(" MORE THAN 2000 NODES!\n");
+                    } else if (way.nodes().is_closed()) {
+                        *m_out += " (closed)\n";
+                    } else {
+                        *m_out += " (open)\n";
+                    }
+
+                    int width = int(log10(way.nodes().size())) + 1;
+                    int n = 0;
+                    for (const auto& node_ref : way.nodes()) {
+                        output_formatted("    %0*d: %10" PRId64, width, n++, node_ref.ref());
+                        if (node_ref.location().valid()) {
+                            output_formatted(" (%.7f,%.7f)", node_ref.location().lon_without_check(), node_ref.location().lat_without_check());
+                        }
+                        *m_out += '\n';
+                    }
+
+                    *m_out += '\n';
+                }
+
+                void relation(const osmium::Relation& relation) {
+                    static const char* short_typename[] = { "node", "way ", "rel " };
+                    write_object_type("relation", relation.visible());
+                    write_meta(relation);
+                    write_tags(relation.tags());
+
+                    write_fieldname("members");
+                    output_formatted("  %d\n", relation.members().size());
+
+                    int width = int(log10(relation.members().size())) + 1;
+                    int n = 0;
+                    for (const auto& member : relation.members()) {
+                        output_formatted("    %0*d: ", width, n++);
+                        *m_out += short_typename[item_type_to_nwr_index(member.type())];
+                        output_formatted(" %10" PRId64 " ", member.ref());
+                        write_string(member.role());
+                        *m_out += '\n';
+                    }
+
+                    *m_out += '\n';
+                }
+
+                void changeset(const osmium::Changeset& changeset) {
+                    write_object_type("changeset");
+                    output_formatted("%d\n", changeset.id());
+                    write_fieldname("num changes");
+                    output_formatted("%d", changeset.num_changes());
+                    if (changeset.num_changes() == 0) {
+                        write_error(" NO CHANGES!");
+                    }
+                    *m_out += '\n';
+                    write_fieldname("created at");
+                    *m_out += ' ';
+                    *m_out += changeset.created_at().to_iso();
+                    output_formatted(" (%d)\n", changeset.created_at());
+                    write_fieldname("closed at");
+                    *m_out += "  ";
+                    if (changeset.closed()) {
+                        *m_out += changeset.closed_at().to_iso();
+                        output_formatted(" (%d)\n", changeset.closed_at());
+                    } else {
+                        write_error("OPEN!\n");
+                    }
+                    write_fieldname("user");
+                    output_formatted("       %d ", changeset.uid());
+                    write_string(changeset.user());
+                    *m_out += '\n';
+
+                    write_box(changeset.bounds());
+                    write_tags(changeset.tags(), "  ");
+
+                    *m_out += '\n';
+                }
+
+            }; // DebugOutputBlock
+
+            class DebugOutputFormat : public osmium::io::detail::OutputFormat {
+
+                bool m_add_metadata;
+                bool m_use_color;
+
+            public:
+
+                DebugOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) :
+                    OutputFormat(file, output_queue),
+                    m_add_metadata(file.get("add_metadata") != "false"),
+                    m_use_color(file.get("color") == "true") {
+                }
+
+                DebugOutputFormat(const DebugOutputFormat&) = delete;
+                DebugOutputFormat& operator=(const DebugOutputFormat&) = delete;
+
+                void write_buffer(osmium::memory::Buffer&& buffer) override final {
+                    m_output_queue.push(osmium::thread::Pool::instance().submit(DebugOutputBlock{std::move(buffer), m_add_metadata, m_use_color}));
+                }
+
+                void write_fieldname(std::string& out, const char* name) {
+                    out += "  ";
+                    if (m_use_color) {
+                        out += color_cyan;
+                    }
+                    out += name;
+                    if (m_use_color) {
+                        out += color_reset;
+                    }
+                    out += ": ";
+                }
+
+                void write_header(const osmium::io::Header& header) override final {
+                    std::string out;
+
+                    if (m_use_color) {
+                        out += color_bold;
+                    }
+                    out += "header\n";
+                    if (m_use_color) {
+                        out += color_reset;
+                    }
+
+                    write_fieldname(out, "multiple object versions");
+                    out += header.has_multiple_object_versions() ? "yes" : "no";
+                    out += '\n';
+                    write_fieldname(out, "bounding boxes");
+                    out += '\n';
+                    for (const auto& box : header.boxes()) {
+                        out += "    ";
+                        box.bottom_left().as_string(std::back_inserter(out), ',');
+                        out += " ";
+                        box.top_right().as_string(std::back_inserter(out), ',');
+                        out += '\n';
+                    }
+                    write_fieldname(out, "options");
+                    out += '\n';
+                    for (const auto& opt : header) {
+                        out += "    ";
+                        out += opt.first;
+                        out += " = ";
+                        out += opt.second;
+                        out += '\n';
+                    }
+                    out += "\n=============================================\n\n";
+
+                    std::promise<std::string> promise;
+                    m_output_queue.push(promise.get_future());
+                    promise.set_value(std::move(out));
+                }
+
+                void close() override final {
+                    std::string out;
+                    std::promise<std::string> promise;
+                    m_output_queue.push(promise.get_future());
+                    promise.set_value(out);
+                }
+
+            }; // class DebugOutputFormat
+
+            namespace {
+
+// we want the register_output_format() function to run, setting the variable
+// is only a side-effect, it will never be used
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
+                const bool registered_debug_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::debug,
+                    [](const osmium::io::File& file, data_queue_type& output_queue) {
+                        return new osmium::io::detail::DebugOutputFormat(file, output_queue);
+                });
+#pragma GCC diagnostic pop
+
+            } // anonymous namespace
+
+        } // namespace detail
+
+    } // namespace io
+
+} // namespace osmium
+
+#endif // OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP

third_party/libosmium/include/osmium/io/detail/opl_output_format.hpp

@@ -46,23 +46,7 @@ DEALINGS IN THE SOFTWARE.
 #include <thread>
 #include <utility>
 
-#include <boost/version.hpp>
+#include <utf8.h>
-
-#ifdef __clang__
-# pragma clang diagnostic push
-# pragma clang diagnostic ignored "-Wmissing-noreturn"
-# pragma clang diagnostic ignored "-Wsign-conversion"
-#endif
-
-#if BOOST_VERSION >= 104800
-# include <boost/regex/pending/unicode_iterator.hpp>
-#else
-# include <boost_unicode_iterator.hpp>
-#endif
-
-#ifdef __clang__
-# pragma clang diagnostic pop
-#endif
 
 #include <osmium/handler.hpp>
 #include <osmium/io/detail/output_format.hpp>
@@ -103,6 +87,8 @@ namespace osmium {
 
                 char m_tmp_buffer[tmp_buffer_size+1];
 
+                bool m_add_metadata;
+
                 template <typename... TArgs>
                 void output_formatted(const char* format, TArgs&&... args) {
 #ifndef NDEBUG
@@ -117,13 +103,12 @@ namespace osmium {
                     *m_out += m_tmp_buffer;
                 }
 
-                void append_encoded_string(const std::string& data) {
+                void append_encoded_string(const char* data) {
-                    boost::u8_to_u32_iterator<std::string::const_iterator> it(data.cbegin(), data.cbegin(), data.cend());
+                    const char* end = data + std::strlen(data);
-                    boost::u8_to_u32_iterator<std::string::const_iterator> end(data.cend(), data.cend(), data.cend());
-                    boost::utf8_output_iterator<std::back_insert_iterator<std::string>> oit(std::back_inserter(*m_out));
 
-                    for (; it != end; ++it) {
+                    while (data != end) {
-                        uint32_t c = *it;
+                        const char* last = data;
+                        uint32_t c = utf8::next(data, end);
 
                         // This is a list of Unicode code points that we let
                         // through instead of escaping them. It is incomplete
@@ -138,21 +123,29 @@ namespace osmium {
                             (0x0041 <= c && c <= 0x007e) ||
                             (0x00a1 <= c && c <= 0x00ac) ||
                             (0x00ae <= c && c <= 0x05ff)) {
-                            *oit = c;
+                            m_out->append(last, data);
                         } else {
                             *m_out += '%';
-                            output_formatted("%04x", c);
+                            if (c <= 0xff) {
+                                output_formatted("%02x", c);
+                            } else {
+                                output_formatted("%04x", c);
+                            }
+                            *m_out += '%';
                         }
                     }
                 }
 
                 void write_meta(const osmium::OSMObject& object) {
-                    output_formatted("%" PRId64 " v%d d", object.id(), object.version());
+                    output_formatted("%" PRId64, object.id());
-                    *m_out += (object.visible() ? 'V' : 'D');
+                    if (m_add_metadata) {
-                    output_formatted(" c%d t", object.changeset());
+                        output_formatted(" v%d d", object.version());
-                    *m_out += object.timestamp().to_iso();
+                        *m_out += (object.visible() ? 'V' : 'D');
-                    output_formatted(" i%d u", object.uid());
+                        output_formatted(" c%d t", object.changeset());
-                    append_encoded_string(object.user());
+                        *m_out += object.timestamp().to_iso();
+                        output_formatted(" i%d u", object.uid());
+                        append_encoded_string(object.user());
+                    }
                     *m_out += " T";
                     bool first = true;
                     for (const auto& tag : object.tags()) {
@@ -180,10 +173,11 @@ namespace osmium {
 
             public:
 
-                explicit OPLOutputBlock(osmium::memory::Buffer&& buffer) :
+                explicit OPLOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata) :
                     m_input_buffer(std::make_shared<osmium::memory::Buffer>(std::move(buffer))),
                     m_out(std::make_shared<std::string>()),
-                    m_tmp_buffer() {
+                    m_tmp_buffer(),
+                    m_add_metadata(add_metadata) {
                 }
 
                 OPLOutputBlock(const OPLOutputBlock&) = default;
@@ -240,7 +234,7 @@ namespace osmium {
                         }
                         *m_out += item_type_to_char(member.type());
                         output_formatted("%" PRId64 "@", member.ref());
-                        *m_out += member.role();
+                        append_encoded_string(member.role());
                     }
                     *m_out += '\n';
                 }
@@ -274,17 +268,20 @@ namespace osmium {
 
             class OPLOutputFormat : public osmium::io::detail::OutputFormat {
 
-                OPLOutputFormat(const OPLOutputFormat&) = delete;
+                bool m_add_metadata;
-                OPLOutputFormat& operator=(const OPLOutputFormat&) = delete;
 
             public:
 
                 OPLOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) :
-                    OutputFormat(file, output_queue) {
+                    OutputFormat(file, output_queue),
+                    m_add_metadata(file.get("add_metadata") != "false") {
                 }
 
+                OPLOutputFormat(const OPLOutputFormat&) = delete;
+                OPLOutputFormat& operator=(const OPLOutputFormat&) = delete;
+
                 void write_buffer(osmium::memory::Buffer&& buffer) override final {
-                    m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer)}));
+                    m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer), m_add_metadata}));
                 }
 
                 void close() override final {
@@ -298,6 +295,8 @@ namespace osmium {
 
             namespace {
 
+// we want the register_output_format() function to run, setting the variable
+// is only a side-effect, it will never be used
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunused-variable"
                 const bool registered_opl_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::opl,

third_party/libosmium/include/osmium/io/detail/pbf.hpp

@@ -33,9 +33,7 @@ DEALINGS IN THE SOFTWARE.
 
 */
 
-#include <stdexcept>
+#include <string>
-
-#include <osmpbf/osmpbf.h>
 
 // needed for htonl and ntohl
 #ifndef _WIN32
@@ -45,38 +43,10 @@ DEALINGS IN THE SOFTWARE.
 #endif
 
 #include <osmium/io/error.hpp>
-#include <osmium/osm/item_type.hpp>
+#include <osmium/osm/location.hpp>
 
 namespace osmium {
 
-// avoid g++ false positive
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wreturn-type"
-    inline item_type osmpbf_membertype_to_item_type(const OSMPBF::Relation::MemberType mt) {
-        switch (mt) {
-            case OSMPBF::Relation::NODE:
-                return item_type::node;
-            case OSMPBF::Relation::WAY:
-                return item_type::way;
-            case OSMPBF::Relation::RELATION:
-                return item_type::relation;
-        }
-    }
-#pragma GCC diagnostic pop
-
-    inline OSMPBF::Relation::MemberType item_type_to_osmpbf_membertype(const item_type type) {
-        switch (type) {
-            case item_type::node:
-                return OSMPBF::Relation::NODE;
-            case item_type::way:
-                return OSMPBF::Relation::WAY;
-            case item_type::relation:
-                return OSMPBF::Relation::RELATION;
-            default:
-                throw std::runtime_error("Unknown relation member type");
-        }
-    }
-
     /**
      * Exception thrown when there was a problem with parsing the PBF format of
      * a file.
@@ -93,6 +63,26 @@ namespace osmium {
 
     }; // struct pbf_error
 
+    namespace io {
+
+        namespace detail {
+
+            // the maximum size of a blob header in bytes
+            const int max_blob_header_size = 64 * 1024; // 64 kB
+
+            // the maximum size of an uncompressed blob in bytes
+            const uint64_t max_uncompressed_blob_size = 32 * 1024 * 1024; // 32 MB
+
+            // resolution for longitude/latitude used for conversion
+            // between representation as double and as int
+            const int64_t lonlat_resolution = 1000 * 1000 * 1000;
+
+            const int64_t resolution_convert = lonlat_resolution / osmium::Location::coordinate_precision;
+
+        }
+
+    }
+
 } // namespace osmium
 
 #endif // OSMIUM_IO_DETAIL_PBF_HPP

third_party/libosmium/include/osmium/io/detail/pbf_decoder.hpp

@@ -0,0 +1,760 @@
+#ifndef OSMIUM_IO_DETAIL_PBF_DECODER_HPP
+#define OSMIUM_IO_DETAIL_PBF_DECODER_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <algorithm>
+#include <iterator>
+#include <limits>
+
+#include <protozero/pbf_message.hpp>
+
+#include <osmium/builder/osm_object_builder.hpp>
+#include <osmium/io/detail/pbf.hpp> // IWYU pragma: export
+#include <osmium/io/detail/protobuf_tags.hpp>
+#include <osmium/io/detail/zlib.hpp>
+#include <osmium/io/header.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/node.hpp>
+#include <osmium/osm/types.hpp>
+#include <osmium/memory/buffer.hpp>
+#include <osmium/osm/entity_bits.hpp>
+#include <osmium/util/cast.hpp>
+#include <osmium/util/delta.hpp>
+
+namespace osmium {
+
+    namespace io {
+
+        namespace detail {
+
+            using ptr_len_type = std::pair<const char*, size_t>;
+
+            class PBFPrimitiveBlockDecoder {
+
+                static constexpr size_t initial_buffer_size = 2 * 1024 * 1024;
+
+                ptr_len_type m_data;
+                std::vector<ptr_len_type> m_stringtable;
+
+                int64_t m_lon_offset = 0;
+                int64_t m_lat_offset = 0;
+                int64_t m_date_factor = 1000;
+                int32_t m_granularity = 100;
+
+                osmium::osm_entity_bits::type m_read_types;
+
+                osmium::memory::Buffer m_buffer { initial_buffer_size };
+
+                void decode_stringtable(const ptr_len_type& data) {
+                    if (!m_stringtable.empty()) {
+                        throw osmium::pbf_error("more than one stringtable in pbf file");
+                    }
+
+                    protozero::pbf_message<OSMFormat::StringTable> pbf_string_table(data);
+                    while (pbf_string_table.next(OSMFormat::StringTable::repeated_bytes_s)) {
+                        m_stringtable.push_back(pbf_string_table.get_data());
+                    }
+                }
+
+                void decode_primitive_block_metadata() {
+                    protozero::pbf_message<OSMFormat::PrimitiveBlock> pbf_primitive_block(m_data);
+                    while (pbf_primitive_block.next()) {
+                        switch (pbf_primitive_block.tag()) {
+                            case OSMFormat::PrimitiveBlock::required_StringTable_stringtable:
+                                decode_stringtable(pbf_primitive_block.get_data());
+                                break;
+                            case OSMFormat::PrimitiveBlock::optional_int32_granularity:
+                                m_granularity = pbf_primitive_block.get_int32();
+                                break;
+                            case OSMFormat::PrimitiveBlock::optional_int32_date_granularity:
+                                m_date_factor = pbf_primitive_block.get_int32();
+                                break;
+                            case OSMFormat::PrimitiveBlock::optional_int64_lat_offset:
+                                m_lat_offset = pbf_primitive_block.get_int64();
+                                break;
+                            case OSMFormat::PrimitiveBlock::optional_int64_lon_offset:
+                                m_lon_offset = pbf_primitive_block.get_int64();
+                                break;
+                            default:
+                                pbf_primitive_block.skip();
+                        }
+                    }
+                }
+
+                void decode_primitive_block_data() {
+                    protozero::pbf_message<OSMFormat::PrimitiveBlock> pbf_primitive_block(m_data);
+                    while (pbf_primitive_block.next(OSMFormat::PrimitiveBlock::repeated_PrimitiveGroup_primitivegroup)) {
+                        protozero::pbf_message<OSMFormat::PrimitiveGroup> pbf_primitive_group = pbf_primitive_block.get_message();
+                        while (pbf_primitive_group.next()) {
+                            switch (pbf_primitive_group.tag()) {
+                                case OSMFormat::PrimitiveGroup::repeated_Node_nodes:
+                                    if (m_read_types & osmium::osm_entity_bits::node) {
+                                        decode_node(pbf_primitive_group.get_data());
+                                    } else {
+                                        pbf_primitive_group.skip();
+                                    }
+                                    break;
+                                case OSMFormat::PrimitiveGroup::optional_DenseNodes_dense:
+                                    if (m_read_types & osmium::osm_entity_bits::node) {
+                                        decode_dense_nodes(pbf_primitive_group.get_data());
+                                    } else {
+                                        pbf_primitive_group.skip();
+                                    }
+                                    break;
+                                case OSMFormat::PrimitiveGroup::repeated_Way_ways:
+                                    if (m_read_types & osmium::osm_entity_bits::way) {
+                                        decode_way(pbf_primitive_group.get_data());
+                                    } else {
+                                        pbf_primitive_group.skip();
+                                    }
+                                    break;
+                                case OSMFormat::PrimitiveGroup::repeated_Relation_relations:
+                                    if (m_read_types & osmium::osm_entity_bits::relation) {
+                                        decode_relation(pbf_primitive_group.get_data());
+                                    } else {
+                                        pbf_primitive_group.skip();
+                                    }
+                                    break;
+                                default:
+                                    pbf_primitive_group.skip();
+                            }
+                        }
+                    }
+                }
+
+                ptr_len_type decode_info(const ptr_len_type& data, osmium::OSMObject& object) {
+                    ptr_len_type user = std::make_pair("", 0);
+
+                    protozero::pbf_message<OSMFormat::Info> pbf_info(data);
+                    while (pbf_info.next()) {
+                        switch (pbf_info.tag()) {
+                            case OSMFormat::Info::optional_int32_version:
+                                {
+                                    auto version = pbf_info.get_int32();
+                                    if (version < 0) {
+                                        throw osmium::pbf_error("object version must not be negative");
+                                    }
+                                    object.set_version(static_cast_with_assert<object_version_type>(version));
+                                }
+                                break;
+                            case OSMFormat::Info::optional_int64_timestamp:
+                                object.set_timestamp(pbf_info.get_int64() * m_date_factor / 1000);
+                                break;
+                            case OSMFormat::Info::optional_int64_changeset:
+                                {
+                                    auto changeset_id = pbf_info.get_int64();
+                                    if (changeset_id < 0) {
+                                        throw osmium::pbf_error("object changeset_id must not be negative");
+                                    }
+                                    object.set_changeset(static_cast_with_assert<changeset_id_type>(changeset_id));
+                                }
+                                break;
+                            case OSMFormat::Info::optional_int32_uid:
+                                object.set_uid_from_signed(pbf_info.get_int32());
+                                break;
+                            case OSMFormat::Info::optional_uint32_user_sid:
+                                user = m_stringtable.at(pbf_info.get_uint32());
+                                break;
+                            case OSMFormat::Info::optional_bool_visible:
+                                object.set_visible(pbf_info.get_bool());
+                                break;
+                            default:
+                                pbf_info.skip();
+                        }
+                    }
+
+                    return user;
+                }
+
+                using kv_type = std::pair<protozero::pbf_reader::const_uint32_iterator, protozero::pbf_reader::const_uint32_iterator>;
+
+                void build_tag_list(osmium::builder::Builder& builder, const kv_type& keys, const kv_type& vals) {
+                    if (keys.first != keys.second) {
+                        osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
+                        auto kit = keys.first;
+                        auto vit = vals.first;
+                        while (kit != keys.second) {
+                            if (vit == vals.second) {
+                                // this is against the spec, must have same number of elements
+                                throw osmium::pbf_error("PBF format error");
+                            }
+                            const auto& k = m_stringtable.at(*kit++);
+                            const auto& v = m_stringtable.at(*vit++);
+                            tl_builder.add_tag(k.first, k.second, v.first, v.second);
+                        }
+                    }
+                }
+
+                int32_t convert_pbf_coordinate(int64_t c) const {
+                    return (c * m_granularity + m_lon_offset) / resolution_convert;
+                }
+
+                void decode_node(const ptr_len_type& data) {
+                    osmium::builder::NodeBuilder builder(m_buffer);
+                    osmium::Node& node = builder.object();
+
+                    kv_type keys;
+                    kv_type vals;
+                    int64_t lon = std::numeric_limits<int64_t>::max();
+                    int64_t lat = std::numeric_limits<int64_t>::max();
+
+                    ptr_len_type user = { "", 0 };
+
+                    protozero::pbf_message<OSMFormat::Node> pbf_node(data);
+                    while (pbf_node.next()) {
+                        switch (pbf_node.tag()) {
+                            case OSMFormat::Node::required_sint64_id:
+                                node.set_id(pbf_node.get_sint64());
+                                break;
+                            case OSMFormat::Node::packed_uint32_keys:
+                                keys = pbf_node.get_packed_uint32();
+                                break;
+                            case OSMFormat::Node::packed_uint32_vals:
+                                vals = pbf_node.get_packed_uint32();
+                                break;
+                            case OSMFormat::Node::optional_Info_info:
+                                user = decode_info(pbf_node.get_data(), builder.object());
+                                break;
+                            case OSMFormat::Node::required_sint64_lat:
+                                lat = pbf_node.get_sint64();
+                                break;
+                            case OSMFormat::Node::required_sint64_lon:
+                                lon = pbf_node.get_sint64();
+                                break;
+                            default:
+                                pbf_node.skip();
+                        }
+                    }
+
+                    if (node.visible()) {
+                        if (lon == std::numeric_limits<int64_t>::max() ||
+                            lat == std::numeric_limits<int64_t>::max()) {
+                            throw osmium::pbf_error("illegal coordinate format");
+                        }
+                        node.set_location(osmium::Location(
+                                convert_pbf_coordinate(lon),
+                                convert_pbf_coordinate(lat)
+                        ));
+                    }
+
+                    builder.add_user(user.first, user.second);
+
+                    build_tag_list(builder, keys, vals);
+
+                    m_buffer.commit();
+                }
+
+                void decode_way(const ptr_len_type& data) {
+                    osmium::builder::WayBuilder builder(m_buffer);
+
+                    kv_type keys;
+                    kv_type vals;
+                    std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> refs;
+
+                    ptr_len_type user = { "", 0 };
+
+                    protozero::pbf_message<OSMFormat::Way> pbf_way(data);
+                    while (pbf_way.next()) {
+                        switch (pbf_way.tag()) {
+                            case OSMFormat::Way::required_int64_id:
+                                builder.object().set_id(pbf_way.get_int64());
+                                break;
+                            case OSMFormat::Way::packed_uint32_keys:
+                                keys = pbf_way.get_packed_uint32();
+                                break;
+                            case OSMFormat::Way::packed_uint32_vals:
+                                vals = pbf_way.get_packed_uint32();
+                                break;
+                            case OSMFormat::Way::optional_Info_info:
+                                user = decode_info(pbf_way.get_data(), builder.object());
+                                break;
+                            case OSMFormat::Way::packed_sint64_refs:
+                                refs = pbf_way.get_packed_sint64();
+                                break;
+                            default:
+                                pbf_way.skip();
+                        }
+                    }
+
+                    builder.add_user(user.first, user.second);
+
+                    if (refs.first != refs.second) {
+                        osmium::builder::WayNodeListBuilder wnl_builder(m_buffer, &builder);
+                        osmium::util::DeltaDecode<int64_t> ref;
+                        while (refs.first != refs.second) {
+                            wnl_builder.add_node_ref(ref.update(*refs.first++));
+                        }
+                    }
+
+                    build_tag_list(builder, keys, vals);
+
+                    m_buffer.commit();
+                }
+
+                void decode_relation(const ptr_len_type& data) {
+                    osmium::builder::RelationBuilder builder(m_buffer);
+
+                    kv_type keys;
+                    kv_type vals;
+                    std::pair<protozero::pbf_reader::const_int32_iterator,  protozero::pbf_reader::const_int32_iterator> roles;
+                    std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> refs;
+                    std::pair<protozero::pbf_reader::const_int32_iterator,  protozero::pbf_reader::const_int32_iterator> types;
+
+                    ptr_len_type user = { "", 0 };
+
+                    protozero::pbf_message<OSMFormat::Relation> pbf_relation(data);
+                    while (pbf_relation.next()) {
+                        switch (pbf_relation.tag()) {
+                            case OSMFormat::Relation::required_int64_id:
+                                builder.object().set_id(pbf_relation.get_int64());
+                                break;
+                            case OSMFormat::Relation::packed_uint32_keys:
+                                keys = pbf_relation.get_packed_uint32();
+                                break;
+                            case OSMFormat::Relation::packed_uint32_vals:
+                                vals = pbf_relation.get_packed_uint32();
+                                break;
+                            case OSMFormat::Relation::optional_Info_info:
+                                user = decode_info(pbf_relation.get_data(), builder.object());
+                                break;
+                            case OSMFormat::Relation::packed_int32_roles_sid:
+                                roles = pbf_relation.get_packed_int32();
+                                break;
+                            case OSMFormat::Relation::packed_sint64_memids:
+                                refs = pbf_relation.get_packed_sint64();
+                                break;
+                            case OSMFormat::Relation::packed_MemberType_types:
+                                types = pbf_relation.get_packed_enum();
+                                break;
+                            default:
+                                pbf_relation.skip();
+                        }
+                    }
+
+                    builder.add_user(user.first, user.second);
+
+                    if (refs.first != refs.second) {
+                        osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder);
+                        osmium::util::DeltaDecode<int64_t> ref;
+                        while (roles.first != roles.second && refs.first != refs.second && types.first != types.second) {
+                            const auto& r = m_stringtable.at(*roles.first++);
+                            int type = *types.first++;
+                            if (type < 0 || type > 2) {
+                                throw osmium::pbf_error("unknown relation member type");
+                            }
+                            rml_builder.add_member(
+                                osmium::item_type(type + 1),
+                                ref.update(*refs.first++),
+                                r.first,
+                                r.second
+                            );
+                        }
+                    }
+
+                    build_tag_list(builder, keys, vals);
+
+                    m_buffer.commit();
+                }
+
+                void decode_dense_nodes(const ptr_len_type& data) {
+                    bool has_info     = false;
+                    bool has_visibles = false;
+
+                    std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> ids;
+                    std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> lats;
+                    std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> lons;
+
+                    std::pair<protozero::pbf_reader::const_int32_iterator,  protozero::pbf_reader::const_int32_iterator>  tags;
+
+                    std::pair<protozero::pbf_reader::const_int32_iterator,  protozero::pbf_reader::const_int32_iterator>  versions;
+                    std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> timestamps;
+                    std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> changesets;
+                    std::pair<protozero::pbf_reader::const_sint32_iterator, protozero::pbf_reader::const_sint32_iterator> uids;
+                    std::pair<protozero::pbf_reader::const_sint32_iterator, protozero::pbf_reader::const_sint32_iterator> user_sids;
+                    std::pair<protozero::pbf_reader::const_int32_iterator,  protozero::pbf_reader::const_int32_iterator>  visibles;
+
+                    protozero::pbf_message<OSMFormat::DenseNodes> pbf_dense_nodes(data);
+                    while (pbf_dense_nodes.next()) {
+                        switch (pbf_dense_nodes.tag()) {
+                            case OSMFormat::DenseNodes::packed_sint64_id:
+                                ids = pbf_dense_nodes.get_packed_sint64();
+                                break;
+                            case OSMFormat::DenseNodes::optional_DenseInfo_denseinfo:
+                                {
+                                    has_info = true;
+                                    protozero::pbf_message<OSMFormat::DenseInfo> pbf_dense_info = pbf_dense_nodes.get_message();
+                                    while (pbf_dense_info.next()) {
+                                        switch (pbf_dense_info.tag()) {
+                                            case OSMFormat::DenseInfo::packed_int32_version:
+                                                versions = pbf_dense_info.get_packed_int32();
+                                                break;
+                                            case OSMFormat::DenseInfo::packed_sint64_timestamp:
+                                                timestamps = pbf_dense_info.get_packed_sint64();
+                                                break;
+                                            case OSMFormat::DenseInfo::packed_sint64_changeset:
+                                                changesets = pbf_dense_info.get_packed_sint64();
+                                                break;
+                                            case OSMFormat::DenseInfo::packed_sint32_uid:
+                                                uids = pbf_dense_info.get_packed_sint32();
+                                                break;
+                                            case OSMFormat::DenseInfo::packed_sint32_user_sid:
+                                                user_sids = pbf_dense_info.get_packed_sint32();
+                                                break;
+                                            case OSMFormat::DenseInfo::packed_bool_visible:
+                                                has_visibles = true;
+                                                visibles = pbf_dense_info.get_packed_bool();
+                                                break;
+                                            default:
+                                                pbf_dense_info.skip();
+                                        }
+                                    }
+                                }
+                                break;
+                            case OSMFormat::DenseNodes::packed_sint64_lat:
+                                lats = pbf_dense_nodes.get_packed_sint64();
+                                break;
+                            case OSMFormat::DenseNodes::packed_sint64_lon:
+                                lons = pbf_dense_nodes.get_packed_sint64();
+                                break;
+                            case OSMFormat::DenseNodes::packed_int32_keys_vals:
+                                tags = pbf_dense_nodes.get_packed_int32();
+                                break;
+                            default:
+                                pbf_dense_nodes.skip();
+                        }
+                    }
+
+                    osmium::util::DeltaDecode<int64_t> dense_id;
+                    osmium::util::DeltaDecode<int64_t> dense_latitude;
+                    osmium::util::DeltaDecode<int64_t> dense_longitude;
+                    osmium::util::DeltaDecode<int64_t> dense_uid;
+                    osmium::util::DeltaDecode<int64_t> dense_user_sid;
+                    osmium::util::DeltaDecode<int64_t> dense_changeset;
+                    osmium::util::DeltaDecode<int64_t> dense_timestamp;
+
+                    auto tag_it = tags.first;
+
+                    while (ids.first != ids.second) {
+                        if (lons.first == lons.second ||
+                            lats.first == lats.second) {
+                            // this is against the spec, must have same number of elements
+                            throw osmium::pbf_error("PBF format error");
+                        }
+
+                        bool visible = true;
+
+                        osmium::builder::NodeBuilder builder(m_buffer);
+                        osmium::Node& node = builder.object();
+
+                        node.set_id(dense_id.update(*ids.first++));
+
+                        if (has_info) {
+                            if (versions.first == versions.second ||
+                                changesets.first == changesets.second ||
+                                timestamps.first == timestamps.second ||
+                                uids.first == uids.second ||
+                                user_sids.first == user_sids.second) {
+                                // this is against the spec, must have same number of elements
+                                throw osmium::pbf_error("PBF format error");
+                            }
+
+                            auto version = *versions.first++;
+                            if (version < 0) {
+                                throw osmium::pbf_error("object version must not be negative");
+                            }
+                            node.set_version(static_cast<osmium::object_version_type>(version));
+
+                            auto changeset_id = dense_changeset.update(*changesets.first++);
+                            if (changeset_id < 0) {
+                                throw osmium::pbf_error("object changeset_id must not be negative");
+                            }
+                            node.set_changeset(static_cast<osmium::changeset_id_type>(changeset_id));
+
+                            node.set_timestamp(dense_timestamp.update(*timestamps.first++) * m_date_factor / 1000);
+                            node.set_uid_from_signed(static_cast<osmium::signed_user_id_type>(dense_uid.update(*uids.first++)));
+
+                            if (has_visibles) {
+                                if (visibles.first == visibles.second) {
+                                    // this is against the spec, must have same number of elements
+                                    throw osmium::pbf_error("PBF format error");
+                                }
+                                visible = *visibles.first++;
+                            }
+                            node.set_visible(visible);
+
+                            const auto& u = m_stringtable.at(dense_user_sid.update(*user_sids.first++));
+                            builder.add_user(u.first, u.second);
+                        } else {
+                            builder.add_user("");
+                        }
+
+                        if (visible) {
+                            builder.object().set_location(osmium::Location(
+                                    convert_pbf_coordinate(dense_longitude.update(*lons.first++)),
+                                    convert_pbf_coordinate(dense_latitude.update(*lats.first++))
+                            ));
+                        }
+
+                        if (tag_it != tags.second) {
+                            osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
+                            while (tag_it != tags.second && *tag_it != 0) {
+                                const auto& k = m_stringtable.at(*tag_it++);
+                                if (tag_it == tags.second) {
+                                    throw osmium::pbf_error("PBF format error"); // this is against the spec, keys/vals must come in pairs
+                                }
+                                const auto& v = m_stringtable.at(*tag_it++);
+                                tl_builder.add_tag(k.first, k.second, v.first, v.second);
+                            }
+
+                            if (tag_it != tags.second) {
+                                ++tag_it;
+                            }
+                        }
+
+                        m_buffer.commit();
+                    }
+
+                }
+
+            public:
+
+                explicit PBFPrimitiveBlockDecoder(const ptr_len_type& data, osmium::osm_entity_bits::type read_types) :
+                    m_data(data),
+                    m_read_types(read_types) {
+                }
+
+                PBFPrimitiveBlockDecoder(const PBFPrimitiveBlockDecoder&) = delete;
+                PBFPrimitiveBlockDecoder& operator=(const PBFPrimitiveBlockDecoder&) = delete;
+
+                PBFPrimitiveBlockDecoder(PBFPrimitiveBlockDecoder&&) = delete;
+                PBFPrimitiveBlockDecoder& operator=(PBFPrimitiveBlockDecoder&&) = delete;
+
+                ~PBFPrimitiveBlockDecoder() = default;
+
+                osmium::memory::Buffer operator()() {
+                    try {
+                        decode_primitive_block_metadata();
+                        decode_primitive_block_data();
+                    } catch (std::out_of_range&) {
+                        throw osmium::pbf_error("string id out of range");
+                    }
+
+                    return std::move(m_buffer);
+                }
+
+            }; // class PBFPrimitiveBlockDecoder
+
+            inline ptr_len_type decode_blob(const std::string& blob_data, std::string& output) {
+                int32_t raw_size;
+                std::pair<const char*, protozero::pbf_length_type> zlib_data;
+
+                protozero::pbf_message<FileFormat::Blob> pbf_blob(blob_data);
+                while (pbf_blob.next()) {
+                    switch (pbf_blob.tag()) {
+                        case FileFormat::Blob::optional_bytes_raw:
+                            {
+                                auto data_len = pbf_blob.get_data();
+                                if (data_len.second > max_uncompressed_blob_size) {
+                                    throw osmium::pbf_error("illegal blob size");
+                                }
+                                return data_len;
+                            }
+                        case FileFormat::Blob::optional_int32_raw_size:
+                            raw_size = pbf_blob.get_int32();
+                            if (raw_size <= 0 || uint32_t(raw_size) > max_uncompressed_blob_size) {
+                                throw osmium::pbf_error("illegal blob size");
+                            }
+                            break;
+                        case FileFormat::Blob::optional_bytes_zlib_data:
+                            zlib_data = pbf_blob.get_data();
+                            break;
+                        case FileFormat::Blob::optional_bytes_lzma_data:
+                            throw osmium::pbf_error("lzma blobs not implemented");
+                        default:
+                            throw osmium::pbf_error("unknown compression");
+                    }
+                }
+
+                if (zlib_data.second != 0) {
+                    return osmium::io::detail::zlib_uncompress_string(
+                        zlib_data.first,
+                        static_cast<unsigned long>(zlib_data.second),
+                        static_cast<unsigned long>(raw_size),
+                        output
+                    );
+                }
+
+                throw osmium::pbf_error("blob contains no data");
+            }
+
+            inline osmium::Box decode_header_bbox(const ptr_len_type& data) {
+                    int64_t left   = std::numeric_limits<int64_t>::max();
+                    int64_t right  = std::numeric_limits<int64_t>::max();
+                    int64_t top    = std::numeric_limits<int64_t>::max();
+                    int64_t bottom = std::numeric_limits<int64_t>::max();
+
+                    protozero::pbf_message<OSMFormat::HeaderBBox> pbf_header_bbox(data);
+                    while (pbf_header_bbox.next()) {
+                        switch (pbf_header_bbox.tag()) {
+                            case OSMFormat::HeaderBBox::required_sint64_left:
+                                left = pbf_header_bbox.get_sint64();
+                                break;
+                            case OSMFormat::HeaderBBox::required_sint64_right:
+                                right = pbf_header_bbox.get_sint64();
+                                break;
+                            case OSMFormat::HeaderBBox::required_sint64_top:
+                                top = pbf_header_bbox.get_sint64();
+                                break;
+                            case OSMFormat::HeaderBBox::required_sint64_bottom:
+                                bottom = pbf_header_bbox.get_sint64();
+                                break;
+                            default:
+                                pbf_header_bbox.skip();
+                        }
+                    }
+
+                    if (left   == std::numeric_limits<int64_t>::max() ||
+                        right  == std::numeric_limits<int64_t>::max() ||
+                        top    == std::numeric_limits<int64_t>::max() ||
+                        bottom == std::numeric_limits<int64_t>::max()) {
+                        throw osmium::pbf_error("invalid bbox");
+                    }
+
+                    osmium::Box box;
+                    box.extend(osmium::Location(left  / resolution_convert, bottom / resolution_convert));
+                    box.extend(osmium::Location(right / resolution_convert, top    / resolution_convert));
+
+                    return box;
+            }
+
+            inline osmium::io::Header decode_header_block(const ptr_len_type& data) {
+                osmium::io::Header header;
+                int i = 0;
+
+                protozero::pbf_message<OSMFormat::HeaderBlock> pbf_header_block(data);
+                while (pbf_header_block.next()) {
+                    switch (pbf_header_block.tag()) {
+                        case OSMFormat::HeaderBlock::optional_HeaderBBox_bbox:
+                            header.add_box(decode_header_bbox(pbf_header_block.get_data()));
+                            break;
+                        case OSMFormat::HeaderBlock::repeated_string_required_features:
+                            {
+                                auto feature = pbf_header_block.get_data();
+                                if (!strncmp("OsmSchema-V0.6", feature.first, feature.second)) {
+                                    // intentionally left blank
+                                } else if (!strncmp("DenseNodes", feature.first, feature.second)) {
+                                    header.set("pbf_dense_nodes", true);
+                                } else if (!strncmp("HistoricalInformation", feature.first, feature.second)) {
+                                    header.set_has_multiple_object_versions(true);
+                                } else {
+                                    std::string msg("required feature not supported: ");
+                                    msg.append(feature.first, feature.second);
+                                    throw osmium::pbf_error(msg);
+                                }
+                            }
+                            break;
+                        case OSMFormat::HeaderBlock::repeated_string_optional_features:
+                            header.set("pbf_optional_feature_" + std::to_string(i++), pbf_header_block.get_string());
+                            break;
+                        case OSMFormat::HeaderBlock::optional_string_writingprogram:
+                            header.set("generator", pbf_header_block.get_string());
+                            break;
+                        case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp:
+                            header.set("osmosis_replication_timestamp", osmium::Timestamp(pbf_header_block.get_int64()).to_iso());
+                            break;
+                        case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_sequence_number:
+                            header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.get_int64()));
+                            break;
+                        case OSMFormat::HeaderBlock::optional_string_osmosis_replication_base_url:
+                            header.set("osmosis_replication_base_url", pbf_header_block.get_string());
+                            break;
+                        default:
+                            pbf_header_block.skip();
+                    }
+                }
+
+                return header;
+            }
+
+            /**
+             * Decode HeaderBlock.
+             *
+             * @param header_block_data Input data
+             * @returns Header object
+             * @throws osmium::pbf_error If there was a parsing error
+             */
+            inline osmium::io::Header decode_header(const std::string& header_block_data) {
+                std::string output;
+
+                return decode_header_block(decode_blob(header_block_data, output));
+            }
+
+            class PBFDataBlobDecoder {
+
+                std::shared_ptr<std::string> m_input_buffer;
+                osmium::osm_entity_bits::type m_read_types;
+
+            public:
+
+                PBFDataBlobDecoder(std::string&& input_buffer, osmium::osm_entity_bits::type read_types) :
+                    m_input_buffer(std::make_shared<std::string>(std::move(input_buffer))),
+                    m_read_types(read_types) {
+                }
+
+                PBFDataBlobDecoder(const PBFDataBlobDecoder&) = default;
+                PBFDataBlobDecoder& operator=(const PBFDataBlobDecoder&) = default;
+
+                PBFDataBlobDecoder(PBFDataBlobDecoder&&) = default;
+                PBFDataBlobDecoder& operator=(PBFDataBlobDecoder&&) = default;
+
+                ~PBFDataBlobDecoder() = default;
+
+                osmium::memory::Buffer operator()() {
+                    std::string output;
+                    PBFPrimitiveBlockDecoder decoder(decode_blob(*m_input_buffer, output), m_read_types);
+                    return decoder();
+                }
+
+            }; // class PBFDataBlobDecoder
+
+        } // namespace detail
+
+    } // namespace io
+
+} // namespace osmium
+
+#endif // OSMIUM_IO_DETAIL_PBF_DECODER_HPP

third_party/libosmium/include/osmium/io/detail/pbf_input_format.hpp

@@ -49,9 +49,12 @@ DEALINGS IN THE SOFTWARE.
 #include <thread>
 #include <type_traits>
 
+#include <protozero/pbf_message.hpp>
+
 #include <osmium/io/detail/input_format.hpp>
 #include <osmium/io/detail/pbf.hpp> // IWYU pragma: export
-#include <osmium/io/detail/pbf_parser.hpp>
+#include <osmium/io/detail/pbf_decoder.hpp>
+#include <osmium/io/detail/protobuf_tags.hpp>
 #include <osmium/io/error.hpp>
 #include <osmium/io/file.hpp>
 #include <osmium/io/file_format.hpp>
@@ -76,13 +79,13 @@ namespace osmium {
 
         namespace detail {
 
-            typedef osmium::thread::Queue<std::future<osmium::memory::Buffer>> queue_type;
-
             /**
              * Class for parsing PBF files.
              */
             class PBFInputFormat : public osmium::io::detail::InputFormat {
 
+                typedef osmium::thread::Queue<std::future<osmium::memory::Buffer>> queue_type;
+
                 bool m_use_thread_pool;
                 bool m_eof { false };
                 queue_type m_queue;
@@ -115,15 +118,10 @@ namespace osmium {
                 }
 
                 /**
-                 * Read BlobHeader by first reading the size and then the
+                 * Read 4 bytes in network byte order from file. They contain
-                 * BlobHeader. The BlobHeader contains a type field (which is
+                 * the length of the following BlobHeader.
-                 * checked against the expected type) and a size field.
-                 *
-                 * @param expected_type Expected type of data ("OSMHeader" or
-                 *                      "OSMData").
-                 * @returns Size of the data read from BlobHeader (0 on EOF).
                  */
-                size_t read_blob_header(const char* expected_type) {
+                uint32_t read_blob_header_size_from_file() {
                     uint32_t size_in_network_byte_order;
 
                     try {
@@ -133,37 +131,76 @@ namespace osmium {
                         return 0; // EOF
                     }
 
-                    uint32_t size = ntohl(size_in_network_byte_order);
+                    const uint32_t size = ntohl(size_in_network_byte_order);
-                    if (size > static_cast<uint32_t>(OSMPBF::max_blob_header_size)) {
+                    if (size > static_cast<uint32_t>(max_blob_header_size)) {
                         throw osmium::pbf_error("invalid BlobHeader size (> max_blob_header_size)");
                     }
 
-                    OSMPBF::BlobHeader blob_header;
+                    return size;
-                    if (!blob_header.ParseFromString(read_from_input_queue(size))) {
+                }
-                        throw osmium::pbf_error("failed to parse BlobHeader");
+
+                /**
+                 * Decode the BlobHeader. Make sure it contains the expected
+                 * type. Return the size of the following Blob.
+                 */
+                size_t decode_blob_header(protozero::pbf_message<FileFormat::BlobHeader>&& pbf_blob_header, const char* expected_type) {
+                    std::pair<const char*, size_t> blob_header_type;
+                    size_t blob_header_datasize = 0;
+
+                    while (pbf_blob_header.next()) {
+                        switch (pbf_blob_header.tag()) {
+                            case FileFormat::BlobHeader::required_string_type:
+                                blob_header_type = pbf_blob_header.get_data();
+                                break;
+                            case FileFormat::BlobHeader::required_int32_datasize:
+                                blob_header_datasize = pbf_blob_header.get_int32();
+                                break;
+                            default:
+                                pbf_blob_header.skip();
+                        }
                     }
 
-                    if (blob_header.type() != expected_type) {
+                    if (blob_header_datasize == 0) {
+                        throw osmium::pbf_error("PBF format error: BlobHeader.datasize missing or zero.");
+                    }
+
+                    if (strncmp(expected_type, blob_header_type.first, blob_header_type.second)) {
                         throw osmium::pbf_error("blob does not have expected type (OSMHeader in first blob, OSMData in following blobs)");
                     }
 
-                    return static_cast<size_t>(blob_header.datasize());
+                    return blob_header_datasize;
+                }
+
+                size_t check_type_and_get_blob_size(const char* expected_type) {
+                    assert(expected_type);
+
+                    auto size = read_blob_header_size_from_file();
+                    if (size == 0) { // EOF
+                        return 0;
+                    }
+
+                    std::string blob_header = read_from_input_queue(size);
+
+                    return decode_blob_header(protozero::pbf_message<FileFormat::BlobHeader>(blob_header), expected_type);
                 }
 
                 void parse_osm_data(osmium::osm_entity_bits::type read_types) {
                     osmium::thread::set_thread_name("_osmium_pbf_in");
-                    int n = 0;
+
-                    while (auto size = read_blob_header("OSMData")) {
+                    while (auto size = check_type_and_get_blob_size("OSMData")) {
+                        std::string input_buffer = read_from_input_queue(size);
+                        if (input_buffer.size() > max_uncompressed_blob_size) {
+                            throw osmium::pbf_error(std::string("invalid blob size: " + std::to_string(input_buffer.size())));
+                        }
 
                         if (m_use_thread_pool) {
-                            m_queue.push(osmium::thread::Pool::instance().submit(DataBlobParser{read_from_input_queue(size), read_types}));
+                            m_queue.push(osmium::thread::Pool::instance().submit(PBFDataBlobDecoder{ std::move(input_buffer), read_types }));
                         } else {
                             std::promise<osmium::memory::Buffer> promise;
                             m_queue.push(promise.get_future());
-                            DataBlobParser data_blob_parser{read_from_input_queue(size), read_types};
+                            PBFDataBlobDecoder data_blob_parser{ std::move(input_buffer), read_types };
                             promise.set_value(data_blob_parser());
                         }
-                        ++n;
 
                         if (m_quit_input_thread) {
                             return;
@@ -197,11 +234,10 @@ namespace osmium {
                     m_quit_input_thread(false),
                     m_input_queue(input_queue),
                     m_input_buffer() {
-                    GOOGLE_PROTOBUF_VERIFY_VERSION;
 
                     // handle OSMHeader
-                    auto size = read_blob_header("OSMHeader");
+                    const auto size = check_type_and_get_blob_size("OSMHeader");
-                    m_header = parse_header_blob(read_from_input_queue(size));
+                    m_header = decode_header(read_from_input_queue(size));
 
                     if (m_read_which_entities != osmium::osm_entity_bits::nothing) {
                         m_reader = std::thread(&PBFInputFormat::parse_osm_data, this, m_read_which_entities);
@@ -246,10 +282,15 @@ namespace osmium {
 
             namespace {
 
+// we want the register_input_format() function to run, setting the variable
+// is only a side-effect, it will never be used
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
                 const bool registered_pbf_input = osmium::io::detail::InputFormatFactory::instance().register_input_format(osmium::io::file_format::pbf,
                     [](const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue<std::string>& input_queue) {
                         return new osmium::io::detail::PBFInputFormat(file, read_which_entities, input_queue);
                 });
+#pragma GCC diagnostic pop
 
             } // anonymous namespace
 

third_party/libosmium/include/osmium/io/detail/pbf_parser.hpp

@@ -1,455 +0,0 @@
-#ifndef OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP
-#define OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP
-
-/*
-
-This file is part of Osmium (http://osmcode.org/libosmium).
-
-Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
-
-Boost Software License - Version 1.0 - August 17th, 2003
-
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-
-*/
-
-#include <cassert>
-#include <cstddef>
-#include <cstdint>
-#include <algorithm>
-
-#include <osmpbf/osmpbf.h>
-
-#include <osmium/builder/osm_object_builder.hpp>
-#include <osmium/io/detail/pbf.hpp> // IWYU pragma: export
-#include <osmium/io/detail/zlib.hpp>
-#include <osmium/io/header.hpp>
-#include <osmium/osm/location.hpp>
-#include <osmium/osm/node.hpp>
-#include <osmium/osm/types.hpp>
-#include <osmium/memory/buffer.hpp>
-#include <osmium/osm/entity_bits.hpp>
-#include <osmium/util/cast.hpp>
-
-namespace osmium {
-
-    namespace io {
-
-        namespace detail {
-
-            class PBFPrimitiveBlockParser {
-
-                static constexpr size_t initial_buffer_size = 2 * 1024 * 1024;
-
-                const std::string& m_data;
-
-                const OSMPBF::StringTable* m_stringtable;
-                int64_t m_lon_offset;
-                int64_t m_lat_offset;
-                int64_t m_date_factor;
-                int32_t m_granularity;
-
-                osmium::osm_entity_bits::type m_read_types;
-
-                osmium::memory::Buffer m_buffer;
-
-                PBFPrimitiveBlockParser(const PBFPrimitiveBlockParser&) = delete;
-                PBFPrimitiveBlockParser(PBFPrimitiveBlockParser&&) = delete;
-
-                PBFPrimitiveBlockParser& operator=(const PBFPrimitiveBlockParser&) = delete;
-                PBFPrimitiveBlockParser& operator=(PBFPrimitiveBlockParser&&) = delete;
-
-            public:
-
-                explicit PBFPrimitiveBlockParser(const std::string& data, osmium::osm_entity_bits::type read_types) :
-                    m_data(data),
-                    m_stringtable(nullptr),
-                    m_lon_offset(0),
-                    m_lat_offset(0),
-                    m_date_factor(1000),
-                    m_granularity(100),
-                    m_read_types(read_types),
-                    m_buffer(initial_buffer_size) {
-                }
-
-                ~PBFPrimitiveBlockParser() = default;
-
-                osmium::memory::Buffer operator()() {
-                    OSMPBF::PrimitiveBlock pbf_primitive_block;
-                    if (!pbf_primitive_block.ParseFromString(m_data)) {
-                        throw osmium::pbf_error("failed to parse PrimitiveBlock");
-                    }
-
-                    m_stringtable = &pbf_primitive_block.stringtable();
-                    m_lon_offset  = pbf_primitive_block.lon_offset();
-                    m_lat_offset  = pbf_primitive_block.lat_offset();
-                    m_date_factor = pbf_primitive_block.date_granularity() / 1000;
-                    m_granularity = pbf_primitive_block.granularity();
-
-                    for (int i = 0; i < pbf_primitive_block.primitivegroup_size(); ++i) {
-                        const OSMPBF::PrimitiveGroup& group = pbf_primitive_block.primitivegroup(i);
-
-                        if (group.has_dense())  {
-                            if (m_read_types & osmium::osm_entity_bits::node) parse_dense_node_group(group);
-                        } else if (group.ways_size() != 0) {
-                            if (m_read_types & osmium::osm_entity_bits::way) parse_way_group(group);
-                        } else if (group.relations_size() != 0) {
-                            if (m_read_types & osmium::osm_entity_bits::relation) parse_relation_group(group);
-                        } else if (group.nodes_size() != 0) {
-                            if (m_read_types & osmium::osm_entity_bits::node) parse_node_group(group);
-                        } else {
-                            throw osmium::pbf_error("group of unknown type");
-                        }
-                    }
-
-                    return std::move(m_buffer);
-                }
-
-            private:
-
-                template <class TBuilder, class TPBFObject>
-                void parse_attributes(TBuilder& builder, const TPBFObject& pbf_object) {
-                    auto& object = builder.object();
-
-                    object.set_id(pbf_object.id());
-
-                    if (pbf_object.has_info()) {
-                        object.set_version(static_cast_with_assert<object_version_type>(pbf_object.info().version()))
-                            .set_changeset(static_cast_with_assert<changeset_id_type>(pbf_object.info().changeset()))
-                            .set_timestamp(pbf_object.info().timestamp() * m_date_factor)
-                            .set_uid_from_signed(pbf_object.info().uid());
-                        if (pbf_object.info().has_visible()) {
-                            object.set_visible(pbf_object.info().visible());
-                        }
-                        builder.add_user(m_stringtable->s(static_cast_with_assert<int>(pbf_object.info().user_sid())));
-                    } else {
-                        builder.add_user("", 1);
-                    }
-                }
-
-                void parse_node_group(const OSMPBF::PrimitiveGroup& group) {
-                    for (int i = 0; i < group.nodes_size(); ++i) {
-                        osmium::builder::NodeBuilder builder(m_buffer);
-                        const OSMPBF::Node& pbf_node = group.nodes(i);
-                        parse_attributes(builder, pbf_node);
-
-                        if (builder.object().visible()) {
-                            builder.object().set_location(osmium::Location(
-                                              (pbf_node.lon() * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision),
-                                              (pbf_node.lat() * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision)));
-                        }
-
-                        if (pbf_node.keys_size() > 0) {
-                            osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
-                            for (int tag = 0; tag < pbf_node.keys_size(); ++tag) {
-                                tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_node.keys(tag))),
-                                                   m_stringtable->s(static_cast<int>(pbf_node.vals(tag))));
-                            }
-                        }
-
-                        m_buffer.commit();
-                    }
-                }
-
-                void parse_way_group(const OSMPBF::PrimitiveGroup& group) {
-                    for (int i = 0; i < group.ways_size(); ++i) {
-                        osmium::builder::WayBuilder builder(m_buffer);
-                        const OSMPBF::Way& pbf_way = group.ways(i);
-                        parse_attributes(builder, pbf_way);
-
-                        if (pbf_way.refs_size() > 0) {
-                            osmium::builder::WayNodeListBuilder wnl_builder(m_buffer, &builder);
-                            int64_t ref = 0;
-                            for (int n = 0; n < pbf_way.refs_size(); ++n) {
-                                ref += pbf_way.refs(n);
-                                wnl_builder.add_node_ref(ref);
-                            }
-                        }
-
-                        if (pbf_way.keys_size() > 0) {
-                            osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
-                            for (int tag = 0; tag < pbf_way.keys_size(); ++tag) {
-                                tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_way.keys(tag))),
-                                                   m_stringtable->s(static_cast<int>(pbf_way.vals(tag))));
-                            }
-                        }
-
-                        m_buffer.commit();
-                    }
-                }
-
-                void parse_relation_group(const OSMPBF::PrimitiveGroup& group) {
-                    for (int i = 0; i < group.relations_size(); ++i) {
-                        osmium::builder::RelationBuilder builder(m_buffer);
-                        const OSMPBF::Relation& pbf_relation = group.relations(i);
-                        parse_attributes(builder, pbf_relation);
-
-                        if (pbf_relation.types_size() > 0) {
-                            osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder);
-                            int64_t ref = 0;
-                            for (int n = 0; n < pbf_relation.types_size(); ++n) {
-                                ref += pbf_relation.memids(n);
-                                rml_builder.add_member(osmpbf_membertype_to_item_type(pbf_relation.types(n)), ref, m_stringtable->s(pbf_relation.roles_sid(n)));
-                            }
-                        }
-
-                        if (pbf_relation.keys_size() > 0) {
-                            osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
-                            for (int tag = 0; tag < pbf_relation.keys_size(); ++tag) {
-                                tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_relation.keys(tag))),
-                                                   m_stringtable->s(static_cast<int>(pbf_relation.vals(tag))));
-                            }
-                        }
-
-                        m_buffer.commit();
-                    }
-                }
-
-                int add_tags(const OSMPBF::DenseNodes& dense, int n, osmium::builder::NodeBuilder* builder) {
-                    if (n >= dense.keys_vals_size()) {
-                        return n;
-                    }
-
-                    if (dense.keys_vals(n) == 0) {
-                        return n+1;
-                    }
-
-                    osmium::builder::TagListBuilder tl_builder(m_buffer, builder);
-
-                    while (n < dense.keys_vals_size()) {
-                        int tag_key_pos = dense.keys_vals(n++);
-
-                        if (tag_key_pos == 0) {
-                            break;
-                        }
-
-                        tl_builder.add_tag(m_stringtable->s(tag_key_pos),
-                                           m_stringtable->s(dense.keys_vals(n)));
-
-                        ++n;
-                    }
-
-                    return n;
-                }
-
-                void parse_dense_node_group(const OSMPBF::PrimitiveGroup& group) {
-                    int64_t last_dense_id        = 0;
-                    int64_t last_dense_latitude  = 0;
-                    int64_t last_dense_longitude = 0;
-                    int64_t last_dense_uid       = 0;
-                    int64_t last_dense_user_sid  = 0;
-                    int64_t last_dense_changeset = 0;
-                    int64_t last_dense_timestamp = 0;
-                    int     last_dense_tag       = 0;
-
-                    const OSMPBF::DenseNodes& dense = group.dense();
-
-                    for (int i = 0; i < dense.id_size(); ++i) {
-                        bool visible = true;
-
-                        last_dense_id        += dense.id(i);
-                        last_dense_latitude  += dense.lat(i);
-                        last_dense_longitude += dense.lon(i);
-
-                        if (dense.has_denseinfo()) {
-                            last_dense_changeset += dense.denseinfo().changeset(i);
-                            last_dense_timestamp += dense.denseinfo().timestamp(i);
-                            last_dense_uid       += dense.denseinfo().uid(i);
-                            last_dense_user_sid  += dense.denseinfo().user_sid(i);
-                            if (dense.denseinfo().visible_size() > 0) {
-                                visible = dense.denseinfo().visible(i);
-                            }
-                            assert(last_dense_changeset >= 0);
-                            assert(last_dense_timestamp >= 0);
-                            assert(last_dense_uid >= -1);
-                            assert(last_dense_user_sid >= 0);
-                        }
-
-                        osmium::builder::NodeBuilder builder(m_buffer);
-                        osmium::Node& node = builder.object();
-
-                        node.set_id(last_dense_id);
-
-                        if (dense.has_denseinfo()) {
-                            auto v = dense.denseinfo().version(i);
-                            assert(v > 0);
-                            node.set_version(static_cast<osmium::object_version_type>(v));
-                            node.set_changeset(static_cast<osmium::changeset_id_type>(last_dense_changeset));
-                            node.set_timestamp(last_dense_timestamp * m_date_factor);
-                            node.set_uid_from_signed(static_cast<osmium::signed_user_id_type>(last_dense_uid));
-                            node.set_visible(visible);
-                            builder.add_user(m_stringtable->s(static_cast<int>(last_dense_user_sid)));
-                        } else {
-                            builder.add_user("", 1);
-                        }
-
-                        if (visible) {
-                            builder.object().set_location(osmium::Location(
-                                              (last_dense_longitude * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision),
-                                              (last_dense_latitude  * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision)));
-                        }
-
-                        last_dense_tag = add_tags(dense, last_dense_tag, &builder);
-                        m_buffer.commit();
-                    }
-                }
-
-            }; // class PBFPrimitiveBlockParser
-
-            /**
-             * PBF blobs can optionally be packed with the zlib algorithm.
-             * This function returns the raw data (if it was unpacked) or
-             * the unpacked data (if it was packed).
-             *
-             * @param input_data Reference to input data.
-             * @returns Unpacked data
-             * @throws osmium::pbf_error If there was a problem parsing the PBF
-             */
-            inline std::unique_ptr<const std::string> unpack_blob(const std::string& input_data) {
-                OSMPBF::Blob pbf_blob;
-                if (!pbf_blob.ParseFromString(input_data)) {
-                    throw osmium::pbf_error("failed to parse blob");
-                }
-
-                if (pbf_blob.has_raw()) {
-                    return std::unique_ptr<std::string>(pbf_blob.release_raw());
-                } else if (pbf_blob.has_zlib_data()) {
-                    auto raw_size = pbf_blob.raw_size();
-                    assert(raw_size >= 0);
-                    assert(raw_size <= OSMPBF::max_uncompressed_blob_size);
-                    return osmium::io::detail::zlib_uncompress(pbf_blob.zlib_data(), static_cast<unsigned long>(raw_size));
-                } else if (pbf_blob.has_lzma_data()) {
-                    throw osmium::pbf_error("lzma blobs not implemented");
-                } else {
-                    throw osmium::pbf_error("blob contains no data");
-                }
-            }
-
-            /**
-             * Parse blob as a HeaderBlock.
-             *
-             * @param input_buffer Blob data
-             * @returns Header object
-             * @throws osmium::pbf_error If there was a parsing error
-             */
-            inline osmium::io::Header parse_header_blob(const std::string& input_buffer) {
-                const std::unique_ptr<const std::string> data = unpack_blob(input_buffer);
-
-                OSMPBF::HeaderBlock pbf_header_block;
-                if (!pbf_header_block.ParseFromString(*data)) {
-                    throw osmium::pbf_error("failed to parse HeaderBlock");
-                }
-
-                osmium::io::Header header;
-                for (int i = 0; i < pbf_header_block.required_features_size(); ++i) {
-                    const std::string& feature = pbf_header_block.required_features(i);
-
-                    if (feature == "OsmSchema-V0.6") continue;
-                    if (feature == "DenseNodes") {
-                        header.set("pbf_dense_nodes", true);
-                        continue;
-                    }
-                    if (feature == "HistoricalInformation") {
-                        header.set_has_multiple_object_versions(true);
-                        continue;
-                    }
-
-                    throw osmium::pbf_error(std::string("required feature not supported: ") + feature);
-                }
-
-                for (int i = 0; i < pbf_header_block.optional_features_size(); ++i) {
-                    const std::string& feature = pbf_header_block.optional_features(i);
-                    header.set("pbf_optional_feature_" + std::to_string(i), feature);
-                }
-
-                if (pbf_header_block.has_writingprogram()) {
-                    header.set("generator", pbf_header_block.writingprogram());
-                }
-
-                if (pbf_header_block.has_bbox()) {
-                    const OSMPBF::HeaderBBox& pbf_bbox = pbf_header_block.bbox();
-                    const int64_t resolution_convert = OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision;
-                    osmium::Box box;
-                    box.extend(osmium::Location(pbf_bbox.left()  / resolution_convert, pbf_bbox.bottom() / resolution_convert));
-                    box.extend(osmium::Location(pbf_bbox.right() / resolution_convert, pbf_bbox.top()    / resolution_convert));
-                    header.add_box(box);
-                }
-
-                if (pbf_header_block.has_osmosis_replication_timestamp()) {
-                    header.set("osmosis_replication_timestamp", osmium::Timestamp(pbf_header_block.osmosis_replication_timestamp()).to_iso());
-                }
-
-                if (pbf_header_block.has_osmosis_replication_sequence_number()) {
-                    header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.osmosis_replication_sequence_number()));
-                }
-
-                if (pbf_header_block.has_osmosis_replication_base_url()) {
-                    header.set("osmosis_replication_base_url", pbf_header_block.osmosis_replication_base_url());
-                }
-
-                return header;
-            }
-
-            class DataBlobParser {
-
-                std::shared_ptr<std::string> m_input_buffer;
-                osmium::osm_entity_bits::type m_read_types;
-
-            public:
-
-                DataBlobParser(std::string&& input_buffer, osmium::osm_entity_bits::type read_types) :
-                    m_input_buffer(std::make_shared<std::string>(std::move(input_buffer))),
-                    m_read_types(read_types) {
-                    if (input_buffer.size() > OSMPBF::max_uncompressed_blob_size) {
-                        throw osmium::pbf_error(std::string("invalid blob size: " + std::to_string(input_buffer.size())));
-                    }
-                }
-/*
-                DataBlobParser(const DataBlobParser& other) :
-                    m_input_buffer(std::move(other.m_input_buffer)),
-                    m_read_types(other.m_read_types) {
-                }*/
-
-                DataBlobParser(const DataBlobParser&) = default;
-                DataBlobParser& operator=(const DataBlobParser&) = default;
-
-                DataBlobParser(DataBlobParser&&) = default;
-                DataBlobParser& operator=(DataBlobParser&&) = default;
-
-                ~DataBlobParser() = default;
-
-                osmium::memory::Buffer operator()() {
-                    const std::unique_ptr<const std::string> data = unpack_blob(*m_input_buffer);
-                    PBFPrimitiveBlockParser parser(*data, m_read_types);
-                    return parser();
-                }
-
-            }; // class DataBlobParser
-
-        } // namespace detail
-
-    } // namespace io
-
-} // namespace osmium
-
-#endif // OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP

third_party/libosmium/include/osmium/io/detail/pbf_stringtable.hpp

@@ -1,218 +0,0 @@
-#ifndef OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP
-#define OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP
-
-/*
-
-This file is part of Osmium (http://osmcode.org/libosmium).
-
-Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
-
-Boost Software License - Version 1.0 - August 17th, 2003
-
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-
-*/
-
-#include <algorithm>
-#include <cstdint>
-#include <iterator>
-#include <map>
-#include <string>
-#include <utility>
-#include <vector>
-
-#include <osmpbf/osmpbf.h>
-
-#include <osmium/util/cast.hpp>
-
-namespace osmium {
-
-    namespace io {
-
-        namespace detail {
-
-            /**
-             * StringTable management for PBF writer
-             *
-             * All strings are stored as indexes to rows in a StringTable. The StringTable contains
-             * one row for each used string, so strings that are used multiple times need to be
-             * stored only once. The StringTable is sorted by usage-count, so the most often used
-             * string is stored at index 1.
-             */
-            class StringTable {
-
-            public:
-
-                /// type for string IDs (interim and final)
-                typedef uint16_t string_id_type;
-
-            private:
-
-                /**
-                 * this is the struct used to build the StringTable. It is stored as
-                 * the value-part in the strings-map.
-                 *
-                 * when a new string is added to the map, its count is set to 0 and
-                 * the interim_id is set to the current size of the map. This interim_id
-                 * is then stored into the pbf-objects.
-                 *
-                 * before the PrimitiveBlock is serialized, the map is sorted by count
-                 * and stored into the pbf-StringTable. Afterwards the interim-ids are
-                 * mapped to the "real" id in the StringTable.
-                 *
-                 * this way often used strings get lower ids in the StringTable. As the
-                 * protobuf-serializer stores numbers in variable bit-lengths, lower
-                 * IDs means less used space in the resulting file.
-                 */
-                struct string_info {
-
-                    /// number of occurrences of this string
-                    uint16_t count;
-
-                    /// an intermediate-id
-                    string_id_type interim_id;
-
-                }; // struct string_info
-
-                /**
-                 * Interim StringTable, storing all strings that should be written to
-                 * the StringTable once the block is written to disk.
-                 */
-                typedef std::map<std::string, string_info> string2string_info_type;
-                string2string_info_type m_strings;
-
-                /**
-                 * This vector is used to map the interim IDs to real StringTable IDs after
-                 * writing all strings to the StringTable.
-                 */
-                typedef std::vector<string_id_type> interim_id2id_type;
-                interim_id2id_type m_id2id_map;
-
-                size_t m_size = 0;
-
-            public:
-
-                StringTable() {
-                }
-
-                friend bool operator<(const string_info& lhs, const string_info& rhs) {
-                    return lhs.count > rhs.count;
-                }
-
-                /**
-                 * record a string in the interim StringTable if it's missing, otherwise just increase its counter,
-                 * return the interim-id assigned to the string.
-                 */
-                string_id_type record_string(const std::string& string) {
-                    string_info& info = m_strings[string];
-                    if (info.interim_id == 0) {
-                        ++m_size;
-                        info.interim_id = static_cast_with_assert<string_id_type>(m_size);
-                    } else {
-                        info.count++;
-                    }
-                    return info.interim_id;
-                }
-
-                /**
-                 * Sort the interim StringTable and store it to the real protobuf StringTable.
-                 * while storing to the real table, this function fills the id2id_map with
-                 * pairs, mapping the interim-ids to final and real StringTable ids.
-                 *
-                 * Note that the m_strings table is a std::map and as such is sorted lexicographically.
-                 * When the transformation into the sortedby multimap is done, it gets sorted by
-                 * the count. The end result (at least with the glibc standard container/algorithm
-                 * implementation) is that the string table is sorted first by reverse count (ie descending)
-                 * and then by reverse lexicographic order.
-                 */
-                void store_stringtable(OSMPBF::StringTable* st, bool sort) {
-                    // add empty StringTable entry at index 0
-                    // StringTable index 0 is reserved as delimiter in the densenodes key/value list
-                    // this line also ensures that there's always a valid StringTable
-                    st->add_s("");
-
-                    if (sort) {
-                        std::multimap<string_info, std::string> sortedbycount;
-
-                        m_id2id_map.resize(m_size+1);
-
-                        std::transform(m_strings.begin(), m_strings.end(),
-                                    std::inserter(sortedbycount, sortedbycount.begin()),
-                                    [](const std::pair<std::string, string_info>& p) {
-                                            return std::pair<string_info, std::string>(p.second, p.first);
-                                    });
-
-                        string_id_type n = 0;
-
-                        for (const auto& mapping : sortedbycount) {
-                            // add the string of the current item to the pbf StringTable
-                            st->add_s(mapping.second);
-
-                            // store the mapping from the interim-id to the real id
-                            m_id2id_map[mapping.first.interim_id] = ++n;
-                        }
-                    } else {
-                        std::vector<std::pair<string_id_type, const char*>> sortedbyid;
-                        sortedbyid.reserve(m_strings.size());
-
-                        for (const auto& p : m_strings) {
-                            sortedbyid.emplace_back(p.second.interim_id, p.first.c_str());
-                        }
-
-                        std::sort(sortedbyid.begin(), sortedbyid.end());
-                        for (const auto& mapping : sortedbyid) {
-                            st->add_s(mapping.second);
-                        }
-                    }
-                }
-
-                /**
-                 * Map from an interim ID to a real string ID.
-                 */
-                string_id_type map_string_id(const string_id_type interim_id) const {
-                    return m_id2id_map[interim_id];
-                }
-
-                template <typename T>
-                string_id_type map_string_id(const T interim_id) const {
-                    return map_string_id(static_cast_with_assert<string_id_type>(interim_id));
-                }
-
-                /**
-                 * Clear the stringtable, preparing for the next block.
-                 */
-                void clear() {
-                    m_strings.clear();
-                    m_id2id_map.clear();
-                    m_size = 0;
-                }
-
-            }; // class StringTable
-
-        } // namespace detail
-
-    } // namespace io
-
-} // namespace osmium
-
-#endif // OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP

third_party/libosmium/include/osmium/io/detail/protobuf_tags.hpp

@@ -0,0 +1,170 @@
+#ifndef OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP
+#define OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <protozero/pbf_types.hpp>
+
+namespace osmium {
+
+    namespace io {
+
+        namespace detail {
+
+            // directly translated from
+            // https://github.com/scrosby/OSM-binary/blob/master/src/fileformat.proto
+
+            namespace FileFormat {
+
+                enum class Blob : protozero::pbf_tag_type {
+                    optional_bytes_raw       = 1,
+                    optional_int32_raw_size  = 2,
+                    optional_bytes_zlib_data = 3,
+                    optional_bytes_lzma_data = 4
+                };
+
+                enum class BlobHeader : protozero::pbf_tag_type {
+                    required_string_type     = 1,
+                    optional_bytes_indexdata = 2,
+                    required_int32_datasize  = 3
+                };
+
+            } // namespace FileFormat
+
+            // directly translated from
+            // https://github.com/scrosby/OSM-binary/blob/master/src/osmformat.proto
+
+            namespace OSMFormat {
+
+                enum class HeaderBlock : protozero::pbf_tag_type {
+                    optional_HeaderBBox_bbox          =  1,
+                    repeated_string_required_features =  4,
+                    repeated_string_optional_features =  5,
+                    optional_string_writingprogram    = 16,
+                    optional_string_source            = 17,
+                    optional_int64_osmosis_replication_timestamp       = 32,
+                    optional_int64_osmosis_replication_sequence_number = 33,
+                    optional_string_osmosis_replication_base_url       = 34
+                };
+
+                enum class HeaderBBox : protozero::pbf_tag_type {
+                    required_sint64_left   = 1,
+                    required_sint64_right  = 2,
+                    required_sint64_top    = 3,
+                    required_sint64_bottom = 4
+                };
+
+                enum class PrimitiveBlock : protozero::pbf_tag_type {
+                    required_StringTable_stringtable       =  1,
+                    repeated_PrimitiveGroup_primitivegroup =  2,
+                    optional_int32_granularity             = 17,
+                    optional_int32_date_granularity        = 18,
+                    optional_int64_lat_offset              = 19,
+                    optional_int64_lon_offset              = 20
+                };
+
+                enum class PrimitiveGroup : protozero::pbf_tag_type {
+                    unknown                       = 0,
+                    repeated_Node_nodes           = 1,
+                    optional_DenseNodes_dense     = 2,
+                    repeated_Way_ways             = 3,
+                    repeated_Relation_relations   = 4,
+                    repeated_ChangeSet_changesets = 5
+                };
+
+                enum class StringTable : protozero::pbf_tag_type {
+                    repeated_bytes_s = 1
+                };
+
+                enum class Info : protozero::pbf_tag_type {
+                    optional_int32_version   = 1,
+                    optional_int64_timestamp = 2,
+                    optional_int64_changeset = 3,
+                    optional_int32_uid       = 4,
+                    optional_uint32_user_sid = 5,
+                    optional_bool_visible    = 6
+                };
+
+                enum class DenseInfo : protozero::pbf_tag_type {
+                    packed_int32_version    = 1,
+                    packed_sint64_timestamp = 2,
+                    packed_sint64_changeset = 3,
+                    packed_sint32_uid       = 4,
+                    packed_sint32_user_sid  = 5,
+                    packed_bool_visible     = 6
+                };
+
+                enum class Node : protozero::pbf_tag_type {
+                    required_sint64_id  = 1,
+                    packed_uint32_keys  = 2,
+                    packed_uint32_vals  = 3,
+                    optional_Info_info  = 4,
+                    required_sint64_lat = 8,
+                    required_sint64_lon = 9
+                };
+
+                enum class DenseNodes : protozero::pbf_tag_type {
+                    packed_sint64_id             =  1,
+                    optional_DenseInfo_denseinfo =  5,
+                    packed_sint64_lat            =  8,
+                    packed_sint64_lon            =  9,
+                    packed_int32_keys_vals       = 10
+                };
+
+                enum class Way : protozero::pbf_tag_type {
+                    required_int64_id  = 1,
+                    packed_uint32_keys = 2,
+                    packed_uint32_vals = 3,
+                    optional_Info_info = 4,
+                    packed_sint64_refs = 8
+                };
+
+                enum class Relation : protozero::pbf_tag_type {
+                    required_int64_id       =  1,
+                    packed_uint32_keys      =  2,
+                    packed_uint32_vals      =  3,
+                    optional_Info_info      =  4,
+                    packed_int32_roles_sid  =  8,
+                    packed_sint64_memids    =  9,
+                    packed_MemberType_types = 10
+                };
+
+            } // namespace OSMFormat
+
+        } // namespace detail
+
+    } // namespace io
+
+} // namespace osmium
+
+#endif //  OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP

third_party/libosmium/include/osmium/io/detail/read_write.hpp

@@ -122,7 +122,7 @@ namespace osmium {
              * @throws std::system_error On error.
              */
             inline void reliable_write(const int fd, const unsigned char* output_buffer, const size_t size) {
-                constexpr size_t max_write = 100 * 1024 * 1024; // Max 100 MByte per write
+                constexpr size_t max_write = 100L * 1024L * 1024L; // Max 100 MByte per write
                 size_t offset = 0;
                 do {
                     auto write_count = size - offset;

third_party/libosmium/include/osmium/io/detail/string_table.hpp

@@ -0,0 +1,250 @@
+#ifndef OSMIUM_IO_DETAIL_STRING_TABLE_HPP
+#define OSMIUM_IO_DETAIL_STRING_TABLE_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cassert>
+#include <cstdlib>
+#include <cstring>
+#include <iterator>
+#include <list>
+#include <map>
+#include <string>
+
+namespace osmium {
+
+    namespace io {
+
+        namespace detail {
+
+            /**
+             * class StringStore
+             *
+             * Storage of lots of strings (const char *). Memory is allocated in chunks.
+             * If a string is added and there is no space in the current chunk, a new
+             * chunk will be allocated. Strings added to the store must not be larger
+             * than the chunk size.
+             *
+             * All memory is released when the destructor is called. There is no other way
+             * to release all or part of the memory.
+             *
+             */
+            class StringStore {
+
+                size_t m_chunk_size;
+
+                std::list<std::string> m_chunks;
+
+                void add_chunk() {
+                    m_chunks.push_front(std::string());
+                    m_chunks.front().reserve(m_chunk_size);
+                }
+
+            public:
+
+                StringStore(size_t chunk_size) :
+                    m_chunk_size(chunk_size),
+                    m_chunks() {
+                    add_chunk();
+                }
+
+                void clear() noexcept {
+                    m_chunks.erase(std::next(m_chunks.begin()), m_chunks.end());
+                    m_chunks.front().clear();
+                }
+
+                /**
+                 * Add a null terminated string to the store. This will
+                 * automatically get more memory if we are out.
+                 * Returns a pointer to the copy of the string we have
+                 * allocated.
+                 */
+                const char* add(const char* string) {
+                    size_t len = std::strlen(string) + 1;
+
+                    assert(len <= m_chunk_size);
+
+                    size_t chunk_len = m_chunks.front().size();
+                    if (chunk_len + len > m_chunks.front().capacity()) {
+                        add_chunk();
+                        chunk_len = 0;
+                    }
+
+                    m_chunks.front().append(string);
+                    m_chunks.front().append(1, '\0');
+
+                    return m_chunks.front().c_str() + chunk_len;
+                }
+
+                class const_iterator : public std::iterator<std::forward_iterator_tag, const char*> {
+
+                    typedef std::list<std::string>::const_iterator it_type;
+                    it_type m_it;
+                    const it_type m_last;
+                    const char* m_pos;
+
+                public:
+
+                    const_iterator(it_type it, it_type last) :
+                        m_it(it),
+                        m_last(last),
+                        m_pos(it == last ? nullptr : m_it->c_str()) {
+                    }
+
+                    const_iterator& operator++() {
+                        assert(m_it != m_last);
+                        auto last_pos = m_it->c_str() + m_it->size();
+                        while (m_pos != last_pos && *m_pos) ++m_pos;
+                        if (m_pos != last_pos) ++m_pos;
+                        if (m_pos == last_pos) {
+                            ++m_it;
+                            if (m_it != m_last) {
+                                m_pos = m_it->c_str();
+                            } else {
+                                m_pos = nullptr;
+                            }
+                        }
+                        return *this;
+                    }
+
+                    const_iterator operator++(int) {
+                        const_iterator tmp(*this);
+                        operator++();
+                        return tmp;
+                    }
+
+                    bool operator==(const const_iterator& rhs) const {
+                        return m_it == rhs.m_it && m_pos == rhs.m_pos;
+                    }
+
+                    bool operator!=(const const_iterator& rhs) const {
+                        return !(*this == rhs);
+                    }
+
+                    const char* operator*() const {
+                        assert(m_it != m_last);
+                        assert(m_pos != nullptr);
+                        return m_pos;
+                    }
+
+                }; // class const_iterator
+
+                const_iterator begin() const {
+                    if (m_chunks.front().empty()) {
+                        return end();
+                    }
+                    return const_iterator(m_chunks.begin(), m_chunks.end());
+                }
+
+                const_iterator end() const {
+                    return const_iterator(m_chunks.end(), m_chunks.end());
+                }
+
+                // These functions get you some idea how much memory was
+                // used.
+                int get_chunk_size() const noexcept {
+                    return m_chunk_size;
+                }
+
+                int get_chunk_count() const noexcept {
+                    return m_chunks.size();
+                }
+
+                int get_used_bytes_in_last_chunk() const noexcept {
+                    return m_chunks.front().size();
+                }
+
+            }; // class StringStore
+
+            struct StrComp {
+
+                bool operator()(const char* lhs, const char* rhs) const {
+                    return strcmp(lhs, rhs) < 0;
+                }
+
+            }; // struct StrComp
+
+            class StringTable {
+
+                StringStore m_strings;
+                std::map<const char*, size_t, StrComp> m_index;
+                size_t m_size;
+
+            public:
+
+                StringTable() :
+                    m_strings(1024 * 1024),
+                    m_index(),
+                    m_size(0) {
+                    m_strings.add("");
+                }
+
+                void clear() {
+                    m_strings.clear();
+                    m_index.clear();
+                    m_size = 0;
+                    m_strings.add("");
+                }
+
+                size_t size() const noexcept {
+                    return m_size + 1;
+                }
+
+                size_t add(const char* s) {
+                    auto f = m_index.find(s);
+                    if (f != m_index.end()) {
+                        return f->second;
+                    }
+
+                    const char* cs = m_strings.add(s);
+                    m_index[cs] = ++m_size;
+                    return m_size;
+                }
+
+                StringStore::const_iterator begin() const {
+                    return m_strings.begin();
+                }
+
+                StringStore::const_iterator end() const {
+                    return m_strings.end();
+                }
+
+            }; // class StringTable
+
+        } // namespace detail
+
+    } // namespace io
+
+} // namespace osmium
+
+#endif // OSMIUM_IO_DETAIL_STRING_TABLE_HPP

third_party/libosmium/include/osmium/io/detail/xml_input_format.hpp

@@ -66,6 +66,7 @@ DEALINGS IN THE SOFTWARE.
 #include <osmium/osm/location.hpp>
 #include <osmium/osm/object.hpp>
 #include <osmium/osm/types.hpp>
+#include <osmium/osm/types_from_string.hpp>
 #include <osmium/thread/queue.hpp>
 #include <osmium/thread/util.hpp>
 #include <osmium/util/cast.hpp>
@@ -191,6 +192,8 @@ namespace osmium {
 
                 std::atomic<bool>& m_done;
 
+                bool m_header_is_done;
+
                 /**
                  * A C++ wrapper for the Expat parser that makes sure no memory is leaked.
                  */
@@ -246,16 +249,25 @@ namespace osmium {
 
                     T& m_data;
                     std::promise<T>& m_promise;
+                    bool m_done;
 
                 public:
 
                     PromiseKeeper(T& data, std::promise<T>& promise) :
                         m_data(data),
-                        m_promise(promise) {
+                        m_promise(promise),
+                        m_done(false) {
+                    }
+
+                    void fullfill_promise() {
+                        if (!m_done) {
+                            m_promise.set_value(m_data);
+                            m_done = true;
+                        }
                     }
 
                     ~PromiseKeeper() {
-                        m_promise.set_value(m_data);
+                        fullfill_promise();
                     }
 
                 }; // class PromiseKeeper
@@ -279,7 +291,8 @@ namespace osmium {
                     m_queue(queue),
                     m_header_promise(header_promise),
                     m_read_types(read_types),
-                    m_done(done) {
+                    m_done(done),
+                    m_header_is_done(false) {
                 }
 
                 /**
@@ -305,7 +318,8 @@ namespace osmium {
                     m_queue(other.m_queue),
                     m_header_promise(other.m_header_promise),
                     m_read_types(other.m_read_types),
-                    m_done(other.m_done) {
+                    m_done(other.m_done),
+                    m_header_is_done(other.m_header_is_done) {
                 }
 
                 XMLParser(XMLParser&&) = default;
@@ -326,6 +340,9 @@ namespace osmium {
                         last = data.empty();
                         try {
                             parser(data, last);
+                            if (m_header_is_done) {
+                                promise_keeper.fullfill_promise();
+                            }
                         } catch (ParserIsDone&) {
                             return true;
                         } catch (...) {
@@ -343,8 +360,7 @@ namespace osmium {
             private:
 
                 const char* init_object(osmium::OSMObject& object, const XML_Char** attrs) {
-                    static const char* empty = "";
+                    const char* user = "";
-                    const char* user = empty;
 
                     if (m_in_delete_section) {
                         object.set_visible(false);
@@ -371,8 +387,7 @@ namespace osmium {
                 }
 
                 void init_changeset(osmium::builder::ChangesetBuilder* builder, const XML_Char** attrs) {
-                    static const char* empty = "";
+                    const char* user = "";
-                    const char* user = empty;
                     osmium::Changeset& new_changeset = builder->object();
 
                     osmium::Location min;
@@ -421,6 +436,7 @@ namespace osmium {
                 }
 
                 void header_is_done() {
+                    m_header_is_done = true;
                     if (m_read_types == osmium::osm_entity_bits::nothing) {
                         throw ParserIsDone();
                     }
@@ -722,10 +738,15 @@ namespace osmium {
 
             namespace {
 
+// we want the register_input_format() function to run, setting the variable
+// is only a side-effect, it will never be used
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
                 const bool registered_xml_input = osmium::io::detail::InputFormatFactory::instance().register_input_format(osmium::io::file_format::xml,
                     [](const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue<std::string>& input_queue) {
                         return new osmium::io::detail::XMLInputFormat(file, read_which_entities, input_queue);
                 });
+#pragma GCC diagnostic pop
 
             } // anonymous namespace
 

third_party/libosmium/include/osmium/io/detail/xml_output_format.hpp

@@ -85,6 +85,9 @@ namespace osmium {
                             case '\'': out += "'"; break;
                             case '<':  out += "&lt;";   break;
                             case '>':  out += "&gt;";   break;
+                            case '\n': out += "&#xA;";  break;
+                            case '\r': out += "&#xD;";  break;
+                            case '\t': out += "&#x9;";  break;
                             default:   out += *in;      break;
                         }
                     }
@@ -126,6 +129,7 @@ namespace osmium {
 
                 operation m_last_op {operation::op_none};
 
+                const bool m_add_metadata;
                 const bool m_write_visible_flag;
                 const bool m_write_change_ops;
 
@@ -146,31 +150,33 @@ namespace osmium {
                 void write_meta(const osmium::OSMObject& object) {
                     oprintf(*m_out, " id=\"%" PRId64 "\"", object.id());
 
-                    if (object.version()) {
+                    if (m_add_metadata) {
-                        oprintf(*m_out, " version=\"%d\"", object.version());
+                        if (object.version()) {
-                    }
+                            oprintf(*m_out, " version=\"%d\"", object.version());
+                        }
 
-                    if (object.timestamp()) {
+                        if (object.timestamp()) {
-                        *m_out += " timestamp=\"";
+                            *m_out += " timestamp=\"";
-                        *m_out += object.timestamp().to_iso();
+                            *m_out += object.timestamp().to_iso();
-                        *m_out += "\"";
+                            *m_out += "\"";
-                    }
+                        }
 
-                    if (!object.user_is_anonymous()) {
+                        if (!object.user_is_anonymous()) {
-                        oprintf(*m_out, " uid=\"%d\" user=\"", object.uid());
+                            oprintf(*m_out, " uid=\"%d\" user=\"", object.uid());
-                        xml_string(*m_out, object.user());
+                            xml_string(*m_out, object.user());
-                        *m_out += "\"";
+                            *m_out += "\"";
-                    }
+                        }
 
-                    if (object.changeset()) {
+                        if (object.changeset()) {
-                        oprintf(*m_out, " changeset=\"%d\"", object.changeset());
+                            oprintf(*m_out, " changeset=\"%d\"", object.changeset());
-                    }
+                        }
 
-                    if (m_write_visible_flag) {
+                        if (m_write_visible_flag) {
-                        if (object.visible()) {
+                            if (object.visible()) {
-                            *m_out += " visible=\"true\"";
+                                *m_out += " visible=\"true\"";
-                        } else {
+                            } else {
-                            *m_out += " visible=\"false\"";
+                                *m_out += " visible=\"false\"";
+                            }
                         }
                     }
                 }
@@ -224,9 +230,10 @@ namespace osmium {
 
             public:
 
-                explicit XMLOutputBlock(osmium::memory::Buffer&& buffer, bool write_visible_flag, bool write_change_ops) :
+                explicit XMLOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata, bool write_visible_flag, bool write_change_ops) :
                     m_input_buffer(std::make_shared<osmium::memory::Buffer>(std::move(buffer))),
                     m_out(std::make_shared<std::string>()),
+                    m_add_metadata(add_metadata),
                     m_write_visible_flag(write_visible_flag && !write_change_ops),
                     m_write_change_ops(write_change_ops) {
                 }
@@ -392,12 +399,14 @@ namespace osmium {
 
             class XMLOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler {
 
+                bool m_add_metadata;
                 bool m_write_visible_flag;
 
             public:
 
                 XMLOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) :
                     OutputFormat(file, output_queue),
+                    m_add_metadata(file.get("add_metadata") != "false"),
                     m_write_visible_flag(file.has_multiple_object_versions() || m_file.is_true("force_visible_flag")) {
                 }
 
@@ -408,7 +417,7 @@ namespace osmium {
                 }
 
                 void write_buffer(osmium::memory::Buffer&& buffer) override final {
-                    m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_write_visible_flag, m_file.is_true("xml_change_format")}));
+                    m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_add_metadata, m_write_visible_flag, m_file.is_true("xml_change_format")}));
                 }
 
                 void write_header(const osmium::io::Header& header) override final {
@@ -468,10 +477,15 @@ namespace osmium {
 
             namespace {
 
+// we want the register_output_format() function to run, setting the variable
+// is only a side-effect, it will never be used
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
                 const bool registered_xml_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::xml,
                     [](const osmium::io::File& file, data_queue_type& output_queue) {
                         return new osmium::io::detail::XMLOutputFormat(file, output_queue);
                 });
+#pragma GCC diagnostic pop
 
             } // anonymous namespace
 

third_party/libosmium/include/osmium/io/detail/zlib.hpp

@@ -85,23 +85,24 @@ namespace osmium {
              *
              * @param input Compressed input data.
              * @param raw_size Size of uncompressed data.
-             * @returns Uncompressed data.
+             * @param output Uncompressed result data.
+             * @returns Pointer and size to incompressed data.
              */
-            inline std::unique_ptr<std::string> zlib_uncompress(const std::string& input, unsigned long raw_size) {
+            inline std::pair<const char*, size_t> zlib_uncompress_string(const char* input, unsigned long input_size, unsigned long raw_size, std::string& output) {
-                auto output = std::unique_ptr<std::string>(new std::string(raw_size, '\0'));
+                output.resize(raw_size);
 
                 auto result = ::uncompress(
-                    reinterpret_cast<unsigned char*>(const_cast<char *>(output->data())),
+                    reinterpret_cast<unsigned char*>(&*output.begin()),
                     &raw_size,
-                    reinterpret_cast<const unsigned char*>(input.data()),
+                    reinterpret_cast<const unsigned char*>(input),
-                    osmium::static_cast_with_assert<unsigned long>(input.size())
+                    input_size
                 );
 
                 if (result != Z_OK) {
                     throw std::runtime_error(std::string("failed to uncompress data: ") + zError(result));
                 }
 
-                return output;
+                return std::make_pair(output.data(), output.size());
             }
 
         } // namespace detail

third_party/libosmium/include/osmium/io/file.hpp

@@ -97,7 +97,9 @@ namespace osmium {
              *                 of the file will be taken from the suffix.
              *                 An empty filename or "-" means stdin or stdout.
              * @param format File format as string. See the description of the
-             *               parse_format() function for details.
+             *               parse_format() function for details. If this is
+             *               empty the format will be deduced from the suffix
+             *               of the filename.
              */
             explicit File(const std::string& filename = "", const std::string& format = "") :
                 Options(),
@@ -107,20 +109,19 @@ namespace osmium {
                 m_format_string(format) {
 
                 // stdin/stdout
-                if (filename == "" || filename == "-") {
+                if (m_filename == "-") {
                     m_filename = "";
-                    default_settings_for_stdinout();
                 }
 
-                // filename is actually a URL
+                // if filename is a URL, default to XML format
                 std::string protocol = m_filename.substr(0, m_filename.find_first_of(':'));
                 if (protocol == "http" || protocol == "https") {
-                    default_settings_for_url();
+                    m_file_format = file_format::xml;
                 }
 
-                detect_format_from_suffix(m_filename);
+                if (format.empty()) {
-
+                    detect_format_from_suffix(m_filename);
-                if (format != "") {
+                } else {
                     parse_format(format);
                 }
             }
@@ -140,9 +141,6 @@ namespace osmium {
                 m_buffer(buffer),
                 m_buffer_size(size),
                 m_format_string(format) {
-
-                default_settings_for_stdinout();
-
                 if (format != "") {
                     parse_format(format);
                 }
@@ -220,6 +218,20 @@ namespace osmium {
                 } else if (suffixes.back() == "opl") {
                     m_file_format = file_format::opl;
                     suffixes.pop_back();
+                } else if (suffixes.back() == "json") {
+                    m_file_format = file_format::json;
+                    suffixes.pop_back();
+                } else if (suffixes.back() == "o5m") {
+                    m_file_format = file_format::o5m;
+                    suffixes.pop_back();
+                } else if (suffixes.back() == "o5c") {
+                    m_file_format = file_format::o5m;
+                    m_has_multiple_object_versions = true;
+                    set("o5c_change_format", true);
+                    suffixes.pop_back();
+                } else if (suffixes.back() == "debug") {
+                    m_file_format = file_format::debug;
+                    suffixes.pop_back();
                 }
 
                 if (suffixes.empty()) return;
@@ -240,8 +252,8 @@ namespace osmium {
             }
 
             /**
-             * Check file format etc. for consistency and throw exception if there
+             * Check file format etc. for consistency and throw exception if
-             * is a problem.
+             * there is a problem.
              *
              * @throws std::runtime_error
              */
@@ -265,36 +277,6 @@ namespace osmium {
                 }
             }
 
-            /**
-             * Set default settings for type and encoding when the filename is
-             * empty or "-". If you want to have a different default setting
-             * override this in a subclass.
-             */
-            void default_settings_for_stdinout() {
-                m_file_format      = file_format::unknown;
-                m_file_compression = file_compression::none;
-            }
-
-            /**
-             * Set default settings for type and encoding when the filename is
-             * a normal file. If you want to have a different default setting
-             * override this in a subclass.
-             */
-            void default_settings_for_file() {
-                m_file_format      = file_format::unknown;
-                m_file_compression = file_compression::none;
-            }
-
-            /**
-             * Set default settings for type and encoding when the filename is a URL.
-             * If you want to have a different default setting override this in a
-             * subclass.
-             */
-            void default_settings_for_url() {
-                m_file_format      = file_format::xml;
-                m_file_compression = file_compression::none;
-            }
-
             file_format format() const noexcept {
                 return m_file_format;
             }

third_party/libosmium/include/osmium/io/file_format.hpp

@@ -44,7 +44,9 @@ namespace osmium {
             xml     = 1,
             pbf     = 2,
             opl     = 3,
-            json    = 4
+            json    = 4,
+            o5m     = 5,
+            debug   = 6
         };
 
 // avoid g++ false positive
@@ -62,6 +64,10 @@ namespace osmium {
                     return "OPL";
                 case file_format::json:
                     return "JSON";
+                case file_format::o5m:
+                    return "O5M";
+                case file_format::debug:
+                    return "DEBUG";
             }
         }
 #pragma GCC diagnostic pop

third_party/libosmium/include/osmium/io/gzip_compression.hpp

@@ -231,11 +231,16 @@ namespace osmium {
 
         namespace {
 
+// we want the register_compression() function to run, setting the variable
+// is only a side-effect, it will never be used
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
             const bool registered_gzip_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::gzip,
                 [](int fd) { return new osmium::io::GzipCompressor(fd); },
                 [](int fd) { return new osmium::io::GzipDecompressor(fd); },
                 [](const char* buffer, size_t size) { return new osmium::io::GzipBufferDecompressor(buffer, size); }
             );
+#pragma GCC diagnostic pop
 
         } // anonymous namespace
 

third_party/libosmium/include/osmium/io/pbf_input.hpp

@@ -39,7 +39,6 @@ DEALINGS IN THE SOFTWARE.
  * Include this file if you want to read OSM PBF files.
  *
  * @attention If you include this file, you'll need to link with
- *            `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only),
  *            `libz`, and enable multithreading.
  */
 

third_party/libosmium/include/osmium/io/pbf_output.hpp

@@ -39,7 +39,6 @@ DEALINGS IN THE SOFTWARE.
  * Include this file if you want to write OSM PBF files.
  *
  * @attention If you include this file, you'll need to link with
- *            `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only),
  *            `libz`, and enable multithreading.
  */
 

third_party/libosmium/include/osmium/memory/buffer.hpp

@@ -37,7 +37,6 @@ DEALINGS IN THE SOFTWARE.
 #include <cassert>
 #include <cstddef>
 #include <cstring>
-#include <exception>
 #include <functional>
 #include <iterator>
 #include <stdexcept>
@@ -83,7 +82,7 @@ namespace osmium {
          * Buffers exist in two flavours, those with external memory management and
          * those with internal memory management. If you already have some memory
          * with data in it (for instance read from disk), you create a Buffer with
-         * external memory managment. It is your job then to free the memory once
+         * external memory management. It is your job then to free the memory once
          * the buffer isn't used any more. If you don't have memory already, you can
          * create a Buffer object and have it manage the memory internally. It will
          * dynamically allocate memory and free it again after use.
@@ -413,6 +412,15 @@ namespace osmium {
                 return iterator(m_data, m_data + m_committed);
             }
 
+            template <class T>
+            t_iterator<T> get_iterator(size_t offset) {
+                return t_iterator<T>(m_data + offset, m_data + m_committed);
+            }
+
+            iterator get_iterator(size_t offset) {
+                return iterator(m_data + offset, m_data + m_committed);
+            }
+
             template <class T>
             t_iterator<T> end() {
                 return t_iterator<T>(m_data + m_committed, m_data + m_committed);
@@ -431,6 +439,15 @@ namespace osmium {
                 return const_iterator(m_data, m_data + m_committed);
             }
 
+            template <class T>
+            t_const_iterator<T> get_iterator(size_t offset) const {
+                return t_const_iterator<T>(m_data + offset, m_data + m_committed);
+            }
+
+            const_iterator get_iterator(size_t offset) const {
+                return const_iterator(m_data + offset, m_data + m_committed);
+            }
+
             template <class T>
             t_const_iterator<T> cend() const {
                 return t_const_iterator<T>(m_data + m_committed, m_data + m_committed);

third_party/libosmium/include/osmium/memory/collection.hpp

@@ -38,7 +38,6 @@ DEALINGS IN THE SOFTWARE.
 #include <type_traits>
 
 #include <osmium/memory/item.hpp>
-#include <osmium/util/compatibility.hpp>
 
 namespace osmium {
 

third_party/libosmium/include/osmium/memory/item.hpp

@@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE.
 
 */
 
-#include <cstddef>
 #include <cstdint>
 #include <type_traits>
 

third_party/libosmium/include/osmium/osm/changeset.hpp

@@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE.
 
 */
 
-#include <cstdint>
 #include <cstring>
 
 #include <osmium/memory/collection.hpp>
@@ -44,6 +43,7 @@ DEALINGS IN THE SOFTWARE.
 #include <osmium/osm/tag.hpp>
 #include <osmium/osm/timestamp.hpp>
 #include <osmium/osm/types.hpp>
+#include <osmium/osm/types_from_string.hpp>
 
 namespace osmium {
 

third_party/libosmium/include/osmium/osm/crc.hpp

@@ -0,0 +1,223 @@
+#ifndef OSMIUM_OSM_CRC_HPP
+#define OSMIUM_OSM_CRC_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstdint>
+
+#include <osmium/osm/area.hpp>
+#include <osmium/osm/changeset.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/node.hpp>
+#include <osmium/osm/node_ref_list.hpp>
+#include <osmium/osm/relation.hpp>
+#include <osmium/osm/way.hpp>
+#include <osmium/util/endian.hpp>
+
+namespace osmium {
+
+    template <class TCRC>
+    class CRC {
+
+        static inline uint16_t byte_swap_16(uint16_t value) noexcept {
+# if defined(__GNUC__) || defined(__clang__)
+            return __builtin_bswap16(value);
+# else
+            return (value >> 8) | (value << 8);
+# endif
+        }
+
+        static inline uint32_t byte_swap_32(uint32_t value) noexcept {
+# if defined(__GNUC__) || defined(__clang__)
+            return __builtin_bswap32(value);
+# else
+            return  (value >> 24) |
+                    ((value >>  8) & 0x0000FF00) |
+                    ((value <<  8) & 0x00FF0000) |
+                    (value << 24);
+# endif
+        }
+
+        static inline uint64_t byte_swap_64(uint64_t value) noexcept {
+# if defined(__GNUC__) || defined(__clang__)
+            return __builtin_bswap64(value);
+# else
+            uint64_t val1 = byte_swap_32(value & 0xFFFFFFFF);
+            uint64_t val2 = byte_swap_32(value >> 32);
+            return (val1 << 32) & val2;
+# endif
+        }
+
+        TCRC m_crc;
+
+    public:
+
+        TCRC& operator()() {
+            return m_crc;
+        }
+
+        const TCRC& operator()() const {
+            return m_crc;
+        }
+
+        void update_bool(bool value) {
+            m_crc.process_byte(value);
+        }
+
+        void update_int8(uint8_t value) {
+            m_crc.process_byte(value);
+        }
+
+        void update_int16(uint16_t value) {
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+            m_crc.process_bytes(&value, sizeof(uint16_t));
+#else
+            uint16_t v = byte_swap_16(value);
+            m_crc.process_bytes(&v, sizeof(uint16_t));
+#endif
+        }
+
+        void update_int32(uint32_t value) {
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+            m_crc.process_bytes(&value, sizeof(uint32_t));
+#else
+            uint32_t v = byte_swap_32(value);
+            m_crc.process_bytes(&v, sizeof(uint32_t));
+#endif
+        }
+
+        void update_int64(uint64_t value) {
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+            m_crc.process_bytes(&value, sizeof(uint64_t));
+#else
+            uint64_t v = byte_swap_64(value);
+            m_crc.process_bytes(&v, sizeof(uint64_t));
+#endif
+        }
+
+        void update_string(const char* str) {
+            while (*str) {
+                m_crc.process_byte(*str++);
+            }
+        }
+
+        void update(const Timestamp& timestamp) {
+            update_int32(uint32_t(timestamp));
+        }
+
+        void update(const osmium::Location& location) {
+            update_int32(location.x());
+            update_int32(location.y());
+        }
+
+        void update(const osmium::Box& box) {
+            update(box.bottom_left());
+            update(box.top_right());
+        }
+
+        void update(const NodeRef& node_ref) {
+            update_int64(node_ref.ref());
+        }
+
+        void update(const NodeRefList& node_refs) {
+            for (const NodeRef& node_ref : node_refs) {
+                update(node_ref);
+            }
+        }
+
+        void update(const TagList& tags) {
+            m_crc.process_bytes(tags.data(), tags.byte_size());
+        }
+
+        void update(const osmium::RelationMember& member) {
+            update_int64(member.ref());
+            update_int16(uint16_t(member.type()));
+            update_string(member.role());
+        }
+
+        void update(const osmium::RelationMemberList& members) {
+            for (const RelationMember& member : members) {
+                update(member);
+            }
+        }
+
+        void update(const osmium::OSMObject& object) {
+            update_int64(object.id());
+            update_bool(object.visible());
+            update_int32(object.version());
+            update(object.timestamp());
+            update_int32(object.uid());
+            update_string(object.user());
+            update(object.tags());
+        }
+
+        void update(const osmium::Node& node) {
+            update(static_cast<const osmium::OSMObject&>(node));
+            update(node.location());
+        }
+
+        void update(const osmium::Way& way) {
+            update(static_cast<const osmium::OSMObject&>(way));
+            update(way.nodes());
+        }
+
+        void update(const osmium::Relation& relation) {
+            update(static_cast<const osmium::OSMObject&>(relation));
+            update(relation.members());
+        }
+
+        void update(const osmium::Area& area) {
+            update(static_cast<const osmium::OSMObject&>(area));
+            for (auto it = area.cbegin(); it != area.cend(); ++it) {
+                if (it->type() == osmium::item_type::outer_ring ||
+                    it->type() == osmium::item_type::inner_ring) {
+                    update(static_cast<const osmium::NodeRefList&>(*it));
+                }
+            }
+        }
+
+        void update(const osmium::Changeset& changeset) {
+            update_int64(changeset.id());
+            update(changeset.created_at());
+            update(changeset.closed_at());
+            update(changeset.bounds());
+            update_int32(changeset.num_changes());
+            update_int32(changeset.uid());
+            update_string(changeset.user());
+        }
+
+    }; // class CRC
+
+} // namespace osmium
+
+#endif // OSMIUM_OSM_CRC

third_party/libosmium/include/osmium/osm/diff_object.hpp

@@ -112,8 +112,35 @@ namespace osmium {
             return m_curr->timestamp();
         }
 
+        /**
+         * Return the timestamp when the current version of the object is
+         * not valid any more, ie the time when the next version of the object
+         * is valid. If this is the last version of the object, this will
+         * return a special "end of time" timestamp that is guaranteed to
+         * be larger than any normal timestamp.
+         */
         const osmium::Timestamp end_time() const noexcept {
-            return last() ? osmium::Timestamp() : m_next->timestamp();
+            return last() ? osmium::end_of_time() : m_next->timestamp();
+        }
+
+        /**
+         * Current object version is valid between time "from" (inclusive) and
+         * time "to" (not inclusive).
+         *
+         * This is a bit more complex than you'd think, because we have to
+         * handle the case properly where the start_time() == end_time().
+         */
+        bool is_between(const osmium::Timestamp& from, const osmium::Timestamp& to) const noexcept {
+            return start_time() < to &&
+                   ((start_time() != end_time() && end_time() >  from) ||
+                    (start_time() == end_time() && end_time() >= from));
+        }
+
+        /**
+         * Current object version is visible at the given timestamp.
+         */
+        bool is_visible_at(const osmium::Timestamp& timestamp) const noexcept {
+            return start_time() <= timestamp && end_time() > timestamp && m_curr->visible();
         }
 
     }; // class DiffObject

third_party/libosmium/include/osmium/osm/entity.hpp

@@ -35,6 +35,7 @@ DEALINGS IN THE SOFTWARE.
 
 #include <osmium/memory/item.hpp>
 #include <osmium/osm/entity_bits.hpp>
+#include <osmium/osm/item_type.hpp>
 
 namespace osmium {
 

third_party/libosmium/include/osmium/osm/item_type.hpp

@@ -33,6 +33,7 @@ DEALINGS IN THE SOFTWARE.
 
 */
 
+#include <cassert>
 #include <cstdint> // IWYU pragma: keep
 #include <iosfwd>
 #include <stdexcept>
@@ -56,6 +57,25 @@ namespace osmium {
 
     }; // enum class item_type
 
+    /**
+     * Return item_type for index:
+     * 0 -> node, 1 -> way, 2 -> relation
+     */
+    inline item_type nwr_index_to_item_type(unsigned int i) noexcept {
+        assert(i <= 2);
+        return item_type(i+1);
+    }
+
+    /**
+     * Return index for item_type:
+     * node -> 0, way -> 1, relation -> 2
+     */
+    inline unsigned int item_type_to_nwr_index(item_type type) noexcept {
+        unsigned int i = static_cast<unsigned int>(type);
+        assert(i >= 1 && i <= 3);
+        return i - 1;
+    }
+
     inline item_type char_to_item_type(const char c) noexcept {
         switch (c) {
             case 'X':

third_party/libosmium/include/osmium/osm/node_ref.hpp

@@ -33,11 +33,11 @@ DEALINGS IN THE SOFTWARE.
 
 */
 
+#include <cstdint>
 #include <cstdlib>
 #include <iosfwd>
 
 #include <osmium/memory/item.hpp>
-#include <osmium/osm/item_type.hpp>
 #include <osmium/osm/location.hpp>
 #include <osmium/osm/types.hpp>
 

third_party/libosmium/include/osmium/osm/object.hpp

@@ -48,6 +48,7 @@ DEALINGS IN THE SOFTWARE.
 #include <osmium/osm/tag.hpp>
 #include <osmium/osm/timestamp.hpp>
 #include <osmium/osm/types.hpp>
+#include <osmium/osm/types_from_string.hpp>
 
 namespace osmium {
 

third_party/libosmium/include/osmium/osm/relation.hpp

@@ -120,6 +120,11 @@ namespace osmium {
             return static_cast<unsigned_object_id_type>(std::abs(m_ref));
         }
 
+        RelationMember& set_ref(const osmium::object_id_type ref) noexcept {
+            m_ref = ref;
+            return *this;
+        }
+
         item_type type() const noexcept {
             return m_type;
         }

third_party/libosmium/include/osmium/osm/timestamp.hpp

@@ -39,9 +39,9 @@ DEALINGS IN THE SOFTWARE.
 #include <limits>
 #include <stdexcept>
 #include <string>
-#include <time.h>
 
 #include <osmium/util/compatibility.hpp>
+#include <osmium/util/minmax.hpp> // IWYU pragma: keep
 
 namespace osmium {
 
@@ -170,6 +170,16 @@ namespace osmium {
         return out;
     }
 
+    template <>
+    inline osmium::Timestamp min_op_start_value<osmium::Timestamp>() {
+        return end_of_time();
+    }
+
+    template <>
+    inline osmium::Timestamp max_op_start_value<osmium::Timestamp>() {
+        return start_of_time();
+    }
+
 } // namespace osmium
 
 #endif // OSMIUM_OSM_TIMESTAMP_HPP

third_party/libosmium/include/osmium/osm/types.hpp

@@ -34,7 +34,6 @@ DEALINGS IN THE SOFTWARE.
 */
 
 #include <cstdint>
-#include <cstdlib>
 
 namespace osmium {
 
@@ -58,26 +57,6 @@ namespace osmium {
      */
     typedef uint16_t string_size_type;
 
-    inline object_id_type string_to_object_id(const char* string) {
-        return std::atoll(string);
-    }
-
-    inline object_version_type string_to_object_version(const char* string) {
-        return static_cast<object_version_type>(std::atol(string));
-    }
-
-    inline changeset_id_type string_to_changeset_id(const char* string) {
-        return static_cast<changeset_id_type>(std::atol(string));
-    }
-
-    inline signed_user_id_type string_to_user_id(const char* string) {
-        return static_cast<signed_user_id_type>(std::atol(string));
-    }
-
-    inline num_changes_type string_to_num_changes(const char* string) {
-        return static_cast<num_changes_type>(std::atol(string));
-    }
-
 } // namespace osmium
 
 #endif // OSMIUM_OSM_TYPES_HPP

third_party/libosmium/include/osmium/osm/types_from_string.hpp

@@ -0,0 +1,116 @@
+#ifndef OSMIUM_OSM_TYPES_FROM_STRING_HPP
+#define OSMIUM_OSM_TYPES_FROM_STRING_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cassert>
+#include <cctype>
+#include <cstdint>
+#include <cstdlib>
+#include <limits>
+#include <string>
+#include <utility>
+
+#include <osmium/osm/entity_bits.hpp>
+#include <osmium/osm/types.hpp>
+
+namespace osmium {
+
+    inline object_id_type string_to_object_id(const char* input) {
+        assert(input);
+        if (*input != '\0' && !std::isspace(*input)) {
+            char* end;
+            auto id = std::strtoll(input, &end, 10);
+            if (id != std::numeric_limits<long long>::min() && id != std::numeric_limits<long long>::max() && *end == '\0') {
+                return id;
+            }
+        }
+        throw std::range_error(std::string("illegal id: '") + input + "'");
+    }
+
+    inline std::pair<osmium::item_type, osmium::object_id_type> string_to_object_id(const char* input, osmium::osm_entity_bits::type types) {
+        assert(input);
+        assert(types != osmium::osm_entity_bits::nothing);
+        if (*input != '\0') {
+            if (std::isdigit(*input)) {
+                return std::make_pair(osmium::item_type::undefined, string_to_object_id(input));
+            }
+            osmium::item_type t = osmium::char_to_item_type(*input);
+            if (osmium::osm_entity_bits::from_item_type(t) & types) {
+                return std::make_pair(t, string_to_object_id(input+1));
+            }
+        }
+        throw std::range_error(std::string("not a valid id: '") + input + "'");
+    }
+
+    namespace detail {
+
+        inline long string_to_ulong(const char* input, const char *name) {
+            if (*input != '\0' && *input != '-' && !std::isspace(*input)) {
+                char* end;
+                auto value = std::strtoul(input, &end, 10);
+                if (value != std::numeric_limits<unsigned long>::max() && *end == '\0') {
+                    return value;
+                }
+            }
+            throw std::range_error(std::string("illegal ") + name + ": '" + input + "'");
+        }
+
+    } // namespace detail
+
+    inline object_version_type string_to_object_version(const char* input) {
+        assert(input);
+        return static_cast<object_version_type>(detail::string_to_ulong(input, "version"));
+    }
+
+    inline changeset_id_type string_to_changeset_id(const char* input) {
+        assert(input);
+        return static_cast<changeset_id_type>(detail::string_to_ulong(input, "changeset"));
+    }
+
+    inline signed_user_id_type string_to_user_id(const char* input) {
+        assert(input);
+        if (input[0] == '-' && input[1] == '1' && input[2] == '\0') {
+            return -1;
+        }
+        return static_cast<signed_user_id_type>(detail::string_to_ulong(input, "user id"));
+    }
+
+    inline num_changes_type string_to_num_changes(const char* input) {
+        assert(input);
+        return static_cast<num_changes_type>(detail::string_to_ulong(input, "value for num changes"));
+    }
+
+} // namespace osmium
+
+#endif // OSMIUM_OSM_TYPES_FROM_STRING_HPP

third_party/libosmium/include/osmium/relations/collector.hpp

@@ -512,7 +512,7 @@ namespace osmium {
                     double percent = static_cast<double>(size_before - size_after);
                     percent /= size_before;
                     percent *= 100;
-                    std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
+//                    std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
                     m_count_complete = 0;
                 }
             }

third_party/libosmium/include/osmium/thread/pool.hpp

@@ -149,6 +149,7 @@ namespace osmium {
 
             ~Pool() {
                 m_done = true;
+                m_work_queue.shutdown();
             }
 
             size_t queue_size() const {

third_party/libosmium/include/osmium/thread/queue.hpp

@@ -41,9 +41,7 @@ DEALINGS IN THE SOFTWARE.
 #include <queue>
 #include <string>
 #include <thread>
-#include <utility>
+#include <utility> // IWYU pragma: keep (for std::move)
-
-#include <osmium/util/compatibility.hpp>
 
 namespace osmium {
 
@@ -71,6 +69,8 @@ namespace osmium {
             /// Used to signal readers when data is available in the queue.
             std::condition_variable m_data_available;
 
+            std::atomic<bool> m_done;
+
 #ifdef OSMIUM_DEBUG_QUEUE_SIZE
             /// The largest size the queue has been so far.
             size_t m_largest_size;
@@ -94,7 +94,8 @@ namespace osmium {
                 m_name(name),
                 m_mutex(),
                 m_queue(),
-                m_data_available()
+                m_data_available(),
+                m_done(false)
 #ifdef OSMIUM_DEBUG_QUEUE_SIZE
                 ,
                 m_largest_size(0),
@@ -104,6 +105,7 @@ namespace osmium {
             }
 
             ~Queue() {
+                shutdown();
 #ifdef OSMIUM_DEBUG_QUEUE_SIZE
                 std::cerr << "queue '" << m_name << "' with max_size=" << m_max_size << " had largest size " << m_largest_size << " and was full " << m_full_counter << " times\n";
 #endif
@@ -132,24 +134,33 @@ namespace osmium {
                 m_data_available.notify_one();
             }
 
+            void shutdown() {
+                m_done = true;
+                m_data_available.notify_all();
+            }
+
             void wait_and_pop(T& value) {
                 std::unique_lock<std::mutex> lock(m_mutex);
                 m_data_available.wait(lock, [this] {
-                    return !m_queue.empty();
+                    return !m_queue.empty() || m_done;
                 });
-                value = std::move(m_queue.front());
+                if (!m_queue.empty()) {
-                m_queue.pop();
+                    value = std::move(m_queue.front());
+                    m_queue.pop();
+                }
             }
 
             void wait_and_pop_with_timeout(T& value) {
                 std::unique_lock<std::mutex> lock(m_mutex);
                 if (!m_data_available.wait_for(lock, std::chrono::seconds(1), [this] {
-                    return !m_queue.empty();
+                    return !m_queue.empty() || m_done;
                 })) {
                     return;
                 }
-                value = std::move(m_queue.front());
+                if (!m_queue.empty()) {
-                m_queue.pop();
+                    value = std::move(m_queue.front());
+                    m_queue.pop();
+                }
             }
 
             bool try_pop(T& value) {

third_party/libosmium/include/osmium/thread/util.hpp

@@ -58,7 +58,7 @@ namespace osmium {
 
         /**
          * Wait until the given future becomes ready. Will block if the future
-         * is not ready. Can be called more than once unless future.get().
+         * is not ready. Can be called more than once unlike future.get().
          */
         template <class T>
         inline void wait_until_done(std::future<T>& future) {

third_party/libosmium/include/osmium/util/data_file.hpp

@@ -0,0 +1,194 @@
+#ifndef OSMIUM_UTIL_DATA_FILE_HPP
+#define OSMIUM_UTIL_DATA_FILE_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cerrno>
+#include <cstddef>
+#include <cstdio>
+#include <stdexcept>
+#include <string>
+#include <system_error>
+
+#ifdef _WIN32
+# include <io.h>
+# include <windows.h>
+#endif
+
+#include <osmium/util/file.hpp>
+
+namespace osmium {
+
+    namespace util {
+
+        /**
+         * Class wrapper for convenient access to some low-level file
+         * functions.
+         */
+        class DataFile {
+
+            FILE* m_file;
+
+        public:
+
+            /**
+             * Create and open a temporary file. It is removed after opening.
+             *
+             * @throws std::system_error if something went wrong.
+             */
+            DataFile() :
+                m_file(::tmpfile()) {
+                if (!m_file) {
+                    throw std::system_error(errno, std::system_category(), "tmpfile failed");
+                }
+            }
+
+            /**
+             * Create and open a temporary file with the specified size. It
+             * is removed after opening.
+             *
+             * @throws std::system_error if something went wrong.
+             */
+            explicit DataFile(size_t size) :
+                DataFile() {
+                grow(size);
+            }
+
+            /**
+             * Create and open a named file.
+             *
+             * @param filename the name of the file
+             * @param writable should the file be writable?
+             * @throws std::system_error if something went wrong.
+             */
+            DataFile(const char* filename, bool writable) :
+                m_file(::fopen(filename, writable ? "wb+" : "rb" )) {
+                if (!m_file) {
+                    throw std::system_error(errno, std::system_category(), "fopen failed");
+                }
+            }
+
+            /**
+             * Create and open a named file.
+             *
+             * @param filename the name of the file
+             * @param writable should the file be writable?
+             * @throws std::system_error if something went wrong.
+             */
+            DataFile(const std::string& filename, bool writable) :
+                DataFile(filename.c_str(), writable) {
+            }
+
+            /**
+             * In boolean context the DataFile class returns true if the file
+             * is open.
+             */
+            operator bool() const noexcept {
+                return m_file != nullptr;
+            }
+
+            /**
+             * Close the file.
+             *
+             * Does nothing if the file is already closed.
+             *
+             * @throws std::system_error if file could not be closed
+             */
+            void close() {
+                if (m_file) {
+                    if (::fclose(m_file) != 0) {
+                        throw std::system_error(errno, std::system_category(), "fclose failed");
+                    }
+                    m_file = nullptr;
+                }
+            }
+
+            ~DataFile() noexcept {
+                try {
+                    close();
+                } catch (std::system_error&) {
+                    // ignore
+                }
+            }
+
+            /**
+             * Get file descriptor of underlying file.
+             *
+             * @throws std::runtime_errro if file is not open
+             * @throws std::system_error if fileno(3) call failed
+             */
+            int fd() const {
+                if (!m_file) {
+                    throw std::runtime_error("no open file");
+                }
+
+                int fd = ::fileno(m_file);
+
+                if (fd == -1) {
+                    throw std::system_error(errno, std::system_category(), "fileno failed");
+                }
+
+                return fd;
+            }
+
+            /**
+             * Ask the operating system for the size of this file.
+             *
+             * @throws std::system_error if fstat(2) call failed
+             */
+            size_t size() const {
+                return osmium::util::file_size(fd());
+            }
+
+            /**
+             * Grow file to given size.
+             *
+             * If the file is large enough already, nothing is done.
+             * The file is never shrunk.
+             *
+             * @throws std::system_error if ftruncate(2) call failed
+             */
+            void grow(size_t new_size) const {
+                if (size() < new_size) {
+                    osmium::util::resize_file(fd(), new_size);
+                }
+            }
+
+        }; // class DataFile
+
+    } // namespace util
+
+} // namespace osmium
+
+
+#endif // OSMIUM_UTIL_DATA_FILE_HPP

third_party/libosmium/include/osmium/util/delta.hpp

@@ -0,0 +1,147 @@
+#ifndef OSMIUM_UTIL_DELTA_HPP
+#define OSMIUM_UTIL_DELTA_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <iterator>
+#include <type_traits>
+#include <utility>
+
+namespace osmium {
+
+    namespace util {
+
+        /**
+         * Helper class for delta encoding.
+         */
+        template <typename T>
+        class DeltaEncode {
+
+            T m_value;
+
+        public:
+
+            DeltaEncode(T value = 0) :
+                m_value(value) {
+            }
+
+            void clear() {
+                m_value = 0;
+            }
+
+            T update(T new_value) {
+                using std::swap;
+                swap(m_value, new_value);
+                return m_value - new_value;
+            }
+
+        }; // class DeltaEncode
+
+        /**
+         * Helper class for delta decoding.
+         */
+        template <typename T>
+        class DeltaDecode {
+
+            T m_value;
+
+        public:
+
+            DeltaDecode() :
+                m_value(0) {
+            }
+
+            void clear() {
+                m_value = 0;
+            }
+
+            T update(T delta) {
+                m_value += delta;
+                return m_value;
+            }
+
+        }; // class DeltaDecode
+
+        template <typename TBaseIterator, typename TTransform, typename TValue>
+        class DeltaEncodeIterator : public std::iterator<std::input_iterator_tag, TValue> {
+
+            typedef TValue value_type;
+
+            TBaseIterator m_it;
+            TBaseIterator m_end;
+            value_type m_delta;
+            DeltaEncode<value_type> m_value;
+            TTransform m_trans;
+
+        public:
+
+            DeltaEncodeIterator(TBaseIterator first, TBaseIterator last, TTransform& trans) :
+                m_it(first),
+                m_end(last),
+                m_delta(m_trans(m_it)),
+                m_value(m_delta),
+                m_trans(trans) {
+            }
+
+            DeltaEncodeIterator& operator++() {
+                if (m_it != m_end) {
+                    m_delta = m_value.update(m_trans(++m_it));
+                }
+                return *this;
+            }
+
+            DeltaEncodeIterator operator++(int) {
+                DeltaEncodeIterator tmp(*this);
+                operator++();
+                return tmp;
+            }
+
+            value_type operator*() {
+                return m_delta;
+            }
+
+            bool operator==(const DeltaEncodeIterator& rhs) const {
+                return m_it == rhs.m_it && m_end == rhs.m_end;
+            }
+
+            bool operator!=(const DeltaEncodeIterator& rhs) const {
+                return !(*this == rhs);
+            }
+
+        }; // class DeltaEncodeIterator
+
+    } // namespace util
+
+} // namespace osmium
+
+#endif // OSMIUM_UTIL_DELTA_HPP

third_party/libosmium/include/osmium/util/endian.hpp

@@ -0,0 +1,45 @@
+#ifndef OSMIUM_UTIL_ENDIAN_HPP
+#define OSMIUM_UTIL_ENDIAN_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+// Windows is only available for little endian architectures
+// http://stackoverflow.com/questions/6449468/can-i-safely-assume-that-windows-installations-will-always-be-little-endian
+#if !defined(_WIN32) && !defined(__APPLE__)
+# include <endian.h>
+#else
+# define __LITTLE_ENDIAN 1234
+# define __BYTE_ORDER __LITTLE_ENDIAN
+#endif
+
+#endif // OSMIUM_UTIL_ENDIAN_HPP

third_party/libosmium/include/osmium/util/file.hpp

@@ -0,0 +1,119 @@
+#ifndef OSMIUM_UTIL_FILE_HPP
+#define OSMIUM_UTIL_FILE_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cerrno>
+#include <cstddef>
+#include <cstdio>
+#include <system_error>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#ifdef _WIN32
+# include <io.h>
+# include <windows.h>
+#endif
+
+#ifndef _MSC_VER
+# include <unistd.h>
+#else
+// https://msdn.microsoft.com/en-us/library/whx354w1.aspx
+# define ftruncate _chsize_s
+#endif
+
+namespace osmium {
+
+    namespace util {
+
+        /**
+         * Get file size.
+         * This is a small wrapper around a system call.
+         *
+         * @param fd File descriptor
+         * @returns file size
+         * @throws std::system_error If system call failed
+         */
+        inline size_t file_size(int fd) {
+#ifdef _MSC_VER
+            // Windows implementation
+            // https://msdn.microsoft.com/en-us/library/dfbc2kec.aspx
+            auto size = ::_filelengthi64(fd);
+            if (size == -1L) {
+                throw std::system_error(errno, std::system_category(), "_filelengthi64 failed");
+            }
+            return size_t(size);
+#else
+            // Unix implementation
+            struct stat s;
+            if (::fstat(fd, &s) != 0) {
+                throw std::system_error(errno, std::system_category(), "fstat failed");
+            }
+            return size_t(s.st_size);
+#endif
+        }
+
+        /**
+         * Resize file.
+         * Small wrapper around ftruncate(2) system call.
+         *
+         * @param fd File descriptor
+         * @param new_size New size
+         * @throws std::system_error If ftruncate(2) call failed
+         */
+        inline void resize_file(int fd, size_t new_size) {
+            if (::ftruncate(fd, new_size) != 0) {
+                throw std::system_error(errno, std::system_category(), "ftruncate failed");
+            }
+        }
+
+        /**
+         * Get the page size for this system.
+         */
+        inline size_t get_pagesize() {
+#ifdef _WIN32
+            // Windows implementation
+            SYSTEM_INFO si;
+            GetSystemInfo(&si);
+            return si.dwPageSize;
+#else
+            // Unix implementation
+            return ::sysconf(_SC_PAGESIZE);
+#endif
+        }
+
+    } // namespace util
+
+} // namespace osmium
+
+#endif // OSMIUM_UTIL_FILE_HPP