Squashed 'third_party/libosmium/' content from commit 910f8f1

git-subtree-dir: third_party/libosmium
git-subtree-split: 910f8f1e992402e0f1acd0132eaffa7539ca83d2

.gitignore

@@ -0,0 +1,3 @@
+core
+*.swp
+build*

.travis.yml

@@ -0,0 +1,43 @@
+language: cpp
+
+compiler:
+ - gcc
+ - clang
+
+before_install:
+ # we need at least g++-4.8 for c++11 features
+ - sudo add-apt-repository --yes ppa:ubuntu-toolchain-r/test
+ - sudo apt-get update --yes --quiet
+
+install:
+ # upgrade compilers
+ - sudo apt-get install --yes gcc-4.8 g++-4.8
+ # make sure 'cpp' is the just installed current one
+ - sudo rm /usr/bin/cpp
+ - sudo ln -s /usr/bin/cpp-4.8 /usr/bin/cpp
+ # upgrade libosmium dependencies
+ - sudo apt-get install --yes make libboost-dev libboost-program-options-dev libsparsehash-dev libprotobuf-dev protobuf-compiler libgeos++-dev libproj-dev
+ - sudo apt-get install --yes make libgdal1h libgdal-dev
+ # OSMPBF is too old, install from git
+ #- sudo apt-get install --yes libosmpbf-dev
+ - git clone https://github.com/scrosby/OSM-binary.git
+ - cd OSM-binary/src
+ - make
+ - sudo make install
+ - cd ../..
+
+#env:
+
+before_script:
+ - true
+
+script:
+ - if [ "${CXX}" = 'g++' ]; then export CXX=g++-4.8; fi;
+ - mkdir build
+ - cd build
+ - cmake -L -DCMAKE_BUILD_TYPE=Dev ..
+ - make VERBOSE=1
+ # Disable multipolygon test because it needs ruby and the 'json' gem, but the
+ # travis ruby installation doesn't find the gem it did install itself.
+ - ctest -V -E testdata-multipolygon
+

CMakeLists.txt

@@ -0,0 +1,259 @@
+#----------------------------------------------------------------------
+#
+#  Libosmium CMakeLists.txt
+#
+#----------------------------------------------------------------------
+
+cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
+
+
+#----------------------------------------------------------------------
+#
+#  Project version
+#
+#----------------------------------------------------------------------
+
+project(libosmium)
+
+set(LIBOSMIUM_VERSION_MAJOR 0)
+set(LIBOSMIUM_VERSION_MINOR 0)
+set(LIBOSMIUM_VERSION_PATCH 1)
+
+set(LIBOSMIUM_VERSION ${LIBOSMIUM_VERSION_MAJOR}.${LIBOSMIUM_VERSION_MINOR}.${LIBOSMIUM_VERSION_PATCH})
+
+
+#----------------------------------------------------------------------
+#
+#  Build options
+#
+#  (Change with -DOPTION=VALUE on cmake command line.)
+#
+#----------------------------------------------------------------------
+
+option(BUILD_EXAMPLES   "compile example programs" ON)
+option(BUILD_UNIT_TESTS "compile unit tests, please run them with ctest" ON)
+option(BUILD_DATA_TESTS "compile data tests, please run them with ctest" ON)
+
+if(CMAKE_BUILD_TYPE STREQUAL "Dev")
+    option(BUILD_HEADERS    "compile every header file on its own" ON)
+else()
+    option(BUILD_HEADERS    "compile every header file on its own" OFF)
+endif()
+
+
+#----------------------------------------------------------------------
+#
+#  Find external dependencies
+#
+#----------------------------------------------------------------------
+
+# check that the essential libraries were found
+if(BUILD_EXAMPLES OR BUILD_TESTING OR BUILD_UNIT_TESTS OR BUILD_DATA_TESTS OR BUILD_HEADERS)
+
+    find_package(Boost 1.38)
+    mark_as_advanced(CLEAR BOOST_ROOT)
+
+    if(Boost_FOUND)
+        include_directories(${Boost_INCLUDE_DIRS})
+    else()
+        set(BOOST_ROOT "NOT FOUND: please choose" CACHE PATH "")
+        message(FATAL_ERROR "PLEASE, specify the directory where the Boost library is installed in BOOST_ROOT")
+    endif()
+
+    set(OSMIUM_INCLUDE_DIR include)
+    find_package(Osmium COMPONENTS io gdal geos proj sparsehash)
+    include_directories(${OSMIUM_INCLUDE_DIRS})
+
+    if(MSVC)
+        find_path(GETOPT_INCLUDE_DIR getopt.h)
+        find_library(GETOPT_LIBRARY NAMES wingetopt)
+        if(GETOPT_INCLUDE_DIR AND GETOPT_LIBRARY)
+            include_directories(${GETOPT_INCLUDE_DIR})
+            list(APPEND OSMIUM_LIBRARIES ${GETOPT_LIBRARY})
+        else()
+            set(GETOPT_MISSING 1)
+        endif()
+    endif()
+
+    include_directories(include)
+endif()
+
+#----------------------------------------------------------------------
+#
+#  Decide which C++ version to use (Minimum/default: C++11).
+#
+#----------------------------------------------------------------------
+
+if(NOT USE_CPP_VERSION)
+    set(USE_CPP_VERSION c++11)
+endif()
+message(STATUS "Use C++ version: ${USE_CPP_VERSION}")
+# following only available from cmake 2.8.12:
+#   add_compile_options(-std=${USE_CPP_VERSION})
+# so using this instead:
+add_definitions(-std=${USE_CPP_VERSION})
+
+#----------------------------------------------------------------------
+
+set(CMAKE_CXX_FLAGS_DEV "-O3 -g"
+    CACHE STRING "Flags used by the compiler during developer builds."
+    FORCE)
+set(CMAKE_EXE_LINKER_FLAGS_DEV ""
+    CACHE STRING "Flags used by the linker during developer builds."
+    FORCE)
+mark_as_advanced(
+    CMAKE_CXX_FLAGS_DEV
+    CMAKE_EXE_LINKER_FLAGS_DEV
+)
+
+set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g"
+    CACHE STRING "Flags used by the compiler during RELWITHDEBINFO builds."
+    FORCE)
+
+set(CMAKE_CONFIGURATION_TYPES "Debug Release RelWithDebInfo MinSizeRel Dev")
+
+# Force RelWithDebInfo build type if none was given
+if (CMAKE_BUILD_TYPE STREQUAL "")
+    set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Choose the type of build, options are: ${CMAKE_CONFIGURATION_TYPES}." FORCE)
+else()
+    set(CMAKE_BUILD_TYPE "${CMAKE_BUILD_TYPE}" CACHE STRING "Choose the type of build, options are: ${CMAKE_CONFIGURATION_TYPES}." FORCE)
+endif()
+
+
+#----------------------------------------------------------------------
+
+if(WIN32)
+    add_definitions(-DWIN32 -D_WIN32 -DMSWIN32 -DBGDWIN32 -DWINVER=0x0500 -D_WIN32_WINNT=0x0500 -D_WIN32_IE=0x0600)
+    set(CPACK_GENERATOR ZIP)
+else()
+    set(CPACK_GENERATOR TGZ)
+endif()
+
+
+#----------------------------------------------------------------------
+#
+#  Set up testing
+#
+#----------------------------------------------------------------------
+enable_testing()
+
+find_program(MEMORYCHECK_COMMAND valgrind)
+set(MEMORYCHECK_COMMAND_OPTIONS "--trace-children=yes --leak-check=full --show-reachable=yes --error-exitcode=1")
+set(MEMORYCHECK_SUPPRESSIONS_FILE "${PROJECT_SOURCE_DIR}/test/valgrind.supp")
+
+if(BUILD_UNIT_TESTS OR BUILD_TESTING)
+    add_subdirectory(test)
+endif()
+
+if(BUILD_DATA_TESTS OR BUILD_TESTING)
+    add_subdirectory(test/osm-testdata)
+endif()
+
+
+#----------------------------------------------------------------------
+#
+#  Optional "cppcheck" target that checks C++ code
+#
+#----------------------------------------------------------------------
+message(STATUS "Looking for cppcheck")
+find_program(CPPCHECK cppcheck)
+
+if(CPPCHECK)
+    message(STATUS "Looking for cppcheck - found")
+    set(CPPCHECK_OPTIONS --enable=warning,style,performance,portability,information,missingInclude)
+
+    # cpp doesn't find system includes for some reason, suppress that report
+    set(CPPCHECK_OPTIONS ${CPPCHECK_OPTIONS} --suppress=missingIncludeSystem)
+
+    file(GLOB_RECURSE ALL_INCLUDES   include/osmium/*.hpp)
+    file(GLOB         ALL_EXAMPLES   examples/*.cpp)
+    file(GLOB         ALL_UNIT_TESTS test/t/*/test_*.cpp)
+    file(GLOB         ALL_DATA_TESTS test/osm-testdata/*.cpp)
+
+    if(Osmium_DEBUG)
+        message(STATUS "Checking includes      : ${ALL_INCLUDES}")
+        message(STATUS "Checking example code  : ${ALL_EXAMPLES}")
+        message(STATUS "Checking unit test code: ${ALL_UNIT_TESTS}")
+        message(STATUS "Checking data test code: ${ALL_DATA_TESTS}")
+    endif()
+
+    set(CPPCHECK_FILES ${ALL_INCLUDES} ${ALL_EXAMPLES} ${ALL_UNIT_TESTS} ${ALL_DATA_TESTS})
+
+    add_custom_target(cppcheck
+        ${CPPCHECK}
+        --std=c++11 ${CPPCHECK_OPTIONS}
+        -I ${CMAKE_SOURCE_DIR}/include
+        ${CPPCHECK_FILES}
+    )
+else()
+    message(STATUS "Looking for cppcheck - not found")
+    message(STATUS "  Make target cppcheck not available")
+endif(CPPCHECK)
+
+
+#----------------------------------------------------------------------
+#
+#  Doxygen-based documentation
+#
+#----------------------------------------------------------------------
+find_package(Doxygen)
+if(DOXYGEN_FOUND)
+    configure_file(doc/Doxyfile.in ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY)
+    add_custom_target(doc
+        ${DOXYGEN_EXECUTABLE}
+        ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+        COMMENT "Generating API documentation with Doxygen" VERBATIM
+    )
+    install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/doc/html" DESTINATION "share/doc/libosmium-dev")
+else()
+    message(STATUS "Doxygen not found, so 'doc' target will not be available.")
+endif()
+
+
+#----------------------------------------------------------------------
+
+if(BUILD_EXAMPLES)
+    add_subdirectory(examples)
+endif()
+
+#----------------------------------------------------------------------
+# This will try to compile include files on their own to detect missing
+# include directives and other dependency-related problems. Note that if this
+# work, it is not enough to be sure it will compile in production code.
+# But if it reports an error we know we are missing something.
+if(BUILD_HEADERS)
+    file(GLOB_RECURSE ALL_HPPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/include" include/osmium/*.hpp)
+
+    # In 'Dev' mode: compile with very strict warnings and turn them into errors.
+    if(CMAKE_BUILD_TYPE STREQUAL "Dev")
+        add_definitions(-Werror ${OSMIUM_WARNING_OPTIONS})
+    endif()
+
+    file(MAKE_DIRECTORY header_check)
+
+    foreach(hpp ${ALL_HPPS})
+        string(REPLACE ".hpp" "" tmp ${hpp})
+        string(REPLACE "/" "__" libname ${tmp})
+
+        # Create a dummy .cpp file that includes the header file we want to
+        # check.
+        set(DUMMYCPP ${CMAKE_BINARY_DIR}/header_check/${libname}.cpp)
+        file(WRITE ${DUMMYCPP} "#include <${hpp}>\n")
+
+        # There is no way in CMake to just compile but not link a C++ file,
+        # so we pretend to build a library here.
+        add_library(${libname} OBJECT ${DUMMYCPP} include/${hpp})
+    endforeach()
+endif()
+
+install(DIRECTORY include/osmium DESTINATION include)
+
+# We only have a copy of this file so we can use older boost versions which
+# don't have it. We probably don't want to install it.
+#install(FILES include/boost_unicode_iterator.hpp DESTINATION include)
+
+#----------------------------------------------------------------------
+include(CPack)
+

CONTRIBUTING.md

@@ -0,0 +1,132 @@
+
+# Notes for Developers
+
+Read this if you want to contribute to Libosmium.
+
+
+## Versioning
+
+Osmium is currently considered in beta and doesn't use versioning yet. Proper
+versions will be introduced as soon as it is somewhat stable.
+
+
+## Namespace
+
+All Osmium code MUST be in the `osmium` namespace or one of its sub-namespaces.
+
+
+## Include-Only
+
+Osmium is a include-only library. You can't compile the library itself. There
+is no libosmium.so.
+
+One drawback ist that you can't have static data in classes, because there
+is no place to put this data.
+
+All free functions must be declared `inline`.
+
+
+## Coding Conventions
+
+These coding conventions have been changing over time and some code is still
+different.
+
+* All include files have `#ifdef` guards around them, macros are the path name
+  in all uppercase where the slashes (`/`) have been changed to underscore (`_`).
+* Class names begin with uppercase chars and use CamelCase. Smaller helper
+  classes are usually defined as struct and have lowercase names.
+* Macros (and only macros) are all uppercase. Use macros sparingly, usually
+  a constexpr is better.
+* Variables, attributes, and function names are lowercase with
+  `underscores_between_words`.
+* Class attribute names start with `m_` (member).
+* Template parameters are single uppercase letters or start with uppercase `T`
+  and use CamelCase.
+* Typedefs have `names_like_this_type` which end in `_type`.
+* Macros should only be used for controlling which parts of the code should be
+  included when compiling.
+* Use `descriptive_variable_names`, exceptions are well-established conventions
+  like `i` for a loop variable. Iterators are usually called `it`.
+* Declare variables where they are first used (C++ style), not at the beginning
+  of a function (old C style).
+* Names from external namespaces (even `std`) are always mentioned explicitly.
+  Do not use `using` (except for `std::swap`). This way we can't even by
+  accident pollute the namespace of the code including Osmium.
+* `#include` directives appear in three "blocks" after the copyright notice.
+  The blocks are separated by blank lines. First block contains `#include`s for
+  standard C/C++ includes, second block for any external libs used, third
+  block for osmium internal includes. Within each block `#include`s are usually
+  sorted by path name. All `#include`s use `<>` syntax not `""`.
+* Names not to be used from outside the library should be in a namespace
+  called `detail` under the namespace where they would otherwise appear. If
+  whole include files are never meant to be included from outside they should
+  be in a subdirectory called `detail`.
+* All files have suffix `.hpp`.
+* Closing } of all classes and namespaces should have a trailing comment
+  with the name of the class/namespace.
+* All constructors with one or more arguments should be declared "explicit"
+  unless there is a reason for them not to be. Document that reason.
+
+Keep to the indentation and other styles used in the code. Use `make indent`
+in the toplevel directory to fix indentation and styling. It calls `astyle`
+with the right parameters. This program is in the `astyle` Debian package.
+
+
+## C++11
+
+Osmium uses C++11 and you can use its features such as auto, lambdas,
+threading, etc. There are a few features we do not use, because even modern
+compilers don't support them yet. This list might change as we get more data
+about which compilers support which feature and what operating system versions
+or distributions have which versions of these compilers installed.
+
+GCC 4.6   - too old, not supported (Ubuntu 12.04 LTS)
+GCC 4.7.2 - can probably not be supported (Debian wheezy/stable)
+GCC 4.7.3 - works
+GCC 4.8   - works
+clang 3.0 - too old, not supported (Debian wheezy/stable, Ubuntu 12.04 LTS)
+clang 3.2 - works
+
+C++11 features you should not use:
+* Inherited Constructors (works only in GCC 4.8+ and clang 3.3+, not in Visual
+  Studio)
+
+
+## Checking your code
+
+The Osmium makefiles use pretty draconian warning options for the compiler.
+This is good. Code MUST never produce any warnings, even with those settings.
+If absolutely necessary pragmas can be used to disable certain warnings in
+specific areas of the code.
+
+If the static code checker `cppcheck` is installed, the CMake configuration
+will add a new build target `cppcheck` that will check all `.cpp` and `.hpp`
+files. Cppcheck finds some bugs that gcc/clang doesn't. But take the result
+with a grain of salt, it also sometimes produces wrong warnings.
+
+Set `BUILD_HEADERS=ON` in the CMake config to enable compiling all include
+files on their own to check whether dependencies are all okay. All include
+files MUST include all other include files they depend on.
+
+Call `cmake/iwyu.sh` to check for proper includes and forward declarations.
+This uses the clang-based `include-what-you-use` program. Note that it does
+produce some false reports and crashes often. The `osmium.imp` file can be
+used to define mappings for iwyu. See the IWYU tool at
+<http://code.google.com/p/include-what-you-use/>.
+
+
+## Testing
+
+There are a unit tests using the Catch Unit Test Framework in the `test`
+directory and some data tests in `test/osm-testdata`. They are built by the
+default cmake config. Run `ctest` to run them. Many more tests are needed.
+
+
+## Documenting the code
+
+All namespaces, classes, functions, attributes, etc. should be documented.
+
+Osmium uses the Doxygen (www.doxygen.org) source code documentation system.
+If it is installed, the CMake configuration will add a new build target, so
+you can build it with `make doc`.
+

LICENSE.txt

@@ -0,0 +1,23 @@
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

Makefile

@@ -0,0 +1,25 @@
+
+all:
+	mkdir -p build && cd build && cmake .. && $(MAKE)
+
+doc:
+	mkdir -p build && cd build && cmake .. && $(MAKE) doc
+
+clean:
+	if test -d build; then cd build && $(MAKE) clean; fi
+
+distclean:
+	rm -fr build
+
+deb:
+	debuild -I -us -uc
+
+deb-clean:
+	debuild clean
+
+indent:
+	astyle --style=java --indent-namespaces --indent-switches --pad-header --lineend=linux --suffix=none --recursive include/\*.hpp examples/\*.cpp test/\*.cpp
+#	astyle --style=java --indent-namespaces --indent-switches --pad-header --unpad-paren --align-pointer=type --lineend=linux --suffix=none --recursive include/\*.hpp examples/\*.cpp test/\*.cpp
+
+.PHONY: clean distclean deb deb-clean doc indent
+

README-changes-from-old-osmium

@@ -0,0 +1,43 @@
+
+Changes from old versions of Osmium
+===================================
+
+This version has some substantial changes and users of Osmium will have to
+rewrite their code. Use the examples provided in the "example" directory
+or in the osmium-contrib repository to get an idea what needs changing.
+These examples are often similar to the examples provided with the old
+Osmium so they should give you an idea how your code has to change.
+
+Here are some of the more important changes:
+
+* Osmium now needs C++11. It will not work with older compilers. You need
+  at least GCC 4.7.3 or clang (LLVM) 3.2.
+
+* Namespaces are now all lower case. Everything is in the "osmium" namespace
+  or sub-namespaces of it. Many classes and functions have been moved to
+  different, more logical or shorter namespaces.
+
+* You can't just instantiate OSM objects such as Nodes, Ways, or Relations.
+  You need a Buffer first to hold them and use the Builder classes to
+  create them. This is a bit more cumbersome, but greatly reduces the need
+  for memory management and makes Osmium faster and easier to parallelize.
+
+* Usually you don't act on single OSM objects any more, but on groups of
+  them in a Buffer.
+
+* Reading and writing of OSM data is much simpler. Use the Reader and Writer
+  classes as they hide much of the detail and have much nicer interfaces
+  than the old Input/Output classes.
+
+* The class Osmium::OSM::Position was renamed to osmium::Location. This
+  better reflects that it is a location on the planet we are talking about.
+  The word "position" has many meanings and is, for instance, often used
+  to denote a position in a file or buffer or so.
+
+* The dependency on boost has been greatly reduced. C++11 offers many
+  features that used to be only available with boost, such as shared_ptr.
+  Osmium now uses the C++11 versions of these.
+
+* Osmium now makes use of the new C++11 threading support when reading and
+  writing OSM files.
+

README.md

@@ -0,0 +1,217 @@
+# Osmium Library
+
+http://osmcode.org/libosmium
+
+A fast and flexible C++ library for working with OpenStreetMap data.
+
+NOTE: This is a beta version of the next-generation Osmium. For production
+use, see the Osmium version at https://github.com/joto/osmium .
+
+There are a few applications that use the Osmium library in the examples
+directory. See the [osmium-contrib](http://github.com/osmcode/osmium-contrib)
+repository for more example code.
+
+[![Build Status](https://secure.travis-ci.org/osmcode/libosmium.png)](http://travis-ci.org/osmcode/libosmium)
+[![Build status](https://ci.appveyor.com/api/projects/status/mkbg6e6stdgq7c1b?svg=true)](https://ci.appveyor.com/project/Mapbox/libosmium)
+
+Libosmium is developed on Linux, but also works on OSX and Windows (with some
+limitations).
+
+## Prerequisites
+
+Because Osmium uses many C++11 features you need a modern compiler and standard
+C++ library. Osmium needs at least GCC 4.8 or clang (LLVM) 3.2. (Some parts may
+work with older versions.)
+
+Different parts of Osmium (and the applications built on top of it) need
+different libraries. You DO NOT NEED to install all of them, just install those
+you need for the programs you need.
+
+    boost-iterator, boost-regex
+        http://www.boost.org/
+        Debian/Ubuntu: libboost-dev
+        openSUSE: boost-devel
+        Homebrew: boost
+
+    boost-program-options (for parsing command line options in some examples)
+        http://www.boost.org/doc/libs/1_54_0/doc/html/program_options.html
+        Debian/Ubuntu: libboost-program-options-dev
+
+    Google protocol buffers (for PBF support)
+        http://code.google.com/p/protobuf/ (at least version 2.3.0 needed)
+        Debian/Ubuntu: libprotobuf-dev protobuf-compiler
+        openSUSE: protobuf-devel
+        Homebrew: protobuf
+        Also see http://wiki.openstreetmap.org/wiki/PBF_Format
+
+    OSMPBF (for PBF support)
+        https://github.com/scrosby/OSM-binary
+        Debian/Ubuntu: libosmpbf-dev
+        (The package in Ubuntu 14.04 and older is too old, install from source
+        in these cases.)
+        Homebrew: osm-pbf
+
+    Expat (for parsing XML files)
+        http://expat.sourceforge.net/
+        Debian/Ubuntu: libexpat1-dev
+        openSUSE: libexpat-devel
+        Homebrew: expat
+
+    zlib (for PBF and for gzip support when reading/writing XML)
+        http://www.zlib.net/
+        Debian/Ubuntu: zlib1g-dev
+        openSUSE: zlib-devel
+
+    bz2lib (for bzip2 support when reading/writing XML)
+        http://www.bzip.org/
+        Debian/Ubuntu: libbz2-dev
+
+    Google sparsehash
+        http://code.google.com/p/google-sparsehash/
+        Debian/Ubuntu: libsparsehash-dev
+        openSUSE: sparsehash
+        Homebrew: google-sparsehash
+
+    GDAL (for OGR support)
+        http://gdal.org/
+        Debian/Ubuntu: libgdal1-dev
+        openSUSE: libgdal-devel
+        Homebrew: gdal
+
+    GEOS (for GEOS support)
+        http://trac.osgeo.org/geos/
+        Debian/Ubuntu: libgeos++-dev
+        openSUSE: libgeos-devel
+        Homebrew: geos
+
+    libproj (for projection support)
+        http://trac.osgeo.org/proj/
+        Debian/Ubuntu: libproj-dev
+
+    Doxygen (to build API documentation) and tools
+        http://www.stack.nl/~dimitri/doxygen/
+        Debian/Ubuntu: doxygen graphviz xmlstarlet
+        Homebrew: doxygen
+
+You need to either install the packages for your distribution or install those
+libraries from source. Most libraries should be available in all distributions.
+
+
+## Directories
+
+* include: C/C++ include files. All of Osmium is in those header files which
+  are needed for building Osmium applications.
+
+* examples: Osmium example applications.
+
+* test: Tests (see below).
+
+* doc: Config for documentation.
+
+
+## Building
+
+Osmium is a header-only library, so there is nothing to build for the
+library itself.
+
+But there are some tests and examples that can be build. Libosmium uses
+cmake:
+
+    mkdir build
+    cd build
+    cmake ..
+    make
+
+This will build the examples and tests. Call `ctest` to run the tests.
+
+To build the documentation you need Doxygen. If cmake can find it it will
+enable the `doc` target so you can build the documentation like this:
+
+    make doc
+
+If the 'cppcheck' binary is found, cmake will add another target to the
+Makfile which allows you to call cppheck on all `*.cpp` and `*.hpp` files:
+
+    make cppcheck
+
+For Mac users: If you have clang 3.2 or newer, use the system compiler.
+If not you have to build the compiler yourself. See the instructions
+on http://clang.llvm.org/ .
+
+Preliminary support for cmake is provided. You can use this instead of "make":
+
+
+## Testing
+
+### Unit Tests
+
+There are a few unit tests using the Catch unit test framework in the "test"
+directory. Many more tests are needed, any help appreciated.
+
+For [Catch](https://github.com/philsquared/Catch/) only one header file is
+needed which is included (`test/include/catch.hpp`).
+
+To compile these unit tests make sure `BUILD_UNIT_TESTS` is set in the cmake
+config, then build the project and call
+
+    ctest
+
+You can run tests matching a pattern by calling
+
+    ctest -R test_name
+
+for instance:
+
+    ctest basic_test_node
+
+will run the `basic_test_node` and `basic_test_node_ref` tests.
+
+### Data Tests
+
+In addition there are some test based on the OSM Test Data Repository at
+http://osmcode.org/osm-testdata/ . Make sure `BUILD_DATA_TESTS` is set in the
+cmake config, then build the project and call `ctest`.
+
+Some of these tests need Ruby and then 'json' gem installed.
+
+### Valgrind
+
+Running tests with valgrind:
+
+    ctest -D ExperimentalMemCheck
+
+
+## Osmium on 32bit Machines
+
+Osmium works well on 64 bit machines, but on 32 bit machines there are some
+problems. Be aware that not everything will work on 32 bit architectures.
+This is mostly due to the 64 bit needed for node IDs. Also Osmium hasn't been
+tested well on 32 bit systems. Here are some issues you might run into:
+
+* Google Sparsehash does not work on 32 bit machines in our use case.
+* The `mmap` system call is called with a `size_t` argument, so it can't
+  give you more than 4GByte of memory on 32 bit systems. This might be a
+  problem.
+
+Please report any issues you have and we might be able to solve them.
+
+
+## Switching from the old Osmium
+
+See `README-changes-from-old-osmium`.
+
+
+## License
+
+The Osmium Library is available under the Boost Software License. See
+LICENSE.txt.
+
+
+## Authors
+
+The Osmium Library was mainly written and is maintained by Jochen Topf
+(jochen@topf.org).
+
+Other authors:
+* Peter Körner (github@mazdermind.de) (PBF writer, ...)
+

appveyor.yml

@@ -0,0 +1,55 @@
+#
+# Configuration for appveyor.com
+#
+
+environment:
+  matrix:
+  - configuration: Dev
+#  - configuration: Release
+
+# branches to build
+branches:
+  # whitelist
+  only:
+    - master
+
+# Operating system (build VM template)
+os: Windows Server 2012 R2
+
+# scripts that are called at very beginning, before repo cloning
+init:
+
+# clone directory
+clone_folder: c:\projects\libosmium
+
+platform: x64
+
+install:
+  # by default, all script lines are interpreted as batch
+  - cd c:\projects
+  - nuget install boost
+  - nuget install bzip2
+  - nuget install zlib
+  - nuget install GDAL
+  - nuget install expat
+  - nuget install protobuf
+  - dir /S c:\projects
+  - git clone https://github.com/scrosby/OSM-binary
+  - cd OSM-binary
+  - mkdir build
+  - cd build
+  - call "%VS120COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64
+  - cmake .. -G "Visual Studio 12 Win64" -DSEARCH_PREFIX=c:\projects -T CTP_Nov2013
+  - msbuild /clp:Verbosity=minimal /nologo OSM-binary.sln
+
+build_script:
+  - cd c:\projects\libosmium
+  - mkdir build
+  - cd build
+  - call "%VS120COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64
+  - SET PATH=C:\Program Files (x86)\MSBuild\12.0\bin\;%PATH%
+  - SET P=c:/projects/libosmium
+  - cmake .. -G "Visual Studio 12 Win64" -DCMAKE_BUILD_TYPE=%Configuration% -DCMAKE_PREFIX_PATH=c:\projects -DBoost_USE_STATIC_LIBS=ON -T CTP_Nov2013
+  - msbuild /clp:Verbosity=minimal /nologo libosmium.sln
+  - msbuild /clp:Verbosity=minimal /nologo tests.vcxproj
+

cmake/FindOSMPBF.cmake

@@ -0,0 +1,50 @@
+#
+# Locate OSMPBF library
+#
+# This module defines
+#  OSMPBF_FOUND        - if false, do not try to link to OSMPBF
+#  OSMPBF_LIBRARIES    - full library path name
+#  OSMPBF_INCLUDE_DIRS - where to find OSMPBF.hpp
+#
+# Note that the expected include convention is
+#  #include <osmpbf/osmpbf.h>
+# and not
+#  #include <osmpbf.h>
+#
+
+find_path(OSMPBF_INCLUDE_DIR osmpbf/osmpbf.h
+    HINTS $ENV{OSMPBF_DIR}
+    PATH_SUFFIXES include
+    PATHS
+        ~/Library/Frameworks
+        /Library/Frameworks
+        /usr/local
+        /usr
+        /opt/local # DarwinPorts
+        /opt
+)
+
+find_library(OSMPBF_LIBRARY
+    NAMES osmpbf
+    HINTS $ENV{OSMPBF_DIR}
+    PATH_SUFFIXES lib64 lib
+    PATHS
+        ~/Library/Frameworks
+        /Library/Frameworks
+        /usr/local
+        /usr
+        /opt/local
+        /opt
+)
+
+# Handle the QUIETLY and REQUIRED arguments and set OSMPBF_FOUND to TRUE if
+# all listed variables are TRUE.
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(OSMPBF DEFAULT_MSG OSMPBF_LIBRARY OSMPBF_INCLUDE_DIR)
+
+# Copy the results to the output variables.
+if(OSMPBF_FOUND)
+    set(OSMPBF_INCLUDE_DIRS ${OSMPBF_INCLUDE_DIR})
+    set(OSMPBF_LIBRARIES ${OSMPBF_LIBRARY})
+endif()
+

cmake/FindOsmium.cmake

@@ -0,0 +1,285 @@
+#----------------------------------------------------------------------
+#
+#  FindOsmium.cmake
+#
+#  Find the Libosmium headers and, optionally, several components needed for
+#  different Libosmium functions.
+#
+#----------------------------------------------------------------------
+#
+#  Usage:
+#
+#    Copy this file (and OsmiumOptions.cmake) somewhere into your project
+#    directory, where cmake can find it. Usually this will be a directory
+#    called "cmake" which you can add to your module search path with the
+#    following line in your CMakeLists.txt:
+#
+#      list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
+#
+#    Then add the following in your CMakeLists.txt:
+#
+#      include(OsmiumOptions)
+#      find_package(Osmium REQUIRED COMPONENTS <XXX>)
+#      include_directories(${OSMIUM_INCLUDE_DIRS})
+#
+#    For the <XXX> substitute a space separated list of one or more of the
+#    following components:
+#
+#      pbf        - include libraries needed for PBF input and output
+#      xml        - include libraries needed for XML input and output
+#      io         - include libraries needed for any type of input/output
+#      geos       - include if you want to use any of the GEOS functions
+#      gdal       - include if you want to use any of the OGR functions
+#      proj       - include if you want to use any of the Proj.4 functions
+#      sparsehash - include if you use the sparsehash index
+#
+#    You can check for success with something like this:
+#
+#      if(NOT OSMIUM_FOUND)
+#          message(WARNING "Libosmium not found!\n")
+#      endif()
+#
+#----------------------------------------------------------------------
+#
+#  Variables:
+#
+#    OSMIUM_FOUND         - True if Osmium found.
+#    OSMIUM_INCLUDE_DIRS  - Where to find include files.
+#    OSMIUM_XML_LIBRARIES - Libraries needed for XML I/O.
+#    OSMIUM_PBF_LIBRARIES - Libraries needed for PBF I/O.
+#    OSMIUM_IO_LIBRARIES  - Libraries needed for XML or PBF I/O.
+#    OSMIUM_LIBRARIES     - All libraries Osmium uses somewhere.
+#
+#----------------------------------------------------------------------
+
+# Look for the header file.
+find_path(OSMIUM_INCLUDE_DIR osmium/osm.hpp
+    PATH_SUFFIXES include
+    PATHS
+        ../libosmium
+        ../../libosmium
+        libosmium
+        ~/Library/Frameworks
+        /Library/Frameworks
+        /usr/local
+        /usr/
+        /opt/local # DarwinPorts
+        /opt
+)
+
+# Handle the QUIETLY and REQUIRED arguments and set OSMIUM_FOUND to TRUE if
+# all listed variables are TRUE.
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(OSMIUM REQUIRED_VARS OSMIUM_INCLUDE_DIR)
+
+# Copy the results to the output variables.
+if(OSMIUM_FOUND)
+    set(OSMIUM_INCLUDE_DIRS ${OSMIUM_INCLUDE_DIR})
+endif()
+
+if(Osmium_FIND_REQUIRED AND NOT OSMIUM_FOUND)
+    message(FATAL_ERROR "Can not find libosmium headers, please install them or configure the paths")
+endif()
+
+#----------------------------------------------------------------------
+#
+#  Check for optional components
+#
+#----------------------------------------------------------------------
+if(Osmium_FIND_COMPONENTS)
+    foreach(_component ${Osmium_FIND_COMPONENTS})
+        string(TOUPPER ${_component} _component_uppercase)
+        set(Osmium_USE_${_component_uppercase} TRUE)
+    endforeach()
+endif()
+
+#----------------------------------------------------------------------
+# Component 'io' is an alias for 'pbf' and 'xml'
+if(Osmium_USE_IO)
+    set(Osmium_USE_PBF TRUE)
+    set(Osmium_USE_XML TRUE)
+endif()
+
+#----------------------------------------------------------------------
+# Component 'ogr' is an alias for 'gdal'
+if(Osmium_USE_OGR)
+    set(Osmium_USE_GDAL TRUE)
+endif()
+
+#----------------------------------------------------------------------
+# Component 'pbf'
+if(Osmium_USE_PBF)
+    find_package(OSMPBF)
+    find_package(Protobuf)
+    find_package(ZLIB)
+    find_package(Threads)
+
+    if(OSMPBF_FOUND AND PROTOBUF_FOUND AND ZLIB_FOUND AND Threads_FOUND)
+        list(APPEND OSMIUM_PBF_LIBRARIES
+            ${OSMPBF_LIBRARIES}
+            ${PROTOBUF_LITE_LIBRARY}
+            ${ZLIB_LIBRARIES}
+            ${CMAKE_THREAD_LIBS_INIT}
+        )
+        if(WIN32)
+            list(APPEND OSMIUM_PBF_LIBRARIES ws2_32)
+        endif()
+        list(APPEND OSMIUM_INCLUDE_DIRS
+            ${OSMPBF_INCLUDE_DIRS}
+            ${PROTOBUF_INCLUDE_DIR}
+            ${ZLIB_INCLUDE_DIR}
+        )
+    else()
+        set(_missing_libraries 1)
+        message(WARNING "Osmium: Can not find some libraries for PBF input/output, please install them or configure the paths.")
+    endif()
+endif()
+
+#----------------------------------------------------------------------
+# Component 'xml'
+if(Osmium_USE_XML)
+    find_package(EXPAT)
+    find_package(BZip2)
+    find_package(ZLIB)
+    find_package(Threads)
+
+    if(EXPAT_FOUND AND BZIP2_FOUND AND ZLIB_FOUND AND Threads_FOUND)
+        list(APPEND OSMIUM_XML_LIBRARIES
+            ${EXPAT_LIBRARIES}
+            ${BZIP2_LIBRARIES}
+            ${ZLIB_LIBRARIES}
+            ${CMAKE_THREAD_LIBS_INIT}
+        )
+        list(APPEND OSMIUM_INCLUDE_DIRS
+            ${EXPAT_INCLUDE_DIR}
+            ${BZIP2_INCLUDE_DIR}
+            ${ZLIB_INCLUDE_DIR}
+        )
+    else()
+        set(_missing_libraries 1)
+        message(WARNING "Osmium: Can not find some libraries for XML input/output, please install them or configure the paths.")
+    endif()
+endif()
+
+#----------------------------------------------------------------------
+list(APPEND OSMIUM_IO_LIBRARIES
+    ${OSMIUM_PBF_LIBRARIES}
+    ${OSMIUM_XML_LIBRARIES}
+)
+
+list(APPEND OSMIUM_LIBRARIES
+    ${OSMIUM_IO_LIBRARIES}
+)
+
+#----------------------------------------------------------------------
+# Component 'geos'
+if(Osmium_USE_GEOS)
+    find_path(GEOS_INCLUDE_DIR geos/geom.h)
+    find_library(GEOS_LIBRARY NAMES geos)
+
+    if(GEOS_INCLUDE_DIR AND GEOS_LIBRARY)
+        SET(GEOS_FOUND 1)
+        list(APPEND OSMIUM_LIBRARIES ${GEOS_LIBRARY})
+        list(APPEND OSMIUM_INCLUDE_DIRS ${GEOS_INCLUDE_DIR})
+    else()
+        set(_missing_libraries 1)
+        message(WARNING "Osmium: GEOS library is required but not found, please install it or configure the paths.")
+    endif()
+endif()
+
+#----------------------------------------------------------------------
+# Component 'gdal' (alias 'ogr')
+if(Osmium_USE_GDAL)
+    find_package(GDAL)
+
+    if(GDAL_FOUND)
+        list(APPEND OSMIUM_LIBRARIES ${GDAL_LIBRARIES})
+        list(APPEND OSMIUM_INCLUDE_DIRS ${GDAL_INCLUDE_DIRS})
+    else()
+        set(_missing_libraries 1)
+        message(WARNING "Osmium: GDAL library is required but not found, please install it or configure the paths.")
+    endif()
+endif()
+
+#----------------------------------------------------------------------
+# Component 'proj'
+if(Osmium_USE_PROJ)
+    find_path(PROJ_INCLUDE_DIR proj_api.h)
+    find_library(PROJ_LIBRARY NAMES proj)
+
+    if(PROJ_INCLUDE_DIR AND PROJ_LIBRARY)
+        set(PROJ_FOUND 1)
+        list(APPEND OSMIUM_LIBRARIES ${PROJ_LIBRARY})
+        list(APPEND OSMIUM_INCLUDE_DIRS ${PROJ_INCLUDE_DIR})
+    else()
+        set(_missing_libraries 1)
+        message(WARNING "Osmium: PROJ.4 library is required but not found, please install it or configure the paths.")
+    endif()
+endif()
+
+#----------------------------------------------------------------------
+# Component 'sparsehash'
+if(Osmium_USE_SPARSEHASH)
+    find_path(SPARSEHASH_INCLUDE_DIR google/sparsetable)
+
+    if(SPARSEHASH_INCLUDE_DIR)
+        set(SPARSEHASH_FOUND 1)
+        list(APPEND OSMIUM_INCLUDE_DIRS ${SPARSEHASH_INCLUDE_DIR})
+    else()
+        set(_missing_libraries 1)
+        message(WARNING "Osmium: Google SparseHash library is required but not found, please install it or configure the paths.")
+    endif()
+endif()
+
+#----------------------------------------------------------------------
+
+list(REMOVE_DUPLICATES OSMIUM_INCLUDE_DIRS)
+list(REMOVE_DUPLICATES OSMIUM_XML_LIBRARIES)
+list(REMOVE_DUPLICATES OSMIUM_PBF_LIBRARIES)
+list(REMOVE_DUPLICATES OSMIUM_IO_LIBRARIES)
+list(REMOVE_DUPLICATES OSMIUM_LIBRARIES)
+
+#----------------------------------------------------------------------
+#
+#  Check that all required libraries are available
+#
+#----------------------------------------------------------------------
+if(Osmium_FIND_REQUIRED AND _missing_libraries)
+    message(FATAL_ERROR "Required library or libraries missing. Aborting.")
+endif()
+
+#----------------------------------------------------------------------
+#
+#  Add compiler flags
+#
+#----------------------------------------------------------------------
+add_definitions(-D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64)
+
+if(MSVC)
+    add_definitions(-wd4996 -DNOMINMAX -DWIN32_LEAN_AND_MEAN -D_CRT_SECURE_NO_WARNINGS)
+endif()
+
+if(APPLE)
+# following only available from cmake 2.8.12:
+#   add_compile_options(-stdlib=libc++)
+# so using this instead:
+    add_definitions(-stdlib=libc++)
+    set(LDFLAGS ${LDFLAGS} -stdlib=libc++)
+endif()
+
+#----------------------------------------------------------------------
+
+# This is a set of recommended warning options that can be added when compiling
+# libosmium code.
+set(OSMIUM_WARNING_OPTIONS "-Wall -Wextra -pedantic -Wredundant-decls -Wdisabled-optimization -Wctor-dtor-privacy -Wnon-virtual-dtor -Woverloaded-virtual -Wsign-promo -Wold-style-cast -Wno-return-type" CACHE STRING "Recommended warning options for libosmium")
+
+set(OSMIUM_DRACONIC_CLANG_OPTIONS "-Wdocumentation -Wunused-exception-parameter -Wmissing-declarations -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-unused-macros -Wno-exit-time-destructors -Wno-global-constructors -Wno-padded -Wno-switch-enum -Wno-missing-prototypes -Wno-weak-vtables -Wno-cast-align -Wno-float-equal")
+
+if(Osmium_DEBUG)
+    message(STATUS "OSMIUM_XML_LIBRARIES=" ${OSMIUM_XML_LIBRARIES})
+    message(STATUS "OSMIUM_PBF_LIBRARIES=" ${OSMIUM_PBF_LIBRARIES})
+    message(STATUS "OSMIUM_IO_LIBRARIES=" ${OSMIUM_IO_LIBRARIES})
+    message(STATUS "OSMIUM_LIBRARIES=" ${OSMIUM_LIBRARIES})
+    message(STATUS "OSMIUM_INCLUDE_DIRS=" ${OSMIUM_INCLUDE_DIRS})
+endif()
+

cmake/build.bat

@@ -0,0 +1,15 @@
+call "%VS120COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64
+set CMAKE_PREFIX_PATH=C:\PROJ
+set VERSION=Debug
+set TESTS=ON
+set ALLHPPS=ON
+set PREFIX=d:\libs18d
+set BOOST_ROOT=d:\boost
+
+cmake .. -G "Visual Studio 12 Win64" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=%PREFIX% -DBOOST_ROOT=%BOOST_ROOT% -DBoost_USE_STATIC_LIBS=ON -DBUILD_TESTING=%TESTS% -DBUILD_TRY_HPPS=%ALLHPPS$ -T CTP_Nov2013
+msbuild /clp:Verbosity=minimal /nologo libosmium.sln /flp1:logfile=build_errors.txt;errorsonly /flp2:logfile=build_warnings.txt;warningsonly
+set PATH=%PATH%;%PREFIX%/bin
+
+del test\osm-testdata\*.db
+del test\osm-testdata\*.json
+if "%TESTS%"=="ON" ctest -VV >build_tests.log

cmake/iwyu.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+#
+# This will run IWYU (Include What You Use) on includes files. The iwyu
+# program isn't very reliable and crashes often, but is still useful.
+#
+# TODO: This script should be integrated with cmake in some way...
+#
+
+cmdline="iwyu -Xiwyu --mapping_file=osmium.imp -std=c++11 -I include"
+
+log=build/iwyu.log
+
+echo "INCLUDE WHAT YOU USE REPORT:" >$log
+
+allok=yes
+
+mkdir -p build/check_reports
+
+for file in `find include/osmium -name \*.hpp`; do
+    mkdir -p `dirname build/check_reports/$file`
+    ifile="build/check_reports/${file%.hpp}.iwyu"
+    $cmdline $file >$ifile 2>&1
+    if grep -q 'has correct #includes/fwd-decls' ${ifile}; then
+        echo "\n\033[1m\033[32m========\033[0m \033[1m${file}\033[0m" >>$log
+        echo "[OK] ${file}"
+    elif grep -q 'Assertion failed' ${ifile}; then
+        echo "\n\033[1m======== ${file}\033[0m" >>$log
+        echo "[--] ${file}"
+        allok=no
+    else
+        echo "\n\033[1m\033[31m========\033[0m \033[1m${file}\033[0m" >>$log
+        echo "[  ] ${file}"
+        allok=no
+    fi
+    cat $ifile >>$log
+done
+
+if [ "$allok" = "yes" ]; then
+    echo "All files OK"
+else
+    echo "There were errors"
+fi
+

doc/doc.txt

@@ -0,0 +1,26 @@
+
+/**
+ * @mainpage
+ *
+ * Osmium is a fast and flexible C++ library for working with OpenStreetMap
+ * data.
+ *
+ * This is the API documentation that was automatically created from the
+ * source code. For more information about the Osmium Library see
+ * http://osmcode.org/libosmium .
+ *
+ * Osmium is free software and available under the Boost Software License.
+ * The source code is available at https://github.com/osmcode/libosmium .
+ *
+ * Osmium is a header-only library. You do not need to compile and link it,
+ * just include the headers you need.
+ *
+ * Everything in namespaces called "detail" is for internal Osmium use only,
+ * do not depend on it in your code. Do not include any include files in
+ * directories named "detail" directly. Include files in directories called
+ * "experimental" and everything in namespaces called "experimental" is
+ * unsupported and may change at any time regardless of the status of the rest
+ * of the library.
+ *
+ */
+

doc/osmium.css

@@ -0,0 +1,22 @@
+
+body {
+    font-family: "Droid Sans",Helvetica,Arial,sans-serif;
+    background-color: #ffffff;
+    color: #202060;
+}
+
+.tabs, .tabs2, .tabs3, .navpath ul, .tablist li {
+    background-image: none;
+}
+
+.tabs, .tabs2, .tabs3 {
+    border-top: 1px solid #202060;
+}
+
+div.contents {
+    margin: 0px;
+    padding-top: 10px;
+    padding-left: 12px;
+    padding-right: 8px;
+}
+

examples/CMakeLists.txt

@@ -0,0 +1,64 @@
+message(STATUS "Configuring examples...")
+
+if(GETOPT_MISSING)
+    message(FATAL_ERROR "On Visual Studio the wingetopt library is needed to compile examples")
+endif()
+
+set(SIMPLE_EXAMPLES
+    osmium_convert
+    osmium_count
+    osmium_create_node_cache
+    osmium_debug
+    osmium_read
+    osmium_serdump
+    osmium_use_node_cache
+)
+
+if(SPARSEHASH_FOUND)
+    list(APPEND SIMPLE_EXAMPLES osmium_area_test)
+else()
+    message("Skipped osmium_area_test example - needed Google SparseHash")
+endif()
+
+# In 'Dev' mode: compile with very strict warnings and turn them into errors.
+if(CMAKE_BUILD_TYPE STREQUAL "Dev")
+    add_definitions(-Werror ${OSMIUM_WARNING_OPTIONS})
+endif()
+
+foreach(example ${SIMPLE_EXAMPLES})
+    message(STATUS "Configuring example ${example}...")
+    add_executable(${example} "${example}.cpp")
+    target_link_libraries(${example} ${OSMIUM_LIBRARIES})
+endforeach(example)
+
+unset(Boost_LIBRARIES)
+unset(Boost_FOUND)
+find_package(Boost 1.38 COMPONENTS program_options)
+
+if(Boost_PROGRAM_OPTIONS_FOUND)
+    set(example
+        osmium_index
+    )
+    message(STATUS "Configuring example ${example}...")
+    add_executable(${example} "${example}.cpp")
+    target_link_libraries(${example} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${OSMIUM_LIBRARIES})
+else()
+    message("Can not build osmium_index example without Boost program_options")
+endif()
+
+set(OGR_EXAMPLES
+    osmium_toogr
+    osmium_toogr2
+    osmium_toogr2_exp
+)
+
+if(GDAL_FOUND AND GEOS_FOUND AND SPARSEHASH_FOUND)
+    foreach(example ${OGR_EXAMPLES})
+        message(STATUS "Configuring example ${example}...")
+        add_executable(${example} "${example}.cpp")
+        target_link_libraries(${example} ${OSMIUM_LIBRARIES})
+    endforeach(example)
+else()
+    message("GDAL+GEOS+SparseHash not found, skipping OGR examples")
+endif()
+

examples/osmium_area_test.cpp

@@ -0,0 +1,138 @@
+/*
+
+  This is an example tool that creates multipolygons from OSM data
+  and dumps them to stdout.
+
+  The code in this example file is released into the Public Domain.
+
+*/
+
+#include <iostream>
+
+#include <getopt.h>
+
+#include <osmium/area/assembler.hpp>
+#include <osmium/area/multipolygon_collector.hpp>
+#include <osmium/dynamic_handler.hpp>
+#include <osmium/geom/wkt.hpp>
+#include <osmium/handler/dump.hpp>
+#include <osmium/handler/node_locations_for_ways.hpp>
+#include <osmium/index/map/dummy.hpp>
+#include <osmium/index/map/sparse_table.hpp>
+#include <osmium/io/any_input.hpp>
+#include <osmium/visitor.hpp>
+
+typedef osmium::index::map::Dummy<osmium::unsigned_object_id_type, osmium::Location> index_neg_type;
+typedef osmium::index::map::SparseTable<osmium::unsigned_object_id_type, osmium::Location> index_pos_type;
+typedef osmium::handler::NodeLocationsForWays<index_pos_type, index_neg_type> location_handler_type;
+
+class WKTDump : public osmium::handler::Handler {
+
+    osmium::geom::WKTFactory<> m_factory ;
+
+    std::ostream& m_out;
+
+public:
+
+    WKTDump(std::ostream& out) :
+        m_out(out) {
+    }
+
+    void area(const osmium::Area& area) {
+        try {
+            m_out << m_factory.create_multipolygon(area) << "\n";
+        } catch (osmium::geometry_error& e) {
+            m_out << "GEOMETRY ERROR: " << e.what() << "\n";
+        }
+    }
+
+}; // class WKTDump
+
+void print_help() {
+    std::cout << "osmium_area_test [OPTIONS] OSMFILE\n\n"
+              << "Read OSMFILE and build multipolygons from it.\n"
+              << "\nOptions:\n"
+              << "  -h, --help           This help message\n"
+              << "  -w, --dump-wkt       Dump area geometries as WKT\n"
+              << "  -o, --dump-objects   Dump area objects\n";
+}
+
+int main(int argc, char* argv[]) {
+    static struct option long_options[] = {
+        {"help",         no_argument, 0, 'h'},
+        {"dump-wkt",     no_argument, 0, 'w'},
+        {"dump-objects", no_argument, 0, 'o'},
+        {0, 0, 0, 0}
+    };
+
+    osmium::handler::DynamicHandler handler;
+
+    while (true) {
+        int c = getopt_long(argc, argv, "hwo", long_options, 0);
+        if (c == -1) {
+            break;
+        }
+
+        switch (c) {
+            case 'h':
+                print_help();
+                exit(0);
+            case 'w':
+                handler.set<WKTDump>(std::cout);
+                break;
+            case 'o':
+                handler.set<osmium::handler::Dump>(std::cout);
+                break;
+            default:
+                exit(1);
+        }
+    }
+
+    int remaining_args = argc - optind;
+    if (remaining_args != 1) {
+        std::cerr << "Usage: " << argv[0] << " [OPTIONS] OSMFILE\n";
+        exit(1);
+    }
+
+    osmium::io::File infile(argv[optind]);
+
+    osmium::area::Assembler::config_type assembler_config;
+    osmium::area::MultipolygonCollector<osmium::area::Assembler> collector(assembler_config);
+
+    std::cout << "Pass 1...\n";
+    osmium::io::Reader reader1(infile, osmium::osm_entity_bits::relation);
+    collector.read_relations(reader1);
+    reader1.close();
+    std::cout << "Pass 1 done\n";
+
+    std::cout << "Memory:\n";
+    collector.used_memory();
+
+    index_pos_type index_pos;
+    index_neg_type index_neg;
+    location_handler_type location_handler(index_pos, index_neg);
+    location_handler.ignore_errors(); // XXX
+
+    std::cout << "Pass 2...\n";
+    osmium::io::Reader reader2(infile);
+    osmium::apply(reader2, location_handler, collector.handler([&handler](osmium::memory::Buffer&& buffer) {
+        osmium::apply(buffer, handler);
+    }));
+    reader2.close();
+    std::cout << "Pass 2 done\n";
+
+    std::cout << "Memory:\n";
+    collector.used_memory();
+
+    std::vector<const osmium::Relation*> incomplete_relations = collector.get_incomplete_relations();
+    if (!incomplete_relations.empty()) {
+        std::cerr << "Warning! Some member ways missing for these multipolygon relations:";
+        for (const auto* relation : incomplete_relations) {
+            std::cerr << " " << relation->id();
+        }
+        std::cerr << "\n";
+    }
+
+    google::protobuf::ShutdownProtobufLibrary();
+}
+

examples/osmium_convert.cpp

@@ -0,0 +1,112 @@
+/*
+
+  Convert OSM files from one format into another.
+
+  The code in this example file is released into the Public Domain.
+
+*/
+
+#include <iostream>
+#include <getopt.h>
+
+#include <osmium/io/any_input.hpp>
+
+#include <osmium/io/any_output.hpp>
+
+void print_help() {
+    std::cout << "osmium_convert [OPTIONS] [INFILE [OUTFILE]]\n\n" \
+              << "If INFILE or OUTFILE is not given stdin/stdout is assumed.\n" \
+              << "File format is autodetected from file name suffix.\n" \
+              << "Use -f and -t options to force file format.\n" \
+              << "\nFile types:\n" \
+              << "  osm        normal OSM file\n" \
+              << "  osc        OSM change file\n" \
+              << "  osh        OSM file with history information\n" \
+              << "\nFile format:\n" \
+              << "  (default)  XML encoding\n" \
+              << "  pbf        binary PBF encoding\n" \
+              << "  opl        OPL encoding\n" \
+              << "\nFile compression\n" \
+              << "  gz         compressed with gzip\n" \
+              << "  bz2        compressed with bzip2\n" \
+              << "\nOptions:\n" \
+              << "  -h, --help                This help message\n" \
+              << "  -f, --from-format=FORMAT  Input format\n" \
+              << "  -t, --to-format=FORMAT    Output format\n";
+}
+
+int main(int argc, char* argv[]) {
+    static struct option long_options[] = {
+        {"help",        no_argument, 0, 'h'},
+        {"from-format", required_argument, 0, 'f'},
+        {"to-format",   required_argument, 0, 't'},
+        {0, 0, 0, 0}
+    };
+
+    std::string input_format;
+    std::string output_format;
+
+    while (true) {
+        int c = getopt_long(argc, argv, "dhf:t:", long_options, 0);
+        if (c == -1) {
+            break;
+        }
+
+        switch (c) {
+            case 'h':
+                print_help();
+                exit(0);
+            case 'f':
+                input_format = optarg;
+                break;
+            case 't':
+                output_format = optarg;
+                break;
+            default:
+                exit(1);
+        }
+    }
+
+    std::string input;
+    std::string output;
+    int remaining_args = argc - optind;
+    if (remaining_args > 2) {
+        std::cerr << "Usage: " << argv[0] << " [OPTIONS] [INFILE [OUTFILE]]" << std::endl;
+        exit(1);
+    } else if (remaining_args == 2) {
+        input =  argv[optind];
+        output = argv[optind+1];
+    } else if (remaining_args == 1) {
+        input =  argv[optind];
+    }
+
+    osmium::io::File infile(input, input_format);
+
+    osmium::io::File outfile(output, output_format);
+
+    if (infile.has_multiple_object_versions() && !outfile.has_multiple_object_versions()) {
+        std::cerr << "Warning! You are converting from an OSM file with (potentially) several versions of the same object to one that is not marked as such.\n";
+    }
+
+    int exit_code = 0;
+
+    try {
+        osmium::io::Reader reader(infile);
+        osmium::io::Header header = reader.header();
+        header.set("generator", "osmium_convert");
+
+        osmium::io::Writer writer(outfile, header, osmium::io::overwrite::allow);
+        while (osmium::memory::Buffer buffer = reader.read()) {
+            writer(std::move(buffer));
+        }
+        writer.close();
+        reader.close();
+    } catch (std::exception& e) {
+        std::cerr << e.what() << "\n";
+        exit_code = 1;
+    }
+
+    google::protobuf::ShutdownProtobufLibrary();
+    return exit_code;
+}
+

examples/osmium_count.cpp

@@ -0,0 +1,57 @@
+/*
+
+  This is a small tool that counts the number of nodes, ways, and relations in
+  the input file.
+
+  The code in this example file is released into the Public Domain.
+
+*/
+
+#include <iostream>
+
+#include <osmium/io/any_input.hpp>
+#include <osmium/handler.hpp>
+#include <osmium/visitor.hpp>
+
+struct CountHandler : public osmium::handler::Handler {
+
+    int nodes = 0;
+    int ways = 0;
+    int relations = 0;
+
+    void node(osmium::Node&) {
+        ++nodes;
+    }
+
+    void way(osmium::Way&) {
+        ++ways;
+    }
+
+    void relation(osmium::Relation&) {
+        ++relations;
+    }
+
+};
+
+
+int main(int argc, char* argv[]) {
+
+    if (argc != 2) {
+        std::cerr << "Usage: " << argv[0] << " OSMFILE\n";
+        exit(1);
+    }
+
+    osmium::io::File infile(argv[1]);
+    osmium::io::Reader reader(infile);
+
+    CountHandler handler;
+    osmium::apply(reader, handler);
+    reader.close();
+
+    std::cout << "Nodes: "     << handler.nodes << "\n";
+    std::cout << "Ways: "      << handler.ways << "\n";
+    std::cout << "Relations: " << handler.relations << "\n";
+
+    google::protobuf::ShutdownProtobufLibrary();
+}
+

examples/osmium_create_node_cache.cpp

@@ -0,0 +1,58 @@
+/*
+
+  This reads an OSM file and writes out the node locations to a cache
+  file.
+
+  The code in this example file is released into the Public Domain.
+
+*/
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <iostream>
+
+#include <osmium/io/pbf_input.hpp>
+#include <osmium/io/xml_input.hpp>
+
+#include <osmium/index/map/dummy.hpp>
+#include <osmium/index/map/mmap_vector_anon.hpp>
+#include <osmium/index/map/mmap_vector_file.hpp>
+
+#include <osmium/handler/node_locations_for_ways.hpp>
+#include <osmium/visitor.hpp>
+
+typedef osmium::index::map::Dummy<osmium::unsigned_object_id_type, osmium::Location> index_neg_type;
+//typedef osmium::index::map::DenseMapMmap<osmium::unsigned_object_id_type, osmium::Location> index_pos_type;
+typedef osmium::index::map::DenseMapFile<osmium::unsigned_object_id_type, osmium::Location> index_pos_type;
+
+typedef osmium::handler::NodeLocationsForWays<index_pos_type, index_neg_type> location_handler_type;
+
+int main(int argc, char* argv[]) {
+    if (argc != 3) {
+        std::cerr << "Usage: " << argv[0] << " OSM_FILE CACHE_FILE\n";
+        return 1;
+    }
+
+    std::string input_filename(argv[1]);
+    osmium::io::Reader reader(input_filename, osmium::osm_entity_bits::node);
+
+    int fd = open(argv[2], O_RDWR | O_CREAT, 0666);
+    if (fd == -1) {
+        std::cerr << "Can not open node cache file '" << argv[2] << "': " << strerror(errno) << "\n";
+        return 1;
+    }
+
+    index_pos_type index_pos {fd};
+    index_neg_type index_neg;
+    location_handler_type location_handler(index_pos, index_neg);
+    location_handler.ignore_errors();
+
+    osmium::apply(reader, location_handler);
+    reader.close();
+
+    google::protobuf::ShutdownProtobufLibrary();
+
+    return 0;
+}
+

examples/osmium_debug.cpp

@@ -0,0 +1,52 @@
+/*
+
+  This is a small tool to dump the contents of the input file.
+
+  The code in this example file is released into the Public Domain.
+
+*/
+
+#include <iostream>
+
+#include <osmium/handler/dump.hpp>
+#include <osmium/io/any_input.hpp>
+
+int main(int argc, char* argv[]) {
+    std::ios_base::sync_with_stdio(false);
+
+    if (argc < 2 || argc > 3) {
+        std::cerr << "Usage: " << argv[0] << " OSMFILE [TYPES]\n";
+        std::cerr << "TYPES can be any combination of 'n', 'w', 'r', and 'c' to indicate what types of OSM entities you want (default: all).\n";
+        exit(1);
+    }
+
+    osmium::osm_entity_bits::type read_types = osmium::osm_entity_bits::all;
+
+    if (argc == 3) {
+        read_types = osmium::osm_entity_bits::nothing;
+        std::string types = argv[2];
+        if (types.find('n') != std::string::npos) read_types |= osmium::osm_entity_bits::node;
+        if (types.find('w') != std::string::npos) read_types |= osmium::osm_entity_bits::way;
+        if (types.find('r') != std::string::npos) read_types |= osmium::osm_entity_bits::relation;
+        if (types.find('c') != std::string::npos) read_types |= osmium::osm_entity_bits::changeset;
+    }
+
+    osmium::io::Reader reader(argv[1], read_types);
+    osmium::io::Header header = reader.header();
+
+    std::cout << "HEADER:\n  generator=" << header.get("generator") << "\n";
+
+    for (auto& bbox : header.boxes()) {
+        std::cout << "  bbox=" << bbox << "\n";
+    }
+
+    osmium::handler::Dump dump(std::cout);
+    while (osmium::memory::Buffer buffer = reader.read()) {
+        osmium::apply(buffer, dump);
+    }
+
+    reader.close();
+
+    google::protobuf::ShutdownProtobufLibrary();
+}
+

examples/osmium_index.cpp

@@ -0,0 +1,236 @@
+/*
+
+  Example program to look at Osmium indexes on disk.
+
+  The code in this example file is released into the Public Domain.
+
+*/
+
+#include <fcntl.h>
+#include <iomanip>
+#include <iostream>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include <boost/program_options.hpp>
+
+#include <osmium/index/map/mmap_vector_file.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/types.hpp>
+
+template <typename TKey, typename TValue>
+class IndexSearch {
+
+    typedef typename osmium::index::map::DenseMapFile<TKey, TValue> dense_index_type;
+    typedef typename osmium::index::map::SparseMapFile<TKey, TValue> sparse_index_type;
+
+    int m_fd;
+    bool m_dense_format;
+
+    void dump_dense() {
+        dense_index_type index(m_fd);
+
+        for (size_t i = 0; i < index.size(); ++i) {
+            if (index.get(i) != TValue()) {
+                std::cout << i << " " << index.get(i) << "\n";
+            }
+        }
+    }
+
+    void dump_sparse() {
+        sparse_index_type index(m_fd);
+
+        for (auto& element : index) {
+            std::cout << element.first << " " << element.second << "\n";
+        }
+    }
+
+    bool search_dense(TKey key) {
+        dense_index_type index(m_fd);
+
+        try {
+            TValue value = index.get(key);
+            std::cout << key << " " << value << std::endl;
+        } catch (...) {
+            std::cout << key << " not found" << std::endl;
+            return false;
+        }
+
+        return true;
+    }
+
+    bool search_sparse(TKey key) {
+        typedef typename sparse_index_type::element_type element_type;
+        sparse_index_type index(m_fd);
+
+        element_type elem {key, TValue()};
+        auto positions = std::equal_range(index.begin(), index.end(), elem, [](const element_type& lhs, const element_type& rhs) {
+            return lhs.first < rhs.first;
+        });
+        if (positions.first == positions.second) {
+            std::cout << key << " not found" << std::endl;
+            return false;
+        }
+
+        for (auto& it = positions.first; it != positions.second; ++it) {
+            std::cout << it->first << " " << it->second << "\n";
+        }
+
+        return true;
+    }
+
+public:
+
+    IndexSearch(int fd, bool dense_format) :
+        m_fd(fd),
+        m_dense_format(dense_format) {
+    }
+
+    void dump() {
+        if (m_dense_format) {
+            dump_dense();
+        } else {
+            dump_sparse();
+        }
+    }
+
+    bool search(TKey key) {
+        if (m_dense_format) {
+            return search_dense(key);
+        } else {
+            return search_sparse(key);
+        }
+    }
+
+    bool search(std::vector<TKey> keys) {
+        bool found_all = true;
+
+        for (const auto key : keys) {
+            if (!search(key)) {
+                found_all = false;
+            }
+        }
+
+        return found_all;
+    }
+
+}; // class IndexSearch
+
+enum return_code : int {
+    okay      = 0,
+    not_found = 1,
+    error     = 2,
+    fatal     = 3
+};
+
+namespace po = boost::program_options;
+
+class Options {
+
+    po::variables_map vm;
+
+public:
+
+    Options(int argc, char* argv[]) {
+        try {
+            po::options_description desc("Allowed options");
+            desc.add_options()
+                ("help,h", "Print this help message")
+                ("array,a", po::value<std::string>(), "Read given index file in array format")
+                ("list,l", po::value<std::string>(), "Read given index file in list format")
+                ("dump,d", "Dump contents of index file to STDOUT")
+                ("search,s", po::value<std::vector<osmium::unsigned_object_id_type>>(), "Search for given id (Option can appear multiple times)")
+                ("type,t", po::value<std::string>(), "Type of value ('location' or 'offset')")
+            ;
+
+            po::store(po::parse_command_line(argc, argv, desc), vm);
+            po::notify(vm);
+
+            if (vm.count("help")) {
+                std::cout << desc << "\n";
+                exit(return_code::okay);
+            }
+
+            if (vm.count("array") && vm.count("list")) {
+                std::cerr << "Only option --array or --list allowed." << std::endl;
+                exit(return_code::fatal);
+            }
+
+            if (!vm.count("array") && !vm.count("list")) {
+                std::cerr << "Need one of option --array or --list." << std::endl;
+                exit(return_code::fatal);
+            }
+
+            if (!vm.count("type")) {
+                std::cerr << "Need --type argument." << std::endl;
+                exit(return_code::fatal);
+            }
+
+            const std::string& type = vm["type"].as<std::string>();
+            if (type != "location" && type != "offset") {
+                std::cerr << "Unknown type '" << type << "'. Must be 'location' or 'offset'." << std::endl;
+                exit(return_code::fatal);
+            }
+        } catch (boost::program_options::error& e) {
+            std::cerr << "Error parsing command line: " << e.what() << std::endl;
+            exit(return_code::fatal);
+        }
+    }
+
+    const std::string& filename() const {
+        if (vm.count("array")) {
+            return vm["array"].as<std::string>();
+        } else {
+            return vm["list"].as<std::string>();
+        }
+    }
+
+    bool dense_format() const {
+        return vm.count("array");
+    }
+
+    bool do_dump() const {
+        return vm.count("dump");
+    }
+
+    std::vector<osmium::unsigned_object_id_type> search_keys() const {
+        return vm["search"].as<std::vector<osmium::unsigned_object_id_type>>();
+    }
+
+    bool type_is(const char* type) const {
+        return vm["type"].as<std::string>() == type;
+    }
+
+}; // class Options
+
+int main(int argc, char* argv[]) {
+    std::ios_base::sync_with_stdio(false);
+
+    Options options(argc, argv);
+
+    std::cout << std::fixed << std::setprecision(7);
+    int fd = open(options.filename().c_str(), O_RDWR);
+
+    bool result_okay = true;
+
+    if (options.type_is("location")) {
+        IndexSearch<osmium::unsigned_object_id_type, osmium::Location> is(fd, options.dense_format());
+
+        if (options.do_dump()) {
+            is.dump();
+        } else {
+            result_okay = is.search(options.search_keys());
+        }
+    } else {
+        IndexSearch<osmium::unsigned_object_id_type, size_t> is(fd, options.dense_format());
+
+        if (options.do_dump()) {
+            is.dump();
+        } else {
+            result_okay = is.search(options.search_keys());
+        }
+    }
+
+    exit(result_okay ? return_code::okay : return_code::not_found);
+}
+

examples/osmium_read.cpp

@@ -0,0 +1,32 @@
+/*
+
+  This is a small tool that reads and discards the contents of the input file.
+  (Used for timing.)
+
+  The code in this example file is released into the Public Domain.
+
+*/
+
+#include <iostream>
+
+#include <osmium/io/any_input.hpp>
+
+int main(int argc, char* argv[]) {
+
+    if (argc != 2) {
+        std::cerr << "Usage: " << argv[0] << " OSMFILE\n";
+        exit(1);
+    }
+
+    osmium::io::File infile(argv[1]);
+    osmium::io::Reader reader(infile);
+
+    while (osmium::memory::Buffer buffer = reader.read()) {
+        // do nothing
+    }
+
+    reader.close();
+
+    google::protobuf::ShutdownProtobufLibrary();
+}
+

examples/osmium_serdump.cpp

@@ -0,0 +1,210 @@
+/*
+
+  This is a small tool to dump the contents of the input file
+  in serialized format to stdout.
+
+  The code in this example file is released into the Public Domain.
+
+*/
+
+#include <cerrno>
+#include <cstring>
+#include <getopt.h>
+#include <iostream>
+#include <string>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#ifdef _MSC_VER
+# include <direct.h>
+#endif
+
+#include <osmium/io/pbf_input.hpp>
+#include <osmium/io/xml_input.hpp>
+#include <osmium/handler/disk_store.hpp>
+#include <osmium/handler/object_relations.hpp>
+
+#include <osmium/index/map/stl_vector.hpp>
+#include <osmium/index/multimap/stl_multimap.hpp>
+#include <osmium/index/multimap/stl_vector.hpp>
+#include <osmium/index/multimap/hybrid.hpp>
+
+// ==============================================================================
+// Choose the following depending on the size of the input OSM files:
+// ==============================================================================
+// for smaller OSM files (extracts)
+typedef osmium::index::map::SparseMapMem<osmium::unsigned_object_id_type, size_t> offset_index_type;
+//typedef osmium::index::map::SparseMapMmap<osmium::unsigned_object_id_type, size_t> offset_index_type;
+//typedef osmium::index::map::SparseMapFile<osmium::unsigned_object_id_type, size_t> offset_index_type;
+
+typedef osmium::index::multimap::SparseMultimapMem<osmium::unsigned_object_id_type, osmium::unsigned_object_id_type> map_type;
+//typedef osmium::index::multimap::StlMultimap<osmium::unsigned_object_id_type, osmium::unsigned_object_id_type> map_type;
+//typedef osmium::index::multimap::Hybrid<osmium::unsigned_object_id_type, osmium::unsigned_object_id_type> map_type;
+
+// ==============================================================================
+// for very large OSM files (planet)
+//typedef osmium::index::map::DenseMapMmap<osmium::unsigned_object_id_type, size_t> offset_index_type;
+//typedef osmium::index::map::DenseMapMem map_type;
+// ==============================================================================
+
+void print_help() {
+    std::cout << "osmium_serdump OSMFILE DIR\n" \
+              << "Serialize content of OSMFILE into data file in DIR.\n" \
+              << "\nOptions:\n" \
+              << "  -h, --help       This help message\n";
+}
+
+int main(int argc, char* argv[]) {
+    std::ios_base::sync_with_stdio(false);
+
+    static struct option long_options[] = {
+        {"help",      no_argument, 0, 'h'},
+        {0, 0, 0, 0}
+    };
+
+    while (true) {
+        int c = getopt_long(argc, argv, "h", long_options, 0);
+        if (c == -1) {
+            break;
+        }
+
+        switch (c) {
+            case 'h':
+                print_help();
+                exit(0);
+            default:
+                exit(2);
+        }
+    }
+
+    int remaining_args = argc - optind;
+
+    if (remaining_args != 2) {
+        std::cerr << "Usage: " << argv[0] << " OSMFILE DIR\n";
+        exit(2);
+    }
+
+    std::string dir(argv[optind+1]);
+#ifndef _WIN32
+    int result = ::mkdir(dir.c_str(), 0777);
+#else
+    int result = mkdir(dir.c_str());
+#endif
+    if (result == -1 && errno != EEXIST) {
+        std::cerr << "Problem creating directory '" << dir << "': " << strerror(errno) << "\n";
+        exit(2);
+    }
+
+    std::string data_file(dir + "/data.osm.ser");
+    int data_fd = ::open(data_file.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
+    if (data_fd < 0) {
+        std::cerr << "Can't open data file '" << data_file << "': " << strerror(errno) << "\n";
+        exit(2);
+    }
+
+    offset_index_type node_index;
+    offset_index_type way_index;
+    offset_index_type relation_index;
+
+    osmium::handler::DiskStore disk_store_handler(data_fd, node_index, way_index, relation_index);
+
+    map_type map_node2way;
+    map_type map_node2relation;
+    map_type map_way2relation;
+    map_type map_relation2relation;
+
+    osmium::handler::ObjectRelations object_relations_handler(map_node2way, map_node2relation, map_way2relation, map_relation2relation);
+
+    osmium::io::Reader reader(argv[1]);
+
+    while (osmium::memory::Buffer buffer = reader.read()) {
+        disk_store_handler(buffer); // XXX
+        osmium::apply(buffer, object_relations_handler);
+    }
+
+    reader.close();
+
+    {
+        std::string index_file(dir + "/nodes.idx");
+        int fd = ::open(index_file.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
+        if (fd < 0) {
+            std::cerr << "Can't open nodes index file '" << index_file << "': " << strerror(errno) << "\n";
+            exit(2);
+        }
+        node_index.dump_as_list(fd);
+        close(fd);
+    }
+
+    {
+        std::string index_file(dir + "/ways.idx");
+        int fd = ::open(index_file.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
+        if (fd < 0) {
+            std::cerr << "Can't open ways index file '" << index_file << "': " << strerror(errno) << "\n";
+            exit(2);
+        }
+        way_index.dump_as_list(fd);
+        close(fd);
+    }
+
+    {
+        std::string index_file(dir + "/relations.idx");
+        int fd = ::open(index_file.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
+        if (fd < 0) {
+            std::cerr << "Can't open relations index file '" << index_file << "': " << strerror(errno) << "\n";
+            exit(2);
+        }
+        relation_index.dump_as_list(fd);
+        close(fd);
+    }
+
+    {
+        map_node2way.sort();
+        std::string index_file(dir + "/node2way.map");
+        int fd = ::open(index_file.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
+        if (fd < 0) {
+            std::cerr << "Can't open node->way map file '" << index_file << "': " << strerror(errno) << "\n";
+            exit(2);
+        }
+        map_node2way.dump_as_list(fd);
+        close(fd);
+    }
+
+    {
+        map_node2relation.sort();
+        std::string index_file(dir + "/node2rel.map");
+        int fd = ::open(index_file.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
+        if (fd < 0) {
+            std::cerr << "Can't open node->rel map file '" << index_file << "': " << strerror(errno) << "\n";
+            exit(2);
+        }
+        map_node2relation.dump_as_list(fd);
+        close(fd);
+    }
+
+    {
+        map_way2relation.sort();
+        std::string index_file(dir + "/way2rel.map");
+        int fd = ::open(index_file.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
+        if (fd < 0) {
+            std::cerr << "Can't open way->rel map file '" << index_file << "': " << strerror(errno) << "\n";
+            exit(2);
+        }
+        map_way2relation.dump_as_list(fd);
+        close(fd);
+    }
+
+    {
+        map_relation2relation.sort();
+        std::string index_file(dir + "/rel2rel.map");
+        int fd = ::open(index_file.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
+        if (fd < 0) {
+            std::cerr << "Can't open rel->rel map file '" << index_file << "': " << strerror(errno) << "\n";
+            exit(2);
+        }
+        map_relation2relation.dump_as_list(fd);
+        close(fd);
+    }
+
+    google::protobuf::ShutdownProtobufLibrary();
+}
+

examples/osmium_toogr.cpp

@@ -0,0 +1,251 @@
+/*
+
+  This is an example tool that converts OSM data to some output format
+  like Spatialite or Shapefiles using the OGR library.
+
+  The code in this example file is released into the Public Domain.
+
+*/
+
+#include <iostream>
+#include <getopt.h>
+
+#pragma GCC diagnostic push
+#ifdef __clang__
+# pragma GCC diagnostic ignored "-Wdocumentation-unknown-command"
+#endif
+#pragma GCC diagnostic ignored "-Wfloat-equal"
+#pragma GCC diagnostic ignored "-Wold-style-cast"
+#pragma GCC diagnostic ignored "-Wpadded"
+#pragma GCC diagnostic ignored "-Wredundant-decls"
+#pragma GCC diagnostic ignored "-Wshadow"
+# include <ogr_api.h>
+# include <ogrsf_frmts.h>
+#pragma GCC diagnostic pop
+
+// usually you only need one or two of these
+#include <osmium/index/map/dummy.hpp>
+#include <osmium/index/map/sparse_table.hpp>
+#include <osmium/index/map/stl_map.hpp>
+#include <osmium/index/map/mmap_vector_anon.hpp>
+
+#include <osmium/handler/node_locations_for_ways.hpp>
+#include <osmium/visitor.hpp>
+
+#include <osmium/geom/ogr.hpp>
+#include <osmium/io/any_input.hpp>
+#include <osmium/handler.hpp>
+
+typedef osmium::index::map::Dummy<osmium::unsigned_object_id_type, osmium::Location> index_neg_type;
+
+//typedef osmium::index::map::StlMap<osmium::unsigned_object_id_type, osmium::Location> index_pos_type;
+//typedef osmium::index::map::SparseMapMmap<osmium::unsigned_object_id_type, osmium::Location> index_pos_type;
+typedef osmium::index::map::SparseTable<osmium::unsigned_object_id_type, osmium::Location> index_pos_type;
+
+typedef osmium::handler::NodeLocationsForWays<index_pos_type, index_neg_type> location_handler_type;
+
+class MyOGRHandler : public osmium::handler::Handler {
+
+    OGRDataSource* m_data_source;
+    OGRLayer* m_layer_point;
+    OGRLayer* m_layer_linestring;
+
+    osmium::geom::OGRFactory<> m_factory;
+
+public:
+
+    MyOGRHandler(const std::string& driver_name, const std::string& filename) {
+
+        OGRRegisterAll();
+
+        OGRSFDriver* driver = OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(driver_name.c_str());
+        if (!driver) {
+            std::cerr << driver_name << " driver not available.\n";
+            exit(1);
+        }
+
+        CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "FALSE");
+        const char* options[] = { "SPATIALITE=TRUE", nullptr };
+        m_data_source = driver->CreateDataSource(filename.c_str(), const_cast<char**>(options));
+        if (!m_data_source) {
+            std::cerr << "Creation of output file failed.\n";
+            exit(1);
+        }
+
+        OGRSpatialReference sparef;
+        sparef.SetWellKnownGeogCS("WGS84");
+        m_layer_point = m_data_source->CreateLayer("postboxes", &sparef, wkbPoint, nullptr);
+        if (!m_layer_point) {
+            std::cerr << "Layer creation failed.\n";
+            exit(1);
+        }
+
+        OGRFieldDefn layer_point_field_id("id", OFTReal);
+        layer_point_field_id.SetWidth(10);
+
+        if (m_layer_point->CreateField(&layer_point_field_id) != OGRERR_NONE) {
+            std::cerr << "Creating id field failed.\n";
+            exit(1);
+        }
+
+        OGRFieldDefn layer_point_field_operator("operator", OFTString);
+        layer_point_field_operator.SetWidth(30);
+
+        if (m_layer_point->CreateField(&layer_point_field_operator) != OGRERR_NONE) {
+            std::cerr << "Creating operator field failed.\n";
+            exit(1);
+        }
+
+        /* Transactions might make things faster, then again they might not.
+           Feel free to experiment and benchmark and report back. */
+        m_layer_point->StartTransaction();
+
+        m_layer_linestring = m_data_source->CreateLayer("roads", &sparef, wkbLineString, nullptr);
+        if (!m_layer_linestring) {
+            std::cerr << "Layer creation failed.\n";
+            exit(1);
+        }
+
+        OGRFieldDefn layer_linestring_field_id("id", OFTReal);
+        layer_linestring_field_id.SetWidth(10);
+
+        if (m_layer_linestring->CreateField(&layer_linestring_field_id) != OGRERR_NONE) {
+            std::cerr << "Creating id field failed.\n";
+            exit(1);
+        }
+
+        OGRFieldDefn layer_linestring_field_type("type", OFTString);
+        layer_linestring_field_type.SetWidth(30);
+
+        if (m_layer_linestring->CreateField(&layer_linestring_field_type) != OGRERR_NONE) {
+            std::cerr << "Creating type field failed.\n";
+            exit(1);
+        }
+
+        m_layer_linestring->StartTransaction();
+    }
+
+    ~MyOGRHandler() {
+        m_layer_linestring->CommitTransaction();
+        m_layer_point->CommitTransaction();
+        OGRDataSource::DestroyDataSource(m_data_source);
+        OGRCleanupAll();
+    }
+
+    void node(const osmium::Node& node) {
+        const char* amenity = node.tags().get_value_by_key("amenity");
+        if (amenity && !strcmp(amenity, "post_box")) {
+            OGRFeature* feature = OGRFeature::CreateFeature(m_layer_point->GetLayerDefn());
+            std::unique_ptr<OGRPoint> ogr_point = m_factory.create_point(node);
+            feature->SetGeometry(ogr_point.get());
+            feature->SetField("id", static_cast<double>(node.id()));
+            feature->SetField("operator", node.tags().get_value_by_key("operator"));
+
+            if (m_layer_point->CreateFeature(feature) != OGRERR_NONE) {
+                std::cerr << "Failed to create feature.\n";
+                exit(1);
+            }
+
+            OGRFeature::DestroyFeature(feature);
+        }
+    }
+
+    void way(const osmium::Way& way) {
+        const char* highway = way.tags().get_value_by_key("highway");
+        if (highway) {
+            try {
+                std::unique_ptr<OGRLineString> ogr_linestring = m_factory.create_linestring(way);
+                OGRFeature* feature = OGRFeature::CreateFeature(m_layer_linestring->GetLayerDefn());
+                feature->SetGeometry(ogr_linestring.get());
+                feature->SetField("id", static_cast<double>(way.id()));
+                feature->SetField("type", highway);
+
+                if (m_layer_linestring->CreateFeature(feature) != OGRERR_NONE) {
+                    std::cerr << "Failed to create feature.\n";
+                    exit(1);
+                }
+
+                OGRFeature::DestroyFeature(feature);
+            } catch (osmium::geometry_error&) {
+                std::cerr << "Ignoring illegal geometry for way " << way.id() << ".\n";
+            }
+        }
+    }
+
+};
+
+/* ================================================== */
+
+void print_help() {
+    std::cout << "osmium_toogr [OPTIONS] [INFILE [OUTFILE]]\n\n" \
+              << "If INFILE is not given stdin is assumed.\n" \
+              << "If OUTFILE is not given 'ogr_out' is used.\n" \
+              << "\nOptions:\n" \
+              << "  -h, --help           This help message\n" \
+              << "  -f, --format=FORMAT  Output OGR format (Default: 'SQLite')\n";
+}
+
+int main(int argc, char* argv[]) {
+    static struct option long_options[] = {
+        {"help",   no_argument, 0, 'h'},
+        {"format", required_argument, 0, 'f'},
+        {0, 0, 0, 0}
+    };
+
+    std::string output_format("SQLite");
+
+    while (true) {
+        int c = getopt_long(argc, argv, "hf:", long_options, 0);
+        if (c == -1) {
+            break;
+        }
+
+        switch (c) {
+            case 'h':
+                print_help();
+                exit(0);
+            case 'f':
+                output_format = optarg;
+                break;
+            default:
+                exit(1);
+        }
+    }
+
+    std::string input_filename;
+    std::string output_filename("ogr_out");
+    int remaining_args = argc - optind;
+    if (remaining_args > 2) {
+        std::cerr << "Usage: " << argv[0] << " [OPTIONS] [INFILE [OUTFILE]]" << std::endl;
+        exit(1);
+    } else if (remaining_args == 2) {
+        input_filename =  argv[optind];
+        output_filename = argv[optind+1];
+    } else if (remaining_args == 1) {
+        input_filename =  argv[optind];
+    } else {
+        input_filename = "-";
+    }
+
+    osmium::io::Reader reader(input_filename);
+
+    index_pos_type index_pos;
+    index_neg_type index_neg;
+    location_handler_type location_handler(index_pos, index_neg);
+    location_handler.ignore_errors();
+
+    MyOGRHandler ogr_handler(output_format, output_filename);
+
+    osmium::apply(reader, location_handler, ogr_handler);
+    reader.close();
+
+    google::protobuf::ShutdownProtobufLibrary();
+
+    int locations_fd = open("locations.dump", O_WRONLY | O_CREAT, 0644);
+    if (locations_fd < 0) {
+        throw std::system_error(errno, std::system_category(), "Open failed");
+    }
+    index_pos.dump_as_list(locations_fd);
+    close(locations_fd);
+}
+

examples/osmium_toogr2.cpp

@@ -0,0 +1,350 @@
+/*
+
+  This is an example tool that converts OSM data to some output format
+  like Spatialite or Shapefiles using the OGR library.
+
+  This version does multipolygon handling (in contrast to the osmium_toogr
+  example which doesn't).
+
+  The code in this example file is released into the Public Domain.
+
+*/
+
+#include <iostream>
+#include <getopt.h>
+
+#pragma GCC diagnostic push
+#ifdef __clang__
+# pragma GCC diagnostic ignored "-Wdocumentation-unknown-command"
+#endif
+#pragma GCC diagnostic ignored "-Wfloat-equal"
+#pragma GCC diagnostic ignored "-Wold-style-cast"
+#pragma GCC diagnostic ignored "-Wpadded"
+#pragma GCC diagnostic ignored "-Wredundant-decls"
+#pragma GCC diagnostic ignored "-Wshadow"
+# include <ogr_api.h>
+# include <ogrsf_frmts.h>
+#pragma GCC diagnostic pop
+
+// usually you only need one or two of these
+#include <osmium/index/map/dummy.hpp>
+#include <osmium/index/map/sparse_table.hpp>
+#include <osmium/index/map/stl_map.hpp>
+#include <osmium/index/map/mmap_vector_anon.hpp>
+
+#include <osmium/handler/node_locations_for_ways.hpp>
+#include <osmium/visitor.hpp>
+#include <osmium/area/multipolygon_collector.hpp>
+#include <osmium/area/assembler.hpp>
+
+#include <osmium/geom/mercator_projection.hpp>
+//#include <osmium/geom/projection.hpp>
+#include <osmium/geom/ogr.hpp>
+#include <osmium/io/any_input.hpp>
+#include <osmium/handler.hpp>
+
+typedef osmium::index::map::Dummy<osmium::unsigned_object_id_type, osmium::Location> index_neg_type;
+
+//typedef osmium::index::map::StlMap<osmium::unsigned_object_id_type, osmium::Location> index_pos_type;
+//typedef osmium::index::map::SparseMapMmap<osmium::unsigned_object_id_type, osmium::Location> index_pos_type;
+typedef osmium::index::map::SparseTable<osmium::unsigned_object_id_type, osmium::Location> index_pos_type;
+
+typedef osmium::handler::NodeLocationsForWays<index_pos_type, index_neg_type> location_handler_type;
+
+class MyOGRHandler : public osmium::handler::Handler {
+
+    OGRDataSource* m_data_source;
+    OGRLayer* m_layer_point;
+    OGRLayer* m_layer_linestring;
+    OGRLayer* m_layer_polygon;
+
+    // Choose one of the following:
+
+    // 1. Use WGS84, do not project coordinates.
+    //osmium::geom::OGRFactory<> m_factory {};
+
+    // 2. Project coordinates into "Web Mercator".
+    osmium::geom::OGRFactory<osmium::geom::MercatorProjection> m_factory;
+
+    // 3. Use any projection that the proj library can handle.
+    //    (Initialize projection with EPSG code or proj string).
+    //    In addition you need to link with "-lproj" and add
+    //    #include <osmium/geom/projection.hpp>.
+    //osmium::geom::OGRFactory<osmium::geom::Projection> m_factory {osmium::geom::Projection(3857)};
+
+public:
+
+    MyOGRHandler(const std::string& driver_name, const std::string& filename) {
+
+        OGRRegisterAll();
+
+        OGRSFDriver* driver = OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(driver_name.c_str());
+        if (!driver) {
+            std::cerr << driver_name << " driver not available.\n";
+            exit(1);
+        }
+
+        CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "FALSE");
+        const char* options[] = { "SPATIALITE=TRUE", nullptr };
+        m_data_source = driver->CreateDataSource(filename.c_str(), const_cast<char**>(options));
+        if (!m_data_source) {
+            std::cerr << "Creation of output file failed.\n";
+            exit(1);
+        }
+
+        OGRSpatialReference sparef;
+        sparef.importFromProj4(m_factory.proj_string().c_str());
+
+        m_layer_point = m_data_source->CreateLayer("postboxes", &sparef, wkbPoint, nullptr);
+        if (!m_layer_point) {
+            std::cerr << "Layer creation failed.\n";
+            exit(1);
+        }
+
+        OGRFieldDefn layer_point_field_id("id", OFTReal);
+        layer_point_field_id.SetWidth(10);
+
+        if (m_layer_point->CreateField(&layer_point_field_id) != OGRERR_NONE) {
+            std::cerr << "Creating id field failed.\n";
+            exit(1);
+        }
+
+        OGRFieldDefn layer_point_field_operator("operator", OFTString);
+        layer_point_field_operator.SetWidth(30);
+
+        if (m_layer_point->CreateField(&layer_point_field_operator) != OGRERR_NONE) {
+            std::cerr << "Creating operator field failed.\n";
+            exit(1);
+        }
+
+        /* Transactions might make things faster, then again they might not.
+           Feel free to experiment and benchmark and report back. */
+        m_layer_point->StartTransaction();
+
+        m_layer_linestring = m_data_source->CreateLayer("roads", &sparef, wkbLineString, nullptr);
+        if (!m_layer_linestring) {
+            std::cerr << "Layer creation failed.\n";
+            exit(1);
+        }
+
+        OGRFieldDefn layer_linestring_field_id("id", OFTReal);
+        layer_linestring_field_id.SetWidth(10);
+
+        if (m_layer_linestring->CreateField(&layer_linestring_field_id) != OGRERR_NONE) {
+            std::cerr << "Creating id field failed.\n";
+            exit(1);
+        }
+
+        OGRFieldDefn layer_linestring_field_type("type", OFTString);
+        layer_linestring_field_type.SetWidth(30);
+
+        if (m_layer_linestring->CreateField(&layer_linestring_field_type) != OGRERR_NONE) {
+            std::cerr << "Creating type field failed.\n";
+            exit(1);
+        }
+
+        m_layer_linestring->StartTransaction();
+
+        m_layer_polygon = m_data_source->CreateLayer("buildings", &sparef, wkbMultiPolygon, nullptr);
+        if (!m_layer_polygon) {
+            std::cerr << "Layer creation failed.\n";
+            exit(1);
+        }
+
+        OGRFieldDefn layer_polygon_field_id("id", OFTInteger);
+        layer_polygon_field_id.SetWidth(10);
+
+        if (m_layer_polygon->CreateField(&layer_polygon_field_id) != OGRERR_NONE) {
+            std::cerr << "Creating id field failed.\n";
+            exit(1);
+        }
+
+        OGRFieldDefn layer_polygon_field_type("type", OFTString);
+        layer_polygon_field_type.SetWidth(30);
+
+        if (m_layer_polygon->CreateField(&layer_polygon_field_type) != OGRERR_NONE) {
+            std::cerr << "Creating type field failed.\n";
+            exit(1);
+        }
+
+        m_layer_polygon->StartTransaction();
+    }
+
+    ~MyOGRHandler() {
+        m_layer_polygon->CommitTransaction();
+        m_layer_linestring->CommitTransaction();
+        m_layer_point->CommitTransaction();
+        OGRDataSource::DestroyDataSource(m_data_source);
+        OGRCleanupAll();
+    }
+
+    void node(const osmium::Node& node) {
+        const char* amenity = node.tags()["amenity"];
+        if (amenity && !strcmp(amenity, "post_box")) {
+            OGRFeature* feature = OGRFeature::CreateFeature(m_layer_point->GetLayerDefn());
+            std::unique_ptr<OGRPoint> ogr_point = m_factory.create_point(node);
+            feature->SetGeometry(ogr_point.get());
+            feature->SetField("id", static_cast<double>(node.id()));
+            feature->SetField("operator", node.tags()["operator"]);
+
+            if (m_layer_point->CreateFeature(feature) != OGRERR_NONE) {
+                std::cerr << "Failed to create feature.\n";
+                exit(1);
+            }
+
+            OGRFeature::DestroyFeature(feature);
+        }
+    }
+
+    void way(const osmium::Way& way) {
+        const char* highway = way.tags()["highway"];
+        if (highway) {
+            try {
+                std::unique_ptr<OGRLineString> ogr_linestring = m_factory.create_linestring(way);
+                OGRFeature* feature = OGRFeature::CreateFeature(m_layer_linestring->GetLayerDefn());
+                feature->SetGeometry(ogr_linestring.get());
+                feature->SetField("id", static_cast<double>(way.id()));
+                feature->SetField("type", highway);
+
+                if (m_layer_linestring->CreateFeature(feature) != OGRERR_NONE) {
+                    std::cerr << "Failed to create feature.\n";
+                    exit(1);
+                }
+
+                OGRFeature::DestroyFeature(feature);
+            } catch (osmium::geometry_error&) {
+                std::cerr << "Ignoring illegal geometry for way " << way.id() << ".\n";
+            }
+        }
+    }
+
+    void area(const osmium::Area& area) {
+        const char* building = area.tags()["building"];
+        if (building) {
+            try {
+                std::unique_ptr<OGRMultiPolygon> ogr_polygon = m_factory.create_multipolygon(area);
+                OGRFeature* feature = OGRFeature::CreateFeature(m_layer_polygon->GetLayerDefn());
+                feature->SetGeometry(ogr_polygon.get());
+                feature->SetField("id", static_cast<int>(area.id()));
+                feature->SetField("type", building);
+
+                std::string type = "";
+                if (area.from_way()) {
+                    type += "w";
+                } else {
+                    type += "r";
+                }
+                feature->SetField("type", type.c_str());
+
+                if (m_layer_polygon->CreateFeature(feature) != OGRERR_NONE) {
+                    std::cerr << "Failed to create feature.\n";
+                    exit(1);
+                }
+
+                OGRFeature::DestroyFeature(feature);
+            } catch (osmium::geometry_error&) {
+                std::cerr << "Ignoring illegal geometry for area " << area.id() << " created from " << (area.from_way() ? "way" : "relation") << " with id=" << area.orig_id() << ".\n";
+            }
+        }
+    }
+
+};
+
+/* ================================================== */
+
+void print_help() {
+    std::cout << "osmium_toogr [OPTIONS] [INFILE [OUTFILE]]\n\n" \
+              << "If INFILE is not given stdin is assumed.\n" \
+              << "If OUTFILE is not given 'ogr_out' is used.\n" \
+              << "\nOptions:\n" \
+              << "  -h, --help           This help message\n" \
+              << "  -d, --debug          Enable debug output\n" \
+              << "  -f, --format=FORMAT  Output OGR format (Default: 'SQLite')\n";
+}
+
+int main(int argc, char* argv[]) {
+    static struct option long_options[] = {
+        {"help",   no_argument, 0, 'h'},
+        {"debug",  no_argument, 0, 'd'},
+        {"format", required_argument, 0, 'f'},
+        {0, 0, 0, 0}
+    };
+
+    std::string output_format("SQLite");
+    bool debug = false;
+
+    while (true) {
+        int c = getopt_long(argc, argv, "hdf:", long_options, 0);
+        if (c == -1) {
+            break;
+        }
+
+        switch (c) {
+            case 'h':
+                print_help();
+                exit(0);
+            case 'd':
+                debug = true;
+                break;
+            case 'f':
+                output_format = optarg;
+                break;
+            default:
+                exit(1);
+        }
+    }
+
+    std::string input_filename;
+    std::string output_filename("ogr_out");
+    int remaining_args = argc - optind;
+    if (remaining_args > 2) {
+        std::cerr << "Usage: " << argv[0] << " [OPTIONS] [INFILE [OUTFILE]]" << std::endl;
+        exit(1);
+    } else if (remaining_args == 2) {
+        input_filename =  argv[optind];
+        output_filename = argv[optind+1];
+    } else if (remaining_args == 1) {
+        input_filename =  argv[optind];
+    } else {
+        input_filename = "-";
+    }
+
+    osmium::area::Assembler::config_type assembler_config;
+    assembler_config.enable_debug_output(debug);
+    osmium::area::MultipolygonCollector<osmium::area::Assembler> collector(assembler_config);
+
+    std::cerr << "Pass 1...\n";
+    osmium::io::Reader reader1(input_filename);
+    collector.read_relations(reader1);
+    reader1.close();
+    std::cerr << "Pass 1 done\n";
+
+    index_pos_type index_pos;
+    index_neg_type index_neg;
+    location_handler_type location_handler(index_pos, index_neg);
+    location_handler.ignore_errors();
+
+    MyOGRHandler ogr_handler(output_format, output_filename);
+
+    std::cerr << "Pass 2...\n";
+    osmium::io::Reader reader2(input_filename);
+
+    osmium::apply(reader2, location_handler, ogr_handler, collector.handler([&ogr_handler](const osmium::memory::Buffer& area_buffer) {
+        osmium::apply(area_buffer, ogr_handler);
+    }));
+
+    reader2.close();
+    std::cerr << "Pass 2 done\n";
+
+    std::vector<const osmium::Relation*> incomplete_relations = collector.get_incomplete_relations();
+    if (!incomplete_relations.empty()) {
+        std::cerr << "Warning! Some member ways missing for these multipolygon relations:";
+        for (const auto* relation : incomplete_relations) {
+            std::cerr << " " << relation->id();
+        }
+        std::cerr << "\n";
+    }
+
+    google::protobuf::ShutdownProtobufLibrary();
+}
+

examples/osmium_toogr2_exp.cpp

@@ -0,0 +1,330 @@
+/*
+
+  This is an example tool that converts OSM data to some output format
+  like Spatialite or Shapefiles using the OGR library.
+
+  This version does multipolygon handling (in contrast to the osmium_toogr
+  example which doesn't).
+
+  This version (..._exp) uses a new experimental unsupported interface.
+
+  The code in this example file is released into the Public Domain.
+
+*/
+
+#include <iostream>
+#include <getopt.h>
+
+#pragma GCC diagnostic push
+#ifdef __clang__
+# pragma GCC diagnostic ignored "-Wdocumentation-unknown-command"
+#endif
+#pragma GCC diagnostic ignored "-Wfloat-equal"
+#pragma GCC diagnostic ignored "-Wold-style-cast"
+#pragma GCC diagnostic ignored "-Wpadded"
+#pragma GCC diagnostic ignored "-Wredundant-decls"
+#pragma GCC diagnostic ignored "-Wshadow"
+# include <ogr_api.h>
+# include <ogrsf_frmts.h>
+#pragma GCC diagnostic pop
+
+// usually you only need one or two of these
+#include <osmium/index/map/dummy.hpp>
+#include <osmium/index/map/sparse_table.hpp>
+#include <osmium/index/map/stl_map.hpp>
+#include <osmium/index/map/mmap_vector_anon.hpp>
+
+#include <osmium/visitor.hpp>
+
+#include <osmium/geom/mercator_projection.hpp>
+//#include <osmium/geom/projection.hpp>
+#include <osmium/geom/ogr.hpp>
+#include <osmium/io/any_input.hpp>
+#include <osmium/handler.hpp>
+#include <osmium/experimental/flex_reader.hpp>
+
+typedef osmium::index::map::Dummy<osmium::unsigned_object_id_type, osmium::Location> index_neg_type;
+
+//typedef osmium::index::map::StlMap<osmium::unsigned_object_id_type, osmium::Location> index_pos_type;
+//typedef osmium::index::map::SparseMapMmap<osmium::unsigned_object_id_type, osmium::Location> index_pos_type;
+typedef osmium::index::map::SparseTable<osmium::unsigned_object_id_type, osmium::Location> index_pos_type;
+
+typedef osmium::handler::NodeLocationsForWays<index_pos_type, index_neg_type> location_handler_type;
+
+class MyOGRHandler : public osmium::handler::Handler {
+
+    OGRDataSource* m_data_source;
+    OGRLayer* m_layer_point;
+    OGRLayer* m_layer_linestring;
+    OGRLayer* m_layer_polygon;
+
+    // Choose one of the following:
+
+    // 1. Use WGS84, do not project coordinates.
+    //osmium::geom::OGRFactory<> m_factory {};
+
+    // 2. Project coordinates into "Web Mercator".
+    osmium::geom::OGRFactory<osmium::geom::MercatorProjection> m_factory;
+
+    // 3. Use any projection that the proj library can handle.
+    //    (Initialize projection with EPSG code or proj string).
+    //    In addition you need to link with "-lproj" and add
+    //    #include <osmium/geom/projection.hpp>.
+    //osmium::geom::OGRFactory<osmium::geom::Projection> m_factory {osmium::geom::Projection(3857)};
+
+public:
+
+    MyOGRHandler(const std::string& driver_name, const std::string& filename) {
+
+        OGRRegisterAll();
+
+        OGRSFDriver* driver = OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(driver_name.c_str());
+        if (!driver) {
+            std::cerr << driver_name << " driver not available.\n";
+            exit(1);
+        }
+
+        CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "FALSE");
+        const char* options[] = { "SPATIALITE=TRUE", nullptr };
+        m_data_source = driver->CreateDataSource(filename.c_str(), const_cast<char**>(options));
+        if (!m_data_source) {
+            std::cerr << "Creation of output file failed.\n";
+            exit(1);
+        }
+
+        OGRSpatialReference sparef;
+        sparef.importFromProj4(m_factory.proj_string().c_str());
+
+        m_layer_point = m_data_source->CreateLayer("postboxes", &sparef, wkbPoint, nullptr);
+        if (!m_layer_point) {
+            std::cerr << "Layer creation failed.\n";
+            exit(1);
+        }
+
+        OGRFieldDefn layer_point_field_id("id", OFTReal);
+        layer_point_field_id.SetWidth(10);
+
+        if (m_layer_point->CreateField(&layer_point_field_id) != OGRERR_NONE) {
+            std::cerr << "Creating id field failed.\n";
+            exit(1);
+        }
+
+        OGRFieldDefn layer_point_field_operator("operator", OFTString);
+        layer_point_field_operator.SetWidth(30);
+
+        if (m_layer_point->CreateField(&layer_point_field_operator) != OGRERR_NONE) {
+            std::cerr << "Creating operator field failed.\n";
+            exit(1);
+        }
+
+        /* Transactions might make things faster, then again they might not.
+           Feel free to experiment and benchmark and report back. */
+        m_layer_point->StartTransaction();
+
+        m_layer_linestring = m_data_source->CreateLayer("roads", &sparef, wkbLineString, nullptr);
+        if (!m_layer_linestring) {
+            std::cerr << "Layer creation failed.\n";
+            exit(1);
+        }
+
+        OGRFieldDefn layer_linestring_field_id("id", OFTReal);
+        layer_linestring_field_id.SetWidth(10);
+
+        if (m_layer_linestring->CreateField(&layer_linestring_field_id) != OGRERR_NONE) {
+            std::cerr << "Creating id field failed.\n";
+            exit(1);
+        }
+
+        OGRFieldDefn layer_linestring_field_type("type", OFTString);
+        layer_linestring_field_type.SetWidth(30);
+
+        if (m_layer_linestring->CreateField(&layer_linestring_field_type) != OGRERR_NONE) {
+            std::cerr << "Creating type field failed.\n";
+            exit(1);
+        }
+
+        m_layer_linestring->StartTransaction();
+
+        m_layer_polygon = m_data_source->CreateLayer("buildings", &sparef, wkbMultiPolygon, nullptr);
+        if (!m_layer_polygon) {
+            std::cerr << "Layer creation failed.\n";
+            exit(1);
+        }
+
+        OGRFieldDefn layer_polygon_field_id("id", OFTInteger);
+        layer_polygon_field_id.SetWidth(10);
+
+        if (m_layer_polygon->CreateField(&layer_polygon_field_id) != OGRERR_NONE) {
+            std::cerr << "Creating id field failed.\n";
+            exit(1);
+        }
+
+        OGRFieldDefn layer_polygon_field_type("type", OFTString);
+        layer_polygon_field_type.SetWidth(30);
+
+        if (m_layer_polygon->CreateField(&layer_polygon_field_type) != OGRERR_NONE) {
+            std::cerr << "Creating type field failed.\n";
+            exit(1);
+        }
+
+        m_layer_polygon->StartTransaction();
+    }
+
+    ~MyOGRHandler() {
+        m_layer_polygon->CommitTransaction();
+        m_layer_linestring->CommitTransaction();
+        m_layer_point->CommitTransaction();
+        OGRDataSource::DestroyDataSource(m_data_source);
+        OGRCleanupAll();
+    }
+
+    void node(const osmium::Node& node) {
+        const char* amenity = node.tags()["amenity"];
+        if (amenity && !strcmp(amenity, "post_box")) {
+            OGRFeature* feature = OGRFeature::CreateFeature(m_layer_point->GetLayerDefn());
+            std::unique_ptr<OGRPoint> ogr_point = m_factory.create_point(node);
+            feature->SetGeometry(ogr_point.get());
+            feature->SetField("id", static_cast<double>(node.id()));
+            feature->SetField("operator", node.tags()["operator"]);
+
+            if (m_layer_point->CreateFeature(feature) != OGRERR_NONE) {
+                std::cerr << "Failed to create feature.\n";
+                exit(1);
+            }
+
+            OGRFeature::DestroyFeature(feature);
+        }
+    }
+
+    void way(const osmium::Way& way) {
+        const char* highway = way.tags()["highway"];
+        if (highway) {
+            try {
+                std::unique_ptr<OGRLineString> ogr_linestring = m_factory.create_linestring(way);
+                OGRFeature* feature = OGRFeature::CreateFeature(m_layer_linestring->GetLayerDefn());
+                feature->SetGeometry(ogr_linestring.get());
+                feature->SetField("id", static_cast<double>(way.id()));
+                feature->SetField("type", highway);
+
+                if (m_layer_linestring->CreateFeature(feature) != OGRERR_NONE) {
+                    std::cerr << "Failed to create feature.\n";
+                    exit(1);
+                }
+
+                OGRFeature::DestroyFeature(feature);
+            } catch (osmium::geometry_error&) {
+                std::cerr << "Ignoring illegal geometry for way " << way.id() << ".\n";
+            }
+        }
+    }
+
+    void area(const osmium::Area& area) {
+        const char* building = area.tags()["building"];
+        if (building) {
+            try {
+                std::unique_ptr<OGRMultiPolygon> ogr_polygon = m_factory.create_multipolygon(area);
+                OGRFeature* feature = OGRFeature::CreateFeature(m_layer_polygon->GetLayerDefn());
+                feature->SetGeometry(ogr_polygon.get());
+                feature->SetField("id", static_cast<int>(area.id()));
+                feature->SetField("type", building);
+
+                std::string type = "";
+                if (area.from_way()) {
+                    type += "w";
+                } else {
+                    type += "r";
+                }
+                feature->SetField("type", type.c_str());
+
+                if (m_layer_polygon->CreateFeature(feature) != OGRERR_NONE) {
+                    std::cerr << "Failed to create feature.\n";
+                    exit(1);
+                }
+
+                OGRFeature::DestroyFeature(feature);
+            } catch (osmium::geometry_error&) {
+                std::cerr << "Ignoring illegal geometry for area " << area.id() << " created from " << (area.from_way() ? "way" : "relation") << " with id=" << area.orig_id() << ".\n";
+            }
+        }
+    }
+
+};
+
+/* ================================================== */
+
+void print_help() {
+    std::cout << "osmium_toogr [OPTIONS] [INFILE [OUTFILE]]\n\n" \
+              << "If INFILE is not given stdin is assumed.\n" \
+              << "If OUTFILE is not given 'ogr_out' is used.\n" \
+              << "\nOptions:\n" \
+              << "  -h, --help           This help message\n" \
+              << "  -f, --format=FORMAT  Output OGR format (Default: 'SQLite')\n";
+}
+
+int main(int argc, char* argv[]) {
+    static struct option long_options[] = {
+        {"help",   no_argument, 0, 'h'},
+        {"format", required_argument, 0, 'f'},
+        {0, 0, 0, 0}
+    };
+
+    std::string output_format("SQLite");
+
+    while (true) {
+        int c = getopt_long(argc, argv, "hf:", long_options, 0);
+        if (c == -1) {
+            break;
+        }
+
+        switch (c) {
+            case 'h':
+                print_help();
+                exit(0);
+            case 'f':
+                output_format = optarg;
+                break;
+            default:
+                exit(1);
+        }
+    }
+
+    std::string input_filename;
+    std::string output_filename("ogr_out");
+    int remaining_args = argc - optind;
+    if (remaining_args > 2) {
+        std::cerr << "Usage: " << argv[0] << " [OPTIONS] [INFILE [OUTFILE]]" << std::endl;
+        exit(1);
+    } else if (remaining_args == 2) {
+        input_filename =  argv[optind];
+        output_filename = argv[optind+1];
+    } else if (remaining_args == 1) {
+        input_filename =  argv[optind];
+    } else {
+        input_filename = "-";
+    }
+
+    index_pos_type index_pos;
+    index_neg_type index_neg;
+    location_handler_type location_handler(index_pos, index_neg);
+    osmium::experimental::FlexReader<location_handler_type> exr(input_filename, osmium::osm_entity_bits::object);
+
+    MyOGRHandler ogr_handler(output_format, output_filename);
+
+    while (auto buffer = exr.read()) {
+        osmium::apply(buffer, ogr_handler);
+    }
+
+    exr.close();
+
+    std::vector<const osmium::Relation*> incomplete_relations = exr.collector().get_incomplete_relations();
+    if (!incomplete_relations.empty()) {
+        std::cerr << "Warning! Some member ways missing for these multipolygon relations:";
+        for (const auto* relation : incomplete_relations) {
+            std::cerr << " " << relation->id();
+        }
+        std::cerr << "\n";
+    }
+
+    google::protobuf::ShutdownProtobufLibrary();
+}
+

examples/osmium_use_node_cache.cpp

@@ -0,0 +1,71 @@
+/*
+
+  This reads ways from an OSM file and writes out the node locations
+  it got from a node cache generated with osmium_create_node_cache.
+
+  The code in this example file is released into the Public Domain.
+
+*/
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <iostream>
+
+#include <osmium/io/pbf_input.hpp>
+#include <osmium/io/xml_input.hpp>
+
+#include <osmium/index/map/dummy.hpp>
+#include <osmium/index/map/mmap_vector_anon.hpp>
+#include <osmium/index/map/mmap_vector_file.hpp>
+
+#include <osmium/handler/node_locations_for_ways.hpp>
+#include <osmium/visitor.hpp>
+
+typedef osmium::index::map::Dummy<osmium::unsigned_object_id_type, osmium::Location> index_neg_type;
+//typedef osmium::index::map::DenseMapMmap<osmium::unsigned_object_id_type, osmium::Location> index_pos_type;
+typedef osmium::index::map::DenseMapFile<osmium::unsigned_object_id_type, osmium::Location> index_pos_type;
+
+typedef osmium::handler::NodeLocationsForWays<index_pos_type, index_neg_type> location_handler_type;
+
+class MyHandler : public osmium::handler::Handler {
+
+public:
+
+    void way(osmium::Way& way) {
+        for (auto& nr : way.nodes()) {
+            std::cout << nr << "\n";
+        }
+    }
+
+}; // class MyHandler
+
+int main(int argc, char* argv[]) {
+    if (argc != 3) {
+        std::cerr << "Usage: " << argv[0] << " OSM_FILE CACHE_FILE\n";
+        return 1;
+    }
+
+    std::string input_filename(argv[1]);
+    osmium::io::Reader reader(input_filename, osmium::osm_entity_bits::way);
+
+    int fd = open(argv[2], O_RDWR);
+    if (fd == -1) {
+        std::cerr << "Can not open node cache file '" << argv[2] << "': " << strerror(errno) << "\n";
+        return 1;
+    }
+
+    index_pos_type index_pos {fd};
+    index_neg_type index_neg;
+    location_handler_type location_handler(index_pos, index_neg);
+    location_handler.ignore_errors();
+
+    MyHandler handler;
+    osmium::apply(reader, location_handler, handler);
+    reader.close();
+
+    google::protobuf::ShutdownProtobufLibrary();
+
+    return 0;
+}
+

include/boost_unicode_iterator.hpp

@@ -0,0 +1,776 @@
+/*
+ *
+ * Copyright (c) 2004
+ * John Maddock
+ *
+ * Use, modification and distribution are subject to the 
+ * Boost Software License, Version 1.0. (See accompanying file 
+ * LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+ *
+ */
+ 
+ /*
+  *   LOCATION:    see http://www.boost.org for most recent version.
+  *   FILE         unicode_iterator.hpp
+  *   VERSION      see <boost/version.hpp>
+  *   DESCRIPTION: Iterator adapters for converting between different Unicode encodings.
+  */
+
+/****************************************************************************
+
+Contents:
+~~~~~~~~~
+
+1) Read Only, Input Adapters:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+template <class BaseIterator, class U8Type = ::boost::uint8_t>
+class u32_to_u8_iterator;
+
+Adapts sequence of UTF-32 code points to "look like" a sequence of UTF-8.
+
+template <class BaseIterator, class U32Type = ::boost::uint32_t>
+class u8_to_u32_iterator;
+
+Adapts sequence of UTF-8 code points to "look like" a sequence of UTF-32.
+
+template <class BaseIterator, class U16Type = ::boost::uint16_t>
+class u32_to_u16_iterator;
+
+Adapts sequence of UTF-32 code points to "look like" a sequence of UTF-16.
+
+template <class BaseIterator, class U32Type = ::boost::uint32_t>
+class u16_to_u32_iterator;
+
+Adapts sequence of UTF-16 code points to "look like" a sequence of UTF-32.
+
+2) Single pass output iterator adapters:
+
+template <class BaseIterator>
+class utf8_output_iterator;
+
+Accepts UTF-32 code points and forwards them on as UTF-8 code points.
+
+template <class BaseIterator>
+class utf16_output_iterator;
+
+Accepts UTF-32 code points and forwards them on as UTF-16 code points.
+
+****************************************************************************/
+
+#ifndef BOOST_REGEX_UNICODE_ITERATOR_HPP
+#define BOOST_REGEX_UNICODE_ITERATOR_HPP
+#include <boost/cstdint.hpp>
+#include <boost/assert.hpp>
+#include <boost/iterator/iterator_facade.hpp>
+#include <boost/static_assert.hpp>
+#include <boost/throw_exception.hpp>
+#include <stdexcept>
+#ifndef BOOST_NO_STD_LOCALE
+#include <sstream>
+#include <ios>
+#endif
+#include <limits.h> // CHAR_BIT
+
+namespace boost{
+
+namespace detail{
+
+static const ::boost::uint16_t high_surrogate_base = 0xD7C0u;
+static const ::boost::uint16_t low_surrogate_base = 0xDC00u;
+static const ::boost::uint32_t ten_bit_mask = 0x3FFu;
+
+inline bool is_high_surrogate(::boost::uint16_t v)
+{
+   return (v & 0xFFFFFC00u) == 0xd800u;
+}
+inline bool is_low_surrogate(::boost::uint16_t v)
+{
+   return (v & 0xFFFFFC00u) == 0xdc00u;
+}
+template <class T>
+inline bool is_surrogate(T v)
+{
+   return (v & 0xFFFFF800u) == 0xd800;
+}
+
+inline unsigned utf8_byte_count(boost::uint8_t c)
+{
+   // if the most significant bit with a zero in it is in position
+   // 8-N then there are N bytes in this UTF-8 sequence:
+   boost::uint8_t mask = 0x80u;
+   unsigned result = 0;
+   while(c & mask)
+   {
+      ++result;
+      mask >>= 1;
+   }
+   return (result == 0) ? 1 : ((result > 4) ? 4 : result);
+}
+
+inline unsigned utf8_trailing_byte_count(boost::uint8_t c)
+{
+   return utf8_byte_count(c) - 1;
+}
+
+#ifdef BOOST_MSVC
+#pragma warning(push)
+#pragma warning(disable:4100)
+#endif
+inline void invalid_utf32_code_point(::boost::uint32_t val)
+{
+#ifndef BOOST_NO_STD_LOCALE
+   std::stringstream ss;
+   ss << "Invalid UTF-32 code point U+" << std::showbase << std::hex << val << " encountered while trying to encode UTF-16 sequence";
+   std::out_of_range e(ss.str());
+#else
+   std::out_of_range e("Invalid UTF-32 code point encountered while trying to encode UTF-16 sequence");
+#endif
+   boost::throw_exception(e);
+}
+#ifdef BOOST_MSVC
+#pragma warning(pop)
+#endif
+
+
+} // namespace detail
+
+template <class BaseIterator, class U16Type = ::boost::uint16_t>
+class u32_to_u16_iterator
+   : public boost::iterator_facade<u32_to_u16_iterator<BaseIterator, U16Type>, U16Type, std::bidirectional_iterator_tag, const U16Type>
+{
+   typedef boost::iterator_facade<u32_to_u16_iterator<BaseIterator, U16Type>, U16Type, std::bidirectional_iterator_tag, const U16Type> base_type;
+
+#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION)
+   typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
+
+   BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32);
+   BOOST_STATIC_ASSERT(sizeof(U16Type)*CHAR_BIT == 16);
+#endif
+
+public:
+   typename base_type::reference
+      dereference()const
+   {
+      if(m_current == 2)
+         extract_current();
+      return m_values[m_current];
+   }
+   bool equal(const u32_to_u16_iterator& that)const
+   {
+      if(m_position == that.m_position)
+      {
+         // Both m_currents must be equal, or both even
+         // this is the same as saying their sum must be even:
+         return (m_current + that.m_current) & 1u ? false : true;
+      }
+      return false;
+   }
+   void increment()
+   {
+      // if we have a pending read then read now, so that we know whether
+      // to skip a position, or move to a low-surrogate:
+      if(m_current == 2)
+      {
+         // pending read:
+         extract_current();
+      }
+      // move to the next surrogate position:
+      ++m_current;
+      // if we've reached the end skip a position:
+      if(m_values[m_current] == 0)
+      {
+         m_current = 2;
+         ++m_position;
+      }
+   }
+   void decrement()
+   {
+      if(m_current != 1)
+      {
+         // decrementing an iterator always leads to a valid position:
+         --m_position;
+         extract_current();
+         m_current = m_values[1] ? 1 : 0;
+      }
+      else
+      {
+         m_current = 0;
+      }
+   }
+   BaseIterator base()const
+   {
+      return m_position;
+   }
+   // construct:
+   u32_to_u16_iterator() : m_position(), m_current(0)
+   {
+      m_values[0] = 0;
+      m_values[1] = 0;
+      m_values[2] = 0;
+   }
+   u32_to_u16_iterator(BaseIterator b) : m_position(b), m_current(2)
+   {
+      m_values[0] = 0;
+      m_values[1] = 0;
+      m_values[2] = 0;
+   }
+private:
+
+   void extract_current()const
+   {
+      // begin by checking for a code point out of range:
+      ::boost::uint32_t v = *m_position;
+      if(v >= 0x10000u)
+      {
+         if(v > 0x10FFFFu)
+            detail::invalid_utf32_code_point(*m_position);
+         // split into two surrogates:
+         m_values[0] = static_cast<U16Type>(v >> 10) + detail::high_surrogate_base;
+         m_values[1] = static_cast<U16Type>(v & detail::ten_bit_mask) + detail::low_surrogate_base;
+         m_current = 0;
+         BOOST_ASSERT(detail::is_high_surrogate(m_values[0]));
+         BOOST_ASSERT(detail::is_low_surrogate(m_values[1]));
+      }
+      else
+      {
+         // 16-bit code point:
+         m_values[0] = static_cast<U16Type>(*m_position);
+         m_values[1] = 0;
+         m_current = 0;
+         // value must not be a surrogate:
+         if(detail::is_surrogate(m_values[0]))
+            detail::invalid_utf32_code_point(*m_position);
+      }
+   }
+   BaseIterator m_position;
+   mutable U16Type m_values[3];
+   mutable unsigned m_current;
+};
+
+template <class BaseIterator, class U32Type = ::boost::uint32_t>
+class u16_to_u32_iterator
+   : public boost::iterator_facade<u16_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type>
+{
+   typedef boost::iterator_facade<u16_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type;
+   // special values for pending iterator reads:
+   BOOST_STATIC_CONSTANT(U32Type, pending_read = 0xffffffffu);
+
+#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION)
+   typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
+
+   BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 16);
+   BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32);
+#endif
+
+public:
+   typename base_type::reference
+      dereference()const
+   {
+      if(m_value == pending_read)
+         extract_current();
+      return m_value;
+   }
+   bool equal(const u16_to_u32_iterator& that)const
+   {
+      return m_position == that.m_position;
+   }
+   void increment()
+   {
+      // skip high surrogate first if there is one:
+      if(detail::is_high_surrogate(*m_position)) ++m_position;
+      ++m_position;
+      m_value = pending_read;
+   }
+   void decrement()
+   {
+      --m_position;
+      // if we have a low surrogate then go back one more:
+      if(detail::is_low_surrogate(*m_position)) 
+         --m_position;
+      m_value = pending_read;
+   }
+   BaseIterator base()const
+   {
+      return m_position;
+   }
+   // construct:
+   u16_to_u32_iterator() : m_position()
+   {
+      m_value = pending_read;
+   }
+   u16_to_u32_iterator(BaseIterator b) : m_position(b)
+   {
+      m_value = pending_read;
+   }
+   //
+   // Range checked version:
+   //
+   u16_to_u32_iterator(BaseIterator b, BaseIterator start, BaseIterator end) : m_position(b)
+   {
+      m_value = pending_read;
+      //
+      // The range must not start with a low surrogate, or end in a high surrogate,
+      // otherwise we run the risk of running outside the underlying input range.
+      // Likewise b must not be located at a low surrogate.
+      //
+      boost::uint16_t val;
+      if(start != end)
+      {
+         if((b != start) && (b != end))
+         {
+            val = *b;
+            if(detail::is_surrogate(val) && ((val & 0xFC00u) == 0xDC00u))
+               invalid_code_point(val);
+         }
+         val = *start;
+         if(detail::is_surrogate(val) && ((val & 0xFC00u) == 0xDC00u))
+            invalid_code_point(val);
+         val = *--end;
+         if(detail::is_high_surrogate(val))
+            invalid_code_point(val);
+      }
+   }
+private:
+   static void invalid_code_point(::boost::uint16_t val)
+   {
+#ifndef BOOST_NO_STD_LOCALE
+      std::stringstream ss;
+      ss << "Misplaced UTF-16 surrogate U+" << std::showbase << std::hex << val << " encountered while trying to encode UTF-32 sequence";
+      std::out_of_range e(ss.str());
+#else
+      std::out_of_range e("Misplaced UTF-16 surrogate encountered while trying to encode UTF-32 sequence");
+#endif
+      boost::throw_exception(e);
+   }
+   void extract_current()const
+   {
+      m_value = static_cast<U32Type>(static_cast< ::boost::uint16_t>(*m_position));
+      // if the last value is a high surrogate then adjust m_position and m_value as needed:
+      if(detail::is_high_surrogate(*m_position))
+      {
+         // precondition; next value must have be a low-surrogate:
+         BaseIterator next(m_position);
+         ::boost::uint16_t t = *++next;
+         if((t & 0xFC00u) != 0xDC00u)
+            invalid_code_point(t);
+         m_value = (m_value - detail::high_surrogate_base) << 10;
+         m_value |= (static_cast<U32Type>(static_cast< ::boost::uint16_t>(t)) & detail::ten_bit_mask);
+      }
+      // postcondition; result must not be a surrogate:
+      if(detail::is_surrogate(m_value))
+         invalid_code_point(static_cast< ::boost::uint16_t>(m_value));
+   }
+   BaseIterator m_position;
+   mutable U32Type m_value;
+};
+
+template <class BaseIterator, class U8Type = ::boost::uint8_t>
+class u32_to_u8_iterator
+   : public boost::iterator_facade<u32_to_u8_iterator<BaseIterator, U8Type>, U8Type, std::bidirectional_iterator_tag, const U8Type>
+{
+   typedef boost::iterator_facade<u32_to_u8_iterator<BaseIterator, U8Type>, U8Type, std::bidirectional_iterator_tag, const U8Type> base_type;
+   
+#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION)
+   typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
+
+   BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32);
+   BOOST_STATIC_ASSERT(sizeof(U8Type)*CHAR_BIT == 8);
+#endif
+
+public:
+   typename base_type::reference
+      dereference()const
+   {
+      if(m_current == 4)
+         extract_current();
+      return m_values[m_current];
+   }
+   bool equal(const u32_to_u8_iterator& that)const
+   {
+      if(m_position == that.m_position)
+      {
+         // either the m_current's must be equal, or one must be 0 and 
+         // the other 4: which means neither must have bits 1 or 2 set:
+         return (m_current == that.m_current)
+            || (((m_current | that.m_current) & 3) == 0);
+      }
+      return false;
+   }
+   void increment()
+   {
+      // if we have a pending read then read now, so that we know whether
+      // to skip a position, or move to a low-surrogate:
+      if(m_current == 4)
+      {
+         // pending read:
+         extract_current();
+      }
+      // move to the next surrogate position:
+      ++m_current;
+      // if we've reached the end skip a position:
+      if(m_values[m_current] == 0)
+      {
+         m_current = 4;
+         ++m_position;
+      }
+   }
+   void decrement()
+   {
+      if((m_current & 3) == 0)
+      {
+         --m_position;
+         extract_current();
+         m_current = 3;
+         while(m_current && (m_values[m_current] == 0))
+            --m_current;
+      }
+      else
+         --m_current;
+   }
+   BaseIterator base()const
+   {
+      return m_position;
+   }
+   // construct:
+   u32_to_u8_iterator() : m_position(), m_current(0)
+   {
+      m_values[0] = 0;
+      m_values[1] = 0;
+      m_values[2] = 0;
+      m_values[3] = 0;
+      m_values[4] = 0;
+   }
+   u32_to_u8_iterator(BaseIterator b) : m_position(b), m_current(4)
+   {
+      m_values[0] = 0;
+      m_values[1] = 0;
+      m_values[2] = 0;
+      m_values[3] = 0;
+      m_values[4] = 0;
+   }
+private:
+
+   void extract_current()const
+   {
+      boost::uint32_t c = *m_position;
+      if(c > 0x10FFFFu)
+         detail::invalid_utf32_code_point(c);
+      if(c < 0x80u)
+      {
+         m_values[0] = static_cast<unsigned char>(c);
+         m_values[1] = static_cast<unsigned char>(0u);
+         m_values[2] = static_cast<unsigned char>(0u);
+         m_values[3] = static_cast<unsigned char>(0u);
+      }
+      else if(c < 0x800u)
+      {
+         m_values[0] = static_cast<unsigned char>(0xC0u + (c >> 6));
+         m_values[1] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
+         m_values[2] = static_cast<unsigned char>(0u);
+         m_values[3] = static_cast<unsigned char>(0u);
+      }
+      else if(c < 0x10000u)
+      {
+         m_values[0] = static_cast<unsigned char>(0xE0u + (c >> 12));
+         m_values[1] = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
+         m_values[2] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
+         m_values[3] = static_cast<unsigned char>(0u);
+      }
+      else
+      {
+         m_values[0] = static_cast<unsigned char>(0xF0u + (c >> 18));
+         m_values[1] = static_cast<unsigned char>(0x80u + ((c >> 12) & 0x3Fu));
+         m_values[2] = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
+         m_values[3] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
+      }
+      m_current= 0;
+   }
+   BaseIterator m_position;
+   mutable U8Type m_values[5];
+   mutable unsigned m_current;
+};
+
+template <class BaseIterator, class U32Type = ::boost::uint32_t>
+class u8_to_u32_iterator
+   : public boost::iterator_facade<u8_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type>
+{
+   typedef boost::iterator_facade<u8_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type;
+   // special values for pending iterator reads:
+   BOOST_STATIC_CONSTANT(U32Type, pending_read = 0xffffffffu);
+
+#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION)
+   typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
+
+   BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 8);
+   BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32);
+#endif
+
+public:
+   typename base_type::reference
+      dereference()const
+   {
+      if(m_value == pending_read)
+         extract_current();
+      return m_value;
+   }
+   bool equal(const u8_to_u32_iterator& that)const
+   {
+      return m_position == that.m_position;
+   }
+   void increment()
+   {
+      // We must not start with a continuation character:
+      if((static_cast<boost::uint8_t>(*m_position) & 0xC0) == 0x80)
+         invalid_sequence();
+      // skip high surrogate first if there is one:
+      unsigned c = detail::utf8_byte_count(*m_position);
+      if(m_value == pending_read)
+      {
+         // Since we haven't read in a value, we need to validate the code points:
+         for(unsigned i = 0; i < c; ++i)
+         {
+            ++m_position;
+            // We must have a continuation byte:
+            if((i != c - 1) && ((static_cast<boost::uint8_t>(*m_position) & 0xC0) != 0x80))
+               invalid_sequence();
+         }
+      }
+      else
+      {
+         std::advance(m_position, c);
+      }
+      m_value = pending_read;
+   }
+   void decrement()
+   {
+      // Keep backtracking until we don't have a trailing character:
+      unsigned count = 0;
+      while((*--m_position & 0xC0u) == 0x80u) ++count;
+      // now check that the sequence was valid:
+      if(count != detail::utf8_trailing_byte_count(*m_position))
+         invalid_sequence();
+      m_value = pending_read;
+   }
+   BaseIterator base()const
+   {
+      return m_position;
+   }
+   // construct:
+   u8_to_u32_iterator() : m_position()
+   {
+      m_value = pending_read;
+   }
+   u8_to_u32_iterator(BaseIterator b) : m_position(b)
+   {
+      m_value = pending_read;
+   }
+   //
+   // Checked constructor:
+   //
+   u8_to_u32_iterator(BaseIterator b, BaseIterator start, BaseIterator end) : m_position(b)
+   {
+      m_value = pending_read;
+      //
+      // We must not start with a continuation character, or end with a 
+      // truncated UTF-8 sequence otherwise we run the risk of going past
+      // the start/end of the underlying sequence:
+      //
+      if(start != end)
+      {
+         unsigned char v = *start;
+         if((v & 0xC0u) == 0x80u)
+            invalid_sequence();
+         if((b != start) && (b != end) && ((*b & 0xC0u) == 0x80u))
+            invalid_sequence();
+         BaseIterator pos = end;
+         do
+         {
+            v = *--pos;
+         }
+         while((start != pos) && ((v & 0xC0u) == 0x80u));
+         std::ptrdiff_t extra = detail::utf8_byte_count(v);
+         if(std::distance(pos, end) < extra)
+            invalid_sequence();
+      }
+   }
+private:
+   static void invalid_sequence()
+   {
+      std::out_of_range e("Invalid UTF-8 sequence encountered while trying to encode UTF-32 character");
+      boost::throw_exception(e);
+   }
+   void extract_current()const
+   {
+      m_value = static_cast<U32Type>(static_cast< ::boost::uint8_t>(*m_position));
+      // we must not have a continuation character:
+      if((m_value & 0xC0u) == 0x80u)
+         invalid_sequence();
+      // see how many extra bytes we have:
+      unsigned extra = detail::utf8_trailing_byte_count(*m_position);
+      // extract the extra bits, 6 from each extra byte:
+      BaseIterator next(m_position);
+      for(unsigned c = 0; c < extra; ++c)
+      {
+         ++next;
+         m_value <<= 6;
+         // We must have a continuation byte:
+         if((static_cast<boost::uint8_t>(*next) & 0xC0) != 0x80)
+            invalid_sequence();
+         m_value += static_cast<boost::uint8_t>(*next) & 0x3Fu;
+      }
+      // we now need to remove a few of the leftmost bits, but how many depends
+      // upon how many extra bytes we've extracted:
+      static const boost::uint32_t masks[4] = 
+      {
+         0x7Fu,
+         0x7FFu,
+         0xFFFFu,
+         0x1FFFFFu,
+      };
+      m_value &= masks[extra];
+      // check the result:
+      if(m_value > static_cast<U32Type>(0x10FFFFu))
+         invalid_sequence();
+   }
+   BaseIterator m_position;
+   mutable U32Type m_value;
+};
+
+template <class BaseIterator>
+class utf16_output_iterator
+{
+public:
+   typedef void                                   difference_type;
+   typedef void                                   value_type;
+   typedef boost::uint32_t*                       pointer;
+   typedef boost::uint32_t&                       reference;
+   typedef std::output_iterator_tag               iterator_category;
+
+   utf16_output_iterator(const BaseIterator& b)
+      : m_position(b){}
+   utf16_output_iterator(const utf16_output_iterator& that)
+      : m_position(that.m_position){}
+   utf16_output_iterator& operator=(const utf16_output_iterator& that)
+   {
+      m_position = that.m_position;
+      return *this;
+   }
+   const utf16_output_iterator& operator*()const
+   {
+      return *this;
+   }
+   void operator=(boost::uint32_t val)const
+   {
+      push(val);
+   }
+   utf16_output_iterator& operator++()
+   {
+      return *this;
+   }
+   utf16_output_iterator& operator++(int)
+   {
+      return *this;
+   }
+   BaseIterator base()const
+   {
+      return m_position;
+   }
+private:
+   void push(boost::uint32_t v)const
+   {
+      if(v >= 0x10000u)
+      {
+         // begin by checking for a code point out of range:
+         if(v > 0x10FFFFu)
+            detail::invalid_utf32_code_point(v);
+         // split into two surrogates:
+         *m_position++ = static_cast<boost::uint16_t>(v >> 10) + detail::high_surrogate_base;
+         *m_position++ = static_cast<boost::uint16_t>(v & detail::ten_bit_mask) + detail::low_surrogate_base;
+      }
+      else
+      {
+         // 16-bit code point:
+         // value must not be a surrogate:
+         if(detail::is_surrogate(v))
+            detail::invalid_utf32_code_point(v);
+         *m_position++ = static_cast<boost::uint16_t>(v);
+      }
+   }
+   mutable BaseIterator m_position;
+};
+
+template <class BaseIterator>
+class utf8_output_iterator
+{
+public:
+   typedef void                                   difference_type;
+   typedef void                                   value_type;
+   typedef boost::uint32_t*                       pointer;
+   typedef boost::uint32_t&                       reference;
+   typedef std::output_iterator_tag               iterator_category;
+
+   utf8_output_iterator(const BaseIterator& b)
+      : m_position(b){}
+   utf8_output_iterator(const utf8_output_iterator& that)
+      : m_position(that.m_position){}
+   utf8_output_iterator& operator=(const utf8_output_iterator& that)
+   {
+      m_position = that.m_position;
+      return *this;
+   }
+   const utf8_output_iterator& operator*()const
+   {
+      return *this;
+   }
+   void operator=(boost::uint32_t val)const
+   {
+      push(val);
+   }
+   utf8_output_iterator& operator++()
+   {
+      return *this;
+   }
+   utf8_output_iterator& operator++(int)
+   {
+      return *this;
+   }
+   BaseIterator base()const
+   {
+      return m_position;
+   }
+private:
+   void push(boost::uint32_t c)const
+   {
+      if(c > 0x10FFFFu)
+         detail::invalid_utf32_code_point(c);
+      if(c < 0x80u)
+      {
+         *m_position++ = static_cast<unsigned char>(c);
+      }
+      else if(c < 0x800u)
+      {
+         *m_position++ = static_cast<unsigned char>(0xC0u + (c >> 6));
+         *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
+      }
+      else if(c < 0x10000u)
+      {
+         *m_position++ = static_cast<unsigned char>(0xE0u + (c >> 12));
+         *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
+         *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
+      }
+      else
+      {
+         *m_position++ = static_cast<unsigned char>(0xF0u + (c >> 18));
+         *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 12) & 0x3Fu));
+         *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
+         *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
+      }
+   }
+   mutable BaseIterator m_position;
+};
+
+} // namespace boost
+
+#endif // BOOST_REGEX_UNICODE_ITERATOR_HPP
+

include/mmap_for_windows.hpp

@@ -0,0 +1,106 @@
+#ifndef MMAP_FOR_WINDOWS_HPP
+#define MMAP_FOR_WINDOWS_HPP
+
+/* mmap() replacement for Windows
+ *
+ * Author: Mike Frysinger <vapier@gentoo.org>
+ * Placed into the public domain
+ */
+
+/* References:
+ * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx
+ * CloseHandle:       http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx
+ * MapViewOfFile:     http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx
+ * UnmapViewOfFile:   http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx
+ */
+
+#include <io.h>
+#include <windows.h>
+#include <sys/types.h>
+
+#define PROT_READ     0x1
+#define PROT_WRITE    0x2
+/* This flag is only available in WinXP+ */
+#ifdef FILE_MAP_EXECUTE
+#define PROT_EXEC     0x4
+#else
+#define PROT_EXEC        0x0
+#define FILE_MAP_EXECUTE 0
+#endif
+
+#define MAP_SHARED    0x01
+#define MAP_PRIVATE   0x02
+#define MAP_ANONYMOUS 0x20
+#define MAP_ANON      MAP_ANONYMOUS
+#define MAP_FAILED    ((void *) -1)
+
+#ifdef __USE_FILE_OFFSET64
+# define DWORD_HI(x) (x >> 32)
+# define DWORD_LO(x) ((x) & 0xffffffff)
+#else
+# define DWORD_HI(x) (0)
+# define DWORD_LO(x) (x)
+#endif
+
+static void *mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset)
+{
+	if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC))
+		return MAP_FAILED;
+	if (fd == -1) {
+		if (!(flags & MAP_ANON) || offset)
+			return MAP_FAILED;
+	} else if (flags & MAP_ANON)
+		return MAP_FAILED;
+
+	DWORD flProtect;
+	if (prot & PROT_WRITE) {
+		if (prot & PROT_EXEC)
+			flProtect = PAGE_EXECUTE_READWRITE;
+		else
+			flProtect = PAGE_READWRITE;
+	} else if (prot & PROT_EXEC) {
+		if (prot & PROT_READ)
+			flProtect = PAGE_EXECUTE_READ;
+		else if (prot & PROT_EXEC)
+			flProtect = PAGE_EXECUTE;
+	} else
+		flProtect = PAGE_READONLY;
+
+	off_t end = length + offset;
+	HANDLE mmap_fd, h;
+	if (fd == -1)
+		mmap_fd = INVALID_HANDLE_VALUE;
+	else
+		mmap_fd = (HANDLE)_get_osfhandle(fd);
+	h = CreateFileMapping(mmap_fd, NULL, flProtect, DWORD_HI(end), DWORD_LO(end), NULL);
+	if (h == NULL)
+		return MAP_FAILED;
+
+	DWORD dwDesiredAccess;
+	if (prot & PROT_WRITE)
+		dwDesiredAccess = FILE_MAP_WRITE;
+	else
+		dwDesiredAccess = FILE_MAP_READ;
+	if (prot & PROT_EXEC)
+		dwDesiredAccess |= FILE_MAP_EXECUTE;
+	if (flags & MAP_PRIVATE)
+		dwDesiredAccess |= FILE_MAP_COPY;
+	void *ret = MapViewOfFile(h, dwDesiredAccess, DWORD_HI(offset), DWORD_LO(offset), length);
+	if (ret == NULL) {
+		CloseHandle(h);
+		ret = MAP_FAILED;
+	}
+	return ret;
+}
+
+static int munmap(void *addr, size_t length)
+{
+	UnmapViewOfFile(addr);
+	return 0;
+	/* ruh-ro, we leaked handle from CreateFileMapping() ... */
+}
+
+#undef DWORD_HI
+#undef DWORD_LO
+
+#endif

include/osmium/area/assembler.hpp

@@ -0,0 +1,783 @@
+#ifndef OSMIUM_AREA_ASSEMBLER_HPP
+#define OSMIUM_AREA_ASSEMBLER_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <algorithm>
+#include <iostream>
+#include <iterator>
+#include <list>
+#include <map>
+#include <vector>
+
+#include <osmium/builder/osm_object_builder.hpp>
+#include <osmium/memory/buffer.hpp>
+#include <osmium/osm/area.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/relation.hpp>
+#include <osmium/tags/filter.hpp>
+
+#include <osmium/area/detail/proto_ring.hpp>
+#include <osmium/area/detail/node_ref_segment.hpp>
+#include <osmium/area/detail/segment_list.hpp>
+#include <osmium/area/problem_reporter.hpp>
+
+namespace osmium {
+
+    namespace area {
+
+        using osmium::area::detail::ProtoRing;
+
+        struct AssemblerConfig {
+
+            osmium::area::ProblemReporter* problem_reporter;
+
+            // Enables debug output to stderr
+            bool debug;
+
+            explicit AssemblerConfig(osmium::area::ProblemReporter* pr = nullptr, bool d=false) :
+                problem_reporter(pr),
+                debug(d) {
+            }
+
+            /**
+             * Enable or disable debug output to stderr. This is for Osmium
+             * developers only.
+             */
+            void enable_debug_output(bool d=true) {
+                debug = d;
+            }
+
+        }; // struct AssemblerConfig
+
+        /**
+         * Assembles area objects from multipolygon relations and their
+         * members. This is called by the MultipolygonCollector object
+         * after all members have been collected.
+         */
+        class Assembler {
+
+            const AssemblerConfig m_config;
+
+            // The way segments
+            osmium::area::detail::SegmentList m_segment_list;
+
+            // The rings we are building from the way segments
+            std::list<ProtoRing> m_rings;
+
+            std::vector<ProtoRing*> m_outer_rings;
+            std::vector<ProtoRing*> m_inner_rings;
+
+            int m_inner_outer_mismatches { 0 };
+
+            bool debug() const {
+                return m_config.debug;
+            }
+
+            /**
+             * Checks whether the given NodeRefs have the same location.
+             * Uses the actual location for the test, not the id. If both
+             * have the same location, but not the same id, a problem
+             * point will be added to the list of problem points.
+             */
+            bool has_same_location(const osmium::NodeRef& nr1, const osmium::NodeRef& nr2) {
+                if (nr1.location() != nr2.location()) {
+                    return false;
+                }
+                if (nr1.ref() != nr2.ref()) {
+                    if (m_config.problem_reporter) {
+                        m_config.problem_reporter->report_duplicate_node(nr1.ref(), nr2.ref(), nr1.location());
+                    }
+                }
+                return true;
+            }
+
+            void add_tags_to_area(osmium::builder::AreaBuilder& builder, const osmium::Way& way) const {
+                osmium::builder::TagListBuilder tl_builder(builder.buffer(), &builder);
+                for (const osmium::Tag& tag : way.tags()) {
+                    tl_builder.add_tag(tag.key(), tag.value());
+                }
+            }
+
+            void add_common_tags(osmium::builder::TagListBuilder& tl_builder, std::set<const osmium::Way*>& ways) const {
+                std::map<std::string, size_t> counter;
+                for (const osmium::Way* way : ways) {
+                    for (const auto& tag : way->tags()) {
+                        std::string kv {tag.key()};
+                        kv.append(1, '\0');
+                        kv.append(tag.value());
+                        ++counter[kv];
+                    }
+                }
+
+                size_t num_ways = ways.size();
+                for (const auto& t_c : counter) {
+                    if (debug()) {
+                        std::cerr << "        tag " << t_c.first << " is used " << t_c.second << " times in " << num_ways << " ways\n";
+                    }
+                    if (t_c.second == num_ways) {
+                        size_t len = std::strlen(t_c.first.c_str());
+                        tl_builder.add_tag(t_c.first.c_str(), t_c.first.c_str() + len + 1);
+                    }
+                }
+            }
+
+            struct MPFilter : public osmium::tags::KeyFilter {
+
+                MPFilter() : osmium::tags::KeyFilter(true) {
+                    add(false, "type");
+                    add(false, "created_by");
+                    add(false, "source");
+                    add(false, "note");
+                    add(false, "test:id");
+                    add(false, "test:section");
+                }
+
+            }; // struct MPFilter
+
+            static MPFilter& filter() {
+                static MPFilter filter;
+                return filter;
+            }
+
+            void add_tags_to_area(osmium::builder::AreaBuilder& builder, const osmium::Relation& relation) const {
+                auto count = std::count_if(relation.tags().begin(), relation.tags().end(), filter());
+
+                if (debug()) {
+                    std::cerr << "  found " << count << " tags on relation (without ignored ones)\n";
+                }
+
+                if (count > 0) {
+                    if (debug()) {
+                        std::cerr << "    use tags from relation\n";
+                    }
+
+                    // write out all tags except type=*
+                    osmium::builder::TagListBuilder tl_builder(builder.buffer(), &builder);
+                    for (const osmium::Tag& tag : relation.tags()) {
+                        if (strcmp(tag.key(), "type")) {
+                            tl_builder.add_tag(tag.key(), tag.value());
+                        }
+                    }
+                } else {
+                    if (debug()) {
+                        std::cerr << "    use tags from outer ways\n";
+                    }
+                    std::set<const osmium::Way*> ways;
+                    for (const auto& ring : m_outer_rings) {
+                        ring->get_ways(ways);
+                    }
+                    if (ways.size() == 1) {
+                        if (debug()) {
+                            std::cerr << "      only one outer way\n";
+                        }
+                        osmium::builder::TagListBuilder tl_builder(builder.buffer(), &builder);
+                        for (const osmium::Tag& tag : (*ways.begin())->tags()) {
+                            tl_builder.add_tag(tag.key(), tag.value());
+                        }
+                    } else {
+                        if (debug()) {
+                            std::cerr << "      multiple outer ways, get common tags\n";
+                        }
+                        osmium::builder::TagListBuilder tl_builder(builder.buffer(), &builder);
+                        add_common_tags(tl_builder, ways);
+                    }
+                }
+            }
+
+            /**
+             * Go through all the rings and find rings that are not closed.
+             * Problems are reported through the problem reporter.
+             *
+             * @returns true if any rings were not closed, false otherwise
+             */
+            bool check_for_open_rings() {
+                bool open_rings = false;
+
+                for (const auto& ring : m_rings) {
+                    if (!ring.closed()) {
+                        open_rings = true;
+                        if (m_config.problem_reporter) {
+                            m_config.problem_reporter->report_ring_not_closed(ring.get_segment_front().first().location(), ring.get_segment_back().second().location());
+                        }
+                    }
+                }
+
+                return open_rings;
+            }
+
+            /**
+             * Check whether there are any rings that can be combined with the
+             * given ring to one larger ring by appending the other ring to
+             * the end of this ring.
+             * If the rings can be combined they are and the function returns
+             * true.
+             */
+            bool possibly_combine_rings_back(ProtoRing& ring) {
+                const osmium::NodeRef& nr = ring.get_segment_back().second();
+
+                if (debug()) {
+                    std::cerr << "      possibly_combine_rings_back()\n";
+                }
+                for (auto it = m_rings.begin(); it != m_rings.end(); ++it) {
+                    if (&*it != &ring && !it->closed()) {
+                        if (has_same_location(nr, it->get_segment_front().first())) {
+                            if (debug()) {
+                                std::cerr << "      ring.last=it->first\n";
+                            }
+                            ring.merge_ring(*it, debug());
+                            m_rings.erase(it);
+                            return true;
+                        }
+                        if (has_same_location(nr, it->get_segment_back().second())) {
+                            if (debug()) {
+                                std::cerr << "      ring.last=it->last\n";
+                            }
+                            ring.merge_ring_reverse(*it, debug());
+                            m_rings.erase(it);
+                            return true;
+                        }
+                    }
+                }
+                return false;
+            }
+
+            /**
+             * Check whether there are any rings that can be combined with the
+             * given ring to one larger ring by prepending the other ring to
+             * the start of this ring.
+             * If the rings can be combined they are and the function returns
+             * true.
+             */
+            bool possibly_combine_rings_front(ProtoRing& ring) {
+                const osmium::NodeRef& nr = ring.get_segment_front().first();
+
+                if (debug()) {
+                    std::cerr << "      possibly_combine_rings_front()\n";
+                }
+                for (auto it = m_rings.begin(); it != m_rings.end(); ++it) {
+                    if (&*it != &ring && !it->closed()) {
+                        if (has_same_location(nr, it->get_segment_back().second())) {
+                            if (debug()) {
+                                std::cerr << "      ring.first=it->last\n";
+                            }
+                            ring.swap_segments(*it);
+                            ring.merge_ring(*it, debug());
+                            m_rings.erase(it);
+                            return true;
+                        }
+                        if (has_same_location(nr, it->get_segment_front().first())) {
+                            if (debug()) {
+                                std::cerr << "      ring.first=it->first\n";
+                            }
+                            ring.reverse();
+                            ring.merge_ring(*it, debug());
+                            m_rings.erase(it);
+                            return true;
+                        }
+                    }
+                }
+                return false;
+            }
+
+            void split_off_subring(osmium::area::detail::ProtoRing& ring, osmium::area::detail::ProtoRing::segments_type::iterator it, osmium::area::detail::ProtoRing::segments_type::iterator it_begin, osmium::area::detail::ProtoRing::segments_type::iterator it_end) {
+                if (debug()) {
+                    std::cerr << "        subring found at: " << *it << "\n";
+                }
+                ProtoRing new_ring(it_begin, it_end);
+                ring.remove_segments(it_begin, it_end);
+                if (debug()) {
+                    std::cerr << "        split into two rings:\n";
+                    std::cerr << "          " << new_ring << "\n";
+                    std::cerr << "          " << ring << "\n";
+                }
+                m_rings.push_back(std::move(new_ring));
+            }
+
+            bool has_closed_subring_back(ProtoRing& ring, const NodeRef& nr) {
+                if (ring.segments().size() < 3) {
+                    return false;
+                }
+                if (debug()) {
+                    std::cerr << "      has_closed_subring_back()\n";
+                }
+                auto end = ring.segments().end();
+                for (auto it = ring.segments().begin() + 1; it != end - 1; ++it) {
+                    if (has_same_location(nr, it->first())) {
+                        split_off_subring(ring, it, it, end);
+                        return true;
+                    }
+                }
+                return false;
+            }
+
+            bool has_closed_subring_front(ProtoRing& ring, const NodeRef& nr) {
+                if (ring.segments().size() < 3) {
+                    return false;
+                }
+                if (debug()) {
+                    std::cerr << "      has_closed_subring_front()\n";
+                }
+                auto end = ring.segments().end();
+                for (auto it = ring.segments().begin() + 1; it != end - 1; ++it) {
+                    if (has_same_location(nr, it->second())) {
+                        split_off_subring(ring, it, ring.segments().begin(), it+1);
+                        return true;
+                    }
+                }
+                return false;
+            }
+
+            bool check_for_closed_subring(ProtoRing& ring) {
+                if (debug()) {
+                    std::cerr << "      check_for_closed_subring()\n";
+                }
+
+                osmium::area::detail::ProtoRing::segments_type segments(ring.segments().size());
+                std::copy(ring.segments().begin(), ring.segments().end(), segments.begin());
+                std::sort(segments.begin(), segments.end());
+                auto it = std::adjacent_find(segments.begin(), segments.end(), [this](const osmium::area::detail::NodeRefSegment& s1, const osmium::area::detail::NodeRefSegment& s2) {
+                    return has_same_location(s1.first(), s2.first());
+                });
+                if (it == segments.end()) {
+                    return false;
+                }
+                auto r1 = std::find_first_of(ring.segments().begin(), ring.segments().end(), it, it+1);
+                assert(r1 != ring.segments().end());
+                auto r2 = std::find_first_of(ring.segments().begin(), ring.segments().end(), it+1, it+2);
+                assert(r2 != ring.segments().end());
+
+                if (debug()) {
+                    std::cerr << "      found subring in ring " << ring << " at " << it->first() << "\n";
+                }
+
+                auto m = std::minmax(r1, r2);
+
+                ProtoRing new_ring(m.first, m.second);
+                ring.remove_segments(m.first, m.second);
+
+                if (debug()) {
+                    std::cerr << "        split ring1=" << new_ring << "\n";
+                    std::cerr << "        split ring2=" << ring << "\n";
+                }
+
+                m_rings.emplace_back(new_ring);
+
+                return true;
+            }
+
+            void combine_rings_front(const osmium::area::detail::NodeRefSegment& segment, ProtoRing& ring) {
+                if (debug()) {
+                    std::cerr << " => match at front of ring\n";
+                }
+                ring.add_segment_front(segment);
+                has_closed_subring_front(ring, segment.first());
+                if (possibly_combine_rings_front(ring)) {
+                    check_for_closed_subring(ring);
+                }
+            }
+
+            void combine_rings_back(const osmium::area::detail::NodeRefSegment& segment, ProtoRing& ring) {
+                if (debug()) {
+                    std::cerr << " => match at back of ring\n";
+                }
+                ring.add_segment_back(segment);
+                has_closed_subring_back(ring, segment.second());
+                if (possibly_combine_rings_back(ring)) {
+                    check_for_closed_subring(ring);
+                }
+            }
+
+            /**
+             * Append each outer ring together with its inner rings to the
+             * area in the buffer.
+             */
+            void add_rings_to_area(osmium::builder::AreaBuilder& builder) const {
+                for (const ProtoRing* ring : m_outer_rings) {
+                    if (debug()) {
+                        std::cerr << "    ring " << *ring << " is outer\n";
+                    }
+                    {
+                        osmium::builder::OuterRingBuilder ring_builder(builder.buffer(), &builder);
+                        ring_builder.add_node_ref(ring->get_segment_front().first());
+                        for (const auto& segment : ring->segments()) {
+                            ring_builder.add_node_ref(segment.second());
+                        }
+                    }
+                    for (ProtoRing* inner : ring->inner_rings()) {
+                        osmium::builder::InnerRingBuilder ring_builder(builder.buffer(), &builder);
+                        ring_builder.add_node_ref(inner->get_segment_front().first());
+                        for (const auto& segment : inner->segments()) {
+                            ring_builder.add_node_ref(segment.second());
+                        }
+                    }
+                }
+            }
+
+            bool add_to_existing_ring(osmium::area::detail::NodeRefSegment segment) {
+                int n=0;
+                for (auto& ring : m_rings) {
+                    if (debug()) {
+                        std::cerr << "    check against ring " << n << " " << ring;
+                    }
+                    if (ring.closed()) {
+                        if (debug()) {
+                            std::cerr << " => ring CLOSED\n";
+                        }
+                    } else {
+                        if (has_same_location(ring.get_segment_back().second(), segment.first())) {
+                            combine_rings_back(segment, ring);
+                            return true;
+                        }
+                        if (has_same_location(ring.get_segment_back().second(), segment.second())) {
+                            segment.swap_locations();
+                            combine_rings_back(segment, ring);
+                            return true;
+                        }
+                        if (has_same_location(ring.get_segment_front().first(), segment.first())) {
+                            segment.swap_locations();
+                            combine_rings_front(segment, ring);
+                            return true;
+                        }
+                        if (has_same_location(ring.get_segment_front().first(), segment.second())) {
+                            combine_rings_front(segment, ring);
+                            return true;
+                        }
+                        if (debug()) {
+                            std::cerr << " => no match\n";
+                        }
+                    }
+
+                    ++n;
+                }
+                return false;
+            }
+
+            void check_inner_outer(ProtoRing& ring) {
+                const osmium::NodeRef& min_node = ring.min_node();
+                if (debug()) {
+                    std::cerr << "    check_inner_outer min_node=" << min_node << "\n";
+                }
+
+                int count = 0;
+                int above = 0;
+
+                for (auto it = m_segment_list.begin(); it != m_segment_list.end() && it->first().location().x() <= min_node.location().x(); ++it) {
+                    if (!ring.contains(*it)) {
+                        if (debug()) {
+                            std::cerr << "      segments for count: " << *it;
+                        }
+                        if (it->to_left_of(min_node.location())) {
+                            ++count;
+                            if (debug()) {
+                                std::cerr << " counted\n";
+                            }
+                        } else {
+                            if (debug()) {
+                                std::cerr << " not counted\n";
+                            }
+                        }
+                        if (it->first().location() == min_node.location()) {
+                            if (it->second().location().y() > min_node.location().y()) {
+                                ++above;
+                            }
+                        }
+                        if (it->second().location() == min_node.location()) {
+                            if (it->first().location().y() > min_node.location().y()) {
+                                ++above;
+                            }
+                        }
+                    }
+                }
+
+                if (debug()) {
+                    std::cerr << "      count=" << count << " above=" << above << "\n";
+                }
+
+                count += above % 2;
+
+                if (count % 2) {
+                    ring.set_inner();
+                }
+            }
+
+            void check_inner_outer_roles() {
+                if (debug()) {
+                    std::cerr << "    check_inner_outer_roles\n";
+                }
+
+                for (const auto ringptr : m_outer_rings) {
+                    for (const auto segment : ringptr->segments()) {
+                        if (!segment.role_outer()) {
+                            ++m_inner_outer_mismatches;
+                            if (debug()) {
+                                std::cerr << "      segment " << segment << " from way " << segment.way()->id() << " should have role 'outer'\n";
+                            }
+                            if (m_config.problem_reporter) {
+                                m_config.problem_reporter->report_role_should_be_outer(segment.way()->id(), segment.first().location(), segment.second().location());
+                            }
+                        }
+                    }
+                }
+                for (const auto ringptr : m_inner_rings) {
+                    for (const auto segment : ringptr->segments()) {
+                        if (!segment.role_inner()) {
+                            ++m_inner_outer_mismatches;
+                            if (debug()) {
+                                std::cerr << "      segment " << segment << " from way " << segment.way()->id() << " should have role 'inner'\n";
+                            }
+                            if (m_config.problem_reporter) {
+                                m_config.problem_reporter->report_role_should_be_inner(segment.way()->id(), segment.first().location(), segment.second().location());
+                            }
+                        }
+                    }
+                }
+            }
+
+            /**
+             * Create rings from segments.
+             */
+            bool create_rings() {
+                m_segment_list.sort();
+                m_segment_list.erase_duplicate_segments();
+
+                // Now we look for segments crossing each other. If there are
+                // any, the multipolygon is invalid.
+                // In the future this could be improved by trying to fix those
+                // cases.
+                if (m_segment_list.find_intersections(m_config.problem_reporter)) {
+                    return false;
+                }
+
+                // Now iterator over all segments and add them to rings. Each segment
+                // is tacked on to either end of an existing ring if possible, or a
+                // new ring is started with it.
+                for (const auto& segment : m_segment_list) {
+                    if (debug()) {
+                        std::cerr << "  checking segment " << segment << "\n";
+                    }
+                    if (!add_to_existing_ring(segment)) {
+                        if (debug()) {
+                            std::cerr << "    new ring for segment " << segment << "\n";
+                        }
+                        m_rings.emplace_back(segment);
+                    }
+                }
+
+                if (debug()) {
+                    std::cerr << "  Rings:\n";
+                    for (const auto& ring : m_rings) {
+                        std::cerr << "    " << ring;
+                        if (ring.closed()) {
+                            std::cerr << " (closed)";
+                        }
+                        std::cerr << "\n";
+                    }
+                }
+
+                if (check_for_open_rings()) {
+                    if (debug()) {
+                        std::cerr << "  not all rings are closed\n";
+                    }
+                    return false;
+                }
+
+                if (debug()) {
+                    std::cerr << "  Find inner/outer...\n";
+                }
+
+                if (m_rings.size() == 1) {
+                    m_outer_rings.push_back(&m_rings.front());
+                } else {
+                    for (auto& ring : m_rings) {
+                        check_inner_outer(ring);
+                        if (ring.outer()) {
+                            if (!ring.is_cw()) {
+                                ring.reverse();
+                            }
+                            m_outer_rings.push_back(&ring);
+                        } else {
+                            if (ring.is_cw()) {
+                                ring.reverse();
+                            }
+                            m_inner_rings.push_back(&ring);
+                        }
+                    }
+
+                    if (m_outer_rings.size() == 1) {
+                        for (auto inner : m_inner_rings) {
+                            m_outer_rings.front()->add_inner_ring(inner);
+                        }
+                    } else {
+                        // sort outer rings by size, smallest first
+                        std::sort(m_outer_rings.begin(), m_outer_rings.end(), [](ProtoRing* a, ProtoRing* b) {
+                            return a->area() < b->area();
+                        });
+                        for (auto inner : m_inner_rings) {
+                            for (auto outer : m_outer_rings) {
+                                if (inner->is_in(outer)) {
+                                    outer->add_inner_ring(inner);
+                                    break;
+                                }
+                            }
+                        }
+                    }
+                }
+
+                check_inner_outer_roles();
+
+                return true;
+            }
+
+        public:
+
+            typedef osmium::area::AssemblerConfig config_type;
+
+            explicit Assembler(const config_type& config) :
+                m_config(config),
+                m_segment_list(config.debug) {
+            }
+
+            ~Assembler() = default;
+
+            /**
+             * Assemble an area from the given way.
+             * The resulting area is put into the out_buffer.
+             */
+            void operator()(const osmium::Way& way, osmium::memory::Buffer& out_buffer) {
+                if (m_config.problem_reporter) {
+                    m_config.problem_reporter->set_object(osmium::item_type::way, way.id());
+                }
+
+                if (!way.ends_have_same_id()) {
+                    if (m_config.problem_reporter) {
+                        m_config.problem_reporter->report_duplicate_node(way.nodes().front().ref(), way.nodes().back().ref(), way.nodes().front().location());
+                    }
+                }
+
+                m_segment_list.extract_segments_from_way(way, "outer");
+
+                if (debug()) {
+                    std::cerr << "\nBuild way id()=" << way.id() << " segments.size()=" << m_segment_list.size() << "\n";
+                }
+
+                // Now create the Area object and add the attributes and tags
+                // from the relation.
+                {
+                    osmium::builder::AreaBuilder builder(out_buffer);
+                    builder.initialize_from_object(way);
+
+                    if (create_rings()) {
+                        add_tags_to_area(builder, way);
+                        add_rings_to_area(builder);
+                    }
+                }
+                out_buffer.commit();
+            }
+
+            /**
+             * Assemble an area from the given relation and its members.
+             * All members are to be found in the in_buffer at the offsets
+             * given by the members parameter.
+             * The resulting area is put into the out_buffer.
+             */
+            void operator()(const osmium::Relation& relation, const std::vector<size_t>& members, const osmium::memory::Buffer& in_buffer, osmium::memory::Buffer& out_buffer) {
+                if (m_config.problem_reporter) {
+                    m_config.problem_reporter->set_object(osmium::item_type::relation, relation.id());
+                }
+
+                m_segment_list.extract_segments_from_ways(relation, members, in_buffer);
+
+                if (debug()) {
+                    std::cerr << "\nBuild relation id()=" << relation.id() << " members.size()=" << members.size() << " segments.size()=" << m_segment_list.size() << "\n";
+                }
+
+                size_t area_offset = out_buffer.committed();
+
+                // Now create the Area object and add the attributes and tags
+                // from the relation.
+                {
+                    osmium::builder::AreaBuilder builder(out_buffer);
+                    builder.initialize_from_object(relation);
+
+                    if (create_rings()) {
+                        add_tags_to_area(builder, relation);
+                        add_rings_to_area(builder);
+                    }
+                }
+                out_buffer.commit();
+
+                const osmium::TagList& area_tags = out_buffer.get<osmium::Area>(area_offset).tags(); // tags of the area we just built
+
+                // Find all closed ways that are inner rings and check their
+                // tags. If they are not the same as the tags of the area we
+                // just built, add them to a list and later build areas for
+                // them, too.
+                std::vector<const osmium::Way*> ways_that_should_be_areas;
+                if (m_inner_outer_mismatches == 0) {
+                    auto memit = relation.members().begin();
+                    for (size_t offset : members) {
+                        if (!std::strcmp(memit->role(), "inner")) {
+                            const osmium::Way& way = in_buffer.get<const osmium::Way>(offset);
+                            if (way.is_closed() && way.tags().size() > 0) {
+                                auto d = std::count_if(way.tags().begin(), way.tags().end(), filter());
+                                if (d > 0) {
+                                    osmium::tags::KeyFilter::iterator way_fi_begin(filter(), way.tags().begin(), way.tags().end());
+                                    osmium::tags::KeyFilter::iterator way_fi_end(filter(), way.tags().end(), way.tags().end());
+                                    osmium::tags::KeyFilter::iterator area_fi_begin(filter(), area_tags.begin(), area_tags.end());
+                                    osmium::tags::KeyFilter::iterator area_fi_end(filter(), area_tags.end(), area_tags.end());
+
+                                    if (!std::equal(way_fi_begin, way_fi_end, area_fi_begin) || d != std::distance(area_fi_begin, area_fi_end)) {
+                                        ways_that_should_be_areas.push_back(&way);
+                                    }
+                                }
+                            }
+                        }
+                        ++memit;
+                    }
+                }
+
+                // Now build areas for all ways found in the last step.
+                for (const osmium::Way* way : ways_that_should_be_areas) {
+                    Assembler assembler(m_config);
+                    assembler(*way, out_buffer);
+                }
+            }
+
+        }; // class Assembler
+
+    } // namespace area
+
+} // namespace osmium
+
+#endif // OSMIUM_AREA_ASSEMBLER_HPP

include/osmium/area/detail/node_ref_segment.hpp

@@ -0,0 +1,262 @@
+#ifndef OSMIUM_AREA_DETAIL_NODE_REF_SEGMENT_HPP
+#define OSMIUM_AREA_DETAIL_NODE_REF_SEGMENT_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <algorithm>
+#include <cstdint>
+#include <cstring>
+#include <iosfwd>
+#include <utility>
+
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/node_ref.hpp>
+
+namespace osmium {
+
+    class Way;
+
+    namespace area {
+
+        /**
+         * @brief Namespace for Osmium internal use
+         */
+        namespace detail {
+
+            /**
+             * This helper class for the Assembler class models a segment.
+             * Segments are the connection between
+             * two nodes and they all have their smaller coordinate at the
+             * beginning of the segment. Smaller, in this case, means smaller x
+             * coordinate, and if they are the same smaller y coordinate.
+             */
+            class NodeRefSegment {
+
+                osmium::NodeRef m_first;
+                osmium::NodeRef m_second;
+
+                /// Role of the member this segment was from.
+                const char* m_role;
+
+                /// Way this segment was from.
+                const osmium::Way* m_way;
+
+            public:
+
+                void swap_locations() {
+                    using std::swap;
+                    swap(m_first, m_second);
+                }
+
+                explicit NodeRefSegment() noexcept :
+                    m_first(),
+                    m_second(),
+                    m_role(nullptr),
+                    m_way(nullptr) {
+                }
+
+                explicit NodeRefSegment(const osmium::NodeRef& nr1, const osmium::NodeRef& nr2, const char* role, const osmium::Way* way) :
+                    m_first(nr1),
+                    m_second(nr2),
+                    m_role(role),
+                    m_way(way) {
+                    if (nr2.location() < nr1.location()) {
+                        swap_locations();
+                    }
+                }
+
+                NodeRefSegment(const NodeRefSegment&) = default;
+                NodeRefSegment(NodeRefSegment&&) = default;
+
+                NodeRefSegment& operator=(const NodeRefSegment&) = default;
+                NodeRefSegment& operator=(NodeRefSegment&&) = default;
+
+                ~NodeRefSegment() = default;
+
+                /// Return first NodeRef of Segment according to sorting order (bottom left to top right).
+                const osmium::NodeRef& first() const noexcept {
+                    return m_first;
+                }
+
+                /// Return second NodeRef of Segment according to sorting order (bottom left to top right).
+                const osmium::NodeRef& second() const noexcept {
+                    return m_second;
+                }
+
+                bool to_left_of(const osmium::Location location) const {
+    //                std::cerr << "segment " << first() << "--" << second() << " to_left_of(" << location << "\n";
+
+                    if (first().location() == location || second().location() == location) {
+                        return false;
+                    }
+
+                    const std::pair<osmium::Location, osmium::Location> mm = std::minmax(first().location(), second().location(), [](const osmium::Location a, const osmium::Location b) {
+                        return a.y() < b.y();
+                    });
+
+                    if (mm.first.y() >= location.y() || mm.second.y() < location.y() || first().location().x() > location.x()) {
+    //                    std::cerr << "  false\n";
+                        return false;
+                    }
+
+                    int64_t ax = mm.first.x();
+                    int64_t bx = mm.second.x();
+                    int64_t lx = location.x();
+                    int64_t ay = mm.first.y();
+                    int64_t by = mm.second.y();
+                    int64_t ly = location.y();
+                    return ((bx - ax)*(ly - ay) - (by - ay)*(lx - ax)) <= 0;
+                }
+
+                bool role_outer() const noexcept {
+                    return !strcmp(m_role, "outer");
+                }
+
+                bool role_inner() const noexcept {
+                    return !strcmp(m_role, "inner");
+                }
+
+                const osmium::Way* way() const noexcept {
+                    return m_way;
+                }
+
+            }; // class NodeRefSegment
+
+            /// NodeRefSegments are equal if both their locations are equal
+            inline bool operator==(const NodeRefSegment& lhs, const NodeRefSegment& rhs) noexcept {
+                return lhs.first().location() == rhs.first().location() && lhs.second().location() == rhs.second().location();
+            }
+
+            inline bool operator!=(const NodeRefSegment& lhs, const NodeRefSegment& rhs) noexcept {
+                return ! (lhs == rhs);
+            }
+
+            /**
+             * NodeRefSegments are "smaller" if they are to the left and down of another
+             * segment. The first() location is checked first() and only if they have the
+             * same first() location the second() location is taken into account.
+             */
+            inline bool operator<(const NodeRefSegment& lhs, const NodeRefSegment& rhs) noexcept {
+                return (lhs.first().location() == rhs.first().location() && lhs.second().location() < rhs.second().location()) || lhs.first().location() < rhs.first().location();
+            }
+
+            inline bool operator>(const NodeRefSegment& lhs, const NodeRefSegment& rhs) noexcept {
+                return rhs < lhs;
+            }
+
+            inline bool operator<=(const NodeRefSegment& lhs, const NodeRefSegment& rhs) noexcept {
+                return ! (rhs < lhs);
+            }
+
+            inline bool operator>=(const NodeRefSegment& lhs, const NodeRefSegment& rhs) noexcept {
+                return ! (lhs < rhs);
+            }
+
+            template <typename TChar, typename TTraits>
+            inline std::basic_ostream<TChar, TTraits>& operator<<(std::basic_ostream<TChar, TTraits>& out, const NodeRefSegment& segment) {
+                return out << segment.first() << "--" << segment.second();
+            }
+
+            inline bool outside_x_range(const NodeRefSegment& s1, const NodeRefSegment& s2) noexcept {
+                if (s1.first().location().x() > s2.second().location().x()) {
+                    return true;
+                }
+                return false;
+            }
+
+            inline bool y_range_overlap(const NodeRefSegment& s1, const NodeRefSegment& s2) {
+                auto m1 = std::minmax(s1.first().location().y(), s1.second().location().y());
+                auto m2 = std::minmax(s2.first().location().y(), s2.second().location().y());
+                if (m1.first > m2.second || m2.first > m1.second) {
+                    return false;
+                }
+                return true;
+            }
+
+            /**
+             * Calculate the intersection between to NodeRefSegments. The result is returned
+             * as a Location. Note that because the Location uses integers with limited
+             * precision internally, the result might be slightly different than the
+             * numerically correct location.
+             *
+             * If the segments touch in one of their endpoints, it doesn't count as an
+             * intersection.
+             *
+             * If the segments intersect not in a single point but in multiple points, ie
+             * if they overlap, this is NOT detected.
+             *
+             * @returns Undefined osmium::Location if there is no intersection or a defined
+             *          Location if the segments intersect.
+             */
+            inline osmium::Location calculate_intersection(const NodeRefSegment& s1, const NodeRefSegment& s2) {
+                if (s1.first().location()  == s2.first().location()  ||
+                    s1.first().location()  == s2.second().location() ||
+                    s1.second().location() == s2.first().location()  ||
+                    s1.second().location() == s2.second().location()) {
+                    return osmium::Location();
+                }
+
+                auto d = (static_cast<int64_t>(s2.second().y()) - static_cast<int64_t>(s2.first().y())) *
+                         (static_cast<int64_t>(s1.second().x()) - static_cast<int64_t>(s1.first().x())) -
+                         (static_cast<int64_t>(s2.second().x()) - static_cast<int64_t>(s2.first().x())) *
+                         (static_cast<int64_t>(s1.second().y()) - static_cast<int64_t>(s1.first().y()));
+
+                if (d != 0) {
+                    double denom  = ((s2.second().lat() - s2.first().lat())*(s1.second().lon() - s1.first().lon())) -
+                                    ((s2.second().lon() - s2.first().lon())*(s1.second().lat() - s1.first().lat()));
+
+                    double nume_a = ((s2.second().lon() - s2.first().lon())*(s1.first().lat() - s2.first().lat())) -
+                                    ((s2.second().lat() - s2.first().lat())*(s1.first().lon() - s2.first().lon()));
+
+                    double nume_b = ((s1.second().lon() - s1.first().lon())*(s1.first().lat() - s2.first().lat())) -
+                                    ((s1.second().lat() - s1.first().lat())*(s1.first().lon() - s2.first().lon()));
+
+                    if ((denom > 0 && nume_a >= 0 && nume_a <= denom && nume_b >= 0 && nume_b <= denom) ||
+                        (denom < 0 && nume_a <= 0 && nume_a >= denom && nume_b <= 0 && nume_b >= denom)) {
+                        double ua = nume_a / denom;
+                        double ix = s1.first().lon() + ua*(s1.second().lon() - s1.first().lon());
+                        double iy = s1.first().lat() + ua*(s1.second().lat() - s1.first().lat());
+                        return osmium::Location(ix, iy);
+                    }
+                }
+
+                return osmium::Location();
+            }
+
+        } // namespace detail
+
+    } // namespace area
+
+} // namespace osmium
+
+#endif // OSMIUM_AREA_DETAIL_NODE_REF_SEGMENT_HPP

include/osmium/area/detail/proto_ring.hpp

@@ -0,0 +1,274 @@
+#ifndef OSMIUM_AREA_DETAIL_PROTO_RING_HPP
+#define OSMIUM_AREA_DETAIL_PROTO_RING_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <algorithm>
+#include <cassert>
+#include <iostream>
+#include <list>
+#include <set>
+#include <vector>
+
+#include <osmium/osm/node_ref.hpp>
+#include <osmium/area/detail/node_ref_segment.hpp>
+
+namespace osmium {
+
+    namespace area {
+
+        namespace detail {
+
+            /**
+             * A ring in the process of being built by the Assembler object.
+             */
+            class ProtoRing {
+
+            public:
+
+                typedef std::vector<NodeRefSegment> segments_type;
+
+            private:
+
+                // segments in this ring
+                segments_type m_segments;
+
+                bool m_outer {true};
+
+                // if this is an outer ring, these point to it's inner rings (if any)
+                std::vector<ProtoRing*> m_inner;
+
+            public:
+
+                explicit ProtoRing(const NodeRefSegment& segment) noexcept :
+                    m_segments() {
+                    add_segment_back(segment);
+                }
+
+                explicit ProtoRing(segments_type::const_iterator sbegin, segments_type::const_iterator send) :
+                    m_segments(static_cast<size_t>(std::distance(sbegin, send))) {
+                    std::copy(sbegin, send, m_segments.begin());
+                }
+
+                bool outer() const noexcept {
+                    return m_outer;
+                }
+
+                void set_inner() noexcept {
+                    m_outer = false;
+                }
+
+                segments_type& segments() noexcept {
+                    return m_segments;
+                }
+
+                const segments_type& segments() const noexcept {
+                    return m_segments;
+                }
+
+                void remove_segments(segments_type::iterator sbegin, segments_type::iterator send) {
+                    m_segments.erase(sbegin, send);
+                }
+
+                void add_segment_front(const NodeRefSegment& segment) {
+                    m_segments.insert(m_segments.begin(), segment);
+                }
+
+                void add_segment_back(const NodeRefSegment& segment) {
+                    m_segments.push_back(segment);
+                }
+
+                const NodeRefSegment& get_segment_front() const {
+                    return m_segments.front();
+                }
+
+                NodeRefSegment& get_segment_front() {
+                    return m_segments.front();
+                }
+
+                const NodeRefSegment& get_segment_back() const {
+                    return m_segments.back();
+                }
+
+                NodeRefSegment& get_segment_back() {
+                    return m_segments.back();
+                }
+
+                bool closed() const {
+                    return m_segments.front().first().location() == m_segments.back().second().location();
+                }
+
+                int64_t sum() const {
+                    int64_t sum = 0;
+
+                    for (const auto& segment : m_segments) {
+                        sum += static_cast<int64_t>(segment.first().location().x()) * static_cast<int64_t>(segment.second().location().y()) -
+                               static_cast<int64_t>(segment.second().location().x()) * static_cast<int64_t>(segment.first().location().y());
+                    }
+
+                    return sum;
+                }
+
+                bool is_cw() const {
+                    return sum() <= 0;
+                }
+
+                int64_t area() const {
+                    return std::abs(sum()) / 2;
+                }
+
+                void swap_segments(ProtoRing& other) {
+                    std::swap(m_segments, other.m_segments);
+                }
+
+                void add_inner_ring(ProtoRing* ring) {
+                    m_inner.push_back(ring);
+                }
+
+                const std::vector<ProtoRing*>& inner_rings() const {
+                    return m_inner;
+                }
+
+                void print(std::ostream& out) const {
+                    out << "[";
+                    bool first = true;
+                    for (const auto& segment : m_segments) {
+                        if (first) {
+                            out << segment.first().ref();
+                        }
+                        out << ',' << segment.second().ref();
+                        first = false;
+                    }
+                    out << "]";
+                }
+
+                void reverse() {
+                    std::for_each(m_segments.begin(), m_segments.end(), [](NodeRefSegment& segment) {
+                        segment.swap_locations();
+                    });
+                    std::reverse(m_segments.begin(), m_segments.end());
+                }
+
+                /**
+                 * Merge other ring to end of this ring.
+                 */
+                void merge_ring(const ProtoRing& other, bool debug) {
+                    if (debug) {
+                        std::cerr << "        MERGE rings ";
+                        print(std::cerr);
+                        std::cerr << " to ";
+                        other.print(std::cerr);
+                        std::cerr << "\n";
+                    }
+                    m_segments.insert(m_segments.end(), other.m_segments.begin(), other.m_segments.end());
+                    if (debug) {
+                        std::cerr << "          result ring: ";
+                        print(std::cerr);
+                        std::cerr << "\n";
+                    }
+                }
+
+                void merge_ring_reverse(const ProtoRing& other, bool debug) {
+                    if (debug) {
+                        std::cerr << "        MERGE rings (reverse) ";
+                        print(std::cerr);
+                        std::cerr << " to ";
+                        other.print(std::cerr);
+                        std::cerr << "\n";
+                    }
+                    size_t n = m_segments.size();
+                    m_segments.resize(n + other.m_segments.size());
+                    std::transform(other.m_segments.rbegin(), other.m_segments.rend(), m_segments.begin() + static_cast<segments_type::difference_type>(n), [](NodeRefSegment segment) {
+                        segment.swap_locations();
+                        return segment;
+                    });
+                    if (debug) {
+                        std::cerr << "          result ring: ";
+                        print(std::cerr);
+                        std::cerr << "\n";
+                    }
+                }
+
+                const NodeRef& min_node() const {
+                    auto it = std::min_element(m_segments.begin(), m_segments.end());
+                    if (location_less()(it->first(), it->second())) {
+                        return it->first();
+                    } else {
+                        return it->second();
+                    }
+                }
+
+                bool is_in(ProtoRing* outer) {
+                    osmium::Location testpoint = segments().front().first().location();
+                    bool is_in = false;
+
+                    for (size_t i = 0, j = outer->segments().size()-1; i < outer->segments().size(); j = i++) {
+                        if (((outer->segments()[i].first().location().y() > testpoint.y()) != (outer->segments()[j].first().location().y() > testpoint.y())) &&
+                            (testpoint.x() < (outer->segments()[j].first().location().x() - outer->segments()[i].first().location().x()) * (testpoint.y() - outer->segments()[i].first().location().y()) / (outer->segments()[j].first().location().y() - outer->segments()[i].first().location().y()) + outer->segments()[i].first().location().x()) ) {
+                            is_in = !is_in;
+                        }
+                    }
+
+                    return is_in;
+                }
+
+                void get_ways(std::set<const osmium::Way*>& ways) {
+                    for (const auto& segment : m_segments) {
+                        ways.insert(segment.way());
+                    }
+                }
+
+                bool contains(const NodeRefSegment& segment) const {
+                    for (const auto& s : m_segments) {
+                        if (s == segment || (s.first() == segment.second() && s.second() == segment.first())) {
+                            return true;
+                        }
+                    }
+                    return false;
+                }
+
+            }; // class ProtoRing
+
+            template <typename TChar, typename TTraits>
+            inline std::basic_ostream<TChar, TTraits>& operator<<(std::basic_ostream<TChar, TTraits>& out, const ProtoRing& ring) {
+                ring.print(out);
+                return out;
+            }
+
+        } // namespace detail
+
+    } // namespace area
+
+} // namespace osmium
+
+#endif // OSMIUM_AREA_DETAIL_PROTO_RING_HPP

include/osmium/area/detail/segment_list.hpp

@@ -0,0 +1,216 @@
+#ifndef OSMIUM_AREA_DETAIL_SEGMENT_LIST_HPP
+#define OSMIUM_AREA_DETAIL_SEGMENT_LIST_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <algorithm>
+#include <cassert>
+#include <iostream>
+#include <vector>
+
+#include <osmium/area/problem_reporter.hpp>
+#include <osmium/area/detail/node_ref_segment.hpp>
+#include <osmium/memory/buffer.hpp>
+#include <osmium/osm/relation.hpp>
+#include <osmium/osm/way.hpp>
+
+namespace osmium {
+
+    namespace area {
+
+        namespace detail {
+
+            /**
+             * This is a helper class for the area assembler. It models
+             * a list of segments.
+             */
+            class SegmentList {
+
+                typedef std::vector<NodeRefSegment> slist_type;
+
+                slist_type m_segments;
+
+                bool m_debug;
+
+            public:
+
+                explicit SegmentList(bool debug) noexcept :
+                    m_debug(debug) {
+                }
+
+                ~SegmentList() = default;
+
+                SegmentList(const SegmentList&) = delete;
+                SegmentList(SegmentList&&) = delete;
+
+                SegmentList& operator=(const SegmentList&) = delete;
+                SegmentList& operator=(SegmentList&&) = delete;
+
+                /// The number of segments in the list.
+                size_t size() const noexcept {
+                    return m_segments.size();
+                }
+
+                bool empty() const noexcept {
+                    return m_segments.empty();
+                }
+
+                typedef slist_type::const_iterator const_iterator;
+
+                const_iterator begin() const noexcept {
+                    return m_segments.begin();
+                }
+
+                const_iterator end() const noexcept {
+                    return m_segments.end();
+                }
+
+                /**
+                 * Enable or disable debug output to stderr. This is for Osmium
+                 * developers only.
+                 */
+                void enable_debug_output(bool debug=true) noexcept {
+                    m_debug = debug;
+                }
+
+                /// Clear the list of segments. All segments are removed.
+                void clear() {
+                    m_segments.clear();
+                }
+
+                /// Sort the list of segments.
+                void sort() {
+                    std::sort(m_segments.begin(), m_segments.end());
+                }
+
+                /**
+                 * Extract segments from given way and add them to the list.
+                 *
+                 * Segments connecting two nodes with the same location (ie same
+                 * node or different node with same location) are removed.
+                 *
+                 * XXX should two nodes with same location be reported?
+                 */
+                void extract_segments_from_way(const osmium::Way& way, const char* role) {
+                    osmium::NodeRef last_nr;
+                    for (const osmium::NodeRef& nr : way.nodes()) {
+                        if (last_nr.location() && last_nr.location() != nr.location()) {
+                            m_segments.emplace_back(last_nr, nr, role, &way);
+                        }
+                        last_nr = nr;
+                    }
+                }
+
+                /**
+                 * Extract all segments from all ways that make up this
+                 * multipolygon relation and add them to the list.
+                 */
+                void extract_segments_from_ways(const osmium::Relation& relation, const std::vector<size_t>& members, const osmium::memory::Buffer& in_buffer) {
+                    auto member_it = relation.members().begin();
+                    for (size_t offset : members) {
+                        const osmium::Way& way = in_buffer.get<const osmium::Way>(offset);
+                        extract_segments_from_way(way, member_it->role());
+                        ++member_it;
+                    }
+                }
+
+                /**
+                 * Find duplicate segments (ie same start and end point) in the
+                 * list and remove them. This will always remove pairs of the same
+                 * segment. So if there are three, for instance, two will be
+                 * removed and one will be left.
+                 */
+                void erase_duplicate_segments() {
+                    while (true) {
+                        auto it = std::adjacent_find(m_segments.begin(), m_segments.end());
+                        if (it == m_segments.end()) {
+                            return;
+                        }
+                        if (m_debug) {
+                            std::cerr << "  erase duplicate segment: " << *it << "\n";
+                        }
+                        m_segments.erase(it, it+2);
+                    }
+                }
+
+                /**
+                 * Find intersection between segments.
+                 *
+                 * @param problem_reporter Any intersections found are reported to this object.
+                 * @returns true if there are intersections.
+                 */
+                bool find_intersections(osmium::area::ProblemReporter* problem_reporter) const {
+                    if (m_segments.empty()) {
+                        return false;
+                    }
+
+                    bool found_intersections = false;
+
+                    for (auto it1 = m_segments.begin(); it1 != m_segments.end()-1; ++it1) {
+                        const NodeRefSegment& s1 = *it1;
+                        for (auto it2 = it1+1; it2 != m_segments.end(); ++it2) {
+                            const NodeRefSegment& s2 = *it2;
+
+                            assert(s1 != s2); // erase_duplicate_segments() should have made sure of that
+
+                            if (outside_x_range(s2, s1)) {
+                                break;
+                            }
+
+                            if (y_range_overlap(s1, s2)) {
+                                osmium::Location intersection = calculate_intersection(s1, s2);
+                                if (intersection) {
+                                    found_intersections = true;
+                                    if (m_debug) {
+                                        std::cerr << "  segments " << s1 << " and " << s2 << " intersecting at " << intersection << "\n";
+                                    }
+                                    if (problem_reporter) {
+                                        problem_reporter->report_intersection(s1.way()->id(), s1.first().location(), s1.second().location(), s2.way()->id(), s2.first().location(), s2.second().location(), intersection);
+                                    }
+                                }
+                            }
+                        }
+                    }
+
+                    return found_intersections;
+                }
+
+            }; // class SegmentList
+
+        } // namespace detail
+
+    } // namespace area
+
+} // namespace osmium
+
+#endif // OSMIUM_AREA_DETAIL_SEGMENT_LIST_HPP

include/osmium/area/multipolygon_collector.hpp

@@ -0,0 +1,212 @@
+#ifndef OSMIUM_AREA_MULTIPOLYGON_COLLECTOR_HPP
+#define OSMIUM_AREA_MULTIPOLYGON_COLLECTOR_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
+#include <cstring>
+#include <vector>
+
+#include <osmium/memory/buffer.hpp>
+#include <osmium/osm/item_type.hpp>
+#include <osmium/osm/relation.hpp>
+#include <osmium/osm/tag.hpp>
+#include <osmium/osm/way.hpp>
+#include <osmium/relations/collector.hpp>
+#include <osmium/relations/detail/member_meta.hpp>
+
+namespace osmium {
+
+    struct invalid_location;
+
+    namespace relations {
+        class RelationMeta;
+    }
+
+    /**
+     * @brief Code related to the building of areas (multipolygons) from relations.
+     */
+    namespace area {
+
+        /**
+         * This class collects all data needed for creating areas from
+         * relations tagged with type=multipolygon or type=boundary.
+         * Most of its functionality is derived from the parent class
+         * osmium::relations::Collector.
+         *
+         * The actual assembling of the areas is done by the assembler
+         * class given as template argument.
+         *
+         * @tparam TAssembler Multipolygon Assembler class.
+         */
+        template <class TAssembler>
+        class MultipolygonCollector : public osmium::relations::Collector<MultipolygonCollector<TAssembler>, false, true, false> {
+
+            typedef typename osmium::relations::Collector<MultipolygonCollector<TAssembler>, false, true, false> collector_type;
+
+            typedef typename TAssembler::config_type assembler_config_type;
+            const assembler_config_type m_assembler_config;
+
+            osmium::memory::Buffer m_output_buffer;
+
+            static constexpr size_t initial_output_buffer_size = 1024 * 1024;
+            static constexpr size_t max_buffer_size_for_flush = 100 * 1024;
+
+            void flush_output_buffer() {
+                if (this->callback()) {
+                    osmium::memory::Buffer buffer(initial_output_buffer_size);
+                    std::swap(buffer, m_output_buffer);
+                    this->callback()(std::move(buffer));
+                }
+            }
+
+            void possibly_flush_output_buffer() {
+                if (m_output_buffer.committed() > max_buffer_size_for_flush) {
+                    flush_output_buffer();
+                }
+            }
+
+        public:
+
+            explicit MultipolygonCollector(const assembler_config_type& assembler_config) :
+                collector_type(),
+                m_assembler_config(assembler_config),
+                m_output_buffer(initial_output_buffer_size, osmium::memory::Buffer::auto_grow::yes) {
+            }
+
+            /**
+             * We are interested in all relations tagged with type=multipolygon or
+             * type=boundary.
+             *
+             * Overwritten from the base class.
+             */
+            bool keep_relation(const osmium::Relation& relation) const {
+                const char* type = relation.tags().get_value_by_key("type");
+
+                // ignore relations without "type" tag
+                if (!type) {
+                    return false;
+                }
+
+                if ((!strcmp(type, "multipolygon")) || (!strcmp(type, "boundary"))) {
+                    return true;
+                }
+
+                return false;
+            }
+
+            /**
+             * Overwritten from the base class.
+             */
+            bool keep_member(const osmium::relations::RelationMeta& /*relation_meta*/, const osmium::RelationMember& member) const {
+                // We are only interested in members of type way.
+                return member.type() == osmium::item_type::way;
+            }
+
+            /**
+             * This is called when a way is not in any multipolygon
+             * relation.
+             *
+             * Overwritten from the base class.
+             */
+            void way_not_in_any_relation(const osmium::Way& way) {
+                if (way.ends_have_same_location() && way.nodes().size() > 3) {
+                    // way is closed and has enough nodes, build simple multipolygon
+                    try {
+                        TAssembler assembler(m_assembler_config);
+                        assembler(way, m_output_buffer);
+                        possibly_flush_output_buffer();
+                    } catch (osmium::invalid_location&) {
+                        // XXX ignore
+                    }
+                }
+            }
+
+            void complete_relation(osmium::relations::RelationMeta& relation_meta) {
+                const osmium::Relation& relation = this->get_relation(relation_meta);
+                std::vector<size_t> offsets;
+                for (const auto& member : relation.members()) {
+                    if (member.ref() != 0) {
+                        offsets.push_back(this->get_offset(member.type(), member.ref()));
+                    }
+                }
+                try {
+                    TAssembler assembler(m_assembler_config);
+                    assembler(relation, offsets, this->members_buffer(), m_output_buffer);
+                    possibly_flush_output_buffer();
+                } catch (osmium::invalid_location&) {
+                    // XXX ignore
+                }
+
+                // clear member metas
+                for (const auto& member : relation.members()) {
+                    if (member.ref() != 0) {
+                        auto& mmv = this->member_meta(member.type());
+                        auto range = std::equal_range(mmv.begin(), mmv.end(), osmium::relations::MemberMeta(member.ref()));
+                        assert(range.first != range.second);
+
+                        // if this is the last time this object was needed
+                        // then mark it as removed
+                        if (osmium::relations::count_not_removed(range.first, range.second) == 1) {
+                            this->get_member(range.first->buffer_offset()).set_removed(true);
+                        }
+
+                        for (auto it = range.first; it != range.second; ++it) {
+                            if (!it->removed() && relation.id() == this->get_relation(it->relation_pos()).id()) {
+                                it->remove();
+                                break;
+                            }
+                        }
+                    }
+                }
+            }
+
+            void flush() {
+                flush_output_buffer();
+            }
+
+            osmium::memory::Buffer read() {
+                osmium::memory::Buffer buffer(initial_output_buffer_size, osmium::memory::Buffer::auto_grow::yes);
+                std::swap(buffer, m_output_buffer);
+                return buffer;
+            }
+
+        }; // class MultipolygonCollector
+
+    } // namespace area
+
+} // namespace osmium
+
+#endif // OSMIUM_AREA_MULTIPOLYGON_COLLECTOR_HPP

include/osmium/area/problem_reporter.hpp

@@ -0,0 +1,149 @@
+#ifndef OSMIUM_AREA_PROBLEM_REPORTER_HPP
+#define OSMIUM_AREA_PROBLEM_REPORTER_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <osmium/osm/item_type.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/types.hpp>
+
+namespace osmium {
+
+    namespace area {
+
+        /**
+         * When assembling a multipolygon/area from a multipolygon relation
+         * or a closed way several problems can be detected. This includes
+         * intersections between lines, wrong role attributes on relation
+         * members etc. These problems are reported by the area::Assembler
+         * class to the ProblemReporter class or one of its child classes.
+         *
+         * This is the parent class which does nothing with the reports.
+         * Child classes are expected to implement different ways of
+         * reporting the problems.
+         */
+        class ProblemReporter {
+
+        protected:
+
+            // Type of object we are currently working on
+            osmium::item_type m_object_type;
+
+            // ID of the relation/way we are currently working on
+            osmium::object_id_type m_object_id;
+
+        public:
+
+            ProblemReporter() = default;
+
+            virtual ~ProblemReporter() = default;
+
+            /**
+             * Set the object the next problem reports will be on.
+             *
+             * @param object_type The type of the object.
+             * @param object_id The ID of the object.
+             */
+            void set_object(osmium::item_type object_type, osmium::object_id_type object_id) noexcept {
+                m_object_type = object_type;
+                m_object_id = object_id;
+            }
+
+// Disable "unused-parameter" warning, so that the compiler will not complain.
+// We can't remove the parameter names, because then doxygen will complain.
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+
+            /**
+             * Report a duplicate node, ie. two nodes with the same location.
+             *
+             * @param node_id1       ID of the first node.
+             * @param node_id2       ID of the second node.
+             * @param location       Location of both nodes.
+             */
+            virtual void report_duplicate_node(osmium::object_id_type node_id1, osmium::object_id_type node_id2, osmium::Location location) {
+            }
+
+            /**
+             * Report an intersection between two segments.
+             *
+             * @param way1_id        ID of the first involved way.
+             * @param way1_seg_start Location where the segment of the first way with the intersection starts
+             * @param way1_seg_end   Location where the segment of the first way with the intersection ends
+             * @param way2_id        ID of the second involved way.
+             * @param way2_seg_start Location where the segment of the second way with the intersection starts
+             * @param way2_seg_end   Location where the segment of the second way with the intersection ends
+             * @param intersection   Location of the intersection. This might be slightly off the correct location due to rounding.
+             */
+            virtual void report_intersection(osmium::object_id_type way1_id, osmium::Location way1_seg_start, osmium::Location way1_seg_end,
+                                             osmium::object_id_type way2_id, osmium::Location way2_seg_start, osmium::Location way2_seg_end, osmium::Location intersection) {
+            }
+
+            /**
+             * Report an open ring.
+             *
+             * @param end1           Location of the first open end.
+             * @param end2           Location of the second open end.
+             */
+            virtual void report_ring_not_closed(osmium::Location end1, osmium::Location end2) {
+            }
+
+            /**
+             * Report a segment that should have role "outer", but has a different role.
+             *
+             * @param way_id         ID of the way this segment is in.
+             * @param seg_start      Start of the segment with the wrong role.
+             * @param seg_end        End of the segment with the wrong role.
+             */
+            virtual void report_role_should_be_outer(osmium::object_id_type way_id, osmium::Location seg_start, osmium::Location seg_end) {
+            }
+
+            /**
+             * Report a segment that should have role "inner", but has a different role.
+             *
+             * @param way_id         ID of the way this segment is in.
+             * @param seg_start      Start of the segment with the wrong role.
+             * @param seg_end        End of the segment with the wrong role.
+             */
+            virtual void report_role_should_be_inner(osmium::object_id_type way_id, osmium::Location seg_start, osmium::Location seg_end) {
+            }
+
+#pragma GCC diagnostic pop
+
+        }; // class ProblemReporter
+
+    } // namespace area
+
+} // namespace osmium
+
+#endif // OSMIUM_AREA_PROBLEM_REPORTER_HPP

include/osmium/area/problem_reporter_exception.hpp

@@ -0,0 +1,96 @@
+#ifndef OSMIUM_AREA_PROBLEM_REPORTER_EXCEPTION_HPP
+#define OSMIUM_AREA_PROBLEM_REPORTER_EXCEPTION_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <sstream>
+#include <stdexcept>
+
+#include <osmium/area/problem_reporter_stream.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/types.hpp>
+
+namespace osmium {
+
+    namespace area {
+
+        class ProblemReporterException : public ProblemReporterStream {
+
+            std::stringstream m_sstream;
+
+        public:
+
+            ProblemReporterException() :
+                ProblemReporterStream(m_sstream) {
+            }
+
+            virtual ~ProblemReporterException() = default;
+
+            void report_duplicate_node(osmium::object_id_type node_id1, osmium::object_id_type node_id2, osmium::Location location) override {
+                m_sstream.str();
+                ProblemReporterStream::report_duplicate_node(node_id1, node_id2, location);
+                throw std::runtime_error(m_sstream.str());
+            }
+
+            void report_intersection(osmium::object_id_type way1_id, osmium::Location way1_seg_start, osmium::Location way1_seg_end,
+                                     osmium::object_id_type way2_id, osmium::Location way2_seg_start, osmium::Location way2_seg_end, osmium::Location intersection) override {
+                m_sstream.str();
+                ProblemReporterStream::report_intersection(way1_id, way1_seg_start, way1_seg_end, way2_id, way2_seg_start, way2_seg_end, intersection);
+                throw std::runtime_error(m_sstream.str());
+            }
+
+            void report_ring_not_closed(osmium::Location end1, osmium::Location end2) override {
+                m_sstream.str();
+                ProblemReporterStream::report_ring_not_closed(end1, end2);
+                throw std::runtime_error(m_sstream.str());
+            }
+
+            void report_role_should_be_outer(osmium::object_id_type way_id, osmium::Location seg_start, osmium::Location seg_end) override {
+                m_sstream.str();
+                ProblemReporterStream::report_role_should_be_outer(way_id, seg_start, seg_end);
+                throw std::runtime_error(m_sstream.str());
+            }
+
+            void report_role_should_be_inner(osmium::object_id_type way_id, osmium::Location seg_start, osmium::Location seg_end) override {
+                m_sstream.str();
+                ProblemReporterStream::report_role_should_be_inner(way_id, seg_start, seg_end);
+                throw std::runtime_error(m_sstream.str());
+            }
+
+        }; // class ProblemReporterException
+
+    } // namespace area
+
+} // namespace osmium
+
+#endif // OSMIUM_AREA_PROBLEM_REPORTER_EXCEPTION_HPP

include/osmium/area/problem_reporter_ogr.hpp

@@ -0,0 +1,203 @@
+#ifndef OSMIUM_AREA_PROBLEM_REPORTER_OGR_HPP
+#define OSMIUM_AREA_PROBLEM_REPORTER_OGR_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#pragma GCC diagnostic push
+#ifdef __clang__
+# pragma GCC diagnostic ignored "-Wdocumentation-unknown-command"
+#endif
+#pragma GCC diagnostic ignored "-Wfloat-equal"
+#pragma GCC diagnostic ignored "-Wold-style-cast"
+#pragma GCC diagnostic ignored "-Wpadded"
+#pragma GCC diagnostic ignored "-Wredundant-decls"
+#pragma GCC diagnostic ignored "-Wshadow"
+# include <ogr_api.h>
+# include <ogrsf_frmts.h>
+#pragma GCC diagnostic pop
+
+#include <memory>
+#include <stdexcept>
+
+#include <osmium/area/problem_reporter.hpp>
+#include <osmium/geom/ogr.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/types.hpp>
+
+namespace osmium {
+
+    namespace area {
+
+        /**
+         * Report problems when assembling areas by adding them to
+         * layers in an OGR datasource.
+         */
+        class ProblemReporterOGR : public ProblemReporter {
+
+            osmium::geom::OGRFactory<> m_ogr_factory;
+
+            OGRDataSource* m_data_source;
+
+            OGRLayer* m_layer_perror;
+            OGRLayer* m_layer_lerror;
+
+            void write_point(const char* problem_type, osmium::object_id_type id1, osmium::object_id_type id2, osmium::Location location) {
+                OGRFeature* feature = OGRFeature::CreateFeature(m_layer_perror->GetLayerDefn());
+                std::unique_ptr<OGRPoint> ogr_point = m_ogr_factory.create_point(location);
+                feature->SetGeometry(ogr_point.get());
+                feature->SetField("id1", static_cast<double>(id1));
+                feature->SetField("id2", static_cast<double>(id2));
+                feature->SetField("problem_type", problem_type);
+
+                if (m_layer_perror->CreateFeature(feature) != OGRERR_NONE) {
+                    std::runtime_error("Failed to create feature on layer 'perrors'");
+                }
+
+                OGRFeature::DestroyFeature(feature);
+            }
+
+            void write_line(const char* problem_type, osmium::object_id_type id1, osmium::object_id_type id2, osmium::Location loc1, osmium::Location loc2) {
+                std::unique_ptr<OGRPoint> ogr_point1 = m_ogr_factory.create_point(loc1);
+                std::unique_ptr<OGRPoint> ogr_point2 = m_ogr_factory.create_point(loc2);
+                std::unique_ptr<OGRLineString> ogr_linestring = std::unique_ptr<OGRLineString>(new OGRLineString());
+                ogr_linestring->addPoint(ogr_point1.get());
+                ogr_linestring->addPoint(ogr_point2.get());
+                OGRFeature* feature = OGRFeature::CreateFeature(m_layer_lerror->GetLayerDefn());
+                feature->SetGeometry(ogr_linestring.get());
+                feature->SetField("id1", static_cast<double>(id1));
+                feature->SetField("id2", static_cast<double>(id2));
+                feature->SetField("problem_type", problem_type);
+
+                if (m_layer_lerror->CreateFeature(feature) != OGRERR_NONE) {
+                    std::runtime_error("Failed to create feature on layer 'lerrors'");
+                }
+
+                OGRFeature::DestroyFeature(feature);
+            }
+
+        public:
+
+            explicit ProblemReporterOGR(OGRDataSource* data_source) :
+                m_data_source(data_source) {
+
+                OGRSpatialReference sparef;
+                sparef.SetWellKnownGeogCS("WGS84");
+
+                m_layer_perror = m_data_source->CreateLayer("perrors", &sparef, wkbPoint, nullptr);
+                if (!m_layer_perror) {
+                    std::runtime_error("Layer creation failed for layer 'perrors'");
+                }
+
+                OGRFieldDefn layer_perror_field_id1("id1", OFTReal);
+                layer_perror_field_id1.SetWidth(10);
+
+                if (m_layer_perror->CreateField(&layer_perror_field_id1) != OGRERR_NONE) {
+                    std::runtime_error("Creating field 'id1' failed for layer 'perrors'");
+                }
+
+                OGRFieldDefn layer_perror_field_id2("id2", OFTReal);
+                layer_perror_field_id2.SetWidth(10);
+
+                if (m_layer_perror->CreateField(&layer_perror_field_id2) != OGRERR_NONE) {
+                    std::runtime_error("Creating field 'id2' failed for layer 'perrors'");
+                }
+
+                OGRFieldDefn layer_perror_field_problem_type("problem_type", OFTString);
+                layer_perror_field_problem_type.SetWidth(30);
+
+                if (m_layer_perror->CreateField(&layer_perror_field_problem_type) != OGRERR_NONE) {
+                    std::runtime_error("Creating field 'problem_type' failed for layer 'perrors'");
+                }
+
+                /**************/
+
+                m_layer_lerror = m_data_source->CreateLayer("lerrors", &sparef, wkbLineString, nullptr);
+                if (!m_layer_lerror) {
+                    std::runtime_error("Layer creation failed for layer 'lerrors'");
+                }
+
+                OGRFieldDefn layer_lerror_field_id1("id1", OFTReal);
+                layer_lerror_field_id1.SetWidth(10);
+
+                if (m_layer_lerror->CreateField(&layer_lerror_field_id1) != OGRERR_NONE) {
+                    std::runtime_error("Creating field 'id1' failed for layer 'lerrors'");
+                }
+
+                OGRFieldDefn layer_lerror_field_id2("id2", OFTReal);
+                layer_lerror_field_id2.SetWidth(10);
+
+                if (m_layer_lerror->CreateField(&layer_lerror_field_id2) != OGRERR_NONE) {
+                    std::runtime_error("Creating field 'id2' failed for layer 'lerrors'");
+                }
+
+                OGRFieldDefn layer_lerror_field_problem_type("problem_type", OFTString);
+                layer_lerror_field_problem_type.SetWidth(30);
+
+                if (m_layer_lerror->CreateField(&layer_lerror_field_problem_type) != OGRERR_NONE) {
+                    std::runtime_error("Creating field 'problem_type' failed for layer 'lerrors'");
+                }
+            }
+
+            virtual ~ProblemReporterOGR() = default;
+
+            void report_duplicate_node(osmium::object_id_type node_id1, osmium::object_id_type node_id2, osmium::Location location) override {
+                write_point("duplicate_node", node_id1, node_id2, location);
+            }
+
+            void report_intersection(osmium::object_id_type way1_id, osmium::Location way1_seg_start, osmium::Location way1_seg_end,
+                                     osmium::object_id_type way2_id, osmium::Location way2_seg_start, osmium::Location way2_seg_end, osmium::Location intersection) override {
+                write_point("intersection", m_object_id, 0, intersection);
+                write_line("intersection", m_object_id, way1_id, way1_seg_start, way1_seg_end);
+                write_line("intersection", m_object_id, way2_id, way2_seg_start, way2_seg_end);
+            }
+
+            void report_ring_not_closed(osmium::Location end1, osmium::Location end2) override {
+                write_point("ring_not_closed", m_object_id, 0, end1);
+                write_point("ring_not_closed", m_object_id, 0, end2);
+            }
+
+            void report_role_should_be_outer(osmium::object_id_type way_id, osmium::Location seg_start, osmium::Location seg_end) override {
+                write_line("role_should_be_outer", m_object_id, way_id, seg_start, seg_end);
+            }
+
+            void report_role_should_be_inner(osmium::object_id_type way_id, osmium::Location seg_start, osmium::Location seg_end) override {
+                write_line("role_should_be_inner", m_object_id, way_id, seg_start, seg_end);
+            }
+
+        }; // class ProblemReporterOGR
+
+    } // namespace area
+
+} // namespace osmium
+
+#endif // OSMIUM_AREA_PROBLEM_REPORTER_OGR_HPP

include/osmium/area/problem_reporter_stream.hpp

@@ -0,0 +1,96 @@
+#ifndef OSMIUM_AREA_PROBLEM_REPORTER_STREAM_HPP
+#define OSMIUM_AREA_PROBLEM_REPORTER_STREAM_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <ostream>
+
+#include <osmium/area/problem_reporter.hpp>
+#include <osmium/osm/item_type.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/types.hpp>
+
+namespace osmium {
+
+    namespace area {
+
+        class ProblemReporterStream : public ProblemReporter {
+
+            std::ostream* m_out;
+
+        public:
+
+            explicit ProblemReporterStream(std::ostream& out) :
+                m_out(&out) {
+            }
+
+            virtual ~ProblemReporterStream() = default;
+
+            void header(const char* msg) {
+                *m_out << "DATA PROBLEM: " << msg << " on " << item_type_to_char(m_object_type) << m_object_id << ": ";
+            }
+
+            void report_duplicate_node(osmium::object_id_type node_id1, osmium::object_id_type node_id2, osmium::Location location) override {
+                header("duplicate node");
+                *m_out << "node_id1=" << node_id1 << " node_id2=" << node_id2 << " location=" << location << "\n";
+            }
+
+            void report_intersection(osmium::object_id_type way1_id, osmium::Location way1_seg_start, osmium::Location way1_seg_end,
+                                     osmium::object_id_type way2_id, osmium::Location way2_seg_start, osmium::Location way2_seg_end, osmium::Location intersection) override {
+                header("intersection");
+                *m_out << "way1_id=" << way1_id << " way1_seg_start=" << way1_seg_start << " way1_seg_end=" << way1_seg_end
+                       << " way2_id=" << way2_id << " way2_seg_start=" << way2_seg_start << " way2_seg_end=" << way2_seg_end << " intersection=" << intersection << "\n";
+            }
+
+            void report_ring_not_closed(osmium::Location end1, osmium::Location end2) override {
+                header("ring not closed");
+                *m_out << "end1=" << end1 << " end2=" << end2 << "\n";
+            }
+
+            void report_role_should_be_outer(osmium::object_id_type way_id, osmium::Location seg_start, osmium::Location seg_end) override {
+                header("role should be outer");
+                *m_out << "way_id=" << way_id << " seg_start=" << seg_start << " seg_end=" << seg_end << "\n";
+            }
+
+            void report_role_should_be_inner(osmium::object_id_type way_id, osmium::Location seg_start, osmium::Location seg_end) override {
+                header("role should be inner");
+                *m_out << "way_id=" << way_id << " seg_start=" << seg_start << " seg_end=" << seg_end << "\n";
+            }
+
+        }; // class ProblemReporterStream
+
+    } // namespace area
+
+} // namespace osmium
+
+#endif // OSMIUM_AREA_PROBLEM_REPORTER_STREAM_HPP

include/osmium/builder/builder.hpp

@@ -0,0 +1,222 @@
+#ifndef OSMIUM_BUILDER_BUILDER_HPP
+#define OSMIUM_BUILDER_BUILDER_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <new>
+#include <type_traits>
+
+#include <osmium/memory/buffer.hpp>
+#include <osmium/memory/item.hpp>
+#include <osmium/osm/types.hpp>
+#include <osmium/util/cast.hpp>
+
+namespace osmium {
+
+    /**
+     * @brief Classes for building OSM objects and other items in buffers
+     */
+    namespace builder {
+
+        class Builder {
+
+            osmium::memory::Buffer& m_buffer;
+            Builder* m_parent;
+            size_t m_item_offset;
+
+            Builder(const Builder&) = delete;
+            Builder(Builder&&) = delete;
+
+            Builder& operator=(const Builder&) = delete;
+            Builder& operator=(Builder&&) = delete;
+
+        protected:
+
+            explicit Builder(osmium::memory::Buffer& buffer, Builder* parent, osmium::memory::item_size_type size) :
+                m_buffer(buffer),
+                m_parent(parent),
+                m_item_offset(buffer.written()) {
+                m_buffer.reserve_space(size);
+                assert(buffer.is_aligned());
+                if (m_parent) {
+                    m_parent->add_size(size);
+                }
+            }
+
+            ~Builder() = default;
+
+            osmium::memory::Item& item() const {
+                return *reinterpret_cast<osmium::memory::Item*>(m_buffer.data() + m_item_offset);
+            }
+
+        public:
+
+            /**
+             * Add padding to buffer (if needed) to align data properly.
+             *
+             * This calculates how many padding bytes are needed and adds
+             * as many zero bytes to the buffer. It also adds this number
+             * to the size of the current item (if the "self" param is
+             * true) and recursively to all the parent items.
+             *
+             * @param self If true add number of padding bytes to size
+             *             of current item. Size is always added to
+             *             parent item (if any).
+             *
+             */
+            void add_padding(bool self=false) {
+                auto padding = osmium::memory::align_bytes - (size() % osmium::memory::align_bytes);
+                if (padding != osmium::memory::align_bytes) {
+                    std::fill_n(m_buffer.reserve_space(padding), padding, 0);
+                    if (self) {
+                        add_size(padding);
+                    } else if (m_parent) {
+                        m_parent->add_size(padding);
+                        assert(m_parent->size() % osmium::memory::align_bytes == 0);
+                    }
+                }
+            }
+
+            void add_size(uint32_t size) {
+                item().add_size(size);
+                if (m_parent) {
+                    m_parent->add_size(size);
+                }
+            }
+
+            uint32_t size() const noexcept {
+                return item().byte_size();
+            }
+
+            void add_item(const osmium::memory::Item* item) {
+                unsigned char* target = m_buffer.reserve_space(item->padded_size());
+                std::copy_n(reinterpret_cast<const unsigned char*>(item), item->padded_size(), target);
+                add_size(item->padded_size());
+            }
+
+            /**
+             * Reserve space for an object of class T in buffer and return
+             * pointer to it.
+             */
+            template <class T>
+            T* reserve_space_for() {
+                assert(m_buffer.is_aligned());
+                return reinterpret_cast<T*>(m_buffer.reserve_space(sizeof(T)));
+            }
+
+            /**
+             * Append data to buffer.
+             *
+             * @param data Pointer to data.
+             * @param length Length of data in bytes. If data is a
+             *               \0-terminated string, length must contain the
+             *               \0 byte.
+             */
+            osmium::memory::item_size_type append(const char* data, const osmium::memory::item_size_type length) {
+                unsigned char* target = m_buffer.reserve_space(length);
+                std::copy_n(reinterpret_cast<const unsigned char*>(data), length, target);
+                return length;
+            }
+
+            /**
+             * Append \0-terminated string to buffer.
+             */
+            osmium::memory::item_size_type append(const char* str) {
+                return append(str, static_cast<osmium::memory::item_size_type>(std::strlen(str) + 1));
+            }
+
+            /// Return the buffer this builder is using.
+            osmium::memory::Buffer& buffer() noexcept {
+                return m_buffer;
+            }
+
+        }; // class Builder
+
+        template <class TItem>
+        class ObjectBuilder : public Builder {
+
+            static_assert(std::is_base_of<osmium::memory::Item, TItem>::value,
+                "ObjectBuilder can only build objects derived from osmium::memory::Item");
+
+        public:
+
+            explicit ObjectBuilder(osmium::memory::Buffer& buffer, Builder* parent=nullptr) :
+                Builder(buffer, parent, sizeof(TItem)) {
+                new (&item()) TItem();
+            }
+
+            TItem& object() noexcept {
+                return static_cast<TItem&>(item());
+            }
+
+            /**
+             * Add user name to buffer.
+             *
+             * @param user Pointer to user name.
+             * @param length Length of user name including \0 byte.
+             */
+            void add_user(const char* user, const string_size_type length) {
+                object().set_user_size(length);
+                add_size(append(user, length));
+                add_padding(true);
+            }
+
+            /**
+             * Add user name to buffer.
+             *
+             * @param user Pointer to \0-terminated user name.
+             */
+            void add_user(const char* user) {
+                add_user(user, static_cast_with_assert<string_size_type>(std::strlen(user) + 1));
+            }
+
+            /**
+             * Add user name to buffer.
+             *
+             * @param user User name.
+             */
+            void add_user(const std::string& user) {
+                add_user(user.data(), static_cast_with_assert<string_size_type>(user.size() + 1));
+            }
+
+        }; // class ObjectBuilder
+
+    } // namespace builder
+
+} // namespace osmium
+
+#endif // OSMIUM_BUILDER_BUILDER_HPP

include/osmium/builder/builder_helper.hpp

@@ -0,0 +1,103 @@
+#ifndef OSMIUM_BUILDER_BUILDER_HELPER_HPP
+#define OSMIUM_BUILDER_BUILDER_HELPER_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstddef>
+#include <initializer_list>
+#include <functional>
+#include <map>
+#include <utility>
+
+#include <osmium/builder/osm_object_builder.hpp>
+#include <osmium/memory/buffer.hpp>
+
+namespace osmium {
+
+    class NodeRef;
+    class TagList;
+    class WayNodeList;
+
+    namespace builder {
+
+        inline const osmium::WayNodeList& build_way_node_list(osmium::memory::Buffer& buffer, const std::initializer_list<osmium::NodeRef>& nodes) {
+            size_t pos = buffer.committed();
+            {
+                osmium::builder::WayNodeListBuilder wnl_builder(buffer);
+                for (const auto& node_ref : nodes) {
+                    wnl_builder.add_node_ref(node_ref);
+                }
+            }
+            buffer.commit();
+            return buffer.get<const osmium::WayNodeList>(pos);
+        }
+
+        inline const osmium::TagList& build_tag_list(osmium::memory::Buffer& buffer, const std::initializer_list<std::pair<const char*, const char*>>& tags) {
+            size_t pos = buffer.committed();
+            {
+                osmium::builder::TagListBuilder tl_builder(buffer);
+                for (const auto& p : tags) {
+                    tl_builder.add_tag(p.first, p.second);
+                }
+            }
+            buffer.commit();
+            return buffer.get<const osmium::TagList>(pos);
+        }
+
+        inline const osmium::TagList& build_tag_list_from_map(osmium::memory::Buffer& buffer, const std::map<const char*, const char*>& tags) {
+            size_t pos = buffer.committed();
+            {
+                osmium::builder::TagListBuilder tl_builder(buffer);
+                for (const auto& p : tags) {
+                    tl_builder.add_tag(p.first, p.second);
+                }
+            }
+            buffer.commit();
+            return buffer.get<const osmium::TagList>(pos);
+        }
+
+        inline const osmium::TagList& build_tag_list_from_func(osmium::memory::Buffer& buffer, std::function<void(osmium::builder::TagListBuilder&)> func) {
+            size_t pos = buffer.committed();
+            {
+                osmium::builder::TagListBuilder tl_builder(buffer);
+                func(tl_builder);
+            }
+            buffer.commit();
+            return buffer.get<const osmium::TagList>(pos);
+        }
+
+    } // namespace builder
+
+} // namespace osmium
+
+#endif // OSMIUM_BUILDER_BUILDER_HELPER_HPP

include/osmium/builder/osm_object_builder.hpp

@@ -0,0 +1,283 @@
+#ifndef OSMIUM_BUILDER_OSM_OBJECT_BUILDER_HPP
+#define OSMIUM_BUILDER_OSM_OBJECT_BUILDER_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstring>
+#include <initializer_list>
+#include <new>
+#include <utility>
+
+#include <osmium/builder/builder.hpp>
+#include <osmium/osm.hpp>
+#include <osmium/osm/item_type.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/node_ref.hpp>
+#include <osmium/osm/object.hpp>
+#include <osmium/osm/tag.hpp>
+#include <osmium/osm/types.hpp>
+#include <osmium/util/cast.hpp>
+
+namespace osmium {
+
+    namespace memory {
+        class Buffer;
+    }
+
+    namespace builder {
+
+        class TagListBuilder : public ObjectBuilder<TagList> {
+
+        public:
+
+            explicit TagListBuilder(osmium::memory::Buffer& buffer, Builder* parent=nullptr) :
+                ObjectBuilder<TagList>(buffer, parent) {
+            }
+
+            ~TagListBuilder() {
+                add_padding();
+            }
+
+            /**
+             * Add tag to buffer.
+             *
+             * @param key Tag key.
+             * @param value Tag value.
+             */
+            void add_tag(const char* key, const char* value) {
+                add_size(append(key) + append(value));
+            }
+
+            /**
+             * Add tag to buffer.
+             *
+             * @param key Tag key.
+             * @param value Tag value.
+             */
+            void add_tag(const std::string& key, const std::string& value) {
+                add_size(append(key.data(),   static_cast_with_assert<string_size_type>(key.size()   + 1)) +
+                         append(value.data(), static_cast_with_assert<string_size_type>(value.size() + 1)));
+            }
+
+        }; // class TagListBuilder
+
+        template <class T>
+        class NodeRefListBuilder : public ObjectBuilder<T> {
+
+        public:
+
+            explicit NodeRefListBuilder(osmium::memory::Buffer& buffer, Builder* parent=nullptr) :
+                ObjectBuilder<T>(buffer, parent) {
+            }
+
+            ~NodeRefListBuilder() {
+                static_cast<Builder*>(this)->add_padding();
+            }
+
+            void add_node_ref(const NodeRef& node_ref) {
+                new (static_cast<Builder*>(this)->reserve_space_for<osmium::NodeRef>()) osmium::NodeRef(node_ref);
+                static_cast<Builder*>(this)->add_size(sizeof(osmium::NodeRef));
+            }
+
+            void add_node_ref(const object_id_type ref, const osmium::Location location=Location()) {
+                add_node_ref(NodeRef(ref, location));
+            }
+
+        }; // class NodeRefListBuilder
+
+        typedef NodeRefListBuilder<WayNodeList> WayNodeListBuilder;
+        typedef NodeRefListBuilder<OuterRing> OuterRingBuilder;
+        typedef NodeRefListBuilder<InnerRing> InnerRingBuilder;
+
+        class RelationMemberListBuilder : public ObjectBuilder<RelationMemberList> {
+
+            /**
+             * Add role to buffer.
+             *
+             * @param member Relation member object where the length of the role
+             *               will be set.
+             * @param role The role.
+             * @param length Length of role string including \0 termination.
+             */
+            void add_role(osmium::RelationMember& member, const char* role, const string_size_type length) {
+                member.set_role_size(length);
+                add_size(append(role, length));
+                add_padding(true);
+            }
+
+            /**
+             * Add role to buffer.
+             *
+             * @param member Relation member object where the length of the role
+             *               will be set.
+             * @param role \0-terminated role.
+             */
+            void add_role(osmium::RelationMember& member, const char* role) {
+                add_role(member, role, static_cast_with_assert<string_size_type>(std::strlen(role) + 1));
+            }
+
+            /**
+             * Add role to buffer.
+             *
+             * @param member Relation member object where the length of the role
+             *               will be set.
+             * @param role Role.
+             */
+            void add_role(osmium::RelationMember& member, const std::string& role) {
+                add_role(member, role.data(), static_cast_with_assert<string_size_type>(role.size() + 1));
+            }
+
+        public:
+
+            explicit RelationMemberListBuilder(osmium::memory::Buffer& buffer, Builder* parent=nullptr) :
+                ObjectBuilder<RelationMemberList>(buffer, parent) {
+            }
+
+            ~RelationMemberListBuilder() {
+                add_padding();
+            }
+
+            /**
+             * Add a member to the relation.
+             *
+             * @param type The type (node, way, or relation).
+             * @param ref The ID of the member.
+             * @param role The role of the member.
+             * @param full_member Optional pointer to the member object. If it
+             *                    is available a copy will be added to the
+             *                    relation.
+             */
+            void add_member(osmium::item_type type, object_id_type ref, const char* role, const osmium::OSMObject* full_member = nullptr) {
+                osmium::RelationMember* member = reserve_space_for<osmium::RelationMember>();
+                new (member) osmium::RelationMember(ref, type, full_member != nullptr);
+                add_size(sizeof(RelationMember));
+                add_role(*member, role);
+                if (full_member) {
+                    add_item(full_member);
+                }
+            }
+
+            /**
+             * Add a member to the relation.
+             *
+             * @param type The type (node, way, or relation).
+             * @param ref The ID of the member.
+             * @param role The role of the member.
+             * @param full_member Optional pointer to the member object. If it
+             *                    is available a copy will be added to the
+             *                    relation.
+             */
+            void add_member(osmium::item_type type, object_id_type ref, const std::string& role, const osmium::OSMObject* full_member = nullptr) {
+                osmium::RelationMember* member = reserve_space_for<osmium::RelationMember>();
+                new (member) osmium::RelationMember(ref, type, full_member != nullptr);
+                add_size(sizeof(RelationMember));
+                add_role(*member, role);
+                if (full_member) {
+                    add_item(full_member);
+                }
+            }
+
+        }; // class RelationMemberListBuilder
+
+        template <class T>
+        class OSMObjectBuilder : public ObjectBuilder<T> {
+
+        public:
+
+            explicit OSMObjectBuilder(osmium::memory::Buffer& buffer, Builder* parent=nullptr) :
+                ObjectBuilder<T>(buffer, parent) {
+                static_cast<Builder*>(this)->reserve_space_for<string_size_type>();
+                static_cast<Builder*>(this)->add_size(sizeof(string_size_type));
+            }
+
+            void add_tags(const std::initializer_list<std::pair<const char*, const char*>>& tags) {
+                osmium::builder::TagListBuilder tl_builder(static_cast<Builder*>(this)->buffer(), this);
+                for (const auto& p : tags) {
+                    tl_builder.add_tag(p.first, p.second);
+                }
+            }
+
+        }; // class OSMObjectBuilder
+
+        typedef OSMObjectBuilder<osmium::Node> NodeBuilder;
+        typedef OSMObjectBuilder<osmium::Relation> RelationBuilder;
+
+        class WayBuilder : public OSMObjectBuilder<osmium::Way> {
+
+        public:
+
+            explicit WayBuilder(osmium::memory::Buffer& buffer, Builder* parent=nullptr) :
+                OSMObjectBuilder<osmium::Way>(buffer, parent) {
+            }
+
+            void add_node_refs(const std::initializer_list<osmium::NodeRef>& nodes) {
+                osmium::builder::WayNodeListBuilder builder(buffer(), this);
+                for (const auto& node_ref : nodes) {
+                    builder.add_node_ref(node_ref);
+                }
+            }
+
+        }; // class WayBuilder
+
+        class AreaBuilder : public OSMObjectBuilder<osmium::Area> {
+
+        public:
+
+            explicit AreaBuilder(osmium::memory::Buffer& buffer, Builder* parent=nullptr) :
+                OSMObjectBuilder<osmium::Area>(buffer, parent) {
+            }
+
+            /**
+             * Initialize area attributes from the attributes of the given object.
+             */
+            void initialize_from_object(const osmium::OSMObject& source) {
+                osmium::Area& area = object();
+                area.set_id(osmium::object_id_to_area_id(source.id(), source.type()));
+                area.set_version(source.version());
+                area.set_changeset(source.changeset());
+                area.set_timestamp(source.timestamp());
+                area.set_visible(source.visible());
+                area.set_uid(source.uid());
+
+                add_user(source.user());
+            }
+
+        }; // class AreaBuilder
+
+        typedef ObjectBuilder<osmium::Changeset> ChangesetBuilder;
+
+    } // namespace builder
+
+} // namespace osmium
+
+#endif // OSMIUM_BUILDER_OSM_OBJECT_BUILDER_HPP

include/osmium/diff_handler.hpp

@@ -0,0 +1,67 @@
+#ifndef OSMIUM_DIFF_HANDLER_HPP
+#define OSMIUM_DIFF_HANDLER_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <osmium/osm/diff_object.hpp>
+
+namespace osmium {
+
+    /**
+     * @brief Osmium diff handlers provide access to differences between OSM object versions
+     */
+    namespace diff_handler {
+
+        class DiffHandler {
+
+        public:
+
+            DiffHandler() {
+            }
+
+            void node(const osmium::DiffNode&) const {
+            }
+
+            void way(const osmium::DiffWay&) const {
+            }
+
+            void relation(const osmium::DiffRelation&) const {
+            }
+
+        }; // class DiffHandler
+
+    } // namespace diff_handler
+
+} // namespace osmium
+
+#endif // OSMIUM_DIFF_HANDLER_HPP

include/osmium/diff_iterator.hpp

@@ -0,0 +1,129 @@
+#ifndef OSMIUM_DIFF_ITERATOR_HPP
+#define OSMIUM_DIFF_ITERATOR_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cassert>
+#include <iterator>
+#include <type_traits>
+
+#include <osmium/osm/diff_object.hpp>
+
+namespace osmium {
+
+    class OSMObject;
+
+    template <class TBasicIterator>
+    class DiffIterator : public std::iterator<std::input_iterator_tag, const osmium::DiffObject> {
+
+        static_assert(std::is_base_of<osmium::OSMObject, typename TBasicIterator::value_type>::value, "TBasicIterator::value_type must derive from osmium::OSMObject");
+
+        TBasicIterator m_prev;
+        TBasicIterator m_curr;
+        TBasicIterator m_next;
+
+        const TBasicIterator m_end;
+
+        mutable osmium::DiffObject m_diff;
+
+        void set_diff() const {
+            assert(m_curr != m_end);
+
+            TBasicIterator prev = m_prev;
+            if (prev->type() != m_curr->type() || prev->id() != m_curr->id()) {
+                prev = m_curr;
+            }
+
+            TBasicIterator next = m_next;
+            if (next == m_end || next->type() != m_curr->type() || next->id() != m_curr->id()) {
+                next = m_curr;
+            }
+
+            m_diff = osmium::DiffObject(*prev, *m_curr, *next);
+        }
+
+    public:
+
+        explicit DiffIterator(TBasicIterator begin, TBasicIterator end) :
+            m_prev(begin),
+            m_curr(begin),
+            m_next(begin == end ? begin : ++begin),
+            m_end(end) {
+        }
+
+        DiffIterator(const DiffIterator&) = default;
+        DiffIterator& operator=(const DiffIterator&) = default;
+
+        DiffIterator(DiffIterator&&) = default;
+        DiffIterator& operator=(DiffIterator&&) = default;
+
+        DiffIterator& operator++() {
+            m_prev = std::move(m_curr);
+            m_curr = m_next;
+
+            if (m_next != m_end) {
+                ++m_next;
+            }
+
+            return *this;
+        }
+
+        DiffIterator operator++(int) {
+            DiffIterator tmp(*this);
+            operator++();
+            return tmp;
+        }
+
+        bool operator==(const DiffIterator& rhs) const {
+            return m_curr == rhs.m_curr && m_end == rhs.m_end;
+        }
+
+        bool operator!=(const DiffIterator& rhs) const {
+            return !(*this == rhs);
+        }
+
+        reference operator*() const {
+            set_diff();
+            return m_diff;
+        }
+
+        pointer operator->() const {
+            set_diff();
+            return &m_diff;
+        }
+
+    }; // class DiffIterator
+
+} // namespace osmium
+
+#endif // OSMIUM_DIFF_ITERATOR_HPP

include/osmium/diff_visitor.hpp

@@ -0,0 +1,104 @@
+#ifndef OSMIUM_DIFF_VISITOR_HPP
+#define OSMIUM_DIFF_VISITOR_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <osmium/diff_iterator.hpp>
+#include <osmium/io/input_iterator.hpp>
+#include <osmium/memory/buffer.hpp>
+#include <osmium/osm/diff_object.hpp>
+#include <osmium/osm/item_type.hpp>
+
+namespace osmium {
+
+    namespace detail {
+
+        template <class THandler>
+        inline void apply_diff_iterator_recurse(const osmium::DiffObject& diff, THandler& handler) {
+            switch (diff.type()) {
+                case osmium::item_type::node:
+                    handler.node(static_cast<const osmium::DiffNode&>(diff));
+                    break;
+                case osmium::item_type::way:
+                    handler.way(static_cast<const osmium::DiffWay&>(diff));
+                    break;
+                case osmium::item_type::relation:
+                    handler.relation(static_cast<const osmium::DiffRelation&>(diff));
+                    break;
+                default:
+                    throw osmium::unknown_type();
+            }
+        }
+
+        template <class THandler, class ...TRest>
+        inline void apply_diff_iterator_recurse(const osmium::DiffObject& diff, THandler& handler, TRest&... more) {
+            apply_diff_iterator_recurse(diff, handler);
+            apply_diff_iterator_recurse(diff, more...);
+        }
+
+    } // namespace detail
+
+    template <class TIterator, class ...THandlers>
+    inline void apply_diff(TIterator it, TIterator end, THandlers&... handlers) {
+        typedef osmium::DiffIterator<TIterator> diff_iterator;
+
+        diff_iterator dit(it, end);
+        diff_iterator dend(end, end);
+
+        for (; dit != dend; ++dit) {
+            detail::apply_diff_iterator_recurse(*dit, handlers...);
+        }
+    }
+
+    class OSMObject;
+
+    template <class TSource, class ...THandlers>
+    inline void apply_diff(TSource& source, THandlers&... handlers) {
+        apply_diff(osmium::io::InputIterator<TSource, osmium::OSMObject> {source},
+                   osmium::io::InputIterator<TSource, osmium::OSMObject> {},
+                   handlers...);
+    }
+
+    template <class ...THandlers>
+    inline void apply_diff(osmium::memory::Buffer& buffer, THandlers&... handlers) {
+        apply_diff(buffer.begin(), buffer.end(), handlers...);
+    }
+
+    template <class ...THandlers>
+    inline void apply_diff(const osmium::memory::Buffer& buffer, THandlers&... handlers) {
+        apply_diff(buffer.cbegin(), buffer.cend(), handlers...);
+    }
+
+} // namespace osmium
+
+#endif // OSMIUM_DIFF_VISITOR_HPP

include/osmium/dynamic_handler.hpp

@@ -0,0 +1,195 @@
+#ifndef OSMIUM_DYNAMIC_HANDLER_HPP
+#define OSMIUM_DYNAMIC_HANDLER_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <memory>
+#include <utility>
+
+#include <osmium/handler.hpp>
+
+namespace osmium {
+
+    class Node;
+    class Way;
+    class Relation;
+    class Area;
+    class Changeset;
+
+    namespace handler {
+
+        namespace detail {
+
+            class HandlerWrapperBase {
+
+            public:
+
+                virtual ~HandlerWrapperBase() {
+                }
+
+                virtual void node(const osmium::Node&) {
+                }
+
+                virtual void way(const osmium::Way&) {
+                }
+
+                virtual void relation(const osmium::Relation&) {
+                }
+
+                virtual void area(const osmium::Area&) {
+                }
+
+                virtual void changeset(const osmium::Changeset&) {
+                }
+
+                virtual void flush() {
+                }
+
+            }; // class HandlerWrapperBase
+
+
+            // The following uses trick from
+            // http://stackoverflow.com/questions/257288/is-it-possible-to-write-a-c-template-to-check-for-a-functions-existence
+            // to either call handler style functions or visitor style operator().
+
+#define OSMIUM_DYNAMIC_HANDLER_DISPATCH(_name_, _type_) \
+template <class THandler> \
+auto _name_##_dispatch(THandler& handler, const osmium::_type_& object, int) -> decltype(handler._name_(object), void()) { \
+    handler._name_(object); \
+} \
+template <class THandler> \
+auto _name_##_dispatch(THandler& handler, const osmium::_type_& object, long) -> decltype(handler(object), void()) { \
+    handler(object); \
+}
+
+            OSMIUM_DYNAMIC_HANDLER_DISPATCH(node, Node)
+            OSMIUM_DYNAMIC_HANDLER_DISPATCH(way, Way)
+            OSMIUM_DYNAMIC_HANDLER_DISPATCH(relation, Relation)
+            OSMIUM_DYNAMIC_HANDLER_DISPATCH(changeset, Changeset)
+            OSMIUM_DYNAMIC_HANDLER_DISPATCH(area, Area)
+
+            template <class THandler>
+            auto flush_dispatch(THandler& handler, int) -> decltype(handler.flush(), void()) {
+                handler.flush();
+            }
+
+            template <class THandler>
+            void flush_dispatch(THandler&, long) {}
+
+            template <class THandler>
+            class HandlerWrapper : public HandlerWrapperBase {
+
+                THandler m_handler;
+
+            public:
+
+                template <class... TArgs>
+                HandlerWrapper(TArgs&&... args) :
+                    m_handler(std::forward<TArgs>(args)...) {
+                }
+
+                void node(const osmium::Node& node) override final {
+                    node_dispatch(m_handler, node, 0);
+                }
+
+                void way(const osmium::Way& way) override final {
+                    way_dispatch(m_handler, way, 0);
+                }
+
+                void relation(const osmium::Relation& relation) override final {
+                    relation_dispatch(m_handler, relation, 0);
+                }
+
+                void area(const osmium::Area& area) override final {
+                    area_dispatch(m_handler, area, 0);
+                }
+
+                void changeset(const osmium::Changeset& changeset) override final {
+                    changeset_dispatch(m_handler, changeset, 0);
+                }
+
+                void flush() override final {
+                    flush_dispatch(m_handler, 0);
+                }
+
+            }; // class HandlerWrapper
+
+        } // namespace detail
+
+        class DynamicHandler : public osmium::handler::Handler {
+
+            typedef std::unique_ptr<osmium::handler::detail::HandlerWrapperBase> impl_ptr;
+            impl_ptr m_impl;
+
+        public:
+
+            DynamicHandler() :
+                m_impl(impl_ptr(new osmium::handler::detail::HandlerWrapperBase)) {
+            }
+
+            template <class THandler, class... TArgs>
+            void set(TArgs&&... args) {
+                m_impl = impl_ptr(new osmium::handler::detail::HandlerWrapper<THandler>(std::forward<TArgs>(args)...));
+            }
+
+            void node(const osmium::Node& node) {
+                m_impl->node(node);
+            }
+
+            void way(const osmium::Way& way) {
+                m_impl->way(way);
+            }
+
+            void relation(const osmium::Relation& relation) {
+                m_impl->relation(relation);
+            }
+
+            void area(const osmium::Area& area) {
+                m_impl->area(area);
+            }
+
+            void changeset(const osmium::Changeset& changeset) {
+                m_impl->changeset(changeset);
+            }
+
+            void flush() {
+                m_impl->flush();
+            }
+
+        }; // class DynamicHandler
+
+    } // namspace handler
+
+} // namespace osmium
+
+#endif // OSMIUM_DYNAMIC_HANDLER_HPP

include/osmium/experimental/flex_reader.hpp

@@ -0,0 +1,134 @@
+#ifndef OSMIUM_EXPERIMENTAL_FLEX_READER_HPP
+#define OSMIUM_EXPERIMENTAL_FLEX_READER_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <osmium/handler/node_locations_for_ways.hpp>
+#include <osmium/visitor.hpp>
+#include <osmium/area/multipolygon_collector.hpp>
+#include <osmium/area/assembler.hpp>
+
+namespace osmium {
+
+    /**
+     * @short Experimental code that is not "officially" supported.
+     */
+    namespace experimental {
+
+        template <class TLocationHandler>
+        class FlexReader {
+
+            bool m_with_areas;
+            osmium::osm_entity_bits::type m_entities;
+
+            typename TLocationHandler::index_pos_type m_index_pos;
+            typename TLocationHandler::index_neg_type m_index_neg;
+            TLocationHandler m_location_handler;
+
+            osmium::io::Reader m_reader;
+            osmium::area::Assembler::config_type m_assembler_config;
+            osmium::area::MultipolygonCollector<osmium::area::Assembler> m_collector;
+
+        public:
+
+            explicit FlexReader(const osmium::io::File& file, osmium::osm_entity_bits::type entities = osmium::osm_entity_bits::nwr) :
+                m_with_areas(entities & osmium::osm_entity_bits::area),
+                m_entities((entities & ~osmium::osm_entity_bits::area) | (m_with_areas ? osmium::osm_entity_bits::node | osmium::osm_entity_bits::way : osmium::osm_entity_bits::nothing)),
+                m_index_pos(),
+                m_index_neg(),
+                m_location_handler(m_index_pos, m_index_neg),
+                m_reader(file, m_entities),
+                m_assembler_config(),
+                m_collector(m_assembler_config)
+            {
+                m_location_handler.ignore_errors();
+                if (m_with_areas) {
+                    osmium::io::Reader reader(file, osmium::osm_entity_bits::relation);
+                    m_collector.read_relations(reader);
+                    reader.close();
+                }
+            }
+
+            explicit FlexReader(const std::string& filename, osmium::osm_entity_bits::type entities = osmium::osm_entity_bits::nwr) :
+                FlexReader(osmium::io::File(filename), entities) {
+            }
+
+            explicit FlexReader(const char* filename, osmium::osm_entity_bits::type entities = osmium::osm_entity_bits::nwr) :
+                FlexReader(osmium::io::File(filename), entities) {
+            }
+
+            osmium::memory::Buffer read() {
+                std::vector<osmium::memory::Buffer> area_buffers;
+
+                osmium::memory::Buffer buffer = m_reader.read();
+
+                if (buffer) {
+                    if (m_with_areas) {
+                        osmium::apply(buffer, m_location_handler, m_collector.handler([&area_buffers](osmium::memory::Buffer&& area_buffer) {
+                            area_buffers.push_back(std::move(area_buffer));
+                        }));
+                        for (const osmium::memory::Buffer& b : area_buffers) {
+                            buffer.add_buffer(b);
+                            buffer.commit();
+                        }
+                    } else if (m_entities & (osmium::osm_entity_bits::node | osmium::osm_entity_bits::way)) {
+                        osmium::apply(buffer, m_location_handler);
+                    }
+                }
+
+                return buffer;
+            }
+
+            osmium::io::Header header() const {
+                return m_reader.header();
+            }
+
+            void close() {
+                return m_reader.close();
+            }
+
+            bool eof() const {
+                return m_reader.eof();
+            }
+
+            const osmium::area::MultipolygonCollector<osmium::area::Assembler>& collector() const {
+                return m_collector;
+            }
+
+        }; // class FlexReader
+
+    } // namespace experimental
+
+} // namespace osmium
+
+#endif // OSMIUM_EXPERIMENTAL_FLEX_READER_HPP

include/osmium/geom/coordinates.hpp

@@ -0,0 +1,97 @@
+#ifndef OSMIUM_GEOM_COORDINATES_HPP
+#define OSMIUM_GEOM_COORDINATES_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstddef>
+#include <iosfwd>
+#include <string>
+
+#include <osmium/osm/location.hpp>
+#include <osmium/util/double.hpp>
+
+namespace osmium {
+
+    namespace geom {
+
+        struct Coordinates {
+
+            double x;
+            double y;
+
+            explicit Coordinates(double cx, double cy) noexcept : x(cx), y(cy) {
+            }
+
+            Coordinates(const osmium::Location& location) : x(location.lon()), y(location.lat()) {
+            }
+
+            void append_to_string(std::string& s, const char infix, int precision) const {
+                osmium::util::double2string(s, x, precision);
+                s += infix;
+                osmium::util::double2string(s, y, precision);
+            }
+
+            void append_to_string(std::string& s, const char prefix, const char infix, const char suffix, int precision) const {
+                s += prefix;
+                append_to_string(s, infix, precision);
+                s += suffix;
+            }
+
+        }; // struct coordinates
+
+        /**
+         * Compare whether two Coordinates are identical. Might not give the
+         * right result if the coordinates have been the result of some
+         * calculation that introduced rounding errors.
+         */
+        inline bool operator==(const Coordinates& lhs, const Coordinates& rhs) noexcept {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wfloat-equal"
+            return lhs.x == rhs.x && lhs.y == rhs.y;
+#pragma GCC diagnostic pop
+        }
+
+        inline bool operator!=(const Coordinates& lhs, const Coordinates& rhs) noexcept {
+            return ! operator==(lhs, rhs);
+        }
+
+        template <typename TChar, typename TTraits>
+        inline std::basic_ostream<TChar, TTraits>& operator<<(std::basic_ostream<TChar, TTraits>& out, const Coordinates& c) {
+            return out << '(' << c.x << ',' << c.y << ')';
+        }
+
+    } // namespace geom
+
+} // namespace osmium
+
+#endif // OSMIUM_GEOM_COORDINATES_HPP

include/osmium/geom/factory.hpp

@@ -0,0 +1,328 @@
+#ifndef OSMIUM_GEOM_FACTORY_HPP
+#define OSMIUM_GEOM_FACTORY_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstddef>
+#include <stdexcept>
+#include <string>
+#include <utility>
+
+#include <osmium/geom/coordinates.hpp>
+#include <osmium/memory/collection.hpp>
+#include <osmium/memory/item.hpp>
+#include <osmium/osm/area.hpp>
+#include <osmium/osm/item_type.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/node.hpp>
+#include <osmium/osm/node_ref.hpp>
+#include <osmium/osm/way.hpp>
+
+namespace osmium {
+
+    /**
+     * Exception thrown when an invalid geometry is encountered. An example
+     * would be a linestring with less than two points.
+     */
+    struct geometry_error : public std::runtime_error {
+
+        geometry_error(const std::string& what) :
+            std::runtime_error(what) {
+        }
+
+        geometry_error(const char* what) :
+            std::runtime_error(what) {
+        }
+
+    }; // struct geometry_error
+
+    /**
+     * @brief Everything related to geometry handling.
+     */
+    namespace geom {
+
+        /**
+         * Which nodes of a way to use for a linestring.
+         */
+        enum class use_nodes : bool {
+            unique = true, ///< Remove consecutive nodes with same location.
+            all    = false ///< Use all nodes.
+        }; // enum class use_nodes
+
+        /**
+         * Which direction the linestring created from a way
+         * should have.
+         */
+        enum class direction : bool {
+            backward = true, ///< Linestring has reverse direction.
+            forward  = false ///< Linestring has same direction as way.
+        }; // enum class direction
+
+        /**
+         * This pseudo projection just returns its WGS84 input unchanged.
+         * Used as a template parameter if a real projection is not needed.
+         */
+        class IdentityProjection {
+
+        public:
+
+            Coordinates operator()(osmium::Location location) const {
+                return Coordinates{location.lon(), location.lat()};
+            }
+
+            int epsg() const noexcept {
+                return 4326;
+            }
+
+            std::string proj_string() const {
+                return "+proj=longlat +datum=WGS84 +no_defs";
+            }
+
+        }; // class IdentityProjection
+
+        /**
+         * Geometry factory.
+         */
+        template <class TGeomImpl, class TProjection = IdentityProjection>
+        class GeometryFactory {
+
+            /**
+             * Add all points of an outer or inner ring to a multipolygon.
+             */
+            void add_points(const osmium::OuterRing& nodes) {
+                osmium::Location last_location;
+                for (const osmium::NodeRef& node_ref : nodes) {
+                    if (last_location != node_ref.location()) {
+                        last_location = node_ref.location();
+                        m_impl.multipolygon_add_location(m_projection(last_location));
+                    }
+                }
+            }
+
+            TProjection m_projection;
+            TGeomImpl m_impl;
+
+        public:
+
+            /**
+             * Constructor for default initialized projection.
+             */
+            template <class... TArgs>
+            GeometryFactory<TGeomImpl, TProjection>(TArgs&&... args) :
+                m_projection(),
+                m_impl(std::forward<TArgs>(args)...) {
+            }
+
+            /**
+             * Constructor for explicitly initialized projection. Note that the
+             * projection is moved into the GeometryFactory.
+             */
+            template <class... TArgs>
+            GeometryFactory<TGeomImpl, TProjection>(TProjection&& projection, TArgs&&... args) :
+                m_projection(std::move(projection)),
+                m_impl(std::forward<TArgs>(args)...) {
+            }
+
+            typedef typename TGeomImpl::point_type        point_type;
+            typedef typename TGeomImpl::linestring_type   linestring_type;
+            typedef typename TGeomImpl::polygon_type      polygon_type;
+            typedef typename TGeomImpl::multipolygon_type multipolygon_type;
+            typedef typename TGeomImpl::ring_type         ring_type;
+
+            int epsg() const {
+                return m_projection.epsg();
+            }
+
+            std::string proj_string() const {
+                return m_projection.proj_string();
+            }
+
+            /* Point */
+
+            point_type create_point(const osmium::Location location) const {
+                return m_impl.make_point(m_projection(location));
+            }
+
+            point_type create_point(const osmium::Node& node) {
+                return create_point(node.location());
+            }
+
+            point_type create_point(const osmium::NodeRef& node_ref) {
+                return create_point(node_ref.location());
+            }
+
+            /* LineString */
+
+            void linestring_start() {
+                m_impl.linestring_start();
+            }
+
+            template <class TIter>
+            size_t fill_linestring(TIter it, TIter end) {
+                size_t num_points = 0;
+                for (; it != end; ++it, ++num_points) {
+                    m_impl.linestring_add_location(m_projection(it->location()));
+                }
+                return num_points;
+            }
+
+            template <class TIter>
+            size_t fill_linestring_unique(TIter it, TIter end) {
+                size_t num_points = 0;
+                osmium::Location last_location;
+                for (; it != end; ++it) {
+                    if (last_location != it->location()) {
+                        last_location = it->location();
+                        m_impl.linestring_add_location(m_projection(last_location));
+                        ++num_points;
+                    }
+                }
+                return num_points;
+            }
+
+            linestring_type linestring_finish(size_t num_points) {
+                return m_impl.linestring_finish(num_points);
+            }
+
+            linestring_type create_linestring(const osmium::WayNodeList& wnl, use_nodes un=use_nodes::unique, direction dir=direction::forward) {
+                linestring_start();
+                size_t num_points = 0;
+
+                if (un == use_nodes::unique) {
+                    osmium::Location last_location;
+                    switch (dir) {
+                        case direction::forward:
+                            num_points = fill_linestring_unique(wnl.cbegin(), wnl.cend());
+                            break;
+                        case direction::backward:
+                            num_points = fill_linestring_unique(wnl.crbegin(), wnl.crend());
+                            break;
+                    }
+                } else {
+                    switch (dir) {
+                        case direction::forward:
+                            num_points = fill_linestring(wnl.cbegin(), wnl.cend());
+                            break;
+                        case direction::backward:
+                            num_points = fill_linestring(wnl.crbegin(), wnl.crend());
+                            break;
+                    }
+                }
+
+                if (num_points < 2) {
+                    throw osmium::geometry_error("not enough points for linestring");
+                }
+
+                return linestring_finish(num_points);
+            }
+
+            linestring_type create_linestring(const osmium::Way& way, use_nodes un=use_nodes::unique, direction dir=direction::forward) {
+                return create_linestring(way.nodes(), un, dir);
+            }
+
+            /* Polygon */
+
+            void polygon_start() {
+                m_impl.polygon_start();
+            }
+
+            template <class TIter>
+            size_t fill_polygon(TIter it, TIter end) {
+                size_t num_points = 0;
+                for (; it != end; ++it, ++num_points) {
+                    m_impl.polygon_add_location(m_projection(it->location()));
+                }
+                return num_points;
+            }
+
+            template <class TIter>
+            size_t fill_polygon_unique(TIter it, TIter end) {
+                size_t num_points = 0;
+                osmium::Location last_location;
+                for (; it != end; ++it) {
+                    if (last_location != it->location()) {
+                        last_location = it->location();
+                        m_impl.polygon_add_location(m_projection(last_location));
+                        ++num_points;
+                    }
+                }
+                return num_points;
+            }
+
+            polygon_type polygon_finish(size_t num_points) {
+                return m_impl.polygon_finish(num_points);
+            }
+
+            /* MultiPolygon */
+
+            multipolygon_type create_multipolygon(const osmium::Area& area) {
+                size_t num_polygons = 0;
+                size_t num_rings = 0;
+                m_impl.multipolygon_start();
+
+                for (auto it = area.cbegin(); it != area.cend(); ++it) {
+                    const osmium::OuterRing& ring = static_cast<const osmium::OuterRing&>(*it);
+                    if (it->type() == osmium::item_type::outer_ring) {
+                        if (num_polygons > 0) {
+                            m_impl.multipolygon_polygon_finish();
+                        }
+                        m_impl.multipolygon_polygon_start();
+                        m_impl.multipolygon_outer_ring_start();
+                        add_points(ring);
+                        m_impl.multipolygon_outer_ring_finish();
+                        ++num_rings;
+                        ++num_polygons;
+                    } else if (it->type() == osmium::item_type::inner_ring) {
+                        m_impl.multipolygon_inner_ring_start();
+                        add_points(ring);
+                        m_impl.multipolygon_inner_ring_finish();
+                        ++num_rings;
+                    }
+                }
+
+                // if there are no rings, this area is invalid
+                if (num_rings == 0) {
+                    throw osmium::geometry_error("invalid area");
+                }
+
+                m_impl.multipolygon_polygon_finish();
+                return m_impl.multipolygon_finish();
+            }
+
+        }; // class GeometryFactory
+
+    } // namespace geom
+
+} // namespace osmium
+
+#endif // OSMIUM_GEOM_FACTORY_HPP

include/osmium/geom/geojson.hpp

@@ -0,0 +1,154 @@
+#ifndef OSMIUM_GEOM_GEOJSON_HPP
+#define OSMIUM_GEOM_GEOJSON_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cassert>
+#include <string>
+#include <utility>
+
+#include <osmium/geom/coordinates.hpp>
+#include <osmium/geom/factory.hpp>
+
+namespace osmium {
+
+    namespace geom {
+
+        namespace detail {
+
+            class GeoJSONFactoryImpl {
+
+                std::string m_str;
+                int m_precision;
+
+            public:
+
+                typedef std::string point_type;
+                typedef std::string linestring_type;
+                typedef std::string polygon_type;
+                typedef std::string multipolygon_type;
+                typedef std::string ring_type;
+
+                GeoJSONFactoryImpl(int precision = 7) :
+                    m_precision(precision) {
+                }
+
+                /* Point */
+
+                // { "type": "Point", "coordinates": [100.0, 0.0] }
+                point_type make_point(const osmium::geom::Coordinates& xy) const {
+                    std::string str {"{\"type\":\"Point\",\"coordinates\":"};
+                    xy.append_to_string(str, '[', ',', ']', m_precision);
+                    str += "}";
+                    return str;
+                }
+
+                /* LineString */
+
+                // { "type": "LineString", "coordinates": [ [100.0, 0.0], [101.0, 1.0] ] }
+                void linestring_start() {
+                    m_str = "{\"type\":\"LineString\",\"coordinates\":[";
+                }
+
+                void linestring_add_location(const osmium::geom::Coordinates& xy) {
+                    xy.append_to_string(m_str, '[', ',', ']', m_precision);
+                    m_str += ',';
+                }
+
+                linestring_type linestring_finish(size_t /* num_points */) {
+                    assert(!m_str.empty());
+                    std::string str;
+                    std::swap(str, m_str);
+                    str.back() = ']';
+                    str += "}";
+                    return str;
+                }
+
+                /* MultiPolygon */
+
+                void multipolygon_start() {
+                    m_str = "{\"type\":\"MultiPolygon\",\"coordinates\":[";
+                }
+
+                void multipolygon_polygon_start() {
+                    m_str += '[';
+                }
+
+                void multipolygon_polygon_finish() {
+                    m_str += "],";
+                }
+
+                void multipolygon_outer_ring_start() {
+                    m_str += '[';
+                }
+
+                void multipolygon_outer_ring_finish() {
+                    assert(!m_str.empty());
+                    m_str.back() = ']';
+                }
+
+                void multipolygon_inner_ring_start() {
+                    m_str += ",[";
+                }
+
+                void multipolygon_inner_ring_finish() {
+                    assert(!m_str.empty());
+                    m_str.back() = ']';
+                }
+
+                void multipolygon_add_location(const osmium::geom::Coordinates& xy) {
+                    xy.append_to_string(m_str, '[', ',', ']', m_precision);
+                    m_str += ',';
+                }
+
+                multipolygon_type multipolygon_finish() {
+                    assert(!m_str.empty());
+                    std::string str;
+                    std::swap(str, m_str);
+                    str.back() = ']';
+                    str += "}";
+                    return str;
+                }
+
+            }; // class GeoJSONFactoryImpl
+
+        } // namespace detail
+
+        template <class TProjection = IdentityProjection>
+        using GeoJSONFactory = GeometryFactory<osmium::geom::detail::GeoJSONFactoryImpl, TProjection>;
+
+    } // namespace geom
+
+} // namespace osmium
+
+#endif // OSMIUM_GEOM_GEOJSON_HPP

include/osmium/geom/geos.hpp

@@ -0,0 +1,221 @@
+#ifndef OSMIUM_GEOM_GEOS_HPP
+#define OSMIUM_GEOM_GEOS_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <utility>
+
+#include <geos/geom/Coordinate.h>
+#include <geos/geom/CoordinateSequence.h>
+#include <geos/geom/CoordinateSequenceFactory.h>
+#include <geos/geom/GeometryFactory.h>
+#include <geos/geom/LinearRing.h>
+#include <geos/geom/MultiPolygon.h>
+#include <geos/geom/Point.h>
+#include <geos/geom/Polygon.h>
+#include <geos/geom/PrecisionModel.h>
+#include <geos/util/GEOSException.h>
+
+#include <osmium/geom/factory.hpp>
+#include <osmium/geom/coordinates.hpp>
+
+// MSVC doesn't support throw_with_nested yet
+#ifdef _MSC_VER
+# define THROW throw
+#else
+# define THROW std::throw_with_nested
+#endif
+
+namespace osmium {
+
+    struct geos_geometry_error : public geometry_error {
+
+        geos_geometry_error() :
+            geometry_error("geometry creation failed in GEOS library, see nested exception for details") {
+        }
+
+    }; // struct geos_geometry_error
+
+    namespace geom {
+
+        namespace detail {
+
+            class GEOSFactoryImpl {
+
+                geos::geom::PrecisionModel m_precision_model;
+                geos::geom::GeometryFactory m_geos_factory;
+
+                std::unique_ptr<geos::geom::CoordinateSequence> m_coordinate_sequence;
+                std::vector<std::unique_ptr<geos::geom::LinearRing>> m_rings;
+                std::vector<std::unique_ptr<geos::geom::Polygon>> m_polygons;
+
+            public:
+
+                typedef std::unique_ptr<geos::geom::Point>        point_type;
+                typedef std::unique_ptr<geos::geom::LineString>   linestring_type;
+                typedef std::unique_ptr<geos::geom::Polygon>      polygon_type;
+                typedef std::unique_ptr<geos::geom::MultiPolygon> multipolygon_type;
+                typedef std::unique_ptr<geos::geom::LinearRing>   ring_type;
+
+                explicit GEOSFactoryImpl(int srid = -1) :
+                    m_precision_model(),
+                    m_geos_factory(&m_precision_model, srid) {
+                }
+
+                /* Point */
+
+                point_type make_point(const osmium::geom::Coordinates& xy) const {
+                    try {
+                        return point_type(m_geos_factory.createPoint(geos::geom::Coordinate(xy.x, xy.y)));
+                    } catch (geos::util::GEOSException&) {
+                        THROW(osmium::geos_geometry_error());
+                    }
+                }
+
+                /* LineString */
+
+                void linestring_start() {
+                    try {
+                        m_coordinate_sequence.reset(m_geos_factory.getCoordinateSequenceFactory()->create(static_cast<size_t>(0), 2));
+                    } catch (geos::util::GEOSException&) {
+                        THROW(osmium::geos_geometry_error());
+                    }
+                }
+
+                void linestring_add_location(const osmium::geom::Coordinates& xy) {
+                    try {
+                        m_coordinate_sequence->add(geos::geom::Coordinate(xy.x, xy.y));
+                    } catch (geos::util::GEOSException&) {
+                        THROW(osmium::geos_geometry_error());
+                    }
+                }
+
+                linestring_type linestring_finish(size_t /* num_points */) {
+                    try {
+                        return linestring_type(m_geos_factory.createLineString(m_coordinate_sequence.release()));
+                    } catch (geos::util::GEOSException&) {
+                        THROW(osmium::geos_geometry_error());
+                    }
+                }
+
+                /* MultiPolygon */
+
+                void multipolygon_start() {
+                    m_polygons.clear();
+                }
+
+                void multipolygon_polygon_start() {
+                    m_rings.clear();
+                }
+
+                void multipolygon_polygon_finish() {
+                    try {
+                        assert(!m_rings.empty());
+                        auto inner_rings = new std::vector<geos::geom::Geometry*>;
+                        std::transform(std::next(m_rings.begin(), 1), m_rings.end(), std::back_inserter(*inner_rings), [](std::unique_ptr<geos::geom::LinearRing>& r) {
+                            return r.release();
+                        });
+                        m_polygons.emplace_back(m_geos_factory.createPolygon(m_rings[0].release(), inner_rings));
+                        m_rings.clear();
+                    } catch (geos::util::GEOSException&) {
+                        THROW(osmium::geos_geometry_error());
+                    }
+                }
+
+                void multipolygon_outer_ring_start() {
+                    try {
+                        m_coordinate_sequence.reset(m_geos_factory.getCoordinateSequenceFactory()->create(static_cast<size_t>(0), 2));
+                    } catch (geos::util::GEOSException&) {
+                        THROW(osmium::geos_geometry_error());
+                    }
+                }
+
+                void multipolygon_outer_ring_finish() {
+                    try {
+                        m_rings.emplace_back(m_geos_factory.createLinearRing(m_coordinate_sequence.release()));
+                    } catch (geos::util::GEOSException&) {
+                        THROW(osmium::geos_geometry_error());
+                    }
+                }
+
+                void multipolygon_inner_ring_start() {
+                    try {
+                        m_coordinate_sequence.reset(m_geos_factory.getCoordinateSequenceFactory()->create(static_cast<size_t>(0), 2));
+                    } catch (geos::util::GEOSException&) {
+                        THROW(osmium::geos_geometry_error());
+                    }
+                }
+
+                void multipolygon_inner_ring_finish() {
+                    try {
+                        m_rings.emplace_back(m_geos_factory.createLinearRing(m_coordinate_sequence.release()));
+                    } catch (geos::util::GEOSException&) {
+                        THROW(osmium::geos_geometry_error());
+                    }
+                }
+
+                void multipolygon_add_location(const osmium::geom::Coordinates& xy) {
+                    try {
+                        m_coordinate_sequence->add(geos::geom::Coordinate(xy.x, xy.y));
+                    } catch (geos::util::GEOSException&) {
+                        THROW(osmium::geos_geometry_error());
+                    }
+                }
+
+                multipolygon_type multipolygon_finish() {
+                    try {
+                        auto polygons = new std::vector<geos::geom::Geometry*>;
+                        std::transform(m_polygons.begin(), m_polygons.end(), std::back_inserter(*polygons), [](std::unique_ptr<geos::geom::Polygon>& p) {
+                            return p.release();
+                        });
+                        m_polygons.clear();
+                        return multipolygon_type(m_geos_factory.createMultiPolygon(polygons));
+                    } catch (geos::util::GEOSException&) {
+                        THROW(osmium::geos_geometry_error());
+                    }
+                }
+
+            }; // class GEOSFactoryImpl
+
+        } // namespace detail
+
+        template <class TProjection = IdentityProjection>
+        using GEOSFactory = GeometryFactory<osmium::geom::detail::GEOSFactoryImpl, TProjection>;
+
+    } // namespace geom
+
+} // namespace osmium
+
+#undef THROW
+
+#endif // OSMIUM_GEOM_GEOS_HPP

include/osmium/geom/haversine.hpp

@@ -0,0 +1,94 @@
+#ifndef OSMIUM_GEOM_HAVERSINE_HPP
+#define OSMIUM_GEOM_HAVERSINE_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cmath>
+#include <iterator>
+
+#include <osmium/geom/coordinates.hpp>
+#include <osmium/geom/util.hpp>
+#include <osmium/osm/node_ref.hpp>
+#include <osmium/osm/way.hpp>
+
+namespace osmium {
+
+    namespace geom {
+
+        /**
+         * @brief Functions to calculate arc distance on Earth using the haversine formula.
+         *
+         * See http://en.wikipedia.org/wiki/Haversine_formula
+         *
+         * Implementation derived from
+         * http://blog.julien.cayzac.name/2008/10/arc-and-distance-between-two-points-on.html
+         */
+        namespace haversine {
+
+            /// @brief Earth's quadratic mean radius for WGS84
+            constexpr double EARTH_RADIUS_IN_METERS = 6372797.560856;
+
+            /**
+             * Calculate distance in meters between two sets of coordinates.
+             */
+            inline double distance(const osmium::geom::Coordinates& c1, const osmium::geom::Coordinates& c2) {
+                double lonh = sin(deg_to_rad(c1.x - c2.x) * 0.5);
+                lonh *= lonh;
+                double lath = sin(deg_to_rad(c1.y - c2.y) * 0.5);
+                lath *= lath;
+                const double tmp = cos(deg_to_rad(c1.y)) * cos(deg_to_rad(c2.y));
+                return 2.0 * EARTH_RADIUS_IN_METERS * asin(sqrt(lath + tmp*lonh));
+            }
+
+            /**
+             * Calculate length of way.
+             */
+            inline double distance(const osmium::WayNodeList& wnl) {
+                double sum_length=0;
+
+                for (auto it = wnl.begin(); it != wnl.end(); ++it) {
+                    if (std::next(it) != wnl.end()) {
+                        sum_length += distance(it->location(), std::next(it)->location());
+                    }
+                }
+
+                return sum_length;
+            }
+
+        } // namespace haversine
+
+    } // namespace geom
+
+} // namespace osmium
+
+#endif // OSMIUM_GEOM_HAVERSINE_HPP

include/osmium/geom/mercator_projection.hpp

@@ -0,0 +1,109 @@
+#ifndef OSMIUM_GEOM_MERCATOR_PROJECTION_HPP
+#define OSMIUM_GEOM_MERCATOR_PROJECTION_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cmath>
+#include <string>
+
+#include <osmium/geom/coordinates.hpp>
+#include <osmium/geom/util.hpp>
+#include <osmium/osm/location.hpp>
+
+namespace osmium {
+
+    namespace geom {
+
+        namespace detail {
+
+            constexpr double earth_radius_for_epsg3857 = 6378137.0;
+
+            constexpr inline double lon_to_x(double lon) {
+                return earth_radius_for_epsg3857 * deg_to_rad(lon);
+            }
+
+            inline double lat_to_y(double lat) { // not constexpr because math functions aren't
+                return earth_radius_for_epsg3857 * std::log(std::tan(osmium::geom::PI/4 + deg_to_rad(lat)/2));
+            }
+
+            constexpr inline double x_to_lon(double x) {
+                return rad_to_deg(x) / earth_radius_for_epsg3857;
+            }
+
+            inline double y_to_lat(double y) { // not constexpr because math functions aren't
+                return rad_to_deg(2 * std::atan(std::exp(y / earth_radius_for_epsg3857)) - osmium::geom::PI/2);
+            }
+
+        } // namespace detail
+
+        /**
+         * The maximum latitude that can be projected with the Web Mercator
+         * (EPSG:3857) projection.
+         */
+        constexpr double MERCATOR_MAX_LAT = 85.0511288;
+
+        inline Coordinates lonlat_to_mercator(const Coordinates& c) {
+            return Coordinates(detail::lon_to_x(c.x), detail::lat_to_y(c.y));
+        }
+
+        inline Coordinates mercator_to_lonlat(const Coordinates& c) {
+            return Coordinates(detail::x_to_lon(c.x), detail::y_to_lat(c.y));
+        }
+
+        /**
+         * Functor that does projection from WGS84 (EPSG:4326) to "Web
+         * Mercator" (EPSG:3857)
+         */
+        class MercatorProjection {
+
+        public:
+
+            Coordinates operator()(osmium::Location location) const {
+                return Coordinates {detail::lon_to_x(location.lon()), detail::lat_to_y(location.lat())};
+            }
+
+            int epsg() const noexcept {
+                return 3857;
+            }
+
+            std::string proj_string() const {
+                return "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs";
+            }
+
+        }; // class MercatorProjection
+
+    } // namespace geom
+
+} // namespace osmium
+
+#endif // OSMIUM_GEOM_MERCATOR_PROJECTION_HPP

include/osmium/geom/ogr.hpp

@@ -0,0 +1,176 @@
+#ifndef OSMIUM_GEOM_OGR_HPP
+#define OSMIUM_GEOM_OGR_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cassert>
+#include <cstddef>
+#include <memory>
+#include <utility>
+
+#pragma GCC diagnostic push
+#ifdef __clang__
+# pragma GCC diagnostic ignored "-Wdocumentation-unknown-command"
+#endif
+#pragma GCC diagnostic ignored "-Wfloat-equal"
+#pragma GCC diagnostic ignored "-Wpadded"
+# include <ogr_geometry.h>
+#pragma GCC diagnostic pop
+
+#include <osmium/geom/coordinates.hpp>
+#include <osmium/geom/factory.hpp>
+
+namespace osmium {
+
+    namespace geom {
+
+        namespace detail {
+
+            class OGRFactoryImpl {
+
+            public:
+
+                typedef std::unique_ptr<OGRPoint>        point_type;
+                typedef std::unique_ptr<OGRLineString>   linestring_type;
+                typedef std::unique_ptr<OGRPolygon>      polygon_type;
+                typedef std::unique_ptr<OGRMultiPolygon> multipolygon_type;
+                typedef std::unique_ptr<OGRLinearRing>   ring_type;
+
+            private:
+
+                linestring_type   m_linestring;
+                multipolygon_type m_multipolygon;
+                polygon_type      m_polygon;
+                ring_type         m_ring;
+
+            public:
+
+                OGRFactoryImpl() = default;
+
+                /* Point */
+
+                point_type make_point(const osmium::geom::Coordinates& xy) const {
+                    return point_type(new OGRPoint(xy.x, xy.y));
+                }
+
+                /* LineString */
+
+                void linestring_start() {
+                    m_linestring = std::unique_ptr<OGRLineString>(new OGRLineString());
+                }
+
+                void linestring_add_location(const osmium::geom::Coordinates& xy) {
+                    assert(!!m_linestring);
+                    m_linestring->addPoint(xy.x, xy.y);
+                }
+
+                linestring_type linestring_finish(size_t /* num_points */) {
+                    return std::move(m_linestring);
+                }
+
+                /* Polygon */
+
+                void polygon_start() {
+                    m_ring = std::unique_ptr<OGRLinearRing>(new OGRLinearRing());
+                }
+
+                void polygon_add_location(const osmium::geom::Coordinates& xy) {
+                    assert(!!m_ring);
+                    m_ring->addPoint(xy.x, xy.y);
+                }
+
+                polygon_type polygon_finish(size_t /* num_points */) {
+                    std::unique_ptr<OGRPolygon> polygon = std::unique_ptr<OGRPolygon>(new OGRPolygon());
+                    polygon->addRingDirectly(m_ring.release());
+                    return polygon;
+                }
+
+                /* MultiPolygon */
+
+                void multipolygon_start() {
+                    m_multipolygon.reset(new OGRMultiPolygon());
+                }
+
+                void multipolygon_polygon_start() {
+                    m_polygon.reset(new OGRPolygon());
+                }
+
+                void multipolygon_polygon_finish() {
+                    assert(!!m_multipolygon);
+                    assert(!!m_polygon);
+                    m_multipolygon->addGeometryDirectly(m_polygon.release());
+                }
+
+                void multipolygon_outer_ring_start() {
+                    m_ring.reset(new OGRLinearRing());
+                }
+
+                void multipolygon_outer_ring_finish() {
+                    assert(!!m_polygon);
+                    assert(!!m_ring);
+                    m_polygon->addRingDirectly(m_ring.release());
+                }
+
+                void multipolygon_inner_ring_start() {
+                    m_ring.reset(new OGRLinearRing());
+                }
+
+                void multipolygon_inner_ring_finish() {
+                    assert(!!m_polygon);
+                    assert(!!m_ring);
+                    m_polygon->addRingDirectly(m_ring.release());
+                }
+
+                void multipolygon_add_location(const osmium::geom::Coordinates& xy) {
+                    assert(!!m_polygon);
+                    assert(!!m_ring);
+                    m_ring->addPoint(xy.x, xy.y);
+                }
+
+                multipolygon_type multipolygon_finish() {
+                    assert(!!m_multipolygon);
+                    return std::move(m_multipolygon);
+                }
+
+            }; // class OGRFactoryImpl
+
+        } // namespace detail
+
+        template <class TProjection = IdentityProjection>
+        using OGRFactory = GeometryFactory<osmium::geom::detail::OGRFactoryImpl, TProjection>;
+
+    } // namespace geom
+
+} // namespace osmium
+
+#endif // OSMIUM_GEOM_OGR_HPP

include/osmium/geom/projection.hpp

@@ -0,0 +1,158 @@
+#ifndef OSMIUM_GEOM_PROJECTION_HPP
+#define OSMIUM_GEOM_PROJECTION_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <memory>
+#include <string>
+
+#include <proj_api.h>
+
+#include <osmium/geom/coordinates.hpp>
+#include <osmium/geom/util.hpp>
+#include <osmium/osm/location.hpp>
+
+namespace osmium {
+
+    namespace geom {
+
+        /**
+         * C++ wrapper for a Coordinate Reference System of the proj library.
+         */
+        class CRS {
+
+            struct ProjCRSDeleter {
+                void operator()(void* crs) {
+                    pj_free(crs);
+                }
+            }; // struct ProjCRSDeleter
+
+            std::unique_ptr<void, ProjCRSDeleter> m_crs;
+
+            projPJ get() const {
+                return m_crs.get();
+            }
+
+        public:
+
+            CRS(const std::string& crs) :
+                m_crs(pj_init_plus(crs.c_str()), ProjCRSDeleter()) {
+                if (!m_crs) {
+                    throw osmium::projection_error(std::string("creation of CRS failed: ") + pj_strerrno(*pj_get_errno_ref()));
+                }
+            }
+
+            CRS(int epsg) :
+                CRS(std::string("+init=epsg:") + std::to_string(epsg)) {
+            }
+
+            bool is_latlong() const {
+                return pj_is_latlong(m_crs.get());
+            }
+
+            bool is_geocent() const {
+                return pj_is_geocent(m_crs.get());
+            }
+
+            /**
+             * Transform coordinates from one CRS into another. Wraps the same function
+             * of the proj library.
+             *
+             * Coordinates have to be in radians and are produced in radians.
+             *
+             * @throws osmmium::projection_error if the projection fails
+             */
+            friend Coordinates transform(const CRS& src, const CRS& dest, Coordinates c) {
+                int result = pj_transform(src.get(), dest.get(), 1, 1, &c.x, &c.y, nullptr);
+                if (result != 0) {
+                    throw osmium::projection_error(std::string("projection failed: ") + pj_strerrno(result));
+                }
+                return c;
+            }
+
+        }; // class CRS
+
+        /**
+         * Functor that does projection from WGS84 (EPSG:4326) to the given
+         * CRS.
+         */
+        class Projection {
+
+            int m_epsg;
+            std::string m_proj_string;
+            CRS m_crs_wgs84 {4326};
+            CRS m_crs_user;
+
+        public:
+
+            Projection(const std::string& proj_string) :
+                m_epsg(-1),
+                m_proj_string(proj_string),
+                m_crs_user(proj_string) {
+            }
+
+            Projection(int epsg) :
+                m_epsg(epsg),
+                m_proj_string(std::string("+init=epsg:") + std::to_string(epsg)),
+                m_crs_user(epsg) {
+            }
+
+            Coordinates operator()(osmium::Location location) const {
+                Coordinates c {location.lon(), location.lat()};
+
+                if (m_epsg != 4326) {
+                    c = transform(m_crs_wgs84, m_crs_user, Coordinates(deg_to_rad(location.lon()), deg_to_rad(location.lat())));
+                    if (m_crs_user.is_latlong()) {
+                        c.x = rad_to_deg(c.x);
+                        c.y = rad_to_deg(c.y);
+                    }
+                }
+
+                return c;
+            }
+
+            int epsg() const noexcept {
+                return m_epsg;
+            }
+
+            std::string proj_string() const {
+                return m_proj_string;
+            }
+
+        }; // class Projection
+
+    } // namespace geom
+
+} // namespace osmium
+
+#endif // OSMIUM_GEOM_PROJECTION_HPP

include/osmium/geom/relations.hpp

@@ -0,0 +1,57 @@
+#ifndef OSMIUM_GEOM_RELATIONS_HPP
+#define OSMIUM_GEOM_RELATIONS_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <osmium/osm/box.hpp>
+#include <osmium/osm/location.hpp>
+
+namespace osmium {
+
+    namespace geom {
+
+        /**
+         * Check whether one geometry contains another.
+         */
+        inline bool contains(const osmium::Box& a, const osmium::Box& b) {
+            return ((a.bottom_left().x() >= b.bottom_left().x()) &&
+                    (a.top_right().x()   <= b.top_right().x())   &&
+                    (a.bottom_left().y() >= b.bottom_left().y()) &&
+                    (a.top_right().y()   <= b.top_right().y()));
+        }
+
+    } // namespace geom
+
+} // namespace osmium
+
+#endif // OSMIUM_GEOM_RELATIONS_HPP

include/osmium/geom/util.hpp

@@ -0,0 +1,75 @@
+#ifndef OSMIUM_GEOM_UTIL_HPP
+#define OSMIUM_GEOM_UTIL_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <stdexcept>
+#include <string>
+
+namespace osmium {
+
+    /**
+     * Exception thrown when a projection object can not be initialized or the
+     * projection of some coordinates can not be calculated.
+     */
+    struct projection_error : public std::runtime_error {
+
+        projection_error(const std::string& what) :
+            std::runtime_error(what) {
+        }
+
+        projection_error(const char* what) :
+            std::runtime_error(what) {
+        }
+
+    }; // struct projection_error
+
+    namespace geom {
+
+        constexpr double PI = 3.14159265358979323846;
+
+        /// Convert angle from degrees to radians.
+        inline constexpr double deg_to_rad(double degree) noexcept {
+            return degree * (PI / 180.0);
+        }
+
+        /// Convert angle from radians to degrees.
+        inline constexpr double rad_to_deg(double radians) noexcept {
+            return radians * (180.0 / PI);
+        }
+
+    } // namespace geom
+
+} // namespace osmium
+
+#endif // OSMIUM_GEOM_UTIL_HPP

include/osmium/geom/wkb.hpp

@@ -0,0 +1,277 @@
+#ifndef OSMIUM_GEOM_WKB_HPP
+#define OSMIUM_GEOM_WKB_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstddef>
+#include <cstdint>
+#include <string>
+
+// Windows is only available for little endian architectures
+// http://stackoverflow.com/questions/6449468/can-i-safely-assume-that-windows-installations-will-always-be-little-endian
+#if !defined(_WIN32) && !defined(__APPLE__)
+# include <endian.h>
+#else
+# define __LITTLE_ENDIAN 1234
+# define __BYTE_ORDER __LITTLE_ENDIAN
+#endif
+
+#include <osmium/geom/coordinates.hpp>
+#include <osmium/geom/factory.hpp>
+#include <osmium/util/cast.hpp>
+
+namespace osmium {
+
+    namespace geom {
+
+        enum class wkb_type : bool {
+            wkb  = false,
+            ewkb = true
+        }; // enum class wkb_type
+
+        enum class out_type : bool {
+            binary = false,
+            hex    = true
+        }; // enum class out_type
+
+        namespace detail {
+
+            template <typename T>
+            inline void str_push(std::string& str, T data) {
+                size_t size = str.size();
+                str.resize(size + sizeof(T));
+                std::copy_n(reinterpret_cast<char*>(&data), sizeof(T), &str[size]);
+            }
+
+            inline std::string convert_to_hex(const std::string& str) {
+                static const char* lookup_hex = "0123456789ABCDEF";
+                std::string out;
+
+                for (char c : str) {
+                    out += lookup_hex[(c >> 4) & 0xf];
+                    out += lookup_hex[c & 0xf];
+                }
+
+                return out;
+            }
+
+            class WKBFactoryImpl {
+
+                /// OSM data always uses SRID 4326 (WGS84).
+                static constexpr uint32_t srid = 4326;
+
+                /**
+                * Type of WKB geometry.
+                * These definitions are from
+                * 99-049_OpenGIS_Simple_Features_Specification_For_SQL_Rev_1.1.pdf (for WKB)
+                * and http://trac.osgeo.org/postgis/browser/trunk/doc/ZMSgeoms.txt (for EWKB).
+                * They are used to encode geometries into the WKB format.
+                */
+                enum wkbGeometryType : uint32_t {
+                    wkbPoint               = 1,
+                    wkbLineString          = 2,
+                    wkbPolygon             = 3,
+                    wkbMultiPoint          = 4,
+                    wkbMultiLineString     = 5,
+                    wkbMultiPolygon        = 6,
+                    wkbGeometryCollection  = 7,
+
+                    // SRID-presence flag (EWKB)
+                    wkbSRID                = 0x20000000
+                }; // enum wkbGeometryType
+
+                /**
+                * Byte order marker in WKB geometry.
+                */
+                enum class wkb_byte_order_type : uint8_t {
+                    XDR = 0,         // Big Endian
+                    NDR = 1          // Little Endian
+                }; // enum class wkb_byte_order_type
+
+                std::string m_data;
+                uint32_t m_points {0};
+                wkb_type m_wkb_type;
+                out_type m_out_type;
+
+                size_t m_linestring_size_offset = 0;
+                size_t m_polygons = 0;
+                size_t m_rings = 0;
+                size_t m_multipolygon_size_offset = 0;
+                size_t m_polygon_size_offset = 0;
+                size_t m_ring_size_offset = 0;
+
+                size_t header(std::string& str, wkbGeometryType type, bool add_length) const {
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+                    str_push(str, wkb_byte_order_type::NDR);
+#else
+                    str_push(str, wkb_byte_order_type::XDR);
+#endif
+                    if (m_wkb_type == wkb_type::ewkb) {
+                        str_push(str, type | wkbSRID);
+                        str_push(str, srid);
+                    } else {
+                        str_push(str, type);
+                    }
+                    size_t offset = str.size();
+                    if (add_length) {
+                        str_push(str, static_cast<uint32_t>(0));
+                    }
+                    return offset;
+                }
+
+                void set_size(const size_t offset, const size_t size) {
+                    *reinterpret_cast<uint32_t*>(&m_data[offset]) = static_cast_with_assert<uint32_t>(size);
+                }
+
+            public:
+
+                typedef std::string point_type;
+                typedef std::string linestring_type;
+                typedef std::string polygon_type;
+                typedef std::string multipolygon_type;
+                typedef std::string ring_type;
+
+                explicit WKBFactoryImpl(wkb_type wtype=wkb_type::wkb, out_type otype=out_type::binary) :
+                    m_wkb_type(wtype),
+                    m_out_type(otype) {
+                }
+
+                /* Point */
+
+                point_type make_point(const osmium::geom::Coordinates& xy) const {
+                    std::string data;
+                    header(data, wkbPoint, false);
+                    str_push(data, xy.x);
+                    str_push(data, xy.y);
+
+                    if (m_out_type == out_type::hex) {
+                        return convert_to_hex(data);
+                    } else {
+                        return data;
+                    }
+                }
+
+                /* LineString */
+
+                void linestring_start() {
+                    m_data.clear();
+                    m_linestring_size_offset = header(m_data, wkbLineString, true);
+                }
+
+                void linestring_add_location(const osmium::geom::Coordinates& xy) {
+                    str_push(m_data, xy.x);
+                    str_push(m_data, xy.y);
+                }
+
+                linestring_type linestring_finish(size_t num_points) {
+                    set_size(m_linestring_size_offset, num_points);
+                    std::string data;
+                    std::swap(data, m_data);
+
+                    if (m_out_type == out_type::hex) {
+                        return convert_to_hex(data);
+                    } else {
+                        return data;
+                    }
+                }
+
+                /* MultiPolygon */
+
+                void multipolygon_start() {
+                    m_data.clear();
+                    m_polygons = 0;
+                    m_multipolygon_size_offset = header(m_data, wkbMultiPolygon, true);
+                }
+
+                void multipolygon_polygon_start() {
+                    ++m_polygons;
+                    m_rings = 0;
+                    m_polygon_size_offset = header(m_data, wkbPolygon, true);
+                }
+
+                void multipolygon_polygon_finish() {
+                    set_size(m_polygon_size_offset, m_rings);
+                }
+
+                void multipolygon_outer_ring_start() {
+                    ++m_rings;
+                    m_points = 0;
+                    m_ring_size_offset = m_data.size();
+                    str_push(m_data, static_cast<uint32_t>(0));
+                }
+
+                void multipolygon_outer_ring_finish() {
+                    set_size(m_ring_size_offset, m_points);
+                }
+
+                void multipolygon_inner_ring_start() {
+                    ++m_rings;
+                    m_points = 0;
+                    m_ring_size_offset = m_data.size();
+                    str_push(m_data, static_cast<uint32_t>(0));
+                }
+
+                void multipolygon_inner_ring_finish() {
+                    set_size(m_ring_size_offset, m_points);
+                }
+
+                void multipolygon_add_location(const osmium::geom::Coordinates& xy) {
+                    str_push(m_data, xy.x);
+                    str_push(m_data, xy.y);
+                    ++m_points;
+                }
+
+                multipolygon_type multipolygon_finish() {
+                    set_size(m_multipolygon_size_offset, m_polygons);
+                    std::string data;
+                    std::swap(data, m_data);
+
+                    if (m_out_type == out_type::hex) {
+                        return convert_to_hex(data);
+                    } else {
+                        return data;
+                    }
+                }
+
+            }; // class WKBFactoryImpl
+
+        } // namespace detail
+
+        template <class TProjection = IdentityProjection>
+        using WKBFactory = GeometryFactory<osmium::geom::detail::WKBFactoryImpl, TProjection>;
+
+    } // namespace geom
+
+} // namespace osmium
+
+#endif // OSMIUM_GEOM_WKB_HPP

include/osmium/geom/wkt.hpp

@@ -0,0 +1,150 @@
+#ifndef OSMIUM_GEOM_WKT_HPP
+#define OSMIUM_GEOM_WKT_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cassert>
+#include <cstddef>
+#include <string>
+#include <utility>
+
+#include <osmium/geom/coordinates.hpp>
+#include <osmium/geom/factory.hpp>
+
+namespace osmium {
+
+    namespace geom {
+
+        namespace detail {
+
+            class WKTFactoryImpl {
+
+                std::string m_str;
+                int m_precision;
+
+            public:
+
+                typedef std::string point_type;
+                typedef std::string linestring_type;
+                typedef std::string polygon_type;
+                typedef std::string multipolygon_type;
+                typedef std::string ring_type;
+
+                WKTFactoryImpl(int precision = 7) :
+                    m_precision(precision) {
+                }
+
+                /* Point */
+
+                point_type make_point(const osmium::geom::Coordinates& xy) const {
+                    std::string str {"POINT"};
+                    xy.append_to_string(str, '(', ' ', ')', m_precision);
+                    return str;
+                }
+
+                /* LineString */
+
+                void linestring_start() {
+                    m_str = "LINESTRING(";
+                }
+
+                void linestring_add_location(const osmium::geom::Coordinates& xy) {
+                    xy.append_to_string(m_str, ' ', m_precision);
+                    m_str += ',';
+                }
+
+                linestring_type linestring_finish(size_t /* num_points */) {
+                    assert(!m_str.empty());
+                    std::string str;
+                    std::swap(str, m_str);
+                    str.back() = ')';
+                    return str;
+                }
+
+                /* MultiPolygon */
+
+                void multipolygon_start() {
+                    m_str = "MULTIPOLYGON(";
+                }
+
+                void multipolygon_polygon_start() {
+                    m_str += '(';
+                }
+
+                void multipolygon_polygon_finish() {
+                    m_str += "),";
+                }
+
+                void multipolygon_outer_ring_start() {
+                    m_str += '(';
+                }
+
+                void multipolygon_outer_ring_finish() {
+                    assert(!m_str.empty());
+                    m_str.back() = ')';
+                }
+
+                void multipolygon_inner_ring_start() {
+                    m_str += ",(";
+                }
+
+                void multipolygon_inner_ring_finish() {
+                    assert(!m_str.empty());
+                    m_str.back() = ')';
+                }
+
+                void multipolygon_add_location(const osmium::geom::Coordinates& xy) {
+                    xy.append_to_string(m_str, ' ', m_precision);
+                    m_str += ',';
+                }
+
+                multipolygon_type multipolygon_finish() {
+                    assert(!m_str.empty());
+                    std::string str;
+                    std::swap(str, m_str);
+                    str.back() = ')';
+                    return str;
+                }
+
+            }; // class WKTFactoryImpl
+
+        } // namespace detail
+
+        template <class TProjection = IdentityProjection>
+        using WKTFactory = GeometryFactory<osmium::geom::detail::WKTFactoryImpl, TProjection>;
+
+    } // namespace geom
+
+} // namespace osmium
+
+#endif // OSMIUM_GEOM_WKT_HPP

include/osmium/handler.hpp

@@ -0,0 +1,101 @@
+#ifndef OSMIUM_HANDLER_HPP
+#define OSMIUM_HANDLER_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+namespace osmium {
+
+    class OSMObject;
+    class Node;
+    class Way;
+    class Relation;
+    class Area;
+    class Changeset;
+    class TagList;
+    class WayNodeList;
+    class RelationMemberList;
+    class OuterRing;
+    class InnerRing;
+
+    /**
+     * @brief Osmium handlers provide callbacks for OSM objects
+     */
+    namespace handler {
+
+        class Handler {
+
+        public:
+
+            void osm_object(const osmium::OSMObject&) const {
+            }
+
+            void node(const osmium::Node&) const {
+            }
+
+            void way(const osmium::Way&) const {
+            }
+
+            void relation(const osmium::Relation&) const {
+            }
+
+            void area(const osmium::Area&) const {
+            }
+
+            void changeset(const osmium::Changeset&) const {
+            }
+
+            void tag_list(const osmium::TagList&) const {
+            }
+
+            void way_node_list(const osmium::WayNodeList&) const {
+            }
+
+            void relation_member_list(const osmium::RelationMemberList&) const {
+            }
+
+            void outer_ring(const osmium::OuterRing&) const {
+            }
+
+            void inner_ring(const osmium::InnerRing&) const {
+            }
+
+            void flush() const {
+            }
+
+        }; // class Handler
+
+    } // namspace handler
+
+} // namespace osmium
+
+#endif // OSMIUM_HANDLER_HPP

include/osmium/handler/chain.hpp

@@ -0,0 +1,128 @@
+#ifndef OSMIUM_HANDLER_CHAIN_HPP
+#define OSMIUM_HANDLER_CHAIN_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <tuple>
+
+#include <osmium/handler.hpp>
+
+#define OSMIUM_CHAIN_HANDLER_CALL(_func_, _type_) \
+    template <int N, int SIZE, class THandlers> \
+    struct call_ ## _func_ { \
+        void operator()(THandlers& handlers, osmium::_type_& object) { \
+            std::get<N>(handlers)._func_(object); \
+            call_ ## _func_<N+1, SIZE, THandlers>()(handlers, object); \
+        } \
+    }; \
+    template <int SIZE, class THandlers> \
+    struct call_ ## _func_<SIZE, SIZE, THandlers> { \
+        void operator()(THandlers&, osmium::_type_&) {} \
+    };
+
+namespace osmium {
+
+    class Node;
+    class Way;
+    class Relation;
+    class Area;
+    class Changeset;
+
+    namespace handler {
+
+        /**
+         * This handler allows chaining of any number of handlers into a single
+         * handler.
+         */
+        template <class ...THandler>
+        class ChainHandler : public osmium::handler::Handler {
+
+            typedef std::tuple<THandler&...> handlers_type;
+            handlers_type m_handlers;
+
+            template <int N, int SIZE, class THandlers>
+            struct call_flush {
+                void operator()(THandlers& handlers) {
+                    std::get<N>(handlers).flush();
+                    call_flush<N+1, SIZE, THandlers>()(handlers);
+                }
+            }; // struct call_flush
+
+            template <int SIZE, class THandlers>
+            struct call_flush<SIZE, SIZE, THandlers> {
+                void operator()(THandlers&) {}
+            }; // struct call_flush
+
+            OSMIUM_CHAIN_HANDLER_CALL(node, Node)
+            OSMIUM_CHAIN_HANDLER_CALL(way, Way)
+            OSMIUM_CHAIN_HANDLER_CALL(relation, Relation)
+            OSMIUM_CHAIN_HANDLER_CALL(changeset, Changeset)
+            OSMIUM_CHAIN_HANDLER_CALL(area, Area)
+
+        public:
+
+            explicit ChainHandler(THandler&... handlers) :
+                m_handlers(handlers...) {
+            }
+
+            void node(osmium::Node& node) {
+                call_node<0, sizeof...(THandler), handlers_type>()(m_handlers, node);
+            }
+
+            void way(osmium::Way& way) {
+                call_way<0, sizeof...(THandler), handlers_type>()(m_handlers, way);
+            }
+
+            void relation(osmium::Relation& relation) {
+                call_relation<0, sizeof...(THandler), handlers_type>()(m_handlers, relation);
+            }
+
+            void changeset( osmium::Changeset& changeset) {
+                call_changeset<0, sizeof...(THandler), handlers_type>()(m_handlers, changeset);
+            }
+
+            void area(osmium::Area& area) {
+                call_area<0, sizeof...(THandler), handlers_type>()(m_handlers, area);
+            }
+
+            void flush() {
+                call_flush<0, sizeof...(THandler), handlers_type>()(m_handlers);
+            }
+
+        }; // class ChainHandler
+
+    } // namespace handler
+
+} // namespace osmium
+
+#endif // OSMIUM_HANDLER_CHAIN_HPP

include/osmium/handler/disk_store.hpp

@@ -0,0 +1,111 @@
+#ifndef OSMIUM_HANDLER_DISK_STORE_HPP
+#define OSMIUM_HANDLER_DISK_STORE_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstddef>
+
+#include <osmium/handler.hpp>
+#include <osmium/index/map.hpp>
+#include <osmium/io/detail/read_write.hpp>
+#include <osmium/memory/buffer.hpp>
+#include <osmium/memory/item_iterator.hpp>
+#include <osmium/osm/node.hpp>
+#include <osmium/osm/relation.hpp>
+#include <osmium/osm/types.hpp>
+#include <osmium/osm/way.hpp>
+#include <osmium/visitor.hpp>
+
+namespace osmium {
+
+    namespace handler {
+
+        /**
+         *
+         * Note: This handler will only work if either all object IDs are
+         *       positive or all object IDs are negative.
+         */
+        class DiskStore : public osmium::handler::Handler {
+
+            typedef osmium::index::map::Map<unsigned_object_id_type, size_t> offset_index_type;
+
+            size_t m_offset = 0;
+            int m_data_fd;
+
+            offset_index_type& m_node_index;
+            offset_index_type& m_way_index;
+            offset_index_type& m_relation_index;
+
+        public:
+
+            explicit DiskStore(int data_fd, offset_index_type& node_index, offset_index_type& way_index, offset_index_type& relation_index) :
+                m_data_fd(data_fd),
+                m_node_index(node_index),
+                m_way_index(way_index),
+                m_relation_index(relation_index) {
+            }
+
+            DiskStore(const DiskStore&) = delete;
+            DiskStore& operator=(const DiskStore&) = delete;
+
+            ~DiskStore() noexcept = default;
+
+            void node(const osmium::Node& node) {
+                m_node_index.set(node.positive_id(), m_offset);
+                m_offset += node.byte_size();
+            }
+
+            void way(const osmium::Way& way) {
+                m_way_index.set(way.positive_id(), m_offset);
+                m_offset += way.byte_size();
+            }
+
+            void relation(const osmium::Relation& relation) {
+                m_relation_index.set(relation.positive_id(), m_offset);
+                m_offset += relation.byte_size();
+            }
+
+            // XXX
+            void operator()(const osmium::memory::Buffer& buffer) {
+                osmium::io::detail::reliable_write(m_data_fd, buffer.data(), buffer.committed());
+
+                osmium::apply(buffer.begin(), buffer.end(), *this);
+            }
+
+        }; // class DiskStore
+
+    } // namespace handler
+
+} // namespace osmium
+
+#endif // OSMIUM_HANDLER_DISK_STORE_HPP

include/osmium/handler/dump.hpp

@@ -0,0 +1,294 @@
+#ifndef OSMIUM_HANDLER_DUMP_HPP
+#define OSMIUM_HANDLER_DUMP_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <iomanip>
+#include <iostream>
+#include <string>
+
+#include <osmium/handler.hpp>
+#include <osmium/memory/collection.hpp>
+#include <osmium/memory/item.hpp>
+#include <osmium/osm/area.hpp>
+#include <osmium/osm/box.hpp>
+#include <osmium/osm/changeset.hpp>
+#include <osmium/osm/item_type.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/node.hpp>
+#include <osmium/osm/node_ref.hpp>
+#include <osmium/osm/object.hpp>
+#include <osmium/osm/relation.hpp>
+#include <osmium/osm/tag.hpp>
+#include <osmium/osm/timestamp.hpp>
+#include <osmium/osm/way.hpp>
+#include <osmium/visitor.hpp>
+
+namespace osmium {
+
+    namespace handler {
+
+        class Dump : public osmium::handler::Handler {
+
+            std::ostream* m_out;
+            bool m_with_size;
+            std::string m_prefix;
+
+            void print_title(const char* title, const osmium::memory::Item& item) {
+                *m_out << m_prefix
+                       << title
+                       << ":";
+
+                if (m_with_size) {
+                    *m_out << " ["
+                           << item.byte_size()
+                           << "]";
+                }
+
+                *m_out << "\n";
+            }
+
+            void print_meta(const osmium::OSMObject& object) {
+                *m_out << m_prefix
+                       << "  id="
+                       << object.id()
+                       << "\n";
+                *m_out << m_prefix
+                       << "  version="
+                       << object.version()
+                       << "\n";
+                *m_out << m_prefix
+                       << "  uid="
+                       << object.uid()
+                       << "\n";
+                *m_out << m_prefix
+                       << "  user=|"
+                       << object.user()
+                       << "|\n";
+                *m_out << m_prefix
+                       << "  changeset="
+                       << object.changeset()
+                       << "\n";
+                *m_out << m_prefix
+                       << "  timestamp="
+                       << object.timestamp().to_iso()
+                       << "\n";
+                *m_out << m_prefix
+                       << "  visible="
+                       << (object.visible() ? "yes" : "no")
+                       << "\n";
+
+                Dump dump(*m_out, m_with_size, m_prefix + "  ");
+                osmium::apply(object.cbegin(), object.cend(), dump);
+            }
+
+            void print_location(const osmium::Node& node) {
+                const osmium::Location& location = node.location();
+
+                if (location) {
+                    *m_out << m_prefix
+                           << "  lon="
+                           << std::fixed
+                           << std::setprecision(7)
+                           << location.lon_without_check()
+                           << "\n";
+                    *m_out << m_prefix
+                           << "  lat="
+                           << location.lat_without_check()
+                           << "\n";
+                } else {
+                    *m_out << m_prefix
+                           << "  lon=\n"
+                           << m_prefix
+                           << "  lat=\n";
+                }
+            }
+
+        public:
+
+            explicit Dump(std::ostream& out, bool with_size=true, const std::string& prefix="") :
+                m_out(&out),
+                m_with_size(with_size),
+                m_prefix(prefix) {
+            }
+
+            void tag_list(const osmium::TagList& tags) {
+                print_title("TAGS", tags);
+                for (const auto& tag : tags) {
+                    *m_out << m_prefix
+                           << "  k=|"
+                           << tag.key()
+                           << "| v=|"
+                           << tag.value()
+                           << "|"
+                           << "\n";
+                }
+            }
+
+            void way_node_list(const osmium::WayNodeList& wnl) {
+                print_title("NODES", wnl);
+                for (const auto& node_ref : wnl) {
+                    *m_out << m_prefix
+                           << "  ref="
+                           << node_ref.ref();
+                    if (node_ref.location()) {
+                        *m_out << " pos="
+                               << node_ref.location();
+                    }
+                    *m_out << "\n";
+                }
+            }
+
+            void relation_member_list(const osmium::RelationMemberList& rml) {
+                print_title("MEMBERS", rml);
+                for (const auto& member : rml) {
+                    *m_out << m_prefix
+                           << "  type="
+                           << item_type_to_name(member.type())
+                           << " ref="
+                           << member.ref()
+                           << " role=|"
+                           << member.role()
+                           << "|\n";
+                    if (member.full_member()) {
+                        Dump dump(*m_out, m_with_size, m_prefix + "  | ");
+                        osmium::apply_item(member.get_object(), dump);
+                    }
+                }
+            }
+
+            void outer_ring(const osmium::OuterRing& ring) {
+                print_title("OUTER RING", ring);
+                for (const auto& node_ref : ring) {
+                    *m_out << m_prefix
+                           << "  ref="
+                           << node_ref.ref();
+                    if (node_ref.location()) {
+                        *m_out << " pos="
+                               << node_ref.location();
+                    }
+                    *m_out << "\n";
+                }
+            }
+
+            void inner_ring(const osmium::InnerRing& ring) {
+                print_title("INNER RING", ring);
+                for (const auto& node_ref : ring) {
+                    *m_out << m_prefix
+                           << "  ref="
+                           << node_ref.ref();
+                    if (node_ref.location()) {
+                        *m_out << " pos="
+                               << node_ref.location();
+                    }
+                    *m_out << "\n";
+                }
+            }
+
+            void node(const osmium::Node& node) {
+                print_title("NODE", node);
+                print_meta(node);
+                print_location(node);
+            }
+
+            void way(const osmium::Way& way) {
+                print_title("WAY", way);
+                print_meta(way);
+            }
+
+            void relation(const osmium::Relation& relation) {
+                print_title("RELATION", relation);
+                print_meta(relation);
+            }
+
+            void area(const osmium::Area& area) {
+                print_title("AREA", area);
+                print_meta(area);
+            }
+
+            void changeset(const osmium::Changeset& changeset) {
+                print_title("CHANGESET", changeset);
+                *m_out << m_prefix
+                       << "  id="
+                       << changeset.id()
+                       << "\n";
+                *m_out << m_prefix
+                       << "  num_changes="
+                       << changeset.num_changes()
+                       << "\n";
+                *m_out << m_prefix
+                       << "  uid="
+                       << changeset.uid()
+                       << "\n";
+                *m_out << m_prefix
+                       << "  user=|"
+                       << changeset.user()
+                       << "|\n";
+                *m_out << m_prefix
+                       << "  created_at="
+                       << changeset.created_at().to_iso()
+                       << "\n";
+                *m_out << m_prefix
+                       << "  closed_at="
+                       << changeset.closed_at().to_iso()
+                       << "\n";
+                *m_out << m_prefix
+                       << "  bounds=";
+
+                if (changeset.bounds()) {
+                    *m_out << '('
+                           << changeset.bounds().bottom_left().lon_without_check()
+                           << ','
+                           << changeset.bounds().bottom_left().lat_without_check()
+                           << ','
+                           << changeset.bounds().top_right().lon_without_check()
+                           << ','
+                           << changeset.bounds().top_right().lat_without_check()
+                           << ')';
+                } else {
+                    *m_out << "(undefined)";
+                }
+
+                *m_out << "\n";
+
+                Dump dump(*m_out, m_with_size, m_prefix + "  ");
+                osmium::apply(changeset.cbegin(), changeset.cend(), dump);
+            }
+
+        }; // class Dump
+
+    } // namespace handler
+
+} // namespace osmium
+
+#endif // OSMIUM_HANDLER_DUMP_HPP

include/osmium/handler/node_locations_for_ways.hpp

@@ -0,0 +1,177 @@
+#ifndef OSMIUM_HANDLER_NODE_LOCATIONS_FOR_WAYS_HPP
+#define OSMIUM_HANDLER_NODE_LOCATIONS_FOR_WAYS_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <type_traits>
+
+#include <osmium/handler.hpp>
+#include <osmium/index/index.hpp>
+#include <osmium/index/map/dummy.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/node.hpp>
+#include <osmium/osm/node_ref.hpp>
+#include <osmium/osm/types.hpp>
+#include <osmium/osm/way.hpp>
+
+namespace osmium {
+
+    namespace handler {
+
+        typedef osmium::index::map::Dummy<osmium::unsigned_object_id_type, osmium::Location> dummy_type;
+
+        /**
+         * Handler to retrieve locations from nodes and add them to ways.
+         *
+         * @tparam TStoragePosIDs Class that handles the actual storage of the node locations
+         *                        (for positive IDs). It must support the set(id, value) and
+         *                        get(id) methods.
+         * @tparam TStorageNegIDs Same but for negative IDs.
+         */
+        template <class TStoragePosIDs, class TStorageNegIDs = dummy_type>
+        class NodeLocationsForWays : public osmium::handler::Handler {
+
+            static_assert(std::is_base_of<osmium::index::map::Map<osmium::unsigned_object_id_type, osmium::Location>, TStoragePosIDs>::value,
+                "Index class must be derived from osmium::index::map::Map<osmium::unsigned_object_id_type, osmium::Location>");
+
+            static_assert(std::is_base_of<osmium::index::map::Map<osmium::unsigned_object_id_type, osmium::Location>, TStorageNegIDs>::value,
+                "Index class must be derived from osmium::index::map::Map<osmium::unsigned_object_id_type, osmium::Location>");
+
+        public:
+
+            typedef TStoragePosIDs index_pos_type;
+            typedef TStorageNegIDs index_neg_type;
+
+        private:
+
+            /// Object that handles the actual storage of the node locations (with positive IDs).
+            TStoragePosIDs& m_storage_pos;
+
+            /// Object that handles the actual storage of the node locations (with negative IDs).
+            TStorageNegIDs& m_storage_neg;
+
+            bool m_ignore_errors {false};
+
+            bool m_must_sort {false};
+
+            // It is okay to have this static dummy instance, even when using several threads,
+            // because it is read-only.
+            static dummy_type& get_dummy() {
+                static dummy_type instance;
+                return instance;
+            }
+
+        public:
+
+            explicit NodeLocationsForWays(TStoragePosIDs& storage_pos,
+                                          TStorageNegIDs& storage_neg = get_dummy()) :
+                m_storage_pos(storage_pos),
+                m_storage_neg(storage_neg) {
+            }
+
+            NodeLocationsForWays(const NodeLocationsForWays&) = delete;
+            NodeLocationsForWays& operator=(const NodeLocationsForWays&) = delete;
+
+            ~NodeLocationsForWays() noexcept = default;
+
+            void ignore_errors() {
+                m_ignore_errors = true;
+            }
+
+            /**
+             * Store the location of the node in the storage.
+             */
+            void node(const osmium::Node& node) {
+                m_must_sort = true;
+                const osmium::object_id_type id = node.id();
+                if (id >= 0) {
+                    m_storage_pos.set(static_cast<osmium::unsigned_object_id_type>( id), node.location());
+                } else {
+                    m_storage_neg.set(static_cast<osmium::unsigned_object_id_type>(-id), node.location());
+                }
+            }
+
+            /**
+             * Get location of node with given id.
+             */
+            osmium::Location get_node_location(const osmium::object_id_type id) const {
+                if (id >= 0) {
+                    return m_storage_pos.get(static_cast<osmium::unsigned_object_id_type>( id));
+                } else {
+                    return m_storage_neg.get(static_cast<osmium::unsigned_object_id_type>(-id));
+                }
+            }
+
+            /**
+             * Retrieve locations of all nodes in the way from storage and add
+             * them to the way object.
+             */
+            void way(osmium::Way& way) {
+                if (m_must_sort) {
+                    m_storage_pos.sort();
+                    m_storage_neg.sort();
+                    m_must_sort = false;
+                }
+                bool error = false;
+                for (auto& node_ref : way.nodes()) {
+                    try {
+                        node_ref.set_location(get_node_location(node_ref.ref()));
+                        if (!node_ref.location()) {
+                            error = true;
+                        }
+                    } catch (osmium::not_found&) {
+                        error = true;
+                    }
+                }
+                if (error && !m_ignore_errors) {
+                    throw osmium::not_found("location for one or more nodes not found in node location index");
+                }
+            }
+
+            /**
+             * Call clear on the location indexes. Makes the
+             * NodeLocationsForWays handler unusable. Used to explicitly free
+             * memory if thats needed.
+             */
+            void clear() {
+                m_storage_pos.clear();
+                m_storage_neg.clear();
+            }
+
+        }; // class NodeLocationsForWays
+
+    } // namespace handler
+
+} // namespace osmium
+
+#endif // OSMIUM_HANDLER_NODE_LOCATIONS_FOR_WAYS_HPP

include/osmium/handler/object_relations.hpp

@@ -0,0 +1,106 @@
+#ifndef OSMIUM_HANDLER_OBJECT_RELATIONS_HPP
+#define OSMIUM_HANDLER_OBJECT_RELATIONS_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <osmium/handler.hpp>
+#include <osmium/index/multimap.hpp>
+#include <osmium/osm/node_ref.hpp>
+#include <osmium/osm/item_type.hpp>
+#include <osmium/osm/relation.hpp>
+#include <osmium/osm/types.hpp>
+#include <osmium/osm/way.hpp>
+
+namespace osmium {
+
+    namespace handler {
+
+        /**
+         *
+         * Note: This handler will only work if either all object IDs are
+         *       positive or all object IDs are negative.
+         */
+        class ObjectRelations : public osmium::handler::Handler {
+
+            typedef osmium::index::multimap::Multimap<unsigned_object_id_type, unsigned_object_id_type> index_type;
+
+            index_type& m_index_n2w;
+            index_type& m_index_n2r;
+            index_type& m_index_w2r;
+            index_type& m_index_r2r;
+
+        public:
+
+            explicit ObjectRelations(index_type& n2w, index_type& n2r, index_type& w2r, index_type& r2r) :
+                m_index_n2w(n2w),
+                m_index_n2r(n2r),
+                m_index_w2r(w2r),
+                m_index_r2r(r2r) {
+            }
+
+            ObjectRelations(const ObjectRelations&) = delete;
+            ObjectRelations& operator=(const ObjectRelations&) = delete;
+
+            ~ObjectRelations() noexcept = default;
+
+            void way(const osmium::Way& way) {
+                for (const auto& node_ref : way.nodes()) {
+                    m_index_n2w.set(node_ref.positive_ref(), way.positive_id());
+                }
+            }
+
+            void relation(const osmium::Relation& relation) {
+                for (const auto& member : relation.members()) {
+                    switch (member.type()) {
+                        case osmium::item_type::node:
+                            m_index_n2r.set(member.positive_ref(), relation.positive_id());
+                            break;
+                        case osmium::item_type::way:
+                            m_index_w2r.set(member.positive_ref(), relation.positive_id());
+                            break;
+                        case osmium::item_type::relation:
+                            m_index_r2r.set(member.positive_ref(), relation.positive_id());
+                            break;
+                        default:
+                            break;
+                    }
+                }
+            }
+
+        }; // class ObjectRelations
+
+    } // namespace handler
+
+} // namespace osmium
+
+#endif // OSMIUM_HANDLER_OBJECT_RELATIONS_HPP

include/osmium/index/detail/mmap_vector_anon.hpp

@@ -0,0 +1,78 @@
+#ifndef OSMIUM_DETAIL_MMAP_VECTOR_ANON_HPP
+#define OSMIUM_DETAIL_MMAP_VECTOR_ANON_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#ifdef __linux__
+
+#include <cstddef>
+
+#include <osmium/index/detail/typed_mmap.hpp>
+#include <osmium/index/detail/mmap_vector_base.hpp>
+
+namespace osmium {
+
+    namespace detail {
+
+        /**
+        * This class looks and behaves like STL vector, but uses mmap internally.
+        */
+        template <typename T>
+        class mmap_vector_anon : public mmap_vector_base<T, mmap_vector_anon> {
+
+        public:
+
+            mmap_vector_anon() :
+                mmap_vector_base<T, osmium::detail::mmap_vector_anon>(
+                    -1,
+                    osmium::detail::mmap_vector_size_increment,
+                    0,
+                    osmium::detail::typed_mmap<T>::map(osmium::detail::mmap_vector_size_increment)) {
+            }
+
+            void reserve(size_t new_capacity) {
+                if (new_capacity > this->capacity()) {
+                    this->data(osmium::detail::typed_mmap<T>::remap(this->data(), this->capacity(), new_capacity));
+                    this->m_capacity = new_capacity;
+                }
+            }
+
+        }; // class mmap_vector_anon
+
+    } // namespace detail
+
+} // namespace osmium
+
+#endif // __linux__
+
+#endif // OSMIUM_DETAIL_MMAP_VECTOR_ANON_HPP

include/osmium/index/detail/mmap_vector_base.hpp

@@ -0,0 +1,183 @@
+#ifndef OSMIUM_DETAIL_MMAP_VECTOR_BASE_HPP
+#define OSMIUM_DETAIL_MMAP_VECTOR_BASE_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstddef>
+#include <new>
+#include <stdexcept>
+
+#include <osmium/index/detail/typed_mmap.hpp>
+
+namespace osmium {
+
+    namespace detail {
+
+        constexpr size_t mmap_vector_size_increment = 1024 * 1024;
+
+        /**
+         * This is a base class for implementing classes that look like
+         * STL vector but use mmap internally. This class can not be used
+         * on it's own. Use the derived classes mmap_vector_anon or
+         * mmap_vector_file.
+         */
+        template <typename T, template <typename> class TDerived>
+        class mmap_vector_base {
+
+        protected:
+
+            int m_fd;
+            size_t m_capacity;
+            size_t m_size;
+            T* m_data;
+
+            explicit mmap_vector_base(int fd, size_t capacity, size_t size, T* data) noexcept :
+                m_fd(fd),
+                m_capacity(capacity),
+                m_size(size),
+                m_data(data) {
+            }
+
+            explicit mmap_vector_base(int fd, size_t capacity, size_t size) :
+                m_fd(fd),
+                m_capacity(capacity),
+                m_size(size),
+                m_data(osmium::detail::typed_mmap<T>::grow_and_map(capacity, m_fd)) {
+            }
+
+            void data(T* data) {
+                m_data = data;
+            }
+
+        public:
+
+            typedef T value_type;
+            typedef T& reference;
+            typedef const T& const_reference;
+            typedef T* pointer;
+            typedef const T* const_pointer;
+            typedef T* iterator;
+            typedef const T* const_iterator;
+
+            ~mmap_vector_base() {
+                osmium::detail::typed_mmap<T>::unmap(m_data, m_capacity);
+            }
+
+            size_t capacity() const noexcept {
+                return m_capacity;
+            }
+
+            size_t size() const noexcept {
+                return m_size;
+            }
+
+            bool empty() const noexcept {
+                return m_size == 0;
+            }
+
+            const T* data() const noexcept {
+                return m_data;
+            }
+
+            T* data() noexcept {
+                return m_data;
+            }
+
+            T& operator[](size_t n) {
+                return m_data[n];
+            }
+
+            T at(size_t n) const {
+                if (n >= m_size) {
+                    throw std::out_of_range("out of range");
+                }
+                return m_data[n];
+            }
+
+            void clear() noexcept {
+                m_size = 0;
+            }
+
+            void shrink_to_fit() {
+                // XXX do something here
+            }
+
+            void push_back(const T& value) {
+                if (m_size >= m_capacity) {
+                    resize(m_size+1);
+                }
+                m_data[m_size] = value;
+                ++m_size;
+            }
+
+            void resize(size_t new_size) {
+                if (new_size > capacity()) {
+                    static_cast<TDerived<T>*>(this)->reserve(new_size + osmium::detail::mmap_vector_size_increment);
+                }
+                if (new_size > size()) {
+                    new (data() + size()) T[new_size - size()];
+                }
+                m_size = new_size;
+            }
+
+            iterator begin() noexcept {
+                return m_data;
+            }
+
+            iterator end() noexcept {
+                return m_data + m_size;
+            }
+
+            const_iterator begin() const noexcept {
+                return m_data;
+            }
+
+            const_iterator end() const noexcept {
+                return m_data + m_size;
+            }
+
+            const_iterator cbegin() noexcept {
+                return m_data;
+            }
+
+            const_iterator cend() noexcept {
+                return m_data + m_size;
+            }
+
+        }; // class mmap_vector_base
+
+    } // namespace detail
+
+} // namespace osmium
+
+#endif // OSMIUM_DETAIL_MMAP_VECTOR_BASE_HPP

include/osmium/index/detail/mmap_vector_file.hpp

@@ -0,0 +1,85 @@
+#ifndef OSMIUM_DETAIL_MMAP_VECTOR_FILE_HPP
+#define OSMIUM_DETAIL_MMAP_VECTOR_FILE_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstddef>
+
+#include <osmium/index/detail/typed_mmap.hpp>
+#include <osmium/index/detail/mmap_vector_base.hpp>
+#include <osmium/index/detail/tmpfile.hpp>
+
+namespace osmium {
+
+    namespace detail {
+
+        /**
+         * This class looks and behaves like STL vector, but mmap's a file
+         * internally.
+         */
+        template <typename T>
+        class mmap_vector_file : public mmap_vector_base<T, mmap_vector_file> {
+
+        public:
+
+            explicit mmap_vector_file() :
+                mmap_vector_base<T, osmium::detail::mmap_vector_file>(
+                    osmium::detail::create_tmp_file(),
+                    osmium::detail::mmap_vector_size_increment,
+                    0) {
+            }
+
+            explicit mmap_vector_file(int fd) :
+                mmap_vector_base<T, osmium::detail::mmap_vector_file>(
+                    fd,
+                    osmium::detail::typed_mmap<T>::file_size(fd) == 0 ?
+                        osmium::detail::mmap_vector_size_increment :
+                        osmium::detail::typed_mmap<T>::file_size(fd),
+                    osmium::detail::typed_mmap<T>::file_size(fd)) {
+            }
+
+            void reserve(size_t new_capacity) {
+                if (new_capacity > this->capacity()) {
+                    typed_mmap<T>::unmap(this->data(), this->capacity());
+                    this->data(typed_mmap<T>::grow_and_map(new_capacity, this->m_fd));
+                    this->m_capacity = new_capacity;
+                }
+            }
+
+        }; // class mmap_vector_file
+
+    } // namespace detail
+
+} // namespace osmium
+
+#endif // OSMIUM_DETAIL_MMAP_VECTOR_FILE_HPP

include/osmium/index/detail/tmpfile.hpp

@@ -0,0 +1,62 @@
+#ifndef OSMIUM_DETAIL_TMPFILE_HPP
+#define OSMIUM_DETAIL_TMPFILE_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cerrno>
+#include <cstdio>
+#include <system_error>
+
+namespace osmium {
+
+    namespace detail {
+
+        /**
+         * Create and open a temporary file. It is removed after opening.
+         *
+         * @returns File descriptor of temporary file.
+         * @throws std::system_error if something went wrong.
+         */
+        inline int create_tmp_file() {
+            FILE* file = ::tmpfile();
+            if (!file) {
+                throw std::system_error(errno, std::system_category(), "tempfile failed");
+            }
+            return fileno(file);
+        }
+
+    } // namespace detail
+
+} // namespace osmium
+
+#endif // OSMIUM_DETAIL_TMPFILE

include/osmium/index/detail/typed_mmap.hpp

@@ -0,0 +1,229 @@
+#ifndef OSMIUM_DETAIL_TYPED_MMAP_HPP
+#define OSMIUM_DETAIL_TYPED_MMAP_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cerrno>
+#include <cstddef>
+#include <stdexcept>
+#include <system_error>
+
+#include <sys/stat.h>
+
+#ifndef _WIN32
+# include <sys/mman.h>
+#else
+# include <mmap_for_windows.hpp>
+#endif
+
+#ifndef _MSC_VER
+# include <unistd.h>
+#else
+# define ftruncate _chsize
+#endif
+
+// for bsd systems
+#ifndef MAP_ANONYMOUS
+# define MAP_ANONYMOUS MAP_ANON
+#endif
+
+#include <osmium/util/cast.hpp>
+
+namespace osmium {
+
+    /**
+     * @brief Namespace for Osmium internal use
+     */
+    namespace detail {
+
+        /**
+         * This is a helper class for working with memory mapped files and
+         * anonymous shared memory. It wraps the necessary system calls
+         * adding:
+         * - error checking: all functions throw exceptions where needed
+         * - internal casts and size calculations allow use with user defined
+         *   type T instead of void*
+         *
+         * This class only contains static functions. It should never be
+         * instantiated.
+         *
+         * @tparam T Type of objects we want to store.
+         */
+        template <typename T>
+        class typed_mmap {
+
+        public:
+
+            /**
+             * Create anonymous private memory mapping with enough space for size
+             * objects of type T.
+             *
+             * Note that no constructor is called for any of the objects in this memory!
+             *
+             * @param size Number of objects of type T that should fit into this memory
+             * @returns Pointer to mapped memory
+             * @throws std::system_error If mmap(2) failed
+             */
+            static T* map(size_t size) {
+                void* addr = ::mmap(nullptr, sizeof(T) * size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wold-style-cast"
+                if (addr == MAP_FAILED) {
+                    throw std::system_error(errno, std::system_category(), "mmap failed");
+                }
+#pragma GCC diagnostic pop
+                return reinterpret_cast<T*>(addr);
+            }
+
+            /**
+             * Create shared memory mapping of a file with enough space for size
+             * objects of type T. The file must already have at least the
+             * required size.
+             *
+             * Note that no constructor is called for any of the objects in this memory!
+             *
+             * @param size Number of objects of type T that should fit into this memory
+             * @param fd File descriptor
+             * @param write True if data should be writable
+             * @returns Pointer to mapped memory
+             * @throws std::system_error If mmap(2) failed
+             */
+            static T* map(size_t size, int fd, bool write = false) {
+                int prot = PROT_READ;
+                if (write) {
+                    prot |= PROT_WRITE;
+                }
+                void* addr = ::mmap(nullptr, sizeof(T) * size, prot, MAP_SHARED, fd, 0);
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wold-style-cast"
+                if (addr == MAP_FAILED) {
+                    throw std::system_error(errno, std::system_category(), "mmap failed");
+                }
+#pragma GCC diagnostic pop
+                return reinterpret_cast<T*>(addr);
+            }
+
+// mremap(2) is only available on linux systems
+#ifdef __linux__
+            /**
+             * Grow memory mapping created with map().
+             *
+             * Note that no constructor is called for any of the objects in this memory!
+             *
+             * @param data Pointer to current mapping (as returned by typed_mmap())
+             * @param old_size Number of objects currently stored in this memory
+             * @param new_size Number of objects we want to have space for
+             * @throws std::system_error If mremap(2) call failed
+             */
+            static T* remap(T* data, size_t old_size, size_t new_size) {
+                void* addr = ::mremap(reinterpret_cast<void*>(data), sizeof(T) * old_size, sizeof(T) * new_size, MREMAP_MAYMOVE);
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wold-style-cast"
+                if (addr == MAP_FAILED) {
+                    throw std::system_error(errno, std::system_category(), "mremap failed");
+                }
+#pragma GCC diagnostic pop
+                return reinterpret_cast<T*>(addr);
+            }
+#endif
+
+            /**
+             * Release memory from map() call.
+             *
+             * Note that no destructor is called for the objects in this memory!
+             *
+             * @param data Pointer to the data
+             * @param size Number of objects of type T stored
+             * @throws std::system_error If munmap(2) call failed
+             */
+            static void unmap(T* data, size_t size) {
+                if (::munmap(reinterpret_cast<void*>(data), sizeof(T) * size) != 0) {
+                    throw std::system_error(errno, std::system_category(), "munmap failed");
+                }
+            }
+
+            /**
+             * Get number of objects of type T that would fit into a file.
+             *
+             * @param fd File descriptor
+             * @returns Number of objects of type T in this file
+             * @throws std::system_error If fstat(2) call failed
+             * @throws std::length_error If size of the file isn't a multiple of sizeof(T)
+             */
+            static size_t file_size(int fd) {
+                struct stat s;
+                if (fstat(fd, &s) < 0) {
+                    throw std::system_error(errno, std::system_category(), "fstat failed");
+                }
+                if (static_cast<size_t>(s.st_size) % sizeof(T) != 0) {
+                    throw std::length_error("file size has to be multiple of object size");
+                }
+                return static_cast<size_t>(s.st_size) / sizeof(T);
+            }
+
+            /**
+             * Grow file so there is enough space for at least new_size objects
+             * of type T. If the file is large enough already, nothing is done.
+             * The file is never shrunk.
+             *
+             * @param new_size Number of objects of type T that should fit into this file
+             * @param fd File descriptor
+             * @throws std::system_error If ftruncate(2) call failed
+             */
+            static void grow_file(size_t new_size, int fd) {
+                if (file_size(fd) < new_size) {
+                    if (::ftruncate(fd, static_cast_with_assert<off_t>(sizeof(T) * new_size)) < 0) {
+                        throw std::system_error(errno, std::system_category(), "ftruncate failed");
+                    }
+                }
+            }
+
+            /**
+             * Grow file to given size (if it is smaller) and mmap it.
+             *
+             * @param size Number of objects of type T that should fit into this file
+             * @param fd File descriptor
+             * @throws Errors thrown by grow_file() or map()
+             */
+            static T* grow_and_map(size_t size, int fd) {
+                grow_file(size, fd);
+                return map(size, fd, true);
+            }
+
+        }; // class typed_mmap
+
+    } // namespace detail
+
+} // namespace osmium
+
+#endif // OSMIUM_DETAIL_TYPED_MMAP_HPP

include/osmium/index/index.hpp

@@ -0,0 +1,100 @@
+#ifndef OSMIUM_INDEX_INDEX_HPP
+#define OSMIUM_INDEX_INDEX_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstddef>
+#include <limits>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+
+#include <osmium/util/compatibility.hpp>
+
+namespace osmium {
+
+    /**
+     * Exception signaling that an element could not be
+     * found in an index.
+     */
+    struct not_found : public std::runtime_error {
+
+        not_found(const std::string& what) :
+            std::runtime_error(what) {
+        }
+
+        not_found(const char* what) :
+            std::runtime_error(what) {
+        }
+
+    }; // struct not_found
+
+    /**
+     * @brief Indexing of OSM data, Locations, etc.
+     */
+    namespace index {
+
+        template <typename TKey>
+        OSMIUM_NORETURN void not_found_error(TKey key) {
+            std::stringstream s;
+            s << "id " << key << " no found";
+            throw not_found(s.str());
+        }
+
+        /**
+         * Some of the index classes need an "empty" value that can
+         * never appear in real data. This function must return this
+         * empty value for any class used as a value in an index.
+         * The default implementation returns a default constructed
+         * object, but it can be specialized.
+         */
+        template <typename T>
+        inline constexpr T empty_value() {
+            return T{};
+        }
+
+        /**
+         * The size_t value in indexes is usually used for offsets
+         * into a buffer or file. It is unlikely that we ever need
+         * the full range, so the max value is a good "empty" value.
+         */
+        template <>
+        inline OSMIUM_CONSTEXPR size_t empty_value<size_t>() {
+            return std::numeric_limits<size_t>::max();
+        }
+
+    } // namespace index
+
+} // namespace osmium
+
+#endif // OSMIUM_INDEX_INDEX_HPP

include/osmium/index/map.hpp

@@ -0,0 +1,155 @@
+#ifndef OSMIUM_INDEX_MAP_HPP
+#define OSMIUM_INDEX_MAP_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstddef>
+#include <type_traits>
+
+#include <osmium/index/index.hpp> // IWYU pragma: export
+
+namespace osmium {
+
+    namespace index {
+
+        /**
+         * @brief Key-value containers with unique integer values for a key
+         */
+        namespace map {
+
+            /**
+             * This abstract class defines an interface to storage classes
+             * intended for storing small pieces of data (such as coordinates)
+             * indexed by a positive integer (such as an object ID). The
+             * storage must be very space efficient and able to scale to billions
+             * of objects.
+             *
+             * Subclasses have different implementations that store the
+             * data in different ways in memory and/or on disk. Some storage
+             * classes are better suited when working with the whole planet,
+             * some are better for data extracts.
+             *
+             * Note that these classes are not required to track "empty" fields.
+             * When reading data you have to be sure you have put something in
+             * there before.
+             *
+             * A typical use for this and derived classes is storage of node
+             * locations indexed by node ID. These indexes will only work
+             * on 64 bit systems if used in this case. 32 bit systems just
+             * can't address that much memory!
+             *
+             * @tparam TId Id type, usually osmium::unsigned_object_id_type,
+             *             must be an unsigned integral type.
+             * @tparam TValue Value type, usually osmium::Location or size_t.
+             *                Copied by value, so should be "small" type.
+             */
+            template <typename TId, typename TValue>
+            class Map {
+
+                static_assert(std::is_integral<TId>::value && std::is_unsigned<TId>::value,
+                              "TId template parameter for class Map must be unsigned integral type");
+
+                Map(const Map&) = delete;
+                Map& operator=(const Map&) = delete;
+
+            protected:
+
+                Map(Map&&) = default;
+                Map& operator=(Map&&) = default;
+
+            public:
+
+                /// The "key" type, usually osmium::unsigned_object_id_type.
+                typedef TId key_type;
+
+                /// The "value" type, usually a Location or size_t.
+                typedef TValue value_type;
+
+                Map() = default;
+
+                virtual ~Map() = default;
+
+                virtual void reserve(const size_t) {
+                    // default implementation is empty
+                }
+
+                /// Set the field with id to value.
+                virtual void set(const TId id, const TValue value) = 0;
+
+                /// Retrieve value by id. Does not check for overflow or empty fields.
+                virtual const TValue get(const TId id) const = 0;
+
+                /**
+                 * Get the approximate number of items in the storage. The storage
+                 * might allocate memory in blocks, so this size might not be
+                 * accurate. You can not use this to find out how much memory the
+                 * storage uses. Use used_memory() for that.
+                 */
+                virtual size_t size() const = 0;
+
+                /**
+                 * Get the memory used for this storage in bytes. Note that this
+                 * is not necessarily entirely accurate but an approximation.
+                 * For storage classes that store the data in memory, this is
+                 * the main memory used, for storage classes storing data on disk
+                 * this is the memory used on disk.
+                 */
+                virtual size_t used_memory() const = 0;
+
+                /**
+                 * Clear memory used for this storage. After this you can not
+                 * use the storage container any more.
+                 */
+                virtual void clear() = 0;
+
+                /**
+                 * Sort data in map. Call this after writing all data and
+                 * before reading. Not all implementations need this.
+                 */
+                virtual void sort() {
+                    // default implementation is empty
+                }
+
+                virtual void dump_as_list(int /*fd*/) const {
+                    std::runtime_error("can't dump as list");
+                }
+
+            }; // class Map
+
+        } // namespace map
+
+    } // namespace index
+
+} // namespace osmium
+
+#endif // OSMIUM_INDEX_MAP_HPP

include/osmium/index/map/dummy.hpp

@@ -0,0 +1,87 @@
+#ifndef OSMIUM_INDEX_MAP_DUMMY_HPP
+#define OSMIUM_INDEX_MAP_DUMMY_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstddef>
+
+#include <osmium/index/map.hpp>
+
+namespace osmium {
+
+    namespace index {
+
+        namespace map {
+
+            /**
+             * Pseudo map.
+             * Use this class if you don't need a map, but you
+             * need an object that behaves like one.
+             */
+            template <typename TId, typename TValue>
+            class Dummy : public osmium::index::map::Map<TId, TValue> {
+
+            public:
+
+                Dummy() = default;
+
+                ~Dummy() override final = default;
+
+                void set(const TId, const TValue) override final {
+                    // intentionally left blank
+                }
+
+                const TValue get(const TId id) const override final {
+                    not_found_error(id);
+                }
+
+                size_t size() const override final {
+                    return 0;
+                }
+
+                size_t used_memory() const override final {
+                    return 0;
+                }
+
+                void clear() override final {
+                }
+
+            }; // class Dummy
+
+        } // namespace map
+
+    } // namespace index
+
+} // namespace osmium
+
+#endif // OSMIUM_INDEX_MAP_DUMMY_HPP

include/osmium/index/map/mmap_vector_anon.hpp

@@ -0,0 +1,61 @@
+#ifndef OSMIUM_INDEX_MAP_MMAP_VECTOR_ANON_HPP
+#define OSMIUM_INDEX_MAP_MMAP_VECTOR_ANON_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#ifdef __linux__
+
+#include <osmium/index/map/vector.hpp>
+#include <osmium/index/detail/mmap_vector_anon.hpp>
+
+namespace osmium {
+
+    namespace index {
+
+        namespace map {
+
+            template <typename TId, typename TValue>
+            using DenseMapMmap = VectorBasedDenseMap<osmium::detail::mmap_vector_anon<TValue>, TId, TValue>;
+
+            template <typename TId, typename TValue>
+            using SparseMapMmap = VectorBasedSparseMap<TId, TValue, osmium::detail::mmap_vector_anon>;
+
+        } // namespace map
+
+    } // namespace index
+
+} // namespace osmium
+
+#endif // __linux__
+
+#endif // OSMIUM_INDEX_MAP_MMAP_VECTOR_ANON_HPP

include/osmium/index/map/mmap_vector_file.hpp

@@ -0,0 +1,57 @@
+#ifndef OSMIUM_INDEX_MAP_MMAP_VECTOR_FILE_HPP
+#define OSMIUM_INDEX_MAP_MMAP_VECTOR_FILE_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <osmium/index/map/vector.hpp>
+#include <osmium/index/detail/mmap_vector_file.hpp>
+
+namespace osmium {
+
+    namespace index {
+
+        namespace map {
+
+            template <typename TId, typename TValue>
+            using DenseMapFile = VectorBasedDenseMap<osmium::detail::mmap_vector_file<TValue>, TId, TValue>;
+
+            template <typename TId, typename TValue>
+            using SparseMapFile = VectorBasedSparseMap<TId, TValue, osmium::detail::mmap_vector_file>;
+
+        } // namespace map
+
+    } // namespace index
+
+} // namespace osmium
+
+#endif // OSMIUM_INDEX_MAP_MMAP_VECTOR_FILE_HPP

include/osmium/index/map/sparse_table.hpp

@@ -0,0 +1,140 @@
+#ifndef OSMIUM_INDEX_MAP_SPARSE_TABLE_HPP
+#define OSMIUM_INDEX_MAP_SPARSE_TABLE_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstddef>
+#include <stdexcept>
+#include <utility>
+#include <vector>
+
+#include <google/sparsetable>
+
+#include <osmium/index/map.hpp>
+#include <osmium/io/detail/read_write.hpp>
+
+namespace osmium {
+
+    namespace index {
+
+        namespace map {
+
+            /**
+            * The SparseTable index stores elements in a Google sparsetable,
+            * a data structure that can hold sparsly filled tables in a
+            * very space efficient way. It will resize automatically.
+            *
+            * Use this index if the ID space is only sparsly
+            * populated, such as when working with smaller OSM files (like
+            * country extracts).
+            *
+            * This will only work on 64 bit machines.
+            */
+            template <typename TId, typename TValue>
+            class SparseTable : public osmium::index::map::Map<TId, TValue> {
+
+                TId m_grow_size;
+
+                google::sparsetable<TValue> m_elements;
+
+                static_assert(sizeof(typename google::sparsetable<TValue>::size_type) >= 8, "google::sparsetable needs 64bit machine");
+
+            public:
+
+                /**
+                * Constructor.
+                *
+                * @param grow_size The initial size of the index (ie number of
+                *                  elements that fit into the index).
+                *                  The storage will grow by at least this size
+                *                  every time it runs out of space.
+                */
+                explicit SparseTable(const TId grow_size=10000) :
+                    m_grow_size(grow_size),
+                    m_elements(grow_size) {
+                }
+
+                ~SparseTable() override final = default;
+
+                void set(const TId id, const TValue value) override final {
+                    if (id >= m_elements.size()) {
+                        m_elements.resize(id + m_grow_size);
+                    }
+                    m_elements[id] = value;
+                }
+
+                const TValue get(const TId id) const override final {
+                    if (id >= m_elements.size()) {
+                        not_found_error(id);
+                    }
+                    if (m_elements[id] == osmium::index::empty_value<TValue>()) {
+                        not_found_error(id);
+                    }
+                    return m_elements[id];
+                }
+
+                size_t size() const override final {
+                    return m_elements.size();
+                }
+
+                size_t used_memory() const override final {
+                    // unused elements use 1 bit, used elements sizeof(TValue) bytes
+                    // http://google-sparsehash.googlecode.com/svn/trunk/doc/sparsetable.html
+                    return (m_elements.size() / 8) + (m_elements.num_nonempty() * sizeof(TValue));
+                }
+
+                void clear() override final {
+                    m_elements.clear();
+                }
+
+                void dump_as_list(const int fd) const override final {
+                    std::vector<std::pair<TId, TValue>> v;
+                    int n=0;
+                    for (const TValue value : m_elements) {
+                        if (value != osmium::index::empty_value<TValue>()) {
+                            v.emplace_back(n, value);
+                        }
+                        ++n;
+                    }
+                    osmium::io::detail::reliable_write(fd, reinterpret_cast<const char*>(v.data()), sizeof(std::pair<TId, TValue>) * v.size());
+                }
+
+            }; // class SparseTable
+
+        } // namespace map
+
+    } // namespace index
+
+} // namespace osmium
+
+#endif // OSMIUM_INDEX_BYID_SPARSE_TABLE_HPP

include/osmium/index/map/stl_map.hpp

@@ -0,0 +1,112 @@
+#ifndef OSMIUM_INDEX_MAP_STL_MAP_HPP
+#define OSMIUM_INDEX_MAP_STL_MAP_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <algorithm>
+#include <cstddef>
+#include <iterator>
+#include <map>
+#include <stdexcept>
+#include <vector>
+
+#include <osmium/index/map.hpp>
+#include <osmium/io/detail/read_write.hpp>
+
+namespace osmium {
+
+    namespace index {
+
+        namespace map {
+
+            /**
+             * This implementation uses std::map internally. It uses rather a
+             * lot of memory, but might make sense for small maps.
+             */
+            template <typename TId, typename TValue>
+            class StlMap : public osmium::index::map::Map<TId, TValue> {
+
+                // This is a rough estimate for the memory needed for each
+                // element in the map (id + value + pointers to left, right,
+                // and parent plus some overhead for color of red-black-tree
+                // or similar).
+                static constexpr size_t element_size = sizeof(TId) + sizeof(TValue) + sizeof(void*) * 4;
+
+                std::map<TId, TValue> m_elements;
+
+            public:
+
+                StlMap() = default;
+
+                ~StlMap() override final = default;
+
+                void set(const TId id, const TValue value) override final {
+                    m_elements[id] = value;
+                }
+
+                const TValue get(const TId id) const override final {
+                    try {
+                        return m_elements.at(id);
+                    } catch (std::out_of_range&) {
+                        not_found_error(id);
+                    }
+                }
+
+                size_t size() const override final {
+                    return m_elements.size();
+                }
+
+                size_t used_memory() const override final {
+                    return element_size * m_elements.size();
+                }
+
+                void clear() override final {
+                    m_elements.clear();
+                }
+
+                void dump_as_list(const int fd) const override final {
+                    typedef typename std::map<TId, TValue>::value_type t;
+                    std::vector<t> v;
+                    std::copy(m_elements.begin(), m_elements.end(), std::back_inserter(v));
+                    osmium::io::detail::reliable_write(fd, reinterpret_cast<const char*>(v.data()), sizeof(t) * v.size());
+                }
+
+            }; // class StlMap
+
+        } // namespace map
+
+    } // namespace index
+
+} // namespace osmium
+
+#endif // OSMIUM_INDEX_MAP_STL_MAP_HPP

include/osmium/index/map/stl_vector.hpp

@@ -0,0 +1,61 @@
+#ifndef OSMIUM_INDEX_MAP_STL_VECTOR_HPP
+#define OSMIUM_INDEX_MAP_STL_VECTOR_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <vector>
+
+#include <osmium/index/map/vector.hpp>
+
+namespace osmium {
+
+    namespace index {
+
+        namespace map {
+
+            template <typename TId, typename TValue>
+            using DenseMapMem = VectorBasedDenseMap<std::vector<TValue>, TId, TValue>;
+
+            template <typename T>
+            using StdVectorWrap = std::vector<T>;
+
+            template <typename TId, typename TValue>
+            using SparseMapMem = VectorBasedSparseMap<TId, TValue, StdVectorWrap>;
+
+        } // namespace map
+
+    } // namespace index
+
+} // namespace osmium
+
+#endif // OSMIUM_INDEX_MAP_STL_VECTOR_HPP

include/osmium/index/map/vector.hpp

@@ -0,0 +1,237 @@
+#ifndef OSMIUM_INDEX_MAP_VECTOR_HPP
+#define OSMIUM_INDEX_MAP_VECTOR_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <algorithm>
+#include <cstddef>
+#include <stdexcept>
+#include <utility>
+
+#include <osmium/index/map.hpp>
+#include <osmium/io/detail/read_write.hpp>
+
+namespace osmium {
+
+    namespace index {
+
+        namespace map {
+
+            template <class TVector, typename TId, typename TValue>
+            class VectorBasedDenseMap : public Map<TId, TValue> {
+
+                TVector m_vector;
+
+            public:
+
+                typedef TValue element_type;
+                typedef TVector vector_type;
+                typedef typename vector_type::iterator iterator;
+                typedef typename vector_type::const_iterator const_iterator;
+
+                VectorBasedDenseMap() :
+                    m_vector() {
+                }
+
+                explicit VectorBasedDenseMap(int fd) :
+                    m_vector(fd) {
+                }
+
+                ~VectorBasedDenseMap() {}
+
+                void reserve(const size_t size) override final {
+                    m_vector.reserve(size);
+                }
+
+                void set(const TId id, const TValue value) override final {
+                    if (size() <= id) {
+                        m_vector.resize(id+1);
+                    }
+                    m_vector[id] = value;
+                }
+
+                const TValue get(const TId id) const override final {
+                    try {
+                        const TValue& value = m_vector.at(id);
+                        if (value == osmium::index::empty_value<TValue>()) {
+                            not_found_error(id);
+                        }
+                        return value;
+                    } catch (std::out_of_range&) {
+                        not_found_error(id);
+                    }
+                }
+
+                size_t size() const override final {
+                    return m_vector.size();
+                }
+
+                size_t used_memory() const override final {
+                    return sizeof(TValue) * size();
+                }
+
+                void clear() override final {
+                    m_vector.clear();
+                    m_vector.shrink_to_fit();
+                }
+
+                iterator begin() {
+                    return m_vector.begin();
+                }
+
+                iterator end() {
+                    return m_vector.end();
+                }
+
+                const_iterator cbegin() const {
+                    return m_vector.cbegin();
+                }
+
+                const_iterator cend() const {
+                    return m_vector.cend();
+                }
+
+                const_iterator begin() const {
+                    return m_vector.cbegin();
+                }
+
+                const_iterator end() const {
+                    return m_vector.cend();
+                }
+
+            }; // class VectorBasedDenseMap
+
+
+            template <typename TId, typename TValue, template<typename...> class TVector>
+            class VectorBasedSparseMap : public Map<TId, TValue> {
+
+            public:
+
+                typedef typename std::pair<TId, TValue> element_type;
+                typedef TVector<element_type> vector_type;
+                typedef typename vector_type::iterator iterator;
+                typedef typename vector_type::const_iterator const_iterator;
+
+            private:
+
+                vector_type m_vector;
+
+            public:
+
+                VectorBasedSparseMap() :
+                    m_vector() {
+                }
+
+                VectorBasedSparseMap(int fd) :
+                    m_vector(fd) {
+                }
+
+                ~VectorBasedSparseMap() override final = default;
+
+                void set(const TId id, const TValue value) override final {
+                    m_vector.push_back(element_type(id, value));
+                }
+
+                const TValue get(const TId id) const override final {
+                    const element_type element {
+                        id,
+                        osmium::index::empty_value<TValue>()
+                    };
+                    const auto result = std::lower_bound(m_vector.begin(), m_vector.end(), element, [](const element_type& a, const element_type& b) {
+                        return a.first < b.first;
+                    });
+                    if (result == m_vector.end() || result->first != id) {
+                        not_found_error(id);
+                    } else {
+                        return result->second;
+                    }
+                }
+
+                size_t size() const override final {
+                    return m_vector.size();
+                }
+
+                size_t byte_size() const {
+                    return m_vector.size() * sizeof(element_type);
+                }
+
+                size_t used_memory() const override final {
+                    return sizeof(element_type) * size();
+                }
+
+                void clear() override final {
+                    m_vector.clear();
+                    m_vector.shrink_to_fit();
+                }
+
+                void sort() override final {
+                    std::sort(m_vector.begin(), m_vector.end());
+                }
+
+                void dump_as_list(int fd) const override final {
+                    osmium::io::detail::reliable_write(fd, reinterpret_cast<const char*>(m_vector.data()), byte_size());
+                }
+
+                iterator begin() {
+                    return m_vector.begin();
+                }
+
+                iterator end() {
+                    return m_vector.end();
+                }
+
+                const_iterator cbegin() const {
+                    return m_vector.cbegin();
+                }
+
+                const_iterator cend() const {
+                    return m_vector.cend();
+                }
+
+                const_iterator begin() const {
+                    return m_vector.cbegin();
+                }
+
+                const_iterator end() const {
+                    return m_vector.cend();
+                }
+
+            }; // class VectorBasedSparseMap
+
+        } // namespace map
+
+    } // namespace index
+
+} // namespace osmium
+
+#endif // OSMIUM_INDEX_MAP_VECTOR_HPP

include/osmium/index/multimap.hpp

@@ -0,0 +1,129 @@
+#ifndef OSMIUM_INDEX_MULTIMAP_HPP
+#define OSMIUM_INDEX_MULTIMAP_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstddef>
+#include <type_traits>
+#include <utility>
+
+#include <osmium/index/index.hpp> // IWYU pragma: export
+
+namespace osmium {
+
+    namespace index {
+
+        /**
+         * @brief Key-value containers with multiple values for an integer key
+         */
+        namespace multimap {
+
+            template <typename TId, typename TValue>
+            class Multimap {
+
+                static_assert(std::is_integral<TId>::value && std::is_unsigned<TId>::value,
+                              "TId template parameter for class Multimap must be unsigned integral type");
+
+                typedef typename std::pair<TId, TValue> element_type;
+
+                Multimap(const Multimap&) = delete;
+                Multimap& operator=(const Multimap&) = delete;
+
+            protected:
+
+                Multimap(Multimap&&) = default;
+                Multimap& operator=(Multimap&&) = default;
+
+            public:
+
+                /// The "key" type, usually osmium::unsigned_object_id_type.
+                typedef TId key_type;
+
+                /// The "value" type, usually a Location or size_t.
+                typedef TValue value_type;
+
+                Multimap() = default;
+
+                virtual ~Multimap() noexcept = default;
+
+                /// Set the field with id to value.
+                virtual void set(const TId id, const TValue value) = 0;
+
+                typedef element_type* iterator;
+
+//                virtual std::pair<iterator, iterator> get_all(const TId id) const = 0;
+
+                /**
+                 * Get the approximate number of items in the storage. The storage
+                 * might allocate memory in blocks, so this size might not be
+                 * accurate. You can not use this to find out how much memory the
+                 * storage uses. Use used_memory() for that.
+                 */
+                virtual size_t size() const = 0;
+
+                /**
+                 * Get the memory used for this storage in bytes. Note that this
+                 * is not necessarily entirely accurate but an approximation.
+                 * For storage classes that store the data in memory, this is
+                 * the main memory used, for storage classes storing data on disk
+                 * this is the memory used on disk.
+                 */
+                virtual size_t used_memory() const = 0;
+
+                /**
+                 * Clear memory used for this storage. After this you can not
+                 * use the storage container any more.
+                 */
+                virtual void clear() = 0;
+
+                /**
+                 * Sort data in map. Call this after writing all data and
+                 * before reading. Not all implementations need this.
+                 */
+                virtual void sort() {
+                    // default implementation is empty
+                }
+
+                virtual void dump_as_list(int /*fd*/) const {
+                    std::runtime_error("can't dump as list");
+                }
+
+            }; // class Multimap
+
+        } // namespace map
+
+    } // namespace index
+
+} // namespace osmium
+
+#endif // OSMIUM_INDEX_MULTIMAP_HPP

include/osmium/index/multimap/hybrid.hpp

@@ -0,0 +1,199 @@
+#ifndef OSMIUM_INDEX_MULTIMAP_HYBRID_HPP
+#define OSMIUM_INDEX_MULTIMAP_HYBRID_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstddef>
+#include <utility>
+
+#include <osmium/index/multimap.hpp>
+#include <osmium/index/multimap/stl_vector.hpp>
+#include <osmium/index/multimap/stl_multimap.hpp>
+
+namespace osmium {
+
+    namespace index {
+
+        namespace multimap {
+
+            template <typename TId, typename TValue>
+            class HybridIterator {
+
+                typedef SparseMultimapMem<TId, TValue> main_map_type;
+                typedef StlMultimap<TId, TValue> extra_map_type;
+
+                typedef typename std::pair<TId, TValue> element_type;
+
+                typename main_map_type::iterator m_begin_main;
+                typename main_map_type::iterator m_end_main;
+                typename extra_map_type::iterator m_begin_extra;
+                typename extra_map_type::iterator m_end_extra;
+
+            public:
+
+                explicit HybridIterator(typename main_map_type::iterator begin_main,
+                               typename main_map_type::iterator end_main,
+                               typename extra_map_type::iterator begin_extra,
+                               typename extra_map_type::iterator end_extra) :
+                    m_begin_main(begin_main),
+                    m_end_main(end_main),
+                    m_begin_extra(begin_extra),
+                    m_end_extra(end_extra) {
+                }
+
+                HybridIterator& operator++() {
+                    if (m_begin_main == m_end_main) {
+                        ++m_begin_extra;
+                    } else {
+                        ++m_begin_main;
+                        while (m_begin_main != m_end_main && m_begin_main->second == osmium::index::empty_value<TValue>()) { // ignore removed elements
+                            ++m_begin_main;
+                        }
+                    }
+                    return *this;
+                }
+
+                HybridIterator<TId, TValue> operator++(int) {
+                    auto tmp(*this);
+                    operator++();
+                    return tmp;
+                }
+
+                bool operator==(const HybridIterator& rhs) const {
+                    return m_begin_main == rhs.m_begin_main &&
+                           m_end_main   == rhs.m_end_main &&
+                           m_begin_extra == rhs.m_begin_extra &&
+                           m_end_extra   == rhs.m_end_extra;
+                }
+
+                bool operator!=(const HybridIterator& rhs) const {
+                    return ! operator==(rhs);
+                }
+
+                const element_type& operator*() {
+                    if (m_begin_main == m_end_main) {
+                        return *m_begin_extra;
+                    } else {
+                        return *m_begin_main;
+                    }
+                }
+
+                const element_type* operator->() {
+                    return &operator*();
+                }
+
+            }; // class HybridIterator
+
+            template <typename TId, typename TValue>
+            class Hybrid : public Multimap<TId, TValue> {
+
+                typedef SparseMultimapMem<TId, TValue> main_map_type;
+                typedef StlMultimap<TId, TValue> extra_map_type;
+
+                main_map_type m_main;
+                extra_map_type m_extra;
+
+            public:
+
+                typedef HybridIterator<TId, TValue> iterator;
+                typedef const HybridIterator<TId, TValue> const_iterator;
+
+                Hybrid() :
+                    m_main(),
+                    m_extra() {
+                }
+
+                size_t size() const override final {
+                    return m_main.size() + m_extra.size();
+                }
+
+                size_t used_memory() const override final {
+                    return m_main.used_memory() + m_extra.used_memory();
+                }
+
+                void reserve(const size_t size) {
+                    m_main.reserve(size);
+                }
+
+                void unsorted_set(const TId id, const TValue value) {
+                    m_main.set(id, value);
+                }
+
+                void set(const TId id, const TValue value) override final {
+                    m_extra.set(id, value);
+                }
+
+                std::pair<iterator, iterator> get_all(const TId id) {
+                    auto result_main = m_main.get_all(id);
+                    auto result_extra = m_extra.get_all(id);
+                    return std::make_pair(iterator(result_main.first, result_main.second, result_extra.first, result_extra.second),
+                                          iterator(result_main.second, result_main.second, result_extra.second, result_extra.second));
+                }
+
+                void remove(const TId id, const TValue value) {
+                    m_main.remove(id, value);
+                    m_extra.remove(id, value);
+                }
+
+                void consolidate() {
+                    m_main.erase_removed();
+                    for (const auto& element : m_extra) {
+                        m_main.set(element.first, element.second);
+                    }
+                    m_extra.clear();
+                    m_main.sort();
+                }
+
+                void dump_as_list(int fd) override final {
+                    consolidate();
+                    m_main.dump_as_list(fd);
+                }
+
+                void clear() override final {
+                    m_main.clear();
+                    m_extra.clear();
+                }
+
+                void sort() override final {
+                    m_main.sort();
+                }
+
+            }; // class Hybrid
+
+        } // namespace multimap
+
+    } // namespace index
+
+} // namespace osmium
+
+#endif // OSMIUM_INDEX_MULTIMAP_HYBRID_HPP

include/osmium/index/multimap/mmap_vector_anon.hpp

@@ -0,0 +1,58 @@
+#ifndef OSMIUM_INDEX_MULTIMAP_MMAP_VECTOR_ANON_HPP
+#define OSMIUM_INDEX_MULTIMAP_MMAP_VECTOR_ANON_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#ifdef __linux__
+
+#include <osmium/index/multimap/vector.hpp>
+#include <osmium/index/detail/mmap_vector_anon.hpp>
+
+namespace osmium {
+
+    namespace index {
+
+        namespace multimap {
+
+            template <typename TId, typename TValue>
+            using SparseMultimapMmap = VectorBasedSparseMultimap<TId, TValue, osmium::detail::mmap_vector_anon>;
+
+        } // namespace multimap
+
+    } // namespace index
+
+} // namespace osmium
+
+#endif // __linux__
+
+#endif // OSMIUM_INDEX_MULTIMAP_MMAP_VECTOR_ANON_HPP

include/osmium/index/multimap/mmap_vector_file.hpp

@@ -0,0 +1,54 @@
+#ifndef OSMIUM_INDEX_MULTIMAP_MMAP_VECTOR_FILE_HPP
+#define OSMIUM_INDEX_MULTIMAP_MMAP_VECTOR_FILE_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <osmium/index/multimap/vector.hpp>
+#include <osmium/index/detail/mmap_vector_file.hpp>
+
+namespace osmium {
+
+    namespace index {
+
+        namespace multimap {
+
+            template <typename TId, typename TValue>
+            using SparseMultimapFile = VectorBasedSparseMultimap<TId, TValue, osmium::detail::mmap_vector_file>;
+
+        } // namespace multimap
+
+    } // namespace index
+
+} // namespace osmium
+
+#endif // OSMIUM_INDEX_MULTIMAP_MMAP_VECTOR_FILE_HPP

include/osmium/index/multimap/stl_multimap.hpp

@@ -0,0 +1,151 @@
+#ifndef OSMIUM_INDEX_MULTIMAP_STL_MULTIMAP_HPP
+#define OSMIUM_INDEX_MULTIMAP_STL_MULTIMAP_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <algorithm>
+#include <cstddef>
+#include <map>
+#include <utility>
+#include <vector>
+
+#include <osmium/index/multimap.hpp>
+#include <osmium/io/detail/read_write.hpp>
+
+namespace osmium {
+
+    namespace index {
+
+        namespace multimap {
+
+            /**
+             * This implementation uses std::multimap internally. It uses rather a
+             * lot of memory, but might make sense for small maps.
+             */
+            template <typename TId, typename TValue>
+            class StlMultimap : public osmium::index::multimap::Multimap<TId, TValue> {
+
+                // This is a rough estimate for the memory needed for each
+                // element in the map (id + value + pointers to left, right,
+                // and parent plus some overhead for color of red-black-tree
+                // or similar).
+                static constexpr size_t element_size = sizeof(TId) + sizeof(TValue) + sizeof(void*) * 4;
+
+            public:
+
+                typedef typename std::multimap<const TId, TValue> collection_type;
+                typedef typename collection_type::iterator iterator;
+                typedef typename collection_type::const_iterator const_iterator;
+                typedef typename collection_type::value_type value_type;
+
+                typedef typename std::pair<TId, TValue> element_type;
+
+            private:
+
+                collection_type m_elements;
+
+            public:
+
+                StlMultimap() = default;
+
+                ~StlMultimap() noexcept override final = default;
+
+                void unsorted_set(const TId id, const TValue value) {
+                    m_elements.emplace(id, value);
+                }
+
+                void set(const TId id, const TValue value) override final {
+                    m_elements.emplace(id, value);
+                }
+
+                std::pair<iterator, iterator> get_all(const TId id) {
+                    return m_elements.equal_range(id);
+                }
+
+                std::pair<const_iterator, const_iterator> get_all(const TId id) const {
+                    return m_elements.equal_range(id);
+                }
+
+                void remove(const TId id, const TValue value) {
+                    std::pair<iterator, iterator> r = get_all(id);
+                    for (iterator it = r.first; it != r.second; ++it) {
+                        if (it->second == value) {
+                            m_elements.erase(it);
+                            return;
+                        }
+                    }
+                }
+
+                iterator begin() {
+                    return m_elements.begin();
+                }
+
+                iterator end() {
+                    return m_elements.end();
+                }
+
+                size_t size() const override final {
+                    return m_elements.size();
+                }
+
+                size_t used_memory() const override final {
+                    return element_size * m_elements.size();
+                }
+
+                void clear() override final {
+                    m_elements.clear();
+                }
+
+                void consolidate() {
+                    // intentionally left blank
+                }
+
+                void dump_as_list(const int fd) const override final {
+                    std::vector<element_type> v;
+                    for (const auto& element : m_elements) {
+                        v.emplace_back(element.first, element.second);
+                    }
+//                    std::copy(m_elements.cbegin(), m_elements.cend(), std::back_inserter(v));
+                    std::sort(v.begin(), v.end());
+                    osmium::io::detail::reliable_write(fd, reinterpret_cast<const char*>(v.data()), sizeof(element_type) * v.size());
+                }
+
+            }; // class StlMultimap
+
+        } // namespace multimap
+
+    } // namespace index
+
+} // namespace osmium
+
+#endif // OSMIUM_INDEX_MULTIMAP_STL_MULTIMAP_HPP

include/osmium/index/multimap/stl_vector.hpp

@@ -0,0 +1,58 @@
+#ifndef OSMIUM_INDEX_MULTIMAP_STL_VECTOR_HPP
+#define OSMIUM_INDEX_MULTIMAP_STL_VECTOR_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <vector>
+
+#include <osmium/index/multimap/vector.hpp>
+
+namespace osmium {
+
+    namespace index {
+
+        namespace multimap {
+
+            template <typename T>
+            using StdVectorWrap = std::vector<T>;
+
+            template <typename TId, typename TValue>
+            using SparseMultimapMem = VectorBasedSparseMultimap<TId, TValue, StdVectorWrap>;
+
+        } // namespace multimap
+
+    } // namespace index
+
+} // namespace osmium
+
+#endif // OSMIUM_INDEX_MULTIMAP_STL_VECTOR_HPP

include/osmium/index/multimap/vector.hpp

@@ -0,0 +1,151 @@
+#ifndef OSMIUM_INDEX_MULTIMAP_VECTOR_HPP
+#define OSMIUM_INDEX_MULTIMAP_VECTOR_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <algorithm>
+#include <cstddef>
+#include <utility>
+
+#include <osmium/index/multimap.hpp>
+#include <osmium/io/detail/read_write.hpp>
+
+namespace osmium {
+
+    namespace index {
+
+        namespace multimap {
+
+            template <typename TId, typename TValue, template<typename...> class TVector>
+            class VectorBasedSparseMultimap : public Multimap<TId, TValue> {
+
+            public:
+
+                typedef typename std::pair<TId, TValue> element_type;
+                typedef TVector<element_type> vector_type;
+                typedef typename vector_type::iterator iterator;
+                typedef typename vector_type::const_iterator const_iterator;
+
+            private:
+
+                vector_type m_vector;
+
+                static bool is_removed(element_type& element) {
+                    return element.second == osmium::index::empty_value<TValue>();
+                }
+
+            public:
+
+                void set(const TId id, const TValue value) override final {
+                    m_vector.push_back(element_type(id, value));
+                }
+
+                void unsorted_set(const TId id, const TValue value) {
+                    m_vector.push_back(element_type(id, value));
+                }
+
+                std::pair<iterator, iterator> get_all(const TId id) {
+                    const element_type element {
+                        id,
+                        osmium::index::empty_value<TValue>()
+                    };
+                    return std::equal_range(m_vector.begin(), m_vector.end(), element, [](const element_type& a, const element_type& b) {
+                        return a.first < b.first;
+                    });
+                }
+
+                std::pair<const_iterator, const_iterator> get_all(const TId id) const {
+                    const element_type element {
+                        id,
+                        osmium::index::empty_value<TValue>()
+                    };
+                    return std::equal_range(m_vector.cbegin(), m_vector.cend(), element, [](const element_type& a, const element_type& b) {
+                        return a.first < b.first;
+                    });
+                }
+
+                size_t size() const override final {
+                    return m_vector.size();
+                }
+
+                size_t byte_size() const {
+                    return m_vector.size() * sizeof(element_type);
+                }
+
+                size_t used_memory() const override final {
+                    return sizeof(element_type) * size();
+                }
+
+                void clear() override final {
+                    m_vector.clear();
+                    m_vector.shrink_to_fit();
+                }
+
+                void sort() override final {
+                    std::sort(m_vector.begin(), m_vector.end());
+                }
+
+                void remove(const TId id, const TValue value) {
+                    auto r = get_all(id);
+                    for (auto it = r.first; it != r.second; ++it) {
+                        if (it->second == value) {
+                            it->second = 0;
+                            return;
+                        }
+                    }
+                }
+
+                void consolidate() {
+                    std::sort(m_vector.begin(), m_vector.end());
+                }
+
+                void erase_removed() {
+                    m_vector.erase(
+                        std::remove_if(m_vector.begin(), m_vector.end(), is_removed),
+                        m_vector.end()
+                    );
+                }
+
+                void dump_as_list(int fd) const override final {
+                    osmium::io::detail::reliable_write(fd, reinterpret_cast<const char*>(m_vector.data()), byte_size());
+                }
+
+            }; // class VectorBasedSparseMultimap
+
+        } // namespace multimap
+
+    } // namespace index
+
+} // namespace osmium
+
+#endif // OSMIUM_INDEX_MULTIMAP_VECTOR_HPP

include/osmium/io/any_compression.hpp

@@ -0,0 +1,39 @@
+#ifndef OSMIUM_IO_ANY_COMPRESSION_HPP
+#define OSMIUM_IO_ANY_COMPRESSION_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <osmium/io/bzip2_compression.hpp> // IWYU pragma: export
+#include <osmium/io/gzip_compression.hpp> // IWYU pragma: export
+
+#endif // OSMIUM_IO_ANY_COMPRESSION_HPP

include/osmium/io/any_input.hpp

@@ -0,0 +1,41 @@
+#ifndef OSMIUM_IO_ANY_INPUT_HPP
+#define OSMIUM_IO_ANY_INPUT_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <osmium/io/any_compression.hpp> // IWYU pragma: export
+
+#include <osmium/io/pbf_input.hpp> // IWYU pragma: export
+#include <osmium/io/xml_input.hpp> // IWYU pragma: export
+
+#endif // OSMIUM_IO_ANY_INPUT_HPP

include/osmium/io/any_output.hpp

@@ -0,0 +1,42 @@
+#ifndef OSMIUM_IO_ANY_OUTPUT_HPP
+#define OSMIUM_IO_ANY_OUTPUT_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <osmium/io/any_compression.hpp> // IWYU pragma: export
+
+#include <osmium/io/opl_output.hpp> // IWYU pragma: export
+#include <osmium/io/pbf_output.hpp> // IWYU pragma: export
+#include <osmium/io/xml_output.hpp> // IWYU pragma: export
+
+#endif // OSMIUM_IO_ANY_OUTPUT_HPP

include/osmium/io/bzip2_compression.hpp

@@ -0,0 +1,278 @@
+#ifndef OSMIUM_IO_BZIP2_COMPRESSION_HPP
+#define OSMIUM_IO_BZIP2_COMPRESSION_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cstdio>
+#include <stdexcept>
+#include <string>
+
+#include <bzlib.h>
+
+#ifndef _MSC_VER
+# include <unistd.h>
+#endif
+
+#include <osmium/io/compression.hpp>
+#include <osmium/io/file_compression.hpp>
+#include <osmium/util/cast.hpp>
+#include <osmium/util/compatibility.hpp>
+
+namespace osmium {
+
+    /**
+     * Exception thrown when there are problems compressing or
+     * decompressing bzip2 files.
+     */
+    struct bzip2_error : public std::runtime_error {
+
+        int bzip2_error_code;
+        int system_errno;
+
+        bzip2_error(const std::string& what, int error_code) :
+            std::runtime_error(what),
+            bzip2_error_code(error_code),
+            system_errno(error_code == BZ_IO_ERROR ? errno : 0) {
+        }
+
+    }; // struct bzip2_error
+
+    namespace io {
+
+        namespace detail {
+
+            OSMIUM_NORETURN inline void throw_bzip2_error(BZFILE* bzfile, const char* msg, int bzlib_error=0) {
+                std::string error("bzip2 error: ");
+                error += msg;
+                error += ": ";
+                int errnum = bzlib_error;
+                if (bzlib_error) {
+                    error += std::to_string(bzlib_error);
+                } else {
+                    error += ::BZ2_bzerror(bzfile, &errnum);
+                }
+                throw osmium::bzip2_error(error, errnum);
+            }
+
+        } // namespace detail
+
+        class Bzip2Compressor : public Compressor {
+
+            FILE* m_file;
+            int m_bzerror;
+            BZFILE* m_bzfile;
+
+        public:
+
+            explicit Bzip2Compressor(int fd) :
+                Compressor(),
+                m_file(fdopen(dup(fd), "wb")),
+                m_bzerror(BZ_OK),
+                m_bzfile(::BZ2_bzWriteOpen(&m_bzerror, m_file, 6, 0, 0)) {
+                if (!m_bzfile) {
+                    detail::throw_bzip2_error(m_bzfile, "write open failed", m_bzerror);
+                }
+            }
+
+            ~Bzip2Compressor() override final {
+                close();
+            }
+
+            void write(const std::string& data) override final {
+                int error;
+                ::BZ2_bzWrite(&error, m_bzfile, const_cast<char*>(data.data()), static_cast_with_assert<int>(data.size()));
+                if (error != BZ_OK && error != BZ_STREAM_END) {
+                    detail::throw_bzip2_error(m_bzfile, "write failed", error);
+                }
+            }
+
+            void close() override final {
+                if (m_bzfile) {
+                    int error;
+                    ::BZ2_bzWriteClose(&error, m_bzfile, 0, nullptr, nullptr);
+                    m_bzfile = nullptr;
+                    if (m_file) {
+                        fclose(m_file);
+                    }
+                    if (error != BZ_OK) {
+                        detail::throw_bzip2_error(m_bzfile, "write close failed", error);
+                    }
+                }
+            }
+
+        }; // class Bzip2Compressor
+
+        class Bzip2Decompressor : public Decompressor {
+
+            FILE* m_file;
+            int m_bzerror;
+            BZFILE* m_bzfile;
+            bool m_stream_end {false};
+
+        public:
+
+            Bzip2Decompressor(int fd) :
+                Decompressor(),
+                m_file(fdopen(dup(fd), "rb")),
+                m_bzerror(BZ_OK),
+                m_bzfile(::BZ2_bzReadOpen(&m_bzerror, m_file, 0, 0, nullptr, 0)) {
+                if (!m_bzfile) {
+                    detail::throw_bzip2_error(m_bzfile, "read open failed", m_bzerror);
+                }
+            }
+
+            ~Bzip2Decompressor() override final {
+                close();
+            }
+
+            std::string read() override final {
+                std::string buffer;
+
+                if (!m_stream_end) {
+                    buffer.resize(osmium::io::Decompressor::input_buffer_size);
+                    int error;
+                    int nread = ::BZ2_bzRead(&error, m_bzfile, const_cast<char*>(buffer.data()), static_cast_with_assert<int>(buffer.size()));
+                    if (error != BZ_OK && error != BZ_STREAM_END) {
+                        detail::throw_bzip2_error(m_bzfile, "read failed", error);
+                    }
+                    if (error == BZ_STREAM_END) {
+                        void* unused;
+                        int nunused;
+                        if (! feof(m_file)) {
+                            ::BZ2_bzReadGetUnused(&error, m_bzfile, &unused, &nunused);
+                            if (error != BZ_OK) {
+                                detail::throw_bzip2_error(m_bzfile, "get unused failed", error);
+                            }
+                            std::string unused_data(static_cast<const char*>(unused), static_cast<std::string::size_type>(nunused));
+                            ::BZ2_bzReadClose(&error, m_bzfile);
+                            if (error != BZ_OK) {
+                                detail::throw_bzip2_error(m_bzfile, "read close failed", error);
+                            }
+                            m_bzfile = ::BZ2_bzReadOpen(&error, m_file, 0, 0, const_cast<void*>(static_cast<const void*>(unused_data.data())), static_cast_with_assert<int>(unused_data.size()));
+                            if (error != BZ_OK) {
+                                detail::throw_bzip2_error(m_bzfile, "read open failed", error);
+                            }
+                        } else {
+                            m_stream_end = true;
+                        }
+                    }
+                    buffer.resize(static_cast<std::string::size_type>(nread));
+                }
+
+                return buffer;
+            }
+
+            void close() override final {
+                if (m_bzfile) {
+                    int error;
+                    ::BZ2_bzReadClose(&error, m_bzfile);
+                    m_bzfile = nullptr;
+                    if (m_file) {
+                        fclose(m_file);
+                    }
+                    if (error != BZ_OK) {
+                        detail::throw_bzip2_error(m_bzfile, "read close failed", error);
+                    }
+                }
+            }
+
+        }; // class Bzip2Decompressor
+
+        class Bzip2BufferDecompressor : public Decompressor {
+
+            const char* m_buffer;
+            size_t m_buffer_size;
+            bz_stream m_bzstream;
+
+        public:
+
+            Bzip2BufferDecompressor(const char* buffer, size_t size) :
+                m_buffer(buffer),
+                m_buffer_size(size),
+                m_bzstream() {
+                m_bzstream.next_in = const_cast<char*>(buffer);
+                m_bzstream.avail_in = static_cast_with_assert<unsigned int>(size);
+                int result = BZ2_bzDecompressInit(&m_bzstream, 0, 0);
+                if (result != BZ_OK) {
+                    std::string message("bzip2 error: decompression init failed: ");
+                    throw bzip2_error(message, result);
+                }
+            }
+
+            ~Bzip2BufferDecompressor() override final {
+                BZ2_bzDecompressEnd(&m_bzstream);
+            }
+
+            std::string read() override final {
+                std::string output;
+
+                if (m_buffer) {
+                    const size_t buffer_size = 10240;
+                    output.resize(buffer_size);
+                    m_bzstream.next_out = const_cast<char*>(output.data());
+                    m_bzstream.avail_out = buffer_size;
+                    int result = BZ2_bzDecompress(&m_bzstream);
+
+                    if (result != BZ_OK) {
+                        m_buffer = nullptr;
+                        m_buffer_size = 0;
+                    }
+
+                    if (result != BZ_OK && result != BZ_STREAM_END) {
+                        std::string message("bzip2 error: decompress failed: ");
+                        throw bzip2_error(message, result);
+                    }
+
+                    output.resize(static_cast<unsigned long>(m_bzstream.next_out - output.data()));
+                }
+
+                return output;
+            }
+
+        }; // class Bzip2BufferDecompressor
+
+        namespace {
+
+            const bool registered_bzip2_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::bzip2,
+                [](int fd) { return new osmium::io::Bzip2Compressor(fd); },
+                [](int fd) { return new osmium::io::Bzip2Decompressor(fd); },
+                [](const char* buffer, size_t size) { return new osmium::io::Bzip2BufferDecompressor(buffer, size); }
+            );
+
+        } // anonymous namespace
+
+    } // namespace io
+
+} // namespace osmium
+
+#endif // OSMIUM_IO_BZIP2_COMPRESSION_HPP

include/osmium/io/compression.hpp

@@ -0,0 +1,281 @@
+#ifndef OSMIUM_IO_COMPRESSION_HPP
+#define OSMIUM_IO_COMPRESSION_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cerrno>
+#include <functional>
+#include <map>
+#include <memory>
+#include <stdexcept>
+#include <string>
+#include <system_error>
+#include <utility>
+
+#ifndef _MSC_VER
+# include <unistd.h>
+#else
+# include <io.h>
+#endif
+
+#include <osmium/io/detail/read_write.hpp>
+#include <osmium/io/file_compression.hpp>
+#include <osmium/util/compatibility.hpp>
+
+namespace osmium {
+
+    namespace io {
+
+        class Compressor {
+
+        public:
+
+            Compressor() = default;
+
+            virtual ~Compressor() {
+            }
+
+            virtual void write(const std::string& data) = 0;
+
+            virtual void close() = 0;
+
+        }; // class Compressor
+
+        class Decompressor {
+
+        public:
+
+            static constexpr size_t input_buffer_size = 256 * 1024;
+
+            Decompressor() = default;
+
+            Decompressor(const Decompressor&) = delete;
+            Decompressor& operator=(const Decompressor&) = delete;
+
+            Decompressor(Decompressor&&) = delete;
+            Decompressor& operator=(Decompressor&&) = delete;
+
+            virtual ~Decompressor() {
+            }
+
+            virtual std::string read() = 0;
+
+            virtual void close() {
+            }
+
+        }; // class Decompressor
+
+        /**
+         * This singleton factory class is used to register compression
+         * algorithms used for reading and writing OSM files.
+         *
+         * For each algorithm we store two functions that construct
+         * a compressor and decompressor object, respectively.
+         */
+        class CompressionFactory {
+
+        public:
+
+            typedef std::function<osmium::io::Compressor*(int)> create_compressor_type;
+            typedef std::function<osmium::io::Decompressor*(int)> create_decompressor_type_fd;
+            typedef std::function<osmium::io::Decompressor*(const char*, size_t)> create_decompressor_type_buffer;
+
+        private:
+
+            typedef std::map<const osmium::io::file_compression, std::tuple<create_compressor_type, create_decompressor_type_fd, create_decompressor_type_buffer>> compression_map_type;
+
+            compression_map_type m_callbacks;
+
+            CompressionFactory() = default;
+
+            CompressionFactory(const CompressionFactory&) = delete;
+            CompressionFactory& operator=(const CompressionFactory&) = delete;
+
+            CompressionFactory(CompressionFactory&&) = delete;
+            CompressionFactory& operator=(CompressionFactory&&) = delete;
+
+            OSMIUM_NORETURN void error(osmium::io::file_compression compression) {
+                std::string error_message {"Support for compression '"};
+                error_message += as_string(compression);
+                error_message += "' not compiled into this binary.";
+                throw std::runtime_error(error_message);
+            }
+
+        public:
+
+            static CompressionFactory& instance() {
+                static CompressionFactory factory;
+                return factory;
+            }
+
+            bool register_compression(
+                osmium::io::file_compression compression,
+                create_compressor_type create_compressor,
+                create_decompressor_type_fd create_decompressor_fd,
+                create_decompressor_type_buffer create_decompressor_buffer) {
+
+                compression_map_type::value_type cc(compression, std::make_tuple(create_compressor, create_decompressor_fd, create_decompressor_buffer));
+                return m_callbacks.insert(cc).second;
+            }
+
+            std::unique_ptr<osmium::io::Compressor> create_compressor(osmium::io::file_compression compression, int fd) {
+                auto it = m_callbacks.find(compression);
+
+                if (it != m_callbacks.end()) {
+                    return std::unique_ptr<osmium::io::Compressor>(std::get<0>(it->second)(fd));
+                }
+
+                error(compression);
+            }
+
+            std::unique_ptr<osmium::io::Decompressor> create_decompressor(osmium::io::file_compression compression, int fd) {
+                auto it = m_callbacks.find(compression);
+
+                if (it != m_callbacks.end()) {
+                    return std::unique_ptr<osmium::io::Decompressor>(std::get<1>(it->second)(fd));
+                }
+
+                error(compression);
+            }
+
+            std::unique_ptr<osmium::io::Decompressor> create_decompressor(osmium::io::file_compression compression, const char* buffer, size_t size) {
+                auto it = m_callbacks.find(compression);
+
+                if (it != m_callbacks.end()) {
+                    return std::unique_ptr<osmium::io::Decompressor>(std::get<2>(it->second)(buffer, size));
+                }
+
+                error(compression);
+            }
+
+        }; // class CompressionFactory
+
+        class NoCompressor : public Compressor {
+
+            int m_fd;
+
+        public:
+
+            NoCompressor(int fd) :
+                Compressor(),
+                m_fd(fd) {
+            }
+
+            ~NoCompressor() override final {
+                close();
+            }
+
+            void write(const std::string& data) override final {
+                osmium::io::detail::reliable_write(m_fd, data.data(), data.size());
+            }
+
+            void close() override final {
+                if (m_fd >= 0) {
+                    ::close(m_fd);
+                    m_fd = -1;
+                }
+            }
+
+        }; // class NoCompressor
+
+        class NoDecompressor : public Decompressor {
+
+            int m_fd;
+            const char *m_buffer;
+            size_t m_buffer_size;
+
+        public:
+
+            NoDecompressor(int fd) :
+                Decompressor(),
+                m_fd(fd),
+                m_buffer(nullptr),
+                m_buffer_size(0) {
+            }
+
+            NoDecompressor(const char* buffer, size_t size) :
+                Decompressor(),
+                m_fd(-1),
+                m_buffer(buffer),
+                m_buffer_size(size) {
+            }
+
+            ~NoDecompressor() override final {
+                close();
+            }
+
+            std::string read() override final {
+                std::string buffer;
+
+                if (m_buffer) {
+                    if (m_buffer_size != 0) {
+                        size_t size = m_buffer_size;
+                        m_buffer_size = 0;
+                        buffer.append(m_buffer, size);
+                    }
+                } else {
+                    buffer.resize(osmium::io::Decompressor::input_buffer_size);
+                    ssize_t nread = ::read(m_fd, const_cast<char*>(buffer.data()), buffer.size());
+                    if (nread < 0) {
+                        throw std::system_error(errno, std::system_category(), "Read failed");
+                    }
+                    buffer.resize(static_cast<size_t>(nread));
+                }
+
+                return buffer;
+            }
+
+            void close() override final {
+                if (m_fd >= 0) {
+                    ::close(m_fd);
+                    m_fd = -1;
+                }
+            }
+
+        }; // class NoDecompressor
+
+        namespace {
+
+            const bool registered_no_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::none,
+                [](int fd) { return new osmium::io::NoCompressor(fd); },
+                [](int fd) { return new osmium::io::NoDecompressor(fd); },
+                [](const char* buffer, size_t size) { return new osmium::io::NoDecompressor(buffer, size); }
+            );
+
+        } // anonymous namespace
+
+    } // namespace io
+
+} // namespace osmium
+
+#endif // OSMIUM_IO_COMPRESSION_HPP

include/osmium/io/detail/input_format.hpp

@@ -0,0 +1,158 @@
+#ifndef OSMIUM_IO_DETAIL_INPUT_FORMAT_HPP
+#define OSMIUM_IO_DETAIL_INPUT_FORMAT_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <functional>
+#include <map>
+#include <memory>
+#include <stdexcept>
+#include <string>
+#include <utility>
+
+#include <osmium/io/file.hpp>
+#include <osmium/io/file_format.hpp>
+#include <osmium/io/header.hpp>
+#include <osmium/memory/buffer.hpp>
+#include <osmium/osm/entity_bits.hpp>
+
+namespace osmium {
+
+    namespace thread {
+        template <typename T> class Queue;
+    } // namespace thread
+
+    namespace io {
+
+        namespace detail {
+
+            /**
+             * Virtual base class for all classes reading OSM files in different
+             * formats.
+             *
+             * Do not use this class or derived classes directly. Use the
+             * osmium::io::Reader class instead.
+             */
+            class InputFormat {
+
+            protected:
+
+                osmium::io::File m_file;
+                osmium::osm_entity_bits::type m_read_which_entities;
+                osmium::io::Header m_header;
+
+                explicit InputFormat(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities) :
+                    m_file(file),
+                    m_read_which_entities(read_which_entities) {
+                    m_header.set_has_multiple_object_versions(m_file.has_multiple_object_versions());
+                }
+
+                InputFormat(const InputFormat&) = delete;
+                InputFormat(InputFormat&&) = delete;
+
+                InputFormat& operator=(const InputFormat&) = delete;
+                InputFormat& operator=(InputFormat&&) = delete;
+
+            public:
+
+                virtual ~InputFormat() {
+                }
+
+                virtual osmium::memory::Buffer read() = 0;
+
+                virtual void close() {
+                }
+
+                virtual osmium::io::Header header() {
+                    return m_header;
+                }
+
+            }; // class InputFormat
+
+            /**
+             * This factory class is used to create objects that read OSM data
+             * written in a specified format.
+             *
+             * Do not use this class directly. Instead use the osmium::io::Reader
+             * class.
+             */
+            class InputFormatFactory {
+
+            public:
+
+                typedef std::function<osmium::io::detail::InputFormat*(const osmium::io::File&, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue<std::string>&)> create_input_type;
+
+            private:
+
+                typedef std::map<osmium::io::file_format, create_input_type> map_type;
+
+                map_type m_callbacks;
+
+                InputFormatFactory() :
+                    m_callbacks() {
+                }
+
+            public:
+
+                static InputFormatFactory& instance() {
+                    static InputFormatFactory factory;
+                    return factory;
+                }
+
+                bool register_input_format(osmium::io::file_format format, create_input_type create_function) {
+                    if (! m_callbacks.insert(map_type::value_type(format, create_function)).second) {
+                        return false;
+                    }
+                    return true;
+                }
+
+                std::unique_ptr<osmium::io::detail::InputFormat> create_input(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue<std::string>& input_queue) {
+                    file.check();
+
+                    auto it = m_callbacks.find(file.format());
+                    if (it != m_callbacks.end()) {
+                        return std::unique_ptr<osmium::io::detail::InputFormat>((it->second)(file, read_which_entities, input_queue));
+                    }
+
+                    throw std::runtime_error(std::string("Support for input format '") + as_string(file.format()) + "' not compiled into this binary.");
+                }
+
+            }; // class InputFormatFactory
+
+        } // namespace detail
+
+    } // namespace io
+
+} // namespace osmium
+
+#endif // OSMIUM_IO_DETAIL_INPUT_FORMAT_HPP

include/osmium/io/detail/opl_output_format.hpp

@@ -0,0 +1,323 @@
+#ifndef OSMIUM_IO_DETAIL_OPL_OUTPUT_FORMAT_HPP
+#define OSMIUM_IO_DETAIL_OPL_OUTPUT_FORMAT_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <chrono>
+#include <cinttypes>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <future>
+#include <iterator>
+#include <memory>
+#include <ratio>
+#include <string>
+#include <thread>
+#include <utility>
+
+#include <boost/version.hpp>
+
+#ifdef __clang__
+# pragma clang diagnostic push
+# pragma clang diagnostic ignored "-Wmissing-noreturn"
+# pragma clang diagnostic ignored "-Wsign-conversion"
+#endif
+
+#if BOOST_VERSION >= 104800
+# include <boost/regex/pending/unicode_iterator.hpp>
+#else
+# include <boost_unicode_iterator.hpp>
+#endif
+
+#ifdef __clang__
+# pragma clang diagnostic pop
+#endif
+
+#include <osmium/handler.hpp>
+#include <osmium/io/detail/output_format.hpp>
+#include <osmium/io/file_format.hpp>
+#include <osmium/memory/buffer.hpp>
+#include <osmium/memory/collection.hpp>
+#include <osmium/osm/box.hpp>
+#include <osmium/osm/changeset.hpp>
+#include <osmium/osm/item_type.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/node.hpp>
+#include <osmium/osm/object.hpp>
+#include <osmium/osm/relation.hpp>
+#include <osmium/osm/tag.hpp>
+#include <osmium/osm/timestamp.hpp>
+#include <osmium/osm/way.hpp>
+#include <osmium/thread/pool.hpp>
+#include <osmium/visitor.hpp>
+
+namespace osmium {
+
+    namespace io {
+
+        class File;
+
+        namespace detail {
+
+            /**
+             * Writes out one buffer with OSM data in OPL format.
+             */
+            class OPLOutputBlock : public osmium::handler::Handler {
+
+                static constexpr size_t tmp_buffer_size = 100;
+
+                osmium::memory::Buffer m_input_buffer;
+
+                std::string m_out;
+
+                char m_tmp_buffer[tmp_buffer_size+1];
+
+                template <typename... TArgs>
+                void output_formatted(const char* format, TArgs&&... args) {
+#ifndef NDEBUG
+                    int len =
+#endif
+#ifndef _MSC_VER
+                    snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward<TArgs>(args)...);
+#else
+                    _snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward<TArgs>(args)...);
+#endif
+                    assert(len > 0 && static_cast<size_t>(len) < tmp_buffer_size);
+                    m_out += m_tmp_buffer;
+                }
+
+                void append_encoded_string(const std::string& data) {
+                    boost::u8_to_u32_iterator<std::string::const_iterator> it(data.cbegin(), data.cbegin(), data.cend());
+                    boost::u8_to_u32_iterator<std::string::const_iterator> end(data.cend(), data.cend(), data.cend());
+                    boost::utf8_output_iterator<std::back_insert_iterator<std::string>> oit(std::back_inserter(m_out));
+
+                    for (; it != end; ++it) {
+                        uint32_t c = *it;
+
+                        // This is a list of Unicode code points that we let
+                        // through instead of escaping them. It is incomplete
+                        // and can be extended later.
+                        // Generally we don't want to let through any character
+                        // that has special meaning in the OPL format such as
+                        // space, comma, @, etc. and any non-printing characters.
+                        if ((0x0021 <= c && c <= 0x0024) ||
+                            (0x0026 <= c && c <= 0x002b) ||
+                            (0x002d <= c && c <= 0x003c) ||
+                            (0x003e <= c && c <= 0x003f) ||
+                            (0x0041 <= c && c <= 0x007e) ||
+                            (0x00a1 <= c && c <= 0x00ac) ||
+                            (0x00ae <= c && c <= 0x05ff)) {
+                            *oit = c;
+                        } else {
+                            m_out += '%';
+                            output_formatted("%04x", c);
+                        }
+                    }
+                }
+
+                void write_meta(const osmium::OSMObject& object) {
+                    output_formatted("%" PRId64 " v%d d", object.id(), object.version());
+                    m_out += (object.visible() ? 'V' : 'D');
+                    output_formatted(" c%d t", object.changeset());
+                    m_out += object.timestamp().to_iso();
+                    output_formatted(" i%d u", object.uid());
+                    append_encoded_string(object.user());
+                    m_out += " T";
+                    bool first = true;
+                    for (const auto& tag : object.tags()) {
+                        if (first) {
+                            first = false;
+                        } else {
+                            m_out += ',';
+                        }
+                        append_encoded_string(tag.key());
+                        m_out += '=';
+                        append_encoded_string(tag.value());
+                    }
+                }
+
+                void write_location(const osmium::Location location, const char x, const char y) {
+                    if (location) {
+                        output_formatted(" %c%.7f %c%.7f", x, location.lon_without_check(), y, location.lat_without_check());
+                    } else {
+                        m_out += ' ';
+                        m_out += x;
+                        m_out += ' ';
+                        m_out += y;
+                    }
+                }
+
+            public:
+
+                explicit OPLOutputBlock(osmium::memory::Buffer&& buffer) :
+                    m_input_buffer(std::move(buffer)),
+                    m_out(),
+                    m_tmp_buffer() {
+                }
+
+                OPLOutputBlock(const OPLOutputBlock&) = delete;
+                OPLOutputBlock& operator=(const OPLOutputBlock&) = delete;
+
+                OPLOutputBlock(OPLOutputBlock&& other) :
+                    m_input_buffer(std::move(other.m_input_buffer)),
+                    m_out(),
+                    m_tmp_buffer() {
+                }
+
+                OPLOutputBlock& operator=(OPLOutputBlock&& other) {
+                    m_input_buffer = std::move(other.m_input_buffer);
+                    return *this;
+                }
+
+                std::string operator()() {
+                    osmium::apply(m_input_buffer.cbegin(), m_input_buffer.cend(), *this);
+
+                    std::string out;
+                    std::swap(out, m_out);
+                    return out;
+                }
+
+                void node(const osmium::Node& node) {
+                    m_out += 'n';
+                    write_meta(node);
+                    write_location(node.location(), 'x', 'y');
+                    m_out += '\n';
+                }
+
+                void way(const osmium::Way& way) {
+                    m_out += 'w';
+                    write_meta(way);
+
+                    m_out += " N";
+                    bool first = true;
+                    for (const auto& node_ref : way.nodes()) {
+                        if (first) {
+                            first = false;
+                        } else {
+                            m_out += ',';
+                        }
+                        output_formatted("n%" PRId64, node_ref.ref());
+                    }
+                    m_out += '\n';
+                }
+
+                void relation(const osmium::Relation& relation) {
+                    m_out += 'r';
+                    write_meta(relation);
+
+                    m_out += " M";
+                    bool first = true;
+                    for (const auto& member : relation.members()) {
+                        if (first) {
+                            first = false;
+                        } else {
+                            m_out += ',';
+                        }
+                        m_out += item_type_to_char(member.type());
+                        output_formatted("%" PRId64 "@", member.ref());
+                        m_out += member.role();
+                    }
+                    m_out += '\n';
+                }
+
+                void changeset(const osmium::Changeset& changeset) {
+                    output_formatted("c%d k%d s", changeset.id(), changeset.num_changes());
+                    m_out += changeset.created_at().to_iso();
+                    m_out += " e";
+                    m_out += changeset.closed_at().to_iso();
+                    output_formatted(" i%d u", changeset.uid());
+                    append_encoded_string(changeset.user());
+                    write_location(changeset.bounds().bottom_left(), 'x', 'y');
+                    write_location(changeset.bounds().top_right(), 'X', 'Y');
+                    m_out += " T";
+                    bool first = true;
+                    for (const auto& tag : changeset.tags()) {
+                        if (first) {
+                            first = false;
+                        } else {
+                            m_out += ',';
+                        }
+                        append_encoded_string(tag.key());
+                        m_out += '=';
+                        append_encoded_string(tag.value());
+                    }
+
+                    m_out += '\n';
+                }
+
+            }; // OPLOutputBlock
+
+            class OPLOutputFormat : public osmium::io::detail::OutputFormat {
+
+                OPLOutputFormat(const OPLOutputFormat&) = delete;
+                OPLOutputFormat& operator=(const OPLOutputFormat&) = delete;
+
+            public:
+
+                OPLOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) :
+                    OutputFormat(file, output_queue) {
+                }
+
+                void write_buffer(osmium::memory::Buffer&& buffer) override final {
+                    m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer)}));
+                }
+
+                void close() override final {
+                    std::string out;
+                    std::promise<std::string> promise;
+                    m_output_queue.push(promise.get_future());
+                    promise.set_value(out);
+                }
+
+            }; // class OPLOutputFormat
+
+            namespace {
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
+                const bool registered_opl_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::opl,
+                    [](const osmium::io::File& file, data_queue_type& output_queue) {
+                        return new osmium::io::detail::OPLOutputFormat(file, output_queue);
+                });
+#pragma GCC diagnostic pop
+
+            } // anonymous namespace
+
+        } // namespace detail
+
+    } // namespace io
+
+} // namespace osmium
+
+#endif // OSMIUM_IO_DETAIL_OPL_OUTPUT_FORMAT_HPP

include/osmium/io/detail/output_format.hpp

@@ -0,0 +1,156 @@
+#ifndef OSMIUM_IO_DETAIL_OUTPUT_FORMAT_HPP
+#define OSMIUM_IO_DETAIL_OUTPUT_FORMAT_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <functional>
+#include <future>
+#include <map>
+#include <memory>
+#include <stdexcept>
+#include <string>
+#include <utility>
+
+#include <osmium/io/file.hpp>
+#include <osmium/io/file_format.hpp>
+#include <osmium/io/header.hpp>
+#include <osmium/thread/queue.hpp>
+
+namespace osmium {
+
+    namespace memory {
+        class Buffer;
+    }
+
+    namespace io {
+
+        namespace detail {
+
+            typedef osmium::thread::Queue<std::future<std::string>> data_queue_type;
+
+            /**
+             * Virtual base class for all classes writing OSM files in different
+             * formats.
+             *
+             * Do not use this class or derived classes directly. Use the
+             * osmium::io::Writer class instead.
+             */
+            class OutputFormat {
+
+            protected:
+
+                osmium::io::File m_file;
+                data_queue_type& m_output_queue;
+
+            public:
+
+                explicit OutputFormat(const osmium::io::File& file, data_queue_type& output_queue) :
+                    m_file(file),
+                    m_output_queue(output_queue) {
+                }
+
+                OutputFormat(const OutputFormat&) = delete;
+                OutputFormat(OutputFormat&&) = delete;
+
+                OutputFormat& operator=(const OutputFormat&) = delete;
+                OutputFormat& operator=(OutputFormat&&) = delete;
+
+                virtual ~OutputFormat() {
+                }
+
+                virtual void write_header(const osmium::io::Header&) {
+                }
+
+                virtual void write_buffer(osmium::memory::Buffer&&) = 0;
+
+                virtual void close() = 0;
+
+            }; // class OutputFormat
+
+            /**
+             * This factory class is used to create objects that write OSM data
+             * into a specified output format.
+             *
+             * Do not use this class directly. Instead use the osmium::io::Writer
+             * class.
+             */
+            class OutputFormatFactory {
+
+            public:
+
+                typedef std::function<osmium::io::detail::OutputFormat*(const osmium::io::File&, data_queue_type&)> create_output_type;
+
+            private:
+
+                typedef std::map<osmium::io::file_format, create_output_type> map_type;
+
+                map_type m_callbacks;
+
+                OutputFormatFactory() :
+                    m_callbacks() {
+                }
+
+            public:
+
+                static OutputFormatFactory& instance() {
+                    static OutputFormatFactory factory;
+                    return factory;
+                }
+
+                bool register_output_format(osmium::io::file_format format, create_output_type create_function) {
+                    if (! m_callbacks.insert(map_type::value_type(format, create_function)).second) {
+                        return false;
+                    }
+                    return true;
+                }
+
+                std::unique_ptr<osmium::io::detail::OutputFormat> create_output(const osmium::io::File& file, data_queue_type& output_queue) {
+                    file.check();
+
+                    auto it = m_callbacks.find(file.format());
+                    if (it != m_callbacks.end()) {
+                        return std::unique_ptr<osmium::io::detail::OutputFormat>((it->second)(file, output_queue));
+                    }
+
+                    throw std::runtime_error(std::string("Support for output format '") + as_string(file.format()) + "' not compiled into this binary.");
+                }
+
+            }; // class OutputFormatFactory
+
+        } // namespace detail
+
+    } // namespace io
+
+} // namespace osmium
+
+#endif // OSMIUM_IO_DETAIL_OUTPUT_FORMAT_HPP

include/osmium/io/detail/pbf.hpp

@@ -0,0 +1,98 @@
+#ifndef OSMIUM_IO_DETAIL_PBF_HPP
+#define OSMIUM_IO_DETAIL_PBF_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <stdexcept>
+
+#include <osmpbf/osmpbf.h>
+
+// needed for htonl and ntohl
+#ifndef _WIN32
+# include <netinet/in.h>
+#else
+# include <winsock2.h>
+#endif
+
+#include <osmium/io/error.hpp>
+#include <osmium/osm/item_type.hpp>
+
+namespace osmium {
+
+// avoid g++ false positive
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wreturn-type"
+    inline item_type osmpbf_membertype_to_item_type(const OSMPBF::Relation::MemberType mt) {
+        switch (mt) {
+            case OSMPBF::Relation::NODE:
+                return item_type::node;
+            case OSMPBF::Relation::WAY:
+                return item_type::way;
+            case OSMPBF::Relation::RELATION:
+                return item_type::relation;
+        }
+    }
+#pragma GCC diagnostic pop
+
+    inline OSMPBF::Relation::MemberType item_type_to_osmpbf_membertype(const item_type type) {
+        switch (type) {
+            case item_type::node:
+                return OSMPBF::Relation::NODE;
+            case item_type::way:
+                return OSMPBF::Relation::WAY;
+            case item_type::relation:
+                return OSMPBF::Relation::RELATION;
+            default:
+                throw std::runtime_error("Unknown relation member type");
+        }
+    }
+
+    /**
+     * Exception thrown when there was a problem with parsing the PBF format of
+     * a file.
+     */
+    struct pbf_error : public io_error {
+
+        pbf_error(const std::string& what) :
+            io_error(std::string("PBF error: ") + what) {
+        }
+
+        pbf_error(const char* what) :
+            io_error(std::string("PBF error: ") + what) {
+        }
+
+    }; // struct pbf_error
+
+} // namespace osmium
+
+#endif // OSMIUM_IO_DETAIL_PBF_HPP

include/osmium/io/detail/pbf_input_format.hpp

@@ -0,0 +1,240 @@
+#ifndef OSMIUM_IO_DETAIL_PBF_INPUT_FORMAT_HPP
+#define OSMIUM_IO_DETAIL_PBF_INPUT_FORMAT_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <algorithm>
+#include <atomic>
+#include <cassert>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <future>
+#include <memory>
+#include <ratio>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <type_traits>
+
+#include <osmium/io/detail/input_format.hpp>
+#include <osmium/io/detail/pbf.hpp> // IWYU pragma: export
+#include <osmium/io/detail/pbf_parser.hpp>
+#include <osmium/io/error.hpp>
+#include <osmium/io/file.hpp>
+#include <osmium/io/file_format.hpp>
+#include <osmium/memory/buffer.hpp>
+#include <osmium/osm.hpp>
+#include <osmium/osm/box.hpp>
+#include <osmium/osm/entity_bits.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/object.hpp>
+#include <osmium/osm/timestamp.hpp>
+#include <osmium/thread/pool.hpp>
+#include <osmium/thread/queue.hpp>
+#include <osmium/thread/util.hpp>
+#include <osmium/util/cast.hpp>
+#include <osmium/util/config.hpp>
+
+namespace osmium {
+
+    namespace io {
+
+        class File;
+
+        namespace detail {
+
+            typedef osmium::thread::Queue<std::future<osmium::memory::Buffer>> queue_type;
+
+            /**
+             * Class for parsing PBF files.
+             */
+            class PBFInputFormat : public osmium::io::detail::InputFormat {
+
+                bool m_use_thread_pool;
+                queue_type m_queue;
+                std::atomic<bool> m_done;
+                std::thread m_reader;
+                osmium::thread::Queue<std::string>& m_input_queue;
+                std::string m_input_buffer;
+
+                /**
+                 * Read the given number of bytes from the input queue.
+                 *
+                 * @param size Number of bytes to read
+                 * @returns String with the data
+                 * @throws osmium::pbf_error If size bytes can't be read
+                 */
+                std::string read_from_input_queue(size_t size) {
+                    while (m_input_buffer.size() < size) {
+                        std::string new_data;
+                        m_input_queue.wait_and_pop(new_data);
+                        if (new_data.empty()) {
+                            throw osmium::pbf_error("truncated data (EOF encountered)");
+                        }
+                        m_input_buffer += new_data;
+                    }
+
+                    std::string output { m_input_buffer.substr(size) };
+                    m_input_buffer.resize(size);
+                    std::swap(output, m_input_buffer);
+                    return output;
+                }
+
+                /**
+                 * Read BlobHeader by first reading the size and then the
+                 * BlobHeader. The BlobHeader contains a type field (which is
+                 * checked against the expected type) and a size field.
+                 *
+                 * @param expected_type Expected type of data ("OSMHeader" or
+                 *                      "OSMData").
+                 * @returns Size of the data read from BlobHeader (0 on EOF).
+                 */
+                size_t read_blob_header(const char* expected_type) {
+                    uint32_t size_in_network_byte_order;
+
+                    try {
+                        std::string input_data = read_from_input_queue(sizeof(size_in_network_byte_order));
+                        size_in_network_byte_order = *reinterpret_cast<const uint32_t*>(input_data.data());
+                    } catch (osmium::pbf_error&) {
+                        return 0; // EOF
+                    }
+
+                    uint32_t size = ntohl(size_in_network_byte_order);
+                    if (size > static_cast<uint32_t>(OSMPBF::max_blob_header_size)) {
+                        throw osmium::pbf_error("invalid BlobHeader size (> max_blob_header_size)");
+                    }
+
+                    OSMPBF::BlobHeader blob_header;
+                    if (!blob_header.ParseFromString(read_from_input_queue(size))) {
+                        throw osmium::pbf_error("failed to parse BlobHeader");
+                    }
+
+                    if (blob_header.type() != expected_type) {
+                        throw osmium::pbf_error("blob does not have expected type (OSMHeader in first blob, OSMData in following blobs)");
+                    }
+
+                    return static_cast<size_t>(blob_header.datasize());
+                }
+
+                void parse_osm_data(osmium::osm_entity_bits::type read_types) {
+                    osmium::thread::set_thread_name("_osmium_pbf_in");
+                    int n=0;
+                    while (auto size = read_blob_header("OSMData")) {
+
+                        if (m_use_thread_pool) {
+                            m_queue.push(osmium::thread::Pool::instance().submit(DataBlobParser{read_from_input_queue(size), read_types}));
+                        } else {
+                            std::promise<osmium::memory::Buffer> promise;
+                            m_queue.push(promise.get_future());
+                            DataBlobParser data_blob_parser{read_from_input_queue(size), read_types};
+                            promise.set_value(data_blob_parser());
+                        }
+                        ++n;
+
+                        if (m_done) {
+                            return;
+                        }
+                    }
+                    m_done = true;
+                }
+
+            public:
+
+                /**
+                 * Instantiate PBF Parser
+                 *
+                 * @param file osmium::io::File instance describing file to be read from.
+                 * @param read_which_entities Which types of OSM entities (nodes, ways, relations, changesets) should be parsed?
+                 * @param input_queue String queue where data is read from.
+                 */
+                PBFInputFormat(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue<std::string>& input_queue) :
+                    osmium::io::detail::InputFormat(file, read_which_entities),
+                    m_use_thread_pool(osmium::config::use_pool_threads_for_pbf_parsing()),
+                    m_queue(20, "pbf_parser_results"), // XXX
+                    m_done(false),
+                    m_input_queue(input_queue),
+                    m_input_buffer() {
+                    GOOGLE_PROTOBUF_VERIFY_VERSION;
+
+                    // handle OSMHeader
+                    auto size = read_blob_header("OSMHeader");
+                    m_header = parse_header_blob(read_from_input_queue(size));
+
+                    if (m_read_which_entities != osmium::osm_entity_bits::nothing) {
+                        m_reader = std::thread(&PBFInputFormat::parse_osm_data, this, m_read_which_entities);
+                    }
+                }
+
+                ~PBFInputFormat() {
+                    m_done = true;
+                    if (m_reader.joinable()) {
+                        m_reader.join();
+                    }
+                }
+
+                /**
+                 * Returns the next buffer with OSM data read from the PBF file.
+                 * Blocks if data is not available yet.
+                 * Returns an empty buffer at end of input.
+                 */
+                osmium::memory::Buffer read() override {
+                    if (!m_done || !m_queue.empty()) {
+                        std::future<osmium::memory::Buffer> buffer_future;
+                        m_queue.wait_and_pop(buffer_future);
+                        return buffer_future.get();
+                    }
+
+                    return osmium::memory::Buffer();
+                }
+
+            }; // class PBFInputFormat
+
+            namespace {
+
+                const bool registered_pbf_input = osmium::io::detail::InputFormatFactory::instance().register_input_format(osmium::io::file_format::pbf,
+                    [](const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue<std::string>& input_queue) {
+                        return new osmium::io::detail::PBFInputFormat(file, read_which_entities, input_queue);
+                });
+
+            } // anonymous namespace
+
+        } // namespace detail
+
+    } // namespace io
+
+} // namespace osmium
+
+#endif // OSMIUM_IO_DETAIL_PBF_INPUT_FORMAT_HPP

include/osmium/io/detail/pbf_output_format.hpp

@@ -0,0 +1,944 @@
+#ifndef OSMIUM_IO_DETAIL_PBF_OUTPUT_FORMAT_HPP
+#define OSMIUM_IO_DETAIL_PBF_OUTPUT_FORMAT_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+/*
+
+About the .osm.pbf file format
+This is an excerpt of <http://wiki.openstreetmap.org/wiki/PBF_Format>
+
+The .osm.pbf format and it's derived formats (.osh.pbf and .osc.pbf) are encoded
+using googles protobuf library for the low-level storage. They are constructed
+by nesting data on two levels:
+
+On the lower level the file is constructed using BlobHeaders and Blobs. A .osm.pbf
+file contains multiple sequences of
+ 1. a 4-byte header size, stored in network-byte-order
+ 2. a BlobHeader of exactly this size
+ 3. a Blob
+
+The BlobHeader tells the reader about the type and size of the following Blob. The
+Blob can contain data in raw or zlib-compressed form. After uncompressing the blob
+it is treated differently depending on the type specified in the BlobHeader.
+
+The contents of the Blob belongs to the higher level. It contains either an HeaderBlock
+(type="OSMHeader") or an PrimitiveBlock (type="OSMData"). The file needs to have
+at least one HeaderBlock before the first PrimitiveBlock.
+
+The HeaderBlock contains meta-information like the writing program or a bbox. It may
+also contain multiple "required features" that describe what kinds of input a
+reading program needs to handle in order to fully understand the files' contents.
+
+The PrimitiveBlock can store multiple types of objects (i.e. 5 nodes, 2 ways and
+1 relation). It contains one or more PrimitiveGroup which in turn contain multiple
+nodes, ways or relations. A PrimitiveGroup should only contain one kind of object.
+
+There's a special kind of "object type" called dense-nodes. It is used to store nodes
+in a very dense format, avoiding message overheads and using delta-encoding for nearly
+all ids.
+
+All Strings are stored as indexes to rows in a StringTable. The StringTable contains
+one row for each used string, so strings that are used multiple times need to be
+stored only once. The StringTable is sorted by usage-count, so the most often used
+string is stored at index 1.
+
+A simple outline of a .osm.pbf file could look like this:
+
+  4-bytes header size
+  BlobHeader
+  Blob
+    HeaderBlock
+  4-bytes header size
+  BlobHeader
+  Blob
+    PrimitiveBlock
+      StringTable
+      PrimitiveGroup
+        5 nodes
+      PrimitiveGroup
+        2 ways
+      PrimitiveGroup
+        1 relation
+
+More complete outlines of real .osm.pbf files can be created using the osmpbf-outline tool:
+ <https://github.com/MaZderMind/OSM-binary/tree/osmpbf-outline>
+*/
+
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <cstdint>
+#include <cstdlib>
+#include <future>
+#include <iostream>
+#include <memory>
+#include <ratio>
+#include <string>
+#include <thread>
+#include <time.h>
+#include <utility>
+
+#include <osmium/handler.hpp>
+#include <osmium/io/detail/output_format.hpp>
+#include <osmium/io/detail/pbf.hpp> // IWYU pragma: export
+#include <osmium/io/detail/pbf_stringtable.hpp>
+#include <osmium/io/detail/zlib.hpp>
+#include <osmium/io/file.hpp>
+#include <osmium/io/file_format.hpp>
+#include <osmium/io/header.hpp>
+#include <osmium/memory/buffer.hpp>
+#include <osmium/memory/collection.hpp>
+#include <osmium/osm/box.hpp>
+#include <osmium/osm/item_type.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/node.hpp>
+#include <osmium/osm/object.hpp>
+#include <osmium/osm/relation.hpp>
+#include <osmium/osm/tag.hpp>
+#include <osmium/osm/timestamp.hpp>
+#include <osmium/osm/way.hpp>
+#include <osmium/util/cast.hpp>
+#include <osmium/visitor.hpp>
+
+namespace osmium {
+
+    namespace io {
+
+        namespace detail {
+
+            namespace {
+
+                /**
+                 * Serialize a protobuf message into a Blob, optionally apply compression
+                 * and return it together with a BlobHeader ready to be written to a file.
+                 *
+                 * @param type Type-string used in the BlobHeader.
+                 * @param msg Protobuf-message.
+                 * @param use_compression Should the output be compressed using zlib?
+                 */
+                std::string serialize_blob(const std::string& type, const google::protobuf::MessageLite& msg, bool use_compression) {
+                    OSMPBF::Blob pbf_blob;
+
+                    {
+                        std::string content;
+                        msg.SerializeToString(&content);
+
+                        pbf_blob.set_raw_size(static_cast_with_assert<::google::protobuf::int32>(content.size()));
+
+                        if (use_compression) {
+                            pbf_blob.set_zlib_data(osmium::io::detail::zlib_compress(content));
+                        } else {
+                            pbf_blob.set_raw(content);
+                        }
+                    }
+
+                    std::string blob_data;
+                    pbf_blob.SerializeToString(&blob_data);
+
+                    OSMPBF::BlobHeader pbf_blob_header;
+                    pbf_blob_header.set_type(type);
+                    pbf_blob_header.set_datasize(static_cast_with_assert<::google::protobuf::int32>(blob_data.size()));
+
+                    std::string blob_header_data;
+                    pbf_blob_header.SerializeToString(&blob_header_data);
+
+                    uint32_t sz = htonl(static_cast_with_assert<uint32_t>(blob_header_data.size()));
+
+                    // write to output: the 4-byte BlobHeader-Size followed by the BlobHeader followed by the Blob
+                    std::string output;
+                    output.reserve(sizeof(sz) + blob_header_data.size() + blob_data.size());
+                    output.append(reinterpret_cast<const char*>(&sz), sizeof(sz));
+                    output.append(blob_header_data);
+                    output.append(blob_data);
+
+                    return output;
+                }
+
+            } // anonymous namespace
+
+            class PBFOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler {
+
+                /**
+                 * This class models a variable that keeps track of the value
+                 * it was last set to and returns the delta between old and
+                 * new value from the update() call.
+                 */
+                template <typename T>
+                class Delta {
+
+                    T m_value;
+
+                public:
+
+                    Delta() :
+                        m_value(0) {
+                    }
+
+                    void clear() {
+                        m_value = 0;
+                    }
+
+                    T update(T new_value) {
+                        using std::swap;
+                        swap(m_value, new_value);
+                        return m_value - new_value;
+                    }
+
+                }; // class Delta
+
+                /**
+                 * Maximum number of items in a primitive block.
+                 *
+                 * The uncompressed length of a Blob *should* be less
+                 * than 16 megabytes and *must* be less than 32 megabytes.
+                 *
+                 * A block may contain any number of entities, as long as
+                 * the size limits for the surrounding blob are obeyed.
+                 * However, for simplicity, the current Osmosis (0.38)
+                 * as well as Osmium implementation always
+                 * uses at most 8k entities in a block.
+                 */
+                static constexpr uint32_t max_block_contents = 8000;
+
+                /**
+                 * The output buffer (block) will be filled to about
+                 * 95% and then written to disk. This leaves more than
+                 * enough space for the string table (which typically
+                 * needs about 0.1 to 0.3% of the block size).
+                 */
+                static constexpr int64_t buffer_fill_percent = 95;
+
+                /**
+                 * protobuf-struct of a HeaderBlock
+                 */
+                OSMPBF::HeaderBlock pbf_header_block;
+
+                /**
+                 * protobuf-struct of a PrimitiveBlock
+                 */
+                OSMPBF::PrimitiveBlock pbf_primitive_block;
+
+                /**
+                 * pointer to PrimitiveGroups inside the current PrimitiveBlock,
+                 * used for writing nodes, ways or relations
+                 */
+                OSMPBF::PrimitiveGroup* pbf_nodes;
+                OSMPBF::PrimitiveGroup* pbf_ways;
+                OSMPBF::PrimitiveGroup* pbf_relations;
+
+                /**
+                 * To flexibly handle multiple resolutions, the granularity, or
+                 * resolution used for representing locations is adjustable in
+                 * multiples of 1 nanodegree. The default scaling factor is 100
+                 * nanodegrees, corresponding to about ~1cm at the equator.
+                 * This is the current resolution of the OSM database.
+                 */
+                int m_location_granularity;
+
+                /**
+                 * The granularity used for representing timestamps is also adjustable in
+                 * multiples of 1 millisecond. The default scaling factor is 1000
+                 * milliseconds, which is the current resolution of the OSM database.
+                 */
+                int m_date_granularity;
+
+                /**
+                 * should nodes be serialized into the dense format?
+                 *
+                 * nodes can be encoded one of two ways, as a Node
+                 * (m_use_dense_nodes = false) and a special dense format.
+                 * In the dense format, all information is stored 'column wise',
+                 * as an array of ID's, array of latitudes, and array of
+                 * longitudes. Each column is delta-encoded. This reduces
+                 * header overheads and allows delta-coding to work very effectively.
+                 */
+                bool m_use_dense_nodes {true};
+
+                /**
+                 * should the PBF blobs contain zlib compressed data?
+                 *
+                 * the zlib compression is optional, it's possible to store the
+                 * blobs in raw format. Disabling the compression can improve the
+                 * writing speed a little but the output will be 2x to 3x bigger.
+                 */
+                bool m_use_compression {true};
+
+                /**
+                 * While the .osm.pbf-format is able to carry all meta information, it is
+                 * also able to omit this information to reduce size.
+                 */
+                bool m_should_add_metadata {true};
+
+                /**
+                 * Should the visible flag be added on objects?
+                 */
+                bool m_add_visible;
+
+                /**
+                 * counter used to quickly check the number of objects stored inside
+                 * the current PrimitiveBlock. When the counter reaches max_block_contents
+                 * the PrimitiveBlock is serialized into a Blob and flushed to the file.
+                 *
+                 * this check is performed in check_block_contents_counter() which is
+                 * called once for each object.
+                 */
+                uint16_t primitive_block_contents;
+                int primitive_block_size;
+
+                // StringTable management
+                StringTable string_table;
+
+                /**
+                 * These variables are used to calculate the
+                 * delta-encoding while storing dense-nodes. It holds the last seen values
+                 * from which the difference is stored into the protobuf.
+                 */
+                Delta<int64_t> m_delta_id;
+                Delta<int64_t> m_delta_lat;
+                Delta<int64_t> m_delta_lon;
+                Delta<int64_t> m_delta_timestamp;
+                Delta<int64_t> m_delta_changeset;
+                Delta<int64_t> m_delta_uid;
+                Delta<::google::protobuf::int32> m_delta_user_sid;
+
+                bool debug;
+
+                bool has_debug_level(int) {
+                    return false;
+                }
+
+                ///// Blob writing /////
+
+                /**
+                 * Before a PrimitiveBlock gets serialized, all interim StringTable-ids needs to be
+                 * mapped to the associated real StringTable ids. This is done in this function.
+                 *
+                 * This function needs to know about the concrete structure of all item types to find
+                 * all occurrences of string-ids.
+                 */
+                void map_string_ids() {
+                    // test, if the node-block has been allocated
+                    if (pbf_nodes) {
+                        // iterate over all nodes, passing them to the map_common_string_ids function
+                        for (int i=0, l=pbf_nodes->nodes_size(); i<l; ++i) {
+                            map_common_string_ids(pbf_nodes->mutable_nodes(i));
+                        }
+
+                        // test, if the node-block has a densenodes structure
+                        if (pbf_nodes->has_dense()) {
+                            // get a pointer to the densenodes structure
+                            OSMPBF::DenseNodes* dense = pbf_nodes->mutable_dense();
+
+                            // in the densenodes structure keys and vals are encoded in an intermixed
+                            // array, individual nodes are seperated by a value of 0 (0 in the StringTable
+                            // is always unused). String-ids of 0 are thus kept alone.
+                            for (int i=0, l=dense->keys_vals_size(); i<l; ++i) {
+                                // map interim string-ids > 0 to real string ids
+                                auto sid = dense->keys_vals(i);
+                                if (sid > 0) {
+                                    dense->set_keys_vals(i, string_table.map_string_id(sid));
+                                }
+                            }
+
+                            // test if the densenodes block has meta infos
+                            if (dense->has_denseinfo()) {
+                                // get a pointer to the denseinfo structure
+                                OSMPBF::DenseInfo* denseinfo = dense->mutable_denseinfo();
+
+                                // iterate over all username string-ids
+                                for (int i=0, l=denseinfo->user_sid_size(); i<l; ++i) {
+                                    // map interim string-ids > 0 to real string ids
+                                    auto user_sid = string_table.map_string_id(denseinfo->user_sid(i));
+
+                                    // delta encode the string-id
+                                    denseinfo->set_user_sid(i, m_delta_user_sid.update(user_sid));
+                                }
+                            }
+                        }
+                    }
+
+                    // test, if the ways-block has been allocated
+                    if (pbf_ways) {
+                        // iterate over all ways, passing them to the map_common_string_ids function
+                        for (int i=0, l=pbf_ways->ways_size(); i<l; ++i) {
+                            map_common_string_ids(pbf_ways->mutable_ways(i));
+                        }
+                    }
+
+                    // test, if the relations-block has been allocated
+                    if (pbf_relations) {
+                        // iterate over all relations
+                        for (int i=0, l=pbf_relations->relations_size(); i<l; ++i) {
+                            // get a pointer to the relation
+                            OSMPBF::Relation* relation = pbf_relations->mutable_relations(i);
+
+                            // pass them to the map_common_string_ids function
+                            map_common_string_ids(relation);
+
+                            // iterate over all relation members, mapping the interim string-ids
+                            // of the role to real string ids
+                            for (int mi=0; mi < relation->roles_sid_size(); ++mi) {
+                                relation->set_roles_sid(mi, string_table.map_string_id(relation->roles_sid(mi)));
+                            }
+                        }
+                    }
+                }
+
+                /**
+                * a helper function used in map_string_ids to map common interim string-ids of the
+                * user name and all tags to real string ids.
+                *
+                * TPBFObject is either OSMPBF::Node, OSMPBF::Way or OSMPBF::Relation.
+                */
+                template <class TPBFObject>
+                void map_common_string_ids(TPBFObject* in) {
+                    // if the object has meta-info attached
+                    if (in->has_info()) {
+                        // map the interim-id of the user name to a real id
+                        OSMPBF::Info* info = in->mutable_info();
+                        info->set_user_sid(string_table.map_string_id(info->user_sid()));
+                    }
+
+                    // iterate over all tags and map the interim-ids of the key and the value to real ids
+                    for (int i=0, l=in->keys_size(); i<l; ++i) {
+                        in->set_keys(i, string_table.map_string_id(in->keys(i)));
+                        in->set_vals(i, string_table.map_string_id(in->vals(i)));
+                    }
+                }
+
+
+                ///// MetaData helper /////
+
+                /**
+                 * convert a double lat or lon value to an int, respecting the current blocks granularity
+                 */
+                int64_t lonlat2int(double lonlat) {
+                    return static_cast<int64_t>(std::round(lonlat * OSMPBF::lonlat_resolution / location_granularity()));
+                }
+
+                /**
+                 * convert a timestamp to an int, respecting the current blocks granularity
+                 */
+                int64_t timestamp2int(time_t timestamp) {
+                    return static_cast<int64_t>(std::round(timestamp * (1000.0 / date_granularity())));
+                }
+
+                /**
+                 * helper function used in the write()-calls to apply common information from an osmium-object
+                 * onto a pbf-object.
+                 *
+                 * TPBFObject is either OSMPBF::Node, OSMPBF::Way or OSMPBF::Relation.
+                 */
+                template <class TPBFObject>
+                void apply_common_info(const osmium::OSMObject& in, TPBFObject* out) {
+                    // set the object-id
+                    out->set_id(in.id());
+
+                    // iterate over all tags and set the keys and vals, recording the strings in the
+                    // interim StringTable and storing the interim ids
+                    for (const auto& tag : in.tags()) {
+                        out->add_keys(string_table.record_string(tag.key()));
+                        out->add_vals(string_table.record_string(tag.value()));
+                    }
+
+                    if (m_should_add_metadata) {
+                        // add an info-section to the pbf object and set the meta-info on it
+                        OSMPBF::Info* out_info = out->mutable_info();
+                        if (m_add_visible) {
+                            out_info->set_visible(in.visible());
+                        }
+                        out_info->set_version(static_cast<::google::protobuf::int32>(in.version()));
+                        out_info->set_timestamp(timestamp2int(in.timestamp()));
+                        out_info->set_changeset(in.changeset());
+                        out_info->set_uid(static_cast<::google::protobuf::int32>(in.uid()));
+                        out_info->set_user_sid(string_table.record_string(in.user()));
+                    }
+                }
+
+
+                ///// High-Level Block writing /////
+
+                /**
+                 * store the current pbf_header_block into a Blob and clear this struct afterwards.
+                 */
+                void store_header_block() {
+                    if (debug && has_debug_level(1)) {
+                        std::cerr << "storing header block" << std::endl;
+                    }
+
+                    std::promise<std::string> promise;
+                    m_output_queue.push(promise.get_future());
+                    promise.set_value(serialize_blob("OSMHeader", pbf_header_block, m_use_compression));
+
+                    pbf_header_block.Clear();
+                }
+
+                /**
+                 * store the interim StringTable to the current pbf_primitive_block, map all interim string ids
+                 * to real StringTable ids and then store the current pbf_primitive_block into a Blob and clear
+                 * this struct and all related pointers and maps afterwards.
+                 */
+                void store_primitive_block() {
+                    if (debug && has_debug_level(1)) {
+                        std::cerr << "storing primitive block with " << primitive_block_contents << " items" << std::endl;
+                    }
+
+                    // set the granularity
+                    pbf_primitive_block.set_granularity(location_granularity());
+                    pbf_primitive_block.set_date_granularity(date_granularity());
+
+                    // store the interim StringTable into the protobuf object
+                    string_table.store_stringtable(pbf_primitive_block.mutable_stringtable());
+
+                    // map all interim string ids to real ids
+                    map_string_ids();
+
+                    std::promise<std::string> promise;
+                    m_output_queue.push(promise.get_future());
+                    promise.set_value(serialize_blob("OSMData", pbf_primitive_block, m_use_compression));
+
+                    // clear the PrimitiveBlock struct
+                    pbf_primitive_block.Clear();
+
+                    // clear the interim StringTable and its id map
+                    string_table.clear();
+
+                    // reset the delta variables
+                    m_delta_id.clear();
+                    m_delta_lat.clear();
+                    m_delta_lon.clear();
+                    m_delta_timestamp.clear();
+                    m_delta_changeset.clear();
+                    m_delta_uid.clear();
+                    m_delta_user_sid.clear();
+
+                    // reset the contents-counter to zero
+                    primitive_block_contents = 0;
+                    primitive_block_size = 0;
+
+                    // reset the node/way/relation pointers to nullptr
+                    pbf_nodes = nullptr;
+                    pbf_ways = nullptr;
+                    pbf_relations = nullptr;
+                }
+
+                /**
+                 * this little function checks primitive_block_contents counter against its maximum and calls
+                 * store_primitive_block to flush the block to the disk when it's reached. It's also responsible
+                 * for increasing this counter.
+                 *
+                 * this function also checks the estimated size of the current block and calls store_primitive_block
+                 * when the estimated size reaches buffer_fill_percent of the maximum uncompressed blob size.
+                 */
+                void check_block_contents_counter() {
+                    if (primitive_block_contents >= max_block_contents) {
+                        store_primitive_block();
+                    } else if (primitive_block_size > OSMPBF::max_uncompressed_blob_size * buffer_fill_percent / 100) {
+                        if (debug && has_debug_level(1)) {
+                            std::cerr << "storing primitive_block with only " << primitive_block_contents << " items, because its ByteSize (" << primitive_block_size << ") reached " <<
+                                      (static_cast<float>(primitive_block_size) / static_cast<float>(OSMPBF::max_uncompressed_blob_size) * 100.0) << "% of the maximum blob-size" << std::endl;
+                        }
+
+                        store_primitive_block();
+                    }
+
+                    ++primitive_block_contents;
+                }
+
+
+                ///// Block content writing /////
+
+                /**
+                 * Add a node to the block.
+                 *
+                 * @param node The node to add.
+                 */
+                void write_node(const osmium::Node& node) {
+                    // add a way to the group
+                    OSMPBF::Node* pbf_node = pbf_nodes->add_nodes();
+
+                    // copy the common meta-info from the osmium-object to the pbf-object
+                    apply_common_info(node, pbf_node);
+
+                    // modify lat & lon to integers, respecting the block's granularity and copy
+                    // the ints to the pbf-object
+                    pbf_node->set_lon(lonlat2int(node.location().lon_without_check()));
+                    pbf_node->set_lat(lonlat2int(node.location().lat_without_check()));
+                }
+
+                /**
+                 * Add a node to the block using DenseNodes.
+                 *
+                 * @param node The node to add.
+                 */
+                void write_dense_node(const osmium::Node& node) {
+                    // add a DenseNodes-Section to the PrimitiveGroup
+                    OSMPBF::DenseNodes* dense = pbf_nodes->mutable_dense();
+
+                    // copy the id, delta encoded
+                    dense->add_id(m_delta_id.update(node.id()));
+
+                    // copy the longitude, delta encoded
+                    dense->add_lon(m_delta_lon.update(lonlat2int(node.location().lon_without_check())));
+
+                    // copy the latitude, delta encoded
+                    dense->add_lat(m_delta_lat.update(lonlat2int(node.location().lat_without_check())));
+
+                    // in the densenodes structure keys and vals are encoded in an intermixed
+                    // array, individual nodes are seperated by a value of 0 (0 in the StringTable
+                    // is always unused)
+                    // so for three nodes the keys_vals array may look like this: 3 5 2 1 0 0 8 5
+                    // the first node has two tags (3=>5 and 2=>1), the second node does not
+                    // have any tags and the third node has a single tag (8=>5)
+                    for (const auto& tag : node.tags()) {
+                        dense->add_keys_vals(string_table.record_string(tag.key()));
+                        dense->add_keys_vals(string_table.record_string(tag.value()));
+                    }
+                    dense->add_keys_vals(0);
+
+                    if (m_should_add_metadata) {
+                        // add a DenseInfo-Section to the PrimitiveGroup
+                        OSMPBF::DenseInfo* denseinfo = dense->mutable_denseinfo();
+
+                        denseinfo->add_version(static_cast<::google::protobuf::int32>(node.version()));
+
+                        if (m_add_visible) {
+                            denseinfo->add_visible(node.visible());
+                        }
+
+                        // copy the timestamp, delta encoded
+                        denseinfo->add_timestamp(m_delta_timestamp.update(timestamp2int(node.timestamp())));
+
+                        // copy the changeset, delta encoded
+                        denseinfo->add_changeset(m_delta_changeset.update(node.changeset()));
+
+                        // copy the user id, delta encoded
+                        denseinfo->add_uid(static_cast<::google::protobuf::int32>(m_delta_uid.update(node.uid())));
+
+                        // record the user-name to the interim stringtable and copy the
+                        // interim string-id to the pbf-object
+                        denseinfo->add_user_sid(string_table.record_string(node.user()));
+                    }
+                }
+
+                /**
+                 * Add a way to the block.
+                 *
+                 * @param way The way to add.
+                 */
+                void write_way(const osmium::Way& way) {
+                    // add a way to the group
+                    OSMPBF::Way* pbf_way = pbf_ways->add_ways();
+
+                    // copy the common meta-info from the osmium-object to the pbf-object
+                    apply_common_info(way, pbf_way);
+
+                    // last way-node-id used for delta-encoding
+                    Delta<int64_t> delta_id;
+
+                    for (const auto& node_ref : way.nodes()) {
+                        // copy the way-node-id, delta encoded
+                        pbf_way->add_refs(delta_id.update(node_ref.ref()));
+                    }
+
+                    // count up blob size by the size of the Way
+                    primitive_block_size += pbf_way->ByteSize();
+                }
+
+                /**
+                 * Add a relation to the block.
+                 *
+                 * @param relation The relation to add.
+                 */
+                void write_relation(const osmium::Relation& relation) {
+                    // add a relation to the group
+                    OSMPBF::Relation* pbf_relation = pbf_relations->add_relations();
+
+                    // copy the common meta-info from the osmium-object to the pbf-object
+                    apply_common_info(relation, pbf_relation);
+
+                    Delta<int64_t> delta_id;
+
+                    for (const auto& member : relation.members()) {
+                        // record the relation-member role to the interim stringtable and copy the
+                        // interim string-id to the pbf-object
+                        pbf_relation->add_roles_sid(string_table.record_string(member.role()));
+
+                        // copy the relation-member-id, delta encoded
+                        pbf_relation->add_memids(delta_id.update(member.ref()));
+
+                        // copy the relation-member-type, mapped to the OSMPBF enum
+                        pbf_relation->add_types(item_type_to_osmpbf_membertype(member.type()));
+                    }
+
+                    // count up blob size by the size of the Relation
+                    primitive_block_size += pbf_relation->ByteSize();
+                }
+
+                // objects of this class can't be copied
+                PBFOutputFormat(const PBFOutputFormat&) = delete;
+                PBFOutputFormat& operator=(const PBFOutputFormat&) = delete;
+
+            public:
+
+                /**
+                 * Create PBFOutputFormat object from File.
+                 */
+                explicit PBFOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) :
+                    OutputFormat(file, output_queue),
+                    pbf_header_block(),
+                    pbf_primitive_block(),
+                    pbf_nodes(nullptr),
+                    pbf_ways(nullptr),
+                    pbf_relations(nullptr),
+                    m_location_granularity(pbf_primitive_block.granularity()),
+                    m_date_granularity(pbf_primitive_block.date_granularity()),
+                    m_add_visible(file.has_multiple_object_versions()),
+                    primitive_block_contents(0),
+                    primitive_block_size(0),
+                    string_table(),
+                    m_delta_id(),
+                    m_delta_lat(),
+                    m_delta_lon(),
+                    m_delta_timestamp(),
+                    m_delta_changeset(),
+                    m_delta_uid(),
+                    m_delta_user_sid(),
+                    debug(true) {
+                    GOOGLE_PROTOBUF_VERIFY_VERSION;
+                    if (file.get("pbf_dense_nodes") == "false") {
+                        m_use_dense_nodes = false;
+                    }
+                    if (file.get("pbf_compression") == "none" || file.get("pbf_compression") == "false") {
+                        m_use_compression = false;
+                    }
+                    if (file.get("pbf_add_metadata") == "false") {
+                        m_should_add_metadata = false;
+                    }
+                }
+
+                void write_buffer(osmium::memory::Buffer&& buffer) override final {
+                    osmium::apply(buffer.cbegin(), buffer.cend(), *this);
+                }
+
+
+                /**
+                 * getter to access the granularity
+                 */
+                int location_granularity() const {
+                    return m_location_granularity;
+                }
+
+                /**
+                 * setter to set the granularity
+                 */
+                PBFOutputFormat& location_granularity(int g) {
+                    m_location_granularity = g;
+                    return *this;
+                }
+
+
+                /**
+                 * getter to access the date_granularity
+                 */
+                int date_granularity() const {
+                    return m_date_granularity;
+                }
+
+                /**
+                 * Set date granularity.
+                 */
+                PBFOutputFormat& date_granularity(int g) {
+                    m_date_granularity = g;
+                    return *this;
+                }
+
+
+                /**
+                 * Initialize the writing process.
+                 *
+                 * This initializes the header-block, sets the required-features and
+                 * the writing-program and adds the obligatory StringTable-Index 0.
+                 */
+                void write_header(const osmium::io::Header& header) override final {
+                    // add the schema version as required feature to the HeaderBlock
+                    pbf_header_block.add_required_features("OsmSchema-V0.6");
+
+                    // when the densenodes-feature is used, add DenseNodes as required feature
+                    if (m_use_dense_nodes) {
+                        pbf_header_block.add_required_features("DenseNodes");
+                    }
+
+                    // when the resulting file will carry history information, add
+                    // HistoricalInformation as required feature
+                    if (m_file.has_multiple_object_versions()) {
+                        pbf_header_block.add_required_features("HistoricalInformation");
+                    }
+
+                    // set the writing program
+                    pbf_header_block.set_writingprogram(header.get("generator"));
+
+                    if (!header.boxes().empty()) {
+                        OSMPBF::HeaderBBox* pbf_bbox = pbf_header_block.mutable_bbox();
+                        osmium::Box box = header.joined_boxes();
+                        pbf_bbox->set_left(static_cast<::google::protobuf::int64>(box.bottom_left().lon() * OSMPBF::lonlat_resolution));
+                        pbf_bbox->set_bottom(static_cast<::google::protobuf::int64>(box.bottom_left().lat() * OSMPBF::lonlat_resolution));
+                        pbf_bbox->set_right(static_cast<::google::protobuf::int64>(box.top_right().lon() * OSMPBF::lonlat_resolution));
+                        pbf_bbox->set_top(static_cast<::google::protobuf::int64>(box.top_right().lat() * OSMPBF::lonlat_resolution));
+                    }
+
+                    std::string osmosis_replication_timestamp = header.get("osmosis_replication_timestamp");
+                    if (!osmosis_replication_timestamp.empty()) {
+                        osmium::Timestamp ts(osmosis_replication_timestamp.c_str());
+                        pbf_header_block.set_osmosis_replication_timestamp(ts);
+                    }
+
+                    std::string osmosis_replication_sequence_number = header.get("osmosis_replication_sequence_number");
+                    if (!osmosis_replication_sequence_number.empty()) {
+                        pbf_header_block.set_osmosis_replication_sequence_number(std::atoll(osmosis_replication_sequence_number.c_str()));
+                    }
+
+                    std::string osmosis_replication_base_url = header.get("osmosis_replication_base_url");
+                    if (!osmosis_replication_base_url.empty()) {
+                        pbf_header_block.set_osmosis_replication_base_url(osmosis_replication_base_url);
+                    }
+
+                    store_header_block();
+                }
+
+                /**
+                 * Add a node to the pbf.
+                 *
+                 * A call to this method won't write the node to the file directly but
+                 * cache it for later bulk-writing. Calling final() ensures that everything
+                 * gets written and every file pointer is closed.
+                 */
+                void node(const osmium::Node& node) {
+                    // first of we check the contents-counter which may flush the cached nodes to
+                    // disk if the limit is reached. This call also increases the contents-counter
+                    check_block_contents_counter();
+
+                    if (debug && has_debug_level(2)) {
+                        std::cerr << "node " << node.id() << " v" << node.version() << std::endl;
+                    }
+
+                    // if no PrimitiveGroup for nodes has been added, add one and save the pointer
+                    if (!pbf_nodes) {
+                        pbf_nodes = pbf_primitive_block.add_primitivegroup();
+                    }
+
+                    if (m_use_dense_nodes) {
+                        write_dense_node(node);
+                    } else {
+                        write_node(node);
+                    }
+                }
+
+                /**
+                 * Add a way to the pbf.
+                 *
+                 * A call to this method won't write the way to the file directly but
+                 * cache it for later bulk-writing. Calling final() ensures that everything
+                 * gets written and every file pointer is closed.
+                 */
+                void way(const osmium::Way& way) {
+                    // first of we check the contents-counter which may flush the cached ways to
+                    // disk if the limit is reached. This call also increases the contents-counter
+                    check_block_contents_counter();
+
+                    // if no PrimitiveGroup for nodes has been added, add one and save the pointer
+                    if (!pbf_ways) {
+                        pbf_ways = pbf_primitive_block.add_primitivegroup();
+                    }
+
+                    write_way(way);
+                }
+
+                /**
+                 * Add a relation to the pbf.
+                 *
+                 * A call to this method won't write the way to the file directly but
+                 * cache it for later bulk-writing. Calling final() ensures that everything
+                 * gets written and every file pointer is closed.
+                 */
+                void relation(const osmium::Relation& relation) {
+                    // first of we check the contents-counter which may flush the cached relations to
+                    // disk if the limit is reached. This call also increases the contents-counter
+                    check_block_contents_counter();
+
+                    // if no PrimitiveGroup for relations has been added, add one and save the pointer
+                    if (!pbf_relations) {
+                        pbf_relations = pbf_primitive_block.add_primitivegroup();
+                    }
+
+                    write_relation(relation);
+                }
+
+                /**
+                 * Finalize the writing process, flush any open primitive blocks to the file and
+                 * close the file.
+                 */
+                void close() override final {
+                    if (debug && has_debug_level(1)) {
+                        std::cerr << "finishing" << std::endl;
+                    }
+
+                    // if the current block contains any elements, flush it to the protobuf
+                    if (primitive_block_contents > 0) {
+                        store_primitive_block();
+                    }
+
+                    std::promise<std::string> promise;
+                    m_output_queue.push(promise.get_future());
+                    promise.set_value(std::string());
+                }
+
+            }; // class PBFOutputFormat
+
+            namespace {
+
+                const bool registered_pbf_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::pbf,
+                    [](const osmium::io::File& file, data_queue_type& output_queue) {
+                        return new osmium::io::detail::PBFOutputFormat(file, output_queue);
+                });
+
+            } // anonymous namespace
+
+        } // namespace detail
+
+    } // namespace io
+
+} // namespace osmium
+
+#endif // OSMIUM_IO_DETAIL_PBF_OUTPUT_FORMAT_HPP

include/osmium/io/detail/pbf_parser.hpp

@@ -0,0 +1,449 @@
+#ifndef OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP
+#define OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <algorithm>
+
+#include <osmpbf/osmpbf.h>
+
+#include <osmium/builder/osm_object_builder.hpp>
+#include <osmium/io/detail/pbf.hpp> // IWYU pragma: export
+#include <osmium/io/detail/zlib.hpp>
+#include <osmium/io/header.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/node.hpp>
+#include <osmium/osm/types.hpp>
+#include <osmium/memory/buffer.hpp>
+#include <osmium/osm/entity_bits.hpp>
+#include <osmium/util/cast.hpp>
+
+namespace osmium {
+
+    namespace io {
+
+        namespace detail {
+
+            class PBFPrimitiveBlockParser {
+
+                static constexpr size_t initial_buffer_size = 2 * 1024 * 1024;
+
+                const std::string& m_data;
+
+                const OSMPBF::StringTable* m_stringtable;
+                int64_t m_lon_offset;
+                int64_t m_lat_offset;
+                int64_t m_date_factor;
+                int32_t m_granularity;
+
+                osmium::osm_entity_bits::type m_read_types;
+
+                osmium::memory::Buffer m_buffer;
+
+                PBFPrimitiveBlockParser(const PBFPrimitiveBlockParser&) = delete;
+                PBFPrimitiveBlockParser(PBFPrimitiveBlockParser&&) = delete;
+
+                PBFPrimitiveBlockParser& operator=(const PBFPrimitiveBlockParser&) = delete;
+                PBFPrimitiveBlockParser& operator=(PBFPrimitiveBlockParser&&) = delete;
+
+            public:
+
+                explicit PBFPrimitiveBlockParser(const std::string& data, osmium::osm_entity_bits::type read_types) :
+                    m_data(data),
+                    m_stringtable(nullptr),
+                    m_lon_offset(0),
+                    m_lat_offset(0),
+                    m_date_factor(1000),
+                    m_granularity(100),
+                    m_read_types(read_types),
+                    m_buffer(initial_buffer_size) {
+                }
+
+                ~PBFPrimitiveBlockParser() = default;
+
+                osmium::memory::Buffer operator()() {
+                    OSMPBF::PrimitiveBlock pbf_primitive_block;
+                    if (!pbf_primitive_block.ParseFromString(m_data)) {
+                        throw osmium::pbf_error("failed to parse PrimitiveBlock");
+                    }
+
+                    m_stringtable = &pbf_primitive_block.stringtable();
+                    m_lon_offset  = pbf_primitive_block.lon_offset();
+                    m_lat_offset  = pbf_primitive_block.lat_offset();
+                    m_date_factor = pbf_primitive_block.date_granularity() / 1000;
+                    m_granularity = pbf_primitive_block.granularity();
+
+                    for (int i=0; i < pbf_primitive_block.primitivegroup_size(); ++i) {
+                        const OSMPBF::PrimitiveGroup& group = pbf_primitive_block.primitivegroup(i);
+
+                        if (group.has_dense())  {
+                            if (m_read_types & osmium::osm_entity_bits::node) parse_dense_node_group(group);
+                        } else if (group.ways_size() != 0) {
+                            if (m_read_types & osmium::osm_entity_bits::way) parse_way_group(group);
+                        } else if (group.relations_size() != 0) {
+                            if (m_read_types & osmium::osm_entity_bits::relation) parse_relation_group(group);
+                        } else if (group.nodes_size() != 0) {
+                            if (m_read_types & osmium::osm_entity_bits::node) parse_node_group(group);
+                        } else {
+                            throw osmium::pbf_error("group of unknown type");
+                        }
+                    }
+
+                    return std::move(m_buffer);
+                }
+
+            private:
+
+                template <class TBuilder, class TPBFObject>
+                void parse_attributes(TBuilder& builder, const TPBFObject& pbf_object) {
+                    auto& object = builder.object();
+
+                    object.set_id(pbf_object.id());
+
+                    if (pbf_object.has_info()) {
+                        object.set_version(static_cast_with_assert<object_version_type>(pbf_object.info().version()))
+                            .set_changeset(static_cast_with_assert<changeset_id_type>(pbf_object.info().changeset()))
+                            .set_timestamp(pbf_object.info().timestamp() * m_date_factor)
+                            .set_uid_from_signed(pbf_object.info().uid());
+                        if (pbf_object.info().has_visible()) {
+                            object.set_visible(pbf_object.info().visible());
+                        }
+                        builder.add_user(m_stringtable->s(static_cast_with_assert<int>(pbf_object.info().user_sid())));
+                    } else {
+                        builder.add_user("", 1);
+                    }
+                }
+
+                void parse_node_group(const OSMPBF::PrimitiveGroup& group) {
+                    for (int i=0; i < group.nodes_size(); ++i) {
+                        osmium::builder::NodeBuilder builder(m_buffer);
+                        const OSMPBF::Node& pbf_node = group.nodes(i);
+                        parse_attributes(builder, pbf_node);
+
+                        if (builder.object().visible()) {
+                            builder.object().set_location(osmium::Location(
+                                              (pbf_node.lon() * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision),
+                                              (pbf_node.lat() * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision)));
+                        }
+
+                        if (pbf_node.keys_size() > 0) {
+                            osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
+                            for (int tag=0; tag < pbf_node.keys_size(); ++tag) {
+                                tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_node.keys(tag))),
+                                                   m_stringtable->s(static_cast<int>(pbf_node.vals(tag))));
+                            }
+                        }
+
+                        m_buffer.commit();
+                    }
+                }
+
+                void parse_way_group(const OSMPBF::PrimitiveGroup& group) {
+                    for (int i=0; i < group.ways_size(); ++i) {
+                        osmium::builder::WayBuilder builder(m_buffer);
+                        const OSMPBF::Way& pbf_way = group.ways(i);
+                        parse_attributes(builder, pbf_way);
+
+                        if (pbf_way.refs_size() > 0) {
+                            osmium::builder::WayNodeListBuilder wnl_builder(m_buffer, &builder);
+                            int64_t ref = 0;
+                            for (int n=0; n < pbf_way.refs_size(); ++n) {
+                                ref += pbf_way.refs(n);
+                                wnl_builder.add_node_ref(ref);
+                            }
+                        }
+
+                        if (pbf_way.keys_size() > 0) {
+                            osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
+                            for (int tag=0; tag < pbf_way.keys_size(); ++tag) {
+                                tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_way.keys(tag))),
+                                                   m_stringtable->s(static_cast<int>(pbf_way.vals(tag))));
+                            }
+                        }
+
+                        m_buffer.commit();
+                    }
+                }
+
+                void parse_relation_group(const OSMPBF::PrimitiveGroup& group) {
+                    for (int i=0; i < group.relations_size(); ++i) {
+                        osmium::builder::RelationBuilder builder(m_buffer);
+                        const OSMPBF::Relation& pbf_relation = group.relations(i);
+                        parse_attributes(builder, pbf_relation);
+
+                        if (pbf_relation.types_size() > 0) {
+                            osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder);
+                            int64_t ref = 0;
+                            for (int n=0; n < pbf_relation.types_size(); ++n) {
+                                ref += pbf_relation.memids(n);
+                                rml_builder.add_member(osmpbf_membertype_to_item_type(pbf_relation.types(n)), ref, m_stringtable->s(pbf_relation.roles_sid(n)));
+                            }
+                        }
+
+                        if (pbf_relation.keys_size() > 0) {
+                            osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
+                            for (int tag=0; tag < pbf_relation.keys_size(); ++tag) {
+                                tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_relation.keys(tag))),
+                                                   m_stringtable->s(static_cast<int>(pbf_relation.vals(tag))));
+                            }
+                        }
+
+                        m_buffer.commit();
+                    }
+                }
+
+                int add_tags(const OSMPBF::DenseNodes& dense, int n, osmium::builder::NodeBuilder* builder) {
+                    if (n >= dense.keys_vals_size()) {
+                        return n;
+                    }
+
+                    if (dense.keys_vals(n) == 0) {
+                        return n+1;
+                    }
+
+                    osmium::builder::TagListBuilder tl_builder(m_buffer, builder);
+
+                    while (n < dense.keys_vals_size()) {
+                        int tag_key_pos = dense.keys_vals(n++);
+
+                        if (tag_key_pos == 0) {
+                            break;
+                        }
+
+                        tl_builder.add_tag(m_stringtable->s(tag_key_pos),
+                                           m_stringtable->s(dense.keys_vals(n)));
+
+                        ++n;
+                    }
+
+                    return n;
+                }
+
+                void parse_dense_node_group(const OSMPBF::PrimitiveGroup& group) {
+                    int64_t last_dense_id        = 0;
+                    int64_t last_dense_latitude  = 0;
+                    int64_t last_dense_longitude = 0;
+                    int64_t last_dense_uid       = 0;
+                    int64_t last_dense_user_sid  = 0;
+                    int64_t last_dense_changeset = 0;
+                    int64_t last_dense_timestamp = 0;
+                    int     last_dense_tag       = 0;
+
+                    const OSMPBF::DenseNodes& dense = group.dense();
+
+                    for (int i=0; i < dense.id_size(); ++i) {
+                        bool visible = true;
+
+                        last_dense_id        += dense.id(i);
+                        last_dense_latitude  += dense.lat(i);
+                        last_dense_longitude += dense.lon(i);
+
+                        if (dense.has_denseinfo()) {
+                            last_dense_changeset += dense.denseinfo().changeset(i);
+                            last_dense_timestamp += dense.denseinfo().timestamp(i);
+                            last_dense_uid       += dense.denseinfo().uid(i);
+                            last_dense_user_sid  += dense.denseinfo().user_sid(i);
+                            if (dense.denseinfo().visible_size() > 0) {
+                                visible = dense.denseinfo().visible(i);
+                            }
+                            assert(last_dense_changeset >= 0);
+                            assert(last_dense_timestamp >= 0);
+                            assert(last_dense_uid >= -1);
+                            assert(last_dense_user_sid >= 0);
+                        }
+
+                        osmium::builder::NodeBuilder builder(m_buffer);
+                        osmium::Node& node = builder.object();
+
+                        node.set_id(last_dense_id);
+
+                        if (dense.has_denseinfo()) {
+                            auto v = dense.denseinfo().version(i);
+                            assert(v > 0);
+                            node.set_version(static_cast<osmium::object_version_type>(v));
+                            node.set_changeset(static_cast<osmium::changeset_id_type>(last_dense_changeset));
+                            node.set_timestamp(last_dense_timestamp * m_date_factor);
+                            node.set_uid_from_signed(static_cast<osmium::signed_user_id_type>(last_dense_uid));
+                            node.set_visible(visible);
+                            builder.add_user(m_stringtable->s(static_cast<int>(last_dense_user_sid)));
+                        } else {
+                            builder.add_user("", 1);
+                        }
+
+                        if (visible) {
+                            builder.object().set_location(osmium::Location(
+                                              (last_dense_longitude * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision),
+                                              (last_dense_latitude  * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision)));
+                        }
+
+                        last_dense_tag = add_tags(dense, last_dense_tag, &builder);
+                        m_buffer.commit();
+                    }
+                }
+
+            }; // class PBFPrimitiveBlockParser
+
+            /**
+             * PBF blobs can optionally be packed with the zlib algorithm.
+             * This function returns the raw data (if it was unpacked) or
+             * the unpacked data (if it was packed).
+             *
+             * @param input_data Reference to input data.
+             * @returns Unpacked data
+             * @throws osmium::pbf_error If there was a problem parsing the PBF
+             */
+            inline std::unique_ptr<const std::string> unpack_blob(const std::string& input_data) {
+                OSMPBF::Blob pbf_blob;
+                if (!pbf_blob.ParseFromString(input_data)) {
+                    throw osmium::pbf_error("failed to parse blob");
+                }
+
+                if (pbf_blob.has_raw()) {
+                    return std::unique_ptr<std::string>(pbf_blob.release_raw());
+                } else if (pbf_blob.has_zlib_data()) {
+                    auto raw_size = pbf_blob.raw_size();
+                    assert(raw_size >= 0);
+                    assert(raw_size <= OSMPBF::max_uncompressed_blob_size);
+                    return osmium::io::detail::zlib_uncompress(pbf_blob.zlib_data(), static_cast<unsigned long>(raw_size));
+                } else if (pbf_blob.has_lzma_data()) {
+                    throw osmium::pbf_error("lzma blobs not implemented");
+                } else {
+                    throw osmium::pbf_error("blob contains no data");
+                }
+            }
+
+            /**
+             * Parse blob as a HeaderBlock.
+             *
+             * @param input_buffer Blob data
+             * @returns Header object
+             * @throws osmium::pbf_error If there was a parsing error
+             */
+            inline osmium::io::Header parse_header_blob(const std::string& input_buffer) {
+                const std::unique_ptr<const std::string> data = unpack_blob(input_buffer);
+
+                OSMPBF::HeaderBlock pbf_header_block;
+                if (!pbf_header_block.ParseFromString(*data)) {
+                    throw osmium::pbf_error("failed to parse HeaderBlock");
+                }
+
+                osmium::io::Header header;
+                for (int i=0; i < pbf_header_block.required_features_size(); ++i) {
+                    const std::string& feature = pbf_header_block.required_features(i);
+
+                    if (feature == "OsmSchema-V0.6") continue;
+                    if (feature == "DenseNodes") {
+                        header.set("pbf_dense_nodes", true);
+                        continue;
+                    }
+                    if (feature == "HistoricalInformation") {
+                        header.set_has_multiple_object_versions(true);
+                        continue;
+                    }
+
+                    throw osmium::pbf_error(std::string("required feature not supported: ") + feature);
+                }
+
+                for (int i=0; i < pbf_header_block.optional_features_size(); ++i) {
+                    const std::string& feature = pbf_header_block.optional_features(i);
+                    header.set("pbf_optional_feature_" + std::to_string(i), feature);
+                }
+
+                if (pbf_header_block.has_writingprogram()) {
+                    header.set("generator", pbf_header_block.writingprogram());
+                }
+
+                if (pbf_header_block.has_bbox()) {
+                    const OSMPBF::HeaderBBox& pbf_bbox = pbf_header_block.bbox();
+                    const int64_t resolution_convert = OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision;
+                    osmium::Box box;
+                    box.extend(osmium::Location(pbf_bbox.left()  / resolution_convert, pbf_bbox.bottom() / resolution_convert));
+                    box.extend(osmium::Location(pbf_bbox.right() / resolution_convert, pbf_bbox.top()    / resolution_convert));
+                    header.add_box(box);
+                }
+
+                if (pbf_header_block.has_osmosis_replication_timestamp()) {
+                    header.set("osmosis_replication_timestamp", osmium::Timestamp(pbf_header_block.osmosis_replication_timestamp()).to_iso());
+                }
+
+                if (pbf_header_block.has_osmosis_replication_sequence_number()) {
+                    header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.osmosis_replication_sequence_number()));
+                }
+
+                if (pbf_header_block.has_osmosis_replication_base_url()) {
+                    header.set("osmosis_replication_base_url", pbf_header_block.osmosis_replication_base_url());
+                }
+
+                return header;
+            }
+
+            class DataBlobParser {
+
+                std::string m_input_buffer;
+                osmium::osm_entity_bits::type m_read_types;
+
+            public:
+
+                DataBlobParser(std::string&& input_buffer, osmium::osm_entity_bits::type read_types) :
+                    m_input_buffer(std::move(input_buffer)),
+                    m_read_types(read_types) {
+                    if (input_buffer.size() > OSMPBF::max_uncompressed_blob_size) {
+                        throw osmium::pbf_error(std::string("invalid blob size: " + std::to_string(input_buffer.size())));
+                    }
+                }
+
+                DataBlobParser(const DataBlobParser& other) :
+                    m_input_buffer(std::move(other.m_input_buffer)),
+                    m_read_types(other.m_read_types) {
+                }
+
+                DataBlobParser& operator=(const DataBlobParser&) = delete;
+
+                osmium::memory::Buffer operator()() {
+                    const std::unique_ptr<const std::string> data = unpack_blob(m_input_buffer);
+                    PBFPrimitiveBlockParser parser(*data, m_read_types);
+                    return parser();
+                }
+
+            }; // class DataBlobParser
+
+        } // namespace detail
+
+    } // namespace io
+
+} // namespace osmium
+
+#endif // OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP

include/osmium/io/detail/pbf_stringtable.hpp

@@ -0,0 +1,204 @@
+#ifndef OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP
+#define OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <algorithm>
+#include <cstdint>
+#include <iterator>
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <osmpbf/osmpbf.h>
+
+#include <osmium/util/cast.hpp>
+
+namespace osmium {
+
+    namespace io {
+
+        namespace detail {
+
+            /**
+             * StringTable management for PBF writer
+             *
+             * All strings are stored as indexes to rows in a StringTable. The StringTable contains
+             * one row for each used string, so strings that are used multiple times need to be
+             * stored only once. The StringTable is sorted by usage-count, so the most often used
+             * string is stored at index 1.
+             */
+            class StringTable {
+
+            public:
+
+                /// type for string IDs (interim and final)
+                typedef uint16_t string_id_type;
+
+            private:
+
+                /**
+                 * this is the struct used to build the StringTable. It is stored as
+                 * the value-part in the strings-map.
+                 *
+                 * when a new string is added to the map, its count is set to 0 and
+                 * the interim_id is set to the current size of the map. This interim_id
+                 * is then stored into the pbf-objects.
+                 *
+                 * before the PrimitiveBlock is serialized, the map is sorted by count
+                 * and stored into the pbf-StringTable. Afterwards the interim-ids are
+                 * mapped to the "real" id in the StringTable.
+                 *
+                 * this way often used strings get lower ids in the StringTable. As the
+                 * protobuf-serializer stores numbers in variable bit-lengths, lower
+                 * IDs means less used space in the resulting file.
+                 */
+                struct string_info {
+
+                    /// number of occurrences of this string
+                    uint16_t count;
+
+                    /// an intermediate-id
+                    string_id_type interim_id;
+
+                }; // struct string_info
+
+                /**
+                 * Interim StringTable, storing all strings that should be written to
+                 * the StringTable once the block is written to disk.
+                 */
+                typedef std::map<std::string, string_info> string2string_info_type;
+                string2string_info_type m_strings;
+
+                /**
+                 * This vector is used to map the interim IDs to real StringTable IDs after
+                 * writing all strings to the StringTable.
+                 */
+                typedef std::vector<string_id_type> interim_id2id_type;
+                interim_id2id_type m_id2id_map;
+
+                size_t m_size = 0;
+
+            public:
+
+                StringTable() {
+                }
+
+                friend bool operator<(const string_info& lhs, const string_info& rhs) {
+                    return lhs.count > rhs.count;
+                }
+
+                /**
+                 * record a string in the interim StringTable if it's missing, otherwise just increase its counter,
+                 * return the interim-id assigned to the string.
+                 */
+                string_id_type record_string(const std::string& string) {
+                    string_info& info = m_strings[string];
+                    if (info.interim_id == 0) {
+                        ++m_size;
+                        info.interim_id = static_cast_with_assert<string_id_type>(m_size);
+                    } else {
+                        info.count++;
+                    }
+                    return info.interim_id;
+                }
+
+                /**
+                 * Sort the interim StringTable and store it to the real protobuf StringTable.
+                 * while storing to the real table, this function fills the id2id_map with
+                 * pairs, mapping the interim-ids to final and real StringTable ids.
+                 *
+                 * Note that the m_strings table is a std::map and as such is sorted lexicographically.
+                 * When the transformation into the sortedby multimap is done, it gets sorted by
+                 * the count. The end result (at least with the glibc standard container/algorithm
+                 * implementation) is that the string table is sorted first by reverse count (ie descending)
+                 * and then by reverse lexicographic order.
+                 */
+                void store_stringtable(OSMPBF::StringTable* st) {
+                    // add empty StringTable entry at index 0
+                    // StringTable index 0 is reserved as delimiter in the densenodes key/value list
+                    // this line also ensures that there's always a valid StringTable
+                    st->add_s("");
+
+                    std::multimap<string_info, std::string> sortedbycount;
+
+                    m_id2id_map.resize(m_size+1);
+
+                    std::transform(m_strings.begin(), m_strings.end(),
+                                std::inserter(sortedbycount, sortedbycount.begin()),
+                                [](const std::pair<std::string, string_info>& p) {
+                                        return std::pair<string_info, std::string>(p.second, p.first);
+                                });
+
+                    string_id_type n=0;
+
+                    for (const auto& mapping : sortedbycount) {
+                        // add the string of the current item to the pbf StringTable
+                        st->add_s(mapping.second);
+
+                        // store the mapping from the interim-id to the real id
+                        m_id2id_map[mapping.first.interim_id] = ++n;
+                    }
+                }
+
+                /**
+                 * Map from an interim ID to a real string ID.
+                 */
+                string_id_type map_string_id(const string_id_type interim_id) const {
+                    return m_id2id_map[interim_id];
+                }
+
+                template <typename T>
+                string_id_type map_string_id(const T interim_id) const {
+                    return map_string_id(static_cast_with_assert<string_id_type>(interim_id));
+                }
+
+                /**
+                 * Clear the stringtable, preparing for the next block.
+                 */
+                void clear() {
+                    m_strings.clear();
+                    m_id2id_map.clear();
+                    m_size = 0;
+                }
+
+            }; // class StringTable
+
+        } // namespace detail
+
+    } // namespace io
+
+} // namespace osmium
+
+#endif // OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP