From afdf8e7b21fbaf597e91d9d8a7542635e60ee9a1 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Tue, 5 Jan 2016 12:00:40 +0100 Subject: [PATCH] Squashed 'third_party/libosmium/' changes from c43f8db..0ff2780 0ff2780 Release v2.5.4 3f1583a Remove workaround for MSVC missing constexpr support. 2a6c80b Rewrite expressions to avoid warnings on MSVC. 9b0602a Make two functions non-constexpr that can't be. ca1e501 Make NodeRef constructor, accessors and comparison ops constexpr. fb48312 Disable some problematic clang_tidy checks. d2d2812 Only do clang_tidy checks on files that are configured to be built. 5af55bb Always use braces after while() and if(). aebb6e3 No else after return. 65a3bf6 Better clang-tidy config. 3a965fa Add typedef Buffer::value_type needed when using std::back_inserter. c20ac26 Add lots of explicits to constructors. 1d12e09 Add clang-tidy make target. 052b1ee Add benchmark programs to files checked with cppcheck. 9ccbb49 Use only "final", not "override final". 0b9df1f User 'override' instead of 'virtual' in overridden functions. 1d0c1c6 Remove unnecessary get() call. 705391b Use consistent namespace closing comment. 25c1f38 Only use "final" on overridden methods, not "override". 8da2553 Setting CMAKE_EXPORT_COMPILE_COMMANDS works only after project() command. 5fec437 Use newest protozero library (v1.2.3). a3d759b Overloaded version of add_tag() with std::pair of strings. 1b4bcf9 Add function to add tag to tag list from existing tag. 7087e62 Update gdalcpp.hpp header to v1.1.1. 0008e8c Update change log. 39fe3a5 Add conversion and comparison operators to Timestamp. e6643a7 Update gdalcpp. 650280f Bugfix: Improved segment intersection function. 1b20597 Make operator bool explicit for (Typed)MemoryMapping. b580b25 Merge pull request #136 from tomhughes/ruby a7c6737 The multipolygon test only needs the ruby interpreter 4a0a9e7 Make conversion from Buffer to bool noexcept. 7cdabbe Set thread name in o5m input format. 6a17a8d Break out computation of thread pool size into function and test it. 1950853 Allow initializing a Timestamp from any integral type. 4103198 Fix link in change log. dc2ed89 Release v2.5.3 a2816b7 Fix end_of_time() Timestamp, add constructor taking std::string&. 3f5eb8e Updated changelog. 9b75c14 Cleanup and better docs for DiffObject and related classes. d0beead Better documentation for NodeRef and NodeRefList classes. 0192292 Cleanup/doc/test osmium::util::Options. b3db055 Use header with forward declarations. 8b3fe16 Improved documentation. cd2ce38 Remove unused typedef Buffer::value_type. 87c9b32 Cleanup and test Buffer::add_buffer() function. e519278 Improved documentation for osmium::memory::Buffer. d721d43 Deprecate set_full_callback(). Better doc for deprecated functions. 58b5fd1 Remove DataFile class which was never used anywhere. 30b806e Simplify some tests. 62958df Update change log. a8a2e68 Merge pull request #134 from zerebubuth/buffer-size-checks 987faab Move some code belonging into there into relations::Collector. da31175 When the buffer isn't big enough, even when empty, to reserve the space that's being requested then it shouldn't reserve it anyway. 5d2f949 Swap the growth flag and any "full" callback as well as all the other members when swapping this buffer object. 1845aa9 The assertions prior to dereference are more strict than the conversion to bool. However, conversion to bool is often used as a predicate for whether dereference is okay, so the behaviour is easier to understand if they match. abdfee1 Remove unused m_want_types in relations collector. 95b387f "Officially" mark two constructors as deprecated. 1d0da5f Release v2.5.2 31c3eaf Copy iterator around less often. 83c93d0 Do not check the write_future for exceptions on every item. 1c805ea Add counting of push() calls on queue in debug mode. aa869c0 Release v2.5.1 212578b Fix documentation of Writer constructor. 0445dd8 Add new header file with forward declarations of commonly used classes. 4b6baac Optionally include external library headers. 712a6d1 Update change log. fc78d04 Update style rules. b590fe3 Unify use of 'typename' in templates. Unify spacing of ellipsis operator. 8c450ac Move osmium/io/overwrite.hpp to writer_options.hpp. 08eed02 Move DEPRECATED macro into compatibility.hpp. b72eb8d Release v2.5.0 5aeba6b Better formatting for invalid timestamps on debug output. 6772413 Add valid() function on Timestamp. 7549b05 Update change log. 84119b0 Add option to fsync files after they are written. db1bd92 Make optional parameters on Writer work in any order. 3482e3e Remove boilerplate. Add explicits to constructors. cf389c6 Rename the wrap() function to the better ensure_cleanup(). 678049e Use reference instead of pointer for decompressor. 7189d28 More consistent use and naming of Function templates. 3d66deb Use const& for parameter that's not changed. 88d65fb Fix warning with a cast. 03e8c9f Fix some misc issues found by cppcheck. 4848676 Options to cppcheck to check everything (--force) and ignore assert. 5f89a8c Add lots of assert() calls to Buffer implementation. c990b43 Use a wrapper function in Writer for error handling. 04d9e3e Refactor of writer to work properly in all error cases. 6daf2d3 Throw when reading from Reader after eof or error. Use io_error everywhere. 713a189 Rename OutputFormat::close() to write_end(). 7905add Refactor Reader/Writer code. c050a05 Make thread_handler class movable. 8e661a2 Make DeltaEncode/Decode more generic and fix signedness issues. da712a9 Fixed a few signedness issues. dc04e67 Avoid possible narrowing conversion. abd44af Make a variable static that should be. 7039fa6 Avoid global variable. 9e9fc0f Add a noreturn attribute. a681a2c Do not pass Timestamp class through forwarding. d37b717 Clean up status handling in Reader and Writer. 3b1f0d8 Add at_end_of_data() helper function to get self-documenting code. bf3cc8c Add add_end_of_data_to_queue() helper function. 94bdd09 Fix test. 531db80 Use valid() on future instead of an extra bool. 7f328b3 Make RVO work for pop() function. c1d726d Add some static_cast_with_assert paranoia checks. 5a064f7 Make sure DeltaDecode/DeltaEncode classes work for all integer types. c69a701 Fix some integer types. 2b2cfc9 Remove extra semicolons at end of function definitions. 2b74aa6 Use workaround for GCC unused variable warning for index::map, too. f7fb94d Different way of supression unused-variable warning. c0813e6 Better handling of threads. 963ff8e Move internal buffer from OutputIterator into Writer. 698d027 Update change log. 24270dd Remove unused variables. 64d6363 Remove unnecessarily fully-qualified name. 5ccacc7 Add support for reading o5m and o5c files. b603904 Fix up includes. 6013a27 Remove pessimizing move. a19e4cf Use queue_wrapper in Reader, too. 74a5174 Refactor thread creation for WriteThread. c480b33 Fix test code. e135597 Do not use promise in two threads at once. 6fa16ca Refactor input format code. 53fc576 More robust implementation of writer/writer_thread. 1285316 Put some queue handling into new wrapper class. a1e6e6f Rename osmium/io/detail/util.hpp to queue_util.hpp. f42d6fc New add_to_queue() helper functions. 3db9b49 Simplify read thread handling. a903561 Move any exception in read thread through queue. ee977cb Add more tests for reader code. 99aaa45 Factor out input handling in classes derived from Parser. 7afa03c Wrap access to m_read_types in InputFormat. f6c5971 Wrap sending to output queue in InputFormat. 26f4170 Consolidate header handling in InputFormat. a0aa3ed Remove unnecessary inline declaration. 894e84a Declare a bunch of destructors noexcept and use consistent comments. 6c49b43 Add hack to append_printf_formatted_string() so it works on Windows. 20c3f20 Remove a move that prevented copy elision. 8192a4c Pull low-level string formatting out of debug output and test it. 190aa46 Move low-level string formatting/encoding functions into own header. 3e35441 Add Option::is_not_false() helper, use and test it. 0a90339 Updated some comments. a44066f Make naming of output format options more consistent and document them. a59b60b Run serialization of PBF blobs in worker threads speeding up PBF writer. 18a739f Remove unused m_file attribute from OutputFormat class. 9b5d3b7 Various output option related cleanups. 36772a0 Consistent ordering of methods in *InputFormat classes. c04a51f Factor out common code in output formats. ebc53d4 Do not use "explicit" on constructors with more than one argument. e1dfcfc Make all destructors in io/detail noexcept. cfd7970 Use consistent handling of output options in all outputs. c4e71f0 Better implementation for output_formatted() and tests for it. 899a061 Extract common code from output formats. b226ae4 Factor out common code from *OutputBlock classes. fe4b287 Cleanup WriteThread class. af421df Consistent naming of queue typedefs. e8253c4 Add missing include. 9f71cd3 Refactor management of read thread into its own class. 4c96e16 Refactor Reader/InputFormat. fa02e6c Refactor input format code. d14ea27 Extract common code from PBF/XMLParser into new Parser class. fe7acd3 Better handling of failures when parsing header. 2e3b6cd Remove unnecessary include of . 3bd18b8 Factor out common code in input format. 2915604 Only get promise and future once, even if header() is called multiple times. cfc980c Make output buffer for XML parser smaller. 4c1ffa7 Orderly shutdown in io::Reader. 8c7aa32 Remember whether the input queue was exhausted. 535bb6a Function call in new thread can be void. e0d5448 Factor out some helper function for queue cleanup. 5a4c6b5 Use std::thread directly for input instead of std::async. c1bdf4f Move common code from InputFormat child classed into base class. 89caa6e Report failures in input_format through the queue. 8f4d300 Rename (m_)queue to (m_)output_queue. 8fc1f5b Reorder XMLParser class making check_attributes() private. df381d7 Make sure we always send end-of-file from PBF parser. 1a178b0 Factor out construction of PBFDataBlobDecoder. cb34f76 Factor out read_from_input_queue_with_check() method. 21b51cc Factor out parse_data_blobs() method. dc957a8 Make some variables const. 1c2812c Make methods private that don't need to be public. 81e5625 Refactor out parse_header_blob() function. f9e5760 Fix test: New signature of XMLParser constructor. 43746d3 Add copy constructor to PBFParser. 8524780 Rename variables and other changes for clarity. 3e9627b Removed now superfluous parameters from InputFormat class. 7eac5cf Simplify input format class. 3ea2ace Set max queue size only in one place. 77ab086 Formatting. ceee837 Use std::async instead of "raw" std::thread for pbf input. 6cafb45 Move and rename PromiseKeeper class: Now in thread/util.hpp. 06eff29 Set thread name for xml parser thread. bd485cd Refactoring of threading code for input. fc03bf6 Make sure (de)compression classes clean up properly. 27af4ea Use special function to shut down pool workers instead of an atomic. 84297b3 Bugfix: auto and std::minmax() don't mix well. 597ecc4 Always use std::swap() in the idiomatic form. 10dd14f Remove threading test that fails when machine is too busy. 2072786 Use reserve() to spead up dumping indexes. 66a344b Declare some index functions noexcept, especially destructors. a0586da Use map::find() instead of awkward try-catch block. d38a7f1 Do not run make tasks in parallel. cd33daa Do not use clever YAML aliases, instead copy dependencies explicitely 4ad6e43 Integrate more compiler and os versions b2c519b Check return code of close() system call and throw. 369057b Update protozero to current 1.2.2. d1db14b Collect debug output options into struct. 54667dd More consistent debug output of way nodes, relation members and changeset comments. 67e1513 Add some paranoia checks to xml parser. 69de191 Refactoring in xml reader: New function check_attributes(). c67f3f3 Add support for changeset discussions (comments). 9c5531c Merge pull request #121 from DerDakon/cmake-find-no-components 44be1a7 Add helper functions to make input iterator ranges and output iterators. 76e2b91 Merge pull request #130 from alex85k/master 5a4fa6b remove assertion messageboxes in tests on Windows bac5a77 Updated change log. 7e7bba4 Updated included protozero library to 1.2.0. 1ae370d Merge pull request #122 from zerebubuth/pbf-decode-non-visible-node-locations 5897468 Merge branch 'master' of github.com:osmcode/libosmium 7f2de1b Bugfix: Delta iterator handling. 65d31e9 Try ; as cmake list separator. 3a9dbc2 Add PBF libraries, now that the test reads PBF too. Thanks @tomhughes for pointing this out. 9a22ea1 Add test case for reading deleted / non-visible nodes in history files. 36098a8 Decode lat/lon even for non-visible nodes. 8279fd1 kick off AppVeyor to test new binary deps package with gdal200 c8244f7 FindOsmium: prevent errors in list(REMOVE_DUPLICATES) when no components are requested a02806a Use https URL to travis. a1b7015 Fix some includes. 468e4d8 Remove pessimizing std::move. 427d2e0 Do iwyu check on header files in alphabetical order. be9a996 Release v2.4.1 95a3bc8 Fixed CRC calculation of tags and changesets. 4e157e3 Release v2.4.0 3da68f0 Fixed setting of binary mode on Windows. 81aa057 Use binary mode for memory mapped file on Windows. 986cb7e Set stdout to binary mode on windows before writing to files. 27d02eb Bugfix: Do not dereference end iterator. e96eeaf Updated change log. 64a55ce FindOsmium: let FPHSA handle all the additionally required things e152057 FindOsmium: pass the proper module name to FPHSA a4acce3 Remove restriction on master branch in appveyor config. 10c8265 FindOsmium: simplify the fallback code for sparsetable::size_type 190ed47 remove the correct include dir from OSMIUM_INCLUDE_DIRS aaa99c1 avoid that FindOsmium finds a random include dir 6406010 Add a magic define fixing a boost problem. 2fa6674 Remove superfluous file paths from cmake config. e081a51 Merge pull request #114 from DerDakon/do-not-cache-version 20e1a24 Use external gdalcpp wrapper for compatibility with GDAL 2. 3b7cc86 Fix initialization order in DeltaEncodeIterator. 0954b0f Fix possibly uninitialized variable. f081942 Take byte swap functions out of CRC class. e085aae Fix byte swap, add test cases for crc. e648b62 Merge pull request #116 from DerDakon/yml-simplify 7912897 properly put bzip2 library in the CMake cache e0ea72b use less variables when defining the test environment cf8ff6c do not cache the version string 38234cd Remove pragmas disabling warnings from gdal includes. 82d8c30 Include headers of external libraries as "system libraries". f721b86 Update protozero to version 1.1.0. a29ef82 Add some magic to enable folding on travis output. 18b2418 Removed toogr examples. They are in their own repository now. 89c8220 AppVeyor: 1st try with VS2015 93a1626 Added recent changes to change log. ce4b45e Bugfix: Program hanging when opening unknown file type. 06ad6ef Rename add_string() to store_in_stringtable() and use right return type. 869058d Add explicit conversion that always works. 0b28f2c Add missing check in TagListBuilder add_tag() overload. 51fa9c0 Check in builder that key/value of a tag is not too long. 9b1da20 Check that string table isn't overflowing. 2c732c6 Add some extra paranoia checks and type conversions to pbf writer. f92096a Fix integer size. a47ddb4 Force conversion to smaller int type, because we know it must fit. f150ff1 Rename variable that was hiding parameter name. ab92064 Use correct size_t as return type. 2f2bf68 Check that roles are no longer than max allowed string length. 4a7df68 Check strings for max length in PBF input. e4b8bb0 Explicit conversion to bool. d18352d Make conversion from double to integer explicit. git-subtree-dir: third_party/libosmium git-subtree-split: 0ff278001f6e0bc79040add736452bef3aa4ff06 --- .travis.yml | 179 ++++- CHANGELOG.md | 137 +++- CMakeLists.txt | 156 ++++- CONTRIBUTING.md | 45 +- README.md | 14 +- appveyor.yml | 31 +- cmake/FindOsmium.cmake | 64 +- cmake/iwyu.sh | 2 +- examples/CMakeLists.txt | 27 +- examples/osmium_filter_discussions.cpp | 72 ++ examples/osmium_toogr.cpp | 244 ------- examples/osmium_toogr2.cpp | 331 --------- examples/osmium_toogr2_exp.cpp | 305 --------- include/gdalcpp.hpp | 406 +++++++++++ include/osmium/area/assembler.hpp | 4 + .../osmium/area/detail/node_ref_segment.hpp | 68 +- include/osmium/area/detail/proto_ring.hpp | 9 +- include/osmium/area/detail/segment_list.hpp | 2 + .../osmium/area/multipolygon_collector.hpp | 34 +- .../area/problem_reporter_exception.hpp | 2 +- include/osmium/area/problem_reporter_ogr.hpp | 134 +--- .../osmium/area/problem_reporter_stream.hpp | 2 +- include/osmium/builder/builder.hpp | 4 +- include/osmium/builder/osm_object_builder.hpp | 162 ++++- include/osmium/diff_handler.hpp | 3 +- include/osmium/diff_iterator.hpp | 54 +- include/osmium/diff_visitor.hpp | 14 +- include/osmium/dynamic_handler.hpp | 35 +- include/osmium/experimental/flex_reader.hpp | 18 +- include/osmium/fwd.hpp | 70 ++ include/osmium/geom/coordinates.hpp | 1 - include/osmium/geom/factory.hpp | 25 +- include/osmium/geom/geojson.hpp | 12 +- include/osmium/geom/geos.hpp | 2 +- include/osmium/geom/ogr.hpp | 32 +- include/osmium/geom/rapid_geojson.hpp | 4 +- include/osmium/geom/tile.hpp | 6 +- include/osmium/geom/wkb.hpp | 10 +- include/osmium/geom/wkt.hpp | 12 +- include/osmium/handler.hpp | 19 +- include/osmium/handler/chain.hpp | 10 +- .../handler/node_locations_for_ways.hpp | 2 +- include/osmium/index/bool_vector.hpp | 4 +- .../index/detail/create_map_with_fd.hpp | 17 +- .../osmium/index/detail/mmap_vector_anon.hpp | 2 + .../osmium/index/detail/mmap_vector_base.hpp | 6 +- .../osmium/index/detail/mmap_vector_file.hpp | 8 +- include/osmium/index/detail/vector_map.hpp | 36 +- .../osmium/index/detail/vector_multimap.hpp | 14 +- include/osmium/index/index.hpp | 2 +- include/osmium/index/map.hpp | 21 +- include/osmium/index/map/dummy.hpp | 12 +- include/osmium/index/map/sparse_mem_map.hpp | 23 +- include/osmium/index/map/sparse_mem_table.hpp | 15 +- include/osmium/index/multimap.hpp | 2 +- include/osmium/index/multimap/hybrid.hpp | 24 +- .../index/multimap/sparse_mem_multimap.hpp | 14 +- include/osmium/io/any_input.hpp | 1 + include/osmium/io/bzip2_compression.hpp | 79 ++- include/osmium/io/compression.hpp | 101 ++- .../osmium/io/detail/debug_output_format.hpp | 235 +++---- include/osmium/io/detail/input_format.hpp | 155 +++-- include/osmium/io/detail/o5m_input_format.hpp | 636 ++++++++++++++++++ .../osmium/io/detail/opl_output_format.hpp | 133 ++-- include/osmium/io/detail/output_format.hpp | 66 +- include/osmium/io/detail/pbf.hpp | 9 +- include/osmium/io/detail/pbf_decoder.hpp | 55 +- include/osmium/io/detail/pbf_input_format.hpp | 189 ++---- .../osmium/io/detail/pbf_output_format.hpp | 332 +++++---- include/osmium/io/detail/queue_util.hpp | 157 +++++ include/osmium/io/detail/read_thread.hpp | 95 ++- include/osmium/io/detail/read_write.hpp | 70 +- include/osmium/io/detail/string_table.hpp | 31 +- include/osmium/io/detail/string_util.hpp | 206 ++++++ include/osmium/io/detail/write_thread.hpp | 59 +- include/osmium/io/detail/xml_input_format.hpp | 562 +++++++--------- .../osmium/io/detail/xml_output_format.hpp | 273 ++++---- include/osmium/io/detail/zlib.hpp | 5 +- include/osmium/io/error.hpp | 16 +- include/osmium/io/file.hpp | 8 +- include/osmium/io/gzip_compression.hpp | 72 +- include/osmium/io/input_iterator.hpp | 40 +- include/osmium/io/o5m_input.hpp | 45 ++ include/osmium/io/output_iterator.hpp | 68 +- include/osmium/io/overwrite.hpp | 17 +- include/osmium/io/reader.hpp | 162 +++-- include/osmium/io/writer.hpp | 281 ++++++-- include/osmium/io/writer_options.hpp | 60 ++ include/osmium/memory/buffer.hpp | 285 ++++++-- include/osmium/memory/collection.hpp | 6 +- include/osmium/memory/item.hpp | 6 +- include/osmium/memory/item_iterator.hpp | 22 +- include/osmium/object_pointer_collection.hpp | 2 +- include/osmium/osm/area.hpp | 3 +- include/osmium/osm/box.hpp | 8 +- include/osmium/osm/changeset.hpp | 135 +++- include/osmium/osm/crc.hpp | 53 +- include/osmium/osm/diff_object.hpp | 147 +++- include/osmium/osm/entity.hpp | 2 +- include/osmium/osm/item_type.hpp | 17 +- include/osmium/osm/location.hpp | 18 +- include/osmium/osm/node.hpp | 4 +- include/osmium/osm/node_ref.hpp | 100 ++- include/osmium/osm/node_ref_list.hpp | 33 +- include/osmium/osm/object.hpp | 18 +- include/osmium/osm/relation.hpp | 12 +- include/osmium/osm/segment.hpp | 8 +- include/osmium/osm/tag.hpp | 5 +- include/osmium/osm/timestamp.hpp | 124 +++- include/osmium/osm/types.hpp | 4 + include/osmium/osm/types_from_string.hpp | 83 ++- include/osmium/osm/way.hpp | 4 +- include/osmium/relations/collector.hpp | 177 ++--- .../osmium/relations/detail/member_meta.hpp | 2 +- include/osmium/tags/filter.hpp | 8 +- include/osmium/tags/taglist.hpp | 6 +- include/osmium/thread/function_wrapper.hpp | 30 +- include/osmium/thread/pool.hpp | 57 +- include/osmium/thread/queue.hpp | 11 +- include/osmium/thread/util.hpp | 37 +- include/osmium/util/compatibility.hpp | 12 +- include/osmium/util/config.hpp | 2 +- include/osmium/util/data_file.hpp | 194 ------ include/osmium/util/delta.hpp | 71 +- include/osmium/util/double.hpp | 8 +- include/osmium/util/file.hpp | 6 +- include/osmium/util/memory_mapping.hpp | 40 +- include/osmium/util/options.hpp | 76 ++- include/osmium/visitor.hpp | 38 +- include/protozero/byteswap.hpp | 34 +- include/protozero/config.hpp | 57 ++ include/protozero/pbf_builder.hpp | 28 +- include/protozero/pbf_message.hpp | 44 ++ include/protozero/pbf_reader.hpp | 46 +- include/protozero/pbf_writer.hpp | 17 +- include/protozero/varint.hpp | 4 - include/protozero/version.hpp | 22 + scripts/travis_install.sh | 20 - scripts/travis_script.sh | 29 - test/CMakeLists.txt | 10 +- test/data-tests/CMakeLists.txt | 6 +- test/data-tests/testdata-multipolygon.cpp | 171 +---- test/data-tests/testdata-overview.cpp | 154 +---- test/data-tests/testdata-xml.cpp | 35 +- test/t/area/test_node_ref_segment.cpp | 14 + test/t/basic/test_changeset.cpp | 62 +- test/t/basic/test_crc.cpp | 21 + test/t/basic/test_node.cpp | 6 +- test/t/basic/test_object_comparisons.cpp | 44 +- test/t/basic/test_relation.cpp | 15 +- test/t/basic/test_timestamp.cpp | 27 +- test/t/basic/test_way.cpp | 4 +- test/t/buffer/test_buffer_basics.cpp | 34 + test/t/buffer/test_buffer_node.cpp | 78 ++- test/t/geom/test_tile_data.hpp | 4 +- test/t/io/deleted_nodes.osh | 5 + test/t/io/deleted_nodes.osh.pbf | Bin 0 -> 189 bytes test/t/io/test_output_utils.cpp | 153 +++++ test/t/io/test_reader.cpp | 105 ++- .../test_reader_with_mock_decompression.cpp | 145 ++++ test/t/io/test_reader_with_mock_parser.cpp | 123 ++++ test/t/io/test_string_table.cpp | 6 +- test/t/io/test_writer.cpp | 117 ++++ .../io/test_writer_with_mock_compression.cpp | 99 +++ test/t/io/test_writer_with_mock_encoder.cpp | 105 +++ test/t/tags/test_tag_list.cpp | 11 + test/t/thread/test_pool.cpp | 62 +- test/t/util/test_data_file.cpp | 81 --- test/t/util/test_delta.cpp | 34 +- test/t/util/test_file.cpp | 2 + test/t/util/test_options.cpp | 49 +- 171 files changed, 7150 insertions(+), 3988 deletions(-) create mode 100644 examples/osmium_filter_discussions.cpp delete mode 100644 examples/osmium_toogr.cpp delete mode 100644 examples/osmium_toogr2.cpp delete mode 100644 examples/osmium_toogr2_exp.cpp create mode 100644 include/gdalcpp.hpp create mode 100644 include/osmium/fwd.hpp create mode 100644 include/osmium/io/detail/o5m_input_format.hpp create mode 100644 include/osmium/io/detail/queue_util.hpp create mode 100644 include/osmium/io/detail/string_util.hpp create mode 100644 include/osmium/io/o5m_input.hpp create mode 100644 include/osmium/io/writer_options.hpp delete mode 100644 include/osmium/util/data_file.hpp create mode 100644 include/protozero/config.hpp create mode 100644 include/protozero/version.hpp delete mode 100755 scripts/travis_install.sh delete mode 100755 scripts/travis_script.sh create mode 100644 test/t/buffer/test_buffer_basics.cpp create mode 100644 test/t/io/deleted_nodes.osh create mode 100644 test/t/io/deleted_nodes.osh.pbf create mode 100644 test/t/io/test_output_utils.cpp create mode 100644 test/t/io/test_reader_with_mock_decompression.cpp create mode 100644 test/t/io/test_reader_with_mock_parser.cpp create mode 100644 test/t/io/test_writer.cpp create mode 100644 test/t/io/test_writer_with_mock_compression.cpp create mode 100644 test/t/io/test_writer_with_mock_encoder.cpp delete mode 100644 test/t/util/test_data_file.cpp diff --git a/.travis.yml b/.travis.yml index 6ebdd7167..ac0d270e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,46 +9,151 @@ language: cpp sudo: false matrix: - include: - - os: linux - compiler: clang - env: BUILD_TYPE=Dev - - os: linux - compiler: clang - env: BUILD_TYPE=Release - - os: linux - compiler: gcc - env: BUILD_TYPE=Dev - - os: linux - compiler: gcc - env: BUILD_TYPE=Release - - os: osx - compiler: clang - env: BUILD_TYPE=Dev - - os: osx - compiler: clang - env: BUILD_TYPE=Release + include: + + # 1/ Linux Clang Builds + - os: linux + compiler: clang + addons: + apt: + sources: ['llvm-toolchain-precise-3.5', 'ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['clang-3.5', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='clang++-3.5' BUILD_TYPE='Release' + + - os: linux + compiler: clang + addons: + apt: + sources: ['llvm-toolchain-precise-3.5', 'ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['clang-3.5', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='clang++-3.5' BUILD_TYPE='Dev' + + + - os: linux + compiler: clang + addons: + apt: + sources: ['llvm-toolchain-precise-3.6', 'ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['clang-3.6', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='clang++-3.6' BUILD_TYPE='Release' + + - os: linux + compiler: clang + addons: + apt: + sources: ['llvm-toolchain-precise-3.6', 'ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['clang-3.6', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='clang++-3.6' BUILD_TYPE='Dev' + + + - os: linux + compiler: clang + addons: + apt: + sources: ['llvm-toolchain-precise-3.7', 'ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['clang-3.7', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='clang++-3.7' BUILD_TYPE='Release' + + - os: linux + compiler: clang + addons: + apt: + sources: ['llvm-toolchain-precise-3.7', 'ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['clang-3.7', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='clang++-3.7' BUILD_TYPE='Dev' + + + # 2/ Linux GCC Builds + - os: linux + compiler: gcc + addons: + apt: + sources: ['ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['g++-4.8', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='g++-4.8' COMPILER_FLAGS='-Wno-return-type' BUILD_TYPE='Release' + + - os: linux + compiler: gcc + addons: + apt: + sources: ['ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['g++-4.8', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='g++-4.8' COMPILER_FLAGS='-Wno-return-type' BUILD_TYPE='Dev' + + + - os: linux + compiler: gcc + addons: + apt: + sources: ['ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['g++-4.9', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='g++-4.9' BUILD_TYPE='Release' + + - os: linux + compiler: gcc + addons: + apt: + sources: ['ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['g++-4.9', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='g++-4.9' BUILD_TYPE='Dev' + + + - os: linux + compiler: gcc + addons: + apt: + sources: ['ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['g++-5', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='g++-5' BUILD_TYPE='Release' + + - os: linux + compiler: gcc + addons: + apt: + sources: ['ubuntu-toolchain-r-test', 'boost-latest'] + packages: ['g++-5', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin'] + env: COMPILER='g++-5' BUILD_TYPE='Dev' + + + # 3/ OSX Clang Builds + - os: osx + osx_image: xcode6.4 + compiler: clang + env: COMPILER='clang++' BUILD_TYPE='Dev' + + - os: osx + osx_image: xcode6.4 + compiler: clang + env: COMPILER='clang++' BUILD_TYPE='Release' + + + - os: osx + osx_image: xcode7 + compiler: clang + env: COMPILER='clang++' BUILD_TYPE='Dev' + + - os: osx + osx_image: xcode7 + compiler: clang + env: COMPILER='clang++' BUILD_TYPE='Release' -# http://docs.travis-ci.com/user/apt/ -addons: - apt: - sources: - - boost-latest - - ubuntu-toolchain-r-test - packages: - - g++-4.8 - - gcc-4.8 - - libboost1.55-dev - - libboost-program-options1.55-dev - - libgdal-dev - - libgeos++-dev - - libproj-dev - - libsparsehash-dev - - spatialite-bin install: - - scripts/travis_install.sh + - DEPS_DIR="${TRAVIS_BUILD_DIR}/deps" + - mkdir -p ${DEPS_DIR} && cd ${DEPS_DIR} + - git clone --quiet --depth 1 https://github.com/osmcode/osm-testdata.git + - | + if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then + brew remove gdal + brew install cmake boost google-sparsehash gdal + fi + +before_script: + - cd ${TRAVIS_BUILD_DIR} + - mkdir build && cd build + - CXX=${COMPILER} CXXFLAGS=${COMPILER_FLAGS} cmake -LA .. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DOSM_TESTDATA="${TRAVIS_BUILD_DIR}/deps/osm-testdata" script: - - scripts/travis_script.sh + - make VERBOSE=1 + - ctest --output-on-failure diff --git a/CHANGELOG.md b/CHANGELOG.md index 22eb06aac..e9377b656 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,132 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Fixed + +## [2.5.4] - 2015-12-03 + +### Changed + +- Included gdalcpp.hpp header was updated to version 1.1.1. +- Included protozero library was updated to version 1.2.3. +- Workarounds for missing constexpr support in Visual Studio removed. All + constexpr features we need are supported now. +- Some code cleanup after running clang-tidy on the code. +- Re-added `Buffer::value_type` typedef. Turns out it is needed when using + `std::back_inserter` on the Buffer. + +### Fixed + +- Bugs with Timestamp code on 32 bit platforms. This necessitated + some changes in Timestamp which might lead to changes in user + code. +- Bug in segment intersection code (which appeared on i686 platform). + + +## [2.5.3] - 2015-11-17 + +### Added + +- `osmium::make_diff_iterator()` helper function. + +### Changed + +- Deprecated `osmium::Buffer::set_full_callback()`. +- Removed DataFile class which was never used anywhere. +- Removed unused and obscure `Buffer::value_type` typedef. + +### Fixed + +- Possible overrun in Buffer when using the full-callback. +- Incorrect swapping of Buffer. + + +## [2.5.2] - 2015-11-06 + +# Fixed + +- Writing data through an OutputIterator was extremly slow due to + lock contention. + + +## [2.5.1] - 2015-11-05 + +### Added + +- Header `osmium/fwd.hpp` with forward declarations of the most commonly + used Osmium classes. + +### Changed + +- Moved `osmium/io/overwrite.hpp` to `osmium/io/writer_options.hpp` + If you still include the old file, you'll get a warning. + + +## [2.5.0] - 2015-11-04 + +### Added + +- Helper functions to make input iterator ranges and output iterators. +- Add support for reading o5m and o5c files. +- Option for osmium::io::Writer to fsync file after writing. +- Lots of internal asserts() and other robustness checks. + +### Changed + +- Updated included protozero library to version 1.2.0. +- Complete overhaul of the I/O system making it much more robust against + wrong data and failures during I/O operations. +- Speed up PBF writing by running parts of it in parallel. +- OutputIterator doesn't hold an internal buffer any more, but it uses + one in Writer. Calling flush() on the OutputIterator isn't needed any + more. +- Reader now throws when trying to read after eof or an error. +- I/O functions that used to throw std::runtime_error now throw + osmium::io_error or derived. +- Optional parameters on osmium::io::Writer now work in any order. + +### Fixed + +- PBF reader now decodes locations of invisible nodes properly. +- Invalid Delta encode iterator dereference. +- Lots of includes fixed to include (only) what's used. +- Dangling reference in area assembly code. + + +## [2.4.1] - 2015-08-29 + +### Fixed + +- CRC calculation of tags and changesets. + + +## [2.4.0] - 2015-08-29 + +### Added + +- Checks that user names, member roles and tag keys and values are not longer + than 256 * 4 bytes. That is the maximum length 256 Unicode characters + can have in UTF-8 encoding. +- Support for GDAL 2. GDAL 1 still works. + +### Changed + +- Improved CMake build scripts. +- Updated internal version of Protozero to 1.1.0. +- Removed `toogr*` examples. They are in their own repository now. + See https://github.com/osmcode/osm-gis-export. +- Files about to be memory-mapped (for instance index files) are now set + to binary mode on Windows so the application doesn't have to do this. + +### Fixed + +- Hanging program when trying to open file with an unknown file format. +- Building problems with old boost versions. +- Initialization errors in PBF writer. +- Bug in byte swap code. +- Output on Windows now always uses binary mode, even when writing to + stdout, so OSM xml and opl files always use LF line endings. + + ## [2.3.0] - 2015-08-18 ### Added @@ -108,8 +234,15 @@ This project adheres to [Semantic Versioning](http://semver.org/). Doxygen (up to version 1.8.8). This version contains a workaround to fix this. -[unreleased]: https://github.com/osmcode/libosmium/compare/v2.3.0...HEAD -[2.3.0]: https://github.com/osmcode/libosmium/compare/v2.3.0...v2.3.0 +[unreleased]: https://github.com/osmcode/libosmium/compare/v2.5.4...HEAD +[2.5.4]: https://github.com/osmcode/libosmium/compare/v2.5.3...v2.5.4 +[2.5.3]: https://github.com/osmcode/libosmium/compare/v2.5.2...v2.5.3 +[2.5.2]: https://github.com/osmcode/libosmium/compare/v2.5.1...v2.5.2 +[2.5.1]: https://github.com/osmcode/libosmium/compare/v2.5.0...v2.5.1 +[2.5.0]: https://github.com/osmcode/libosmium/compare/v2.4.1...v2.5.0 +[2.4.1]: https://github.com/osmcode/libosmium/compare/v2.4.0...v2.4.1 +[2.4.0]: https://github.com/osmcode/libosmium/compare/v2.3.0...v2.4.0 +[2.3.0]: https://github.com/osmcode/libosmium/compare/v2.2.0...v2.3.0 [2.2.0]: https://github.com/osmcode/libosmium/compare/v2.1.0...v2.2.0 [2.1.0]: https://github.com/osmcode/libosmium/compare/v2.0.0...v2.1.0 diff --git a/CMakeLists.txt b/CMakeLists.txt index fba967a4e..0764915ea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,8 +9,6 @@ cmake_minimum_required(VERSION 2.8 FATAL_ERROR) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - #----------------------------------------------------------------------------- # @@ -26,13 +24,13 @@ set(CMAKE_CONFIGURATION_TYPES "Debug;Release;RelWithDebInfo;MinSizeRel;Dev;Cover project(libosmium) set(LIBOSMIUM_VERSION_MAJOR 2) -set(LIBOSMIUM_VERSION_MINOR 3) -set(LIBOSMIUM_VERSION_PATCH 0) +set(LIBOSMIUM_VERSION_MINOR 5) +set(LIBOSMIUM_VERSION_PATCH 4) set(LIBOSMIUM_VERSION - "${LIBOSMIUM_VERSION_MAJOR}.${LIBOSMIUM_VERSION_MINOR}.${LIBOSMIUM_VERSION_PATCH}" - CACHE STRING - "Libosmium version") + "${LIBOSMIUM_VERSION_MAJOR}.${LIBOSMIUM_VERSION_MINOR}.${LIBOSMIUM_VERSION_PATCH}") + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) #----------------------------------------------------------------------------- @@ -56,6 +54,10 @@ option(BUILD_HEADERS "compile every header file on its own" ${dev_build}) option(BUILD_BENCHMARKS "compile benchmark programs" ${dev_build}) option(BUILD_DATA_TESTS "compile data tests, please run them with ctest" ${dev_build}) +option(INSTALL_GDALCPP "also install gdalcpp headers" OFF) +option(INSTALL_PROTOZERO "also install protozero headers" OFF) +option(INSTALL_UTFCPP "also install utfcpp headers" OFF) + #----------------------------------------------------------------------------- # @@ -118,29 +120,39 @@ find_package(Boost 1.38) mark_as_advanced(CLEAR BOOST_ROOT) if(Boost_FOUND) - include_directories(${Boost_INCLUDE_DIRS}) + include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) else() set(BOOST_ROOT "NOT FOUND: please choose" CACHE PATH "") message(FATAL_ERROR "PLEASE, specify the directory where the Boost library is installed in BOOST_ROOT") endif() -set(OSMIUM_INCLUDE_DIR include) +# set OSMIUM_INCLUDE_DIR so FindOsmium will not set anything different +set(OSMIUM_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include") + +include_directories(${OSMIUM_INCLUDE_DIR}) + find_package(Osmium COMPONENTS io gdal geos proj sparsehash) -include_directories(${OSMIUM_INCLUDE_DIRS}) + +# The find_package put the directory where it found the libosmium includes +# into OSMIUM_INCLUDE_DIRS. We remove it again, because we want to make +# sure to use our own include directory already set up above. +list(FIND OSMIUM_INCLUDE_DIRS "${OSMIUM_INCLUDE_DIR}" _own_index) +list(REMOVE_AT OSMIUM_INCLUDE_DIRS ${_own_index}) +set(_own_index) + +include_directories(SYSTEM ${OSMIUM_INCLUDE_DIRS}) if(MSVC) find_path(GETOPT_INCLUDE_DIR getopt.h) find_library(GETOPT_LIBRARY NAMES wingetopt) if(GETOPT_INCLUDE_DIR AND GETOPT_LIBRARY) - include_directories(${GETOPT_INCLUDE_DIR}) + include_directories(SYSTEM ${GETOPT_INCLUDE_DIR}) list(APPEND OSMIUM_LIBRARIES ${GETOPT_LIBRARY}) else() set(GETOPT_MISSING 1) endif() endif() -include_directories(include) - #----------------------------------------------------------------------------- # @@ -205,6 +217,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Dev") add_definitions(-Werror) endif() add_definitions(${OSMIUM_WARNING_OPTIONS}) +# add_definitions(${OSMIUM_WARNING_OPTIONS} ${OSMIUM_DRACONIC_CLANG_OPTIONS} -Wno-documentation -Wno-format-nonliteral -Wno-deprecated -Wno-covered-switch-default -Wno-shadow) endif() # Force RelWithDebInfo build type if none was given @@ -256,19 +269,21 @@ find_program(CPPCHECK cppcheck) if(CPPCHECK) message(STATUS "Looking for cppcheck - found") set(CPPCHECK_OPTIONS - --enable=warning,style,performance,portability,information,missingInclude) + --enable=warning,style,performance,portability,information,missingInclude --force -Uassert) # cpp doesn't find system includes for some reason, suppress that report set(CPPCHECK_OPTIONS ${CPPCHECK_OPTIONS} --suppress=missingIncludeSystem) file(GLOB_RECURSE ALL_INCLUDES include/osmium/*.hpp) file(GLOB ALL_EXAMPLES examples/*.cpp) + file(GLOB ALL_BENCHMARKS benchmarks/*.cpp) file(GLOB ALL_UNIT_TESTS test/t/*/test_*.cpp) file(GLOB ALL_DATA_TESTS test/data-tests/*.cpp) if(Osmium_DEBUG) message(STATUS "Checking includes : ${ALL_INCLUDES}") message(STATUS "Checking example code : ${ALL_EXAMPLES}") + message(STATUS "Checking benchmarks : ${ALL_BENCHMARKS}") message(STATUS "Checking unit test code: ${ALL_UNIT_TESTS}") message(STATUS "Checking data test code: ${ALL_DATA_TESTS}") endif() @@ -276,6 +291,7 @@ if(CPPCHECK) set(CPPCHECK_FILES ${ALL_INCLUDES} ${ALL_EXAMPLES} + ${ALL_BENCHMARKS} ${ALL_UNIT_TESTS} ${ALL_DATA_TESTS}) @@ -288,7 +304,7 @@ if(CPPCHECK) else() message(STATUS "Looking for cppcheck - not found") message(STATUS " Build target 'cppcheck' will not be available.") -endif(CPPCHECK) +endif() #----------------------------------------------------------------------------- @@ -345,11 +361,115 @@ if(BUILD_HEADERS) endforeach() endif() + +#----------------------------------------------------------------------------- +# +# Optional "clang-tidy" target +# +#----------------------------------------------------------------------------- +message(STATUS "Looking for clang-tidy") +find_program(CLANG_TIDY NAMES clang-tidy clang-tidy-3.9 clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy-3.5) + +if(CLANG_TIDY) + message(STATUS "Looking for clang-tidy - found") + + if(BUILD_EXAMPLES) + file(GLOB CT_ALL_EXAMPLES examples/*.cpp) + endif() + + if(BUILD_TESTING) + file(GLOB CT_ALL_UNIT_TESTS test/t/*/test_*.cpp) + endif() + + if(BUILD_HEADERS) + file(GLOB_RECURSE CT_ALL_INCLUDES ${CMAKE_BINARY_DIR}/header_check/osmium__*.cpp) + endif() + + if(BUILD_BENCHMARKS) + file(GLOB CT_ALL_BENCHMARKS benchmarks/*.cpp) + endif() + + if(BUILD_DATA_TESTS) + file(GLOB CT_ALL_DATA_TESTS test/data-tests/*.cpp) + endif() + + if(Osmium_DEBUG) + message(STATUS "Checking example code : ${CT_ALL_EXAMPLES}") + message(STATUS "Checking unit test code: ${CT_ALL_UNIT_TESTS}") + message(STATUS "Checking includes : ${CT_ALL_INCLUDES}") + message(STATUS "Checking benchmarks : ${CT_ALL_BENCHMARKS}") + message(STATUS "Checking data test code: ${CT_ALL_DATA_TESTS}") + endif() + + set(CT_CHECK_FILES + ${CT_ALL_EXAMPLES} + ${CT_ALL_UNIT_TESTS} + ${CT_ALL_INCLUDES} + ${CT_ALL_BENCHMARKS} + ${CT_ALL_DATA_TESTS}) + + # For a list of check options, see: + # http://clang.llvm.org/extra/clang-tidy/checks/list.html + + list(APPEND CT_CHECKS "cert-*" + "-cert-err60-cpp") # even the std lib doesn't do this + + # disabled, because it is slow +# list(APPEND CT_CHECKS "clang-analyzer-*") + + list(APPEND CT_CHECKS "google-*" + "-google-explicit-constructor" + "-google-readability-casting" + "-google-readability-function") + + list(APPEND CT_CHECKS "llvm-*" + "-llvm-include-order") + + list(APPEND CT_CHECKS "misc-*" + "-misc-argument-comment") + + list(APPEND CT_CHECKS "modernize-*") + + list(APPEND CT_CHECKS "readability-*" + "-readability-identifier-naming" + "-readability-named-parameter") + + string(REPLACE ";" "," ALL_CHECKS "${CT_CHECKS}") + + add_custom_target(clang-tidy + ${CLANG_TIDY} + -p ${CMAKE_BINARY_DIR} + -header-filter='include/osmium/.*' + -checks="${ALL_CHECKS}" + ${CT_CHECK_FILES} + ) +else() + message(STATUS "Looking for clang-tidy - not found") + message(STATUS " Build target 'clang-tidy' will not be available.") +endif() + +#----------------------------------------------------------------------------- +# +# Installation +# +# External libraries are only installed if the options are set in case they +# are installed from somewhere else. +# +#----------------------------------------------------------------------------- install(DIRECTORY include/osmium DESTINATION include) -# We only have a copy of this file so we can use older boost versions which -# don't have it. We probably don't want to install it. -#install(FILES include/boost_unicode_iterator.hpp DESTINATION include) +if(INSTALL_GDALCPP) + install(include/gdalcpp.hpp DESTINATION include) +endif() + +if(INSTALL_PROTOZERO) + install(DIRECTORY include/protozero DESTINATION include) +endif() + +if(INSTALL_UTFCPP) + install(include/utf8.hpp DESTINATION include) + install(DIRECTORY include/utf8 DESTINATION include) +endif() #----------------------------------------------------------------------------- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 323c84744..1064b94de 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,22 +36,21 @@ different. * Class names begin with uppercase chars and use CamelCase. Smaller helper classes are usually defined as struct and have lowercase names. * Macros (and only macros) are all uppercase. Use macros sparingly, usually - a constexpr is better. + a simple (maybe constexpr) inline function is better. Undef macros after use + if possible. +* Macros should only be used for controlling which parts of the code should be + included when compiling or to avoid major code repetitions. * Variables, attributes, and function names are lowercase with `underscores_between_words`. * Class attribute names start with `m_` (member). -* Template parameters are single uppercase letters or start with uppercase `T` - and use CamelCase. -* Typedefs have `names_like_this_type` which end in `_type`. -* Macros should only be used for controlling which parts of the code should be - included when compiling. * Use `descriptive_variable_names`, exceptions are well-established conventions like `i` for a loop variable. Iterators are usually called `it`. * Declare variables where they are first used (C++ style), not at the beginning of a function (old C style). * Names from external namespaces (even `std`) are always mentioned explicitly. Do not use `using` (except for `std::swap`). This way we can't even by - accident pollute the namespace of the code including Osmium. + accident pollute the namespace of the code using Osmium. +* Always use the standard swap idiom: `using std::swap; swap(foo, bar);`. * `#include` directives appear in three "blocks" after the copyright notice. The blocks are separated by blank lines. First block contains `#include`s for standard C/C++ includes, second block for any external libs used, third @@ -64,8 +63,20 @@ different. * All files have suffix `.hpp`. * Closing } of all classes and namespaces should have a trailing comment with the name of the class/namespace. -* All constructors with one or more arguments should be declared "explicit" - unless there is a reason for them not to be. Document that reason. +* All constructors with one (or more arguments if they have a default) should + be declared "explicit" unless there is a reason for them not to be. Document + that reason. +* If a class has any of the special methods (copy/move constructor/assigment, + destructor) it should have all of them, possibly marking them as default or + deleted. +* Typedefs have `names_like_this_type` which end in `_type`. Typedefs should + use the new `using foo_type = bar` syntax instead of the old + `typedef bar foo_type`. +* Template parameters are single uppercase letters or start with uppercase `T` + and use CamelCase. +* Always use `typename` in templates, not `class`: `template `. +* The ellipsis in variadic template never has a space to the left of it and + always has a space to the right: `template ` etc. Keep to the indentation and other styles used in the code. Use `make indent` in the toplevel directory to fix indentation and styling. It calls `astyle` @@ -81,15 +92,15 @@ about which compilers support which feature and what operating system versions or distributions have which versions of these compilers installed. GCC 4.6 - too old, not supported (Ubuntu 12.04 LTS) -GCC 4.7.2 - can probably not be supported (Debian wheezy/stable) -GCC 4.7.3 - works -GCC 4.8 - works -clang 3.0 - too old, not supported (Debian wheezy/stable, Ubuntu 12.04 LTS) -clang 3.2 - works +GCC 4.7.2 - can probably not be supported (Debian wheezy) +GCC 4.7.3 - probably works +GCC 4.8 - works and is supported from here on +clang 3.0 - too old, not supported (Debian wheezy, Ubuntu 12.04 LTS) +clang 3.2 - probably works +clang 3.5 - works and is supported from here on -C++11 features you should not use: -* Inherited Constructors (works only in GCC 4.8+ and clang 3.3+, not in Visual - Studio) +Use `include/osmium/util/compatibility.hpp` if there are compatibility problems +between compilers due to different C++11 support. ## Checking your code diff --git a/README.md b/README.md index 9676d80b7..68fc2f61b 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ http://osmcode.org/libosmium A fast and flexible C++ library for working with OpenStreetMap data. -[![Build Status](https://secure.travis-ci.org/osmcode/libosmium.png)](http://travis-ci.org/osmcode/libosmium) +[![Build Status](https://secure.travis-ci.org/osmcode/libosmium.png)](https://travis-ci.org/osmcode/libosmium) [![Build status](https://ci.appveyor.com/api/projects/status/mkbg6e6stdgq7c1b?svg=true)](https://ci.appveyor.com/project/Mapbox/libosmium) Libosmium is developed on Linux, but also works on OSX and Windows (with some @@ -27,9 +27,15 @@ you need for your programs. For details see the [list of dependencies](https://github.com/osmcode/libosmium/wiki/Libosmium-dependencies). -The [protozero](https://github.com/mapbox/protozero) and -[utf8-cpp](http://utfcpp.sourceforge.net/) header-only libraries are included -in the libosmium repository. +The following external (header-only) libraries are included in the libosmium +repository: +* [gdalcpp](https://github.com/joto/gdalcpp) +* [protozero](https://github.com/mapbox/protozero) +* [utfcpp](http://utfcpp.sourceforge.net/) + +If you want (some of) those libraries to be installed along with libosmium +itself when calling `make install`, you have to use the CMake options +`INSTALL_GDALCPP`, `INSTALL_PROTOZERO`, and/or `INSTALL_UTFCPP`. ## Directories diff --git a/appveyor.yml b/appveyor.yml index a05c396cc..8244d98e7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,16 +9,10 @@ environment: - config: Dev - config: RelWithDebInfo -# branches to build -branches: - # whitelist - only: - - master - shallow_clone: true # Operating system (build VM template) -os: Visual Studio 2014 CTP4 +os: Visual Studio 2015 # scripts that are called at very beginning, before repo cloning init: @@ -46,6 +40,8 @@ install: - set PATH=%LODEPSDIR%\expat\lib;%PATH% #libtiff.dll - set PATH=%LODEPSDIR%\libtiff\lib;%PATH% + #jpeg.dll + - set PATH=%LODEPSDIR%\jpeg\lib;%PATH% #zlibwapi.dll - set PATH=%LODEPSDIR%\zlib\lib;%PATH% #convert backslashes in bzip2 path to forward slashes @@ -71,27 +67,16 @@ build_script: # This will produce lots of LNK4099 warnings which can be ignored. # Unfortunately they can't be disabled, see # http://stackoverflow.com/questions/661606/visual-c-how-to-disable-specific-linker-warnings - - cmake .. -LA -G "Visual Studio 14 Win64" + - cmake -LA -G "Visual Studio 14 Win64" -DOsmium_DEBUG=TRUE -DCMAKE_BUILD_TYPE=%config% -DBUILD_HEADERS=OFF -DBOOST_ROOT=%LODEPSDIR%\boost - -DBoost_PROGRAM_OPTIONS_LIBRARY=%LODEPSDIR%\boost\lib\libboost_program_options-vc140-mt-1_57.lib + -DBoost_PROGRAM_OPTIONS_LIBRARY=%LODEPSDIR%\boost\lib\libboost_program_options-vc140-mt-1_58.lib -DZLIB_LIBRARY=%LODEPSDIR%\zlib\lib\zlibwapi.lib - -DZLIB_INCLUDE_DIR=%LODEPSDIR%\zlib\include - -DEXPAT_LIBRARY=%LODEPSDIR%\expat\lib\libexpat.lib - -DEXPAT_INCLUDE_DIR=%LODEPSDIR%\expat\include - -DBZIP2_LIBRARIES=%LIBBZIP2% - -DBZIP2_INCLUDE_DIR=%LODEPSDIR%\bzip2\include - -DGDAL_LIBRARY=%LODEPSDIR%\gdal\lib\gdal_i.lib - -DGDAL_INCLUDE_DIR=%LODEPSDIR%\gdal\include - -DGEOS_LIBRARY=%LODEPSDIR%\geos\lib\geos.lib - -DGEOS_INCLUDE_DIR=%LODEPSDIR%\geos\include - -DPROJ_LIBRARY=%LODEPSDIR%\proj\lib\proj.lib - -DPROJ_INCLUDE_DIR=%LODEPSDIR%\proj\include - -DSPARSEHASH_INCLUDE_DIR=%LODEPSDIR%\sparsehash\include - -DGETOPT_LIBRARY=%LODEPSDIR%\wingetopt\lib\wingetopt.lib - -DGETOPT_INCLUDE_DIR=%LODEPSDIR%\wingetopt\include + -DBZIP2_LIBRARY_RELEASE=%LIBBZIP2% + -DCMAKE_PREFIX_PATH=%LODEPSDIR%\zlib;%LODEPSDIR%\expat;%LODEPSDIR%\bzip2;%LODEPSDIR%\geos;%LODEPSDIR%\gdal;%LODEPSDIR%\proj;%LODEPSDIR%\sparsehash;%LODEPSDIR%\wingetopt + .. - msbuild libosmium.sln /p:Configuration=%config% /toolsversion:14.0 /p:Platform=x64 /p:PlatformToolset=v140 #- cmake .. -LA -G "NMake Makefiles" # -DOsmium_DEBUG=TRUE diff --git a/cmake/FindOsmium.cmake b/cmake/FindOsmium.cmake index bb140718b..fba8ffb92 100644 --- a/cmake/FindOsmium.cmake +++ b/cmake/FindOsmium.cmake @@ -19,7 +19,7 @@ # Then add the following in your CMakeLists.txt: # # find_package(Osmium REQUIRED COMPONENTS ) -# include_directories(${OSMIUM_INCLUDE_DIRS}) +# include_directories(SYSTEM ${OSMIUM_INCLUDE_DIRS}) # # For the substitute a space separated list of one or more of the # following components: @@ -56,31 +56,13 @@ find_path(OSMIUM_INCLUDE_DIR osmium/osm.hpp PATH_SUFFIXES include PATHS ../libosmium - ../../libosmium - libosmium ~/Library/Frameworks /Library/Frameworks - /usr/local - /usr/ /opt/local # DarwinPorts /opt ) -# Handle the QUIETLY and REQUIRED arguments and set OSMIUM_FOUND to TRUE if -# all listed variables are TRUE. -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(OSMIUM REQUIRED_VARS OSMIUM_INCLUDE_DIR) - -# Copy the results to the output variables. -if(OSMIUM_FOUND) - set(OSMIUM_INCLUDE_DIRS ${OSMIUM_INCLUDE_DIR}) -else() - set(OSMIUM_INCLUDE_DIRS "") -endif() - -if(Osmium_FIND_REQUIRED AND NOT OSMIUM_FOUND) - message(FATAL_ERROR "Can not find libosmium headers, please install them or configure the paths") -endif() +set(OSMIUM_INCLUDE_DIRS "${OSMIUM_INCLUDE_DIR}") #---------------------------------------------------------------------- # @@ -113,6 +95,7 @@ if(Osmium_USE_PBF) find_package(ZLIB) find_package(Threads) + list(APPEND OSMIUM_EXTRA_FIND_VARS ZLIB_FOUND Threads_FOUND) if(ZLIB_FOUND AND Threads_FOUND) list(APPEND OSMIUM_PBF_LIBRARIES ${ZLIB_LIBRARIES} @@ -125,7 +108,6 @@ if(Osmium_USE_PBF) ${ZLIB_INCLUDE_DIR} ) else() - set(_missing_libraries 1) message(WARNING "Osmium: Can not find some libraries for PBF input/output, please install them or configure the paths.") endif() endif() @@ -138,6 +120,7 @@ if(Osmium_USE_XML) find_package(ZLIB) find_package(Threads) + list(APPEND OSMIUM_EXTRA_FIND_VARS EXPAT_FOUND BZIP2_FOUND ZLIB_FOUND Threads_FOUND) if(EXPAT_FOUND AND BZIP2_FOUND AND ZLIB_FOUND AND Threads_FOUND) list(APPEND OSMIUM_XML_LIBRARIES ${EXPAT_LIBRARIES} @@ -151,7 +134,6 @@ if(Osmium_USE_XML) ${ZLIB_INCLUDE_DIR} ) else() - set(_missing_libraries 1) message(WARNING "Osmium: Can not find some libraries for XML input/output, please install them or configure the paths.") endif() endif() @@ -172,12 +154,12 @@ if(Osmium_USE_GEOS) find_path(GEOS_INCLUDE_DIR geos/geom.h) find_library(GEOS_LIBRARY NAMES geos) + list(APPEND OSMIUM_EXTRA_FIND_VARS GEOS_INCLUDE_DIR GEOS_LIBRARY) if(GEOS_INCLUDE_DIR AND GEOS_LIBRARY) SET(GEOS_FOUND 1) list(APPEND OSMIUM_LIBRARIES ${GEOS_LIBRARY}) list(APPEND OSMIUM_INCLUDE_DIRS ${GEOS_INCLUDE_DIR}) else() - set(_missing_libraries 1) message(WARNING "Osmium: GEOS library is required but not found, please install it or configure the paths.") endif() endif() @@ -187,11 +169,11 @@ endif() if(Osmium_USE_GDAL) find_package(GDAL) + list(APPEND OSMIUM_EXTRA_FIND_VARS GDAL_FOUND) if(GDAL_FOUND) list(APPEND OSMIUM_LIBRARIES ${GDAL_LIBRARIES}) list(APPEND OSMIUM_INCLUDE_DIRS ${GDAL_INCLUDE_DIRS}) else() - set(_missing_libraries 1) message(WARNING "Osmium: GDAL library is required but not found, please install it or configure the paths.") endif() endif() @@ -202,12 +184,12 @@ if(Osmium_USE_PROJ) find_path(PROJ_INCLUDE_DIR proj_api.h) find_library(PROJ_LIBRARY NAMES proj) + list(APPEND OSMIUM_EXTRA_FIND_VARS PROJ_INCLUDE_DIR PROJ_LIBRARY) if(PROJ_INCLUDE_DIR AND PROJ_LIBRARY) set(PROJ_FOUND 1) list(APPEND OSMIUM_LIBRARIES ${PROJ_LIBRARY}) list(APPEND OSMIUM_INCLUDE_DIRS ${PROJ_INCLUDE_DIR}) else() - set(_missing_libraries 1) message(WARNING "Osmium: PROJ.4 library is required but not found, please install it or configure the paths.") endif() endif() @@ -217,21 +199,19 @@ endif() if(Osmium_USE_SPARSEHASH) find_path(SPARSEHASH_INCLUDE_DIR google/sparsetable) + list(APPEND OSMIUM_EXTRA_FIND_VARS SPARSEHASH_INCLUDE_DIR) if(SPARSEHASH_INCLUDE_DIR) # Find size of sparsetable::size_type. This does not work on older # CMake versions because they can do this check only in C, not in C++. - include(CheckTypeSize) - set(CMAKE_REQUIRED_INCLUDES ${SPARSEHASH_INCLUDE_DIR}) - set(CMAKE_EXTRA_INCLUDE_FILES "google/sparsetable") - check_type_size("google::sparsetable::size_type" SPARSETABLE_SIZE_TYPE LANGUAGE CXX) - set(CMAKE_EXTRA_INCLUDE_FILES) - set(CMAKE_REQUIRED_INCLUDES) - - # Falling back to checking size_t if google::sparsetable::size_type - # could not be checked. - if(SPARSETABLE_SIZE_TYPE STREQUAL "") - check_type_size("void*" VOID_PTR_SIZE) - set(SPARSETABLE_SIZE_TYPE ${VOID_PTR_SIZE}) + if (NOT CMAKE_VERSION VERSION_LESS 3.0) + include(CheckTypeSize) + set(CMAKE_REQUIRED_INCLUDES ${SPARSEHASH_INCLUDE_DIR}) + set(CMAKE_EXTRA_INCLUDE_FILES "google/sparsetable") + check_type_size("google::sparsetable::size_type" SPARSETABLE_SIZE_TYPE LANGUAGE CXX) + set(CMAKE_EXTRA_INCLUDE_FILES) + set(CMAKE_REQUIRED_INCLUDES) + else() + set(SPARSETABLE_SIZE_TYPE ${CMAKE_SIZEOF_VOID_P}) endif() # Sparsetable::size_type must be at least 8 bytes (64bit), otherwise @@ -244,7 +224,6 @@ if(Osmium_USE_SPARSEHASH) message(WARNING "Osmium: Disabled Google SparseHash library on 32bit system (size_type=${SPARSETABLE_SIZE_TYPE}).") endif() else() - set(_missing_libraries 1) message(WARNING "Osmium: Google SparseHash library is required but not found, please install it or configure the paths.") endif() endif() @@ -274,9 +253,14 @@ endif() # Check that all required libraries are available # #---------------------------------------------------------------------- -if(Osmium_FIND_REQUIRED AND _missing_libraries) - message(FATAL_ERROR "Required library or libraries missing. Aborting.") +if (OSMIUM_EXTRA_FIND_VARS) + list(REMOVE_DUPLICATES OSMIUM_EXTRA_FIND_VARS) endif() +# Handle the QUIETLY and REQUIRED arguments and set OSMIUM_FOUND to TRUE if +# all listed variables are TRUE. +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Osmium REQUIRED_VARS OSMIUM_INCLUDE_DIR ${OSMIUM_EXTRA_FIND_VARS}) +unset(OSMIUM_EXTRA_FIND_VARS) #---------------------------------------------------------------------- # diff --git a/cmake/iwyu.sh b/cmake/iwyu.sh index f7d8a15e8..ceea106c3 100755 --- a/cmake/iwyu.sh +++ b/cmake/iwyu.sh @@ -16,7 +16,7 @@ echo "INCLUDE WHAT YOU USE REPORT:" >$log allok=yes -for file in `find include/osmium -name \*.hpp`; do +for file in `find include/osmium -name \*.hpp | sort`; do mkdir -p `dirname build/check_reports/$file` ifile="build/check_reports/${file%.hpp}.iwyu" $cmdline $file >$ifile 2>&1 diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index c9f59603d..a04a843f2 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -14,12 +14,10 @@ set(EXAMPLES count create_node_cache debug + filter_discussions index read serdump - toogr - toogr2 - toogr2_exp use_node_cache CACHE STRING "Example programs" ) @@ -30,7 +28,7 @@ set(EXAMPLES # Examples depending on wingetopt # #----------------------------------------------------------------------------- -set(GETOPT_EXAMPLES area_test convert serdump toogr toogr2 toogr2_exp) +set(GETOPT_EXAMPLES area_test convert serdump) if(NOT GETOPT_MISSING) foreach(example ${GETOPT_EXAMPLES}) list(APPEND EXAMPLE_LIBS_${example} ${GETOPT_LIBRARY}) @@ -74,27 +72,6 @@ else() endif() -#----------------------------------------------------------------------------- -# -# Examples depending on GDAL/PROJ.4/SparseHash -# -#----------------------------------------------------------------------------- -set(OGR_EXAMPLES toogr toogr2 toogr2_exp) - -if(GDAL_FOUND AND PROJ_FOUND AND SPARSEHASH_FOUND) - foreach(example ${OGR_EXAMPLES}) - list(APPEND EXAMPLE_LIBS_${example} ${GDAL_LIBRARIES}) - list(APPEND EXAMPLE_LIBS_${example} ${PROJ_LIBRARIES}) - endforeach() -else() - message(STATUS "Configuring examples - Skipping examples because GDAL and/or Proj.4 and/or SparseHash not found:") - foreach(example ${OGR_EXAMPLES}) - message(STATUS " - osmium_${example}") - list(REMOVE_ITEM EXAMPLES ${example}) - endforeach() -endif() - - #----------------------------------------------------------------------------- # # Configure examples diff --git a/examples/osmium_filter_discussions.cpp b/examples/osmium_filter_discussions.cpp new file mode 100644 index 000000000..bba25b752 --- /dev/null +++ b/examples/osmium_filter_discussions.cpp @@ -0,0 +1,72 @@ +/* + + Read OSM changesets with discussions from a changeset dump like the one + you get from http://planet.osm.org/planet/discussions-latest.osm.bz2 + and write out only those changesets which have discussions (ie comments). + + The code in this example file is released into the Public Domain. + +*/ + +#include // for std::copy_if +#include // for std::cout, std::cerr + +// we want to read OSM files in XML format +// (other formats don't support full changesets, so only XML is needed here) +#include +#include + +// we want to write OSM files in XML format +#include +#include + +// we want to support any compressioon (.gz2 and .bz2) +#include + +int main(int argc, char* argv[]) { + if (argc != 3) { + std::cout << "Usage: " << argv[0] << " INFILE OUTFILE\n"; + exit(1); + } + + // The input file, deduce file format from file suffix + osmium::io::File infile(argv[1]); + + // The output file, force class XML OSM file format + osmium::io::File outfile(argv[2], "osm"); + + // Initialize Reader for the input file. + // Read only changesets (will ignore nodes, ways, and + // relations if there are any). + osmium::io::Reader reader(infile, osmium::osm_entity_bits::changeset); + + // Get the header from the input file + osmium::io::Header header = reader.header(); + + // Initialize writer for the output file. Use the header from the input + // file for the output file. This will copy over some header information. + // The last parameter will tell the writer that it is allowed to overwrite + // an existing file. Without it, it will refuse to do so. + osmium::io::Writer writer(outfile, header, osmium::io::overwrite::allow); + + // Create range of input iterators that will iterator over all changesets + // delivered from input file through the "reader". + auto input_range = osmium::io::make_input_iterator_range(reader); + + // Create an output iterator writing through the "writer" object to the + // output file. + auto output_iterator = osmium::io::make_output_iterator(writer); + + // Copy all changesets from input to output that have at least one comment. + std::copy_if(input_range.begin(), input_range.end(), output_iterator, [](const osmium::Changeset& changeset) { + return changeset.num_comments() > 0; + }); + + // Explicitly close the writer and reader. Will throw an exception if + // there is a problem. If you wait for the destructor to close the writer + // and reader, you will not notice the problem, because destructors must + // not throw. + writer.close(); + reader.close(); +} + diff --git a/examples/osmium_toogr.cpp b/examples/osmium_toogr.cpp deleted file mode 100644 index 7c5a965c5..000000000 --- a/examples/osmium_toogr.cpp +++ /dev/null @@ -1,244 +0,0 @@ -/* - - This is an example tool that converts OSM data to some output format - like Spatialite or Shapefiles using the OGR library. - - The code in this example file is released into the Public Domain. - -*/ - -#include -#include - -#include -#include -#include - -#include -#include -#include - -typedef osmium::index::map::Dummy index_neg_type; -typedef osmium::index::map::Map index_pos_type; - -typedef osmium::handler::NodeLocationsForWays location_handler_type; - -class MyOGRHandler : public osmium::handler::Handler { - - OGRDataSource* m_data_source; - OGRLayer* m_layer_point; - OGRLayer* m_layer_linestring; - - osmium::geom::OGRFactory<> m_factory; - -public: - - MyOGRHandler(const std::string& driver_name, const std::string& filename) { - - OGRRegisterAll(); - - OGRSFDriver* driver = OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(driver_name.c_str()); - if (!driver) { - std::cerr << driver_name << " driver not available.\n"; - exit(1); - } - - CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "FALSE"); - const char* options[] = { "SPATIALITE=TRUE", nullptr }; - m_data_source = driver->CreateDataSource(filename.c_str(), const_cast(options)); - if (!m_data_source) { - std::cerr << "Creation of output file failed.\n"; - exit(1); - } - - OGRSpatialReference sparef; - sparef.SetWellKnownGeogCS("WGS84"); - m_layer_point = m_data_source->CreateLayer("postboxes", &sparef, wkbPoint, nullptr); - if (!m_layer_point) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_point_field_id("id", OFTReal); - layer_point_field_id.SetWidth(10); - - if (m_layer_point->CreateField(&layer_point_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_point_field_operator("operator", OFTString); - layer_point_field_operator.SetWidth(30); - - if (m_layer_point->CreateField(&layer_point_field_operator) != OGRERR_NONE) { - std::cerr << "Creating operator field failed.\n"; - exit(1); - } - - /* Transactions might make things faster, then again they might not. - Feel free to experiment and benchmark and report back. */ - m_layer_point->StartTransaction(); - - m_layer_linestring = m_data_source->CreateLayer("roads", &sparef, wkbLineString, nullptr); - if (!m_layer_linestring) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_linestring_field_id("id", OFTReal); - layer_linestring_field_id.SetWidth(10); - - if (m_layer_linestring->CreateField(&layer_linestring_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_linestring_field_type("type", OFTString); - layer_linestring_field_type.SetWidth(30); - - if (m_layer_linestring->CreateField(&layer_linestring_field_type) != OGRERR_NONE) { - std::cerr << "Creating type field failed.\n"; - exit(1); - } - - m_layer_linestring->StartTransaction(); - } - - ~MyOGRHandler() { - m_layer_linestring->CommitTransaction(); - m_layer_point->CommitTransaction(); - OGRDataSource::DestroyDataSource(m_data_source); - OGRCleanupAll(); - } - - void node(const osmium::Node& node) { - const char* amenity = node.tags().get_value_by_key("amenity"); - if (amenity && !strcmp(amenity, "post_box")) { - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_point->GetLayerDefn()); - std::unique_ptr ogr_point = m_factory.create_point(node); - feature->SetGeometry(ogr_point.get()); - feature->SetField("id", static_cast(node.id())); - feature->SetField("operator", node.tags().get_value_by_key("operator")); - - if (m_layer_point->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); - } - } - - void way(const osmium::Way& way) { - const char* highway = way.tags().get_value_by_key("highway"); - if (highway) { - try { - std::unique_ptr ogr_linestring = m_factory.create_linestring(way); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_linestring->GetLayerDefn()); - feature->SetGeometry(ogr_linestring.get()); - feature->SetField("id", static_cast(way.id())); - feature->SetField("type", highway); - - if (m_layer_linestring->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); - } catch (osmium::geometry_error&) { - std::cerr << "Ignoring illegal geometry for way " << way.id() << ".\n"; - } - } - } - -}; - -/* ================================================== */ - -void print_help() { - std::cout << "osmium_toogr [OPTIONS] [INFILE [OUTFILE]]\n\n" \ - << "If INFILE is not given stdin is assumed.\n" \ - << "If OUTFILE is not given 'ogr_out' is used.\n" \ - << "\nOptions:\n" \ - << " -h, --help This help message\n" \ - << " -l, --location_store=TYPE Set location store\n" \ - << " -f, --format=FORMAT Output OGR format (Default: 'SQLite')\n" \ - << " -L See available location stores\n"; -} - -int main(int argc, char* argv[]) { - const auto& map_factory = osmium::index::MapFactory::instance(); - - static struct option long_options[] = { - {"help", no_argument, 0, 'h'}, - {"format", required_argument, 0, 'f'}, - {"location_store", required_argument, 0, 'l'}, - {"list_location_stores", no_argument, 0, 'L'}, - {0, 0, 0, 0} - }; - - std::string output_format { "SQLite" }; - std::string location_store { "sparse_mem_array" }; - - while (true) { - int c = getopt_long(argc, argv, "hf:l:L", long_options, 0); - if (c == -1) { - break; - } - - switch (c) { - case 'h': - print_help(); - exit(0); - case 'f': - output_format = optarg; - break; - case 'l': - location_store = optarg; - break; - case 'L': - std::cout << "Available map types:\n"; - for (const auto& map_type : map_factory.map_types()) { - std::cout << " " << map_type << "\n"; - } - exit(0); - default: - exit(1); - } - } - - std::string input_filename; - std::string output_filename("ogr_out"); - int remaining_args = argc - optind; - if (remaining_args > 2) { - std::cerr << "Usage: " << argv[0] << " [OPTIONS] [INFILE [OUTFILE]]" << std::endl; - exit(1); - } else if (remaining_args == 2) { - input_filename = argv[optind]; - output_filename = argv[optind+1]; - } else if (remaining_args == 1) { - input_filename = argv[optind]; - } else { - input_filename = "-"; - } - - osmium::io::Reader reader(input_filename); - - std::unique_ptr index_pos = map_factory.create_map(location_store); - index_neg_type index_neg; - location_handler_type location_handler(*index_pos, index_neg); - location_handler.ignore_errors(); - - MyOGRHandler ogr_handler(output_format, output_filename); - - osmium::apply(reader, location_handler, ogr_handler); - reader.close(); - - int locations_fd = open("locations.dump", O_WRONLY | O_CREAT, 0644); - if (locations_fd < 0) { - throw std::system_error(errno, std::system_category(), "Open failed"); - } - index_pos->dump_as_list(locations_fd); - close(locations_fd); -} - diff --git a/examples/osmium_toogr2.cpp b/examples/osmium_toogr2.cpp deleted file mode 100644 index e1b505688..000000000 --- a/examples/osmium_toogr2.cpp +++ /dev/null @@ -1,331 +0,0 @@ -/* - - This is an example tool that converts OSM data to some output format - like Spatialite or Shapefiles using the OGR library. - - This version does multipolygon handling (in contrast to the osmium_toogr - example which doesn't). - - The code in this example file is released into the Public Domain. - -*/ - -#include -#include - -// usually you only need one or two of these -#include -#include - -#include -#include -#include -#include - -#include -//#include -#include -#include -#include - -typedef osmium::index::map::Dummy index_neg_type; - -typedef osmium::index::map::SparseMemArray index_pos_type; - -typedef osmium::handler::NodeLocationsForWays location_handler_type; - -class MyOGRHandler : public osmium::handler::Handler { - - OGRDataSource* m_data_source; - OGRLayer* m_layer_point; - OGRLayer* m_layer_linestring; - OGRLayer* m_layer_polygon; - - // Choose one of the following: - - // 1. Use WGS84, do not project coordinates. - //osmium::geom::OGRFactory<> m_factory {}; - - // 2. Project coordinates into "Web Mercator". - osmium::geom::OGRFactory m_factory; - - // 3. Use any projection that the proj library can handle. - // (Initialize projection with EPSG code or proj string). - // In addition you need to link with "-lproj" and add - // #include . - //osmium::geom::OGRFactory m_factory {osmium::geom::Projection(3857)}; - -public: - - MyOGRHandler(const std::string& driver_name, const std::string& filename) { - - OGRRegisterAll(); - - OGRSFDriver* driver = OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(driver_name.c_str()); - if (!driver) { - std::cerr << driver_name << " driver not available.\n"; - exit(1); - } - - CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "FALSE"); - const char* options[] = { "SPATIALITE=TRUE", nullptr }; - m_data_source = driver->CreateDataSource(filename.c_str(), const_cast(options)); - if (!m_data_source) { - std::cerr << "Creation of output file failed.\n"; - exit(1); - } - - OGRSpatialReference sparef; - sparef.importFromProj4(m_factory.proj_string().c_str()); - - m_layer_point = m_data_source->CreateLayer("postboxes", &sparef, wkbPoint, nullptr); - if (!m_layer_point) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_point_field_id("id", OFTReal); - layer_point_field_id.SetWidth(10); - - if (m_layer_point->CreateField(&layer_point_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_point_field_operator("operator", OFTString); - layer_point_field_operator.SetWidth(30); - - if (m_layer_point->CreateField(&layer_point_field_operator) != OGRERR_NONE) { - std::cerr << "Creating operator field failed.\n"; - exit(1); - } - - /* Transactions might make things faster, then again they might not. - Feel free to experiment and benchmark and report back. */ - m_layer_point->StartTransaction(); - - m_layer_linestring = m_data_source->CreateLayer("roads", &sparef, wkbLineString, nullptr); - if (!m_layer_linestring) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_linestring_field_id("id", OFTReal); - layer_linestring_field_id.SetWidth(10); - - if (m_layer_linestring->CreateField(&layer_linestring_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_linestring_field_type("type", OFTString); - layer_linestring_field_type.SetWidth(30); - - if (m_layer_linestring->CreateField(&layer_linestring_field_type) != OGRERR_NONE) { - std::cerr << "Creating type field failed.\n"; - exit(1); - } - - m_layer_linestring->StartTransaction(); - - m_layer_polygon = m_data_source->CreateLayer("buildings", &sparef, wkbMultiPolygon, nullptr); - if (!m_layer_polygon) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_polygon_field_id("id", OFTInteger); - layer_polygon_field_id.SetWidth(10); - - if (m_layer_polygon->CreateField(&layer_polygon_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_polygon_field_type("type", OFTString); - layer_polygon_field_type.SetWidth(30); - - if (m_layer_polygon->CreateField(&layer_polygon_field_type) != OGRERR_NONE) { - std::cerr << "Creating type field failed.\n"; - exit(1); - } - - m_layer_polygon->StartTransaction(); - } - - ~MyOGRHandler() { - m_layer_polygon->CommitTransaction(); - m_layer_linestring->CommitTransaction(); - m_layer_point->CommitTransaction(); - OGRDataSource::DestroyDataSource(m_data_source); - OGRCleanupAll(); - } - - void node(const osmium::Node& node) { - const char* amenity = node.tags()["amenity"]; - if (amenity && !strcmp(amenity, "post_box")) { - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_point->GetLayerDefn()); - std::unique_ptr ogr_point = m_factory.create_point(node); - feature->SetGeometry(ogr_point.get()); - feature->SetField("id", static_cast(node.id())); - feature->SetField("operator", node.tags()["operator"]); - - if (m_layer_point->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); - } - } - - void way(const osmium::Way& way) { - const char* highway = way.tags()["highway"]; - if (highway) { - try { - std::unique_ptr ogr_linestring = m_factory.create_linestring(way); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_linestring->GetLayerDefn()); - feature->SetGeometry(ogr_linestring.get()); - feature->SetField("id", static_cast(way.id())); - feature->SetField("type", highway); - - if (m_layer_linestring->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); - } catch (osmium::geometry_error&) { - std::cerr << "Ignoring illegal geometry for way " << way.id() << ".\n"; - } - } - } - - void area(const osmium::Area& area) { - const char* building = area.tags()["building"]; - if (building) { - try { - std::unique_ptr ogr_polygon = m_factory.create_multipolygon(area); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_polygon->GetLayerDefn()); - feature->SetGeometry(ogr_polygon.get()); - feature->SetField("id", static_cast(area.id())); - feature->SetField("type", building); - - std::string type = ""; - if (area.from_way()) { - type += "w"; - } else { - type += "r"; - } - feature->SetField("type", type.c_str()); - - if (m_layer_polygon->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); - } catch (osmium::geometry_error&) { - std::cerr << "Ignoring illegal geometry for area " << area.id() << " created from " << (area.from_way() ? "way" : "relation") << " with id=" << area.orig_id() << ".\n"; - } - } - } - -}; - -/* ================================================== */ - -void print_help() { - std::cout << "osmium_toogr [OPTIONS] [INFILE [OUTFILE]]\n\n" \ - << "If INFILE is not given stdin is assumed.\n" \ - << "If OUTFILE is not given 'ogr_out' is used.\n" \ - << "\nOptions:\n" \ - << " -h, --help This help message\n" \ - << " -d, --debug Enable debug output\n" \ - << " -f, --format=FORMAT Output OGR format (Default: 'SQLite')\n"; -} - -int main(int argc, char* argv[]) { - static struct option long_options[] = { - {"help", no_argument, 0, 'h'}, - {"debug", no_argument, 0, 'd'}, - {"format", required_argument, 0, 'f'}, - {0, 0, 0, 0} - }; - - std::string output_format("SQLite"); - bool debug = false; - - while (true) { - int c = getopt_long(argc, argv, "hdf:", long_options, 0); - if (c == -1) { - break; - } - - switch (c) { - case 'h': - print_help(); - exit(0); - case 'd': - debug = true; - break; - case 'f': - output_format = optarg; - break; - default: - exit(1); - } - } - - std::string input_filename; - std::string output_filename("ogr_out"); - int remaining_args = argc - optind; - if (remaining_args > 2) { - std::cerr << "Usage: " << argv[0] << " [OPTIONS] [INFILE [OUTFILE]]" << std::endl; - exit(1); - } else if (remaining_args == 2) { - input_filename = argv[optind]; - output_filename = argv[optind+1]; - } else if (remaining_args == 1) { - input_filename = argv[optind]; - } else { - input_filename = "-"; - } - - osmium::area::Assembler::config_type assembler_config; - assembler_config.enable_debug_output(debug); - osmium::area::MultipolygonCollector collector(assembler_config); - - std::cerr << "Pass 1...\n"; - osmium::io::Reader reader1(input_filename); - collector.read_relations(reader1); - reader1.close(); - std::cerr << "Pass 1 done\n"; - - index_pos_type index_pos; - index_neg_type index_neg; - location_handler_type location_handler(index_pos, index_neg); - location_handler.ignore_errors(); - - MyOGRHandler ogr_handler(output_format, output_filename); - - std::cerr << "Pass 2...\n"; - osmium::io::Reader reader2(input_filename); - - osmium::apply(reader2, location_handler, ogr_handler, collector.handler([&ogr_handler](const osmium::memory::Buffer& area_buffer) { - osmium::apply(area_buffer, ogr_handler); - })); - - reader2.close(); - std::cerr << "Pass 2 done\n"; - - std::vector incomplete_relations = collector.get_incomplete_relations(); - if (!incomplete_relations.empty()) { - std::cerr << "Warning! Some member ways missing for these multipolygon relations:"; - for (const auto* relation : incomplete_relations) { - std::cerr << " " << relation->id(); - } - std::cerr << "\n"; - } -} - diff --git a/examples/osmium_toogr2_exp.cpp b/examples/osmium_toogr2_exp.cpp deleted file mode 100644 index db8d5cf4e..000000000 --- a/examples/osmium_toogr2_exp.cpp +++ /dev/null @@ -1,305 +0,0 @@ -/* - - This is an example tool that converts OSM data to some output format - like Spatialite or Shapefiles using the OGR library. - - This version does multipolygon handling (in contrast to the osmium_toogr - example which doesn't). - - This version (..._exp) uses a new experimental unsupported interface. - - The code in this example file is released into the Public Domain. - -*/ - -#include -#include - -#include - -#include - -#include -//#include -#include -#include -#include -#include - -typedef osmium::index::map::SparseMemArray index_type; -typedef osmium::handler::NodeLocationsForWays location_handler_type; - -class MyOGRHandler : public osmium::handler::Handler { - - OGRDataSource* m_data_source; - OGRLayer* m_layer_point; - OGRLayer* m_layer_linestring; - OGRLayer* m_layer_polygon; - - // Choose one of the following: - - // 1. Use WGS84, do not project coordinates. - //osmium::geom::OGRFactory<> m_factory {}; - - // 2. Project coordinates into "Web Mercator". - osmium::geom::OGRFactory m_factory; - - // 3. Use any projection that the proj library can handle. - // (Initialize projection with EPSG code or proj string). - // In addition you need to link with "-lproj" and add - // #include . - //osmium::geom::OGRFactory m_factory {osmium::geom::Projection(3857)}; - -public: - - MyOGRHandler(const std::string& driver_name, const std::string& filename) { - - OGRRegisterAll(); - - OGRSFDriver* driver = OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(driver_name.c_str()); - if (!driver) { - std::cerr << driver_name << " driver not available.\n"; - exit(1); - } - - CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "FALSE"); - const char* options[] = { "SPATIALITE=TRUE", nullptr }; - m_data_source = driver->CreateDataSource(filename.c_str(), const_cast(options)); - if (!m_data_source) { - std::cerr << "Creation of output file failed.\n"; - exit(1); - } - - OGRSpatialReference sparef; - sparef.importFromProj4(m_factory.proj_string().c_str()); - - m_layer_point = m_data_source->CreateLayer("postboxes", &sparef, wkbPoint, nullptr); - if (!m_layer_point) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_point_field_id("id", OFTReal); - layer_point_field_id.SetWidth(10); - - if (m_layer_point->CreateField(&layer_point_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_point_field_operator("operator", OFTString); - layer_point_field_operator.SetWidth(30); - - if (m_layer_point->CreateField(&layer_point_field_operator) != OGRERR_NONE) { - std::cerr << "Creating operator field failed.\n"; - exit(1); - } - - /* Transactions might make things faster, then again they might not. - Feel free to experiment and benchmark and report back. */ - m_layer_point->StartTransaction(); - - m_layer_linestring = m_data_source->CreateLayer("roads", &sparef, wkbLineString, nullptr); - if (!m_layer_linestring) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_linestring_field_id("id", OFTReal); - layer_linestring_field_id.SetWidth(10); - - if (m_layer_linestring->CreateField(&layer_linestring_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_linestring_field_type("type", OFTString); - layer_linestring_field_type.SetWidth(30); - - if (m_layer_linestring->CreateField(&layer_linestring_field_type) != OGRERR_NONE) { - std::cerr << "Creating type field failed.\n"; - exit(1); - } - - m_layer_linestring->StartTransaction(); - - m_layer_polygon = m_data_source->CreateLayer("buildings", &sparef, wkbMultiPolygon, nullptr); - if (!m_layer_polygon) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_polygon_field_id("id", OFTInteger); - layer_polygon_field_id.SetWidth(10); - - if (m_layer_polygon->CreateField(&layer_polygon_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_polygon_field_type("type", OFTString); - layer_polygon_field_type.SetWidth(30); - - if (m_layer_polygon->CreateField(&layer_polygon_field_type) != OGRERR_NONE) { - std::cerr << "Creating type field failed.\n"; - exit(1); - } - - m_layer_polygon->StartTransaction(); - } - - ~MyOGRHandler() { - m_layer_polygon->CommitTransaction(); - m_layer_linestring->CommitTransaction(); - m_layer_point->CommitTransaction(); - OGRDataSource::DestroyDataSource(m_data_source); - OGRCleanupAll(); - } - - void node(const osmium::Node& node) { - const char* amenity = node.tags()["amenity"]; - if (amenity && !strcmp(amenity, "post_box")) { - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_point->GetLayerDefn()); - std::unique_ptr ogr_point = m_factory.create_point(node); - feature->SetGeometry(ogr_point.get()); - feature->SetField("id", static_cast(node.id())); - feature->SetField("operator", node.tags()["operator"]); - - if (m_layer_point->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); - } - } - - void way(const osmium::Way& way) { - const char* highway = way.tags()["highway"]; - if (highway) { - try { - std::unique_ptr ogr_linestring = m_factory.create_linestring(way); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_linestring->GetLayerDefn()); - feature->SetGeometry(ogr_linestring.get()); - feature->SetField("id", static_cast(way.id())); - feature->SetField("type", highway); - - if (m_layer_linestring->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); - } catch (osmium::geometry_error&) { - std::cerr << "Ignoring illegal geometry for way " << way.id() << ".\n"; - } - } - } - - void area(const osmium::Area& area) { - const char* building = area.tags()["building"]; - if (building) { - try { - std::unique_ptr ogr_polygon = m_factory.create_multipolygon(area); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_polygon->GetLayerDefn()); - feature->SetGeometry(ogr_polygon.get()); - feature->SetField("id", static_cast(area.id())); - feature->SetField("type", building); - - std::string type = ""; - if (area.from_way()) { - type += "w"; - } else { - type += "r"; - } - feature->SetField("type", type.c_str()); - - if (m_layer_polygon->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); - } catch (osmium::geometry_error&) { - std::cerr << "Ignoring illegal geometry for area " << area.id() << " created from " << (area.from_way() ? "way" : "relation") << " with id=" << area.orig_id() << ".\n"; - } - } - } - -}; - -/* ================================================== */ - -void print_help() { - std::cout << "osmium_toogr [OPTIONS] [INFILE [OUTFILE]]\n\n" \ - << "If INFILE is not given stdin is assumed.\n" \ - << "If OUTFILE is not given 'ogr_out' is used.\n" \ - << "\nOptions:\n" \ - << " -h, --help This help message\n" \ - << " -f, --format=FORMAT Output OGR format (Default: 'SQLite')\n"; -} - -int main(int argc, char* argv[]) { - static struct option long_options[] = { - {"help", no_argument, 0, 'h'}, - {"format", required_argument, 0, 'f'}, - {0, 0, 0, 0} - }; - - std::string output_format("SQLite"); - - while (true) { - int c = getopt_long(argc, argv, "hf:", long_options, 0); - if (c == -1) { - break; - } - - switch (c) { - case 'h': - print_help(); - exit(0); - case 'f': - output_format = optarg; - break; - default: - exit(1); - } - } - - std::string input_filename; - std::string output_filename("ogr_out"); - int remaining_args = argc - optind; - if (remaining_args > 2) { - std::cerr << "Usage: " << argv[0] << " [OPTIONS] [INFILE [OUTFILE]]" << std::endl; - exit(1); - } else if (remaining_args == 2) { - input_filename = argv[optind]; - output_filename = argv[optind+1]; - } else if (remaining_args == 1) { - input_filename = argv[optind]; - } else { - input_filename = "-"; - } - - index_type index_pos; - location_handler_type location_handler(index_pos); - osmium::experimental::FlexReader exr(input_filename, location_handler, osmium::osm_entity_bits::object); - - MyOGRHandler ogr_handler(output_format, output_filename); - - while (auto buffer = exr.read()) { - osmium::apply(buffer, ogr_handler); - } - - exr.close(); - - std::vector incomplete_relations = exr.collector().get_incomplete_relations(); - if (!incomplete_relations.empty()) { - std::cerr << "Warning! Some member ways missing for these multipolygon relations:"; - for (const auto* relation : incomplete_relations) { - std::cerr << " " << relation->id(); - } - std::cerr << "\n"; - } -} - diff --git a/include/gdalcpp.hpp b/include/gdalcpp.hpp new file mode 100644 index 000000000..1502f2f0c --- /dev/null +++ b/include/gdalcpp.hpp @@ -0,0 +1,406 @@ +#ifndef GDALCPP_HPP +#define GDALCPP_HPP + +/* + +C++11 wrapper classes for GDAL/OGR. + +Version 1.1.1 + +https://github.com/joto/gdalcpp + +Copyright 2015 Jochen Topf + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace gdalcpp { + +#if GDAL_VERSION_MAJOR >= 2 + using gdal_driver_type = GDALDriver; + using gdal_dataset_type = GDALDataset; +#else + using gdal_driver_type = OGRSFDriver; + using gdal_dataset_type = OGRDataSource; +#endif + + /** + * Exception thrown for all errors in this class. + */ + class gdal_error : public std::runtime_error { + + std::string m_driver; + std::string m_dataset; + std::string m_layer; + std::string m_field; + OGRErr m_error; + + public: + + gdal_error(const std::string& message, + OGRErr error, + const std::string& driver = "", + const std::string& dataset = "", + const std::string& layer = "", + const std::string& field = "") : + std::runtime_error(message), + m_driver(driver), + m_dataset(dataset), + m_layer(layer), + m_field(field), + m_error(error) { + } + + const std::string& driver() const { + return m_driver; + } + + const std::string& dataset() const { + return m_dataset; + } + + const std::string& layer() const { + return m_layer; + } + + const std::string& field() const { + return m_field; + } + + OGRErr error() const { + return m_error; + } + + }; // class gdal_error + + namespace detail { + + struct init_wrapper { + init_wrapper() { OGRRegisterAll(); } + ~init_wrapper() { OGRCleanupAll(); } + }; + + struct init_library { + init_library() { + static init_wrapper iw; + } + }; + + class Driver : private init_library { + + gdal_driver_type* m_driver; + + public: + + Driver(const std::string& driver_name) : + init_library(), +#if GDAL_VERSION_MAJOR >= 2 + m_driver(GetGDALDriverManager()->GetDriverByName(driver_name.c_str())) { +#else + m_driver(OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(driver_name.c_str())) { +#endif + if (!m_driver) { + throw gdal_error(std::string("unknown driver: '") + driver_name + "'", OGRERR_NONE, driver_name); + } + } + + gdal_driver_type& get() const { + return *m_driver; + } + + }; // struct Driver + + struct Options { + + std::vector m_options; + std::unique_ptr m_ptrs; + + Options(const std::vector& options) : + m_options(options), + m_ptrs(new const char*[options.size()+1]) { + std::transform(m_options.begin(), m_options.end(), m_ptrs.get(), [&](const std::string& s) { + return s.data(); + }); + m_ptrs[options.size()] = nullptr; + } + + char** get() const { + return const_cast(m_ptrs.get()); + } + + }; // struct Options + + } // namespace detail + + class SRS { + + OGRSpatialReference m_spatial_reference; + + public: + + SRS() : + m_spatial_reference() { + auto result = m_spatial_reference.SetWellKnownGeogCS("WGS84"); + if (result != OGRERR_NONE) { + throw gdal_error(std::string("can not initialize spatial reference system WGS84"), result); + } + } + + explicit SRS(int epsg) : + m_spatial_reference() { + auto result = m_spatial_reference.importFromEPSG(epsg); + if (result != OGRERR_NONE) { + throw gdal_error(std::string("can not initialize spatial reference system for EPSG:") + std::to_string(epsg), result); + } + } + + explicit SRS(const char* name) : + m_spatial_reference() { + auto result = m_spatial_reference.importFromProj4(name); + if (result != OGRERR_NONE) { + throw gdal_error(std::string("can not initialize spatial reference system '") + name + "'", result); + } + } + + explicit SRS(const std::string& name) : + m_spatial_reference() { + auto result = m_spatial_reference.importFromProj4(name.c_str()); + if (result != OGRERR_NONE) { + throw gdal_error(std::string("can not initialize spatial reference system '") + name + "'", result); + } + } + + explicit SRS(const OGRSpatialReference& spatial_reference) : + m_spatial_reference(spatial_reference) { + } + + OGRSpatialReference& get() { + return m_spatial_reference; + } + + const OGRSpatialReference& get() const { + return m_spatial_reference; + } + + }; // class SRS + + class Dataset { + + struct gdal_dataset_deleter { + + void operator()(gdal_dataset_type* ds) { +#if GDAL_VERSION_MAJOR >= 2 + GDALClose(ds); +#else + OGRDataSource::DestroyDataSource(ds); +#endif + } + + }; // struct gdal_dataset_deleter + + std::string m_driver_name; + std::string m_dataset_name; + detail::Options m_options; + SRS m_srs; + std::unique_ptr m_dataset; + + public: + + Dataset(const std::string& driver_name, const std::string& dataset_name, const SRS& srs = SRS{}, const std::vector& options = {}) : + m_driver_name(driver_name), + m_dataset_name(dataset_name), + m_options(options), + m_srs(srs), +#if GDAL_VERSION_MAJOR >= 2 + m_dataset(detail::Driver(driver_name).get().Create(dataset_name.c_str(), 0, 0, 0, GDT_Unknown, m_options.get())) { +#else + m_dataset(detail::Driver(driver_name).get().CreateDataSource(dataset_name.c_str(), m_options.get())) { +#endif + if (!m_dataset) { + throw gdal_error(std::string("failed to create dataset '") + dataset_name + "'", OGRERR_NONE, driver_name, dataset_name); + } + } + + const std::string& driver_name() const { + return m_driver_name; + } + + const std::string& dataset_name() const { + return m_dataset_name; + } + + gdal_dataset_type& get() const { + return *m_dataset; + } + + SRS& srs() { + return m_srs; + } + + void exec(const char* sql) { + auto result = m_dataset->ExecuteSQL(sql, nullptr, nullptr); + if (result) { + m_dataset->ReleaseResultSet(result); + } + } + + void exec(const std::string& sql) { + exec(sql.c_str()); + } + + + Dataset& start_transaction() { +#if GDAL_VERSION_MAJOR >= 2 + m_dataset->StartTransaction(); +#endif + return *this; + } + + Dataset& commit_transaction() { +#if GDAL_VERSION_MAJOR >= 2 + m_dataset->CommitTransaction(); +#endif + return *this; + } + + }; // class Dataset + + class Layer { + + detail::Options m_options; + Dataset& m_dataset; + OGRLayer* m_layer; + + public: + + Layer(Dataset& dataset, const std::string& layer_name, OGRwkbGeometryType type, const std::vector& options = {}) : + m_options(options), + m_dataset(dataset), + m_layer(dataset.get().CreateLayer(layer_name.c_str(), &dataset.srs().get(), type, m_options.get())) { + if (!m_layer) { + throw gdal_error(std::string("failed to create layer '") + layer_name + "'", OGRERR_NONE, + dataset.driver_name(), dataset.dataset_name(), layer_name); + } + } + + OGRLayer& get() { + return *m_layer; + } + + const OGRLayer& get() const { + return *m_layer; + } + + Dataset& dataset() const { + return m_dataset; + } + + const char* name() const { + return m_layer->GetName(); + } + + Layer& add_field(const std::string& field_name, OGRFieldType type, int width, int precision=0) { + OGRFieldDefn field(field_name.c_str(), type); + field.SetWidth(width); + field.SetPrecision(precision); + + if (m_layer->CreateField(&field) != OGRERR_NONE) { + throw gdal_error(std::string("failed to create field '") + field_name + "' in layer '" + name() + "'", OGRERR_NONE, + m_dataset.driver_name(), m_dataset.dataset_name(), name(), field_name); + } + + return *this; + } + + Layer& start_transaction() { + OGRErr result = m_layer->StartTransaction(); + if (result != OGRERR_NONE) { + throw gdal_error(std::string("starting transaction on layer '") + name() + "' failed", result, m_dataset.driver_name(), m_dataset.dataset_name(), name()); + } + return *this; + } + + Layer& commit_transaction() { + OGRErr result = m_layer->CommitTransaction(); + if (result != OGRERR_NONE) { + throw gdal_error(std::string("committing transaction on layer '") + name() + "' failed", result, m_dataset.driver_name(), m_dataset.dataset_name(), name()); + } + return *this; + } + + }; // class Layer + + class Feature { + + Layer& m_layer; + OGRFeature m_feature; + + public: + + Feature(Layer& layer, std::unique_ptr&& geometry) : + m_layer(layer), + m_feature(m_layer.get().GetLayerDefn()) { + OGRErr result = m_feature.SetGeometryDirectly(geometry.release()); + if (result != OGRERR_NONE) { + throw gdal_error(std::string("setting feature geometry in layer '") + m_layer.name() + "' failed", result, m_layer.dataset().driver_name(), m_layer.dataset().dataset_name()); + } + } + + void add_to_layer() { + OGRErr result = m_layer.get().CreateFeature(&m_feature); + if (result != OGRERR_NONE) { + throw gdal_error(std::string("creating feature in layer '") + m_layer.name() + "' failed", result, m_layer.dataset().driver_name(), m_layer.dataset().dataset_name()); + } + } + + template + Feature& set_field(int n, T&& arg) { + m_feature.SetField(n, std::forward(arg)); + return *this; + } + + template + Feature& set_field(const char* name, T&& arg) { + m_feature.SetField(name, std::forward(arg)); + return *this; + } + + }; // class Feature + +} // namespace gdalcpp + +#endif // GDALCPP_HPP diff --git a/include/osmium/area/assembler.hpp b/include/osmium/area/assembler.hpp index f17299109..87feea29e 100644 --- a/include/osmium/area/assembler.hpp +++ b/include/osmium/area/assembler.hpp @@ -34,9 +34,13 @@ DEALINGS IN THE SOFTWARE. */ #include +#include +#include #include #include #include +#include +#include #include #include diff --git a/include/osmium/area/detail/node_ref_segment.hpp b/include/osmium/area/detail/node_ref_segment.hpp index 43569a865..aab5b8fa3 100644 --- a/include/osmium/area/detail/node_ref_segment.hpp +++ b/include/osmium/area/detail/node_ref_segment.hpp @@ -113,7 +113,7 @@ namespace osmium { return m_second; } - bool to_left_of(const osmium::Location location) const { + bool to_left_of(const osmium::Location& location) const { // std::cerr << "segment " << first() << "--" << second() << " to_left_of(" << location << "\n"; if (first().location() == location || second().location() == location) { @@ -195,8 +195,8 @@ namespace osmium { } inline bool y_range_overlap(const NodeRefSegment& s1, const NodeRefSegment& s2) { - auto m1 = std::minmax(s1.first().location().y(), s1.second().location().y()); - auto m2 = std::minmax(s2.first().location().y(), s2.second().location().y()); + const std::pair m1 = std::minmax(s1.first().location().y(), s1.second().location().y()); + const std::pair m2 = std::minmax(s2.first().location().y(), s2.second().location().y()); if (m1.first > m2.second || m2.first > m1.second) { return false; } @@ -204,19 +204,25 @@ namespace osmium { } /** - * Calculate the intersection between to NodeRefSegments. The result is returned - * as a Location. Note that because the Location uses integers with limited - * precision internally, the result might be slightly different than the - * numerically correct location. + * Calculate the intersection between two NodeRefSegments. The + * result is returned as a Location. Note that because the Location + * uses integers with limited precision internally, the result + * might be slightly different than the numerically correct + * location. * - * If the segments touch in one of their endpoints, it doesn't count as an - * intersection. + * This function uses integer arithmentic as much as possible and + * will not work if the segments are longer than about half the + * planet. This shouldn't happen with real data, so it isn't a big + * problem. * - * If the segments intersect not in a single point but in multiple points, ie - * if they overlap, this is NOT detected. + * If the segments touch in one of their endpoints, it doesn't + * count as an intersection. * - * @returns Undefined osmium::Location if there is no intersection or a defined - * Location if the segments intersect. + * If the segments intersect not in a single point but in multiple + * points, ie if they overlap, this is NOT detected. + * + * @returns Undefined osmium::Location if there is no intersection + * or a defined Location if the segments intersect. */ inline osmium::Location calculate_intersection(const NodeRefSegment& s1, const NodeRefSegment& s2) { if (s1.first().location() == s2.first().location() || @@ -226,26 +232,32 @@ namespace osmium { return osmium::Location(); } - auto d = (static_cast(s2.second().y()) - static_cast(s2.first().y())) * - (static_cast(s1.second().x()) - static_cast(s1.first().x())) - - (static_cast(s2.second().x()) - static_cast(s2.first().x())) * - (static_cast(s1.second().y()) - static_cast(s1.first().y())); + int64_t s1ax = s1.first().x(); + int64_t s1ay = s1.first().y(); + int64_t s1bx = s1.second().x(); + int64_t s1by = s1.second().y(); + int64_t s2ax = s2.first().x(); + int64_t s2ay = s2.first().y(); + int64_t s2bx = s2.second().x(); + int64_t s2by = s2.second().y(); + + int64_t d = (s2by - s2ay) * (s1bx - s1ax) - + (s2bx - s2ax) * (s1by - s1ay); if (d != 0) { - double denom = ((s2.second().lat() - s2.first().lat())*(s1.second().lon() - s1.first().lon())) - - ((s2.second().lon() - s2.first().lon())*(s1.second().lat() - s1.first().lat())); + int64_t na = (s2bx - s2ax) * (s1ay - s2ay) - + (s2by - s2ay) * (s1ax - s2ax); - double nume_a = ((s2.second().lon() - s2.first().lon())*(s1.first().lat() - s2.first().lat())) - - ((s2.second().lat() - s2.first().lat())*(s1.first().lon() - s2.first().lon())); + int64_t nb = (s1bx - s1ax) * (s1ay - s2ay) - + (s1by - s1ay) * (s1ax - s2ax); - double nume_b = ((s1.second().lon() - s1.first().lon())*(s1.first().lat() - s2.first().lat())) - - ((s1.second().lat() - s1.first().lat())*(s1.first().lon() - s2.first().lon())); + if ((d > 0 && na >= 0 && na <= d && nb >= 0 && nb <= d) || + (d < 0 && na <= 0 && na >= d && nb <= 0 && nb >= d)) { + + double ua = double(na) / d; + int32_t ix = int32_t(s1ax + ua*(s1bx - s1ax)); + int32_t iy = int32_t(s1ay + ua*(s1by - s1ay)); - if ((denom > 0 && nume_a >= 0 && nume_a <= denom && nume_b >= 0 && nume_b <= denom) || - (denom < 0 && nume_a <= 0 && nume_a >= denom && nume_b <= 0 && nume_b >= denom)) { - double ua = nume_a / denom; - double ix = s1.first().lon() + ua*(s1.second().lon() - s1.first().lon()); - double iy = s1.first().lat() + ua*(s1.second().lat() - s1.first().lat()); return osmium::Location(ix, iy); } } diff --git a/include/osmium/area/detail/proto_ring.hpp b/include/osmium/area/detail/proto_ring.hpp index c0f545c98..162e2896d 100644 --- a/include/osmium/area/detail/proto_ring.hpp +++ b/include/osmium/area/detail/proto_ring.hpp @@ -34,12 +34,14 @@ DEALINGS IN THE SOFTWARE. */ #include -#include +#include +#include #include -#include +#include #include #include +#include #include #include @@ -148,7 +150,8 @@ namespace osmium { } void swap_segments(ProtoRing& other) { - std::swap(m_segments, other.m_segments); + using std::swap; + swap(m_segments, other.m_segments); } void add_inner_ring(ProtoRing* ring) { diff --git a/include/osmium/area/detail/segment_list.hpp b/include/osmium/area/detail/segment_list.hpp index ca6071ede..289ecf0c9 100644 --- a/include/osmium/area/detail/segment_list.hpp +++ b/include/osmium/area/detail/segment_list.hpp @@ -41,6 +41,8 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include +#include #include #include diff --git a/include/osmium/area/multipolygon_collector.hpp b/include/osmium/area/multipolygon_collector.hpp index bf2a4ce86..2881597b0 100644 --- a/include/osmium/area/multipolygon_collector.hpp +++ b/include/osmium/area/multipolygon_collector.hpp @@ -53,7 +53,7 @@ namespace osmium { namespace relations { class RelationMeta; - } + } // namespace relations /** * @brief Code related to the building of areas (multipolygons) from relations. @@ -71,7 +71,7 @@ namespace osmium { * * @tparam TAssembler Multipolygon Assembler class. */ - template + template class MultipolygonCollector : public osmium::relations::Collector, false, true, false> { typedef typename osmium::relations::Collector, false, true, false> collector_type; @@ -87,7 +87,8 @@ namespace osmium { void flush_output_buffer() { if (this->callback()) { osmium::memory::Buffer buffer(initial_output_buffer_size); - std::swap(buffer, m_output_buffer); + using std::swap; + swap(buffer, m_output_buffer); this->callback()(std::move(buffer)); } } @@ -176,28 +177,6 @@ namespace osmium { } catch (osmium::invalid_location&) { // XXX ignore } - - // clear member metas - for (const auto& member : relation.members()) { - if (member.ref() != 0) { - auto& mmv = this->member_meta(member.type()); - auto range = std::equal_range(mmv.begin(), mmv.end(), osmium::relations::MemberMeta(member.ref())); - assert(range.first != range.second); - - // if this is the last time this object was needed - // then mark it as removed - if (osmium::relations::count_not_removed(range.first, range.second) == 1) { - this->get_member(range.first->buffer_offset()).set_removed(true); - } - - for (auto it = range.first; it != range.second; ++it) { - if (!it->removed() && relation.id() == this->get_relation(it->relation_pos()).id()) { - it->remove(); - break; - } - } - } - } } void flush() { @@ -206,7 +185,10 @@ namespace osmium { osmium::memory::Buffer read() { osmium::memory::Buffer buffer(initial_output_buffer_size, osmium::memory::Buffer::auto_grow::yes); - std::swap(buffer, m_output_buffer); + + using std::swap; + swap(buffer, m_output_buffer); + return buffer; } diff --git a/include/osmium/area/problem_reporter_exception.hpp b/include/osmium/area/problem_reporter_exception.hpp index 5e743c6d8..7c9a5e393 100644 --- a/include/osmium/area/problem_reporter_exception.hpp +++ b/include/osmium/area/problem_reporter_exception.hpp @@ -54,7 +54,7 @@ namespace osmium { ProblemReporterStream(m_sstream) { } - virtual ~ProblemReporterException() = default; + ~ProblemReporterException() override = default; void report_duplicate_node(osmium::object_id_type node_id1, osmium::object_id_type node_id2, osmium::Location location) override { m_sstream.str(); diff --git a/include/osmium/area/problem_reporter_ogr.hpp b/include/osmium/area/problem_reporter_ogr.hpp index c437a3f57..68fae3bb8 100644 --- a/include/osmium/area/problem_reporter_ogr.hpp +++ b/include/osmium/area/problem_reporter_ogr.hpp @@ -42,34 +42,12 @@ DEALINGS IN THE SOFTWARE. * @attention If you include this file, you'll need to link with `libgdal`. */ -#ifdef _MSC_VER -# pragma warning(push) -# pragma warning(disable : 4458) -#else -# pragma GCC diagnostic push -# ifdef __clang__ -# pragma GCC diagnostic ignored "-Wdocumentation-unknown-command" -# endif -# pragma GCC diagnostic ignored "-Wfloat-equal" -# pragma GCC diagnostic ignored "-Wold-style-cast" -# pragma GCC diagnostic ignored "-Wpadded" -# pragma GCC diagnostic ignored "-Wredundant-decls" -# pragma GCC diagnostic ignored "-Wshadow" -#endif - -#include -#include - -#ifdef _MSC_VER -# pragma warning(pop) -#else -# pragma GCC diagnostic pop -#endif - #include -#include + +#include #include +#include #include #include #include @@ -86,24 +64,15 @@ namespace osmium { osmium::geom::OGRFactory<> m_ogr_factory; - OGRDataSource* m_data_source; - - OGRLayer* m_layer_perror; - OGRLayer* m_layer_lerror; + gdalcpp::Layer m_layer_perror; + gdalcpp::Layer m_layer_lerror; void write_point(const char* problem_type, osmium::object_id_type id1, osmium::object_id_type id2, osmium::Location location) { - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_perror->GetLayerDefn()); - std::unique_ptr ogr_point = m_ogr_factory.create_point(location); - feature->SetGeometry(ogr_point.get()); - feature->SetField("id1", static_cast(id1)); - feature->SetField("id2", static_cast(id2)); - feature->SetField("problem_type", problem_type); - - if (m_layer_perror->CreateFeature(feature) != OGRERR_NONE) { - std::runtime_error("Failed to create feature on layer 'perrors'"); - } - - OGRFeature::DestroyFeature(feature); + gdalcpp::Feature feature(m_layer_perror, m_ogr_factory.create_point(location)); + feature.set_field("id1", static_cast(id1)); + feature.set_field("id2", static_cast(id2)); + feature.set_field("problem_type", problem_type); + feature.add_to_layer(); } void write_line(const char* problem_type, osmium::object_id_type id1, osmium::object_id_type id2, osmium::Location loc1, osmium::Location loc2) { @@ -112,83 +81,30 @@ namespace osmium { std::unique_ptr ogr_linestring = std::unique_ptr(new OGRLineString()); ogr_linestring->addPoint(ogr_point1.get()); ogr_linestring->addPoint(ogr_point2.get()); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_lerror->GetLayerDefn()); - feature->SetGeometry(ogr_linestring.get()); - feature->SetField("id1", static_cast(id1)); - feature->SetField("id2", static_cast(id2)); - feature->SetField("problem_type", problem_type); - if (m_layer_lerror->CreateFeature(feature) != OGRERR_NONE) { - std::runtime_error("Failed to create feature on layer 'lerrors'"); - } - - OGRFeature::DestroyFeature(feature); + gdalcpp::Feature feature(m_layer_lerror, std::move(ogr_linestring)); + feature.set_field("id1", static_cast(id1)); + feature.set_field("id2", static_cast(id2)); + feature.set_field("problem_type", problem_type); + feature.add_to_layer(); } public: - explicit ProblemReporterOGR(OGRDataSource* data_source) : - m_data_source(data_source) { + explicit ProblemReporterOGR(gdalcpp::Dataset& dataset) : + m_layer_perror(dataset, "perrors", wkbPoint), + m_layer_lerror(dataset, "lerrors", wkbLineString) { - OGRSpatialReference sparef; - sparef.SetWellKnownGeogCS("WGS84"); + m_layer_perror.add_field("id1", OFTReal, 10); + m_layer_perror.add_field("id2", OFTReal, 10); + m_layer_perror.add_field("problem_type", OFTString, 30); - m_layer_perror = m_data_source->CreateLayer("perrors", &sparef, wkbPoint, nullptr); - if (!m_layer_perror) { - std::runtime_error("Layer creation failed for layer 'perrors'"); - } - - OGRFieldDefn layer_perror_field_id1("id1", OFTReal); - layer_perror_field_id1.SetWidth(10); - - if (m_layer_perror->CreateField(&layer_perror_field_id1) != OGRERR_NONE) { - std::runtime_error("Creating field 'id1' failed for layer 'perrors'"); - } - - OGRFieldDefn layer_perror_field_id2("id2", OFTReal); - layer_perror_field_id2.SetWidth(10); - - if (m_layer_perror->CreateField(&layer_perror_field_id2) != OGRERR_NONE) { - std::runtime_error("Creating field 'id2' failed for layer 'perrors'"); - } - - OGRFieldDefn layer_perror_field_problem_type("problem_type", OFTString); - layer_perror_field_problem_type.SetWidth(30); - - if (m_layer_perror->CreateField(&layer_perror_field_problem_type) != OGRERR_NONE) { - std::runtime_error("Creating field 'problem_type' failed for layer 'perrors'"); - } - - /**************/ - - m_layer_lerror = m_data_source->CreateLayer("lerrors", &sparef, wkbLineString, nullptr); - if (!m_layer_lerror) { - std::runtime_error("Layer creation failed for layer 'lerrors'"); - } - - OGRFieldDefn layer_lerror_field_id1("id1", OFTReal); - layer_lerror_field_id1.SetWidth(10); - - if (m_layer_lerror->CreateField(&layer_lerror_field_id1) != OGRERR_NONE) { - std::runtime_error("Creating field 'id1' failed for layer 'lerrors'"); - } - - OGRFieldDefn layer_lerror_field_id2("id2", OFTReal); - layer_lerror_field_id2.SetWidth(10); - - if (m_layer_lerror->CreateField(&layer_lerror_field_id2) != OGRERR_NONE) { - std::runtime_error("Creating field 'id2' failed for layer 'lerrors'"); - } - - OGRFieldDefn layer_lerror_field_problem_type("problem_type", OFTString); - layer_lerror_field_problem_type.SetWidth(30); - - if (m_layer_lerror->CreateField(&layer_lerror_field_problem_type) != OGRERR_NONE) { - std::runtime_error("Creating field 'problem_type' failed for layer 'lerrors'"); - } + m_layer_lerror.add_field("id1", OFTReal, 10); + m_layer_lerror.add_field("id2", OFTReal, 10); + m_layer_lerror.add_field("problem_type", OFTString, 30); } - virtual ~ProblemReporterOGR() = default; + ~ProblemReporterOGR() override = default; void report_duplicate_node(osmium::object_id_type node_id1, osmium::object_id_type node_id2, osmium::Location location) override { write_point("duplicate_node", node_id1, node_id2, location); diff --git a/include/osmium/area/problem_reporter_stream.hpp b/include/osmium/area/problem_reporter_stream.hpp index ddcb343e8..b6a004cdf 100644 --- a/include/osmium/area/problem_reporter_stream.hpp +++ b/include/osmium/area/problem_reporter_stream.hpp @@ -54,7 +54,7 @@ namespace osmium { m_out(&out) { } - virtual ~ProblemReporterStream() = default; + ~ProblemReporterStream() override = default; void header(const char* msg) { *m_out << "DATA PROBLEM: " << msg << " on " << item_type_to_char(m_object_type) << m_object_id << ": "; diff --git a/include/osmium/builder/builder.hpp b/include/osmium/builder/builder.hpp index 4424d88e1..63eb4bb2d 100644 --- a/include/osmium/builder/builder.hpp +++ b/include/osmium/builder/builder.hpp @@ -134,7 +134,7 @@ namespace osmium { * Reserve space for an object of class T in buffer and return * pointer to it. */ - template + template T* reserve_space_for() { assert(m_buffer.is_aligned()); return reinterpret_cast(m_buffer.reserve_space(sizeof(T))); @@ -182,7 +182,7 @@ namespace osmium { }; // class Builder - template + template class ObjectBuilder : public Builder { static_assert(std::is_base_of::value, "ObjectBuilder can only build objects derived from osmium::memory::Item"); diff --git a/include/osmium/builder/osm_object_builder.hpp b/include/osmium/builder/osm_object_builder.hpp index 074076ce6..82b4b23ce 100644 --- a/include/osmium/builder/osm_object_builder.hpp +++ b/include/osmium/builder/osm_object_builder.hpp @@ -33,9 +33,11 @@ DEALINGS IN THE SOFTWARE. */ +#include #include #include #include +#include #include #include @@ -53,7 +55,7 @@ namespace osmium { namespace memory { class Buffer; - } + } // namespace memory namespace builder { @@ -76,6 +78,12 @@ namespace osmium { * @param value Tag value (0-terminated string). */ void add_tag(const char* key, const char* value) { + if (std::strlen(key) > osmium::max_osm_string_length) { + throw std::length_error("OSM tag key is too long"); + } + if (std::strlen(value) > osmium::max_osm_string_length) { + throw std::length_error("OSM tag value is too long"); + } add_size(append(key) + append(value)); } @@ -87,8 +95,15 @@ namespace osmium { * @param value Pointer to tag value. * @param value_length Length of value (not including the \0 byte). */ - void add_tag(const char* key, const string_size_type key_length, const char* value, const string_size_type value_length) { - add_size(append(key, key_length) + append_zero() + append(value, value_length) + append_zero()); + void add_tag(const char* key, const size_t key_length, const char* value, const size_t value_length) { + if (key_length > osmium::max_osm_string_length) { + throw std::length_error("OSM tag key is too long"); + } + if (value_length > osmium::max_osm_string_length) { + throw std::length_error("OSM tag value is too long"); + } + add_size(append(key, osmium::memory::item_size_type(key_length)) + append_zero() + + append(value, osmium::memory::item_size_type(value_length)) + append_zero()); } /** @@ -98,13 +113,46 @@ namespace osmium { * @param value Tag value. */ void add_tag(const std::string& key, const std::string& value) { - add_size(append(key.data(), static_cast_with_assert(key.size() + 1)) + - append(value.data(), static_cast_with_assert(value.size() + 1))); + if (key.size() > osmium::max_osm_string_length) { + throw std::length_error("OSM tag key is too long"); + } + if (value.size() > osmium::max_osm_string_length) { + throw std::length_error("OSM tag value is too long"); + } + add_size(append(key.data(), osmium::memory::item_size_type(key.size()) + 1) + + append(value.data(), osmium::memory::item_size_type(value.size()) + 1)); + } + + /** + * Add tag to buffer. + * + * @param tag Tag. + */ + void add_tag(const osmium::Tag& tag) { + add_size(append(tag.key()) + append(tag.value())); + } + + /** + * Add tag to buffer. + * + * @param tag Pair of key/value 0-terminated strings. + */ + void add_tag(const std::pair& tag) { + add_tag(tag.first, tag.second); + } + + /** + * Add tag to buffer. + * + * @param tag Pair of std::string references. + */ + void add_tag(const std::pair& tag) { + add_tag(tag.first, tag.second); } }; // class TagListBuilder - template + template class NodeRefListBuilder : public ObjectBuilder { public: @@ -122,7 +170,7 @@ namespace osmium { static_cast(this)->add_size(sizeof(osmium::NodeRef)); } - void add_node_ref(const object_id_type ref, const osmium::Location location = Location()) { + void add_node_ref(const object_id_type ref, const osmium::Location& location = Location{}) { add_node_ref(NodeRef(ref, location)); } @@ -141,35 +189,17 @@ namespace osmium { * will be set. * @param role The role. * @param length Length of role (without \0 termination). + * @throws std:length_error If role is longer than osmium::max_osm_string_length */ - void add_role(osmium::RelationMember& member, const char* role, const string_size_type length) { - member.set_role_size(length + 1); - add_size(append(role, length) + append_zero()); + void add_role(osmium::RelationMember& member, const char* role, const size_t length) { + if (length > osmium::max_osm_string_length) { + throw std::length_error("OSM relation member role is too long"); + } + member.set_role_size(osmium::string_size_type(length) + 1); + add_size(append(role, osmium::memory::item_size_type(length)) + append_zero()); add_padding(true); } - /** - * Add role to buffer. - * - * @param member Relation member object where the length of the role - * will be set. - * @param role \0-terminated role. - */ - void add_role(osmium::RelationMember& member, const char* role) { - add_role(member, role, static_cast_with_assert(std::strlen(role))); - } - - /** - * Add role to buffer. - * - * @param member Relation member object where the length of the role - * will be set. - * @param role Role. - */ - void add_role(osmium::RelationMember& member, const std::string& role) { - add_role(member, role.data(), static_cast_with_assert(role.size())); - } - public: explicit RelationMemberListBuilder(osmium::memory::Buffer& buffer, Builder* parent = nullptr) : @@ -190,8 +220,10 @@ namespace osmium { * @param full_member Optional pointer to the member object. If it * is available a copy will be added to the * relation. + * @throws std:length_error If role_length is greater than + * osmium::max_osm_string_length */ - void add_member(osmium::item_type type, object_id_type ref, const char* role, const string_size_type role_length, const osmium::OSMObject* full_member = nullptr) { + void add_member(osmium::item_type type, object_id_type ref, const char* role, const size_t role_length, const osmium::OSMObject* full_member = nullptr) { osmium::RelationMember* member = reserve_space_for(); new (member) osmium::RelationMember(ref, type, full_member != nullptr); add_size(sizeof(RelationMember)); @@ -210,9 +242,10 @@ namespace osmium { * @param full_member Optional pointer to the member object. If it * is available a copy will be added to the * relation. + * @throws std:length_error If role is longer than osmium::max_osm_string_length */ void add_member(osmium::item_type type, object_id_type ref, const char* role, const osmium::OSMObject* full_member = nullptr) { - add_member(type, ref, role, strlen(role), full_member); + add_member(type, ref, role, std::strlen(role), full_member); } /** @@ -224,6 +257,7 @@ namespace osmium { * @param full_member Optional pointer to the member object. If it * is available a copy will be added to the * relation. + * @throws std:length_error If role is longer than osmium::max_osm_string_length */ void add_member(osmium::item_type type, object_id_type ref, const std::string& role, const osmium::OSMObject* full_member = nullptr) { add_member(type, ref, role.data(), role.size(), full_member); @@ -231,7 +265,65 @@ namespace osmium { }; // class RelationMemberListBuilder - template + class ChangesetDiscussionBuilder : public ObjectBuilder { + + osmium::ChangesetComment* m_comment = nullptr; + + void add_user(osmium::ChangesetComment& comment, const char* user, const size_t length) { + if (length > osmium::max_osm_string_length) { + throw std::length_error("OSM user name is too long"); + } + comment.set_user_size(osmium::string_size_type(length) + 1); + add_size(append(user, osmium::memory::item_size_type(length)) + append_zero()); + } + + void add_text(osmium::ChangesetComment& comment, const char* text, const size_t length) { + // XXX There is no limit on the length of a comment text. We + // limit it here to 2^16-2 characters, because that's all that + // will fit into our internal data structure. This is not ideal, + // and will have to be discussed and cleared up. + if (length > std::numeric_limits::max() - 1) { + throw std::length_error("OSM changeset comment is too long"); + } + comment.set_text_size(osmium::string_size_type(length) + 1); + add_size(append(text, osmium::memory::item_size_type(length)) + append_zero()); + add_padding(true); + } + + public: + + explicit ChangesetDiscussionBuilder(osmium::memory::Buffer& buffer, Builder* parent = nullptr) : + ObjectBuilder(buffer, parent) { + } + + ~ChangesetDiscussionBuilder() { + assert(!m_comment && "You have to always call both add_comment() and then add_comment_text() in that order for each comment!"); + add_padding(); + } + + void add_comment(osmium::Timestamp date, osmium::user_id_type uid, const char* user) { + assert(!m_comment && "You have to always call both add_comment() and then add_comment_text() in that order for each comment!"); + m_comment = reserve_space_for(); + new (m_comment) osmium::ChangesetComment(date, uid); + add_size(sizeof(ChangesetComment)); + add_user(*m_comment, user, std::strlen(user)); + } + + void add_comment_text(const char* text) { + assert(m_comment && "You have to always call both add_comment() and then add_comment_text() in that order for each comment!"); + add_text(*m_comment, text, std::strlen(text)); + m_comment = nullptr; + } + + void add_comment_text(const std::string& text) { + assert(m_comment && "You have to always call both add_comment() and then add_comment_text() in that order for each comment!"); + add_text(*m_comment, text.c_str(), text.size()); + m_comment = nullptr; + } + + }; // class ChangesetDiscussionBuilder + + template class OSMObjectBuilder : public ObjectBuilder { public: diff --git a/include/osmium/diff_handler.hpp b/include/osmium/diff_handler.hpp index 4f9b3a1a0..06d8a93ed 100644 --- a/include/osmium/diff_handler.hpp +++ b/include/osmium/diff_handler.hpp @@ -46,8 +46,7 @@ namespace osmium { public: - DiffHandler() { - } + DiffHandler() = default; void node(const osmium::DiffNode&) const { } diff --git a/include/osmium/diff_iterator.hpp b/include/osmium/diff_iterator.hpp index 0ddf7ff7a..6725fecc8 100644 --- a/include/osmium/diff_iterator.hpp +++ b/include/osmium/diff_iterator.hpp @@ -43,7 +43,12 @@ namespace osmium { class OSMObject; - template + /** + * An input iterator wrapping any iterator over OSMObjects. When + * dereferenced it will yield DiffObject objects pointing to the + * underlying OSMObjects. + */ + template class DiffIterator : public std::iterator { static_assert(std::is_base_of::value, "TBasicIterator::value_type must derive from osmium::OSMObject"); @@ -56,37 +61,29 @@ namespace osmium { mutable osmium::DiffObject m_diff; - void set_diff() const { + void set_diff() const noexcept { assert(m_curr != m_end); - TBasicIterator prev = m_prev; - if (prev->type() != m_curr->type() || prev->id() != m_curr->id()) { - prev = m_curr; - } + bool use_curr_for_prev = m_prev->type() != m_curr->type() || m_prev->id() != m_curr->id(); + bool use_curr_for_next = m_next == m_end || m_next->type() != m_curr->type() || m_next->id() != m_curr->id(); - TBasicIterator next = m_next; - if (next == m_end || next->type() != m_curr->type() || next->id() != m_curr->id()) { - next = m_curr; - } - - m_diff = osmium::DiffObject(*prev, *m_curr, *next); + m_diff = std::move(osmium::DiffObject{ + *(use_curr_for_prev ? m_curr : m_prev), + *m_curr, + *(use_curr_for_next ? m_curr : m_next) + }); } public: - explicit DiffIterator(TBasicIterator begin, TBasicIterator end) : + DiffIterator(TBasicIterator begin, TBasicIterator end) : m_prev(begin), m_curr(begin), m_next(begin == end ? begin : ++begin), - m_end(end) { + m_end(std::move(end)), + m_diff() { } - DiffIterator(const DiffIterator&) = default; - DiffIterator& operator=(const DiffIterator&) = default; - - DiffIterator(DiffIterator&&) = default; - DiffIterator& operator=(DiffIterator&&) = default; - DiffIterator& operator++() { m_prev = std::move(m_curr); m_curr = m_next; @@ -104,26 +101,35 @@ namespace osmium { return tmp; } - bool operator==(const DiffIterator& rhs) const { + bool operator==(const DiffIterator& rhs) const noexcept { return m_curr == rhs.m_curr && m_end == rhs.m_end; } - bool operator!=(const DiffIterator& rhs) const { + bool operator!=(const DiffIterator& rhs) const noexcept { return !(*this == rhs); } - reference operator*() const { + reference operator*() const noexcept { set_diff(); return m_diff; } - pointer operator->() const { + pointer operator->() const noexcept { set_diff(); return &m_diff; } }; // class DiffIterator + /** + * Create a DiffIterator based on the given iterators. + */ + template + inline DiffIterator make_diff_iterator(TBasicIterator begin, + TBasicIterator end) { + return DiffIterator{begin, end}; + } + } // namespace osmium #endif // OSMIUM_DIFF_ITERATOR_HPP diff --git a/include/osmium/diff_visitor.hpp b/include/osmium/diff_visitor.hpp index 5e72a7bb7..ac16a8ed4 100644 --- a/include/osmium/diff_visitor.hpp +++ b/include/osmium/diff_visitor.hpp @@ -43,7 +43,7 @@ namespace osmium { namespace detail { - template + template inline void apply_diff_iterator_recurse(const osmium::DiffObject& diff, THandler& handler) { switch (diff.type()) { case osmium::item_type::node: @@ -60,7 +60,7 @@ namespace osmium { } } - template + template inline void apply_diff_iterator_recurse(const osmium::DiffObject& diff, THandler& handler, TRest&... more) { apply_diff_iterator_recurse(diff, handler); apply_diff_iterator_recurse(diff, more...); @@ -68,9 +68,9 @@ namespace osmium { } // namespace detail - template + template inline void apply_diff(TIterator it, TIterator end, THandlers&... handlers) { - typedef osmium::DiffIterator diff_iterator; + using diff_iterator = osmium::DiffIterator; diff_iterator dit(it, end); diff_iterator dend(end, end); @@ -82,19 +82,19 @@ namespace osmium { class OSMObject; - template + template inline void apply_diff(TSource& source, THandlers&... handlers) { apply_diff(osmium::io::InputIterator {source}, osmium::io::InputIterator {}, handlers...); } - template + template inline void apply_diff(osmium::memory::Buffer& buffer, THandlers&... handlers) { apply_diff(buffer.begin(), buffer.end(), handlers...); } - template + template inline void apply_diff(const osmium::memory::Buffer& buffer, THandlers&... handlers) { apply_diff(buffer.cbegin(), buffer.cend(), handlers...); } diff --git a/include/osmium/dynamic_handler.hpp b/include/osmium/dynamic_handler.hpp index 9d0bd66f5..7077554ff 100644 --- a/include/osmium/dynamic_handler.hpp +++ b/include/osmium/dynamic_handler.hpp @@ -36,16 +36,11 @@ DEALINGS IN THE SOFTWARE. #include #include +#include #include namespace osmium { - class Node; - class Way; - class Relation; - class Area; - class Changeset; - namespace handler { namespace detail { @@ -83,11 +78,11 @@ namespace osmium { // to either call handler style functions or visitor style operator(). #define OSMIUM_DYNAMIC_HANDLER_DISPATCH(_name_, _type_) \ -template \ +template \ auto _name_##_dispatch(THandler& handler, const osmium::_type_& object, int) -> decltype(handler._name_(object), void()) { \ handler._name_(object); \ } \ -template \ +template \ auto _name_##_dispatch(THandler& handler, const osmium::_type_& object, long) -> decltype(handler(object), void()) { \ handler(object); \ } @@ -98,47 +93,47 @@ auto _name_##_dispatch(THandler& handler, const osmium::_type_& object, long) -> OSMIUM_DYNAMIC_HANDLER_DISPATCH(changeset, Changeset) OSMIUM_DYNAMIC_HANDLER_DISPATCH(area, Area) - template + template auto flush_dispatch(THandler& handler, int) -> decltype(handler.flush(), void()) { handler.flush(); } - template + template void flush_dispatch(THandler&, long) {} - template + template class HandlerWrapper : public HandlerWrapperBase { THandler m_handler; public: - template + template HandlerWrapper(TArgs&&... args) : m_handler(std::forward(args)...) { } - void node(const osmium::Node& node) override final { + void node(const osmium::Node& node) final { node_dispatch(m_handler, node, 0); } - void way(const osmium::Way& way) override final { + void way(const osmium::Way& way) final { way_dispatch(m_handler, way, 0); } - void relation(const osmium::Relation& relation) override final { + void relation(const osmium::Relation& relation) final { relation_dispatch(m_handler, relation, 0); } - void area(const osmium::Area& area) override final { + void area(const osmium::Area& area) final { area_dispatch(m_handler, area, 0); } - void changeset(const osmium::Changeset& changeset) override final { + void changeset(const osmium::Changeset& changeset) final { changeset_dispatch(m_handler, changeset, 0); } - void flush() override final { + void flush() final { flush_dispatch(m_handler, 0); } @@ -157,7 +152,7 @@ auto _name_##_dispatch(THandler& handler, const osmium::_type_& object, long) -> m_impl(impl_ptr(new osmium::handler::detail::HandlerWrapperBase)) { } - template + template void set(TArgs&&... args) { m_impl = impl_ptr(new osmium::handler::detail::HandlerWrapper(std::forward(args)...)); } @@ -188,7 +183,7 @@ auto _name_##_dispatch(THandler& handler, const osmium::_type_& object, long) -> }; // class DynamicHandler - } // namspace handler + } // namespace handler } // namespace osmium diff --git a/include/osmium/experimental/flex_reader.hpp b/include/osmium/experimental/flex_reader.hpp index f00a5ecaf..c1d235746 100644 --- a/include/osmium/experimental/flex_reader.hpp +++ b/include/osmium/experimental/flex_reader.hpp @@ -33,10 +33,18 @@ DEALINGS IN THE SOFTWARE. */ -#include -#include -#include +#include +#include + #include +#include +#include +#include +#include +#include +#include +#include +#include namespace osmium { @@ -45,7 +53,7 @@ namespace osmium { */ namespace experimental { - template + template class FlexReader { bool m_with_areas; @@ -104,7 +112,7 @@ namespace osmium { return buffer; } - osmium::io::Header header() const { + osmium::io::Header header() { return m_reader.header(); } diff --git a/include/osmium/fwd.hpp b/include/osmium/fwd.hpp new file mode 100644 index 000000000..bfcb5f57f --- /dev/null +++ b/include/osmium/fwd.hpp @@ -0,0 +1,70 @@ +#ifndef OSMIUM_FWD_HPP +#define OSMIUM_FWD_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +/** + * + * @file + * + * This file contains forward declarations for commonly used Osmium classes. + * + */ + +namespace osmium { + + class Area; + class Box; + class Changeset; + class ChangesetComment; + class ChangesetDiscussion; + class InnerRing; + class Location; + class Node; + class NodeRef; + class NodeRefList; + class OSMEntity; + class OSMObject; + class OuterRing; + class Relation; + class RelationMemberList; + class Segment; + class Tag; + class TagList; + class Timestamp; + class Way; + class WayNodeList; + +} // namespace osmium + +#endif // OSMIUM_FWD_HPP diff --git a/include/osmium/geom/coordinates.hpp b/include/osmium/geom/coordinates.hpp index 2bad57e0d..6544e68aa 100644 --- a/include/osmium/geom/coordinates.hpp +++ b/include/osmium/geom/coordinates.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include diff --git a/include/osmium/geom/factory.hpp b/include/osmium/geom/factory.hpp index 9be050d25..19375806b 100644 --- a/include/osmium/geom/factory.hpp +++ b/include/osmium/geom/factory.hpp @@ -61,7 +61,7 @@ namespace osmium { public: - geometry_error(const std::string& message, const char* object_type = "", osmium::object_id_type id = 0) : + explicit geometry_error(const std::string& message, const char* object_type = "", osmium::object_id_type id = 0) : std::runtime_error(message), m_message(message), m_id(id) { @@ -89,7 +89,7 @@ namespace osmium { return m_id; } - virtual const char* what() const noexcept override { + const char* what() const noexcept override { return m_message.c_str(); } @@ -142,7 +142,7 @@ namespace osmium { /** * Geometry factory. */ - template + template class GeometryFactory { /** @@ -166,8 +166,8 @@ namespace osmium { /** * Constructor for default initialized projection. */ - template - GeometryFactory(TArgs&&... args) : + template + explicit GeometryFactory(TArgs&&... args) : m_projection(), m_impl(std::forward(args)...) { } @@ -176,12 +176,13 @@ namespace osmium { * Constructor for explicitly initialized projection. Note that the * projection is moved into the GeometryFactory. */ - template - GeometryFactory(TProjection&& projection, TArgs&&... args) : + template + explicit GeometryFactory(TProjection&& projection, TArgs&&... args) : m_projection(std::move(projection)), m_impl(std::forward(args)...) { } + typedef TProjection projection_type; typedef typename TGeomImpl::point_type point_type; typedef typename TGeomImpl::linestring_type linestring_type; typedef typename TGeomImpl::polygon_type polygon_type; @@ -198,7 +199,7 @@ namespace osmium { /* Point */ - point_type create_point(const osmium::Location location) const { + point_type create_point(const osmium::Location& location) const { return m_impl.make_point(m_projection(location)); } @@ -226,7 +227,7 @@ namespace osmium { m_impl.linestring_start(); } - template + template size_t fill_linestring(TIter it, TIter end) { size_t num_points = 0; for (; it != end; ++it, ++num_points) { @@ -235,7 +236,7 @@ namespace osmium { return num_points; } - template + template size_t fill_linestring_unique(TIter it, TIter end) { size_t num_points = 0; osmium::Location last_location; @@ -300,7 +301,7 @@ namespace osmium { m_impl.polygon_start(); } - template + template size_t fill_polygon(TIter it, TIter end) { size_t num_points = 0; for (; it != end; ++it, ++num_points) { @@ -309,7 +310,7 @@ namespace osmium { return num_points; } - template + template size_t fill_polygon_unique(TIter it, TIter end) { size_t num_points = 0; osmium::Location last_location; diff --git a/include/osmium/geom/geojson.hpp b/include/osmium/geom/geojson.hpp index 7d5953535..e5b5a9cfc 100644 --- a/include/osmium/geom/geojson.hpp +++ b/include/osmium/geom/geojson.hpp @@ -88,7 +88,10 @@ namespace osmium { linestring_type linestring_finish(size_t /* num_points */) { assert(!m_str.empty()); std::string str; - std::swap(str, m_str); + + using std::swap; + swap(str, m_str); + str.back() = ']'; str += "}"; return str; @@ -134,7 +137,10 @@ namespace osmium { multipolygon_type multipolygon_finish() { assert(!m_str.empty()); std::string str; - std::swap(str, m_str); + + using std::swap; + swap(str, m_str); + str.back() = ']'; str += "}"; return str; @@ -144,7 +150,7 @@ namespace osmium { } // namespace detail - template + template using GeoJSONFactory = GeometryFactory; } // namespace geom diff --git a/include/osmium/geom/geos.hpp b/include/osmium/geom/geos.hpp index 771b08736..49b1fd7c0 100644 --- a/include/osmium/geom/geos.hpp +++ b/include/osmium/geom/geos.hpp @@ -228,7 +228,7 @@ namespace osmium { } // namespace detail - template + template using GEOSFactory = GeometryFactory; } // namespace geom diff --git a/include/osmium/geom/ogr.hpp b/include/osmium/geom/ogr.hpp index f33971c7d..4d5995cf7 100644 --- a/include/osmium/geom/ogr.hpp +++ b/include/osmium/geom/ogr.hpp @@ -47,35 +47,7 @@ DEALINGS IN THE SOFTWARE. #include #include -#ifdef _MSC_VER -# pragma warning(push) -# pragma warning(disable : 4458) -# pragma warning(disable : 4251) -#else -# pragma GCC diagnostic push -# ifdef __clang__ -# pragma GCC diagnostic ignored "-Wdocumentation-unknown-command" -# endif -# pragma GCC diagnostic ignored "-Wfloat-equal" -# pragma GCC diagnostic ignored "-Wold-style-cast" -# pragma GCC diagnostic ignored "-Wpadded" -# pragma GCC diagnostic ignored "-Wredundant-decls" -# pragma GCC diagnostic ignored "-Wshadow" -#endif - -/* Strictly speaking the following include would be enough here, - but everybody using this file will very likely need the other includes, - so we are adding them here, so that not everybody will need all those - pragmas to disable warnings. */ -//#include -#include -#include - -#ifdef _MSC_VER -# pragma warning(pop) -#else -# pragma GCC diagnostic pop -#endif +#include #include #include @@ -196,7 +168,7 @@ namespace osmium { } // namespace detail - template + template using OGRFactory = GeometryFactory; } // namespace geom diff --git a/include/osmium/geom/rapid_geojson.hpp b/include/osmium/geom/rapid_geojson.hpp index a3d46870c..87e479bf1 100644 --- a/include/osmium/geom/rapid_geojson.hpp +++ b/include/osmium/geom/rapid_geojson.hpp @@ -46,7 +46,7 @@ namespace osmium { * A geometry factory implementation that can be used with the * RapidJSON (https://github.com/miloyip/rapidjson) JSON writer. */ - template + template class RapidGeoJSONFactoryImpl { TWriter* m_writer; @@ -180,7 +180,7 @@ namespace osmium { } // namespace detail - template + template using RapidGeoJSONFactory = GeometryFactory, TProjection>; } // namespace geom diff --git a/include/osmium/geom/tile.hpp b/include/osmium/geom/tile.hpp index 9cd0b282b..6ca068262 100644 --- a/include/osmium/geom/tile.hpp +++ b/include/osmium/geom/tile.hpp @@ -67,10 +67,10 @@ namespace osmium { explicit Tile(uint32_t zoom, const osmium::Location& location) : z(zoom) { osmium::geom::Coordinates c = lonlat_to_mercator(location); - const int32_t n = 1LL << zoom; + const int32_t n = 1 << zoom; const double scale = detail::max_coordinate_epsg3857 * 2 / n; - x = detail::restrict_to_range((c.x + detail::max_coordinate_epsg3857) / scale, 0, n-1); - y = detail::restrict_to_range((detail::max_coordinate_epsg3857 - c.y) / scale, 0, n-1); + x = uint32_t(detail::restrict_to_range(int32_t((c.x + detail::max_coordinate_epsg3857) / scale), 0, n-1)); + y = uint32_t(detail::restrict_to_range(int32_t((detail::max_coordinate_epsg3857 - c.y) / scale), 0, n-1)); } }; // struct Tile diff --git a/include/osmium/geom/wkb.hpp b/include/osmium/geom/wkb.hpp index a290c02d0..49833e66b 100644 --- a/include/osmium/geom/wkb.hpp +++ b/include/osmium/geom/wkb.hpp @@ -188,7 +188,9 @@ namespace osmium { linestring_type linestring_finish(size_t num_points) { set_size(m_linestring_size_offset, num_points); std::string data; - std::swap(data, m_data); + + using std::swap; + swap(data, m_data); if (m_out_type == out_type::hex) { return convert_to_hex(data); @@ -246,7 +248,9 @@ namespace osmium { multipolygon_type multipolygon_finish() { set_size(m_multipolygon_size_offset, m_polygons); std::string data; - std::swap(data, m_data); + + using std::swap; + swap(data, m_data); if (m_out_type == out_type::hex) { return convert_to_hex(data); @@ -259,7 +263,7 @@ namespace osmium { } // namespace detail - template + template using WKBFactory = GeometryFactory; } // namespace geom diff --git a/include/osmium/geom/wkt.hpp b/include/osmium/geom/wkt.hpp index 4fea96baa..9cf5371a9 100644 --- a/include/osmium/geom/wkt.hpp +++ b/include/osmium/geom/wkt.hpp @@ -86,7 +86,10 @@ namespace osmium { linestring_type linestring_finish(size_t /* num_points */) { assert(!m_str.empty()); std::string str; - std::swap(str, m_str); + + using std::swap; + swap(str, m_str); + str.back() = ')'; return str; } @@ -131,7 +134,10 @@ namespace osmium { multipolygon_type multipolygon_finish() { assert(!m_str.empty()); std::string str; - std::swap(str, m_str); + + using std::swap; + swap(str, m_str); + str.back() = ')'; return str; } @@ -140,7 +146,7 @@ namespace osmium { } // namespace detail - template + template using WKTFactory = GeometryFactory; } // namespace geom diff --git a/include/osmium/handler.hpp b/include/osmium/handler.hpp index 34d878511..47e997efc 100644 --- a/include/osmium/handler.hpp +++ b/include/osmium/handler.hpp @@ -33,19 +33,9 @@ DEALINGS IN THE SOFTWARE. */ -namespace osmium { +#include - class OSMObject; - class Node; - class Way; - class Relation; - class Area; - class Changeset; - class TagList; - class WayNodeList; - class RelationMemberList; - class OuterRing; - class InnerRing; +namespace osmium { /** * @brief Osmium handlers provide callbacks for OSM objects @@ -89,12 +79,15 @@ namespace osmium { void inner_ring(const osmium::InnerRing&) const { } + void changeset_discussion(const osmium::ChangesetDiscussion&) const { + } + void flush() const { } }; // class Handler - } // namspace handler + } // namespace handler } // namespace osmium diff --git a/include/osmium/handler/chain.hpp b/include/osmium/handler/chain.hpp index 1af3962fd..4f3291ca6 100644 --- a/include/osmium/handler/chain.hpp +++ b/include/osmium/handler/chain.hpp @@ -38,14 +38,14 @@ DEALINGS IN THE SOFTWARE. #include #define OSMIUM_CHAIN_HANDLER_CALL(_func_, _type_) \ - template \ + template \ struct call_ ## _func_ { \ void operator()(THandlers& handlers, osmium::_type_& object) { \ std::get(handlers)._func_(object); \ call_ ## _func_()(handlers, object); \ } \ }; \ - template \ + template \ struct call_ ## _func_ { \ void operator()(THandlers&, osmium::_type_&) {} \ }; @@ -64,13 +64,13 @@ namespace osmium { * This handler allows chaining of any number of handlers into a single * handler. */ - template + template class ChainHandler : public osmium::handler::Handler { typedef std::tuple handlers_type; handlers_type m_handlers; - template + template struct call_flush { void operator()(THandlers& handlers) { std::get(handlers).flush(); @@ -78,7 +78,7 @@ namespace osmium { } }; // struct call_flush - template + template struct call_flush { void operator()(THandlers&) {} }; // struct call_flush diff --git a/include/osmium/handler/node_locations_for_ways.hpp b/include/osmium/handler/node_locations_for_ways.hpp index 9b9fcbf3f..8d31310c3 100644 --- a/include/osmium/handler/node_locations_for_ways.hpp +++ b/include/osmium/handler/node_locations_for_ways.hpp @@ -60,7 +60,7 @@ namespace osmium { * get(id) methods. * @tparam TStorageNegIDs Same but for negative IDs. */ - template + template class NodeLocationsForWays : public osmium::handler::Handler { static_assert(std::is_base_of, TStoragePosIDs>::value, "Index class must be derived from osmium::index::map::Map"); diff --git a/include/osmium/index/bool_vector.hpp b/include/osmium/index/bool_vector.hpp index 94e1f7268..04850a5fc 100644 --- a/include/osmium/index/bool_vector.hpp +++ b/include/osmium/index/bool_vector.hpp @@ -56,11 +56,13 @@ namespace osmium { public: BoolVector() = default; + BoolVector(const BoolVector&) = default; BoolVector(BoolVector&&) = default; BoolVector& operator=(const BoolVector&) = default; BoolVector& operator=(BoolVector&&) = default; - ~BoolVector() = default; + + ~BoolVector() noexcept = default; void set(T id, bool value = true) { if (m_bits.size() <= id) { diff --git a/include/osmium/index/detail/create_map_with_fd.hpp b/include/osmium/index/detail/create_map_with_fd.hpp index 5ccbfc809..a2e6b768a 100644 --- a/include/osmium/index/detail/create_map_with_fd.hpp +++ b/include/osmium/index/detail/create_map_with_fd.hpp @@ -47,19 +47,18 @@ namespace osmium { namespace detail { - template + template inline T* create_map_with_fd(const std::vector& config) { if (config.size() == 1) { return new T(); - } else { - assert(config.size() > 1); - const std::string& filename = config[1]; - int fd = ::open(filename.c_str(), O_CREAT | O_RDWR, 0644); - if (fd == -1) { - throw std::runtime_error(std::string("can't open file '") + filename + "': " + strerror(errno)); - } - return new T(fd); } + assert(config.size() > 1); + const std::string& filename = config[1]; + int fd = ::open(filename.c_str(), O_CREAT | O_RDWR, 0644); + if (fd == -1) { + throw std::runtime_error(std::string("can't open file '") + filename + "': " + strerror(errno)); + } + return new T(fd); } } // namespace detail diff --git a/include/osmium/index/detail/mmap_vector_anon.hpp b/include/osmium/index/detail/mmap_vector_anon.hpp index fc016261e..12a18036b 100644 --- a/include/osmium/index/detail/mmap_vector_anon.hpp +++ b/include/osmium/index/detail/mmap_vector_anon.hpp @@ -54,6 +54,8 @@ namespace osmium { mmap_vector_base() { } + ~mmap_vector_anon() noexcept = default; + }; // class mmap_vector_anon } // namespace detail diff --git a/include/osmium/index/detail/mmap_vector_base.hpp b/include/osmium/index/detail/mmap_vector_base.hpp index 9b6476812..e5f28e9b9 100644 --- a/include/osmium/index/detail/mmap_vector_base.hpp +++ b/include/osmium/index/detail/mmap_vector_base.hpp @@ -60,7 +60,7 @@ namespace osmium { public: - explicit mmap_vector_base(int fd, size_t capacity, size_t size = 0) : + mmap_vector_base(int fd, size_t capacity, size_t size = 0) : m_size(size), m_mapping(capacity, osmium::util::MemoryMapping::mapping_mode::write_shared, fd) { } @@ -70,6 +70,8 @@ namespace osmium { m_mapping(capacity) { } + ~mmap_vector_base() noexcept = default; + typedef T value_type; typedef T& reference; typedef const T& const_reference; @@ -78,8 +80,6 @@ namespace osmium { typedef T* iterator; typedef const T* const_iterator; - ~mmap_vector_base() = default; - void close() { m_mapping.unmap(); } diff --git a/include/osmium/index/detail/mmap_vector_file.hpp b/include/osmium/index/detail/mmap_vector_file.hpp index 1dadbcb45..54ef5137e 100644 --- a/include/osmium/index/detail/mmap_vector_file.hpp +++ b/include/osmium/index/detail/mmap_vector_file.hpp @@ -50,17 +50,21 @@ namespace osmium { public: - explicit mmap_vector_file() : mmap_vector_base( + mmap_vector_file() : + mmap_vector_base( osmium::detail::create_tmp_file(), osmium::detail::mmap_vector_size_increment) { } - explicit mmap_vector_file(int fd) : mmap_vector_base( + explicit mmap_vector_file(int fd) : + mmap_vector_base( fd, osmium::util::file_size(fd) / sizeof(T), osmium::util::file_size(fd) / sizeof(T)) { } + ~mmap_vector_file() noexcept = default; + }; // class mmap_vector_file } // namespace detail diff --git a/include/osmium/index/detail/vector_map.hpp b/include/osmium/index/detail/vector_map.hpp index 9c1cf4ed8..2a13061e2 100644 --- a/include/osmium/index/detail/vector_map.hpp +++ b/include/osmium/index/detail/vector_map.hpp @@ -48,7 +48,7 @@ namespace osmium { namespace map { - template + template class VectorBasedDenseMap : public Map { TVector m_vector; @@ -68,20 +68,20 @@ namespace osmium { m_vector(fd) { } - ~VectorBasedDenseMap() = default; + ~VectorBasedDenseMap() noexcept final = default; - void reserve(const size_t size) override final { + void reserve(const size_t size) final { m_vector.reserve(size); } - void set(const TId id, const TValue value) override final { + void set(const TId id, const TValue value) final { if (size() <= id) { m_vector.resize(id+1); } m_vector[id] = value; } - const TValue get(const TId id) const override final { + const TValue get(const TId id) const final { try { const TValue& value = m_vector.at(id); if (value == osmium::index::empty_value()) { @@ -93,7 +93,7 @@ namespace osmium { } } - size_t size() const override final { + size_t size() const final { return m_vector.size(); } @@ -101,16 +101,16 @@ namespace osmium { return m_vector.size() * sizeof(element_type); } - size_t used_memory() const override final { + size_t used_memory() const final { return sizeof(TValue) * size(); } - void clear() override final { + void clear() final { m_vector.clear(); m_vector.shrink_to_fit(); } - void dump_as_array(const int fd) override final { + void dump_as_array(const int fd) final { osmium::io::detail::reliable_write(fd, reinterpret_cast(m_vector.data()), byte_size()); } @@ -161,17 +161,17 @@ namespace osmium { m_vector() { } - VectorBasedSparseMap(int fd) : + explicit VectorBasedSparseMap(int fd) : m_vector(fd) { } - ~VectorBasedSparseMap() override final = default; + ~VectorBasedSparseMap() final = default; - void set(const TId id, const TValue value) override final { + void set(const TId id, const TValue value) final { m_vector.push_back(element_type(id, value)); } - const TValue get(const TId id) const override final { + const TValue get(const TId id) const final { const element_type element { id, osmium::index::empty_value() @@ -186,7 +186,7 @@ namespace osmium { } } - size_t size() const override final { + size_t size() const final { return m_vector.size(); } @@ -194,20 +194,20 @@ namespace osmium { return m_vector.size() * sizeof(element_type); } - size_t used_memory() const override final { + size_t used_memory() const final { return sizeof(element_type) * size(); } - void clear() override final { + void clear() final { m_vector.clear(); m_vector.shrink_to_fit(); } - void sort() override final { + void sort() final { std::sort(m_vector.begin(), m_vector.end()); } - void dump_as_list(const int fd) override final { + void dump_as_list(const int fd) final { osmium::io::detail::reliable_write(fd, reinterpret_cast(m_vector.data()), byte_size()); } diff --git a/include/osmium/index/detail/vector_multimap.hpp b/include/osmium/index/detail/vector_multimap.hpp index 1d875fcba..789fc9499 100644 --- a/include/osmium/index/detail/vector_multimap.hpp +++ b/include/osmium/index/detail/vector_multimap.hpp @@ -75,9 +75,9 @@ namespace osmium { m_vector(fd) { } - ~VectorBasedSparseMultimap() = default; + ~VectorBasedSparseMultimap() noexcept final = default; - void set(const TId id, const TValue value) override final { + void set(const TId id, const TValue value) final { m_vector.push_back(element_type(id, value)); } @@ -105,7 +105,7 @@ namespace osmium { }); } - size_t size() const override final { + size_t size() const final { return m_vector.size(); } @@ -113,16 +113,16 @@ namespace osmium { return m_vector.size() * sizeof(element_type); } - size_t used_memory() const override final { + size_t used_memory() const final { return sizeof(element_type) * size(); } - void clear() override final { + void clear() final { m_vector.clear(); m_vector.shrink_to_fit(); } - void sort() override final { + void sort() final { std::sort(m_vector.begin(), m_vector.end()); } @@ -147,7 +147,7 @@ namespace osmium { ); } - void dump_as_list(const int fd) override final { + void dump_as_list(const int fd) final { osmium::io::detail::reliable_write(fd, reinterpret_cast(m_vector.data()), byte_size()); } diff --git a/include/osmium/index/index.hpp b/include/osmium/index/index.hpp index f415192d2..78c142d82 100644 --- a/include/osmium/index/index.hpp +++ b/include/osmium/index/index.hpp @@ -89,7 +89,7 @@ namespace osmium { * the full range, so the max value is a good "empty" value. */ template <> - inline OSMIUM_CONSTEXPR size_t empty_value() { + inline constexpr size_t empty_value() { return std::numeric_limits::max(); } diff --git a/include/osmium/index/map.hpp b/include/osmium/index/map.hpp index 61af672cd..68a9c201c 100644 --- a/include/osmium/index/map.hpp +++ b/include/osmium/index/map.hpp @@ -84,7 +84,8 @@ namespace osmium { template class Map { - static_assert(std::is_integral::value && std::is_unsigned::value, "TId template parameter for class Map must be unsigned integral type"); + static_assert(std::is_integral::value && std::is_unsigned::value, + "TId template parameter for class Map must be unsigned integral type"); Map(const Map&) = delete; Map& operator=(const Map&) = delete; @@ -104,7 +105,7 @@ namespace osmium { Map() = default; - virtual ~Map() = default; + virtual ~Map() noexcept = default; virtual void reserve(const size_t) { // default implementation is empty @@ -147,10 +148,16 @@ namespace osmium { // default implementation is empty } + // This function could usually be const in derived classes, + // but not always. It could, for instance, sort internal data. + // This is why it is not declared const here. virtual void dump_as_list(const int /*fd*/) { throw std::runtime_error("can't dump as list"); } + // This function could usually be const in derived classes, + // but not always. It could, for instance, sort internal data. + // This is why it is not declared const here. virtual void dump_as_array(const int /*fd*/) { throw std::runtime_error("can't dump as array"); } @@ -252,12 +259,14 @@ namespace osmium { #define OSMIUM_CONCATENATE_DETAIL_(x, y) x##y #define OSMIUM_CONCATENATE_(x, y) OSMIUM_CONCATENATE_DETAIL_(x, y) -#define OSMIUM_MAKE_UNIQUE_(x) OSMIUM_CONCATENATE_(x, __COUNTER__) #define REGISTER_MAP(id, value, klass, name) \ -namespace { \ - const bool OSMIUM_MAKE_UNIQUE_(registered_index_map_##name) = osmium::index::register_map(#name); \ -} +namespace osmium { namespace index { namespace detail { \ + const bool OSMIUM_CONCATENATE_(registered_, name) = osmium::index::register_map(#name); \ + inline bool OSMIUM_CONCATENATE_(get_registered_, name)() noexcept { \ + return OSMIUM_CONCATENATE_(registered_, name); \ + } \ +} } } } // namespace index diff --git a/include/osmium/index/map/dummy.hpp b/include/osmium/index/map/dummy.hpp index de05d1d69..d6a360e56 100644 --- a/include/osmium/index/map/dummy.hpp +++ b/include/osmium/index/map/dummy.hpp @@ -56,25 +56,25 @@ namespace osmium { Dummy() = default; - ~Dummy() override final = default; + ~Dummy() noexcept final = default; - void set(const TId, const TValue) override final { + void set(const TId, const TValue) final { // intentionally left blank } - const TValue get(const TId id) const override final { + const TValue get(const TId id) const final { not_found_error(id); } - size_t size() const override final { + size_t size() const final { return 0; } - size_t used_memory() const override final { + size_t used_memory() const final { return 0; } - void clear() override final { + void clear() final { } }; // class Dummy diff --git a/include/osmium/index/map/sparse_mem_map.hpp b/include/osmium/index/map/sparse_mem_map.hpp index 2b9048b3b..d001b6791 100644 --- a/include/osmium/index/map/sparse_mem_map.hpp +++ b/include/osmium/index/map/sparse_mem_map.hpp @@ -71,36 +71,37 @@ namespace osmium { SparseMemMap() = default; - ~SparseMemMap() override final = default; + ~SparseMemMap() noexcept final = default; - void set(const TId id, const TValue value) override final { + void set(const TId id, const TValue value) final { m_elements[id] = value; } - const TValue get(const TId id) const override final { - try { - return m_elements.at(id); - } catch (std::out_of_range&) { + const TValue get(const TId id) const final { + auto it = m_elements.find(id); + if (it == m_elements.end()) { not_found_error(id); } + return it->second; } - size_t size() const override final { + size_t size() const noexcept final { return m_elements.size(); } - size_t used_memory() const override final { + size_t used_memory() const noexcept final { return element_size * m_elements.size(); } - void clear() override final { + void clear() final { m_elements.clear(); } - void dump_as_list(const int fd) override final { + void dump_as_list(const int fd) final { typedef typename std::map::value_type t; std::vector v; - std::copy(m_elements.begin(), m_elements.end(), std::back_inserter(v)); + v.reserve(m_elements.size()); + std::copy(m_elements.cbegin(), m_elements.cend(), std::back_inserter(v)); osmium::io::detail::reliable_write(fd, reinterpret_cast(v.data()), sizeof(t) * v.size()); } diff --git a/include/osmium/index/map/sparse_mem_table.hpp b/include/osmium/index/map/sparse_mem_table.hpp index 09ee81bbc..797a9265e 100644 --- a/include/osmium/index/map/sparse_mem_table.hpp +++ b/include/osmium/index/map/sparse_mem_table.hpp @@ -88,16 +88,16 @@ namespace osmium { m_elements(grow_size) { } - ~SparseMemTable() override final = default; + ~SparseMemTable() noexcept final = default; - void set(const TId id, const TValue value) override final { + void set(const TId id, const TValue value) final { if (id >= m_elements.size()) { m_elements.resize(id + m_grow_size); } m_elements[id] = value; } - const TValue get(const TId id) const override final { + const TValue get(const TId id) const final { if (id >= m_elements.size()) { not_found_error(id); } @@ -107,22 +107,23 @@ namespace osmium { return m_elements[id]; } - size_t size() const override final { + size_t size() const final { return m_elements.size(); } - size_t used_memory() const override final { + size_t used_memory() const final { // unused elements use 1 bit, used elements sizeof(TValue) bytes // http://google-sparsehash.googlecode.com/svn/trunk/doc/sparsetable.html return (m_elements.size() / 8) + (m_elements.num_nonempty() * sizeof(TValue)); } - void clear() override final { + void clear() final { m_elements.clear(); } - void dump_as_list(const int fd) override final { + void dump_as_list(const int fd) final { std::vector> v; + v.reserve(m_elements.size()); int n = 0; for (const TValue value : m_elements) { if (value != osmium::index::empty_value()) { diff --git a/include/osmium/index/multimap.hpp b/include/osmium/index/multimap.hpp index c817b6fb6..a7e1aadb3 100644 --- a/include/osmium/index/multimap.hpp +++ b/include/osmium/index/multimap.hpp @@ -118,7 +118,7 @@ namespace osmium { }; // class Multimap - } // namespace map + } // namespace multimap } // namespace index diff --git a/include/osmium/index/multimap/hybrid.hpp b/include/osmium/index/multimap/hybrid.hpp index ac2d96452..e13ee680f 100644 --- a/include/osmium/index/multimap/hybrid.hpp +++ b/include/osmium/index/multimap/hybrid.hpp @@ -62,16 +62,18 @@ namespace osmium { public: - explicit HybridIterator(typename main_map_type::iterator begin_main, - typename main_map_type::iterator end_main, - typename extra_map_type::iterator begin_extra, - typename extra_map_type::iterator end_extra) : + HybridIterator(typename main_map_type::iterator begin_main, + typename main_map_type::iterator end_main, + typename extra_map_type::iterator begin_extra, + typename extra_map_type::iterator end_extra) : m_begin_main(begin_main), m_end_main(end_main), m_begin_extra(begin_extra), m_end_extra(end_extra) { } + ~HybridIterator() noexcept = default; + HybridIterator& operator++() { if (m_begin_main == m_end_main) { ++m_begin_extra; @@ -134,11 +136,13 @@ namespace osmium { m_extra() { } - size_t size() const override final { + ~Hybrid() noexcept = default; + + size_t size() const final { return m_main.size() + m_extra.size(); } - size_t used_memory() const override final { + size_t used_memory() const final { return m_main.used_memory() + m_extra.used_memory(); } @@ -150,7 +154,7 @@ namespace osmium { m_main.set(id, value); } - void set(const TId id, const TValue value) override final { + void set(const TId id, const TValue value) final { m_extra.set(id, value); } @@ -175,17 +179,17 @@ namespace osmium { m_main.sort(); } - void dump_as_list(const int fd) override final { + void dump_as_list(const int fd) final { consolidate(); m_main.dump_as_list(fd); } - void clear() override final { + void clear() final { m_main.clear(); m_extra.clear(); } - void sort() override final { + void sort() final { m_main.sort(); } diff --git a/include/osmium/index/multimap/sparse_mem_multimap.hpp b/include/osmium/index/multimap/sparse_mem_multimap.hpp index 5b4715279..84cb640d9 100644 --- a/include/osmium/index/multimap/sparse_mem_multimap.hpp +++ b/include/osmium/index/multimap/sparse_mem_multimap.hpp @@ -78,13 +78,13 @@ namespace osmium { SparseMemMultimap() = default; - ~SparseMemMultimap() noexcept override final = default; + ~SparseMemMultimap() noexcept final = default; void unsorted_set(const TId id, const TValue value) { m_elements.emplace(id, value); } - void set(const TId id, const TValue value) override final { + void set(const TId id, const TValue value) final { m_elements.emplace(id, value); } @@ -114,15 +114,15 @@ namespace osmium { return m_elements.end(); } - size_t size() const override final { + size_t size() const final { return m_elements.size(); } - size_t used_memory() const override final { + size_t used_memory() const final { return element_size * m_elements.size(); } - void clear() override final { + void clear() final { m_elements.clear(); } @@ -130,12 +130,12 @@ namespace osmium { // intentionally left blank } - void dump_as_list(const int fd) override final { + void dump_as_list(const int fd) final { std::vector v; + v.reserve(m_elements.size()); for (const auto& element : m_elements) { v.emplace_back(element.first, element.second); } -// std::copy(m_elements.cbegin(), m_elements.cend(), std::back_inserter(v)); std::sort(v.begin(), v.end()); osmium::io::detail::reliable_write(fd, reinterpret_cast(v.data()), sizeof(element_type) * v.size()); } diff --git a/include/osmium/io/any_input.hpp b/include/osmium/io/any_input.hpp index d16d069a5..36f43b785 100644 --- a/include/osmium/io/any_input.hpp +++ b/include/osmium/io/any_input.hpp @@ -47,5 +47,6 @@ DEALINGS IN THE SOFTWARE. #include // IWYU pragma: export #include // IWYU pragma: export +#include // IWYU pragma: export #endif // OSMIUM_IO_ANY_INPUT_HPP diff --git a/include/osmium/io/bzip2_compression.hpp b/include/osmium/io/bzip2_compression.hpp index e961a87ab..fc0e33c53 100644 --- a/include/osmium/io/bzip2_compression.hpp +++ b/include/osmium/io/bzip2_compression.hpp @@ -55,7 +55,9 @@ DEALINGS IN THE SOFTWARE. #endif #include +#include #include +#include #include #include @@ -65,13 +67,13 @@ namespace osmium { * Exception thrown when there are problems compressing or * decompressing bzip2 files. */ - struct bzip2_error : public std::runtime_error { + struct bzip2_error : public io_error { int bzip2_error_code; int system_errno; bzip2_error(const std::string& what, int error_code) : - std::runtime_error(what), + io_error(what), bzip2_error_code(error_code), system_errno(error_code == BZ_IO_ERROR ? errno : 0) { } @@ -105,8 +107,8 @@ namespace osmium { public: - explicit Bzip2Compressor(int fd) : - Compressor(), + explicit Bzip2Compressor(int fd, fsync sync) : + Compressor(sync), m_file(fdopen(dup(fd), "wb")), m_bzerror(BZ_OK), m_bzfile(::BZ2_bzWriteOpen(&m_bzerror, m_file, 6, 0, 0)) { @@ -115,11 +117,15 @@ namespace osmium { } } - ~Bzip2Compressor() override final { - close(); + ~Bzip2Compressor() noexcept final { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } } - void write(const std::string& data) override final { + void write(const std::string& data) final { int error; ::BZ2_bzWrite(&error, m_bzfile, const_cast(data.data()), static_cast_with_assert(data.size())); if (error != BZ_OK && error != BZ_STREAM_END) { @@ -127,13 +133,18 @@ namespace osmium { } } - void close() override final { + void close() final { if (m_bzfile) { int error; ::BZ2_bzWriteClose(&error, m_bzfile, 0, nullptr, nullptr); m_bzfile = nullptr; if (m_file) { - fclose(m_file); + if (do_fsync()) { + osmium::io::detail::reliable_fsync(::fileno(m_file)); + } + if (fclose(m_file) != 0) { + throw std::system_error(errno, std::system_category(), "Close failed"); + } } if (error != BZ_OK) { detail::throw_bzip2_error(m_bzfile, "write close failed", error); @@ -152,7 +163,7 @@ namespace osmium { public: - Bzip2Decompressor(int fd) : + explicit Bzip2Decompressor(int fd) : Decompressor(), m_file(fdopen(dup(fd), "rb")), m_bzerror(BZ_OK), @@ -162,11 +173,15 @@ namespace osmium { } } - ~Bzip2Decompressor() override final { - close(); + ~Bzip2Decompressor() noexcept final { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } } - std::string read() override final { + std::string read() final { std::string buffer; if (!m_stream_end) { @@ -203,13 +218,15 @@ namespace osmium { return buffer; } - void close() override final { + void close() final { if (m_bzfile) { int error; ::BZ2_bzReadClose(&error, m_bzfile); m_bzfile = nullptr; if (m_file) { - fclose(m_file); + if (fclose(m_file) != 0) { + throw std::system_error(errno, std::system_category(), "Close failed"); + } } if (error != BZ_OK) { detail::throw_bzip2_error(m_bzfile, "read close failed", error); @@ -240,11 +257,15 @@ namespace osmium { } } - ~Bzip2BufferDecompressor() override final { - BZ2_bzDecompressEnd(&m_bzstream); + ~Bzip2BufferDecompressor() noexcept final { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } } - std::string read() override final { + std::string read() final { std::string output; if (m_buffer) { @@ -270,22 +291,28 @@ namespace osmium { return output; } + void close() final { + BZ2_bzDecompressEnd(&m_bzstream); + } + }; // class Bzip2BufferDecompressor - namespace { + namespace detail { -// we want the register_compression() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" + // we want the register_compression() function to run, setting + // the variable is only a side-effect, it will never be used const bool registered_bzip2_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::bzip2, - [](int fd) { return new osmium::io::Bzip2Compressor(fd); }, + [](int fd, fsync sync) { return new osmium::io::Bzip2Compressor(fd, sync); }, [](int fd) { return new osmium::io::Bzip2Decompressor(fd); }, [](const char* buffer, size_t size) { return new osmium::io::Bzip2BufferDecompressor(buffer, size); } ); -#pragma GCC diagnostic pop - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_bzip2_compression() noexcept { + return registered_bzip2_compression; + } + + } // namespace detail } // namespace io diff --git a/include/osmium/io/compression.hpp b/include/osmium/io/compression.hpp index 252976181..b337503ca 100644 --- a/include/osmium/io/compression.hpp +++ b/include/osmium/io/compression.hpp @@ -40,6 +40,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include #include #ifndef _MSC_VER @@ -49,7 +50,9 @@ DEALINGS IN THE SOFTWARE. #endif #include +#include #include +#include #include namespace osmium { @@ -58,11 +61,21 @@ namespace osmium { class Compressor { + fsync m_fsync; + + protected: + + bool do_fsync() const { + return m_fsync == fsync::yes; + } + public: - Compressor() = default; + explicit Compressor(fsync sync) : + m_fsync(sync) { + } - virtual ~Compressor() { + virtual ~Compressor() noexcept { } virtual void write(const std::string& data) = 0; @@ -85,13 +98,12 @@ namespace osmium { Decompressor(Decompressor&&) = delete; Decompressor& operator=(Decompressor&&) = delete; - virtual ~Decompressor() { + virtual ~Decompressor() noexcept { } virtual std::string read() = 0; - virtual void close() { - } + virtual void close() = 0; }; // class Decompressor @@ -106,13 +118,16 @@ namespace osmium { public: - typedef std::function create_compressor_type; + typedef std::function create_compressor_type; typedef std::function create_decompressor_type_fd; typedef std::function create_decompressor_type_buffer; private: - typedef std::map> compression_map_type; + typedef std::map> compression_map_type; compression_map_type m_callbacks; @@ -128,7 +143,7 @@ namespace osmium { std::string error_message {"Support for compression '"}; error_message += as_string(compression); error_message += "' not compiled into this binary."; - throw std::runtime_error(error_message); + throw unsupported_file_format_error(error_message); } public: @@ -144,15 +159,20 @@ namespace osmium { create_decompressor_type_fd create_decompressor_fd, create_decompressor_type_buffer create_decompressor_buffer) { - compression_map_type::value_type cc(compression, std::make_tuple(create_compressor, create_decompressor_fd, create_decompressor_buffer)); + compression_map_type::value_type cc(compression, + std::make_tuple(create_compressor, + create_decompressor_fd, + create_decompressor_buffer)); + return m_callbacks.insert(cc).second; } - std::unique_ptr create_compressor(osmium::io::file_compression compression, int fd) { + template + std::unique_ptr create_compressor(osmium::io::file_compression compression, TArgs&&... args) { auto it = m_callbacks.find(compression); if (it != m_callbacks.end()) { - return std::unique_ptr(std::get<0>(it->second)(fd)); + return std::unique_ptr(std::get<0>(it->second)(std::forward(args)...)); } error(compression); @@ -186,23 +206,31 @@ namespace osmium { public: - NoCompressor(int fd) : - Compressor(), + NoCompressor(int fd, fsync sync) : + Compressor(sync), m_fd(fd) { } - ~NoCompressor() override final { - close(); + ~NoCompressor() noexcept final { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } } - void write(const std::string& data) override final { + void write(const std::string& data) final { osmium::io::detail::reliable_write(m_fd, data.data(), data.size()); } - void close() override final { + void close() final { if (m_fd >= 0) { - ::close(m_fd); + int fd = m_fd; m_fd = -1; + if (do_fsync()) { + osmium::io::detail::reliable_fsync(fd); + } + osmium::io::detail::reliable_close(fd); } } @@ -216,7 +244,7 @@ namespace osmium { public: - NoDecompressor(int fd) : + explicit NoDecompressor(int fd) : Decompressor(), m_fd(fd), m_buffer(nullptr), @@ -230,11 +258,15 @@ namespace osmium { m_buffer_size(size) { } - ~NoDecompressor() override final { - close(); + ~NoDecompressor() noexcept final { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } } - std::string read() override final { + std::string read() final { std::string buffer; if (m_buffer) { @@ -249,35 +281,38 @@ namespace osmium { if (nread < 0) { throw std::system_error(errno, std::system_category(), "Read failed"); } - buffer.resize(nread); + buffer.resize(std::string::size_type(nread)); } return buffer; } - void close() override final { + void close() final { if (m_fd >= 0) { - ::close(m_fd); + int fd = m_fd; m_fd = -1; + osmium::io::detail::reliable_close(fd); } } }; // class NoDecompressor - namespace { + namespace detail { -// we want the register_compression() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" + // we want the register_compression() function to run, setting + // the variable is only a side-effect, it will never be used const bool registered_no_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::none, - [](int fd) { return new osmium::io::NoCompressor(fd); }, + [](int fd, fsync sync) { return new osmium::io::NoCompressor(fd, sync); }, [](int fd) { return new osmium::io::NoDecompressor(fd); }, [](const char* buffer, size_t size) { return new osmium::io::NoDecompressor(buffer, size); } ); -#pragma GCC diagnostic pop - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_no_compression() noexcept { + return registered_no_compression; + } + + } // namespace detail } // namespace io diff --git a/include/osmium/io/detail/debug_output_format.hpp b/include/osmium/io/detail/debug_output_format.hpp index efecc5833..f1766dea6 100644 --- a/include/osmium/io/detail/debug_output_format.hpp +++ b/include/osmium/io/detail/debug_output_format.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include #include @@ -41,14 +40,10 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include #include #include #include -#include - -#include #include #include #include @@ -87,65 +82,32 @@ namespace osmium { constexpr const char* color_white = "\x1b[37m"; constexpr const char* color_reset = "\x1b[0m"; + struct debug_output_options { + + /// Should metadata of objects be added? + bool add_metadata; + + /// Output with ANSI colors? + bool use_color; + + }; + /** * Writes out one buffer with OSM data in Debug format. */ - class DebugOutputBlock : public osmium::handler::Handler { + class DebugOutputBlock : public OutputBlock { - static constexpr size_t tmp_buffer_size = 50; + debug_output_options m_options; - std::shared_ptr m_input_buffer; - - std::shared_ptr m_out; - - char m_tmp_buffer[tmp_buffer_size+1]; - - bool m_add_metadata; - bool m_use_color; - - template - void output_formatted(const char* format, TArgs&&... args) { -#ifndef NDEBUG - int len = -#endif -#ifndef _MSC_VER - snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward(args)...); -#else - _snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward(args)...); -#endif - assert(len > 0 && static_cast(len) < tmp_buffer_size); - *m_out += m_tmp_buffer; - } + const char* m_utf8_prefix = ""; + const char* m_utf8_suffix = ""; void append_encoded_string(const char* data) { - const char* end = data + std::strlen(data); - - while (data != end) { - const char* last = data; - uint32_t c = utf8::next(data, end); - - // This is a list of Unicode code points that we let - // through instead of escaping them. It is incomplete - // and can be extended later. - // Generally we don't want to let through any - // non-printing characters. - if ((0x0020 <= c && c <= 0x0021) || - (0x0023 <= c && c <= 0x003b) || - (0x003d == c) || - (0x003f <= c && c <= 0x007e) || - (0x00a1 <= c && c <= 0x00ac) || - (0x00ae <= c && c <= 0x05ff)) { - m_out->append(last, data); - } else { - write_color(color_red); - output_formatted("", c); - write_color(color_blue); - } - } + append_debug_encoded_string(*m_out, data, m_utf8_prefix, m_utf8_suffix); } void write_color(const char* color) { - if (m_use_color) { + if (m_options.use_color) { *m_out += color; } } @@ -177,15 +139,38 @@ namespace osmium { *m_out += ": "; } + void write_comment_field(const char* name) { + write_color(color_cyan); + *m_out += name; + write_color(color_reset); + *m_out += ": "; + } + + void write_counter(int width, int n) { + write_color(color_white); + output_formatted(" %0*d: ", width, n++); + write_color(color_reset); + } + void write_error(const char* msg) { write_color(color_red); *m_out += msg; write_color(color_reset); } + void write_timestamp(const osmium::Timestamp& timestamp) { + if (timestamp.valid()) { + *m_out += timestamp.to_iso(); + output_formatted(" (%d)", timestamp.seconds_since_epoch()); + } else { + write_error("NOT SET"); + } + *m_out += '\n'; + } + void write_meta(const osmium::OSMObject& object) { output_formatted("%" PRId64 "\n", object.id()); - if (m_add_metadata) { + if (m_options.add_metadata) { write_fieldname("version"); output_formatted(" %d", object.version()); if (object.visible()) { @@ -196,8 +181,7 @@ namespace osmium { write_fieldname("changeset"); output_formatted("%d\n", object.changeset()); write_fieldname("timestamp"); - *m_out += object.timestamp().to_iso(); - output_formatted(" (%d)\n", object.timestamp()); + write_timestamp(object.timestamp()); write_fieldname("user"); output_formatted(" %d ", object.uid()); write_string(object.user()); @@ -211,14 +195,14 @@ namespace osmium { *m_out += padding; output_formatted(" %d\n", tags.size()); - osmium::max_op max; + osmium::max_op max; for (const auto& tag : tags) { max.update(std::strlen(tag.key())); } for (const auto& tag : tags) { *m_out += " "; write_string(tag.key()); - int spacing = max() - std::strlen(tag.key()); + auto spacing = max() - std::strlen(tag.key()); while (spacing--) { *m_out += " "; } @@ -255,12 +239,11 @@ namespace osmium { public: - explicit DebugOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata, bool use_color) : - m_input_buffer(std::make_shared(std::move(buffer))), - m_out(std::make_shared()), - m_tmp_buffer(), - m_add_metadata(add_metadata), - m_use_color(use_color) { + DebugOutputBlock(osmium::memory::Buffer&& buffer, const debug_output_options& options) : + OutputBlock(std::move(buffer)), + m_options(options), + m_utf8_prefix(options.use_color ? color_red : ""), + m_utf8_suffix(options.use_color ? color_blue : "") { } DebugOutputBlock(const DebugOutputBlock&) = default; @@ -269,13 +252,15 @@ namespace osmium { DebugOutputBlock(DebugOutputBlock&&) = default; DebugOutputBlock& operator=(DebugOutputBlock&&) = default; - ~DebugOutputBlock() = default; + ~DebugOutputBlock() noexcept = default; std::string operator()() { osmium::apply(m_input_buffer->cbegin(), m_input_buffer->cend(), *this); std::string out; - std::swap(out, *m_out); + using std::swap; + swap(out, *m_out); + return out; } @@ -313,7 +298,8 @@ namespace osmium { int width = int(log10(way.nodes().size())) + 1; int n = 0; for (const auto& node_ref : way.nodes()) { - output_formatted(" %0*d: %10" PRId64, width, n++, node_ref.ref()); + write_counter(width, n++); + output_formatted("%10" PRId64, node_ref.ref()); if (node_ref.location().valid()) { output_formatted(" (%.7f,%.7f)", node_ref.location().lon_without_check(), node_ref.location().lat_without_check()); } @@ -335,7 +321,7 @@ namespace osmium { int width = int(log10(relation.members().size())) + 1; int n = 0; for (const auto& member : relation.members()) { - output_formatted(" %0*d: ", width, n++); + write_counter(width, n++); *m_out += short_typename[item_type_to_nwr_index(member.type())]; output_formatted(" %10" PRId64 " ", member.ref()); write_string(member.role()); @@ -348,24 +334,26 @@ namespace osmium { void changeset(const osmium::Changeset& changeset) { write_object_type("changeset"); output_formatted("%d\n", changeset.id()); + write_fieldname("num changes"); output_formatted("%d", changeset.num_changes()); if (changeset.num_changes() == 0) { write_error(" NO CHANGES!"); } *m_out += '\n'; + write_fieldname("created at"); *m_out += ' '; - *m_out += changeset.created_at().to_iso(); - output_formatted(" (%d)\n", changeset.created_at()); + write_timestamp(changeset.created_at()); + write_fieldname("closed at"); *m_out += " "; if (changeset.closed()) { - *m_out += changeset.closed_at().to_iso(); - output_formatted(" (%d)\n", changeset.closed_at()); + write_timestamp(changeset.closed_at()); } else { write_error("OPEN!\n"); } + write_fieldname("user"); output_formatted(" %d ", changeset.uid()); write_string(changeset.user()); @@ -374,51 +362,73 @@ namespace osmium { write_box(changeset.bounds()); write_tags(changeset.tags(), " "); + if (changeset.num_comments() > 0) { + write_fieldname("comments"); + output_formatted(" %d\n", changeset.num_comments()); + + int width = int(log10(changeset.num_comments())) + 1; + int n = 0; + for (const auto& comment : changeset.discussion()) { + write_counter(width, n++); + + write_comment_field("date"); + write_timestamp(comment.date()); + output_formatted(" %*s", width, ""); + + write_comment_field("user"); + output_formatted("%d ", comment.uid()); + write_string(comment.user()); + output_formatted("\n %*s", width, ""); + + write_comment_field("text"); + write_string(comment.text()); + *m_out += '\n'; + } + } + *m_out += '\n'; } - }; // DebugOutputBlock + }; // class DebugOutputBlock class DebugOutputFormat : public osmium::io::detail::OutputFormat { - bool m_add_metadata; - bool m_use_color; - - public: - - DebugOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : - OutputFormat(file, output_queue), - m_add_metadata(file.get("add_metadata") != "false"), - m_use_color(file.get("color") == "true") { - } - - DebugOutputFormat(const DebugOutputFormat&) = delete; - DebugOutputFormat& operator=(const DebugOutputFormat&) = delete; - - void write_buffer(osmium::memory::Buffer&& buffer) override final { - m_output_queue.push(osmium::thread::Pool::instance().submit(DebugOutputBlock{std::move(buffer), m_add_metadata, m_use_color})); - } + debug_output_options m_options; void write_fieldname(std::string& out, const char* name) { out += " "; - if (m_use_color) { + if (m_options.use_color) { out += color_cyan; } out += name; - if (m_use_color) { + if (m_options.use_color) { out += color_reset; } out += ": "; } - void write_header(const osmium::io::Header& header) override final { + public: + + DebugOutputFormat(const osmium::io::File& file, future_string_queue_type& output_queue) : + OutputFormat(output_queue), + m_options() { + m_options.add_metadata = file.is_not_false("add_metadata"); + m_options.use_color = file.is_true("color"); + } + + DebugOutputFormat(const DebugOutputFormat&) = delete; + DebugOutputFormat& operator=(const DebugOutputFormat&) = delete; + + ~DebugOutputFormat() noexcept final = default; + + void write_header(const osmium::io::Header& header) final { std::string out; - if (m_use_color) { + if (m_options.use_color) { out += color_bold; } out += "header\n"; - if (m_use_color) { + if (m_options.use_color) { out += color_reset; } @@ -445,33 +455,26 @@ namespace osmium { } out += "\n=============================================\n\n"; - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(std::move(out)); + send_to_output_queue(std::move(out)); } - void close() override final { - std::string out; - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(out); + void write_buffer(osmium::memory::Buffer&& buffer) final { + m_output_queue.push(osmium::thread::Pool::instance().submit(DebugOutputBlock{std::move(buffer), m_options})); } }; // class DebugOutputFormat - namespace { + // we want the register_output_format() function to run, setting + // the variable is only a side-effect, it will never be used + const bool registered_debug_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::debug, + [](const osmium::io::File& file, future_string_queue_type& output_queue) { + return new osmium::io::detail::DebugOutputFormat(file, output_queue); + }); -// we want the register_output_format() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" - const bool registered_debug_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::debug, - [](const osmium::io::File& file, data_queue_type& output_queue) { - return new osmium::io::detail::DebugOutputFormat(file, output_queue); - }); -#pragma GCC diagnostic pop - - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_debug_output() noexcept { + return registered_debug_output; + } } // namespace detail diff --git a/include/osmium/io/detail/input_format.hpp b/include/osmium/io/detail/input_format.hpp index 03e1190c3..d26b1ee92 100644 --- a/include/osmium/io/detail/input_format.hpp +++ b/include/osmium/io/detail/input_format.hpp @@ -33,13 +33,16 @@ DEALINGS IN THE SOFTWARE. */ +#include #include +#include #include #include #include #include #include +#include #include #include #include @@ -48,106 +51,156 @@ DEALINGS IN THE SOFTWARE. namespace osmium { - namespace thread { - template class Queue; - } // namespace thread - namespace io { namespace detail { - /** - * Virtual base class for all classes reading OSM files in different - * formats. - * - * Do not use this class or derived classes directly. Use the - * osmium::io::Reader class instead. - */ - class InputFormat { + class Parser { + + future_buffer_queue_type& m_output_queue; + std::promise& m_header_promise; + queue_wrapper m_input_queue; + osmium::osm_entity_bits::type m_read_types; + bool m_header_is_done; protected: - osmium::io::File m_file; - osmium::osm_entity_bits::type m_read_which_entities; - osmium::io::Header m_header; - - explicit InputFormat(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities) : - m_file(file), - m_read_which_entities(read_which_entities) { - m_header.set_has_multiple_object_versions(m_file.has_multiple_object_versions()); + std::string get_input() { + return m_input_queue.pop(); } - InputFormat(const InputFormat&) = delete; - InputFormat(InputFormat&&) = delete; + bool input_done() const { + return m_input_queue.has_reached_end_of_data(); + } - InputFormat& operator=(const InputFormat&) = delete; - InputFormat& operator=(InputFormat&&) = delete; + osmium::osm_entity_bits::type read_types() const { + return m_read_types; + } + + bool header_is_done() const { + return m_header_is_done; + } + + void set_header_value(const osmium::io::Header& header) { + if (!m_header_is_done) { + m_header_is_done = true; + m_header_promise.set_value(header); + } + } + + void set_header_exception(const std::exception_ptr& exception) { + if (!m_header_is_done) { + m_header_is_done = true; + m_header_promise.set_exception(exception); + } + } + + /** + * Wrap the buffer into a future and add it to the output queue. + */ + void send_to_output_queue(osmium::memory::Buffer&& buffer) { + add_to_queue(m_output_queue, std::move(buffer)); + } + + void send_to_output_queue(std::future&& future) { + m_output_queue.push(std::move(future)); + } public: - virtual ~InputFormat() { + Parser(future_string_queue_type& input_queue, + future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_types) : + m_output_queue(output_queue), + m_header_promise(header_promise), + m_input_queue(input_queue), + m_read_types(read_types), + m_header_is_done(false) { } - virtual osmium::memory::Buffer read() = 0; + Parser(const Parser&) = delete; + Parser& operator=(const Parser&) = delete; - virtual void close() { + Parser(Parser&&) = delete; + Parser& operator=(Parser&&) = delete; + + virtual ~Parser() noexcept = default; + + virtual void run() = 0; + + void parse() { + try { + run(); + } catch (...) { + std::exception_ptr exception = std::current_exception(); + set_header_exception(exception); + add_to_queue(m_output_queue, std::move(exception)); + } + + add_end_of_data_to_queue(m_output_queue); } - virtual osmium::io::Header header() { - return m_header; - } - - }; // class InputFormat + }; // class Parser /** - * This factory class is used to create objects that read OSM data - * written in a specified format. + * This factory class is used to create objects that decode OSM + * data written in a specified format. * - * Do not use this class directly. Instead use the osmium::io::Reader - * class. + * Do not use this class directly. Use the osmium::io::Reader + * class instead. */ - class InputFormatFactory { + class ParserFactory { public: - typedef std::function&)> create_input_type; + typedef std::function< + std::unique_ptr( + future_string_queue_type&, + future_buffer_queue_type&, + std::promise& header_promise, + osmium::osm_entity_bits::type read_which_entities + ) + > create_parser_type; private: - typedef std::map map_type; + typedef std::map map_type; map_type m_callbacks; - InputFormatFactory() : + ParserFactory() : m_callbacks() { } public: - static InputFormatFactory& instance() { - static InputFormatFactory factory; + static ParserFactory& instance() { + static ParserFactory factory; return factory; } - bool register_input_format(osmium::io::file_format format, create_input_type create_function) { + bool register_parser(osmium::io::file_format format, create_parser_type create_function) { if (! m_callbacks.insert(map_type::value_type(format, create_function)).second) { return false; } return true; } - std::unique_ptr create_input(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue& input_queue) { - file.check(); - + create_parser_type get_creator_function(const osmium::io::File& file) { auto it = m_callbacks.find(file.format()); - if (it != m_callbacks.end()) { - return std::unique_ptr((it->second)(file, read_which_entities, input_queue)); + if (it == m_callbacks.end()) { + throw unsupported_file_format_error( + std::string("Can not open file '") + + file.filename() + + "' with type '" + + as_string(file.format()) + + "'. No support for reading this format in this program."); } - - throw std::runtime_error(std::string("Support for input format '") + as_string(file.format()) + "' not compiled into this binary."); + return it->second; } - }; // class InputFormatFactory + }; // class ParserFactory } // namespace detail diff --git a/include/osmium/io/detail/o5m_input_format.hpp b/include/osmium/io/detail/o5m_input_format.hpp new file mode 100644 index 000000000..0318432e8 --- /dev/null +++ b/include/osmium/io/detail/o5m_input_format.hpp @@ -0,0 +1,636 @@ +#ifndef OSMIUM_IO_DETAIL_O5M_INPUT_FORMAT_HPP +#define OSMIUM_IO_DETAIL_O5M_INPUT_FORMAT_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace osmium { + + /** + * Exception thrown when the o5m deocder failed. The exception contains + * (if available) information about the place where the error happened + * and the type of error. + */ + struct o5m_error : public io_error { + + explicit o5m_error(const char* what) : + io_error(std::string("o5m format error: ") + what) { + } + + }; // struct o5m_error + + namespace io { + + namespace detail { + + // Implementation of the o5m/o5c file formats according to the + // description at http://wiki.openstreetmap.org/wiki/O5m . + + class ReferenceTable { + + // The following settings are from the o5m description: + + // The maximum number of entries in this table. + const uint64_t number_of_entries = 15000; + + // The size of one entry in the table. + const unsigned int entry_size = 256; + + // The maximum length of a string in the table including + // two \0 bytes. + const unsigned int max_length = 250 + 2; + + // The data is stored in this string. It is default constructed + // and then resized on demand the first time something is added. + // This is done because the ReferenceTable is in a O5mParser + // object which will be copied from one thread to another. This + // way the string is still small when it is copied. + std::string m_table; + + unsigned int current_entry = 0; + + public: + + void clear() { + current_entry = 0; + } + + void add(const char* string, size_t size) { + if (m_table.empty()) { + m_table.resize(entry_size * number_of_entries); + } + if (size <= max_length) { + std::copy_n(string, size, &m_table[current_entry * entry_size]); + if (++current_entry == number_of_entries) { + current_entry = 0; + } + } + } + + const char* get(uint64_t index) const { + if (m_table.empty() || index == 0 || index > number_of_entries) { + throw o5m_error("reference to non-existing string in table"); + } + auto entry = (current_entry + number_of_entries - index) % number_of_entries; + return &m_table[entry * entry_size]; + } + + }; // class ReferenceTable + + class O5mParser : public Parser { + + static constexpr int buffer_size = 2 * 1000 * 1000; + + osmium::io::Header m_header; + + osmium::memory::Buffer m_buffer; + + std::string m_input; + + const char* m_data; + const char* m_end; + + ReferenceTable m_reference_table; + + static int64_t zvarint(const char** data, const char* end) { + return protozero::decode_zigzag64(protozero::decode_varint(data, end)); + } + + bool ensure_bytes_available(size_t need_bytes) { + if ((m_end - m_data) >= long(need_bytes)) { + return true; + } + + if (input_done() && (m_input.size() < need_bytes)) { + return false; + } + + m_input.erase(0, m_data - m_input.data()); + + while (m_input.size() < need_bytes) { + std::string data = get_input(); + if (input_done()) { + return false; + } + m_input.append(data); + } + + m_data = m_input.data(); + m_end = m_input.data() + m_input.size(); + + return true; + } + + void check_header_magic() { + static const unsigned char header_magic[] = { 0xff, 0xe0, 0x04, 'o', '5' }; + + if (std::strncmp(reinterpret_cast(header_magic), m_data, sizeof(header_magic))) { + throw o5m_error("wrong header magic"); + } + + m_data += sizeof(header_magic); + } + + void check_file_type() { + if (*m_data == 'm') { // o5m data file + m_header.set_has_multiple_object_versions(false); + } else if (*m_data == 'c') { // o5c change file + m_header.set_has_multiple_object_versions(true); + } else { + throw o5m_error("wrong header magic"); + } + + m_data++; + } + + void check_file_format_version() { + if (*m_data != '2') { + throw o5m_error("wrong header magic"); + } + + m_data++; + } + + void decode_header() { + if (! ensure_bytes_available(7)) { // overall length of header + throw o5m_error("file too short (incomplete header info)"); + } + + check_header_magic(); + check_file_type(); + check_file_format_version(); + } + + void mark_header_as_done() { + set_header_value(m_header); + } + + osmium::util::DeltaDecode m_delta_id; + + osmium::util::DeltaDecode m_delta_timestamp; + osmium::util::DeltaDecode m_delta_changeset; + osmium::util::DeltaDecode m_delta_lon; + osmium::util::DeltaDecode m_delta_lat; + + osmium::util::DeltaDecode m_delta_way_node_id; + osmium::util::DeltaDecode m_delta_member_ids[3]; + + void reset() { + m_reference_table.clear(); + + m_delta_id.clear(); + m_delta_timestamp.clear(); + m_delta_changeset.clear(); + m_delta_lon.clear(); + m_delta_lat.clear(); + + m_delta_way_node_id.clear(); + m_delta_member_ids[0].clear(); + m_delta_member_ids[1].clear(); + m_delta_member_ids[2].clear(); + } + + const char* decode_string(const char** dataptr, const char* const end) { + if (**dataptr == 0x00) { // get inline string + (*dataptr)++; + if (*dataptr == end) { + throw o5m_error("string format error"); + } + return *dataptr; + } else { // get from reference table + auto index = protozero::decode_varint(dataptr, end); + return m_reference_table.get(index); + } + } + + std::pair decode_user(const char** dataptr, const char* const end) { + bool update_pointer = (**dataptr == 0x00); + const char* data = decode_string(dataptr, end); + const char* start = data; + + auto uid = protozero::decode_varint(&data, end); + + if (data == end) { + throw o5m_error("missing user name"); + } + + const char* user = ++data; + + if (uid == 0 && update_pointer) { + m_reference_table.add("\0\0", 2); + *dataptr = data; + return std::make_pair(0, ""); + } + + while (*data++) { + if (data == end) { + throw o5m_error("no null byte in user name"); + } + } + + if (update_pointer) { + m_reference_table.add(start, data - start); + *dataptr = data; + } + + return std::make_pair(static_cast_with_assert(uid), user); + } + + void decode_tags(osmium::builder::Builder* builder, const char** dataptr, const char* const end) { + osmium::builder::TagListBuilder tl_builder(m_buffer, builder); + + while(*dataptr != end) { + bool update_pointer = (**dataptr == 0x00); + const char* data = decode_string(dataptr, end); + const char* start = data; + + while (*data++) { + if (data == end) { + throw o5m_error("no null byte in tag key"); + } + } + + const char* value = data; + while (*data++) { + if (data == end) { + throw o5m_error("no null byte in tag value"); + } + } + + if (update_pointer) { + m_reference_table.add(start, data - start); + *dataptr = data; + } + + tl_builder.add_tag(start, value); + } + } + + const char* decode_info(osmium::OSMObject& object, const char** dataptr, const char* const end) { + const char* user = ""; + + if (**dataptr == 0x00) { // no info section + ++*dataptr; + } else { // has info section + object.set_version(static_cast_with_assert(protozero::decode_varint(dataptr, end))); + auto timestamp = m_delta_timestamp.update(zvarint(dataptr, end)); + if (timestamp != 0) { // has timestamp + object.set_timestamp(timestamp); + object.set_changeset(m_delta_changeset.update(zvarint(dataptr, end))); + if (*dataptr != end) { + auto uid_user = decode_user(dataptr, end); + object.set_uid(uid_user.first); + user = uid_user.second; + } else { + object.set_uid(user_id_type(0)); + } + } + } + + return user; + } + + void decode_node(const char* data, const char* const end) { + osmium::builder::NodeBuilder builder(m_buffer); + osmium::Node& node = builder.object(); + + node.set_id(m_delta_id.update(zvarint(&data, end))); + + builder.add_user(decode_info(node, &data, end)); + + if (data == end) { + // no location, object is deleted + builder.object().set_visible(false); + builder.object().set_location(osmium::Location{}); + } else { + auto lon = m_delta_lon.update(zvarint(&data, end)); + auto lat = m_delta_lat.update(zvarint(&data, end)); + builder.object().set_location(osmium::Location{lon, lat}); + + if (data != end) { + decode_tags(&builder, &data, end); + } + } + + m_buffer.commit(); + } + + void decode_way(const char* data, const char* const end) { + osmium::builder::WayBuilder builder(m_buffer); + osmium::Way& way = builder.object(); + + way.set_id(m_delta_id.update(zvarint(&data, end))); + + builder.add_user(decode_info(way, &data, end)); + + if (data == end) { + // no reference section, object is deleted + builder.object().set_visible(false); + } else { + auto reference_section_length = protozero::decode_varint(&data, end); + if (reference_section_length > 0) { + const char* const end_refs = data + reference_section_length; + if (end_refs > end) { + throw o5m_error("way nodes ref section too long"); + } + + osmium::builder::WayNodeListBuilder wn_builder(m_buffer, &builder); + + while (data < end_refs) { + wn_builder.add_node_ref(m_delta_way_node_id.update(zvarint(&data, end))); + } + } + + if (data != end) { + decode_tags(&builder, &data, end); + } + } + + m_buffer.commit(); + } + + osmium::item_type decode_member_type(char c) { + if (c < '0' || c > '2') { + throw o5m_error("unknown member type"); + } + return osmium::nwr_index_to_item_type(c - '0'); + } + + std::pair decode_role(const char** dataptr, const char* const end) { + bool update_pointer = (**dataptr == 0x00); + const char* data = decode_string(dataptr, end); + const char* start = data; + + auto member_type = decode_member_type(*data++); + if (data == end) { + throw o5m_error("missing role"); + } + const char* role = data; + + while (*data++) { + if (data == end) { + throw o5m_error("no null byte in role"); + } + } + + if (update_pointer) { + m_reference_table.add(start, data - start); + *dataptr = data; + } + + return std::make_pair(member_type, role); + } + + void decode_relation(const char* data, const char* const end) { + osmium::builder::RelationBuilder builder(m_buffer); + osmium::Relation& relation = builder.object(); + + relation.set_id(m_delta_id.update(zvarint(&data, end))); + + builder.add_user(decode_info(relation, &data, end)); + + if (data == end) { + // no reference section, object is deleted + builder.object().set_visible(false); + } else { + auto reference_section_length = protozero::decode_varint(&data, end); + if (reference_section_length > 0) { + const char* const end_refs = data + reference_section_length; + if (end_refs > end) { + throw o5m_error("relation format error"); + } + + osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder); + + while (data < end_refs) { + auto delta_id = zvarint(&data, end); + if (data == end) { + throw o5m_error("relation member format error"); + } + auto type_role = decode_role(&data, end); + auto i = osmium::item_type_to_nwr_index(type_role.first); + auto ref = m_delta_member_ids[i].update(delta_id); + rml_builder.add_member(type_role.first, ref, type_role.second); + } + } + + if (data != end) { + decode_tags(&builder, &data, end); + } + } + + m_buffer.commit(); + } + + void decode_bbox(const char* data, const char* const end) { + auto sw_lon = zvarint(&data, end); + auto sw_lat = zvarint(&data, end); + auto ne_lon = zvarint(&data, end); + auto ne_lat = zvarint(&data, end); + + m_header.add_box(osmium::Box{osmium::Location{sw_lon, sw_lat}, + osmium::Location{ne_lon, ne_lat}}); + } + + void decode_timestamp(const char* data, const char* const end) { + auto timestamp = osmium::Timestamp(zvarint(&data, end)).to_iso(); + m_header.set("o5m_timestamp", timestamp); + m_header.set("timestamp", timestamp); + } + + void flush() { + osmium::memory::Buffer buffer(buffer_size); + using std::swap; + swap(m_buffer, buffer); + send_to_output_queue(std::move(buffer)); + } + + enum class dataset_type : unsigned char { + node = 0x10, + way = 0x11, + relation = 0x12, + bounding_box = 0xdb, + timestamp = 0xdc, + header = 0xe0, + sync = 0xee, + jump = 0xef, + reset = 0xff + }; + + void decode_data() { + while (ensure_bytes_available(1)) { + dataset_type ds_type = dataset_type(*m_data++); + if (ds_type > dataset_type::jump) { + if (ds_type == dataset_type::reset) { + reset(); + } + } else { + ensure_bytes_available(protozero::max_varint_length); + + uint64_t length = 0; + try { + length = protozero::decode_varint(&m_data, m_end); + } catch (protozero::end_of_buffer_exception&) { + throw o5m_error("premature end of file"); + } + + if (! ensure_bytes_available(length)) { + throw o5m_error("premature end of file"); + } + + switch (ds_type) { + case dataset_type::node: + mark_header_as_done(); + if (read_types() & osmium::osm_entity_bits::node) { + decode_node(m_data, m_data + length); + } + break; + case dataset_type::way: + mark_header_as_done(); + if (read_types() & osmium::osm_entity_bits::way) { + decode_way(m_data, m_data + length); + } + break; + case dataset_type::relation: + mark_header_as_done(); + if (read_types() & osmium::osm_entity_bits::relation) { + decode_relation(m_data, m_data + length); + } + break; + case dataset_type::bounding_box: + decode_bbox(m_data, m_data + length); + break; + case dataset_type::timestamp: + decode_timestamp(m_data, m_data + length); + break; + default: + // ignore unknown datasets + break; + } + + if (read_types() == osmium::osm_entity_bits::nothing && header_is_done()) { + break; + } + + m_data += length; + + if (m_buffer.committed() > buffer_size / 10 * 9) { + flush(); + } + } + } + + if (m_buffer.committed()) { + flush(); + } + + mark_header_as_done(); + } + + public: + + O5mParser(future_string_queue_type& input_queue, + future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_types) : + Parser(input_queue, output_queue, header_promise, read_types), + m_header(), + m_buffer(buffer_size), + m_input(), + m_data(m_input.data()), + m_end(m_data) { + } + + ~O5mParser() noexcept final = default; + + void run() final { + osmium::thread::set_thread_name("_osmium_o5m_in"); + + decode_header(); + decode_data(); + } + + }; // class O5mParser + + // we want the register_parser() function to run, setting + // the variable is only a side-effect, it will never be used + const bool registered_o5m_parser = ParserFactory::instance().register_parser( + file_format::o5m, + [](future_string_queue_type& input_queue, + future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_which_entities) { + return std::unique_ptr(new O5mParser(input_queue, output_queue, header_promise, read_which_entities)); + }); + + // dummy function to silence the unused variable warning from above + inline bool get_registered_o5m_parser() noexcept { + return registered_o5m_parser; + } + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_O5M_INPUT_FORMAT_HPP diff --git a/include/osmium/io/detail/opl_output_format.hpp b/include/osmium/io/detail/opl_output_format.hpp index a3103d9bf..2d863ea82 100644 --- a/include/osmium/io/detail/opl_output_format.hpp +++ b/include/osmium/io/detail/opl_output_format.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include #include @@ -41,14 +40,10 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include #include #include #include -#include - -#include #include #include #include @@ -74,71 +69,27 @@ namespace osmium { namespace detail { + struct opl_output_options { + + /// Should metadata of objects be added? + bool add_metadata; + + }; + /** * Writes out one buffer with OSM data in OPL format. */ - class OPLOutputBlock : public osmium::handler::Handler { + class OPLOutputBlock : public OutputBlock { - static constexpr size_t tmp_buffer_size = 100; - - std::shared_ptr m_input_buffer; - - std::shared_ptr m_out; - - char m_tmp_buffer[tmp_buffer_size+1]; - - bool m_add_metadata; - - template - void output_formatted(const char* format, TArgs&&... args) { -#ifndef NDEBUG - int len = -#endif -#ifndef _MSC_VER - snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward(args)...); -#else - _snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward(args)...); -#endif - assert(len > 0 && static_cast(len) < tmp_buffer_size); - *m_out += m_tmp_buffer; - } + opl_output_options m_options; void append_encoded_string(const char* data) { - const char* end = data + std::strlen(data); - - while (data != end) { - const char* last = data; - uint32_t c = utf8::next(data, end); - - // This is a list of Unicode code points that we let - // through instead of escaping them. It is incomplete - // and can be extended later. - // Generally we don't want to let through any character - // that has special meaning in the OPL format such as - // space, comma, @, etc. and any non-printing characters. - if ((0x0021 <= c && c <= 0x0024) || - (0x0026 <= c && c <= 0x002b) || - (0x002d <= c && c <= 0x003c) || - (0x003e <= c && c <= 0x003f) || - (0x0041 <= c && c <= 0x007e) || - (0x00a1 <= c && c <= 0x00ac) || - (0x00ae <= c && c <= 0x05ff)) { - m_out->append(last, data); - } else { - *m_out += '%'; - if (c <= 0xff) { - output_formatted("%02x", c); - } else { - output_formatted("%04x", c); - } - *m_out += '%'; - } - } + osmium::io::detail::append_utf8_encoded_string(*m_out, data); } void write_meta(const osmium::OSMObject& object) { output_formatted("%" PRId64, object.id()); - if (m_add_metadata) { + if (m_options.add_metadata) { output_formatted(" v%d d", object.version()); *m_out += (object.visible() ? 'V' : 'D'); output_formatted(" c%d t", object.changeset()); @@ -160,7 +111,7 @@ namespace osmium { } } - void write_location(const osmium::Location location, const char x, const char y) { + void write_location(const osmium::Location& location, const char x, const char y) { if (location) { output_formatted(" %c%.7f %c%.7f", x, location.lon_without_check(), y, location.lat_without_check()); } else { @@ -173,11 +124,9 @@ namespace osmium { public: - explicit OPLOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata) : - m_input_buffer(std::make_shared(std::move(buffer))), - m_out(std::make_shared()), - m_tmp_buffer(), - m_add_metadata(add_metadata) { + OPLOutputBlock(osmium::memory::Buffer&& buffer, const opl_output_options& options) : + OutputBlock(std::move(buffer)), + m_options(options) { } OPLOutputBlock(const OPLOutputBlock&) = default; @@ -186,13 +135,15 @@ namespace osmium { OPLOutputBlock(OPLOutputBlock&&) = default; OPLOutputBlock& operator=(OPLOutputBlock&&) = default; - ~OPLOutputBlock() = default; + ~OPLOutputBlock() noexcept = default; std::string operator()() { osmium::apply(m_input_buffer->cbegin(), m_input_buffer->cend(), *this); std::string out; - std::swap(out, *m_out); + using std::swap; + swap(out, *m_out); + return out; } @@ -244,7 +195,7 @@ namespace osmium { *m_out += changeset.created_at().to_iso(); *m_out += " e"; *m_out += changeset.closed_at().to_iso(); - output_formatted(" i%d u", changeset.uid()); + output_formatted(" d%d i%d u", changeset.num_comments(), changeset.uid()); append_encoded_string(changeset.user()); write_location(changeset.bounds().bottom_left(), 'x', 'y'); write_location(changeset.bounds().top_right(), 'X', 'Y'); @@ -264,48 +215,42 @@ namespace osmium { *m_out += '\n'; } - }; // OPLOutputBlock + }; // class OPLOutputBlock class OPLOutputFormat : public osmium::io::detail::OutputFormat { - bool m_add_metadata; + opl_output_options m_options; public: - OPLOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : - OutputFormat(file, output_queue), - m_add_metadata(file.get("add_metadata") != "false") { + OPLOutputFormat(const osmium::io::File& file, future_string_queue_type& output_queue) : + OutputFormat(output_queue), + m_options() { + m_options.add_metadata = file.is_not_false("add_metadata"); } OPLOutputFormat(const OPLOutputFormat&) = delete; OPLOutputFormat& operator=(const OPLOutputFormat&) = delete; - void write_buffer(osmium::memory::Buffer&& buffer) override final { - m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer), m_add_metadata})); - } + ~OPLOutputFormat() noexcept final = default; - void close() override final { - std::string out; - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(out); + void write_buffer(osmium::memory::Buffer&& buffer) final { + m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer), m_options})); } }; // class OPLOutputFormat - namespace { + // we want the register_output_format() function to run, setting + // the variable is only a side-effect, it will never be used + const bool registered_opl_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::opl, + [](const osmium::io::File& file, future_string_queue_type& output_queue) { + return new osmium::io::detail::OPLOutputFormat(file, output_queue); + }); -// we want the register_output_format() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" - const bool registered_opl_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::opl, - [](const osmium::io::File& file, data_queue_type& output_queue) { - return new osmium::io::detail::OPLOutputFormat(file, output_queue); - }); -#pragma GCC diagnostic pop - - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_opl_output() noexcept { + return registered_opl_output; + } } // namespace detail diff --git a/include/osmium/io/detail/output_format.hpp b/include/osmium/io/detail/output_format.hpp index 529a1890f..4c38c4d1e 100644 --- a/include/osmium/io/detail/output_format.hpp +++ b/include/osmium/io/detail/output_format.hpp @@ -34,29 +34,48 @@ DEALINGS IN THE SOFTWARE. */ #include -#include #include #include #include #include #include +#include +#include +#include #include #include -#include -#include +#include namespace osmium { - namespace memory { - class Buffer; - } + namespace io { + class Header; + } // namespace io namespace io { namespace detail { - typedef osmium::thread::Queue> data_queue_type; + class OutputBlock : public osmium::handler::Handler { + + protected: + + std::shared_ptr m_input_buffer; + + std::shared_ptr m_out; + + explicit OutputBlock(osmium::memory::Buffer&& buffer) : + m_input_buffer(std::make_shared(std::move(buffer))), + m_out(std::make_shared()) { + } + + template + void output_formatted(const char* format, TArgs&&... args) { + append_printf_formatted_string(*m_out, format, std::forward(args)...); + } + + }; // class OutputBlock; /** * Virtual base class for all classes writing OSM files in different @@ -69,13 +88,19 @@ namespace osmium { protected: - osmium::io::File m_file; - data_queue_type& m_output_queue; + future_string_queue_type& m_output_queue; + + /** + * Wrap the string into a future and add it to the output + * queue. + */ + void send_to_output_queue(std::string&& data) { + add_to_queue(m_output_queue, std::move(data)); + } public: - explicit OutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : - m_file(file), + explicit OutputFormat(future_string_queue_type& output_queue) : m_output_queue(output_queue) { } @@ -85,15 +110,15 @@ namespace osmium { OutputFormat& operator=(const OutputFormat&) = delete; OutputFormat& operator=(OutputFormat&&) = delete; - virtual ~OutputFormat() { - } + virtual ~OutputFormat() noexcept = default; virtual void write_header(const osmium::io::Header&) { } virtual void write_buffer(osmium::memory::Buffer&&) = 0; - virtual void close() = 0; + virtual void write_end() { + } }; // class OutputFormat @@ -108,7 +133,7 @@ namespace osmium { public: - typedef std::function create_output_type; + typedef std::function create_output_type; private: @@ -134,15 +159,18 @@ namespace osmium { return true; } - std::unique_ptr create_output(const osmium::io::File& file, data_queue_type& output_queue) { - file.check(); - + std::unique_ptr create_output(const osmium::io::File& file, future_string_queue_type& output_queue) { auto it = m_callbacks.find(file.format()); if (it != m_callbacks.end()) { return std::unique_ptr((it->second)(file, output_queue)); } - throw std::runtime_error(std::string("Support for output format '") + as_string(file.format()) + "' not compiled into this binary."); + throw unsupported_file_format_error( + std::string("Can not open file '") + + file.filename() + + "' with type '" + + as_string(file.format()) + + "'. No support for writing this format in this program."); } }; // class OutputFormatFactory diff --git a/include/osmium/io/detail/pbf.hpp b/include/osmium/io/detail/pbf.hpp index 15e457a12..13d552910 100644 --- a/include/osmium/io/detail/pbf.hpp +++ b/include/osmium/io/detail/pbf.hpp @@ -33,6 +33,7 @@ DEALINGS IN THE SOFTWARE. */ +#include #include // needed for htonl and ntohl @@ -53,11 +54,11 @@ namespace osmium { */ struct pbf_error : public io_error { - pbf_error(const std::string& what) : + explicit pbf_error(const std::string& what) : io_error(std::string("PBF error: ") + what) { } - pbf_error(const char* what) : + explicit pbf_error(const char* what) : io_error(std::string("PBF error: ") + what) { } @@ -79,9 +80,9 @@ namespace osmium { const int64_t resolution_convert = lonlat_resolution / osmium::Location::coordinate_precision; - } + } // namespace detail - } + } // namespace io } // namespace osmium diff --git a/include/osmium/io/detail/pbf_decoder.hpp b/include/osmium/io/detail/pbf_decoder.hpp index 79e899ff8..09e09bf02 100644 --- a/include/osmium/io/detail/pbf_decoder.hpp +++ b/include/osmium/io/detail/pbf_decoder.hpp @@ -37,8 +37,12 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include #include +#include +#include +#include +#include +#include #include @@ -62,13 +66,14 @@ namespace osmium { namespace detail { using ptr_len_type = std::pair; + using osm_string_len_type = std::pair; class PBFPrimitiveBlockDecoder { static constexpr size_t initial_buffer_size = 2 * 1024 * 1024; ptr_len_type m_data; - std::vector m_stringtable; + std::vector m_stringtable; int64_t m_lon_offset = 0; int64_t m_lat_offset = 0; @@ -86,7 +91,11 @@ namespace osmium { protozero::pbf_message pbf_string_table(data); while (pbf_string_table.next(OSMFormat::StringTable::repeated_bytes_s)) { - m_stringtable.push_back(pbf_string_table.get_data()); + auto str_len = pbf_string_table.get_data(); + if (str_len.second > osmium::max_osm_string_length) { + throw osmium::pbf_error("overlong string in string table"); + } + m_stringtable.emplace_back(str_len.first, osmium::string_size_type(str_len.second)); } } @@ -156,8 +165,8 @@ namespace osmium { } } - ptr_len_type decode_info(const ptr_len_type& data, osmium::OSMObject& object) { - ptr_len_type user = std::make_pair("", 0); + osm_string_len_type decode_info(const ptr_len_type& data, osmium::OSMObject& object) { + osm_string_len_type user = std::make_pair("", 0); protozero::pbf_message pbf_info(data); while (pbf_info.next()) { @@ -220,7 +229,7 @@ namespace osmium { } int32_t convert_pbf_coordinate(int64_t c) const { - return (c * m_granularity + m_lon_offset) / resolution_convert; + return int32_t((c * m_granularity + m_lon_offset) / resolution_convert); } void decode_node(const ptr_len_type& data) { @@ -232,7 +241,7 @@ namespace osmium { int64_t lon = std::numeric_limits::max(); int64_t lat = std::numeric_limits::max(); - ptr_len_type user = { "", 0 }; + osm_string_len_type user = { "", 0 }; protozero::pbf_message pbf_node(data); while (pbf_node.next()) { @@ -285,7 +294,7 @@ namespace osmium { kv_type vals; std::pair refs; - ptr_len_type user = { "", 0 }; + osm_string_len_type user = { "", 0 }; protozero::pbf_message pbf_way(data); while (pbf_way.next()) { @@ -334,7 +343,7 @@ namespace osmium { std::pair refs; std::pair types; - ptr_len_type user = { "", 0 }; + osm_string_len_type user = { "", 0 }; protozero::pbf_message pbf_relation(data); while (pbf_relation.next()) { @@ -512,7 +521,7 @@ namespace osmium { // this is against the spec, must have same number of elements throw osmium::pbf_error("PBF format error"); } - visible = *visibles.first++; + visible = (*visibles.first++) != 0; } node.set_visible(visible); @@ -522,10 +531,14 @@ namespace osmium { builder.add_user(""); } + // even if the node isn't visible, there's still a record + // of its lat/lon in the dense arrays. + const auto lon = dense_longitude.update(*lons.first++); + const auto lat = dense_latitude.update(*lats.first++); if (visible) { builder.object().set_location(osmium::Location( - convert_pbf_coordinate(dense_longitude.update(*lons.first++)), - convert_pbf_coordinate(dense_latitude.update(*lats.first++)) + convert_pbf_coordinate(lon), + convert_pbf_coordinate(lat) )); } @@ -552,7 +565,7 @@ namespace osmium { public: - explicit PBFPrimitiveBlockDecoder(const ptr_len_type& data, osmium::osm_entity_bits::type read_types) : + PBFPrimitiveBlockDecoder(const ptr_len_type& data, osmium::osm_entity_bits::type read_types) : m_data(data), m_read_types(read_types) { } @@ -563,7 +576,7 @@ namespace osmium { PBFPrimitiveBlockDecoder(PBFPrimitiveBlockDecoder&&) = delete; PBFPrimitiveBlockDecoder& operator=(PBFPrimitiveBlockDecoder&&) = delete; - ~PBFPrimitiveBlockDecoder() = default; + ~PBFPrimitiveBlockDecoder() noexcept = default; osmium::memory::Buffer operator()() { try { @@ -579,8 +592,8 @@ namespace osmium { }; // class PBFPrimitiveBlockDecoder inline ptr_len_type decode_blob(const std::string& blob_data, std::string& output) { - int32_t raw_size; - std::pair zlib_data; + int32_t raw_size = 0; + std::pair zlib_data = {nullptr, 0}; protozero::pbf_message pbf_blob(blob_data); while (pbf_blob.next()) { @@ -609,7 +622,7 @@ namespace osmium { } } - if (zlib_data.second != 0) { + if (zlib_data.second != 0 && raw_size != 0) { return osmium::io::detail::zlib_uncompress_string( zlib_data.first, static_cast(zlib_data.second), @@ -694,7 +707,11 @@ namespace osmium { header.set("generator", pbf_header_block.get_string()); break; case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp: - header.set("osmosis_replication_timestamp", osmium::Timestamp(pbf_header_block.get_int64()).to_iso()); + { + auto timestamp = osmium::Timestamp(pbf_header_block.get_int64()).to_iso(); + header.set("osmosis_replication_timestamp", timestamp); + header.set("timestamp", timestamp); + } break; case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_sequence_number: header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.get_int64())); @@ -741,7 +758,7 @@ namespace osmium { PBFDataBlobDecoder(PBFDataBlobDecoder&&) = default; PBFDataBlobDecoder& operator=(PBFDataBlobDecoder&&) = default; - ~PBFDataBlobDecoder() = default; + ~PBFDataBlobDecoder() noexcept = default; osmium::memory::Buffer operator()() { std::string output; diff --git a/include/osmium/io/detail/pbf_input_format.hpp b/include/osmium/io/detail/pbf_input_format.hpp index 7817d27d1..e9d0e71d1 100644 --- a/include/osmium/io/detail/pbf_input_format.hpp +++ b/include/osmium/io/detail/pbf_input_format.hpp @@ -34,17 +34,12 @@ DEALINGS IN THE SOFTWARE. */ #include -#include #include -#include #include #include #include -#include #include -#include #include -#include #include #include #include @@ -58,40 +53,22 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include #include -#include #include -#include #include #include #include -#include #include -#include #include namespace osmium { namespace io { - class File; - namespace detail { - /** - * Class for parsing PBF files. - */ - class PBFInputFormat : public osmium::io::detail::InputFormat { + class PBFParser : public Parser { - typedef osmium::thread::Queue> queue_type; - - bool m_use_thread_pool; - bool m_eof { false }; - queue_type m_queue; - std::atomic m_quit_input_thread; - std::thread m_reader; - osmium::thread::Queue& m_input_queue; std::string m_input_buffer; /** @@ -103,9 +80,8 @@ namespace osmium { */ std::string read_from_input_queue(size_t size) { while (m_input_buffer.size() < size) { - std::string new_data; - m_input_queue.wait_and_pop(new_data); - if (new_data.empty()) { + std::string new_data = get_input(); + if (input_done()) { throw osmium::pbf_error("truncated data (EOF encountered)"); } m_input_buffer += new_data; @@ -113,7 +89,10 @@ namespace osmium { std::string output { m_input_buffer.substr(size) }; m_input_buffer.resize(size); - std::swap(output, m_input_buffer); + + using std::swap; + swap(output, m_input_buffer); + return output; } @@ -125,7 +104,7 @@ namespace osmium { uint32_t size_in_network_byte_order; try { - std::string input_data = read_from_input_queue(sizeof(size_in_network_byte_order)); + const std::string input_data = read_from_input_queue(sizeof(size_in_network_byte_order)); size_in_network_byte_order = *reinterpret_cast(input_data.data()); } catch (osmium::pbf_error&) { return 0; // EOF @@ -174,125 +153,85 @@ namespace osmium { size_t check_type_and_get_blob_size(const char* expected_type) { assert(expected_type); - auto size = read_blob_header_size_from_file(); + const auto size = read_blob_header_size_from_file(); if (size == 0) { // EOF return 0; } - std::string blob_header = read_from_input_queue(size); + const std::string blob_header = read_from_input_queue(size); return decode_blob_header(protozero::pbf_message(blob_header), expected_type); } - void parse_osm_data(osmium::osm_entity_bits::type read_types) { - osmium::thread::set_thread_name("_osmium_pbf_in"); - - while (auto size = check_type_and_get_blob_size("OSMData")) { - std::string input_buffer = read_from_input_queue(size); - if (input_buffer.size() > max_uncompressed_blob_size) { - throw osmium::pbf_error(std::string("invalid blob size: " + std::to_string(input_buffer.size()))); - } - - if (m_use_thread_pool) { - m_queue.push(osmium::thread::Pool::instance().submit(PBFDataBlobDecoder{ std::move(input_buffer), read_types })); - } else { - std::promise promise; - m_queue.push(promise.get_future()); - PBFDataBlobDecoder data_blob_parser{ std::move(input_buffer), read_types }; - promise.set_value(data_blob_parser()); - } - - if (m_quit_input_thread) { - return; - } + std::string read_from_input_queue_with_check(size_t size) { + if (size > max_uncompressed_blob_size) { + throw osmium::pbf_error(std::string("invalid blob size: " + + std::to_string(size))); } - - // Send an empty buffer to signal the reader that we are - // done. - std::promise promise; - m_queue.push(promise.get_future()); - promise.set_value(osmium::memory::Buffer{}); + return read_from_input_queue(size); } - void signal_input_thread_to_quit() { - m_quit_input_thread = true; + // Parse the header in the PBF OSMHeader blob. + void parse_header_blob() { + osmium::io::Header header; + const auto size = check_type_and_get_blob_size("OSMHeader"); + header = decode_header(read_from_input_queue_with_check(size)); + set_header_value(header); + } + + void parse_data_blobs() { + while (const auto size = check_type_and_get_blob_size("OSMData")) { + std::string input_buffer = read_from_input_queue_with_check(size); + + PBFDataBlobDecoder data_blob_parser{ std::move(input_buffer), read_types() }; + + if (osmium::config::use_pool_threads_for_pbf_parsing()) { + send_to_output_queue(osmium::thread::Pool::instance().submit(std::move(data_blob_parser))); + } else { + send_to_output_queue(data_blob_parser()); + } + } } public: - /** - * Instantiate PBF Parser - * - * @param file osmium::io::File instance describing file to be read from. - * @param read_which_entities Which types of OSM entities (nodes, ways, relations, changesets) should be parsed? - * @param input_queue String queue where data is read from. - */ - PBFInputFormat(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue& input_queue) : - osmium::io::detail::InputFormat(file, read_which_entities), - m_use_thread_pool(osmium::config::use_pool_threads_for_pbf_parsing()), - m_queue(20, "pbf_parser_results"), // XXX - m_quit_input_thread(false), - m_input_queue(input_queue), + PBFParser(future_string_queue_type& input_queue, + future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_types) : + Parser(input_queue, output_queue, header_promise, read_types), m_input_buffer() { + } - // handle OSMHeader - const auto size = check_type_and_get_blob_size("OSMHeader"); - m_header = decode_header(read_from_input_queue(size)); + ~PBFParser() noexcept final = default; - if (m_read_which_entities != osmium::osm_entity_bits::nothing) { - m_reader = std::thread(&PBFInputFormat::parse_osm_data, this, m_read_which_entities); + void run() final { + osmium::thread::set_thread_name("_osmium_pbf_in"); + + parse_header_blob(); + + if (read_types() != osmium::osm_entity_bits::nothing) { + parse_data_blobs(); } } - ~PBFInputFormat() { - signal_input_thread_to_quit(); - if (m_reader.joinable()) { - m_reader.join(); - } - } + }; // class PBFParser - /** - * Returns the next buffer with OSM data read from the PBF - * file. Blocks if data is not available yet. - * Returns an empty buffer at end of input. - */ - osmium::memory::Buffer read() override { - osmium::memory::Buffer buffer; - if (m_eof) { - return buffer; - } + // we want the register_parser() function to run, setting + // the variable is only a side-effect, it will never be used + const bool registered_pbf_parser = ParserFactory::instance().register_parser( + file_format::pbf, + [](future_string_queue_type& input_queue, + future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_which_entities) { + return std::unique_ptr(new PBFParser(input_queue, output_queue, header_promise, read_which_entities)); + }); - std::future buffer_future; - m_queue.wait_and_pop(buffer_future); - - try { - buffer = std::move(buffer_future.get()); - if (!buffer) { - m_eof = true; - } - return buffer; - } catch (...) { - m_eof = true; - signal_input_thread_to_quit(); - throw; - } - } - - }; // class PBFInputFormat - - namespace { - -// we want the register_input_format() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" - const bool registered_pbf_input = osmium::io::detail::InputFormatFactory::instance().register_input_format(osmium::io::file_format::pbf, - [](const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue& input_queue) { - return new osmium::io::detail::PBFInputFormat(file, read_which_entities, input_queue); - }); -#pragma GCC diagnostic pop - - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_pbf_parser() noexcept { + return registered_pbf_parser; + } } // namespace detail diff --git a/include/osmium/io/detail/pbf_output_format.hpp b/include/osmium/io/detail/pbf_output_format.hpp index 8d8a079b4..88c1cf489 100644 --- a/include/osmium/io/detail/pbf_output_format.hpp +++ b/include/osmium/io/detail/pbf_output_format.hpp @@ -34,19 +34,18 @@ DEALINGS IN THE SOFTWARE. */ #include -#include +#include #include #include #include -#include #include #include -#include #include -#include #include #include +// needed for older boost libraries +#define BOOST_RESULT_OF_USE_DECLTYPE #include #include @@ -71,6 +70,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include #include #include #include @@ -81,6 +81,32 @@ namespace osmium { namespace detail { + struct pbf_output_options { + + /// Should nodes be encoded in DenseNodes? + bool use_dense_nodes; + + /** + * Should the PBF blobs contain zlib compressed data? + * + * The zlib compression is optional, it's possible to store the + * blobs in raw format. Disabling the compression can improve + * the writing speed a little but the output will be 2x to 3x + * bigger. + */ + bool use_compression; + + /// Should metadata of objects be written? + bool add_metadata; + + /// Add the "HistoricalInformation" header flag. + bool add_historical_information_flag; + + /// Should the visible flag be added to all OSM objects? + bool add_visible_flag; + + }; + /** * Maximum number of items in a primitive block. * @@ -104,43 +130,81 @@ namespace osmium { return static_cast(std::round(lonlat * lonlat_resolution / location_granularity)); } - /** - * Serialize a protobuf message into a Blob, optionally apply compression - * and return it together with a BlobHeader ready to be written to a file. - * - * @param type Type-string used in the BlobHeader. - * @param msg Protobuf-message. - * @param use_compression Should the output be compressed using zlib? - */ - inline std::string serialize_blob(const std::string& type, const std::string& msg, bool use_compression) { - std::string blob_data; - protozero::pbf_builder pbf_blob(blob_data); + enum class pbf_blob_type { + header = 0, + data = 1 + }; - if (use_compression) { - pbf_blob.add_int32(FileFormat::Blob::optional_int32_raw_size, msg.size()); - pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_zlib_data, osmium::io::detail::zlib_compress(msg)); - } else { - pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_raw, msg); + class SerializeBlob { + + std::string m_msg; + + pbf_blob_type m_blob_type; + + bool m_use_compression; + + public: + + /** + * Initialize a blob serializer. + * + * @param msg Protobuf-message containing the blob data + * @param type Type of blob. + * @param use_compression Should the output be compressed using + * zlib? + */ + SerializeBlob(std::string&& msg, pbf_blob_type type, bool use_compression) : + m_msg(std::move(msg)), + m_blob_type(type), + m_use_compression(use_compression) { } - std::string blob_header_data; - protozero::pbf_builder pbf_blob_header(blob_header_data); + /** + * Serialize a protobuf message into a Blob, optionally apply + * compression and return it together with a BlobHeader ready + * to be written to a file. + */ + std::string operator()() { + assert(m_msg.size() <= max_uncompressed_blob_size); - pbf_blob_header.add_string(FileFormat::BlobHeader::required_string_type, type); - pbf_blob_header.add_int32(FileFormat::BlobHeader::required_int32_datasize, blob_data.size()); + std::string blob_data; + protozero::pbf_builder pbf_blob(blob_data); - uint32_t sz = htonl(static_cast_with_assert(blob_header_data.size())); + if (m_use_compression) { + pbf_blob.add_int32(FileFormat::Blob::optional_int32_raw_size, int32_t(m_msg.size())); + pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_zlib_data, osmium::io::detail::zlib_compress(m_msg)); + } else { + pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_raw, m_msg); + } - // write to output: the 4-byte BlobHeader-Size followed by the BlobHeader followed by the Blob - std::string output; - output.reserve(sizeof(sz) + blob_header_data.size() + blob_data.size()); - output.append(reinterpret_cast(&sz), sizeof(sz)); - output.append(blob_header_data); - output.append(blob_data); + std::string blob_header_data; + protozero::pbf_builder pbf_blob_header(blob_header_data); - return output; - } + pbf_blob_header.add_string(FileFormat::BlobHeader::required_string_type, m_blob_type == pbf_blob_type::data ? "OSMData" : "OSMHeader"); + pbf_blob_header.add_int32(FileFormat::BlobHeader::required_int32_datasize, static_cast_with_assert(blob_data.size())); + uint32_t sz = htonl(static_cast_with_assert(blob_header_data.size())); + + // write to output: the 4-byte BlobHeader-Size followed by the BlobHeader followed by the Blob + std::string output; + output.reserve(sizeof(sz) + blob_header_data.size() + blob_data.size()); + output.append(reinterpret_cast(&sz), sizeof(sz)); + output.append(blob_header_data); + output.append(blob_data); + + return output; + } + + }; // class SerializeBlob + + /** + * Contains the code to pack any number of nodes into a DenseNode + * structure. + * + * Because this needs to allocate a lot of memory on the heap, + * only one object of this class will be created and then re-used + * after calling clear() on it. + */ class DenseNodes { StringTable& m_stringtable; @@ -158,27 +222,26 @@ namespace osmium { std::vector m_lons; std::vector m_tags; - osmium::util::DeltaEncode m_delta_id; + osmium::util::DeltaEncode m_delta_id; - osmium::util::DeltaEncode m_delta_timestamp; - osmium::util::DeltaEncode m_delta_changeset; - osmium::util::DeltaEncode m_delta_uid; - osmium::util::DeltaEncode m_delta_user_sid; + osmium::util::DeltaEncode m_delta_timestamp; + osmium::util::DeltaEncode m_delta_changeset; + osmium::util::DeltaEncode m_delta_uid; + osmium::util::DeltaEncode m_delta_user_sid; - osmium::util::DeltaEncode m_delta_lat; - osmium::util::DeltaEncode m_delta_lon; + osmium::util::DeltaEncode m_delta_lat; + osmium::util::DeltaEncode m_delta_lon; - bool m_add_metadata; - bool m_add_visible; + const pbf_output_options& m_options; public: - DenseNodes(StringTable& stringtable, bool add_metadata, bool add_visible) : + DenseNodes(StringTable& stringtable, const pbf_output_options& options) : m_stringtable(stringtable), - m_add_metadata(add_metadata), - m_add_visible(add_visible) { + m_options(options) { } + /// Clear object for re-use. Keep the allocated memory. void clear() { m_ids.clear(); @@ -211,13 +274,13 @@ namespace osmium { void add_node(const osmium::Node& node) { m_ids.push_back(m_delta_id.update(node.id())); - if (m_add_metadata) { - m_versions.push_back(node.version()); - m_timestamps.push_back(m_delta_timestamp.update(node.timestamp())); + if (m_options.add_metadata) { + m_versions.push_back(static_cast_with_assert(node.version())); + m_timestamps.push_back(m_delta_timestamp.update(uint32_t(node.timestamp()))); m_changesets.push_back(m_delta_changeset.update(node.changeset())); m_uids.push_back(m_delta_uid.update(node.uid())); m_user_sids.push_back(m_delta_user_sid.update(m_stringtable.add(node.user()))); - if (m_add_visible) { + if (m_options.add_visible_flag) { m_visibles.push_back(node.visible()); } } @@ -226,8 +289,8 @@ namespace osmium { m_lons.push_back(m_delta_lon.update(lonlat2int(node.location().lon_without_check()))); for (const auto& tag : node.tags()) { - m_tags.push_back(m_stringtable.add(tag.key())); - m_tags.push_back(m_stringtable.add(tag.value())); + m_tags.push_back(static_cast_with_assert(m_stringtable.add(tag.key()))); + m_tags.push_back(static_cast_with_assert(m_stringtable.add(tag.value()))); } m_tags.push_back(0); } @@ -238,7 +301,7 @@ namespace osmium { pbf_dense_nodes.add_packed_sint64(OSMFormat::DenseNodes::packed_sint64_id, m_ids.cbegin(), m_ids.cend()); - if (m_add_metadata) { + if (m_options.add_metadata) { protozero::pbf_builder pbf_dense_info(pbf_dense_nodes, OSMFormat::DenseNodes::optional_DenseInfo_denseinfo); pbf_dense_info.add_packed_int32(OSMFormat::DenseInfo::packed_int32_version, m_versions.cbegin(), m_versions.cend()); pbf_dense_info.add_packed_sint64(OSMFormat::DenseInfo::packed_sint64_timestamp, m_timestamps.cbegin(), m_timestamps.cend()); @@ -246,7 +309,7 @@ namespace osmium { pbf_dense_info.add_packed_sint32(OSMFormat::DenseInfo::packed_sint32_uid, m_uids.cbegin(), m_uids.cend()); pbf_dense_info.add_packed_sint32(OSMFormat::DenseInfo::packed_sint32_user_sid, m_user_sids.cbegin(), m_user_sids.cend()); - if (m_add_visible) { + if (m_options.add_visible_flag) { pbf_dense_info.add_packed_bool(OSMFormat::DenseInfo::packed_bool_visible, m_visibles.cbegin(), m_visibles.cend()); } } @@ -272,11 +335,11 @@ namespace osmium { public: - PrimitiveBlock(bool add_metadata, bool add_visible) : + explicit PrimitiveBlock(const pbf_output_options& options) : m_pbf_primitive_group_data(), m_pbf_primitive_group(m_pbf_primitive_group_data), m_stringtable(), - m_dense_nodes(m_stringtable, add_metadata, add_visible), + m_dense_nodes(m_stringtable, options), m_type(OSMFormat::PrimitiveGroup::unknown), m_count(0) { } @@ -312,7 +375,7 @@ namespace osmium { ++m_count; } - size_t add_string(const char* s) { + uint32_t store_in_stringtable(const char* s) { return m_stringtable.add(s); } @@ -350,24 +413,7 @@ namespace osmium { class PBFOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler { - /// Should nodes be encoded in DenseNodes? - bool m_use_dense_nodes; - - /** - * Should the PBF blobs contain zlib compressed data? - * - * The zlib compression is optional, it's possible to store the - * blobs in raw format. Disabling the compression can improve - * the writing speed a little but the output will be 2x to 3x - * bigger. - */ - bool m_use_compression; - - /// Should metadata of objects be written? - bool m_add_metadata; - - /// Should the visible flag be added to objects? - bool m_add_visible; + pbf_output_options m_options; PrimitiveBlock m_primitive_block; @@ -386,20 +432,22 @@ namespace osmium { primitive_block.add_message(OSMFormat::PrimitiveBlock::repeated_PrimitiveGroup_primitivegroup, m_primitive_block.group_data()); - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(serialize_blob("OSMData", primitive_block_data, m_use_compression)); + m_output_queue.push(osmium::thread::Pool::instance().submit( + SerializeBlob{std::move(primitive_block_data), + pbf_blob_type::data, + m_options.use_compression} + )); } template void add_meta(const osmium::OSMObject& object, T& pbf_object) { const osmium::TagList& tags = object.tags(); - auto map_tag_key = [this](const osmium::Tag& tag) -> size_t { - return m_primitive_block.add_string(tag.key()); + auto map_tag_key = [this](const osmium::Tag& tag) -> uint32_t { + return m_primitive_block.store_in_stringtable(tag.key()); }; - auto map_tag_value = [this](const osmium::Tag& tag) -> size_t { - return m_primitive_block.add_string(tag.value()); + auto map_tag_value = [this](const osmium::Tag& tag) -> uint32_t { + return m_primitive_block.store_in_stringtable(tag.value()); }; pbf_object.add_packed_uint32(T::enum_type::packed_uint32_keys, @@ -410,39 +458,46 @@ namespace osmium { boost::make_transform_iterator(tags.begin(), map_tag_value), boost::make_transform_iterator(tags.end(), map_tag_value)); - if (m_add_metadata) { + if (m_options.add_metadata) { protozero::pbf_builder pbf_info(pbf_object, T::enum_type::optional_Info_info); - pbf_info.add_int32(OSMFormat::Info::optional_int32_version, object.version()); - pbf_info.add_int64(OSMFormat::Info::optional_int64_timestamp, object.timestamp()); + pbf_info.add_int32(OSMFormat::Info::optional_int32_version, static_cast_with_assert(object.version())); + pbf_info.add_int64(OSMFormat::Info::optional_int64_timestamp, uint32_t(object.timestamp())); pbf_info.add_int64(OSMFormat::Info::optional_int64_changeset, object.changeset()); - pbf_info.add_int32(OSMFormat::Info::optional_int32_uid, object.uid()); - pbf_info.add_uint32(OSMFormat::Info::optional_uint32_user_sid, m_primitive_block.add_string(object.user())); - if (m_add_visible) { + pbf_info.add_int32(OSMFormat::Info::optional_int32_uid, static_cast_with_assert(object.uid())); + pbf_info.add_uint32(OSMFormat::Info::optional_uint32_user_sid, m_primitive_block.store_in_stringtable(object.user())); + if (m_options.add_visible_flag) { pbf_info.add_bool(OSMFormat::Info::optional_bool_visible, object.visible()); } } } - PBFOutputFormat(const PBFOutputFormat&) = delete; - PBFOutputFormat& operator=(const PBFOutputFormat&) = delete; + void switch_primitive_block_type(OSMFormat::PrimitiveGroup type) { + if (!m_primitive_block.can_add(type)) { + store_primitive_block(); + m_primitive_block.reset(type); + } + } public: - explicit PBFOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : - OutputFormat(file, output_queue), - m_use_dense_nodes(file.get("pbf_dense_nodes") != "false"), - m_use_compression(file.get("pbf_compression") != "none" && file.get("pbf_compression") != "false"), - m_add_metadata(file.get("pbf_add_metadata") != "false" && file.get("add_metadata") != "false"), - m_add_visible(file.has_multiple_object_versions()), - m_primitive_block(m_add_metadata, m_add_visible) { + PBFOutputFormat(const osmium::io::File& file, future_string_queue_type& output_queue) : + OutputFormat(output_queue), + m_options(), + m_primitive_block(m_options) { + m_options.use_dense_nodes = file.is_not_false("pbf_dense_nodes"); + m_options.use_compression = file.get("pbf_compression") != "none" && file.is_not_false("pbf_compression"); + m_options.add_metadata = file.is_not_false("pbf_add_metadata") && file.is_not_false("add_metadata"); + m_options.add_historical_information_flag = file.has_multiple_object_versions(); + m_options.add_visible_flag = file.has_multiple_object_versions(); } - void write_buffer(osmium::memory::Buffer&& buffer) override final { - osmium::apply(buffer.cbegin(), buffer.cend(), *this); - } + PBFOutputFormat(const PBFOutputFormat&) = delete; + PBFOutputFormat& operator=(const PBFOutputFormat&) = delete; - void write_header(const osmium::io::Header& header) override final { + ~PBFOutputFormat() noexcept final = default; + + void write_header(const osmium::io::Header& header) final { std::string data; protozero::pbf_builder pbf_header_block(data); @@ -450,19 +505,19 @@ namespace osmium { protozero::pbf_builder pbf_header_bbox(pbf_header_block, OSMFormat::HeaderBlock::optional_HeaderBBox_bbox); osmium::Box box = header.joined_boxes(); - pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_left, box.bottom_left().lon() * lonlat_resolution); - pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_right, box.top_right().lon() * lonlat_resolution); - pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_top, box.top_right().lat() * lonlat_resolution); - pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_bottom, box.bottom_left().lat() * lonlat_resolution); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_left, int64_t(box.bottom_left().lon() * lonlat_resolution)); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_right, int64_t(box.top_right().lon() * lonlat_resolution)); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_top, int64_t(box.top_right().lat() * lonlat_resolution)); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_bottom, int64_t(box.bottom_left().lat() * lonlat_resolution)); } pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "OsmSchema-V0.6"); - if (m_use_dense_nodes) { + if (m_options.use_dense_nodes) { pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "DenseNodes"); } - if (m_file.has_multiple_object_versions()) { + if (m_options.add_historical_information_flag) { pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "HistoricalInformation"); } @@ -471,7 +526,7 @@ namespace osmium { std::string osmosis_replication_timestamp = header.get("osmosis_replication_timestamp"); if (!osmosis_replication_timestamp.empty()) { osmium::Timestamp ts(osmosis_replication_timestamp.c_str()); - pbf_header_block.add_int64(OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp, ts); + pbf_header_block.add_int64(OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp, uint32_t(ts)); } std::string osmosis_replication_sequence_number = header.get("osmosis_replication_sequence_number"); @@ -484,20 +539,23 @@ namespace osmium { pbf_header_block.add_string(OSMFormat::HeaderBlock::optional_string_osmosis_replication_base_url, osmosis_replication_base_url); } - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(serialize_blob("OSMHeader", data, m_use_compression)); + m_output_queue.push(osmium::thread::Pool::instance().submit( + SerializeBlob{std::move(data), + pbf_blob_type::header, + m_options.use_compression} + )); } - void switch_primitive_block_type(OSMFormat::PrimitiveGroup type) { - if (!m_primitive_block.can_add(type)) { - store_primitive_block(); - m_primitive_block.reset(type); - } + void write_buffer(osmium::memory::Buffer&& buffer) final { + osmium::apply(buffer.cbegin(), buffer.cend(), *this); + } + + void write_end() final { + store_primitive_block(); } void node(const osmium::Node& node) { - if (m_use_dense_nodes) { + if (m_options.use_dense_nodes) { switch_primitive_block_type(OSMFormat::PrimitiveGroup::optional_DenseNodes_dense); m_primitive_block.add_dense_node(node); return; @@ -538,8 +596,8 @@ namespace osmium { pbf_relation.add_int64(OSMFormat::Relation::required_int64_id, relation.id()); add_meta(relation, pbf_relation); - auto map_member_role = [this](const osmium::RelationMember& member) -> size_t { - return m_primitive_block.add_string(member.role()); + auto map_member_role = [this](const osmium::RelationMember& member) -> uint32_t { + return m_primitive_block.store_in_stringtable(member.role()); }; pbf_relation.add_packed_int32(OSMFormat::Relation::packed_int32_roles_sid, boost::make_transform_iterator(relation.members().begin(), map_member_role), @@ -554,41 +612,27 @@ namespace osmium { it_type last { members.cend(), members.cend(), map_member_ref }; pbf_relation.add_packed_sint64(OSMFormat::Relation::packed_sint64_memids, first, last); - static auto map_member_type = [](const osmium::RelationMember& member) noexcept -> int { - return osmium::item_type_to_nwr_index(member.type()); + static auto map_member_type = [](const osmium::RelationMember& member) noexcept -> int32_t { + return int32_t(osmium::item_type_to_nwr_index(member.type())); }; pbf_relation.add_packed_int32(OSMFormat::Relation::packed_MemberType_types, boost::make_transform_iterator(relation.members().begin(), map_member_type), boost::make_transform_iterator(relation.members().end(), map_member_type)); } - /** - * Finalize the writing process, flush any open primitive - * blocks to the file and close the file. - */ - void close() override final { - store_primitive_block(); - - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(std::string()); - } - }; // class PBFOutputFormat - namespace { + // we want the register_output_format() function to run, setting + // the variable is only a side-effect, it will never be used + const bool registered_pbf_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::pbf, + [](const osmium::io::File& file, future_string_queue_type& output_queue) { + return new osmium::io::detail::PBFOutputFormat(file, output_queue); + }); -// we want the register_output_format() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" - const bool registered_pbf_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::pbf, - [](const osmium::io::File& file, data_queue_type& output_queue) { - return new osmium::io::detail::PBFOutputFormat(file, output_queue); - }); -#pragma GCC diagnostic pop - - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_pbf_output() noexcept { + return registered_pbf_output; + } } // namespace detail diff --git a/include/osmium/io/detail/queue_util.hpp b/include/osmium/io/detail/queue_util.hpp new file mode 100644 index 000000000..6c9f071bc --- /dev/null +++ b/include/osmium/io/detail/queue_util.hpp @@ -0,0 +1,157 @@ +#ifndef OSMIUM_IO_DETAIL_QUEUE_UTIL_HPP +#define OSMIUM_IO_DETAIL_QUEUE_UTIL_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include + +#include +#include + +namespace osmium { + + namespace io { + + namespace detail { + + /** + * This type of queue contains buffers with OSM data in them. + * The "end of file" is marked by an invalid Buffer. + * The buffers are wrapped in a std::future so that they can also + * transport exceptions. The future also helps with keeping the + * data in order. + */ + using future_buffer_queue_type = osmium::thread::Queue>; + + /** + * This type of queue contains OSM file data in the form it is + * stored on disk, ie encoded as XML, PBF, etc. + * The "end of file" is marked by an empty string. + */ + using string_queue_type = osmium::thread::Queue; + + /** + * This type of queue contains OSM file data in the form it is + * stored on disk, ie encoded as XML, PBF, etc. + * The "end of file" is marked by an empty string. + * The strings are wrapped in a std::future so that they can also + * transport exceptions. The future also helps with keeping the + * data in order. + */ + using future_string_queue_type = osmium::thread::Queue>; + + template + inline void add_to_queue(osmium::thread::Queue>& queue, T&& data) { + std::promise promise; + queue.push(promise.get_future()); + promise.set_value(std::forward(data)); + } + + template + inline void add_to_queue(osmium::thread::Queue>& queue, std::exception_ptr&& exception) { + std::promise promise; + queue.push(promise.get_future()); + promise.set_exception(std::move(exception)); + } + + template + inline void add_end_of_data_to_queue(osmium::thread::Queue>& queue) { + add_to_queue(queue, T{}); + } + + inline bool at_end_of_data(const std::string& data) { + return data.empty(); + } + + inline bool at_end_of_data(osmium::memory::Buffer& buffer) { + return !buffer; + } + + template + class queue_wrapper { + + using queue_type = osmium::thread::Queue>; + + queue_type& m_queue; + bool m_has_reached_end_of_data; + + public: + + explicit queue_wrapper(queue_type& queue) : + m_queue(queue), + m_has_reached_end_of_data(false) { + } + + ~queue_wrapper() noexcept { + drain(); + } + + void drain() { + while (!m_has_reached_end_of_data) { + try { + pop(); + } catch (...) { + // Ignore any exceptions. + } + } + } + + bool has_reached_end_of_data() const noexcept { + return m_has_reached_end_of_data; + } + + T pop() { + T data; + if (!m_has_reached_end_of_data) { + std::future data_future; + m_queue.wait_and_pop(data_future); + data = std::move(data_future.get()); + if (at_end_of_data(data)) { + m_has_reached_end_of_data = true; + } + } + return data; + } + + }; // class queue_wrapper + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_QUEUE_UTIL_HPP diff --git a/include/osmium/io/detail/read_thread.hpp b/include/osmium/io/detail/read_thread.hpp index bce4f5507..6f96c0b65 100644 --- a/include/osmium/io/detail/read_thread.hpp +++ b/include/osmium/io/detail/read_thread.hpp @@ -34,14 +34,13 @@ DEALINGS IN THE SOFTWARE. */ #include -#include -#include +#include #include #include #include #include -#include +#include #include namespace osmium { @@ -50,52 +49,80 @@ namespace osmium { namespace detail { - class ReadThread { + /** + * This code uses an internally managed thread to read data from + * the input file and (optionally) decompress it. The result is + * sent to the given queue. Any exceptions will also be send to + * the queue. + */ + class ReadThreadManager { - osmium::thread::Queue& m_queue; - osmium::io::Decompressor* m_decompressor; + // only used in the sub-thread + osmium::io::Decompressor& m_decompressor; + future_string_queue_type& m_queue; - // If this is set in the main thread, we have to wrap up at the - // next possible moment. - std::atomic& m_done; + // used in both threads + std::atomic m_done; - public: + // only used in the main thread + std::thread m_thread; - explicit ReadThread(osmium::thread::Queue& queue, osmium::io::Decompressor* decompressor, std::atomic& done) : - m_queue(queue), - m_decompressor(decompressor), - m_done(done) { - } - - bool operator()() { - osmium::thread::set_thread_name("_osmium_input"); + void run_in_thread() { + osmium::thread::set_thread_name("_osmium_read"); try { while (!m_done) { - std::string data {m_decompressor->read()}; - if (data.empty()) { - m_queue.push(std::move(data)); + std::string data {m_decompressor.read()}; + if (at_end_of_data(data)) { break; } - m_queue.push(std::move(data)); - while (m_queue.size() > 10 && !m_done) { - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } + add_to_queue(m_queue, std::move(data)); } - m_decompressor->close(); + m_decompressor.close(); } catch (...) { - // If there is an exception in this thread, we make sure - // to push an empty string onto the queue to signal the - // end-of-data to the reading thread so that it will not - // hang. Then we re-throw the exception. - m_queue.push(std::string()); - throw; + add_to_queue(m_queue, std::current_exception()); } - return true; + + add_end_of_data_to_queue(m_queue); } - }; // class ReadThread + public: + + ReadThreadManager(osmium::io::Decompressor& decompressor, + future_string_queue_type& queue) : + m_decompressor(decompressor), + m_queue(queue), + m_done(false), + m_thread(std::thread(&ReadThreadManager::run_in_thread, this)) { + } + + ReadThreadManager(const ReadThreadManager&) = delete; + ReadThreadManager& operator=(const ReadThreadManager&) = delete; + + ReadThreadManager(ReadThreadManager&&) = delete; + ReadThreadManager& operator=(ReadThreadManager&&) = delete; + + ~ReadThreadManager() noexcept { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } + } + + void stop() noexcept { + m_done = true; + } + + void close() { + stop(); + if (m_thread.joinable()) { + m_thread.join(); + } + } + + }; // class ReadThreadManager } // namespace detail diff --git a/include/osmium/io/detail/read_write.hpp b/include/osmium/io/detail/read_write.hpp index 9863bd719..6db6d8aa8 100644 --- a/include/osmium/io/detail/read_write.hpp +++ b/include/osmium/io/detail/read_write.hpp @@ -35,6 +35,7 @@ DEALINGS IN THE SOFTWARE. #include #include +#include #include #include #include @@ -45,7 +46,7 @@ DEALINGS IN THE SOFTWARE. # include #endif -#include +#include namespace osmium { @@ -68,23 +69,26 @@ namespace osmium { */ inline int open_for_writing(const std::string& filename, osmium::io::overwrite allow_overwrite = osmium::io::overwrite::no) { if (filename == "" || filename == "-") { - return 1; // stdout - } else { - int flags = O_WRONLY | O_CREAT; - if (allow_overwrite == osmium::io::overwrite::allow) { - flags |= O_TRUNC; - } else { - flags |= O_EXCL; - } #ifdef _WIN32 - flags |= O_BINARY; + _setmode(1, _O_BINARY); #endif - int fd = ::open(filename.c_str(), flags, 0666); - if (fd < 0) { - throw std::system_error(errno, std::system_category(), std::string("Open failed for '") + filename + "'"); - } - return fd; + return 1; // stdout } + + int flags = O_WRONLY | O_CREAT; + if (allow_overwrite == osmium::io::overwrite::allow) { + flags |= O_TRUNC; + } else { + flags |= O_EXCL; + } +#ifdef _WIN32 + flags |= O_BINARY; +#endif + int fd = ::open(filename.c_str(), flags, 0666); + if (fd < 0) { + throw std::system_error(errno, std::system_category(), std::string("Open failed for '") + filename + "'"); + } + return fd; } /** @@ -98,17 +102,17 @@ namespace osmium { inline int open_for_reading(const std::string& filename) { if (filename == "" || filename == "-") { return 0; // stdin - } else { - int flags = O_RDONLY; -#ifdef _WIN32 - flags |= O_BINARY; -#endif - int fd = ::open(filename.c_str(), flags); - if (fd < 0) { - throw std::system_error(errno, std::system_category(), std::string("Open failed for '") + filename + "'"); - } - return fd; } + + int flags = O_RDONLY; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + int fd = ::open(filename.c_str(), flags); + if (fd < 0) { + throw std::system_error(errno, std::system_category(), std::string("Open failed for '") + filename + "'"); + } + return fd; } /** @@ -151,6 +155,22 @@ namespace osmium { reliable_write(fd, reinterpret_cast(output_buffer), size); } + inline void reliable_fsync(const int fd) { +#ifdef _WIN32 + if (_commit(fd) != 0) { +#else + if (::fsync(fd) != 0) { +#endif + throw std::system_error(errno, std::system_category(), "Fsync failed"); + } + } + + inline void reliable_close(const int fd) { + if (::close(fd) != 0) { + throw std::system_error(errno, std::system_category(), "Close failed"); + } + } + } // namespace detail } // namespace io diff --git a/include/osmium/io/detail/string_table.hpp b/include/osmium/io/detail/string_table.hpp index ae9d5f0ce..1cb142d00 100644 --- a/include/osmium/io/detail/string_table.hpp +++ b/include/osmium/io/detail/string_table.hpp @@ -34,6 +34,7 @@ DEALINGS IN THE SOFTWARE. */ #include +#include #include #include #include @@ -41,6 +42,8 @@ DEALINGS IN THE SOFTWARE. #include #include +#include + namespace osmium { namespace io { @@ -72,7 +75,7 @@ namespace osmium { public: - StringStore(size_t chunk_size) : + explicit StringStore(size_t chunk_size) : m_chunk_size(chunk_size), m_chunks() { add_chunk(); @@ -172,15 +175,15 @@ namespace osmium { // These functions get you some idea how much memory was // used. - int get_chunk_size() const noexcept { + size_t get_chunk_size() const noexcept { return m_chunk_size; } - int get_chunk_count() const noexcept { + size_t get_chunk_count() const noexcept { return m_chunks.size(); } - int get_used_bytes_in_last_chunk() const noexcept { + size_t get_used_bytes_in_last_chunk() const noexcept { return m_chunks.front().size(); } @@ -196,9 +199,16 @@ namespace osmium { class StringTable { + // This is the maximum number of entries in a string table. + // This should never be reached in practice but we better + // make sure it doesn't. If we had max_uncompressed_blob_size + // many entries, we are sure they would never fit into a PBF + // Blob. + static constexpr const uint32_t max_entries = max_uncompressed_blob_size; + StringStore m_strings; std::map m_index; - size_t m_size; + uint32_t m_size; public: @@ -216,18 +226,23 @@ namespace osmium { m_strings.add(""); } - size_t size() const noexcept { + uint32_t size() const noexcept { return m_size + 1; } - size_t add(const char* s) { + uint32_t add(const char* s) { auto f = m_index.find(s); if (f != m_index.end()) { - return f->second; + return uint32_t(f->second); } const char* cs = m_strings.add(s); m_index[cs] = ++m_size; + + if (m_size > max_entries) { + throw osmium::pbf_error("string table has too many entries"); + } + return m_size; } diff --git a/include/osmium/io/detail/string_util.hpp b/include/osmium/io/detail/string_util.hpp new file mode 100644 index 000000000..672266a92 --- /dev/null +++ b/include/osmium/io/detail/string_util.hpp @@ -0,0 +1,206 @@ +#ifndef OSMIUM_IO_DETAIL_STRING_UTIL_HPP +#define OSMIUM_IO_DETAIL_STRING_UTIL_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include + +#include + +namespace osmium { + + namespace io { + + namespace detail { + +#ifndef _MSC_VER +# define SNPRINTF std::snprintf +#else +# define SNPRINTF _snprintf +#endif + + template + inline int string_snprintf(std::string& out, + size_t old_size, + size_t max_size, + const char* format, + TArgs&&... args) { + out.resize(old_size + max_size); + + return SNPRINTF(max_size ? const_cast(out.c_str()) + old_size : nullptr, + max_size, + format, + std::forward(args)...); + } + +#undef SNPRINTF + + /** + * This is a helper function for writing printf-like formatted + * data into a std::string. + * + * @param out The data will be appended to this string. + * @param format A string with formatting instructions a la printf. + * @param args Any further arguments like in printf. + * @throws std::bad_alloc If the string needed to grow and there + * wasn't enough memory. + */ + template + inline void append_printf_formatted_string(std::string& out, + const char* format, + TArgs&&... args) { + + // First try to write string with the max_size, if that doesn't + // work snprintf will tell us how much space it needs. We + // reserve that much space and try again. So this will always + // work, even if the output is larger than the given max_size. + // + // Unfortunately this trick doesn't work on Windows, because + // the _snprintf() function there only returns the length it + // needs if max_size==0 and the buffer pointer is the null + // pointer. So we have to take this into account. + +#ifndef _MSC_VER + static const size_t max_size = 100; +#else + static const size_t max_size = 0; +#endif + + size_t old_size = out.size(); + + int len = string_snprintf(out, + old_size, + max_size, + format, + std::forward(args)...); + assert(len > 0); + + if (size_t(len) >= max_size) { + int len2 = string_snprintf(out, + old_size, + size_t(len) + 1, + format, + std::forward(args)...); + assert(len2 == len); + } + + out.resize(old_size + size_t(len)); + } + + inline void append_utf8_encoded_string(std::string& out, const char* data) { + const char* end = data + std::strlen(data); + + while (data != end) { + const char* last = data; + uint32_t c = utf8::next(data, end); + + // This is a list of Unicode code points that we let + // through instead of escaping them. It is incomplete + // and can be extended later. + // Generally we don't want to let through any character + // that has special meaning in the OPL format such as + // space, comma, @, etc. and any non-printing characters. + if ((0x0021 <= c && c <= 0x0024) || + (0x0026 <= c && c <= 0x002b) || + (0x002d <= c && c <= 0x003c) || + (0x003e <= c && c <= 0x003f) || + (0x0041 <= c && c <= 0x007e) || + (0x00a1 <= c && c <= 0x00ac) || + (0x00ae <= c && c <= 0x05ff)) { + out.append(last, data); + } else { + out += '%'; + if (c <= 0xff) { + append_printf_formatted_string(out, "%02x", c); + } else { + append_printf_formatted_string(out, "%04x", c); + } + out += '%'; + } + } + } + + inline void append_xml_encoded_string(std::string& out, const char* data) { + for (; *data != '\0'; ++data) { + switch(*data) { + case '&': out += "&"; break; + case '\"': out += """; break; + case '\'': out += "'"; break; + case '<': out += "<"; break; + case '>': out += ">"; break; + case '\n': out += " "; break; + case '\r': out += " "; break; + case '\t': out += " "; break; + default: out += *data; break; + } + } + } + + inline void append_debug_encoded_string(std::string& out, const char* data, const char* prefix, const char* suffix) { + const char* end = data + std::strlen(data); + + while (data != end) { + const char* last = data; + uint32_t c = utf8::next(data, end); + + // This is a list of Unicode code points that we let + // through instead of escaping them. It is incomplete + // and can be extended later. + // Generally we don't want to let through any + // non-printing characters. + if ((0x0020 <= c && c <= 0x0021) || + (0x0023 <= c && c <= 0x003b) || + (0x003d == c) || + (0x003f <= c && c <= 0x007e) || + (0x00a1 <= c && c <= 0x00ac) || + (0x00ae <= c && c <= 0x05ff)) { + out.append(last, data); + } else { + out.append(prefix); + append_printf_formatted_string(out, "", c); + out.append(suffix); + } + } + } + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_STRING_UTIL_HPP diff --git a/include/osmium/io/detail/write_thread.hpp b/include/osmium/io/detail/write_thread.hpp index fad22ed69..81ab1947f 100644 --- a/include/osmium/io/detail/write_thread.hpp +++ b/include/osmium/io/detail/write_thread.hpp @@ -33,11 +33,13 @@ DEALINGS IN THE SOFTWARE. */ +#include +#include #include #include #include -#include +#include #include namespace osmium { @@ -46,33 +48,52 @@ namespace osmium { namespace detail { + /** + * This codes runs in its own thread, getting data from the given + * queue, (optionally) compressing it, and writing it to the output + * file. + */ class WriteThread { - typedef osmium::io::detail::data_queue_type data_queue_type; - - data_queue_type& m_input_queue; - osmium::io::Compressor* m_compressor; + queue_wrapper m_queue; + std::unique_ptr m_compressor; + std::promise m_promise; public: - explicit WriteThread(data_queue_type& input_queue, osmium::io::Compressor* compressor) : - m_input_queue(input_queue), - m_compressor(compressor) { + WriteThread(future_string_queue_type& input_queue, + std::unique_ptr&& compressor, + std::promise&& promise) : + m_queue(input_queue), + m_compressor(std::move(compressor)), + m_promise(std::move(promise)) { } - bool operator()() { - osmium::thread::set_thread_name("_osmium_output"); + WriteThread(const WriteThread&) = delete; + WriteThread& operator=(const WriteThread&) = delete; - std::future data_future; - std::string data; - do { - m_input_queue.wait_and_pop(data_future); - data = data_future.get(); - m_compressor->write(data); - } while (!data.empty()); + WriteThread(WriteThread&&) = delete; + WriteThread& operator=(WriteThread&&) = delete; - m_compressor->close(); - return true; + ~WriteThread() noexcept = default; + + void operator()() { + osmium::thread::set_thread_name("_osmium_write"); + + try { + while (true) { + std::string data = m_queue.pop(); + if (at_end_of_data(data)) { + break; + } + m_compressor->write(data); + } + m_compressor->close(); + m_promise.set_value(true); + } catch (...) { + m_promise.set_exception(std::current_exception()); + m_queue.drain(); + } } }; // class WriteThread diff --git a/include/osmium/io/detail/xml_input_format.hpp b/include/osmium/io/detail/xml_input_format.hpp index b0f3da339..11d3cba08 100644 --- a/include/osmium/io/detail/xml_input_format.hpp +++ b/include/osmium/io/detail/xml_input_format.hpp @@ -33,21 +33,14 @@ DEALINGS IN THE SOFTWARE. */ -#include #include -#include #include #include #include #include #include -#include #include -#include -#include -#include #include -#include #include #include @@ -55,6 +48,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include #include #include #include @@ -130,23 +124,11 @@ namespace osmium { namespace io { - class File; - namespace detail { - /** - * Once the header is fully parsed this exception will be thrown if - * the caller is not interested in anything else except the header. - * It will break off the parsing at this point. - * - * This exception is never seen by user code, it is caught internally. - */ - class ParserIsDone : std::exception { - }; + class XMLParser : public Parser { - class XMLParser { - - static constexpr int buffer_size = 10 * 1000 * 1000; + static constexpr int buffer_size = 2 * 1000 * 1000; enum class context { root, @@ -155,6 +137,9 @@ namespace osmium { way, relation, changeset, + discussion, + comment, + comment_text, ignored_node, ignored_way, ignored_relation, @@ -175,29 +160,22 @@ namespace osmium { osmium::memory::Buffer m_buffer; - std::unique_ptr m_node_builder; - std::unique_ptr m_way_builder; - std::unique_ptr m_relation_builder; - std::unique_ptr m_changeset_builder; + std::unique_ptr m_node_builder; + std::unique_ptr m_way_builder; + std::unique_ptr m_relation_builder; + std::unique_ptr m_changeset_builder; + std::unique_ptr m_changeset_discussion_builder; - std::unique_ptr m_tl_builder; - std::unique_ptr m_wnl_builder; - std::unique_ptr m_rml_builder; + std::unique_ptr m_tl_builder; + std::unique_ptr m_wnl_builder; + std::unique_ptr m_rml_builder; - osmium::thread::Queue& m_input_queue; - osmium::thread::Queue& m_queue; - std::promise& m_header_promise; - - osmium::osm_entity_bits::type m_read_types; - - std::atomic& m_done; - - bool m_header_is_done; + std::string m_comment_text; /** * A C++ wrapper for the Expat parser that makes sure no memory is leaked. */ - template + template class ExpatXMLParser { XML_Parser m_parser; @@ -210,15 +188,20 @@ namespace osmium { static_cast(data)->end_element(element); } + static void XMLCALL character_data_wrapper(void* data, const XML_Char* text, int len) { + static_cast(data)->characters(text, len); + } + public: - ExpatXMLParser(T* callback_object) : + explicit ExpatXMLParser(T* callback_object) : m_parser(XML_ParserCreate(nullptr)) { if (!m_parser) { throw osmium::io_error("Internal error: Can not create parser"); } XML_SetUserData(m_parser, callback_object); XML_SetElementHandler(m_parser, start_element_wrapper, end_element_wrapper); + XML_SetCharacterDataHandler(m_parser, character_data_wrapper); } ExpatXMLParser(const ExpatXMLParser&) = delete; @@ -227,7 +210,7 @@ namespace osmium { ExpatXMLParser& operator=(const ExpatXMLParser&) = delete; ExpatXMLParser& operator=(ExpatXMLParser&&) = delete; - ~ExpatXMLParser() { + ~ExpatXMLParser() noexcept { XML_ParserFree(m_parser); } @@ -239,126 +222,14 @@ namespace osmium { }; // class ExpatXMLParser - /** - * A helper class that makes sure a promise is kept. It stores - * a reference to some piece of data and to a promise and, on - * destruction, sets the value of the promise from the data. - */ - template - class PromiseKeeper { - - T& m_data; - std::promise& m_promise; - bool m_done; - - public: - - PromiseKeeper(T& data, std::promise& promise) : - m_data(data), - m_promise(promise), - m_done(false) { + template + static void check_attributes(const XML_Char** attrs, T check) { + while (*attrs) { + check(attrs[0], attrs[1]); + attrs += 2; } - - void fullfill_promise() { - if (!m_done) { - m_promise.set_value(m_data); - m_done = true; - } - } - - ~PromiseKeeper() { - fullfill_promise(); - } - - }; // class PromiseKeeper - - public: - - explicit XMLParser(osmium::thread::Queue& input_queue, osmium::thread::Queue& queue, std::promise& header_promise, osmium::osm_entity_bits::type read_types, std::atomic& done) : - m_context(context::root), - m_last_context(context::root), - m_in_delete_section(false), - m_header(), - m_buffer(buffer_size), - m_node_builder(), - m_way_builder(), - m_relation_builder(), - m_changeset_builder(), - m_tl_builder(), - m_wnl_builder(), - m_rml_builder(), - m_input_queue(input_queue), - m_queue(queue), - m_header_promise(header_promise), - m_read_types(read_types), - m_done(done), - m_header_is_done(false) { } - /** - * The copy constructor is needed for storing XMLParser in a std::function. - * The copy will look the same as if it has been initialized with the - * same parameters as the original. Any state changes in the original will - * not be reflected in the copy. - */ - XMLParser(const XMLParser& other) : - m_context(context::root), - m_last_context(context::root), - m_in_delete_section(false), - m_header(), - m_buffer(buffer_size), - m_node_builder(), - m_way_builder(), - m_relation_builder(), - m_changeset_builder(), - m_tl_builder(), - m_wnl_builder(), - m_rml_builder(), - m_input_queue(other.m_input_queue), - m_queue(other.m_queue), - m_header_promise(other.m_header_promise), - m_read_types(other.m_read_types), - m_done(other.m_done), - m_header_is_done(other.m_header_is_done) { - } - - XMLParser(XMLParser&&) = default; - - XMLParser& operator=(const XMLParser&) = delete; - - XMLParser& operator=(XMLParser&&) = default; - - ~XMLParser() = default; - - bool operator()() { - ExpatXMLParser parser(this); - PromiseKeeper promise_keeper(m_header, m_header_promise); - bool last; - do { - std::string data; - m_input_queue.wait_and_pop(data); - last = data.empty(); - try { - parser(data, last); - if (m_header_is_done) { - promise_keeper.fullfill_promise(); - } - } catch (ParserIsDone&) { - return true; - } catch (...) { - m_queue.push(osmium::memory::Buffer()); // empty buffer to signify eof - throw; - } - } while (!last && !m_done); - if (m_buffer.committed() > 0) { - m_queue.push(std::move(m_buffer)); - } - m_queue.push(osmium::memory::Buffer()); // empty buffer to signify eof - return true; - } - - private: - const char* init_object(osmium::OSMObject& object, const XML_Char** attrs) { const char* user = ""; @@ -367,17 +238,18 @@ namespace osmium { } osmium::Location location; - for (int count = 0; attrs[count]; count += 2) { - if (!strcmp(attrs[count], "lon")) { - location.set_lon(std::atof(attrs[count+1])); // XXX doesn't detect garbage after the number - } else if (!strcmp(attrs[count], "lat")) { - location.set_lat(std::atof(attrs[count+1])); // XXX doesn't detect garbage after the number - } else if (!strcmp(attrs[count], "user")) { - user = attrs[count+1]; + + check_attributes(attrs, [&location, &user, &object](const XML_Char* name, const XML_Char* value) { + if (!strcmp(name, "lon")) { + location.set_lon(std::atof(value)); // XXX doesn't detect garbage after the number + } else if (!strcmp(name, "lat")) { + location.set_lat(std::atof(value)); // XXX doesn't detect garbage after the number + } else if (!strcmp(name, "user")) { + user = value; } else { - object.set_attribute(attrs[count], attrs[count+1]); + object.set_attribute(name, value); } - } + }); if (location && object.type() == osmium::item_type::node) { static_cast(object).set_location(location); @@ -392,21 +264,21 @@ namespace osmium { osmium::Location min; osmium::Location max; - for (int count = 0; attrs[count]; count += 2) { - if (!strcmp(attrs[count], "min_lon")) { - min.set_lon(atof(attrs[count+1])); - } else if (!strcmp(attrs[count], "min_lat")) { - min.set_lat(atof(attrs[count+1])); - } else if (!strcmp(attrs[count], "max_lon")) { - max.set_lon(atof(attrs[count+1])); - } else if (!strcmp(attrs[count], "max_lat")) { - max.set_lat(atof(attrs[count+1])); - } else if (!strcmp(attrs[count], "user")) { - user = attrs[count+1]; + check_attributes(attrs, [&min, &max, &user, &new_changeset](const XML_Char* name, const XML_Char* value) { + if (!strcmp(name, "min_lon")) { + min.set_lon(atof(value)); + } else if (!strcmp(name, "min_lat")) { + min.set_lat(atof(value)); + } else if (!strcmp(name, "max_lon")) { + max.set_lon(atof(value)); + } else if (!strcmp(name, "max_lat")) { + max.set_lat(atof(value)); + } else if (!strcmp(name, "user")) { + user = value; } else { - new_changeset.set_attribute(attrs[count], attrs[count+1]); + new_changeset.set_attribute(name, value); } - } + }); new_changeset.bounds().extend(min); new_changeset.bounds().extend(max); @@ -414,32 +286,24 @@ namespace osmium { builder->add_user(user); } - void check_tag(osmium::builder::Builder* builder, const XML_Char* element, const XML_Char** attrs) { - if (!strcmp(element, "tag")) { - m_wnl_builder.reset(); - m_rml_builder.reset(); - - const char* key = ""; - const char* value = ""; - for (int count = 0; attrs[count]; count += 2) { - if (attrs[count][0] == 'k' && attrs[count][1] == 0) { - key = attrs[count+1]; - } else if (attrs[count][0] == 'v' && attrs[count][1] == 0) { - value = attrs[count+1]; - } + void get_tag(osmium::builder::Builder* builder, const XML_Char** attrs) { + const char* k = ""; + const char* v = ""; + check_attributes(attrs, [&k, &v](const XML_Char* name, const XML_Char* value) { + if (name[0] == 'k' && name[1] == 0) { + k = value; + } else if (name[0] == 'v' && name[1] == 0) { + v = value; } - if (!m_tl_builder) { - m_tl_builder = std::unique_ptr(new osmium::builder::TagListBuilder(m_buffer, builder)); - } - m_tl_builder->add_tag(key, value); + }); + if (!m_tl_builder) { + m_tl_builder = std::unique_ptr(new osmium::builder::TagListBuilder(m_buffer, builder)); } + m_tl_builder->add_tag(k, v); } - void header_is_done() { - m_header_is_done = true; - if (m_read_types == osmium::osm_entity_bits::nothing) { - throw ParserIsDone(); - } + void mark_header_as_done() { + set_header_value(m_header); } void start_element(const XML_Char* element, const XML_Char** attrs) { @@ -449,16 +313,16 @@ namespace osmium { if (!strcmp(element, "osmChange")) { m_header.set_has_multiple_object_versions(true); } - for (int count = 0; attrs[count]; count += 2) { - if (!strcmp(attrs[count], "version")) { - m_header.set("version", attrs[count+1]); - if (strcmp(attrs[count+1], "0.6")) { - throw osmium::format_version_error(attrs[count+1]); + check_attributes(attrs, [this](const XML_Char* name, const XML_Char* value) { + if (!strcmp(name, "version")) { + m_header.set("version", value); + if (strcmp(value, "0.6")) { + throw osmium::format_version_error(value); } - } else if (!strcmp(attrs[count], "generator")) { - m_header.set("generator", attrs[count+1]); + } else if (!strcmp(name, "generator")) { + m_header.set("generator", value); } - } + }); if (m_header.get("version") == "") { throw osmium::format_version_error(); } @@ -470,8 +334,8 @@ namespace osmium { case context::top: assert(!m_tl_builder); if (!strcmp(element, "node")) { - header_is_done(); - if (m_read_types & osmium::osm_entity_bits::node) { + mark_header_as_done(); + if (read_types() & osmium::osm_entity_bits::node) { m_node_builder = std::unique_ptr(new osmium::builder::NodeBuilder(m_buffer)); m_node_builder->add_user(init_object(m_node_builder->object(), attrs)); m_context = context::node; @@ -479,8 +343,8 @@ namespace osmium { m_context = context::ignored_node; } } else if (!strcmp(element, "way")) { - header_is_done(); - if (m_read_types & osmium::osm_entity_bits::way) { + mark_header_as_done(); + if (read_types() & osmium::osm_entity_bits::way) { m_way_builder = std::unique_ptr(new osmium::builder::WayBuilder(m_buffer)); m_way_builder->add_user(init_object(m_way_builder->object(), attrs)); m_context = context::way; @@ -488,8 +352,8 @@ namespace osmium { m_context = context::ignored_way; } } else if (!strcmp(element, "relation")) { - header_is_done(); - if (m_read_types & osmium::osm_entity_bits::relation) { + mark_header_as_done(); + if (read_types() & osmium::osm_entity_bits::relation) { m_relation_builder = std::unique_ptr(new osmium::builder::RelationBuilder(m_buffer)); m_relation_builder->add_user(init_object(m_relation_builder->object(), attrs)); m_context = context::relation; @@ -497,8 +361,8 @@ namespace osmium { m_context = context::ignored_relation; } } else if (!strcmp(element, "changeset")) { - header_is_done(); - if (m_read_types & osmium::osm_entity_bits::changeset) { + mark_header_as_done(); + if (read_types() & osmium::osm_entity_bits::changeset) { m_changeset_builder = std::unique_ptr(new osmium::builder::ChangesetBuilder(m_buffer)); init_changeset(m_changeset_builder.get(), attrs); m_context = context::changeset; @@ -508,17 +372,17 @@ namespace osmium { } else if (!strcmp(element, "bounds")) { osmium::Location min; osmium::Location max; - for (int count = 0; attrs[count]; count += 2) { - if (!strcmp(attrs[count], "minlon")) { - min.set_lon(atof(attrs[count+1])); - } else if (!strcmp(attrs[count], "minlat")) { - min.set_lat(atof(attrs[count+1])); - } else if (!strcmp(attrs[count], "maxlon")) { - max.set_lon(atof(attrs[count+1])); - } else if (!strcmp(attrs[count], "maxlat")) { - max.set_lat(atof(attrs[count+1])); + check_attributes(attrs, [&min, &max](const XML_Char* name, const XML_Char* value) { + if (!strcmp(name, "minlon")) { + min.set_lon(atof(value)); + } else if (!strcmp(name, "minlat")) { + min.set_lat(atof(value)); + } else if (!strcmp(name, "maxlon")) { + max.set_lon(atof(value)); + } else if (!strcmp(name, "maxlat")) { + max.set_lat(atof(value)); } - } + }); osmium::Box box; box.extend(min).extend(max); m_header.add_box(box); @@ -529,7 +393,9 @@ namespace osmium { case context::node: m_last_context = context::node; m_context = context::in_object; - check_tag(m_node_builder.get(), element, attrs); + if (!strcmp(element, "tag")) { + get_tag(m_node_builder.get(), attrs); + } break; case context::way: m_last_context = context::way; @@ -541,13 +407,14 @@ namespace osmium { m_wnl_builder = std::unique_ptr(new osmium::builder::WayNodeListBuilder(m_buffer, m_way_builder.get())); } - for (int count = 0; attrs[count]; count += 2) { - if (!strcmp(attrs[count], "ref")) { - m_wnl_builder->add_node_ref(osmium::string_to_object_id(attrs[count+1])); + check_attributes(attrs, [this](const XML_Char* name, const XML_Char* value) { + if (!strcmp(name, "ref")) { + m_wnl_builder->add_node_ref(osmium::string_to_object_id(value)); } - } - } else { - check_tag(m_way_builder.get(), element, attrs); + }); + } else if (!strcmp(element, "tag")) { + m_wnl_builder.reset(); + get_tag(m_way_builder.get(), attrs); } break; case context::relation: @@ -560,28 +427,68 @@ namespace osmium { m_rml_builder = std::unique_ptr(new osmium::builder::RelationMemberListBuilder(m_buffer, m_relation_builder.get())); } - char type = 'x'; - object_id_type ref = 0; + item_type type = item_type::undefined; + object_id_type ref = 0; const char* role = ""; - for (int count = 0; attrs[count]; count += 2) { - if (!strcmp(attrs[count], "type")) { - type = static_cast(attrs[count+1][0]); - } else if (!strcmp(attrs[count], "ref")) { - ref = osmium::string_to_object_id(attrs[count+1]); - } else if (!strcmp(attrs[count], "role")) { - role = static_cast(attrs[count+1]); + check_attributes(attrs, [&type, &ref, &role](const XML_Char* name, const XML_Char* value) { + if (!strcmp(name, "type")) { + type = char_to_item_type(value[0]); + } else if (!strcmp(name, "ref")) { + ref = osmium::string_to_object_id(value); + } else if (!strcmp(name, "role")) { + role = static_cast(value); } + }); + if (type != item_type::node && type != item_type::way && type != item_type::relation) { + throw osmium::xml_error("Unknown type on relation member"); } - // XXX assert type, ref, role are set - m_rml_builder->add_member(char_to_item_type(type), ref, role); - } else { - check_tag(m_relation_builder.get(), element, attrs); + if (ref == 0) { + throw osmium::xml_error("Missing ref on relation member"); + } + m_rml_builder->add_member(type, ref, role); + } else if (!strcmp(element, "tag")) { + m_rml_builder.reset(); + get_tag(m_relation_builder.get(), attrs); } break; case context::changeset: m_last_context = context::changeset; - m_context = context::in_object; - check_tag(m_changeset_builder.get(), element, attrs); + if (!strcmp(element, "discussion")) { + m_context = context::discussion; + m_tl_builder.reset(); + if (!m_changeset_discussion_builder) { + m_changeset_discussion_builder = std::unique_ptr(new osmium::builder::ChangesetDiscussionBuilder(m_buffer, m_changeset_builder.get())); + } + } else if (!strcmp(element, "tag")) { + m_context = context::in_object; + m_changeset_discussion_builder.reset(); + get_tag(m_changeset_builder.get(), attrs); + } + break; + case context::discussion: + if (!strcmp(element, "comment")) { + m_context = context::comment; + osmium::Timestamp date; + osmium::user_id_type uid = 0; + const char* user = ""; + check_attributes(attrs, [&date, &uid, &user](const XML_Char* name, const XML_Char* value) { + if (!strcmp(name, "date")) { + date = osmium::Timestamp(value); + } else if (!strcmp(name, "uid")) { + uid = osmium::string_to_user_id(value); + } else if (!strcmp(name, "user")) { + user = static_cast(value); + } + }); + m_changeset_discussion_builder->add_comment(date, uid, user); + } + break; + case context::comment: + if (!strcmp(element, "text")) { + m_context = context::comment_text; + } + break; + case context::comment_text: break; case context::ignored_node: break; @@ -604,7 +511,7 @@ namespace osmium { break; case context::top: if (!strcmp(element, "osm") || !strcmp(element, "osmChange")) { - header_is_done(); + mark_header_as_done(); m_context = context::root; } else if (!strcmp(element, "delete")) { m_in_delete_section = false; @@ -639,11 +546,25 @@ namespace osmium { case context::changeset: assert(!strcmp(element, "changeset")); m_tl_builder.reset(); + m_changeset_discussion_builder.reset(); m_changeset_builder.reset(); m_buffer.commit(); m_context = context::top; flush_buffer(); break; + case context::discussion: + assert(!strcmp(element, "discussion")); + m_context = context::changeset; + break; + case context::comment: + assert(!strcmp(element, "comment")); + m_context = context::discussion; + break; + case context::comment_text: + assert(!strcmp(element, "text")); + m_context = context::comment; + m_changeset_discussion_builder->add_comment_text(m_comment_text); + break; case context::in_object: m_context = m_last_context; break; @@ -670,85 +591,84 @@ namespace osmium { } } + void characters(const XML_Char* text, int len) { + if (m_context == context::comment_text) { + m_comment_text.append(text, len); + } else { + m_comment_text.resize(0); + } + } + void flush_buffer() { - if (m_buffer.capacity() - m_buffer.committed() < 1000 * 1000) { - m_queue.push(std::move(m_buffer)); + if (m_buffer.committed() > buffer_size / 10 * 9) { + send_to_output_queue(std::move(m_buffer)); osmium::memory::Buffer buffer(buffer_size); - std::swap(m_buffer, buffer); + using std::swap; + swap(m_buffer, buffer); + } + } + + public: + + XMLParser(future_string_queue_type& input_queue, + future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_types) : + Parser(input_queue, output_queue, header_promise, read_types), + m_context(context::root), + m_last_context(context::root), + m_in_delete_section(false), + m_header(), + m_buffer(buffer_size), + m_node_builder(), + m_way_builder(), + m_relation_builder(), + m_changeset_builder(), + m_changeset_discussion_builder(), + m_tl_builder(), + m_wnl_builder(), + m_rml_builder() { + } + + ~XMLParser() noexcept final = default; + + void run() final { + osmium::thread::set_thread_name("_osmium_xml_in"); + + ExpatXMLParser parser(this); + + while (!input_done()) { + std::string data = get_input(); + parser(data, input_done()); + if (read_types() == osmium::osm_entity_bits::nothing && header_is_done()) { + break; + } + } + + mark_header_as_done(); + + if (m_buffer.committed() > 0) { + send_to_output_queue(std::move(m_buffer)); } } }; // class XMLParser - class XMLInputFormat : public osmium::io::detail::InputFormat { + // we want the register_parser() function to run, setting + // the variable is only a side-effect, it will never be used + const bool registered_xml_parser = ParserFactory::instance().register_parser( + file_format::xml, + [](future_string_queue_type& input_queue, + future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_which_entities) { + return std::unique_ptr(new XMLParser(input_queue, output_queue, header_promise, read_which_entities)); + }); - static constexpr size_t max_queue_size = 100; - - osmium::thread::Queue m_queue; - std::atomic m_done; - std::promise m_header_promise; - std::future m_parser_future; - - public: - - /** - * Instantiate XML Parser - * - * @param file osmium::io::File instance describing file to be read from. - * @param read_which_entities Which types of OSM entities (nodes, ways, relations, changesets) should be parsed? - * @param input_queue String queue where data is read from. - */ - explicit XMLInputFormat(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue& input_queue) : - osmium::io::detail::InputFormat(file, read_which_entities), - m_queue(max_queue_size, "xml_parser_results"), - m_done(false), - m_header_promise(), - m_parser_future(std::async(std::launch::async, XMLParser(input_queue, m_queue, m_header_promise, read_which_entities, m_done))) { - } - - ~XMLInputFormat() { - try { - close(); - } catch (...) { - // ignore any exceptions at this point because destructor should not throw - } - } - - virtual osmium::io::Header header() override final { - osmium::thread::check_for_exception(m_parser_future); - return m_header_promise.get_future().get(); - } - - osmium::memory::Buffer read() override { - osmium::memory::Buffer buffer; - if (!m_done || !m_queue.empty()) { - m_queue.wait_and_pop(buffer); - } - - osmium::thread::check_for_exception(m_parser_future); - return buffer; - } - - void close() override { - m_done = true; - osmium::thread::wait_until_done(m_parser_future); - } - - }; // class XMLInputFormat - - namespace { - -// we want the register_input_format() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" - const bool registered_xml_input = osmium::io::detail::InputFormatFactory::instance().register_input_format(osmium::io::file_format::xml, - [](const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue& input_queue) { - return new osmium::io::detail::XMLInputFormat(file, read_which_entities, input_queue); - }); -#pragma GCC diagnostic pop - - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_xml_parser() noexcept { + return registered_xml_parser; + } } // namespace detail diff --git a/include/osmium/io/detail/xml_output_format.hpp b/include/osmium/io/detail/xml_output_format.hpp index 2a381d5ab..3d7878c8e 100644 --- a/include/osmium/io/detail/xml_output_format.hpp +++ b/include/osmium/io/detail/xml_output_format.hpp @@ -33,20 +33,17 @@ DEALINGS IN THE SOFTWARE. */ -#include -#include +#include #include #include #include #include #include #include -#include #include #include #include -#include #include #include #include @@ -75,45 +72,23 @@ namespace osmium { struct XMLWriteError {}; - namespace { + struct xml_output_options { - void xml_string(std::string& out, const char* in) { - for (; *in != '\0'; ++in) { - switch(*in) { - case '&': out += "&"; break; - case '\"': out += """; break; - case '\'': out += "'"; break; - case '<': out += "<"; break; - case '>': out += ">"; break; - case '\n': out += " "; break; - case '\r': out += " "; break; - case '\t': out += " "; break; - default: out += *in; break; - } - } - } + /// Should metadata of objects be added? + bool add_metadata; - const size_t tmp_buffer_size = 100; + /// Should the visible flag be added to all OSM objects? + bool add_visible_flag; - template - void oprintf(std::string& out, const char* format, T value) { - char buffer[tmp_buffer_size+1]; - size_t max_size = sizeof(buffer)/sizeof(char); -#ifndef NDEBUG - int len = -#endif -#ifndef _MSC_VER - snprintf(buffer, max_size, format, value); -#else - _snprintf(buffer, max_size, format, value); -#endif - assert(len > 0 && static_cast(len) < max_size); - out += buffer; - } + /** + * Should , , "operations" be added? + * (This is used for .osc files.) + */ + bool use_change_ops; - } // anonymous namespace + }; - class XMLOutputBlock : public osmium::handler::Handler { + class XMLOutputBlock : public OutputBlock { // operation (create, modify, delete) for osc files enum class operation { @@ -123,15 +98,9 @@ namespace osmium { op_delete = 3 }; // enum class operation - std::shared_ptr m_input_buffer; - - std::shared_ptr m_out; - operation m_last_op {operation::op_none}; - const bool m_add_metadata; - const bool m_write_visible_flag; - const bool m_write_change_ops; + xml_output_options m_options; void write_spaces(int num) { for (; num != 0; --num) { @@ -139,20 +108,20 @@ namespace osmium { } } + int prefix_spaces() { + return m_options.use_change_ops ? 4 : 2; + } + void write_prefix() { - if (m_write_change_ops) { - write_spaces(4); - } else { - write_spaces(2); - } + write_spaces(prefix_spaces()); } void write_meta(const osmium::OSMObject& object) { - oprintf(*m_out, " id=\"%" PRId64 "\"", object.id()); + output_formatted(" id=\"%" PRId64 "\"", object.id()); - if (m_add_metadata) { + if (m_options.add_metadata) { if (object.version()) { - oprintf(*m_out, " version=\"%d\"", object.version()); + output_formatted(" version=\"%d\"", object.version()); } if (object.timestamp()) { @@ -162,16 +131,16 @@ namespace osmium { } if (!object.user_is_anonymous()) { - oprintf(*m_out, " uid=\"%d\" user=\"", object.uid()); - xml_string(*m_out, object.user()); + output_formatted(" uid=\"%d\" user=\"", object.uid()); + append_xml_encoded_string(*m_out, object.user()); *m_out += "\""; } if (object.changeset()) { - oprintf(*m_out, " changeset=\"%d\"", object.changeset()); + output_formatted(" changeset=\"%d\"", object.changeset()); } - if (m_write_visible_flag) { + if (m_options.add_visible_flag) { if (object.visible()) { *m_out += " visible=\"true\""; } else { @@ -181,17 +150,31 @@ namespace osmium { } } - void write_tags(const osmium::TagList& tags) { + void write_tags(const osmium::TagList& tags, int spaces) { for (const auto& tag : tags) { - write_prefix(); + write_spaces(spaces); *m_out += " \n"; } } + void write_discussion(const osmium::ChangesetDiscussion& comments) { + for (const auto& comment : comments) { + output_formatted(" \n"; + *m_out += " "; + append_xml_encoded_string(*m_out, comment.text()); + *m_out += "\n \n"; + } + *m_out += " \n"; + } + void open_close_op_tag(const operation op = operation::op_none) { if (op == m_last_op) { return; @@ -230,12 +213,9 @@ namespace osmium { public: - explicit XMLOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata, bool write_visible_flag, bool write_change_ops) : - m_input_buffer(std::make_shared(std::move(buffer))), - m_out(std::make_shared()), - m_add_metadata(add_metadata), - m_write_visible_flag(write_visible_flag && !write_change_ops), - m_write_change_ops(write_change_ops) { + XMLOutputBlock(osmium::memory::Buffer&& buffer, const xml_output_options& options) : + OutputBlock(std::move(buffer)), + m_options(options) { } XMLOutputBlock(const XMLOutputBlock&) = default; @@ -244,22 +224,24 @@ namespace osmium { XMLOutputBlock(XMLOutputBlock&&) = default; XMLOutputBlock& operator=(XMLOutputBlock&&) = default; - ~XMLOutputBlock() = default; + ~XMLOutputBlock() noexcept = default; std::string operator()() { osmium::apply(m_input_buffer->cbegin(), m_input_buffer->cend(), *this); - if (m_write_change_ops) { + if (m_options.use_change_ops) { open_close_op_tag(); } std::string out; - std::swap(out, *m_out); + using std::swap; + swap(out, *m_out); + return out; } void node(const osmium::Node& node) { - if (m_write_change_ops) { + if (m_options.use_change_ops) { open_close_op_tag(node.visible() ? (node.version() == 1 ? operation::op_create : operation::op_modify) : operation::op_delete); } @@ -283,14 +265,14 @@ namespace osmium { *m_out += ">\n"; - write_tags(node.tags()); + write_tags(node.tags(), prefix_spaces()); write_prefix(); *m_out += "\n"; } void way(const osmium::Way& way) { - if (m_write_change_ops) { + if (m_options.use_change_ops) { open_close_op_tag(way.visible() ? (way.version() == 1 ? operation::op_create : operation::op_modify) : operation::op_delete); } @@ -307,17 +289,17 @@ namespace osmium { for (const auto& node_ref : way.nodes()) { write_prefix(); - oprintf(*m_out, " \n", node_ref.ref()); + output_formatted(" \n", node_ref.ref()); } - write_tags(way.tags()); + write_tags(way.tags(), prefix_spaces()); write_prefix(); *m_out += "\n"; } void relation(const osmium::Relation& relation) { - if (m_write_change_ops) { + if (m_options.use_change_ops) { open_close_op_tag(relation.visible() ? (relation.version() == 1 ? operation::op_create : operation::op_modify) : operation::op_delete); } @@ -336,22 +318,21 @@ namespace osmium { write_prefix(); *m_out += " \n"; } - write_tags(relation.tags()); + write_tags(relation.tags(), prefix_spaces()); write_prefix(); *m_out += "\n"; } void changeset(const osmium::Changeset& changeset) { - write_prefix(); - *m_out += " 0) { + *m_out += " \n"; + write_discussion(changeset.discussion()); + } + + *m_out += " \n"; } }; // class XMLOutputBlock class XMLOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler { - bool m_add_metadata; - bool m_write_visible_flag; + xml_output_options m_options; public: - XMLOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : - OutputFormat(file, output_queue), - m_add_metadata(file.get("add_metadata") != "false"), - m_write_visible_flag(file.has_multiple_object_versions() || m_file.is_true("force_visible_flag")) { + XMLOutputFormat(const osmium::io::File& file, future_string_queue_type& output_queue) : + OutputFormat(output_queue), + m_options() { + m_options.add_metadata = file.is_not_false("add_metadata"); + m_options.use_change_ops = file.is_true("xml_change_format"); + m_options.add_visible_flag = (file.has_multiple_object_versions() || file.is_true("force_visible_flag")) && !m_options.use_change_ops; } XMLOutputFormat(const XMLOutputFormat&) = delete; XMLOutputFormat& operator=(const XMLOutputFormat&) = delete; - ~XMLOutputFormat() override final { - } + ~XMLOutputFormat() noexcept final = default; - void write_buffer(osmium::memory::Buffer&& buffer) override final { - m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_add_metadata, m_write_visible_flag, m_file.is_true("xml_change_format")})); - } - - void write_header(const osmium::io::Header& header) override final { + void write_header(const osmium::io::Header& header) final { std::string out = "\n"; - if (m_file.is_true("xml_change_format")) { + if (m_options.use_change_ops) { out += "\n"; } else { out += "\n"; } + append_xml_encoded_string(out, header.get("generator").c_str()); + out += "\">\n"; for (const auto& box : header.boxes()) { out += " \n", box.top_right().lat()); + append_printf_formatted_string(out, " minlon=\"%.7f\"", box.bottom_left().lon()); + append_printf_formatted_string(out, " minlat=\"%.7f\"", box.bottom_left().lat()); + append_printf_formatted_string(out, " maxlon=\"%.7f\"", box.top_right().lon()); + append_printf_formatted_string(out, " maxlat=\"%.7f\"/>\n", box.top_right().lat()); } - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(std::move(out)); + send_to_output_queue(std::move(out)); } - void close() override final { - { - std::string out; - if (m_file.is_true("xml_change_format")) { - out += "\n"; - } else { - out += "\n"; - } + void write_buffer(osmium::memory::Buffer&& buffer) final { + m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_options})); + } - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(std::move(out)); + void write_end() final { + std::string out; + + if (m_options.use_change_ops) { + out += "\n"; + } else { + out += "\n"; } - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(std::string()); + send_to_output_queue(std::move(out)); } }; // class XMLOutputFormat - namespace { + // we want the register_output_format() function to run, setting + // the variable is only a side-effect, it will never be used + const bool registered_xml_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::xml, + [](const osmium::io::File& file, future_string_queue_type& output_queue) { + return new osmium::io::detail::XMLOutputFormat(file, output_queue); + }); -// we want the register_output_format() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" - const bool registered_xml_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::xml, - [](const osmium::io::File& file, data_queue_type& output_queue) { - return new osmium::io::detail::XMLOutputFormat(file, output_queue); - }); -#pragma GCC diagnostic pop - - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_xml_output() noexcept { + return registered_xml_output; + } } // namespace detail - } // namespace output + } // namespace io } // namespace osmium diff --git a/include/osmium/io/detail/zlib.hpp b/include/osmium/io/detail/zlib.hpp index fc0baf344..4d86168d2 100644 --- a/include/osmium/io/detail/zlib.hpp +++ b/include/osmium/io/detail/zlib.hpp @@ -39,6 +39,7 @@ DEALINGS IN THE SOFTWARE. #include +#include #include namespace osmium { @@ -69,7 +70,7 @@ namespace osmium { ); if (result != Z_OK) { - throw std::runtime_error(std::string("failed to compress data: ") + zError(result)); + throw io_error(std::string("failed to compress data: ") + zError(result)); } output.resize(output_size); @@ -99,7 +100,7 @@ namespace osmium { ); if (result != Z_OK) { - throw std::runtime_error(std::string("failed to uncompress data: ") + zError(result)); + throw io_error(std::string("failed to uncompress data: ") + zError(result)); } return std::make_pair(output.data(), output.size()); diff --git a/include/osmium/io/error.hpp b/include/osmium/io/error.hpp index 07652bc59..6b5c677c0 100644 --- a/include/osmium/io/error.hpp +++ b/include/osmium/io/error.hpp @@ -43,16 +43,28 @@ namespace osmium { */ struct io_error : public std::runtime_error { - io_error(const std::string& what) : + explicit io_error(const std::string& what) : std::runtime_error(what) { } - io_error(const char* what) : + explicit io_error(const char* what) : std::runtime_error(what) { } }; // struct io_error + struct unsupported_file_format_error : public io_error { + + explicit unsupported_file_format_error(const std::string& what) : + io_error(what) { + } + + explicit unsupported_file_format_error(const char* what) : + io_error(what) { + } + + }; // struct unsupported_file_format_error + } // namespace osmium #endif // OSMIUM_IO_ERROR_HPP diff --git a/include/osmium/io/file.hpp b/include/osmium/io/file.hpp index 3bbfacc6d..87fcc3707 100644 --- a/include/osmium/io/file.hpp +++ b/include/osmium/io/file.hpp @@ -39,6 +39,7 @@ DEALINGS IN THE SOFTWARE. #include #include +#include #include #include #include @@ -255,9 +256,9 @@ namespace osmium { * Check file format etc. for consistency and throw exception if * there is a problem. * - * @throws std::runtime_error + * @throws osmium::io_error */ - void check() const { + const File& check() const { if (m_file_format == file_format::unknown) { std::string msg = "Could not detect file format"; if (!m_format_string.empty()) { @@ -273,8 +274,9 @@ namespace osmium { msg += "'"; } msg += "."; - throw std::runtime_error(msg); + throw io_error(msg); } + return *this; } file_format format() const noexcept { diff --git a/include/osmium/io/gzip_compression.hpp b/include/osmium/io/gzip_compression.hpp index 204bfd521..184787539 100644 --- a/include/osmium/io/gzip_compression.hpp +++ b/include/osmium/io/gzip_compression.hpp @@ -49,7 +49,9 @@ DEALINGS IN THE SOFTWARE. #include #include +#include #include +#include #include #include @@ -59,13 +61,13 @@ namespace osmium { * Exception thrown when there are problems compressing or * decompressing gzip files. */ - struct gzip_error : public std::runtime_error { + struct gzip_error : public io_error { int gzip_error_code; int system_errno; gzip_error(const std::string& what, int error_code) : - std::runtime_error(what), + io_error(what), gzip_error_code(error_code), system_errno(error_code == Z_ERRNO ? errno : 0) { } @@ -93,23 +95,29 @@ namespace osmium { class GzipCompressor : public Compressor { + int m_fd; gzFile m_gzfile; public: - explicit GzipCompressor(int fd) : - Compressor(), + explicit GzipCompressor(int fd, fsync sync) : + Compressor(sync), + m_fd(dup(fd)), m_gzfile(::gzdopen(fd, "w")) { if (!m_gzfile) { detail::throw_gzip_error(m_gzfile, "write initialization failed"); } } - ~GzipCompressor() override final { - close(); + ~GzipCompressor() noexcept final { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } } - void write(const std::string& data) override final { + void write(const std::string& data) final { if (!data.empty()) { int nwrite = ::gzwrite(m_gzfile, data.data(), static_cast_with_assert(data.size())); if (nwrite == 0) { @@ -118,13 +126,17 @@ namespace osmium { } } - void close() override final { + void close() final { if (m_gzfile) { int result = ::gzclose(m_gzfile); m_gzfile = nullptr; if (result != Z_OK) { detail::throw_gzip_error(m_gzfile, "write close failed", result); } + if (do_fsync()) { + osmium::io::detail::reliable_fsync(m_fd); + } + osmium::io::detail::reliable_close(m_fd); } } @@ -144,11 +156,15 @@ namespace osmium { } } - ~GzipDecompressor() override final { - close(); + ~GzipDecompressor() noexcept final { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } } - std::string read() override final { + std::string read() final { std::string buffer(osmium::io::Decompressor::input_buffer_size, '\0'); int nread = ::gzread(m_gzfile, const_cast(buffer.data()), static_cast_with_assert(buffer.size())); if (nread < 0) { @@ -158,7 +174,7 @@ namespace osmium { return buffer; } - void close() override final { + void close() final { if (m_gzfile) { int result = ::gzclose(m_gzfile); m_gzfile = nullptr; @@ -194,11 +210,15 @@ namespace osmium { } } - ~GzipBufferDecompressor() override final { - inflateEnd(&m_zstream); + ~GzipBufferDecompressor() noexcept final { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. + } } - std::string read() override final { + std::string read() final { std::string output; if (m_buffer) { @@ -227,22 +247,28 @@ namespace osmium { return output; } + void close() final { + inflateEnd(&m_zstream); + } + }; // class GzipBufferDecompressor - namespace { + namespace detail { -// we want the register_compression() function to run, setting the variable -// is only a side-effect, it will never be used -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" + // we want the register_compression() function to run, setting + // the variable is only a side-effect, it will never be used const bool registered_gzip_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::gzip, - [](int fd) { return new osmium::io::GzipCompressor(fd); }, + [](int fd, fsync sync) { return new osmium::io::GzipCompressor(fd, sync); }, [](int fd) { return new osmium::io::GzipDecompressor(fd); }, [](const char* buffer, size_t size) { return new osmium::io::GzipBufferDecompressor(buffer, size); } ); -#pragma GCC diagnostic pop - } // anonymous namespace + // dummy function to silence the unused variable warning from above + inline bool get_registered_gzip_compression() noexcept { + return registered_gzip_compression; + } + + } // namespace detail } // namespace io diff --git a/include/osmium/io/input_iterator.hpp b/include/osmium/io/input_iterator.hpp index f6197299d..864776303 100644 --- a/include/osmium/io/input_iterator.hpp +++ b/include/osmium/io/input_iterator.hpp @@ -52,7 +52,7 @@ namespace osmium { * source. It hides all the buffer handling and makes the contents of a * source accessible as a normal STL input iterator. */ - template + template class InputIterator { static_assert(std::is_base_of::value, "TItem must derive from osmium::buffer::Item"); @@ -133,6 +133,44 @@ namespace osmium { }; // class InputIterator + template + class InputIteratorRange { + + InputIterator m_begin; + InputIterator m_end; + + public: + + InputIteratorRange(InputIterator&& begin, + InputIterator&& end) : + m_begin(std::move(begin)), + m_end(std::move(end)) { + } + + InputIterator begin() const noexcept { + return m_begin; + } + + InputIterator end() const noexcept { + return m_end; + } + + const InputIterator cbegin() const noexcept { + return m_begin; + } + + const InputIterator cend() const noexcept { + return m_end; + } + + }; // class InputIteratorRange + + template + InputIteratorRange make_input_iterator_range(TSource& source) { + using it_type = InputIterator; + return InputIteratorRange(it_type{source}, it_type{}); + } + } // namespace io } // namespace osmium diff --git a/include/osmium/io/o5m_input.hpp b/include/osmium/io/o5m_input.hpp new file mode 100644 index 000000000..b63f72853 --- /dev/null +++ b/include/osmium/io/o5m_input.hpp @@ -0,0 +1,45 @@ +#ifndef OSMIUM_IO_O5M_INPUT_HPP +#define OSMIUM_IO_O5M_INPUT_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +/** + * @file + * + * Include this file if you want to read OSM o5m and o5c files. + */ + +#include // IWYU pragma: export +#include // IWYU pragma: export + +#endif // OSMIUM_IO_O5M_INPUT_HPP diff --git a/include/osmium/io/output_iterator.hpp b/include/osmium/io/output_iterator.hpp index 608852fa9..1cf1d1dcb 100644 --- a/include/osmium/io/output_iterator.hpp +++ b/include/osmium/io/output_iterator.hpp @@ -40,6 +40,7 @@ DEALINGS IN THE SOFTWARE. #include #include +#include namespace osmium { @@ -49,30 +50,26 @@ namespace osmium { namespace io { - template + template class OutputIterator : public std::iterator { - struct buffer_wrapper { - - osmium::memory::Buffer buffer; - - buffer_wrapper(size_t buffer_size) : - buffer(buffer_size, osmium::memory::Buffer::auto_grow::no) { - } - - }; // struct buffer_wrapper - - static constexpr size_t default_buffer_size = 10 * 1024 * 1024; - TDest* m_destination; - std::shared_ptr m_buffer_wrapper; - public: - explicit OutputIterator(TDest& destination, const size_t buffer_size = default_buffer_size) : - m_destination(&destination), - m_buffer_wrapper(std::make_shared(buffer_size)) { + explicit OutputIterator(TDest& destination) : + m_destination(&destination) { + } + + /** + * @deprecated + * Use of buffer size argument on OutputIterator + * constructor is deprecated. Call Writer::set_buffer_size() + * instead if you want to change the default. + */ + OSMIUM_DEPRECATED OutputIterator(TDest& destination, const size_t buffer_size) : + m_destination(&destination) { + destination.set_buffer_size(buffer_size); } OutputIterator(const OutputIterator&) = default; @@ -83,19 +80,17 @@ namespace osmium { ~OutputIterator() = default; - void flush() { - osmium::memory::Buffer buffer(m_buffer_wrapper->buffer.capacity(), osmium::memory::Buffer::auto_grow::no); - std::swap(m_buffer_wrapper->buffer, buffer); - (*m_destination)(std::move(buffer)); + /** + * @deprecated + * Calling OutputIterator::flush() is usually not + * needed any more. Call flush() on the Writer instead if needed. + */ + OSMIUM_DEPRECATED void flush() { + m_destination->flush(); } OutputIterator& operator=(const osmium::memory::Item& item) { - try { - m_buffer_wrapper->buffer.push_back(item); - } catch (osmium::buffer_is_full&) { - flush(); - m_buffer_wrapper->buffer.push_back(item); - } + (*m_destination)(item); return *this; } @@ -117,6 +112,23 @@ namespace osmium { }; // class OutputIterator + template + OutputIterator make_output_iterator(TDest& destination) { + return OutputIterator{destination}; + } + + /** + * @deprecated + * Use of buffer size argument on make_output_iterator is deprecated. + * Call Writer::set_buffer_size() instead if you want to change the + * default. + */ + template + OSMIUM_DEPRECATED OutputIterator make_output_iterator(TDest& destination, const size_t buffer_size) { + destination.set_buffer_size(buffer_size); + return OutputIterator{destination}; + } + } // namespace io } // namespace osmium diff --git a/include/osmium/io/overwrite.hpp b/include/osmium/io/overwrite.hpp index e33894bd8..f9fbc71af 100644 --- a/include/osmium/io/overwrite.hpp +++ b/include/osmium/io/overwrite.hpp @@ -33,20 +33,7 @@ DEALINGS IN THE SOFTWARE. */ -namespace osmium { - - namespace io { - - /** - * Allow overwriting of existing file. - */ - enum class overwrite : bool { - no = false, - allow = true - }; - - } // namespace io - -} // namespace osmium +#pragma message("Including overwrite.hpp is deprecated, #include instead.") +#include #endif // OSMIUM_IO_OVERWRITE_HPP diff --git a/include/osmium/io/reader.hpp b/include/osmium/io/reader.hpp index c68a8e180..fd2a6d240 100644 --- a/include/osmium/io/reader.hpp +++ b/include/osmium/io/reader.hpp @@ -33,10 +33,10 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include #include +#include #include #include #include @@ -55,12 +55,13 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include +#include #include #include #include #include #include -#include namespace osmium { @@ -74,17 +75,46 @@ namespace osmium { */ class Reader { + static constexpr size_t max_input_queue_size = 20; // XXX + static constexpr size_t max_osmdata_queue_size = 20; // XXX + osmium::io::File m_file; osmium::osm_entity_bits::type m_read_which_entities; - std::atomic m_input_done; + + enum class status { + okay = 0, // normal reading + error = 1, // some error occurred while reading + closed = 2, // close() called successfully after eof + eof = 3 // eof of file was reached without error + } m_status; + int m_childpid; - osmium::thread::Queue m_input_queue; + detail::future_string_queue_type m_input_queue; std::unique_ptr m_decompressor; - std::future m_read_future; - std::unique_ptr m_input; + osmium::io::detail::ReadThreadManager m_read_thread_manager; + + detail::future_buffer_queue_type m_osmdata_queue; + detail::queue_wrapper m_osmdata_queue_wrapper; + + std::future m_header_future; + osmium::io::Header m_header; + + osmium::thread::thread_handler m_thread; + + // This function will run in a separate thread. + static void parser_thread(const osmium::io::File& file, + detail::future_string_queue_type& input_queue, + detail::future_buffer_queue_type& osmdata_queue, + std::promise&& header_promise, + osmium::osm_entity_bits::type read_which_entities) { + std::promise promise = std::move(header_promise); + auto creator = detail::ParserFactory::instance().get_creator_function(file); + auto parser = creator(input_queue, osmdata_queue, promise, read_which_entities); + parser->parse(); + } #ifndef _WIN32 /** @@ -149,7 +179,7 @@ namespace osmium { #ifndef _WIN32 return execute("curl", filename, childpid); #else - throw std::runtime_error("Reading OSM files from the network currently not supported on Windows."); + throw io_error("Reading OSM files from the network currently not supported on Windows."); #endif } else { return osmium::io::detail::open_for_reading(filename); @@ -168,16 +198,23 @@ namespace osmium { * parsed. */ explicit Reader(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities = osmium::osm_entity_bits::all) : - m_file(file), + m_file(file.check()), m_read_which_entities(read_which_entities), - m_input_done(false), + m_status(status::okay), m_childpid(0), - m_input_queue(20, "raw_input"), // XXX + m_input_queue(max_input_queue_size, "raw_input"), m_decompressor(m_file.buffer() ? osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), m_file.buffer(), m_file.buffer_size()) : osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), open_input_file_or_url(m_file.filename(), &m_childpid))), - m_read_future(std::async(std::launch::async, detail::ReadThread(m_input_queue, m_decompressor.get(), m_input_done))), - m_input(osmium::io::detail::InputFormatFactory::instance().create_input(m_file, m_read_which_entities, m_input_queue)) { + m_read_thread_manager(*m_decompressor, m_input_queue), + m_osmdata_queue(max_osmdata_queue_size, "parser_results"), + m_osmdata_queue_wrapper(m_osmdata_queue), + m_header_future(), + m_header(), + m_thread() { + std::promise header_promise; + m_header_future = header_promise.get_future(); + m_thread = osmium::thread::thread_handler{parser_thread, std::ref(m_file), std::ref(m_input_queue), std::ref(m_osmdata_queue), std::move(header_promise), read_which_entities}; } explicit Reader(const std::string& filename, osmium::osm_entity_bits::type read_types = osmium::osm_entity_bits::all) : @@ -191,27 +228,37 @@ namespace osmium { Reader(const Reader&) = delete; Reader& operator=(const Reader&) = delete; - ~Reader() { + Reader(Reader&&) = default; + Reader& operator=(Reader&&) = default; + + ~Reader() noexcept { try { close(); - } - catch (...) { + } catch (...) { + // Ignore any exceptions because destructor must not throw. } } /** * Close down the Reader. A call to this is optional, because the * destructor of Reader will also call this. But if you don't call - * this function first, the destructor might throw an exception - * which is not good. + * this function first, you might miss an exception, because the + * destructor is not allowed to throw. * - * @throws Some form of std::runtime_error when there is a problem. + * @throws Some form of osmium::io_error when there is a problem. */ void close() { - // Signal to input child process that it should wrap up. - m_input_done = true; + m_status = status::closed; - m_input->close(); + m_read_thread_manager.stop(); + + m_osmdata_queue_wrapper.drain(); + + try { + m_read_thread_manager.close(); + } catch (...) { + // Ignore any exceptions. + } #ifndef _WIN32 if (m_childpid) { @@ -226,15 +273,32 @@ namespace osmium { m_childpid = 0; } #endif - - osmium::thread::wait_until_done(m_read_future); } /** * Get the header data from the file. + * + * @returns Header. + * @throws Some form of osmium::io_error if there is an error. */ - osmium::io::Header header() const { - return m_input->header(); + osmium::io::Header header() { + if (m_status == status::error) { + throw io_error("Can not get header from reader when in status 'error'"); + } + + try { + if (m_header_future.valid()) { + m_header = m_header_future.get(); + if (m_read_which_entities == osmium::osm_entity_bits::nothing) { + m_status = status::eof; + } + } + } catch (...) { + close(); + m_status = status::error; + throw; + } + return m_header; } /** @@ -245,32 +309,36 @@ namespace osmium { * constructed. * * @returns Buffer. - * @throws Some form of std::runtime_error if there is an error. + * @throws Some form of osmium::io_error if there is an error. */ osmium::memory::Buffer read() { - // If an exception happened in the input thread, re-throw - // it in this (the main) thread. - osmium::thread::check_for_exception(m_read_future); + osmium::memory::Buffer buffer; - if (m_read_which_entities == osmium::osm_entity_bits::nothing || m_input_done) { - // If the caller didn't want anything but the header, it will - // always get an empty buffer here. - return osmium::memory::Buffer(); + if (m_status != status::okay || + m_read_which_entities == osmium::osm_entity_bits::nothing) { + throw io_error("Can not read from reader when in status 'closed', 'eof', or 'error'"); } - // m_input->read() can return an invalid buffer to signal EOF, - // or a valid buffer with or without data. A valid buffer - // without data is not an error, it just means we have to - // keep getting the next buffer until there is one with data. - while (true) { - osmium::memory::Buffer buffer = m_input->read(); - if (!buffer) { - m_input_done = true; - return buffer; - } - if (buffer.committed() > 0) { - return buffer; + try { + // m_input_format.read() can return an invalid buffer to signal EOF, + // or a valid buffer with or without data. A valid buffer + // without data is not an error, it just means we have to + // keep getting the next buffer until there is one with data. + while (true) { + buffer = m_osmdata_queue_wrapper.pop(); + if (detail::at_end_of_data(buffer)) { + m_status = status::eof; + m_read_thread_manager.close(); + return buffer; + } + if (buffer.committed() > 0) { + return buffer; + } } + } catch (...) { + close(); + m_status = status::error; + throw; } } @@ -279,7 +347,7 @@ namespace osmium { * data has been read. It is also set by calling close(). */ bool eof() const { - return m_input_done; + return m_status == status::eof || m_status == status::closed; } }; // class Reader @@ -292,7 +360,7 @@ namespace osmium { * unless you are working with small OSM files and/or have lots of * RAM. */ - template + template osmium::memory::Buffer read_file(TArgs&&... args) { osmium::memory::Buffer buffer(1024*1024, osmium::memory::Buffer::auto_grow::yes); diff --git a/include/osmium/io/writer.hpp b/include/osmium/io/writer.hpp index 64afe2066..7cd133d0e 100644 --- a/include/osmium/io/writer.hpp +++ b/include/osmium/io/writer.hpp @@ -33,17 +33,23 @@ DEALINGS IN THE SOFTWARE. */ +#include +#include #include +#include #include +#include #include #include #include +#include #include #include +#include #include #include -#include +#include #include #include @@ -54,21 +60,112 @@ namespace osmium { /** * This is the user-facing interface for writing OSM files. Instantiate * an object of this class with a file name or osmium::io::File object - * and optionally the data for the header and then call operator() on it - * to write Buffers. Call close() to finish up. + * and optionally the data for the header and then call operator() on + * it to write Buffers or Items. + * + * The writer uses multithreading internally to do the actual encoding + * of the data into the intended format, possible compress the data and + * then write it out. But this is intentionally hidden from the user + * of this class who can use it without knowing those details. + * + * If you are done call the close() method to finish up. Only if you + * don't get an exception from the close() method, you can be sure + * the data is written correctly (modulo operating system buffering). + * The destructor of this class will also do the right thing if you + * forget to call close(), but because the destructor can't throw you + * will not get informed about any problems. + * + * The writer is usually used to write complete blocks of data stored + * in osmium::memory::Buffers. But you can also write single + * osmium::memory::Items. In this case the Writer uses an internal + * Buffer. */ class Writer { + static constexpr size_t default_buffer_size = 10 * 1024 * 1024; + osmium::io::File m_file; - osmium::io::detail::data_queue_type m_output_queue; + detail::future_string_queue_type m_output_queue; std::unique_ptr m_output; - std::unique_ptr m_compressor; + osmium::memory::Buffer m_buffer; + + size_t m_buffer_size; std::future m_write_future; + osmium::thread::thread_handler m_thread; + + enum class status { + okay = 0, // normal writing + error = 1, // some error occurred while writing + closed = 2 // close() called successfully + } m_status; + + // This function will run in a separate thread. + static void write_thread(detail::future_string_queue_type& output_queue, + std::unique_ptr&& compressor, + std::promise&& write_promise) { + detail::WriteThread write_thread{output_queue, + std::move(compressor), + std::move(write_promise)}; + write_thread(); + } + + void do_write(osmium::memory::Buffer&& buffer) { + if (buffer && buffer.committed() > 0) { + m_output->write_buffer(std::move(buffer)); + } + } + + void do_flush() { + osmium::thread::check_for_exception(m_write_future); + if (m_buffer && m_buffer.committed() > 0) { + osmium::memory::Buffer buffer{m_buffer_size, + osmium::memory::Buffer::auto_grow::no}; + using std::swap; + swap(m_buffer, buffer); + + m_output->write_buffer(std::move(buffer)); + } + } + + template + void ensure_cleanup(TFunction func, TArgs&&... args) { + if (m_status != status::okay) { + throw io_error("Can not write to writer when in status 'closed' or 'error'"); + } + + try { + func(std::forward(args)...); + } catch (...) { + m_status = status::error; + detail::add_to_queue(m_output_queue, std::current_exception()); + detail::add_end_of_data_to_queue(m_output_queue); + throw; + } + } + + struct options_type { + osmium::io::Header header; + overwrite allow_overwrite = overwrite::no; + fsync sync = fsync::no; + }; + + static void set_option(options_type& options, const osmium::io::Header& header) { + options.header = header; + } + + static void set_option(options_type& options, overwrite value) { + options.allow_overwrite = value; + } + + static void set_option(options_type& options, fsync value) { + options.sync = value; + } + public: /** @@ -76,64 +173,166 @@ namespace osmium { * header to it. * * @param file File (contains name and format info) to open. - * @param header Optional header data. If this is not given sensible - * defaults will be used. See the default constructor - * of osmium::io::Header for details. - * @param allow_overwrite Allow overwriting of existing file? Can be - * osmium::io::overwrite::allow or osmium::io::overwrite::no - * (default). + * @param args All further arguments are optional and can appear + * in any order: * - * @throws std::runtime_error If the file could not be opened. + * * osmium::io::Header: Optional header data. If this is + * not given, a default constructed osmium::io::Header + * object will be used. + * + * * osmium::io::overwrite: Allow overwriting of existing file? + * Can be osmium::io::overwrite::allow or + * osmium::io::overwrite::no (default). + * + * * osmium::io::fsync: Should fsync be called on the file + * before closing it? Can be osmium::io::fsync::yes or + * osmium::io::fsync::no (default). + * + * @throws osmium::io_error If there was an error. * @throws std::system_error If the file could not be opened. */ - explicit Writer(const osmium::io::File& file, const osmium::io::Header& header = osmium::io::Header(), overwrite allow_overwrite = overwrite::no) : - m_file(file), + template + explicit Writer(const osmium::io::File& file, TArgs&&... args) : + m_file(file.check()), m_output_queue(20, "raw_output"), // XXX m_output(osmium::io::detail::OutputFormatFactory::instance().create_output(m_file, m_output_queue)), - m_compressor(osmium::io::CompressionFactory::instance().create_compressor(file.compression(), osmium::io::detail::open_for_writing(m_file.filename(), allow_overwrite))), - m_write_future(std::async(std::launch::async, detail::WriteThread(m_output_queue, m_compressor.get()))) { - assert(!m_file.buffer()); - m_output->write_header(header); + m_buffer(), + m_buffer_size(default_buffer_size), + m_write_future(), + m_thread(), + m_status(status::okay) { + assert(!m_file.buffer()); // XXX can't handle pseudo-files + + options_type options; + (void)std::initializer_list{ + (set_option(options, args), 0)... + }; + + std::unique_ptr compressor = + CompressionFactory::instance().create_compressor(file.compression(), + osmium::io::detail::open_for_writing(m_file.filename(), options.allow_overwrite), + options.sync); + + std::promise write_promise; + m_write_future = write_promise.get_future(); + m_thread = osmium::thread::thread_handler{write_thread, std::ref(m_output_queue), std::move(compressor), std::move(write_promise)}; + + ensure_cleanup([&](){ + m_output->write_header(options.header); + }); } - explicit Writer(const std::string& filename, const osmium::io::Header& header = osmium::io::Header(), overwrite allow_overwrite = overwrite::no) : - Writer(osmium::io::File(filename), header, allow_overwrite) { + template + explicit Writer(const std::string& filename, TArgs&&... args) : + Writer(osmium::io::File(filename), std::forward(args)...) { } - explicit Writer(const char* filename, const osmium::io::Header& header = osmium::io::Header(), overwrite allow_overwrite = overwrite::no) : - Writer(osmium::io::File(filename), header, allow_overwrite) { + template + explicit Writer(const char* filename, TArgs&&... args) : + Writer(osmium::io::File(filename), std::forward(args)...) { } Writer(const Writer&) = delete; Writer& operator=(const Writer&) = delete; - ~Writer() { - close(); - } + Writer(Writer&&) = default; + Writer& operator=(Writer&&) = default; - /** - * Write contents of a buffer to the output file. - * - * @throws Some form of std::runtime_error when there is a problem. - */ - void operator()(osmium::memory::Buffer&& buffer) { - osmium::thread::check_for_exception(m_write_future); - if (buffer.committed() > 0) { - m_output->write_buffer(std::move(buffer)); + ~Writer() noexcept { + try { + close(); + } catch (...) { + // Ignore any exceptions because destructor must not throw. } } /** - * Flush writes to output file and closes it. If you do not - * call this, the destructor of Writer will also do the same - * thing. But because this call might thrown an exception, - * it is better to call close() explicitly. + * Get the currently configured size of the internal buffer. + */ + size_t buffer_size() const noexcept { + return m_buffer_size; + } + + /** + * Set the size of the internal buffer. This will only take effect + * if you have not yet written anything or after the next flush(). + */ + void set_buffer_size(size_t size) noexcept { + m_buffer_size = size; + } + + /** + * Flush the internal buffer if it contains any data. This is + * usually not needed as the buffer gets flushed on close() + * automatically. * - * @throws Some form of std::runtime_error when there is a problem. + * @throws Some form of osmium::io_error when there is a problem. + */ + void flush() { + ensure_cleanup([&](){ + do_flush(); + }); + } + + /** + * Write contents of a buffer to the output file. The buffer is + * moved into this function and will be in an undefined moved-from + * state afterwards. + * + * @param buffer Buffer that is being written out. + * @throws Some form of osmium::io_error when there is a problem. + */ + void operator()(osmium::memory::Buffer&& buffer) { + ensure_cleanup([&](){ + do_flush(); + do_write(std::move(buffer)); + }); + } + + /** + * Add item to the internal buffer for eventual writing to the + * output file. + * + * @param item Item to write (usually an OSM object). + * @throws Some form of osmium::io_error when there is a problem. + */ + void operator()(const osmium::memory::Item& item) { + ensure_cleanup([&](){ + if (!m_buffer) { + m_buffer = osmium::memory::Buffer{m_buffer_size, + osmium::memory::Buffer::auto_grow::no}; + } + try { + m_buffer.push_back(item); + } catch (osmium::buffer_is_full&) { + do_flush(); + m_buffer.push_back(item); + } + }); + } + + /** + * Flushes internal buffer and closes output file. If you do not + * call this, the destructor of Writer will also do the same + * thing. But because this call might throw an exception, which + * the destructor will ignore, it is better to call close() + * explicitly. + * + * @throws Some form of osmium::io_error when there is a problem. */ void close() { - m_output->close(); - osmium::thread::wait_until_done(m_write_future); + if (m_status == status::okay) { + ensure_cleanup([&](){ + do_write(std::move(m_buffer)); + m_output->write_end(); + m_status = status::closed; + detail::add_end_of_data_to_queue(m_output_queue); + }); + } + + if (m_write_future.valid()) { + m_write_future.get(); + } } }; // class Writer diff --git a/include/osmium/io/writer_options.hpp b/include/osmium/io/writer_options.hpp new file mode 100644 index 000000000..ef1955312 --- /dev/null +++ b/include/osmium/io/writer_options.hpp @@ -0,0 +1,60 @@ +#ifndef OSMIUM_IO_WRITER_OPTIONS_HPP +#define OSMIUM_IO_WRITER_OPTIONS_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +namespace osmium { + + namespace io { + + /** + * Allow overwriting of existing file? + */ + enum class overwrite : bool { + no = false, + allow = true + }; + + /** + * Should writer do an fsync before closing the file? + */ + enum class fsync : bool { + no = false, + yes = true + }; + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_WRITER_OPTIONS_HPP diff --git a/include/osmium/memory/buffer.hpp b/include/osmium/memory/buffer.hpp index d800c685c..bc6e9f8b9 100644 --- a/include/osmium/memory/buffer.hpp +++ b/include/osmium/memory/buffer.hpp @@ -46,6 +46,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include namespace osmium { @@ -89,12 +90,17 @@ namespace osmium { * * By default, if a buffer gets full it will throw a buffer_is_full exception. * You can use the set_full_callback() method to set a callback functor - * which will be called instead of throwing an exception. + * which will be called instead of throwing an exception. The full + * callback functionality is deprecated and will be removed in the + * future. See the documentation for set_full_callback() for alternatives. */ class Buffer { public: + // This is needed so we can call std::back_inserter() on a Buffer. + using value_type = Item; + enum class auto_grow : bool { yes = true, no = false @@ -112,12 +118,13 @@ namespace osmium { public: - typedef Item value_type; - /** - * The constructor without any parameters creates a non-initialized + * The constructor without any parameters creates an invalid, * buffer, ie an empty hull of a buffer that has no actual memory - * associated with it. It can be used to signify end-of-input. + * associated with it. It can be used to signify end-of-data. + * + * Most methods of the Buffer class will not work with an invalid + * buffer. */ Buffer() noexcept : m_memory(), @@ -128,12 +135,14 @@ namespace osmium { } /** - * Constructs an externally memory-managed buffer using the given - * memory and size. + * Constructs a valid externally memory-managed buffer using the + * given memory and size. * * @param data A pointer to some already initialized data. * @param size The size of the initialized data. - * @throws std::invalid_argument When the size isn't a multiple of the alignment. + * + * @throws std::invalid_argument if the size isn't a multiple of + * the alignment. */ explicit Buffer(unsigned char* data, size_t size) : m_memory(), @@ -147,13 +156,15 @@ namespace osmium { } /** - * Constructs an externally memory-managed buffer with the given - * capacity that already contains 'committed' bytes of data. + * Constructs a valid externally memory-managed buffer with the + * given capacity that already contains 'committed' bytes of data. * * @param data A pointer to some (possibly initialized) data. * @param capacity The size of the memory for this buffer. * @param committed The size of the initialized data. If this is 0, the buffer startes out empty. - * @throws std::invalid_argument When the capacity or committed isn't a multiple of the alignment. + * + * @throws std::invalid_argument if the capacity or committed isn't + * a multiple of the alignment. */ explicit Buffer(unsigned char* data, size_t capacity, size_t committed) : m_memory(), @@ -170,10 +181,18 @@ namespace osmium { } /** - * Create an internally memory-managed buffer with the given capacity. - * different in that it internally gets dynamic memory of the - * required size. The dynamic memory will be automatically - * freed when the Buffer is destroyed. + * Constructs a valid internally memory-managed buffer with the + * given capacity. + * Will internally get dynamic memory of the required size. + * The dynamic memory will be automatically freed when the Buffer + * is destroyed. + * + * @param capacity The (initial) size of the memory for this buffer. + * @param auto_grow Should this buffer automatically grow when it + * becomes to small? + * + * @throws std::invalid_argument if the capacity isn't a multiple + * of the alignment. */ explicit Buffer(size_t capacity, auto_grow auto_grow = auto_grow::yes) : m_memory(capacity), @@ -199,13 +218,17 @@ namespace osmium { /** * Return a pointer to data inside the buffer. + * + * @pre The buffer must be valid. */ unsigned char* data() const noexcept { + assert(m_data); return m_data; } /** - * Returns the capacity of the buffer, ie how many bytes it can contain. + * Returns the capacity of the buffer, ie how many bytes it can + * contain. Always returns 0 on invalid buffers. */ size_t capacity() const noexcept { return m_capacity; @@ -213,6 +236,7 @@ namespace osmium { /** * Returns the number of bytes already filled in this buffer. + * Always returns 0 on invalid buffers. */ size_t committed() const noexcept { return m_committed; @@ -221,6 +245,7 @@ namespace osmium { /** * Returns the number of bytes currently filled in this buffer that * are not yet committed. + * Always returns 0 on invalid buffers. */ size_t written() const noexcept { return m_written; @@ -229,28 +254,57 @@ namespace osmium { /** * This tests if the current state of the buffer is aligned * properly. Can be used for asserts. + * + * @pre The buffer must be valid. */ bool is_aligned() const noexcept { + assert(m_data); return (m_written % align_bytes == 0) && (m_committed % align_bytes == 0); } /** * Set functor to be called whenever the buffer is full * instead of throwing buffer_is_full. + * + * The behaviour is undefined if you call this on an invalid + * buffer. + * + * @pre The buffer must be valid. + * + * @deprecated + * Callback functionality will be removed in the future. Either + * detect the buffer_is_full exception or use a buffer with + * auto_grow::yes. If you want to avoid growing buffers, check + * that the used size of the buffer (committed()) is small enough + * compared to the capacity (for instance small than 90% of the + * capacity) before adding anything to the Buffer. If the buffer + * is initialized with auto_grow::yes, it will still grow in the + * rare case that a very large object will be added taking more + * than the difference between committed() and capacity(). */ - void set_full_callback(std::function full) { + OSMIUM_DEPRECATED void set_full_callback(std::function full) { + assert(m_data); m_full = full; } /** * Grow capacity of this buffer to the given size. * This works only with internally memory-managed buffers. - * If the given size is not larger than the current capacity, nothing is done. + * If the given size is not larger than the current capacity, + * nothing is done. * Already written but not committed data is discarded. * + * @pre The buffer must be valid. + * * @param size New capacity. + * + * @throws std::logic_error if the buffer doesn't use internal + * memory management. + * @throws std::invalid_argument if the size isn't a multiple + * of the alignment. */ void grow(size_t size) { + assert(m_data); if (m_memory.empty()) { throw std::logic_error("Can't grow Buffer if it doesn't use internal memory management."); } @@ -267,9 +321,15 @@ namespace osmium { /** * Mark currently written bytes in the buffer as committed. * - * @returns Last number of committed bytes before this commit. + * @pre The buffer must be valid and aligned properly (as indicated + * by is_aligned(). + * + * @returns Number of committed bytes before this commit. Can be + * used as an offset into the buffer to get to the + * object being committed by this call. */ size_t commit() { + assert(m_data); assert(is_aligned()); const size_t offset = m_committed; @@ -279,14 +339,19 @@ namespace osmium { /** * Roll back changes in buffer to last committed state. + * + * @pre The buffer must be valid. */ void rollback() { + assert(m_data); m_written = m_committed; } /** * Clear the buffer. * + * No-op on an invalid buffer. + * * @returns Number of bytes in the buffer before it was cleared. */ size_t clear() { @@ -299,11 +364,16 @@ namespace osmium { /** * Get the data in the buffer at the given offset. * + * @pre The buffer must be valid. + * * @tparam T Type we want to the data to be interpreted as. - * @returns Reference of given type pointing to the data in the buffer. + * + * @returns Reference of given type pointing to the data in the + * buffer. */ - template + template T& get(const size_t offset) const { + assert(m_data); return *reinterpret_cast(&m_data[offset]); } @@ -320,23 +390,35 @@ namespace osmium { * * * If you have set a callback with set_full_callback(), it is * called. After the call returns, you must have either grown - * the buffer or cleared it by calling buffer.clear(). + * the buffer or cleared it by calling buffer.clear(). (Usage + * of the full callback is deprecated and this functionality + * will be removed in the future. See the documentation for + * set_full_callback() for alternatives. * * If no callback is defined and this buffer uses internal * memory management, the buffers capacity is grown, so that * the new data will fit. * * Else the buffer_is_full exception is thrown. * + * @pre The buffer must be valid. + * * @param size Number of bytes to reserve. + * * @returns Pointer to reserved space. Note that this pointer is - * only guaranteed to be valid until the next call to - * reserve_space(). - * @throws osmium::buffer_is_full Might be thrown if the buffer is full. + * only guaranteed to be valid until the next call to + * reserve_space(). + * + * @throws osmium::buffer_is_full if the buffer is full there is + * no callback defined and the buffer isn't auto-growing. */ unsigned char* reserve_space(const size_t size) { + assert(m_data); + // try to flush the buffer empty first. + if (m_written + size > m_capacity && m_full) { + m_full(*this); + } + // if there's still not enough space, then try growing the buffer. if (m_written + size > m_capacity) { - if (m_full) { - m_full(*this); - } else if (!m_memory.empty() && (m_auto_grow == auto_grow::yes)) { + if (!m_memory.empty() && (m_auto_grow == auto_grow::yes)) { // double buffer size until there is enough space size_t new_capacity = m_capacity * 2; while (m_written + size > new_capacity) { @@ -359,12 +441,17 @@ namespace osmium { * Note that you have to eventually call commit() to actually * commit this data. * + * @pre The buffer must be valid. + * * @tparam T Class of the item to be copied. + * * @param item Reference to the item to be copied. + * * @returns Reference to newly copied data in the buffer. */ - template + template T& add_item(const T& item) { + assert(m_data); unsigned char* target = reserve_space(item.padded_size()); std::copy_n(reinterpret_cast(&item), item.padded_size(), target); return *reinterpret_cast(target); @@ -373,91 +460,176 @@ namespace osmium { /** * Add committed contents of the given buffer to this buffer. * + * @pre The buffer must be valid. + * * Note that you have to eventually call commit() to actually * commit this data. + * + * @param buffer The source of the copy. Must be valid. */ void add_buffer(const Buffer& buffer) { + assert(m_data && buffer); unsigned char* target = reserve_space(buffer.committed()); - std::copy_n(reinterpret_cast(buffer.data()), buffer.committed(), target); + std::copy_n(buffer.data(), buffer.committed(), target); } /** * Add an item to the buffer. This function is provided so that * you can use std::back_inserter. + * + * @pre The buffer must be valid. + * + * @param item The item to be added. */ void push_back(const osmium::memory::Item& item) { + assert(m_data); add_item(item); commit(); } /** - * These iterators can be used to iterate over all items in - * a buffer. + * An iterator that can be used to iterate over all items of + * type T in a buffer. */ - template + template using t_iterator = osmium::memory::ItemIterator; - template + /** + * A const iterator that can be used to iterate over all items of + * type T in a buffer. + */ + template using t_const_iterator = osmium::memory::ItemIterator; - typedef t_iterator iterator; - typedef t_const_iterator const_iterator; + /** + * An iterator that can be used to iterate over all OSMEntity + * objects in a buffer. + */ + using iterator = t_iterator; - template + /** + * A const iterator that can be used to iterate over all OSMEntity + * objects in a buffer. + */ + using const_iterator = t_const_iterator; + + /** + * Get iterator for iterating over all items of type T in the + * buffer. + * + * @pre The buffer must be valid. + * + * @returns Iterator to first item of type T in the buffer. + */ + template t_iterator begin() { + assert(m_data); return t_iterator(m_data, m_data + m_committed); } + /** + * Get iterator for iterating over all objects of class OSMEntity + * in the buffer. + * + * @pre The buffer must be valid. + * + * @returns Iterator to first OSMEntity in the buffer. + */ iterator begin() { + assert(m_data); return iterator(m_data, m_data + m_committed); } - template + /** + * Get iterator for iterating over all items of type T in the + * buffer. + * + * @pre The buffer must be valid. + * + * @returns Iterator to first item of type T after given offset + * in the buffer. + */ + template t_iterator get_iterator(size_t offset) { + assert(m_data); return t_iterator(m_data + offset, m_data + m_committed); } + /** + * Get iterator for iterating over all objects of class OSMEntity + * in the buffer. + * + * @pre The buffer must be valid. + * + * @returns Iterator to first OSMEntity after given offset in the + * buffer. + */ iterator get_iterator(size_t offset) { + assert(m_data); return iterator(m_data + offset, m_data + m_committed); } - template + /** + * Get iterator for iterating over all items of type T in the + * buffer. + * + * @pre The buffer must be valid. + * + * @returns End iterator. + */ + template t_iterator end() { + assert(m_data); return t_iterator(m_data + m_committed, m_data + m_committed); } + /** + * Get iterator for iterating over all objects of class OSMEntity + * in the buffer. + * + * @pre The buffer must be valid. + * + * @returns End iterator. + */ iterator end() { + assert(m_data); return iterator(m_data + m_committed, m_data + m_committed); } - template + template t_const_iterator cbegin() const { + assert(m_data); return t_const_iterator(m_data, m_data + m_committed); } const_iterator cbegin() const { + assert(m_data); return const_iterator(m_data, m_data + m_committed); } - template + template t_const_iterator get_iterator(size_t offset) const { + assert(m_data); return t_const_iterator(m_data + offset, m_data + m_committed); } const_iterator get_iterator(size_t offset) const { + assert(m_data); return const_iterator(m_data + offset, m_data + m_committed); } - template + template t_const_iterator cend() const { + assert(m_data); return t_const_iterator(m_data + m_committed, m_data + m_committed); } const_iterator cend() const { + assert(m_data); return const_iterator(m_data + m_committed, m_data + m_committed); } - template + template t_const_iterator begin() const { return cbegin(); } @@ -466,7 +638,7 @@ namespace osmium { return cbegin(); } - template + template t_const_iterator end() const { return cend(); } @@ -476,9 +648,9 @@ namespace osmium { } /** - * In a bool context any initialized buffer is true. + * In a bool context any valid buffer is true. */ - explicit operator bool() const { + explicit operator bool() const noexcept { return m_data != nullptr; } @@ -490,6 +662,8 @@ namespace osmium { swap(lhs.m_capacity, rhs.m_capacity); swap(lhs.m_written, rhs.m_written); swap(lhs.m_committed, rhs.m_committed); + swap(lhs.m_auto_grow, rhs.m_auto_grow); + swap(lhs.m_full, rhs.m_full); } /** @@ -497,17 +671,20 @@ namespace osmium { * non-removed items forward in the buffer overwriting removed * items and then correcting the m_written and m_committed numbers. * - * Note that calling this function invalidates all iterators on this - * buffer and all offsets in this buffer. + * Note that calling this function invalidates all iterators on + * this buffer and all offsets in this buffer. * * For every non-removed item that moves its position, the function * 'moving_in_buffer' is called on the given callback object with * the old and new offsets in the buffer where the object used to * be and is now, respectively. This call can be used to update any * indexes. + * + * @pre The buffer must be valid. */ - template + template void purge_removed(TCallbackClass* callback) { + assert(m_data); if (begin() == end()) { return; } @@ -537,7 +714,17 @@ namespace osmium { }; // class Buffer + /** + * Compare two buffers for equality. + * + * Buffers are equal if they are both invalid or if they are both + * valid and have the same data pointer, capacity and committed + * data. + */ inline bool operator==(const Buffer& lhs, const Buffer& rhs) noexcept { + if (!lhs || !rhs) { + return !lhs && !rhs; + } return lhs.data() == rhs.data() && lhs.capacity() == rhs.capacity() && lhs.committed() == rhs.committed(); } diff --git a/include/osmium/memory/collection.hpp b/include/osmium/memory/collection.hpp index 5cf3cc6e1..3878b9a3a 100644 --- a/include/osmium/memory/collection.hpp +++ b/include/osmium/memory/collection.hpp @@ -43,7 +43,7 @@ namespace osmium { namespace memory { - template + template class CollectionIterator : public std::iterator { // This data_type is either 'unsigned char*' or 'const unsigned char*' depending @@ -59,7 +59,7 @@ namespace osmium { m_data(nullptr) { } - CollectionIterator(data_type data) noexcept : + explicit CollectionIterator(data_type data) noexcept : m_data(data) { } @@ -101,7 +101,7 @@ namespace osmium { }; // class CollectionIterator - template + template class Collection : public Item { public: diff --git a/include/osmium/memory/item.hpp b/include/osmium/memory/item.hpp index dc544049a..f95ce8807 100644 --- a/include/osmium/memory/item.hpp +++ b/include/osmium/memory/item.hpp @@ -43,7 +43,7 @@ namespace osmium { namespace builder { class Builder; - } + } // namespace builder namespace memory { @@ -102,10 +102,10 @@ namespace osmium { uint16_t m_removed : 1; uint16_t m_padding : 15; - template + template friend class CollectionIterator; - template + template friend class ItemIterator; friend class osmium::builder::Builder; diff --git a/include/osmium/memory/item_iterator.hpp b/include/osmium/memory/item_iterator.hpp index 3e5b5fa8b..87af56830 100644 --- a/include/osmium/memory/item_iterator.hpp +++ b/include/osmium/memory/item_iterator.hpp @@ -38,29 +38,17 @@ DEALINGS IN THE SOFTWARE. #include #include +#include #include #include namespace osmium { - class Node; - class Way; - class Relation; - class Area; - class Changeset; - class OSMObject; - class OSMEntity; - class TagList; - class WayNodeList; - class RelationMemberList; - class InnerRing; - class OuterRing; - namespace memory { namespace detail { - template + template inline bool type_is_compatible(osmium::item_type) noexcept { return true; } @@ -127,7 +115,7 @@ namespace osmium { } // namespace detail - template + template class ItemIterator : public std::iterator { static_assert(std::is_base_of::value, "TMember must derive from osmium::memory::Item"); @@ -160,7 +148,7 @@ namespace osmium { advance_to_next_item_of_right_type(); } - template + template ItemIterator cast() const { return ItemIterator(m_data, m_end); } @@ -217,7 +205,7 @@ namespace osmium { } explicit operator bool() const { - return m_data != nullptr; + return (m_data != nullptr) && (m_data != m_end); } template diff --git a/include/osmium/object_pointer_collection.hpp b/include/osmium/object_pointer_collection.hpp index 752470305..85566b6a4 100644 --- a/include/osmium/object_pointer_collection.hpp +++ b/include/osmium/object_pointer_collection.hpp @@ -84,7 +84,7 @@ namespace osmium { /** * Sort objects according to the given order functor. */ - template + template void sort(TCompare&& compare) { std::sort(m_objects.begin(), m_objects.end(), std::forward(compare)); } diff --git a/include/osmium/osm/area.hpp b/include/osmium/osm/area.hpp index 3e129d0fb..7fb2a798b 100644 --- a/include/osmium/osm/area.hpp +++ b/include/osmium/osm/area.hpp @@ -48,7 +48,7 @@ namespace osmium { namespace builder { template class ObjectBuilder; - } + } // namespace builder /** * An outer ring of an Area. @@ -167,6 +167,7 @@ namespace osmium { case osmium::item_type::way_node_list: case osmium::item_type::relation_member_list: case osmium::item_type::relation_member_list_with_full_members: + case osmium::item_type::changeset_discussion: assert(false && "Children of Area can only be outer/inner_ring and tag_list."); break; } diff --git a/include/osmium/osm/box.hpp b/include/osmium/osm/box.hpp index 631f91911..155f5e902 100644 --- a/include/osmium/osm/box.hpp +++ b/include/osmium/osm/box.hpp @@ -154,14 +154,14 @@ namespace osmium { * Box is valid, ie. defined and inside usual bounds * (-180<=lon<=180, -90<=lat<=90). */ - OSMIUM_CONSTEXPR bool valid() const noexcept { + constexpr bool valid() const noexcept { return bottom_left().valid() && top_right().valid(); } /** * Access bottom-left location. */ - OSMIUM_CONSTEXPR Location bottom_left() const noexcept { + constexpr Location bottom_left() const noexcept { return m_bottom_left; } @@ -175,7 +175,7 @@ namespace osmium { /** * Access top-right location. */ - OSMIUM_CONSTEXPR Location top_right() const noexcept { + constexpr Location top_right() const noexcept { return m_top_right; } @@ -216,7 +216,7 @@ namespace osmium { * Boxes are equal if both locations are equal. Undefined boxes will * compare equal. */ - inline OSMIUM_CONSTEXPR bool operator==(const Box& lhs, const Box& rhs) noexcept { + inline constexpr bool operator==(const Box& lhs, const Box& rhs) noexcept { return lhs.bottom_left() == rhs.bottom_left() && lhs.top_right() == rhs.top_right(); } diff --git a/include/osmium/osm/changeset.hpp b/include/osmium/osm/changeset.hpp index 07bc0dd95..db3f717da 100644 --- a/include/osmium/osm/changeset.hpp +++ b/include/osmium/osm/changeset.hpp @@ -48,8 +48,102 @@ DEALINGS IN THE SOFTWARE. namespace osmium { namespace builder { - template class ObjectBuilder; - } + class ChangesetDiscussionBuilder; + template class ObjectBuilder; + } // namespace builder + + class Changeset; + + class ChangesetComment : public osmium::memory::detail::ItemHelper { + + friend class osmium::builder::ChangesetDiscussionBuilder; + + osmium::Timestamp m_date; + osmium::user_id_type m_uid {0}; + string_size_type m_user_size; + string_size_type m_text_size; + + ChangesetComment(const ChangesetComment&) = delete; + ChangesetComment(ChangesetComment&&) = delete; + + ChangesetComment& operator=(const ChangesetComment&) = delete; + ChangesetComment& operator=(ChangesetComment&&) = delete; + + unsigned char* endpos() { + return data() + osmium::memory::padded_length(sizeof(ChangesetComment) + m_user_size + m_text_size); + } + + const unsigned char* endpos() const { + return data() + osmium::memory::padded_length(sizeof(ChangesetComment) + m_user_size + m_text_size); + } + + template + friend class osmium::memory::CollectionIterator; + + unsigned char* next() { + return endpos(); + } + + unsigned const char* next() const { + return endpos(); + } + + void set_user_size(string_size_type size) noexcept { + m_user_size = size; + } + + void set_text_size(string_size_type size) noexcept { + m_text_size = size; + } + + public: + + static constexpr item_type collection_type = item_type::changeset_discussion; + + ChangesetComment(osmium::Timestamp date, osmium::user_id_type uid) noexcept : + m_date(date), + m_uid(uid), + m_user_size(0), + m_text_size(0) { + } + + osmium::Timestamp date() const noexcept { + return m_date; + } + + osmium::user_id_type uid() const noexcept { + return m_uid; + } + + const char* user() const noexcept { + return reinterpret_cast(data() + sizeof(ChangesetComment)); + } + + const char* text() const noexcept { + return reinterpret_cast(data() + sizeof(ChangesetComment) + m_user_size); + } + + }; // class ChangesetComment + + class ChangesetDiscussion : public osmium::memory::Collection { + + friend class osmium::builder::ObjectBuilder; + + public: + + typedef size_t size_type; + + ChangesetDiscussion() : + osmium::memory::Collection() { + } + + size_type size() const noexcept { + return static_cast(std::distance(begin(), end())); + } + + }; // class ChangesetDiscussion + + static_assert(sizeof(ChangesetDiscussion) % osmium::memory::align_bytes == 0, "Class osmium::ChangesetDiscussion has wrong size to be aligned properly!"); /** * \brief An OSM Changeset, a group of changes made by a single user over @@ -62,13 +156,16 @@ namespace osmium { friend class osmium::builder::ObjectBuilder; + osmium::Box m_bounds; osmium::Timestamp m_created_at; osmium::Timestamp m_closed_at; - osmium::Box m_bounds; changeset_id_type m_id {0}; num_changes_type m_num_changes {0}; + num_comments_type m_num_comments {0}; user_id_type m_uid {0}; string_size_type m_user_size; + int16_t m_padding1 {0}; + int32_t m_padding2 {0}; Changeset() : OSMEntity(sizeof(Changeset), osmium::item_type::changeset) { @@ -188,7 +285,7 @@ namespace osmium { * @param timestamp Timestamp * @returns Reference to changeset to make calls chainable. */ - Changeset& set_created_at(const osmium::Timestamp timestamp) { + Changeset& set_created_at(const osmium::Timestamp& timestamp) { m_created_at = timestamp; return *this; } @@ -199,7 +296,7 @@ namespace osmium { * @param timestamp Timestamp * @returns Reference to changeset to make calls chainable. */ - Changeset& set_closed_at(const osmium::Timestamp timestamp) { + Changeset& set_closed_at(const osmium::Timestamp& timestamp) { m_closed_at = timestamp; return *this; } @@ -216,10 +313,26 @@ namespace osmium { } /// Set the number of changes in this changeset - Changeset& set_num_changes(const char* num_changes) noexcept { + Changeset& set_num_changes(const char* num_changes) { return set_num_changes(osmium::string_to_num_changes(num_changes)); } + /// Get the number of comments in this changeset + num_comments_type num_comments() const noexcept { + return m_num_comments; + } + + /// Set the number of comments in this changeset + Changeset& set_num_comments(num_comments_type num_comments) noexcept { + m_num_comments = num_comments; + return *this; + } + + /// Set the number of comments in this changeset + Changeset& set_num_comments(const char* num_comments) { + return set_num_comments(osmium::string_to_num_comments(num_comments)); + } + /** * Get the bounding box of this changeset. * @@ -260,6 +373,8 @@ namespace osmium { set_id(value); } else if (!strcmp(attr, "num_changes")) { set_num_changes(value); + } else if (!strcmp(attr, "comments_count")) { + set_num_comments(value); } else if (!strcmp(attr, "created_at")) { set_created_at(osmium::Timestamp(value)); } else if (!strcmp(attr, "closed_at")) { @@ -296,6 +411,14 @@ namespace osmium { return cend(); } + ChangesetDiscussion& discussion() { + return osmium::detail::subitem_of_type(begin(), end()); + } + + const ChangesetDiscussion& discussion() const { + return osmium::detail::subitem_of_type(cbegin(), cend()); + } + }; // class Changeset static_assert(sizeof(Changeset) % osmium::memory::align_bytes == 0, "Class osmium::Changeset has wrong size to be aligned properly!"); diff --git a/include/osmium/osm/crc.hpp b/include/osmium/osm/crc.hpp index eefa4a13e..f5e01e47b 100644 --- a/include/osmium/osm/crc.hpp +++ b/include/osmium/osm/crc.hpp @@ -46,10 +46,9 @@ DEALINGS IN THE SOFTWARE. namespace osmium { - template - class CRC { + namespace util { - static inline uint16_t byte_swap_16(uint16_t value) noexcept { + inline uint16_t byte_swap_16(uint16_t value) noexcept { # if defined(__GNUC__) || defined(__clang__) return __builtin_bswap16(value); # else @@ -57,27 +56,32 @@ namespace osmium { # endif } - static inline uint32_t byte_swap_32(uint32_t value) noexcept { + inline uint32_t byte_swap_32(uint32_t value) noexcept { # if defined(__GNUC__) || defined(__clang__) return __builtin_bswap32(value); # else return (value >> 24) | - ((value >> 8) & 0x0000FF00) | - ((value << 8) & 0x00FF0000) | + ((value >> 8) & 0x0000FF00) | + ((value << 8) & 0x00FF0000) | (value << 24); # endif } - static inline uint64_t byte_swap_64(uint64_t value) noexcept { + inline uint64_t byte_swap_64(uint64_t value) noexcept { # if defined(__GNUC__) || defined(__clang__) return __builtin_bswap64(value); # else uint64_t val1 = byte_swap_32(value & 0xFFFFFFFF); uint64_t val2 = byte_swap_32(value >> 32); - return (val1 << 32) & val2; + return (val1 << 32) | val2; # endif } + } // namespace util + + template + class CRC { + TCRC m_crc; public: @@ -90,37 +94,37 @@ namespace osmium { return m_crc; } - void update_bool(bool value) { + void update_bool(const bool value) { m_crc.process_byte(value); } - void update_int8(uint8_t value) { + void update_int8(const uint8_t value) { m_crc.process_byte(value); } - void update_int16(uint16_t value) { + void update_int16(const uint16_t value) { #if __BYTE_ORDER == __LITTLE_ENDIAN m_crc.process_bytes(&value, sizeof(uint16_t)); #else - uint16_t v = byte_swap_16(value); + uint16_t v = osmium::util::byte_swap_16(value); m_crc.process_bytes(&v, sizeof(uint16_t)); #endif } - void update_int32(uint32_t value) { + void update_int32(const uint32_t value) { #if __BYTE_ORDER == __LITTLE_ENDIAN m_crc.process_bytes(&value, sizeof(uint32_t)); #else - uint32_t v = byte_swap_32(value); + uint32_t v = osmium::util::byte_swap_32(value); m_crc.process_bytes(&v, sizeof(uint32_t)); #endif } - void update_int64(uint64_t value) { + void update_int64(const uint64_t value) { #if __BYTE_ORDER == __LITTLE_ENDIAN m_crc.process_bytes(&value, sizeof(uint64_t)); #else - uint64_t v = byte_swap_64(value); + uint64_t v = osmium::util::byte_swap_64(value); m_crc.process_bytes(&v, sizeof(uint64_t)); #endif } @@ -156,7 +160,10 @@ namespace osmium { } void update(const TagList& tags) { - m_crc.process_bytes(tags.data(), tags.byte_size()); + for (const Tag& tag : tags) { + update_string(tag.key()); + update_string(tag.value()); + } } void update(const osmium::RelationMember& member) { @@ -206,14 +213,26 @@ namespace osmium { } } + void update(const osmium::ChangesetDiscussion& discussion) { + for (const auto& comment : discussion) { + update(comment.date()); + update_int32(comment.uid()); + update_string(comment.user()); + update_string(comment.text()); + } + } + void update(const osmium::Changeset& changeset) { update_int64(changeset.id()); update(changeset.created_at()); update(changeset.closed_at()); update(changeset.bounds()); update_int32(changeset.num_changes()); + update_int32(changeset.num_comments()); update_int32(changeset.uid()); update_string(changeset.user()); + update(changeset.tags()); + update(changeset.discussion()); } }; // class CRC diff --git a/include/osmium/osm/diff_object.hpp b/include/osmium/osm/diff_object.hpp index 1e053fdda..d9872eaea 100644 --- a/include/osmium/osm/diff_object.hpp +++ b/include/osmium/osm/diff_object.hpp @@ -33,6 +33,9 @@ DEALINGS IN THE SOFTWARE. */ +#include + +#include #include #include #include @@ -40,75 +43,158 @@ DEALINGS IN THE SOFTWARE. namespace osmium { - class Node; - class Way; - class Relation; - + /** + * A DiffObject holds pointers to three OSMObjects, the current object, + * the previous, and the next. They always have the same type (Node, Way, + * or Relation) and the same ID, but may have different versions. + * + * It is used when iterating over OSM files with history data to make + * working with versioned OSM objects easier. Because you have access to + * the previous and next objects as well as the current one, comparisons + * between object versions is easy. + * + * If the current object is the first version available, the previous + * pointer must be the same as the current one. If the current object is + * the last version available, the next pointer must be the same as the + * current one. + * + * DiffObjects are immutable. + */ class DiffObject { - protected: - - osmium::OSMObject* m_prev; - osmium::OSMObject* m_curr; - osmium::OSMObject* m_next; + const osmium::OSMObject* m_prev; + const osmium::OSMObject* m_curr; + const osmium::OSMObject* m_next; public: + /** + * Default construct an empty DiffObject. Most methods of this class + * can not be called on empty DiffObjects. + */ DiffObject() noexcept : m_prev(nullptr), m_curr(nullptr), m_next(nullptr) { } - explicit DiffObject(osmium::OSMObject& prev, osmium::OSMObject& curr, osmium::OSMObject& next) noexcept : + /** + * Construct a non-empty DiffObject from the given OSMObjects. All + * OSMObjects must be of the same type (Node, Way, or Relation) and + * have the same ID. + */ + DiffObject(const osmium::OSMObject& prev, const osmium::OSMObject& curr, const osmium::OSMObject& next) noexcept : m_prev(&prev), m_curr(&curr), m_next(&next) { + assert(prev.type() == curr.type() && curr.type() == next.type()); + assert(prev.id() == curr.id() && curr.id() == next.id()); } - DiffObject(const DiffObject&) = default; - DiffObject& operator=(const DiffObject&) = default; - - DiffObject(DiffObject&&) = default; - DiffObject& operator=(DiffObject&&) = default; + /** + * Check whether the DiffObject was created empty. + */ + bool empty() const noexcept { + return m_prev == nullptr; + } + /** + * Get the previous object stored. + * + * @pre DiffObject must not be empty. + */ const osmium::OSMObject& prev() const noexcept { + assert(m_prev && m_curr && m_next); return *m_prev; } + /** + * Get the current object stored. + * + * @pre DiffObject must not be empty. + */ const osmium::OSMObject& curr() const noexcept { + assert(m_prev && m_curr && m_next); return *m_curr; } + /** + * Get the next object stored. + * + * @pre DiffObject must not be empty. + */ const osmium::OSMObject& next() const noexcept { + assert(m_prev && m_curr && m_next); return *m_next; } + /** + * Is the current object version the first (with this type and ID)? + * + * @pre DiffObject must not be empty. + */ bool first() const noexcept { + assert(m_prev && m_curr && m_next); return m_prev == m_curr; } + /** + * Is the current object version the last (with this type and ID)? + * + * @pre DiffObject must not be empty. + */ bool last() const noexcept { + assert(m_prev && m_curr && m_next); return m_curr == m_next; } + /** + * Return the type of the current object. + * + * @pre DiffObject must not be empty. + */ osmium::item_type type() const noexcept { + assert(m_prev && m_curr && m_next); return m_curr->type(); } + /** + * Return the ID of the current object. + * + * @pre DiffObject must not be empty. + */ osmium::object_id_type id() const noexcept { + assert(m_prev && m_curr && m_next); return m_curr->id(); } + /** + * Return the version of the current object. + * + * @pre DiffObject must not be empty. + */ osmium::object_version_type version() const noexcept { + assert(m_prev && m_curr && m_next); return m_curr->version(); } + /** + * Return the changeset ID of the current object. + * + * @pre DiffObject must not be empty. + */ osmium::changeset_id_type changeset() const noexcept { + assert(m_prev && m_curr && m_next); return m_curr->changeset(); } + /** + * Return the timestamp when the current object version was created. + * + * @pre DiffObject must not be empty. + */ const osmium::Timestamp start_time() const noexcept { + assert(m_prev && m_curr && m_next); return m_curr->timestamp(); } @@ -118,8 +204,11 @@ namespace osmium { * is valid. If this is the last version of the object, this will * return a special "end of time" timestamp that is guaranteed to * be larger than any normal timestamp. + * + * @pre DiffObject must not be empty. */ const osmium::Timestamp end_time() const noexcept { + assert(m_prev && m_curr && m_next); return last() ? osmium::end_of_time() : m_next->timestamp(); } @@ -129,8 +218,11 @@ namespace osmium { * * This is a bit more complex than you'd think, because we have to * handle the case properly where the start_time() == end_time(). + * + * @pre DiffObject must not be empty. */ bool is_between(const osmium::Timestamp& from, const osmium::Timestamp& to) const noexcept { + assert(m_prev && m_curr && m_next); return start_time() < to && ((start_time() != end_time() && end_time() > from) || (start_time() == end_time() && end_time() >= from)); @@ -138,45 +230,42 @@ namespace osmium { /** * Current object version is visible at the given timestamp. + * + * @pre DiffObject must not be empty. */ bool is_visible_at(const osmium::Timestamp& timestamp) const noexcept { + assert(m_prev && m_curr && m_next); return start_time() <= timestamp && end_time() > timestamp && m_curr->visible(); } }; // class DiffObject - template + template class DiffObjectDerived : public DiffObject { public: - DiffObjectDerived(T& prev, T& curr, T& next) noexcept : + DiffObjectDerived(const T& prev, const T& curr, const T& next) noexcept : DiffObject(prev, curr, next) { } - DiffObjectDerived(const DiffObjectDerived&) = default; - DiffObjectDerived& operator=(const DiffObjectDerived&) = default; - - DiffObjectDerived(DiffObjectDerived&&) = default; - DiffObjectDerived& operator=(DiffObjectDerived&&) = default; - const T& prev() const noexcept { - return *static_cast(m_prev); + return static_cast(DiffObject::prev()); } const T& curr() const noexcept { - return *static_cast(m_curr); + return static_cast(DiffObject::curr()); } const T& next() const noexcept { - return *static_cast(m_next); + return static_cast(DiffObject::next()); } }; // class DiffObjectDerived - typedef DiffObjectDerived DiffNode; - typedef DiffObjectDerived DiffWay; - typedef DiffObjectDerived DiffRelation; + using DiffNode = DiffObjectDerived; + using DiffWay = DiffObjectDerived; + using DiffRelation = DiffObjectDerived; } // namespace osmium diff --git a/include/osmium/osm/entity.hpp b/include/osmium/osm/entity.hpp index ce292c8d6..c7f70553c 100644 --- a/include/osmium/osm/entity.hpp +++ b/include/osmium/osm/entity.hpp @@ -41,7 +41,7 @@ namespace osmium { namespace detail { - template + template inline TSubitem& subitem_of_type(TIter it, TIter end) { for (; it != end; ++it) { if (it->type() == TSubitem::itemtype) { diff --git a/include/osmium/osm/item_type.hpp b/include/osmium/osm/item_type.hpp index 54975e326..95826bc98 100644 --- a/include/osmium/osm/item_type.hpp +++ b/include/osmium/osm/item_type.hpp @@ -53,13 +53,18 @@ namespace osmium { relation_member_list = 0x13, relation_member_list_with_full_members = 0x23, outer_ring = 0x40, - inner_ring = 0x41 + inner_ring = 0x41, + changeset_discussion = 0x80 }; // enum class item_type /** * Return item_type for index: * 0 -> node, 1 -> way, 2 -> relation + * + * @param i Index. Must be between 0 and 2. + * + * @returns Item type. */ inline item_type nwr_index_to_item_type(unsigned int i) noexcept { assert(i <= 2); @@ -69,6 +74,10 @@ namespace osmium { /** * Return index for item_type: * node -> 0, way -> 1, relation -> 2 + * + * @param type Item type. Must be node, way, or relation. + * + * @returns Index. */ inline unsigned int item_type_to_nwr_index(item_type type) noexcept { unsigned int i = static_cast(type); @@ -102,6 +111,8 @@ namespace osmium { return item_type::outer_ring; case 'I': return item_type::inner_ring; + case 'D': + return item_type::changeset_discussion; default: return item_type::undefined; } @@ -136,6 +147,8 @@ namespace osmium { return 'O'; case item_type::inner_ring: return 'I'; + case item_type::changeset_discussion: + return 'D'; } } @@ -165,6 +178,8 @@ namespace osmium { return "outer_ring"; case item_type::inner_ring: return "inner_ring"; + case item_type::changeset_discussion: + return "changeset_discussion"; } } #pragma GCC diagnostic pop diff --git a/include/osmium/osm/location.hpp b/include/osmium/osm/location.hpp index 0d4fdc13d..f79117e0a 100644 --- a/include/osmium/osm/location.hpp +++ b/include/osmium/osm/location.hpp @@ -52,11 +52,11 @@ namespace osmium { */ struct invalid_location : public std::range_error { - invalid_location(const std::string& what) : + explicit invalid_location(const std::string& what) : std::range_error(what) { } - invalid_location(const char* what) : + explicit invalid_location(const char* what) : std::range_error(what) { } @@ -95,7 +95,7 @@ namespace osmium { return static_cast(std::round(c * coordinate_precision)); } - static OSMIUM_CONSTEXPR double fix_to_double(const int32_t c) noexcept { + static constexpr double fix_to_double(const int32_t c) noexcept { return static_cast(c) / coordinate_precision; } @@ -238,11 +238,11 @@ namespace osmium { /** * Locations are equal if both coordinates are equal. */ - inline OSMIUM_CONSTEXPR bool operator==(const Location& lhs, const Location& rhs) noexcept { + inline constexpr bool operator==(const Location& lhs, const Location& rhs) noexcept { return lhs.x() == rhs.x() && lhs.y() == rhs.y(); } - inline OSMIUM_CONSTEXPR bool operator!=(const Location& lhs, const Location& rhs) noexcept { + inline constexpr bool operator!=(const Location& lhs, const Location& rhs) noexcept { return ! (lhs == rhs); } @@ -251,19 +251,19 @@ namespace osmium { * the y coordinate. If either of the locations is * undefined the result is undefined. */ - inline OSMIUM_CONSTEXPR bool operator<(const Location& lhs, const Location& rhs) noexcept { + inline constexpr bool operator<(const Location& lhs, const Location& rhs) noexcept { return (lhs.x() == rhs.x() && lhs.y() < rhs.y()) || lhs.x() < rhs.x(); } - inline OSMIUM_CONSTEXPR bool operator>(const Location& lhs, const Location& rhs) noexcept { + inline constexpr bool operator>(const Location& lhs, const Location& rhs) noexcept { return rhs < lhs; } - inline OSMIUM_CONSTEXPR bool operator<=(const Location& lhs, const Location& rhs) noexcept { + inline constexpr bool operator<=(const Location& lhs, const Location& rhs) noexcept { return ! (rhs < lhs); } - inline OSMIUM_CONSTEXPR bool operator>=(const Location& lhs, const Location& rhs) noexcept { + inline constexpr bool operator>=(const Location& lhs, const Location& rhs) noexcept { return ! (lhs < rhs); } diff --git a/include/osmium/osm/node.hpp b/include/osmium/osm/node.hpp index 123bfc4fa..1ff7d1c28 100644 --- a/include/osmium/osm/node.hpp +++ b/include/osmium/osm/node.hpp @@ -41,8 +41,8 @@ DEALINGS IN THE SOFTWARE. namespace osmium { namespace builder { - template class ObjectBuilder; - } + template class ObjectBuilder; + } // namespace builder class Node : public OSMObject { diff --git a/include/osmium/osm/node_ref.hpp b/include/osmium/osm/node_ref.hpp index 72359cd0f..e1c9c12e1 100644 --- a/include/osmium/osm/node_ref.hpp +++ b/include/osmium/osm/node_ref.hpp @@ -54,15 +54,21 @@ namespace osmium { public: - NodeRef(const osmium::object_id_type ref = 0, const osmium::Location& location = Location()) noexcept : + constexpr NodeRef(const osmium::object_id_type ref = 0, const osmium::Location& location = Location()) noexcept : m_ref(ref), m_location(location) { } - osmium::object_id_type ref() const noexcept { + /** + * Get reference ID of this NodeRef. + */ + constexpr osmium::object_id_type ref() const noexcept { return m_ref; } + /** + * Get absolute value of the reference ID of this NodeRef. + */ osmium::unsigned_object_id_type positive_ref() const noexcept { return static_cast(std::abs(m_ref)); } @@ -74,31 +80,60 @@ namespace osmium { return m_location; } - osmium::Location location() const noexcept { + /** + * Get location of this NodeRef. + */ + constexpr osmium::Location location() const noexcept { return m_location; } + /** + * Get longitude of the location in this NodeRef. + * + * @throws osmium::invalid_location if the location is not set. + */ double lon() const { return m_location.lon(); } + /** + * Get latitude of the location in this NodeRef. + * + * @throws osmium::invalid_location if the location is not set. + */ double lat() const { return m_location.lat(); } - int32_t x() const noexcept { + /** + * Get internal x value of the location in this NodeRef. + */ + constexpr int32_t x() const noexcept { return m_location.x(); } - int32_t y() const noexcept { + /** + * Get internal y value of the location in this NodeRef. + */ + constexpr int32_t y() const noexcept { return m_location.y(); } + /** + * Set the referenced ID. + * + * @returns Reference to this NodeRef for chaining calls. + */ NodeRef& set_ref(const osmium::object_id_type ref) noexcept { m_ref = ref; return *this; } + /** + * Set the location. + * + * @returns Reference to this NodeRef for chaining calls. + */ NodeRef& set_location(const osmium::Location& location) noexcept { m_location = location; return *this; @@ -106,27 +141,50 @@ namespace osmium { }; // class NodeRef - inline bool operator==(const NodeRef& lhs, const NodeRef& rhs) noexcept { + /** + * Compare two NodeRefs. They are equal if they reference the same Node ID. + */ + inline constexpr bool operator==(const NodeRef& lhs, const NodeRef& rhs) noexcept { return lhs.ref() == rhs.ref(); } - inline bool operator!=(const NodeRef& lhs, const NodeRef& rhs) noexcept { + /** + * Compare two NodeRefs. They are not equal if they reference different + * Node IDs. + */ + inline constexpr bool operator!=(const NodeRef& lhs, const NodeRef& rhs) noexcept { return ! (lhs == rhs); } - inline bool operator<(const NodeRef& lhs, const NodeRef& rhs) noexcept { + /** + * Compare two NodeRefs. NodeRefs are ordered according to the Node ID + * they reference. + */ + inline constexpr bool operator<(const NodeRef& lhs, const NodeRef& rhs) noexcept { return lhs.ref() < rhs.ref(); } - inline bool operator>(const NodeRef& lhs, const NodeRef& rhs) noexcept { + /** + * Compare two NodeRefs. NodeRefs are ordered according to the Node ID + * they reference. + */ + inline constexpr bool operator>(const NodeRef& lhs, const NodeRef& rhs) noexcept { return rhs < lhs; } - inline bool operator<=(const NodeRef& lhs, const NodeRef& rhs) noexcept { + /** + * Compare two NodeRefs. NodeRefs are ordered according to the Node ID + * they reference. + */ + inline constexpr bool operator<=(const NodeRef& lhs, const NodeRef& rhs) noexcept { return ! (rhs < lhs); } - inline bool operator>=(const NodeRef& lhs, const NodeRef& rhs) noexcept { + /** + * Compare two NodeRefs. NodeRefs are ordered according to the Node ID + * they reference. + */ + inline constexpr bool operator>=(const NodeRef& lhs, const NodeRef& rhs) noexcept { return ! (lhs < rhs); } @@ -139,32 +197,32 @@ namespace osmium { } /** - * Functor to compare NodeRefs by Location instead of id. + * Functor to compare NodeRefs by Location instead of ID. */ struct location_equal { - bool operator()(const NodeRef& lhs, const NodeRef& rhs) const noexcept { + constexpr bool operator()(const NodeRef& lhs, const NodeRef& rhs) const noexcept { return lhs.location() == rhs.location(); } - typedef NodeRef first_argument_type; - typedef NodeRef second_argument_type; - typedef bool result_type; + using first_argument_type = NodeRef; + using second_argument_type = NodeRef; + using result_type = bool; }; // struct location_equal /** - * Functor to compare NodeRefs by Location instead of id. + * Functor to compare NodeRefs by Location instead of ID. */ struct location_less { - bool operator()(const NodeRef& lhs, const NodeRef& rhs) const noexcept { + constexpr bool operator()(const NodeRef& lhs, const NodeRef& rhs) const noexcept { return lhs.location() < rhs.location(); } - typedef NodeRef first_argument_type; - typedef NodeRef second_argument_type; - typedef bool result_type; + using first_argument_type = NodeRef; + using second_argument_type = NodeRef; + using result_type = bool; }; // struct location_less diff --git a/include/osmium/osm/node_ref_list.hpp b/include/osmium/osm/node_ref_list.hpp index f0dfedbc1..f990b88c0 100644 --- a/include/osmium/osm/node_ref_list.hpp +++ b/include/osmium/osm/node_ref_list.hpp @@ -44,29 +44,29 @@ DEALINGS IN THE SOFTWARE. namespace osmium { /** - * A vector of NodeRef objects. Usually this is not instantiated directly, - * but one of its subclasses are used. + * An ordered collection of NodeRef objects. Usually this is not + * instantiated directly, but one of its subclasses are used. */ class NodeRefList : public osmium::memory::Item { public: - NodeRefList(osmium::item_type itemtype) noexcept : + explicit NodeRefList(osmium::item_type itemtype) noexcept : osmium::memory::Item(sizeof(NodeRefList), itemtype) { } /** - * Checks whether the node list is empty. + * Checks whether the collection is empty. */ bool empty() const noexcept { return sizeof(NodeRefList) == byte_size(); } /** - * Returns the number of nodes in the list. + * Returns the number of NodeRefs in the collection. */ size_t size() const noexcept { - auto size_node_refs = osmium::memory::Item::byte_size() - sizeof(NodeRefList); + auto size_node_refs = byte_size() - sizeof(NodeRefList); assert(size_node_refs % sizeof(NodeRef) == 0); return size_node_refs / sizeof(NodeRef); } @@ -74,8 +74,9 @@ namespace osmium { /** * Access specified element. * - * @param n Get this element of the list. * @pre @code n < size() @endcode + * + * @param n Get the n-th element of the collection. */ const NodeRef& operator[](size_t n) const noexcept { assert(n < size()); @@ -104,16 +105,18 @@ namespace osmium { } /** - * Checks whether the first and last node in the list have the same ID. + * Checks whether the first and last node in the collection have the + * same ID. The locations are not checked. * * @pre @code !empty() @endcode */ bool is_closed() const noexcept { - return front().ref() == back().ref(); + return ends_have_same_id(); } /** - * Checks whether the first and last node in the list have the same ID. + * Checks whether the first and last node in the collection have the + * same ID. The locations are not checked. * * @pre @code !empty() @endcode */ @@ -122,8 +125,8 @@ namespace osmium { } /** - * Checks whether the first and last node in the list have the same - * location. The ID is not checked. + * Checks whether the first and last node in the collection have the + * same location. The IDs are not checked. * * @pre @code !empty() @endcode * @pre @code front().location() && back().location() @endcode @@ -133,9 +136,9 @@ namespace osmium { return front().location() == back().location(); } - typedef NodeRef* iterator; - typedef const NodeRef* const_iterator; - typedef std::reverse_iterator const_reverse_iterator; + using iterator = NodeRef*; + using const_iterator = const NodeRef*; + using const_reverse_iterator = std::reverse_iterator; /// Returns an iterator to the beginning. iterator begin() noexcept { diff --git a/include/osmium/osm/object.hpp b/include/osmium/osm/object.hpp index 8c745ce9b..c0f46adbd 100644 --- a/include/osmium/osm/object.hpp +++ b/include/osmium/osm/object.hpp @@ -281,7 +281,7 @@ namespace osmium { * @param timestamp Timestamp * @returns Reference to object to make calls chainable. */ - OSMObject& set_timestamp(const osmium::Timestamp timestamp) noexcept { + OSMObject& set_timestamp(const osmium::Timestamp& timestamp) noexcept { m_timestamp = timestamp; return *this; } @@ -355,38 +355,38 @@ namespace osmium { return cend(); } - template + template using t_iterator = osmium::memory::ItemIterator; - template + template using t_const_iterator = osmium::memory::ItemIterator; - template + template t_iterator begin() { return t_iterator(subitems_position(), next()); } - template + template t_iterator end() { return t_iterator(next(), next()); } - template + template t_const_iterator cbegin() const { return t_const_iterator(subitems_position(), next()); } - template + template t_const_iterator cend() const { return t_const_iterator(next(), next()); } - template + template t_const_iterator begin() const { return cbegin(); } - template + template t_const_iterator end() const { return cend(); } diff --git a/include/osmium/osm/relation.hpp b/include/osmium/osm/relation.hpp index 99a4f4cd1..eec51193d 100644 --- a/include/osmium/osm/relation.hpp +++ b/include/osmium/osm/relation.hpp @@ -47,9 +47,9 @@ DEALINGS IN THE SOFTWARE. namespace osmium { namespace builder { - template class ObjectBuilder; + template class ObjectBuilder; class RelationMemberListBuilder; - } + } // namespace builder class RelationMember : public osmium::memory::detail::ItemHelper { @@ -74,23 +74,21 @@ namespace osmium { return data() + osmium::memory::padded_length(sizeof(RelationMember) + m_role_size); } - template + template friend class osmium::memory::CollectionIterator; unsigned char* next() { if (full_member()) { return endpos() + reinterpret_cast(endpos())->byte_size(); - } else { - return endpos(); } + return endpos(); } unsigned const char* next() const { if (full_member()) { return endpos() + reinterpret_cast(endpos())->byte_size(); - } else { - return endpos(); } + return endpos(); } void set_role_size(string_size_type size) noexcept { diff --git a/include/osmium/osm/segment.hpp b/include/osmium/osm/segment.hpp index f3a82c97f..fe43102fc 100644 --- a/include/osmium/osm/segment.hpp +++ b/include/osmium/osm/segment.hpp @@ -65,12 +65,12 @@ namespace osmium { ~Segment() = default; /// Return first Location of Segment. - OSMIUM_CONSTEXPR osmium::Location first() const noexcept { + constexpr osmium::Location first() const noexcept { return m_first; } /// Return second Location of Segment. - OSMIUM_CONSTEXPR osmium::Location second() const noexcept { + constexpr osmium::Location second() const noexcept { return m_second; } @@ -84,11 +84,11 @@ namespace osmium { }; // class Segment /// Segments are equal if both their locations are equal - inline OSMIUM_CONSTEXPR bool operator==(const Segment& lhs, const Segment& rhs) noexcept { + inline constexpr bool operator==(const Segment& lhs, const Segment& rhs) noexcept { return lhs.first() == rhs.first() && lhs.second() == rhs.second(); } - inline OSMIUM_CONSTEXPR bool operator!=(const Segment& lhs, const Segment& rhs) noexcept { + inline constexpr bool operator!=(const Segment& lhs, const Segment& rhs) noexcept { return ! (lhs == rhs); } diff --git a/include/osmium/osm/tag.hpp b/include/osmium/osm/tag.hpp index 2e93ede24..4eba2210e 100644 --- a/include/osmium/osm/tag.hpp +++ b/include/osmium/osm/tag.hpp @@ -53,7 +53,7 @@ namespace osmium { Tag& operator=(const Tag&) = delete; Tag& operator=(Tag&&) = delete; - template + template friend class osmium::memory::CollectionIterator; static unsigned char* after_null(unsigned char* ptr) { @@ -122,9 +122,8 @@ namespace osmium { }); if (result == cend()) { return default_value; - } else { - return result->value(); } + return result->value(); } const char* operator[](const char* key) const noexcept { diff --git a/include/osmium/osm/timestamp.hpp b/include/osmium/osm/timestamp.hpp index 390f0e745..651b43f68 100644 --- a/include/osmium/osm/timestamp.hpp +++ b/include/osmium/osm/timestamp.hpp @@ -47,17 +47,18 @@ namespace osmium { /** * A timestamp. Internal representation is an unsigned 32bit integer - * holding seconds since epoch, so this will overflow in 2038. + * holding seconds since epoch (1970-01-01T00:00:00Z), so this will + * overflow in 2106. We can use an unsigned integer here, because the + * OpenStreetMap project was started long after 1970, so there will + * never be dates before that. */ class Timestamp { // length of ISO timestamp string yyyy-mm-ddThh:mm:ssZ\0 static constexpr int timestamp_length = 20 + 1; - /** - * The timestamp format for OSM timestamps in strftime(3) format. - * This is the ISO-Format yyyy-mm-ddThh:mm:ssZ - */ + // The timestamp format for OSM timestamps in strftime(3) format. + // This is the ISO-Format "yyyy-mm-ddThh:mm:ssZ". static const char* timestamp_format() { static const char f[timestamp_length] = "%Y-%m-%dT%H:%M:%SZ"; return f; @@ -67,19 +68,32 @@ namespace osmium { public: + /** + * Default construct an invalid Timestamp. + */ constexpr Timestamp() noexcept : m_timestamp(0) { } - // Not "explicit" so that conversions from time_t work - // like in node.timestamp(123); - constexpr Timestamp(time_t timestamp) noexcept : - m_timestamp(static_cast(timestamp)) { + /** + * Construct a Timestamp from any integer type containing the seconds + * since the epoch. This will not check for overruns, you have to + * make sure the value fits into a uint32_t which is used internally + * in the Timestamp. + * + * The constructor is not declared "explicit" so that conversions + * like @code node.set_timestamp(123); @endcode work. + */ + template ::value, int>::type = 0> + constexpr Timestamp(T timestamp) noexcept : + m_timestamp(uint32_t(timestamp)) { } /** - * Construct timestamp from ISO date/time string. - * Throws std::invalid_argument, if the timestamp can not be parsed. + * Construct timestamp from ISO date/time string in the format + * "yyyy-mm-ddThh:mm:ssZ". + * + * @throws std::invalid_argument if the timestamp can not be parsed. */ explicit Timestamp(const char* timestamp) { #ifndef _WIN32 @@ -105,16 +119,51 @@ namespace osmium { #endif } + /** + * Construct timestamp from ISO date/time string in the format + * "yyyy-mm-ddThh:mm:ssZ". + * + * @throws std::invalid_argument if the timestamp can not be parsed. + */ + explicit Timestamp(const std::string& timestamp) : + Timestamp(timestamp.c_str()) { + } + + /** + * Returns true if this timestamp is valid (ie set to something other + * than 0). + */ + bool valid() const noexcept { + return m_timestamp != 0; + } + + /// Explicit conversion into bool. + explicit constexpr operator bool() const noexcept { + return m_timestamp != 0; + } + + /// Explicit conversion into time_t. constexpr time_t seconds_since_epoch() const noexcept { - return static_cast(m_timestamp); - } - - constexpr operator time_t() const noexcept { - return static_cast(m_timestamp); + return time_t(m_timestamp); } + /// Explicit conversion into uint32_t. explicit constexpr operator uint32_t() const noexcept { - return m_timestamp; + return uint32_t(m_timestamp); + } + + /// Explicit conversion into uint64_t. + explicit constexpr operator uint64_t() const noexcept { + return uint64_t(m_timestamp); + } + + /** + * Implicit conversion into time_t. + * + * @deprecated You should call seconds_since_epoch() explicitly instead. + */ + OSMIUM_DEPRECATED constexpr operator time_t() const noexcept { + return static_cast(m_timestamp); } template @@ -128,7 +177,8 @@ namespace osmium { } /** - * Return UTC Unix time as string in ISO date/time format. + * Return UTC Unix time as string in ISO date/time + * ("yyyy-mm-ddThh:mm:ssZ") format. */ std::string to_iso() const { std::string s; @@ -156,12 +206,20 @@ namespace osmium { }; // class Timestamp - inline OSMIUM_CONSTEXPR Timestamp start_of_time() noexcept { + /** + * A special Timestamp guaranteed to be ordered before any other valid + * Timestamp. + */ + inline constexpr Timestamp start_of_time() noexcept { return Timestamp(1); } - inline OSMIUM_CONSTEXPR Timestamp end_of_time() noexcept { - return Timestamp(std::numeric_limits::max()); + /** + * A special Timestamp guaranteed to be ordered after any other valid + * Timestamp. + */ + inline constexpr Timestamp end_of_time() noexcept { + return Timestamp(std::numeric_limits::max()); } template @@ -170,6 +228,30 @@ namespace osmium { return out; } + inline bool operator==(const Timestamp& lhs, const Timestamp& rhs) noexcept { + return uint32_t(lhs) == uint32_t(rhs); + } + + inline bool operator!=(const Timestamp& lhs, const Timestamp& rhs) noexcept { + return !(lhs == rhs); + } + + inline bool operator<(const Timestamp& lhs, const Timestamp& rhs) noexcept { + return uint32_t(lhs) < uint32_t(rhs); + } + + inline bool operator>(const Timestamp& lhs, const Timestamp& rhs) noexcept { + return rhs < lhs; + } + + inline bool operator<=(const Timestamp& lhs, const Timestamp& rhs) noexcept { + return ! (rhs < lhs); + } + + inline bool operator>=(const Timestamp& lhs, const Timestamp& rhs) noexcept { + return ! (lhs < rhs); + } + template <> inline osmium::Timestamp min_op_start_value() { return end_of_time(); diff --git a/include/osmium/osm/types.hpp b/include/osmium/osm/types.hpp index b3414e594..e4250d966 100644 --- a/include/osmium/osm/types.hpp +++ b/include/osmium/osm/types.hpp @@ -49,6 +49,7 @@ namespace osmium { typedef uint32_t user_id_type; ///< Type for OSM user IDs. typedef int32_t signed_user_id_type; ///< Type for signed OSM user IDs. typedef uint32_t num_changes_type; ///< Type for changeset num_changes. + typedef uint32_t num_comments_type; ///< Type for changeset num_comments. /** * Size for strings in OSM data such as user names, tag keys, roles, etc. @@ -57,6 +58,9 @@ namespace osmium { */ typedef uint16_t string_size_type; + // maximum of 256 characters of max 4 bytes each (in UTF-8 encoding) + constexpr const int max_osm_string_length = 256 * 4; + } // namespace osmium #endif // OSMIUM_OSM_TYPES_HPP diff --git a/include/osmium/osm/types_from_string.hpp b/include/osmium/osm/types_from_string.hpp index b8de14c90..67ab2c157 100644 --- a/include/osmium/osm/types_from_string.hpp +++ b/include/osmium/osm/types_from_string.hpp @@ -43,9 +43,19 @@ DEALINGS IN THE SOFTWARE. #include #include +#include namespace osmium { + /** + * Convert string with object id to object_id_type. + * + * @pre input must not be nullptr. + * + * @param input Input string. + * + * @throws std::range_error if the value is out of range. + */ inline object_id_type string_to_object_id(const char* input) { assert(input); if (*input != '\0' && !std::isspace(*input)) { @@ -58,6 +68,19 @@ namespace osmium { throw std::range_error(std::string("illegal id: '") + input + "'"); } + /** + * Parse string with object type identifier followed by object id. This + * reads strings like "n1234" and "w10". + * + * @pre input must not be nullptr. + * + * @param input Input string. + * @param types Allowed types. Must not be osmium::osm_entity_bits::nothing. + * + * @returns std::pair of type and id. + * + * @throws std::range_error if the value is out of range. + */ inline std::pair string_to_object_id(const char* input, osmium::osm_entity_bits::type types) { assert(input); assert(types != osmium::osm_entity_bits::nothing); @@ -75,7 +98,7 @@ namespace osmium { namespace detail { - inline long string_to_ulong(const char* input, const char *name) { + inline unsigned long string_to_ulong(const char* input, const char *name) { if (*input != '\0' && *input != '-' && !std::isspace(*input)) { char* end; auto value = std::strtoul(input, &end, 10); @@ -88,27 +111,77 @@ namespace osmium { } // namespace detail + /** + * Convert string with object version to object_version_type. + * + * @pre input must not be nullptr. + * + * @param input Input string. + * + * @throws std::range_error if the value is out of range. + */ inline object_version_type string_to_object_version(const char* input) { assert(input); - return static_cast(detail::string_to_ulong(input, "version")); + return static_cast_with_assert(detail::string_to_ulong(input, "version")); } + /** + * Convert string with object version to object_version_type. + * + * @pre input must not be nullptr. + * + * @param input Input string. + * + * @throws std::range_error if the value is out of range. + */ inline changeset_id_type string_to_changeset_id(const char* input) { assert(input); - return static_cast(detail::string_to_ulong(input, "changeset")); + return static_cast_with_assert(detail::string_to_ulong(input, "changeset")); } + /** + * Convert string with user id to signed_user_id_type. + * + * @pre input must not be nullptr. + * + * @param input Input string. + * + * @throws std::range_error if the value is out of range. + */ inline signed_user_id_type string_to_user_id(const char* input) { assert(input); if (input[0] == '-' && input[1] == '1' && input[2] == '\0') { return -1; } - return static_cast(detail::string_to_ulong(input, "user id")); + return static_cast_with_assert(detail::string_to_ulong(input, "user id")); } + /** + * Convert string with number of changes to num_changes_type. + * + * @pre input must not be nullptr. + * + * @param input Input string. + * + * @throws std::range_error if the value is out of range. + */ inline num_changes_type string_to_num_changes(const char* input) { assert(input); - return static_cast(detail::string_to_ulong(input, "value for num changes")); + return static_cast_with_assert(detail::string_to_ulong(input, "value for num changes")); + } + + /** + * Convert string with number of comments to num_comments_type. + * + * @pre input must not be nullptr. + * + * @param input Input string. + * + * @throws std::range_error if the value is out of range. + */ + inline num_comments_type string_to_num_comments(const char* input) { + assert(input); + return static_cast_with_assert(detail::string_to_ulong(input, "value for num comments")); } } // namespace osmium diff --git a/include/osmium/osm/way.hpp b/include/osmium/osm/way.hpp index 3c5f1f6dc..90cde8c3c 100644 --- a/include/osmium/osm/way.hpp +++ b/include/osmium/osm/way.hpp @@ -43,8 +43,8 @@ DEALINGS IN THE SOFTWARE. namespace osmium { namespace builder { - template class ObjectBuilder; - } + template class ObjectBuilder; + } // namespace builder /** * List of node references (id and location) in a way. diff --git a/include/osmium/relations/collector.hpp b/include/osmium/relations/collector.hpp index 40e377393..7d27b3398 100644 --- a/include/osmium/relations/collector.hpp +++ b/include/osmium/relations/collector.hpp @@ -39,9 +39,10 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include +//#include #include +#include #include #include #include // IWYU pragma: keep @@ -55,9 +56,6 @@ DEALINGS IN THE SOFTWARE. namespace osmium { - class Node; - class Way; - /** * @brief Code related to the assembly of OSM relations */ @@ -91,7 +89,7 @@ namespace osmium { * * @tparam TRelations Are we interested in member relations? */ - template + template class Collector { /** @@ -124,82 +122,15 @@ namespace osmium { TCollector& m_collector; - /** - * This variable is initialized with the number of different - * kinds of OSM objects we are interested in. If we only need - * way members (for instance for the multipolygon collector) - * it is intialized with 1 for instance. If node and way - * members are needed, it is initialized with 2. - * - * In the after_* methods of this handler, it is decremented - * and once it reaches 0, we know we have all members available - * that we are ever going to get. - */ - int m_want_types; - - /** - * Find this object in the member vectors and add it to all - * relations that need it. - * - * @returns true if the member was added to at least one - * relation and false otherwise - */ - bool find_and_add_object(const osmium::OSMObject& object) { - auto& mmv = m_collector.member_meta(object.type()); - auto range = std::equal_range(mmv.begin(), mmv.end(), MemberMeta(object.id())); - - if (osmium::relations::count_not_removed(range.first, range.second) == 0) { - // nothing found - return false; - } - - { - m_collector.members_buffer().add_item(object); - const size_t member_offset = m_collector.members_buffer().commit(); - - for (auto it = range.first; it != range.second; ++it) { - it->set_buffer_offset(member_offset); - } - } - - for (auto it = range.first; it != range.second; ++it) { - MemberMeta& member_meta = *it; - if (member_meta.removed()) { - break; - } - assert(member_meta.member_id() == object.id()); - assert(member_meta.relation_pos() < m_collector.m_relations.size()); - RelationMeta& relation_meta = m_collector.m_relations[member_meta.relation_pos()]; -// std::cerr << " => " << member_meta.member_pos() << " < " << m_collector.get_relation(relation_meta).members().size() << " (id=" << m_collector.get_relation(relation_meta).id() << ")\n"; - assert(member_meta.member_pos() < m_collector.get_relation(relation_meta).members().size()); -// std::cerr << " add way " << member_meta.member_id() << " to rel " << m_collector.get_relation(relation_meta).id() << " at pos " << member_meta.member_pos() << "\n"; - relation_meta.got_one_member(); - if (relation_meta.has_all_members()) { - const size_t relation_offset = member_meta.relation_pos(); - m_collector.complete_relation(relation_meta); - m_collector.m_relations[relation_offset] = RelationMeta(); - m_collector.possibly_purge_removed_members(); - } - } - - // Remove MemberMetas that were marked as removed. - mmv.erase(std::remove_if(mmv.begin(), mmv.end(), [](MemberMeta& mm) { - return mm.removed(); - }), mmv.end()); - - return true; - } - public: HandlerPass2(TCollector& collector) noexcept : - m_collector(collector), - m_want_types((TNodes?1:0) + (TWays?1:0) + (TRelations?1:0)) { + m_collector(collector) { } void node(const osmium::Node& node) { if (TNodes) { - if (! find_and_add_object(node)) { + if (! m_collector.find_and_add_object(node)) { m_collector.node_not_in_any_relation(node); } } @@ -207,7 +138,7 @@ namespace osmium { void way(const osmium::Way& way) { if (TWays) { - if (! find_and_add_object(way)) { + if (! m_collector.find_and_add_object(way)) { m_collector.way_not_in_any_relation(way); } } @@ -215,7 +146,7 @@ namespace osmium { void relation(const osmium::Relation& relation) { if (TRelations) { - if (! find_and_add_object(relation)) { + if (! m_collector.find_and_add_object(relation)) { m_collector.relation_not_in_any_relation(relation); } } @@ -227,6 +158,8 @@ namespace osmium { }; // class HandlerPass2 + private: + HandlerPass2 m_handler_pass2; // All relations we are interested in will be kept in this buffer @@ -375,6 +308,8 @@ namespace osmium { return m_members_buffer.get(offset); } + private: + /** * Tell the Collector that you are interested in this relation * and want it kept until all members have been assembled and @@ -424,6 +359,84 @@ namespace osmium { std::sort(m_member_meta[2].begin(), m_member_meta[2].end()); } + /** + * Find this object in the member vectors and add it to all + * relations that need it. + * + * @returns true if the member was added to at least one + * relation and false otherwise + */ + bool find_and_add_object(const osmium::OSMObject& object) { + auto& mmv = member_meta(object.type()); + auto range = std::equal_range(mmv.begin(), mmv.end(), MemberMeta(object.id())); + + if (osmium::relations::count_not_removed(range.first, range.second) == 0) { + // nothing found + return false; + } + + { + members_buffer().add_item(object); + const size_t member_offset = members_buffer().commit(); + + for (auto it = range.first; it != range.second; ++it) { + it->set_buffer_offset(member_offset); + } + } + + for (auto it = range.first; it != range.second; ++it) { + MemberMeta& member_meta = *it; + if (member_meta.removed()) { + break; + } + assert(member_meta.member_id() == object.id()); + assert(member_meta.relation_pos() < m_relations.size()); + RelationMeta& relation_meta = m_relations[member_meta.relation_pos()]; +// std::cerr << " => " << member_meta.member_pos() << " < " << get_relation(relation_meta).members().size() << " (id=" << get_relation(relation_meta).id() << ")\n"; + assert(member_meta.member_pos() < get_relation(relation_meta).members().size()); +// std::cerr << " add way " << member_meta.member_id() << " to rel " << get_relation(relation_meta).id() << " at pos " << member_meta.member_pos() << "\n"; + relation_meta.got_one_member(); + if (relation_meta.has_all_members()) { + const size_t relation_offset = member_meta.relation_pos(); + static_cast(this)->complete_relation(relation_meta); + clear_member_metas(relation_meta); + m_relations[relation_offset] = RelationMeta(); + possibly_purge_removed_members(); + } + } + + // Remove MemberMetas that were marked as removed. + mmv.erase(std::remove_if(mmv.begin(), mmv.end(), [](MemberMeta& mm) { + return mm.removed(); + }), mmv.end()); + + return true; + } + + void clear_member_metas(const osmium::relations::RelationMeta& relation_meta) { + const osmium::Relation& relation = get_relation(relation_meta); + for (const auto& member : relation.members()) { + if (member.ref() != 0) { + auto& mmv = member_meta(member.type()); + auto range = std::equal_range(mmv.begin(), mmv.end(), MemberMeta(member.ref())); + assert(range.first != range.second); + + // if this is the last time this object was needed + // then mark it as removed + if (osmium::relations::count_not_removed(range.first, range.second) == 1) { + get_member(range.first->buffer_offset()).set_removed(true); + } + + for (auto it = range.first; it != range.second; ++it) { + if (!it->removed() && relation.id() == get_relation(it->relation_pos()).id()) { + it->remove(); + break; + } + } + } + } + } + public: uint64_t used_memory() const { @@ -474,14 +487,14 @@ namespace osmium { return range.first->buffer_offset(); } - template + template void read_relations(TIter begin, TIter end) { HandlerPass1 handler(*static_cast(this)); osmium::apply(begin, end, handler); sort_member_meta(); } - template + template void read_relations(TSource& source) { read_relations(std::begin(source), std::end(source)); source.close(); @@ -490,7 +503,7 @@ namespace osmium { void moving_in_buffer(size_t old_offset, size_t new_offset) { const osmium::OSMObject& object = m_members_buffer.get(old_offset); auto& mmv = member_meta(object.type()); - auto range = std::equal_range(mmv.begin(), mmv.end(), osmium::relations::MemberMeta(object.id())); + auto range = std::equal_range(mmv.begin(), mmv.end(), MemberMeta(object.id())); for (auto it = range.first; it != range.second; ++it) { assert(it->buffer_offset() == old_offset); it->set_buffer_offset(new_offset); @@ -506,13 +519,15 @@ namespace osmium { void possibly_purge_removed_members() { ++m_count_complete; if (m_count_complete > 10000) { // XXX - const size_t size_before = m_members_buffer.committed(); +// const size_t size_before = m_members_buffer.committed(); m_members_buffer.purge_removed(this); +/* const size_t size_after = m_members_buffer.committed(); double percent = static_cast(size_before - size_after); percent /= size_before; percent *= 100; -// std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast(percent) << "%)\n"; + std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast(percent) << "%)\n"; +*/ m_count_complete = 0; } } diff --git a/include/osmium/relations/detail/member_meta.hpp b/include/osmium/relations/detail/member_meta.hpp index a45088eab..ea86734ea 100644 --- a/include/osmium/relations/detail/member_meta.hpp +++ b/include/osmium/relations/detail/member_meta.hpp @@ -144,7 +144,7 @@ namespace osmium { * @param begin Begin of iterator range * @param end End of iterator range */ - template + template inline typename std::iterator_traits::difference_type count_not_removed(TIter begin, TIter end) { return std::count_if(begin, end, [](MemberMeta& mm) { return !mm.removed(); diff --git a/include/osmium/tags/filter.hpp b/include/osmium/tags/filter.hpp index 3c1946c54..0d3fc4ee2 100644 --- a/include/osmium/tags/filter.hpp +++ b/include/osmium/tags/filter.hpp @@ -46,7 +46,7 @@ namespace osmium { namespace tags { - template + template struct match_key { bool operator()(const TKey& rule_key, const char* tag_key) { return rule_key == tag_key; @@ -59,7 +59,7 @@ namespace osmium { } }; // struct match_key_prefix - template + template struct match_value { bool operator()(const TValue& rule_value, const char* tag_value) { return rule_value == tag_value; @@ -73,7 +73,7 @@ namespace osmium { } }; // struct match_value - template , class TValueComp=match_value> + template , typename TValueComp=match_value> class Filter { typedef TKey key_type; @@ -115,7 +115,7 @@ namespace osmium { m_default_result(default_result) { } - template ::value, int>::type = 0> + template ::value, int>::type = 0> Filter& add(bool result, const key_type& key, const value_type& value) { m_rules.emplace_back(result, false, key, value); return *this; diff --git a/include/osmium/tags/taglist.hpp b/include/osmium/tags/taglist.hpp index d7c78dc79..8fc9c68cd 100644 --- a/include/osmium/tags/taglist.hpp +++ b/include/osmium/tags/taglist.hpp @@ -45,17 +45,17 @@ namespace osmium { */ namespace tags { - template + template inline bool match_any_of(const osmium::TagList& tag_list, TFilter&& filter) { return std::any_of(tag_list.cbegin(), tag_list.cend(), std::forward(filter)); } - template + template inline bool match_all_of(const osmium::TagList& tag_list, TFilter&& filter) { return std::all_of(tag_list.cbegin(), tag_list.cend(), std::forward(filter)); } - template + template inline bool match_none_of(const osmium::TagList& tag_list, TFilter&& filter) { return std::none_of(tag_list.cbegin(), tag_list.cend(), std::forward(filter)); } diff --git a/include/osmium/thread/function_wrapper.hpp b/include/osmium/thread/function_wrapper.hpp index fe0a49257..2fc0b7e22 100644 --- a/include/osmium/thread/function_wrapper.hpp +++ b/include/osmium/thread/function_wrapper.hpp @@ -50,7 +50,9 @@ namespace osmium { struct impl_base { virtual ~impl_base() = default; - virtual void call() = 0; + virtual bool call() { + return true; + } }; // struct impl_base @@ -58,28 +60,38 @@ namespace osmium { template struct impl_type : impl_base { + F m_functor; - impl_type(F&& functor) : - m_functor(std::move(functor)) { + explicit impl_type(F&& functor) : + m_functor(std::forward(functor)) { } - void call() override { + bool call() override { m_functor(); + return false; } + }; // struct impl_type public: // Constructor must not be "explicit" for wrapper // to work seemlessly. - template - function_wrapper(F&& f) : - impl(new impl_type(std::move(f))) { + template + function_wrapper(TFunction&& f) : + impl(new impl_type(std::forward(f))) { } - void operator()() { - impl->call(); + // The integer parameter is only used to signal that we want + // the special function wrapper that makes the worker thread + // shut down. + function_wrapper(int) : + impl(new impl_base()) { + } + + bool operator()() { + return impl->call(); } function_wrapper() = default; diff --git a/include/osmium/thread/pool.hpp b/include/osmium/thread/pool.hpp index 391603108..f7b4f4038 100644 --- a/include/osmium/thread/pool.hpp +++ b/include/osmium/thread/pool.hpp @@ -54,6 +54,32 @@ namespace osmium { */ namespace thread { + namespace detail { + + // Maximum number of allowed pool threads (just to keep the user + // from setting something silly). + constexpr const int max_pool_threads = 256; + + inline int get_pool_size(int num_threads, int user_setting, unsigned hardware_concurrency) { + if (num_threads == 0) { + num_threads = user_setting ? user_setting : -2; + } + + if (num_threads < 0) { + num_threads += hardware_concurrency; + } + + if (num_threads < 1) { + num_threads = 1; + } else if (num_threads > max_pool_threads) { + num_threads = max_pool_threads; + } + + return num_threads; + } + + } // namespace detail + /** * Thread pool. */ @@ -83,7 +109,6 @@ namespace osmium { }; // class thread_joiner - std::atomic m_done; osmium::thread::Queue m_work_queue; std::vector m_threads; thread_joiner m_joiner; @@ -91,11 +116,15 @@ namespace osmium { void worker_thread() { osmium::thread::set_thread_name("_osmium_worker"); - while (!m_done) { + while (true) { function_wrapper task; m_work_queue.wait_and_pop_with_timeout(task); if (task) { - task(); + if (task()) { + // The called tasks returns true only when the + // worker thread should shut down. + return; + } } } } @@ -113,26 +142,17 @@ namespace osmium { * In all cases the minimum number of threads in the pool is 1. */ explicit Pool(int num_threads, size_t max_queue_size) : - m_done(false), m_work_queue(max_queue_size, "work"), m_threads(), m_joiner(m_threads), - m_num_threads(num_threads) { - - if (m_num_threads == 0) { - m_num_threads = osmium::config::get_pool_threads(); - } - - if (m_num_threads <= 0) { - m_num_threads = std::max(1, static_cast(std::thread::hardware_concurrency()) + m_num_threads); - } + m_num_threads(detail::get_pool_size(num_threads, osmium::config::get_pool_threads(), std::thread::hardware_concurrency())) { try { for (int i = 0; i < m_num_threads; ++i) { m_threads.push_back(std::thread(&Pool::worker_thread, this)); } } catch (...) { - m_done = true; + shutdown_all_workers(); throw; } } @@ -147,8 +167,15 @@ namespace osmium { return pool; } + void shutdown_all_workers() { + for (int i = 0; i < m_num_threads; ++i) { + // The special function wrapper makes a worker shut down. + m_work_queue.push(function_wrapper{0}); + } + } + ~Pool() { - m_done = true; + shutdown_all_workers(); m_work_queue.shutdown(); } diff --git a/include/osmium/thread/queue.hpp b/include/osmium/thread/queue.hpp index 76ad9a020..65b18475c 100644 --- a/include/osmium/thread/queue.hpp +++ b/include/osmium/thread/queue.hpp @@ -75,6 +75,9 @@ namespace osmium { /// The largest size the queue has been so far. size_t m_largest_size; + /// The number of times push() was called on the queue. + std::atomic m_push_counter; + /// The number of times the queue was full and a thread pushing /// to the queue was blocked. std::atomic m_full_counter; @@ -89,7 +92,7 @@ namespace osmium { * 0 for an unlimited size. * @param name Optional name for this queue. (Used for debugging.) */ - Queue(size_t max_size = 0, const std::string& name = "") : + explicit Queue(size_t max_size = 0, const std::string& name = "") : m_max_size(max_size), m_name(name), m_mutex(), @@ -99,6 +102,7 @@ namespace osmium { #ifdef OSMIUM_DEBUG_QUEUE_SIZE , m_largest_size(0), + m_push_counter(0), m_full_counter(0) #endif { @@ -107,7 +111,7 @@ namespace osmium { ~Queue() { shutdown(); #ifdef OSMIUM_DEBUG_QUEUE_SIZE - std::cerr << "queue '" << m_name << "' with max_size=" << m_max_size << " had largest size " << m_largest_size << " and was full " << m_full_counter << " times\n"; + std::cerr << "queue '" << m_name << "' with max_size=" << m_max_size << " had largest size " << m_largest_size << " and was full " << m_full_counter << " times in " << m_push_counter << " push() calls\n"; #endif } @@ -116,6 +120,9 @@ namespace osmium { * call will block if the queue is full. */ void push(T value) { +#ifdef OSMIUM_DEBUG_QUEUE_SIZE + ++m_push_counter; +#endif if (m_max_size) { while (size() >= m_max_size) { std::this_thread::sleep_for(full_queue_sleep_duration); diff --git a/include/osmium/thread/util.hpp b/include/osmium/thread/util.hpp index ca4f6dd53..a20e618a0 100644 --- a/include/osmium/thread/util.hpp +++ b/include/osmium/thread/util.hpp @@ -49,7 +49,7 @@ namespace osmium { * the exception stored in the future if there was one. Otherwise it * will just return. */ - template + template inline void check_for_exception(std::future& future) { if (future.valid() && future.wait_for(std::chrono::seconds(0)) == std::future_status::ready) { future.get(); @@ -60,7 +60,7 @@ namespace osmium { * Wait until the given future becomes ready. Will block if the future * is not ready. Can be called more than once unlike future.get(). */ - template + template inline void wait_until_done(std::future& future) { if (future.valid()) { future.get(); @@ -71,15 +71,44 @@ namespace osmium { * Set name of current thread for debugging. This only works on Linux. */ #ifdef __linux__ - inline void set_thread_name(const char* name) { + inline void set_thread_name(const char* name) noexcept { prctl(PR_SET_NAME, name, 0, 0, 0); } #else - inline void set_thread_name(const char*) { + inline void set_thread_name(const char*) noexcept { // intentionally left blank } #endif + class thread_handler { + + std::thread m_thread; + + public: + + thread_handler() : + m_thread() { + } + + template + explicit thread_handler(TFunction&& f, TArgs&&... args) : + m_thread(std::forward(f), std::forward(args)...) { + } + + thread_handler(const thread_handler&) = delete; + thread_handler& operator=(const thread_handler&) = delete; + + thread_handler(thread_handler&&) = default; + thread_handler& operator=(thread_handler&&) = default; + + ~thread_handler() { + if (m_thread.joinable()) { + m_thread.join(); + } + } + + }; // class thread_handler + } // namespace thread } // namespace osmium diff --git a/include/osmium/util/compatibility.hpp b/include/osmium/util/compatibility.hpp index 90d85c502..27adca7b7 100644 --- a/include/osmium/util/compatibility.hpp +++ b/include/osmium/util/compatibility.hpp @@ -34,14 +34,20 @@ DEALINGS IN THE SOFTWARE. */ // Workarounds for MSVC which doesn't support -// * constexpr in all cases yet // * [[noreturn]] #ifdef _MSC_VER -# define OSMIUM_CONSTEXPR # define OSMIUM_NORETURN __declspec(noreturn) #else -# define OSMIUM_CONSTEXPR constexpr # define OSMIUM_NORETURN [[noreturn]] #endif +// [[deprecated]] is only available in C++14, use this for the time being +#ifdef __GNUC__ +# define OSMIUM_DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) +# define OSMIUM_DEPRECATED __declspec(deprecated) +#else +# define OSMIUM_DEPRECATED +#endif + #endif // OSMIUM_UTIL_COMPATIBILITY_HPP diff --git a/include/osmium/util/config.hpp b/include/osmium/util/config.hpp index 3285eedbb..e31cd6a9b 100644 --- a/include/osmium/util/config.hpp +++ b/include/osmium/util/config.hpp @@ -49,7 +49,7 @@ namespace osmium { if (env) { return std::atoi(env); } - return -2; + return 0; } inline bool use_pool_threads_for_pbf_parsing() { diff --git a/include/osmium/util/data_file.hpp b/include/osmium/util/data_file.hpp deleted file mode 100644 index 22bf1910a..000000000 --- a/include/osmium/util/data_file.hpp +++ /dev/null @@ -1,194 +0,0 @@ -#ifndef OSMIUM_UTIL_DATA_FILE_HPP -#define OSMIUM_UTIL_DATA_FILE_HPP - -/* - -This file is part of Osmium (http://osmcode.org/libosmium). - -Copyright 2013-2015 Jochen Topf and others (see README). - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -*/ - -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 -# include -# include -#endif - -#include - -namespace osmium { - - namespace util { - - /** - * Class wrapper for convenient access to some low-level file - * functions. - */ - class DataFile { - - FILE* m_file; - - public: - - /** - * Create and open a temporary file. It is removed after opening. - * - * @throws std::system_error if something went wrong. - */ - DataFile() : - m_file(::tmpfile()) { - if (!m_file) { - throw std::system_error(errno, std::system_category(), "tmpfile failed"); - } - } - - /** - * Create and open a temporary file with the specified size. It - * is removed after opening. - * - * @throws std::system_error if something went wrong. - */ - explicit DataFile(size_t size) : - DataFile() { - grow(size); - } - - /** - * Create and open a named file. - * - * @param filename the name of the file - * @param writable should the file be writable? - * @throws std::system_error if something went wrong. - */ - DataFile(const char* filename, bool writable) : - m_file(::fopen(filename, writable ? "wb+" : "rb" )) { - if (!m_file) { - throw std::system_error(errno, std::system_category(), "fopen failed"); - } - } - - /** - * Create and open a named file. - * - * @param filename the name of the file - * @param writable should the file be writable? - * @throws std::system_error if something went wrong. - */ - DataFile(const std::string& filename, bool writable) : - DataFile(filename.c_str(), writable) { - } - - /** - * In boolean context the DataFile class returns true if the file - * is open. - */ - operator bool() const noexcept { - return m_file != nullptr; - } - - /** - * Close the file. - * - * Does nothing if the file is already closed. - * - * @throws std::system_error if file could not be closed - */ - void close() { - if (m_file) { - if (::fclose(m_file) != 0) { - throw std::system_error(errno, std::system_category(), "fclose failed"); - } - m_file = nullptr; - } - } - - ~DataFile() noexcept { - try { - close(); - } catch (std::system_error&) { - // ignore - } - } - - /** - * Get file descriptor of underlying file. - * - * @throws std::runtime_errro if file is not open - * @throws std::system_error if fileno(3) call failed - */ - int fd() const { - if (!m_file) { - throw std::runtime_error("no open file"); - } - - int fd = ::fileno(m_file); - - if (fd == -1) { - throw std::system_error(errno, std::system_category(), "fileno failed"); - } - - return fd; - } - - /** - * Ask the operating system for the size of this file. - * - * @throws std::system_error if fstat(2) call failed - */ - size_t size() const { - return osmium::util::file_size(fd()); - } - - /** - * Grow file to given size. - * - * If the file is large enough already, nothing is done. - * The file is never shrunk. - * - * @throws std::system_error if ftruncate(2) call failed - */ - void grow(size_t new_size) const { - if (size() < new_size) { - osmium::util::resize_file(fd(), new_size); - } - } - - }; // class DataFile - - } // namespace util - -} // namespace osmium - - -#endif // OSMIUM_UTIL_DATA_FILE_HPP diff --git a/include/osmium/util/delta.hpp b/include/osmium/util/delta.hpp index 0c77e5242..558a1d4f9 100644 --- a/include/osmium/util/delta.hpp +++ b/include/osmium/util/delta.hpp @@ -37,6 +37,8 @@ DEALINGS IN THE SOFTWARE. #include #include +#include + namespace osmium { namespace util { @@ -44,25 +46,39 @@ namespace osmium { /** * Helper class for delta encoding. */ - template + template class DeltaEncode { - T m_value; + static_assert(std::is_integral::value, + "DeltaEncode value type must be some integer"); + + static_assert(std::is_integral::value && std::is_signed::value, + "DeltaEncode delta type must be some signed integer"); + + TValue m_value; public: - DeltaEncode(T value = 0) : + using value_type = TValue; + using delta_type = TDelta; + + explicit DeltaEncode(TValue value = 0) : m_value(value) { } - void clear() { + void clear() noexcept { m_value = 0; } - T update(T new_value) { + TValue value() const noexcept { + return m_value; + } + + TDelta update(TValue new_value) noexcept { using std::swap; swap(m_value, new_value); - return m_value - new_value; + return static_cast_with_assert(m_value) - + static_cast_with_assert(new_value); } }; // class DeltaEncode @@ -70,52 +86,63 @@ namespace osmium { /** * Helper class for delta decoding. */ - template + template class DeltaDecode { - T m_value; + static_assert(std::is_integral::value, + "DeltaDecode value type must be some integer"); + + static_assert(std::is_integral::value && std::is_signed::value, + "DeltaDecode delta type must be some signed integer"); + + TValue m_value; public: + using value_type = TValue; + using delta_type = TDelta; + DeltaDecode() : m_value(0) { } - void clear() { + void clear() noexcept { m_value = 0; } - T update(T delta) { - m_value += delta; + TValue update(TDelta delta) noexcept { + m_value = static_cast_with_assert( + static_cast_with_assert(m_value) + delta); return m_value; } }; // class DeltaDecode - template + template class DeltaEncodeIterator : public std::iterator { - typedef TValue value_type; - TBaseIterator m_it; TBaseIterator m_end; - value_type m_delta; - DeltaEncode m_value; TTransform m_trans; + DeltaEncode m_value; + TDelta m_delta; public: + using value_type = TValue; + using delta_type = TDelta; + DeltaEncodeIterator(TBaseIterator first, TBaseIterator last, TTransform& trans) : m_it(first), m_end(last), - m_delta(m_trans(m_it)), - m_value(m_delta), - m_trans(trans) { + m_trans(trans), + m_value(m_it != m_end ? m_trans(m_it) : 0), + m_delta(static_cast_with_assert(m_value.value())) { } DeltaEncodeIterator& operator++() { - if (m_it != m_end) { - m_delta = m_value.update(m_trans(++m_it)); + if (++m_it != m_end) { + m_delta = m_value.update(m_trans(m_it)); } return *this; } @@ -126,7 +153,7 @@ namespace osmium { return tmp; } - value_type operator*() { + TDelta operator*() { return m_delta; } diff --git a/include/osmium/util/double.hpp b/include/osmium/util/double.hpp index 85a250807..e0701394c 100644 --- a/include/osmium/util/double.hpp +++ b/include/osmium/util/double.hpp @@ -68,8 +68,12 @@ namespace osmium { #endif assert(len > 0 && len < max_double_length); - while (buffer[len-1] == '0') --len; - if (buffer[len-1] == '.') --len; + while (buffer[len-1] == '0') { + --len; + } + if (buffer[len-1] == '.') { + --len; + } return std::copy_n(buffer, len, iterator); } diff --git a/include/osmium/util/file.hpp b/include/osmium/util/file.hpp index 461f4e642..39e01af83 100644 --- a/include/osmium/util/file.hpp +++ b/include/osmium/util/file.hpp @@ -52,6 +52,8 @@ DEALINGS IN THE SOFTWARE. # define ftruncate _chsize_s #endif +#include + namespace osmium { namespace util { @@ -92,7 +94,7 @@ namespace osmium { * @throws std::system_error If ftruncate(2) call failed */ inline void resize_file(int fd, size_t new_size) { - if (::ftruncate(fd, new_size) != 0) { + if (::ftruncate(fd, static_cast_with_assert(new_size)) != 0) { throw std::system_error(errno, std::system_category(), "ftruncate failed"); } } @@ -108,7 +110,7 @@ namespace osmium { return si.dwPageSize; #else // Unix implementation - return ::sysconf(_SC_PAGESIZE); + return size_t(::sysconf(_SC_PAGESIZE)); #endif } diff --git a/include/osmium/util/memory_mapping.hpp b/include/osmium/util/memory_mapping.hpp index e48aff282..b187c3c80 100644 --- a/include/osmium/util/memory_mapping.hpp +++ b/include/osmium/util/memory_mapping.hpp @@ -38,11 +38,13 @@ DEALINGS IN THE SOFTWARE. #include #include +#include #include #ifndef _WIN32 # include #else +# include # include # include # include @@ -85,6 +87,9 @@ namespace osmium { * On Unix systems this wraps the mmap(), munmap(), and the mremap() * system calls. On Windows it wraps the CreateFileMapping(), * CloseHandle(), MapViewOfFile(), and UnmapViewOfFile() functions. + * + * On Windows the file will be set to binary mode before the memory + * mapping. */ class MemoryMapping { @@ -169,7 +174,8 @@ private: * created, otherwise a mapping based on the file descriptor will * be created. * - * @pre size > 0 or mode == write_shared oder write_private + * @pre @code size > 0 @endcode or + * @code mode == write_shared || mode == write_private @endcode * * @param size Size of the mapping in bytes * @param mode Mapping mode: readonly, or writable (shared or private) @@ -179,8 +185,12 @@ private: */ MemoryMapping(size_t size, mapping_mode mode, int fd=-1, off_t offset=0); - /// DEPRECATED: For backwards compatibility - MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0) : + /** + * @deprecated + * For backwards compatibility only. Use the constructor taking + * a mapping_mode as second argument instead. + */ + OSMIUM_DEPRECATED MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0) : MemoryMapping(size, writable ? mapping_mode::write_shared : mapping_mode::readonly, fd, offset) { } @@ -209,7 +219,7 @@ private: try { unmap(); } catch (std::system_error&) { - // ignore + // Ignore any exceptions because destructor must not throw. } } @@ -228,8 +238,9 @@ private: * systems it will unmap and remap the memory. This can only be * done for file-based mappings, not anonymous mappings! * - * @param new_size Number of bytes to resize to - * @throws std::system_error if the remapping fails + * @param new_size Number of bytes to resize to (must be > 0). + * + * @throws std::system_error if the remapping fails. */ void resize(size_t new_size); @@ -237,7 +248,7 @@ private: * In a boolean context a MemoryMapping is true when it is a valid * existing mapping. */ - operator bool() const noexcept { + explicit operator bool() const noexcept { return is_valid(); } @@ -349,8 +360,12 @@ private: m_mapping(sizeof(T) * size, mode, fd, sizeof(T) * offset) { } - /// DEPRECATED: For backwards compatibility - TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset = 0) : + /** + * @deprecated + * For backwards compatibility only. Use the constructor taking + * a mapping_mode as second argument instead. + */ + OSMIUM_DEPRECATED TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset = 0) : m_mapping(sizeof(T) * size, writable ? MemoryMapping::mapping_mode::write_shared : MemoryMapping::mapping_mode::readonly, fd, sizeof(T) * offset) { } @@ -375,7 +390,7 @@ private: * Releases the mapping by calling unmap(). Will never throw. * Call unmap() instead if you want to be notified of any error. */ - ~TypedMemoryMapping() = default; + ~TypedMemoryMapping() noexcept = default; /** * Unmap a mapping. If the mapping is not valid, it will do @@ -405,7 +420,7 @@ private: * In a boolean context a TypedMemoryMapping is true when it is * a valid existing mapping. */ - operator bool() const noexcept { + explicit operator bool() const noexcept { return !!m_mapping; } @@ -655,6 +670,9 @@ inline HANDLE osmium::util::MemoryMapping::get_handle() const noexcept { } inline HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept { + if (m_fd != -1) { + _setmode(m_fd, _O_BINARY); + } return CreateFileMapping(get_handle(), nullptr, get_protection(), osmium::util::dword_hi(static_cast(m_size) + m_offset), osmium::util::dword_lo(static_cast(m_size) + m_offset), nullptr); } diff --git a/include/osmium/util/options.hpp b/include/osmium/util/options.hpp index fea075230..1019c8bbf 100644 --- a/include/osmium/util/options.hpp +++ b/include/osmium/util/options.hpp @@ -47,46 +47,65 @@ namespace osmium { * as a base class. Options are stored and retrieved by key using the * different set() and get() methods. * + * Both keys and values are stored as strings. The values "true", + * "yes", "false", and "no" are interpreted as boolean values in some + * functions. + * * You can iterate over all set options. Dereferencing an iterator * yields a std::pair of the key and value strings. */ class Options { - typedef std::map option_map; + using option_map = std::map; option_map m_options; public: - typedef option_map::iterator iterator; - typedef option_map::const_iterator const_iterator; - typedef option_map::value_type value_type; + using iterator = option_map::iterator; + using const_iterator = option_map::const_iterator; + using value_type = option_map::value_type; + /** + * Construct empty option set. + */ Options() = default; + /** + * Construct option set from initializer list: + * @code + * Options options{ { "foo", "true" }, { "bar", "17" } }; + * @endcode + */ explicit Options(const std::initializer_list& values) : m_options(values) { } - Options(const Options&) = default; - Options& operator=(const Options&) = default; - - Options(Options&&) = default; - Options& operator=(Options&&) = default; - - ~Options() = default; - + /** + * Set option 'key' to 'value'. + */ void set(const std::string& key, const std::string& value) { m_options[key] = value; } + /** + * Set option 'key' to 'value'. + */ void set(const std::string& key, const char* value) { m_options[key] = value; } + /** + * Set option 'key' to 'value'. + */ void set(const std::string& key, bool value) { m_options[key] = value ? "true" : "false"; } + /** + * Set option from string in the form 'key=value'. If the string + * contains no equal sign, the whole string is the key and it will + * be set to "true". + */ void set(std::string data) { size_t pos = data.find_first_of('='); if (pos == std::string::npos) { @@ -99,7 +118,7 @@ namespace osmium { } /** - * Get value of "key" option. If not set the default_value (or + * Get value of "key" option. If not set, the default_value (or * empty string) is returned. */ std::string get(const std::string& key, const std::string& default_value="") const noexcept { @@ -112,36 +131,67 @@ namespace osmium { /** * Is this option set to a true value ("true" or "yes")? + * Will return false if the value is unset. */ bool is_true(const std::string& key) const noexcept { std::string value = get(key); return (value == "true" || value == "yes"); } + /** + * Is this option not set to a false value ("false" or "no")? + * Will return true if the value is unset. + */ + bool is_not_false(const std::string& key) const noexcept { + std::string value = get(key); + return !(value == "false" || value == "no"); + } + + /** + * The number of options set. + */ size_t size() const noexcept { return m_options.size(); } + /** + * Returns an iterator to the beginning. + */ iterator begin() noexcept { return m_options.begin(); } + /** + * Returns an iterator to the end. + */ iterator end() noexcept { return m_options.end(); } + /** + * Returns an iterator to the beginning. + */ const_iterator begin() const noexcept { return m_options.cbegin(); } + /** + * Returns an iterator to the end. + */ const_iterator end() const noexcept { return m_options.cend(); } + /** + * Returns an iterator to the beginning. + */ const_iterator cbegin() const noexcept { return m_options.cbegin(); } + /** + * Returns a iterator to the end. + */ const_iterator cend() const noexcept { return m_options.cend(); } diff --git a/include/osmium/visitor.hpp b/include/osmium/visitor.hpp index 0250f11d4..c76eb17df 100644 --- a/include/osmium/visitor.hpp +++ b/include/osmium/visitor.hpp @@ -35,6 +35,7 @@ DEALINGS IN THE SOFTWARE. #include +#include #include // IWYU pragma: keep #include #include @@ -43,22 +44,16 @@ DEALINGS IN THE SOFTWARE. namespace osmium { - class TagList; - class WayNodeList; - class RelationMemberList; - class OuterRing; - class InnerRing; - namespace memory { class Item; - } + } // namespace memory namespace detail { template using ConstIfConst = typename std::conditional::value, typename std::add_const::type, U>::type; - template + template inline void apply_item_recurse(TItem& item, THandler& handler) { switch (item.type()) { case osmium::item_type::undefined: @@ -98,10 +93,13 @@ namespace osmium { case osmium::item_type::inner_ring: handler.inner_ring(static_cast&>(item)); break; + case osmium::item_type::changeset_discussion: + handler.changeset_discussion(static_cast&>(item)); + break; } } - template + template inline void apply_item_recurse(const osmium::OSMEntity& item, THandler& handler) { switch (item.type()) { case osmium::item_type::node: @@ -128,7 +126,7 @@ namespace osmium { } } - template + template inline void apply_item_recurse(osmium::OSMEntity& item, THandler& handler) { switch (item.type()) { case osmium::item_type::node: @@ -155,7 +153,7 @@ namespace osmium { } } - template + template inline void apply_item_recurse(const osmium::OSMObject& item, THandler& handler) { switch (item.type()) { case osmium::item_type::node: @@ -179,7 +177,7 @@ namespace osmium { } } - template + template inline void apply_item_recurse(osmium::OSMObject& item, THandler& handler) { switch (item.type()) { case osmium::item_type::node: @@ -203,18 +201,18 @@ namespace osmium { } } - template + template inline void apply_item_recurse(TItem& item, THandler& handler, TRest&... more) { apply_item_recurse(item, handler); apply_item_recurse(item, more...); } - template + template inline void flush_recurse(THandler& handler) { handler.flush(); } - template + template inline void flush_recurse(THandler& handler, TRest&... more) { flush_recurse(handler); flush_recurse(more...); @@ -222,17 +220,17 @@ namespace osmium { } // namespace detail - template + template inline void apply_item(const osmium::memory::Item& item, THandlers&... handlers) { detail::apply_item_recurse(item, handlers...); } - template + template inline void apply_item(osmium::memory::Item& item, THandlers&... handlers) { detail::apply_item_recurse(item, handlers...); } - template + template inline void apply(TIterator it, TIterator end, THandlers&... handlers) { for (; it != end; ++it) { detail::apply_item_recurse(*it, handlers...); @@ -240,12 +238,12 @@ namespace osmium { detail::flush_recurse(handlers...); } - template + template inline void apply(TContainer& c, THandlers&... handlers) { apply(std::begin(c), std::end(c), handlers...); } - template + template inline void apply(const osmium::memory::Buffer& buffer, THandlers&... handlers) { apply(buffer.cbegin(), buffer.cend(), handlers...); } diff --git a/include/protozero/byteswap.hpp b/include/protozero/byteswap.hpp index d019c28c5..a018c1c17 100644 --- a/include/protozero/byteswap.hpp +++ b/include/protozero/byteswap.hpp @@ -10,30 +10,51 @@ documentation. *****************************************************************************/ +/** + * @file byteswap.hpp + * + * @brief Contains functions to swap bytes in values (for different endianness). + */ + +#include #include +#include + namespace protozero { +/** + * Swap N byte value between endianness formats. This template function must + * be specialized to actually work. + */ template inline void byteswap(const char* /*data*/, char* /*result*/) { - assert(false); -} - -template <> -inline void byteswap<1>(const char* data, char* result) { - result[0] = data[0]; + static_assert(N == 1, "Can only swap 4 or 8 byte values"); } +/** + * Swap 4 byte value (int32_t, uint32_t, float) between endianness formats. + */ template <> inline void byteswap<4>(const char* data, char* result) { +#ifdef PROTOZERO_USE_BUILTIN_BSWAP + *reinterpret_cast(result) = __builtin_bswap32(*reinterpret_cast(data)); +#else result[3] = data[0]; result[2] = data[1]; result[1] = data[2]; result[0] = data[3]; +#endif } +/** + * Swap 8 byte value (int64_t, uint64_t, double) between endianness formats. + */ template <> inline void byteswap<8>(const char* data, char* result) { +#ifdef PROTOZERO_USE_BUILTIN_BSWAP + *reinterpret_cast(result) = __builtin_bswap64(*reinterpret_cast(data)); +#else result[7] = data[0]; result[6] = data[1]; result[5] = data[2]; @@ -42,6 +63,7 @@ inline void byteswap<8>(const char* data, char* result) { result[2] = data[5]; result[1] = data[6]; result[0] = data[7]; +#endif } } // end namespace protozero diff --git a/include/protozero/config.hpp b/include/protozero/config.hpp new file mode 100644 index 000000000..4086994fb --- /dev/null +++ b/include/protozero/config.hpp @@ -0,0 +1,57 @@ +#ifndef PROTOZERO_CONFIG_HPP +#define PROTOZERO_CONFIG_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +#include + +/** + * @file config.hpp + * + * @brief Contains macro checks for different configurations. + */ + +#define PROTOZERO_LITTLE_ENDIAN 1234 +#define PROTOZERO_BIG_ENDIAN 4321 + +// Find out which byte order the machine has. +#if defined(__BYTE_ORDER) +# if (__BYTE_ORDER == __LITTLE_ENDIAN) +# define PROTOZERO_BYTE_ORDER PROTOZERO_LITTLE_ENDIAN +# endif +# if (__BYTE_ORDER == __BIG_ENDIAN) +# define PROTOZERO_BYTE_ORDER PROTOZERO_BIG_ENDIAN +# endif +#else +// This probably isn't a very good default, but might do until we figure +// out something better. +# define PROTOZERO_BYTE_ORDER PROTOZERO_LITTLE_ENDIAN +#endif + +// On some ARM machines and depending on compiler settings access to unaligned +// floating point values will result in a SIGBUS. Do not use the bare pointers +// in this case. +#if PROTOZERO_BYTE_ORDER == PROTOZERO_LITTLE_ENDIAN +# if !defined(__arm__) && !defined(_M_ARM) +# define PROTOZERO_USE_BARE_POINTER_FOR_PACKED_FIXED +# endif +#endif + +// Check whether __builtin_bswap is available +#if defined(__GNUC__) || defined(__clang__) +# define PROTOZERO_USE_BUILTIN_BSWAP +#endif + +// Wrapper for assert() used for testing +#ifndef protozero_assert +# define protozero_assert(x) assert(x) +#endif + +#endif // PROTOZERO_CONFIG_HPP diff --git a/include/protozero/pbf_builder.hpp b/include/protozero/pbf_builder.hpp index d49a7ba59..063fa9cb9 100644 --- a/include/protozero/pbf_builder.hpp +++ b/include/protozero/pbf_builder.hpp @@ -10,6 +10,12 @@ documentation. *****************************************************************************/ +/** + * @file pbf_builder.hpp + * + * @brief Contains the pbf_builder template class. + */ + #include #include @@ -17,10 +23,22 @@ documentation. namespace protozero { +/** + * The pbf_builder is used to write PBF formatted messages into a buffer. It + * is based on the pbf_writer class and has all the same methods. The + * difference is that whereever the pbf_writer class takes an integer tag, + * this template class takes a tag of the template type T. + * + * Almost all methods in this class can throw an std::bad_alloc exception if + * the std::string used as a buffer wants to resize. + * + * Read the tutorial to understand how this class is used. + */ template class pbf_builder : public pbf_writer { - static_assert(std::is_same::type>::value, "T must be enum with underlying type protozero::pbf_tag_type"); + static_assert(std::is_same::type>::value, + "T must be enum with underlying type protozero::pbf_tag_type"); public: @@ -35,6 +53,7 @@ public: pbf_writer(parent_writer, pbf_tag_type(tag)) { } +/// @cond INTERNAL #define PROTOZERO_WRITER_WRAP_ADD_SCALAR(name, type) \ inline void add_##name(T tag, type value) { \ pbf_writer::add_##name(pbf_tag_type(tag), value); \ @@ -55,6 +74,9 @@ public: PROTOZERO_WRITER_WRAP_ADD_SCALAR(float, float) PROTOZERO_WRITER_WRAP_ADD_SCALAR(double, double) +#undef PROTOZERO_WRITER_WRAP_ADD_SCALAR +/// @endcond + inline void add_bytes(T tag, const char* value, size_t size) { pbf_writer::add_bytes(pbf_tag_type(tag), value, size); } @@ -83,6 +105,7 @@ public: pbf_writer::add_message(pbf_tag_type(tag), value); } +/// @cond INTERNAL #define PROTOZERO_WRITER_WRAP_ADD_PACKED(name) \ template \ inline void add_packed_##name(T tag, InputIterator first, InputIterator last) { \ @@ -104,6 +127,9 @@ public: PROTOZERO_WRITER_WRAP_ADD_PACKED(float) PROTOZERO_WRITER_WRAP_ADD_PACKED(double) +#undef PROTOZERO_WRITER_WRAP_ADD_PACKED +/// @endcond + }; } // end namespace protozero diff --git a/include/protozero/pbf_message.hpp b/include/protozero/pbf_message.hpp index af29a00f4..7fef06f81 100644 --- a/include/protozero/pbf_message.hpp +++ b/include/protozero/pbf_message.hpp @@ -10,6 +10,12 @@ documentation. *****************************************************************************/ +/** + * @file pbf_message.hpp + * + * @brief Contains the pbf_message class. + */ + #include #include @@ -17,6 +23,44 @@ documentation. namespace protozero { +/** + * This class represents a protobuf message. Either a top-level message or + * a nested sub-message. Top-level messages can be created from any buffer + * with a pointer and length: + * + * @code + * enum class Message : protozero::pbf_tag_type { + * ... + * }; + * + * std::string buffer; + * // fill buffer... + * pbf_message message(buffer.data(), buffer.size()); + * @endcode + * + * Sub-messages are created using get_message(): + * + * @code + * enum class SubMessage : protozero::pbf_tag_type { + * ... + * }; + * + * pbf_message message(...); + * message.next(); + * pbf_message submessage = message.get_message(); + * @endcode + * + * All methods of the pbf_message class except get_bytes() and get_string() + * provide the strong exception guarantee, ie they either succeed or do not + * change the pbf_message object they are called on. Use the get_data() method + * instead of get_bytes() or get_string(), if you need this guarantee. + * + * This template class is based on the pbf_reader class and has all the same + * methods. The difference is that whereever the pbf_reader class takes an + * integer tag, this template class takes a tag of the template type T. + * + * Read the tutorial to understand how this class is used. + */ template class pbf_message : public pbf_reader { diff --git a/include/protozero/pbf_reader.hpp b/include/protozero/pbf_reader.hpp index 1c5ed0d70..aced901c6 100644 --- a/include/protozero/pbf_reader.hpp +++ b/include/protozero/pbf_reader.hpp @@ -16,7 +16,6 @@ documentation. * @brief Contains the pbf_reader class. */ -#include #include #include #include @@ -24,19 +23,15 @@ documentation. #include #include -#include +#include #include +#include #include -#if __BYTE_ORDER != __LITTLE_ENDIAN +#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN # include #endif -/// Wrapper for assert() used for testing -#ifndef protozero_assert -# define protozero_assert(x) assert(x) -#endif - namespace protozero { /** @@ -77,19 +72,27 @@ class pbf_reader { // The tag of the current field. pbf_tag_type m_tag = 0; + // Copy N bytes from src to dest on little endian machines, on big endian + // swap the bytes in the process. + template + static void copy_or_byteswap(const char* src, void* dest) noexcept { +#if PROTOZERO_BYTE_ORDER == PROTOZERO_LITTLE_ENDIAN + memcpy(dest, src, N); +#else + byteswap(src, reinterpret_cast(dest)); +#endif + } + template inline T get_fixed() { T result; skip_bytes(sizeof(T)); -#if __BYTE_ORDER == __LITTLE_ENDIAN - memcpy(&result, m_data - sizeof(T), sizeof(T)); -#else - byteswap(m_data - sizeof(T), reinterpret_cast(&result)); -#endif + copy_or_byteswap(m_data - sizeof(T), &result); return result; } -#if __BYTE_ORDER == __LITTLE_ENDIAN +#ifdef PROTOZERO_USE_BARE_POINTER_FOR_PACKED_FIXED + template inline std::pair packed_fixed() { protozero_assert(tag() != 0 && "call next() before accessing field value"); @@ -128,7 +131,7 @@ class pbf_reader { T operator*() { T result; - byteswap(m_data, reinterpret_cast(&result)); + copy_or_byteswap(m_data , &result); return result; } @@ -161,6 +164,7 @@ class pbf_reader { return std::make_pair(const_fixed_iterator(m_data-len, m_data), const_fixed_iterator(m_data, m_data)); } + #endif template inline T get_varint(); @@ -866,8 +870,16 @@ bool pbf_reader::next() { protozero_assert(((m_tag > 0 && m_tag < 19000) || (m_tag > 19999 && m_tag <= ((1 << 29) - 1))) && "tag out of range"); m_wire_type = pbf_wire_type(value & 0x07); -// XXX do we want this check? or should it throw an exception? -// protozero_assert((m_wire_type <=2 || m_wire_type == 5) && "illegal wire type"); + switch (m_wire_type) { + case pbf_wire_type::varint: + case pbf_wire_type::fixed64: + case pbf_wire_type::length_delimited: + case pbf_wire_type::fixed32: + break; + default: + throw unknown_pbf_wire_type_exception(); + } + return true; } diff --git a/include/protozero/pbf_writer.hpp b/include/protozero/pbf_writer.hpp index 53cbfdf1c..2b78cb889 100644 --- a/include/protozero/pbf_writer.hpp +++ b/include/protozero/pbf_writer.hpp @@ -16,7 +16,6 @@ documentation. * @brief Contains the pbf_writer class. */ -#include #include #include #include @@ -24,18 +23,14 @@ documentation. #include #include +#include #include #include -#if __BYTE_ORDER != __LITTLE_ENDIAN +#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN # include #endif -/// Wrapper for assert() used for testing -#ifndef protozero_assert -# define protozero_assert(x) assert(x) -#endif - namespace protozero { /** @@ -71,7 +66,7 @@ class pbf_writer { inline void add_fixed(T value) { protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); protozero_assert(m_data); -#if __BYTE_ORDER == __LITTLE_ENDIAN +#if PROTOZERO_BYTE_ORDER == PROTOZERO_LITTLE_ENDIAN m_data->append(reinterpret_cast(&value), sizeof(T)); #else auto size = m_data->size(); @@ -229,7 +224,9 @@ public: */ inline void add_bool(pbf_tag_type tag, bool value) { add_field(tag, pbf_wire_type::varint); - add_fixed(value); + protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); + protozero_assert(m_data); + m_data->append(1, value); } /** @@ -378,7 +375,7 @@ public: inline void add_bytes(pbf_tag_type tag, const char* value, size_t size) { protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); protozero_assert(m_data); - assert(size <= std::numeric_limits::max()); + protozero_assert(size <= std::numeric_limits::max()); add_length_varint(tag, pbf_length_type(size)); m_data->append(value, size); } diff --git a/include/protozero/varint.hpp b/include/protozero/varint.hpp index bc9c3296d..27536fd39 100644 --- a/include/protozero/varint.hpp +++ b/include/protozero/varint.hpp @@ -16,10 +16,6 @@ documentation. * @brief Contains low-level varint and zigzag encoding and decoding functions. */ -#if __BYTE_ORDER != __LITTLE_ENDIAN -# error "This code only works on little endian machines." -#endif - #include #include diff --git a/include/protozero/version.hpp b/include/protozero/version.hpp new file mode 100644 index 000000000..4f129acc6 --- /dev/null +++ b/include/protozero/version.hpp @@ -0,0 +1,22 @@ +#ifndef PROTOZERO_VERSION_HPP +#define PROTOZERO_VERSION_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +#define PROTOZERO_VERSION_MAJOR 1 +#define PROTOZERO_VERSION_MINOR 2 +#define PROTOZERO_VERSION_PATCH 3 + +#define PROTOZERO_VERSION_CODE (PROTOZERO_VERSION_MAJOR * 10000 + PROTOZERO_VERSION_MINOR * 100 + PROTOZERO_VERSION_PATCH) + +#define PROTOZERO_VERSION_STRING "1.2.3" + + +#endif // PROTOZERO_VERSION_HPP diff --git a/scripts/travis_install.sh b/scripts/travis_install.sh deleted file mode 100755 index 119e9fd15..000000000 --- a/scripts/travis_install.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh -# -# travis_install.sh -# - -if [ "$TRAVIS_OS_NAME" = "osx" ]; then - - brew install google-sparsehash || true - - brew install --without-python boost || true - - # workaround for gdal homebrew problem - brew remove gdal - brew install gdal - -fi - -cd .. -git clone --quiet --depth 1 https://github.com/osmcode/osm-testdata.git - diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh deleted file mode 100755 index 75b3b3657..000000000 --- a/scripts/travis_script.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -# -# travis_script.sh -# - -mkdir build -cd build - -# GCC ignores the pragmas in the code that disable the "return-type" warning -# selectively, so use this workaround. -if [ "${CXX}" = "g++" ]; then - WORKAROUND="-DCMAKE_CXX_FLAGS=-Wno-return-type" -else - WORKAROUND="" -fi - -if [ "${CXX}" = "g++" ]; then - CXX=g++-4.8 - CC=gcc-4.8 -fi - -cmake -LA \ - -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ - ${WORKAROUND} \ - .. - -make VERBOSE=1 -ctest --output-on-failure - diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 00474577a..f57416120 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -106,6 +106,7 @@ add_unit_test(basic test_timestamp) add_unit_test(basic test_types_from_string) add_unit_test(basic test_way) +add_unit_test(buffer test_buffer_basics) add_unit_test(buffer test_buffer_node) add_unit_test(buffer test_buffer_purge) @@ -133,9 +134,15 @@ add_unit_test(index test_id_to_location ENABLE_IF ${SPARSEHASH_FOUND}) add_unit_test(io test_bzip2 ENABLE_IF ${BZIP2_FOUND} LIBS ${BZIP2_LIBRARIES}) add_unit_test(io test_file_formats) -add_unit_test(io test_reader LIBS "${OSMIUM_XML_LIBRARIES}") +add_unit_test(io test_reader LIBS "${OSMIUM_XML_LIBRARIES};${OSMIUM_PBF_LIBRARIES}") +add_unit_test(io test_reader_with_mock_decompression ENABLE_IF ${Threads_FOUND} LIBS ${OSMIUM_XML_LIBRARIES}) +add_unit_test(io test_reader_with_mock_parser ENABLE_IF ${Threads_FOUND} LIBS ${CMAKE_THREAD_LIBS_INIT}) +add_unit_test(io test_output_utils) add_unit_test(io test_output_iterator ENABLE_IF ${Threads_FOUND} LIBS ${CMAKE_THREAD_LIBS_INIT}) add_unit_test(io test_string_table) +add_unit_test(io test_writer ENABLE_IF ${Threads_FOUND} LIBS ${OSMIUM_XML_LIBRARIES}) +add_unit_test(io test_writer_with_mock_compression ENABLE_IF ${Threads_FOUND} LIBS ${OSMIUM_XML_LIBRARIES}) +add_unit_test(io test_writer_with_mock_encoder ENABLE_IF ${Threads_FOUND} LIBS ${OSMIUM_XML_LIBRARIES}) add_unit_test(tags test_filter) add_unit_test(tags test_operators) @@ -144,7 +151,6 @@ add_unit_test(tags test_tag_list) add_unit_test(thread test_pool ENABLE_IF ${Threads_FOUND} LIBS ${CMAKE_THREAD_LIBS_INIT}) add_unit_test(util test_cast_with_assert) -add_unit_test(util test_data_file) add_unit_test(util test_delta) add_unit_test(util test_double) add_unit_test(util test_file) diff --git a/test/data-tests/CMakeLists.txt b/test/data-tests/CMakeLists.txt index 89aead98e..a36c31cfd 100644 --- a/test/data-tests/CMakeLists.txt +++ b/test/data-tests/CMakeLists.txt @@ -88,11 +88,11 @@ set_tests_properties(testdata-overview PROPERTIES # #----------------------------------------------------------------------------- -find_package(Ruby 1.9) +find_program(RUBY ruby) find_package(Gem COMPONENTS json) find_program(SPATIALITE spatialite) -if(RUBY_FOUND AND GEM_json_FOUND AND SPATIALITE) +if(RUBY AND GEM_json_FOUND AND SPATIALITE) add_executable(testdata-multipolygon testdata-multipolygon.cpp) target_link_libraries(testdata-multipolygon ${OSMIUM_XML_LIBRARIES} @@ -102,7 +102,7 @@ if(RUBY_FOUND AND GEM_json_FOUND AND SPATIALITE) add_test(NAME testdata-multipolygon COMMAND ${CMAKE_COMMAND} -D OSM_TESTDATA=${OSM_TESTDATA} - -D RUBY=${RUBY_EXECUTABLE} + -D RUBY=${RUBY} -P ${CMAKE_CURRENT_SOURCE_DIR}/run-testdata-multipolygon.cmake) set_tests_properties(testdata-multipolygon PROPERTIES LABELS "data;slow") diff --git a/test/data-tests/testdata-multipolygon.cpp b/test/data-tests/testdata-multipolygon.cpp index 0fd0d9849..cf4fc521a 100644 --- a/test/data-tests/testdata-multipolygon.cpp +++ b/test/data-tests/testdata-multipolygon.cpp @@ -3,6 +3,8 @@ #include #include +#include + #include #include @@ -41,10 +43,9 @@ inline tagmap_type create_map(const osmium::TagList& taglist) { class TestHandler : public osmium::handler::Handler { - OGRDataSource* m_data_source; - OGRLayer* m_layer_point; - OGRLayer* m_layer_linestring; - OGRLayer* m_layer_polygon; + gdalcpp::Layer m_layer_point; + gdalcpp::Layer m_layer_lines; + gdalcpp::Layer m_layer_mpoly; osmium::geom::OGRFactory<> m_ogr_factory; osmium::geom::WKTFactory<> m_wkt_factory; @@ -55,84 +56,20 @@ class TestHandler : public osmium::handler::Handler { public: - TestHandler(OGRDataSource* data_source) : - m_data_source(data_source), + explicit TestHandler(gdalcpp::Dataset& dataset) : + m_layer_point(dataset, "points", wkbPoint), + m_layer_lines(dataset, "lines", wkbLineString), + m_layer_mpoly(dataset, "multipolygons", wkbMultiPolygon), m_out("multipolygon-tests.json") { - OGRSpatialReference sparef; - sparef.SetWellKnownGeogCS("WGS84"); + m_layer_point.add_field("id", OFTReal, 10); + m_layer_point.add_field("type", OFTString, 30); - /**************/ + m_layer_lines.add_field("id", OFTReal, 10); + m_layer_lines.add_field("type", OFTString, 30); - m_layer_point = m_data_source->CreateLayer("points", &sparef, wkbPoint, nullptr); - if (!m_layer_point) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_point_field_id("id", OFTReal); - layer_point_field_id.SetWidth(10); - - if (m_layer_point->CreateField(&layer_point_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_point_field_type("type", OFTString); - layer_point_field_type.SetWidth(30); - - if (m_layer_point->CreateField(&layer_point_field_type) != OGRERR_NONE) { - std::cerr << "Creating type field failed.\n"; - exit(1); - } - - /**************/ - - m_layer_linestring = m_data_source->CreateLayer("lines", &sparef, wkbLineString, nullptr); - if (!m_layer_linestring) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_linestring_field_id("id", OFTReal); - layer_linestring_field_id.SetWidth(10); - - if (m_layer_linestring->CreateField(&layer_linestring_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_linestring_field_type("type", OFTString); - layer_linestring_field_type.SetWidth(30); - - if (m_layer_linestring->CreateField(&layer_linestring_field_type) != OGRERR_NONE) { - std::cerr << "Creating type field failed.\n"; - exit(1); - } - - /**************/ - - m_layer_polygon = m_data_source->CreateLayer("multipolygons", &sparef, wkbMultiPolygon, nullptr); - if (!m_layer_polygon) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_polygon_field_id("id", OFTInteger); - layer_polygon_field_id.SetWidth(10); - - if (m_layer_polygon->CreateField(&layer_polygon_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_polygon_field_from_type("from_type", OFTString); - layer_polygon_field_from_type.SetWidth(1); - - if (m_layer_polygon->CreateField(&layer_polygon_field_from_type) != OGRERR_NONE) { - std::cerr << "Creating from_type field failed.\n"; - exit(1); - } + m_layer_mpoly.add_field("id", OFTReal, 10); + m_layer_mpoly.add_field("from_type", OFTString, 1); } ~TestHandler() { @@ -140,34 +77,18 @@ public: } void node(const osmium::Node& node) { - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_point->GetLayerDefn()); - std::unique_ptr ogr_point = m_ogr_factory.create_point(node); - feature->SetGeometry(ogr_point.get()); - feature->SetField("id", static_cast(node.id())); - feature->SetField("type", node.tags().get_value_by_key("type")); - - if (m_layer_point->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); + gdalcpp::Feature feature(m_layer_point, m_ogr_factory.create_point(node)); + feature.set_field("id", static_cast(node.id())); + feature.set_field("type", node.tags().get_value_by_key("type")); + feature.add_to_layer(); } void way(const osmium::Way& way) { try { - std::unique_ptr ogr_linestring = m_ogr_factory.create_linestring(way); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_linestring->GetLayerDefn()); - feature->SetGeometry(ogr_linestring.get()); - feature->SetField("id", static_cast(way.id())); - feature->SetField("type", way.tags().get_value_by_key("type")); - - if (m_layer_linestring->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); + gdalcpp::Feature feature(m_layer_lines, m_ogr_factory.create_linestring(way)); + feature.set_field("id", static_cast(way.id())); + feature.set_field("type", way.tags().get_value_by_key("type")); + feature.add_to_layer(); } catch (osmium::geometry_error&) { std::cerr << "Ignoring illegal geometry for way " << way.id() << ".\n"; } @@ -200,10 +121,8 @@ public: m_out << "INVALID\"\n}"; } try { - std::unique_ptr ogr_polygon = m_ogr_factory.create_multipolygon(area); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_polygon->GetLayerDefn()); - feature->SetGeometry(ogr_polygon.get()); - feature->SetField("id", static_cast(area.orig_id())); + gdalcpp::Feature feature(m_layer_mpoly, m_ogr_factory.create_multipolygon(area)); + feature.set_field("id", static_cast(area.orig_id())); std::string from_type; if (area.from_way()) { @@ -211,14 +130,8 @@ public: } else { from_type = "r"; } - feature->SetField("from_type", from_type.c_str()); - - if (m_layer_polygon->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); + feature.set_field("from_type", from_type.c_str()); + feature.add_to_layer(); } catch (osmium::geometry_error&) { std::cerr << "Ignoring illegal geometry for area " << area.id() << " created from " << (area.from_way() ? "way" : "relation") << " with id=" << area.orig_id() << ".\n"; } @@ -228,26 +141,6 @@ public: /* ================================================== */ -OGRDataSource* initialize_database(const std::string& output_format, const std::string& output_filename) { - OGRRegisterAll(); - - OGRSFDriver* driver = OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(output_format.c_str()); - if (!driver) { - std::cerr << output_format << " driver not available.\n"; - exit(1); - } - - CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "FALSE"); - const char* options[] = { "SPATIALITE=TRUE", nullptr }; - OGRDataSource* data_source = driver->CreateDataSource(output_filename.c_str(), const_cast(options)); - if (!data_source) { - std::cerr << "Creation of output file failed.\n"; - exit(1); - } - - return data_source; -} - int main(int argc, char* argv[]) { if (argc != 2) { std::cerr << "Usage: " << argv[0] << " INFILE\n"; @@ -258,9 +151,10 @@ int main(int argc, char* argv[]) { std::string input_filename(argv[1]); std::string output_filename("multipolygon.db"); - OGRDataSource* data_source = initialize_database(output_format, output_filename); + CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "FALSE"); + gdalcpp::Dataset dataset{output_format, output_filename, gdalcpp::SRS{}, { "SPATIALITE=TRUE" }}; - osmium::area::ProblemReporterOGR problem_reporter(data_source); + osmium::area::ProblemReporterOGR problem_reporter(dataset); osmium::area::Assembler::config_type assembler_config(&problem_reporter); assembler_config.enable_debug_output(); osmium::area::MultipolygonCollector collector(assembler_config); @@ -275,7 +169,7 @@ int main(int argc, char* argv[]) { location_handler_type location_handler(index); location_handler.ignore_errors(); - TestHandler test_handler(data_source); + TestHandler test_handler(dataset); std::cerr << "Pass 2...\n"; osmium::io::Reader reader2(input_filename); @@ -284,8 +178,5 @@ int main(int argc, char* argv[]) { })); reader2.close(); std::cerr << "Pass 2 done\n"; - - OGRDataSource::DestroyDataSource(data_source); - OGRCleanupAll(); } diff --git a/test/data-tests/testdata-overview.cpp b/test/data-tests/testdata-overview.cpp index 2d63dc684..43d672dd1 100644 --- a/test/data-tests/testdata-overview.cpp +++ b/test/data-tests/testdata-overview.cpp @@ -2,6 +2,8 @@ #include +#include + #include #include @@ -15,154 +17,53 @@ typedef osmium::handler::NodeLocationsForWays location_handler_type; class TestOverviewHandler : public osmium::handler::Handler { - OGRDataSource* m_data_source; - - OGRLayer* m_layer_nodes; - OGRLayer* m_layer_labels; - OGRLayer* m_layer_ways; + gdalcpp::Layer m_layer_nodes; + gdalcpp::Layer m_layer_labels; + gdalcpp::Layer m_layer_ways; osmium::geom::OGRFactory<> m_factory; public: - TestOverviewHandler(const std::string& driver_name, const std::string& filename) { + explicit TestOverviewHandler(gdalcpp::Dataset& dataset) : + m_layer_nodes(dataset, "nodes", wkbPoint), + m_layer_labels(dataset, "labels", wkbPoint), + m_layer_ways(dataset, "ways", wkbLineString) { - OGRRegisterAll(); + m_layer_nodes.add_field("id", OFTReal, 10); - OGRSFDriver* driver = OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(driver_name.c_str()); - if (!driver) { - std::cerr << driver_name << " driver not available.\n"; - exit(1); - } + m_layer_labels.add_field("id", OFTReal, 10); + m_layer_labels.add_field("label", OFTString, 30); - CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "FALSE"); - const char* options[] = { "SPATIALITE=TRUE", nullptr }; - m_data_source = driver->CreateDataSource(filename.c_str(), const_cast(options)); - if (!m_data_source) { - std::cerr << "Creation of output file failed.\n"; - exit(1); - } - - OGRSpatialReference sparef; - sparef.SetWellKnownGeogCS("WGS84"); - - // nodes layer - - m_layer_nodes = m_data_source->CreateLayer("nodes", &sparef, wkbPoint, nullptr); - if (!m_layer_nodes) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_nodes_field_id("id", OFTReal); - layer_nodes_field_id.SetWidth(10); - - if (m_layer_nodes->CreateField(&layer_nodes_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - // labels layer - - m_layer_labels = m_data_source->CreateLayer("labels", &sparef, wkbPoint, nullptr); - if (!m_layer_labels) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_labels_field_id("id", OFTReal); - layer_labels_field_id.SetWidth(10); - - if (m_layer_labels->CreateField(&layer_labels_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_labels_field_label("label", OFTString); - layer_labels_field_label.SetWidth(30); - - if (m_layer_labels->CreateField(&layer_labels_field_label) != OGRERR_NONE) { - std::cerr << "Creating label field failed.\n"; - exit(1); - } - - // ways layer - - m_layer_ways = m_data_source->CreateLayer("ways", &sparef, wkbLineString, nullptr); - if (!m_layer_ways) { - std::cerr << "Layer creation failed.\n"; - exit(1); - } - - OGRFieldDefn layer_way_field_id("id", OFTReal); - layer_way_field_id.SetWidth(10); - - if (m_layer_ways->CreateField(&layer_way_field_id) != OGRERR_NONE) { - std::cerr << "Creating id field failed.\n"; - exit(1); - } - - OGRFieldDefn layer_way_field_test("test", OFTInteger); - layer_way_field_test.SetWidth(3); - - if (m_layer_ways->CreateField(&layer_way_field_test) != OGRERR_NONE) { - std::cerr << "Creating test field failed.\n"; - exit(1); - } - } - - ~TestOverviewHandler() { - OGRDataSource::DestroyDataSource(m_data_source); - OGRCleanupAll(); + m_layer_ways.add_field("id", OFTReal, 10); + m_layer_ways.add_field("test", OFTInteger, 3); } void node(const osmium::Node& node) { const char* label = node.tags().get_value_by_key("label"); if (label) { - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_labels->GetLayerDefn()); - std::unique_ptr ogr_point = m_factory.create_point(node); - feature->SetGeometry(ogr_point.get()); - feature->SetField("id", static_cast(node.id())); - feature->SetField("label", label); - - if (m_layer_labels->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); + gdalcpp::Feature feature(m_layer_labels, m_factory.create_point(node)); + feature.set_field("id", static_cast(node.id())); + feature.set_field("label", label); + feature.add_to_layer(); } else { - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_nodes->GetLayerDefn()); - std::unique_ptr ogr_point = m_factory.create_point(node); - feature->SetGeometry(ogr_point.get()); - feature->SetField("id", static_cast(node.id())); - - if (m_layer_nodes->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - OGRFeature::DestroyFeature(feature); + gdalcpp::Feature feature(m_layer_nodes, m_factory.create_point(node)); + feature.set_field("id", static_cast(node.id())); + feature.add_to_layer(); } } void way(const osmium::Way& way) { try { - std::unique_ptr ogr_linestring = m_factory.create_linestring(way); - OGRFeature* feature = OGRFeature::CreateFeature(m_layer_ways->GetLayerDefn()); - feature->SetGeometry(ogr_linestring.get()); - feature->SetField("id", static_cast(way.id())); + gdalcpp::Feature feature(m_layer_ways, m_factory.create_linestring(way)); + feature.set_field("id", static_cast(way.id())); const char* test = way.tags().get_value_by_key("test"); if (test) { - feature->SetField("test", test); + feature.set_field("test", test); } - if (m_layer_ways->CreateFeature(feature) != OGRERR_NONE) { - std::cerr << "Failed to create feature.\n"; - exit(1); - } - - OGRFeature::DestroyFeature(feature); + feature.add_to_layer(); } catch (osmium::geometry_error&) { std::cerr << "Ignoring illegal geometry for way " << way.id() << ".\n"; } @@ -183,13 +84,16 @@ int main(int argc, char* argv[]) { std::string output_filename("testdata-overview.db"); ::unlink(output_filename.c_str()); + CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "FALSE"); + gdalcpp::Dataset dataset(output_format, output_filename, gdalcpp::SRS{}, { "SPATIALITE=TRUE" }); + osmium::io::Reader reader(input_filename); index_type index; location_handler_type location_handler(index); location_handler.ignore_errors(); - TestOverviewHandler handler(output_format, output_filename); + TestOverviewHandler handler(dataset); osmium::apply(reader, location_handler, handler); reader.close(); diff --git a/test/data-tests/testdata-xml.cpp b/test/data-tests/testdata-xml.cpp index 8102759d1..b5a0e9028 100644 --- a/test/data-tests/testdata-xml.cpp +++ b/test/data-tests/testdata-xml.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -72,24 +73,27 @@ std::string read_gz_file(const char* test_id, const char* suffix) { header_buffer_type parse_xml(std::string input) { - osmium::thread::Queue input_queue; - osmium::thread::Queue output_queue; + osmium::io::detail::future_string_queue_type input_queue; + osmium::io::detail::future_buffer_queue_type output_queue; std::promise header_promise; - std::atomic done {false}; - input_queue.push(input); - input_queue.push(std::string()); // EOF marker + std::future header_future = header_promise.get_future(); - osmium::io::detail::XMLParser parser(input_queue, output_queue, header_promise, osmium::osm_entity_bits::all, done); - parser(); + osmium::io::detail::add_to_queue(input_queue, std::move(input)); + osmium::io::detail::add_to_queue(input_queue, std::string{}); + + osmium::io::detail::XMLParser parser(input_queue, output_queue, header_promise, osmium::osm_entity_bits::all); + parser.parse(); header_buffer_type result; - result.header = header_promise.get_future().get(); - output_queue.wait_and_pop(result.buffer); + result.header = header_future.get(); + std::future future_buffer; + output_queue.wait_and_pop(future_buffer); + result.buffer = future_buffer.get(); if (result.buffer) { - osmium::memory::Buffer buffer; - output_queue.wait_and_pop(buffer); - assert(!buffer); + std::future future_buffer2; + output_queue.wait_and_pop(future_buffer2); + assert(!future_buffer2.get()); } return result; @@ -534,9 +538,10 @@ TEST_CASE("Reading OSM XML 200") { osmium::io::Header header = reader.header(); REQUIRE(header.get("generator") == "testdata"); - osmium::memory::Buffer buffer = reader.read(); - REQUIRE(0 == buffer.committed()); - REQUIRE(! buffer); + REQUIRE_THROWS({ + reader.read(); + }); + reader.close(); } diff --git a/test/t/area/test_node_ref_segment.cpp b/test/t/area/test_node_ref_segment.cpp index 309768744..3261c24e1 100644 --- a/test/t/area/test_node_ref_segment.cpp +++ b/test/t/area/test_node_ref_segment.cpp @@ -52,6 +52,20 @@ TEST_CASE("NodeRefSegmentClass") { REQUIRE(calculate_intersection(s1, s7) == osmium::Location()); } + SECTION("intersection of very long segments") { + NodeRefSegment s1({ 1, {90.0, 90.0}}, { 2, {-90.0, -90.0}}, nullptr, nullptr); + NodeRefSegment s2({ 1, {-90.0, 90.0}}, { 2, {90.0, -90.0}}, nullptr, nullptr); + REQUIRE(calculate_intersection(s1, s2) == osmium::Location(0.0, 0.0)); + + NodeRefSegment s3({ 1, {-90.0, -90.0}}, { 2, {90.0, 90.0}}, nullptr, nullptr); + NodeRefSegment s4({ 1, {-90.0, 90.0}}, { 2, {90.0, -90.0}}, nullptr, nullptr); + REQUIRE(calculate_intersection(s3, s4) == osmium::Location(0.0, 0.0)); + + NodeRefSegment s5({ 1, {-90.0000001, -90.0}}, { 2, {90.0, 90.0}}, nullptr, nullptr); + NodeRefSegment s6({ 1, {-90.0, 90.0}}, { 2, {90.0, -90.0}}, nullptr, nullptr); + REQUIRE(calculate_intersection(s5, s6) == osmium::Location(0.0, 0.0)); + } + SECTION("to_left_of") { osmium::Location loc { 2.0, 2.0 }; diff --git a/test/t/basic/test_changeset.cpp b/test/t/basic/test_changeset.cpp index fc9f1bdee..d1f3fde87 100644 --- a/test/t/basic/test_changeset.cpp +++ b/test/t/basic/test_changeset.cpp @@ -21,11 +21,13 @@ TEST_CASE("Basic Changeset") { .set_created_at(100) .set_closed_at(200) .set_num_changes(7) + .set_num_comments(3) .set_uid(9); REQUIRE(42 == cs1.id()); REQUIRE(9 == cs1.uid()); REQUIRE(7 == cs1.num_changes()); + REQUIRE(3 == cs1.num_comments()); REQUIRE(true == cs1.closed()); REQUIRE(osmium::Timestamp(100) == cs1.created_at()); REQUIRE(osmium::Timestamp(200) == cs1.closed_at()); @@ -33,7 +35,7 @@ TEST_CASE("Basic Changeset") { REQUIRE(std::string("user") == cs1.user()); crc32.update(cs1); - REQUIRE(crc32().checksum() == 0xf44aff25); + REQUIRE(crc32().checksum() == 0x502e8c0e); osmium::Changeset& cs2 = buffer_add_changeset(buffer, "user", @@ -42,11 +44,13 @@ TEST_CASE("Basic Changeset") { cs2.set_id(43) .set_created_at(120) .set_num_changes(21) + .set_num_comments(osmium::num_comments_type(0)) .set_uid(9); REQUIRE(43 == cs2.id()); REQUIRE(9 == cs2.uid()); REQUIRE(21 == cs2.num_changes()); + REQUIRE(0 == cs2.num_comments()); REQUIRE(false == cs2.closed()); REQUIRE(osmium::Timestamp(120) == cs2.created_at()); REQUIRE(osmium::Timestamp() == cs2.closed_at()); @@ -61,3 +65,59 @@ TEST_CASE("Basic Changeset") { REQUIRE(false == (cs1 >= cs2)); } + +TEST_CASE("Create changeset without helper") { + osmium::memory::Buffer buffer(10 * 1000); + osmium::builder::ChangesetBuilder builder(buffer); + + osmium::Changeset& cs1 = builder.object(); + cs1.set_id(42) + .set_created_at(100) + .set_closed_at(200) + .set_num_changes(7) + .set_num_comments(2) + .set_uid(9); + + builder.add_user("user"); + add_tags(buffer, builder, { + {"key1", "val1"}, + {"key2", "val2"} + }); + + { + osmium::builder::ChangesetDiscussionBuilder disc_builder(buffer, &builder); + disc_builder.add_comment(osmium::Timestamp(300), 10, "user2"); + disc_builder.add_comment_text("foo"); + disc_builder.add_comment(osmium::Timestamp(400), 9, "user"); + disc_builder.add_comment_text("bar"); + } + + buffer.commit(); + + REQUIRE(42 == cs1.id()); + REQUIRE(9 == cs1.uid()); + REQUIRE(7 == cs1.num_changes()); + REQUIRE(2 == cs1.num_comments()); + REQUIRE(true == cs1.closed()); + REQUIRE(osmium::Timestamp(100) == cs1.created_at()); + REQUIRE(osmium::Timestamp(200) == cs1.closed_at()); + REQUIRE(2 == cs1.tags().size()); + REQUIRE(std::string("user") == cs1.user()); + + auto cit = cs1.discussion().begin(); + + REQUIRE(cit != cs1.discussion().end()); + REQUIRE(cit->date() == osmium::Timestamp(300)); + REQUIRE(cit->uid() == 10); + REQUIRE(std::string("user2") == cit->user()); + REQUIRE(std::string("foo") == cit->text()); + + REQUIRE(++cit != cs1.discussion().end()); + REQUIRE(cit->date() == osmium::Timestamp(400)); + REQUIRE(cit->uid() == 9); + REQUIRE(std::string("user") == cit->user()); + REQUIRE(std::string("bar") == cit->text()); + + REQUIRE(++cit == cs1.discussion().end()); +} + diff --git a/test/t/basic/test_crc.cpp b/test/t/basic/test_crc.cpp index aab1013f4..fcd50a13c 100644 --- a/test/t/basic/test_crc.cpp +++ b/test/t/basic/test_crc.cpp @@ -24,6 +24,27 @@ TEST_CASE("CRC of basic datatypes") { REQUIRE(crc32().checksum() == 0x8fe62899); } + SECTION("Int16") { + crc32.update_int16(0x0123U); + crc32.update_int16(0x1234U); + + REQUIRE(crc32().checksum() == 0xda923744); + } + + SECTION("Int32") { + crc32.update_int32(0x01234567UL); + crc32.update_int32(0x12345678UL); + + REQUIRE(crc32().checksum() == 0x9b4e2af3); + } + + SECTION("Int64") { + crc32.update_int64(0x0123456789abcdefULL); + crc32.update_int64(0x123456789abcdef0ULL); + + REQUIRE(crc32().checksum() == 0x6d8b7267); + } + SECTION("String") { const char* str = "foobar"; crc32.update_string(str); diff --git a/test/t/basic/test_node.cpp b/test/t/basic/test_node.cpp index db5b4cd53..9f8b181fe 100644 --- a/test/t/basic/test_node.cpp +++ b/test/t/basic/test_node.cpp @@ -37,12 +37,12 @@ SECTION("node_builder") { REQUIRE(333 == node.changeset()); REQUIRE(21 == node.uid()); REQUIRE(std::string("foo") == node.user()); - REQUIRE(123 == node.timestamp()); + REQUIRE(123 == uint32_t(node.timestamp())); REQUIRE(osmium::Location(3.5, 4.7) == node.location()); REQUIRE(2 == node.tags().size()); crc32.update(node); - REQUIRE(crc32().checksum() == 0xc696802f); + REQUIRE(crc32().checksum() == 0x7dc553f9); node.set_visible(false); REQUIRE(false == node.visible()); @@ -61,7 +61,7 @@ SECTION("node_default_attributes") { REQUIRE(0 == node.changeset()); REQUIRE(0 == node.uid()); REQUIRE(std::string("") == node.user()); - REQUIRE(0 == node.timestamp()); + REQUIRE(0 == uint32_t(node.timestamp())); REQUIRE(osmium::Location() == node.location()); REQUIRE(0 == node.tags().size()); } diff --git a/test/t/basic/test_object_comparisons.cpp b/test/t/basic/test_object_comparisons.cpp index 2bfdcad90..ec9e6fae8 100644 --- a/test/t/basic/test_object_comparisons.cpp +++ b/test/t/basic/test_object_comparisons.cpp @@ -31,20 +31,20 @@ TEST_CASE("Object_Comparisons") { node1.set_version(1); node2.set_id(15); node2.set_version(2); - REQUIRE(true == (node1 < node2)); - REQUIRE(false == (node1 > node2)); + REQUIRE(node1 < node2); + REQUIRE_FALSE(node1 > node2); node1.set_id(20); node1.set_version(1); node2.set_id(20); node2.set_version(2); - REQUIRE(true == (node1 < node2)); - REQUIRE(false == (node1 > node2)); + REQUIRE(node1 < node2); + REQUIRE_FALSE(node1 > node2); node1.set_id(-10); node1.set_version(2); node2.set_id(-15); node2.set_version(1); - REQUIRE(true == (node1 < node2)); - REQUIRE(false == (node1 > node2)); + REQUIRE(node1 < node2); + REQUIRE_FALSE(node1 > node2); } SECTION("order_types") { @@ -122,26 +122,26 @@ TEST_CASE("Object_Comparisons") { const osmium::Way& way = static_cast(*(++it)); const osmium::Relation& relation = static_cast(*(++it)); - REQUIRE(true == (node1 < node2)); - REQUIRE(true == (node2 < way)); - REQUIRE(false == (node2 > way)); - REQUIRE(true == (way < relation)); - REQUIRE(true == (node1 < relation)); + REQUIRE(node1 < node2); + REQUIRE(node2 < way); + REQUIRE_FALSE(node2 > way); + REQUIRE(way < relation); + REQUIRE(node1 < relation); - REQUIRE(true == osmium::object_order_type_id_version()(node1, node2)); - REQUIRE(true == osmium::object_order_type_id_reverse_version()(node2, node1)); - REQUIRE(true == osmium::object_order_type_id_version()(node1, way)); - REQUIRE(true == osmium::object_order_type_id_reverse_version()(node1, way)); + REQUIRE(osmium::object_order_type_id_version()(node1, node2)); + REQUIRE(osmium::object_order_type_id_reverse_version()(node2, node1)); + REQUIRE(osmium::object_order_type_id_version()(node1, way)); + REQUIRE(osmium::object_order_type_id_reverse_version()(node1, way)); - REQUIRE(false == osmium::object_equal_type_id_version()(node1, node2)); - REQUIRE(true == osmium::object_equal_type_id_version()(node2, node3)); + REQUIRE_FALSE(osmium::object_equal_type_id_version()(node1, node2)); + REQUIRE(osmium::object_equal_type_id_version()(node2, node3)); - REQUIRE(true == osmium::object_equal_type_id()(node1, node2)); - REQUIRE(true == osmium::object_equal_type_id()(node2, node3)); + REQUIRE(osmium::object_equal_type_id()(node1, node2)); + REQUIRE(osmium::object_equal_type_id()(node2, node3)); - REQUIRE(false == osmium::object_equal_type_id_version()(node1, way)); - REQUIRE(false == osmium::object_equal_type_id_version()(node1, relation)); - REQUIRE(false == osmium::object_equal_type_id()(node1, relation)); + REQUIRE_FALSE(osmium::object_equal_type_id_version()(node1, way)); + REQUIRE_FALSE(osmium::object_equal_type_id_version()(node1, relation)); + REQUIRE_FALSE(osmium::object_equal_type_id()(node1, relation)); } } diff --git a/test/t/basic/test_relation.cpp b/test/t/basic/test_relation.cpp index fd5c7b4ad..66b201c90 100644 --- a/test/t/basic/test_relation.cpp +++ b/test/t/basic/test_relation.cpp @@ -36,7 +36,7 @@ TEST_CASE("Build relation") { REQUIRE(333 == relation.changeset()); REQUIRE(21 == relation.uid()); REQUIRE(std::string("foo") == relation.user()); - REQUIRE(123 == relation.timestamp()); + REQUIRE(123 == uint32_t(relation.timestamp())); REQUIRE(2 == relation.tags().size()); REQUIRE(3 == relation.members().size()); @@ -61,5 +61,16 @@ TEST_CASE("Build relation") { } crc32.update(relation); - REQUIRE(crc32().checksum() == 0xebcd836d); + REQUIRE(crc32().checksum() == 0x2c2352e); } + +TEST_CASE("Member role too long") { + osmium::memory::Buffer buffer(10000); + + osmium::builder::RelationMemberListBuilder builder(buffer); + + const char role[2000] = ""; + builder.add_member(osmium::item_type::node, 1, role, 1024); + REQUIRE_THROWS(builder.add_member(osmium::item_type::node, 1, role, 1025)); +} + diff --git a/test/t/basic/test_timestamp.cpp b/test/t/basic/test_timestamp.cpp index f015730ea..f80ffcf5f 100644 --- a/test/t/basic/test_timestamp.cpp +++ b/test/t/basic/test_timestamp.cpp @@ -8,34 +8,45 @@ TEST_CASE("Timestamp") { SECTION("can be default initialized to invalid value") { osmium::Timestamp t; - REQUIRE(0 == t); + REQUIRE(0 == uint32_t(t)); REQUIRE("" == t.to_iso()); + REQUIRE_FALSE(t.valid()); } SECTION("invalid value is zero") { osmium::Timestamp t(static_cast(0)); - REQUIRE(0 == t); + REQUIRE(0 == uint32_t(t)); REQUIRE("" == t.to_iso()); + REQUIRE_FALSE(t.valid()); } SECTION("can be initialized from time_t") { osmium::Timestamp t(static_cast(1)); - REQUIRE(1 == t); + REQUIRE(1 == uint32_t(t)); REQUIRE("1970-01-01T00:00:01Z" == t.to_iso()); + REQUIRE(t.valid()); + } + + SECTION("can be initialized from const char*") { + osmium::Timestamp t("2000-01-01T00:00:00Z"); + REQUIRE("2000-01-01T00:00:00Z" == t.to_iso()); + REQUIRE(t.valid()); } SECTION("can be initialized from string") { - osmium::Timestamp t("2000-01-01T00:00:00Z"); + std::string s = "2000-01-01T00:00:00Z"; + osmium::Timestamp t(s); REQUIRE("2000-01-01T00:00:00Z" == t.to_iso()); + REQUIRE(t.valid()); } SECTION("throws if initialized from bad string") { REQUIRE_THROWS_AS(osmium::Timestamp("x"), std::invalid_argument); } - SECTION("can be implicitly cast to time_t") { + SECTION("can be explicitly cast to time_t") { osmium::Timestamp t(4242); - time_t x = t; + time_t x = t.seconds_since_epoch(); REQUIRE(x == 4242); } @@ -50,6 +61,10 @@ TEST_CASE("Timestamp") { osmium::Timestamp t1(10); osmium::Timestamp t2(50); REQUIRE(t1 < t2); + REQUIRE(t1 > osmium::start_of_time()); + REQUIRE(t2 > osmium::start_of_time()); + REQUIRE(t1 < osmium::end_of_time()); + REQUIRE(t2 < osmium::end_of_time()); } SECTION("can be written to stream") { diff --git a/test/t/basic/test_way.cpp b/test/t/basic/test_way.cpp index 7c7bc2148..370cd0105 100644 --- a/test/t/basic/test_way.cpp +++ b/test/t/basic/test_way.cpp @@ -36,7 +36,7 @@ SECTION("way_builder") { REQUIRE(333 == way.changeset()); REQUIRE(21 == way.uid()); REQUIRE(std::string("foo") == way.user()); - REQUIRE(123 == way.timestamp()); + REQUIRE(123 == uint32_t(way.timestamp())); REQUIRE(2 == way.tags().size()); REQUIRE(3 == way.nodes().size()); REQUIRE(1 == way.nodes()[0].ref()); @@ -45,7 +45,7 @@ SECTION("way_builder") { REQUIRE(! way.is_closed()); crc32.update(way); - REQUIRE(crc32().checksum() == 0x20fe7a30); + REQUIRE(crc32().checksum() == 0x7676d0c2); } SECTION("closed_way") { diff --git a/test/t/buffer/test_buffer_basics.cpp b/test/t/buffer/test_buffer_basics.cpp new file mode 100644 index 000000000..ffe725168 --- /dev/null +++ b/test/t/buffer/test_buffer_basics.cpp @@ -0,0 +1,34 @@ +#include "catch.hpp" + +#include + +TEST_CASE("Buffer basics") { + + osmium::memory::Buffer invalid_buffer1; + osmium::memory::Buffer invalid_buffer2; + osmium::memory::Buffer empty_buffer1(1024); + osmium::memory::Buffer empty_buffer2(2048); + + REQUIRE(!invalid_buffer1); + REQUIRE(!invalid_buffer2); + REQUIRE(empty_buffer1); + REQUIRE(empty_buffer2); + + REQUIRE(invalid_buffer1 == invalid_buffer2); + REQUIRE(invalid_buffer1 != empty_buffer1); + REQUIRE(empty_buffer1 != empty_buffer2); + + REQUIRE(invalid_buffer1.capacity() == 0); + REQUIRE(invalid_buffer1.written() == 0); + REQUIRE(invalid_buffer1.committed() == 0); + + REQUIRE(empty_buffer1.capacity() == 1024); + REQUIRE(empty_buffer1.written() == 0); + REQUIRE(empty_buffer1.committed() == 0); + + REQUIRE(empty_buffer2.capacity() == 2048); + REQUIRE(empty_buffer2.written() == 0); + REQUIRE(empty_buffer2.committed() == 0); + +} + diff --git a/test/t/buffer/test_buffer_node.cpp b/test/t/buffer/test_buffer_node.cpp index 9bc8f701d..ba2431b80 100644 --- a/test/t/buffer/test_buffer_node.cpp +++ b/test/t/buffer/test_buffer_node.cpp @@ -9,7 +9,7 @@ void check_node_1(osmium::Node& node) { REQUIRE(true == node.visible()); REQUIRE(333 == node.changeset()); REQUIRE(21 == node.uid()); - REQUIRE(123 == node.timestamp()); + REQUIRE(123 == uint32_t(node.timestamp())); REQUIRE(osmium::Location(3.5, 4.7) == node.location()); REQUIRE(std::string("testuser") == node.user()); @@ -28,7 +28,7 @@ void check_node_2(osmium::Node& node) { REQUIRE(true == node.visible()); REQUIRE(333 == node.changeset()); REQUIRE(21 == node.uid()); - REQUIRE(123 == node.timestamp()); + REQUIRE(123 == uint32_t(node.timestamp())); REQUIRE(osmium::Location(3.5, 4.7) == node.location()); REQUIRE(std::string("testuser") == node.user()); @@ -56,13 +56,14 @@ void check_node_2(osmium::Node& node) { REQUIRE(2 == n); } -TEST_CASE("Buffer_Node") { +TEST_CASE("Node in Buffer") { - SECTION("buffer_node") { - constexpr size_t buffer_size = 10000; - unsigned char data[buffer_size]; + constexpr size_t buffer_size = 10000; + unsigned char data[buffer_size]; - osmium::memory::Buffer buffer(data, buffer_size, 0); + osmium::memory::Buffer buffer(data, buffer_size, 0); + + SECTION("Add node to buffer") { { // add node 1 @@ -132,4 +133,67 @@ TEST_CASE("Buffer_Node") { } + SECTION("Add buffer to another one") { + + { + // add node 1 + osmium::builder::NodeBuilder node_builder(buffer); + osmium::Node& node = node_builder.object(); + REQUIRE(osmium::item_type::node == node.type()); + + node.set_id(1); + node.set_version(3); + node.set_visible(true); + node.set_changeset(333); + node.set_uid(21); + node.set_timestamp(123); + node.set_location(osmium::Location(3.5, 4.7)); + + node_builder.add_user("testuser"); + + buffer.commit(); + } + + osmium::memory::Buffer buffer2(buffer_size, osmium::memory::Buffer::auto_grow::yes); + + buffer2.add_buffer(buffer); + buffer2.commit(); + + REQUIRE(buffer.committed() == buffer2.committed()); + const osmium::Node& node = buffer2.get(0); + REQUIRE(node.id() == 1); + REQUIRE(123 == uint32_t(node.timestamp())); + } + + SECTION("Use back_inserter on buffer") { + + { + // add node 1 + osmium::builder::NodeBuilder node_builder(buffer); + osmium::Node& node = node_builder.object(); + REQUIRE(osmium::item_type::node == node.type()); + + node.set_id(1); + node.set_version(3); + node.set_visible(true); + node.set_changeset(333); + node.set_uid(21); + node.set_timestamp(123); + node.set_location(osmium::Location(3.5, 4.7)); + + node_builder.add_user("testuser"); + + buffer.commit(); + } + + osmium::memory::Buffer buffer2(buffer_size, osmium::memory::Buffer::auto_grow::yes); + + std::copy(buffer.begin(), buffer.end(), std::back_inserter(buffer2)); + + REQUIRE(buffer.committed() == buffer2.committed()); + const osmium::Node& node = buffer2.get(0); + REQUIRE(node.id() == 1); + REQUIRE(123 == uint32_t(node.timestamp())); + } } + diff --git a/test/t/geom/test_tile_data.hpp b/test/t/geom/test_tile_data.hpp index e5c0953c0..8a22cfa51 100644 --- a/test/t/geom/test_tile_data.hpp +++ b/test/t/geom/test_tile_data.hpp @@ -1,5 +1,7 @@ -std::string s = R"(127.4864358 16.8380041 223904 118630 18 +#include + +static std::string s = R"(127.4864358 16.8380041 223904 118630 18 163.1103174 39.4760232 121 48 7 -4.1372725 -22.5105386 31 36 6 98.7193066 -36.2312406 1 1 1 diff --git a/test/t/io/deleted_nodes.osh b/test/t/io/deleted_nodes.osh new file mode 100644 index 000000000..639e05149 --- /dev/null +++ b/test/t/io/deleted_nodes.osh @@ -0,0 +1,5 @@ + + + + + diff --git a/test/t/io/deleted_nodes.osh.pbf b/test/t/io/deleted_nodes.osh.pbf new file mode 100644 index 0000000000000000000000000000000000000000..8a94870d14ad3d7a17b0a79dc1441cf51afa3ba1 GIT binary patch literal 189 zcmV;u07Cx&000dN2~Sf^NM&JUWpWr!5JDPCc$`z>^DoW~PR>ZpP1FrD&@)rwa!JiA zPW8)ANi9|q^~fwP$uG)GPR#MlOUp0HO)SaG&ue1j$uG{$EX~z7)HBjE1OU}a74rZ9 z01FBSPg6}qVRT^_PY_NTOL&~);^Jc9Vk<39Es8f3auBlPVq#*_lHp=vWE5gMu=MJE rE(R$kCI%%S)&gQ9CI&_)u7Cgk|L6a^boV!YFC=yl69WSPqA(XSe-ljE literal 0 HcmV?d00001 diff --git a/test/t/io/test_output_utils.cpp b/test/t/io/test_output_utils.cpp new file mode 100644 index 000000000..a76068348 --- /dev/null +++ b/test/t/io/test_output_utils.cpp @@ -0,0 +1,153 @@ + +#include "catch.hpp" + +#include + +#include + +TEST_CASE("output formatted") { + + std::string out; + + SECTION("small results") { + osmium::io::detail::append_printf_formatted_string(out, "%d", 17); + REQUIRE(out == "17"); + } + + SECTION("several parameters") { + osmium::io::detail::append_printf_formatted_string(out, "%d %s", 17, "foo"); + REQUIRE(out == "17 foo"); + + } + + SECTION("string already containing something") { + out += "foo"; + osmium::io::detail::append_printf_formatted_string(out, " %d", 23); + REQUIRE(out == "foo 23"); + } + + SECTION("large results") { + const char* str = + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"; + + osmium::io::detail::append_printf_formatted_string(out, "%s", str); + + REQUIRE(out == str); + } + +} + +TEST_CASE("UTF8 encoding") { + + std::string out; + + SECTION("append to string") { + out += "1234"; + osmium::io::detail::append_utf8_encoded_string(out, "abc"); + REQUIRE(out == "1234abc"); + } + + SECTION("don't encode alphabetic characters") { + const char* s = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + osmium::io::detail::append_utf8_encoded_string(out, s); + REQUIRE(out == s); + } + + SECTION("don't encode numeric characters") { + const char* s = "0123456789"; + osmium::io::detail::append_utf8_encoded_string(out, s); + REQUIRE(out == s); + } + + SECTION("don't encode lots of often used characters characters") { + const char* s = ".-;:_#+"; + osmium::io::detail::append_utf8_encoded_string(out, s); + REQUIRE(out == s); + } + + SECTION("encode characters that are special in OPL") { + osmium::io::detail::append_utf8_encoded_string(out, " \n,=@"); + REQUIRE(out == "%20%%0a%%2c%%3d%%40%"); + } + +// workaround for missing support for u8 string literals on Windows +#if !defined(_MSC_VER) + + SECTION("encode multibyte character") { + osmium::io::detail::append_utf8_encoded_string(out, u8"\u30dc_\U0001d11e_\U0001f6eb"); + REQUIRE(out == "%30dc%_%1d11e%_%1f6eb%"); + } + +#endif + +} + +TEST_CASE("html encoding") { + + std::string out; + + SECTION("do not encode normal characters") { + const char* s = "abc123,.-"; + osmium::io::detail::append_xml_encoded_string(out, s); + REQUIRE(out == s); + } + + SECTION("encode special XML characters") { + const char* s = "& \" \' < > \n \r \t"; + osmium::io::detail::append_xml_encoded_string(out, s); + REQUIRE(out == "& " ' < > "); + } + +} + +TEST_CASE("debug encoding") { + + std::string out; + + SECTION("do not encode normal characters") { + const char* s = "abc123,.-"; + osmium::io::detail::append_debug_encoded_string(out, s, "[", "]"); + REQUIRE(out == s); + } + + SECTION("encode some unicode characters") { + const char* s = u8"\n_\u30dc_\U0001d11e_\U0001f6eb"; + osmium::io::detail::append_debug_encoded_string(out, s, "[", "]"); + REQUIRE(out == "[]_[]_[]_[]"); + } + +} + +TEST_CASE("encoding of non-printable characters in the first 127 characters") { + + std::locale cloc("C"); + char s[] = "a\0"; + + for (char c = 1; c < 0x7f; ++c) { + std::string out; + s[0] = c; + + SECTION("utf8 encode") { + osmium::io::detail::append_utf8_encoded_string(out, s); + + if (!std::isprint(c, cloc)) { + REQUIRE(out[0] == '%'); + } + } + + SECTION("debug encode") { + osmium::io::detail::append_debug_encoded_string(out, s, "", ""); + + if (!std::isprint(c, cloc)) { + REQUIRE(out[0] == '<'); + } + } + + } + +} + diff --git a/test/t/io/test_reader.cpp b/test/t/io/test_reader.cpp index 9a06d8408..a83af52ab 100644 --- a/test/t/io/test_reader.cpp +++ b/test/t/io/test_reader.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -17,6 +18,27 @@ struct CountHandler : public osmium::handler::Handler { }; // class CountHandler +struct ZeroPositionNodeCountHandler : public osmium::handler::Handler { + + // number of nodes seen at zero position, or visible with undefined + // location. + int count = 0; + int total_count = 0; // total number of nodes seen + const osmium::Location zero = osmium::Location(int32_t(0), int32_t(0)); + + void node(osmium::Node &n) { + // no nodes in the history file have a zero location, and + // no visible nodes should have an undefined location. + if ((n.location() == zero) || + (n.visible() && !n.location())) { + ++count; + } + ++total_count; + } + +}; // class ZeroPositionNodeCountHandler + + TEST_CASE("Reader") { SECTION("reader can be initialized with file") { @@ -34,7 +56,7 @@ TEST_CASE("Reader") { osmium::apply(reader, handler); } - SECTION("should return invalid buffer after eof") { + SECTION("should throw after eof") { osmium::io::File file(with_data_dir("t/io/data.osm")); osmium::io::Reader reader(file); @@ -45,9 +67,9 @@ TEST_CASE("Reader") { REQUIRE(reader.eof()); - // extra read always returns invalid buffer - osmium::memory::Buffer buffer = reader.read(); - REQUIRE(!buffer); + REQUIRE_THROWS_AS({ + reader.read(); + }, osmium::io_error); } SECTION("should not hang when apply() is called twice on reader") { @@ -56,7 +78,9 @@ TEST_CASE("Reader") { osmium::handler::Handler handler; osmium::apply(reader, handler); - osmium::apply(reader, handler); + REQUIRE_THROWS_AS({ + osmium::apply(reader, handler); + }, osmium::io_error); } SECTION("should work with a buffer with uncompressed data") { @@ -113,5 +137,76 @@ TEST_CASE("Reader") { REQUIRE(handler.count == 1); } + SECTION("should decode zero node positions in history (XML)") { + osmium::io::Reader reader(with_data_dir("t/io/deleted_nodes.osh"), + osmium::osm_entity_bits::node); + ZeroPositionNodeCountHandler handler; + + REQUIRE(handler.count == 0); + REQUIRE(handler.total_count == 0); + + osmium::apply(reader, handler); + + REQUIRE(handler.count == 0); + REQUIRE(handler.total_count == 2); + } + + SECTION("should decode zero node positions in history (PBF)") { + osmium::io::Reader reader(with_data_dir("t/io/deleted_nodes.osh.pbf"), + osmium::osm_entity_bits::node); + ZeroPositionNodeCountHandler handler; + + REQUIRE(handler.count == 0); + REQUIRE(handler.total_count == 0); + + osmium::apply(reader, handler); + + REQUIRE(handler.count == 0); + REQUIRE(handler.total_count == 2); + } + +} + +TEST_CASE("Reader failure modes") { + + SECTION("should fail with nonexistent file") { + REQUIRE_THROWS({ + osmium::io::Reader reader(with_data_dir("t/io/nonexistent-file.osm")); + }); + } + + SECTION("should fail with nonexistent file (gz)") { + REQUIRE_THROWS({ + osmium::io::Reader reader(with_data_dir("t/io/nonexistent-file.osm.gz")); + }); + } + + SECTION("should fail with nonexistent file (pbf)") { + REQUIRE_THROWS({ + osmium::io::Reader reader(with_data_dir("t/io/nonexistent-file.osm.pbf")); + }); + } + + SECTION("should work when there is an exception in main thread before getting header") { + try { + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + REQUIRE(!reader.eof()); + throw std::runtime_error("foo"); + } catch (...) { + } + + } + + SECTION("should work when there is an exception in main thread while reading") { + try { + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + REQUIRE(!reader.eof()); + auto header = reader.header(); + throw std::runtime_error("foo"); + } catch (...) { + } + + } + } diff --git a/test/t/io/test_reader_with_mock_decompression.cpp b/test/t/io/test_reader_with_mock_decompression.cpp new file mode 100644 index 000000000..566295aff --- /dev/null +++ b/test/t/io/test_reader_with_mock_decompression.cpp @@ -0,0 +1,145 @@ + +#include "catch.hpp" +#include "utils.hpp" + +#include + +#include +#include + +// The MockDecompressor behaves like other Decompressor classes, but "invents" +// OSM data in XML format that can be read. Through a parameter to the +// constructor it can be instructed to throw an exception in specific parts +// of its code. This is then used to test the internals of the Reader. + +class MockDecompressor : public osmium::io::Decompressor { + + std::string m_fail_in; + int m_read_count = 0; + +public: + + MockDecompressor(const std::string& fail_in) : + Decompressor(), + m_fail_in(fail_in) { + if (m_fail_in == "constructor") { + throw std::runtime_error("error constructor"); + } + } + + ~MockDecompressor() noexcept final = default; + + void add_node(std::string& s, int i) { + s += "\n"; + } + + std::string read() final { + std::string buffer; + ++m_read_count; + + if (m_read_count == 1) { + if (m_fail_in == "first read") { + throw std::runtime_error("error first read"); + } else { + buffer += "\n\n"; + for (int i = 0; i < 1000; ++i) { + add_node(buffer, i); + } + } + } else if (m_read_count == 2) { + if (m_fail_in == "second read") { + throw std::runtime_error("error second read"); + } else { + for (int i = 1000; i < 2000; ++i) { + add_node(buffer, i); + } + } + } else if (m_read_count == 3) { + buffer += ""; + } + + return buffer; + } + + void close() final { + if (m_fail_in == "close") { + throw std::runtime_error("error close"); + } + } + +}; // class MockDecompressor + +TEST_CASE("Test Reader using MockDecompressor") { + + std::string fail_in; + + osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::gzip, + [](int, osmium::io::fsync) { return nullptr; }, + [&](int) { return new MockDecompressor(fail_in); }, + [](const char*, size_t) { return nullptr; } + ); + + SECTION("fail in constructor") { + fail_in = "constructor"; + + try { + osmium::io::Reader reader(with_data_dir("t/io/data.osm.gz")); + REQUIRE(false); + } catch (std::runtime_error& e) { + REQUIRE(std::string{e.what()} == "error constructor"); + } + } + + SECTION("fail in first read") { + fail_in = "first read"; + + try { + osmium::io::Reader reader(with_data_dir("t/io/data.osm.gz")); + reader.read(); + REQUIRE(false); + } catch (std::runtime_error& e) { + REQUIRE(std::string{e.what()} == "error first read"); + } + } + + SECTION("fail in second read") { + fail_in = "second read"; + + try { + osmium::io::Reader reader(with_data_dir("t/io/data.osm.gz")); + reader.read(); + reader.read(); + REQUIRE(false); + } catch (std::runtime_error& e) { + REQUIRE(std::string{e.what()} == "error second read"); + } + } + + SECTION("fail in close") { + fail_in = "close"; + + try { + osmium::io::Reader reader(with_data_dir("t/io/data.osm.gz")); + reader.read(); + reader.read(); + reader.read(); + reader.close(); + REQUIRE(false); + } catch (std::runtime_error& e) { + REQUIRE(std::string{e.what()} == "error close"); + } + } + + SECTION("not failing") { + fail_in = "not"; + + osmium::io::Reader reader(with_data_dir("t/io/data.osm.gz")); + reader.read(); + reader.close(); + REQUIRE(true); + } + +} + diff --git a/test/t/io/test_reader_with_mock_parser.cpp b/test/t/io/test_reader_with_mock_parser.cpp new file mode 100644 index 000000000..7c7dcddba --- /dev/null +++ b/test/t/io/test_reader_with_mock_parser.cpp @@ -0,0 +1,123 @@ + +#include "catch.hpp" +#include "utils.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +class MockParser : public osmium::io::detail::Parser { + + std::string m_fail_in; + +public: + + MockParser(osmium::io::detail::future_string_queue_type& input_queue, + osmium::io::detail::future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_types, + const std::string& fail_in) : + Parser(input_queue, output_queue, header_promise, read_types), + m_fail_in(fail_in) { + } + + osmium::memory::Buffer create_testdata() { + osmium::memory::Buffer buffer(1000); + + { + osmium::builder::NodeBuilder nb(buffer); + nb.add_user("foo"); + } + buffer.commit(); + + return buffer; + } + + void run() final { + osmium::thread::set_thread_name("_osmium_mock_in"); + + if (m_fail_in == "header") { + throw std::runtime_error("error in header"); + } + + set_header_value(osmium::io::Header{}); + + send_to_output_queue(create_testdata()); + + if (m_fail_in == "read") { + throw std::runtime_error("error in read"); + } + } + +}; // class MockParser + +TEST_CASE("Test Reader using MockParser") { + + std::string fail_in; + + osmium::io::detail::ParserFactory::instance().register_parser( + osmium::io::file_format::xml, + [&](osmium::io::detail::future_string_queue_type& input_queue, + osmium::io::detail::future_buffer_queue_type& output_queue, + std::promise& header_promise, + osmium::osm_entity_bits::type read_which_entities) { + return std::unique_ptr(new MockParser(input_queue, output_queue, header_promise, read_which_entities, fail_in)); + }); + + SECTION("no failure") { + fail_in = ""; + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + auto header = reader.header(); + REQUIRE(reader.read()); + REQUIRE(!reader.read()); + REQUIRE(reader.eof()); + reader.close(); + } + + SECTION("throw in header") { + fail_in = "header"; + try { + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + reader.header(); + } catch (std::runtime_error& e) { + REQUIRE(std::string{e.what()} == "error in header"); + } + } + + SECTION("throw in read") { + fail_in = "read"; + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + reader.header(); + try { + reader.read(); + } catch (std::runtime_error& e) { + REQUIRE(std::string{e.what()} == "error in read"); + } + reader.close(); + } + + SECTION("throw in user code") { + fail_in = ""; + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + reader.header(); + try { + throw std::runtime_error("error in user code"); + } catch (std::runtime_error& e) { + REQUIRE(std::string{e.what()} == "error in user code"); + } + REQUIRE(reader.read()); + REQUIRE(!reader.read()); + REQUIRE(reader.eof()); + reader.close(); + } + +} + diff --git a/test/t/io/test_string_table.cpp b/test/t/io/test_string_table.cpp index 7fedfcfca..ab977e8bc 100644 --- a/test/t/io/test_string_table.cpp +++ b/test/t/io/test_string_table.cpp @@ -33,9 +33,9 @@ TEST_CASE("String store") { } SECTION("add zero-length string and longer strings") { - const char* s1 = ss.add(""); - const char* s2 = ss.add("xxx"); - const char* s3 = ss.add("yyyyy"); + ss.add(""); + ss.add("xxx"); + ss.add("yyyyy"); auto it = ss.begin(); REQUIRE(std::string(*it++) == ""); diff --git a/test/t/io/test_writer.cpp b/test/t/io/test_writer.cpp new file mode 100644 index 000000000..45593cf60 --- /dev/null +++ b/test/t/io/test_writer.cpp @@ -0,0 +1,117 @@ +#include "catch.hpp" +#include "utils.hpp" + +#include + +#include +#include +#include +#include +#include + +TEST_CASE("Writer") { + + osmium::io::Header header; + header.set("generator", "test_writer.cpp"); + + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + osmium::memory::Buffer buffer = reader.read(); + REQUIRE(buffer); + REQUIRE(buffer.committed() > 0); + auto num = std::distance(buffer.cbegin(), buffer.cend()); + REQUIRE(num > 0); + REQUIRE(buffer.cbegin()->id() == 1); + + std::string filename; + + SECTION("Empty writes") { + + SECTION("Empty buffer") { + filename = "test-writer-out-empty-buffer.osm"; + osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow); + osmium::memory::Buffer empty_buffer(1024); + writer(std::move(empty_buffer)); + writer.close(); + } + + SECTION("Invalid buffer") { + filename = "test-writer-out-invalid-buffer.osm"; + osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow); + osmium::memory::Buffer invalid_buffer; + writer(std::move(invalid_buffer)); + writer.close(); + } + + osmium::io::Reader reader_check(filename); + osmium::memory::Buffer buffer_check = reader_check.read(); + REQUIRE(!buffer_check); + } + + SECTION("Successfull writes") { + + SECTION("Writer buffer") { + filename = "test-writer-out-buffer.osm"; + osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow); + writer(std::move(buffer)); + writer.close(); + + REQUIRE_THROWS_AS({ + writer(osmium::memory::Buffer{}); + }, osmium::io_error); + } + + SECTION("Writer item") { + filename = "test-writer-out-item.osm"; + osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow); + for (const auto& item : buffer) { + writer(item); + } + writer.close(); + } + + SECTION("Writer output iterator") { + filename = "test-writer-out-iterator.osm"; + osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow); + auto it = osmium::io::make_output_iterator(writer); + std::copy(buffer.cbegin(), buffer.cend(), it); + writer.close(); + } + + osmium::io::Reader reader_check(filename); + osmium::memory::Buffer buffer_check = reader_check.read(); + REQUIRE(buffer_check); + REQUIRE(buffer_check.committed() > 0); + REQUIRE(std::distance(buffer_check.cbegin(), buffer_check.cend()) == num); + REQUIRE(buffer_check.cbegin()->id() == 1); + + } + + SECTION("Interrupted write") { + + int error = 0; + try { + + SECTION("fail after open") { + filename = "test-writer-out-fail1.osm"; + osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow); + throw 1; + } + + SECTION("fail after write") { + filename = "test-writer-out-fail2.osm"; + osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow); + writer(std::move(buffer)); + throw 2; + } + + } catch (int e) { + error = e; + } + + REQUIRE(error > 0); + + } + +} + + diff --git a/test/t/io/test_writer_with_mock_compression.cpp b/test/t/io/test_writer_with_mock_compression.cpp new file mode 100644 index 000000000..c2d3bbd49 --- /dev/null +++ b/test/t/io/test_writer_with_mock_compression.cpp @@ -0,0 +1,99 @@ + +#include "catch.hpp" +#include "utils.hpp" + +#include +#include + +#include +#include +#include + +class MockCompressor : public osmium::io::Compressor { + + std::string m_fail_in; + +public: + + MockCompressor(const std::string& fail_in) : + Compressor(osmium::io::fsync::no), + m_fail_in(fail_in) { + if (m_fail_in == "constructor") { + throw std::logic_error("constructor"); + } + } + + ~MockCompressor() noexcept final = default; + + void write(const std::string&) final { + if (m_fail_in == "write") { + throw std::logic_error("write"); + } + } + + void close() final { + if (m_fail_in == "close") { + throw std::logic_error("close"); + } + } + +}; // class MockCompressor + +TEST_CASE("Write with mock compressor") { + + std::string fail_in; + + osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::gzip, + [&](int, osmium::io::fsync) { return new MockCompressor(fail_in); }, + [](int) { return nullptr; }, + [](const char*, size_t) { return nullptr; } + ); + + osmium::io::Header header; + header.set("generator", "test_writer_with_mock_compression.cpp"); + + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + osmium::memory::Buffer buffer = reader.read(); + REQUIRE(buffer); + REQUIRE(buffer.committed() > 0); + auto num = std::distance(buffer.cbegin(), buffer.cend()); + REQUIRE(num > 0); + + SECTION("fail on construction") { + + fail_in = "constructor"; + + REQUIRE_THROWS_AS({ + osmium::io::Writer writer("test-writer-mock-fail-on-construction.osm.gz", header, osmium::io::overwrite::allow); + writer(std::move(buffer)); + writer.close(); + }, std::logic_error); + + } + + SECTION("fail on write") { + + fail_in = "write"; + + REQUIRE_THROWS_AS({ + osmium::io::Writer writer("test-writer-mock-fail-on-write.osm.gz", header, osmium::io::overwrite::allow); + writer(std::move(buffer)); + writer.close(); + }, std::logic_error); + + } + + SECTION("fail on close") { + + fail_in = "close"; + + REQUIRE_THROWS_AS({ + osmium::io::Writer writer("test-writer-mock-fail-on-close.osm.gz", header, osmium::io::overwrite::allow); + writer(std::move(buffer)); + writer.close(); + }, std::logic_error); + + } + +} + diff --git a/test/t/io/test_writer_with_mock_encoder.cpp b/test/t/io/test_writer_with_mock_encoder.cpp new file mode 100644 index 000000000..a43d59183 --- /dev/null +++ b/test/t/io/test_writer_with_mock_encoder.cpp @@ -0,0 +1,105 @@ + +#include "catch.hpp" +#include "utils.hpp" + +#include +#include + +#include +#include +#include +#include +#include + +class MockOutputFormat : public osmium::io::detail::OutputFormat { + + std::string m_fail_in; + +public: + + MockOutputFormat(const osmium::io::File&, osmium::io::detail::future_string_queue_type& output_queue, const std::string& fail_in) : + OutputFormat(output_queue), + m_fail_in(fail_in) { + } + + void write_header(const osmium::io::Header&) final { + if (m_fail_in == "header") { + throw std::logic_error("header"); + } + send_to_output_queue(std::string{"header"}); + } + + void write_buffer(osmium::memory::Buffer&&) final { + if (m_fail_in == "write") { + throw std::logic_error("write"); + } + send_to_output_queue(std::string{"write"}); + } + + void write_end() final { + if (m_fail_in == "write_end") { + throw std::logic_error("write_end"); + } + send_to_output_queue(std::string{"end"}); + } + +}; // class MockOutputFormat + +TEST_CASE("Test Writer with MockOutputFormat") { + + std::string fail_in; + + osmium::io::detail::OutputFormatFactory::instance().register_output_format( + osmium::io::file_format::xml, + [&](const osmium::io::File& file, osmium::io::detail::future_string_queue_type& output_queue) { + return new MockOutputFormat(file, output_queue, fail_in); + }); + + osmium::io::Header header; + header.set("generator", "test_writer_with_mock_encoder.cpp"); + + osmium::io::Reader reader(with_data_dir("t/io/data.osm")); + osmium::memory::Buffer buffer = reader.read(); + REQUIRE(buffer); + REQUIRE(buffer.committed() > 0); + auto num = std::distance(buffer.cbegin(), buffer.cend()); + REQUIRE(num > 0); + + SECTION("error in header") { + + fail_in = "header"; + + REQUIRE_THROWS_AS({ + osmium::io::Writer writer("test-writer-mock-fail-on-construction.osm", header, osmium::io::overwrite::allow); + writer(std::move(buffer)); + writer.close(); + }, std::logic_error); + + } + + SECTION("error in write") { + + fail_in = "write"; + + REQUIRE_THROWS_AS({ + osmium::io::Writer writer("test-writer-mock-fail-on-construction.osm", header, osmium::io::overwrite::allow); + writer(std::move(buffer)); + writer.close(); + }, std::logic_error); + + } + + SECTION("error in write_end") { + + fail_in = "write_end"; + + REQUIRE_THROWS_AS({ + osmium::io::Writer writer("test-writer-mock-fail-on-construction.osm", header, osmium::io::overwrite::allow); + writer(std::move(buffer)); + writer.close(); + }, std::logic_error); + + } + +} + diff --git a/test/t/tags/test_tag_list.cpp b/test/t/tags/test_tag_list.cpp index 77523e796..295f51af8 100644 --- a/test/t/tags/test_tag_list.cpp +++ b/test/t/tags/test_tag_list.cpp @@ -100,3 +100,14 @@ TEST_CASE("empty keys and values are okay") { REQUIRE(std::string("") == tl.get_value_by_key("empty value")); REQUIRE(std::string("empty key") == tl.get_value_by_key("")); } + +TEST_CASE("tag key or value is too long") { + osmium::memory::Buffer buffer(10240); + osmium::builder::TagListBuilder builder(buffer); + + const char kv[2000] = ""; + builder.add_tag(kv, 1, kv, 1000); + REQUIRE_THROWS(builder.add_tag(kv, 1500, kv, 1)); + REQUIRE_THROWS(builder.add_tag(kv, 1, kv, 1500)); +} + diff --git a/test/t/thread/test_pool.cpp b/test/t/thread/test_pool.cpp index 5fa6bbab5..c1047db49 100644 --- a/test/t/thread/test_pool.cpp +++ b/test/t/thread/test_pool.cpp @@ -5,14 +5,7 @@ #include #include - -static std::atomic result; - -struct test_job_ok { - void operator()() const { - result = 1; - } -}; +#include struct test_job_with_result { int operator()() const { @@ -21,44 +14,55 @@ struct test_job_with_result { }; struct test_job_throw { - void operator()() const { + OSMIUM_NORETURN void operator()() const { throw std::runtime_error("exception in pool thread"); } }; +TEST_CASE("number of threads in pool") { + + // hardcoded setting + REQUIRE(osmium::thread::detail::get_pool_size( 1, 0, 2) == 1); + REQUIRE(osmium::thread::detail::get_pool_size( 4, 0, 2) == 4); + REQUIRE(osmium::thread::detail::get_pool_size( 4, 0, 4) == 4); + REQUIRE(osmium::thread::detail::get_pool_size(16, 0, 4) == 16); + REQUIRE(osmium::thread::detail::get_pool_size(16, 0, 16) == 16); + REQUIRE(osmium::thread::detail::get_pool_size( 8, 4, 2) == 8); + REQUIRE(osmium::thread::detail::get_pool_size( 8, 16, 2) == 8); + REQUIRE(osmium::thread::detail::get_pool_size(-2, 16, 2) == 1); + REQUIRE(osmium::thread::detail::get_pool_size(-2, 16, 8) == 6); + + // user decides through OSMIUM_POOL_THREADS env variable + REQUIRE(osmium::thread::detail::get_pool_size( 0, 0, 2) == 1); + REQUIRE(osmium::thread::detail::get_pool_size( 0, -2, 4) == 2); + REQUIRE(osmium::thread::detail::get_pool_size( 0, -1, 8) == 7); + REQUIRE(osmium::thread::detail::get_pool_size( 0, 0, 16) == 14); + REQUIRE(osmium::thread::detail::get_pool_size( 0, 1, 16) == 1); + REQUIRE(osmium::thread::detail::get_pool_size( 0, 2, 16) == 2); + REQUIRE(osmium::thread::detail::get_pool_size( 0, 4, 16) == 4); + REQUIRE(osmium::thread::detail::get_pool_size( 0, 8, 16) == 8); + + // outliers + REQUIRE(osmium::thread::detail::get_pool_size(-100, 0, 16) == 1); + REQUIRE(osmium::thread::detail::get_pool_size(1000, 0, 16) == 256); + +} + TEST_CASE("thread") { + auto& pool = osmium::thread::Pool::instance(); + SECTION("can get access to thread pool") { - auto& pool = osmium::thread::Pool::instance(); REQUIRE(pool.queue_empty()); } SECTION("can send job to thread pool") { - auto& pool = osmium::thread::Pool::instance(); - result = 0; - auto future = pool.submit(test_job_ok {}); - - // wait a bit for the other thread to get a chance to run - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - - REQUIRE(result == 1); - - future.get(); - - REQUIRE(true); - } - - SECTION("can send job to thread pool") { - auto& pool = osmium::thread::Pool::instance(); auto future = pool.submit(test_job_with_result {}); REQUIRE(future.get() == 42); } SECTION("can throw from job in thread pool") { - auto& pool = osmium::thread::Pool::instance(); - result = 0; - auto future = pool.submit(test_job_throw {}); REQUIRE_THROWS_AS(future.get(), std::runtime_error); diff --git a/test/t/util/test_data_file.cpp b/test/t/util/test_data_file.cpp deleted file mode 100644 index 3f432f943..000000000 --- a/test/t/util/test_data_file.cpp +++ /dev/null @@ -1,81 +0,0 @@ -#include "catch.hpp" - -#include - -#include - -TEST_CASE("temporary file") { - - SECTION("create/open") { - osmium::util::DataFile file; - - REQUIRE(!!file); - int fd = file.fd(); - - REQUIRE(fd > 0); - - const char buf[] = "foobar"; - REQUIRE(::write(fd, buf, sizeof(buf)) == sizeof(buf)); - - file.close(); - - REQUIRE(!file); - } - -} - -TEST_CASE("named file") { - - SECTION("create/open") { - { - osmium::util::DataFile file("test.data", true); - - REQUIRE(!!file); - int fd = file.fd(); - - REQUIRE(fd > 0); - - REQUIRE(file.size() == 0); - - const char buf[] = "foobar"; - REQUIRE(::write(fd, buf, sizeof(buf) - 1) == sizeof(buf) - 1); - - file.close(); - - REQUIRE(!file); - } - { - osmium::util::DataFile file("test.data", false); - - REQUIRE(!!file); - int fd = file.fd(); - - REQUIRE(fd > 0); - - REQUIRE(file.size() == 6); - - char buf[10]; - int len = ::read(fd, buf, sizeof(buf)); - - REQUIRE(len == 6); - REQUIRE(!strncmp(buf, "foobar", 6)); - - file.close(); - - REQUIRE(!file); - REQUIRE(unlink("test.data") == 0); - } - } - - SECTION("grow file") { - osmium::util::DataFile file("test.data", true); - - REQUIRE(!!file); - - REQUIRE(file.size() == 0); - file.grow(10); - REQUIRE(file.size() == 10); - } - -} - diff --git a/test/t/util/test_delta.cpp b/test/t/util/test_delta.cpp index cebcca8ed..667c9b443 100644 --- a/test/t/util/test_delta.cpp +++ b/test/t/util/test_delta.cpp @@ -4,24 +4,50 @@ #include -TEST_CASE("delta encode") { +TEST_CASE("delta encode int") { osmium::util::DeltaEncode x; SECTION("int") { REQUIRE(x.update(17) == 17); REQUIRE(x.update(10) == -7); + REQUIRE(x.update(-10) == -20); } } -TEST_CASE("delta decode") { +TEST_CASE("delta decode int") { osmium::util::DeltaDecode x; SECTION("int") { REQUIRE(x.update(17) == 17); REQUIRE(x.update(10) == 27); + REQUIRE(x.update(-40) == -13); + } + +} + +TEST_CASE("delta encode unsigned int") { + + osmium::util::DeltaEncode x; + + SECTION("int") { + REQUIRE(x.update(17) == 17); + REQUIRE(x.update(10) == -7); + REQUIRE(x.update(0) == -10); + } + +} + +TEST_CASE("delta decode unsigned int") { + + osmium::util::DeltaDecode x; + + SECTION("int") { + REQUIRE(x.update(17) == 17); + REQUIRE(x.update(10) == 27); + REQUIRE(x.update(-15) == 12); } } @@ -30,13 +56,13 @@ TEST_CASE("delta encode and decode") { std::vector a = { 5, -9, 22, 13, 0, 23 }; - osmium::util::DeltaEncode de; + osmium::util::DeltaEncode de; std::vector b; for (int x : a) { b.push_back(de.update(x)); } - osmium::util::DeltaDecode dd; + osmium::util::DeltaDecode dd; std::vector c; for (int x : b) { c.push_back(dd.update(x)); diff --git a/test/t/util/test_file.cpp b/test/t/util/test_file.cpp index 2787261a5..475f28596 100644 --- a/test/t/util/test_file.cpp +++ b/test/t/util/test_file.cpp @@ -3,6 +3,7 @@ #include #ifdef _WIN32 +#include // https://msdn.microsoft.com/en-us/library/ksazx244.aspx // https://msdn.microsoft.com/en-us/library/a9yf33zb.aspx class DoNothingInvalidParameterHandler { @@ -23,6 +24,7 @@ public: DoNothingInvalidParameterHandler() : old_handler(_set_invalid_parameter_handler(invalid_parameter_handler)) { + _CrtSetReportMode(_CRT_ASSERT, 0); } ~DoNothingInvalidParameterHandler() { diff --git a/test/t/util/test_options.cpp b/test/t/util/test_options.cpp index 969f20103..8cba0950b 100644 --- a/test/t/util/test_options.cpp +++ b/test/t/util/test_options.cpp @@ -6,43 +6,76 @@ TEST_CASE("Options") { - SECTION("set_simple") { - osmium::util::Options o; + osmium::util::Options o; + + SECTION("set a single value from string") { o.set("foo", "bar"); REQUIRE("bar" == o.get("foo")); REQUIRE("" == o.get("empty")); REQUIRE("default" == o.get("empty", "default")); + REQUIRE(!o.is_true("foo")); REQUIRE(!o.is_true("empty")); + + REQUIRE(o.is_not_false("foo")); + REQUIRE(o.is_not_false("empty")); + REQUIRE(1 == o.size()); } - SECTION("set_from_bool") { - osmium::util::Options o; + SECTION("set values from booleans") { o.set("t", true); o.set("f", false); REQUIRE("true" == o.get("t")); REQUIRE("false" == o.get("f")); REQUIRE("" == o.get("empty")); + REQUIRE(o.is_true("t")); REQUIRE(!o.is_true("f")); + + REQUIRE(o.is_not_false("t")); + REQUIRE(!o.is_not_false("f")); + REQUIRE(2 == o.size()); } - SECTION("set_from_single_string_with_equals") { - osmium::util::Options o; + SECTION("set value from string with equal sign") { o.set("foo=bar"); REQUIRE("bar" == o.get("foo")); REQUIRE(1 == o.size()); } - SECTION("set_from_single_string_without_equals") { - osmium::util::Options o; + SECTION("set value from string without equal sign") { o.set("foo"); REQUIRE("true" == o.get("foo")); + REQUIRE(o.is_true("foo")); + REQUIRE(o.is_not_false("foo")); + REQUIRE(1 == o.size()); } } +TEST_CASE("Options with initializer list") { + + osmium::util::Options o{ { "foo", "true" }, { "bar", "17" } }; + + REQUIRE(o.get("foo") == "true"); + REQUIRE(o.get("bar") == "17"); + REQUIRE(o.is_true("foo")); + REQUIRE_FALSE(o.is_true("bar")); + REQUIRE(o.size() == 2); + + SECTION("Change existing value") { + o.set("foo", "false"); + REQUIRE_FALSE(o.is_true("foo")); + } + + SECTION("Add new value") { + o.set("new", "something"); + REQUIRE_FALSE(o.is_true("new")); + REQUIRE(o.get("new") == "something"); + } +} +