Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Remove libosmium

Browse Source
This commit is contained in:
Patrick Niklaus
2016-03-01 17:54:39 +01:00
parent 042740877c
commit ae85d86d8f
229 changed files with 0 additions and 40229 deletions
Download Patch File Download Diff File Expand all files Collapse all files
third_party/libosmium/include/osmium/io/any_compression.hpp
Vendored
-48
View File
@@ -1,48 +0,0 @@
#ifndef OSMIUM_IO_ANY_COMPRESSION_HPP
#define OSMIUM_IO_ANY_COMPRESSION_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
/**
* @file
*
* Include this file if you want to read or write compressed OSM XML files.
*
* @attention If you include this file, you'll need to link with `libz`
* and `libbz2`.
*/
#include <osmium/io/bzip2_compression.hpp> // IWYU pragma: export
#include <osmium/io/gzip_compression.hpp> // IWYU pragma: export
#endif // OSMIUM_IO_ANY_COMPRESSION_HPP
third_party/libosmium/include/osmium/io/any_input.hpp
Vendored
-52
View File
@@ -1,52 +0,0 @@
#ifndef OSMIUM_IO_ANY_INPUT_HPP
#define OSMIUM_IO_ANY_INPUT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
/**
* @file
*
* Include this file if you want to read all kinds of OSM files.
*
* @attention If you include this file, you'll need to link with
* `ws2_32` (Windows only), `libexpat`, `libz`, `libbz2`,
* and enable multithreading.
*/
#include <osmium/io/any_compression.hpp> // IWYU pragma: export
#include <osmium/io/pbf_input.hpp> // IWYU pragma: export
#include <osmium/io/xml_input.hpp> // IWYU pragma: export
#include <osmium/io/o5m_input.hpp> // IWYU pragma: export
#endif // OSMIUM_IO_ANY_INPUT_HPP
third_party/libosmium/include/osmium/io/any_output.hpp
Vendored
-53
View File
@@ -1,53 +0,0 @@
#ifndef OSMIUM_IO_ANY_OUTPUT_HPP
#define OSMIUM_IO_ANY_OUTPUT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
/**
* @file
*
* Include this file if you want to write all kinds of OSM files.
*
* @attention If you include this file, you'll need to link with
* `ws2_32` (Windows only), `libz`, `libbz2`, and enable
* multithreading.
*/
#include <osmium/io/any_compression.hpp> // IWYU pragma: export
#include <osmium/io/debug_output.hpp> // IWYU pragma: export
#include <osmium/io/opl_output.hpp> // IWYU pragma: export
#include <osmium/io/pbf_output.hpp> // IWYU pragma: export
#include <osmium/io/xml_output.hpp> // IWYU pragma: export
#endif // OSMIUM_IO_ANY_OUTPUT_HPP
third_party/libosmium/include/osmium/io/bzip2_compression.hpp
Vendored
-321
View File
@@ -1,321 +0,0 @@
#ifndef OSMIUM_IO_BZIP2_COMPRESSION_HPP
#define OSMIUM_IO_BZIP2_COMPRESSION_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
/**
* @file
*
* Include this file if you want to read or write bzip2-compressed OSM XML
* files.
*
* @attention If you include this file, you'll need to link with `libbz2`.
*/
#include <cerrno>
#include <cstddef>
#include <cstdio>
#include <stdexcept>
#include <string>
#include <bzlib.h>
#ifndef _MSC_VER
# include <unistd.h>
#endif
#include <osmium/io/compression.hpp>
#include <osmium/io/error.hpp>
#include <osmium/io/file_compression.hpp>
#include <osmium/io/writer_options.hpp>
#include <osmium/util/cast.hpp>
#include <osmium/util/compatibility.hpp>
namespace osmium {
/**
* Exception thrown when there are problems compressing or
* decompressing bzip2 files.
*/
struct bzip2_error : public io_error {
int bzip2_error_code;
int system_errno;
bzip2_error(const std::string& what, int error_code) :
io_error(what),
bzip2_error_code(error_code),
system_errno(error_code == BZ_IO_ERROR ? errno : 0) {
}
}; // struct bzip2_error
namespace io {
namespace detail {
OSMIUM_NORETURN inline void throw_bzip2_error(BZFILE* bzfile, const char* msg, int bzlib_error = 0) {
std::string error("bzip2 error: ");
error += msg;
error += ": ";
int errnum = bzlib_error;
if (bzlib_error) {
error += std::to_string(bzlib_error);
} else {
error += ::BZ2_bzerror(bzfile, &errnum);
}
throw osmium::bzip2_error(error, errnum);
}
} // namespace detail
class Bzip2Compressor : public Compressor {
FILE* m_file;
int m_bzerror;
BZFILE* m_bzfile;
public:
explicit Bzip2Compressor(int fd, fsync sync) :
Compressor(sync),
m_file(fdopen(dup(fd), "wb")),
m_bzerror(BZ_OK),
m_bzfile(::BZ2_bzWriteOpen(&m_bzerror, m_file, 6, 0, 0)) {
if (!m_bzfile) {
detail::throw_bzip2_error(m_bzfile, "write open failed", m_bzerror);
}
}
~Bzip2Compressor() noexcept final {
try {
close();
} catch (...) {
// Ignore any exceptions because destructor must not throw.
}
}
void write(const std::string& data) final {
int error;
::BZ2_bzWrite(&error, m_bzfile, const_cast<char*>(data.data()), static_cast_with_assert<int>(data.size()));
if (error != BZ_OK && error != BZ_STREAM_END) {
detail::throw_bzip2_error(m_bzfile, "write failed", error);
}
}
void close() final {
if (m_bzfile) {
int error;
::BZ2_bzWriteClose(&error, m_bzfile, 0, nullptr, nullptr);
m_bzfile = nullptr;
if (m_file) {
if (do_fsync()) {
osmium::io::detail::reliable_fsync(::fileno(m_file));
}
if (fclose(m_file) != 0) {
throw std::system_error(errno, std::system_category(), "Close failed");
}
}
if (error != BZ_OK) {
detail::throw_bzip2_error(m_bzfile, "write close failed", error);
}
}
}
}; // class Bzip2Compressor
class Bzip2Decompressor : public Decompressor {
FILE* m_file;
int m_bzerror;
BZFILE* m_bzfile;
bool m_stream_end {false};
public:
explicit Bzip2Decompressor(int fd) :
Decompressor(),
m_file(fdopen(dup(fd), "rb")),
m_bzerror(BZ_OK),
m_bzfile(::BZ2_bzReadOpen(&m_bzerror, m_file, 0, 0, nullptr, 0)) {
if (!m_bzfile) {
detail::throw_bzip2_error(m_bzfile, "read open failed", m_bzerror);
}
}
~Bzip2Decompressor() noexcept final {
try {
close();
} catch (...) {
// Ignore any exceptions because destructor must not throw.
}
}
std::string read() final {
std::string buffer;
if (!m_stream_end) {
buffer.resize(osmium::io::Decompressor::input_buffer_size);
int error;
int nread = ::BZ2_bzRead(&error, m_bzfile, const_cast<char*>(buffer.data()), static_cast_with_assert<int>(buffer.size()));
if (error != BZ_OK && error != BZ_STREAM_END) {
detail::throw_bzip2_error(m_bzfile, "read failed", error);
}
if (error == BZ_STREAM_END) {
void* unused;
int nunused;
if (! feof(m_file)) {
::BZ2_bzReadGetUnused(&error, m_bzfile, &unused, &nunused);
if (error != BZ_OK) {
detail::throw_bzip2_error(m_bzfile, "get unused failed", error);
}
std::string unused_data(static_cast<const char*>(unused), static_cast<std::string::size_type>(nunused));
::BZ2_bzReadClose(&error, m_bzfile);
if (error != BZ_OK) {
detail::throw_bzip2_error(m_bzfile, "read close failed", error);
}
m_bzfile = ::BZ2_bzReadOpen(&error, m_file, 0, 0, const_cast<void*>(static_cast<const void*>(unused_data.data())), static_cast_with_assert<int>(unused_data.size()));
if (error != BZ_OK) {
detail::throw_bzip2_error(m_bzfile, "read open failed", error);
}
} else {
m_stream_end = true;
}
}
buffer.resize(static_cast<std::string::size_type>(nread));
}
return buffer;
}
void close() final {
if (m_bzfile) {
int error;
::BZ2_bzReadClose(&error, m_bzfile);
m_bzfile = nullptr;
if (m_file) {
if (fclose(m_file) != 0) {
throw std::system_error(errno, std::system_category(), "Close failed");
}
}
if (error != BZ_OK) {
detail::throw_bzip2_error(m_bzfile, "read close failed", error);
}
}
}
}; // class Bzip2Decompressor
class Bzip2BufferDecompressor : public Decompressor {
const char* m_buffer;
size_t m_buffer_size;
bz_stream m_bzstream;
public:
Bzip2BufferDecompressor(const char* buffer, size_t size) :
m_buffer(buffer),
m_buffer_size(size),
m_bzstream() {
m_bzstream.next_in = const_cast<char*>(buffer);
m_bzstream.avail_in = static_cast_with_assert<unsigned int>(size);
int result = BZ2_bzDecompressInit(&m_bzstream, 0, 0);
if (result != BZ_OK) {
std::string message("bzip2 error: decompression init failed: ");
throw bzip2_error(message, result);
}
}
~Bzip2BufferDecompressor() noexcept final {
try {
close();
} catch (...) {
// Ignore any exceptions because destructor must not throw.
}
}
std::string read() final {
std::string output;
if (m_buffer) {
const size_t buffer_size = 10240;
output.resize(buffer_size);
m_bzstream.next_out = const_cast<char*>(output.data());
m_bzstream.avail_out = buffer_size;
int result = BZ2_bzDecompress(&m_bzstream);
if (result != BZ_OK) {
m_buffer = nullptr;
m_buffer_size = 0;
}
if (result != BZ_OK && result != BZ_STREAM_END) {
std::string message("bzip2 error: decompress failed: ");
throw bzip2_error(message, result);
}
output.resize(static_cast<unsigned long>(m_bzstream.next_out - output.data()));
}
return output;
}
void close() final {
BZ2_bzDecompressEnd(&m_bzstream);
}
}; // class Bzip2BufferDecompressor
namespace detail {
// we want the register_compression() function to run, setting
// the variable is only a side-effect, it will never be used
const bool registered_bzip2_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::bzip2,
[](int fd, fsync sync) { return new osmium::io::Bzip2Compressor(fd, sync); },
[](int fd) { return new osmium::io::Bzip2Decompressor(fd); },
[](const char* buffer, size_t size) { return new osmium::io::Bzip2BufferDecompressor(buffer, size); }
);
// dummy function to silence the unused variable warning from above
inline bool get_registered_bzip2_compression() noexcept {
return registered_bzip2_compression;
}
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_BZIP2_COMPRESSION_HPP
third_party/libosmium/include/osmium/io/compression.hpp
Vendored
-321
View File
@@ -1,321 +0,0 @@
#ifndef OSMIUM_IO_COMPRESSION_HPP
#define OSMIUM_IO_COMPRESSION_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cerrno>
#include <functional>
#include <map>
#include <memory>
#include <stdexcept>
#include <string>
#include <system_error>
#include <tuple>
#include <utility>
#ifndef _MSC_VER
# include <unistd.h>
#else
# include <io.h>
#endif
#include <osmium/io/detail/read_write.hpp>
#include <osmium/io/error.hpp>
#include <osmium/io/file_compression.hpp>
#include <osmium/io/writer_options.hpp>
#include <osmium/util/compatibility.hpp>
namespace osmium {
namespace io {
class Compressor {
fsync m_fsync;
protected:
bool do_fsync() const {
return m_fsync == fsync::yes;
}
public:
explicit Compressor(fsync sync) :
m_fsync(sync) {
}
virtual ~Compressor() noexcept {
}
virtual void write(const std::string& data) = 0;
virtual void close() = 0;
}; // class Compressor
class Decompressor {
public:
static constexpr unsigned int input_buffer_size = 1024 * 1024;
Decompressor() = default;
Decompressor(const Decompressor&) = delete;
Decompressor& operator=(const Decompressor&) = delete;
Decompressor(Decompressor&&) = delete;
Decompressor& operator=(Decompressor&&) = delete;
virtual ~Decompressor() noexcept {
}
virtual std::string read() = 0;
virtual void close() = 0;
}; // class Decompressor
/**
* This singleton factory class is used to register compression
* algorithms used for reading and writing OSM files.
*
* For each algorithm we store two functions that construct
* a compressor and decompressor object, respectively.
*/
class CompressionFactory {
public:
typedef std::function<osmium::io::Compressor*(int, fsync)> create_compressor_type;
typedef std::function<osmium::io::Decompressor*(int)> create_decompressor_type_fd;
typedef std::function<osmium::io::Decompressor*(const char*, size_t)> create_decompressor_type_buffer;
private:
typedef std::map<const osmium::io::file_compression,
std::tuple<create_compressor_type,
create_decompressor_type_fd,
create_decompressor_type_buffer>> compression_map_type;
compression_map_type m_callbacks;
CompressionFactory() = default;
CompressionFactory(const CompressionFactory&) = delete;
CompressionFactory& operator=(const CompressionFactory&) = delete;
CompressionFactory(CompressionFactory&&) = delete;
CompressionFactory& operator=(CompressionFactory&&) = delete;
OSMIUM_NORETURN void error(osmium::io::file_compression compression) {
std::string error_message {"Support for compression '"};
error_message += as_string(compression);
error_message += "' not compiled into this binary.";
throw unsupported_file_format_error(error_message);
}
public:
static CompressionFactory& instance() {
static CompressionFactory factory;
return factory;
}
bool register_compression(
osmium::io::file_compression compression,
create_compressor_type create_compressor,
create_decompressor_type_fd create_decompressor_fd,
create_decompressor_type_buffer create_decompressor_buffer) {
compression_map_type::value_type cc(compression,
std::make_tuple(create_compressor,
create_decompressor_fd,
create_decompressor_buffer));
return m_callbacks.insert(cc).second;
}
template <typename... TArgs>
std::unique_ptr<osmium::io::Compressor> create_compressor(osmium::io::file_compression compression, TArgs&&... args) {
auto it = m_callbacks.find(compression);
if (it != m_callbacks.end()) {
return std::unique_ptr<osmium::io::Compressor>(std::get<0>(it->second)(std::forward<TArgs>(args)...));
}
error(compression);
}
std::unique_ptr<osmium::io::Decompressor> create_decompressor(osmium::io::file_compression compression, int fd) {
auto it = m_callbacks.find(compression);
if (it != m_callbacks.end()) {
return std::unique_ptr<osmium::io::Decompressor>(std::get<1>(it->second)(fd));
}
error(compression);
}
std::unique_ptr<osmium::io::Decompressor> create_decompressor(osmium::io::file_compression compression, const char* buffer, size_t size) {
auto it = m_callbacks.find(compression);
if (it != m_callbacks.end()) {
return std::unique_ptr<osmium::io::Decompressor>(std::get<2>(it->second)(buffer, size));
}
error(compression);
}
}; // class CompressionFactory
class NoCompressor : public Compressor {
int m_fd;
public:
NoCompressor(int fd, fsync sync) :
Compressor(sync),
m_fd(fd) {
}
~NoCompressor() noexcept final {
try {
close();
} catch (...) {
// Ignore any exceptions because destructor must not throw.
}
}
void write(const std::string& data) final {
osmium::io::detail::reliable_write(m_fd, data.data(), data.size());
}
void close() final {
if (m_fd >= 0) {
int fd = m_fd;
m_fd = -1;
if (do_fsync()) {
osmium::io::detail::reliable_fsync(fd);
}
osmium::io::detail::reliable_close(fd);
}
}
}; // class NoCompressor
class NoDecompressor : public Decompressor {
int m_fd;
const char *m_buffer;
size_t m_buffer_size;
public:
explicit NoDecompressor(int fd) :
Decompressor(),
m_fd(fd),
m_buffer(nullptr),
m_buffer_size(0) {
}
NoDecompressor(const char* buffer, size_t size) :
Decompressor(),
m_fd(-1),
m_buffer(buffer),
m_buffer_size(size) {
}
~NoDecompressor() noexcept final {
try {
close();
} catch (...) {
// Ignore any exceptions because destructor must not throw.
}
}
std::string read() final {
std::string buffer;
if (m_buffer) {
if (m_buffer_size != 0) {
size_t size = m_buffer_size;
m_buffer_size = 0;
buffer.append(m_buffer, size);
}
} else {
buffer.resize(osmium::io::Decompressor::input_buffer_size);
auto nread = ::read(m_fd, const_cast<char*>(buffer.data()), osmium::io::Decompressor::input_buffer_size);
if (nread < 0) {
throw std::system_error(errno, std::system_category(), "Read failed");
}
buffer.resize(std::string::size_type(nread));
}
return buffer;
}
void close() final {
if (m_fd >= 0) {
int fd = m_fd;
m_fd = -1;
osmium::io::detail::reliable_close(fd);
}
}
}; // class NoDecompressor
namespace detail {
// we want the register_compression() function to run, setting
// the variable is only a side-effect, it will never be used
const bool registered_no_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::none,
[](int fd, fsync sync) { return new osmium::io::NoCompressor(fd, sync); },
[](int fd) { return new osmium::io::NoDecompressor(fd); },
[](const char* buffer, size_t size) { return new osmium::io::NoDecompressor(buffer, size); }
);
// dummy function to silence the unused variable warning from above
inline bool get_registered_no_compression() noexcept {
return registered_no_compression;
}
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_COMPRESSION_HPP
third_party/libosmium/include/osmium/io/debug_output.hpp
Vendored
-39
View File
@@ -1,39 +0,0 @@
#ifndef OSMIUM_IO_DEBUG_OUTPUT_HPP
#define OSMIUM_IO_DEBUG_OUTPUT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <osmium/io/writer.hpp> // IWYU pragma: export
#include <osmium/io/detail/debug_output_format.hpp> // IWYU pragma: export
#endif // OSMIUM_IO_DEBUG_OUTPUT_HPP
third_party/libosmium/include/osmium/io/detail/debug_output_format.hpp
Vendored
-485
View File
@@ -1,485 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP
#define OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cinttypes>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <future>
#include <iterator>
#include <memory>
#include <string>
#include <thread>
#include <utility>
#include <osmium/io/detail/output_format.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/memory/collection.hpp>
#include <osmium/osm/box.hpp>
#include <osmium/osm/changeset.hpp>
#include <osmium/osm/item_type.hpp>
#include <osmium/osm/location.hpp>
#include <osmium/osm/node.hpp>
#include <osmium/osm/object.hpp>
#include <osmium/osm/relation.hpp>
#include <osmium/osm/tag.hpp>
#include <osmium/osm/timestamp.hpp>
#include <osmium/osm/way.hpp>
#include <osmium/thread/pool.hpp>
#include <osmium/util/minmax.hpp>
#include <osmium/visitor.hpp>
namespace osmium {
namespace io {
class File;
namespace detail {
constexpr const char* color_bold = "\x1b[1m";
constexpr const char* color_black = "\x1b[30m";
constexpr const char* color_gray = "\x1b[30;1m";
constexpr const char* color_red = "\x1b[31m";
constexpr const char* color_green = "\x1b[32m";
constexpr const char* color_yellow = "\x1b[33m";
constexpr const char* color_blue = "\x1b[34m";
constexpr const char* color_magenta = "\x1b[35m";
constexpr const char* color_cyan = "\x1b[36m";
constexpr const char* color_white = "\x1b[37m";
constexpr const char* color_reset = "\x1b[0m";
struct debug_output_options {
/// Should metadata of objects be added?
bool add_metadata;
/// Output with ANSI colors?
bool use_color;
};
/**
* Writes out one buffer with OSM data in Debug format.
*/
class DebugOutputBlock : public OutputBlock {
debug_output_options m_options;
const char* m_utf8_prefix = "";
const char* m_utf8_suffix = "";
void append_encoded_string(const char* data) {
append_debug_encoded_string(*m_out, data, m_utf8_prefix, m_utf8_suffix);
}
void write_color(const char* color) {
if (m_options.use_color) {
*m_out += color;
}
}
void write_string(const char* string) {
*m_out += '"';
write_color(color_blue);
append_encoded_string(string);
write_color(color_reset);
*m_out += '"';
}
void write_object_type(const char* object_type, bool visible = true) {
if (visible) {
write_color(color_bold);
} else {
write_color(color_white);
}
*m_out += object_type;
write_color(color_reset);
*m_out += ' ';
}
void write_fieldname(const char* name) {
*m_out += " ";
write_color(color_cyan);
*m_out += name;
write_color(color_reset);
*m_out += ": ";
}
void write_comment_field(const char* name) {
write_color(color_cyan);
*m_out += name;
write_color(color_reset);
*m_out += ": ";
}
void write_counter(int width, int n) {
write_color(color_white);
output_formatted(" %0*d: ", width, n++);
write_color(color_reset);
}
void write_error(const char* msg) {
write_color(color_red);
*m_out += msg;
write_color(color_reset);
}
void write_timestamp(const osmium::Timestamp& timestamp) {
if (timestamp.valid()) {
*m_out += timestamp.to_iso();
output_formatted(" (%d)", timestamp.seconds_since_epoch());
} else {
write_error("NOT SET");
}
*m_out += '\n';
}
void write_meta(const osmium::OSMObject& object) {
output_formatted("%" PRId64 "\n", object.id());
if (m_options.add_metadata) {
write_fieldname("version");
output_formatted(" %d", object.version());
if (object.visible()) {
*m_out += " visible\n";
} else {
write_error(" deleted\n");
}
write_fieldname("changeset");
output_formatted("%d\n", object.changeset());
write_fieldname("timestamp");
write_timestamp(object.timestamp());
write_fieldname("user");
output_formatted(" %d ", object.uid());
write_string(object.user());
*m_out += '\n';
}
}
void write_tags(const osmium::TagList& tags, const char* padding="") {
if (!tags.empty()) {
write_fieldname("tags");
*m_out += padding;
output_formatted(" %d\n", tags.size());
osmium::max_op<size_t> max;
for (const auto& tag : tags) {
max.update(std::strlen(tag.key()));
}
for (const auto& tag : tags) {
*m_out += " ";
write_string(tag.key());
auto spacing = max() - std::strlen(tag.key());
while (spacing--) {
*m_out += " ";
}
*m_out += " = ";
write_string(tag.value());
*m_out += '\n';
}
}
}
void write_location(const osmium::Location& location) {
write_fieldname("lon/lat");
output_formatted(" %.7f,%.7f", location.lon_without_check(), location.lat_without_check());
if (!location.valid()) {
write_error(" INVALID LOCATION!");
}
*m_out += '\n';
}
void write_box(const osmium::Box& box) {
write_fieldname("box l/b/r/t");
if (!box) {
write_error("BOX NOT SET!\n");
return;
}
const auto& bl = box.bottom_left();
const auto& tr = box.top_right();
output_formatted("%.7f,%.7f %.7f,%.7f", bl.lon_without_check(), bl.lat_without_check(), tr.lon_without_check(), tr.lat_without_check());
if (!box.valid()) {
write_error(" INVALID BOX!");
}
*m_out += '\n';
}
public:
DebugOutputBlock(osmium::memory::Buffer&& buffer, const debug_output_options& options) :
OutputBlock(std::move(buffer)),
m_options(options),
m_utf8_prefix(options.use_color ? color_red : ""),
m_utf8_suffix(options.use_color ? color_blue : "") {
}
DebugOutputBlock(const DebugOutputBlock&) = default;
DebugOutputBlock& operator=(const DebugOutputBlock&) = default;
DebugOutputBlock(DebugOutputBlock&&) = default;
DebugOutputBlock& operator=(DebugOutputBlock&&) = default;
~DebugOutputBlock() noexcept = default;
std::string operator()() {
osmium::apply(m_input_buffer->cbegin(), m_input_buffer->cend(), *this);
std::string out;
using std::swap;
swap(out, *m_out);
return out;
}
void node(const osmium::Node& node) {
write_object_type("node", node.visible());
write_meta(node);
if (node.visible()) {
write_location(node.location());
}
write_tags(node.tags());
*m_out += '\n';
}
void way(const osmium::Way& way) {
write_object_type("way", way.visible());
write_meta(way);
write_tags(way.tags());
write_fieldname("nodes");
output_formatted(" %d", way.nodes().size());
if (way.nodes().size() < 2) {
write_error(" LESS THAN 2 NODES!\n");
} else if (way.nodes().size() > 2000) {
write_error(" MORE THAN 2000 NODES!\n");
} else if (way.nodes().is_closed()) {
*m_out += " (closed)\n";
} else {
*m_out += " (open)\n";
}
int width = int(log10(way.nodes().size())) + 1;
int n = 0;
for (const auto& node_ref : way.nodes()) {
write_counter(width, n++);
output_formatted("%10" PRId64, node_ref.ref());
if (node_ref.location().valid()) {
output_formatted(" (%.7f,%.7f)", node_ref.location().lon_without_check(), node_ref.location().lat_without_check());
}
*m_out += '\n';
}
*m_out += '\n';
}
void relation(const osmium::Relation& relation) {
static const char* short_typename[] = { "node", "way ", "rel " };
write_object_type("relation", relation.visible());
write_meta(relation);
write_tags(relation.tags());
write_fieldname("members");
output_formatted(" %d\n", relation.members().size());
int width = int(log10(relation.members().size())) + 1;
int n = 0;
for (const auto& member : relation.members()) {
write_counter(width, n++);
*m_out += short_typename[item_type_to_nwr_index(member.type())];
output_formatted(" %10" PRId64 " ", member.ref());
write_string(member.role());
*m_out += '\n';
}
*m_out += '\n';
}
void changeset(const osmium::Changeset& changeset) {
write_object_type("changeset");
output_formatted("%d\n", changeset.id());
write_fieldname("num changes");
output_formatted("%d", changeset.num_changes());
if (changeset.num_changes() == 0) {
write_error(" NO CHANGES!");
}
*m_out += '\n';
write_fieldname("created at");
*m_out += ' ';
write_timestamp(changeset.created_at());
write_fieldname("closed at");
*m_out += " ";
if (changeset.closed()) {
write_timestamp(changeset.closed_at());
} else {
write_error("OPEN!\n");
}
write_fieldname("user");
output_formatted(" %d ", changeset.uid());
write_string(changeset.user());
*m_out += '\n';
write_box(changeset.bounds());
write_tags(changeset.tags(), " ");
if (changeset.num_comments() > 0) {
write_fieldname("comments");
output_formatted(" %d\n", changeset.num_comments());
int width = int(log10(changeset.num_comments())) + 1;
int n = 0;
for (const auto& comment : changeset.discussion()) {
write_counter(width, n++);
write_comment_field("date");
write_timestamp(comment.date());
output_formatted(" %*s", width, "");
write_comment_field("user");
output_formatted("%d ", comment.uid());
write_string(comment.user());
output_formatted("\n %*s", width, "");
write_comment_field("text");
write_string(comment.text());
*m_out += '\n';
}
}
*m_out += '\n';
}
}; // class DebugOutputBlock
class DebugOutputFormat : public osmium::io::detail::OutputFormat {
debug_output_options m_options;
void write_fieldname(std::string& out, const char* name) {
out += " ";
if (m_options.use_color) {
out += color_cyan;
}
out += name;
if (m_options.use_color) {
out += color_reset;
}
out += ": ";
}
public:
DebugOutputFormat(const osmium::io::File& file, future_string_queue_type& output_queue) :
OutputFormat(output_queue),
m_options() {
m_options.add_metadata = file.is_not_false("add_metadata");
m_options.use_color = file.is_true("color");
}
DebugOutputFormat(const DebugOutputFormat&) = delete;
DebugOutputFormat& operator=(const DebugOutputFormat&) = delete;
~DebugOutputFormat() noexcept final = default;
void write_header(const osmium::io::Header& header) final {
std::string out;
if (m_options.use_color) {
out += color_bold;
}
out += "header\n";
if (m_options.use_color) {
out += color_reset;
}
write_fieldname(out, "multiple object versions");
out += header.has_multiple_object_versions() ? "yes" : "no";
out += '\n';
write_fieldname(out, "bounding boxes");
out += '\n';
for (const auto& box : header.boxes()) {
out += " ";
box.bottom_left().as_string(std::back_inserter(out), ',');
out += " ";
box.top_right().as_string(std::back_inserter(out), ',');
out += '\n';
}
write_fieldname(out, "options");
out += '\n';
for (const auto& opt : header) {
out += " ";
out += opt.first;
out += " = ";
out += opt.second;
out += '\n';
}
out += "\n=============================================\n\n";
send_to_output_queue(std::move(out));
}
void write_buffer(osmium::memory::Buffer&& buffer) final {
m_output_queue.push(osmium::thread::Pool::instance().submit(DebugOutputBlock{std::move(buffer), m_options}));
}
}; // class DebugOutputFormat
// we want the register_output_format() function to run, setting
// the variable is only a side-effect, it will never be used
const bool registered_debug_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::debug,
[](const osmium::io::File& file, future_string_queue_type& output_queue) {
return new osmium::io::detail::DebugOutputFormat(file, output_queue);
});
// dummy function to silence the unused variable warning from above
inline bool get_registered_debug_output() noexcept {
return registered_debug_output;
}
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP
third_party/libosmium/include/osmium/io/detail/input_format.hpp
Vendored
-211
View File
@@ -1,211 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_INPUT_FORMAT_HPP
#define OSMIUM_IO_DETAIL_INPUT_FORMAT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <exception>
#include <functional>
#include <future>
#include <map>
#include <memory>
#include <stdexcept>
#include <string>
#include <utility>
#include <osmium/io/detail/queue_util.hpp>
#include <osmium/io/file.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/io/header.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/osm/entity_bits.hpp>
namespace osmium {
namespace io {
namespace detail {
class Parser {
future_buffer_queue_type& m_output_queue;
std::promise<osmium::io::Header>& m_header_promise;
queue_wrapper<std::string> m_input_queue;
osmium::osm_entity_bits::type m_read_types;
bool m_header_is_done;
protected:
std::string get_input() {
return m_input_queue.pop();
}
bool input_done() const {
return m_input_queue.has_reached_end_of_data();
}
osmium::osm_entity_bits::type read_types() const {
return m_read_types;
}
bool header_is_done() const {
return m_header_is_done;
}
void set_header_value(const osmium::io::Header& header) {
if (!m_header_is_done) {
m_header_is_done = true;
m_header_promise.set_value(header);
}
}
void set_header_exception(const std::exception_ptr& exception) {
if (!m_header_is_done) {
m_header_is_done = true;
m_header_promise.set_exception(exception);
}
}
/**
* Wrap the buffer into a future and add it to the output queue.
*/
void send_to_output_queue(osmium::memory::Buffer&& buffer) {
add_to_queue(m_output_queue, std::move(buffer));
}
void send_to_output_queue(std::future<osmium::memory::Buffer>&& future) {
m_output_queue.push(std::move(future));
}
public:
Parser(future_string_queue_type& input_queue,
future_buffer_queue_type& output_queue,
std::promise<osmium::io::Header>& header_promise,
osmium::osm_entity_bits::type read_types) :
m_output_queue(output_queue),
m_header_promise(header_promise),
m_input_queue(input_queue),
m_read_types(read_types),
m_header_is_done(false) {
}
Parser(const Parser&) = delete;
Parser& operator=(const Parser&) = delete;
Parser(Parser&&) = delete;
Parser& operator=(Parser&&) = delete;
virtual ~Parser() noexcept = default;
virtual void run() = 0;
void parse() {
try {
run();
} catch (...) {
std::exception_ptr exception = std::current_exception();
set_header_exception(exception);
add_to_queue(m_output_queue, std::move(exception));
}
add_end_of_data_to_queue(m_output_queue);
}
}; // class Parser
/**
* This factory class is used to create objects that decode OSM
* data written in a specified format.
*
* Do not use this class directly. Use the osmium::io::Reader
* class instead.
*/
class ParserFactory {
public:
typedef std::function<
std::unique_ptr<Parser>(
future_string_queue_type&,
future_buffer_queue_type&,
std::promise<osmium::io::Header>& header_promise,
osmium::osm_entity_bits::type read_which_entities
)
> create_parser_type;
private:
typedef std::map<osmium::io::file_format, create_parser_type> map_type;
map_type m_callbacks;
ParserFactory() :
m_callbacks() {
}
public:
static ParserFactory& instance() {
static ParserFactory factory;
return factory;
}
bool register_parser(osmium::io::file_format format, create_parser_type create_function) {
if (! m_callbacks.insert(map_type::value_type(format, create_function)).second) {
return false;
}
return true;
}
create_parser_type get_creator_function(const osmium::io::File& file) {
auto it = m_callbacks.find(file.format());
if (it == m_callbacks.end()) {
throw unsupported_file_format_error(
std::string("Can not open file '") +
file.filename() +
"' with type '" +
as_string(file.format()) +
"'. No support for reading this format in this program.");
}
return it->second;
}
}; // class ParserFactory
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_INPUT_FORMAT_HPP
third_party/libosmium/include/osmium/io/detail/o5m_input_format.hpp
Vendored
-636
View File
@@ -1,636 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_O5M_INPUT_FORMAT_HPP
#define OSMIUM_IO_DETAIL_O5M_INPUT_FORMAT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <future>
#include <memory>
#include <string>
#include <utility>
#include <protozero/varint.hpp>
#include <osmium/builder/builder.hpp>
#include <osmium/builder/osm_object_builder.hpp>
#include <osmium/io/detail/input_format.hpp>
#include <osmium/io/detail/queue_util.hpp>
#include <osmium/io/error.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/io/header.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/osm.hpp>
#include <osmium/osm/box.hpp>
#include <osmium/osm/entity_bits.hpp>
#include <osmium/osm/item_type.hpp>
#include <osmium/osm/location.hpp>
#include <osmium/osm/object.hpp>
#include <osmium/osm/types.hpp>
#include <osmium/thread/util.hpp>
#include <osmium/util/cast.hpp>
#include <osmium/util/delta.hpp>
namespace osmium {
/**
* Exception thrown when the o5m deocder failed. The exception contains
* (if available) information about the place where the error happened
* and the type of error.
*/
struct o5m_error : public io_error {
explicit o5m_error(const char* what) :
io_error(std::string("o5m format error: ") + what) {
}
}; // struct o5m_error
namespace io {
namespace detail {
// Implementation of the o5m/o5c file formats according to the
// description at http://wiki.openstreetmap.org/wiki/O5m .
class ReferenceTable {
// The following settings are from the o5m description:
// The maximum number of entries in this table.
const uint64_t number_of_entries = 15000;
// The size of one entry in the table.
const unsigned int entry_size = 256;
// The maximum length of a string in the table including
// two \0 bytes.
const unsigned int max_length = 250 + 2;
// The data is stored in this string. It is default constructed
// and then resized on demand the first time something is added.
// This is done because the ReferenceTable is in a O5mParser
// object which will be copied from one thread to another. This
// way the string is still small when it is copied.
std::string m_table;
unsigned int current_entry = 0;
public:
void clear() {
current_entry = 0;
}
void add(const char* string, size_t size) {
if (m_table.empty()) {
m_table.resize(entry_size * number_of_entries);
}
if (size <= max_length) {
std::copy_n(string, size, &m_table[current_entry * entry_size]);
if (++current_entry == number_of_entries) {
current_entry = 0;
}
}
}
const char* get(uint64_t index) const {
if (m_table.empty() || index == 0 || index > number_of_entries) {
throw o5m_error("reference to non-existing string in table");
}
auto entry = (current_entry + number_of_entries - index) % number_of_entries;
return &m_table[entry * entry_size];
}
}; // class ReferenceTable
class O5mParser : public Parser {
static constexpr int buffer_size = 2 * 1000 * 1000;
osmium::io::Header m_header;
osmium::memory::Buffer m_buffer;
std::string m_input;
const char* m_data;
const char* m_end;
ReferenceTable m_reference_table;
static int64_t zvarint(const char** data, const char* end) {
return protozero::decode_zigzag64(protozero::decode_varint(data, end));
}
bool ensure_bytes_available(size_t need_bytes) {
if ((m_end - m_data) >= long(need_bytes)) {
return true;
}
if (input_done() && (m_input.size() < need_bytes)) {
return false;
}
m_input.erase(0, m_data - m_input.data());
while (m_input.size() < need_bytes) {
std::string data = get_input();
if (input_done()) {
return false;
}
m_input.append(data);
}
m_data = m_input.data();
m_end = m_input.data() + m_input.size();
return true;
}
void check_header_magic() {
static const unsigned char header_magic[] = { 0xff, 0xe0, 0x04, 'o', '5' };
if (std::strncmp(reinterpret_cast<const char*>(header_magic), m_data, sizeof(header_magic))) {
throw o5m_error("wrong header magic");
}
m_data += sizeof(header_magic);
}
void check_file_type() {
if (*m_data == 'm') { // o5m data file
m_header.set_has_multiple_object_versions(false);
} else if (*m_data == 'c') { // o5c change file
m_header.set_has_multiple_object_versions(true);
} else {
throw o5m_error("wrong header magic");
}
m_data++;
}
void check_file_format_version() {
if (*m_data != '2') {
throw o5m_error("wrong header magic");
}
m_data++;
}
void decode_header() {
if (! ensure_bytes_available(7)) { // overall length of header
throw o5m_error("file too short (incomplete header info)");
}
check_header_magic();
check_file_type();
check_file_format_version();
}
void mark_header_as_done() {
set_header_value(m_header);
}
osmium::util::DeltaDecode<osmium::object_id_type> m_delta_id;
osmium::util::DeltaDecode<int64_t> m_delta_timestamp;
osmium::util::DeltaDecode<osmium::changeset_id_type> m_delta_changeset;
osmium::util::DeltaDecode<int64_t> m_delta_lon;
osmium::util::DeltaDecode<int64_t> m_delta_lat;
osmium::util::DeltaDecode<osmium::object_id_type> m_delta_way_node_id;
osmium::util::DeltaDecode<osmium::object_id_type> m_delta_member_ids[3];
void reset() {
m_reference_table.clear();
m_delta_id.clear();
m_delta_timestamp.clear();
m_delta_changeset.clear();
m_delta_lon.clear();
m_delta_lat.clear();
m_delta_way_node_id.clear();
m_delta_member_ids[0].clear();
m_delta_member_ids[1].clear();
m_delta_member_ids[2].clear();
}
const char* decode_string(const char** dataptr, const char* const end) {
if (**dataptr == 0x00) { // get inline string
(*dataptr)++;
if (*dataptr == end) {
throw o5m_error("string format error");
}
return *dataptr;
} else { // get from reference table
auto index = protozero::decode_varint(dataptr, end);
return m_reference_table.get(index);
}
}
std::pair<osmium::user_id_type, const char*> decode_user(const char** dataptr, const char* const end) {
bool update_pointer = (**dataptr == 0x00);
const char* data = decode_string(dataptr, end);
const char* start = data;
auto uid = protozero::decode_varint(&data, end);
if (data == end) {
throw o5m_error("missing user name");
}
const char* user = ++data;
if (uid == 0 && update_pointer) {
m_reference_table.add("\0\0", 2);
*dataptr = data;
return std::make_pair(0, "");
}
while (*data++) {
if (data == end) {
throw o5m_error("no null byte in user name");
}
}
if (update_pointer) {
m_reference_table.add(start, data - start);
*dataptr = data;
}
return std::make_pair(static_cast_with_assert<osmium::user_id_type>(uid), user);
}
void decode_tags(osmium::builder::Builder* builder, const char** dataptr, const char* const end) {
osmium::builder::TagListBuilder tl_builder(m_buffer, builder);
while(*dataptr != end) {
bool update_pointer = (**dataptr == 0x00);
const char* data = decode_string(dataptr, end);
const char* start = data;
while (*data++) {
if (data == end) {
throw o5m_error("no null byte in tag key");
}
}
const char* value = data;
while (*data++) {
if (data == end) {
throw o5m_error("no null byte in tag value");
}
}
if (update_pointer) {
m_reference_table.add(start, data - start);
*dataptr = data;
}
tl_builder.add_tag(start, value);
}
}
const char* decode_info(osmium::OSMObject& object, const char** dataptr, const char* const end) {
const char* user = "";
if (**dataptr == 0x00) { // no info section
++*dataptr;
} else { // has info section
object.set_version(static_cast_with_assert<object_version_type>(protozero::decode_varint(dataptr, end)));
auto timestamp = m_delta_timestamp.update(zvarint(dataptr, end));
if (timestamp != 0) { // has timestamp
object.set_timestamp(timestamp);
object.set_changeset(m_delta_changeset.update(zvarint(dataptr, end)));
if (*dataptr != end) {
auto uid_user = decode_user(dataptr, end);
object.set_uid(uid_user.first);
user = uid_user.second;
} else {
object.set_uid(user_id_type(0));
}
}
}
return user;
}
void decode_node(const char* data, const char* const end) {
osmium::builder::NodeBuilder builder(m_buffer);
osmium::Node& node = builder.object();
node.set_id(m_delta_id.update(zvarint(&data, end)));
builder.add_user(decode_info(node, &data, end));
if (data == end) {
// no location, object is deleted
builder.object().set_visible(false);
builder.object().set_location(osmium::Location{});
} else {
auto lon = m_delta_lon.update(zvarint(&data, end));
auto lat = m_delta_lat.update(zvarint(&data, end));
builder.object().set_location(osmium::Location{lon, lat});
if (data != end) {
decode_tags(&builder, &data, end);
}
}
m_buffer.commit();
}
void decode_way(const char* data, const char* const end) {
osmium::builder::WayBuilder builder(m_buffer);
osmium::Way& way = builder.object();
way.set_id(m_delta_id.update(zvarint(&data, end)));
builder.add_user(decode_info(way, &data, end));
if (data == end) {
// no reference section, object is deleted
builder.object().set_visible(false);
} else {
auto reference_section_length = protozero::decode_varint(&data, end);
if (reference_section_length > 0) {
const char* const end_refs = data + reference_section_length;
if (end_refs > end) {
throw o5m_error("way nodes ref section too long");
}
osmium::builder::WayNodeListBuilder wn_builder(m_buffer, &builder);
while (data < end_refs) {
wn_builder.add_node_ref(m_delta_way_node_id.update(zvarint(&data, end)));
}
}
if (data != end) {
decode_tags(&builder, &data, end);
}
}
m_buffer.commit();
}
osmium::item_type decode_member_type(char c) {
if (c < '0' || c > '2') {
throw o5m_error("unknown member type");
}
return osmium::nwr_index_to_item_type(c - '0');
}
std::pair<osmium::item_type, const char*> decode_role(const char** dataptr, const char* const end) {
bool update_pointer = (**dataptr == 0x00);
const char* data = decode_string(dataptr, end);
const char* start = data;
auto member_type = decode_member_type(*data++);
if (data == end) {
throw o5m_error("missing role");
}
const char* role = data;
while (*data++) {
if (data == end) {
throw o5m_error("no null byte in role");
}
}
if (update_pointer) {
m_reference_table.add(start, data - start);
*dataptr = data;
}
return std::make_pair(member_type, role);
}
void decode_relation(const char* data, const char* const end) {
osmium::builder::RelationBuilder builder(m_buffer);
osmium::Relation& relation = builder.object();
relation.set_id(m_delta_id.update(zvarint(&data, end)));
builder.add_user(decode_info(relation, &data, end));
if (data == end) {
// no reference section, object is deleted
builder.object().set_visible(false);
} else {
auto reference_section_length = protozero::decode_varint(&data, end);
if (reference_section_length > 0) {
const char* const end_refs = data + reference_section_length;
if (end_refs > end) {
throw o5m_error("relation format error");
}
osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder);
while (data < end_refs) {
auto delta_id = zvarint(&data, end);
if (data == end) {
throw o5m_error("relation member format error");
}
auto type_role = decode_role(&data, end);
auto i = osmium::item_type_to_nwr_index(type_role.first);
auto ref = m_delta_member_ids[i].update(delta_id);
rml_builder.add_member(type_role.first, ref, type_role.second);
}
}
if (data != end) {
decode_tags(&builder, &data, end);
}
}
m_buffer.commit();
}
void decode_bbox(const char* data, const char* const end) {
auto sw_lon = zvarint(&data, end);
auto sw_lat = zvarint(&data, end);
auto ne_lon = zvarint(&data, end);
auto ne_lat = zvarint(&data, end);
m_header.add_box(osmium::Box{osmium::Location{sw_lon, sw_lat},
osmium::Location{ne_lon, ne_lat}});
}
void decode_timestamp(const char* data, const char* const end) {
auto timestamp = osmium::Timestamp(zvarint(&data, end)).to_iso();
m_header.set("o5m_timestamp", timestamp);
m_header.set("timestamp", timestamp);
}
void flush() {
osmium::memory::Buffer buffer(buffer_size);
using std::swap;
swap(m_buffer, buffer);
send_to_output_queue(std::move(buffer));
}
enum class dataset_type : unsigned char {
node = 0x10,
way = 0x11,
relation = 0x12,
bounding_box = 0xdb,
timestamp = 0xdc,
header = 0xe0,
sync = 0xee,
jump = 0xef,
reset = 0xff
};
void decode_data() {
while (ensure_bytes_available(1)) {
dataset_type ds_type = dataset_type(*m_data++);
if (ds_type > dataset_type::jump) {
if (ds_type == dataset_type::reset) {
reset();
}
} else {
ensure_bytes_available(protozero::max_varint_length);
uint64_t length = 0;
try {
length = protozero::decode_varint(&m_data, m_end);
} catch (protozero::end_of_buffer_exception&) {
throw o5m_error("premature end of file");
}
if (! ensure_bytes_available(length)) {
throw o5m_error("premature end of file");
}
switch (ds_type) {
case dataset_type::node:
mark_header_as_done();
if (read_types() & osmium::osm_entity_bits::node) {
decode_node(m_data, m_data + length);
}
break;
case dataset_type::way:
mark_header_as_done();
if (read_types() & osmium::osm_entity_bits::way) {
decode_way(m_data, m_data + length);
}
break;
case dataset_type::relation:
mark_header_as_done();
if (read_types() & osmium::osm_entity_bits::relation) {
decode_relation(m_data, m_data + length);
}
break;
case dataset_type::bounding_box:
decode_bbox(m_data, m_data + length);
break;
case dataset_type::timestamp:
decode_timestamp(m_data, m_data + length);
break;
default:
// ignore unknown datasets
break;
}
if (read_types() == osmium::osm_entity_bits::nothing && header_is_done()) {
break;
}
m_data += length;
if (m_buffer.committed() > buffer_size / 10 * 9) {
flush();
}
}
}
if (m_buffer.committed()) {
flush();
}
mark_header_as_done();
}
public:
O5mParser(future_string_queue_type& input_queue,
future_buffer_queue_type& output_queue,
std::promise<osmium::io::Header>& header_promise,
osmium::osm_entity_bits::type read_types) :
Parser(input_queue, output_queue, header_promise, read_types),
m_header(),
m_buffer(buffer_size),
m_input(),
m_data(m_input.data()),
m_end(m_data) {
}
~O5mParser() noexcept final = default;
void run() final {
osmium::thread::set_thread_name("_osmium_o5m_in");
decode_header();
decode_data();
}
}; // class O5mParser
// we want the register_parser() function to run, setting
// the variable is only a side-effect, it will never be used
const bool registered_o5m_parser = ParserFactory::instance().register_parser(
file_format::o5m,
[](future_string_queue_type& input_queue,
future_buffer_queue_type& output_queue,
std::promise<osmium::io::Header>& header_promise,
osmium::osm_entity_bits::type read_which_entities) {
return std::unique_ptr<Parser>(new O5mParser(input_queue, output_queue, header_promise, read_which_entities));
});
// dummy function to silence the unused variable warning from above
inline bool get_registered_o5m_parser() noexcept {
return registered_o5m_parser;
}
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_O5M_INPUT_FORMAT_HPP
third_party/libosmium/include/osmium/io/detail/opl_output_format.hpp
Vendored
-261
View File
@@ -1,261 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_OPL_OUTPUT_FORMAT_HPP
#define OSMIUM_IO_DETAIL_OPL_OUTPUT_FORMAT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cinttypes>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <future>
#include <iterator>
#include <memory>
#include <string>
#include <thread>
#include <utility>
#include <osmium/io/detail/output_format.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/memory/collection.hpp>
#include <osmium/osm/box.hpp>
#include <osmium/osm/changeset.hpp>
#include <osmium/osm/item_type.hpp>
#include <osmium/osm/location.hpp>
#include <osmium/osm/node.hpp>
#include <osmium/osm/object.hpp>
#include <osmium/osm/relation.hpp>
#include <osmium/osm/tag.hpp>
#include <osmium/osm/timestamp.hpp>
#include <osmium/osm/way.hpp>
#include <osmium/thread/pool.hpp>
#include <osmium/visitor.hpp>
namespace osmium {
namespace io {
class File;
namespace detail {
struct opl_output_options {
/// Should metadata of objects be added?
bool add_metadata;
};
/**
* Writes out one buffer with OSM data in OPL format.
*/
class OPLOutputBlock : public OutputBlock {
opl_output_options m_options;
void append_encoded_string(const char* data) {
osmium::io::detail::append_utf8_encoded_string(*m_out, data);
}
void write_meta(const osmium::OSMObject& object) {
output_formatted("%" PRId64, object.id());
if (m_options.add_metadata) {
output_formatted(" v%d d", object.version());
*m_out += (object.visible() ? 'V' : 'D');
output_formatted(" c%d t", object.changeset());
*m_out += object.timestamp().to_iso();
output_formatted(" i%d u", object.uid());
append_encoded_string(object.user());
}
*m_out += " T";
bool first = true;
for (const auto& tag : object.tags()) {
if (first) {
first = false;
} else {
*m_out += ',';
}
append_encoded_string(tag.key());
*m_out += '=';
append_encoded_string(tag.value());
}
}
void write_location(const osmium::Location& location, const char x, const char y) {
if (location) {
output_formatted(" %c%.7f %c%.7f", x, location.lon_without_check(), y, location.lat_without_check());
} else {
*m_out += ' ';
*m_out += x;
*m_out += ' ';
*m_out += y;
}
}
public:
OPLOutputBlock(osmium::memory::Buffer&& buffer, const opl_output_options& options) :
OutputBlock(std::move(buffer)),
m_options(options) {
}
OPLOutputBlock(const OPLOutputBlock&) = default;
OPLOutputBlock& operator=(const OPLOutputBlock&) = default;
OPLOutputBlock(OPLOutputBlock&&) = default;
OPLOutputBlock& operator=(OPLOutputBlock&&) = default;
~OPLOutputBlock() noexcept = default;
std::string operator()() {
osmium::apply(m_input_buffer->cbegin(), m_input_buffer->cend(), *this);
std::string out;
using std::swap;
swap(out, *m_out);
return out;
}
void node(const osmium::Node& node) {
*m_out += 'n';
write_meta(node);
write_location(node.location(), 'x', 'y');
*m_out += '\n';
}
void way(const osmium::Way& way) {
*m_out += 'w';
write_meta(way);
*m_out += " N";
bool first = true;
for (const auto& node_ref : way.nodes()) {
if (first) {
first = false;
} else {
*m_out += ',';
}
output_formatted("n%" PRId64, node_ref.ref());
}
*m_out += '\n';
}
void relation(const osmium::Relation& relation) {
*m_out += 'r';
write_meta(relation);
*m_out += " M";
bool first = true;
for (const auto& member : relation.members()) {
if (first) {
first = false;
} else {
*m_out += ',';
}
*m_out += item_type_to_char(member.type());
output_formatted("%" PRId64 "@", member.ref());
append_encoded_string(member.role());
}
*m_out += '\n';
}
void changeset(const osmium::Changeset& changeset) {
output_formatted("c%d k%d s", changeset.id(), changeset.num_changes());
*m_out += changeset.created_at().to_iso();
*m_out += " e";
*m_out += changeset.closed_at().to_iso();
output_formatted(" d%d i%d u", changeset.num_comments(), changeset.uid());
append_encoded_string(changeset.user());
write_location(changeset.bounds().bottom_left(), 'x', 'y');
write_location(changeset.bounds().top_right(), 'X', 'Y');
*m_out += " T";
bool first = true;
for (const auto& tag : changeset.tags()) {
if (first) {
first = false;
} else {
*m_out += ',';
}
append_encoded_string(tag.key());
*m_out += '=';
append_encoded_string(tag.value());
}
*m_out += '\n';
}
}; // class OPLOutputBlock
class OPLOutputFormat : public osmium::io::detail::OutputFormat {
opl_output_options m_options;
public:
OPLOutputFormat(const osmium::io::File& file, future_string_queue_type& output_queue) :
OutputFormat(output_queue),
m_options() {
m_options.add_metadata = file.is_not_false("add_metadata");
}
OPLOutputFormat(const OPLOutputFormat&) = delete;
OPLOutputFormat& operator=(const OPLOutputFormat&) = delete;
~OPLOutputFormat() noexcept final = default;
void write_buffer(osmium::memory::Buffer&& buffer) final {
m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer), m_options}));
}
}; // class OPLOutputFormat
// we want the register_output_format() function to run, setting
// the variable is only a side-effect, it will never be used
const bool registered_opl_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::opl,
[](const osmium::io::File& file, future_string_queue_type& output_queue) {
return new osmium::io::detail::OPLOutputFormat(file, output_queue);
});
// dummy function to silence the unused variable warning from above
inline bool get_registered_opl_output() noexcept {
return registered_opl_output;
}
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_OPL_OUTPUT_FORMAT_HPP
third_party/libosmium/include/osmium/io/detail/output_format.hpp
Vendored
-184
View File
@@ -1,184 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_OUTPUT_FORMAT_HPP
#define OSMIUM_IO_DETAIL_OUTPUT_FORMAT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <functional>
#include <map>
#include <memory>
#include <stdexcept>
#include <string>
#include <utility>
#include <osmium/handler.hpp>
#include <osmium/io/detail/queue_util.hpp>
#include <osmium/io/detail/string_util.hpp>
#include <osmium/io/file.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/memory/buffer.hpp>
namespace osmium {
namespace io {
class Header;
} // namespace io
namespace io {
namespace detail {
class OutputBlock : public osmium::handler::Handler {
protected:
std::shared_ptr<osmium::memory::Buffer> m_input_buffer;
std::shared_ptr<std::string> m_out;
explicit OutputBlock(osmium::memory::Buffer&& buffer) :
m_input_buffer(std::make_shared<osmium::memory::Buffer>(std::move(buffer))),
m_out(std::make_shared<std::string>()) {
}
template <typename... TArgs>
void output_formatted(const char* format, TArgs&&... args) {
append_printf_formatted_string(*m_out, format, std::forward<TArgs>(args)...);
}
}; // class OutputBlock;
/**
* Virtual base class for all classes writing OSM files in different
* formats.
*
* Do not use this class or derived classes directly. Use the
* osmium::io::Writer class instead.
*/
class OutputFormat {
protected:
future_string_queue_type& m_output_queue;
/**
* Wrap the string into a future and add it to the output
* queue.
*/
void send_to_output_queue(std::string&& data) {
add_to_queue(m_output_queue, std::move(data));
}
public:
explicit OutputFormat(future_string_queue_type& output_queue) :
m_output_queue(output_queue) {
}
OutputFormat(const OutputFormat&) = delete;
OutputFormat(OutputFormat&&) = delete;
OutputFormat& operator=(const OutputFormat&) = delete;
OutputFormat& operator=(OutputFormat&&) = delete;
virtual ~OutputFormat() noexcept = default;
virtual void write_header(const osmium::io::Header&) {
}
virtual void write_buffer(osmium::memory::Buffer&&) = 0;
virtual void write_end() {
}
}; // class OutputFormat
/**
* This factory class is used to create objects that write OSM data
* into a specified output format.
*
* Do not use this class directly. Instead use the osmium::io::Writer
* class.
*/
class OutputFormatFactory {
public:
typedef std::function<osmium::io::detail::OutputFormat*(const osmium::io::File&, future_string_queue_type&)> create_output_type;
private:
typedef std::map<osmium::io::file_format, create_output_type> map_type;
map_type m_callbacks;
OutputFormatFactory() :
m_callbacks() {
}
public:
static OutputFormatFactory& instance() {
static OutputFormatFactory factory;
return factory;
}
bool register_output_format(osmium::io::file_format format, create_output_type create_function) {
if (! m_callbacks.insert(map_type::value_type(format, create_function)).second) {
return false;
}
return true;
}
std::unique_ptr<osmium::io::detail::OutputFormat> create_output(const osmium::io::File& file, future_string_queue_type& output_queue) {
auto it = m_callbacks.find(file.format());
if (it != m_callbacks.end()) {
return std::unique_ptr<osmium::io::detail::OutputFormat>((it->second)(file, output_queue));
}
throw unsupported_file_format_error(
std::string("Can not open file '") +
file.filename() +
"' with type '" +
as_string(file.format()) +
"'. No support for writing this format in this program.");
}
}; // class OutputFormatFactory
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_OUTPUT_FORMAT_HPP
third_party/libosmium/include/osmium/io/detail/pbf.hpp
Vendored
-89
View File
@@ -1,89 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_PBF_HPP
#define OSMIUM_IO_DETAIL_PBF_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cstdint>
#include <string>
// needed for htonl and ntohl
#ifndef _WIN32
# include <netinet/in.h>
#else
# include <winsock2.h>
#endif
#include <osmium/io/error.hpp>
#include <osmium/osm/location.hpp>
namespace osmium {
/**
* Exception thrown when there was a problem with parsing the PBF format of
* a file.
*/
struct pbf_error : public io_error {
explicit pbf_error(const std::string& what) :
io_error(std::string("PBF error: ") + what) {
}
explicit pbf_error(const char* what) :
io_error(std::string("PBF error: ") + what) {
}
}; // struct pbf_error
namespace io {
namespace detail {
// the maximum size of a blob header in bytes
const int max_blob_header_size = 64 * 1024; // 64 kB
// the maximum size of an uncompressed blob in bytes
const uint64_t max_uncompressed_blob_size = 32 * 1024 * 1024; // 32 MB
// resolution for longitude/latitude used for conversion
// between representation as double and as int
const int64_t lonlat_resolution = 1000 * 1000 * 1000;
const int64_t resolution_convert = lonlat_resolution / osmium::Location::coordinate_precision;
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_PBF_HPP
third_party/libosmium/include/osmium/io/detail/pbf_decoder.hpp
Vendored
-777
View File
@@ -1,777 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_PBF_DECODER_HPP
#define OSMIUM_IO_DETAIL_PBF_DECODER_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <algorithm>
#include <limits>
#include <memory>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
#include <protozero/pbf_message.hpp>
#include <osmium/builder/osm_object_builder.hpp>
#include <osmium/io/detail/pbf.hpp> // IWYU pragma: export
#include <osmium/io/detail/protobuf_tags.hpp>
#include <osmium/io/detail/zlib.hpp>
#include <osmium/io/header.hpp>
#include <osmium/osm/location.hpp>
#include <osmium/osm/node.hpp>
#include <osmium/osm/types.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/osm/entity_bits.hpp>
#include <osmium/util/cast.hpp>
#include <osmium/util/delta.hpp>
namespace osmium {
namespace io {
namespace detail {
using ptr_len_type = std::pair<const char*, size_t>;
using osm_string_len_type = std::pair<const char*, osmium::string_size_type>;
class PBFPrimitiveBlockDecoder {
static constexpr size_t initial_buffer_size = 2 * 1024 * 1024;
ptr_len_type m_data;
std::vector<osm_string_len_type> m_stringtable;
int64_t m_lon_offset = 0;
int64_t m_lat_offset = 0;
int64_t m_date_factor = 1000;
int32_t m_granularity = 100;
osmium::osm_entity_bits::type m_read_types;
osmium::memory::Buffer m_buffer { initial_buffer_size };
void decode_stringtable(const ptr_len_type& data) {
if (!m_stringtable.empty()) {
throw osmium::pbf_error("more than one stringtable in pbf file");
}
protozero::pbf_message<OSMFormat::StringTable> pbf_string_table(data);
while (pbf_string_table.next(OSMFormat::StringTable::repeated_bytes_s)) {
auto str_len = pbf_string_table.get_data();
if (str_len.second > osmium::max_osm_string_length) {
throw osmium::pbf_error("overlong string in string table");
}
m_stringtable.emplace_back(str_len.first, osmium::string_size_type(str_len.second));
}
}
void decode_primitive_block_metadata() {
protozero::pbf_message<OSMFormat::PrimitiveBlock> pbf_primitive_block(m_data);
while (pbf_primitive_block.next()) {
switch (pbf_primitive_block.tag()) {
case OSMFormat::PrimitiveBlock::required_StringTable_stringtable:
decode_stringtable(pbf_primitive_block.get_data());
break;
case OSMFormat::PrimitiveBlock::optional_int32_granularity:
m_granularity = pbf_primitive_block.get_int32();
break;
case OSMFormat::PrimitiveBlock::optional_int32_date_granularity:
m_date_factor = pbf_primitive_block.get_int32();
break;
case OSMFormat::PrimitiveBlock::optional_int64_lat_offset:
m_lat_offset = pbf_primitive_block.get_int64();
break;
case OSMFormat::PrimitiveBlock::optional_int64_lon_offset:
m_lon_offset = pbf_primitive_block.get_int64();
break;
default:
pbf_primitive_block.skip();
}
}
}
void decode_primitive_block_data() {
protozero::pbf_message<OSMFormat::PrimitiveBlock> pbf_primitive_block(m_data);
while (pbf_primitive_block.next(OSMFormat::PrimitiveBlock::repeated_PrimitiveGroup_primitivegroup)) {
protozero::pbf_message<OSMFormat::PrimitiveGroup> pbf_primitive_group = pbf_primitive_block.get_message();
while (pbf_primitive_group.next()) {
switch (pbf_primitive_group.tag()) {
case OSMFormat::PrimitiveGroup::repeated_Node_nodes:
if (m_read_types & osmium::osm_entity_bits::node) {
decode_node(pbf_primitive_group.get_data());
} else {
pbf_primitive_group.skip();
}
break;
case OSMFormat::PrimitiveGroup::optional_DenseNodes_dense:
if (m_read_types & osmium::osm_entity_bits::node) {
decode_dense_nodes(pbf_primitive_group.get_data());
} else {
pbf_primitive_group.skip();
}
break;
case OSMFormat::PrimitiveGroup::repeated_Way_ways:
if (m_read_types & osmium::osm_entity_bits::way) {
decode_way(pbf_primitive_group.get_data());
} else {
pbf_primitive_group.skip();
}
break;
case OSMFormat::PrimitiveGroup::repeated_Relation_relations:
if (m_read_types & osmium::osm_entity_bits::relation) {
decode_relation(pbf_primitive_group.get_data());
} else {
pbf_primitive_group.skip();
}
break;
default:
pbf_primitive_group.skip();
}
}
}
}
osm_string_len_type decode_info(const ptr_len_type& data, osmium::OSMObject& object) {
osm_string_len_type user = std::make_pair("", 0);
protozero::pbf_message<OSMFormat::Info> pbf_info(data);
while (pbf_info.next()) {
switch (pbf_info.tag()) {
case OSMFormat::Info::optional_int32_version:
{
auto version = pbf_info.get_int32();
if (version < 0) {
throw osmium::pbf_error("object version must not be negative");
}
object.set_version(static_cast_with_assert<object_version_type>(version));
}
break;
case OSMFormat::Info::optional_int64_timestamp:
object.set_timestamp(pbf_info.get_int64() * m_date_factor / 1000);
break;
case OSMFormat::Info::optional_int64_changeset:
{
auto changeset_id = pbf_info.get_int64();
if (changeset_id < 0) {
throw osmium::pbf_error("object changeset_id must not be negative");
}
object.set_changeset(static_cast_with_assert<changeset_id_type>(changeset_id));
}
break;
case OSMFormat::Info::optional_int32_uid:
object.set_uid_from_signed(pbf_info.get_int32());
break;
case OSMFormat::Info::optional_uint32_user_sid:
user = m_stringtable.at(pbf_info.get_uint32());
break;
case OSMFormat::Info::optional_bool_visible:
object.set_visible(pbf_info.get_bool());
break;
default:
pbf_info.skip();
}
}
return user;
}
using kv_type = std::pair<protozero::pbf_reader::const_uint32_iterator, protozero::pbf_reader::const_uint32_iterator>;
void build_tag_list(osmium::builder::Builder& builder, const kv_type& keys, const kv_type& vals) {
if (keys.first != keys.second) {
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
auto kit = keys.first;
auto vit = vals.first;
while (kit != keys.second) {
if (vit == vals.second) {
// this is against the spec, must have same number of elements
throw osmium::pbf_error("PBF format error");
}
const auto& k = m_stringtable.at(*kit++);
const auto& v = m_stringtable.at(*vit++);
tl_builder.add_tag(k.first, k.second, v.first, v.second);
}
}
}
int32_t convert_pbf_coordinate(int64_t c) const {
return int32_t((c * m_granularity + m_lon_offset) / resolution_convert);
}
void decode_node(const ptr_len_type& data) {
osmium::builder::NodeBuilder builder(m_buffer);
osmium::Node& node = builder.object();
kv_type keys;
kv_type vals;
int64_t lon = std::numeric_limits<int64_t>::max();
int64_t lat = std::numeric_limits<int64_t>::max();
osm_string_len_type user = { "", 0 };
protozero::pbf_message<OSMFormat::Node> pbf_node(data);
while (pbf_node.next()) {
switch (pbf_node.tag()) {
case OSMFormat::Node::required_sint64_id:
node.set_id(pbf_node.get_sint64());
break;
case OSMFormat::Node::packed_uint32_keys:
keys = pbf_node.get_packed_uint32();
break;
case OSMFormat::Node::packed_uint32_vals:
vals = pbf_node.get_packed_uint32();
break;
case OSMFormat::Node::optional_Info_info:
user = decode_info(pbf_node.get_data(), builder.object());
break;
case OSMFormat::Node::required_sint64_lat:
lat = pbf_node.get_sint64();
break;
case OSMFormat::Node::required_sint64_lon:
lon = pbf_node.get_sint64();
break;
default:
pbf_node.skip();
}
}
if (node.visible()) {
if (lon == std::numeric_limits<int64_t>::max() ||
lat == std::numeric_limits<int64_t>::max()) {
throw osmium::pbf_error("illegal coordinate format");
}
node.set_location(osmium::Location(
convert_pbf_coordinate(lon),
convert_pbf_coordinate(lat)
));
}
builder.add_user(user.first, user.second);
build_tag_list(builder, keys, vals);
m_buffer.commit();
}
void decode_way(const ptr_len_type& data) {
osmium::builder::WayBuilder builder(m_buffer);
kv_type keys;
kv_type vals;
std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> refs;
osm_string_len_type user = { "", 0 };
protozero::pbf_message<OSMFormat::Way> pbf_way(data);
while (pbf_way.next()) {
switch (pbf_way.tag()) {
case OSMFormat::Way::required_int64_id:
builder.object().set_id(pbf_way.get_int64());
break;
case OSMFormat::Way::packed_uint32_keys:
keys = pbf_way.get_packed_uint32();
break;
case OSMFormat::Way::packed_uint32_vals:
vals = pbf_way.get_packed_uint32();
break;
case OSMFormat::Way::optional_Info_info:
user = decode_info(pbf_way.get_data(), builder.object());
break;
case OSMFormat::Way::packed_sint64_refs:
refs = pbf_way.get_packed_sint64();
break;
default:
pbf_way.skip();
}
}
builder.add_user(user.first, user.second);
if (refs.first != refs.second) {
osmium::builder::WayNodeListBuilder wnl_builder(m_buffer, &builder);
osmium::util::DeltaDecode<int64_t> ref;
while (refs.first != refs.second) {
wnl_builder.add_node_ref(ref.update(*refs.first++));
}
}
build_tag_list(builder, keys, vals);
m_buffer.commit();
}
void decode_relation(const ptr_len_type& data) {
osmium::builder::RelationBuilder builder(m_buffer);
kv_type keys;
kv_type vals;
std::pair<protozero::pbf_reader::const_int32_iterator, protozero::pbf_reader::const_int32_iterator> roles;
std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> refs;
std::pair<protozero::pbf_reader::const_int32_iterator, protozero::pbf_reader::const_int32_iterator> types;
osm_string_len_type user = { "", 0 };
protozero::pbf_message<OSMFormat::Relation> pbf_relation(data);
while (pbf_relation.next()) {
switch (pbf_relation.tag()) {
case OSMFormat::Relation::required_int64_id:
builder.object().set_id(pbf_relation.get_int64());
break;
case OSMFormat::Relation::packed_uint32_keys:
keys = pbf_relation.get_packed_uint32();
break;
case OSMFormat::Relation::packed_uint32_vals:
vals = pbf_relation.get_packed_uint32();
break;
case OSMFormat::Relation::optional_Info_info:
user = decode_info(pbf_relation.get_data(), builder.object());
break;
case OSMFormat::Relation::packed_int32_roles_sid:
roles = pbf_relation.get_packed_int32();
break;
case OSMFormat::Relation::packed_sint64_memids:
refs = pbf_relation.get_packed_sint64();
break;
case OSMFormat::Relation::packed_MemberType_types:
types = pbf_relation.get_packed_enum();
break;
default:
pbf_relation.skip();
}
}
builder.add_user(user.first, user.second);
if (refs.first != refs.second) {
osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder);
osmium::util::DeltaDecode<int64_t> ref;
while (roles.first != roles.second && refs.first != refs.second && types.first != types.second) {
const auto& r = m_stringtable.at(*roles.first++);
int type = *types.first++;
if (type < 0 || type > 2) {
throw osmium::pbf_error("unknown relation member type");
}
rml_builder.add_member(
osmium::item_type(type + 1),
ref.update(*refs.first++),
r.first,
r.second
);
}
}
build_tag_list(builder, keys, vals);
m_buffer.commit();
}
void decode_dense_nodes(const ptr_len_type& data) {
bool has_info = false;
bool has_visibles = false;
std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> ids;
std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> lats;
std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> lons;
std::pair<protozero::pbf_reader::const_int32_iterator, protozero::pbf_reader::const_int32_iterator> tags;
std::pair<protozero::pbf_reader::const_int32_iterator, protozero::pbf_reader::const_int32_iterator> versions;
std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> timestamps;
std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> changesets;
std::pair<protozero::pbf_reader::const_sint32_iterator, protozero::pbf_reader::const_sint32_iterator> uids;
std::pair<protozero::pbf_reader::const_sint32_iterator, protozero::pbf_reader::const_sint32_iterator> user_sids;
std::pair<protozero::pbf_reader::const_int32_iterator, protozero::pbf_reader::const_int32_iterator> visibles;
protozero::pbf_message<OSMFormat::DenseNodes> pbf_dense_nodes(data);
while (pbf_dense_nodes.next()) {
switch (pbf_dense_nodes.tag()) {
case OSMFormat::DenseNodes::packed_sint64_id:
ids = pbf_dense_nodes.get_packed_sint64();
break;
case OSMFormat::DenseNodes::optional_DenseInfo_denseinfo:
{
has_info = true;
protozero::pbf_message<OSMFormat::DenseInfo> pbf_dense_info = pbf_dense_nodes.get_message();
while (pbf_dense_info.next()) {
switch (pbf_dense_info.tag()) {
case OSMFormat::DenseInfo::packed_int32_version:
versions = pbf_dense_info.get_packed_int32();
break;
case OSMFormat::DenseInfo::packed_sint64_timestamp:
timestamps = pbf_dense_info.get_packed_sint64();
break;
case OSMFormat::DenseInfo::packed_sint64_changeset:
changesets = pbf_dense_info.get_packed_sint64();
break;
case OSMFormat::DenseInfo::packed_sint32_uid:
uids = pbf_dense_info.get_packed_sint32();
break;
case OSMFormat::DenseInfo::packed_sint32_user_sid:
user_sids = pbf_dense_info.get_packed_sint32();
break;
case OSMFormat::DenseInfo::packed_bool_visible:
has_visibles = true;
visibles = pbf_dense_info.get_packed_bool();
break;
default:
pbf_dense_info.skip();
}
}
}
break;
case OSMFormat::DenseNodes::packed_sint64_lat:
lats = pbf_dense_nodes.get_packed_sint64();
break;
case OSMFormat::DenseNodes::packed_sint64_lon:
lons = pbf_dense_nodes.get_packed_sint64();
break;
case OSMFormat::DenseNodes::packed_int32_keys_vals:
tags = pbf_dense_nodes.get_packed_int32();
break;
default:
pbf_dense_nodes.skip();
}
}
osmium::util::DeltaDecode<int64_t> dense_id;
osmium::util::DeltaDecode<int64_t> dense_latitude;
osmium::util::DeltaDecode<int64_t> dense_longitude;
osmium::util::DeltaDecode<int64_t> dense_uid;
osmium::util::DeltaDecode<int64_t> dense_user_sid;
osmium::util::DeltaDecode<int64_t> dense_changeset;
osmium::util::DeltaDecode<int64_t> dense_timestamp;
auto tag_it = tags.first;
while (ids.first != ids.second) {
if (lons.first == lons.second ||
lats.first == lats.second) {
// this is against the spec, must have same number of elements
throw osmium::pbf_error("PBF format error");
}
bool visible = true;
osmium::builder::NodeBuilder builder(m_buffer);
osmium::Node& node = builder.object();
node.set_id(dense_id.update(*ids.first++));
if (has_info) {
if (versions.first == versions.second ||
changesets.first == changesets.second ||
timestamps.first == timestamps.second ||
uids.first == uids.second ||
user_sids.first == user_sids.second) {
// this is against the spec, must have same number of elements
throw osmium::pbf_error("PBF format error");
}
auto version = *versions.first++;
if (version < 0) {
throw osmium::pbf_error("object version must not be negative");
}
node.set_version(static_cast<osmium::object_version_type>(version));
auto changeset_id = dense_changeset.update(*changesets.first++);
if (changeset_id < 0) {
throw osmium::pbf_error("object changeset_id must not be negative");
}
node.set_changeset(static_cast<osmium::changeset_id_type>(changeset_id));
node.set_timestamp(dense_timestamp.update(*timestamps.first++) * m_date_factor / 1000);
node.set_uid_from_signed(static_cast<osmium::signed_user_id_type>(dense_uid.update(*uids.first++)));
if (has_visibles) {
if (visibles.first == visibles.second) {
// this is against the spec, must have same number of elements
throw osmium::pbf_error("PBF format error");
}
visible = (*visibles.first++) != 0;
}
node.set_visible(visible);
const auto& u = m_stringtable.at(dense_user_sid.update(*user_sids.first++));
builder.add_user(u.first, u.second);
} else {
builder.add_user("");
}
// even if the node isn't visible, there's still a record
// of its lat/lon in the dense arrays.
const auto lon = dense_longitude.update(*lons.first++);
const auto lat = dense_latitude.update(*lats.first++);
if (visible) {
builder.object().set_location(osmium::Location(
convert_pbf_coordinate(lon),
convert_pbf_coordinate(lat)
));
}
if (tag_it != tags.second) {
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
while (tag_it != tags.second && *tag_it != 0) {
const auto& k = m_stringtable.at(*tag_it++);
if (tag_it == tags.second) {
throw osmium::pbf_error("PBF format error"); // this is against the spec, keys/vals must come in pairs
}
const auto& v = m_stringtable.at(*tag_it++);
tl_builder.add_tag(k.first, k.second, v.first, v.second);
}
if (tag_it != tags.second) {
++tag_it;
}
}
m_buffer.commit();
}
}
public:
PBFPrimitiveBlockDecoder(const ptr_len_type& data, osmium::osm_entity_bits::type read_types) :
m_data(data),
m_read_types(read_types) {
}
PBFPrimitiveBlockDecoder(const PBFPrimitiveBlockDecoder&) = delete;
PBFPrimitiveBlockDecoder& operator=(const PBFPrimitiveBlockDecoder&) = delete;
PBFPrimitiveBlockDecoder(PBFPrimitiveBlockDecoder&&) = delete;
PBFPrimitiveBlockDecoder& operator=(PBFPrimitiveBlockDecoder&&) = delete;
~PBFPrimitiveBlockDecoder() noexcept = default;
osmium::memory::Buffer operator()() {
try {
decode_primitive_block_metadata();
decode_primitive_block_data();
} catch (std::out_of_range&) {
throw osmium::pbf_error("string id out of range");
}
return std::move(m_buffer);
}
}; // class PBFPrimitiveBlockDecoder
inline ptr_len_type decode_blob(const std::string& blob_data, std::string& output) {
int32_t raw_size = 0;
std::pair<const char*, protozero::pbf_length_type> zlib_data = {nullptr, 0};
protozero::pbf_message<FileFormat::Blob> pbf_blob(blob_data);
while (pbf_blob.next()) {
switch (pbf_blob.tag()) {
case FileFormat::Blob::optional_bytes_raw:
{
auto data_len = pbf_blob.get_data();
if (data_len.second > max_uncompressed_blob_size) {
throw osmium::pbf_error("illegal blob size");
}
return data_len;
}
case FileFormat::Blob::optional_int32_raw_size:
raw_size = pbf_blob.get_int32();
if (raw_size <= 0 || uint32_t(raw_size) > max_uncompressed_blob_size) {
throw osmium::pbf_error("illegal blob size");
}
break;
case FileFormat::Blob::optional_bytes_zlib_data:
zlib_data = pbf_blob.get_data();
break;
case FileFormat::Blob::optional_bytes_lzma_data:
throw osmium::pbf_error("lzma blobs not implemented");
default:
throw osmium::pbf_error("unknown compression");
}
}
if (zlib_data.second != 0 && raw_size != 0) {
return osmium::io::detail::zlib_uncompress_string(
zlib_data.first,
static_cast<unsigned long>(zlib_data.second),
static_cast<unsigned long>(raw_size),
output
);
}
throw osmium::pbf_error("blob contains no data");
}
inline osmium::Box decode_header_bbox(const ptr_len_type& data) {
int64_t left = std::numeric_limits<int64_t>::max();
int64_t right = std::numeric_limits<int64_t>::max();
int64_t top = std::numeric_limits<int64_t>::max();
int64_t bottom = std::numeric_limits<int64_t>::max();
protozero::pbf_message<OSMFormat::HeaderBBox> pbf_header_bbox(data);
while (pbf_header_bbox.next()) {
switch (pbf_header_bbox.tag()) {
case OSMFormat::HeaderBBox::required_sint64_left:
left = pbf_header_bbox.get_sint64();
break;
case OSMFormat::HeaderBBox::required_sint64_right:
right = pbf_header_bbox.get_sint64();
break;
case OSMFormat::HeaderBBox::required_sint64_top:
top = pbf_header_bbox.get_sint64();
break;
case OSMFormat::HeaderBBox::required_sint64_bottom:
bottom = pbf_header_bbox.get_sint64();
break;
default:
pbf_header_bbox.skip();
}
}
if (left == std::numeric_limits<int64_t>::max() ||
right == std::numeric_limits<int64_t>::max() ||
top == std::numeric_limits<int64_t>::max() ||
bottom == std::numeric_limits<int64_t>::max()) {
throw osmium::pbf_error("invalid bbox");
}
osmium::Box box;
box.extend(osmium::Location(left / resolution_convert, bottom / resolution_convert));
box.extend(osmium::Location(right / resolution_convert, top / resolution_convert));
return box;
}
inline osmium::io::Header decode_header_block(const ptr_len_type& data) {
osmium::io::Header header;
int i = 0;
protozero::pbf_message<OSMFormat::HeaderBlock> pbf_header_block(data);
while (pbf_header_block.next()) {
switch (pbf_header_block.tag()) {
case OSMFormat::HeaderBlock::optional_HeaderBBox_bbox:
header.add_box(decode_header_bbox(pbf_header_block.get_data()));
break;
case OSMFormat::HeaderBlock::repeated_string_required_features:
{
auto feature = pbf_header_block.get_data();
if (!strncmp("OsmSchema-V0.6", feature.first, feature.second)) {
// intentionally left blank
} else if (!strncmp("DenseNodes", feature.first, feature.second)) {
header.set("pbf_dense_nodes", true);
} else if (!strncmp("HistoricalInformation", feature.first, feature.second)) {
header.set_has_multiple_object_versions(true);
} else {
std::string msg("required feature not supported: ");
msg.append(feature.first, feature.second);
throw osmium::pbf_error(msg);
}
}
break;
case OSMFormat::HeaderBlock::repeated_string_optional_features:
header.set("pbf_optional_feature_" + std::to_string(i++), pbf_header_block.get_string());
break;
case OSMFormat::HeaderBlock::optional_string_writingprogram:
header.set("generator", pbf_header_block.get_string());
break;
case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp:
{
auto timestamp = osmium::Timestamp(pbf_header_block.get_int64()).to_iso();
header.set("osmosis_replication_timestamp", timestamp);
header.set("timestamp", timestamp);
}
break;
case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_sequence_number:
header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.get_int64()));
break;
case OSMFormat::HeaderBlock::optional_string_osmosis_replication_base_url:
header.set("osmosis_replication_base_url", pbf_header_block.get_string());
break;
default:
pbf_header_block.skip();
}
}
return header;
}
/**
* Decode HeaderBlock.
*
* @param header_block_data Input data
* @returns Header object
* @throws osmium::pbf_error If there was a parsing error
*/
inline osmium::io::Header decode_header(const std::string& header_block_data) {
std::string output;
return decode_header_block(decode_blob(header_block_data, output));
}
class PBFDataBlobDecoder {
std::shared_ptr<std::string> m_input_buffer;
osmium::osm_entity_bits::type m_read_types;
public:
PBFDataBlobDecoder(std::string&& input_buffer, osmium::osm_entity_bits::type read_types) :
m_input_buffer(std::make_shared<std::string>(std::move(input_buffer))),
m_read_types(read_types) {
}
PBFDataBlobDecoder(const PBFDataBlobDecoder&) = default;
PBFDataBlobDecoder& operator=(const PBFDataBlobDecoder&) = default;
PBFDataBlobDecoder(PBFDataBlobDecoder&&) = default;
PBFDataBlobDecoder& operator=(PBFDataBlobDecoder&&) = default;
~PBFDataBlobDecoder() noexcept = default;
osmium::memory::Buffer operator()() {
std::string output;
PBFPrimitiveBlockDecoder decoder(decode_blob(*m_input_buffer, output), m_read_types);
return decoder();
}
}; // class PBFDataBlobDecoder
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_PBF_DECODER_HPP
third_party/libosmium/include/osmium/io/detail/pbf_input_format.hpp
Vendored
-242
View File
@@ -1,242 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_PBF_INPUT_FORMAT_HPP
#define OSMIUM_IO_DETAIL_PBF_INPUT_FORMAT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <memory>
#include <sstream>
#include <string>
#include <thread>
#include <type_traits>
#include <protozero/pbf_message.hpp>
#include <osmium/io/detail/input_format.hpp>
#include <osmium/io/detail/pbf.hpp> // IWYU pragma: export
#include <osmium/io/detail/pbf_decoder.hpp>
#include <osmium/io/detail/protobuf_tags.hpp>
#include <osmium/io/error.hpp>
#include <osmium/io/file.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/osm.hpp>
#include <osmium/osm/entity_bits.hpp>
#include <osmium/osm/object.hpp>
#include <osmium/osm/timestamp.hpp>
#include <osmium/thread/pool.hpp>
#include <osmium/thread/util.hpp>
#include <osmium/util/config.hpp>
namespace osmium {
namespace io {
namespace detail {
class PBFParser : public Parser {
std::string m_input_buffer;
/**
* Read the given number of bytes from the input queue.
*
* @param size Number of bytes to read
* @returns String with the data
* @throws osmium::pbf_error If size bytes can't be read
*/
std::string read_from_input_queue(size_t size) {
while (m_input_buffer.size() < size) {
std::string new_data = get_input();
if (input_done()) {
throw osmium::pbf_error("truncated data (EOF encountered)");
}
m_input_buffer += new_data;
}
std::string output { m_input_buffer.substr(size) };
m_input_buffer.resize(size);
using std::swap;
swap(output, m_input_buffer);
return output;
}
/**
* Read 4 bytes in network byte order from file. They contain
* the length of the following BlobHeader.
*/
uint32_t read_blob_header_size_from_file() {
uint32_t size_in_network_byte_order;
try {
const std::string input_data = read_from_input_queue(sizeof(size_in_network_byte_order));
size_in_network_byte_order = *reinterpret_cast<const uint32_t*>(input_data.data());
} catch (osmium::pbf_error&) {
return 0; // EOF
}
const uint32_t size = ntohl(size_in_network_byte_order);
if (size > static_cast<uint32_t>(max_blob_header_size)) {
throw osmium::pbf_error("invalid BlobHeader size (> max_blob_header_size)");
}
return size;
}
/**
* Decode the BlobHeader. Make sure it contains the expected
* type. Return the size of the following Blob.
*/
size_t decode_blob_header(protozero::pbf_message<FileFormat::BlobHeader>&& pbf_blob_header, const char* expected_type) {
std::pair<const char*, size_t> blob_header_type;
size_t blob_header_datasize = 0;
while (pbf_blob_header.next()) {
switch (pbf_blob_header.tag()) {
case FileFormat::BlobHeader::required_string_type:
blob_header_type = pbf_blob_header.get_data();
break;
case FileFormat::BlobHeader::required_int32_datasize:
blob_header_datasize = pbf_blob_header.get_int32();
break;
default:
pbf_blob_header.skip();
}
}
if (blob_header_datasize == 0) {
throw osmium::pbf_error("PBF format error: BlobHeader.datasize missing or zero.");
}
if (strncmp(expected_type, blob_header_type.first, blob_header_type.second)) {
throw osmium::pbf_error("blob does not have expected type (OSMHeader in first blob, OSMData in following blobs)");
}
return blob_header_datasize;
}
size_t check_type_and_get_blob_size(const char* expected_type) {
assert(expected_type);
const auto size = read_blob_header_size_from_file();
if (size == 0) { // EOF
return 0;
}
const std::string blob_header = read_from_input_queue(size);
return decode_blob_header(protozero::pbf_message<FileFormat::BlobHeader>(blob_header), expected_type);
}
std::string read_from_input_queue_with_check(size_t size) {
if (size > max_uncompressed_blob_size) {
throw osmium::pbf_error(std::string("invalid blob size: " +
std::to_string(size)));
}
return read_from_input_queue(size);
}
// Parse the header in the PBF OSMHeader blob.
void parse_header_blob() {
osmium::io::Header header;
const auto size = check_type_and_get_blob_size("OSMHeader");
header = decode_header(read_from_input_queue_with_check(size));
set_header_value(header);
}
void parse_data_blobs() {
while (const auto size = check_type_and_get_blob_size("OSMData")) {
std::string input_buffer = read_from_input_queue_with_check(size);
PBFDataBlobDecoder data_blob_parser{ std::move(input_buffer), read_types() };
if (osmium::config::use_pool_threads_for_pbf_parsing()) {
send_to_output_queue(osmium::thread::Pool::instance().submit(std::move(data_blob_parser)));
} else {
send_to_output_queue(data_blob_parser());
}
}
}
public:
PBFParser(future_string_queue_type& input_queue,
future_buffer_queue_type& output_queue,
std::promise<osmium::io::Header>& header_promise,
osmium::osm_entity_bits::type read_types) :
Parser(input_queue, output_queue, header_promise, read_types),
m_input_buffer() {
}
~PBFParser() noexcept final = default;
void run() final {
osmium::thread::set_thread_name("_osmium_pbf_in");
parse_header_blob();
if (read_types() != osmium::osm_entity_bits::nothing) {
parse_data_blobs();
}
}
}; // class PBFParser
// we want the register_parser() function to run, setting
// the variable is only a side-effect, it will never be used
const bool registered_pbf_parser = ParserFactory::instance().register_parser(
file_format::pbf,
[](future_string_queue_type& input_queue,
future_buffer_queue_type& output_queue,
std::promise<osmium::io::Header>& header_promise,
osmium::osm_entity_bits::type read_which_entities) {
return std::unique_ptr<Parser>(new PBFParser(input_queue, output_queue, header_promise, read_which_entities));
});
// dummy function to silence the unused variable warning from above
inline bool get_registered_pbf_parser() noexcept {
return registered_pbf_parser;
}
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_PBF_INPUT_FORMAT_HPP
third_party/libosmium/include/osmium/io/detail/pbf_output_format.hpp
Vendored
-643
View File
@@ -1,643 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_PBF_OUTPUT_FORMAT_HPP
#define OSMIUM_IO_DETAIL_PBF_OUTPUT_FORMAT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <algorithm>
#include <cassert>
#include <cmath>
#include <cstdint>
#include <cstdlib>
#include <iterator>
#include <memory>
#include <string>
#include <time.h>
#include <utility>
// needed for older boost libraries
#define BOOST_RESULT_OF_USE_DECLTYPE
#include <boost/iterator/transform_iterator.hpp>
#include <protozero/pbf_builder.hpp>
#include <osmium/handler.hpp>
#include <osmium/io/detail/output_format.hpp>
#include <osmium/io/detail/pbf.hpp> // IWYU pragma: export
#include <osmium/io/detail/protobuf_tags.hpp>
#include <osmium/io/detail/string_table.hpp>
#include <osmium/io/detail/zlib.hpp>
#include <osmium/io/file.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/io/header.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/memory/collection.hpp>
#include <osmium/osm/box.hpp>
#include <osmium/osm/item_type.hpp>
#include <osmium/osm/location.hpp>
#include <osmium/osm/node.hpp>
#include <osmium/osm/object.hpp>
#include <osmium/osm/relation.hpp>
#include <osmium/osm/tag.hpp>
#include <osmium/osm/timestamp.hpp>
#include <osmium/osm/way.hpp>
#include <osmium/thread/pool.hpp>
#include <osmium/util/cast.hpp>
#include <osmium/util/delta.hpp>
#include <osmium/visitor.hpp>
namespace osmium {
namespace io {
namespace detail {
struct pbf_output_options {
/// Should nodes be encoded in DenseNodes?
bool use_dense_nodes;
/**
* Should the PBF blobs contain zlib compressed data?
*
* The zlib compression is optional, it's possible to store the
* blobs in raw format. Disabling the compression can improve
* the writing speed a little but the output will be 2x to 3x
* bigger.
*/
bool use_compression;
/// Should metadata of objects be written?
bool add_metadata;
/// Add the "HistoricalInformation" header flag.
bool add_historical_information_flag;
/// Should the visible flag be added to all OSM objects?
bool add_visible_flag;
};
/**
* Maximum number of items in a primitive block.
*
* The uncompressed length of a Blob *should* be less
* than 16 megabytes and *must* be less than 32 megabytes.
*
* A block may contain any number of entities, as long as
* the size limits for the surrounding blob are obeyed.
* However, for simplicity, the current Osmosis (0.38)
* as well as Osmium implementation always
* uses at most 8k entities in a block.
*/
constexpr int32_t max_entities_per_block = 8000;
constexpr int location_granularity = 100;
/**
* convert a double lat or lon value to an int, respecting the granularity
*/
inline int64_t lonlat2int(double lonlat) {
return static_cast<int64_t>(std::round(lonlat * lonlat_resolution / location_granularity));
}
enum class pbf_blob_type {
header = 0,
data = 1
};
class SerializeBlob {
std::string m_msg;
pbf_blob_type m_blob_type;
bool m_use_compression;
public:
/**
* Initialize a blob serializer.
*
* @param msg Protobuf-message containing the blob data
* @param type Type of blob.
* @param use_compression Should the output be compressed using
* zlib?
*/
SerializeBlob(std::string&& msg, pbf_blob_type type, bool use_compression) :
m_msg(std::move(msg)),
m_blob_type(type),
m_use_compression(use_compression) {
}
/**
* Serialize a protobuf message into a Blob, optionally apply
* compression and return it together with a BlobHeader ready
* to be written to a file.
*/
std::string operator()() {
assert(m_msg.size() <= max_uncompressed_blob_size);
std::string blob_data;
protozero::pbf_builder<FileFormat::Blob> pbf_blob(blob_data);
if (m_use_compression) {
pbf_blob.add_int32(FileFormat::Blob::optional_int32_raw_size, int32_t(m_msg.size()));
pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_zlib_data, osmium::io::detail::zlib_compress(m_msg));
} else {
pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_raw, m_msg);
}
std::string blob_header_data;
protozero::pbf_builder<FileFormat::BlobHeader> pbf_blob_header(blob_header_data);
pbf_blob_header.add_string(FileFormat::BlobHeader::required_string_type, m_blob_type == pbf_blob_type::data ? "OSMData" : "OSMHeader");
pbf_blob_header.add_int32(FileFormat::BlobHeader::required_int32_datasize, static_cast_with_assert<int32_t>(blob_data.size()));
uint32_t sz = htonl(static_cast_with_assert<uint32_t>(blob_header_data.size()));
// write to output: the 4-byte BlobHeader-Size followed by the BlobHeader followed by the Blob
std::string output;
output.reserve(sizeof(sz) + blob_header_data.size() + blob_data.size());
output.append(reinterpret_cast<const char*>(&sz), sizeof(sz));
output.append(blob_header_data);
output.append(blob_data);
return output;
}
}; // class SerializeBlob
/**
* Contains the code to pack any number of nodes into a DenseNode
* structure.
*
* Because this needs to allocate a lot of memory on the heap,
* only one object of this class will be created and then re-used
* after calling clear() on it.
*/
class DenseNodes {
StringTable& m_stringtable;
std::vector<int64_t> m_ids;
std::vector<int32_t> m_versions;
std::vector<int64_t> m_timestamps;
std::vector<int64_t> m_changesets;
std::vector<int32_t> m_uids;
std::vector<int32_t> m_user_sids;
std::vector<bool> m_visibles;
std::vector<int64_t> m_lats;
std::vector<int64_t> m_lons;
std::vector<int32_t> m_tags;
osmium::util::DeltaEncode<object_id_type, int64_t> m_delta_id;
osmium::util::DeltaEncode<uint32_t, int64_t> m_delta_timestamp;
osmium::util::DeltaEncode<changeset_id_type, int64_t> m_delta_changeset;
osmium::util::DeltaEncode<user_id_type, int32_t> m_delta_uid;
osmium::util::DeltaEncode<uint32_t, int32_t> m_delta_user_sid;
osmium::util::DeltaEncode<int64_t, int64_t> m_delta_lat;
osmium::util::DeltaEncode<int64_t, int64_t> m_delta_lon;
const pbf_output_options& m_options;
public:
DenseNodes(StringTable& stringtable, const pbf_output_options& options) :
m_stringtable(stringtable),
m_options(options) {
}
/// Clear object for re-use. Keep the allocated memory.
void clear() {
m_ids.clear();
m_versions.clear();
m_timestamps.clear();
m_changesets.clear();
m_uids.clear();
m_user_sids.clear();
m_visibles.clear();
m_lats.clear();
m_lons.clear();
m_tags.clear();
m_delta_id.clear();
m_delta_timestamp.clear();
m_delta_changeset.clear();
m_delta_uid.clear();
m_delta_user_sid.clear();
m_delta_lat.clear();
m_delta_lon.clear();
}
size_t size() const {
return m_ids.size() * 3 * sizeof(int64_t);
}
void add_node(const osmium::Node& node) {
m_ids.push_back(m_delta_id.update(node.id()));
if (m_options.add_metadata) {
m_versions.push_back(static_cast_with_assert<int32_t>(node.version()));
m_timestamps.push_back(m_delta_timestamp.update(uint32_t(node.timestamp())));
m_changesets.push_back(m_delta_changeset.update(node.changeset()));
m_uids.push_back(m_delta_uid.update(node.uid()));
m_user_sids.push_back(m_delta_user_sid.update(m_stringtable.add(node.user())));
if (m_options.add_visible_flag) {
m_visibles.push_back(node.visible());
}
}
m_lats.push_back(m_delta_lat.update(lonlat2int(node.location().lat_without_check())));
m_lons.push_back(m_delta_lon.update(lonlat2int(node.location().lon_without_check())));
for (const auto& tag : node.tags()) {
m_tags.push_back(static_cast_with_assert<int32_t>(m_stringtable.add(tag.key())));
m_tags.push_back(static_cast_with_assert<int32_t>(m_stringtable.add(tag.value())));
}
m_tags.push_back(0);
}
std::string serialize() const {
std::string data;
protozero::pbf_builder<OSMFormat::DenseNodes> pbf_dense_nodes(data);
pbf_dense_nodes.add_packed_sint64(OSMFormat::DenseNodes::packed_sint64_id, m_ids.cbegin(), m_ids.cend());
if (m_options.add_metadata) {
protozero::pbf_builder<OSMFormat::DenseInfo> pbf_dense_info(pbf_dense_nodes, OSMFormat::DenseNodes::optional_DenseInfo_denseinfo);
pbf_dense_info.add_packed_int32(OSMFormat::DenseInfo::packed_int32_version, m_versions.cbegin(), m_versions.cend());
pbf_dense_info.add_packed_sint64(OSMFormat::DenseInfo::packed_sint64_timestamp, m_timestamps.cbegin(), m_timestamps.cend());
pbf_dense_info.add_packed_sint64(OSMFormat::DenseInfo::packed_sint64_changeset, m_changesets.cbegin(), m_changesets.cend());
pbf_dense_info.add_packed_sint32(OSMFormat::DenseInfo::packed_sint32_uid, m_uids.cbegin(), m_uids.cend());
pbf_dense_info.add_packed_sint32(OSMFormat::DenseInfo::packed_sint32_user_sid, m_user_sids.cbegin(), m_user_sids.cend());
if (m_options.add_visible_flag) {
pbf_dense_info.add_packed_bool(OSMFormat::DenseInfo::packed_bool_visible, m_visibles.cbegin(), m_visibles.cend());
}
}
pbf_dense_nodes.add_packed_sint64(OSMFormat::DenseNodes::packed_sint64_lat, m_lats.cbegin(), m_lats.cend());
pbf_dense_nodes.add_packed_sint64(OSMFormat::DenseNodes::packed_sint64_lon, m_lons.cbegin(), m_lons.cend());
pbf_dense_nodes.add_packed_int32(OSMFormat::DenseNodes::packed_int32_keys_vals, m_tags.cbegin(), m_tags.cend());
return data;
}
}; // class DenseNodes
class PrimitiveBlock {
std::string m_pbf_primitive_group_data;
protozero::pbf_builder<OSMFormat::PrimitiveGroup> m_pbf_primitive_group;
StringTable m_stringtable;
DenseNodes m_dense_nodes;
OSMFormat::PrimitiveGroup m_type;
int m_count;
public:
explicit PrimitiveBlock(const pbf_output_options& options) :
m_pbf_primitive_group_data(),
m_pbf_primitive_group(m_pbf_primitive_group_data),
m_stringtable(),
m_dense_nodes(m_stringtable, options),
m_type(OSMFormat::PrimitiveGroup::unknown),
m_count(0) {
}
const std::string& group_data() {
if (type() == OSMFormat::PrimitiveGroup::optional_DenseNodes_dense) {
m_pbf_primitive_group.add_message(OSMFormat::PrimitiveGroup::optional_DenseNodes_dense, m_dense_nodes.serialize());
}
return m_pbf_primitive_group_data;
}
void reset(OSMFormat::PrimitiveGroup type) {
m_pbf_primitive_group_data.clear();
m_stringtable.clear();
m_dense_nodes.clear();
m_type = type;
m_count = 0;
}
void write_stringtable(protozero::pbf_builder<OSMFormat::StringTable>& pbf_string_table) {
for (const char* s : m_stringtable) {
pbf_string_table.add_bytes(OSMFormat::StringTable::repeated_bytes_s, s);
}
}
protozero::pbf_builder<OSMFormat::PrimitiveGroup>& group() {
++m_count;
return m_pbf_primitive_group;
}
void add_dense_node(const osmium::Node& node) {
m_dense_nodes.add_node(node);
++m_count;
}
uint32_t store_in_stringtable(const char* s) {
return m_stringtable.add(s);
}
int count() const {
return m_count;
}
OSMFormat::PrimitiveGroup type() const {
return m_type;
}
size_t size() const {
return m_pbf_primitive_group_data.size() + m_stringtable.size() + m_dense_nodes.size();
}
/**
* The output buffer (block) will be filled to about
* 95% and then written to disk. This leaves more than
* enough space for the string table (which typically
* needs about 0.1 to 0.3% of the block size).
*/
constexpr static size_t max_used_blob_size = max_uncompressed_blob_size * 95 / 100;
bool can_add(OSMFormat::PrimitiveGroup type) const {
if (type != m_type) {
return false;
}
if (count() >= max_entities_per_block) {
return false;
}
return size() < max_used_blob_size;
}
}; // class PrimitiveBlock
class PBFOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler {
pbf_output_options m_options;
PrimitiveBlock m_primitive_block;
void store_primitive_block() {
if (m_primitive_block.count() == 0) {
return;
}
std::string primitive_block_data;
protozero::pbf_builder<OSMFormat::PrimitiveBlock> primitive_block(primitive_block_data);
{
protozero::pbf_builder<OSMFormat::StringTable> pbf_string_table(primitive_block, OSMFormat::PrimitiveBlock::required_StringTable_stringtable);
m_primitive_block.write_stringtable(pbf_string_table);
}
primitive_block.add_message(OSMFormat::PrimitiveBlock::repeated_PrimitiveGroup_primitivegroup, m_primitive_block.group_data());
m_output_queue.push(osmium::thread::Pool::instance().submit(
SerializeBlob{std::move(primitive_block_data),
pbf_blob_type::data,
m_options.use_compression}
));
}
template <typename T>
void add_meta(const osmium::OSMObject& object, T& pbf_object) {
const osmium::TagList& tags = object.tags();
auto map_tag_key = [this](const osmium::Tag& tag) -> uint32_t {
return m_primitive_block.store_in_stringtable(tag.key());
};
auto map_tag_value = [this](const osmium::Tag& tag) -> uint32_t {
return m_primitive_block.store_in_stringtable(tag.value());
};
pbf_object.add_packed_uint32(T::enum_type::packed_uint32_keys,
boost::make_transform_iterator(tags.begin(), map_tag_key),
boost::make_transform_iterator(tags.end(), map_tag_key));
pbf_object.add_packed_uint32(T::enum_type::packed_uint32_vals,
boost::make_transform_iterator(tags.begin(), map_tag_value),
boost::make_transform_iterator(tags.end(), map_tag_value));
if (m_options.add_metadata) {
protozero::pbf_builder<OSMFormat::Info> pbf_info(pbf_object, T::enum_type::optional_Info_info);
pbf_info.add_int32(OSMFormat::Info::optional_int32_version, static_cast_with_assert<int32_t>(object.version()));
pbf_info.add_int64(OSMFormat::Info::optional_int64_timestamp, uint32_t(object.timestamp()));
pbf_info.add_int64(OSMFormat::Info::optional_int64_changeset, object.changeset());
pbf_info.add_int32(OSMFormat::Info::optional_int32_uid, static_cast_with_assert<int32_t>(object.uid()));
pbf_info.add_uint32(OSMFormat::Info::optional_uint32_user_sid, m_primitive_block.store_in_stringtable(object.user()));
if (m_options.add_visible_flag) {
pbf_info.add_bool(OSMFormat::Info::optional_bool_visible, object.visible());
}
}
}
void switch_primitive_block_type(OSMFormat::PrimitiveGroup type) {
if (!m_primitive_block.can_add(type)) {
store_primitive_block();
m_primitive_block.reset(type);
}
}
public:
PBFOutputFormat(const osmium::io::File& file, future_string_queue_type& output_queue) :
OutputFormat(output_queue),
m_options(),
m_primitive_block(m_options) {
m_options.use_dense_nodes = file.is_not_false("pbf_dense_nodes");
m_options.use_compression = file.get("pbf_compression") != "none" && file.is_not_false("pbf_compression");
m_options.add_metadata = file.is_not_false("pbf_add_metadata") && file.is_not_false("add_metadata");
m_options.add_historical_information_flag = file.has_multiple_object_versions();
m_options.add_visible_flag = file.has_multiple_object_versions();
}
PBFOutputFormat(const PBFOutputFormat&) = delete;
PBFOutputFormat& operator=(const PBFOutputFormat&) = delete;
~PBFOutputFormat() noexcept final = default;
void write_header(const osmium::io::Header& header) final {
std::string data;
protozero::pbf_builder<OSMFormat::HeaderBlock> pbf_header_block(data);
if (!header.boxes().empty()) {
protozero::pbf_builder<OSMFormat::HeaderBBox> pbf_header_bbox(pbf_header_block, OSMFormat::HeaderBlock::optional_HeaderBBox_bbox);
osmium::Box box = header.joined_boxes();
pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_left, int64_t(box.bottom_left().lon() * lonlat_resolution));
pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_right, int64_t(box.top_right().lon() * lonlat_resolution));
pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_top, int64_t(box.top_right().lat() * lonlat_resolution));
pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_bottom, int64_t(box.bottom_left().lat() * lonlat_resolution));
}
pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "OsmSchema-V0.6");
if (m_options.use_dense_nodes) {
pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "DenseNodes");
}
if (m_options.add_historical_information_flag) {
pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "HistoricalInformation");
}
pbf_header_block.add_string(OSMFormat::HeaderBlock::optional_string_writingprogram, header.get("generator"));
std::string osmosis_replication_timestamp = header.get("osmosis_replication_timestamp");
if (!osmosis_replication_timestamp.empty()) {
osmium::Timestamp ts(osmosis_replication_timestamp.c_str());
pbf_header_block.add_int64(OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp, uint32_t(ts));
}
std::string osmosis_replication_sequence_number = header.get("osmosis_replication_sequence_number");
if (!osmosis_replication_sequence_number.empty()) {
pbf_header_block.add_int64(OSMFormat::HeaderBlock::optional_int64_osmosis_replication_sequence_number, std::atoll(osmosis_replication_sequence_number.c_str()));
}
std::string osmosis_replication_base_url = header.get("osmosis_replication_base_url");
if (!osmosis_replication_base_url.empty()) {
pbf_header_block.add_string(OSMFormat::HeaderBlock::optional_string_osmosis_replication_base_url, osmosis_replication_base_url);
}
m_output_queue.push(osmium::thread::Pool::instance().submit(
SerializeBlob{std::move(data),
pbf_blob_type::header,
m_options.use_compression}
));
}
void write_buffer(osmium::memory::Buffer&& buffer) final {
osmium::apply(buffer.cbegin(), buffer.cend(), *this);
}
void write_end() final {
store_primitive_block();
}
void node(const osmium::Node& node) {
if (m_options.use_dense_nodes) {
switch_primitive_block_type(OSMFormat::PrimitiveGroup::optional_DenseNodes_dense);
m_primitive_block.add_dense_node(node);
return;
}
switch_primitive_block_type(OSMFormat::PrimitiveGroup::repeated_Node_nodes);
protozero::pbf_builder<OSMFormat::Node> pbf_node{ m_primitive_block.group(), OSMFormat::PrimitiveGroup::repeated_Node_nodes };
pbf_node.add_sint64(OSMFormat::Node::required_sint64_id, node.id());
add_meta(node, pbf_node);
pbf_node.add_sint64(OSMFormat::Node::required_sint64_lat, lonlat2int(node.location().lat_without_check()));
pbf_node.add_sint64(OSMFormat::Node::required_sint64_lon, lonlat2int(node.location().lon_without_check()));
}
void way(const osmium::Way& way) {
switch_primitive_block_type(OSMFormat::PrimitiveGroup::repeated_Way_ways);
protozero::pbf_builder<OSMFormat::Way> pbf_way{ m_primitive_block.group(), OSMFormat::PrimitiveGroup::repeated_Way_ways };
pbf_way.add_int64(OSMFormat::Way::required_int64_id, way.id());
add_meta(way, pbf_way);
static auto map_node_ref = [](osmium::NodeRefList::const_iterator node_ref) noexcept -> osmium::object_id_type {
return node_ref->ref();
};
typedef osmium::util::DeltaEncodeIterator<osmium::NodeRefList::const_iterator, decltype(map_node_ref), osmium::object_id_type> it_type;
const auto& nodes = way.nodes();
it_type first { nodes.cbegin(), nodes.cend(), map_node_ref };
it_type last { nodes.cend(), nodes.cend(), map_node_ref };
pbf_way.add_packed_sint64(OSMFormat::Way::packed_sint64_refs, first, last);
}
void relation(const osmium::Relation& relation) {
switch_primitive_block_type(OSMFormat::PrimitiveGroup::repeated_Relation_relations);
protozero::pbf_builder<OSMFormat::Relation> pbf_relation { m_primitive_block.group(), OSMFormat::PrimitiveGroup::repeated_Relation_relations };
pbf_relation.add_int64(OSMFormat::Relation::required_int64_id, relation.id());
add_meta(relation, pbf_relation);
auto map_member_role = [this](const osmium::RelationMember& member) -> uint32_t {
return m_primitive_block.store_in_stringtable(member.role());
};
pbf_relation.add_packed_int32(OSMFormat::Relation::packed_int32_roles_sid,
boost::make_transform_iterator(relation.members().begin(), map_member_role),
boost::make_transform_iterator(relation.members().end(), map_member_role));
static auto map_member_ref = [](osmium::RelationMemberList::const_iterator member) noexcept -> osmium::object_id_type {
return member->ref();
};
typedef osmium::util::DeltaEncodeIterator<osmium::RelationMemberList::const_iterator, decltype(map_member_ref), osmium::object_id_type> it_type;
const auto& members = relation.members();
it_type first { members.cbegin(), members.cend(), map_member_ref };
it_type last { members.cend(), members.cend(), map_member_ref };
pbf_relation.add_packed_sint64(OSMFormat::Relation::packed_sint64_memids, first, last);
static auto map_member_type = [](const osmium::RelationMember& member) noexcept -> int32_t {
return int32_t(osmium::item_type_to_nwr_index(member.type()));
};
pbf_relation.add_packed_int32(OSMFormat::Relation::packed_MemberType_types,
boost::make_transform_iterator(relation.members().begin(), map_member_type),
boost::make_transform_iterator(relation.members().end(), map_member_type));
}
}; // class PBFOutputFormat
// we want the register_output_format() function to run, setting
// the variable is only a side-effect, it will never be used
const bool registered_pbf_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::pbf,
[](const osmium::io::File& file, future_string_queue_type& output_queue) {
return new osmium::io::detail::PBFOutputFormat(file, output_queue);
});
// dummy function to silence the unused variable warning from above
inline bool get_registered_pbf_output() noexcept {
return registered_pbf_output;
}
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_PBF_OUTPUT_FORMAT_HPP
third_party/libosmium/include/osmium/io/detail/protobuf_tags.hpp
Vendored
-170
View File
@@ -1,170 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP
#define OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <protozero/pbf_types.hpp>
namespace osmium {
namespace io {
namespace detail {
// directly translated from
// https://github.com/scrosby/OSM-binary/blob/master/src/fileformat.proto
namespace FileFormat {
enum class Blob : protozero::pbf_tag_type {
optional_bytes_raw = 1,
optional_int32_raw_size = 2,
optional_bytes_zlib_data = 3,
optional_bytes_lzma_data = 4
};
enum class BlobHeader : protozero::pbf_tag_type {
required_string_type = 1,
optional_bytes_indexdata = 2,
required_int32_datasize = 3
};
} // namespace FileFormat
// directly translated from
// https://github.com/scrosby/OSM-binary/blob/master/src/osmformat.proto
namespace OSMFormat {
enum class HeaderBlock : protozero::pbf_tag_type {
optional_HeaderBBox_bbox = 1,
repeated_string_required_features = 4,
repeated_string_optional_features = 5,
optional_string_writingprogram = 16,
optional_string_source = 17,
optional_int64_osmosis_replication_timestamp = 32,
optional_int64_osmosis_replication_sequence_number = 33,
optional_string_osmosis_replication_base_url = 34
};
enum class HeaderBBox : protozero::pbf_tag_type {
required_sint64_left = 1,
required_sint64_right = 2,
required_sint64_top = 3,
required_sint64_bottom = 4
};
enum class PrimitiveBlock : protozero::pbf_tag_type {
required_StringTable_stringtable = 1,
repeated_PrimitiveGroup_primitivegroup = 2,
optional_int32_granularity = 17,
optional_int32_date_granularity = 18,
optional_int64_lat_offset = 19,
optional_int64_lon_offset = 20
};
enum class PrimitiveGroup : protozero::pbf_tag_type {
unknown = 0,
repeated_Node_nodes = 1,
optional_DenseNodes_dense = 2,
repeated_Way_ways = 3,
repeated_Relation_relations = 4,
repeated_ChangeSet_changesets = 5
};
enum class StringTable : protozero::pbf_tag_type {
repeated_bytes_s = 1
};
enum class Info : protozero::pbf_tag_type {
optional_int32_version = 1,
optional_int64_timestamp = 2,
optional_int64_changeset = 3,
optional_int32_uid = 4,
optional_uint32_user_sid = 5,
optional_bool_visible = 6
};
enum class DenseInfo : protozero::pbf_tag_type {
packed_int32_version = 1,
packed_sint64_timestamp = 2,
packed_sint64_changeset = 3,
packed_sint32_uid = 4,
packed_sint32_user_sid = 5,
packed_bool_visible = 6
};
enum class Node : protozero::pbf_tag_type {
required_sint64_id = 1,
packed_uint32_keys = 2,
packed_uint32_vals = 3,
optional_Info_info = 4,
required_sint64_lat = 8,
required_sint64_lon = 9
};
enum class DenseNodes : protozero::pbf_tag_type {
packed_sint64_id = 1,
optional_DenseInfo_denseinfo = 5,
packed_sint64_lat = 8,
packed_sint64_lon = 9,
packed_int32_keys_vals = 10
};
enum class Way : protozero::pbf_tag_type {
required_int64_id = 1,
packed_uint32_keys = 2,
packed_uint32_vals = 3,
optional_Info_info = 4,
packed_sint64_refs = 8
};
enum class Relation : protozero::pbf_tag_type {
required_int64_id = 1,
packed_uint32_keys = 2,
packed_uint32_vals = 3,
optional_Info_info = 4,
packed_int32_roles_sid = 8,
packed_sint64_memids = 9,
packed_MemberType_types = 10
};
} // namespace OSMFormat
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP
third_party/libosmium/include/osmium/io/detail/queue_util.hpp
Vendored
-157
View File
@@ -1,157 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_QUEUE_UTIL_HPP
#define OSMIUM_IO_DETAIL_QUEUE_UTIL_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <algorithm>
#include <exception>
#include <future>
#include <string>
#include <osmium/memory/buffer.hpp>
#include <osmium/thread/queue.hpp>
namespace osmium {
namespace io {
namespace detail {
/**
* This type of queue contains buffers with OSM data in them.
* The "end of file" is marked by an invalid Buffer.
* The buffers are wrapped in a std::future so that they can also
* transport exceptions. The future also helps with keeping the
* data in order.
*/
using future_buffer_queue_type = osmium::thread::Queue<std::future<osmium::memory::Buffer>>;
/**
* This type of queue contains OSM file data in the form it is
* stored on disk, ie encoded as XML, PBF, etc.
* The "end of file" is marked by an empty string.
*/
using string_queue_type = osmium::thread::Queue<std::string>;
/**
* This type of queue contains OSM file data in the form it is
* stored on disk, ie encoded as XML, PBF, etc.
* The "end of file" is marked by an empty string.
* The strings are wrapped in a std::future so that they can also
* transport exceptions. The future also helps with keeping the
* data in order.
*/
using future_string_queue_type = osmium::thread::Queue<std::future<std::string>>;
template <typename T>
inline void add_to_queue(osmium::thread::Queue<std::future<T>>& queue, T&& data) {
std::promise<T> promise;
queue.push(promise.get_future());
promise.set_value(std::forward<T>(data));
}
template <typename T>
inline void add_to_queue(osmium::thread::Queue<std::future<T>>& queue, std::exception_ptr&& exception) {
std::promise<T> promise;
queue.push(promise.get_future());
promise.set_exception(std::move(exception));
}
template <typename T>
inline void add_end_of_data_to_queue(osmium::thread::Queue<std::future<T>>& queue) {
add_to_queue<T>(queue, T{});
}
inline bool at_end_of_data(const std::string& data) {
return data.empty();
}
inline bool at_end_of_data(osmium::memory::Buffer& buffer) {
return !buffer;
}
template <typename T>
class queue_wrapper {
using queue_type = osmium::thread::Queue<std::future<T>>;
queue_type& m_queue;
bool m_has_reached_end_of_data;
public:
explicit queue_wrapper(queue_type& queue) :
m_queue(queue),
m_has_reached_end_of_data(false) {
}
~queue_wrapper() noexcept {
drain();
}
void drain() {
while (!m_has_reached_end_of_data) {
try {
pop();
} catch (...) {
// Ignore any exceptions.
}
}
}
bool has_reached_end_of_data() const noexcept {
return m_has_reached_end_of_data;
}
T pop() {
T data;
if (!m_has_reached_end_of_data) {
std::future<T> data_future;
m_queue.wait_and_pop(data_future);
data = std::move(data_future.get());
if (at_end_of_data(data)) {
m_has_reached_end_of_data = true;
}
}
return data;
}
}; // class queue_wrapper
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_QUEUE_UTIL_HPP
third_party/libosmium/include/osmium/io/detail/read_thread.hpp
Vendored
-133
View File
@@ -1,133 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_READ_THREAD_HPP
#define OSMIUM_IO_DETAIL_READ_THREAD_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <atomic>
#include <exception>
#include <string>
#include <thread>
#include <utility>
#include <osmium/io/compression.hpp>
#include <osmium/io/detail/queue_util.hpp>
#include <osmium/thread/util.hpp>
namespace osmium {
namespace io {
namespace detail {
/**
* This code uses an internally managed thread to read data from
* the input file and (optionally) decompress it. The result is
* sent to the given queue. Any exceptions will also be send to
* the queue.
*/
class ReadThreadManager {
// only used in the sub-thread
osmium::io::Decompressor& m_decompressor;
future_string_queue_type& m_queue;
// used in both threads
std::atomic<bool> m_done;
// only used in the main thread
std::thread m_thread;
void run_in_thread() {
osmium::thread::set_thread_name("_osmium_read");
try {
while (!m_done) {
std::string data {m_decompressor.read()};
if (at_end_of_data(data)) {
break;
}
add_to_queue(m_queue, std::move(data));
}
m_decompressor.close();
} catch (...) {
add_to_queue(m_queue, std::current_exception());
}
add_end_of_data_to_queue(m_queue);
}
public:
ReadThreadManager(osmium::io::Decompressor& decompressor,
future_string_queue_type& queue) :
m_decompressor(decompressor),
m_queue(queue),
m_done(false),
m_thread(std::thread(&ReadThreadManager::run_in_thread, this)) {
}
ReadThreadManager(const ReadThreadManager&) = delete;
ReadThreadManager& operator=(const ReadThreadManager&) = delete;
ReadThreadManager(ReadThreadManager&&) = delete;
ReadThreadManager& operator=(ReadThreadManager&&) = delete;
~ReadThreadManager() noexcept {
try {
close();
} catch (...) {
// Ignore any exceptions because destructor must not throw.
}
}
void stop() noexcept {
m_done = true;
}
void close() {
stop();
if (m_thread.joinable()) {
m_thread.join();
}
}
}; // class ReadThreadManager
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_READ_THREAD_HPP
third_party/libosmium/include/osmium/io/detail/read_write.hpp
Vendored
-180
View File
@@ -1,180 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_READ_WRITE_HPP
#define OSMIUM_IO_DETAIL_READ_WRITE_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cerrno>
#include <cstddef>
#include <errno.h>
#include <fcntl.h>
#include <string>
#include <system_error>
#ifndef _MSC_VER
# include <unistd.h>
#else
# include <io.h>
#endif
#include <osmium/io/writer_options.hpp>
namespace osmium {
namespace io {
/**
* @brief Namespace for Osmium internal use
*/
namespace detail {
/**
* Open file for writing. If the file exists, it is truncated, if
* not, it is created. If the file name is empty or "-", no file
* is opened and the stdout file descriptor (1) is returned.
*
* @param filename Name of file to be opened.
* @param allow_overwrite If the file exists, should it be overwritten?
* @returns File descriptor of open file.
* @throws system_error if the file can't be opened.
*/
inline int open_for_writing(const std::string& filename, osmium::io::overwrite allow_overwrite = osmium::io::overwrite::no) {
if (filename == "" || filename == "-") {
#ifdef _WIN32
_setmode(1, _O_BINARY);
#endif
return 1; // stdout
}
int flags = O_WRONLY | O_CREAT;
if (allow_overwrite == osmium::io::overwrite::allow) {
flags |= O_TRUNC;
} else {
flags |= O_EXCL;
}
#ifdef _WIN32
flags |= O_BINARY;
#endif
int fd = ::open(filename.c_str(), flags, 0666);
if (fd < 0) {
throw std::system_error(errno, std::system_category(), std::string("Open failed for '") + filename + "'");
}
return fd;
}
/**
* Open file for reading. If the file name is empty or "-", no file
* is opened and the stdin file descriptor (0) is returned.
*
* @param filename Name of file to be opened.
* @returns File descriptor of open file.
* @throws system_error if the file can't be opened.
*/
inline int open_for_reading(const std::string& filename) {
if (filename == "" || filename == "-") {
return 0; // stdin
}
int flags = O_RDONLY;
#ifdef _WIN32
flags |= O_BINARY;
#endif
int fd = ::open(filename.c_str(), flags);
if (fd < 0) {
throw std::system_error(errno, std::system_category(), std::string("Open failed for '") + filename + "'");
}
return fd;
}
/**
* Writes the given number of bytes from the output_buffer to the file descriptor.
* This is just a wrapper around write(2), because write(2) can write less than
* the given number of bytes.
*
* @param fd File descriptor.
* @param output_buffer Buffer with data to be written. Must be at least size bytes long.
* @param size Number of bytes to write.
* @throws std::system_error On error.
*/
inline void reliable_write(const int fd, const unsigned char* output_buffer, const size_t size) {
constexpr size_t max_write = 100L * 1024L * 1024L; // Max 100 MByte per write
size_t offset = 0;
do {
auto write_count = size - offset;
if (write_count > max_write) {
write_count = max_write;
}
auto length = ::write(fd, output_buffer + offset, static_cast<unsigned int>(write_count));
if (length < 0) {
throw std::system_error(errno, std::system_category(), "Write failed");
}
offset += static_cast<size_t>(length);
} while (offset < size);
}
/**
* Writes the given number of bytes from the output_buffer to the file descriptor.
* This is just a wrapper around write(2), because write(2) can write less than
* the given number of bytes.
*
* @param fd File descriptor.
* @param output_buffer Buffer with data to be written. Must be at least size bytes long.
* @param size Number of bytes to write.
* @throws std::system_error On error.
*/
inline void reliable_write(const int fd, const char* output_buffer, const size_t size) {
reliable_write(fd, reinterpret_cast<const unsigned char*>(output_buffer), size);
}
inline void reliable_fsync(const int fd) {
#ifdef _WIN32
if (_commit(fd) != 0) {
#else
if (::fsync(fd) != 0) {
#endif
throw std::system_error(errno, std::system_category(), "Fsync failed");
}
}
inline void reliable_close(const int fd) {
if (::close(fd) != 0) {
throw std::system_error(errno, std::system_category(), "Close failed");
}
}
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_READ_WRITE_HPP
third_party/libosmium/include/osmium/io/detail/string_table.hpp
Vendored
-265
View File
@@ -1,265 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_STRING_TABLE_HPP
#define OSMIUM_IO_DETAIL_STRING_TABLE_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cassert>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <iterator>
#include <list>
#include <map>
#include <string>
#include <osmium/io/detail/pbf.hpp>
namespace osmium {
namespace io {
namespace detail {
/**
* class StringStore
*
* Storage of lots of strings (const char *). Memory is allocated in chunks.
* If a string is added and there is no space in the current chunk, a new
* chunk will be allocated. Strings added to the store must not be larger
* than the chunk size.
*
* All memory is released when the destructor is called. There is no other way
* to release all or part of the memory.
*
*/
class StringStore {
size_t m_chunk_size;
std::list<std::string> m_chunks;
void add_chunk() {
m_chunks.push_front(std::string());
m_chunks.front().reserve(m_chunk_size);
}
public:
explicit StringStore(size_t chunk_size) :
m_chunk_size(chunk_size),
m_chunks() {
add_chunk();
}
void clear() noexcept {
m_chunks.erase(std::next(m_chunks.begin()), m_chunks.end());
m_chunks.front().clear();
}
/**
* Add a null terminated string to the store. This will
* automatically get more memory if we are out.
* Returns a pointer to the copy of the string we have
* allocated.
*/
const char* add(const char* string) {
size_t len = std::strlen(string) + 1;
assert(len <= m_chunk_size);
size_t chunk_len = m_chunks.front().size();
if (chunk_len + len > m_chunks.front().capacity()) {
add_chunk();
chunk_len = 0;
}
m_chunks.front().append(string);
m_chunks.front().append(1, '\0');
return m_chunks.front().c_str() + chunk_len;
}
class const_iterator : public std::iterator<std::forward_iterator_tag, const char*> {
typedef std::list<std::string>::const_iterator it_type;
it_type m_it;
const it_type m_last;
const char* m_pos;
public:
const_iterator(it_type it, it_type last) :
m_it(it),
m_last(last),
m_pos(it == last ? nullptr : m_it->c_str()) {
}
const_iterator& operator++() {
assert(m_it != m_last);
auto last_pos = m_it->c_str() + m_it->size();
while (m_pos != last_pos && *m_pos) ++m_pos;
if (m_pos != last_pos) ++m_pos;
if (m_pos == last_pos) {
++m_it;
if (m_it != m_last) {
m_pos = m_it->c_str();
} else {
m_pos = nullptr;
}
}
return *this;
}
const_iterator operator++(int) {
const_iterator tmp(*this);
operator++();
return tmp;
}
bool operator==(const const_iterator& rhs) const {
return m_it == rhs.m_it && m_pos == rhs.m_pos;
}
bool operator!=(const const_iterator& rhs) const {
return !(*this == rhs);
}
const char* operator*() const {
assert(m_it != m_last);
assert(m_pos != nullptr);
return m_pos;
}
}; // class const_iterator
const_iterator begin() const {
if (m_chunks.front().empty()) {
return end();
}
return const_iterator(m_chunks.begin(), m_chunks.end());
}
const_iterator end() const {
return const_iterator(m_chunks.end(), m_chunks.end());
}
// These functions get you some idea how much memory was
// used.
size_t get_chunk_size() const noexcept {
return m_chunk_size;
}
size_t get_chunk_count() const noexcept {
return m_chunks.size();
}
size_t get_used_bytes_in_last_chunk() const noexcept {
return m_chunks.front().size();
}
}; // class StringStore
struct StrComp {
bool operator()(const char* lhs, const char* rhs) const {
return strcmp(lhs, rhs) < 0;
}
}; // struct StrComp
class StringTable {
// This is the maximum number of entries in a string table.
// This should never be reached in practice but we better
// make sure it doesn't. If we had max_uncompressed_blob_size
// many entries, we are sure they would never fit into a PBF
// Blob.
static constexpr const uint32_t max_entries = max_uncompressed_blob_size;
StringStore m_strings;
std::map<const char*, size_t, StrComp> m_index;
uint32_t m_size;
public:
StringTable() :
m_strings(1024 * 1024),
m_index(),
m_size(0) {
m_strings.add("");
}
void clear() {
m_strings.clear();
m_index.clear();
m_size = 0;
m_strings.add("");
}
uint32_t size() const noexcept {
return m_size + 1;
}
uint32_t add(const char* s) {
auto f = m_index.find(s);
if (f != m_index.end()) {
return uint32_t(f->second);
}
const char* cs = m_strings.add(s);
m_index[cs] = ++m_size;
if (m_size > max_entries) {
throw osmium::pbf_error("string table has too many entries");
}
return m_size;
}
StringStore::const_iterator begin() const {
return m_strings.begin();
}
StringStore::const_iterator end() const {
return m_strings.end();
}
}; // class StringTable
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_STRING_TABLE_HPP
third_party/libosmium/include/osmium/io/detail/string_util.hpp
Vendored
-209
View File
@@ -1,209 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_STRING_UTIL_HPP
#define OSMIUM_IO_DETAIL_STRING_UTIL_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cassert>
#include <cstdint>
#include <cstring>
#include <string>
#include <utility>
#include <utf8.h>
namespace osmium {
namespace io {
namespace detail {
#ifndef _MSC_VER
# define SNPRINTF std::snprintf
#else
# define SNPRINTF _snprintf
#endif
template <typename... TArgs>
inline int string_snprintf(std::string& out,
size_t old_size,
size_t max_size,
const char* format,
TArgs&&... args) {
out.resize(old_size + max_size);
return SNPRINTF(max_size ? const_cast<char*>(out.c_str()) + old_size : nullptr,
max_size,
format,
std::forward<TArgs>(args)...);
}
#undef SNPRINTF
/**
* This is a helper function for writing printf-like formatted
* data into a std::string.
*
* @param out The data will be appended to this string.
* @param format A string with formatting instructions a la printf.
* @param args Any further arguments like in printf.
* @throws std::bad_alloc If the string needed to grow and there
* wasn't enough memory.
*/
template <typename... TArgs>
inline void append_printf_formatted_string(std::string& out,
const char* format,
TArgs&&... args) {
// First try to write string with the max_size, if that doesn't
// work snprintf will tell us how much space it needs. We
// reserve that much space and try again. So this will always
// work, even if the output is larger than the given max_size.
//
// Unfortunately this trick doesn't work on Windows, because
// the _snprintf() function there only returns the length it
// needs if max_size==0 and the buffer pointer is the null
// pointer. So we have to take this into account.
#ifndef _MSC_VER
static const size_t max_size = 100;
#else
static const size_t max_size = 0;
#endif
size_t old_size = out.size();
int len = string_snprintf(out,
old_size,
max_size,
format,
std::forward<TArgs>(args)...);
assert(len > 0);
if (size_t(len) >= max_size) {
#ifndef NDEBUG
int len2 =
#endif
string_snprintf(out,
old_size,
size_t(len) + 1,
format,
std::forward<TArgs>(args)...);
assert(len2 == len);
}
out.resize(old_size + size_t(len));
}
inline void append_utf8_encoded_string(std::string& out, const char* data) {
const char* end = data + std::strlen(data);
while (data != end) {
const char* last = data;
uint32_t c = utf8::next(data, end);
// This is a list of Unicode code points that we let
// through instead of escaping them. It is incomplete
// and can be extended later.
// Generally we don't want to let through any character
// that has special meaning in the OPL format such as
// space, comma, @, etc. and any non-printing characters.
if ((0x0021 <= c && c <= 0x0024) ||
(0x0026 <= c && c <= 0x002b) ||
(0x002d <= c && c <= 0x003c) ||
(0x003e <= c && c <= 0x003f) ||
(0x0041 <= c && c <= 0x007e) ||
(0x00a1 <= c && c <= 0x00ac) ||
(0x00ae <= c && c <= 0x05ff)) {
out.append(last, data);
} else {
out += '%';
if (c <= 0xff) {
append_printf_formatted_string(out, "%02x", c);
} else {
append_printf_formatted_string(out, "%04x", c);
}
out += '%';
}
}
}
inline void append_xml_encoded_string(std::string& out, const char* data) {
for (; *data != '\0'; ++data) {
switch(*data) {
case '&': out += "&amp;"; break;
case '\"': out += "&quot;"; break;
case '\'': out += "&apos;"; break;
case '<': out += "&lt;"; break;
case '>': out += "&gt;"; break;
case '\n': out += "&#xA;"; break;
case '\r': out += "&#xD;"; break;
case '\t': out += "&#x9;"; break;
default: out += *data; break;
}
}
}
inline void append_debug_encoded_string(std::string& out, const char* data, const char* prefix, const char* suffix) {
const char* end = data + std::strlen(data);
while (data != end) {
const char* last = data;
uint32_t c = utf8::next(data, end);
// This is a list of Unicode code points that we let
// through instead of escaping them. It is incomplete
// and can be extended later.
// Generally we don't want to let through any
// non-printing characters.
if ((0x0020 <= c && c <= 0x0021) ||
(0x0023 <= c && c <= 0x003b) ||
(0x003d == c) ||
(0x003f <= c && c <= 0x007e) ||
(0x00a1 <= c && c <= 0x00ac) ||
(0x00ae <= c && c <= 0x05ff)) {
out.append(last, data);
} else {
out.append(prefix);
append_printf_formatted_string(out, "<U+%04X>", c);
out.append(suffix);
}
}
}
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_STRING_UTIL_HPP
third_party/libosmium/include/osmium/io/detail/write_thread.hpp
Vendored
-107
View File
@@ -1,107 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_WRITE_THREAD_HPP
#define OSMIUM_IO_DETAIL_WRITE_THREAD_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <algorithm>
#include <exception>
#include <future>
#include <string>
#include <osmium/io/compression.hpp>
#include <osmium/io/detail/queue_util.hpp>
#include <osmium/thread/util.hpp>
namespace osmium {
namespace io {
namespace detail {
/**
* This codes runs in its own thread, getting data from the given
* queue, (optionally) compressing it, and writing it to the output
* file.
*/
class WriteThread {
queue_wrapper<std::string> m_queue;
std::unique_ptr<osmium::io::Compressor> m_compressor;
std::promise<bool> m_promise;
public:
WriteThread(future_string_queue_type& input_queue,
std::unique_ptr<osmium::io::Compressor>&& compressor,
std::promise<bool>&& promise) :
m_queue(input_queue),
m_compressor(std::move(compressor)),
m_promise(std::move(promise)) {
}
WriteThread(const WriteThread&) = delete;
WriteThread& operator=(const WriteThread&) = delete;
WriteThread(WriteThread&&) = delete;
WriteThread& operator=(WriteThread&&) = delete;
~WriteThread() noexcept = default;
void operator()() {
osmium::thread::set_thread_name("_osmium_write");
try {
while (true) {
std::string data = m_queue.pop();
if (at_end_of_data(data)) {
break;
}
m_compressor->write(data);
}
m_compressor->close();
m_promise.set_value(true);
} catch (...) {
m_promise.set_exception(std::current_exception());
m_queue.drain();
}
}
}; // class WriteThread
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_WRITE_THREAD_HPP
third_party/libosmium/include/osmium/io/detail/xml_input_format.hpp
Vendored
-679
View File
@@ -1,679 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_XML_INPUT_FORMAT_HPP
#define OSMIUM_IO_DETAIL_XML_INPUT_FORMAT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cassert>
#include <cstddef>
#include <cstdlib>
#include <cstring>
#include <exception>
#include <future>
#include <memory>
#include <string>
#include <utility>
#include <expat.h>
#include <osmium/builder/builder.hpp>
#include <osmium/builder/osm_object_builder.hpp>
#include <osmium/io/detail/input_format.hpp>
#include <osmium/io/detail/queue_util.hpp>
#include <osmium/io/error.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/io/header.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/osm.hpp>
#include <osmium/osm/box.hpp>
#include <osmium/osm/entity_bits.hpp>
#include <osmium/osm/item_type.hpp>
#include <osmium/osm/location.hpp>
#include <osmium/osm/object.hpp>
#include <osmium/osm/types.hpp>
#include <osmium/osm/types_from_string.hpp>
#include <osmium/thread/queue.hpp>
#include <osmium/thread/util.hpp>
#include <osmium/util/cast.hpp>
namespace osmium {
/**
* Exception thrown when the XML parser failed. The exception contains
* (if available) information about the place where the error happened
* and the type of error.
*/
struct xml_error : public io_error {
unsigned long line;
unsigned long column;
XML_Error error_code;
std::string error_string;
explicit xml_error(XML_Parser parser) :
io_error(std::string("XML parsing error at line ")
+ std::to_string(XML_GetCurrentLineNumber(parser))
+ ", column "
+ std::to_string(XML_GetCurrentColumnNumber(parser))
+ ": "
+ XML_ErrorString(XML_GetErrorCode(parser))),
line(XML_GetCurrentLineNumber(parser)),
column(XML_GetCurrentColumnNumber(parser)),
error_code(XML_GetErrorCode(parser)),
error_string(XML_ErrorString(error_code)) {
}
explicit xml_error(const std::string& message) :
io_error(message),
line(0),
column(0),
error_code(),
error_string(message) {
}
}; // struct xml_error
/**
* Exception thrown when an OSM XML files contains no version attribute
* on the 'osm' element or if the version is unknown.
*/
struct format_version_error : public io_error {
std::string version;
explicit format_version_error() :
io_error("Can not read file without version (missing version attribute on osm element)."),
version() {
}
explicit format_version_error(const char* v) :
io_error(std::string("Can not read file with version ") + v),
version(v) {
}
}; // struct format_version_error
namespace io {
namespace detail {
class XMLParser : public Parser {
static constexpr int buffer_size = 2 * 1000 * 1000;
enum class context {
root,
top,
node,
way,
relation,
changeset,
discussion,
comment,
comment_text,
ignored_node,
ignored_way,
ignored_relation,
ignored_changeset,
in_object
}; // enum class context
context m_context;
context m_last_context;
/**
* This is used only for change files which contain create, modify,
* and delete sections.
*/
bool m_in_delete_section;
osmium::io::Header m_header;
osmium::memory::Buffer m_buffer;
std::unique_ptr<osmium::builder::NodeBuilder> m_node_builder;
std::unique_ptr<osmium::builder::WayBuilder> m_way_builder;
std::unique_ptr<osmium::builder::RelationBuilder> m_relation_builder;
std::unique_ptr<osmium::builder::ChangesetBuilder> m_changeset_builder;
std::unique_ptr<osmium::builder::ChangesetDiscussionBuilder> m_changeset_discussion_builder;
std::unique_ptr<osmium::builder::TagListBuilder> m_tl_builder;
std::unique_ptr<osmium::builder::WayNodeListBuilder> m_wnl_builder;
std::unique_ptr<osmium::builder::RelationMemberListBuilder> m_rml_builder;
std::string m_comment_text;
/**
* A C++ wrapper for the Expat parser that makes sure no memory is leaked.
*/
template <typename T>
class ExpatXMLParser {
XML_Parser m_parser;
static void XMLCALL start_element_wrapper(void* data, const XML_Char* element, const XML_Char** attrs) {
static_cast<XMLParser*>(data)->start_element(element, attrs);
}
static void XMLCALL end_element_wrapper(void* data, const XML_Char* element) {
static_cast<XMLParser*>(data)->end_element(element);
}
static void XMLCALL character_data_wrapper(void* data, const XML_Char* text, int len) {
static_cast<XMLParser*>(data)->characters(text, len);
}
public:
explicit ExpatXMLParser(T* callback_object) :
m_parser(XML_ParserCreate(nullptr)) {
if (!m_parser) {
throw osmium::io_error("Internal error: Can not create parser");
}
XML_SetUserData(m_parser, callback_object);
XML_SetElementHandler(m_parser, start_element_wrapper, end_element_wrapper);
XML_SetCharacterDataHandler(m_parser, character_data_wrapper);
}
ExpatXMLParser(const ExpatXMLParser&) = delete;
ExpatXMLParser(ExpatXMLParser&&) = delete;
ExpatXMLParser& operator=(const ExpatXMLParser&) = delete;
ExpatXMLParser& operator=(ExpatXMLParser&&) = delete;
~ExpatXMLParser() noexcept {
XML_ParserFree(m_parser);
}
void operator()(const std::string& data, bool last) {
if (XML_Parse(m_parser, data.data(), static_cast_with_assert<int>(data.size()), last) == XML_STATUS_ERROR) {
throw osmium::xml_error(m_parser);
}
}
}; // class ExpatXMLParser
template <typename T>
static void check_attributes(const XML_Char** attrs, T check) {
while (*attrs) {
check(attrs[0], attrs[1]);
attrs += 2;
}
}
const char* init_object(osmium::OSMObject& object, const XML_Char** attrs) {
const char* user = "";
if (m_in_delete_section) {
object.set_visible(false);
}
osmium::Location location;
check_attributes(attrs, [&location, &user, &object](const XML_Char* name, const XML_Char* value) {
if (!strcmp(name, "lon")) {
location.set_lon(std::atof(value)); // XXX doesn't detect garbage after the number
} else if (!strcmp(name, "lat")) {
location.set_lat(std::atof(value)); // XXX doesn't detect garbage after the number
} else if (!strcmp(name, "user")) {
user = value;
} else {
object.set_attribute(name, value);
}
});
if (location && object.type() == osmium::item_type::node) {
static_cast<osmium::Node&>(object).set_location(location);
}
return user;
}
void init_changeset(osmium::builder::ChangesetBuilder* builder, const XML_Char** attrs) {
const char* user = "";
osmium::Changeset& new_changeset = builder->object();
osmium::Location min;
osmium::Location max;
check_attributes(attrs, [&min, &max, &user, &new_changeset](const XML_Char* name, const XML_Char* value) {
if (!strcmp(name, "min_lon")) {
min.set_lon(atof(value));
} else if (!strcmp(name, "min_lat")) {
min.set_lat(atof(value));
} else if (!strcmp(name, "max_lon")) {
max.set_lon(atof(value));
} else if (!strcmp(name, "max_lat")) {
max.set_lat(atof(value));
} else if (!strcmp(name, "user")) {
user = value;
} else {
new_changeset.set_attribute(name, value);
}
});
new_changeset.bounds().extend(min);
new_changeset.bounds().extend(max);
builder->add_user(user);
}
void get_tag(osmium::builder::Builder* builder, const XML_Char** attrs) {
const char* k = "";
const char* v = "";
check_attributes(attrs, [&k, &v](const XML_Char* name, const XML_Char* value) {
if (name[0] == 'k' && name[1] == 0) {
k = value;
} else if (name[0] == 'v' && name[1] == 0) {
v = value;
}
});
if (!m_tl_builder) {
m_tl_builder = std::unique_ptr<osmium::builder::TagListBuilder>(new osmium::builder::TagListBuilder(m_buffer, builder));
}
m_tl_builder->add_tag(k, v);
}
void mark_header_as_done() {
set_header_value(m_header);
}
void start_element(const XML_Char* element, const XML_Char** attrs) {
switch (m_context) {
case context::root:
if (!strcmp(element, "osm") || !strcmp(element, "osmChange")) {
if (!strcmp(element, "osmChange")) {
m_header.set_has_multiple_object_versions(true);
}
check_attributes(attrs, [this](const XML_Char* name, const XML_Char* value) {
if (!strcmp(name, "version")) {
m_header.set("version", value);
if (strcmp(value, "0.6")) {
throw osmium::format_version_error(value);
}
} else if (!strcmp(name, "generator")) {
m_header.set("generator", value);
}
});
if (m_header.get("version") == "") {
throw osmium::format_version_error();
}
} else {
throw osmium::xml_error(std::string("Unknown top-level element: ") + element);
}
m_context = context::top;
break;
case context::top:
assert(!m_tl_builder);
if (!strcmp(element, "node")) {
mark_header_as_done();
if (read_types() & osmium::osm_entity_bits::node) {
m_node_builder = std::unique_ptr<osmium::builder::NodeBuilder>(new osmium::builder::NodeBuilder(m_buffer));
m_node_builder->add_user(init_object(m_node_builder->object(), attrs));
m_context = context::node;
} else {
m_context = context::ignored_node;
}
} else if (!strcmp(element, "way")) {
mark_header_as_done();
if (read_types() & osmium::osm_entity_bits::way) {
m_way_builder = std::unique_ptr<osmium::builder::WayBuilder>(new osmium::builder::WayBuilder(m_buffer));
m_way_builder->add_user(init_object(m_way_builder->object(), attrs));
m_context = context::way;
} else {
m_context = context::ignored_way;
}
} else if (!strcmp(element, "relation")) {
mark_header_as_done();
if (read_types() & osmium::osm_entity_bits::relation) {
m_relation_builder = std::unique_ptr<osmium::builder::RelationBuilder>(new osmium::builder::RelationBuilder(m_buffer));
m_relation_builder->add_user(init_object(m_relation_builder->object(), attrs));
m_context = context::relation;
} else {
m_context = context::ignored_relation;
}
} else if (!strcmp(element, "changeset")) {
mark_header_as_done();
if (read_types() & osmium::osm_entity_bits::changeset) {
m_changeset_builder = std::unique_ptr<osmium::builder::ChangesetBuilder>(new osmium::builder::ChangesetBuilder(m_buffer));
init_changeset(m_changeset_builder.get(), attrs);
m_context = context::changeset;
} else {
m_context = context::ignored_changeset;
}
} else if (!strcmp(element, "bounds")) {
osmium::Location min;
osmium::Location max;
check_attributes(attrs, [&min, &max](const XML_Char* name, const XML_Char* value) {
if (!strcmp(name, "minlon")) {
min.set_lon(atof(value));
} else if (!strcmp(name, "minlat")) {
min.set_lat(atof(value));
} else if (!strcmp(name, "maxlon")) {
max.set_lon(atof(value));
} else if (!strcmp(name, "maxlat")) {
max.set_lat(atof(value));
}
});
osmium::Box box;
box.extend(min).extend(max);
m_header.add_box(box);
} else if (!strcmp(element, "delete")) {
m_in_delete_section = true;
}
break;
case context::node:
m_last_context = context::node;
m_context = context::in_object;
if (!strcmp(element, "tag")) {
get_tag(m_node_builder.get(), attrs);
}
break;
case context::way:
m_last_context = context::way;
m_context = context::in_object;
if (!strcmp(element, "nd")) {
m_tl_builder.reset();
if (!m_wnl_builder) {
m_wnl_builder = std::unique_ptr<osmium::builder::WayNodeListBuilder>(new osmium::builder::WayNodeListBuilder(m_buffer, m_way_builder.get()));
}
check_attributes(attrs, [this](const XML_Char* name, const XML_Char* value) {
if (!strcmp(name, "ref")) {
m_wnl_builder->add_node_ref(osmium::string_to_object_id(value));
}
});
} else if (!strcmp(element, "tag")) {
m_wnl_builder.reset();
get_tag(m_way_builder.get(), attrs);
}
break;
case context::relation:
m_last_context = context::relation;
m_context = context::in_object;
if (!strcmp(element, "member")) {
m_tl_builder.reset();
if (!m_rml_builder) {
m_rml_builder = std::unique_ptr<osmium::builder::RelationMemberListBuilder>(new osmium::builder::RelationMemberListBuilder(m_buffer, m_relation_builder.get()));
}
item_type type = item_type::undefined;
object_id_type ref = 0;
const char* role = "";
check_attributes(attrs, [&type, &ref, &role](const XML_Char* name, const XML_Char* value) {
if (!strcmp(name, "type")) {
type = char_to_item_type(value[0]);
} else if (!strcmp(name, "ref")) {
ref = osmium::string_to_object_id(value);
} else if (!strcmp(name, "role")) {
role = static_cast<const char*>(value);
}
});
if (type != item_type::node && type != item_type::way && type != item_type::relation) {
throw osmium::xml_error("Unknown type on relation member");
}
if (ref == 0) {
throw osmium::xml_error("Missing ref on relation member");
}
m_rml_builder->add_member(type, ref, role);
} else if (!strcmp(element, "tag")) {
m_rml_builder.reset();
get_tag(m_relation_builder.get(), attrs);
}
break;
case context::changeset:
m_last_context = context::changeset;
if (!strcmp(element, "discussion")) {
m_context = context::discussion;
m_tl_builder.reset();
if (!m_changeset_discussion_builder) {
m_changeset_discussion_builder = std::unique_ptr<osmium::builder::ChangesetDiscussionBuilder>(new osmium::builder::ChangesetDiscussionBuilder(m_buffer, m_changeset_builder.get()));
}
} else if (!strcmp(element, "tag")) {
m_context = context::in_object;
m_changeset_discussion_builder.reset();
get_tag(m_changeset_builder.get(), attrs);
}
break;
case context::discussion:
if (!strcmp(element, "comment")) {
m_context = context::comment;
osmium::Timestamp date;
osmium::user_id_type uid = 0;
const char* user = "";
check_attributes(attrs, [&date, &uid, &user](const XML_Char* name, const XML_Char* value) {
if (!strcmp(name, "date")) {
date = osmium::Timestamp(value);
} else if (!strcmp(name, "uid")) {
uid = osmium::string_to_user_id(value);
} else if (!strcmp(name, "user")) {
user = static_cast<const char*>(value);
}
});
m_changeset_discussion_builder->add_comment(date, uid, user);
}
break;
case context::comment:
if (!strcmp(element, "text")) {
m_context = context::comment_text;
}
break;
case context::comment_text:
break;
case context::ignored_node:
break;
case context::ignored_way:
break;
case context::ignored_relation:
break;
case context::ignored_changeset:
break;
case context::in_object:
assert(false); // should never be here
break;
}
}
void end_element(const XML_Char* element) {
switch (m_context) {
case context::root:
assert(false); // should never be here
break;
case context::top:
if (!strcmp(element, "osm") || !strcmp(element, "osmChange")) {
mark_header_as_done();
m_context = context::root;
} else if (!strcmp(element, "delete")) {
m_in_delete_section = false;
}
break;
case context::node:
assert(!strcmp(element, "node"));
m_tl_builder.reset();
m_node_builder.reset();
m_buffer.commit();
m_context = context::top;
flush_buffer();
break;
case context::way:
assert(!strcmp(element, "way"));
m_tl_builder.reset();
m_wnl_builder.reset();
m_way_builder.reset();
m_buffer.commit();
m_context = context::top;
flush_buffer();
break;
case context::relation:
assert(!strcmp(element, "relation"));
m_tl_builder.reset();
m_rml_builder.reset();
m_relation_builder.reset();
m_buffer.commit();
m_context = context::top;
flush_buffer();
break;
case context::changeset:
assert(!strcmp(element, "changeset"));
m_tl_builder.reset();
m_changeset_discussion_builder.reset();
m_changeset_builder.reset();
m_buffer.commit();
m_context = context::top;
flush_buffer();
break;
case context::discussion:
assert(!strcmp(element, "discussion"));
m_context = context::changeset;
break;
case context::comment:
assert(!strcmp(element, "comment"));
m_context = context::discussion;
break;
case context::comment_text:
assert(!strcmp(element, "text"));
m_context = context::comment;
m_changeset_discussion_builder->add_comment_text(m_comment_text);
break;
case context::in_object:
m_context = m_last_context;
break;
case context::ignored_node:
if (!strcmp(element, "node")) {
m_context = context::top;
}
break;
case context::ignored_way:
if (!strcmp(element, "way")) {
m_context = context::top;
}
break;
case context::ignored_relation:
if (!strcmp(element, "relation")) {
m_context = context::top;
}
break;
case context::ignored_changeset:
if (!strcmp(element, "changeset")) {
m_context = context::top;
}
break;
}
}
void characters(const XML_Char* text, int len) {
if (m_context == context::comment_text) {
m_comment_text.append(text, len);
} else {
m_comment_text.resize(0);
}
}
void flush_buffer() {
if (m_buffer.committed() > buffer_size / 10 * 9) {
send_to_output_queue(std::move(m_buffer));
osmium::memory::Buffer buffer(buffer_size);
using std::swap;
swap(m_buffer, buffer);
}
}
public:
XMLParser(future_string_queue_type& input_queue,
future_buffer_queue_type& output_queue,
std::promise<osmium::io::Header>& header_promise,
osmium::osm_entity_bits::type read_types) :
Parser(input_queue, output_queue, header_promise, read_types),
m_context(context::root),
m_last_context(context::root),
m_in_delete_section(false),
m_header(),
m_buffer(buffer_size),
m_node_builder(),
m_way_builder(),
m_relation_builder(),
m_changeset_builder(),
m_changeset_discussion_builder(),
m_tl_builder(),
m_wnl_builder(),
m_rml_builder() {
}
~XMLParser() noexcept final = default;
void run() final {
osmium::thread::set_thread_name("_osmium_xml_in");
ExpatXMLParser<XMLParser> parser(this);
while (!input_done()) {
std::string data = get_input();
parser(data, input_done());
if (read_types() == osmium::osm_entity_bits::nothing && header_is_done()) {
break;
}
}
mark_header_as_done();
if (m_buffer.committed() > 0) {
send_to_output_queue(std::move(m_buffer));
}
}
}; // class XMLParser
// we want the register_parser() function to run, setting
// the variable is only a side-effect, it will never be used
const bool registered_xml_parser = ParserFactory::instance().register_parser(
file_format::xml,
[](future_string_queue_type& input_queue,
future_buffer_queue_type& output_queue,
std::promise<osmium::io::Header>& header_promise,
osmium::osm_entity_bits::type read_which_entities) {
return std::unique_ptr<Parser>(new XMLParser(input_queue, output_queue, header_promise, read_which_entities));
});
// dummy function to silence the unused variable warning from above
inline bool get_registered_xml_parser() noexcept {
return registered_xml_parser;
}
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_XML_INPUT_FORMAT_HPP
third_party/libosmium/include/osmium/io/detail/xml_output_format.hpp
Vendored
-473
View File
@@ -1,473 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_XML_OUTPUT_FORMAT_HPP
#define OSMIUM_IO_DETAIL_XML_OUTPUT_FORMAT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <algorithm>
#include <cinttypes>
#include <cstddef>
#include <cstdio>
#include <future>
#include <iterator>
#include <memory>
#include <string>
#include <thread>
#include <utility>
#include <osmium/io/detail/output_format.hpp>
#include <osmium/io/file.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/io/header.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/memory/collection.hpp>
#include <osmium/osm/box.hpp>
#include <osmium/osm/changeset.hpp>
#include <osmium/osm/item_type.hpp>
#include <osmium/osm/location.hpp>
#include <osmium/osm/node.hpp>
#include <osmium/osm/object.hpp>
#include <osmium/osm/relation.hpp>
#include <osmium/osm/tag.hpp>
#include <osmium/osm/timestamp.hpp>
#include <osmium/osm/types.hpp>
#include <osmium/osm/way.hpp>
#include <osmium/thread/pool.hpp>
#include <osmium/visitor.hpp>
namespace osmium {
namespace io {
namespace detail {
struct XMLWriteError {};
struct xml_output_options {
/// Should metadata of objects be added?
bool add_metadata;
/// Should the visible flag be added to all OSM objects?
bool add_visible_flag;
/**
* Should <create>, <modify>, <delete> "operations" be added?
* (This is used for .osc files.)
*/
bool use_change_ops;
};
class XMLOutputBlock : public OutputBlock {
// operation (create, modify, delete) for osc files
enum class operation {
op_none = 0,
op_create = 1,
op_modify = 2,
op_delete = 3
}; // enum class operation
operation m_last_op {operation::op_none};
xml_output_options m_options;
void write_spaces(int num) {
for (; num != 0; --num) {
*m_out += ' ';
}
}
int prefix_spaces() {
return m_options.use_change_ops ? 4 : 2;
}
void write_prefix() {
write_spaces(prefix_spaces());
}
void write_meta(const osmium::OSMObject& object) {
output_formatted(" id=\"%" PRId64 "\"", object.id());
if (m_options.add_metadata) {
if (object.version()) {
output_formatted(" version=\"%d\"", object.version());
}
if (object.timestamp()) {
*m_out += " timestamp=\"";
*m_out += object.timestamp().to_iso();
*m_out += "\"";
}
if (!object.user_is_anonymous()) {
output_formatted(" uid=\"%d\" user=\"", object.uid());
append_xml_encoded_string(*m_out, object.user());
*m_out += "\"";
}
if (object.changeset()) {
output_formatted(" changeset=\"%d\"", object.changeset());
}
if (m_options.add_visible_flag) {
if (object.visible()) {
*m_out += " visible=\"true\"";
} else {
*m_out += " visible=\"false\"";
}
}
}
}
void write_tags(const osmium::TagList& tags, int spaces) {
for (const auto& tag : tags) {
write_spaces(spaces);
*m_out += " <tag k=\"";
append_xml_encoded_string(*m_out, tag.key());
*m_out += "\" v=\"";
append_xml_encoded_string(*m_out, tag.value());
*m_out += "\"/>\n";
}
}
void write_discussion(const osmium::ChangesetDiscussion& comments) {
for (const auto& comment : comments) {
output_formatted(" <comment uid=\"%d\" user=\"", comment.uid());
append_xml_encoded_string(*m_out, comment.user());
*m_out += "\" date=\"";
*m_out += comment.date().to_iso();
*m_out += "\">\n";
*m_out += " <text>";
append_xml_encoded_string(*m_out, comment.text());
*m_out += "</text>\n </comment>\n";
}
*m_out += " </discussion>\n";
}
void open_close_op_tag(const operation op = operation::op_none) {
if (op == m_last_op) {
return;
}
switch (m_last_op) {
case operation::op_none:
break;
case operation::op_create:
*m_out += " </create>\n";
break;
case operation::op_modify:
*m_out += " </modify>\n";
break;
case operation::op_delete:
*m_out += " </delete>\n";
break;
}
switch (op) {
case operation::op_none:
break;
case operation::op_create:
*m_out += " <create>\n";
break;
case operation::op_modify:
*m_out += " <modify>\n";
break;
case operation::op_delete:
*m_out += " <delete>\n";
break;
}
m_last_op = op;
}
public:
XMLOutputBlock(osmium::memory::Buffer&& buffer, const xml_output_options& options) :
OutputBlock(std::move(buffer)),
m_options(options) {
}
XMLOutputBlock(const XMLOutputBlock&) = default;
XMLOutputBlock& operator=(const XMLOutputBlock&) = default;
XMLOutputBlock(XMLOutputBlock&&) = default;
XMLOutputBlock& operator=(XMLOutputBlock&&) = default;
~XMLOutputBlock() noexcept = default;
std::string operator()() {
osmium::apply(m_input_buffer->cbegin(), m_input_buffer->cend(), *this);
if (m_options.use_change_ops) {
open_close_op_tag();
}
std::string out;
using std::swap;
swap(out, *m_out);
return out;
}
void node(const osmium::Node& node) {
if (m_options.use_change_ops) {
open_close_op_tag(node.visible() ? (node.version() == 1 ? operation::op_create : operation::op_modify) : operation::op_delete);
}
write_prefix();
*m_out += "<node";
write_meta(node);
if (node.location()) {
*m_out += " lat=\"";
osmium::util::double2string(std::back_inserter(*m_out), node.location().lat_without_check(), 7);
*m_out += "\" lon=\"";
osmium::util::double2string(std::back_inserter(*m_out), node.location().lon_without_check(), 7);
*m_out += "\"";
}
if (node.tags().empty()) {
*m_out += "/>\n";
return;
}
*m_out += ">\n";
write_tags(node.tags(), prefix_spaces());
write_prefix();
*m_out += "</node>\n";
}
void way(const osmium::Way& way) {
if (m_options.use_change_ops) {
open_close_op_tag(way.visible() ? (way.version() == 1 ? operation::op_create : operation::op_modify) : operation::op_delete);
}
write_prefix();
*m_out += "<way";
write_meta(way);
if (way.tags().empty() && way.nodes().empty()) {
*m_out += "/>\n";
return;
}
*m_out += ">\n";
for (const auto& node_ref : way.nodes()) {
write_prefix();
output_formatted(" <nd ref=\"%" PRId64 "\"/>\n", node_ref.ref());
}
write_tags(way.tags(), prefix_spaces());
write_prefix();
*m_out += "</way>\n";
}
void relation(const osmium::Relation& relation) {
if (m_options.use_change_ops) {
open_close_op_tag(relation.visible() ? (relation.version() == 1 ? operation::op_create : operation::op_modify) : operation::op_delete);
}
write_prefix();
*m_out += "<relation";
write_meta(relation);
if (relation.tags().empty() && relation.members().empty()) {
*m_out += "/>\n";
return;
}
*m_out += ">\n";
for (const auto& member : relation.members()) {
write_prefix();
*m_out += " <member type=\"";
*m_out += item_type_to_name(member.type());
output_formatted("\" ref=\"%" PRId64 "\" role=\"", member.ref());
append_xml_encoded_string(*m_out, member.role());
*m_out += "\"/>\n";
}
write_tags(relation.tags(), prefix_spaces());
write_prefix();
*m_out += "</relation>\n";
}
void changeset(const osmium::Changeset& changeset) {
*m_out += " <changeset";
output_formatted(" id=\"%" PRId32 "\"", changeset.id());
if (changeset.created_at()) {
*m_out += " created_at=\"";
*m_out += changeset.created_at().to_iso();
*m_out += "\"";
}
if (changeset.closed_at()) {
*m_out += " closed_at=\"";
*m_out += changeset.closed_at().to_iso();
*m_out += "\" open=\"false\"";
} else {
*m_out += " open=\"true\"";
}
if (!changeset.user_is_anonymous()) {
*m_out += " user=\"";
append_xml_encoded_string(*m_out, changeset.user());
output_formatted("\" uid=\"%d\"", changeset.uid());
}
if (changeset.bounds()) {
output_formatted(" min_lat=\"%.7f\"", changeset.bounds().bottom_left().lat_without_check());
output_formatted(" min_lon=\"%.7f\"", changeset.bounds().bottom_left().lon_without_check());
output_formatted(" max_lat=\"%.7f\"", changeset.bounds().top_right().lat_without_check());
output_formatted(" max_lon=\"%.7f\"", changeset.bounds().top_right().lon_without_check());
}
output_formatted(" num_changes=\"%" PRId32 "\"", changeset.num_changes());
output_formatted(" comments_count=\"%" PRId32 "\"", changeset.num_comments());
// If there are no tags and no comments, we can close the
// tag right here and are done.
if (changeset.tags().empty() && changeset.num_comments() == 0) {
*m_out += "/>\n";
return;
}
*m_out += ">\n";
write_tags(changeset.tags(), 0);
if (changeset.num_comments() > 0) {
*m_out += " <discussion>\n";
write_discussion(changeset.discussion());
}
*m_out += " </changeset>\n";
}
}; // class XMLOutputBlock
class XMLOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler {
xml_output_options m_options;
public:
XMLOutputFormat(const osmium::io::File& file, future_string_queue_type& output_queue) :
OutputFormat(output_queue),
m_options() {
m_options.add_metadata = file.is_not_false("add_metadata");
m_options.use_change_ops = file.is_true("xml_change_format");
m_options.add_visible_flag = (file.has_multiple_object_versions() || file.is_true("force_visible_flag")) && !m_options.use_change_ops;
}
XMLOutputFormat(const XMLOutputFormat&) = delete;
XMLOutputFormat& operator=(const XMLOutputFormat&) = delete;
~XMLOutputFormat() noexcept final = default;
void write_header(const osmium::io::Header& header) final {
std::string out = "<?xml version='1.0' encoding='UTF-8'?>\n";
if (m_options.use_change_ops) {
out += "<osmChange version=\"0.6\" generator=\"";
} else {
out += "<osm version=\"0.6\"";
std::string xml_josm_upload = header.get("xml_josm_upload");
if (xml_josm_upload == "true" || xml_josm_upload == "false") {
out += " upload=\"";
out += xml_josm_upload;
out += "\"";
}
out += " generator=\"";
}
append_xml_encoded_string(out, header.get("generator").c_str());
out += "\">\n";
for (const auto& box : header.boxes()) {
out += " <bounds";
append_printf_formatted_string(out, " minlon=\"%.7f\"", box.bottom_left().lon());
append_printf_formatted_string(out, " minlat=\"%.7f\"", box.bottom_left().lat());
append_printf_formatted_string(out, " maxlon=\"%.7f\"", box.top_right().lon());
append_printf_formatted_string(out, " maxlat=\"%.7f\"/>\n", box.top_right().lat());
}
send_to_output_queue(std::move(out));
}
void write_buffer(osmium::memory::Buffer&& buffer) final {
m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_options}));
}
void write_end() final {
std::string out;
if (m_options.use_change_ops) {
out += "</osmChange>\n";
} else {
out += "</osm>\n";
}
send_to_output_queue(std::move(out));
}
}; // class XMLOutputFormat
// we want the register_output_format() function to run, setting
// the variable is only a side-effect, it will never be used
const bool registered_xml_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::xml,
[](const osmium::io::File& file, future_string_queue_type& output_queue) {
return new osmium::io::detail::XMLOutputFormat(file, output_queue);
});
// dummy function to silence the unused variable warning from above
inline bool get_registered_xml_output() noexcept {
return registered_xml_output;
}
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_XML_OUTPUT_FORMAT_HPP
third_party/libosmium/include/osmium/io/detail/zlib.hpp
Vendored
-115
View File
@@ -1,115 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_ZLIB_HPP
#define OSMIUM_IO_DETAIL_ZLIB_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <memory>
#include <stdexcept>
#include <string>
#include <zlib.h>
#include <osmium/io/error.hpp>
#include <osmium/util/cast.hpp>
namespace osmium {
namespace io {
namespace detail {
/**
* Compress data using zlib.
*
* Note that this function can not compress data larger than
* what fits in an unsigned long, on Windows this is usually 32bit.
*
* @param input Data to compress.
* @returns Compressed data.
*/
inline std::string zlib_compress(const std::string& input) {
unsigned long output_size = ::compressBound(osmium::static_cast_with_assert<unsigned long>(input.size()));
std::string output(output_size, '\0');
auto result = ::compress(
reinterpret_cast<unsigned char*>(const_cast<char *>(output.data())),
&output_size,
reinterpret_cast<const unsigned char*>(input.data()),
osmium::static_cast_with_assert<unsigned long>(input.size())
);
if (result != Z_OK) {
throw io_error(std::string("failed to compress data: ") + zError(result));
}
output.resize(output_size);
return output;
}
/**
* Uncompress data using zlib.
*
* Note that this function can not uncompress data larger than
* what fits in an unsigned long, on Windows this is usually 32bit.
*
* @param input Compressed input data.
* @param raw_size Size of uncompressed data.
* @param output Uncompressed result data.
* @returns Pointer and size to incompressed data.
*/
inline std::pair<const char*, size_t> zlib_uncompress_string(const char* input, unsigned long input_size, unsigned long raw_size, std::string& output) {
output.resize(raw_size);
auto result = ::uncompress(
reinterpret_cast<unsigned char*>(&*output.begin()),
&raw_size,
reinterpret_cast<const unsigned char*>(input),
input_size
);
if (result != Z_OK) {
throw io_error(std::string("failed to uncompress data: ") + zError(result));
}
return std::make_pair(output.data(), output.size());
}
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_ZLIB_HPP
third_party/libosmium/include/osmium/io/error.hpp
Vendored
-70
View File
@@ -1,70 +0,0 @@
#ifndef OSMIUM_IO_ERROR_HPP
#define OSMIUM_IO_ERROR_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <stdexcept>
#include <string>
namespace osmium {
/**
* Exception thrown when some kind of input/output operation failed.
*/
struct io_error : public std::runtime_error {
explicit io_error(const std::string& what) :
std::runtime_error(what) {
}
explicit io_error(const char* what) :
std::runtime_error(what) {
}
}; // struct io_error
struct unsupported_file_format_error : public io_error {
explicit unsupported_file_format_error(const std::string& what) :
io_error(what) {
}
explicit unsupported_file_format_error(const char* what) :
io_error(what) {
}
}; // struct unsupported_file_format_error
} // namespace osmium
#endif // OSMIUM_IO_ERROR_HPP
third_party/libosmium/include/osmium/io/file.hpp
Vendored
-328
View File
@@ -1,328 +0,0 @@
#ifndef OSMIUM_IO_FILE_HPP
#define OSMIUM_IO_FILE_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cstddef>
#include <stdexcept>
#include <sstream>
#include <string>
#include <vector>
#include <osmium/io/error.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/io/file_compression.hpp>
#include <osmium/util/options.hpp>
#include <osmium/util/compatibility.hpp>
namespace osmium {
/**
* @brief Everything related to input and output of OSM data.
*/
namespace io {
namespace detail {
inline std::vector<std::string> split(const std::string& in, const char delim) {
std::vector<std::string> result;
std::stringstream ss(in);
std::string item;
while (std::getline(ss, item, delim)) {
result.push_back(item);
}
return result;
}
} // namespace detail
/**
* This class describes an OSM file in one of several different formats.
*
* If the filename is empty or "-", this means stdin or stdout is used.
*/
class File : public osmium::util::Options {
private:
std::string m_filename;
const char* m_buffer;
size_t m_buffer_size;
std::string m_format_string;
file_format m_file_format {file_format::unknown};
file_compression m_file_compression {file_compression::none};
bool m_has_multiple_object_versions {false};
public:
/**
* Create File using type and encoding from filename or given
* format specification.
*
* @param filename Filename including suffix. The type and encoding
* of the file will be taken from the suffix.
* An empty filename or "-" means stdin or stdout.
* @param format File format as string. See the description of the
* parse_format() function for details. If this is
* empty the format will be deduced from the suffix
* of the filename.
*/
explicit File(const std::string& filename = "", const std::string& format = "") :
Options(),
m_filename(filename),
m_buffer(nullptr),
m_buffer_size(0),
m_format_string(format) {
// stdin/stdout
if (m_filename == "-") {
m_filename = "";
}
// if filename is a URL, default to XML format
std::string protocol = m_filename.substr(0, m_filename.find_first_of(':'));
if (protocol == "http" || protocol == "https") {
m_file_format = file_format::xml;
}
if (format.empty()) {
detect_format_from_suffix(m_filename);
} else {
parse_format(format);
}
}
/**
* Create File using buffer pointer and size and type and encoding
* from given format specification.
*
* @param buffer Pointer to buffer with data.
* @param size Size of buffer.
* @param format File format as string. See the description of the
* parse_format() function for details.
*/
explicit File(const char* buffer, size_t size, const std::string& format = "") :
Options(),
m_filename(),
m_buffer(buffer),
m_buffer_size(size),
m_format_string(format) {
if (format != "") {
parse_format(format);
}
}
File(const File&) = default;
File& operator=(const File&) = default;
File(File&&) = default;
File& operator=(File&&) = default;
~File() = default;
const char* buffer() const noexcept {
return m_buffer;
}
size_t buffer_size() const noexcept {
return m_buffer_size;
}
void parse_format(const std::string& format) {
std::vector<std::string> options = detail::split(format, ',');
// if the first item in the format list doesn't contain
// an equals sign, it is a format
if (!options.empty() && options[0].find_first_of('=') == std::string::npos) {
detect_format_from_suffix(options[0]);
options.erase(options.begin());
}
for (auto& option : options) {
size_t pos = option.find_first_of('=');
if (pos == std::string::npos) {
set(option, true);
} else {
std::string value = option.substr(pos+1);
option.erase(pos);
set(option, value);
}
}
if (get("history") == "true") {
m_has_multiple_object_versions = true;
} else if (get("history") == "false") {
m_has_multiple_object_versions = false;
}
}
void detect_format_from_suffix(const std::string& name) {
std::vector<std::string> suffixes = detail::split(name, '.');
if (suffixes.empty()) return;
// if the last suffix is one of a known set of compressions,
// set that compression
if (suffixes.back() == "gz") {
m_file_compression = file_compression::gzip;
suffixes.pop_back();
} else if (suffixes.back() == "bz2") {
m_file_compression = file_compression::bzip2;
suffixes.pop_back();
}
if (suffixes.empty()) return;
// if the last suffix is one of a known set of formats,
// set that format
if (suffixes.back() == "pbf") {
m_file_format = file_format::pbf;
suffixes.pop_back();
} else if (suffixes.back() == "xml") {
m_file_format = file_format::xml;
suffixes.pop_back();
} else if (suffixes.back() == "opl") {
m_file_format = file_format::opl;
suffixes.pop_back();
} else if (suffixes.back() == "json") {
m_file_format = file_format::json;
suffixes.pop_back();
} else if (suffixes.back() == "o5m") {
m_file_format = file_format::o5m;
suffixes.pop_back();
} else if (suffixes.back() == "o5c") {
m_file_format = file_format::o5m;
m_has_multiple_object_versions = true;
set("o5c_change_format", true);
suffixes.pop_back();
} else if (suffixes.back() == "debug") {
m_file_format = file_format::debug;
suffixes.pop_back();
}
if (suffixes.empty()) return;
if (suffixes.back() == "osm") {
if (m_file_format == file_format::unknown) m_file_format = file_format::xml;
suffixes.pop_back();
} else if (suffixes.back() == "osh") {
if (m_file_format == file_format::unknown) m_file_format = file_format::xml;
m_has_multiple_object_versions = true;
suffixes.pop_back();
} else if (suffixes.back() == "osc") {
if (m_file_format == file_format::unknown) m_file_format = file_format::xml;
m_has_multiple_object_versions = true;
set("xml_change_format", true);
suffixes.pop_back();
}
}
/**
* Check file format etc. for consistency and throw exception if
* there is a problem.
*
* @throws osmium::io_error
*/
const File& check() const {
if (m_file_format == file_format::unknown) {
std::string msg = "Could not detect file format";
if (!m_format_string.empty()) {
msg += " from format string '";
msg += m_format_string;
msg += "'";
}
if (m_filename.empty()) {
msg += " for stdin/stdout";
} else {
msg += " for filename '";
msg += m_filename;
msg += "'";
}
msg += ".";
throw io_error(msg);
}
return *this;
}
file_format format() const noexcept {
return m_file_format;
}
File& set_format(file_format format) noexcept {
m_file_format = format;
return *this;
}
file_compression compression() const noexcept {
return m_file_compression;
}
File& set_compression(file_compression compression) noexcept {
m_file_compression = compression;
return *this;
}
bool has_multiple_object_versions() const noexcept {
return m_has_multiple_object_versions;
}
File& set_has_multiple_object_versions(bool value) noexcept {
m_has_multiple_object_versions = value;
return *this;
}
File& filename(const std::string& filename) {
if (filename == "-") {
m_filename = "";
} else {
m_filename = filename;
}
return *this;
}
const std::string& filename() const noexcept {
return m_filename;
}
}; // class File
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_FILE_HPP
third_party/libosmium/include/osmium/io/file_compression.hpp
Vendored
-72
View File
@@ -1,72 +0,0 @@
#ifndef OSMIUM_IO_FILE_COMPRESSION_HPP
#define OSMIUM_IO_FILE_COMPRESSION_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <iosfwd>
namespace osmium {
namespace io {
enum class file_compression {
none = 0,
gzip = 1,
bzip2 = 2
};
// avoid g++ false positive
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wreturn-type"
inline const char* as_string(file_compression compression) {
switch (compression) {
case file_compression::none:
return "none";
case file_compression::gzip:
return "gzip";
case file_compression::bzip2:
return "bzip2";
}
}
#pragma GCC diagnostic pop
template <typename TChar, typename TTraits>
inline std::basic_ostream<TChar, TTraits>& operator<<(std::basic_ostream<TChar, TTraits>& out, const file_compression compression) {
return out << as_string(compression);
}
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_FILE_COMPRESSION_HPP
third_party/libosmium/include/osmium/io/file_format.hpp
Vendored
-84
View File
@@ -1,84 +0,0 @@
#ifndef OSMIUM_IO_FILE_FORMAT_HPP
#define OSMIUM_IO_FILE_FORMAT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <iosfwd>
namespace osmium {
namespace io {
enum class file_format {
unknown = 0,
xml = 1,
pbf = 2,
opl = 3,
json = 4,
o5m = 5,
debug = 6
};
// avoid g++ false positive
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wreturn-type"
inline const char* as_string(file_format format) {
switch (format) {
case file_format::unknown:
return "unknown";
case file_format::xml:
return "XML";
case file_format::pbf:
return "PBF";
case file_format::opl:
return "OPL";
case file_format::json:
return "JSON";
case file_format::o5m:
return "O5M";
case file_format::debug:
return "DEBUG";
}
}
#pragma GCC diagnostic pop
template <typename TChar, typename TTraits>
inline std::basic_ostream<TChar, TTraits>& operator<<(std::basic_ostream<TChar, TTraits>& out, const file_format format) {
return out << as_string(format);
}
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_FILE_FORMAT_HPP
third_party/libosmium/include/osmium/io/gzip_compression.hpp
Vendored
-277
View File
@@ -1,277 +0,0 @@
#ifndef OSMIUM_IO_GZIP_COMPRESSION_HPP
#define OSMIUM_IO_GZIP_COMPRESSION_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
/**
* @file
*
* Include this file if you want to read or write gzip-compressed OSM XML
* files.
*
* @attention If you include this file, you'll need to link with `libz`.
*/
#include <stdexcept>
#include <string>
#include <errno.h>
#include <zlib.h>
#include <osmium/io/compression.hpp>
#include <osmium/io/error.hpp>
#include <osmium/io/file_compression.hpp>
#include <osmium/io/writer_options.hpp>
#include <osmium/util/cast.hpp>
#include <osmium/util/compatibility.hpp>
namespace osmium {
/**
* Exception thrown when there are problems compressing or
* decompressing gzip files.
*/
struct gzip_error : public io_error {
int gzip_error_code;
int system_errno;
gzip_error(const std::string& what, int error_code) :
io_error(what),
gzip_error_code(error_code),
system_errno(error_code == Z_ERRNO ? errno : 0) {
}
}; // struct gzip_error
namespace io {
namespace detail {
OSMIUM_NORETURN inline void throw_gzip_error(gzFile gzfile, const char* msg, int zlib_error = 0) {
std::string error("gzip error: ");
error += msg;
error += ": ";
int errnum = zlib_error;
if (zlib_error) {
error += std::to_string(zlib_error);
} else {
error += ::gzerror(gzfile, &errnum);
}
throw osmium::gzip_error(error, errnum);
}
} // namespace detail
class GzipCompressor : public Compressor {
int m_fd;
gzFile m_gzfile;
public:
explicit GzipCompressor(int fd, fsync sync) :
Compressor(sync),
m_fd(dup(fd)),
m_gzfile(::gzdopen(fd, "w")) {
if (!m_gzfile) {
detail::throw_gzip_error(m_gzfile, "write initialization failed");
}
}
~GzipCompressor() noexcept final {
try {
close();
} catch (...) {
// Ignore any exceptions because destructor must not throw.
}
}
void write(const std::string& data) final {
if (!data.empty()) {
int nwrite = ::gzwrite(m_gzfile, data.data(), static_cast_with_assert<unsigned int>(data.size()));
if (nwrite == 0) {
detail::throw_gzip_error(m_gzfile, "write failed");
}
}
}
void close() final {
if (m_gzfile) {
int result = ::gzclose(m_gzfile);
m_gzfile = nullptr;
if (result != Z_OK) {
detail::throw_gzip_error(m_gzfile, "write close failed", result);
}
if (do_fsync()) {
osmium::io::detail::reliable_fsync(m_fd);
}
osmium::io::detail::reliable_close(m_fd);
}
}
}; // class GzipCompressor
class GzipDecompressor : public Decompressor {
gzFile m_gzfile;
public:
explicit GzipDecompressor(int fd) :
Decompressor(),
m_gzfile(::gzdopen(fd, "r")) {
if (!m_gzfile) {
detail::throw_gzip_error(m_gzfile, "read initialization failed");
}
}
~GzipDecompressor() noexcept final {
try {
close();
} catch (...) {
// Ignore any exceptions because destructor must not throw.
}
}
std::string read() final {
std::string buffer(osmium::io::Decompressor::input_buffer_size, '\0');
int nread = ::gzread(m_gzfile, const_cast<char*>(buffer.data()), static_cast_with_assert<unsigned int>(buffer.size()));
if (nread < 0) {
detail::throw_gzip_error(m_gzfile, "read failed");
}
buffer.resize(static_cast<std::string::size_type>(nread));
return buffer;
}
void close() final {
if (m_gzfile) {
int result = ::gzclose(m_gzfile);
m_gzfile = nullptr;
if (result != Z_OK) {
detail::throw_gzip_error(m_gzfile, "read close failed", result);
}
}
}
}; // class GzipDecompressor
class GzipBufferDecompressor : public Decompressor {
const char* m_buffer;
size_t m_buffer_size;
z_stream m_zstream;
public:
GzipBufferDecompressor(const char* buffer, size_t size) :
m_buffer(buffer),
m_buffer_size(size),
m_zstream() {
m_zstream.next_in = reinterpret_cast<unsigned char*>(const_cast<char*>(buffer));
m_zstream.avail_in = static_cast_with_assert<unsigned int>(size);
int result = inflateInit2(&m_zstream, MAX_WBITS | 32);
if (result != Z_OK) {
std::string message("gzip error: decompression init failed: ");
if (m_zstream.msg) {
message.append(m_zstream.msg);
}
throw osmium::gzip_error(message, result);
}
}
~GzipBufferDecompressor() noexcept final {
try {
close();
} catch (...) {
// Ignore any exceptions because destructor must not throw.
}
}
std::string read() final {
std::string output;
if (m_buffer) {
const size_t buffer_size = 10240;
output.append(buffer_size, '\0');
m_zstream.next_out = reinterpret_cast<unsigned char*>(const_cast<char*>(output.data()));
m_zstream.avail_out = buffer_size;
int result = inflate(&m_zstream, Z_SYNC_FLUSH);
if (result != Z_OK) {
m_buffer = nullptr;
m_buffer_size = 0;
}
if (result != Z_OK && result != Z_STREAM_END) {
std::string message("gzip error: inflate failed: ");
if (m_zstream.msg) {
message.append(m_zstream.msg);
}
throw osmium::gzip_error(message, result);
}
output.resize(static_cast<unsigned long>(m_zstream.next_out - reinterpret_cast<const unsigned char*>(output.data())));
}
return output;
}
void close() final {
inflateEnd(&m_zstream);
}
}; // class GzipBufferDecompressor
namespace detail {
// we want the register_compression() function to run, setting
// the variable is only a side-effect, it will never be used
const bool registered_gzip_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::gzip,
[](int fd, fsync sync) { return new osmium::io::GzipCompressor(fd, sync); },
[](int fd) { return new osmium::io::GzipDecompressor(fd); },
[](const char* buffer, size_t size) { return new osmium::io::GzipBufferDecompressor(buffer, size); }
);
// dummy function to silence the unused variable warning from above
inline bool get_registered_gzip_compression() noexcept {
return registered_gzip_compression;
}
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_GZIP_COMPRESSION_HPP
third_party/libosmium/include/osmium/io/header.hpp
Vendored
-122
View File
@@ -1,122 +0,0 @@
#ifndef OSMIUM_IO_HEADER_HPP
#define OSMIUM_IO_HEADER_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <initializer_list>
#include <vector>
#include <osmium/osm/box.hpp>
#include <osmium/util/options.hpp>
namespace osmium {
namespace io {
/**
* Meta information from the header of an OSM file.
*/
class Header : public osmium::util::Options {
/// Bounding boxes
std::vector<osmium::Box> m_boxes;
/**
* Are there possibly multiple versions of the same object in this stream of objects?
* This is true for history files and for change files, but not for normal OSM files.
*/
bool m_has_multiple_object_versions = false;
public:
Header() = default;
explicit Header(const std::initializer_list<osmium::util::Options::value_type>& values) :
Options(values) {
}
Header(const Header&) = default;
Header& operator=(const Header&) = default;
Header(Header&&) = default;
Header& operator=(Header&&) = default;
~Header() = default;
std::vector<osmium::Box>& boxes() noexcept {
return m_boxes;
}
const std::vector<osmium::Box>& boxes() const noexcept {
return m_boxes;
}
Header& boxes(const std::vector<osmium::Box>& boxes) noexcept {
m_boxes = boxes;
return *this;
}
osmium::Box box() const {
return m_boxes.empty() ? osmium::Box() : m_boxes.front();
}
osmium::Box joined_boxes() const {
osmium::Box box;
for (const auto& b : m_boxes) {
box.extend(b.bottom_left());
box.extend(b.top_right());
}
return box;
}
Header& add_box(const osmium::Box& box) {
m_boxes.push_back(box);
return *this;
}
bool has_multiple_object_versions() const noexcept {
return m_has_multiple_object_versions;
}
Header& set_has_multiple_object_versions(bool value) noexcept {
m_has_multiple_object_versions = value;
return *this;
}
}; // class Header
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_HEADER_HPP
third_party/libosmium/include/osmium/io/input_iterator.hpp
Vendored
-178
View File
@@ -1,178 +0,0 @@
#ifndef OSMIUM_IO_INPUT_ITERATOR_HPP
#define OSMIUM_IO_INPUT_ITERATOR_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cassert>
#include <cstddef>
#include <iterator>
#include <memory>
#include <type_traits>
#include <osmium/memory/buffer.hpp>
#include <osmium/memory/item.hpp>
#include <osmium/util/compatibility.hpp>
namespace osmium {
namespace io {
/**
* This iterator class allows you to iterate over all items from a
* source. It hides all the buffer handling and makes the contents of a
* source accessible as a normal STL input iterator.
*/
template <typename TSource, typename TItem = osmium::memory::Item>
class InputIterator {
static_assert(std::is_base_of<osmium::memory::Item, TItem>::value, "TItem must derive from osmium::buffer::Item");
typedef typename osmium::memory::Buffer::t_iterator<TItem> item_iterator;
TSource* m_source;
std::shared_ptr<osmium::memory::Buffer> m_buffer;
item_iterator m_iter {};
void update_buffer() {
do {
m_buffer = std::make_shared<osmium::memory::Buffer>(std::move(m_source->read()));
if (!m_buffer || !*m_buffer) { // end of input
m_source = nullptr;
m_buffer.reset();
m_iter = item_iterator();
return;
}
m_iter = m_buffer->begin<TItem>();
} while (m_iter == m_buffer->end<TItem>());
}
public:
typedef std::input_iterator_tag iterator_category;
typedef TItem value_type;
typedef ptrdiff_t difference_type;
typedef TItem* pointer;
typedef TItem& reference;
explicit InputIterator(TSource& source) :
m_source(&source) {
update_buffer();
}
// end iterator
InputIterator() noexcept :
m_source(nullptr) {
}
InputIterator& operator++() {
assert(m_source);
assert(m_buffer);
assert(m_iter);
++m_iter;
if (m_iter == m_buffer->end<TItem>()) {
update_buffer();
}
return *this;
}
InputIterator operator++(int) {
InputIterator tmp(*this);
operator++();
return tmp;
}
bool operator==(const InputIterator& rhs) const noexcept {
return m_source == rhs.m_source &&
m_buffer == rhs.m_buffer &&
m_iter == rhs.m_iter;
}
bool operator!=(const InputIterator& rhs) const noexcept {
return !(*this == rhs);
}
reference operator*() const {
assert(m_iter);
return static_cast<reference>(*m_iter);
}
pointer operator->() const {
assert(m_iter);
return &static_cast<reference>(*m_iter);
}
}; // class InputIterator
template <typename TSource, typename TItem = osmium::memory::Item>
class InputIteratorRange {
InputIterator<TSource, TItem> m_begin;
InputIterator<TSource, TItem> m_end;
public:
InputIteratorRange(InputIterator<TSource, TItem>&& begin,
InputIterator<TSource, TItem>&& end) :
m_begin(std::move(begin)),
m_end(std::move(end)) {
}
InputIterator<TSource, TItem> begin() const noexcept {
return m_begin;
}
InputIterator<TSource, TItem> end() const noexcept {
return m_end;
}
const InputIterator<TSource, TItem> cbegin() const noexcept {
return m_begin;
}
const InputIterator<TSource, TItem> cend() const noexcept {
return m_end;
}
}; // class InputIteratorRange
template <typename TItem, typename TSource>
InputIteratorRange<TSource, TItem> make_input_iterator_range(TSource& source) {
using it_type = InputIterator<TSource, TItem>;
return InputIteratorRange<TSource, TItem>(it_type{source}, it_type{});
}
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_INPUT_ITERATOR_HPP
third_party/libosmium/include/osmium/io/o5m_input.hpp
Vendored
-45
View File
@@ -1,45 +0,0 @@
#ifndef OSMIUM_IO_O5M_INPUT_HPP
#define OSMIUM_IO_O5M_INPUT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
/**
* @file
*
* Include this file if you want to read OSM o5m and o5c files.
*/
#include <osmium/io/reader.hpp> // IWYU pragma: export
#include <osmium/io/detail/o5m_input_format.hpp> // IWYU pragma: export
#endif // OSMIUM_IO_O5M_INPUT_HPP
third_party/libosmium/include/osmium/io/opl_output.hpp
Vendored
-39
View File
@@ -1,39 +0,0 @@
#ifndef OSMIUM_IO_OPL_OUTPUT_HPP
#define OSMIUM_IO_OPL_OUTPUT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <osmium/io/writer.hpp> // IWYU pragma: export
#include <osmium/io/detail/opl_output_format.hpp> // IWYU pragma: export
#endif // OSMIUM_IO_OPL_OUTPUT_HPP
third_party/libosmium/include/osmium/io/output_iterator.hpp
Vendored
-136
View File
@@ -1,136 +0,0 @@
#ifndef OSMIUM_IO_OUTPUT_ITERATOR_HPP
#define OSMIUM_IO_OUTPUT_ITERATOR_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cstddef>
#include <iterator>
#include <memory>
#include <utility>
#include <osmium/memory/buffer.hpp>
#include <osmium/osm/diff_object.hpp>
#include <osmium/util/compatibility.hpp>
namespace osmium {
namespace memory {
class Item;
} // namespace memory
namespace io {
template <typename TDest>
class OutputIterator : public std::iterator<std::output_iterator_tag, osmium::memory::Item> {
TDest* m_destination;
public:
explicit OutputIterator(TDest& destination) :
m_destination(&destination) {
}
/**
* @deprecated
* Use of buffer size argument on OutputIterator
* constructor is deprecated. Call Writer::set_buffer_size()
* instead if you want to change the default.
*/
OSMIUM_DEPRECATED OutputIterator(TDest& destination, const size_t buffer_size) :
m_destination(&destination) {
destination.set_buffer_size(buffer_size);
}
OutputIterator(const OutputIterator&) = default;
OutputIterator(OutputIterator&&) = default;
OutputIterator& operator=(const OutputIterator&) = default;
OutputIterator& operator=(OutputIterator&&) = default;
~OutputIterator() = default;
/**
* @deprecated
* Calling OutputIterator<Writer>::flush() is usually not
* needed any more. Call flush() on the Writer instead if needed.
*/
OSMIUM_DEPRECATED void flush() {
m_destination->flush();
}
OutputIterator& operator=(const osmium::memory::Item& item) {
(*m_destination)(item);
return *this;
}
OutputIterator& operator=(const osmium::DiffObject& diff) {
return this->operator=(diff.curr());
}
OutputIterator& operator*() {
return *this;
}
OutputIterator& operator++() {
return *this;
}
OutputIterator& operator++(int) {
return *this;
}
}; // class OutputIterator
template <typename TDest>
OutputIterator<TDest> make_output_iterator(TDest& destination) {
return OutputIterator<TDest>{destination};
}
/**
* @deprecated
* Use of buffer size argument on make_output_iterator is deprecated.
* Call Writer::set_buffer_size() instead if you want to change the
* default.
*/
template <typename TDest>
OSMIUM_DEPRECATED OutputIterator<TDest> make_output_iterator(TDest& destination, const size_t buffer_size) {
destination.set_buffer_size(buffer_size);
return OutputIterator<TDest>{destination};
}
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_OUTPUT_ITERATOR_HPP
third_party/libosmium/include/osmium/io/overwrite.hpp
Vendored
-39
View File
@@ -1,39 +0,0 @@
#ifndef OSMIUM_IO_OVERWRITE_HPP
#define OSMIUM_IO_OVERWRITE_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#pragma message("Including overwrite.hpp is deprecated, #include <osmium/io/writer_options.hpp> instead.")
#include <osmium/io/writer_options.hpp>
#endif // OSMIUM_IO_OVERWRITE_HPP
third_party/libosmium/include/osmium/io/pbf_input.hpp
Vendored
-48
View File
@@ -1,48 +0,0 @@
#ifndef OSMIUM_IO_PBF_INPUT_HPP
#define OSMIUM_IO_PBF_INPUT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
/**
* @file
*
* Include this file if you want to read OSM PBF files.
*
* @attention If you include this file, you'll need to link with
* `libz`, and enable multithreading.
*/
#include <osmium/io/reader.hpp> // IWYU pragma: export
#include <osmium/io/detail/pbf_input_format.hpp> // IWYU pragma: export
#endif // OSMIUM_IO_PBF_INPUT_HPP
third_party/libosmium/include/osmium/io/pbf_output.hpp
Vendored
-48
View File
@@ -1,48 +0,0 @@
#ifndef OSMIUM_IO_PBF_OUTPUT_HPP
#define OSMIUM_IO_PBF_OUTPUT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
/**
* @file
*
* Include this file if you want to write OSM PBF files.
*
* @attention If you include this file, you'll need to link with
* `libz`, and enable multithreading.
*/
#include <osmium/io/writer.hpp> // IWYU pragma: export
#include <osmium/io/detail/pbf_output_format.hpp> // IWYU pragma: export
#endif // OSMIUM_IO_PBF_OUTPUT_HPP
third_party/libosmium/include/osmium/io/reader.hpp
Vendored
-380
View File
@@ -1,380 +0,0 @@
#ifndef OSMIUM_IO_READER_HPP
#define OSMIUM_IO_READER_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cerrno>
#include <cstdlib>
#include <fcntl.h>
#include <future>
#include <memory>
#include <string>
#include <system_error>
#include <thread>
#include <utility>
#ifndef _WIN32
# include <sys/wait.h>
#endif
#ifndef _MSC_VER
# include <unistd.h>
#endif
#include <osmium/io/compression.hpp>
#include <osmium/io/detail/input_format.hpp>
#include <osmium/io/detail/read_thread.hpp>
#include <osmium/io/detail/read_write.hpp>
#include <osmium/io/detail/queue_util.hpp>
#include <osmium/io/error.hpp>
#include <osmium/io/file.hpp>
#include <osmium/io/header.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/osm/entity_bits.hpp>
#include <osmium/thread/util.hpp>
namespace osmium {
namespace io {
/**
* This is the user-facing interface for reading OSM files. Instantiate
* an object of this class with a file name or osmium::io::File object
* and then call read() on it in a loop until it returns an invalid
* Buffer.
*/
class Reader {
static constexpr size_t max_input_queue_size = 20; // XXX
static constexpr size_t max_osmdata_queue_size = 20; // XXX
osmium::io::File m_file;
osmium::osm_entity_bits::type m_read_which_entities;
enum class status {
okay = 0, // normal reading
error = 1, // some error occurred while reading
closed = 2, // close() called successfully after eof
eof = 3 // eof of file was reached without error
} m_status;
int m_childpid;
detail::future_string_queue_type m_input_queue;
std::unique_ptr<osmium::io::Decompressor> m_decompressor;
osmium::io::detail::ReadThreadManager m_read_thread_manager;
detail::future_buffer_queue_type m_osmdata_queue;
detail::queue_wrapper<osmium::memory::Buffer> m_osmdata_queue_wrapper;
std::future<osmium::io::Header> m_header_future;
osmium::io::Header m_header;
osmium::thread::thread_handler m_thread;
// This function will run in a separate thread.
static void parser_thread(const osmium::io::File& file,
detail::future_string_queue_type& input_queue,
detail::future_buffer_queue_type& osmdata_queue,
std::promise<osmium::io::Header>&& header_promise,
osmium::osm_entity_bits::type read_which_entities) {
std::promise<osmium::io::Header> promise = std::move(header_promise);
auto creator = detail::ParserFactory::instance().get_creator_function(file);
auto parser = creator(input_queue, osmdata_queue, promise, read_which_entities);
parser->parse();
}
#ifndef _WIN32
/**
* Fork and execute the given command in the child.
* A pipe is created between the child and the parent.
* The child writes to the pipe, the parent reads from it.
* This function never returns in the child.
*
* @param command Command to execute in the child.
* @param filename Filename to give to command as argument.
* @returns File descriptor of pipe in the parent.
* @throws std::system_error if a system call fails.
*/
static int execute(const std::string& command, const std::string& filename, int* childpid) {
int pipefd[2];
if (pipe(pipefd) < 0) {
throw std::system_error(errno, std::system_category(), "opening pipe failed");
}
pid_t pid = fork();
if (pid < 0) {
throw std::system_error(errno, std::system_category(), "fork failed");
}
if (pid == 0) { // child
// close all file descriptors except one end of the pipe
for (int i = 0; i < 32; ++i) {
if (i != pipefd[1]) {
::close(i);
}
}
if (dup2(pipefd[1], 1) < 0) { // put end of pipe as stdout/stdin
exit(1);
}
::open("/dev/null", O_RDONLY); // stdin
::open("/dev/null", O_WRONLY); // stderr
// hack: -g switches off globbing in curl which allows [] to be used in file names
// this is important for XAPI URLs
// in theory this execute() function could be used for other commands, but it is
// only used for curl at the moment, so this is okay.
if (::execlp(command.c_str(), command.c_str(), "-g", filename.c_str(), nullptr) < 0) {
exit(1);
}
}
// parent
*childpid = pid;
::close(pipefd[1]);
return pipefd[0];
}
#endif
/**
* Open File for reading. Handles URLs or normal files. URLs
* are opened by executing the "curl" program (which must be installed)
* and reading from its output.
*
* @returns File descriptor of open file or pipe.
* @throws std::system_error if a system call fails.
*/
static int open_input_file_or_url(const std::string& filename, int* childpid) {
std::string protocol = filename.substr(0, filename.find_first_of(':'));
if (protocol == "http" || protocol == "https" || protocol == "ftp" || protocol == "file") {
#ifndef _WIN32
return execute("curl", filename, childpid);
#else
throw io_error("Reading OSM files from the network currently not supported on Windows.");
#endif
} else {
return osmium::io::detail::open_for_reading(filename);
}
}
public:
/**
* Create new Reader object.
*
* @param file The file we want to open.
* @param read_which_entities Which OSM entities (nodes, ways, relations, and/or changesets)
* should be read from the input file. It can speed the read up
* significantly if objects that are not needed anyway are not
* parsed.
*/
explicit Reader(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities = osmium::osm_entity_bits::all) :
m_file(file.check()),
m_read_which_entities(read_which_entities),
m_status(status::okay),
m_childpid(0),
m_input_queue(max_input_queue_size, "raw_input"),
m_decompressor(m_file.buffer() ?
osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), m_file.buffer(), m_file.buffer_size()) :
osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), open_input_file_or_url(m_file.filename(), &m_childpid))),
m_read_thread_manager(*m_decompressor, m_input_queue),
m_osmdata_queue(max_osmdata_queue_size, "parser_results"),
m_osmdata_queue_wrapper(m_osmdata_queue),
m_header_future(),
m_header(),
m_thread() {
std::promise<osmium::io::Header> header_promise;
m_header_future = header_promise.get_future();
m_thread = osmium::thread::thread_handler{parser_thread, std::ref(m_file), std::ref(m_input_queue), std::ref(m_osmdata_queue), std::move(header_promise), read_which_entities};
}
explicit Reader(const std::string& filename, osmium::osm_entity_bits::type read_types = osmium::osm_entity_bits::all) :
Reader(osmium::io::File(filename), read_types) {
}
explicit Reader(const char* filename, osmium::osm_entity_bits::type read_types = osmium::osm_entity_bits::all) :
Reader(osmium::io::File(filename), read_types) {
}
Reader(const Reader&) = delete;
Reader& operator=(const Reader&) = delete;
Reader(Reader&&) = default;
Reader& operator=(Reader&&) = default;
~Reader() noexcept {
try {
close();
} catch (...) {
// Ignore any exceptions because destructor must not throw.
}
}
/**
* Close down the Reader. A call to this is optional, because the
* destructor of Reader will also call this. But if you don't call
* this function first, you might miss an exception, because the
* destructor is not allowed to throw.
*
* @throws Some form of osmium::io_error when there is a problem.
*/
void close() {
m_status = status::closed;
m_read_thread_manager.stop();
m_osmdata_queue_wrapper.drain();
try {
m_read_thread_manager.close();
} catch (...) {
// Ignore any exceptions.
}
#ifndef _WIN32
if (m_childpid) {
int status;
pid_t pid = ::waitpid(m_childpid, &status, 0);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
if (pid < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) {
throw std::system_error(errno, std::system_category(), "subprocess returned error");
}
#pragma GCC diagnostic pop
m_childpid = 0;
}
#endif
}
/**
* Get the header data from the file.
*
* @returns Header.
* @throws Some form of osmium::io_error if there is an error.
*/
osmium::io::Header header() {
if (m_status == status::error) {
throw io_error("Can not get header from reader when in status 'error'");
}
try {
if (m_header_future.valid()) {
m_header = m_header_future.get();
if (m_read_which_entities == osmium::osm_entity_bits::nothing) {
m_status = status::eof;
}
}
} catch (...) {
close();
m_status = status::error;
throw;
}
return m_header;
}
/**
* Reads the next buffer from the input. An invalid buffer signals
* end-of-file. After end-of-file all read() calls will return an
* invalid buffer. An invalid buffer is also always returned if
* osmium::osm_entity_bits::nothing was set when the Reader was
* constructed.
*
* @returns Buffer.
* @throws Some form of osmium::io_error if there is an error.
*/
osmium::memory::Buffer read() {
osmium::memory::Buffer buffer;
if (m_status != status::okay ||
m_read_which_entities == osmium::osm_entity_bits::nothing) {
throw io_error("Can not read from reader when in status 'closed', 'eof', or 'error'");
}
try {
// m_input_format.read() can return an invalid buffer to signal EOF,
// or a valid buffer with or without data. A valid buffer
// without data is not an error, it just means we have to
// keep getting the next buffer until there is one with data.
while (true) {
buffer = m_osmdata_queue_wrapper.pop();
if (detail::at_end_of_data(buffer)) {
m_status = status::eof;
m_read_thread_manager.close();
return buffer;
}
if (buffer.committed() > 0) {
return buffer;
}
}
} catch (...) {
close();
m_status = status::error;
throw;
}
}
/**
* Has the end of file been reached? This is set after the last
* data has been read. It is also set by calling close().
*/
bool eof() const {
return m_status == status::eof || m_status == status::closed;
}
}; // class Reader
/**
* Read contents of the given file into a buffer in one go. Takes
* the same arguments as any of the Reader constructors.
*
* The buffer can take up quite a lot of memory, so don't do this
* unless you are working with small OSM files and/or have lots of
* RAM.
*/
template <typename... TArgs>
osmium::memory::Buffer read_file(TArgs&&... args) {
osmium::memory::Buffer buffer(1024*1024, osmium::memory::Buffer::auto_grow::yes);
Reader reader(std::forward<TArgs>(args)...);
while (osmium::memory::Buffer read_buffer = reader.read()) {
buffer.add_buffer(read_buffer);
buffer.commit();
}
return buffer;
}
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_READER_HPP
third_party/libosmium/include/osmium/io/reader_iterator.hpp
Vendored
-51
View File
@@ -1,51 +0,0 @@
#ifndef OSMIUM_IO_READER_ITERATOR_HPP
#define OSMIUM_IO_READER_ITERATOR_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <osmium/io/reader.hpp>
#include <osmium/io/input_iterator.hpp>
namespace std {
inline osmium::io::InputIterator<osmium::io::Reader> begin(osmium::io::Reader& reader) {
return osmium::io::InputIterator<osmium::io::Reader>(reader);
}
inline osmium::io::InputIterator<osmium::io::Reader> end(osmium::io::Reader&) {
return osmium::io::InputIterator<osmium::io::Reader>();
}
} // namespace std
#endif // OSMIUM_IO_READER_ITERATOR_HPP
third_party/libosmium/include/osmium/io/writer.hpp
Vendored
-344
View File
@@ -1,344 +0,0 @@
#ifndef OSMIUM_IO_WRITER_HPP
#define OSMIUM_IO_WRITER_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cassert>
#include <future>
#include <memory>
#include <stdexcept>
#include <string>
#include <thread>
#include <utility>
#include <osmium/io/compression.hpp>
#include <osmium/io/detail/output_format.hpp>
#include <osmium/io/detail/queue_util.hpp>
#include <osmium/io/detail/read_write.hpp>
#include <osmium/io/detail/write_thread.hpp>
#include <osmium/io/error.hpp>
#include <osmium/io/file.hpp>
#include <osmium/io/header.hpp>
#include <osmium/io/writer_options.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/thread/util.hpp>
namespace osmium {
namespace io {
/**
* This is the user-facing interface for writing OSM files. Instantiate
* an object of this class with a file name or osmium::io::File object
* and optionally the data for the header and then call operator() on
* it to write Buffers or Items.
*
* The writer uses multithreading internally to do the actual encoding
* of the data into the intended format, possible compress the data and
* then write it out. But this is intentionally hidden from the user
* of this class who can use it without knowing those details.
*
* If you are done call the close() method to finish up. Only if you
* don't get an exception from the close() method, you can be sure
* the data is written correctly (modulo operating system buffering).
* The destructor of this class will also do the right thing if you
* forget to call close(), but because the destructor can't throw you
* will not get informed about any problems.
*
* The writer is usually used to write complete blocks of data stored
* in osmium::memory::Buffers. But you can also write single
* osmium::memory::Items. In this case the Writer uses an internal
* Buffer.
*/
class Writer {
static constexpr size_t default_buffer_size = 10 * 1024 * 1024;
osmium::io::File m_file;
detail::future_string_queue_type m_output_queue;
std::unique_ptr<osmium::io::detail::OutputFormat> m_output;
osmium::memory::Buffer m_buffer;
size_t m_buffer_size;
std::future<bool> m_write_future;
osmium::thread::thread_handler m_thread;
enum class status {
okay = 0, // normal writing
error = 1, // some error occurred while writing
closed = 2 // close() called successfully
} m_status;
// This function will run in a separate thread.
static void write_thread(detail::future_string_queue_type& output_queue,
std::unique_ptr<osmium::io::Compressor>&& compressor,
std::promise<bool>&& write_promise) {
detail::WriteThread write_thread{output_queue,
std::move(compressor),
std::move(write_promise)};
write_thread();
}
void do_write(osmium::memory::Buffer&& buffer) {
if (buffer && buffer.committed() > 0) {
m_output->write_buffer(std::move(buffer));
}
}
void do_flush() {
osmium::thread::check_for_exception(m_write_future);
if (m_buffer && m_buffer.committed() > 0) {
osmium::memory::Buffer buffer{m_buffer_size,
osmium::memory::Buffer::auto_grow::no};
using std::swap;
swap(m_buffer, buffer);
m_output->write_buffer(std::move(buffer));
}
}
template <typename TFunction, typename... TArgs>
void ensure_cleanup(TFunction func, TArgs&&... args) {
if (m_status != status::okay) {
throw io_error("Can not write to writer when in status 'closed' or 'error'");
}
try {
func(std::forward<TArgs>(args)...);
} catch (...) {
m_status = status::error;
detail::add_to_queue(m_output_queue, std::current_exception());
detail::add_end_of_data_to_queue(m_output_queue);
throw;
}
}
struct options_type {
osmium::io::Header header;
overwrite allow_overwrite = overwrite::no;
fsync sync = fsync::no;
};
static void set_option(options_type& options, const osmium::io::Header& header) {
options.header = header;
}
static void set_option(options_type& options, overwrite value) {
options.allow_overwrite = value;
}
static void set_option(options_type& options, fsync value) {
options.sync = value;
}
public:
/**
* The constructor of the Writer object opens a file and writes the
* header to it.
*
* @param file File (contains name and format info) to open.
* @param args All further arguments are optional and can appear
* in any order:
*
* * osmium::io::Header: Optional header data. If this is
* not given, a default constructed osmium::io::Header
* object will be used.
*
* * osmium::io::overwrite: Allow overwriting of existing file?
* Can be osmium::io::overwrite::allow or
* osmium::io::overwrite::no (default).
*
* * osmium::io::fsync: Should fsync be called on the file
* before closing it? Can be osmium::io::fsync::yes or
* osmium::io::fsync::no (default).
*
* @throws osmium::io_error If there was an error.
* @throws std::system_error If the file could not be opened.
*/
template <typename... TArgs>
explicit Writer(const osmium::io::File& file, TArgs&&... args) :
m_file(file.check()),
m_output_queue(20, "raw_output"), // XXX
m_output(osmium::io::detail::OutputFormatFactory::instance().create_output(m_file, m_output_queue)),
m_buffer(),
m_buffer_size(default_buffer_size),
m_write_future(),
m_thread(),
m_status(status::okay) {
assert(!m_file.buffer()); // XXX can't handle pseudo-files
options_type options;
(void)std::initializer_list<int>{
(set_option(options, args), 0)...
};
std::unique_ptr<osmium::io::Compressor> compressor =
CompressionFactory::instance().create_compressor(file.compression(),
osmium::io::detail::open_for_writing(m_file.filename(), options.allow_overwrite),
options.sync);
std::promise<bool> write_promise;
m_write_future = write_promise.get_future();
m_thread = osmium::thread::thread_handler{write_thread, std::ref(m_output_queue), std::move(compressor), std::move(write_promise)};
ensure_cleanup([&](){
m_output->write_header(options.header);
});
}
template <typename... TArgs>
explicit Writer(const std::string& filename, TArgs&&... args) :
Writer(osmium::io::File(filename), std::forward<TArgs>(args)...) {
}
template <typename... TArgs>
explicit Writer(const char* filename, TArgs&&... args) :
Writer(osmium::io::File(filename), std::forward<TArgs>(args)...) {
}
Writer(const Writer&) = delete;
Writer& operator=(const Writer&) = delete;
Writer(Writer&&) = default;
Writer& operator=(Writer&&) = default;
~Writer() noexcept {
try {
close();
} catch (...) {
// Ignore any exceptions because destructor must not throw.
}
}
/**
* Get the currently configured size of the internal buffer.
*/
size_t buffer_size() const noexcept {
return m_buffer_size;
}
/**
* Set the size of the internal buffer. This will only take effect
* if you have not yet written anything or after the next flush().
*/
void set_buffer_size(size_t size) noexcept {
m_buffer_size = size;
}
/**
* Flush the internal buffer if it contains any data. This is
* usually not needed as the buffer gets flushed on close()
* automatically.
*
* @throws Some form of osmium::io_error when there is a problem.
*/
void flush() {
ensure_cleanup([&](){
do_flush();
});
}
/**
* Write contents of a buffer to the output file. The buffer is
* moved into this function and will be in an undefined moved-from
* state afterwards.
*
* @param buffer Buffer that is being written out.
* @throws Some form of osmium::io_error when there is a problem.
*/
void operator()(osmium::memory::Buffer&& buffer) {
ensure_cleanup([&](){
do_flush();
do_write(std::move(buffer));
});
}
/**
* Add item to the internal buffer for eventual writing to the
* output file.
*
* @param item Item to write (usually an OSM object).
* @throws Some form of osmium::io_error when there is a problem.
*/
void operator()(const osmium::memory::Item& item) {
ensure_cleanup([&](){
if (!m_buffer) {
m_buffer = osmium::memory::Buffer{m_buffer_size,
osmium::memory::Buffer::auto_grow::no};
}
try {
m_buffer.push_back(item);
} catch (osmium::buffer_is_full&) {
do_flush();
m_buffer.push_back(item);
}
});
}
/**
* Flushes internal buffer and closes output file. If you do not
* call this, the destructor of Writer will also do the same
* thing. But because this call might throw an exception, which
* the destructor will ignore, it is better to call close()
* explicitly.
*
* @throws Some form of osmium::io_error when there is a problem.
*/
void close() {
if (m_status == status::okay) {
ensure_cleanup([&](){
do_write(std::move(m_buffer));
m_output->write_end();
m_status = status::closed;
detail::add_end_of_data_to_queue(m_output_queue);
});
}
if (m_write_future.valid()) {
m_write_future.get();
}
}
}; // class Writer
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_WRITER_HPP
third_party/libosmium/include/osmium/io/writer_options.hpp
Vendored
-60
View File
@@ -1,60 +0,0 @@
#ifndef OSMIUM_IO_WRITER_OPTIONS_HPP
#define OSMIUM_IO_WRITER_OPTIONS_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
namespace osmium {
namespace io {
/**
* Allow overwriting of existing file?
*/
enum class overwrite : bool {
no = false,
allow = true
};
/**
* Should writer do an fsync before closing the file?
*/
enum class fsync : bool {
no = false,
yes = true
};
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_WRITER_OPTIONS_HPP
third_party/libosmium/include/osmium/io/xml_input.hpp
Vendored
-48
View File
@@ -1,48 +0,0 @@
#ifndef OSMIUM_IO_XML_INPUT_HPP
#define OSMIUM_IO_XML_INPUT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
/**
* @file
*
* Include this file if you want to read OSM XML files.
*
* @attention If you include this file, you'll need to link with
* `libexpat`, and enable multithreading.
*/
#include <osmium/io/reader.hpp> // IWYU pragma: export
#include <osmium/io/detail/xml_input_format.hpp> // IWYU pragma: export
#endif // OSMIUM_IO_XML_INPUT_HPP
third_party/libosmium/include/osmium/io/xml_output.hpp
Vendored
-47
View File
@@ -1,47 +0,0 @@
#ifndef OSMIUM_IO_XML_OUTPUT_HPP
#define OSMIUM_IO_XML_OUTPUT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
/**
* @file
*
* Include this file if you want to write OSM XML files.
*
* @attention If you include this file, you'll need to enable multithreading.
*/
#include <osmium/io/writer.hpp> // IWYU pragma: export
#include <osmium/io/detail/xml_output_format.hpp> // IWYU pragma: export
#endif // OSMIUM_IO_XML_OUTPUT_HPP