Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Merge commit '788bc67faa7738cf7c6b2a192ecf3e3567d1c20e' into develop

Browse Source
This commit is contained in:
Patrick Niklaus
2015-08-28 12:42:03 +02:00
parent 8e1f70865e 788bc67faa
commit 048be2da2c
150 changed files with 12325 additions and 4321 deletions
Download Patch File Download Diff File Expand all files Collapse all files
third_party/libosmium/include/osmium/io/any_input.hpp
Vendored
+2 -2
View File
@@ -39,8 +39,8 @@ DEALINGS IN THE SOFTWARE.
* Include this file if you want to read all kinds of OSM files.
*
* @attention If you include this file, you'll need to link with
* `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only),
* `libexpat`, `libz`, `libbz2`, and enable multithreading.
* `ws2_32` (Windows only), `libexpat`, `libz`, `libbz2`,
* and enable multithreading.
*/
#include <osmium/io/any_compression.hpp> // IWYU pragma: export
third_party/libosmium/include/osmium/io/any_output.hpp
Vendored
+3 -2
View File
@@ -39,12 +39,13 @@ DEALINGS IN THE SOFTWARE.
* Include this file if you want to write all kinds of OSM files.
*
* @attention If you include this file, you'll need to link with
* `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only),
* `libz`, `libbz2`, and enable multithreading.
* `ws2_32` (Windows only), `libz`, `libbz2`, and enable
* multithreading.
*/
#include <osmium/io/any_compression.hpp> // IWYU pragma: export
#include <osmium/io/debug_output.hpp> // IWYU pragma: export
#include <osmium/io/opl_output.hpp> // IWYU pragma: export
#include <osmium/io/pbf_output.hpp> // IWYU pragma: export
#include <osmium/io/xml_output.hpp> // IWYU pragma: export
third_party/libosmium/include/osmium/io/bzip2_compression.hpp
Vendored
+5
View File
@@ -274,11 +274,16 @@ namespace osmium {
namespace {
// we want the register_compression() function to run, setting the variable
// is only a side-effect, it will never be used
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
const bool registered_bzip2_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::bzip2,
[](int fd) { return new osmium::io::Bzip2Compressor(fd); },
[](int fd) { return new osmium::io::Bzip2Decompressor(fd); },
[](const char* buffer, size_t size) { return new osmium::io::Bzip2BufferDecompressor(buffer, size); }
);
#pragma GCC diagnostic pop
} // anonymous namespace
third_party/libosmium/include/osmium/io/compression.hpp
Vendored
+5
View File
@@ -266,11 +266,16 @@ namespace osmium {
namespace {
// we want the register_compression() function to run, setting the variable
// is only a side-effect, it will never be used
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
const bool registered_no_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::none,
[](int fd) { return new osmium::io::NoCompressor(fd); },
[](int fd) { return new osmium::io::NoDecompressor(fd); },
[](const char* buffer, size_t size) { return new osmium::io::NoDecompressor(buffer, size); }
);
#pragma GCC diagnostic pop
} // anonymous namespace
third_party/libosmium/include/osmium/io/debug_output.hpp
Vendored
+39
View File
@@ -0,0 +1,39 @@
#ifndef OSMIUM_IO_DEBUG_OUTPUT_HPP
#define OSMIUM_IO_DEBUG_OUTPUT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <osmium/io/writer.hpp> // IWYU pragma: export
#include <osmium/io/detail/debug_output_format.hpp> // IWYU pragma: export
#endif // OSMIUM_IO_DEBUG_OUTPUT_HPP
third_party/libosmium/include/osmium/io/detail/debug_output_format.hpp
Vendored
+482
View File
@@ -0,0 +1,482 @@
#ifndef OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP
#define OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <chrono>
#include <cinttypes>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <future>
#include <iterator>
#include <memory>
#include <ratio>
#include <string>
#include <thread>
#include <utility>
#include <utf8.h>
#include <osmium/handler.hpp>
#include <osmium/io/detail/output_format.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/memory/collection.hpp>
#include <osmium/osm/box.hpp>
#include <osmium/osm/changeset.hpp>
#include <osmium/osm/item_type.hpp>
#include <osmium/osm/location.hpp>
#include <osmium/osm/node.hpp>
#include <osmium/osm/object.hpp>
#include <osmium/osm/relation.hpp>
#include <osmium/osm/tag.hpp>
#include <osmium/osm/timestamp.hpp>
#include <osmium/osm/way.hpp>
#include <osmium/thread/pool.hpp>
#include <osmium/util/minmax.hpp>
#include <osmium/visitor.hpp>
namespace osmium {
namespace io {
class File;
namespace detail {
constexpr const char* color_bold = "\x1b[1m";
constexpr const char* color_black = "\x1b[30m";
constexpr const char* color_gray = "\x1b[30;1m";
constexpr const char* color_red = "\x1b[31m";
constexpr const char* color_green = "\x1b[32m";
constexpr const char* color_yellow = "\x1b[33m";
constexpr const char* color_blue = "\x1b[34m";
constexpr const char* color_magenta = "\x1b[35m";
constexpr const char* color_cyan = "\x1b[36m";
constexpr const char* color_white = "\x1b[37m";
constexpr const char* color_reset = "\x1b[0m";
/**
* Writes out one buffer with OSM data in Debug format.
*/
class DebugOutputBlock : public osmium::handler::Handler {
static constexpr size_t tmp_buffer_size = 50;
std::shared_ptr<osmium::memory::Buffer> m_input_buffer;
std::shared_ptr<std::string> m_out;
char m_tmp_buffer[tmp_buffer_size+1];
bool m_add_metadata;
bool m_use_color;
template <typename... TArgs>
void output_formatted(const char* format, TArgs&&... args) {
#ifndef NDEBUG
int len =
#endif
#ifndef _MSC_VER
snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward<TArgs>(args)...);
#else
_snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward<TArgs>(args)...);
#endif
assert(len > 0 && static_cast<size_t>(len) < tmp_buffer_size);
*m_out += m_tmp_buffer;
}
void append_encoded_string(const char* data) {
const char* end = data + std::strlen(data);
while (data != end) {
const char* last = data;
uint32_t c = utf8::next(data, end);
// This is a list of Unicode code points that we let
// through instead of escaping them. It is incomplete
// and can be extended later.
// Generally we don't want to let through any
// non-printing characters.
if ((0x0020 <= c && c <= 0x0021) ||
(0x0023 <= c && c <= 0x003b) ||
(0x003d == c) ||
(0x003f <= c && c <= 0x007e) ||
(0x00a1 <= c && c <= 0x00ac) ||
(0x00ae <= c && c <= 0x05ff)) {
m_out->append(last, data);
} else {
write_color(color_red);
output_formatted("<U+%04X>", c);
write_color(color_blue);
}
}
}
void write_color(const char* color) {
if (m_use_color) {
*m_out += color;
}
}
void write_string(const char* string) {
*m_out += '"';
write_color(color_blue);
append_encoded_string(string);
write_color(color_reset);
*m_out += '"';
}
void write_object_type(const char* object_type, bool visible = true) {
if (visible) {
write_color(color_bold);
} else {
write_color(color_white);
}
*m_out += object_type;
write_color(color_reset);
*m_out += ' ';
}
void write_fieldname(const char* name) {
*m_out += " ";
write_color(color_cyan);
*m_out += name;
write_color(color_reset);
*m_out += ": ";
}
void write_error(const char* msg) {
write_color(color_red);
*m_out += msg;
write_color(color_reset);
}
void write_meta(const osmium::OSMObject& object) {
output_formatted("%" PRId64 "\n", object.id());
if (m_add_metadata) {
write_fieldname("version");
output_formatted(" %d", object.version());
if (object.visible()) {
*m_out += " visible\n";
} else {
write_error(" deleted\n");
}
write_fieldname("changeset");
output_formatted("%d\n", object.changeset());
write_fieldname("timestamp");
*m_out += object.timestamp().to_iso();
output_formatted(" (%d)\n", object.timestamp());
write_fieldname("user");
output_formatted(" %d ", object.uid());
write_string(object.user());
*m_out += '\n';
}
}
void write_tags(const osmium::TagList& tags, const char* padding="") {
if (!tags.empty()) {
write_fieldname("tags");
*m_out += padding;
output_formatted(" %d\n", tags.size());
osmium::max_op<int> max;
for (const auto& tag : tags) {
max.update(std::strlen(tag.key()));
}
for (const auto& tag : tags) {
*m_out += " ";
write_string(tag.key());
int spacing = max() - std::strlen(tag.key());
while (spacing--) {
*m_out += " ";
}
*m_out += " = ";
write_string(tag.value());
*m_out += '\n';
}
}
}
void write_location(const osmium::Location& location) {
write_fieldname("lon/lat");
output_formatted(" %.7f,%.7f", location.lon_without_check(), location.lat_without_check());
if (!location.valid()) {
write_error(" INVALID LOCATION!");
}
*m_out += '\n';
}
void write_box(const osmium::Box& box) {
write_fieldname("box l/b/r/t");
if (!box) {
write_error("BOX NOT SET!\n");
return;
}
const auto& bl = box.bottom_left();
const auto& tr = box.top_right();
output_formatted("%.7f,%.7f %.7f,%.7f", bl.lon_without_check(), bl.lat_without_check(), tr.lon_without_check(), tr.lat_without_check());
if (!box.valid()) {
write_error(" INVALID BOX!");
}
*m_out += '\n';
}
public:
explicit DebugOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata, bool use_color) :
m_input_buffer(std::make_shared<osmium::memory::Buffer>(std::move(buffer))),
m_out(std::make_shared<std::string>()),
m_tmp_buffer(),
m_add_metadata(add_metadata),
m_use_color(use_color) {
}
DebugOutputBlock(const DebugOutputBlock&) = default;
DebugOutputBlock& operator=(const DebugOutputBlock&) = default;
DebugOutputBlock(DebugOutputBlock&&) = default;
DebugOutputBlock& operator=(DebugOutputBlock&&) = default;
~DebugOutputBlock() = default;
std::string operator()() {
osmium::apply(m_input_buffer->cbegin(), m_input_buffer->cend(), *this);
std::string out;
std::swap(out, *m_out);
return out;
}
void node(const osmium::Node& node) {
write_object_type("node", node.visible());
write_meta(node);
if (node.visible()) {
write_location(node.location());
}
write_tags(node.tags());
*m_out += '\n';
}
void way(const osmium::Way& way) {
write_object_type("way", way.visible());
write_meta(way);
write_tags(way.tags());
write_fieldname("nodes");
output_formatted(" %d", way.nodes().size());
if (way.nodes().size() < 2) {
write_error(" LESS THAN 2 NODES!\n");
} else if (way.nodes().size() > 2000) {
write_error(" MORE THAN 2000 NODES!\n");
} else if (way.nodes().is_closed()) {
*m_out += " (closed)\n";
} else {
*m_out += " (open)\n";
}
int width = int(log10(way.nodes().size())) + 1;
int n = 0;
for (const auto& node_ref : way.nodes()) {
output_formatted(" %0*d: %10" PRId64, width, n++, node_ref.ref());
if (node_ref.location().valid()) {
output_formatted(" (%.7f,%.7f)", node_ref.location().lon_without_check(), node_ref.location().lat_without_check());
}
*m_out += '\n';
}
*m_out += '\n';
}
void relation(const osmium::Relation& relation) {
static const char* short_typename[] = { "node", "way ", "rel " };
write_object_type("relation", relation.visible());
write_meta(relation);
write_tags(relation.tags());
write_fieldname("members");
output_formatted(" %d\n", relation.members().size());
int width = int(log10(relation.members().size())) + 1;
int n = 0;
for (const auto& member : relation.members()) {
output_formatted(" %0*d: ", width, n++);
*m_out += short_typename[item_type_to_nwr_index(member.type())];
output_formatted(" %10" PRId64 " ", member.ref());
write_string(member.role());
*m_out += '\n';
}
*m_out += '\n';
}
void changeset(const osmium::Changeset& changeset) {
write_object_type("changeset");
output_formatted("%d\n", changeset.id());
write_fieldname("num changes");
output_formatted("%d", changeset.num_changes());
if (changeset.num_changes() == 0) {
write_error(" NO CHANGES!");
}
*m_out += '\n';
write_fieldname("created at");
*m_out += ' ';
*m_out += changeset.created_at().to_iso();
output_formatted(" (%d)\n", changeset.created_at());
write_fieldname("closed at");
*m_out += " ";
if (changeset.closed()) {
*m_out += changeset.closed_at().to_iso();
output_formatted(" (%d)\n", changeset.closed_at());
} else {
write_error("OPEN!\n");
}
write_fieldname("user");
output_formatted(" %d ", changeset.uid());
write_string(changeset.user());
*m_out += '\n';
write_box(changeset.bounds());
write_tags(changeset.tags(), " ");
*m_out += '\n';
}
}; // DebugOutputBlock
class DebugOutputFormat : public osmium::io::detail::OutputFormat {
bool m_add_metadata;
bool m_use_color;
public:
DebugOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) :
OutputFormat(file, output_queue),
m_add_metadata(file.get("add_metadata") != "false"),
m_use_color(file.get("color") == "true") {
}
DebugOutputFormat(const DebugOutputFormat&) = delete;
DebugOutputFormat& operator=(const DebugOutputFormat&) = delete;
void write_buffer(osmium::memory::Buffer&& buffer) override final {
m_output_queue.push(osmium::thread::Pool::instance().submit(DebugOutputBlock{std::move(buffer), m_add_metadata, m_use_color}));
}
void write_fieldname(std::string& out, const char* name) {
out += " ";
if (m_use_color) {
out += color_cyan;
}
out += name;
if (m_use_color) {
out += color_reset;
}
out += ": ";
}
void write_header(const osmium::io::Header& header) override final {
std::string out;
if (m_use_color) {
out += color_bold;
}
out += "header\n";
if (m_use_color) {
out += color_reset;
}
write_fieldname(out, "multiple object versions");
out += header.has_multiple_object_versions() ? "yes" : "no";
out += '\n';
write_fieldname(out, "bounding boxes");
out += '\n';
for (const auto& box : header.boxes()) {
out += " ";
box.bottom_left().as_string(std::back_inserter(out), ',');
out += " ";
box.top_right().as_string(std::back_inserter(out), ',');
out += '\n';
}
write_fieldname(out, "options");
out += '\n';
for (const auto& opt : header) {
out += " ";
out += opt.first;
out += " = ";
out += opt.second;
out += '\n';
}
out += "\n=============================================\n\n";
std::promise<std::string> promise;
m_output_queue.push(promise.get_future());
promise.set_value(std::move(out));
}
void close() override final {
std::string out;
std::promise<std::string> promise;
m_output_queue.push(promise.get_future());
promise.set_value(out);
}
}; // class DebugOutputFormat
namespace {
// we want the register_output_format() function to run, setting the variable
// is only a side-effect, it will never be used
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
const bool registered_debug_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::debug,
[](const osmium::io::File& file, data_queue_type& output_queue) {
return new osmium::io::detail::DebugOutputFormat(file, output_queue);
});
#pragma GCC diagnostic pop
} // anonymous namespace
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP
third_party/libosmium/include/osmium/io/detail/opl_output_format.hpp
Vendored
+37 -38
View File
@@ -46,23 +46,7 @@ DEALINGS IN THE SOFTWARE.
#include <thread>
#include <utility>
#include <boost/version.hpp>
#ifdef __clang__
# pragma clang diagnostic push
# pragma clang diagnostic ignored "-Wmissing-noreturn"
# pragma clang diagnostic ignored "-Wsign-conversion"
#endif
#if BOOST_VERSION >= 104800
# include <boost/regex/pending/unicode_iterator.hpp>
#else
# include <boost_unicode_iterator.hpp>
#endif
#ifdef __clang__
# pragma clang diagnostic pop
#endif
#include <utf8.h>
#include <osmium/handler.hpp>
#include <osmium/io/detail/output_format.hpp>
@@ -103,6 +87,8 @@ namespace osmium {
char m_tmp_buffer[tmp_buffer_size+1];
bool m_add_metadata;
template <typename... TArgs>
void output_formatted(const char* format, TArgs&&... args) {
#ifndef NDEBUG
@@ -117,13 +103,12 @@ namespace osmium {
*m_out += m_tmp_buffer;
}
void append_encoded_string(const std::string& data) {
boost::u8_to_u32_iterator<std::string::const_iterator> it(data.cbegin(), data.cbegin(), data.cend());
boost::u8_to_u32_iterator<std::string::const_iterator> end(data.cend(), data.cend(), data.cend());
boost::utf8_output_iterator<std::back_insert_iterator<std::string>> oit(std::back_inserter(*m_out));
void append_encoded_string(const char* data) {
const char* end = data + std::strlen(data);
for (; it != end; ++it) {
uint32_t c = *it;
while (data != end) {
const char* last = data;
uint32_t c = utf8::next(data, end);
// This is a list of Unicode code points that we let
// through instead of escaping them. It is incomplete
@@ -138,21 +123,29 @@ namespace osmium {
(0x0041 <= c && c <= 0x007e) ||
(0x00a1 <= c && c <= 0x00ac) ||
(0x00ae <= c && c <= 0x05ff)) {
*oit = c;
m_out->append(last, data);
} else {
*m_out += '%';
output_formatted("%04x", c);
if (c <= 0xff) {
output_formatted("%02x", c);
} else {
output_formatted("%04x", c);
}
*m_out += '%';
}
}
}
void write_meta(const osmium::OSMObject& object) {
output_formatted("%" PRId64 " v%d d", object.id(), object.version());
*m_out += (object.visible() ? 'V' : 'D');
output_formatted(" c%d t", object.changeset());
*m_out += object.timestamp().to_iso();
output_formatted(" i%d u", object.uid());
append_encoded_string(object.user());
output_formatted("%" PRId64, object.id());
if (m_add_metadata) {
output_formatted(" v%d d", object.version());
*m_out += (object.visible() ? 'V' : 'D');
output_formatted(" c%d t", object.changeset());
*m_out += object.timestamp().to_iso();
output_formatted(" i%d u", object.uid());
append_encoded_string(object.user());
}
*m_out += " T";
bool first = true;
for (const auto& tag : object.tags()) {
@@ -180,10 +173,11 @@ namespace osmium {
public:
explicit OPLOutputBlock(osmium::memory::Buffer&& buffer) :
explicit OPLOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata) :
m_input_buffer(std::make_shared<osmium::memory::Buffer>(std::move(buffer))),
m_out(std::make_shared<std::string>()),
m_tmp_buffer() {
m_tmp_buffer(),
m_add_metadata(add_metadata) {
}
OPLOutputBlock(const OPLOutputBlock&) = default;
@@ -240,7 +234,7 @@ namespace osmium {
}
*m_out += item_type_to_char(member.type());
output_formatted("%" PRId64 "@", member.ref());
*m_out += member.role();
append_encoded_string(member.role());
}
*m_out += '\n';
}
@@ -274,17 +268,20 @@ namespace osmium {
class OPLOutputFormat : public osmium::io::detail::OutputFormat {
OPLOutputFormat(const OPLOutputFormat&) = delete;
OPLOutputFormat& operator=(const OPLOutputFormat&) = delete;
bool m_add_metadata;
public:
OPLOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) :
OutputFormat(file, output_queue) {
OutputFormat(file, output_queue),
m_add_metadata(file.get("add_metadata") != "false") {
}
OPLOutputFormat(const OPLOutputFormat&) = delete;
OPLOutputFormat& operator=(const OPLOutputFormat&) = delete;
void write_buffer(osmium::memory::Buffer&& buffer) override final {
m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer)}));
m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer), m_add_metadata}));
}
void close() override final {
@@ -298,6 +295,8 @@ namespace osmium {
namespace {
// we want the register_output_format() function to run, setting the variable
// is only a side-effect, it will never be used
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
const bool registered_opl_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::opl,
third_party/libosmium/include/osmium/io/detail/pbf.hpp
Vendored
+22 -32
View File
@@ -33,9 +33,7 @@ DEALINGS IN THE SOFTWARE.
*/
#include <stdexcept>
#include <osmpbf/osmpbf.h>
#include <string>
// needed for htonl and ntohl
#ifndef _WIN32
@@ -45,38 +43,10 @@ DEALINGS IN THE SOFTWARE.
#endif
#include <osmium/io/error.hpp>
#include <osmium/osm/item_type.hpp>
#include <osmium/osm/location.hpp>
namespace osmium {
// avoid g++ false positive
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wreturn-type"
inline item_type osmpbf_membertype_to_item_type(const OSMPBF::Relation::MemberType mt) {
switch (mt) {
case OSMPBF::Relation::NODE:
return item_type::node;
case OSMPBF::Relation::WAY:
return item_type::way;
case OSMPBF::Relation::RELATION:
return item_type::relation;
}
}
#pragma GCC diagnostic pop
inline OSMPBF::Relation::MemberType item_type_to_osmpbf_membertype(const item_type type) {
switch (type) {
case item_type::node:
return OSMPBF::Relation::NODE;
case item_type::way:
return OSMPBF::Relation::WAY;
case item_type::relation:
return OSMPBF::Relation::RELATION;
default:
throw std::runtime_error("Unknown relation member type");
}
}
/**
* Exception thrown when there was a problem with parsing the PBF format of
* a file.
@@ -93,6 +63,26 @@ namespace osmium {
}; // struct pbf_error
namespace io {
namespace detail {
// the maximum size of a blob header in bytes
const int max_blob_header_size = 64 * 1024; // 64 kB
// the maximum size of an uncompressed blob in bytes
const uint64_t max_uncompressed_blob_size = 32 * 1024 * 1024; // 32 MB
// resolution for longitude/latitude used for conversion
// between representation as double and as int
const int64_t lonlat_resolution = 1000 * 1000 * 1000;
const int64_t resolution_convert = lonlat_resolution / osmium::Location::coordinate_precision;
}
}
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_PBF_HPP
third_party/libosmium/include/osmium/io/detail/pbf_decoder.hpp
Vendored
+760
View File
@@ -0,0 +1,760 @@
#ifndef OSMIUM_IO_DETAIL_PBF_DECODER_HPP
#define OSMIUM_IO_DETAIL_PBF_DECODER_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <algorithm>
#include <iterator>
#include <limits>
#include <protozero/pbf_message.hpp>
#include <osmium/builder/osm_object_builder.hpp>
#include <osmium/io/detail/pbf.hpp> // IWYU pragma: export
#include <osmium/io/detail/protobuf_tags.hpp>
#include <osmium/io/detail/zlib.hpp>
#include <osmium/io/header.hpp>
#include <osmium/osm/location.hpp>
#include <osmium/osm/node.hpp>
#include <osmium/osm/types.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/osm/entity_bits.hpp>
#include <osmium/util/cast.hpp>
#include <osmium/util/delta.hpp>
namespace osmium {
namespace io {
namespace detail {
using ptr_len_type = std::pair<const char*, size_t>;
class PBFPrimitiveBlockDecoder {
static constexpr size_t initial_buffer_size = 2 * 1024 * 1024;
ptr_len_type m_data;
std::vector<ptr_len_type> m_stringtable;
int64_t m_lon_offset = 0;
int64_t m_lat_offset = 0;
int64_t m_date_factor = 1000;
int32_t m_granularity = 100;
osmium::osm_entity_bits::type m_read_types;
osmium::memory::Buffer m_buffer { initial_buffer_size };
void decode_stringtable(const ptr_len_type& data) {
if (!m_stringtable.empty()) {
throw osmium::pbf_error("more than one stringtable in pbf file");
}
protozero::pbf_message<OSMFormat::StringTable> pbf_string_table(data);
while (pbf_string_table.next(OSMFormat::StringTable::repeated_bytes_s)) {
m_stringtable.push_back(pbf_string_table.get_data());
}
}
void decode_primitive_block_metadata() {
protozero::pbf_message<OSMFormat::PrimitiveBlock> pbf_primitive_block(m_data);
while (pbf_primitive_block.next()) {
switch (pbf_primitive_block.tag()) {
case OSMFormat::PrimitiveBlock::required_StringTable_stringtable:
decode_stringtable(pbf_primitive_block.get_data());
break;
case OSMFormat::PrimitiveBlock::optional_int32_granularity:
m_granularity = pbf_primitive_block.get_int32();
break;
case OSMFormat::PrimitiveBlock::optional_int32_date_granularity:
m_date_factor = pbf_primitive_block.get_int32();
break;
case OSMFormat::PrimitiveBlock::optional_int64_lat_offset:
m_lat_offset = pbf_primitive_block.get_int64();
break;
case OSMFormat::PrimitiveBlock::optional_int64_lon_offset:
m_lon_offset = pbf_primitive_block.get_int64();
break;
default:
pbf_primitive_block.skip();
}
}
}
void decode_primitive_block_data() {
protozero::pbf_message<OSMFormat::PrimitiveBlock> pbf_primitive_block(m_data);
while (pbf_primitive_block.next(OSMFormat::PrimitiveBlock::repeated_PrimitiveGroup_primitivegroup)) {
protozero::pbf_message<OSMFormat::PrimitiveGroup> pbf_primitive_group = pbf_primitive_block.get_message();
while (pbf_primitive_group.next()) {
switch (pbf_primitive_group.tag()) {
case OSMFormat::PrimitiveGroup::repeated_Node_nodes:
if (m_read_types & osmium::osm_entity_bits::node) {
decode_node(pbf_primitive_group.get_data());
} else {
pbf_primitive_group.skip();
}
break;
case OSMFormat::PrimitiveGroup::optional_DenseNodes_dense:
if (m_read_types & osmium::osm_entity_bits::node) {
decode_dense_nodes(pbf_primitive_group.get_data());
} else {
pbf_primitive_group.skip();
}
break;
case OSMFormat::PrimitiveGroup::repeated_Way_ways:
if (m_read_types & osmium::osm_entity_bits::way) {
decode_way(pbf_primitive_group.get_data());
} else {
pbf_primitive_group.skip();
}
break;
case OSMFormat::PrimitiveGroup::repeated_Relation_relations:
if (m_read_types & osmium::osm_entity_bits::relation) {
decode_relation(pbf_primitive_group.get_data());
} else {
pbf_primitive_group.skip();
}
break;
default:
pbf_primitive_group.skip();
}
}
}
}
ptr_len_type decode_info(const ptr_len_type& data, osmium::OSMObject& object) {
ptr_len_type user = std::make_pair("", 0);
protozero::pbf_message<OSMFormat::Info> pbf_info(data);
while (pbf_info.next()) {
switch (pbf_info.tag()) {
case OSMFormat::Info::optional_int32_version:
{
auto version = pbf_info.get_int32();
if (version < 0) {
throw osmium::pbf_error("object version must not be negative");
}
object.set_version(static_cast_with_assert<object_version_type>(version));
}
break;
case OSMFormat::Info::optional_int64_timestamp:
object.set_timestamp(pbf_info.get_int64() * m_date_factor / 1000);
break;
case OSMFormat::Info::optional_int64_changeset:
{
auto changeset_id = pbf_info.get_int64();
if (changeset_id < 0) {
throw osmium::pbf_error("object changeset_id must not be negative");
}
object.set_changeset(static_cast_with_assert<changeset_id_type>(changeset_id));
}
break;
case OSMFormat::Info::optional_int32_uid:
object.set_uid_from_signed(pbf_info.get_int32());
break;
case OSMFormat::Info::optional_uint32_user_sid:
user = m_stringtable.at(pbf_info.get_uint32());
break;
case OSMFormat::Info::optional_bool_visible:
object.set_visible(pbf_info.get_bool());
break;
default:
pbf_info.skip();
}
}
return user;
}
using kv_type = std::pair<protozero::pbf_reader::const_uint32_iterator, protozero::pbf_reader::const_uint32_iterator>;
void build_tag_list(osmium::builder::Builder& builder, const kv_type& keys, const kv_type& vals) {
if (keys.first != keys.second) {
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
auto kit = keys.first;
auto vit = vals.first;
while (kit != keys.second) {
if (vit == vals.second) {
// this is against the spec, must have same number of elements
throw osmium::pbf_error("PBF format error");
}
const auto& k = m_stringtable.at(*kit++);
const auto& v = m_stringtable.at(*vit++);
tl_builder.add_tag(k.first, k.second, v.first, v.second);
}
}
}
int32_t convert_pbf_coordinate(int64_t c) const {
return (c * m_granularity + m_lon_offset) / resolution_convert;
}
void decode_node(const ptr_len_type& data) {
osmium::builder::NodeBuilder builder(m_buffer);
osmium::Node& node = builder.object();
kv_type keys;
kv_type vals;
int64_t lon = std::numeric_limits<int64_t>::max();
int64_t lat = std::numeric_limits<int64_t>::max();
ptr_len_type user = { "", 0 };
protozero::pbf_message<OSMFormat::Node> pbf_node(data);
while (pbf_node.next()) {
switch (pbf_node.tag()) {
case OSMFormat::Node::required_sint64_id:
node.set_id(pbf_node.get_sint64());
break;
case OSMFormat::Node::packed_uint32_keys:
keys = pbf_node.get_packed_uint32();
break;
case OSMFormat::Node::packed_uint32_vals:
vals = pbf_node.get_packed_uint32();
break;
case OSMFormat::Node::optional_Info_info:
user = decode_info(pbf_node.get_data(), builder.object());
break;
case OSMFormat::Node::required_sint64_lat:
lat = pbf_node.get_sint64();
break;
case OSMFormat::Node::required_sint64_lon:
lon = pbf_node.get_sint64();
break;
default:
pbf_node.skip();
}
}
if (node.visible()) {
if (lon == std::numeric_limits<int64_t>::max() ||
lat == std::numeric_limits<int64_t>::max()) {
throw osmium::pbf_error("illegal coordinate format");
}
node.set_location(osmium::Location(
convert_pbf_coordinate(lon),
convert_pbf_coordinate(lat)
));
}
builder.add_user(user.first, user.second);
build_tag_list(builder, keys, vals);
m_buffer.commit();
}
void decode_way(const ptr_len_type& data) {
osmium::builder::WayBuilder builder(m_buffer);
kv_type keys;
kv_type vals;
std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> refs;
ptr_len_type user = { "", 0 };
protozero::pbf_message<OSMFormat::Way> pbf_way(data);
while (pbf_way.next()) {
switch (pbf_way.tag()) {
case OSMFormat::Way::required_int64_id:
builder.object().set_id(pbf_way.get_int64());
break;
case OSMFormat::Way::packed_uint32_keys:
keys = pbf_way.get_packed_uint32();
break;
case OSMFormat::Way::packed_uint32_vals:
vals = pbf_way.get_packed_uint32();
break;
case OSMFormat::Way::optional_Info_info:
user = decode_info(pbf_way.get_data(), builder.object());
break;
case OSMFormat::Way::packed_sint64_refs:
refs = pbf_way.get_packed_sint64();
break;
default:
pbf_way.skip();
}
}
builder.add_user(user.first, user.second);
if (refs.first != refs.second) {
osmium::builder::WayNodeListBuilder wnl_builder(m_buffer, &builder);
osmium::util::DeltaDecode<int64_t> ref;
while (refs.first != refs.second) {
wnl_builder.add_node_ref(ref.update(*refs.first++));
}
}
build_tag_list(builder, keys, vals);
m_buffer.commit();
}
void decode_relation(const ptr_len_type& data) {
osmium::builder::RelationBuilder builder(m_buffer);
kv_type keys;
kv_type vals;
std::pair<protozero::pbf_reader::const_int32_iterator, protozero::pbf_reader::const_int32_iterator> roles;
std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> refs;
std::pair<protozero::pbf_reader::const_int32_iterator, protozero::pbf_reader::const_int32_iterator> types;
ptr_len_type user = { "", 0 };
protozero::pbf_message<OSMFormat::Relation> pbf_relation(data);
while (pbf_relation.next()) {
switch (pbf_relation.tag()) {
case OSMFormat::Relation::required_int64_id:
builder.object().set_id(pbf_relation.get_int64());
break;
case OSMFormat::Relation::packed_uint32_keys:
keys = pbf_relation.get_packed_uint32();
break;
case OSMFormat::Relation::packed_uint32_vals:
vals = pbf_relation.get_packed_uint32();
break;
case OSMFormat::Relation::optional_Info_info:
user = decode_info(pbf_relation.get_data(), builder.object());
break;
case OSMFormat::Relation::packed_int32_roles_sid:
roles = pbf_relation.get_packed_int32();
break;
case OSMFormat::Relation::packed_sint64_memids:
refs = pbf_relation.get_packed_sint64();
break;
case OSMFormat::Relation::packed_MemberType_types:
types = pbf_relation.get_packed_enum();
break;
default:
pbf_relation.skip();
}
}
builder.add_user(user.first, user.second);
if (refs.first != refs.second) {
osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder);
osmium::util::DeltaDecode<int64_t> ref;
while (roles.first != roles.second && refs.first != refs.second && types.first != types.second) {
const auto& r = m_stringtable.at(*roles.first++);
int type = *types.first++;
if (type < 0 || type > 2) {
throw osmium::pbf_error("unknown relation member type");
}
rml_builder.add_member(
osmium::item_type(type + 1),
ref.update(*refs.first++),
r.first,
r.second
);
}
}
build_tag_list(builder, keys, vals);
m_buffer.commit();
}
void decode_dense_nodes(const ptr_len_type& data) {
bool has_info = false;
bool has_visibles = false;
std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> ids;
std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> lats;
std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> lons;
std::pair<protozero::pbf_reader::const_int32_iterator, protozero::pbf_reader::const_int32_iterator> tags;
std::pair<protozero::pbf_reader::const_int32_iterator, protozero::pbf_reader::const_int32_iterator> versions;
std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> timestamps;
std::pair<protozero::pbf_reader::const_sint64_iterator, protozero::pbf_reader::const_sint64_iterator> changesets;
std::pair<protozero::pbf_reader::const_sint32_iterator, protozero::pbf_reader::const_sint32_iterator> uids;
std::pair<protozero::pbf_reader::const_sint32_iterator, protozero::pbf_reader::const_sint32_iterator> user_sids;
std::pair<protozero::pbf_reader::const_int32_iterator, protozero::pbf_reader::const_int32_iterator> visibles;
protozero::pbf_message<OSMFormat::DenseNodes> pbf_dense_nodes(data);
while (pbf_dense_nodes.next()) {
switch (pbf_dense_nodes.tag()) {
case OSMFormat::DenseNodes::packed_sint64_id:
ids = pbf_dense_nodes.get_packed_sint64();
break;
case OSMFormat::DenseNodes::optional_DenseInfo_denseinfo:
{
has_info = true;
protozero::pbf_message<OSMFormat::DenseInfo> pbf_dense_info = pbf_dense_nodes.get_message();
while (pbf_dense_info.next()) {
switch (pbf_dense_info.tag()) {
case OSMFormat::DenseInfo::packed_int32_version:
versions = pbf_dense_info.get_packed_int32();
break;
case OSMFormat::DenseInfo::packed_sint64_timestamp:
timestamps = pbf_dense_info.get_packed_sint64();
break;
case OSMFormat::DenseInfo::packed_sint64_changeset:
changesets = pbf_dense_info.get_packed_sint64();
break;
case OSMFormat::DenseInfo::packed_sint32_uid:
uids = pbf_dense_info.get_packed_sint32();
break;
case OSMFormat::DenseInfo::packed_sint32_user_sid:
user_sids = pbf_dense_info.get_packed_sint32();
break;
case OSMFormat::DenseInfo::packed_bool_visible:
has_visibles = true;
visibles = pbf_dense_info.get_packed_bool();
break;
default:
pbf_dense_info.skip();
}
}
}
break;
case OSMFormat::DenseNodes::packed_sint64_lat:
lats = pbf_dense_nodes.get_packed_sint64();
break;
case OSMFormat::DenseNodes::packed_sint64_lon:
lons = pbf_dense_nodes.get_packed_sint64();
break;
case OSMFormat::DenseNodes::packed_int32_keys_vals:
tags = pbf_dense_nodes.get_packed_int32();
break;
default:
pbf_dense_nodes.skip();
}
}
osmium::util::DeltaDecode<int64_t> dense_id;
osmium::util::DeltaDecode<int64_t> dense_latitude;
osmium::util::DeltaDecode<int64_t> dense_longitude;
osmium::util::DeltaDecode<int64_t> dense_uid;
osmium::util::DeltaDecode<int64_t> dense_user_sid;
osmium::util::DeltaDecode<int64_t> dense_changeset;
osmium::util::DeltaDecode<int64_t> dense_timestamp;
auto tag_it = tags.first;
while (ids.first != ids.second) {
if (lons.first == lons.second ||
lats.first == lats.second) {
// this is against the spec, must have same number of elements
throw osmium::pbf_error("PBF format error");
}
bool visible = true;
osmium::builder::NodeBuilder builder(m_buffer);
osmium::Node& node = builder.object();
node.set_id(dense_id.update(*ids.first++));
if (has_info) {
if (versions.first == versions.second ||
changesets.first == changesets.second ||
timestamps.first == timestamps.second ||
uids.first == uids.second ||
user_sids.first == user_sids.second) {
// this is against the spec, must have same number of elements
throw osmium::pbf_error("PBF format error");
}
auto version = *versions.first++;
if (version < 0) {
throw osmium::pbf_error("object version must not be negative");
}
node.set_version(static_cast<osmium::object_version_type>(version));
auto changeset_id = dense_changeset.update(*changesets.first++);
if (changeset_id < 0) {
throw osmium::pbf_error("object changeset_id must not be negative");
}
node.set_changeset(static_cast<osmium::changeset_id_type>(changeset_id));
node.set_timestamp(dense_timestamp.update(*timestamps.first++) * m_date_factor / 1000);
node.set_uid_from_signed(static_cast<osmium::signed_user_id_type>(dense_uid.update(*uids.first++)));
if (has_visibles) {
if (visibles.first == visibles.second) {
// this is against the spec, must have same number of elements
throw osmium::pbf_error("PBF format error");
}
visible = *visibles.first++;
}
node.set_visible(visible);
const auto& u = m_stringtable.at(dense_user_sid.update(*user_sids.first++));
builder.add_user(u.first, u.second);
} else {
builder.add_user("");
}
if (visible) {
builder.object().set_location(osmium::Location(
convert_pbf_coordinate(dense_longitude.update(*lons.first++)),
convert_pbf_coordinate(dense_latitude.update(*lats.first++))
));
}
if (tag_it != tags.second) {
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
while (tag_it != tags.second && *tag_it != 0) {
const auto& k = m_stringtable.at(*tag_it++);
if (tag_it == tags.second) {
throw osmium::pbf_error("PBF format error"); // this is against the spec, keys/vals must come in pairs
}
const auto& v = m_stringtable.at(*tag_it++);
tl_builder.add_tag(k.first, k.second, v.first, v.second);
}
if (tag_it != tags.second) {
++tag_it;
}
}
m_buffer.commit();
}
}
public:
explicit PBFPrimitiveBlockDecoder(const ptr_len_type& data, osmium::osm_entity_bits::type read_types) :
m_data(data),
m_read_types(read_types) {
}
PBFPrimitiveBlockDecoder(const PBFPrimitiveBlockDecoder&) = delete;
PBFPrimitiveBlockDecoder& operator=(const PBFPrimitiveBlockDecoder&) = delete;
PBFPrimitiveBlockDecoder(PBFPrimitiveBlockDecoder&&) = delete;
PBFPrimitiveBlockDecoder& operator=(PBFPrimitiveBlockDecoder&&) = delete;
~PBFPrimitiveBlockDecoder() = default;
osmium::memory::Buffer operator()() {
try {
decode_primitive_block_metadata();
decode_primitive_block_data();
} catch (std::out_of_range&) {
throw osmium::pbf_error("string id out of range");
}
return std::move(m_buffer);
}
}; // class PBFPrimitiveBlockDecoder
inline ptr_len_type decode_blob(const std::string& blob_data, std::string& output) {
int32_t raw_size;
std::pair<const char*, protozero::pbf_length_type> zlib_data;
protozero::pbf_message<FileFormat::Blob> pbf_blob(blob_data);
while (pbf_blob.next()) {
switch (pbf_blob.tag()) {
case FileFormat::Blob::optional_bytes_raw:
{
auto data_len = pbf_blob.get_data();
if (data_len.second > max_uncompressed_blob_size) {
throw osmium::pbf_error("illegal blob size");
}
return data_len;
}
case FileFormat::Blob::optional_int32_raw_size:
raw_size = pbf_blob.get_int32();
if (raw_size <= 0 || uint32_t(raw_size) > max_uncompressed_blob_size) {
throw osmium::pbf_error("illegal blob size");
}
break;
case FileFormat::Blob::optional_bytes_zlib_data:
zlib_data = pbf_blob.get_data();
break;
case FileFormat::Blob::optional_bytes_lzma_data:
throw osmium::pbf_error("lzma blobs not implemented");
default:
throw osmium::pbf_error("unknown compression");
}
}
if (zlib_data.second != 0) {
return osmium::io::detail::zlib_uncompress_string(
zlib_data.first,
static_cast<unsigned long>(zlib_data.second),
static_cast<unsigned long>(raw_size),
output
);
}
throw osmium::pbf_error("blob contains no data");
}
inline osmium::Box decode_header_bbox(const ptr_len_type& data) {
int64_t left = std::numeric_limits<int64_t>::max();
int64_t right = std::numeric_limits<int64_t>::max();
int64_t top = std::numeric_limits<int64_t>::max();
int64_t bottom = std::numeric_limits<int64_t>::max();
protozero::pbf_message<OSMFormat::HeaderBBox> pbf_header_bbox(data);
while (pbf_header_bbox.next()) {
switch (pbf_header_bbox.tag()) {
case OSMFormat::HeaderBBox::required_sint64_left:
left = pbf_header_bbox.get_sint64();
break;
case OSMFormat::HeaderBBox::required_sint64_right:
right = pbf_header_bbox.get_sint64();
break;
case OSMFormat::HeaderBBox::required_sint64_top:
top = pbf_header_bbox.get_sint64();
break;
case OSMFormat::HeaderBBox::required_sint64_bottom:
bottom = pbf_header_bbox.get_sint64();
break;
default:
pbf_header_bbox.skip();
}
}
if (left == std::numeric_limits<int64_t>::max() ||
right == std::numeric_limits<int64_t>::max() ||
top == std::numeric_limits<int64_t>::max() ||
bottom == std::numeric_limits<int64_t>::max()) {
throw osmium::pbf_error("invalid bbox");
}
osmium::Box box;
box.extend(osmium::Location(left / resolution_convert, bottom / resolution_convert));
box.extend(osmium::Location(right / resolution_convert, top / resolution_convert));
return box;
}
inline osmium::io::Header decode_header_block(const ptr_len_type& data) {
osmium::io::Header header;
int i = 0;
protozero::pbf_message<OSMFormat::HeaderBlock> pbf_header_block(data);
while (pbf_header_block.next()) {
switch (pbf_header_block.tag()) {
case OSMFormat::HeaderBlock::optional_HeaderBBox_bbox:
header.add_box(decode_header_bbox(pbf_header_block.get_data()));
break;
case OSMFormat::HeaderBlock::repeated_string_required_features:
{
auto feature = pbf_header_block.get_data();
if (!strncmp("OsmSchema-V0.6", feature.first, feature.second)) {
// intentionally left blank
} else if (!strncmp("DenseNodes", feature.first, feature.second)) {
header.set("pbf_dense_nodes", true);
} else if (!strncmp("HistoricalInformation", feature.first, feature.second)) {
header.set_has_multiple_object_versions(true);
} else {
std::string msg("required feature not supported: ");
msg.append(feature.first, feature.second);
throw osmium::pbf_error(msg);
}
}
break;
case OSMFormat::HeaderBlock::repeated_string_optional_features:
header.set("pbf_optional_feature_" + std::to_string(i++), pbf_header_block.get_string());
break;
case OSMFormat::HeaderBlock::optional_string_writingprogram:
header.set("generator", pbf_header_block.get_string());
break;
case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp:
header.set("osmosis_replication_timestamp", osmium::Timestamp(pbf_header_block.get_int64()).to_iso());
break;
case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_sequence_number:
header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.get_int64()));
break;
case OSMFormat::HeaderBlock::optional_string_osmosis_replication_base_url:
header.set("osmosis_replication_base_url", pbf_header_block.get_string());
break;
default:
pbf_header_block.skip();
}
}
return header;
}
/**
* Decode HeaderBlock.
*
* @param header_block_data Input data
* @returns Header object
* @throws osmium::pbf_error If there was a parsing error
*/
inline osmium::io::Header decode_header(const std::string& header_block_data) {
std::string output;
return decode_header_block(decode_blob(header_block_data, output));
}
class PBFDataBlobDecoder {
std::shared_ptr<std::string> m_input_buffer;
osmium::osm_entity_bits::type m_read_types;
public:
PBFDataBlobDecoder(std::string&& input_buffer, osmium::osm_entity_bits::type read_types) :
m_input_buffer(std::make_shared<std::string>(std::move(input_buffer))),
m_read_types(read_types) {
}
PBFDataBlobDecoder(const PBFDataBlobDecoder&) = default;
PBFDataBlobDecoder& operator=(const PBFDataBlobDecoder&) = default;
PBFDataBlobDecoder(PBFDataBlobDecoder&&) = default;
PBFDataBlobDecoder& operator=(PBFDataBlobDecoder&&) = default;
~PBFDataBlobDecoder() = default;
osmium::memory::Buffer operator()() {
std::string output;
PBFPrimitiveBlockDecoder decoder(decode_blob(*m_input_buffer, output), m_read_types);
return decoder();
}
}; // class PBFDataBlobDecoder
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_PBF_DECODER_HPP
third_party/libosmium/include/osmium/io/detail/pbf_input_format.hpp
Vendored
+67 -26
View File
@@ -49,9 +49,12 @@ DEALINGS IN THE SOFTWARE.
#include <thread>
#include <type_traits>
#include <protozero/pbf_message.hpp>
#include <osmium/io/detail/input_format.hpp>
#include <osmium/io/detail/pbf.hpp> // IWYU pragma: export
#include <osmium/io/detail/pbf_parser.hpp>
#include <osmium/io/detail/pbf_decoder.hpp>
#include <osmium/io/detail/protobuf_tags.hpp>
#include <osmium/io/error.hpp>
#include <osmium/io/file.hpp>
#include <osmium/io/file_format.hpp>
@@ -76,13 +79,13 @@ namespace osmium {
namespace detail {
typedef osmium::thread::Queue<std::future<osmium::memory::Buffer>> queue_type;
/**
* Class for parsing PBF files.
*/
class PBFInputFormat : public osmium::io::detail::InputFormat {
typedef osmium::thread::Queue<std::future<osmium::memory::Buffer>> queue_type;
bool m_use_thread_pool;
bool m_eof { false };
queue_type m_queue;
@@ -115,15 +118,10 @@ namespace osmium {
}
/**
* Read BlobHeader by first reading the size and then the
* BlobHeader. The BlobHeader contains a type field (which is
* checked against the expected type) and a size field.
*
* @param expected_type Expected type of data ("OSMHeader" or
* "OSMData").
* @returns Size of the data read from BlobHeader (0 on EOF).
* Read 4 bytes in network byte order from file. They contain
* the length of the following BlobHeader.
*/
size_t read_blob_header(const char* expected_type) {
uint32_t read_blob_header_size_from_file() {
uint32_t size_in_network_byte_order;
try {
@@ -133,37 +131,76 @@ namespace osmium {
return 0; // EOF
}
uint32_t size = ntohl(size_in_network_byte_order);
if (size > static_cast<uint32_t>(OSMPBF::max_blob_header_size)) {
const uint32_t size = ntohl(size_in_network_byte_order);
if (size > static_cast<uint32_t>(max_blob_header_size)) {
throw osmium::pbf_error("invalid BlobHeader size (> max_blob_header_size)");
}
OSMPBF::BlobHeader blob_header;
if (!blob_header.ParseFromString(read_from_input_queue(size))) {
throw osmium::pbf_error("failed to parse BlobHeader");
return size;
}
/**
* Decode the BlobHeader. Make sure it contains the expected
* type. Return the size of the following Blob.
*/
size_t decode_blob_header(protozero::pbf_message<FileFormat::BlobHeader>&& pbf_blob_header, const char* expected_type) {
std::pair<const char*, size_t> blob_header_type;
size_t blob_header_datasize = 0;
while (pbf_blob_header.next()) {
switch (pbf_blob_header.tag()) {
case FileFormat::BlobHeader::required_string_type:
blob_header_type = pbf_blob_header.get_data();
break;
case FileFormat::BlobHeader::required_int32_datasize:
blob_header_datasize = pbf_blob_header.get_int32();
break;
default:
pbf_blob_header.skip();
}
}
if (blob_header.type() != expected_type) {
if (blob_header_datasize == 0) {
throw osmium::pbf_error("PBF format error: BlobHeader.datasize missing or zero.");
}
if (strncmp(expected_type, blob_header_type.first, blob_header_type.second)) {
throw osmium::pbf_error("blob does not have expected type (OSMHeader in first blob, OSMData in following blobs)");
}
return static_cast<size_t>(blob_header.datasize());
return blob_header_datasize;
}
size_t check_type_and_get_blob_size(const char* expected_type) {
assert(expected_type);
auto size = read_blob_header_size_from_file();
if (size == 0) { // EOF
return 0;
}
std::string blob_header = read_from_input_queue(size);
return decode_blob_header(protozero::pbf_message<FileFormat::BlobHeader>(blob_header), expected_type);
}
void parse_osm_data(osmium::osm_entity_bits::type read_types) {
osmium::thread::set_thread_name("_osmium_pbf_in");
int n = 0;
while (auto size = read_blob_header("OSMData")) {
while (auto size = check_type_and_get_blob_size("OSMData")) {
std::string input_buffer = read_from_input_queue(size);
if (input_buffer.size() > max_uncompressed_blob_size) {
throw osmium::pbf_error(std::string("invalid blob size: " + std::to_string(input_buffer.size())));
}
if (m_use_thread_pool) {
m_queue.push(osmium::thread::Pool::instance().submit(DataBlobParser{read_from_input_queue(size), read_types}));
m_queue.push(osmium::thread::Pool::instance().submit(PBFDataBlobDecoder{ std::move(input_buffer), read_types }));
} else {
std::promise<osmium::memory::Buffer> promise;
m_queue.push(promise.get_future());
DataBlobParser data_blob_parser{read_from_input_queue(size), read_types};
PBFDataBlobDecoder data_blob_parser{ std::move(input_buffer), read_types };
promise.set_value(data_blob_parser());
}
++n;
if (m_quit_input_thread) {
return;
@@ -197,11 +234,10 @@ namespace osmium {
m_quit_input_thread(false),
m_input_queue(input_queue),
m_input_buffer() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
// handle OSMHeader
auto size = read_blob_header("OSMHeader");
m_header = parse_header_blob(read_from_input_queue(size));
const auto size = check_type_and_get_blob_size("OSMHeader");
m_header = decode_header(read_from_input_queue(size));
if (m_read_which_entities != osmium::osm_entity_bits::nothing) {
m_reader = std::thread(&PBFInputFormat::parse_osm_data, this, m_read_which_entities);
@@ -246,10 +282,15 @@ namespace osmium {
namespace {
// we want the register_input_format() function to run, setting the variable
// is only a side-effect, it will never be used
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
const bool registered_pbf_input = osmium::io::detail::InputFormatFactory::instance().register_input_format(osmium::io::file_format::pbf,
[](const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue<std::string>& input_queue) {
return new osmium::io::detail::PBFInputFormat(file, read_which_entities, input_queue);
});
#pragma GCC diagnostic pop
} // anonymous namespace
third_party/libosmium/include/osmium/io/detail/pbf_output_format.hpp
Vendored
+391 -764
View File
File diff suppressed because it is too large Load Diff
third_party/libosmium/include/osmium/io/detail/pbf_parser.hpp
Vendored
-455
View File
@@ -1,455 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP
#define OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <algorithm>
#include <osmpbf/osmpbf.h>
#include <osmium/builder/osm_object_builder.hpp>
#include <osmium/io/detail/pbf.hpp> // IWYU pragma: export
#include <osmium/io/detail/zlib.hpp>
#include <osmium/io/header.hpp>
#include <osmium/osm/location.hpp>
#include <osmium/osm/node.hpp>
#include <osmium/osm/types.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/osm/entity_bits.hpp>
#include <osmium/util/cast.hpp>
namespace osmium {
namespace io {
namespace detail {
class PBFPrimitiveBlockParser {
static constexpr size_t initial_buffer_size = 2 * 1024 * 1024;
const std::string& m_data;
const OSMPBF::StringTable* m_stringtable;
int64_t m_lon_offset;
int64_t m_lat_offset;
int64_t m_date_factor;
int32_t m_granularity;
osmium::osm_entity_bits::type m_read_types;
osmium::memory::Buffer m_buffer;
PBFPrimitiveBlockParser(const PBFPrimitiveBlockParser&) = delete;
PBFPrimitiveBlockParser(PBFPrimitiveBlockParser&&) = delete;
PBFPrimitiveBlockParser& operator=(const PBFPrimitiveBlockParser&) = delete;
PBFPrimitiveBlockParser& operator=(PBFPrimitiveBlockParser&&) = delete;
public:
explicit PBFPrimitiveBlockParser(const std::string& data, osmium::osm_entity_bits::type read_types) :
m_data(data),
m_stringtable(nullptr),
m_lon_offset(0),
m_lat_offset(0),
m_date_factor(1000),
m_granularity(100),
m_read_types(read_types),
m_buffer(initial_buffer_size) {
}
~PBFPrimitiveBlockParser() = default;
osmium::memory::Buffer operator()() {
OSMPBF::PrimitiveBlock pbf_primitive_block;
if (!pbf_primitive_block.ParseFromString(m_data)) {
throw osmium::pbf_error("failed to parse PrimitiveBlock");
}
m_stringtable = &pbf_primitive_block.stringtable();
m_lon_offset = pbf_primitive_block.lon_offset();
m_lat_offset = pbf_primitive_block.lat_offset();
m_date_factor = pbf_primitive_block.date_granularity() / 1000;
m_granularity = pbf_primitive_block.granularity();
for (int i = 0; i < pbf_primitive_block.primitivegroup_size(); ++i) {
const OSMPBF::PrimitiveGroup& group = pbf_primitive_block.primitivegroup(i);
if (group.has_dense()) {
if (m_read_types & osmium::osm_entity_bits::node) parse_dense_node_group(group);
} else if (group.ways_size() != 0) {
if (m_read_types & osmium::osm_entity_bits::way) parse_way_group(group);
} else if (group.relations_size() != 0) {
if (m_read_types & osmium::osm_entity_bits::relation) parse_relation_group(group);
} else if (group.nodes_size() != 0) {
if (m_read_types & osmium::osm_entity_bits::node) parse_node_group(group);
} else {
throw osmium::pbf_error("group of unknown type");
}
}
return std::move(m_buffer);
}
private:
template <class TBuilder, class TPBFObject>
void parse_attributes(TBuilder& builder, const TPBFObject& pbf_object) {
auto& object = builder.object();
object.set_id(pbf_object.id());
if (pbf_object.has_info()) {
object.set_version(static_cast_with_assert<object_version_type>(pbf_object.info().version()))
.set_changeset(static_cast_with_assert<changeset_id_type>(pbf_object.info().changeset()))
.set_timestamp(pbf_object.info().timestamp() * m_date_factor)
.set_uid_from_signed(pbf_object.info().uid());
if (pbf_object.info().has_visible()) {
object.set_visible(pbf_object.info().visible());
}
builder.add_user(m_stringtable->s(static_cast_with_assert<int>(pbf_object.info().user_sid())));
} else {
builder.add_user("", 1);
}
}
void parse_node_group(const OSMPBF::PrimitiveGroup& group) {
for (int i = 0; i < group.nodes_size(); ++i) {
osmium::builder::NodeBuilder builder(m_buffer);
const OSMPBF::Node& pbf_node = group.nodes(i);
parse_attributes(builder, pbf_node);
if (builder.object().visible()) {
builder.object().set_location(osmium::Location(
(pbf_node.lon() * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision),
(pbf_node.lat() * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision)));
}
if (pbf_node.keys_size() > 0) {
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
for (int tag = 0; tag < pbf_node.keys_size(); ++tag) {
tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_node.keys(tag))),
m_stringtable->s(static_cast<int>(pbf_node.vals(tag))));
}
}
m_buffer.commit();
}
}
void parse_way_group(const OSMPBF::PrimitiveGroup& group) {
for (int i = 0; i < group.ways_size(); ++i) {
osmium::builder::WayBuilder builder(m_buffer);
const OSMPBF::Way& pbf_way = group.ways(i);
parse_attributes(builder, pbf_way);
if (pbf_way.refs_size() > 0) {
osmium::builder::WayNodeListBuilder wnl_builder(m_buffer, &builder);
int64_t ref = 0;
for (int n = 0; n < pbf_way.refs_size(); ++n) {
ref += pbf_way.refs(n);
wnl_builder.add_node_ref(ref);
}
}
if (pbf_way.keys_size() > 0) {
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
for (int tag = 0; tag < pbf_way.keys_size(); ++tag) {
tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_way.keys(tag))),
m_stringtable->s(static_cast<int>(pbf_way.vals(tag))));
}
}
m_buffer.commit();
}
}
void parse_relation_group(const OSMPBF::PrimitiveGroup& group) {
for (int i = 0; i < group.relations_size(); ++i) {
osmium::builder::RelationBuilder builder(m_buffer);
const OSMPBF::Relation& pbf_relation = group.relations(i);
parse_attributes(builder, pbf_relation);
if (pbf_relation.types_size() > 0) {
osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder);
int64_t ref = 0;
for (int n = 0; n < pbf_relation.types_size(); ++n) {
ref += pbf_relation.memids(n);
rml_builder.add_member(osmpbf_membertype_to_item_type(pbf_relation.types(n)), ref, m_stringtable->s(pbf_relation.roles_sid(n)));
}
}
if (pbf_relation.keys_size() > 0) {
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
for (int tag = 0; tag < pbf_relation.keys_size(); ++tag) {
tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_relation.keys(tag))),
m_stringtable->s(static_cast<int>(pbf_relation.vals(tag))));
}
}
m_buffer.commit();
}
}
int add_tags(const OSMPBF::DenseNodes& dense, int n, osmium::builder::NodeBuilder* builder) {
if (n >= dense.keys_vals_size()) {
return n;
}
if (dense.keys_vals(n) == 0) {
return n+1;
}
osmium::builder::TagListBuilder tl_builder(m_buffer, builder);
while (n < dense.keys_vals_size()) {
int tag_key_pos = dense.keys_vals(n++);
if (tag_key_pos == 0) {
break;
}
tl_builder.add_tag(m_stringtable->s(tag_key_pos),
m_stringtable->s(dense.keys_vals(n)));
++n;
}
return n;
}
void parse_dense_node_group(const OSMPBF::PrimitiveGroup& group) {
int64_t last_dense_id = 0;
int64_t last_dense_latitude = 0;
int64_t last_dense_longitude = 0;
int64_t last_dense_uid = 0;
int64_t last_dense_user_sid = 0;
int64_t last_dense_changeset = 0;
int64_t last_dense_timestamp = 0;
int last_dense_tag = 0;
const OSMPBF::DenseNodes& dense = group.dense();
for (int i = 0; i < dense.id_size(); ++i) {
bool visible = true;
last_dense_id += dense.id(i);
last_dense_latitude += dense.lat(i);
last_dense_longitude += dense.lon(i);
if (dense.has_denseinfo()) {
last_dense_changeset += dense.denseinfo().changeset(i);
last_dense_timestamp += dense.denseinfo().timestamp(i);
last_dense_uid += dense.denseinfo().uid(i);
last_dense_user_sid += dense.denseinfo().user_sid(i);
if (dense.denseinfo().visible_size() > 0) {
visible = dense.denseinfo().visible(i);
}
assert(last_dense_changeset >= 0);
assert(last_dense_timestamp >= 0);
assert(last_dense_uid >= -1);
assert(last_dense_user_sid >= 0);
}
osmium::builder::NodeBuilder builder(m_buffer);
osmium::Node& node = builder.object();
node.set_id(last_dense_id);
if (dense.has_denseinfo()) {
auto v = dense.denseinfo().version(i);
assert(v > 0);
node.set_version(static_cast<osmium::object_version_type>(v));
node.set_changeset(static_cast<osmium::changeset_id_type>(last_dense_changeset));
node.set_timestamp(last_dense_timestamp * m_date_factor);
node.set_uid_from_signed(static_cast<osmium::signed_user_id_type>(last_dense_uid));
node.set_visible(visible);
builder.add_user(m_stringtable->s(static_cast<int>(last_dense_user_sid)));
} else {
builder.add_user("", 1);
}
if (visible) {
builder.object().set_location(osmium::Location(
(last_dense_longitude * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision),
(last_dense_latitude * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision)));
}
last_dense_tag = add_tags(dense, last_dense_tag, &builder);
m_buffer.commit();
}
}
}; // class PBFPrimitiveBlockParser
/**
* PBF blobs can optionally be packed with the zlib algorithm.
* This function returns the raw data (if it was unpacked) or
* the unpacked data (if it was packed).
*
* @param input_data Reference to input data.
* @returns Unpacked data
* @throws osmium::pbf_error If there was a problem parsing the PBF
*/
inline std::unique_ptr<const std::string> unpack_blob(const std::string& input_data) {
OSMPBF::Blob pbf_blob;
if (!pbf_blob.ParseFromString(input_data)) {
throw osmium::pbf_error("failed to parse blob");
}
if (pbf_blob.has_raw()) {
return std::unique_ptr<std::string>(pbf_blob.release_raw());
} else if (pbf_blob.has_zlib_data()) {
auto raw_size = pbf_blob.raw_size();
assert(raw_size >= 0);
assert(raw_size <= OSMPBF::max_uncompressed_blob_size);
return osmium::io::detail::zlib_uncompress(pbf_blob.zlib_data(), static_cast<unsigned long>(raw_size));
} else if (pbf_blob.has_lzma_data()) {
throw osmium::pbf_error("lzma blobs not implemented");
} else {
throw osmium::pbf_error("blob contains no data");
}
}
/**
* Parse blob as a HeaderBlock.
*
* @param input_buffer Blob data
* @returns Header object
* @throws osmium::pbf_error If there was a parsing error
*/
inline osmium::io::Header parse_header_blob(const std::string& input_buffer) {
const std::unique_ptr<const std::string> data = unpack_blob(input_buffer);
OSMPBF::HeaderBlock pbf_header_block;
if (!pbf_header_block.ParseFromString(*data)) {
throw osmium::pbf_error("failed to parse HeaderBlock");
}
osmium::io::Header header;
for (int i = 0; i < pbf_header_block.required_features_size(); ++i) {
const std::string& feature = pbf_header_block.required_features(i);
if (feature == "OsmSchema-V0.6") continue;
if (feature == "DenseNodes") {
header.set("pbf_dense_nodes", true);
continue;
}
if (feature == "HistoricalInformation") {
header.set_has_multiple_object_versions(true);
continue;
}
throw osmium::pbf_error(std::string("required feature not supported: ") + feature);
}
for (int i = 0; i < pbf_header_block.optional_features_size(); ++i) {
const std::string& feature = pbf_header_block.optional_features(i);
header.set("pbf_optional_feature_" + std::to_string(i), feature);
}
if (pbf_header_block.has_writingprogram()) {
header.set("generator", pbf_header_block.writingprogram());
}
if (pbf_header_block.has_bbox()) {
const OSMPBF::HeaderBBox& pbf_bbox = pbf_header_block.bbox();
const int64_t resolution_convert = OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision;
osmium::Box box;
box.extend(osmium::Location(pbf_bbox.left() / resolution_convert, pbf_bbox.bottom() / resolution_convert));
box.extend(osmium::Location(pbf_bbox.right() / resolution_convert, pbf_bbox.top() / resolution_convert));
header.add_box(box);
}
if (pbf_header_block.has_osmosis_replication_timestamp()) {
header.set("osmosis_replication_timestamp", osmium::Timestamp(pbf_header_block.osmosis_replication_timestamp()).to_iso());
}
if (pbf_header_block.has_osmosis_replication_sequence_number()) {
header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.osmosis_replication_sequence_number()));
}
if (pbf_header_block.has_osmosis_replication_base_url()) {
header.set("osmosis_replication_base_url", pbf_header_block.osmosis_replication_base_url());
}
return header;
}
class DataBlobParser {
std::shared_ptr<std::string> m_input_buffer;
osmium::osm_entity_bits::type m_read_types;
public:
DataBlobParser(std::string&& input_buffer, osmium::osm_entity_bits::type read_types) :
m_input_buffer(std::make_shared<std::string>(std::move(input_buffer))),
m_read_types(read_types) {
if (input_buffer.size() > OSMPBF::max_uncompressed_blob_size) {
throw osmium::pbf_error(std::string("invalid blob size: " + std::to_string(input_buffer.size())));
}
}
/*
DataBlobParser(const DataBlobParser& other) :
m_input_buffer(std::move(other.m_input_buffer)),
m_read_types(other.m_read_types) {
}*/
DataBlobParser(const DataBlobParser&) = default;
DataBlobParser& operator=(const DataBlobParser&) = default;
DataBlobParser(DataBlobParser&&) = default;
DataBlobParser& operator=(DataBlobParser&&) = default;
~DataBlobParser() = default;
osmium::memory::Buffer operator()() {
const std::unique_ptr<const std::string> data = unpack_blob(*m_input_buffer);
PBFPrimitiveBlockParser parser(*data, m_read_types);
return parser();
}
}; // class DataBlobParser
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP
third_party/libosmium/include/osmium/io/detail/pbf_stringtable.hpp
Vendored
-218
View File
@@ -1,218 +0,0 @@
#ifndef OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP
#define OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <algorithm>
#include <cstdint>
#include <iterator>
#include <map>
#include <string>
#include <utility>
#include <vector>
#include <osmpbf/osmpbf.h>
#include <osmium/util/cast.hpp>
namespace osmium {
namespace io {
namespace detail {
/**
* StringTable management for PBF writer
*
* All strings are stored as indexes to rows in a StringTable. The StringTable contains
* one row for each used string, so strings that are used multiple times need to be
* stored only once. The StringTable is sorted by usage-count, so the most often used
* string is stored at index 1.
*/
class StringTable {
public:
/// type for string IDs (interim and final)
typedef uint16_t string_id_type;
private:
/**
* this is the struct used to build the StringTable. It is stored as
* the value-part in the strings-map.
*
* when a new string is added to the map, its count is set to 0 and
* the interim_id is set to the current size of the map. This interim_id
* is then stored into the pbf-objects.
*
* before the PrimitiveBlock is serialized, the map is sorted by count
* and stored into the pbf-StringTable. Afterwards the interim-ids are
* mapped to the "real" id in the StringTable.
*
* this way often used strings get lower ids in the StringTable. As the
* protobuf-serializer stores numbers in variable bit-lengths, lower
* IDs means less used space in the resulting file.
*/
struct string_info {
/// number of occurrences of this string
uint16_t count;
/// an intermediate-id
string_id_type interim_id;
}; // struct string_info
/**
* Interim StringTable, storing all strings that should be written to
* the StringTable once the block is written to disk.
*/
typedef std::map<std::string, string_info> string2string_info_type;
string2string_info_type m_strings;
/**
* This vector is used to map the interim IDs to real StringTable IDs after
* writing all strings to the StringTable.
*/
typedef std::vector<string_id_type> interim_id2id_type;
interim_id2id_type m_id2id_map;
size_t m_size = 0;
public:
StringTable() {
}
friend bool operator<(const string_info& lhs, const string_info& rhs) {
return lhs.count > rhs.count;
}
/**
* record a string in the interim StringTable if it's missing, otherwise just increase its counter,
* return the interim-id assigned to the string.
*/
string_id_type record_string(const std::string& string) {
string_info& info = m_strings[string];
if (info.interim_id == 0) {
++m_size;
info.interim_id = static_cast_with_assert<string_id_type>(m_size);
} else {
info.count++;
}
return info.interim_id;
}
/**
* Sort the interim StringTable and store it to the real protobuf StringTable.
* while storing to the real table, this function fills the id2id_map with
* pairs, mapping the interim-ids to final and real StringTable ids.
*
* Note that the m_strings table is a std::map and as such is sorted lexicographically.
* When the transformation into the sortedby multimap is done, it gets sorted by
* the count. The end result (at least with the glibc standard container/algorithm
* implementation) is that the string table is sorted first by reverse count (ie descending)
* and then by reverse lexicographic order.
*/
void store_stringtable(OSMPBF::StringTable* st, bool sort) {
// add empty StringTable entry at index 0
// StringTable index 0 is reserved as delimiter in the densenodes key/value list
// this line also ensures that there's always a valid StringTable
st->add_s("");
if (sort) {
std::multimap<string_info, std::string> sortedbycount;
m_id2id_map.resize(m_size+1);
std::transform(m_strings.begin(), m_strings.end(),
std::inserter(sortedbycount, sortedbycount.begin()),
[](const std::pair<std::string, string_info>& p) {
return std::pair<string_info, std::string>(p.second, p.first);
});
string_id_type n = 0;
for (const auto& mapping : sortedbycount) {
// add the string of the current item to the pbf StringTable
st->add_s(mapping.second);
// store the mapping from the interim-id to the real id
m_id2id_map[mapping.first.interim_id] = ++n;
}
} else {
std::vector<std::pair<string_id_type, const char*>> sortedbyid;
sortedbyid.reserve(m_strings.size());
for (const auto& p : m_strings) {
sortedbyid.emplace_back(p.second.interim_id, p.first.c_str());
}
std::sort(sortedbyid.begin(), sortedbyid.end());
for (const auto& mapping : sortedbyid) {
st->add_s(mapping.second);
}
}
}
/**
* Map from an interim ID to a real string ID.
*/
string_id_type map_string_id(const string_id_type interim_id) const {
return m_id2id_map[interim_id];
}
template <typename T>
string_id_type map_string_id(const T interim_id) const {
return map_string_id(static_cast_with_assert<string_id_type>(interim_id));
}
/**
* Clear the stringtable, preparing for the next block.
*/
void clear() {
m_strings.clear();
m_id2id_map.clear();
m_size = 0;
}
}; // class StringTable
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP
third_party/libosmium/include/osmium/io/detail/protobuf_tags.hpp
Vendored
+170
View File
@@ -0,0 +1,170 @@
#ifndef OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP
#define OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <protozero/pbf_types.hpp>
namespace osmium {
namespace io {
namespace detail {
// directly translated from
// https://github.com/scrosby/OSM-binary/blob/master/src/fileformat.proto
namespace FileFormat {
enum class Blob : protozero::pbf_tag_type {
optional_bytes_raw = 1,
optional_int32_raw_size = 2,
optional_bytes_zlib_data = 3,
optional_bytes_lzma_data = 4
};
enum class BlobHeader : protozero::pbf_tag_type {
required_string_type = 1,
optional_bytes_indexdata = 2,
required_int32_datasize = 3
};
} // namespace FileFormat
// directly translated from
// https://github.com/scrosby/OSM-binary/blob/master/src/osmformat.proto
namespace OSMFormat {
enum class HeaderBlock : protozero::pbf_tag_type {
optional_HeaderBBox_bbox = 1,
repeated_string_required_features = 4,
repeated_string_optional_features = 5,
optional_string_writingprogram = 16,
optional_string_source = 17,
optional_int64_osmosis_replication_timestamp = 32,
optional_int64_osmosis_replication_sequence_number = 33,
optional_string_osmosis_replication_base_url = 34
};
enum class HeaderBBox : protozero::pbf_tag_type {
required_sint64_left = 1,
required_sint64_right = 2,
required_sint64_top = 3,
required_sint64_bottom = 4
};
enum class PrimitiveBlock : protozero::pbf_tag_type {
required_StringTable_stringtable = 1,
repeated_PrimitiveGroup_primitivegroup = 2,
optional_int32_granularity = 17,
optional_int32_date_granularity = 18,
optional_int64_lat_offset = 19,
optional_int64_lon_offset = 20
};
enum class PrimitiveGroup : protozero::pbf_tag_type {
unknown = 0,
repeated_Node_nodes = 1,
optional_DenseNodes_dense = 2,
repeated_Way_ways = 3,
repeated_Relation_relations = 4,
repeated_ChangeSet_changesets = 5
};
enum class StringTable : protozero::pbf_tag_type {
repeated_bytes_s = 1
};
enum class Info : protozero::pbf_tag_type {
optional_int32_version = 1,
optional_int64_timestamp = 2,
optional_int64_changeset = 3,
optional_int32_uid = 4,
optional_uint32_user_sid = 5,
optional_bool_visible = 6
};
enum class DenseInfo : protozero::pbf_tag_type {
packed_int32_version = 1,
packed_sint64_timestamp = 2,
packed_sint64_changeset = 3,
packed_sint32_uid = 4,
packed_sint32_user_sid = 5,
packed_bool_visible = 6
};
enum class Node : protozero::pbf_tag_type {
required_sint64_id = 1,
packed_uint32_keys = 2,
packed_uint32_vals = 3,
optional_Info_info = 4,
required_sint64_lat = 8,
required_sint64_lon = 9
};
enum class DenseNodes : protozero::pbf_tag_type {
packed_sint64_id = 1,
optional_DenseInfo_denseinfo = 5,
packed_sint64_lat = 8,
packed_sint64_lon = 9,
packed_int32_keys_vals = 10
};
enum class Way : protozero::pbf_tag_type {
required_int64_id = 1,
packed_uint32_keys = 2,
packed_uint32_vals = 3,
optional_Info_info = 4,
packed_sint64_refs = 8
};
enum class Relation : protozero::pbf_tag_type {
required_int64_id = 1,
packed_uint32_keys = 2,
packed_uint32_vals = 3,
optional_Info_info = 4,
packed_int32_roles_sid = 8,
packed_sint64_memids = 9,
packed_MemberType_types = 10
};
} // namespace OSMFormat
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP
third_party/libosmium/include/osmium/io/detail/read_write.hpp
Vendored
+1 -1
View File
@@ -122,7 +122,7 @@ namespace osmium {
* @throws std::system_error On error.
*/
inline void reliable_write(const int fd, const unsigned char* output_buffer, const size_t size) {
constexpr size_t max_write = 100 * 1024 * 1024; // Max 100 MByte per write
constexpr size_t max_write = 100L * 1024L * 1024L; // Max 100 MByte per write
size_t offset = 0;
do {
auto write_count = size - offset;
third_party/libosmium/include/osmium/io/detail/string_table.hpp
Vendored
+250
View File
@@ -0,0 +1,250 @@
#ifndef OSMIUM_IO_DETAIL_STRING_TABLE_HPP
#define OSMIUM_IO_DETAIL_STRING_TABLE_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cassert>
#include <cstdlib>
#include <cstring>
#include <iterator>
#include <list>
#include <map>
#include <string>
namespace osmium {
namespace io {
namespace detail {
/**
* class StringStore
*
* Storage of lots of strings (const char *). Memory is allocated in chunks.
* If a string is added and there is no space in the current chunk, a new
* chunk will be allocated. Strings added to the store must not be larger
* than the chunk size.
*
* All memory is released when the destructor is called. There is no other way
* to release all or part of the memory.
*
*/
class StringStore {
size_t m_chunk_size;
std::list<std::string> m_chunks;
void add_chunk() {
m_chunks.push_front(std::string());
m_chunks.front().reserve(m_chunk_size);
}
public:
StringStore(size_t chunk_size) :
m_chunk_size(chunk_size),
m_chunks() {
add_chunk();
}
void clear() noexcept {
m_chunks.erase(std::next(m_chunks.begin()), m_chunks.end());
m_chunks.front().clear();
}
/**
* Add a null terminated string to the store. This will
* automatically get more memory if we are out.
* Returns a pointer to the copy of the string we have
* allocated.
*/
const char* add(const char* string) {
size_t len = std::strlen(string) + 1;
assert(len <= m_chunk_size);
size_t chunk_len = m_chunks.front().size();
if (chunk_len + len > m_chunks.front().capacity()) {
add_chunk();
chunk_len = 0;
}
m_chunks.front().append(string);
m_chunks.front().append(1, '\0');
return m_chunks.front().c_str() + chunk_len;
}
class const_iterator : public std::iterator<std::forward_iterator_tag, const char*> {
typedef std::list<std::string>::const_iterator it_type;
it_type m_it;
const it_type m_last;
const char* m_pos;
public:
const_iterator(it_type it, it_type last) :
m_it(it),
m_last(last),
m_pos(it == last ? nullptr : m_it->c_str()) {
}
const_iterator& operator++() {
assert(m_it != m_last);
auto last_pos = m_it->c_str() + m_it->size();
while (m_pos != last_pos && *m_pos) ++m_pos;
if (m_pos != last_pos) ++m_pos;
if (m_pos == last_pos) {
++m_it;
if (m_it != m_last) {
m_pos = m_it->c_str();
} else {
m_pos = nullptr;
}
}
return *this;
}
const_iterator operator++(int) {
const_iterator tmp(*this);
operator++();
return tmp;
}
bool operator==(const const_iterator& rhs) const {
return m_it == rhs.m_it && m_pos == rhs.m_pos;
}
bool operator!=(const const_iterator& rhs) const {
return !(*this == rhs);
}
const char* operator*() const {
assert(m_it != m_last);
assert(m_pos != nullptr);
return m_pos;
}
}; // class const_iterator
const_iterator begin() const {
if (m_chunks.front().empty()) {
return end();
}
return const_iterator(m_chunks.begin(), m_chunks.end());
}
const_iterator end() const {
return const_iterator(m_chunks.end(), m_chunks.end());
}
// These functions get you some idea how much memory was
// used.
int get_chunk_size() const noexcept {
return m_chunk_size;
}
int get_chunk_count() const noexcept {
return m_chunks.size();
}
int get_used_bytes_in_last_chunk() const noexcept {
return m_chunks.front().size();
}
}; // class StringStore
struct StrComp {
bool operator()(const char* lhs, const char* rhs) const {
return strcmp(lhs, rhs) < 0;
}
}; // struct StrComp
class StringTable {
StringStore m_strings;
std::map<const char*, size_t, StrComp> m_index;
size_t m_size;
public:
StringTable() :
m_strings(1024 * 1024),
m_index(),
m_size(0) {
m_strings.add("");
}
void clear() {
m_strings.clear();
m_index.clear();
m_size = 0;
m_strings.add("");
}
size_t size() const noexcept {
return m_size + 1;
}
size_t add(const char* s) {
auto f = m_index.find(s);
if (f != m_index.end()) {
return f->second;
}
const char* cs = m_strings.add(s);
m_index[cs] = ++m_size;
return m_size;
}
StringStore::const_iterator begin() const {
return m_strings.begin();
}
StringStore::const_iterator end() const {
return m_strings.end();
}
}; // class StringTable
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_STRING_TABLE_HPP
third_party/libosmium/include/osmium/io/detail/xml_input_format.hpp
Vendored
+29 -8
View File
@@ -66,6 +66,7 @@ DEALINGS IN THE SOFTWARE.
#include <osmium/osm/location.hpp>
#include <osmium/osm/object.hpp>
#include <osmium/osm/types.hpp>
#include <osmium/osm/types_from_string.hpp>
#include <osmium/thread/queue.hpp>
#include <osmium/thread/util.hpp>
#include <osmium/util/cast.hpp>
@@ -191,6 +192,8 @@ namespace osmium {
std::atomic<bool>& m_done;
bool m_header_is_done;
/**
* A C++ wrapper for the Expat parser that makes sure no memory is leaked.
*/
@@ -246,16 +249,25 @@ namespace osmium {
T& m_data;
std::promise<T>& m_promise;
bool m_done;
public:
PromiseKeeper(T& data, std::promise<T>& promise) :
m_data(data),
m_promise(promise) {
m_promise(promise),
m_done(false) {
}
void fullfill_promise() {
if (!m_done) {
m_promise.set_value(m_data);
m_done = true;
}
}
~PromiseKeeper() {
m_promise.set_value(m_data);
fullfill_promise();
}
}; // class PromiseKeeper
@@ -279,7 +291,8 @@ namespace osmium {
m_queue(queue),
m_header_promise(header_promise),
m_read_types(read_types),
m_done(done) {
m_done(done),
m_header_is_done(false) {
}
/**
@@ -305,7 +318,8 @@ namespace osmium {
m_queue(other.m_queue),
m_header_promise(other.m_header_promise),
m_read_types(other.m_read_types),
m_done(other.m_done) {
m_done(other.m_done),
m_header_is_done(other.m_header_is_done) {
}
XMLParser(XMLParser&&) = default;
@@ -326,6 +340,9 @@ namespace osmium {
last = data.empty();
try {
parser(data, last);
if (m_header_is_done) {
promise_keeper.fullfill_promise();
}
} catch (ParserIsDone&) {
return true;
} catch (...) {
@@ -343,8 +360,7 @@ namespace osmium {
private:
const char* init_object(osmium::OSMObject& object, const XML_Char** attrs) {
static const char* empty = "";
const char* user = empty;
const char* user = "";
if (m_in_delete_section) {
object.set_visible(false);
@@ -371,8 +387,7 @@ namespace osmium {
}
void init_changeset(osmium::builder::ChangesetBuilder* builder, const XML_Char** attrs) {
static const char* empty = "";
const char* user = empty;
const char* user = "";
osmium::Changeset& new_changeset = builder->object();
osmium::Location min;
@@ -421,6 +436,7 @@ namespace osmium {
}
void header_is_done() {
m_header_is_done = true;
if (m_read_types == osmium::osm_entity_bits::nothing) {
throw ParserIsDone();
}
@@ -722,10 +738,15 @@ namespace osmium {
namespace {
// we want the register_input_format() function to run, setting the variable
// is only a side-effect, it will never be used
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
const bool registered_xml_input = osmium::io::detail::InputFormatFactory::instance().register_input_format(osmium::io::file_format::xml,
[](const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue<std::string>& input_queue) {
return new osmium::io::detail::XMLInputFormat(file, read_which_entities, input_queue);
});
#pragma GCC diagnostic pop
} // anonymous namespace
third_party/libosmium/include/osmium/io/detail/xml_output_format.hpp
Vendored
+37 -23
View File
@@ -85,6 +85,9 @@ namespace osmium {
case '\'': out += "&apos;"; break;
case '<': out += "&lt;"; break;
case '>': out += "&gt;"; break;
case '\n': out += "&#xA;"; break;
case '\r': out += "&#xD;"; break;
case '\t': out += "&#x9;"; break;
default: out += *in; break;
}
}
@@ -126,6 +129,7 @@ namespace osmium {
operation m_last_op {operation::op_none};
const bool m_add_metadata;
const bool m_write_visible_flag;
const bool m_write_change_ops;
@@ -146,31 +150,33 @@ namespace osmium {
void write_meta(const osmium::OSMObject& object) {
oprintf(*m_out, " id=\"%" PRId64 "\"", object.id());
if (object.version()) {
oprintf(*m_out, " version=\"%d\"", object.version());
}
if (m_add_metadata) {
if (object.version()) {
oprintf(*m_out, " version=\"%d\"", object.version());
}
if (object.timestamp()) {
*m_out += " timestamp=\"";
*m_out += object.timestamp().to_iso();
*m_out += "\"";
}
if (object.timestamp()) {
*m_out += " timestamp=\"";
*m_out += object.timestamp().to_iso();
*m_out += "\"";
}
if (!object.user_is_anonymous()) {
oprintf(*m_out, " uid=\"%d\" user=\"", object.uid());
xml_string(*m_out, object.user());
*m_out += "\"";
}
if (!object.user_is_anonymous()) {
oprintf(*m_out, " uid=\"%d\" user=\"", object.uid());
xml_string(*m_out, object.user());
*m_out += "\"";
}
if (object.changeset()) {
oprintf(*m_out, " changeset=\"%d\"", object.changeset());
}
if (object.changeset()) {
oprintf(*m_out, " changeset=\"%d\"", object.changeset());
}
if (m_write_visible_flag) {
if (object.visible()) {
*m_out += " visible=\"true\"";
} else {
*m_out += " visible=\"false\"";
if (m_write_visible_flag) {
if (object.visible()) {
*m_out += " visible=\"true\"";
} else {
*m_out += " visible=\"false\"";
}
}
}
}
@@ -224,9 +230,10 @@ namespace osmium {
public:
explicit XMLOutputBlock(osmium::memory::Buffer&& buffer, bool write_visible_flag, bool write_change_ops) :
explicit XMLOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata, bool write_visible_flag, bool write_change_ops) :
m_input_buffer(std::make_shared<osmium::memory::Buffer>(std::move(buffer))),
m_out(std::make_shared<std::string>()),
m_add_metadata(add_metadata),
m_write_visible_flag(write_visible_flag && !write_change_ops),
m_write_change_ops(write_change_ops) {
}
@@ -392,12 +399,14 @@ namespace osmium {
class XMLOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler {
bool m_add_metadata;
bool m_write_visible_flag;
public:
XMLOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) :
OutputFormat(file, output_queue),
m_add_metadata(file.get("add_metadata") != "false"),
m_write_visible_flag(file.has_multiple_object_versions() || m_file.is_true("force_visible_flag")) {
}
@@ -408,7 +417,7 @@ namespace osmium {
}
void write_buffer(osmium::memory::Buffer&& buffer) override final {
m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_write_visible_flag, m_file.is_true("xml_change_format")}));
m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_add_metadata, m_write_visible_flag, m_file.is_true("xml_change_format")}));
}
void write_header(const osmium::io::Header& header) override final {
@@ -468,10 +477,15 @@ namespace osmium {
namespace {
// we want the register_output_format() function to run, setting the variable
// is only a side-effect, it will never be used
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
const bool registered_xml_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::xml,
[](const osmium::io::File& file, data_queue_type& output_queue) {
return new osmium::io::detail::XMLOutputFormat(file, output_queue);
});
#pragma GCC diagnostic pop
} // anonymous namespace
third_party/libosmium/include/osmium/io/detail/zlib.hpp
Vendored
+8 -7
View File
@@ -85,23 +85,24 @@ namespace osmium {
*
* @param input Compressed input data.
* @param raw_size Size of uncompressed data.
* @returns Uncompressed data.
* @param output Uncompressed result data.
* @returns Pointer and size to incompressed data.
*/
inline std::unique_ptr<std::string> zlib_uncompress(const std::string& input, unsigned long raw_size) {
auto output = std::unique_ptr<std::string>(new std::string(raw_size, '\0'));
inline std::pair<const char*, size_t> zlib_uncompress_string(const char* input, unsigned long input_size, unsigned long raw_size, std::string& output) {
output.resize(raw_size);
auto result = ::uncompress(
reinterpret_cast<unsigned char*>(const_cast<char *>(output->data())),
reinterpret_cast<unsigned char*>(&*output.begin()),
&raw_size,
reinterpret_cast<const unsigned char*>(input.data()),
osmium::static_cast_with_assert<unsigned long>(input.size())
reinterpret_cast<const unsigned char*>(input),
input_size
);
if (result != Z_OK) {
throw std::runtime_error(std::string("failed to uncompress data: ") + zError(result));
}
return output;
return std::make_pair(output.data(), output.size());
}
} // namespace detail
third_party/libosmium/include/osmium/io/file.hpp
Vendored
+25 -43
View File
@@ -97,7 +97,9 @@ namespace osmium {
* of the file will be taken from the suffix.
* An empty filename or "-" means stdin or stdout.
* @param format File format as string. See the description of the
* parse_format() function for details.
* parse_format() function for details. If this is
* empty the format will be deduced from the suffix
* of the filename.
*/
explicit File(const std::string& filename = "", const std::string& format = "") :
Options(),
@@ -107,20 +109,19 @@ namespace osmium {
m_format_string(format) {
// stdin/stdout
if (filename == "" || filename == "-") {
if (m_filename == "-") {
m_filename = "";
default_settings_for_stdinout();
}
// filename is actually a URL
// if filename is a URL, default to XML format
std::string protocol = m_filename.substr(0, m_filename.find_first_of(':'));
if (protocol == "http" || protocol == "https") {
default_settings_for_url();
m_file_format = file_format::xml;
}
detect_format_from_suffix(m_filename);
if (format != "") {
if (format.empty()) {
detect_format_from_suffix(m_filename);
} else {
parse_format(format);
}
}
@@ -140,9 +141,6 @@ namespace osmium {
m_buffer(buffer),
m_buffer_size(size),
m_format_string(format) {
default_settings_for_stdinout();
if (format != "") {
parse_format(format);
}
@@ -220,6 +218,20 @@ namespace osmium {
} else if (suffixes.back() == "opl") {
m_file_format = file_format::opl;
suffixes.pop_back();
} else if (suffixes.back() == "json") {
m_file_format = file_format::json;
suffixes.pop_back();
} else if (suffixes.back() == "o5m") {
m_file_format = file_format::o5m;
suffixes.pop_back();
} else if (suffixes.back() == "o5c") {
m_file_format = file_format::o5m;
m_has_multiple_object_versions = true;
set("o5c_change_format", true);
suffixes.pop_back();
} else if (suffixes.back() == "debug") {
m_file_format = file_format::debug;
suffixes.pop_back();
}
if (suffixes.empty()) return;
@@ -240,8 +252,8 @@ namespace osmium {
}
/**
* Check file format etc. for consistency and throw exception if there
* is a problem.
* Check file format etc. for consistency and throw exception if
* there is a problem.
*
* @throws std::runtime_error
*/
@@ -265,36 +277,6 @@ namespace osmium {
}
}
/**
* Set default settings for type and encoding when the filename is
* empty or "-". If you want to have a different default setting
* override this in a subclass.
*/
void default_settings_for_stdinout() {
m_file_format = file_format::unknown;
m_file_compression = file_compression::none;
}
/**
* Set default settings for type and encoding when the filename is
* a normal file. If you want to have a different default setting
* override this in a subclass.
*/
void default_settings_for_file() {
m_file_format = file_format::unknown;
m_file_compression = file_compression::none;
}
/**
* Set default settings for type and encoding when the filename is a URL.
* If you want to have a different default setting override this in a
* subclass.
*/
void default_settings_for_url() {
m_file_format = file_format::xml;
m_file_compression = file_compression::none;
}
file_format format() const noexcept {
return m_file_format;
}
third_party/libosmium/include/osmium/io/file_format.hpp
Vendored
+7 -1
View File
@@ -44,7 +44,9 @@ namespace osmium {
xml = 1,
pbf = 2,
opl = 3,
json = 4
json = 4,
o5m = 5,
debug = 6
};
// avoid g++ false positive
@@ -62,6 +64,10 @@ namespace osmium {
return "OPL";
case file_format::json:
return "JSON";
case file_format::o5m:
return "O5M";
case file_format::debug:
return "DEBUG";
}
}
#pragma GCC diagnostic pop
third_party/libosmium/include/osmium/io/gzip_compression.hpp
Vendored
+5
View File
@@ -231,11 +231,16 @@ namespace osmium {
namespace {
// we want the register_compression() function to run, setting the variable
// is only a side-effect, it will never be used
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
const bool registered_gzip_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::gzip,
[](int fd) { return new osmium::io::GzipCompressor(fd); },
[](int fd) { return new osmium::io::GzipDecompressor(fd); },
[](const char* buffer, size_t size) { return new osmium::io::GzipBufferDecompressor(buffer, size); }
);
#pragma GCC diagnostic pop
} // anonymous namespace
third_party/libosmium/include/osmium/io/pbf_input.hpp
Vendored
-1
View File
@@ -39,7 +39,6 @@ DEALINGS IN THE SOFTWARE.
* Include this file if you want to read OSM PBF files.
*
* @attention If you include this file, you'll need to link with
* `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only),
* `libz`, and enable multithreading.
*/
third_party/libosmium/include/osmium/io/pbf_output.hpp
Vendored
-1
View File
@@ -39,7 +39,6 @@ DEALINGS IN THE SOFTWARE.
* Include this file if you want to write OSM PBF files.
*
* @attention If you include this file, you'll need to link with
* `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only),
* `libz`, and enable multithreading.
*/