905 lines
45 KiB
C++
905 lines
45 KiB
C++
|
#ifndef OSMIUM_IO_DETAIL_PBF_DECODER_HPP
|
||
|
|
#define OSMIUM_IO_DETAIL_PBF_DECODER_HPP
|
||
|
|
|
||
|
|
/*
|
||
|
|
|
||
|
|
This file is part of Osmium (http://osmcode.org/libosmium).
|
||
|
|
|
||
|
|
Copyright 2013-2017 Jochen Topf <jochen@topf.org> and others (see README).
|
||
|
|
|
||
|
|
Boost Software License - Version 1.0 - August 17th, 2003
|
||
|
|
|
||
|
|
Permission is hereby granted, free of charge, to any person or organization
|
||
|
|
obtaining a copy of the software and accompanying documentation covered by
|
||
|
|
this license (the "Software") to use, reproduce, display, distribute,
|
||
|
|
execute, and transmit the Software, and to prepare derivative works of the
|
||
|
|
Software, and to permit third-parties to whom the Software is furnished to
|
||
|
|
do so, all subject to the following:
|
||
|
|
|
||
|
|
The copyright notices in the Software and this entire statement, including
|
||
|
|
the above license grant, this restriction and the following disclaimer,
|
||
|
|
must be included in all copies of the Software, in whole or in part, and
|
||
|
|
all derivative works of the Software, unless such copies or derivative
|
||
|
|
works are solely in the form of machine-executable object code generated by
|
||
|
|
a source language processor.
|
||
|
|
|
||
|
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||
|
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||
|
|
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
||
|
|
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
||
|
|
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
||
|
|
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||
|
|
DEALINGS IN THE SOFTWARE.
|
||
|
|
|
||
|
|
*/
|
||
|
|
|
||
|
|
#include <cstdint>
|
||
|
|
#include <cstring>
|
||
|
|
#include <limits>
|
||
|
|
#include <memory>
|
||
|
|
#include <stdexcept>
|
||
|
|
#include <string>
|
||
|
|
#include <utility>
|
||
|
|
#include <vector>
|
||
|
|
|
||
|
|
#include <protozero/iterators.hpp>
|
||
|
|
#include <protozero/pbf_message.hpp>
|
||
|
|
#include <protozero/types.hpp>
|
||
|
|
|
||
|
|
#include <osmium/builder/osm_object_builder.hpp>
|
||
|
|
#include <osmium/io/detail/pbf.hpp> // IWYU pragma: export
|
||
|
|
#include <osmium/io/detail/protobuf_tags.hpp>
|
||
|
|
#include <osmium/io/detail/zlib.hpp>
|
||
|
|
#include <osmium/io/file_format.hpp>
|
||
|
|
#include <osmium/io/header.hpp>
|
||
|
|
#include <osmium/memory/buffer.hpp>
|
||
|
|
#include <osmium/osm/box.hpp>
|
||
|
|
#include <osmium/osm/entity_bits.hpp>
|
||
|
|
#include <osmium/osm/item_type.hpp>
|
||
|
|
#include <osmium/osm/location.hpp>
|
||
|
|
#include <osmium/osm/node.hpp>
|
||
|
|
#include <osmium/osm/object.hpp>
|
||
|
|
#include <osmium/osm/relation.hpp>
|
||
|
|
#include <osmium/osm/timestamp.hpp>
|
||
|
|
#include <osmium/osm/types.hpp>
|
||
|
|
#include <osmium/osm/way.hpp>
|
||
|
|
#include <osmium/util/cast.hpp>
|
||
|
|
#include <osmium/util/delta.hpp>
|
||
|
|
|
||
|
|
namespace osmium {
|
||
|
|
|
||
|
|
namespace builder {
|
||
|
|
class Builder;
|
||
|
|
} // namespace builder
|
||
|
|
|
||
|
|
namespace io {
|
||
|
|
|
||
|
|
namespace detail {
|
||
|
|
|
||
|
|
using protozero::data_view;
|
||
|
|
using osm_string_len_type = std::pair<const char*, osmium::string_size_type>;
|
||
|
|
|
||
|
|
class PBFPrimitiveBlockDecoder {
|
||
|
|
|
||
|
|
static constexpr const size_t initial_buffer_size = 2 * 1024 * 1024;
|
||
|
|
|
||
|
|
data_view m_data;
|
||
|
|
std::vector<osm_string_len_type> m_stringtable;
|
||
|
|
|
||
|
|
int64_t m_lon_offset = 0;
|
||
|
|
int64_t m_lat_offset = 0;
|
||
|
|
int64_t m_date_factor = 1000;
|
||
|
|
int32_t m_granularity = 100;
|
||
|
|
|
||
|
|
osmium::osm_entity_bits::type m_read_types;
|
||
|
|
|
||
|
|
osmium::memory::Buffer m_buffer { initial_buffer_size };
|
||
|
|
|
||
|
|
osmium::io::read_meta m_read_metadata;
|
||
|
|
|
||
|
|
void decode_stringtable(const data_view& data) {
|
||
|
|
if (!m_stringtable.empty()) {
|
||
|
|
throw osmium::pbf_error{"more than one stringtable in pbf file"};
|
||
|
|
}
|
||
|
|
|
||
|
|
protozero::pbf_message<OSMFormat::StringTable> pbf_string_table{data};
|
||
|
|
while (pbf_string_table.next(OSMFormat::StringTable::repeated_bytes_s)) {
|
||
|
|
const auto str_view = pbf_string_table.get_view();
|
||
|
|
if (str_view.size() > osmium::max_osm_string_length) {
|
||
|
|
throw osmium::pbf_error{"overlong string in string table"};
|
||
|
|
}
|
||
|
|
m_stringtable.emplace_back(str_view.data(), osmium::string_size_type(str_view.size()));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
void decode_primitive_block_metadata() {
|
||
|
|
protozero::pbf_message<OSMFormat::PrimitiveBlock> pbf_primitive_block{m_data};
|
||
|
|
while (pbf_primitive_block.next()) {
|
||
|
|
switch (pbf_primitive_block.tag()) {
|
||
|
|
case OSMFormat::PrimitiveBlock::required_StringTable_stringtable:
|
||
|
|
decode_stringtable(pbf_primitive_block.get_view());
|
||
|
|
break;
|
||
|
|
case OSMFormat::PrimitiveBlock::optional_int32_granularity:
|
||
|
|
m_granularity = pbf_primitive_block.get_int32();
|
||
|
|
break;
|
||
|
|
case OSMFormat::PrimitiveBlock::optional_int32_date_granularity:
|
||
|
|
m_date_factor = pbf_primitive_block.get_int32();
|
||
|
|
break;
|
||
|
|
case OSMFormat::PrimitiveBlock::optional_int64_lat_offset:
|
||
|
|
m_lat_offset = pbf_primitive_block.get_int64();
|
||
|
|
break;
|
||
|
|
case OSMFormat::PrimitiveBlock::optional_int64_lon_offset:
|
||
|
|
m_lon_offset = pbf_primitive_block.get_int64();
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
pbf_primitive_block.skip();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
void decode_primitive_block_data() {
|
||
|
|
protozero::pbf_message<OSMFormat::PrimitiveBlock> pbf_primitive_block{m_data};
|
||
|
|
while (pbf_primitive_block.next(OSMFormat::PrimitiveBlock::repeated_PrimitiveGroup_primitivegroup)) {
|
||
|
|
protozero::pbf_message<OSMFormat::PrimitiveGroup> pbf_primitive_group = pbf_primitive_block.get_message();
|
||
|
|
while (pbf_primitive_group.next()) {
|
||
|
|
switch (pbf_primitive_group.tag()) {
|
||
|
|
case OSMFormat::PrimitiveGroup::repeated_Node_nodes:
|
||
|
|
if (m_read_types & osmium::osm_entity_bits::node) {
|
||
|
|
decode_node(pbf_primitive_group.get_view());
|
||
|
|
m_buffer.commit();
|
||
|
|
} else {
|
||
|
|
pbf_primitive_group.skip();
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
case OSMFormat::PrimitiveGroup::optional_DenseNodes_dense:
|
||
|
|
if (m_read_types & osmium::osm_entity_bits::node) {
|
||
|
|
if (m_read_metadata == osmium::io::read_meta::yes) {
|
||
|
|
decode_dense_nodes(pbf_primitive_group.get_view());
|
||
|
|
} else {
|
||
|
|
decode_dense_nodes_without_metadata(pbf_primitive_group.get_view());
|
||
|
|
}
|
||
|
|
m_buffer.commit();
|
||
|
|
} else {
|
||
|
|
pbf_primitive_group.skip();
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
case OSMFormat::PrimitiveGroup::repeated_Way_ways:
|
||
|
|
if (m_read_types & osmium::osm_entity_bits::way) {
|
||
|
|
decode_way(pbf_primitive_group.get_view());
|
||
|
|
m_buffer.commit();
|
||
|
|
} else {
|
||
|
|
pbf_primitive_group.skip();
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
case OSMFormat::PrimitiveGroup::repeated_Relation_relations:
|
||
|
|
if (m_read_types & osmium::osm_entity_bits::relation) {
|
||
|
|
decode_relation(pbf_primitive_group.get_view());
|
||
|
|
m_buffer.commit();
|
||
|
|
} else {
|
||
|
|
pbf_primitive_group.skip();
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
pbf_primitive_group.skip();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
osm_string_len_type decode_info(const data_view& data, osmium::OSMObject& object) {
|
||
|
|
osm_string_len_type user{"", 0};
|
||
|
|
|
||
|
|
protozero::pbf_message<OSMFormat::Info> pbf_info{data};
|
||
|
|
while (pbf_info.next()) {
|
||
|
|
switch (pbf_info.tag()) {
|
||
|
|
case OSMFormat::Info::optional_int32_version:
|
||
|
|
{
|
||
|
|
const auto version = pbf_info.get_int32();
|
||
|
|
if (version < 0) {
|
||
|
|
throw osmium::pbf_error{"object version must not be negative"};
|
||
|
|
}
|
||
|
|
object.set_version(static_cast_with_assert<object_version_type>(version));
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
case OSMFormat::Info::optional_int64_timestamp:
|
||
|
|
object.set_timestamp(pbf_info.get_int64() * m_date_factor / 1000);
|
||
|
|
break;
|
||
|
|
case OSMFormat::Info::optional_int64_changeset:
|
||
|
|
{
|
||
|
|
const auto changeset_id = pbf_info.get_int64();
|
||
|
|
if (changeset_id < 0) {
|
||
|
|
throw osmium::pbf_error{"object changeset_id must not be negative"};
|
||
|
|
}
|
||
|
|
object.set_changeset(static_cast_with_assert<changeset_id_type>(changeset_id));
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
case OSMFormat::Info::optional_int32_uid:
|
||
|
|
object.set_uid_from_signed(pbf_info.get_int32());
|
||
|
|
break;
|
||
|
|
case OSMFormat::Info::optional_uint32_user_sid:
|
||
|
|
user = m_stringtable.at(pbf_info.get_uint32());
|
||
|
|
break;
|
||
|
|
case OSMFormat::Info::optional_bool_visible:
|
||
|
|
object.set_visible(pbf_info.get_bool());
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
pbf_info.skip();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return user;
|
||
|
|
}
|
||
|
|
|
||
|
|
using kv_type = protozero::iterator_range<protozero::pbf_reader::const_uint32_iterator>;
|
||
|
|
|
||
|
|
void build_tag_list(osmium::builder::Builder& parent, const kv_type& keys, const kv_type& vals) {
|
||
|
|
if (!keys.empty()) {
|
||
|
|
osmium::builder::TagListBuilder builder{parent};
|
||
|
|
auto kit = keys.begin();
|
||
|
|
auto vit = vals.begin();
|
||
|
|
while (kit != keys.end()) {
|
||
|
|
if (vit == vals.end()) {
|
||
|
|
// this is against the spec, must have same number of elements
|
||
|
|
throw osmium::pbf_error{"PBF format error"};
|
||
|
|
}
|
||
|
|
const auto& k = m_stringtable.at(*kit++);
|
||
|
|
const auto& v = m_stringtable.at(*vit++);
|
||
|
|
builder.add_tag(k.first, k.second, v.first, v.second);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
int32_t convert_pbf_coordinate(int64_t c) const {
|
||
|
|
return int32_t((c * m_granularity + m_lon_offset) / resolution_convert);
|
||
|
|
}
|
||
|
|
|
||
|
|
void decode_node(const data_view& data) {
|
||
|
|
osmium::builder::NodeBuilder builder{m_buffer};
|
||
|
|
osmium::Node& node = builder.object();
|
||
|
|
|
||
|
|
kv_type keys;
|
||
|
|
kv_type vals;
|
||
|
|
int64_t lon = std::numeric_limits<int64_t>::max();
|
||
|
|
int64_t lat = std::numeric_limits<int64_t>::max();
|
||
|
|
|
||
|
|
osm_string_len_type user{"", 0};
|
||
|
|
|
||
|
|
protozero::pbf_message<OSMFormat::Node> pbf_node{data};
|
||
|
|
while (pbf_node.next()) {
|
||
|
|
switch (pbf_node.tag()) {
|
||
|
|
case OSMFormat::Node::required_sint64_id:
|
||
|
|
node.set_id(pbf_node.get_sint64());
|
||
|
|
break;
|
||
|
|
case OSMFormat::Node::packed_uint32_keys:
|
||
|
|
keys = pbf_node.get_packed_uint32();
|
||
|
|
break;
|
||
|
|
case OSMFormat::Node::packed_uint32_vals:
|
||
|
|
vals = pbf_node.get_packed_uint32();
|
||
|
|
break;
|
||
|
|
case OSMFormat::Node::optional_Info_info:
|
||
|
|
if (m_read_metadata == osmium::io::read_meta::yes) {
|
||
|
|
user = decode_info(pbf_node.get_view(), builder.object());
|
||
|
|
} else {
|
||
|
|
pbf_node.skip();
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
case OSMFormat::Node::required_sint64_lat:
|
||
|
|
lat = pbf_node.get_sint64();
|
||
|
|
break;
|
||
|
|
case OSMFormat::Node::required_sint64_lon:
|
||
|
|
lon = pbf_node.get_sint64();
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
pbf_node.skip();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (node.visible()) {
|
||
|
|
if (lon == std::numeric_limits<int64_t>::max() ||
|
||
|
|
lat == std::numeric_limits<int64_t>::max()) {
|
||
|
|
throw osmium::pbf_error{"illegal coordinate format"};
|
||
|
|
}
|
||
|
|
node.set_location(osmium::Location{
|
||
|
|
convert_pbf_coordinate(lon),
|
||
|
|
convert_pbf_coordinate(lat)
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
builder.set_user(user.first, user.second);
|
||
|
|
|
||
|
|
build_tag_list(builder, keys, vals);
|
||
|
|
}
|
||
|
|
|
||
|
|
void decode_way(const data_view& data) {
|
||
|
|
osmium::builder::WayBuilder builder{m_buffer};
|
||
|
|
|
||
|
|
kv_type keys;
|
||
|
|
kv_type vals;
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_sint64_iterator> refs;
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_sint64_iterator> lats;
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_sint64_iterator> lons;
|
||
|
|
|
||
|
|
osm_string_len_type user{"", 0};
|
||
|
|
|
||
|
|
protozero::pbf_message<OSMFormat::Way> pbf_way{data};
|
||
|
|
while (pbf_way.next()) {
|
||
|
|
switch (pbf_way.tag()) {
|
||
|
|
case OSMFormat::Way::required_int64_id:
|
||
|
|
builder.object().set_id(pbf_way.get_int64());
|
||
|
|
break;
|
||
|
|
case OSMFormat::Way::packed_uint32_keys:
|
||
|
|
keys = pbf_way.get_packed_uint32();
|
||
|
|
break;
|
||
|
|
case OSMFormat::Way::packed_uint32_vals:
|
||
|
|
vals = pbf_way.get_packed_uint32();
|
||
|
|
break;
|
||
|
|
case OSMFormat::Way::optional_Info_info:
|
||
|
|
if (m_read_metadata == osmium::io::read_meta::yes) {
|
||
|
|
user = decode_info(pbf_way.get_view(), builder.object());
|
||
|
|
} else {
|
||
|
|
pbf_way.skip();
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
case OSMFormat::Way::packed_sint64_refs:
|
||
|
|
refs = pbf_way.get_packed_sint64();
|
||
|
|
break;
|
||
|
|
case OSMFormat::Way::packed_sint64_lat:
|
||
|
|
lats = pbf_way.get_packed_sint64();
|
||
|
|
break;
|
||
|
|
case OSMFormat::Way::packed_sint64_lon:
|
||
|
|
lons = pbf_way.get_packed_sint64();
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
pbf_way.skip();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
builder.set_user(user.first, user.second);
|
||
|
|
|
||
|
|
if (!refs.empty()) {
|
||
|
|
osmium::builder::WayNodeListBuilder wnl_builder{builder};
|
||
|
|
osmium::util::DeltaDecode<int64_t> ref;
|
||
|
|
if (lats.empty()) {
|
||
|
|
for (const auto& ref_value : refs) {
|
||
|
|
wnl_builder.add_node_ref(ref.update(ref_value));
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
osmium::util::DeltaDecode<int64_t> lon;
|
||
|
|
osmium::util::DeltaDecode<int64_t> lat;
|
||
|
|
while (!refs.empty() && !lons.empty() && !lats.empty()) {
|
||
|
|
wnl_builder.add_node_ref(
|
||
|
|
ref.update(refs.front()),
|
||
|
|
osmium::Location{convert_pbf_coordinate(lon.update(lons.front())),
|
||
|
|
convert_pbf_coordinate(lat.update(lats.front()))}
|
||
|
|
);
|
||
|
|
refs.drop_front();
|
||
|
|
lons.drop_front();
|
||
|
|
lats.drop_front();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
build_tag_list(builder, keys, vals);
|
||
|
|
}
|
||
|
|
|
||
|
|
void decode_relation(const data_view& data) {
|
||
|
|
osmium::builder::RelationBuilder builder{m_buffer};
|
||
|
|
|
||
|
|
kv_type keys;
|
||
|
|
kv_type vals;
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_int32_iterator> roles;
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_sint64_iterator> refs;
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_int32_iterator> types;
|
||
|
|
|
||
|
|
osm_string_len_type user{"", 0};
|
||
|
|
|
||
|
|
protozero::pbf_message<OSMFormat::Relation> pbf_relation{data};
|
||
|
|
while (pbf_relation.next()) {
|
||
|
|
switch (pbf_relation.tag()) {
|
||
|
|
case OSMFormat::Relation::required_int64_id:
|
||
|
|
builder.object().set_id(pbf_relation.get_int64());
|
||
|
|
break;
|
||
|
|
case OSMFormat::Relation::packed_uint32_keys:
|
||
|
|
keys = pbf_relation.get_packed_uint32();
|
||
|
|
break;
|
||
|
|
case OSMFormat::Relation::packed_uint32_vals:
|
||
|
|
vals = pbf_relation.get_packed_uint32();
|
||
|
|
break;
|
||
|
|
case OSMFormat::Relation::optional_Info_info:
|
||
|
|
if (m_read_metadata == osmium::io::read_meta::yes) {
|
||
|
|
user = decode_info(pbf_relation.get_view(), builder.object());
|
||
|
|
} else {
|
||
|
|
pbf_relation.skip();
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
case OSMFormat::Relation::packed_int32_roles_sid:
|
||
|
|
roles = pbf_relation.get_packed_int32();
|
||
|
|
break;
|
||
|
|
case OSMFormat::Relation::packed_sint64_memids:
|
||
|
|
refs = pbf_relation.get_packed_sint64();
|
||
|
|
break;
|
||
|
|
case OSMFormat::Relation::packed_MemberType_types:
|
||
|
|
types = pbf_relation.get_packed_enum();
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
pbf_relation.skip();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
builder.set_user(user.first, user.second);
|
||
|
|
|
||
|
|
if (!refs.empty()) {
|
||
|
|
osmium::builder::RelationMemberListBuilder rml_builder{builder};
|
||
|
|
osmium::util::DeltaDecode<int64_t> ref;
|
||
|
|
while (!roles.empty() && !refs.empty() && !types.empty()) {
|
||
|
|
const auto& r = m_stringtable.at(roles.front());
|
||
|
|
const int type = types.front();
|
||
|
|
if (type < 0 || type > 2) {
|
||
|
|
throw osmium::pbf_error{"unknown relation member type"};
|
||
|
|
}
|
||
|
|
rml_builder.add_member(
|
||
|
|
osmium::item_type(type + 1),
|
||
|
|
ref.update(refs.front()),
|
||
|
|
r.first,
|
||
|
|
r.second
|
||
|
|
);
|
||
|
|
roles.drop_front();
|
||
|
|
refs.drop_front();
|
||
|
|
types.drop_front();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
build_tag_list(builder, keys, vals);
|
||
|
|
}
|
||
|
|
|
||
|
|
void build_tag_list_from_dense_nodes(osmium::builder::NodeBuilder& builder, protozero::pbf_reader::const_int32_iterator& it, protozero::pbf_reader::const_int32_iterator last) {
|
||
|
|
osmium::builder::TagListBuilder tl_builder{builder};
|
||
|
|
while (it != last && *it != 0) {
|
||
|
|
const auto& k = m_stringtable.at(*it++);
|
||
|
|
if (it == last) {
|
||
|
|
throw osmium::pbf_error{"PBF format error"}; // this is against the spec, keys/vals must come in pairs
|
||
|
|
}
|
||
|
|
const auto& v = m_stringtable.at(*it++);
|
||
|
|
tl_builder.add_tag(k.first, k.second, v.first, v.second);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (it != last) {
|
||
|
|
++it;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
void decode_dense_nodes_without_metadata(const data_view& data) {
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_sint64_iterator> ids;
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_sint64_iterator> lats;
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_sint64_iterator> lons;
|
||
|
|
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_int32_iterator> tags;
|
||
|
|
|
||
|
|
protozero::pbf_message<OSMFormat::DenseNodes> pbf_dense_nodes{data};
|
||
|
|
while (pbf_dense_nodes.next()) {
|
||
|
|
switch (pbf_dense_nodes.tag()) {
|
||
|
|
case OSMFormat::DenseNodes::packed_sint64_id:
|
||
|
|
ids = pbf_dense_nodes.get_packed_sint64();
|
||
|
|
break;
|
||
|
|
case OSMFormat::DenseNodes::packed_sint64_lat:
|
||
|
|
lats = pbf_dense_nodes.get_packed_sint64();
|
||
|
|
break;
|
||
|
|
case OSMFormat::DenseNodes::packed_sint64_lon:
|
||
|
|
lons = pbf_dense_nodes.get_packed_sint64();
|
||
|
|
break;
|
||
|
|
case OSMFormat::DenseNodes::packed_int32_keys_vals:
|
||
|
|
tags = pbf_dense_nodes.get_packed_int32();
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
pbf_dense_nodes.skip();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
osmium::util::DeltaDecode<int64_t> dense_id;
|
||
|
|
osmium::util::DeltaDecode<int64_t> dense_latitude;
|
||
|
|
osmium::util::DeltaDecode<int64_t> dense_longitude;
|
||
|
|
|
||
|
|
auto tag_it = tags.begin();
|
||
|
|
|
||
|
|
while (!ids.empty()) {
|
||
|
|
if (lons.empty() ||
|
||
|
|
lats.empty()) {
|
||
|
|
// this is against the spec, must have same number of elements
|
||
|
|
throw osmium::pbf_error{"PBF format error"};
|
||
|
|
}
|
||
|
|
|
||
|
|
osmium::builder::NodeBuilder builder{m_buffer};
|
||
|
|
osmium::Node& node = builder.object();
|
||
|
|
|
||
|
|
node.set_id(dense_id.update(ids.front()));
|
||
|
|
ids.drop_front();
|
||
|
|
|
||
|
|
const auto lon = dense_longitude.update(lons.front());
|
||
|
|
lons.drop_front();
|
||
|
|
const auto lat = dense_latitude.update(lats.front());
|
||
|
|
lats.drop_front();
|
||
|
|
builder.object().set_location(osmium::Location(
|
||
|
|
convert_pbf_coordinate(lon),
|
||
|
|
convert_pbf_coordinate(lat)
|
||
|
|
));
|
||
|
|
|
||
|
|
if (tag_it != tags.end()) {
|
||
|
|
build_tag_list_from_dense_nodes(builder, tag_it, tags.end());
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
void decode_dense_nodes(const data_view& data) {
|
||
|
|
bool has_info = false;
|
||
|
|
bool has_visibles = false;
|
||
|
|
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_sint64_iterator> ids;
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_sint64_iterator> lats;
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_sint64_iterator> lons;
|
||
|
|
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_int32_iterator> tags;
|
||
|
|
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_int32_iterator> versions;
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_sint64_iterator> timestamps;
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_sint64_iterator> changesets;
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_sint32_iterator> uids;
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_sint32_iterator> user_sids;
|
||
|
|
protozero::iterator_range<protozero::pbf_reader::const_int32_iterator> visibles;
|
||
|
|
|
||
|
|
protozero::pbf_message<OSMFormat::DenseNodes> pbf_dense_nodes{data};
|
||
|
|
while (pbf_dense_nodes.next()) {
|
||
|
|
switch (pbf_dense_nodes.tag()) {
|
||
|
|
case OSMFormat::DenseNodes::packed_sint64_id:
|
||
|
|
ids = pbf_dense_nodes.get_packed_sint64();
|
||
|
|
break;
|
||
|
|
case OSMFormat::DenseNodes::optional_DenseInfo_denseinfo:
|
||
|
|
{
|
||
|
|
has_info = true;
|
||
|
|
protozero::pbf_message<OSMFormat::DenseInfo> pbf_dense_info{pbf_dense_nodes.get_message()};
|
||
|
|
while (pbf_dense_info.next()) {
|
||
|
|
switch (pbf_dense_info.tag()) {
|
||
|
|
case OSMFormat::DenseInfo::packed_int32_version:
|
||
|
|
versions = pbf_dense_info.get_packed_int32();
|
||
|
|
break;
|
||
|
|
case OSMFormat::DenseInfo::packed_sint64_timestamp:
|
||
|
|
timestamps = pbf_dense_info.get_packed_sint64();
|
||
|
|
break;
|
||
|
|
case OSMFormat::DenseInfo::packed_sint64_changeset:
|
||
|
|
changesets = pbf_dense_info.get_packed_sint64();
|
||
|
|
break;
|
||
|
|
case OSMFormat::DenseInfo::packed_sint32_uid:
|
||
|
|
uids = pbf_dense_info.get_packed_sint32();
|
||
|
|
break;
|
||
|
|
case OSMFormat::DenseInfo::packed_sint32_user_sid:
|
||
|
|
user_sids = pbf_dense_info.get_packed_sint32();
|
||
|
|
break;
|
||
|
|
case OSMFormat::DenseInfo::packed_bool_visible:
|
||
|
|
has_visibles = true;
|
||
|
|
visibles = pbf_dense_info.get_packed_bool();
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
pbf_dense_info.skip();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
case OSMFormat::DenseNodes::packed_sint64_lat:
|
||
|
|
lats = pbf_dense_nodes.get_packed_sint64();
|
||
|
|
break;
|
||
|
|
case OSMFormat::DenseNodes::packed_sint64_lon:
|
||
|
|
lons = pbf_dense_nodes.get_packed_sint64();
|
||
|
|
break;
|
||
|
|
case OSMFormat::DenseNodes::packed_int32_keys_vals:
|
||
|
|
tags = pbf_dense_nodes.get_packed_int32();
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
pbf_dense_nodes.skip();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
osmium::util::DeltaDecode<int64_t> dense_id;
|
||
|
|
osmium::util::DeltaDecode<int64_t> dense_latitude;
|
||
|
|
osmium::util::DeltaDecode<int64_t> dense_longitude;
|
||
|
|
osmium::util::DeltaDecode<int64_t> dense_uid;
|
||
|
|
osmium::util::DeltaDecode<int64_t> dense_user_sid;
|
||
|
|
osmium::util::DeltaDecode<int64_t> dense_changeset;
|
||
|
|
osmium::util::DeltaDecode<int64_t> dense_timestamp;
|
||
|
|
|
||
|
|
auto tag_it = tags.begin();
|
||
|
|
|
||
|
|
while (!ids.empty()) {
|
||
|
|
if (lons.empty() ||
|
||
|
|
lats.empty()) {
|
||
|
|
// this is against the spec, must have same number of elements
|
||
|
|
throw osmium::pbf_error{"PBF format error"};
|
||
|
|
}
|
||
|
|
|
||
|
|
bool visible = true;
|
||
|
|
|
||
|
|
osmium::builder::NodeBuilder builder{m_buffer};
|
||
|
|
osmium::Node& node = builder.object();
|
||
|
|
|
||
|
|
node.set_id(dense_id.update(ids.front()));
|
||
|
|
ids.drop_front();
|
||
|
|
|
||
|
|
if (has_info) {
|
||
|
|
if (versions.empty() ||
|
||
|
|
changesets.empty() ||
|
||
|
|
timestamps.empty() ||
|
||
|
|
uids.empty() ||
|
||
|
|
user_sids.empty()) {
|
||
|
|
// this is against the spec, must have same number of elements
|
||
|
|
throw osmium::pbf_error{"PBF format error"};
|
||
|
|
}
|
||
|
|
|
||
|
|
const auto version = versions.front();
|
||
|
|
versions.drop_front();
|
||
|
|
if (version < 0) {
|
||
|
|
throw osmium::pbf_error{"object version must not be negative"};
|
||
|
|
}
|
||
|
|
node.set_version(static_cast<osmium::object_version_type>(version));
|
||
|
|
|
||
|
|
const auto changeset_id = dense_changeset.update(changesets.front());
|
||
|
|
changesets.drop_front();
|
||
|
|
if (changeset_id < 0) {
|
||
|
|
throw osmium::pbf_error{"object changeset_id must not be negative"};
|
||
|
|
}
|
||
|
|
node.set_changeset(static_cast<osmium::changeset_id_type>(changeset_id));
|
||
|
|
|
||
|
|
node.set_timestamp(dense_timestamp.update(timestamps.front()) * m_date_factor / 1000);
|
||
|
|
timestamps.drop_front();
|
||
|
|
node.set_uid_from_signed(static_cast<osmium::signed_user_id_type>(dense_uid.update(uids.front())));
|
||
|
|
uids.drop_front();
|
||
|
|
|
||
|
|
if (has_visibles) {
|
||
|
|
if (visibles.empty()) {
|
||
|
|
// this is against the spec, must have same number of elements
|
||
|
|
throw osmium::pbf_error{"PBF format error"};
|
||
|
|
}
|
||
|
|
visible = (visibles.front() != 0);
|
||
|
|
visibles.drop_front();
|
||
|
|
}
|
||
|
|
node.set_visible(visible);
|
||
|
|
|
||
|
|
const auto& u = m_stringtable.at(dense_user_sid.update(user_sids.front()));
|
||
|
|
user_sids.drop_front();
|
||
|
|
builder.set_user(u.first, u.second);
|
||
|
|
}
|
||
|
|
|
||
|
|
// even if the node isn't visible, there's still a record
|
||
|
|
// of its lat/lon in the dense arrays.
|
||
|
|
const auto lon = dense_longitude.update(lons.front());
|
||
|
|
lons.drop_front();
|
||
|
|
const auto lat = dense_latitude.update(lats.front());
|
||
|
|
lats.drop_front();
|
||
|
|
if (visible) {
|
||
|
|
builder.object().set_location(osmium::Location{
|
||
|
|
convert_pbf_coordinate(lon),
|
||
|
|
convert_pbf_coordinate(lat)
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
if (tag_it != tags.end()) {
|
||
|
|
build_tag_list_from_dense_nodes(builder, tag_it, tags.end());
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
public:
|
||
|
|
|
||
|
|
PBFPrimitiveBlockDecoder(const data_view& data, osmium::osm_entity_bits::type read_types, osmium::io::read_meta read_metadata) :
|
||
|
|
m_data(data),
|
||
|
|
m_read_types(read_types),
|
||
|
|
m_read_metadata(read_metadata) {
|
||
|
|
}
|
||
|
|
|
||
|
|
PBFPrimitiveBlockDecoder(const PBFPrimitiveBlockDecoder&) = delete;
|
||
|
|
PBFPrimitiveBlockDecoder& operator=(const PBFPrimitiveBlockDecoder&) = delete;
|
||
|
|
|
||
|
|
PBFPrimitiveBlockDecoder(PBFPrimitiveBlockDecoder&&) = delete;
|
||
|
|
PBFPrimitiveBlockDecoder& operator=(PBFPrimitiveBlockDecoder&&) = delete;
|
||
|
|
|
||
|
|
~PBFPrimitiveBlockDecoder() noexcept = default;
|
||
|
|
|
||
|
|
osmium::memory::Buffer operator()() {
|
||
|
|
try {
|
||
|
|
decode_primitive_block_metadata();
|
||
|
|
decode_primitive_block_data();
|
||
|
|
} catch (const std::out_of_range&) {
|
||
|
|
throw osmium::pbf_error{"string id out of range"};
|
||
|
|
}
|
||
|
|
|
||
|
|
return std::move(m_buffer);
|
||
|
|
}
|
||
|
|
|
||
|
|
}; // class PBFPrimitiveBlockDecoder
|
||
|
|
|
||
|
|
inline data_view decode_blob(const std::string& blob_data, std::string& output) {
|
||
|
|
int32_t raw_size = 0;
|
||
|
|
protozero::data_view zlib_data;
|
||
|
|
|
||
|
|
protozero::pbf_message<FileFormat::Blob> pbf_blob{blob_data};
|
||
|
|
while (pbf_blob.next()) {
|
||
|
|
switch (pbf_blob.tag()) {
|
||
|
|
case FileFormat::Blob::optional_bytes_raw:
|
||
|
|
{
|
||
|
|
const auto data_len = pbf_blob.get_view();
|
||
|
|
if (data_len.size() > max_uncompressed_blob_size) {
|
||
|
|
throw osmium::pbf_error{"illegal blob size"};
|
||
|
|
}
|
||
|
|
return data_len;
|
||
|
|
}
|
||
|
|
case FileFormat::Blob::optional_int32_raw_size:
|
||
|
|
raw_size = pbf_blob.get_int32();
|
||
|
|
if (raw_size <= 0 || uint32_t(raw_size) > max_uncompressed_blob_size) {
|
||
|
|
throw osmium::pbf_error{"illegal blob size"};
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
case FileFormat::Blob::optional_bytes_zlib_data:
|
||
|
|
zlib_data = pbf_blob.get_view();
|
||
|
|
break;
|
||
|
|
case FileFormat::Blob::optional_bytes_lzma_data:
|
||
|
|
throw osmium::pbf_error{"lzma blobs not implemented"};
|
||
|
|
default:
|
||
|
|
throw osmium::pbf_error{"unknown compression"};
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (zlib_data.size() != 0 && raw_size != 0) {
|
||
|
|
return osmium::io::detail::zlib_uncompress_string(
|
||
|
|
zlib_data.data(),
|
||
|
|
static_cast<unsigned long>(zlib_data.size()),
|
||
|
|
static_cast<unsigned long>(raw_size),
|
||
|
|
output
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
throw osmium::pbf_error{"blob contains no data"};
|
||
|
|
}
|
||
|
|
|
||
|
|
inline osmium::Box decode_header_bbox(const data_view& data) {
|
||
|
|
int64_t left = std::numeric_limits<int64_t>::max();
|
||
|
|
int64_t right = std::numeric_limits<int64_t>::max();
|
||
|
|
int64_t top = std::numeric_limits<int64_t>::max();
|
||
|
|
int64_t bottom = std::numeric_limits<int64_t>::max();
|
||
|
|
|
||
|
|
protozero::pbf_message<OSMFormat::HeaderBBox> pbf_header_bbox{data};
|
||
|
|
while (pbf_header_bbox.next()) {
|
||
|
|
switch (pbf_header_bbox.tag()) {
|
||
|
|
case OSMFormat::HeaderBBox::required_sint64_left:
|
||
|
|
left = pbf_header_bbox.get_sint64();
|
||
|
|
break;
|
||
|
|
case OSMFormat::HeaderBBox::required_sint64_right:
|
||
|
|
right = pbf_header_bbox.get_sint64();
|
||
|
|
break;
|
||
|
|
case OSMFormat::HeaderBBox::required_sint64_top:
|
||
|
|
top = pbf_header_bbox.get_sint64();
|
||
|
|
break;
|
||
|
|
case OSMFormat::HeaderBBox::required_sint64_bottom:
|
||
|
|
bottom = pbf_header_bbox.get_sint64();
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
pbf_header_bbox.skip();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (left == std::numeric_limits<int64_t>::max() ||
|
||
|
|
right == std::numeric_limits<int64_t>::max() ||
|
||
|
|
top == std::numeric_limits<int64_t>::max() ||
|
||
|
|
bottom == std::numeric_limits<int64_t>::max()) {
|
||
|
|
throw osmium::pbf_error{"invalid bbox"};
|
||
|
|
}
|
||
|
|
|
||
|
|
osmium::Box box;
|
||
|
|
box.extend(osmium::Location(left / resolution_convert, bottom / resolution_convert));
|
||
|
|
box.extend(osmium::Location(right / resolution_convert, top / resolution_convert));
|
||
|
|
|
||
|
|
return box;
|
||
|
|
}
|
||
|
|
|
||
|
|
inline osmium::io::Header decode_header_block(const data_view& data) {
|
||
|
|
osmium::io::Header header;
|
||
|
|
int i = 0;
|
||
|
|
|
||
|
|
protozero::pbf_message<OSMFormat::HeaderBlock> pbf_header_block{data};
|
||
|
|
while (pbf_header_block.next()) {
|
||
|
|
switch (pbf_header_block.tag()) {
|
||
|
|
case OSMFormat::HeaderBlock::optional_HeaderBBox_bbox:
|
||
|
|
header.add_box(decode_header_bbox(pbf_header_block.get_view()));
|
||
|
|
break;
|
||
|
|
case OSMFormat::HeaderBlock::repeated_string_required_features:
|
||
|
|
{
|
||
|
|
auto feature = pbf_header_block.get_view();
|
||
|
|
if (!std::strncmp("OsmSchema-V0.6", feature.data(), feature.size())) {
|
||
|
|
// intentionally left blank
|
||
|
|
} else if (!std::strncmp("DenseNodes", feature.data(), feature.size())) {
|
||
|
|
header.set("pbf_dense_nodes", true);
|
||
|
|
} else if (!std::strncmp("HistoricalInformation", feature.data(), feature.size())) {
|
||
|
|
header.set_has_multiple_object_versions(true);
|
||
|
|
} else {
|
||
|
|
std::string msg{"required feature not supported: "};
|
||
|
|
msg.append(feature.data(), feature.size());
|
||
|
|
throw osmium::pbf_error{msg};
|
||
|
|
}
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
case OSMFormat::HeaderBlock::repeated_string_optional_features:
|
||
|
|
header.set("pbf_optional_feature_" + std::to_string(i++), pbf_header_block.get_string());
|
||
|
|
break;
|
||
|
|
case OSMFormat::HeaderBlock::optional_string_writingprogram:
|
||
|
|
header.set("generator", pbf_header_block.get_string());
|
||
|
|
break;
|
||
|
|
case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp:
|
||
|
|
{
|
||
|
|
const auto timestamp = osmium::Timestamp{pbf_header_block.get_int64()}.to_iso();
|
||
|
|
header.set("osmosis_replication_timestamp", timestamp);
|
||
|
|
header.set("timestamp", timestamp);
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_sequence_number:
|
||
|
|
header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.get_int64()));
|
||
|
|
break;
|
||
|
|
case OSMFormat::HeaderBlock::optional_string_osmosis_replication_base_url:
|
||
|
|
header.set("osmosis_replication_base_url", pbf_header_block.get_string());
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
pbf_header_block.skip();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return header;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Decode HeaderBlock.
|
||
|
|
*
|
||
|
|
* @param header_block_data Input data
|
||
|
|
* @returns Header object
|
||
|
|
* @throws osmium::pbf_error If there was a parsing error
|
||
|
|
*/
|
||
|
|
inline osmium::io::Header decode_header(const std::string& header_block_data) {
|
||
|
|
std::string output;
|
||
|
|
|
||
|
|
return decode_header_block(decode_blob(header_block_data, output));
|
||
|
|
}
|
||
|
|
|
||
|
|
class PBFDataBlobDecoder {
|
||
|
|
|
||
|
|
std::shared_ptr<std::string> m_input_buffer;
|
||
|
|
osmium::osm_entity_bits::type m_read_types;
|
||
|
|
osmium::io::read_meta m_read_metadata;
|
||
|
|
|
||
|
|
public:
|
||
|
|
|
||
|
|
PBFDataBlobDecoder(std::string&& input_buffer, osmium::osm_entity_bits::type read_types, osmium::io::read_meta read_metadata) :
|
||
|
|
m_input_buffer(std::make_shared<std::string>(std::move(input_buffer))),
|
||
|
|
m_read_types(read_types),
|
||
|
|
m_read_metadata(read_metadata) {
|
||
|
|
}
|
||
|
|
|
||
|
|
PBFDataBlobDecoder(const PBFDataBlobDecoder&) = default;
|
||
|
|
PBFDataBlobDecoder& operator=(const PBFDataBlobDecoder&) = default;
|
||
|
|
|
||
|
|
PBFDataBlobDecoder(PBFDataBlobDecoder&&) = default;
|
||
|
|
PBFDataBlobDecoder& operator=(PBFDataBlobDecoder&&) = default;
|
||
|
|
|
||
|
|
~PBFDataBlobDecoder() noexcept = default;
|
||
|
|
|
||
|
|
osmium::memory::Buffer operator()() {
|
||
|
|
std::string output;
|
||
|
|
PBFPrimitiveBlockDecoder decoder{decode_blob(*m_input_buffer, output), m_read_types, m_read_metadata};
|
||
|
|
return decoder();
|
||
|
|
}
|
||
|
|
|
||
|
|
}; // class PBFDataBlobDecoder
|
||
|
|
|
||
|
|
} // namespace detail
|
||
|
|
|
||
|
|
} // namespace io
|
||
|
|
|
||
|
|
} // namespace osmium
|
||
|
|
|
||
|
|
#endif // OSMIUM_IO_DETAIL_PBF_DECODER_HPP
|