c0402345f7
8bcd4ea Add explicit cast operator from osmium::Timestamp to uint32_t. 0b24814 Fixed the full_queue_sleep_duration const (again). 5e19dd2 Try different workaround for windows... df5d6c9 Fix build error on Windows. 1553adf Workaround for missing support for u8 string literals on Windows. aa5e44a Do not build benchmarks in Appveyor to speed up build. 2b22b31 Remove warning generated by assert by casting to largest type. 148c5e3 Fix inclusion of our own css file into doxygen documentation. 13bce6f Split out test thats fails on Windows and do not run it on appveyor. ca04757 Make CMake add_unit_test() function more flexible. 6c04a63 Tell CMake to output json file with compile commands. c0dd848 Fix indentation in some tests. dfa7e4b Formatting: Consistently use spaces around equal signs. 08fe6db Add change log, release version 2.1.0. 667192e Add XML tests. 28acfc7 Make sorting the PBF stringtables optional. 8184781 Fix PBFInputFormat. Use member variable instead of static variable. 2b48945 Fix comment. 6d37054 Rename m_done to m_quit_input_thread to clarify what it is for. bc23083 Fix race condition in PBF reader. 7fc380e Add various docs, noexcepts, asserts, and tests. aeaf4d7 Not a good idea to how overbroad patterns in .gitignore. c1ef2f9 Bugfix: Multipolygon collector was accessing non-existent NodeRef. 0ef9a13 Add noexcept and docs to some functions in NodeRefList class. da4d764 Workaround for Doxygen bug. e67d5d9 use absolute paths to osmium in YouCompleteMe configuration 38abeac remove template parameter from NodeRefList c47adf0 Add check that osm xml file has <osm> or <osmChange> as top-level element. 5e4af90 Updated version number to 2.0.0. 5b2bc3e Workaround in cmake test for sparsehash size. git-subtree-dir: third_party/libosmium git-subtree-split: 8bcd4ea771696812bbb08ebc58e3ee22d0538070
456 lines
21 KiB
C++
456 lines
21 KiB
C++
#ifndef OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP
|
|
#define OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP
|
|
|
|
/*
|
|
|
|
This file is part of Osmium (http://osmcode.org/libosmium).
|
|
|
|
Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
|
|
|
|
Boost Software License - Version 1.0 - August 17th, 2003
|
|
|
|
Permission is hereby granted, free of charge, to any person or organization
|
|
obtaining a copy of the software and accompanying documentation covered by
|
|
this license (the "Software") to use, reproduce, display, distribute,
|
|
execute, and transmit the Software, and to prepare derivative works of the
|
|
Software, and to permit third-parties to whom the Software is furnished to
|
|
do so, all subject to the following:
|
|
|
|
The copyright notices in the Software and this entire statement, including
|
|
the above license grant, this restriction and the following disclaimer,
|
|
must be included in all copies of the Software, in whole or in part, and
|
|
all derivative works of the Software, unless such copies or derivative
|
|
works are solely in the form of machine-executable object code generated by
|
|
a source language processor.
|
|
|
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
|
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
|
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
|
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
DEALINGS IN THE SOFTWARE.
|
|
|
|
*/
|
|
|
|
#include <cassert>
|
|
#include <cstddef>
|
|
#include <cstdint>
|
|
#include <algorithm>
|
|
|
|
#include <osmpbf/osmpbf.h>
|
|
|
|
#include <osmium/builder/osm_object_builder.hpp>
|
|
#include <osmium/io/detail/pbf.hpp> // IWYU pragma: export
|
|
#include <osmium/io/detail/zlib.hpp>
|
|
#include <osmium/io/header.hpp>
|
|
#include <osmium/osm/location.hpp>
|
|
#include <osmium/osm/node.hpp>
|
|
#include <osmium/osm/types.hpp>
|
|
#include <osmium/memory/buffer.hpp>
|
|
#include <osmium/osm/entity_bits.hpp>
|
|
#include <osmium/util/cast.hpp>
|
|
|
|
namespace osmium {
|
|
|
|
namespace io {
|
|
|
|
namespace detail {
|
|
|
|
class PBFPrimitiveBlockParser {
|
|
|
|
static constexpr size_t initial_buffer_size = 2 * 1024 * 1024;
|
|
|
|
const std::string& m_data;
|
|
|
|
const OSMPBF::StringTable* m_stringtable;
|
|
int64_t m_lon_offset;
|
|
int64_t m_lat_offset;
|
|
int64_t m_date_factor;
|
|
int32_t m_granularity;
|
|
|
|
osmium::osm_entity_bits::type m_read_types;
|
|
|
|
osmium::memory::Buffer m_buffer;
|
|
|
|
PBFPrimitiveBlockParser(const PBFPrimitiveBlockParser&) = delete;
|
|
PBFPrimitiveBlockParser(PBFPrimitiveBlockParser&&) = delete;
|
|
|
|
PBFPrimitiveBlockParser& operator=(const PBFPrimitiveBlockParser&) = delete;
|
|
PBFPrimitiveBlockParser& operator=(PBFPrimitiveBlockParser&&) = delete;
|
|
|
|
public:
|
|
|
|
explicit PBFPrimitiveBlockParser(const std::string& data, osmium::osm_entity_bits::type read_types) :
|
|
m_data(data),
|
|
m_stringtable(nullptr),
|
|
m_lon_offset(0),
|
|
m_lat_offset(0),
|
|
m_date_factor(1000),
|
|
m_granularity(100),
|
|
m_read_types(read_types),
|
|
m_buffer(initial_buffer_size) {
|
|
}
|
|
|
|
~PBFPrimitiveBlockParser() = default;
|
|
|
|
osmium::memory::Buffer operator()() {
|
|
OSMPBF::PrimitiveBlock pbf_primitive_block;
|
|
if (!pbf_primitive_block.ParseFromString(m_data)) {
|
|
throw osmium::pbf_error("failed to parse PrimitiveBlock");
|
|
}
|
|
|
|
m_stringtable = &pbf_primitive_block.stringtable();
|
|
m_lon_offset = pbf_primitive_block.lon_offset();
|
|
m_lat_offset = pbf_primitive_block.lat_offset();
|
|
m_date_factor = pbf_primitive_block.date_granularity() / 1000;
|
|
m_granularity = pbf_primitive_block.granularity();
|
|
|
|
for (int i = 0; i < pbf_primitive_block.primitivegroup_size(); ++i) {
|
|
const OSMPBF::PrimitiveGroup& group = pbf_primitive_block.primitivegroup(i);
|
|
|
|
if (group.has_dense()) {
|
|
if (m_read_types & osmium::osm_entity_bits::node) parse_dense_node_group(group);
|
|
} else if (group.ways_size() != 0) {
|
|
if (m_read_types & osmium::osm_entity_bits::way) parse_way_group(group);
|
|
} else if (group.relations_size() != 0) {
|
|
if (m_read_types & osmium::osm_entity_bits::relation) parse_relation_group(group);
|
|
} else if (group.nodes_size() != 0) {
|
|
if (m_read_types & osmium::osm_entity_bits::node) parse_node_group(group);
|
|
} else {
|
|
throw osmium::pbf_error("group of unknown type");
|
|
}
|
|
}
|
|
|
|
return std::move(m_buffer);
|
|
}
|
|
|
|
private:
|
|
|
|
template <class TBuilder, class TPBFObject>
|
|
void parse_attributes(TBuilder& builder, const TPBFObject& pbf_object) {
|
|
auto& object = builder.object();
|
|
|
|
object.set_id(pbf_object.id());
|
|
|
|
if (pbf_object.has_info()) {
|
|
object.set_version(static_cast_with_assert<object_version_type>(pbf_object.info().version()))
|
|
.set_changeset(static_cast_with_assert<changeset_id_type>(pbf_object.info().changeset()))
|
|
.set_timestamp(pbf_object.info().timestamp() * m_date_factor)
|
|
.set_uid_from_signed(pbf_object.info().uid());
|
|
if (pbf_object.info().has_visible()) {
|
|
object.set_visible(pbf_object.info().visible());
|
|
}
|
|
builder.add_user(m_stringtable->s(static_cast_with_assert<int>(pbf_object.info().user_sid())));
|
|
} else {
|
|
builder.add_user("", 1);
|
|
}
|
|
}
|
|
|
|
void parse_node_group(const OSMPBF::PrimitiveGroup& group) {
|
|
for (int i = 0; i < group.nodes_size(); ++i) {
|
|
osmium::builder::NodeBuilder builder(m_buffer);
|
|
const OSMPBF::Node& pbf_node = group.nodes(i);
|
|
parse_attributes(builder, pbf_node);
|
|
|
|
if (builder.object().visible()) {
|
|
builder.object().set_location(osmium::Location(
|
|
(pbf_node.lon() * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision),
|
|
(pbf_node.lat() * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision)));
|
|
}
|
|
|
|
if (pbf_node.keys_size() > 0) {
|
|
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
|
|
for (int tag = 0; tag < pbf_node.keys_size(); ++tag) {
|
|
tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_node.keys(tag))),
|
|
m_stringtable->s(static_cast<int>(pbf_node.vals(tag))));
|
|
}
|
|
}
|
|
|
|
m_buffer.commit();
|
|
}
|
|
}
|
|
|
|
void parse_way_group(const OSMPBF::PrimitiveGroup& group) {
|
|
for (int i = 0; i < group.ways_size(); ++i) {
|
|
osmium::builder::WayBuilder builder(m_buffer);
|
|
const OSMPBF::Way& pbf_way = group.ways(i);
|
|
parse_attributes(builder, pbf_way);
|
|
|
|
if (pbf_way.refs_size() > 0) {
|
|
osmium::builder::WayNodeListBuilder wnl_builder(m_buffer, &builder);
|
|
int64_t ref = 0;
|
|
for (int n = 0; n < pbf_way.refs_size(); ++n) {
|
|
ref += pbf_way.refs(n);
|
|
wnl_builder.add_node_ref(ref);
|
|
}
|
|
}
|
|
|
|
if (pbf_way.keys_size() > 0) {
|
|
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
|
|
for (int tag = 0; tag < pbf_way.keys_size(); ++tag) {
|
|
tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_way.keys(tag))),
|
|
m_stringtable->s(static_cast<int>(pbf_way.vals(tag))));
|
|
}
|
|
}
|
|
|
|
m_buffer.commit();
|
|
}
|
|
}
|
|
|
|
void parse_relation_group(const OSMPBF::PrimitiveGroup& group) {
|
|
for (int i = 0; i < group.relations_size(); ++i) {
|
|
osmium::builder::RelationBuilder builder(m_buffer);
|
|
const OSMPBF::Relation& pbf_relation = group.relations(i);
|
|
parse_attributes(builder, pbf_relation);
|
|
|
|
if (pbf_relation.types_size() > 0) {
|
|
osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder);
|
|
int64_t ref = 0;
|
|
for (int n = 0; n < pbf_relation.types_size(); ++n) {
|
|
ref += pbf_relation.memids(n);
|
|
rml_builder.add_member(osmpbf_membertype_to_item_type(pbf_relation.types(n)), ref, m_stringtable->s(pbf_relation.roles_sid(n)));
|
|
}
|
|
}
|
|
|
|
if (pbf_relation.keys_size() > 0) {
|
|
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
|
|
for (int tag = 0; tag < pbf_relation.keys_size(); ++tag) {
|
|
tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_relation.keys(tag))),
|
|
m_stringtable->s(static_cast<int>(pbf_relation.vals(tag))));
|
|
}
|
|
}
|
|
|
|
m_buffer.commit();
|
|
}
|
|
}
|
|
|
|
int add_tags(const OSMPBF::DenseNodes& dense, int n, osmium::builder::NodeBuilder* builder) {
|
|
if (n >= dense.keys_vals_size()) {
|
|
return n;
|
|
}
|
|
|
|
if (dense.keys_vals(n) == 0) {
|
|
return n+1;
|
|
}
|
|
|
|
osmium::builder::TagListBuilder tl_builder(m_buffer, builder);
|
|
|
|
while (n < dense.keys_vals_size()) {
|
|
int tag_key_pos = dense.keys_vals(n++);
|
|
|
|
if (tag_key_pos == 0) {
|
|
break;
|
|
}
|
|
|
|
tl_builder.add_tag(m_stringtable->s(tag_key_pos),
|
|
m_stringtable->s(dense.keys_vals(n)));
|
|
|
|
++n;
|
|
}
|
|
|
|
return n;
|
|
}
|
|
|
|
void parse_dense_node_group(const OSMPBF::PrimitiveGroup& group) {
|
|
int64_t last_dense_id = 0;
|
|
int64_t last_dense_latitude = 0;
|
|
int64_t last_dense_longitude = 0;
|
|
int64_t last_dense_uid = 0;
|
|
int64_t last_dense_user_sid = 0;
|
|
int64_t last_dense_changeset = 0;
|
|
int64_t last_dense_timestamp = 0;
|
|
int last_dense_tag = 0;
|
|
|
|
const OSMPBF::DenseNodes& dense = group.dense();
|
|
|
|
for (int i = 0; i < dense.id_size(); ++i) {
|
|
bool visible = true;
|
|
|
|
last_dense_id += dense.id(i);
|
|
last_dense_latitude += dense.lat(i);
|
|
last_dense_longitude += dense.lon(i);
|
|
|
|
if (dense.has_denseinfo()) {
|
|
last_dense_changeset += dense.denseinfo().changeset(i);
|
|
last_dense_timestamp += dense.denseinfo().timestamp(i);
|
|
last_dense_uid += dense.denseinfo().uid(i);
|
|
last_dense_user_sid += dense.denseinfo().user_sid(i);
|
|
if (dense.denseinfo().visible_size() > 0) {
|
|
visible = dense.denseinfo().visible(i);
|
|
}
|
|
assert(last_dense_changeset >= 0);
|
|
assert(last_dense_timestamp >= 0);
|
|
assert(last_dense_uid >= -1);
|
|
assert(last_dense_user_sid >= 0);
|
|
}
|
|
|
|
osmium::builder::NodeBuilder builder(m_buffer);
|
|
osmium::Node& node = builder.object();
|
|
|
|
node.set_id(last_dense_id);
|
|
|
|
if (dense.has_denseinfo()) {
|
|
auto v = dense.denseinfo().version(i);
|
|
assert(v > 0);
|
|
node.set_version(static_cast<osmium::object_version_type>(v));
|
|
node.set_changeset(static_cast<osmium::changeset_id_type>(last_dense_changeset));
|
|
node.set_timestamp(last_dense_timestamp * m_date_factor);
|
|
node.set_uid_from_signed(static_cast<osmium::signed_user_id_type>(last_dense_uid));
|
|
node.set_visible(visible);
|
|
builder.add_user(m_stringtable->s(static_cast<int>(last_dense_user_sid)));
|
|
} else {
|
|
builder.add_user("", 1);
|
|
}
|
|
|
|
if (visible) {
|
|
builder.object().set_location(osmium::Location(
|
|
(last_dense_longitude * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision),
|
|
(last_dense_latitude * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision)));
|
|
}
|
|
|
|
last_dense_tag = add_tags(dense, last_dense_tag, &builder);
|
|
m_buffer.commit();
|
|
}
|
|
}
|
|
|
|
}; // class PBFPrimitiveBlockParser
|
|
|
|
/**
|
|
* PBF blobs can optionally be packed with the zlib algorithm.
|
|
* This function returns the raw data (if it was unpacked) or
|
|
* the unpacked data (if it was packed).
|
|
*
|
|
* @param input_data Reference to input data.
|
|
* @returns Unpacked data
|
|
* @throws osmium::pbf_error If there was a problem parsing the PBF
|
|
*/
|
|
inline std::unique_ptr<const std::string> unpack_blob(const std::string& input_data) {
|
|
OSMPBF::Blob pbf_blob;
|
|
if (!pbf_blob.ParseFromString(input_data)) {
|
|
throw osmium::pbf_error("failed to parse blob");
|
|
}
|
|
|
|
if (pbf_blob.has_raw()) {
|
|
return std::unique_ptr<std::string>(pbf_blob.release_raw());
|
|
} else if (pbf_blob.has_zlib_data()) {
|
|
auto raw_size = pbf_blob.raw_size();
|
|
assert(raw_size >= 0);
|
|
assert(raw_size <= OSMPBF::max_uncompressed_blob_size);
|
|
return osmium::io::detail::zlib_uncompress(pbf_blob.zlib_data(), static_cast<unsigned long>(raw_size));
|
|
} else if (pbf_blob.has_lzma_data()) {
|
|
throw osmium::pbf_error("lzma blobs not implemented");
|
|
} else {
|
|
throw osmium::pbf_error("blob contains no data");
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Parse blob as a HeaderBlock.
|
|
*
|
|
* @param input_buffer Blob data
|
|
* @returns Header object
|
|
* @throws osmium::pbf_error If there was a parsing error
|
|
*/
|
|
inline osmium::io::Header parse_header_blob(const std::string& input_buffer) {
|
|
const std::unique_ptr<const std::string> data = unpack_blob(input_buffer);
|
|
|
|
OSMPBF::HeaderBlock pbf_header_block;
|
|
if (!pbf_header_block.ParseFromString(*data)) {
|
|
throw osmium::pbf_error("failed to parse HeaderBlock");
|
|
}
|
|
|
|
osmium::io::Header header;
|
|
for (int i = 0; i < pbf_header_block.required_features_size(); ++i) {
|
|
const std::string& feature = pbf_header_block.required_features(i);
|
|
|
|
if (feature == "OsmSchema-V0.6") continue;
|
|
if (feature == "DenseNodes") {
|
|
header.set("pbf_dense_nodes", true);
|
|
continue;
|
|
}
|
|
if (feature == "HistoricalInformation") {
|
|
header.set_has_multiple_object_versions(true);
|
|
continue;
|
|
}
|
|
|
|
throw osmium::pbf_error(std::string("required feature not supported: ") + feature);
|
|
}
|
|
|
|
for (int i = 0; i < pbf_header_block.optional_features_size(); ++i) {
|
|
const std::string& feature = pbf_header_block.optional_features(i);
|
|
header.set("pbf_optional_feature_" + std::to_string(i), feature);
|
|
}
|
|
|
|
if (pbf_header_block.has_writingprogram()) {
|
|
header.set("generator", pbf_header_block.writingprogram());
|
|
}
|
|
|
|
if (pbf_header_block.has_bbox()) {
|
|
const OSMPBF::HeaderBBox& pbf_bbox = pbf_header_block.bbox();
|
|
const int64_t resolution_convert = OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision;
|
|
osmium::Box box;
|
|
box.extend(osmium::Location(pbf_bbox.left() / resolution_convert, pbf_bbox.bottom() / resolution_convert));
|
|
box.extend(osmium::Location(pbf_bbox.right() / resolution_convert, pbf_bbox.top() / resolution_convert));
|
|
header.add_box(box);
|
|
}
|
|
|
|
if (pbf_header_block.has_osmosis_replication_timestamp()) {
|
|
header.set("osmosis_replication_timestamp", osmium::Timestamp(pbf_header_block.osmosis_replication_timestamp()).to_iso());
|
|
}
|
|
|
|
if (pbf_header_block.has_osmosis_replication_sequence_number()) {
|
|
header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.osmosis_replication_sequence_number()));
|
|
}
|
|
|
|
if (pbf_header_block.has_osmosis_replication_base_url()) {
|
|
header.set("osmosis_replication_base_url", pbf_header_block.osmosis_replication_base_url());
|
|
}
|
|
|
|
return header;
|
|
}
|
|
|
|
class DataBlobParser {
|
|
|
|
std::shared_ptr<std::string> m_input_buffer;
|
|
osmium::osm_entity_bits::type m_read_types;
|
|
|
|
public:
|
|
|
|
DataBlobParser(std::string&& input_buffer, osmium::osm_entity_bits::type read_types) :
|
|
m_input_buffer(std::make_shared<std::string>(std::move(input_buffer))),
|
|
m_read_types(read_types) {
|
|
if (input_buffer.size() > OSMPBF::max_uncompressed_blob_size) {
|
|
throw osmium::pbf_error(std::string("invalid blob size: " + std::to_string(input_buffer.size())));
|
|
}
|
|
}
|
|
/*
|
|
DataBlobParser(const DataBlobParser& other) :
|
|
m_input_buffer(std::move(other.m_input_buffer)),
|
|
m_read_types(other.m_read_types) {
|
|
}*/
|
|
|
|
DataBlobParser(const DataBlobParser&) = default;
|
|
DataBlobParser& operator=(const DataBlobParser&) = default;
|
|
|
|
DataBlobParser(DataBlobParser&&) = default;
|
|
DataBlobParser& operator=(DataBlobParser&&) = default;
|
|
|
|
~DataBlobParser() = default;
|
|
|
|
osmium::memory::Buffer operator()() {
|
|
const std::unique_ptr<const std::string> data = unpack_blob(*m_input_buffer);
|
|
PBFPrimitiveBlockParser parser(*data, m_read_types);
|
|
return parser();
|
|
}
|
|
|
|
}; // class DataBlobParser
|
|
|
|
} // namespace detail
|
|
|
|
} // namespace io
|
|
|
|
} // namespace osmium
|
|
|
|
#endif // OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP
|