Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

bump osmium version

Browse Source
This commit is contained in:
Dennis Luxen 2014-12-17 11:19:08 +01:00
parent 40ff7ad999
commit 3dddd16ec7
47 changed files with 2620 additions and 2251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
third_party/osmium/area/detail/segment_list.hpp vendored
View File

@ -70,11 +70,11 @@ namespace osmium {
~SegmentList() = default;
SegmentList(const SegmentList& other) = delete;
SegmentList(SegmentList&& other) = delete;
SegmentList(const SegmentList&) = delete;
SegmentList(SegmentList&&) = delete;
SegmentList& operator=(const SegmentList& other) = delete;
SegmentList& operator=(SegmentList&& other) = delete;
SegmentList& operator=(const SegmentList&) = delete;
SegmentList& operator=(SegmentList&&) = delete;
/// The number of segments in the list.
size_t size() const noexcept {

5
third_party/osmium/area/multipolygon_collector.hpp vendored
View File

@ -86,8 +86,9 @@ namespace osmium {
void flush_output_buffer() {
if (this->callback()) {
this->callback()(m_output_buffer);
m_output_buffer.clear();
osmium::memory::Buffer buffer(initial_output_buffer_size);
std::swap(buffer, m_output_buffer);
this->callback()(std::move(buffer));
}
}

8
third_party/osmium/diff_iterator.hpp vendored
View File

@ -81,11 +81,11 @@ namespace osmium {
m_end(end) {
}
DiffIterator(const DiffIterator& other) = default;
DiffIterator& operator=(const DiffIterator& other) = default;
DiffIterator(const DiffIterator&) = default;
DiffIterator& operator=(const DiffIterator&) = default;
DiffIterator(DiffIterator&& other) = default;
DiffIterator& operator=(DiffIterator&& other) = default;
DiffIterator(DiffIterator&&) = default;
DiffIterator& operator=(DiffIterator&&) = default;
DiffIterator& operator++() {
m_prev = std::move(m_curr);

131
third_party/osmium/experimental/flex_reader.hpp vendored Normal file
View File

@ -0,0 +1,131 @@
#ifndef OSMIUM_EXPERIMENTAL_FLEX_READER_HPP
#define OSMIUM_EXPERIMENTAL_FLEX_READER_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2014 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <osmium/handler/node_locations_for_ways.hpp>
#include <osmium/visitor.hpp>
#include <osmium/area/multipolygon_collector.hpp>
#include <osmium/area/assembler.hpp>
namespace osmium {
/**
* Experimental code that is not "officially" supported.
*/
namespace experimental {
template <class TLocationHandler>
class FlexReader {
bool m_with_areas;
osmium::osm_entity_bits::type m_entities;
typename TLocationHandler::index_pos_type m_index_pos;
typename TLocationHandler::index_neg_type m_index_neg;
TLocationHandler m_location_handler;
osmium::io::Reader m_reader;
osmium::area::Assembler::config_type m_assembler_config;
osmium::area::MultipolygonCollector<osmium::area::Assembler> m_collector;
public:
explicit FlexReader(const osmium::io::File& file, osmium::osm_entity_bits::type entities = osmium::osm_entity_bits::nwr) :
m_with_areas(entities & osmium::osm_entity_bits::area),
m_entities((entities & ~osmium::osm_entity_bits::area) | (m_with_areas ? osmium::osm_entity_bits::node | osmium::osm_entity_bits::way : osmium::osm_entity_bits::nothing)),
m_index_pos(),
m_index_neg(),
m_location_handler(m_index_pos, m_index_neg),
m_reader(file, m_entities),
m_assembler_config(),
m_collector(m_assembler_config)
{
m_location_handler.ignore_errors();
if (m_with_areas) {
osmium::io::Reader reader(file, osmium::osm_entity_bits::relation);
m_collector.read_relations(reader);
reader.close();
}
}
explicit FlexReader(const std::string& filename, osmium::osm_entity_bits::type entities = osmium::osm_entity_bits::nwr) :
FlexReader(osmium::io::File(filename), entities) {
}
explicit FlexReader(const char* filename, osmium::osm_entity_bits::type entities = osmium::osm_entity_bits::nwr) :
FlexReader(osmium::io::File(filename), entities) {
}
std::vector<osmium::memory::Buffer> read() {
std::vector<osmium::memory::Buffer> buffers;
osmium::memory::Buffer buffer = m_reader.read();
if (buffer) {
buffers.push_back(std::move(buffer));
if (m_with_areas) {
osmium::apply(buffers[0], m_location_handler, m_collector.handler([&buffers](osmium::memory::Buffer&& area_buffer) {
buffers.push_back(std::move(area_buffer));
}));
} else if (m_entities & (osmium::osm_entity_bits::node | osmium::osm_entity_bits::way)) {
osmium::apply(buffers[0], m_location_handler);
}
}
return buffers;
}
osmium::io::Header header() const {
return m_reader.header();
}
void close() {
return m_reader.close();
}
bool eof() const {
return m_reader.eof();
}
const osmium::area::MultipolygonCollector<osmium::area::Assembler>& collector() const {
return m_collector;
}
}; // class FlexReader
} // namespace experimental
} // namespace osmium
#endif // OSMIUM_EXPERIMENTAL_FLEX_READER_HPP

7
third_party/osmium/handler/node_locations_for_ways.hpp vendored
View File

@ -67,6 +67,13 @@ namespace osmium {
static_assert(std::is_base_of<osmium::index::map::Map<osmium::unsigned_object_id_type, osmium::Location>, TStorageNegIDs>::value,
"Index class must be derived from osmium::index::map::Map<osmium::unsigned_object_id_type, osmium::Location>");
public:
typedef TStoragePosIDs index_pos_type;
typedef TStorageNegIDs index_neg_type;
private:
/// Object that handles the actual storage of the node locations (with positive IDs).
TStoragePosIDs& m_storage_pos;

10
third_party/osmium/io/detail/opl_output_format.hpp vendored
View File

@ -189,8 +189,8 @@ namespace osmium {
OPLOutputBlock(const OPLOutputBlock&) = delete;
OPLOutputBlock& operator=(const OPLOutputBlock&) = delete;
OPLOutputBlock(OPLOutputBlock&& other) = default;
OPLOutputBlock& operator=(OPLOutputBlock&& other) = default;
OPLOutputBlock(OPLOutputBlock&&) = default;
OPLOutputBlock& operator=(OPLOutputBlock&&) = default;
std::string operator()() {
osmium::apply(m_input_buffer.cbegin(), m_input_buffer.cend(), *this);
@ -282,11 +282,7 @@ namespace osmium {
}
void write_buffer(osmium::memory::Buffer&& buffer) override final {
osmium::thread::SharedPtrWrapper<OPLOutputBlock> output_block(std::move(buffer));
m_output_queue.push(osmium::thread::Pool::instance().submit(std::move(output_block)));
while (m_output_queue.size() > 10) {
std::this_thread::sleep_for(std::chrono::milliseconds(100)); // XXX
}
m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer)}));
}
void close() override final {

17
third_party/osmium/io/detail/pbf.hpp vendored
View File

@ -46,6 +46,7 @@ DEALINGS IN THE SOFTWARE.
# include <winsock2.h>
#endif
#include <osmium/io/error.hpp>
#include <osmium/osm/item_type.hpp>
namespace osmium {
@ -78,6 +79,22 @@ namespace osmium {
}
}
/**
* Exception thrown when there was a problem with parsing the PBF format of
* a file.
*/
struct pbf_error : public io_error {
pbf_error(const std::string& what) :
io_error(std::string("PBF error: ") + what) {
}
pbf_error(const char* what) :
io_error(std::string("PBF error: ") + what) {
}
}; // struct pbf_error
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_PBF_HPP

564
third_party/osmium/io/detail/pbf_input_format.hpp vendored
View File

@ -47,14 +47,14 @@ DEALINGS IN THE SOFTWARE.
#include <stdexcept>
#include <string>
#include <thread>
#include <type_traits>
#include <osmium/builder/osm_object_builder.hpp>
#include <osmium/io/detail/input_format.hpp>
#include <osmium/io/detail/pbf.hpp> // IWYU pragma: export
#include <osmium/io/detail/zlib.hpp>
#include <osmium/io/detail/pbf_parser.hpp>
#include <osmium/io/error.hpp>
#include <osmium/io/file.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/io/header.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/osm.hpp>
#include <osmium/osm/box.hpp>
@ -62,486 +62,22 @@ DEALINGS IN THE SOFTWARE.
#include <osmium/osm/location.hpp>
#include <osmium/osm/object.hpp>
#include <osmium/osm/timestamp.hpp>
#include <osmium/thread/name.hpp>
#include <osmium/thread/pool.hpp>
#include <osmium/thread/queue.hpp>
#include <osmium/thread/util.hpp>
#include <osmium/util/cast.hpp>
#include <osmium/util/config.hpp>
namespace osmium {
/**
* Exception thrown when there was a problem with parsing the PBF format of
* a file.
*/
struct pbf_error : public io_error {
pbf_error(const std::string& what) :
io_error(std::string("PBF error: ") + what) {
}
pbf_error(const char* what) :
io_error(std::string("PBF error: ") + what) {
}
}; // struct pbf_error
namespace io {
class File;
namespace detail {
class PBFPrimitiveBlockParser {
static constexpr size_t initial_buffer_size = 2 * 1024 * 1024;
const void* m_data;
const int m_size;
const OSMPBF::StringTable* m_stringtable;
int64_t m_lon_offset;
int64_t m_lat_offset;
int64_t m_date_factor;
int32_t m_granularity;
osmium::osm_entity_bits::type m_read_types;
osmium::memory::Buffer m_buffer;
PBFPrimitiveBlockParser(const PBFPrimitiveBlockParser&) = delete;
PBFPrimitiveBlockParser(PBFPrimitiveBlockParser&&) = delete;
PBFPrimitiveBlockParser& operator=(const PBFPrimitiveBlockParser&) = delete;
PBFPrimitiveBlockParser& operator=(PBFPrimitiveBlockParser&&) = delete;
public:
explicit PBFPrimitiveBlockParser(const void* data, const int size, osmium::osm_entity_bits::type read_types) :
m_data(data),
m_size(size),
m_stringtable(nullptr),
m_lon_offset(0),
m_lat_offset(0),
m_date_factor(1000),
m_granularity(100),
m_read_types(read_types),
m_buffer(initial_buffer_size) {
}
~PBFPrimitiveBlockParser() = default;
osmium::memory::Buffer operator()() {
OSMPBF::PrimitiveBlock pbf_primitive_block;
if (!pbf_primitive_block.ParseFromArray(m_data, m_size)) {
throw osmium::pbf_error("failed to parse PrimitiveBlock");
}
m_stringtable = &pbf_primitive_block.stringtable();
m_lon_offset = pbf_primitive_block.lon_offset();
m_lat_offset = pbf_primitive_block.lat_offset();
m_date_factor = pbf_primitive_block.date_granularity() / 1000;
m_granularity = pbf_primitive_block.granularity();
for (int i=0; i < pbf_primitive_block.primitivegroup_size(); ++i) {
const OSMPBF::PrimitiveGroup& group = pbf_primitive_block.primitivegroup(i);
if (group.has_dense()) {
if (m_read_types & osmium::osm_entity_bits::node) parse_dense_node_group(group);
} else if (group.ways_size() != 0) {
if (m_read_types & osmium::osm_entity_bits::way) parse_way_group(group);
} else if (group.relations_size() != 0) {
if (m_read_types & osmium::osm_entity_bits::relation) parse_relation_group(group);
} else if (group.nodes_size() != 0) {
if (m_read_types & osmium::osm_entity_bits::node) parse_node_group(group);
} else {
throw osmium::pbf_error("group of unknown type");
}
}
return std::move(m_buffer);
}
private:
template <class TBuilder, class TPBFObject>
void parse_attributes(TBuilder& builder, const TPBFObject& pbf_object) {
auto& object = builder.object();
object.set_id(pbf_object.id());
if (pbf_object.has_info()) {
object.set_version(static_cast_with_assert<object_version_type>(pbf_object.info().version()))
.set_changeset(static_cast_with_assert<changeset_id_type>(pbf_object.info().changeset()))
.set_timestamp(pbf_object.info().timestamp() * m_date_factor)
.set_uid_from_signed(pbf_object.info().uid());
if (pbf_object.info().has_visible()) {
object.set_visible(pbf_object.info().visible());
}
builder.add_user(m_stringtable->s(static_cast_with_assert<int>(pbf_object.info().user_sid())));
} else {
builder.add_user("", 1);
}
}
void parse_node_group(const OSMPBF::PrimitiveGroup& group) {
for (int i=0; i < group.nodes_size(); ++i) {
osmium::builder::NodeBuilder builder(m_buffer);
const OSMPBF::Node& pbf_node = group.nodes(i);
parse_attributes(builder, pbf_node);
if (builder.object().visible()) {
builder.object().set_location(osmium::Location(
(pbf_node.lon() * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision),
(pbf_node.lat() * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision)));
}
if (pbf_node.keys_size() > 0) {
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
for (int tag=0; tag < pbf_node.keys_size(); ++tag) {
tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_node.keys(tag))),
m_stringtable->s(static_cast<int>(pbf_node.vals(tag))));
}
}
m_buffer.commit();
}
}
void parse_way_group(const OSMPBF::PrimitiveGroup& group) {
for (int i=0; i < group.ways_size(); ++i) {
osmium::builder::WayBuilder builder(m_buffer);
const OSMPBF::Way& pbf_way = group.ways(i);
parse_attributes(builder, pbf_way);
if (pbf_way.refs_size() > 0) {
osmium::builder::WayNodeListBuilder wnl_builder(m_buffer, &builder);
int64_t ref = 0;
for (int n=0; n < pbf_way.refs_size(); ++n) {
ref += pbf_way.refs(n);
wnl_builder.add_node_ref(ref);
}
}
if (pbf_way.keys_size() > 0) {
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
for (int tag=0; tag < pbf_way.keys_size(); ++tag) {
tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_way.keys(tag))),
m_stringtable->s(static_cast<int>(pbf_way.vals(tag))));
}
}
m_buffer.commit();
}
}
void parse_relation_group(const OSMPBF::PrimitiveGroup& group) {
for (int i=0; i < group.relations_size(); ++i) {
osmium::builder::RelationBuilder builder(m_buffer);
const OSMPBF::Relation& pbf_relation = group.relations(i);
parse_attributes(builder, pbf_relation);
if (pbf_relation.types_size() > 0) {
osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder);
int64_t ref = 0;
for (int n=0; n < pbf_relation.types_size(); ++n) {
ref += pbf_relation.memids(n);
rml_builder.add_member(osmpbf_membertype_to_item_type(pbf_relation.types(n)), ref, m_stringtable->s(pbf_relation.roles_sid(n)));
}
}
if (pbf_relation.keys_size() > 0) {
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
for (int tag=0; tag < pbf_relation.keys_size(); ++tag) {
tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_relation.keys(tag))),
m_stringtable->s(static_cast<int>(pbf_relation.vals(tag))));
}
}
m_buffer.commit();
}
}
int add_tags(const OSMPBF::DenseNodes& dense, int n, osmium::builder::NodeBuilder* builder) {
if (n >= dense.keys_vals_size()) {
return n;
}
if (dense.keys_vals(n) == 0) {
return n+1;
}
osmium::builder::TagListBuilder tl_builder(m_buffer, builder);
while (n < dense.keys_vals_size()) {
int tag_key_pos = dense.keys_vals(n++);
if (tag_key_pos == 0) {
break;
}
tl_builder.add_tag(m_stringtable->s(tag_key_pos),
m_stringtable->s(dense.keys_vals(n)));
++n;
}
return n;
}
void parse_dense_node_group(const OSMPBF::PrimitiveGroup& group) {
int64_t last_dense_id = 0;
int64_t last_dense_latitude = 0;
int64_t last_dense_longitude = 0;
int64_t last_dense_uid = 0;
int64_t last_dense_user_sid = 0;
int64_t last_dense_changeset = 0;
int64_t last_dense_timestamp = 0;
int last_dense_tag = 0;
const OSMPBF::DenseNodes& dense = group.dense();
for (int i=0; i < dense.id_size(); ++i) {
bool visible = true;
last_dense_id += dense.id(i);
last_dense_latitude += dense.lat(i);
last_dense_longitude += dense.lon(i);
if (dense.has_denseinfo()) {
last_dense_changeset += dense.denseinfo().changeset(i);
last_dense_timestamp += dense.denseinfo().timestamp(i);
last_dense_uid += dense.denseinfo().uid(i);
last_dense_user_sid += dense.denseinfo().user_sid(i);
if (dense.denseinfo().visible_size() > 0) {
visible = dense.denseinfo().visible(i);
}
assert(last_dense_changeset >= 0);
assert(last_dense_timestamp >= 0);
assert(last_dense_uid >= -1);
assert(last_dense_user_sid >= 0);
}
osmium::builder::NodeBuilder builder(m_buffer);
osmium::Node& node = builder.object();
node.set_id(last_dense_id);
if (dense.has_denseinfo()) {
auto v = dense.denseinfo().version(i);
assert(v > 0);
node.set_version(static_cast<osmium::object_version_type>(v));
node.set_changeset(static_cast<osmium::changeset_id_type>(last_dense_changeset));
node.set_timestamp(last_dense_timestamp * m_date_factor);
node.set_uid_from_signed(static_cast<osmium::signed_user_id_type>(last_dense_uid));
node.set_visible(visible);
builder.add_user(m_stringtable->s(static_cast<int>(last_dense_user_sid)));
} else {
builder.add_user("", 1);
}
if (visible) {
builder.object().set_location(osmium::Location(
(last_dense_longitude * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision),
(last_dense_latitude * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision)));
}
last_dense_tag = add_tags(dense, last_dense_tag, &builder);
m_buffer.commit();
}
}
}; // class PBFPrimitiveBlockParser
typedef osmium::thread::Queue<std::future<osmium::memory::Buffer>> queue_type;
class InputQueueReader {
osmium::thread::Queue<std::string>& m_queue;
std::string m_buffer;
public:
InputQueueReader(osmium::thread::Queue<std::string>& queue) :
m_queue(queue) {
}
bool operator()(unsigned char* data, size_t size) {
while (m_buffer.size() < size) {
std::string new_data;
m_queue.wait_and_pop(new_data);
if (new_data.empty()) {
return false;
}
m_buffer += new_data;
}
memcpy(data, m_buffer.data(), size);
m_buffer.erase(0, size);
return true;
}
}; // class InputQueueReader
template <class TDerived>
class BlobParser {
protected:
std::shared_ptr<unsigned char> m_input_buffer;
const int m_size;
const int m_blob_num;
InputQueueReader& m_input_queue_reader;
BlobParser(const int size, const int blob_num, InputQueueReader& input_queue_reader) :
m_input_buffer(new unsigned char[size], [](unsigned char* ptr) { delete[] ptr; }),
m_size(size),
m_blob_num(blob_num),
m_input_queue_reader(input_queue_reader) {
if (size < 0 || size > OSMPBF::max_uncompressed_blob_size) {
throw osmium::pbf_error(std::string("invalid blob size: " + std::to_string(size)));
}
if (! input_queue_reader(m_input_buffer.get(), static_cast<size_t>(size))) {
throw osmium::pbf_error("truncated data (EOF encountered)");
}
}
public:
void doit() {
OSMPBF::Blob pbf_blob;
if (!pbf_blob.ParseFromArray(m_input_buffer.get(), m_size)) {
throw osmium::pbf_error("failed to parse blob");
}
if (pbf_blob.has_raw()) {
static_cast<TDerived*>(this)->handle_blob(pbf_blob.raw());
return;
} else if (pbf_blob.has_zlib_data()) {
auto raw_size = pbf_blob.raw_size();
assert(raw_size >= 0);
assert(raw_size <= OSMPBF::max_uncompressed_blob_size);
std::string unpack_buffer { osmium::io::detail::zlib_uncompress(pbf_blob.zlib_data(), static_cast<unsigned long>(raw_size)) };
static_cast<TDerived*>(this)->handle_blob(unpack_buffer);
return;
} else if (pbf_blob.has_lzma_data()) {
throw osmium::pbf_error("lzma blobs not implemented");
} else {
throw osmium::pbf_error("blob contains no data");
}
}
osmium::memory::Buffer operator()() {
OSMPBF::Blob pbf_blob;
if (!pbf_blob.ParseFromArray(m_input_buffer.get(), m_size)) {
throw osmium::pbf_error("failed to parse blob");
}
if (pbf_blob.has_raw()) {
return static_cast<TDerived*>(this)->handle_blob(pbf_blob.raw());
} else if (pbf_blob.has_zlib_data()) {
auto raw_size = pbf_blob.raw_size();
assert(raw_size >= 0);
assert(raw_size <= OSMPBF::max_uncompressed_blob_size);
std::string unpack_buffer { osmium::io::detail::zlib_uncompress(pbf_blob.zlib_data(), static_cast<unsigned long>(raw_size)) };
return static_cast<TDerived*>(this)->handle_blob(unpack_buffer);
} else if (pbf_blob.has_lzma_data()) {
throw osmium::pbf_error("lzma blobs not implemented");
} else {
throw osmium::pbf_error("blob contains no data");
}
}
}; // class BlobParser;
class HeaderBlobParser : public BlobParser<HeaderBlobParser> {
osmium::io::Header& m_header;
void handle_blob(const std::string& data) {
OSMPBF::HeaderBlock pbf_header_block;
if (!pbf_header_block.ParseFromArray(data.data(), static_cast_with_assert<int>(data.size()))) {
throw osmium::pbf_error("failed to parse HeaderBlock");
}
for (int i=0; i < pbf_header_block.required_features_size(); ++i) {
const std::string& feature = pbf_header_block.required_features(i);
if (feature == "OsmSchema-V0.6") continue;
if (feature == "DenseNodes") {
m_header.set("pbf_dense_nodes", true);
continue;
}
if (feature == "HistoricalInformation") {
m_header.set_has_multiple_object_versions(true);
continue;
}
throw osmium::pbf_error(std::string("required feature not supported: ") + feature);
}
for (int i=0; i < pbf_header_block.optional_features_size(); ++i) {
const std::string& feature = pbf_header_block.optional_features(i);
m_header.set("pbf_optional_feature_" + std::to_string(i), feature);
}
if (pbf_header_block.has_writingprogram()) {
m_header.set("generator", pbf_header_block.writingprogram());
}
if (pbf_header_block.has_bbox()) {
const OSMPBF::HeaderBBox& pbf_bbox = pbf_header_block.bbox();
const int64_t resolution_convert = OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision;
osmium::Box box;
box.extend(osmium::Location(pbf_bbox.left() / resolution_convert, pbf_bbox.bottom() / resolution_convert));
box.extend(osmium::Location(pbf_bbox.right() / resolution_convert, pbf_bbox.top() / resolution_convert));
m_header.add_box(box);
}
if (pbf_header_block.has_osmosis_replication_timestamp()) {
m_header.set("osmosis_replication_timestamp", osmium::Timestamp(pbf_header_block.osmosis_replication_timestamp()).to_iso());
}
if (pbf_header_block.has_osmosis_replication_sequence_number()) {
m_header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.osmosis_replication_sequence_number()));
}
if (pbf_header_block.has_osmosis_replication_base_url()) {
m_header.set("osmosis_replication_base_url", pbf_header_block.osmosis_replication_base_url());
}
}
public:
friend class BlobParser<HeaderBlobParser>;
HeaderBlobParser(const int size, InputQueueReader& input_queue_reader, osmium::io::Header& header) :
BlobParser(size, 0, input_queue_reader),
m_header(header) {
}
}; // class HeaderBlobParser
class DataBlobParser : public BlobParser<DataBlobParser> {
osmium::osm_entity_bits::type m_read_types;
osmium::memory::Buffer handle_blob(const std::string& data) {
PBFPrimitiveBlockParser parser(data.data(), static_cast_with_assert<int>(data.size()), m_read_types);
return std::move(parser());
}
public:
friend class BlobParser<DataBlobParser>;
DataBlobParser(const int size, const int blob_num, InputQueueReader& input_queue_reader, osmium::osm_entity_bits::type read_types) :
BlobParser(size, blob_num, input_queue_reader),
m_read_types(read_types) {
}
}; // class DataBlobParser
/**
* Class for parsing PBF files.
*/
@ -549,25 +85,50 @@ namespace osmium {
bool m_use_thread_pool;
queue_type m_queue;
const size_t m_max_work_queue_size;
const size_t m_max_buffer_queue_size;
std::atomic<bool> m_done;
std::thread m_reader;
OSMPBF::BlobHeader m_blob_header;
InputQueueReader m_input_queue_reader;
osmium::thread::Queue<std::string>& m_input_queue;
std::string m_input_buffer;
/**
* Read BlobHeader by first reading the size and then the BlobHeader.
* The BlobHeader contains a type field (which is checked against
* the expected type) and a size field.
* Read the given number of bytes from the input queue.
*
* @param expected_type Expected type of data ("OSMHeader" or "OSMData").
* @param size Number of bytes to read
* @returns String with the data
* @throws osmium::pbf_error If size bytes can't be read
*/
std::string read_from_input_queue(size_t size) {
while (m_input_buffer.size() < size) {
std::string new_data;
m_input_queue.wait_and_pop(new_data);
if (new_data.empty()) {
throw osmium::pbf_error("truncated data (EOF encountered)");
}
m_input_buffer += new_data;
}
std::string output { m_input_buffer.substr(size) };
m_input_buffer.resize(size);
std::swap(output, m_input_buffer);
return output;
}
/**
* Read BlobHeader by first reading the size and then the
* BlobHeader. The BlobHeader contains a type field (which is
* checked against the expected type) and a size field.
*
* @param expected_type Expected type of data ("OSMHeader" or
* "OSMData").
* @returns Size of the data read from BlobHeader (0 on EOF).
*/
int read_blob_header(const char* expected_type) {
size_t read_blob_header(const char* expected_type) {
uint32_t size_in_network_byte_order;
if (! m_input_queue_reader(reinterpret_cast<unsigned char*>(&size_in_network_byte_order), sizeof(size_in_network_byte_order))) {
try {
std::string input_data = read_from_input_queue(sizeof(size_in_network_byte_order));
size_in_network_byte_order = *reinterpret_cast<const uint32_t*>(input_data.data());
} catch (osmium::pbf_error&) {
return 0; // EOF
}
@ -576,47 +137,33 @@ namespace osmium {
throw osmium::pbf_error("invalid BlobHeader size (> max_blob_header_size)");
}
unsigned char blob_header_buffer[OSMPBF::max_blob_header_size];
if (! m_input_queue_reader(blob_header_buffer, size)) {
throw osmium::pbf_error("read error");
}
if (!m_blob_header.ParseFromArray(blob_header_buffer, static_cast<int>(size))) {
OSMPBF::BlobHeader blob_header;
if (!blob_header.ParseFromString(read_from_input_queue(size))) {
throw osmium::pbf_error("failed to parse BlobHeader");
}
if (std::strcmp(m_blob_header.type().c_str(), expected_type)) {
if (blob_header.type() != expected_type) {
throw osmium::pbf_error("blob does not have expected type (OSMHeader in first blob, OSMData in following blobs)");
}
return m_blob_header.datasize();
return static_cast<size_t>(blob_header.datasize());
}
void parse_osm_data(osmium::osm_entity_bits::type read_types) {
osmium::thread::set_thread_name("_osmium_pbf_in");
int n=0;
while (int size = read_blob_header("OSMData")) {
DataBlobParser data_blob_parser(size, n, m_input_queue_reader, read_types);
while (auto size = read_blob_header("OSMData")) {
if (m_use_thread_pool) {
m_queue.push(osmium::thread::Pool::instance().submit(data_blob_parser));
// if the work queue is getting too large, wait for a while
while (!m_done && osmium::thread::Pool::instance().queue_size() >= m_max_work_queue_size) {
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
m_queue.push(osmium::thread::Pool::instance().submit(DataBlobParser{read_from_input_queue(size), read_types}));
} else {
std::promise<osmium::memory::Buffer> promise;
m_queue.push(promise.get_future());
DataBlobParser data_blob_parser{read_from_input_queue(size), read_types};
promise.set_value(data_blob_parser());
}
++n;
// wait if the backlog of buffers with parsed data is too large
while (!m_done && m_queue.size() > m_max_buffer_queue_size) {
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
if (m_done) {
return;
}
@ -635,21 +182,16 @@ namespace osmium {
*/
PBFInputFormat(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue<std::string>& input_queue) :
osmium::io::detail::InputFormat(file, read_which_entities, input_queue),
m_use_thread_pool(true),
m_queue(),
m_max_work_queue_size(10), // XXX tune these settings
m_max_buffer_queue_size(20), // XXX tune these settings
m_use_thread_pool(osmium::config::use_pool_threads_for_pbf_parsing()),
m_queue(20, "pbf_parser_results"), // XXX
m_done(false),
m_input_queue_reader(input_queue) {
m_input_queue(input_queue),
m_input_buffer() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
// handle OSMHeader
int size = read_blob_header("OSMHeader");
{
HeaderBlobParser header_blob_parser(size, m_input_queue_reader, m_header);
header_blob_parser.doit();
}
auto size = read_blob_header("OSMHeader");
m_header = parse_header_blob(read_from_input_queue(size));
if (m_read_which_entities != osmium::osm_entity_bits::nothing) {
m_reader = std::thread(&PBFInputFormat::parse_osm_data, this, m_read_which_entities);

3
third_party/osmium/io/detail/pbf_output_format.hpp vendored
View File

@ -527,9 +527,6 @@ namespace osmium {
std::promise<std::string> promise;
m_output_queue.push(promise.get_future());
promise.set_value(serialize_blob("OSMData", pbf_primitive_block, m_use_compression));
while (m_output_queue.size() > 10) {
std::this_thread::sleep_for(std::chrono::milliseconds(100)); // XXX
}
// clear the PrimitiveBlock struct
pbf_primitive_block.Clear();

449
third_party/osmium/io/detail/pbf_parser.hpp vendored Normal file
View File

@ -0,0 +1,449 @@
#ifndef OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP
#define OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <algorithm>
#include <osmpbf/osmpbf.h>
#include <osmium/builder/osm_object_builder.hpp>
#include <osmium/io/detail/pbf.hpp> // IWYU pragma: export
#include <osmium/io/detail/zlib.hpp>
#include <osmium/io/header.hpp>
#include <osmium/osm/location.hpp>
#include <osmium/osm/node.hpp>
#include <osmium/osm/types.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/osm/entity_bits.hpp>
#include <osmium/util/cast.hpp>
namespace osmium {
namespace io {
namespace detail {
class PBFPrimitiveBlockParser {
static constexpr size_t initial_buffer_size = 2 * 1024 * 1024;
const std::string& m_data;
const OSMPBF::StringTable* m_stringtable;
int64_t m_lon_offset;
int64_t m_lat_offset;
int64_t m_date_factor;
int32_t m_granularity;
osmium::osm_entity_bits::type m_read_types;
osmium::memory::Buffer m_buffer;
PBFPrimitiveBlockParser(const PBFPrimitiveBlockParser&) = delete;
PBFPrimitiveBlockParser(PBFPrimitiveBlockParser&&) = delete;
PBFPrimitiveBlockParser& operator=(const PBFPrimitiveBlockParser&) = delete;
PBFPrimitiveBlockParser& operator=(PBFPrimitiveBlockParser&&) = delete;
public:
explicit PBFPrimitiveBlockParser(const std::string& data, osmium::osm_entity_bits::type read_types) :
m_data(data),
m_stringtable(nullptr),
m_lon_offset(0),
m_lat_offset(0),
m_date_factor(1000),
m_granularity(100),
m_read_types(read_types),
m_buffer(initial_buffer_size) {
}
~PBFPrimitiveBlockParser() = default;
osmium::memory::Buffer operator()() {
OSMPBF::PrimitiveBlock pbf_primitive_block;
if (!pbf_primitive_block.ParseFromString(m_data)) {
throw osmium::pbf_error("failed to parse PrimitiveBlock");
}
m_stringtable = &pbf_primitive_block.stringtable();
m_lon_offset = pbf_primitive_block.lon_offset();
m_lat_offset = pbf_primitive_block.lat_offset();
m_date_factor = pbf_primitive_block.date_granularity() / 1000;
m_granularity = pbf_primitive_block.granularity();
for (int i=0; i < pbf_primitive_block.primitivegroup_size(); ++i) {
const OSMPBF::PrimitiveGroup& group = pbf_primitive_block.primitivegroup(i);
if (group.has_dense()) {
if (m_read_types & osmium::osm_entity_bits::node) parse_dense_node_group(group);
} else if (group.ways_size() != 0) {
if (m_read_types & osmium::osm_entity_bits::way) parse_way_group(group);
} else if (group.relations_size() != 0) {
if (m_read_types & osmium::osm_entity_bits::relation) parse_relation_group(group);
} else if (group.nodes_size() != 0) {
if (m_read_types & osmium::osm_entity_bits::node) parse_node_group(group);
} else {
throw osmium::pbf_error("group of unknown type");
}
}
return std::move(m_buffer);
}
private:
template <class TBuilder, class TPBFObject>
void parse_attributes(TBuilder& builder, const TPBFObject& pbf_object) {
auto& object = builder.object();
object.set_id(pbf_object.id());
if (pbf_object.has_info()) {
object.set_version(static_cast_with_assert<object_version_type>(pbf_object.info().version()))
.set_changeset(static_cast_with_assert<changeset_id_type>(pbf_object.info().changeset()))
.set_timestamp(pbf_object.info().timestamp() * m_date_factor)
.set_uid_from_signed(pbf_object.info().uid());
if (pbf_object.info().has_visible()) {
object.set_visible(pbf_object.info().visible());
}
builder.add_user(m_stringtable->s(static_cast_with_assert<int>(pbf_object.info().user_sid())));
} else {
builder.add_user("", 1);
}
}
void parse_node_group(const OSMPBF::PrimitiveGroup& group) {
for (int i=0; i < group.nodes_size(); ++i) {
osmium::builder::NodeBuilder builder(m_buffer);
const OSMPBF::Node& pbf_node = group.nodes(i);
parse_attributes(builder, pbf_node);
if (builder.object().visible()) {
builder.object().set_location(osmium::Location(
(pbf_node.lon() * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision),
(pbf_node.lat() * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision)));
}
if (pbf_node.keys_size() > 0) {
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
for (int tag=0; tag < pbf_node.keys_size(); ++tag) {
tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_node.keys(tag))),
m_stringtable->s(static_cast<int>(pbf_node.vals(tag))));
}
}
m_buffer.commit();
}
}
void parse_way_group(const OSMPBF::PrimitiveGroup& group) {
for (int i=0; i < group.ways_size(); ++i) {
osmium::builder::WayBuilder builder(m_buffer);
const OSMPBF::Way& pbf_way = group.ways(i);
parse_attributes(builder, pbf_way);
if (pbf_way.refs_size() > 0) {
osmium::builder::WayNodeListBuilder wnl_builder(m_buffer, &builder);
int64_t ref = 0;
for (int n=0; n < pbf_way.refs_size(); ++n) {
ref += pbf_way.refs(n);
wnl_builder.add_node_ref(ref);
}
}
if (pbf_way.keys_size() > 0) {
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
for (int tag=0; tag < pbf_way.keys_size(); ++tag) {
tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_way.keys(tag))),
m_stringtable->s(static_cast<int>(pbf_way.vals(tag))));
}
}
m_buffer.commit();
}
}
void parse_relation_group(const OSMPBF::PrimitiveGroup& group) {
for (int i=0; i < group.relations_size(); ++i) {
osmium::builder::RelationBuilder builder(m_buffer);
const OSMPBF::Relation& pbf_relation = group.relations(i);
parse_attributes(builder, pbf_relation);
if (pbf_relation.types_size() > 0) {
osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder);
int64_t ref = 0;
for (int n=0; n < pbf_relation.types_size(); ++n) {
ref += pbf_relation.memids(n);
rml_builder.add_member(osmpbf_membertype_to_item_type(pbf_relation.types(n)), ref, m_stringtable->s(pbf_relation.roles_sid(n)));
}
}
if (pbf_relation.keys_size() > 0) {
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
for (int tag=0; tag < pbf_relation.keys_size(); ++tag) {
tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_relation.keys(tag))),
m_stringtable->s(static_cast<int>(pbf_relation.vals(tag))));
}
}
m_buffer.commit();
}
}
int add_tags(const OSMPBF::DenseNodes& dense, int n, osmium::builder::NodeBuilder* builder) {
if (n >= dense.keys_vals_size()) {
return n;
}
if (dense.keys_vals(n) == 0) {
return n+1;
}
osmium::builder::TagListBuilder tl_builder(m_buffer, builder);
while (n < dense.keys_vals_size()) {
int tag_key_pos = dense.keys_vals(n++);
if (tag_key_pos == 0) {
break;
}
tl_builder.add_tag(m_stringtable->s(tag_key_pos),
m_stringtable->s(dense.keys_vals(n)));
++n;
}
return n;
}
void parse_dense_node_group(const OSMPBF::PrimitiveGroup& group) {
int64_t last_dense_id = 0;
int64_t last_dense_latitude = 0;
int64_t last_dense_longitude = 0;
int64_t last_dense_uid = 0;
int64_t last_dense_user_sid = 0;
int64_t last_dense_changeset = 0;
int64_t last_dense_timestamp = 0;
int last_dense_tag = 0;
const OSMPBF::DenseNodes& dense = group.dense();
for (int i=0; i < dense.id_size(); ++i) {
bool visible = true;
last_dense_id += dense.id(i);
last_dense_latitude += dense.lat(i);
last_dense_longitude += dense.lon(i);
if (dense.has_denseinfo()) {
last_dense_changeset += dense.denseinfo().changeset(i);
last_dense_timestamp += dense.denseinfo().timestamp(i);
last_dense_uid += dense.denseinfo().uid(i);
last_dense_user_sid += dense.denseinfo().user_sid(i);
if (dense.denseinfo().visible_size() > 0) {
visible = dense.denseinfo().visible(i);
}
assert(last_dense_changeset >= 0);
assert(last_dense_timestamp >= 0);
assert(last_dense_uid >= -1);
assert(last_dense_user_sid >= 0);
}
osmium::builder::NodeBuilder builder(m_buffer);
osmium::Node& node = builder.object();
node.set_id(last_dense_id);
if (dense.has_denseinfo()) {
auto v = dense.denseinfo().version(i);
assert(v > 0);
node.set_version(static_cast<osmium::object_version_type>(v));
node.set_changeset(static_cast<osmium::changeset_id_type>(last_dense_changeset));
node.set_timestamp(last_dense_timestamp * m_date_factor);
node.set_uid_from_signed(static_cast<osmium::signed_user_id_type>(last_dense_uid));
node.set_visible(visible);
builder.add_user(m_stringtable->s(static_cast<int>(last_dense_user_sid)));
} else {
builder.add_user("", 1);
}
if (visible) {
builder.object().set_location(osmium::Location(
(last_dense_longitude * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision),
(last_dense_latitude * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision)));
}
last_dense_tag = add_tags(dense, last_dense_tag, &builder);
m_buffer.commit();
}
}
}; // class PBFPrimitiveBlockParser
/**
* PBF blobs can optionally be packed with the zlib algorithm.
* This function returns the raw data (if it was unpacked) or
* the unpacked data (if it was packed).
*
* @param input_data Reference to input data.
* @returns Unpacked data
* @throws osmium::pbf_error If there was a problem parsing the PBF
*/
inline std::unique_ptr<const std::string> unpack_blob(const std::string& input_data) {
OSMPBF::Blob pbf_blob;
if (!pbf_blob.ParseFromString(input_data)) {
throw osmium::pbf_error("failed to parse blob");
}
if (pbf_blob.has_raw()) {
return std::unique_ptr<std::string>(pbf_blob.release_raw());
} else if (pbf_blob.has_zlib_data()) {
auto raw_size = pbf_blob.raw_size();
assert(raw_size >= 0);
assert(raw_size <= OSMPBF::max_uncompressed_blob_size);
return osmium::io::detail::zlib_uncompress(pbf_blob.zlib_data(), static_cast<unsigned long>(raw_size));
} else if (pbf_blob.has_lzma_data()) {
throw osmium::pbf_error("lzma blobs not implemented");
} else {
throw osmium::pbf_error("blob contains no data");
}
}
/**
* Parse blob as a HeaderBlock.
*
* @param input_buffer Blob data
* @returns Header object
* @throws osmium::pbf_error If there was a parsing error
*/
inline osmium::io::Header parse_header_blob(const std::string& input_buffer) {
const std::unique_ptr<const std::string> data = unpack_blob(input_buffer);
OSMPBF::HeaderBlock pbf_header_block;
if (!pbf_header_block.ParseFromString(*data)) {
throw osmium::pbf_error("failed to parse HeaderBlock");
}
osmium::io::Header header;
for (int i=0; i < pbf_header_block.required_features_size(); ++i) {
const std::string& feature = pbf_header_block.required_features(i);
if (feature == "OsmSchema-V0.6") continue;
if (feature == "DenseNodes") {
header.set("pbf_dense_nodes", true);
continue;
}
if (feature == "HistoricalInformation") {
header.set_has_multiple_object_versions(true);
continue;
}
throw osmium::pbf_error(std::string("required feature not supported: ") + feature);
}
for (int i=0; i < pbf_header_block.optional_features_size(); ++i) {
const std::string& feature = pbf_header_block.optional_features(i);
header.set("pbf_optional_feature_" + std::to_string(i), feature);
}
if (pbf_header_block.has_writingprogram()) {
header.set("generator", pbf_header_block.writingprogram());
}
if (pbf_header_block.has_bbox()) {
const OSMPBF::HeaderBBox& pbf_bbox = pbf_header_block.bbox();
const int64_t resolution_convert = OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision;
osmium::Box box;
box.extend(osmium::Location(pbf_bbox.left() / resolution_convert, pbf_bbox.bottom() / resolution_convert));
box.extend(osmium::Location(pbf_bbox.right() / resolution_convert, pbf_bbox.top() / resolution_convert));
header.add_box(box);
}
if (pbf_header_block.has_osmosis_replication_timestamp()) {
header.set("osmosis_replication_timestamp", osmium::Timestamp(pbf_header_block.osmosis_replication_timestamp()).to_iso());
}
if (pbf_header_block.has_osmosis_replication_sequence_number()) {
header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.osmosis_replication_sequence_number()));
}
if (pbf_header_block.has_osmosis_replication_base_url()) {
header.set("osmosis_replication_base_url", pbf_header_block.osmosis_replication_base_url());
}
return header;
}
class DataBlobParser {
std::string m_input_buffer;
osmium::osm_entity_bits::type m_read_types;
public:
DataBlobParser(std::string&& input_buffer, osmium::osm_entity_bits::type read_types) :
m_input_buffer(std::move(input_buffer)),
m_read_types(read_types) {
if (input_buffer.size() > OSMPBF::max_uncompressed_blob_size) {
throw osmium::pbf_error(std::string("invalid blob size: " + std::to_string(input_buffer.size())));
}
}
DataBlobParser(const DataBlobParser& other) :
m_input_buffer(std::move(other.m_input_buffer)),
m_read_types(other.m_read_types) {
}
DataBlobParser& operator=(const DataBlobParser&) = delete;
osmium::memory::Buffer operator()() {
const std::unique_ptr<const std::string> data = unpack_blob(m_input_buffer);
PBFPrimitiveBlockParser parser(*data, m_read_types);
return std::move(parser());
}
}; // class DataBlobParser
} // namespace detail
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP

2
third_party/osmium/io/detail/read_thread.hpp vendored
View File

@ -41,8 +41,8 @@ DEALINGS IN THE SOFTWARE.
#include <utility>
#include <osmium/io/compression.hpp>
#include <osmium/thread/name.hpp>
#include <osmium/thread/queue.hpp>
#include <osmium/thread/util.hpp>
namespace osmium {

2
third_party/osmium/io/detail/write_thread.hpp vendored
View File

@ -38,7 +38,7 @@ DEALINGS IN THE SOFTWARE.
#include <osmium/io/compression.hpp>
#include <osmium/io/detail/output_format.hpp>
#include <osmium/thread/name.hpp>
#include <osmium/thread/util.hpp>
namespace osmium {

29
third_party/osmium/io/detail/xml_input_format.hpp vendored
View File

@ -57,6 +57,7 @@ DEALINGS IN THE SOFTWARE.
#include <osmium/builder/builder.hpp>
#include <osmium/builder/osm_object_builder.hpp>
#include <osmium/io/detail/input_format.hpp>
#include <osmium/io/error.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/io/header.hpp>
#include <osmium/memory/buffer.hpp>
@ -68,7 +69,7 @@ DEALINGS IN THE SOFTWARE.
#include <osmium/osm/object.hpp>
#include <osmium/osm/types.hpp>
#include <osmium/thread/queue.hpp>
#include <osmium/thread/checked_task.hpp>
#include <osmium/thread/util.hpp>
#include <osmium/util/cast.hpp>
namespace osmium {
@ -178,8 +179,6 @@ namespace osmium {
osmium::osm_entity_bits::type m_read_types;
size_t m_max_queue_size;
std::atomic<bool>& m_done;
/**
@ -270,7 +269,6 @@ namespace osmium {
m_queue(queue),
m_header_promise(header_promise),
m_read_types(read_types),
m_max_queue_size(100),
m_done(done) {
}
@ -297,15 +295,14 @@ namespace osmium {
m_queue(other.m_queue),
m_header_promise(other.m_header_promise),
m_read_types(other.m_read_types),
m_max_queue_size(100),
m_done(other.m_done) {
}
XMLParser(XMLParser&& other) = default;
XMLParser(XMLParser&&) = default;
XMLParser& operator=(const XMLParser&) = delete;
XMLParser& operator=(XMLParser&& other) = default;
XMLParser& operator=(XMLParser&&) = default;
~XMLParser() = default;
@ -650,10 +647,6 @@ namespace osmium {
m_queue.push(std::move(m_buffer));
osmium::memory::Buffer buffer(buffer_size);
std::swap(m_buffer, buffer);
while (m_queue.size() > m_max_queue_size) {
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
}
}
@ -661,12 +654,12 @@ namespace osmium {
class XMLInputFormat : public osmium::io::detail::InputFormat {
static constexpr size_t m_max_queue_size = 100;
static constexpr size_t max_queue_size = 100;
osmium::thread::Queue<osmium::memory::Buffer> m_queue;
std::atomic<bool> m_done;
std::promise<osmium::io::Header> m_header_promise;
osmium::thread::CheckedTask<XMLParser> m_parser_task;
std::future<bool> m_parser_future;
public:
@ -679,10 +672,10 @@ namespace osmium {
*/
explicit XMLInputFormat(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue<std::string>& input_queue) :
osmium::io::detail::InputFormat(file, read_which_entities, input_queue),
m_queue(),
m_queue(max_queue_size, "xml_parser_results"),
m_done(false),
m_header_promise(),
m_parser_task(input_queue, m_queue, m_header_promise, read_which_entities, m_done) {
m_parser_future(std::async(std::launch::async, XMLParser(input_queue, m_queue, m_header_promise, read_which_entities, m_done))) {
}
~XMLInputFormat() {
@ -694,7 +687,7 @@ namespace osmium {
}
virtual osmium::io::Header header() override final {
m_parser_task.check_for_exception();
osmium::thread::check_for_exception(m_parser_future);
return m_header_promise.get_future().get();
}
@ -704,13 +697,13 @@ namespace osmium {
m_queue.wait_and_pop(buffer);
}
m_parser_task.check_for_exception();
osmium::thread::check_for_exception(m_parser_future);
return buffer;
}
void close() {
m_done = true;
m_parser_task.close();
osmium::thread::wait_until_done(m_parser_future);
}
}; // class XMLInputFormat

10
third_party/osmium/io/detail/xml_output_format.hpp vendored
View File

@ -233,8 +233,8 @@ namespace osmium {
XMLOutputBlock(const XMLOutputBlock&) = delete;
XMLOutputBlock& operator=(const XMLOutputBlock&) = delete;
XMLOutputBlock(XMLOutputBlock&& other) = default;
XMLOutputBlock& operator=(XMLOutputBlock&& other) = default;
XMLOutputBlock(XMLOutputBlock&&) = default;
XMLOutputBlock& operator=(XMLOutputBlock&&) = default;
std::string operator()() {
osmium::apply(m_input_buffer.cbegin(), m_input_buffer.cend(), *this);
@ -405,11 +405,7 @@ namespace osmium {
}
void write_buffer(osmium::memory::Buffer&& buffer) override final {
osmium::thread::SharedPtrWrapper<XMLOutputBlock> output_block(std::move(buffer), m_write_visible_flag, m_file.is_true("xml_change_format"));
m_output_queue.push(osmium::thread::Pool::instance().submit(std::move(output_block)));
while (m_output_queue.size() > 10) {
std::this_thread::sleep_for(std::chrono::milliseconds(100)); // XXX
}
m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_write_visible_flag, m_file.is_true("xml_change_format")}));
}
void write_header(const osmium::io::Header& header) override final {

7
third_party/osmium/io/detail/zlib.hpp vendored
View File

@ -35,6 +35,7 @@ DEALINGS IN THE SOFTWARE.
#define OSMIUM_LINK_WITH_LIBS_ZLIB -lz
#include <memory>
#include <stdexcept>
#include <string>
@ -76,10 +77,10 @@ namespace osmium {
* @param raw_size Size of uncompressed data.
* @returns Uncompressed data.
*/
inline std::string zlib_uncompress(const std::string& input, unsigned long raw_size) {
std::string output(raw_size, '\0');
inline std::unique_ptr<std::string> zlib_uncompress(const std::string& input, unsigned long raw_size) {
auto output = std::unique_ptr<std::string>(new std::string(raw_size, '\0'));
if (::uncompress(reinterpret_cast<unsigned char*>(const_cast<char *>(output.data())),
if (::uncompress(reinterpret_cast<unsigned char*>(const_cast<char *>(output->data())),
&raw_size,
reinterpret_cast<const unsigned char*>(input.data()),
input.size()) != Z_OK) {

57
third_party/osmium/io/error.hpp vendored Normal file
View File

@ -0,0 +1,57 @@
#ifndef OSMIUM_IO_ERROR_HPP
#define OSMIUM_IO_ERROR_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <stdexcept>
namespace osmium {
/**
* Exception thrown when some kind of input/output operation failed.
*/
struct io_error : public std::runtime_error {
io_error(const std::string& what) :
std::runtime_error(what) {
}
io_error(const char* what) :
std::runtime_error(what) {
}
}; // struct io_error
} // namespace osmium
#endif // OSMIUM_IO_ERROR_HPP

23
third_party/osmium/io/file.hpp vendored
View File

@ -45,21 +45,6 @@ DEALINGS IN THE SOFTWARE.
namespace osmium {
/**
* Exception thrown when some kind of input/output operation failed.
*/
struct io_error : public std::runtime_error {
io_error(const std::string& what) :
std::runtime_error(what) {
}
io_error(const char* what) :
std::runtime_error(what) {
}
}; // struct io_error
/**
* @brief Everything related to input and output of OSM data.
*/
@ -162,11 +147,11 @@ namespace osmium {
}
}
File(const File& other) = default;
File& operator=(const File& other) = default;
File(const File&) = default;
File& operator=(const File&) = default;
File(File&& other) = default;
File& operator=(File&& other) = default;
File(File&&) = default;
File& operator=(File&&) = default;
~File() = default;

15
third_party/osmium/io/reader.hpp vendored
View File

@ -59,8 +59,7 @@ DEALINGS IN THE SOFTWARE.
#include <osmium/io/header.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/osm/entity_bits.hpp>
#include <osmium/thread/checked_task.hpp>
#include <osmium/thread/name.hpp>
#include <osmium/thread/util.hpp>
#include <osmium/thread/queue.hpp>
namespace osmium {
@ -83,7 +82,7 @@ namespace osmium {
osmium::thread::Queue<std::string> m_input_queue;
std::unique_ptr<osmium::io::Decompressor> m_decompressor;
osmium::thread::CheckedTask<detail::ReadThread> m_read_task;
std::future<bool> m_read_future;
std::unique_ptr<osmium::io::detail::InputFormat> m_input;
@ -173,11 +172,11 @@ namespace osmium {
m_read_which_entities(read_which_entities),
m_input_done(false),
m_childpid(0),
m_input_queue(),
m_input_queue(20, "raw_input"), // XXX
m_decompressor(m_file.buffer() ?
osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), m_file.buffer(), m_file.buffer_size()) :
osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), open_input_file_or_url(m_file.filename(), &m_childpid))),
m_read_task(m_input_queue, m_decompressor.get(), m_input_done),
m_read_future(std::async(std::launch::async, detail::ReadThread(m_input_queue, m_decompressor.get(), m_input_done))),
m_input(osmium::io::detail::InputFormatFactory::instance().create_input(m_file, m_read_which_entities, m_input_queue)) {
}
@ -228,7 +227,7 @@ namespace osmium {
}
#endif
m_read_task.close();
osmium::thread::wait_until_done(m_read_future);
}
/**
@ -251,7 +250,7 @@ namespace osmium {
osmium::memory::Buffer read() {
// If an exception happened in the input thread, re-throw
// it in this (the main) thread.
m_read_task.check_for_exception();
osmium::thread::check_for_exception(m_read_future);
if (m_read_which_entities == osmium::osm_entity_bits::nothing || m_input_done) {
// If the caller didn't want anything but the header, it will
@ -270,7 +269,7 @@ namespace osmium {
* Has the end of file been reached? This is set after the last
* data has been read. It is also set by calling close().
*/
bool eof() {
bool eof() const {
return m_input_done;
}

14
third_party/osmium/io/writer.hpp vendored
View File

@ -45,7 +45,7 @@ DEALINGS IN THE SOFTWARE.
#include <osmium/io/header.hpp>
#include <osmium/io/overwrite.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/thread/checked_task.hpp>
#include <osmium/thread/util.hpp>
namespace osmium {
@ -61,12 +61,13 @@ namespace osmium {
osmium::io::File m_file;
std::unique_ptr<osmium::io::detail::OutputFormat> m_output;
osmium::io::detail::data_queue_type m_output_queue;
std::unique_ptr<osmium::io::detail::OutputFormat> m_output;
std::unique_ptr<osmium::io::Compressor> m_compressor;
osmium::thread::CheckedTask<detail::WriteThread> m_write_task;
std::future<bool> m_write_future;
public:
@ -87,9 +88,10 @@ namespace osmium {
*/
explicit Writer(const osmium::io::File& file, const osmium::io::Header& header = osmium::io::Header(), overwrite allow_overwrite = overwrite::no) :
m_file(file),
m_output_queue(20, "raw_output"), // XXX
m_output(osmium::io::detail::OutputFormatFactory::instance().create_output(m_file, m_output_queue)),
m_compressor(osmium::io::CompressionFactory::instance().create_compressor(file.compression(), osmium::io::detail::open_for_writing(m_file.filename(), allow_overwrite))),
m_write_task(m_output_queue, m_compressor.get()) {
m_write_future(std::async(std::launch::async, detail::WriteThread(m_output_queue, m_compressor.get()))) {
assert(!m_file.buffer());
m_output->write_header(header);
}
@ -115,7 +117,7 @@ namespace osmium {
* @throws Some form of std::runtime_error when there is a problem.
*/
void operator()(osmium::memory::Buffer&& buffer) {
m_write_task.check_for_exception();
osmium::thread::check_for_exception(m_write_future);
if (buffer.committed() > 0) {
m_output->write_buffer(std::move(buffer));
}
@ -131,7 +133,7 @@ namespace osmium {
*/
void close() {
m_output->close();
m_write_task.close();
osmium::thread::wait_until_done(m_write_future);
}
}; // class Writer

13
third_party/osmium/memory/item.hpp vendored
View File

@ -35,6 +35,7 @@ DEALINGS IN THE SOFTWARE.
#include <cstddef>
#include <cstdint>
#include <type_traits>
namespace osmium {
@ -52,11 +53,10 @@ namespace osmium {
// align datastructures to this many bytes
constexpr item_size_type align_bytes = 8;
inline size_t padded_length(size_t length) noexcept {
return (length + align_bytes - 1) & ~(align_bytes - 1);
}
inline item_size_type padded_length(item_size_type length) noexcept {
template <typename T>
inline T padded_length(T length) noexcept {
static_assert(std::is_integral<T>::value && std::is_unsigned<T>::value,
"Template parameter must be unsigned integral type");
return (length + align_bytes - 1) & ~(align_bytes - 1);
}
@ -122,7 +122,8 @@ namespace osmium {
explicit Item(item_size_type size=0, item_type type=item_type()) noexcept :
m_size(size),
m_type(type),
m_removed(false) {
m_removed(false),
m_padding(0) {
}
Item(const Item&) = delete;

16
third_party/osmium/osm/diff_object.hpp vendored
View File

@ -66,11 +66,11 @@ namespace osmium {
m_next(&next) {
}
DiffObject(const DiffObject& other) = default;
DiffObject& operator=(const DiffObject& other) = default;
DiffObject(const DiffObject&) = default;
DiffObject& operator=(const DiffObject&) = default;
DiffObject(DiffObject&& other) = default;
DiffObject& operator=(DiffObject&& other) = default;
DiffObject(DiffObject&&) = default;
DiffObject& operator=(DiffObject&&) = default;
const osmium::OSMObject& prev() const noexcept {
return *m_prev;
@ -127,11 +127,11 @@ namespace osmium {
DiffObject(prev, curr, next) {
}
DiffObjectDerived(const DiffObjectDerived& other) = default;
DiffObjectDerived& operator=(const DiffObjectDerived& other) = default;
DiffObjectDerived(const DiffObjectDerived&) = default;
DiffObjectDerived& operator=(const DiffObjectDerived&) = default;
DiffObjectDerived(DiffObjectDerived&& other) = default;
DiffObjectDerived& operator=(DiffObjectDerived&& other) = default;
DiffObjectDerived(DiffObjectDerived&&) = default;
DiffObjectDerived& operator=(DiffObjectDerived&&) = default;
const T& prev() const noexcept {
return *static_cast<const T*>(m_prev);

6
third_party/osmium/osm/entity_bits.hpp vendored
View File

@ -61,7 +61,9 @@ namespace osmium {
node = 0x01,
way = 0x02,
relation = 0x04,
nwr = 0x07, ///< node, way, or relation object
area = 0x08,
nwra = 0x0f, ///< node, way, relation, or area object
object = 0x0f, ///< node, way, relation, or area object
changeset = 0x10,
all = 0x1f ///< object or changeset
@ -81,6 +83,10 @@ namespace osmium {
return static_cast<type>(static_cast<int>(lhs) & static_cast<int> (rhs));
}
inline type operator~(const type value) noexcept {
return static_cast<type>(~static_cast<int>(value));
}
inline type operator&=(type& lhs, const type rhs) noexcept {
lhs = lhs & rhs;
return lhs;

4
third_party/osmium/relations/collector.hpp vendored
View File

@ -115,6 +115,8 @@ namespace osmium {
}; // class HandlerPass1
public:
/**
* This is the handler class for the second pass of the Collector.
*/
@ -244,7 +246,7 @@ namespace osmium {
int m_count_complete = 0;
typedef std::function<void(const osmium::memory::Buffer&)> callback_func_type;
typedef std::function<void(osmium::memory::Buffer&&)> callback_func_type;
callback_func_type m_callback;
static constexpr size_t initial_buffer_size = 1024 * 1024;

12
third_party/osmium/thread/function_wrapper.hpp vendored
View File

@ -40,12 +40,18 @@ namespace osmium {
namespace thread {
/**
* This function wrapper can collect move-only functions unlike
* std::function which needs copyable functions.
* Taken from the book "C++ Concurrency in Action".
*/
class function_wrapper {
struct impl_base {
virtual ~impl_base() = default;
virtual void call() = 0;
virtual ~impl_base() {
}
}; // struct impl_base
std::unique_ptr<impl_base> impl;
@ -58,7 +64,7 @@ namespace osmium {
m_functor(std::move(functor)) {
}
void call() {
void call() override {
m_functor();
}
}; // struct impl_type

48
third_party/osmium/thread/pool.hpp vendored
View File

@ -42,9 +42,10 @@ DEALINGS IN THE SOFTWARE.
#include <type_traits>
#include <vector>
#include <osmium/thread/queue.hpp>
#include <osmium/thread/name.hpp>
#include <osmium/thread/function_wrapper.hpp>
#include <osmium/thread/queue.hpp>
#include <osmium/thread/util.hpp>
#include <osmium/util/config.hpp>
namespace osmium {
@ -58,7 +59,10 @@ namespace osmium {
*/
class Pool {
// This class makes sure pool threads are joined when the pool is destructed
/**
* This class makes sure all pool threads will be joined when
* the pool is destructed.
*/
class thread_joiner {
std::vector<std::thread>& m_threads;
@ -108,20 +112,15 @@ namespace osmium {
*
* In all cases the minimum number of threads in the pool is 1.
*/
explicit Pool(int num_threads) :
explicit Pool(int num_threads, size_t max_queue_size) :
m_done(false),
m_work_queue(),
m_work_queue(max_queue_size, "work"),
m_threads(),
m_joiner(m_threads),
m_num_threads(num_threads) {
if (m_num_threads == 0) {
const char* env_threads = getenv("OSMIUM_POOL_THREADS");
if (env_threads) {
m_num_threads = std::atoi(env_threads);
} else {
m_num_threads = -2;
}
m_num_threads = osmium::config::get_pool_threads();
}
if (m_num_threads <= 0) {
@ -141,9 +140,10 @@ namespace osmium {
public:
static constexpr int default_num_threads = 0;
static constexpr size_t max_work_queue_size = 10;
static Pool& instance() {
static Pool pool(default_num_threads);
static Pool pool(default_num_threads, max_work_queue_size);
return pool;
}
@ -173,30 +173,6 @@ namespace osmium {
}; // class Pool
/**
* Wrapper for classes that can't be copied but need to be copyable for
* putting them in the pool.
*/
template <class TWrapped>
class SharedPtrWrapper {
std::shared_ptr<TWrapped> m_task;
public:
typedef typename std::result_of<TWrapped()>::type result_type;
template <typename... TArgs>
SharedPtrWrapper(TArgs&&... args) :
m_task(std::make_shared<TWrapped>(std::forward<TArgs>(args)...)) {
}
result_type operator()() {
return m_task->operator()();
}
}; // class SharedPtrWrapper
} // namespace thread
} // namespace osmium

74
third_party/osmium/thread/queue.hpp vendored
View File

@ -33,50 +33,100 @@ DEALINGS IN THE SOFTWARE.
*/
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <cstddef>
#include <mutex>
#include <queue>
#include <thread>
#include <utility>
namespace osmium {
namespace thread {
constexpr std::chrono::milliseconds full_queue_sleep_duration { 10 }; // XXX
/**
* A thread-safe queue.
*/
template <typename T>
class Queue {
/// Maximum size of this queue. If the queue is full pushing to
/// the queue will block.
const size_t m_max_size;
/// Name of this queue (for debugging only).
const std::string m_name;
mutable std::mutex m_mutex;
std::queue<T> m_queue;
/// Used to signal readers when data is available in the queue.
std::condition_variable m_data_available;
#ifdef OSMIUM_DEBUG_QUEUE_SIZE
/// The largest size the queue has been so far.
size_t m_largest_size;
/// The number of times the queue was full and a thread pushing
/// to the queue was blocked.
std::atomic<int> m_full_counter;
#endif
public:
Queue() :
/**
* Construct a multithreaded queue.
*
* @param max_size Maximum number of elements in the queue. Set to
* 0 for an unlimited size.
* @param name Optional name for this queue. (Used for debugging.)
*/
Queue(size_t max_size = 0, const std::string& name = "") :
m_max_size(max_size),
m_name(name),
m_mutex(),
m_queue(),
m_data_available() {
m_data_available()
#ifdef OSMIUM_DEBUG_QUEUE_SIZE
,
m_largest_size(0),
m_full_counter(0)
#endif
{
}
~Queue() {
#ifdef OSMIUM_DEBUG_QUEUE_SIZE
std::cerr << "queue '" << m_name << "' with max_size=" << m_max_size << " had largest size " << m_largest_size << " and was full " << m_full_counter << " times\n";
#endif
}
/**
* Push an element onto the queue. If the queue has a max size, this
* call will block if the queue is full.
*/
void push(T value) {
if (m_max_size) {
while (size() >= m_max_size) {
std::this_thread::sleep_for(full_queue_sleep_duration);
#ifdef OSMIUM_DEBUG_QUEUE_SIZE
++m_full_counter;
#endif
}
}
std::lock_guard<std::mutex> lock(m_mutex);
m_queue.push(std::move(value));
m_data_available.notify_one();
#ifdef OSMIUM_DEBUG_QUEUE_SIZE
if (m_largest_size < m_queue.size()) {
m_largest_size = m_queue.size();
}
size_t push_and_get_size(T&& value) {
std::lock_guard<std::mutex> lock(m_mutex);
m_queue.push(std::forward<T>(value));
#endif
m_data_available.notify_one();
return m_queue.size();
}
void push(T value, int) {
push(value);
}
void wait_and_pop(T& value) {

87
third_party/osmium/thread/util.hpp vendored Normal file
View File

@ -0,0 +1,87 @@
#ifndef OSMIUM_THREAD_UTIL_HPP
#define OSMIUM_THREAD_UTIL_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <chrono>
#include <future>
#ifdef __linux__
# include <sys/prctl.h>
#endif
namespace osmium {
namespace thread {
/**
* Check if the future resulted in an exception. This will re-throw
* the exception stored in the future if there was one. Otherwise it
* will just return.
*/
template <class T>
inline void check_for_exception(std::future<T>& future) {
if (future.valid() && future.wait_for(std::chrono::seconds(0)) == std::future_status::ready) {
future.get();
}
}
/**
* Wait until the given future becomes ready. Will block if the future
* is not ready. Can be called more than once unless future.get().
*/
template <class T>
inline void wait_until_done(std::future<T>& future) {
if (future.valid()) {
future.get();
}
}
/**
* Set name of current thread for debugging. This only works on Linux.
*/
#ifdef __linux__
inline void set_thread_name(const char* name) {
prctl(PR_SET_NAME, name, 0, 0, 0);
}
#else
inline void set_thread_name(const char*) {
// intentionally left blank
}
#endif
} // namespace thread
} // namespace osmium
#endif // OSMIUM_THREAD_UTIL_HPP

68
third_party/osmium/util/config.hpp vendored Normal file
View File

@ -0,0 +1,68 @@
#ifndef OSMIUM_UTIL_CONFIG_HPP
#define OSMIUM_UTIL_CONFIG_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013,2014 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cstdlib>
#include <cstring>
namespace osmium {
namespace config {
inline int get_pool_threads() {
const char* env = getenv("OSMIUM_POOL_THREADS");
if (env) {
return std::atoi(env);
}
return -2;
}
inline bool use_pool_threads_for_pbf_parsing() {
const char* env = getenv("OSMIUM_USE_POOL_THREADS_FOR_PBF_PARSING");
if (env) {
if (!strcasecmp(env, "off") ||
!strcasecmp(env, "false") ||
!strcasecmp(env, "no") ||
!strcasecmp(env, "0")) {
return false;
}
}
return true;
}
} // namespace config
} // namespace osmium
#endif // OSMIUM_UTIL_CONFIG_HPP