Squashed 'third_party/libosmium/' changes from f074d94..8bcd4ea
8bcd4ea Add explicit cast operator from osmium::Timestamp to uint32_t. 0b24814 Fixed the full_queue_sleep_duration const (again). 5e19dd2 Try different workaround for windows... df5d6c9 Fix build error on Windows. 1553adf Workaround for missing support for u8 string literals on Windows. aa5e44a Do not build benchmarks in Appveyor to speed up build. 2b22b31 Remove warning generated by assert by casting to largest type. 148c5e3 Fix inclusion of our own css file into doxygen documentation. 13bce6f Split out test thats fails on Windows and do not run it on appveyor. ca04757 Make CMake add_unit_test() function more flexible. 6c04a63 Tell CMake to output json file with compile commands. c0dd848 Fix indentation in some tests. dfa7e4b Formatting: Consistently use spaces around equal signs. 08fe6db Add change log, release version 2.1.0. 667192e Add XML tests. 28acfc7 Make sorting the PBF stringtables optional. 8184781 Fix PBFInputFormat. Use member variable instead of static variable. 2b48945 Fix comment. 6d37054 Rename m_done to m_quit_input_thread to clarify what it is for. bc23083 Fix race condition in PBF reader. 7fc380e Add various docs, noexcepts, asserts, and tests. aeaf4d7 Not a good idea to how overbroad patterns in .gitignore. c1ef2f9 Bugfix: Multipolygon collector was accessing non-existent NodeRef. 0ef9a13 Add noexcept and docs to some functions in NodeRefList class. da4d764 Workaround for Doxygen bug. e67d5d9 use absolute paths to osmium in YouCompleteMe configuration 38abeac remove template parameter from NodeRefList c47adf0 Add check that osm xml file has <osm> or <osmChange> as top-level element. 5e4af90 Updated version number to 2.0.0. 5b2bc3e Workaround in cmake test for sparsehash size. git-subtree-dir: third_party/libosmium git-subtree-split: 8bcd4ea771696812bbb08ebc58e3ee22d0538070
This commit is contained in:
Dennis Luxen
@@ -84,8 +84,9 @@ namespace osmium {
|
||||
class PBFInputFormat : public osmium::io::detail::InputFormat {
|
||||
|
||||
bool m_use_thread_pool;
|
||||
bool m_eof { false };
|
||||
queue_type m_queue;
|
||||
std::atomic<bool> m_done;
|
||||
std::atomic<bool> m_quit_input_thread;
|
||||
std::thread m_reader;
|
||||
osmium::thread::Queue<std::string>& m_input_queue;
|
||||
std::string m_input_buffer;
|
||||
@@ -151,7 +152,7 @@ namespace osmium {
|
||||
|
||||
void parse_osm_data(osmium::osm_entity_bits::type read_types) {
|
||||
osmium::thread::set_thread_name("_osmium_pbf_in");
|
||||
int n=0;
|
||||
int n = 0;
|
||||
while (auto size = read_blob_header("OSMData")) {
|
||||
|
||||
if (m_use_thread_pool) {
|
||||
@@ -164,11 +165,20 @@ namespace osmium {
|
||||
}
|
||||
++n;
|
||||
|
||||
if (m_done) {
|
||||
if (m_quit_input_thread) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
m_done = true;
|
||||
|
||||
// Send an empty buffer to signal the reader that we are
|
||||
// done.
|
||||
std::promise<osmium::memory::Buffer> promise;
|
||||
m_queue.push(promise.get_future());
|
||||
promise.set_value(osmium::memory::Buffer{});
|
||||
}
|
||||
|
||||
void signal_input_thread_to_quit() {
|
||||
m_quit_input_thread = true;
|
||||
}
|
||||
|
||||
public:
|
||||
@@ -184,7 +194,7 @@ namespace osmium {
|
||||
osmium::io::detail::InputFormat(file, read_which_entities),
|
||||
m_use_thread_pool(osmium::config::use_pool_threads_for_pbf_parsing()),
|
||||
m_queue(20, "pbf_parser_results"), // XXX
|
||||
m_done(false),
|
||||
m_quit_input_thread(false),
|
||||
m_input_queue(input_queue),
|
||||
m_input_buffer() {
|
||||
GOOGLE_PROTOBUF_VERIFY_VERSION;
|
||||
@@ -199,30 +209,37 @@ namespace osmium {
|
||||
}
|
||||
|
||||
~PBFInputFormat() {
|
||||
m_done = true;
|
||||
signal_input_thread_to_quit();
|
||||
if (m_reader.joinable()) {
|
||||
m_reader.join();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the next buffer with OSM data read from the PBF file.
|
||||
* Blocks if data is not available yet.
|
||||
* Returns the next buffer with OSM data read from the PBF
|
||||
* file. Blocks if data is not available yet.
|
||||
* Returns an empty buffer at end of input.
|
||||
*/
|
||||
osmium::memory::Buffer read() override {
|
||||
if (!m_done || !m_queue.empty()) {
|
||||
std::future<osmium::memory::Buffer> buffer_future;
|
||||
m_queue.wait_and_pop(buffer_future);
|
||||
try {
|
||||
return buffer_future.get();
|
||||
} catch (...) {
|
||||
m_done = true;
|
||||
throw;
|
||||
}
|
||||
osmium::memory::Buffer buffer;
|
||||
if (m_eof) {
|
||||
return buffer;
|
||||
}
|
||||
|
||||
return osmium::memory::Buffer();
|
||||
std::future<osmium::memory::Buffer> buffer_future;
|
||||
m_queue.wait_and_pop(buffer_future);
|
||||
|
||||
try {
|
||||
buffer = std::move(buffer_future.get());
|
||||
if (!buffer) {
|
||||
m_eof = true;
|
||||
}
|
||||
return buffer;
|
||||
} catch (...) {
|
||||
m_eof = true;
|
||||
signal_input_thread_to_quit();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
}; // class PBFInputFormat
|
||||
|
||||
@@ -294,6 +294,14 @@ namespace osmium {
|
||||
*/
|
||||
bool m_use_compression {true};
|
||||
|
||||
/**
|
||||
* Should the string tables in the data blocks be sorted?
|
||||
*
|
||||
* Not sorting the string tables makes writing PBF files
|
||||
* slightly faster.
|
||||
*/
|
||||
bool m_sort_stringtables { true };
|
||||
|
||||
/**
|
||||
* While the .osm.pbf-format is able to carry all meta information, it is
|
||||
* also able to omit this information to reduce size.
|
||||
@@ -340,6 +348,21 @@ namespace osmium {
|
||||
|
||||
///// Blob writing /////
|
||||
|
||||
void delta_encode_string_ids() {
|
||||
if (pbf_nodes && pbf_nodes->has_dense()) {
|
||||
OSMPBF::DenseNodes* dense = pbf_nodes->mutable_dense();
|
||||
|
||||
if (dense->has_denseinfo()) {
|
||||
OSMPBF::DenseInfo* denseinfo = dense->mutable_denseinfo();
|
||||
|
||||
for (int i = 0, l=denseinfo->user_sid_size(); i<l; ++i) {
|
||||
auto user_sid = denseinfo->user_sid(i);
|
||||
denseinfo->set_user_sid(i, m_delta_user_sid.update(user_sid));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Before a PrimitiveBlock gets serialized, all interim StringTable-ids needs to be
|
||||
* mapped to the associated real StringTable ids. This is done in this function.
|
||||
@@ -351,7 +374,7 @@ namespace osmium {
|
||||
// test, if the node-block has been allocated
|
||||
if (pbf_nodes) {
|
||||
// iterate over all nodes, passing them to the map_common_string_ids function
|
||||
for (int i=0, l=pbf_nodes->nodes_size(); i<l; ++i) {
|
||||
for (int i = 0, l=pbf_nodes->nodes_size(); i<l; ++i) {
|
||||
map_common_string_ids(pbf_nodes->mutable_nodes(i));
|
||||
}
|
||||
|
||||
@@ -363,7 +386,7 @@ namespace osmium {
|
||||
// in the densenodes structure keys and vals are encoded in an intermixed
|
||||
// array, individual nodes are seperated by a value of 0 (0 in the StringTable
|
||||
// is always unused). String-ids of 0 are thus kept alone.
|
||||
for (int i=0, l=dense->keys_vals_size(); i<l; ++i) {
|
||||
for (int i = 0, l=dense->keys_vals_size(); i<l; ++i) {
|
||||
// map interim string-ids > 0 to real string ids
|
||||
auto sid = dense->keys_vals(i);
|
||||
if (sid > 0) {
|
||||
@@ -377,7 +400,7 @@ namespace osmium {
|
||||
OSMPBF::DenseInfo* denseinfo = dense->mutable_denseinfo();
|
||||
|
||||
// iterate over all username string-ids
|
||||
for (int i=0, l=denseinfo->user_sid_size(); i<l; ++i) {
|
||||
for (int i = 0, l=denseinfo->user_sid_size(); i<l; ++i) {
|
||||
// map interim string-ids > 0 to real string ids
|
||||
auto user_sid = string_table.map_string_id(denseinfo->user_sid(i));
|
||||
|
||||
@@ -391,7 +414,7 @@ namespace osmium {
|
||||
// test, if the ways-block has been allocated
|
||||
if (pbf_ways) {
|
||||
// iterate over all ways, passing them to the map_common_string_ids function
|
||||
for (int i=0, l=pbf_ways->ways_size(); i<l; ++i) {
|
||||
for (int i = 0, l=pbf_ways->ways_size(); i<l; ++i) {
|
||||
map_common_string_ids(pbf_ways->mutable_ways(i));
|
||||
}
|
||||
}
|
||||
@@ -399,7 +422,7 @@ namespace osmium {
|
||||
// test, if the relations-block has been allocated
|
||||
if (pbf_relations) {
|
||||
// iterate over all relations
|
||||
for (int i=0, l=pbf_relations->relations_size(); i<l; ++i) {
|
||||
for (int i = 0, l=pbf_relations->relations_size(); i<l; ++i) {
|
||||
// get a pointer to the relation
|
||||
OSMPBF::Relation* relation = pbf_relations->mutable_relations(i);
|
||||
|
||||
@@ -408,7 +431,7 @@ namespace osmium {
|
||||
|
||||
// iterate over all relation members, mapping the interim string-ids
|
||||
// of the role to real string ids
|
||||
for (int mi=0; mi < relation->roles_sid_size(); ++mi) {
|
||||
for (int mi = 0; mi < relation->roles_sid_size(); ++mi) {
|
||||
relation->set_roles_sid(mi, string_table.map_string_id(relation->roles_sid(mi)));
|
||||
}
|
||||
}
|
||||
@@ -431,7 +454,7 @@ namespace osmium {
|
||||
}
|
||||
|
||||
// iterate over all tags and map the interim-ids of the key and the value to real ids
|
||||
for (int i=0, l=in->keys_size(); i<l; ++i) {
|
||||
for (int i = 0, l=in->keys_size(); i<l; ++i) {
|
||||
in->set_keys(i, string_table.map_string_id(in->keys(i)));
|
||||
in->set_vals(i, string_table.map_string_id(in->vals(i)));
|
||||
}
|
||||
@@ -518,11 +541,13 @@ namespace osmium {
|
||||
pbf_primitive_block.set_granularity(location_granularity());
|
||||
pbf_primitive_block.set_date_granularity(date_granularity());
|
||||
|
||||
// store the interim StringTable into the protobuf object
|
||||
string_table.store_stringtable(pbf_primitive_block.mutable_stringtable());
|
||||
string_table.store_stringtable(pbf_primitive_block.mutable_stringtable(), m_sort_stringtables);
|
||||
|
||||
// map all interim string ids to real ids
|
||||
map_string_ids();
|
||||
if (m_sort_stringtables) {
|
||||
map_string_ids();
|
||||
} else {
|
||||
delta_encode_string_ids();
|
||||
}
|
||||
|
||||
std::promise<std::string> promise;
|
||||
m_output_queue.push(promise.get_future());
|
||||
@@ -743,6 +768,9 @@ namespace osmium {
|
||||
if (file.get("pbf_compression") == "none" || file.get("pbf_compression") == "false") {
|
||||
m_use_compression = false;
|
||||
}
|
||||
if (file.get("pbf_sort_stringtables") == "false") {
|
||||
m_sort_stringtables = false;
|
||||
}
|
||||
if (file.get("pbf_add_metadata") == "false") {
|
||||
m_should_add_metadata = false;
|
||||
}
|
||||
|
||||
@@ -106,7 +106,7 @@ namespace osmium {
|
||||
m_date_factor = pbf_primitive_block.date_granularity() / 1000;
|
||||
m_granularity = pbf_primitive_block.granularity();
|
||||
|
||||
for (int i=0; i < pbf_primitive_block.primitivegroup_size(); ++i) {
|
||||
for (int i = 0; i < pbf_primitive_block.primitivegroup_size(); ++i) {
|
||||
const OSMPBF::PrimitiveGroup& group = pbf_primitive_block.primitivegroup(i);
|
||||
|
||||
if (group.has_dense()) {
|
||||
@@ -148,7 +148,7 @@ namespace osmium {
|
||||
}
|
||||
|
||||
void parse_node_group(const OSMPBF::PrimitiveGroup& group) {
|
||||
for (int i=0; i < group.nodes_size(); ++i) {
|
||||
for (int i = 0; i < group.nodes_size(); ++i) {
|
||||
osmium::builder::NodeBuilder builder(m_buffer);
|
||||
const OSMPBF::Node& pbf_node = group.nodes(i);
|
||||
parse_attributes(builder, pbf_node);
|
||||
@@ -161,7 +161,7 @@ namespace osmium {
|
||||
|
||||
if (pbf_node.keys_size() > 0) {
|
||||
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
|
||||
for (int tag=0; tag < pbf_node.keys_size(); ++tag) {
|
||||
for (int tag = 0; tag < pbf_node.keys_size(); ++tag) {
|
||||
tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_node.keys(tag))),
|
||||
m_stringtable->s(static_cast<int>(pbf_node.vals(tag))));
|
||||
}
|
||||
@@ -172,7 +172,7 @@ namespace osmium {
|
||||
}
|
||||
|
||||
void parse_way_group(const OSMPBF::PrimitiveGroup& group) {
|
||||
for (int i=0; i < group.ways_size(); ++i) {
|
||||
for (int i = 0; i < group.ways_size(); ++i) {
|
||||
osmium::builder::WayBuilder builder(m_buffer);
|
||||
const OSMPBF::Way& pbf_way = group.ways(i);
|
||||
parse_attributes(builder, pbf_way);
|
||||
@@ -180,7 +180,7 @@ namespace osmium {
|
||||
if (pbf_way.refs_size() > 0) {
|
||||
osmium::builder::WayNodeListBuilder wnl_builder(m_buffer, &builder);
|
||||
int64_t ref = 0;
|
||||
for (int n=0; n < pbf_way.refs_size(); ++n) {
|
||||
for (int n = 0; n < pbf_way.refs_size(); ++n) {
|
||||
ref += pbf_way.refs(n);
|
||||
wnl_builder.add_node_ref(ref);
|
||||
}
|
||||
@@ -188,7 +188,7 @@ namespace osmium {
|
||||
|
||||
if (pbf_way.keys_size() > 0) {
|
||||
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
|
||||
for (int tag=0; tag < pbf_way.keys_size(); ++tag) {
|
||||
for (int tag = 0; tag < pbf_way.keys_size(); ++tag) {
|
||||
tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_way.keys(tag))),
|
||||
m_stringtable->s(static_cast<int>(pbf_way.vals(tag))));
|
||||
}
|
||||
@@ -199,7 +199,7 @@ namespace osmium {
|
||||
}
|
||||
|
||||
void parse_relation_group(const OSMPBF::PrimitiveGroup& group) {
|
||||
for (int i=0; i < group.relations_size(); ++i) {
|
||||
for (int i = 0; i < group.relations_size(); ++i) {
|
||||
osmium::builder::RelationBuilder builder(m_buffer);
|
||||
const OSMPBF::Relation& pbf_relation = group.relations(i);
|
||||
parse_attributes(builder, pbf_relation);
|
||||
@@ -207,7 +207,7 @@ namespace osmium {
|
||||
if (pbf_relation.types_size() > 0) {
|
||||
osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder);
|
||||
int64_t ref = 0;
|
||||
for (int n=0; n < pbf_relation.types_size(); ++n) {
|
||||
for (int n = 0; n < pbf_relation.types_size(); ++n) {
|
||||
ref += pbf_relation.memids(n);
|
||||
rml_builder.add_member(osmpbf_membertype_to_item_type(pbf_relation.types(n)), ref, m_stringtable->s(pbf_relation.roles_sid(n)));
|
||||
}
|
||||
@@ -215,7 +215,7 @@ namespace osmium {
|
||||
|
||||
if (pbf_relation.keys_size() > 0) {
|
||||
osmium::builder::TagListBuilder tl_builder(m_buffer, &builder);
|
||||
for (int tag=0; tag < pbf_relation.keys_size(); ++tag) {
|
||||
for (int tag = 0; tag < pbf_relation.keys_size(); ++tag) {
|
||||
tl_builder.add_tag(m_stringtable->s(static_cast<int>(pbf_relation.keys(tag))),
|
||||
m_stringtable->s(static_cast<int>(pbf_relation.vals(tag))));
|
||||
}
|
||||
@@ -264,7 +264,7 @@ namespace osmium {
|
||||
|
||||
const OSMPBF::DenseNodes& dense = group.dense();
|
||||
|
||||
for (int i=0; i < dense.id_size(); ++i) {
|
||||
for (int i = 0; i < dense.id_size(); ++i) {
|
||||
bool visible = true;
|
||||
|
||||
last_dense_id += dense.id(i);
|
||||
@@ -361,7 +361,7 @@ namespace osmium {
|
||||
}
|
||||
|
||||
osmium::io::Header header;
|
||||
for (int i=0; i < pbf_header_block.required_features_size(); ++i) {
|
||||
for (int i = 0; i < pbf_header_block.required_features_size(); ++i) {
|
||||
const std::string& feature = pbf_header_block.required_features(i);
|
||||
|
||||
if (feature == "OsmSchema-V0.6") continue;
|
||||
@@ -377,7 +377,7 @@ namespace osmium {
|
||||
throw osmium::pbf_error(std::string("required feature not supported: ") + feature);
|
||||
}
|
||||
|
||||
for (int i=0; i < pbf_header_block.optional_features_size(); ++i) {
|
||||
for (int i = 0; i < pbf_header_block.optional_features_size(); ++i) {
|
||||
const std::string& feature = pbf_header_block.optional_features(i);
|
||||
header.set("pbf_optional_feature_" + std::to_string(i), feature);
|
||||
}
|
||||
|
||||
@@ -145,30 +145,44 @@ namespace osmium {
|
||||
* implementation) is that the string table is sorted first by reverse count (ie descending)
|
||||
* and then by reverse lexicographic order.
|
||||
*/
|
||||
void store_stringtable(OSMPBF::StringTable* st) {
|
||||
void store_stringtable(OSMPBF::StringTable* st, bool sort) {
|
||||
// add empty StringTable entry at index 0
|
||||
// StringTable index 0 is reserved as delimiter in the densenodes key/value list
|
||||
// this line also ensures that there's always a valid StringTable
|
||||
st->add_s("");
|
||||
|
||||
std::multimap<string_info, std::string> sortedbycount;
|
||||
if (sort) {
|
||||
std::multimap<string_info, std::string> sortedbycount;
|
||||
|
||||
m_id2id_map.resize(m_size+1);
|
||||
m_id2id_map.resize(m_size+1);
|
||||
|
||||
std::transform(m_strings.begin(), m_strings.end(),
|
||||
std::inserter(sortedbycount, sortedbycount.begin()),
|
||||
[](const std::pair<std::string, string_info>& p) {
|
||||
return std::pair<string_info, std::string>(p.second, p.first);
|
||||
});
|
||||
std::transform(m_strings.begin(), m_strings.end(),
|
||||
std::inserter(sortedbycount, sortedbycount.begin()),
|
||||
[](const std::pair<std::string, string_info>& p) {
|
||||
return std::pair<string_info, std::string>(p.second, p.first);
|
||||
});
|
||||
|
||||
string_id_type n=0;
|
||||
string_id_type n = 0;
|
||||
|
||||
for (const auto& mapping : sortedbycount) {
|
||||
// add the string of the current item to the pbf StringTable
|
||||
st->add_s(mapping.second);
|
||||
for (const auto& mapping : sortedbycount) {
|
||||
// add the string of the current item to the pbf StringTable
|
||||
st->add_s(mapping.second);
|
||||
|
||||
// store the mapping from the interim-id to the real id
|
||||
m_id2id_map[mapping.first.interim_id] = ++n;
|
||||
// store the mapping from the interim-id to the real id
|
||||
m_id2id_map[mapping.first.interim_id] = ++n;
|
||||
}
|
||||
} else {
|
||||
std::vector<std::pair<string_id_type, const char*>> sortedbyid;
|
||||
sortedbyid.reserve(m_strings.size());
|
||||
|
||||
for (const auto& p : m_strings) {
|
||||
sortedbyid.emplace_back(p.second.interim_id, p.first.c_str());
|
||||
}
|
||||
|
||||
std::sort(sortedbyid.begin(), sortedbyid.end());
|
||||
for (const auto& mapping : sortedbyid) {
|
||||
st->add_s(mapping.second);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -74,8 +74,8 @@ namespace osmium {
|
||||
|
||||
/**
|
||||
* Exception thrown when the XML parser failed. The exception contains
|
||||
* information about the place where the error happened and the type of
|
||||
* error.
|
||||
* (if available) information about the place where the error happened
|
||||
* and the type of error.
|
||||
*/
|
||||
struct xml_error : public io_error {
|
||||
|
||||
@@ -84,7 +84,7 @@ namespace osmium {
|
||||
XML_Error error_code;
|
||||
std::string error_string;
|
||||
|
||||
xml_error(XML_Parser parser) :
|
||||
explicit xml_error(XML_Parser parser) :
|
||||
io_error(std::string("XML parsing error at line ")
|
||||
+ std::to_string(XML_GetCurrentLineNumber(parser))
|
||||
+ ", column "
|
||||
@@ -97,8 +97,20 @@ namespace osmium {
|
||||
error_string(XML_ErrorString(error_code)) {
|
||||
}
|
||||
|
||||
explicit xml_error(const std::string& message) :
|
||||
io_error(message),
|
||||
line(0),
|
||||
column(0),
|
||||
error_code(),
|
||||
error_string(message) {
|
||||
}
|
||||
|
||||
}; // struct xml_error
|
||||
|
||||
/**
|
||||
* Exception thrown when an OSM XML files contains no version attribute
|
||||
* on the 'osm' element or if the version is unknown.
|
||||
*/
|
||||
struct format_version_error : public io_error {
|
||||
|
||||
std::string version;
|
||||
@@ -434,6 +446,8 @@ namespace osmium {
|
||||
if (m_header.get("version") == "") {
|
||||
throw osmium::format_version_error();
|
||||
}
|
||||
} else {
|
||||
throw osmium::xml_error(std::string("Unknown top-level element: ") + element);
|
||||
}
|
||||
m_context = context::top;
|
||||
break;
|
||||
|
||||
@@ -130,7 +130,7 @@ namespace osmium {
|
||||
const bool m_write_change_ops;
|
||||
|
||||
void write_spaces(int num) {
|
||||
for (; num!=0; --num) {
|
||||
for (; num != 0; --num) {
|
||||
*m_out += ' ';
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user