From d24ba6d13d66b0afbf42be6fab18e188dd855d97 Mon Sep 17 00:00:00 2001 From: Dennis Luxen Date: Fri, 16 Aug 2013 17:09:27 +0200 Subject: [PATCH] Mild perfomance fixed for PBF extraction --- Extractor/BaseParser.cpp | 5 +--- Extractor/ExtractorStructs.h | 2 +- Extractor/PBFParser.cpp | 44 ++++++++++++++++++++++-------------- Extractor/PBFParser.h | 33 ++++++++++++++------------- 4 files changed, 46 insertions(+), 38 deletions(-) diff --git a/Extractor/BaseParser.cpp b/Extractor/BaseParser.cpp index cf5530a4d..7885b095b 100644 --- a/Extractor/BaseParser.cpp +++ b/Extractor/BaseParser.cpp @@ -72,13 +72,10 @@ void BaseParser::ParseNodeInLua(ImportNode& n, lua_State* localLuaState) { } void BaseParser::ParseWayInLua(ExtractionWay& w, lua_State* localLuaState) { - if(2 > w.path.size()) { - return; - } luabind::call_function( localLuaState, "way_function", boost::ref(w) ); } -bool BaseParser::ShouldIgnoreRestriction(const std::string& except_tag_string) const { +bool BaseParser::ShouldIgnoreRestriction(const std::string & except_tag_string) const { //should this restriction be ignored? yes if there's an overlap between: //a) the list of modes in the except tag of the restriction (except_tag_string), ex: except=bus;bicycle //b) the lua profile defines a hierachy of modes, ex: [access, vehicle, bicycle] diff --git a/Extractor/ExtractorStructs.h b/Extractor/ExtractorStructs.h index 778778ed0..90251fcb0 100644 --- a/Extractor/ExtractorStructs.h +++ b/Extractor/ExtractorStructs.h @@ -154,7 +154,7 @@ struct CmpWayByID : public std::binary_function<_WayIDStartAndEndEdge, _WayIDSta struct Cmp : public std::binary_function { typedef NodeID value_type; - bool operator () (const NodeID & a, const NodeID & b) const { + bool operator () (const NodeID a, const NodeID b) const { return a < b; } value_type max_value() { diff --git a/Extractor/PBFParser.cpp b/Extractor/PBFParser.cpp index 413217cc8..a6085a08c 100644 --- a/Extractor/PBFParser.cpp +++ b/Extractor/PBFParser.cpp @@ -173,14 +173,14 @@ inline void PBFParser::parseDenseNode(_ThreadData * threadData) { extracted_nodes_vector[i].lon = COORDINATE_PRECISION*( ( double ) m_lastDenseLongitude * threadData->PBFprimitiveBlock.granularity() + threadData->PBFprimitiveBlock.lon_offset() ) / NANO; while (denseTagIndex < dense.keys_vals_size()) { const int tagValue = dense.keys_vals( denseTagIndex ); - if( 0==tagValue ) { + if( 0 == tagValue ) { ++denseTagIndex; break; } const int keyValue = dense.keys_vals ( denseTagIndex+1 ); - const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(tagValue).data(); - const std::string & value = threadData->PBFprimitiveBlock.stringtable().s(keyValue).data(); - extracted_nodes_vector[i].keyVals.insert(std::make_pair(key, value)); + const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(tagValue); + const std::string & value = threadData->PBFprimitiveBlock.stringtable().s(keyValue); + extracted_nodes_vector[i].keyVals.emplace(key, value); denseTagIndex += 2; } } @@ -191,7 +191,7 @@ inline void PBFParser::parseDenseNode(_ThreadData * threadData) { ParseNodeInLua( n, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) ); } - BOOST_FOREACH(ImportNode &n, extracted_nodes_vector) { + BOOST_FOREACH(const ImportNode &n, extracted_nodes_vector) { extractor_callbacks->nodeFunction(n); } } @@ -209,7 +209,8 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) { return; } const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ); - for(int i = 0; i < group.relations_size(); ++i ) { + + for(int i = 0, relation_size = group.relations_size(); i < relation_size; ++i ) { std::string except_tag_string; const OSMPBF::Relation& inputRelation = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).relations(i); bool isRestriction = false; @@ -241,8 +242,12 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) { if(isRestriction) { int64_t lastRef = 0; _RawRestrictionContainer currentRestrictionContainer(isOnlyRestriction); - for(int rolesIndex = 0; rolesIndex < inputRelation.roles_sid_size(); ++rolesIndex) { - std::string role(threadData->PBFprimitiveBlock.stringtable().s( inputRelation.roles_sid( rolesIndex ) ).data()); + for( + int rolesIndex = 0, last_role = inputRelation.roles_sid_size(); + rolesIndex < last_role; + ++rolesIndex + ) { + const std::string & role = threadData->PBFprimitiveBlock.stringtable().s( inputRelation.roles_sid( rolesIndex ) ); lastRef += inputRelation.memids(rolesIndex); if(!("from" == role || "to" == role || "via" == role)) { @@ -280,7 +285,6 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) { break; default: //should not happen - //cout << "unknown"; assert(false); break; } @@ -309,17 +313,23 @@ inline void PBFParser::parseWay(_ThreadData * threadData) { for(int j = 0; j < number_of_keys; ++j) { const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(inputWay.keys(j)); const std::string & val = threadData->PBFprimitiveBlock.stringtable().s(inputWay.vals(j)); - parsed_way_vector[i].keyVals.insert(std::make_pair(key, val)); + parsed_way_vector[i].keyVals.emplace(key, val); } } #pragma omp parallel for schedule ( guided ) for(int i = 0; i < number_of_ways; ++i) { ExtractionWay & w = parsed_way_vector[i]; - ParseWayInLua( w, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) ); + if(2 > w.path.size()) { + continue; + } + ParseWayInLua( w, scriptingEnvironment.getLuaStateForThreadID( omp_get_thread_num()) ); } BOOST_FOREACH(ExtractionWay & w, parsed_way_vector) { + if(2 > w.path.size()) { + continue; + } extractor_callbacks->wayFunction(w); } } @@ -330,21 +340,21 @@ inline void PBFParser::loadGroup(_ThreadData * threadData) { #endif const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ); - threadData->entityTypeIndicator = 0; - if ( group.nodes_size() != 0 ) { + threadData->entityTypeIndicator = TypeDummy; + if ( 0 != group.nodes_size() ) { threadData->entityTypeIndicator = TypeNode; } - if ( group.ways_size() != 0 ) { + if ( 0 != group.ways_size() ) { threadData->entityTypeIndicator = TypeWay; } - if ( group.relations_size() != 0 ) { + if ( 0 != group.relations_size() ) { threadData->entityTypeIndicator = TypeRelation; } if ( group.has_dense() ) { threadData->entityTypeIndicator = TypeDenseNode; - assert( group.dense().id_size() != 0 ); + assert( 0 != group.dense().id_size() ); } - assert( threadData->entityTypeIndicator != 0 ); + assert( threadData->entityTypeIndicator != TypeDummy ); } inline void PBFParser::loadBlock(_ThreadData * threadData) { diff --git a/Extractor/PBFParser.h b/Extractor/PBFParser.h index ec5035f3d..c8282f26b 100644 --- a/Extractor/PBFParser.h +++ b/Extractor/PBFParser.h @@ -44,16 +44,17 @@ class PBFParser : public BaseParser { enum EntityType { - TypeNode = 1, - TypeWay = 2, - TypeRelation = 4, + TypeDummy = 0, + TypeNode = 1, + TypeWay = 2, + TypeRelation = 4, TypeDenseNode = 8 - } ; + }; struct _ThreadData { int currentGroupID; int currentEntityID; - short entityTypeIndicator; + EntityType entityTypeIndicator; OSMPBF::BlobHeader PBFBlobHeader; OSMPBF::Blob PBFBlob; @@ -74,18 +75,18 @@ public: private: inline void ReadData(); inline void ParseData(); - inline void parseDenseNode(_ThreadData * threadData); - inline void parseNode(_ThreadData * ); - inline void parseRelation(_ThreadData * threadData); - inline void parseWay(_ThreadData * threadData); + inline void parseDenseNode (_ThreadData * threadData); + inline void parseNode (_ThreadData * threadData); + inline void parseRelation (_ThreadData * threadData); + inline void parseWay (_ThreadData * threadData); - inline void loadGroup(_ThreadData * threadData); - inline void loadBlock(_ThreadData * threadData); - inline bool readPBFBlobHeader(std::fstream& stream, _ThreadData * threadData); - inline bool unpackZLIB(std::fstream &, _ThreadData * threadData); - inline bool unpackLZMA(std::fstream &, _ThreadData * ); - inline bool readBlob(std::fstream& stream, _ThreadData * threadData) ; - inline bool readNextBlock(std::fstream& stream, _ThreadData * threadData); + inline void loadGroup (_ThreadData * threadData); + inline void loadBlock (_ThreadData * threadData); + inline bool readPBFBlobHeader(std::fstream & stream, _ThreadData * threadData); + inline bool unpackZLIB (std::fstream & stream, _ThreadData * threadData); + inline bool unpackLZMA (std::fstream & stream, _ThreadData * threadData); + inline bool readBlob (std::fstream & stream, _ThreadData * threadData); + inline bool readNextBlock (std::fstream & stream, _ThreadData * threadData); static const int NANO = 1000 * 1000 * 1000; static const int MAX_BLOB_HEADER_SIZE = 64 * 1024;