Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Mild perfomance fixed for PBF extraction

Browse Source
This commit is contained in:
Dennis Luxen 2013-08-16 17:09:27 +02:00
parent aba078a9d8
commit d24ba6d13d
4 changed files with 46 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Extractor/BaseParser.cpp
View File

@ -72,9 +72,6 @@ void BaseParser::ParseNodeInLua(ImportNode& n, lua_State* localLuaState) {
} }
void BaseParser::ParseWayInLua(ExtractionWay& w, lua_State* localLuaState) { void BaseParser::ParseWayInLua(ExtractionWay& w, lua_State* localLuaState) {
if(2 > w.path.size()) {
return;
}
luabind::call_function<void>( localLuaState, "way_function", boost::ref(w) ); luabind::call_function<void>( localLuaState, "way_function", boost::ref(w) );
} }

2
Extractor/ExtractorStructs.h
View File

@ -154,7 +154,7 @@ struct CmpWayByID : public std::binary_function<_WayIDStartAndEndEdge, _WayIDSta
struct Cmp : public std::binary_function<NodeID, NodeID, bool> { struct Cmp : public std::binary_function<NodeID, NodeID, bool> {
typedef NodeID value_type; typedef NodeID value_type;
bool operator () (const NodeID & a, const NodeID & b) const { bool operator () (const NodeID a, const NodeID b) const {
return a < b; return a < b;
} }
value_type max_value() { value_type max_value() {

40
Extractor/PBFParser.cpp
View File

@ -178,9 +178,9 @@ inline void PBFParser::parseDenseNode(_ThreadData * threadData) {
break; break;
} }
const int keyValue = dense.keys_vals ( denseTagIndex+1 ); const int keyValue = dense.keys_vals ( denseTagIndex+1 );
const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(tagValue).data(); const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(tagValue);
const std::string & value = threadData->PBFprimitiveBlock.stringtable().s(keyValue).data(); const std::string & value = threadData->PBFprimitiveBlock.stringtable().s(keyValue);
extracted_nodes_vector[i].keyVals.insert(std::make_pair(key, value)); extracted_nodes_vector[i].keyVals.emplace(key, value);
denseTagIndex += 2; denseTagIndex += 2;
} }
} }
@ -191,7 +191,7 @@ inline void PBFParser::parseDenseNode(_ThreadData * threadData) {
ParseNodeInLua( n, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) ); ParseNodeInLua( n, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) );
} }
BOOST_FOREACH(ImportNode &n, extracted_nodes_vector) { BOOST_FOREACH(const ImportNode &n, extracted_nodes_vector) {
extractor_callbacks->nodeFunction(n); extractor_callbacks->nodeFunction(n);
} }
} }
@ -209,7 +209,8 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
return; return;
} }
const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ); const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID );
for(int i = 0; i < group.relations_size(); ++i ) {
for(int i = 0, relation_size = group.relations_size(); i < relation_size; ++i ) {
std::string except_tag_string; std::string except_tag_string;
const OSMPBF::Relation& inputRelation = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).relations(i); const OSMPBF::Relation& inputRelation = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).relations(i);
bool isRestriction = false; bool isRestriction = false;
@ -241,8 +242,12 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
if(isRestriction) { if(isRestriction) {
int64_t lastRef = 0; int64_t lastRef = 0;
_RawRestrictionContainer currentRestrictionContainer(isOnlyRestriction); _RawRestrictionContainer currentRestrictionContainer(isOnlyRestriction);
for(int rolesIndex = 0; rolesIndex < inputRelation.roles_sid_size(); ++rolesIndex) { for(
std::string role(threadData->PBFprimitiveBlock.stringtable().s( inputRelation.roles_sid( rolesIndex ) ).data()); int rolesIndex = 0, last_role = inputRelation.roles_sid_size();
rolesIndex < last_role;
++rolesIndex
) {
const std::string & role = threadData->PBFprimitiveBlock.stringtable().s( inputRelation.roles_sid( rolesIndex ) );
lastRef += inputRelation.memids(rolesIndex); lastRef += inputRelation.memids(rolesIndex);
if(!("from" == role || "to" == role || "via" == role)) { if(!("from" == role || "to" == role || "via" == role)) {
@ -280,7 +285,6 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
break; break;
default: //should not happen default: //should not happen
//cout << "unknown";
assert(false); assert(false);
break; break;
} }
@ -309,17 +313,23 @@ inline void PBFParser::parseWay(_ThreadData * threadData) {
for(int j = 0; j < number_of_keys; ++j) { for(int j = 0; j < number_of_keys; ++j) {
const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(inputWay.keys(j)); const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(inputWay.keys(j));
const std::string & val = threadData->PBFprimitiveBlock.stringtable().s(inputWay.vals(j)); const std::string & val = threadData->PBFprimitiveBlock.stringtable().s(inputWay.vals(j));
parsed_way_vector[i].keyVals.insert(std::make_pair(key, val)); parsed_way_vector[i].keyVals.emplace(key, val);
} }
} }
#pragma omp parallel for schedule ( guided ) #pragma omp parallel for schedule ( guided )
for(int i = 0; i < number_of_ways; ++i) { for(int i = 0; i < number_of_ways; ++i) {
ExtractionWay & w = parsed_way_vector[i]; ExtractionWay & w = parsed_way_vector[i];
if(2 > w.path.size()) {
continue;
}
ParseWayInLua( w, scriptingEnvironment.getLuaStateForThreadID( omp_get_thread_num()) ); ParseWayInLua( w, scriptingEnvironment.getLuaStateForThreadID( omp_get_thread_num()) );
} }
BOOST_FOREACH(ExtractionWay & w, parsed_way_vector) { BOOST_FOREACH(ExtractionWay & w, parsed_way_vector) {
if(2 > w.path.size()) {
continue;
}
extractor_callbacks->wayFunction(w); extractor_callbacks->wayFunction(w);
} }
} }
@ -330,21 +340,21 @@ inline void PBFParser::loadGroup(_ThreadData * threadData) {
#endif #endif
const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ); const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID );
threadData->entityTypeIndicator = 0; threadData->entityTypeIndicator = TypeDummy;
if ( group.nodes_size() != 0 ) { if ( 0 != group.nodes_size() ) {
threadData->entityTypeIndicator = TypeNode; threadData->entityTypeIndicator = TypeNode;
} }
if ( group.ways_size() != 0 ) { if ( 0 != group.ways_size() ) {
threadData->entityTypeIndicator = TypeWay; threadData->entityTypeIndicator = TypeWay;
} }
if ( group.relations_size() != 0 ) { if ( 0 != group.relations_size() ) {
threadData->entityTypeIndicator = TypeRelation; threadData->entityTypeIndicator = TypeRelation;
} }
if ( group.has_dense() ) { if ( group.has_dense() ) {
threadData->entityTypeIndicator = TypeDenseNode; threadData->entityTypeIndicator = TypeDenseNode;
assert( group.dense().id_size() != 0 ); assert( 0 != group.dense().id_size() );
} }
assert( threadData->entityTypeIndicator != 0 ); assert( threadData->entityTypeIndicator != TypeDummy );
} }
inline void PBFParser::loadBlock(_ThreadData * threadData) { inline void PBFParser::loadBlock(_ThreadData * threadData) {

9
Extractor/PBFParser.h
View File

@ -44,6 +44,7 @@
class PBFParser : public BaseParser { class PBFParser : public BaseParser {
enum EntityType { enum EntityType {
TypeDummy = 0,
TypeNode = 1, TypeNode = 1,
TypeWay = 2, TypeWay = 2,
TypeRelation = 4, TypeRelation = 4,
@ -53,7 +54,7 @@ class PBFParser : public BaseParser {
struct _ThreadData { struct _ThreadData {
int currentGroupID; int currentGroupID;
int currentEntityID; int currentEntityID;
short entityTypeIndicator; EntityType entityTypeIndicator;
OSMPBF::BlobHeader PBFBlobHeader; OSMPBF::BlobHeader PBFBlobHeader;
OSMPBF::Blob PBFBlob; OSMPBF::Blob PBFBlob;
@ -75,15 +76,15 @@ private:
inline void ReadData(); inline void ReadData();
inline void ParseData(); inline void ParseData();
inline void parseDenseNode (_ThreadData * threadData); inline void parseDenseNode (_ThreadData * threadData);
inline void parseNode(_ThreadData * ); inline void parseNode (_ThreadData * threadData);
inline void parseRelation (_ThreadData * threadData); inline void parseRelation (_ThreadData * threadData);
inline void parseWay (_ThreadData * threadData); inline void parseWay (_ThreadData * threadData);
inline void loadGroup (_ThreadData * threadData); inline void loadGroup (_ThreadData * threadData);
inline void loadBlock (_ThreadData * threadData); inline void loadBlock (_ThreadData * threadData);
inline bool readPBFBlobHeader(std::fstream & stream, _ThreadData * threadData); inline bool readPBFBlobHeader(std::fstream & stream, _ThreadData * threadData);
inline bool unpackZLIB(std::fstream &, _ThreadData * threadData); inline bool unpackZLIB (std::fstream & stream, _ThreadData * threadData);
inline bool unpackLZMA(std::fstream &, _ThreadData * ); inline bool unpackLZMA (std::fstream & stream, _ThreadData * threadData);
inline bool readBlob (std::fstream & stream, _ThreadData * threadData); inline bool readBlob (std::fstream & stream, _ThreadData * threadData);
inline bool readNextBlock (std::fstream & stream, _ThreadData * threadData); inline bool readNextBlock (std::fstream & stream, _ThreadData * threadData);