Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Mild perfomance fixed for PBF extraction

Browse Source
This commit is contained in:
Dennis Luxen 2013-08-16 17:09:27 +02:00
parent aba078a9d8
commit d24ba6d13d
4 changed files with 46 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Extractor/BaseParser.cpp
View File

@ -72,13 +72,10 @@ void BaseParser::ParseNodeInLua(ImportNode& n, lua_State* localLuaState) {
}
void BaseParser::ParseWayInLua(ExtractionWay& w, lua_State* localLuaState) {
if(2 > w.path.size()) {
return;
}
luabind::call_function<void>( localLuaState, "way_function", boost::ref(w) );
}
bool BaseParser::ShouldIgnoreRestriction(const std::string& except_tag_string) const {
bool BaseParser::ShouldIgnoreRestriction(const std::string & except_tag_string) const {
//should this restriction be ignored? yes if there's an overlap between:
//a) the list of modes in the except tag of the restriction (except_tag_string), ex: except=bus;bicycle
//b) the lua profile defines a hierachy of modes, ex: [access, vehicle, bicycle]

2
Extractor/ExtractorStructs.h
View File

@ -154,7 +154,7 @@ struct CmpWayByID : public std::binary_function<_WayIDStartAndEndEdge, _WayIDSta
struct Cmp : public std::binary_function<NodeID, NodeID, bool> {
typedef NodeID value_type;
bool operator () (const NodeID & a, const NodeID & b) const {
bool operator () (const NodeID a, const NodeID b) const {
return a < b;
}
value_type max_value() {

44
Extractor/PBFParser.cpp
View File

@ -173,14 +173,14 @@ inline void PBFParser::parseDenseNode(_ThreadData * threadData) {
extracted_nodes_vector[i].lon = COORDINATE_PRECISION*( ( double ) m_lastDenseLongitude * threadData->PBFprimitiveBlock.granularity() + threadData->PBFprimitiveBlock.lon_offset() ) / NANO;
while (denseTagIndex < dense.keys_vals_size()) {
const int tagValue = dense.keys_vals( denseTagIndex );
if( 0==tagValue ) {
if( 0 == tagValue ) {
++denseTagIndex;
break;
}
const int keyValue = dense.keys_vals ( denseTagIndex+1 );
const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(tagValue).data();
const std::string & value = threadData->PBFprimitiveBlock.stringtable().s(keyValue).data();
extracted_nodes_vector[i].keyVals.insert(std::make_pair(key, value));
const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(tagValue);
const std::string & value = threadData->PBFprimitiveBlock.stringtable().s(keyValue);
extracted_nodes_vector[i].keyVals.emplace(key, value);
denseTagIndex += 2;
}
}
@ -191,7 +191,7 @@ inline void PBFParser::parseDenseNode(_ThreadData * threadData) {
ParseNodeInLua( n, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) );
}
BOOST_FOREACH(ImportNode &n, extracted_nodes_vector) {
BOOST_FOREACH(const ImportNode &n, extracted_nodes_vector) {
extractor_callbacks->nodeFunction(n);
}
}
@ -209,7 +209,8 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
return;
}
const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID );
for(int i = 0; i < group.relations_size(); ++i ) {
for(int i = 0, relation_size = group.relations_size(); i < relation_size; ++i ) {
std::string except_tag_string;
const OSMPBF::Relation& inputRelation = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).relations(i);
bool isRestriction = false;
@ -241,8 +242,12 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
if(isRestriction) {
int64_t lastRef = 0;
_RawRestrictionContainer currentRestrictionContainer(isOnlyRestriction);
for(int rolesIndex = 0; rolesIndex < inputRelation.roles_sid_size(); ++rolesIndex) {
std::string role(threadData->PBFprimitiveBlock.stringtable().s( inputRelation.roles_sid( rolesIndex ) ).data());
for(
int rolesIndex = 0, last_role = inputRelation.roles_sid_size();
rolesIndex < last_role;
++rolesIndex
) {
const std::string & role = threadData->PBFprimitiveBlock.stringtable().s( inputRelation.roles_sid( rolesIndex ) );
lastRef += inputRelation.memids(rolesIndex);
if(!("from" == role || "to" == role || "via" == role)) {
@ -280,7 +285,6 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
break;
default: //should not happen
//cout << "unknown";
assert(false);
break;
}
@ -309,17 +313,23 @@ inline void PBFParser::parseWay(_ThreadData * threadData) {
for(int j = 0; j < number_of_keys; ++j) {
const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(inputWay.keys(j));
const std::string & val = threadData->PBFprimitiveBlock.stringtable().s(inputWay.vals(j));
parsed_way_vector[i].keyVals.insert(std::make_pair(key, val));
parsed_way_vector[i].keyVals.emplace(key, val);
}
}
#pragma omp parallel for schedule ( guided )
for(int i = 0; i < number_of_ways; ++i) {
ExtractionWay & w = parsed_way_vector[i];
ParseWayInLua( w, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) );
if(2 > w.path.size()) {
continue;
}
ParseWayInLua( w, scriptingEnvironment.getLuaStateForThreadID( omp_get_thread_num()) );
}
BOOST_FOREACH(ExtractionWay & w, parsed_way_vector) {
if(2 > w.path.size()) {
continue;
}
extractor_callbacks->wayFunction(w);
}
}
@ -330,21 +340,21 @@ inline void PBFParser::loadGroup(_ThreadData * threadData) {
#endif
const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID );
threadData->entityTypeIndicator = 0;
if ( group.nodes_size() != 0 ) {
threadData->entityTypeIndicator = TypeDummy;
if ( 0 != group.nodes_size() ) {
threadData->entityTypeIndicator = TypeNode;
}
if ( group.ways_size() != 0 ) {
if ( 0 != group.ways_size() ) {
threadData->entityTypeIndicator = TypeWay;
}
if ( group.relations_size() != 0 ) {
if ( 0 != group.relations_size() ) {
threadData->entityTypeIndicator = TypeRelation;
}
if ( group.has_dense() ) {
threadData->entityTypeIndicator = TypeDenseNode;
assert( group.dense().id_size() != 0 );
assert( 0 != group.dense().id_size() );
}
assert( threadData->entityTypeIndicator != 0 );
assert( threadData->entityTypeIndicator != TypeDummy );
}
inline void PBFParser::loadBlock(_ThreadData * threadData) {

33
Extractor/PBFParser.h
View File

@ -44,16 +44,17 @@
class PBFParser : public BaseParser {
enum EntityType {
TypeNode = 1,
TypeWay = 2,
TypeRelation = 4,
TypeDummy = 0,
TypeNode = 1,
TypeWay = 2,
TypeRelation = 4,
TypeDenseNode = 8
} ;
};
struct _ThreadData {
int currentGroupID;
int currentEntityID;
short entityTypeIndicator;
EntityType entityTypeIndicator;
OSMPBF::BlobHeader PBFBlobHeader;
OSMPBF::Blob PBFBlob;
@ -74,18 +75,18 @@ public:
private:
inline void ReadData();
inline void ParseData();
inline void parseDenseNode(_ThreadData * threadData);
inline void parseNode(_ThreadData * );
inline void parseRelation(_ThreadData * threadData);
inline void parseWay(_ThreadData * threadData);
inline void parseDenseNode (_ThreadData * threadData);
inline void parseNode (_ThreadData * threadData);
inline void parseRelation (_ThreadData * threadData);
inline void parseWay (_ThreadData * threadData);
inline void loadGroup(_ThreadData * threadData);
inline void loadBlock(_ThreadData * threadData);
inline bool readPBFBlobHeader(std::fstream& stream, _ThreadData * threadData);
inline bool unpackZLIB(std::fstream &, _ThreadData * threadData);
inline bool unpackLZMA(std::fstream &, _ThreadData * );
inline bool readBlob(std::fstream& stream, _ThreadData * threadData) ;
inline bool readNextBlock(std::fstream& stream, _ThreadData * threadData);
inline void loadGroup (_ThreadData * threadData);
inline void loadBlock (_ThreadData * threadData);
inline bool readPBFBlobHeader(std::fstream & stream, _ThreadData * threadData);
inline bool unpackZLIB (std::fstream & stream, _ThreadData * threadData);
inline bool unpackLZMA (std::fstream & stream, _ThreadData * threadData);
inline bool readBlob (std::fstream & stream, _ThreadData * threadData);
inline bool readNextBlock (std::fstream & stream, _ThreadData * threadData);
static const int NANO = 1000 * 1000 * 1000;
static const int MAX_BLOB_HEADER_SIZE = 64 * 1024;