diff --git a/Extractor/BaseParser.cpp b/Extractor/BaseParser.cpp index d84ebe034..d57c4c5cf 100644 --- a/Extractor/BaseParser.cpp +++ b/Extractor/BaseParser.cpp @@ -20,15 +20,15 @@ or see http://www.gnu.org/licenses/agpl.txt. #include "BaseParser.h" -BaseParser::BaseParser(ExtractorCallbacks* em, ScriptingEnvironment& se) : -externalMemory(em), scriptingEnvironment(se), luaState(NULL), use_turn_restrictions(true) { +BaseParser::BaseParser(ExtractorCallbacks* ec, ScriptingEnvironment& se) : +externalMemory(ec), scriptingEnvironment(se), luaState(NULL), use_turn_restrictions(true) { luaState = se.getLuaStateForThreadID(0); ReadUseRestrictionsSetting(); ReadRestrictionExceptions(); } void BaseParser::ReadUseRestrictionsSetting() { - if(0 != luaL_dostring( luaState, "return use_turn_restrictions\n")) { + if( 0 != luaL_dostring( luaState, "return use_turn_restrictions\n") ) { ERR(lua_tostring( luaState,-1)<< " occured in scripting block"); } if( lua_isboolean( luaState, -1) ) { @@ -64,8 +64,8 @@ void BaseParser::ReadRestrictionExceptions() { } } -void BaseParser::report_errors(lua_State *L, int status) { - if ( status!=0 ) { +void BaseParser::report_errors(lua_State *L, const int status) const { + if( 0!=status ) { std::cerr << "-- " << lua_tostring(L, -1) << std::endl; lua_pop(L, 1); // remove error message } @@ -91,22 +91,22 @@ inline void BaseParser::ParseWayInLua(ExtractionWay& w, lua_State* localLuaState } } -inline bool BaseParser::ShouldIgnoreRestriction(std::string& exception_string) { +inline bool BaseParser::ShouldIgnoreRestriction(const std::string& except_tag_string) const { //should this restriction be ignored? yes if there's an overlap between: - //a) the list of modes in the except tag of the restriction (exception_string), ex: except=bus;bicycle + //a) the list of modes in the except tag of the restriction (except_tag_string), ex: except=bus;bicycle //b) the lua profile defines a hierachy of modes, ex: [access, vehicle, bicycle] - if( "" == exception_string ) + if( "" == except_tag_string ) { return false; + } //Be warned, this is quadratic work here, but we assume that //only a few exceptions are actually defined. std::vector exceptions; - boost::algorithm::split_regex(exceptions, exception_string, boost::regex("[;][ ]*")); + boost::algorithm::split_regex(exceptions, except_tag_string, boost::regex("[;][ ]*")); BOOST_FOREACH(std::string& str, exceptions) { if( restriction_exceptions.end() != std::find(restriction_exceptions.begin(), restriction_exceptions.end(), str) ) { return true; - break; //BOOST_FOREACH } } return false; diff --git a/Extractor/BaseParser.h b/Extractor/BaseParser.h index 4c23bc78d..891af320b 100644 --- a/Extractor/BaseParser.h +++ b/Extractor/BaseParser.h @@ -34,19 +34,19 @@ extern "C" { class BaseParser : boost::noncopyable { public: - BaseParser(ExtractorCallbacks* em, ScriptingEnvironment& se); + BaseParser(ExtractorCallbacks* ec, ScriptingEnvironment& se); virtual ~BaseParser() {} virtual bool ReadHeader() = 0; virtual bool Parse() = 0; inline virtual void ParseNodeInLua(ImportNode& n, lua_State* luaStateForThread); inline virtual void ParseWayInLua(ExtractionWay& n, lua_State* luaStateForThread); - virtual void report_errors(lua_State *L, int status); + virtual void report_errors(lua_State *L, const int status) const; protected: virtual void ReadUseRestrictionsSetting(); virtual void ReadRestrictionExceptions(); - inline virtual bool ShouldIgnoreRestriction(std::string& exception_string); + inline virtual bool ShouldIgnoreRestriction(const std::string& except_tag_string) const; ExtractorCallbacks* externalMemory; ScriptingEnvironment& scriptingEnvironment; diff --git a/Extractor/PBFParser.cpp b/Extractor/PBFParser.cpp index de8acad14..30ad1feb6 100644 --- a/Extractor/PBFParser.cpp +++ b/Extractor/PBFParser.cpp @@ -20,7 +20,7 @@ #include "PBFParser.h" -PBFParser::PBFParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const char * fileName) : BaseParser( em, se ) { +PBFParser::PBFParser(const char * fileName, ExtractorCallbacks* ec, ScriptingEnvironment& se) : BaseParser( ec, se ) { GOOGLE_PROTOBUF_VERIFY_VERSION; //TODO: What is the bottleneck here? Filling the queue or reading the stuff from disk? //NOTE: With Lua scripting, it is parsing the stuff. I/O is virtually for free. @@ -38,8 +38,9 @@ PBFParser::PBFParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const cha } PBFParser::~PBFParser() { - if(input.is_open()) + if(input.is_open()) { input.close(); + } // Clean up any leftover ThreadData objects in the queue _ThreadData* td; @@ -69,11 +70,13 @@ inline bool PBFParser::ReadHeader() { for(int i = 0, featureSize = initData.PBFHeaderBlock.required_features_size(); i < featureSize; ++i) { const std::string& feature = initData.PBFHeaderBlock.required_features( i ); bool supported = false; - if ( "OsmSchema-V0.6" == feature ) + if ( "OsmSchema-V0.6" == feature ) { supported = true; - else if ( "DenseNodes" == feature ) + } + else if ( "DenseNodes" == feature ) { supported = true; - + } + if ( !supported ) { std::cerr << "[error] required feature not supported: " << feature.data() << std::endl; return false; @@ -91,9 +94,9 @@ inline void PBFParser::ReadData() { _ThreadData *threadData = new _ThreadData(); keepRunning = readNextBlock(input, threadData); - if (keepRunning) + if (keepRunning) { threadDataQueue->push(threadData); - else { + } else { threadDataQueue->push(NULL); // No more data to read, parse stops when NULL encountered delete threadData; } @@ -104,7 +107,7 @@ inline void PBFParser::ParseData() { while (true) { _ThreadData *threadData; threadDataQueue->wait_and_pop(threadData); - if (threadData == NULL) { + if( NULL==threadData ) { INFO("Parse Data Thread Finished"); threadDataQueue->push(NULL); // Signal end of data for other threads break; @@ -116,14 +119,18 @@ inline void PBFParser::ParseData() { threadData->currentGroupID = i; loadGroup(threadData); - if(threadData->entityTypeIndicator == TypeNode) + if(threadData->entityTypeIndicator == TypeNode) { parseNode(threadData); - if(threadData->entityTypeIndicator == TypeWay) + } + if(threadData->entityTypeIndicator == TypeWay) { parseWay(threadData); - if(threadData->entityTypeIndicator == TypeRelation) + } + if(threadData->entityTypeIndicator == TypeRelation) { parseRelation(threadData); - if(threadData->entityTypeIndicator == TypeDenseNode) + } + if(threadData->entityTypeIndicator == TypeDenseNode) { parseDenseNode(threadData); + } } delete threadData; @@ -164,7 +171,7 @@ inline void PBFParser::parseDenseNode(_ThreadData * threadData) { n.lon = 100000*( ( double ) m_lastDenseLongitude * threadData->PBFprimitiveBlock.granularity() + threadData->PBFprimitiveBlock.lon_offset() ) / NANO; while (denseTagIndex < dense.keys_vals_size()) { const int tagValue = dense.keys_vals( denseTagIndex ); - if(tagValue == 0) { + if( 0==tagValue ) { ++denseTagIndex; break; } @@ -197,12 +204,12 @@ inline void PBFParser::parseNode(_ThreadData * ) { inline void PBFParser::parseRelation(_ThreadData * threadData) { //TODO: leave early, if relation is not a restriction //TODO: reuse rawRestriction container - if( !use_turn_restrictions ) - return; - + if( !use_turn_restrictions ) { + return; + } const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ); for(int i = 0; i < group.relations_size(); ++i ) { - std::string restriction_exceptions; + std::string except_tag_string; const OSMPBF::Relation& inputRelation = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).relations(i); bool isRestriction = false; bool isOnlyRestriction = false; @@ -210,22 +217,25 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) { const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(inputRelation.keys(k)); const std::string & val = threadData->PBFprimitiveBlock.stringtable().s(inputRelation.vals(k)); if ("type" == key) { - if( "restriction" == val) + if( "restriction" == val) { isRestriction = true; - else + } else { break; + } } if ("restriction" == key) { - if(val.find("only_") == 0) + if(val.find("only_") == 0) { isOnlyRestriction = true; + } } if ("except" == key) { - restriction_exceptions = val; + except_tag_string = val; } } - if( isRestriction && ShouldIgnoreRestriction(restriction_exceptions) ) - isRestriction = false; + if( isRestriction && ShouldIgnoreRestriction(except_tag_string) ) { + isRestriction = false; + } if(isRestriction) { int64_t lastRef = 0; @@ -240,11 +250,13 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) { switch(inputRelation.types(rolesIndex)) { case 0: //node - if("from" == role || "to" == role) //Only via should be a node + if("from" == role || "to" == role) { //Only via should be a node continue; + } assert("via" == role); - if(UINT_MAX != currentRestrictionContainer.viaNode) + if(UINT_MAX != currentRestrictionContainer.viaNode) { currentRestrictionContainer.viaNode = UINT_MAX; + } assert(UINT_MAX == currentRestrictionContainer.viaNode); currentRestrictionContainer.restriction.viaNode = lastRef; break; @@ -272,8 +284,9 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) { break; } } - if(!externalMemory->restrictionFunction(currentRestrictionContainer)) + if(!externalMemory->restrictionFunction(currentRestrictionContainer)) { std::cerr << "[PBFParser] relation not parsed" << std::endl; + } } } } @@ -406,9 +419,10 @@ inline bool PBFParser::unpackLZMA(std::fstream &, _ThreadData * ) { } inline bool PBFParser::readBlob(std::fstream& stream, _ThreadData * threadData) { - if(stream.eof()) + if(stream.eof()) { return false; - + } + const int size = threadData->PBFBlobHeader.datasize(); if ( size < 0 || size > MAX_BLOB_SIZE ) { std::cerr << "[error] invalid Blob size:" << size << std::endl; @@ -436,8 +450,9 @@ inline bool PBFParser::readBlob(std::fstream& stream, _ThreadData * threadData) return false; } } else if ( threadData->PBFBlob.has_lzma_data() ) { - if ( !unpackLZMA(stream, threadData) ) + if ( !unpackLZMA(stream, threadData) ) { std::cerr << "[error] lzma data encountered that could not be unpacked" << std::endl; + } delete[] data; return false; } else { diff --git a/Extractor/PBFParser.h b/Extractor/PBFParser.h index a483baf19..587fbc96c 100644 --- a/Extractor/PBFParser.h +++ b/Extractor/PBFParser.h @@ -62,7 +62,7 @@ class PBFParser : public BaseParser { }; public: - PBFParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const char * fileName); + PBFParser(const char * fileName, ExtractorCallbacks* ec, ScriptingEnvironment& se); virtual ~PBFParser(); inline bool ReadHeader(); @@ -95,7 +95,7 @@ private: #endif std::fstream input; // the input stream to parse - boost::shared_ptr > threadDataQueue; // ThreadData Queue + boost::shared_ptr > threadDataQueue; }; #endif /* PBFPARSER_H_ */ diff --git a/Extractor/XMLParser.cpp b/Extractor/XMLParser.cpp index b1225cc5c..f28440d36 100644 --- a/Extractor/XMLParser.cpp +++ b/Extractor/XMLParser.cpp @@ -27,7 +27,7 @@ #include "../DataStructures/InputReaderFactory.h" -XMLParser::XMLParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const char * filename) : BaseParser( em, se ) { +XMLParser::XMLParser(const char * filename, ExtractorCallbacks* ec, ScriptingEnvironment& se) : BaseParser(ec, se) { WARN("Parsing plain .osm/.osm.bz2 is deprecated. Switch to .pbf"); inputReader = inputReaderFactory(filename); } @@ -40,16 +40,18 @@ bool XMLParser::Parse() { const int type = xmlTextReaderNodeType( inputReader ); //1 is Element - if ( type != 1 ) + if ( type != 1 ) { continue; - + } + xmlChar* currentName = xmlTextReaderName( inputReader ); - if ( currentName == NULL ) + if ( currentName == NULL ) { continue; - + } + if ( xmlStrEqual( currentName, ( const xmlChar* ) "node" ) == 1 ) { ImportNode n = _ReadXMLNode(); - ParseNodeInLua( n, luaState ); + ParseNodeInLua( n, luaState ); if(!externalMemory->nodeFunction(n)) std::cerr << "[XMLParser] dense node not parsed" << std::endl; @@ -57,7 +59,7 @@ bool XMLParser::Parse() { if ( xmlStrEqual( currentName, ( const xmlChar* ) "way" ) == 1 ) { ExtractionWay way = _ReadXMLWay( ); - ParseWayInLua( way, luaState ); + ParseWayInLua( way, luaState ); if(!externalMemory->wayFunction(way)) std::cerr << "[PBFParser] way not parsed" << std::endl; } @@ -75,19 +77,20 @@ bool XMLParser::Parse() { } _RawRestrictionContainer XMLParser::_ReadXMLRestriction() { - _RawRestrictionContainer restriction; - std::string exception_string; + _RawRestrictionContainer restriction; + std::string except_tag_string; if ( xmlTextReaderIsEmptyElement( inputReader ) != 1 ) { const int depth = xmlTextReaderDepth( inputReader );while ( xmlTextReaderRead( inputReader ) == 1 ) { const int childType = xmlTextReaderNodeType( inputReader ); - if ( childType != 1 && childType != 15 ) + if ( childType != 1 && childType != 15 ) { continue; + } const int childDepth = xmlTextReaderDepth( inputReader ); xmlChar* childName = xmlTextReaderName( inputReader ); - if ( childName == NULL ) + if ( childName == NULL ) { continue; - + } if ( depth == childDepth && childType == 15 && xmlStrEqual( childName, ( const xmlChar* ) "relation" ) == 1 ) { xmlFree( childName ); break; @@ -102,18 +105,21 @@ _RawRestrictionContainer XMLParser::_ReadXMLRestriction() { xmlChar* value = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "v" ); if ( k != NULL && value != NULL ) { if(xmlStrEqual(k, ( const xmlChar* ) "restriction" )){ - if(0 == std::string((const char *) value).find("only_")) + if(0 == std::string((const char *) value).find("only_")) { restriction.restriction.flags.isOnly = true; + } } if ( xmlStrEqual(k, (const xmlChar *) "except") ) { - exception_string = (const char*) value; + except_tag_string = (const char*) value; } } - if ( k != NULL ) + if ( k != NULL ) { xmlFree( k ); - if ( value != NULL ) + } + if ( value != NULL ) { xmlFree( value ); + } } else if ( xmlStrEqual( childName, ( const xmlChar* ) "member" ) == 1 ) { xmlChar* ref = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "ref" ); if ( ref != NULL ) { @@ -130,21 +136,24 @@ _RawRestrictionContainer XMLParser::_ReadXMLRestriction() { restriction.restriction.viaNode = atoi((const char*) ref); } - if(NULL != type) + if(NULL != type) { xmlFree( type ); - if(NULL != role) + } + if(NULL != role) { xmlFree( role ); - if(NULL != ref) + } + if(NULL != ref) { xmlFree( ref ); + } } } xmlFree( childName ); } } - if( ShouldIgnoreRestriction(exception_string) ) - restriction.fromWay = UINT_MAX; //workaround to ignore the restriction - + if( ShouldIgnoreRestriction(except_tag_string) ) { + restriction.fromWay = UINT_MAX; //workaround to ignore the restriction + } return restriction; } @@ -154,12 +163,14 @@ ExtractionWay XMLParser::_ReadXMLWay() { const int depth = xmlTextReaderDepth( inputReader ); while ( xmlTextReaderRead( inputReader ) == 1 ) { const int childType = xmlTextReaderNodeType( inputReader ); - if ( childType != 1 && childType != 15 ) + if ( childType != 1 && childType != 15 ) { continue; + } const int childDepth = xmlTextReaderDepth( inputReader ); xmlChar* childName = xmlTextReaderName( inputReader ); - if ( childName == NULL ) + if ( childName == NULL ) { continue; + } if ( depth == childDepth && childType == 15 && xmlStrEqual( childName, ( const xmlChar* ) "way" ) == 1 ) { xmlChar* id = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "id" ); @@ -176,15 +187,16 @@ ExtractionWay XMLParser::_ReadXMLWay() { if ( xmlStrEqual( childName, ( const xmlChar* ) "tag" ) == 1 ) { xmlChar* k = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "k" ); xmlChar* value = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "v" ); - // cout << "->k=" << k << ", v=" << value << endl; + // cout << "->k=" << k << ", v=" << value << endl; if ( k != NULL && value != NULL ) { - way.keyVals.Add(std::string( (char *) k ), std::string( (char *) value)); } - if ( k != NULL ) + if ( k != NULL ) { xmlFree( k ); - if ( value != NULL ) + } + if ( value != NULL ) { xmlFree( value ); + } } else if ( xmlStrEqual( childName, ( const xmlChar* ) "nd" ) == 1 ) { xmlChar* ref = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "ref" ); if ( ref != NULL ) { @@ -222,12 +234,14 @@ ImportNode XMLParser::_ReadXMLNode() { while ( xmlTextReaderRead( inputReader ) == 1 ) { const int childType = xmlTextReaderNodeType( inputReader ); // 1 = Element, 15 = EndElement - if ( childType != 1 && childType != 15 ) + if ( childType != 1 && childType != 15 ) { continue; + } const int childDepth = xmlTextReaderDepth( inputReader ); xmlChar* childName = xmlTextReaderName( inputReader ); - if ( childName == NULL ) + if ( childName == NULL ) { continue; + } if ( depth == childDepth && childType == 15 && xmlStrEqual( childName, ( const xmlChar* ) "node" ) == 1 ) { xmlFree( childName ); @@ -244,10 +258,12 @@ ImportNode XMLParser::_ReadXMLNode() { if ( k != NULL && value != NULL ) { node.keyVals.Add(std::string( reinterpret_cast(k) ), std::string( reinterpret_cast(value))); } - if ( k != NULL ) + if ( k != NULL ) { xmlFree( k ); - if ( value != NULL ) + } + if ( value != NULL ) { xmlFree( value ); + } } xmlFree( childName ); diff --git a/Extractor/XMLParser.h b/Extractor/XMLParser.h index cc96e65e1..a2af2fb86 100644 --- a/Extractor/XMLParser.h +++ b/Extractor/XMLParser.h @@ -28,7 +28,7 @@ class XMLParser : public BaseParser { public: - XMLParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const char* filename); + XMLParser(const char* filename, ExtractorCallbacks* ec, ScriptingEnvironment& se); bool ReadHeader(); bool Parse(); diff --git a/extractor.cpp b/extractor.cpp index fb52961e6..4912c9b67 100644 --- a/extractor.cpp +++ b/extractor.cpp @@ -96,13 +96,14 @@ int main (int argc, char *argv[]) { extractCallBacks = new ExtractorCallbacks(&externalMemory, &stringMap); BaseParser* parser; if(isPBF) { - parser = new PBFParser(extractCallBacks, scriptingEnvironment, argv[1]); + parser = new PBFParser(argv[1], extractCallBacks, scriptingEnvironment); } else { - parser = new XMLParser(extractCallBacks, scriptingEnvironment, argv[1]); + parser = new XMLParser(argv[1], extractCallBacks, scriptingEnvironment); } - if(!parser->ReadHeader()) + if(!parser->ReadHeader()) { ERR("Parser not initialized!"); + } INFO("Parsing in progress.."); double time = get_timestamp(); parser->Parse();