Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

code cleanup

Browse Source
This commit is contained in:
Emil Tin 2013-02-10 18:28:04 +01:00 committed by DennisOSRM
parent 1ecad20a0d
commit db148741e9
7 changed files with 112 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Extractor/BaseParser.cpp
View File

@ -20,15 +20,15 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "BaseParser.h"
BaseParser::BaseParser(ExtractorCallbacks* em, ScriptingEnvironment& se) :
externalMemory(em), scriptingEnvironment(se), luaState(NULL), use_turn_restrictions(true) {
BaseParser::BaseParser(ExtractorCallbacks* ec, ScriptingEnvironment& se) :
externalMemory(ec), scriptingEnvironment(se), luaState(NULL), use_turn_restrictions(true) {
luaState = se.getLuaStateForThreadID(0);
ReadUseRestrictionsSetting();
ReadRestrictionExceptions();
}
void BaseParser::ReadUseRestrictionsSetting() {
if(0 != luaL_dostring( luaState, "return use_turn_restrictions\n")) {
if( 0 != luaL_dostring( luaState, "return use_turn_restrictions\n") ) {
ERR(lua_tostring( luaState,-1)<< " occured in scripting block");
}
if( lua_isboolean( luaState, -1) ) {
@ -64,8 +64,8 @@ void BaseParser::ReadRestrictionExceptions() {
}
}
void BaseParser::report_errors(lua_State *L, int status) {
if ( status!=0 ) {
void BaseParser::report_errors(lua_State *L, const int status) const {
if( 0!=status ) {
std::cerr << "-- " << lua_tostring(L, -1) << std::endl;
lua_pop(L, 1); // remove error message
}
@ -91,22 +91,22 @@ inline void BaseParser::ParseWayInLua(ExtractionWay& w, lua_State* localLuaState
}
}
inline bool BaseParser::ShouldIgnoreRestriction(std::string& exception_string) {
inline bool BaseParser::ShouldIgnoreRestriction(const std::string& except_tag_string) const {
//should this restriction be ignored? yes if there's an overlap between:
//a) the list of modes in the except tag of the restriction (exception_string), ex: except=bus;bicycle
//a) the list of modes in the except tag of the restriction (except_tag_string), ex: except=bus;bicycle
//b) the lua profile defines a hierachy of modes, ex: [access, vehicle, bicycle]
if( "" == exception_string )
if( "" == except_tag_string ) {
return false;
}
//Be warned, this is quadratic work here, but we assume that
//only a few exceptions are actually defined.
std::vector<std::string> exceptions;
boost::algorithm::split_regex(exceptions, exception_string, boost::regex("[;][ ]*"));
boost::algorithm::split_regex(exceptions, except_tag_string, boost::regex("[;][ ]*"));
BOOST_FOREACH(std::string& str, exceptions) {
if( restriction_exceptions.end() != std::find(restriction_exceptions.begin(), restriction_exceptions.end(), str) ) {
return true;
break; //BOOST_FOREACH
}
}
return false;

6
Extractor/BaseParser.h
View File

@ -34,19 +34,19 @@ extern "C" {
class BaseParser : boost::noncopyable {
public:
BaseParser(ExtractorCallbacks* em, ScriptingEnvironment& se);
BaseParser(ExtractorCallbacks* ec, ScriptingEnvironment& se);
virtual ~BaseParser() {}
virtual bool ReadHeader() = 0;
virtual bool Parse() = 0;
inline virtual void ParseNodeInLua(ImportNode& n, lua_State* luaStateForThread);
inline virtual void ParseWayInLua(ExtractionWay& n, lua_State* luaStateForThread);
virtual void report_errors(lua_State *L, int status);
virtual void report_errors(lua_State *L, const int status) const;
protected:
virtual void ReadUseRestrictionsSetting();
virtual void ReadRestrictionExceptions();
inline virtual bool ShouldIgnoreRestriction(std::string& exception_string);
inline virtual bool ShouldIgnoreRestriction(const std::string& except_tag_string) const;
ExtractorCallbacks* externalMemory;
ScriptingEnvironment& scriptingEnvironment;

73
Extractor/PBFParser.cpp
View File

@ -20,7 +20,7 @@
#include "PBFParser.h"
PBFParser::PBFParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const char * fileName) : BaseParser( em, se ) {
PBFParser::PBFParser(const char * fileName, ExtractorCallbacks* ec, ScriptingEnvironment& se) : BaseParser( ec, se ) {
GOOGLE_PROTOBUF_VERIFY_VERSION;
//TODO: What is the bottleneck here? Filling the queue or reading the stuff from disk?
//NOTE: With Lua scripting, it is parsing the stuff. I/O is virtually for free.
@ -38,8 +38,9 @@ PBFParser::PBFParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const cha
}
PBFParser::~PBFParser() {
if(input.is_open())
if(input.is_open()) {
input.close();
}
// Clean up any leftover ThreadData objects in the queue
_ThreadData* td;
@ -69,11 +70,13 @@ inline bool PBFParser::ReadHeader() {
for(int i = 0, featureSize = initData.PBFHeaderBlock.required_features_size(); i < featureSize; ++i) {
const std::string& feature = initData.PBFHeaderBlock.required_features( i );
bool supported = false;
if ( "OsmSchema-V0.6" == feature )
if ( "OsmSchema-V0.6" == feature ) {
supported = true;
else if ( "DenseNodes" == feature )
}
else if ( "DenseNodes" == feature ) {
supported = true;
}
if ( !supported ) {
std::cerr << "[error] required feature not supported: " << feature.data() << std::endl;
return false;
@ -91,9 +94,9 @@ inline void PBFParser::ReadData() {
_ThreadData *threadData = new _ThreadData();
keepRunning = readNextBlock(input, threadData);
if (keepRunning)
if (keepRunning) {
threadDataQueue->push(threadData);
else {
} else {
threadDataQueue->push(NULL); // No more data to read, parse stops when NULL encountered
delete threadData;
}
@ -104,7 +107,7 @@ inline void PBFParser::ParseData() {
while (true) {
_ThreadData *threadData;
threadDataQueue->wait_and_pop(threadData);
if (threadData == NULL) {
if( NULL==threadData ) {
INFO("Parse Data Thread Finished");
threadDataQueue->push(NULL); // Signal end of data for other threads
break;
@ -116,14 +119,18 @@ inline void PBFParser::ParseData() {
threadData->currentGroupID = i;
loadGroup(threadData);
if(threadData->entityTypeIndicator == TypeNode)
if(threadData->entityTypeIndicator == TypeNode) {
parseNode(threadData);
if(threadData->entityTypeIndicator == TypeWay)
}
if(threadData->entityTypeIndicator == TypeWay) {
parseWay(threadData);
if(threadData->entityTypeIndicator == TypeRelation)
}
if(threadData->entityTypeIndicator == TypeRelation) {
parseRelation(threadData);
if(threadData->entityTypeIndicator == TypeDenseNode)
}
if(threadData->entityTypeIndicator == TypeDenseNode) {
parseDenseNode(threadData);
}
}
delete threadData;
@ -164,7 +171,7 @@ inline void PBFParser::parseDenseNode(_ThreadData * threadData) {
n.lon = 100000*( ( double ) m_lastDenseLongitude * threadData->PBFprimitiveBlock.granularity() + threadData->PBFprimitiveBlock.lon_offset() ) / NANO;
while (denseTagIndex < dense.keys_vals_size()) {
const int tagValue = dense.keys_vals( denseTagIndex );
if(tagValue == 0) {
if( 0==tagValue ) {
++denseTagIndex;
break;
}
@ -197,12 +204,12 @@ inline void PBFParser::parseNode(_ThreadData * ) {
inline void PBFParser::parseRelation(_ThreadData * threadData) {
//TODO: leave early, if relation is not a restriction
//TODO: reuse rawRestriction container
if( !use_turn_restrictions )
return;
if( !use_turn_restrictions ) {
return;
}
const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID );
for(int i = 0; i < group.relations_size(); ++i ) {
std::string restriction_exceptions;
std::string except_tag_string;
const OSMPBF::Relation& inputRelation = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).relations(i);
bool isRestriction = false;
bool isOnlyRestriction = false;
@ -210,22 +217,25 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(inputRelation.keys(k));
const std::string & val = threadData->PBFprimitiveBlock.stringtable().s(inputRelation.vals(k));
if ("type" == key) {
if( "restriction" == val)
if( "restriction" == val) {
isRestriction = true;
else
} else {
break;
}
}
if ("restriction" == key) {
if(val.find("only_") == 0)
if(val.find("only_") == 0) {
isOnlyRestriction = true;
}
}
if ("except" == key) {
restriction_exceptions = val;
except_tag_string = val;
}
}
if( isRestriction && ShouldIgnoreRestriction(restriction_exceptions) )
isRestriction = false;
if( isRestriction && ShouldIgnoreRestriction(except_tag_string) ) {
isRestriction = false;
}
if(isRestriction) {
int64_t lastRef = 0;
@ -240,11 +250,13 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
switch(inputRelation.types(rolesIndex)) {
case 0: //node
if("from" == role || "to" == role) //Only via should be a node
if("from" == role || "to" == role) { //Only via should be a node
continue;
}
assert("via" == role);
if(UINT_MAX != currentRestrictionContainer.viaNode)
if(UINT_MAX != currentRestrictionContainer.viaNode) {
currentRestrictionContainer.viaNode = UINT_MAX;
}
assert(UINT_MAX == currentRestrictionContainer.viaNode);
currentRestrictionContainer.restriction.viaNode = lastRef;
break;
@ -272,8 +284,9 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
break;
}
}
if(!externalMemory->restrictionFunction(currentRestrictionContainer))
if(!externalMemory->restrictionFunction(currentRestrictionContainer)) {
std::cerr << "[PBFParser] relation not parsed" << std::endl;
}
}
}
}
@ -406,9 +419,10 @@ inline bool PBFParser::unpackLZMA(std::fstream &, _ThreadData * ) {
}
inline bool PBFParser::readBlob(std::fstream& stream, _ThreadData * threadData) {
if(stream.eof())
if(stream.eof()) {
return false;
}
const int size = threadData->PBFBlobHeader.datasize();
if ( size < 0 || size > MAX_BLOB_SIZE ) {
std::cerr << "[error] invalid Blob size:" << size << std::endl;
@ -436,8 +450,9 @@ inline bool PBFParser::readBlob(std::fstream& stream, _ThreadData * threadData)
return false;
}
} else if ( threadData->PBFBlob.has_lzma_data() ) {
if ( !unpackLZMA(stream, threadData) )
if ( !unpackLZMA(stream, threadData) ) {
std::cerr << "[error] lzma data encountered that could not be unpacked" << std::endl;
}
delete[] data;
return false;
} else {

4
Extractor/PBFParser.h
View File

@ -62,7 +62,7 @@ class PBFParser : public BaseParser {
};
public:
PBFParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const char * fileName);
PBFParser(const char * fileName, ExtractorCallbacks* ec, ScriptingEnvironment& se);
virtual ~PBFParser();
inline bool ReadHeader();
@ -95,7 +95,7 @@ private:
#endif
std::fstream input; // the input stream to parse
boost::shared_ptr<ConcurrentQueue < _ThreadData* > > threadDataQueue; // ThreadData Queue
boost::shared_ptr<ConcurrentQueue < _ThreadData* > > threadDataQueue;
};
#endif /* PBFPARSER_H_ */

80
Extractor/XMLParser.cpp
View File

@ -27,7 +27,7 @@
#include "../DataStructures/InputReaderFactory.h"
XMLParser::XMLParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const char * filename) : BaseParser( em, se ) {
XMLParser::XMLParser(const char * filename, ExtractorCallbacks* ec, ScriptingEnvironment& se) : BaseParser(ec, se) {
WARN("Parsing plain .osm/.osm.bz2 is deprecated. Switch to .pbf");
inputReader = inputReaderFactory(filename);
}
@ -40,16 +40,18 @@ bool XMLParser::Parse() {
const int type = xmlTextReaderNodeType( inputReader );
//1 is Element
if ( type != 1 )
if ( type != 1 ) {
continue;
}
xmlChar* currentName = xmlTextReaderName( inputReader );
if ( currentName == NULL )
if ( currentName == NULL ) {
continue;
}
if ( xmlStrEqual( currentName, ( const xmlChar* ) "node" ) == 1 ) {
ImportNode n = _ReadXMLNode();
ParseNodeInLua( n, luaState );
ParseNodeInLua( n, luaState );
if(!externalMemory->nodeFunction(n))
std::cerr << "[XMLParser] dense node not parsed" << std::endl;
@ -57,7 +59,7 @@ bool XMLParser::Parse() {
if ( xmlStrEqual( currentName, ( const xmlChar* ) "way" ) == 1 ) {
ExtractionWay way = _ReadXMLWay( );
ParseWayInLua( way, luaState );
ParseWayInLua( way, luaState );
if(!externalMemory->wayFunction(way))
std::cerr << "[PBFParser] way not parsed" << std::endl;
}
@ -75,19 +77,20 @@ bool XMLParser::Parse() {
}
_RawRestrictionContainer XMLParser::_ReadXMLRestriction() {
_RawRestrictionContainer restriction;
std::string exception_string;
_RawRestrictionContainer restriction;
std::string except_tag_string;
if ( xmlTextReaderIsEmptyElement( inputReader ) != 1 ) {
const int depth = xmlTextReaderDepth( inputReader );while ( xmlTextReaderRead( inputReader ) == 1 ) {
const int childType = xmlTextReaderNodeType( inputReader );
if ( childType != 1 && childType != 15 )
if ( childType != 1 && childType != 15 ) {
continue;
}
const int childDepth = xmlTextReaderDepth( inputReader );
xmlChar* childName = xmlTextReaderName( inputReader );
if ( childName == NULL )
if ( childName == NULL ) {
continue;
}
if ( depth == childDepth && childType == 15 && xmlStrEqual( childName, ( const xmlChar* ) "relation" ) == 1 ) {
xmlFree( childName );
break;
@ -102,18 +105,21 @@ _RawRestrictionContainer XMLParser::_ReadXMLRestriction() {
xmlChar* value = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "v" );
if ( k != NULL && value != NULL ) {
if(xmlStrEqual(k, ( const xmlChar* ) "restriction" )){
if(0 == std::string((const char *) value).find("only_"))
if(0 == std::string((const char *) value).find("only_")) {
restriction.restriction.flags.isOnly = true;
}
}
if ( xmlStrEqual(k, (const xmlChar *) "except") ) {
exception_string = (const char*) value;
except_tag_string = (const char*) value;
}
}
if ( k != NULL )
if ( k != NULL ) {
xmlFree( k );
if ( value != NULL )
}
if ( value != NULL ) {
xmlFree( value );
}
} else if ( xmlStrEqual( childName, ( const xmlChar* ) "member" ) == 1 ) {
xmlChar* ref = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "ref" );
if ( ref != NULL ) {
@ -130,21 +136,24 @@ _RawRestrictionContainer XMLParser::_ReadXMLRestriction() {
restriction.restriction.viaNode = atoi((const char*) ref);
}
if(NULL != type)
if(NULL != type) {
xmlFree( type );
if(NULL != role)
}
if(NULL != role) {
xmlFree( role );
if(NULL != ref)
}
if(NULL != ref) {
xmlFree( ref );
}
}
}
xmlFree( childName );
}
}
if( ShouldIgnoreRestriction(exception_string) )
restriction.fromWay = UINT_MAX; //workaround to ignore the restriction
if( ShouldIgnoreRestriction(except_tag_string) ) {
restriction.fromWay = UINT_MAX; //workaround to ignore the restriction
}
return restriction;
}
@ -154,12 +163,14 @@ ExtractionWay XMLParser::_ReadXMLWay() {
const int depth = xmlTextReaderDepth( inputReader );
while ( xmlTextReaderRead( inputReader ) == 1 ) {
const int childType = xmlTextReaderNodeType( inputReader );
if ( childType != 1 && childType != 15 )
if ( childType != 1 && childType != 15 ) {
continue;
}
const int childDepth = xmlTextReaderDepth( inputReader );
xmlChar* childName = xmlTextReaderName( inputReader );
if ( childName == NULL )
if ( childName == NULL ) {
continue;
}
if ( depth == childDepth && childType == 15 && xmlStrEqual( childName, ( const xmlChar* ) "way" ) == 1 ) {
xmlChar* id = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "id" );
@ -176,15 +187,16 @@ ExtractionWay XMLParser::_ReadXMLWay() {
if ( xmlStrEqual( childName, ( const xmlChar* ) "tag" ) == 1 ) {
xmlChar* k = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "k" );
xmlChar* value = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "v" );
// cout << "->k=" << k << ", v=" << value << endl;
// cout << "->k=" << k << ", v=" << value << endl;
if ( k != NULL && value != NULL ) {
way.keyVals.Add(std::string( (char *) k ), std::string( (char *) value));
}
if ( k != NULL )
if ( k != NULL ) {
xmlFree( k );
if ( value != NULL )
}
if ( value != NULL ) {
xmlFree( value );
}
} else if ( xmlStrEqual( childName, ( const xmlChar* ) "nd" ) == 1 ) {
xmlChar* ref = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "ref" );
if ( ref != NULL ) {
@ -222,12 +234,14 @@ ImportNode XMLParser::_ReadXMLNode() {
while ( xmlTextReaderRead( inputReader ) == 1 ) {
const int childType = xmlTextReaderNodeType( inputReader );
// 1 = Element, 15 = EndElement
if ( childType != 1 && childType != 15 )
if ( childType != 1 && childType != 15 ) {
continue;
}
const int childDepth = xmlTextReaderDepth( inputReader );
xmlChar* childName = xmlTextReaderName( inputReader );
if ( childName == NULL )
if ( childName == NULL ) {
continue;
}
if ( depth == childDepth && childType == 15 && xmlStrEqual( childName, ( const xmlChar* ) "node" ) == 1 ) {
xmlFree( childName );
@ -244,10 +258,12 @@ ImportNode XMLParser::_ReadXMLNode() {
if ( k != NULL && value != NULL ) {
node.keyVals.Add(std::string( reinterpret_cast<char*>(k) ), std::string( reinterpret_cast<char*>(value)));
}
if ( k != NULL )
if ( k != NULL ) {
xmlFree( k );
if ( value != NULL )
}
if ( value != NULL ) {
xmlFree( value );
}
}
xmlFree( childName );

2
Extractor/XMLParser.h
View File

@ -28,7 +28,7 @@
class XMLParser : public BaseParser {
public:
XMLParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const char* filename);
XMLParser(const char* filename, ExtractorCallbacks* ec, ScriptingEnvironment& se);
bool ReadHeader();
bool Parse();

7
extractor.cpp
View File

@ -96,13 +96,14 @@ int main (int argc, char *argv[]) {
extractCallBacks = new ExtractorCallbacks(&externalMemory, &stringMap);
BaseParser* parser;
if(isPBF) {
parser = new PBFParser(extractCallBacks, scriptingEnvironment, argv[1]);
parser = new PBFParser(argv[1], extractCallBacks, scriptingEnvironment);
} else {
parser = new XMLParser(extractCallBacks, scriptingEnvironment, argv[1]);
parser = new XMLParser(argv[1], extractCallBacks, scriptingEnvironment);
}
if(!parser->ReadHeader())
if(!parser->ReadHeader()) {
ERR("Parser not initialized!");
}
INFO("Parsing in progress..");
double time = get_timestamp();
parser->Parse();