support xml, move duplicated xml/pbf code to base

Extractor/BaseParser.cpp

@@ -0,0 +1,113 @@
+/*
+open source routing machine
+Copyright (C) Dennis Luxen, others 2010
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU AFFERO General Public License as published by
+the Free Software Foundation; either version 3 of the License, or
+any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+or see http://www.gnu.org/licenses/agpl.txt.
+*/
+
+#include "BaseParser.h"
+
+BaseParser::BaseParser(ExtractorCallbacks* em, ScriptingEnvironment& se) :
+externalMemory(em), scriptingEnvironment(se), luaState(NULL), use_turn_restrictions(true) {
+    luaState = se.getLuaStateForThreadID(0);
+    ReadUseRestrictionsSetting();
+    ReadRestrictionExceptions();
+}
+
+void BaseParser::ReadUseRestrictionsSetting() {
+    if(0 != luaL_dostring( luaState, "return use_turn_restrictions\n")) {
+        ERR(lua_tostring( luaState,-1)<< " occured in scripting block");
+    }
+    if( lua_isboolean( luaState, -1) ) {
+        use_turn_restrictions = lua_toboolean(luaState, -1);
+    }
+    if( use_turn_restrictions ) {
+        INFO("Using turn restrictions" );
+    } else {
+        INFO("Ignoring turn restrictions" );
+    }
+}
+
+void BaseParser::ReadRestrictionExceptions() {
+    if(lua_function_exists(luaState, "get_exceptions" )) {
+        //get list of turn restriction exceptions
+        try {
+            luabind::call_function<void>(
+                luaState,
+                "get_exceptions",
+                boost::ref(restriction_exceptions)
+                );
+            INFO("Found " << restriction_exceptions.size() << " exceptions to turn restriction");
+            BOOST_FOREACH(std::string & str, restriction_exceptions) {
+                INFO("   " << str);
+            }
+        } catch (const luabind::error &er) {
+            lua_State* Ler=er.state();
+            report_errors(Ler, -1);
+            ERR(er.what());
+        }
+    } else {
+        INFO("Found no exceptions to turn restrictions");
+    }
+}
+
+void BaseParser::report_errors(lua_State *L, int status) {
+    if ( status!=0 ) {
+        std::cerr << "-- " << lua_tostring(L, -1) << std::endl;
+        lua_pop(L, 1); // remove error message
+    }
+}
+
+inline void BaseParser::ParseNodeInLua(ImportNode& n, lua_State* localLuaState) {
+    try {
+        luabind::call_function<int>( localLuaState, "node_function", boost::ref(n) );
+    } catch (const luabind::error &er) {
+        lua_State* Ler=er.state();
+        report_errors(Ler, -1);
+        ERR(er.what());
+    }
+}
+
+inline void BaseParser::ParseWayInLua(ExtractionWay& w, lua_State* localLuaState) {
+    try {
+        luabind::call_function<int>( localLuaState, "way_function", boost::ref(w), w.path.size() );
+    } catch (const luabind::error &er) {
+        lua_State* Ler=er.state();
+        report_errors(Ler, -1);
+        ERR(er.what());
+    }
+}
+
+inline bool BaseParser::ShouldIgnoreRestriction(std::string& exception_string) {
+    //should this restriction be ignored? yes if there's an overlap between:
+    //a) the list of modes in the except tag of the restriction (exception_string), ex: except=bus;bicycle
+    //b) the lua profile defines a hierachy of modes, ex: [access, vehicle, bicycle]
+    
+    if( "" == exception_string )
+        return false;
+    
+    //Be warned, this is quadratic work here, but we assume that
+    //only a few exceptions are actually defined.
+    std::vector<std::string> exceptions;
+    boost::algorithm::split_regex(exceptions, exception_string, boost::regex("[;][ ]*"));
+    BOOST_FOREACH(std::string& str, exceptions) {
+        if( restriction_exceptions.end() != std::find(restriction_exceptions.begin(), restriction_exceptions.end(), str) ) {
+            return true;
+            break; //BOOST_FOREACH
+        }
+    }
+    return false;
+}

Extractor/BaseParser.h

@@ -29,23 +29,30 @@ extern "C" {
 
 #include <boost/noncopyable.hpp>
 
+#include "ExtractorCallbacks.h"
 #include "ScriptingEnvironment.h"
 
-template<class ExternalMemoryT, typename NodeT, typename RestrictionT, typename WayT>
 class BaseParser : boost::noncopyable {
 public:
+    BaseParser(ExtractorCallbacks* em, ScriptingEnvironment& se);
     virtual ~BaseParser() {}
-    virtual bool Init() = 0;
+    virtual bool ReadHeader() = 0;
-    virtual void RegisterCallbacks(ExternalMemoryT * externalMemory) = 0;
-    virtual void RegisterScriptingEnvironment(ScriptingEnvironment & _se) = 0;
     virtual bool Parse() = 0;
 
-    void report_errors(lua_State *L, int status) {
+    inline virtual void ParseNodeInLua(ImportNode& n, lua_State* luaStateForThread);
-        if ( status!=0 ) {
+    inline virtual void ParseWayInLua(ExtractionWay& n, lua_State* luaStateForThread);
-            std::cerr << "-- " << lua_tostring(L, -1) << std::endl;
+    virtual void report_errors(lua_State *L, int status);
-            lua_pop(L, 1); // remove error message
+
-        }
+protected:   
-    }
+    virtual void ReadUseRestrictionsSetting();
+    virtual void ReadRestrictionExceptions();
+    inline virtual bool ShouldIgnoreRestriction(std::string& exception_string);
+    
+    ExtractorCallbacks* externalMemory;
+    ScriptingEnvironment& scriptingEnvironment;
+    lua_State* luaState;
+    std::vector<std::string> restriction_exceptions;
+    bool use_turn_restrictions;
 
 };
 

Extractor/PBFParser.cpp

@@ -20,7 +20,7 @@
 
 #include "PBFParser.h"
 
-PBFParser::PBFParser(const char * fileName) : externalMemory(NULL), use_turn_restrictions(true) {
+PBFParser::PBFParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const char * fileName) : BaseParser( em, se ) {
 	GOOGLE_PROTOBUF_VERIFY_VERSION;
 	//TODO: What is the bottleneck here? Filling the queue or reading the stuff from disk?
 	//NOTE: With Lua scripting, it is parsing the stuff. I/O is virtually for free.
@@ -37,43 +37,6 @@ PBFParser::PBFParser(const char * fileName) : externalMemory(NULL), use_turn_res
 #endif
 }
 
-void PBFParser::RegisterCallbacks(ExtractorCallbacks * em) {
-	externalMemory = em;
-}
-
-void PBFParser::RegisterScriptingEnvironment(ScriptingEnvironment & _se) {
-	scriptingEnvironment = _se;
-    
-    if(0 != luaL_dostring( scriptingEnvironment.getLuaStateForThreadID(0), "return use_turn_restrictions\n")) {
-        ERR(lua_tostring(scriptingEnvironment.getLuaStateForThreadID(0),-1)<< " occured in scripting block");
-    }
-    if( lua_isboolean(scriptingEnvironment.getLuaStateForThreadID(0), -1) ) {
-        use_turn_restrictions = lua_toboolean(scriptingEnvironment.getLuaStateForThreadID(0), -1);
-    }
-	INFO("Use turn restrictions: " << (use_turn_restrictions ? "yes" : "no"));
-	
-	if(lua_function_exists(scriptingEnvironment.getLuaStateForThreadID(0), "get_exceptions" )) {
-		//get list of turn restriction exceptions
-		try {
-			luabind::call_function<void>(
-					scriptingEnvironment.getLuaStateForThreadID(0),
-					"get_exceptions",
-					boost::ref(restriction_exceptions_vector)
-			);
-			INFO("Found " << restriction_exceptions_vector.size() << " exceptions to turn restriction");
-			BOOST_FOREACH(std::string & str, restriction_exceptions_vector) {
-				INFO("ignoring: " << str);
-			}
-		} catch (const luabind::error &er) {
-			lua_State* Ler=er.state();
-			report_errors(Ler, -1);
-			ERR(er.what());
-		}
-	} else {
-		INFO("Found no exceptions to turn restrictions");
-	}
-}
-
 PBFParser::~PBFParser() {
 	if(input.is_open())
 		input.close();
@@ -90,7 +53,7 @@ PBFParser::~PBFParser() {
 #endif
 }
 
-inline bool PBFParser::Init() {
+inline bool PBFParser::ReadHeader() {
 	_ThreadData initData;
 	/** read Header */
 	if(!readPBFBlobHeader(input, &initData)) {
@@ -218,23 +181,8 @@ inline void PBFParser::parseDenseNode(_ThreadData * threadData) {
 #pragma omp parallel for schedule ( guided )
     		for(unsigned i = 0; i < endi_nodes; ++i) {
     			ImportNode &n = nodesToParse[i];
-    			/** Pass the unpacked node to the LUA call back **/
+                ParseNodeInLua( n, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) );
-    			try {
-    				luabind::call_function<int>(
-    						scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()),
-    						"node_function",
-    						boost::ref(n)
-    				);
-    			} catch (const luabind::error &er) {
-    				lua_State* Ler=er.state();
-    				report_errors(Ler, -1);
-    				ERR(er.what());
     		}
-    			//            catch (...) {
-    			//                ERR("Unknown error occurred during PBF dense node parsing!");
-    			//            }
-    		}
-
 
     		BOOST_FOREACH(ImportNode &n, nodesToParse) {
     			if(!externalMemory->nodeFunction(n))
@@ -249,12 +197,12 @@ inline void PBFParser::parseNode(_ThreadData * ) {
 inline void PBFParser::parseRelation(_ThreadData * threadData) {
 	//TODO: leave early, if relation is not a restriction
 	//TODO: reuse rawRestriction container
-	if( use_turn_restrictions==false )
+	if( !use_turn_restrictions )
         return;
     
 	const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID );
 	for(int i = 0; i < group.relations_size(); ++i ) {
-		std::string exception_of_restriction_tag;
+		std::string restriction_exceptions;
 		const OSMPBF::Relation& inputRelation = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).relations(i);
 		bool isRestriction = false;
 		bool isOnlyRestriction = false;
@@ -272,24 +220,12 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
 					isOnlyRestriction = true;
 			}
 			if ("except" == key) {
-				exception_of_restriction_tag = val;
+				restriction_exceptions = val;
 			}
 		}
 
-		//Check if restriction shall be ignored
+		if( isRestriction && ShouldIgnoreRestriction(restriction_exceptions) )
-		if(isRestriction && ("" != exception_of_restriction_tag) ) {
-			//Be warned, this is quadratic work here, but we assume that
-			//only a few exceptions are actually defined.
-			std::vector<std::string> tokenized_exception_tags_of_restriction;
-			boost::algorithm::split_regex(tokenized_exception_tags_of_restriction, exception_of_restriction_tag, boost::regex("[;][ ]*"));
-			BOOST_FOREACH(std::string & str, tokenized_exception_tags_of_restriction) {
-				if(restriction_exceptions_vector.end() != std::find(restriction_exceptions_vector.begin(), restriction_exceptions_vector.end(), str)) {
             isRestriction = false;
-					break; //BOOST_FOREACH
-				}
-			}
-		}
-
 
 		if(isRestriction) {
 			int64_t lastRef = 0;
@@ -368,31 +304,14 @@ inline void PBFParser::parseWay(_ThreadData * threadData) {
 #pragma omp parallel for schedule ( guided )
     		for(unsigned i = 0; i < endi_ways; ++i) {
     			ExtractionWay & w = waysToParse[i];
-    			/** Pass the unpacked way to the LUA call back **/
+                ParseWayInLua( w, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) );
-    			try {
-    				luabind::call_function<int>(
-    						scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()),
-    						"way_function",
-    						boost::ref(w),
-    						w.path.size()
-    				);
-
-    			} catch (const luabind::error &er) {
-    				lua_State* Ler=er.state();
-    				report_errors(Ler, -1);
-    				ERR(er.what());
-    			}
-    			//                catch (...) {
-    			//                    ERR("Unknown error!");
-    			//                }
     		}
 
     		BOOST_FOREACH(ExtractionWay & w, waysToParse) {
-    			if(!externalMemory->wayFunction(w)) {
+    			if(!externalMemory->wayFunction(w))
     				std::cerr << "[PBFParser] way not parsed" << std::endl;
     		}
 }
-}
 
 inline void PBFParser::loadGroup(_ThreadData * threadData) {
 #ifndef NDEBUG

Extractor/PBFParser.h

@@ -37,11 +37,8 @@
 #include "../Util/OpenMPWrapper.h"
 
 #include "BaseParser.h"
-#include "ExtractorCallbacks.h"
-#include "ExtractorStructs.h"
-#include "ScriptingEnvironment.h"
 
-class PBFParser : public BaseParser<ExtractorCallbacks, _Node, _RawRestrictionContainer, ExtractionWay> {
+class PBFParser : public BaseParser {
     
     enum EntityType {
         TypeNode = 1,
@@ -65,13 +62,10 @@ class PBFParser : public BaseParser<ExtractorCallbacks, _Node, _RawRestrictionCo
     };
     
 public:
-    PBFParser(const char * fileName);
+    PBFParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const char * fileName);
     virtual ~PBFParser();
     
-    void RegisterCallbacks(ExtractorCallbacks * em);
+    inline bool ReadHeader();
-    void RegisterScriptingEnvironment(ScriptingEnvironment & _se);
-    
-    inline bool Init();
 	inline bool Parse();
     
 private:
@@ -100,15 +94,8 @@ private:
     unsigned blockCount;
 #endif
 	
-    ExtractorCallbacks * externalMemory;
+    std::fstream input;     // the input stream to parse
-    /* the input stream to parse */
+    boost::shared_ptr<ConcurrentQueue < _ThreadData* > > threadDataQueue;       // ThreadData Queue
-    std::fstream input;
-    /* ThreadData Queue */
-    boost::shared_ptr<ConcurrentQueue < _ThreadData* > > threadDataQueue;
-    ScriptingEnvironment scriptingEnvironment;
-    
-    bool use_turn_restrictions;
-    std::vector<std::string> restriction_exceptions_vector;
 };
 
 #endif /* PBFPARSER_H_ */

Extractor/XMLParser.cpp

@@ -27,42 +27,12 @@
 #include "../DataStructures/InputReaderFactory.h"
 
 
-XMLParser::XMLParser(const char * filename) : externalMemory(NULL), myLuaState(NULL){
+XMLParser::XMLParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const char * filename) : BaseParser( em, se ) {
 	WARN("Parsing plain .osm/.osm.bz2 is deprecated. Switch to .pbf");
 	inputReader = inputReaderFactory(filename);
 }
 
-XMLParser::~XMLParser() {}
+bool XMLParser::ReadHeader() {
-
-void XMLParser::RegisterCallbacks(ExtractorCallbacks * em) {
-	externalMemory = em;
-}
-
-void XMLParser::RegisterScriptingEnvironment(ScriptingEnvironment & _se) {
-	myLuaState = _se.getLuaStateForThreadID(0);
-	if(lua_function_exists(myLuaState, "get_exceptions" )) {
-		//get list of turn restriction exceptions
-		try {
-			luabind::call_function<void>(
-					myLuaState,
-					"get_exceptions",
-					boost::ref(restriction_exceptions_vector)
-			);
-			INFO("Found " << restriction_exceptions_vector.size() << " exceptions to turn restriction");
-			BOOST_FOREACH(std::string & str, restriction_exceptions_vector) {
-				INFO("   " << str);
-			}
-		} catch (const luabind::error &er) {
-			lua_State* Ler=er.state();
-			report_errors(Ler, -1);
-			ERR(er.what());
-		}
-	} else {
-		INFO("Found no exceptions to turn restrictions");
-	}
-}
-
-bool XMLParser::Init() {
 	return (xmlTextReaderRead( inputReader ) == 1);
 }
 bool XMLParser::Parse() {
@@ -79,50 +49,18 @@ bool XMLParser::Parse() {
 
 		if ( xmlStrEqual( currentName, ( const xmlChar* ) "node" ) == 1 ) {
 			ImportNode n = _ReadXMLNode();
-			/** Pass the unpacked node to the LUA call back **/
+            ParseNodeInLua( n, luaState );
-			try {
+			
-				luabind::call_function<int>(
-						myLuaState,
-						"node_function",
-						boost::ref(n)
-				);
 			if(!externalMemory->nodeFunction(n))
 				std::cerr << "[XMLParser] dense node not parsed" << std::endl;
-			} catch (const luabind::error &er) {
-				std::cerr << er.what() << std::endl;
-				lua_State* Ler=er.state();
-				report_errors(Ler, -1);
-			} catch (std::exception & e) {
-				ERR(e.what());
-			} catch (...) {
-				ERR("Unknown error occurred during XML node parsing!");
-			}
 		}
 
 		if ( xmlStrEqual( currentName, ( const xmlChar* ) "way" ) == 1 ) {
 			ExtractionWay way = _ReadXMLWay( );
-
+            ParseWayInLua( way, luaState );
-			/** Pass the unpacked way to the LUA call back **/
+			if(!externalMemory->wayFunction(way))
-			try {
-				luabind::call_function<int>(
-						myLuaState,
-						"way_function",
-						boost::ref(way),
-						way.path.size()
-				);
-				if(!externalMemory->wayFunction(way)) {
 				std::cerr << "[PBFParser] way not parsed" << std::endl;
 		}
-			} catch (const luabind::error &er) {
-				std::cerr << er.what() << std::endl;
-				lua_State* Ler=er.state();
-				report_errors(Ler, -1);
-			} catch (std::exception & e) {
-				ERR(e.what());
-			} catch (...) {
-				ERR("Unknown error occurred during XML way parsing!");
-			}
-		}
 		if ( xmlStrEqual( currentName, ( const xmlChar* ) "relation" ) == 1 ) {
 			_RawRestrictionContainer r = _ReadXMLRestriction();
 			if(r.fromWay != UINT_MAX) {
@@ -138,7 +76,7 @@ bool XMLParser::Parse() {
 
 _RawRestrictionContainer XMLParser::_ReadXMLRestriction() {
 	_RawRestrictionContainer restriction;
-	std::string exception_of_restriction_tag;
+	std::string exception_string;
 
 	if ( xmlTextReaderIsEmptyElement( inputReader ) != 1 ) {
 		const int depth = xmlTextReaderDepth( inputReader );while ( xmlTextReaderRead( inputReader ) == 1 ) {
@@ -168,7 +106,7 @@ _RawRestrictionContainer XMLParser::_ReadXMLRestriction() {
 							restriction.restriction.flags.isOnly = true;
 					}
 					if ( xmlStrEqual(k, (const xmlChar *) "except") ) {
-						exception_of_restriction_tag = (const char*) value;
+						exception_string = (const char*) value;
 					}
 				}
 
@@ -204,19 +142,8 @@ _RawRestrictionContainer XMLParser::_ReadXMLRestriction() {
 		}
 	}
 
-	//Check if restriction shall be ignored
+    if( ShouldIgnoreRestriction(exception_string) )
-	if( "" != exception_of_restriction_tag ) {
-		//Be warned, this is quadratic work here, but we assume that
-		//only a few exceptions are actually defined.
-		std::vector<std::string> tokenized_exception_tags_of_restriction;
-		boost::algorithm::split_regex(tokenized_exception_tags_of_restriction, exception_of_restriction_tag, boost::regex("[;][ ]*"));
-		BOOST_FOREACH(std::string & str, tokenized_exception_tags_of_restriction) {
-			if(restriction_exceptions_vector.end() != std::find(restriction_exceptions_vector.begin(), restriction_exceptions_vector.end(), str)) {
         restriction.fromWay = UINT_MAX;                 //workaround to ignore the restriction
-				break; //BOOST_FOREACH
-			}
-		}
-	}
 
 	return restriction;
 }

Extractor/XMLParser.h

@@ -25,28 +25,18 @@
 
 #include "../typedefs.h"
 #include "BaseParser.h"
-#include "ExtractorCallbacks.h"
-#include "ScriptingEnvironment.h"
 
-class XMLParser : public BaseParser<ExtractorCallbacks, _Node, _RawRestrictionContainer, ExtractionWay> {
+class XMLParser : public BaseParser {
 public:
-    XMLParser(const char * filename);
+    XMLParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const char* filename);
-    virtual ~XMLParser();
+    bool ReadHeader();
-    void RegisterCallbacks(ExtractorCallbacks * em);
-    void RegisterScriptingEnvironment(ScriptingEnvironment & _se);
-    bool Init();
     bool Parse();
     
 private:
     _RawRestrictionContainer _ReadXMLRestriction();
     ExtractionWay _ReadXMLWay();
     ImportNode _ReadXMLNode();
-    /* Input Reader */
     xmlTextReaderPtr inputReader;
-    ExtractorCallbacks * externalMemory;
-    lua_State *myLuaState;
-
-    std::vector<std::string> restriction_exceptions_vector;
 };
 
 #endif /* XMLPARSER_H_ */

extractor.cpp

@@ -94,20 +94,19 @@ int main (int argc, char *argv[]) {
 
     stringMap[""] = 0;
     extractCallBacks = new ExtractorCallbacks(&externalMemory, &stringMap);
-    BaseParser<ExtractorCallbacks, _Node, _RawRestrictionContainer, ExtractionWay> * parser;
+    BaseParser* parser;
     if(isPBF) {
-        parser = new PBFParser(argv[1]);
+        parser = new PBFParser(extractCallBacks, scriptingEnvironment, argv[1]);
     } else {
-        parser = new XMLParser(argv[1]);
+        parser = new XMLParser(extractCallBacks, scriptingEnvironment, argv[1]);
     }
-    parser->RegisterCallbacks(extractCallBacks);
-    parser->RegisterScriptingEnvironment(scriptingEnvironment);
     
-    if(!parser->Init())
+    if(!parser->ReadHeader())
         ERR("Parser not initialized!");
+    INFO("Parsing in progress..");
     double time = get_timestamp();
     parser->Parse();
-    INFO("parsing finished after " << get_timestamp() - time << " seconds");
+    INFO("Parsing finished after " << get_timestamp() - time << " seconds");
 
     externalMemory.PrepareData(outputFileName, restrictionsFileName, amountOfRAM);
 

features/support/data.rb

@@ -205,14 +205,15 @@ def write_timestamp
 end
 
 def reprocess
+  use_pbf = true
   Dir.chdir TEST_FOLDER do
     write_osm
     write_timestamp
-    convert_osm_to_pbf
+    convert_osm_to_pbf if use_pbf
     unless extracted?
       log_preprocess_info
       log "== Extracting #{@osm_file}.osm...", :preprocess
-      unless system "../osrm-extract #{@osm_file}.osm.pbf 1>>#{PREPROCESS_LOG_FILE} 2>>#{PREPROCESS_LOG_FILE} #{PROFILES_PATH}/#{@profile}.lua"
+      unless system "../osrm-extract #{@osm_file}.osm#{'.pbf' if use_pbf} 1>>#{PREPROCESS_LOG_FILE} 2>>#{PREPROCESS_LOG_FILE} #{PROFILES_PATH}/#{@profile}.lua"
         log "*** Exited with code #{$?.exitstatus}.", :preprocess
         raise ExtractError.new $?.exitstatus, "osrm-extract exited with code #{$?.exitstatus}."
       end