support xml, move duplicated xml/pbf code to base
This commit is contained in:
parent
22c5c539c2
commit
1ecad20a0d
113
Extractor/BaseParser.cpp
Normal file
113
Extractor/BaseParser.cpp
Normal file
@ -0,0 +1,113 @@
|
|||||||
|
/*
|
||||||
|
open source routing machine
|
||||||
|
Copyright (C) Dennis Luxen, others 2010
|
||||||
|
|
||||||
|
This program is free software; you can redistribute it and/or modify
|
||||||
|
it under the terms of the GNU AFFERO General Public License as published by
|
||||||
|
the Free Software Foundation; either version 3 of the License, or
|
||||||
|
any later version.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU Affero General Public License
|
||||||
|
along with this program; if not, write to the Free Software
|
||||||
|
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||||
|
or see http://www.gnu.org/licenses/agpl.txt.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "BaseParser.h"
|
||||||
|
|
||||||
|
BaseParser::BaseParser(ExtractorCallbacks* em, ScriptingEnvironment& se) :
|
||||||
|
externalMemory(em), scriptingEnvironment(se), luaState(NULL), use_turn_restrictions(true) {
|
||||||
|
luaState = se.getLuaStateForThreadID(0);
|
||||||
|
ReadUseRestrictionsSetting();
|
||||||
|
ReadRestrictionExceptions();
|
||||||
|
}
|
||||||
|
|
||||||
|
void BaseParser::ReadUseRestrictionsSetting() {
|
||||||
|
if(0 != luaL_dostring( luaState, "return use_turn_restrictions\n")) {
|
||||||
|
ERR(lua_tostring( luaState,-1)<< " occured in scripting block");
|
||||||
|
}
|
||||||
|
if( lua_isboolean( luaState, -1) ) {
|
||||||
|
use_turn_restrictions = lua_toboolean(luaState, -1);
|
||||||
|
}
|
||||||
|
if( use_turn_restrictions ) {
|
||||||
|
INFO("Using turn restrictions" );
|
||||||
|
} else {
|
||||||
|
INFO("Ignoring turn restrictions" );
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void BaseParser::ReadRestrictionExceptions() {
|
||||||
|
if(lua_function_exists(luaState, "get_exceptions" )) {
|
||||||
|
//get list of turn restriction exceptions
|
||||||
|
try {
|
||||||
|
luabind::call_function<void>(
|
||||||
|
luaState,
|
||||||
|
"get_exceptions",
|
||||||
|
boost::ref(restriction_exceptions)
|
||||||
|
);
|
||||||
|
INFO("Found " << restriction_exceptions.size() << " exceptions to turn restriction");
|
||||||
|
BOOST_FOREACH(std::string & str, restriction_exceptions) {
|
||||||
|
INFO(" " << str);
|
||||||
|
}
|
||||||
|
} catch (const luabind::error &er) {
|
||||||
|
lua_State* Ler=er.state();
|
||||||
|
report_errors(Ler, -1);
|
||||||
|
ERR(er.what());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
INFO("Found no exceptions to turn restrictions");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void BaseParser::report_errors(lua_State *L, int status) {
|
||||||
|
if ( status!=0 ) {
|
||||||
|
std::cerr << "-- " << lua_tostring(L, -1) << std::endl;
|
||||||
|
lua_pop(L, 1); // remove error message
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
inline void BaseParser::ParseNodeInLua(ImportNode& n, lua_State* localLuaState) {
|
||||||
|
try {
|
||||||
|
luabind::call_function<int>( localLuaState, "node_function", boost::ref(n) );
|
||||||
|
} catch (const luabind::error &er) {
|
||||||
|
lua_State* Ler=er.state();
|
||||||
|
report_errors(Ler, -1);
|
||||||
|
ERR(er.what());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
inline void BaseParser::ParseWayInLua(ExtractionWay& w, lua_State* localLuaState) {
|
||||||
|
try {
|
||||||
|
luabind::call_function<int>( localLuaState, "way_function", boost::ref(w), w.path.size() );
|
||||||
|
} catch (const luabind::error &er) {
|
||||||
|
lua_State* Ler=er.state();
|
||||||
|
report_errors(Ler, -1);
|
||||||
|
ERR(er.what());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
inline bool BaseParser::ShouldIgnoreRestriction(std::string& exception_string) {
|
||||||
|
//should this restriction be ignored? yes if there's an overlap between:
|
||||||
|
//a) the list of modes in the except tag of the restriction (exception_string), ex: except=bus;bicycle
|
||||||
|
//b) the lua profile defines a hierachy of modes, ex: [access, vehicle, bicycle]
|
||||||
|
|
||||||
|
if( "" == exception_string )
|
||||||
|
return false;
|
||||||
|
|
||||||
|
//Be warned, this is quadratic work here, but we assume that
|
||||||
|
//only a few exceptions are actually defined.
|
||||||
|
std::vector<std::string> exceptions;
|
||||||
|
boost::algorithm::split_regex(exceptions, exception_string, boost::regex("[;][ ]*"));
|
||||||
|
BOOST_FOREACH(std::string& str, exceptions) {
|
||||||
|
if( restriction_exceptions.end() != std::find(restriction_exceptions.begin(), restriction_exceptions.end(), str) ) {
|
||||||
|
return true;
|
||||||
|
break; //BOOST_FOREACH
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
@ -29,23 +29,30 @@ extern "C" {
|
|||||||
|
|
||||||
#include <boost/noncopyable.hpp>
|
#include <boost/noncopyable.hpp>
|
||||||
|
|
||||||
|
#include "ExtractorCallbacks.h"
|
||||||
#include "ScriptingEnvironment.h"
|
#include "ScriptingEnvironment.h"
|
||||||
|
|
||||||
template<class ExternalMemoryT, typename NodeT, typename RestrictionT, typename WayT>
|
|
||||||
class BaseParser : boost::noncopyable {
|
class BaseParser : boost::noncopyable {
|
||||||
public:
|
public:
|
||||||
|
BaseParser(ExtractorCallbacks* em, ScriptingEnvironment& se);
|
||||||
virtual ~BaseParser() {}
|
virtual ~BaseParser() {}
|
||||||
virtual bool Init() = 0;
|
virtual bool ReadHeader() = 0;
|
||||||
virtual void RegisterCallbacks(ExternalMemoryT * externalMemory) = 0;
|
|
||||||
virtual void RegisterScriptingEnvironment(ScriptingEnvironment & _se) = 0;
|
|
||||||
virtual bool Parse() = 0;
|
virtual bool Parse() = 0;
|
||||||
|
|
||||||
void report_errors(lua_State *L, int status) {
|
inline virtual void ParseNodeInLua(ImportNode& n, lua_State* luaStateForThread);
|
||||||
if ( status!=0 ) {
|
inline virtual void ParseWayInLua(ExtractionWay& n, lua_State* luaStateForThread);
|
||||||
std::cerr << "-- " << lua_tostring(L, -1) << std::endl;
|
virtual void report_errors(lua_State *L, int status);
|
||||||
lua_pop(L, 1); // remove error message
|
|
||||||
}
|
protected:
|
||||||
}
|
virtual void ReadUseRestrictionsSetting();
|
||||||
|
virtual void ReadRestrictionExceptions();
|
||||||
|
inline virtual bool ShouldIgnoreRestriction(std::string& exception_string);
|
||||||
|
|
||||||
|
ExtractorCallbacks* externalMemory;
|
||||||
|
ScriptingEnvironment& scriptingEnvironment;
|
||||||
|
lua_State* luaState;
|
||||||
|
std::vector<std::string> restriction_exceptions;
|
||||||
|
bool use_turn_restrictions;
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -20,7 +20,7 @@
|
|||||||
|
|
||||||
#include "PBFParser.h"
|
#include "PBFParser.h"
|
||||||
|
|
||||||
PBFParser::PBFParser(const char * fileName) : externalMemory(NULL), use_turn_restrictions(true) {
|
PBFParser::PBFParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const char * fileName) : BaseParser( em, se ) {
|
||||||
GOOGLE_PROTOBUF_VERIFY_VERSION;
|
GOOGLE_PROTOBUF_VERIFY_VERSION;
|
||||||
//TODO: What is the bottleneck here? Filling the queue or reading the stuff from disk?
|
//TODO: What is the bottleneck here? Filling the queue or reading the stuff from disk?
|
||||||
//NOTE: With Lua scripting, it is parsing the stuff. I/O is virtually for free.
|
//NOTE: With Lua scripting, it is parsing the stuff. I/O is virtually for free.
|
||||||
@ -37,43 +37,6 @@ PBFParser::PBFParser(const char * fileName) : externalMemory(NULL), use_turn_res
|
|||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
void PBFParser::RegisterCallbacks(ExtractorCallbacks * em) {
|
|
||||||
externalMemory = em;
|
|
||||||
}
|
|
||||||
|
|
||||||
void PBFParser::RegisterScriptingEnvironment(ScriptingEnvironment & _se) {
|
|
||||||
scriptingEnvironment = _se;
|
|
||||||
|
|
||||||
if(0 != luaL_dostring( scriptingEnvironment.getLuaStateForThreadID(0), "return use_turn_restrictions\n")) {
|
|
||||||
ERR(lua_tostring(scriptingEnvironment.getLuaStateForThreadID(0),-1)<< " occured in scripting block");
|
|
||||||
}
|
|
||||||
if( lua_isboolean(scriptingEnvironment.getLuaStateForThreadID(0), -1) ) {
|
|
||||||
use_turn_restrictions = lua_toboolean(scriptingEnvironment.getLuaStateForThreadID(0), -1);
|
|
||||||
}
|
|
||||||
INFO("Use turn restrictions: " << (use_turn_restrictions ? "yes" : "no"));
|
|
||||||
|
|
||||||
if(lua_function_exists(scriptingEnvironment.getLuaStateForThreadID(0), "get_exceptions" )) {
|
|
||||||
//get list of turn restriction exceptions
|
|
||||||
try {
|
|
||||||
luabind::call_function<void>(
|
|
||||||
scriptingEnvironment.getLuaStateForThreadID(0),
|
|
||||||
"get_exceptions",
|
|
||||||
boost::ref(restriction_exceptions_vector)
|
|
||||||
);
|
|
||||||
INFO("Found " << restriction_exceptions_vector.size() << " exceptions to turn restriction");
|
|
||||||
BOOST_FOREACH(std::string & str, restriction_exceptions_vector) {
|
|
||||||
INFO("ignoring: " << str);
|
|
||||||
}
|
|
||||||
} catch (const luabind::error &er) {
|
|
||||||
lua_State* Ler=er.state();
|
|
||||||
report_errors(Ler, -1);
|
|
||||||
ERR(er.what());
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
INFO("Found no exceptions to turn restrictions");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
PBFParser::~PBFParser() {
|
PBFParser::~PBFParser() {
|
||||||
if(input.is_open())
|
if(input.is_open())
|
||||||
input.close();
|
input.close();
|
||||||
@ -90,7 +53,7 @@ PBFParser::~PBFParser() {
|
|||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
inline bool PBFParser::Init() {
|
inline bool PBFParser::ReadHeader() {
|
||||||
_ThreadData initData;
|
_ThreadData initData;
|
||||||
/** read Header */
|
/** read Header */
|
||||||
if(!readPBFBlobHeader(input, &initData)) {
|
if(!readPBFBlobHeader(input, &initData)) {
|
||||||
@ -218,24 +181,9 @@ inline void PBFParser::parseDenseNode(_ThreadData * threadData) {
|
|||||||
#pragma omp parallel for schedule ( guided )
|
#pragma omp parallel for schedule ( guided )
|
||||||
for(unsigned i = 0; i < endi_nodes; ++i) {
|
for(unsigned i = 0; i < endi_nodes; ++i) {
|
||||||
ImportNode &n = nodesToParse[i];
|
ImportNode &n = nodesToParse[i];
|
||||||
/** Pass the unpacked node to the LUA call back **/
|
ParseNodeInLua( n, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) );
|
||||||
try {
|
|
||||||
luabind::call_function<int>(
|
|
||||||
scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()),
|
|
||||||
"node_function",
|
|
||||||
boost::ref(n)
|
|
||||||
);
|
|
||||||
} catch (const luabind::error &er) {
|
|
||||||
lua_State* Ler=er.state();
|
|
||||||
report_errors(Ler, -1);
|
|
||||||
ERR(er.what());
|
|
||||||
}
|
|
||||||
// catch (...) {
|
|
||||||
// ERR("Unknown error occurred during PBF dense node parsing!");
|
|
||||||
// }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
BOOST_FOREACH(ImportNode &n, nodesToParse) {
|
BOOST_FOREACH(ImportNode &n, nodesToParse) {
|
||||||
if(!externalMemory->nodeFunction(n))
|
if(!externalMemory->nodeFunction(n))
|
||||||
std::cerr << "[PBFParser] dense node not parsed" << std::endl;
|
std::cerr << "[PBFParser] dense node not parsed" << std::endl;
|
||||||
@ -249,12 +197,12 @@ inline void PBFParser::parseNode(_ThreadData * ) {
|
|||||||
inline void PBFParser::parseRelation(_ThreadData * threadData) {
|
inline void PBFParser::parseRelation(_ThreadData * threadData) {
|
||||||
//TODO: leave early, if relation is not a restriction
|
//TODO: leave early, if relation is not a restriction
|
||||||
//TODO: reuse rawRestriction container
|
//TODO: reuse rawRestriction container
|
||||||
if( use_turn_restrictions==false )
|
if( !use_turn_restrictions )
|
||||||
return;
|
return;
|
||||||
|
|
||||||
const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID );
|
const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID );
|
||||||
for(int i = 0; i < group.relations_size(); ++i ) {
|
for(int i = 0; i < group.relations_size(); ++i ) {
|
||||||
std::string exception_of_restriction_tag;
|
std::string restriction_exceptions;
|
||||||
const OSMPBF::Relation& inputRelation = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).relations(i);
|
const OSMPBF::Relation& inputRelation = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).relations(i);
|
||||||
bool isRestriction = false;
|
bool isRestriction = false;
|
||||||
bool isOnlyRestriction = false;
|
bool isOnlyRestriction = false;
|
||||||
@ -272,24 +220,12 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
|
|||||||
isOnlyRestriction = true;
|
isOnlyRestriction = true;
|
||||||
}
|
}
|
||||||
if ("except" == key) {
|
if ("except" == key) {
|
||||||
exception_of_restriction_tag = val;
|
restriction_exceptions = val;
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//Check if restriction shall be ignored
|
|
||||||
if(isRestriction && ("" != exception_of_restriction_tag) ) {
|
|
||||||
//Be warned, this is quadratic work here, but we assume that
|
|
||||||
//only a few exceptions are actually defined.
|
|
||||||
std::vector<std::string> tokenized_exception_tags_of_restriction;
|
|
||||||
boost::algorithm::split_regex(tokenized_exception_tags_of_restriction, exception_of_restriction_tag, boost::regex("[;][ ]*"));
|
|
||||||
BOOST_FOREACH(std::string & str, tokenized_exception_tags_of_restriction) {
|
|
||||||
if(restriction_exceptions_vector.end() != std::find(restriction_exceptions_vector.begin(), restriction_exceptions_vector.end(), str)) {
|
|
||||||
isRestriction = false;
|
|
||||||
break; //BOOST_FOREACH
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if( isRestriction && ShouldIgnoreRestriction(restriction_exceptions) )
|
||||||
|
isRestriction = false;
|
||||||
|
|
||||||
if(isRestriction) {
|
if(isRestriction) {
|
||||||
int64_t lastRef = 0;
|
int64_t lastRef = 0;
|
||||||
@ -368,29 +304,12 @@ inline void PBFParser::parseWay(_ThreadData * threadData) {
|
|||||||
#pragma omp parallel for schedule ( guided )
|
#pragma omp parallel for schedule ( guided )
|
||||||
for(unsigned i = 0; i < endi_ways; ++i) {
|
for(unsigned i = 0; i < endi_ways; ++i) {
|
||||||
ExtractionWay & w = waysToParse[i];
|
ExtractionWay & w = waysToParse[i];
|
||||||
/** Pass the unpacked way to the LUA call back **/
|
ParseWayInLua( w, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) );
|
||||||
try {
|
|
||||||
luabind::call_function<int>(
|
|
||||||
scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()),
|
|
||||||
"way_function",
|
|
||||||
boost::ref(w),
|
|
||||||
w.path.size()
|
|
||||||
);
|
|
||||||
|
|
||||||
} catch (const luabind::error &er) {
|
|
||||||
lua_State* Ler=er.state();
|
|
||||||
report_errors(Ler, -1);
|
|
||||||
ERR(er.what());
|
|
||||||
}
|
|
||||||
// catch (...) {
|
|
||||||
// ERR("Unknown error!");
|
|
||||||
// }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
BOOST_FOREACH(ExtractionWay & w, waysToParse) {
|
BOOST_FOREACH(ExtractionWay & w, waysToParse) {
|
||||||
if(!externalMemory->wayFunction(w)) {
|
if(!externalMemory->wayFunction(w))
|
||||||
std::cerr << "[PBFParser] way not parsed" << std::endl;
|
std::cerr << "[PBFParser] way not parsed" << std::endl;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -37,11 +37,8 @@
|
|||||||
#include "../Util/OpenMPWrapper.h"
|
#include "../Util/OpenMPWrapper.h"
|
||||||
|
|
||||||
#include "BaseParser.h"
|
#include "BaseParser.h"
|
||||||
#include "ExtractorCallbacks.h"
|
|
||||||
#include "ExtractorStructs.h"
|
|
||||||
#include "ScriptingEnvironment.h"
|
|
||||||
|
|
||||||
class PBFParser : public BaseParser<ExtractorCallbacks, _Node, _RawRestrictionContainer, ExtractionWay> {
|
class PBFParser : public BaseParser {
|
||||||
|
|
||||||
enum EntityType {
|
enum EntityType {
|
||||||
TypeNode = 1,
|
TypeNode = 1,
|
||||||
@ -65,13 +62,10 @@ class PBFParser : public BaseParser<ExtractorCallbacks, _Node, _RawRestrictionCo
|
|||||||
};
|
};
|
||||||
|
|
||||||
public:
|
public:
|
||||||
PBFParser(const char * fileName);
|
PBFParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const char * fileName);
|
||||||
virtual ~PBFParser();
|
virtual ~PBFParser();
|
||||||
|
|
||||||
void RegisterCallbacks(ExtractorCallbacks * em);
|
inline bool ReadHeader();
|
||||||
void RegisterScriptingEnvironment(ScriptingEnvironment & _se);
|
|
||||||
|
|
||||||
inline bool Init();
|
|
||||||
inline bool Parse();
|
inline bool Parse();
|
||||||
|
|
||||||
private:
|
private:
|
||||||
@ -100,15 +94,8 @@ private:
|
|||||||
unsigned blockCount;
|
unsigned blockCount;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
ExtractorCallbacks * externalMemory;
|
std::fstream input; // the input stream to parse
|
||||||
/* the input stream to parse */
|
boost::shared_ptr<ConcurrentQueue < _ThreadData* > > threadDataQueue; // ThreadData Queue
|
||||||
std::fstream input;
|
|
||||||
/* ThreadData Queue */
|
|
||||||
boost::shared_ptr<ConcurrentQueue < _ThreadData* > > threadDataQueue;
|
|
||||||
ScriptingEnvironment scriptingEnvironment;
|
|
||||||
|
|
||||||
bool use_turn_restrictions;
|
|
||||||
std::vector<std::string> restriction_exceptions_vector;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
#endif /* PBFPARSER_H_ */
|
#endif /* PBFPARSER_H_ */
|
||||||
|
@ -27,42 +27,12 @@
|
|||||||
#include "../DataStructures/InputReaderFactory.h"
|
#include "../DataStructures/InputReaderFactory.h"
|
||||||
|
|
||||||
|
|
||||||
XMLParser::XMLParser(const char * filename) : externalMemory(NULL), myLuaState(NULL){
|
XMLParser::XMLParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const char * filename) : BaseParser( em, se ) {
|
||||||
WARN("Parsing plain .osm/.osm.bz2 is deprecated. Switch to .pbf");
|
WARN("Parsing plain .osm/.osm.bz2 is deprecated. Switch to .pbf");
|
||||||
inputReader = inputReaderFactory(filename);
|
inputReader = inputReaderFactory(filename);
|
||||||
}
|
}
|
||||||
|
|
||||||
XMLParser::~XMLParser() {}
|
bool XMLParser::ReadHeader() {
|
||||||
|
|
||||||
void XMLParser::RegisterCallbacks(ExtractorCallbacks * em) {
|
|
||||||
externalMemory = em;
|
|
||||||
}
|
|
||||||
|
|
||||||
void XMLParser::RegisterScriptingEnvironment(ScriptingEnvironment & _se) {
|
|
||||||
myLuaState = _se.getLuaStateForThreadID(0);
|
|
||||||
if(lua_function_exists(myLuaState, "get_exceptions" )) {
|
|
||||||
//get list of turn restriction exceptions
|
|
||||||
try {
|
|
||||||
luabind::call_function<void>(
|
|
||||||
myLuaState,
|
|
||||||
"get_exceptions",
|
|
||||||
boost::ref(restriction_exceptions_vector)
|
|
||||||
);
|
|
||||||
INFO("Found " << restriction_exceptions_vector.size() << " exceptions to turn restriction");
|
|
||||||
BOOST_FOREACH(std::string & str, restriction_exceptions_vector) {
|
|
||||||
INFO(" " << str);
|
|
||||||
}
|
|
||||||
} catch (const luabind::error &er) {
|
|
||||||
lua_State* Ler=er.state();
|
|
||||||
report_errors(Ler, -1);
|
|
||||||
ERR(er.what());
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
INFO("Found no exceptions to turn restrictions");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
bool XMLParser::Init() {
|
|
||||||
return (xmlTextReaderRead( inputReader ) == 1);
|
return (xmlTextReaderRead( inputReader ) == 1);
|
||||||
}
|
}
|
||||||
bool XMLParser::Parse() {
|
bool XMLParser::Parse() {
|
||||||
@ -78,50 +48,18 @@ bool XMLParser::Parse() {
|
|||||||
continue;
|
continue;
|
||||||
|
|
||||||
if ( xmlStrEqual( currentName, ( const xmlChar* ) "node" ) == 1 ) {
|
if ( xmlStrEqual( currentName, ( const xmlChar* ) "node" ) == 1 ) {
|
||||||
ImportNode n = _ReadXMLNode( );
|
ImportNode n = _ReadXMLNode();
|
||||||
/** Pass the unpacked node to the LUA call back **/
|
ParseNodeInLua( n, luaState );
|
||||||
try {
|
|
||||||
luabind::call_function<int>(
|
if(!externalMemory->nodeFunction(n))
|
||||||
myLuaState,
|
std::cerr << "[XMLParser] dense node not parsed" << std::endl;
|
||||||
"node_function",
|
|
||||||
boost::ref(n)
|
|
||||||
);
|
|
||||||
if(!externalMemory->nodeFunction(n))
|
|
||||||
std::cerr << "[XMLParser] dense node not parsed" << std::endl;
|
|
||||||
} catch (const luabind::error &er) {
|
|
||||||
std::cerr << er.what() << std::endl;
|
|
||||||
lua_State* Ler=er.state();
|
|
||||||
report_errors(Ler, -1);
|
|
||||||
} catch (std::exception & e) {
|
|
||||||
ERR(e.what());
|
|
||||||
} catch (...) {
|
|
||||||
ERR("Unknown error occurred during XML node parsing!");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if ( xmlStrEqual( currentName, ( const xmlChar* ) "way" ) == 1 ) {
|
if ( xmlStrEqual( currentName, ( const xmlChar* ) "way" ) == 1 ) {
|
||||||
ExtractionWay way = _ReadXMLWay( );
|
ExtractionWay way = _ReadXMLWay( );
|
||||||
|
ParseWayInLua( way, luaState );
|
||||||
/** Pass the unpacked way to the LUA call back **/
|
if(!externalMemory->wayFunction(way))
|
||||||
try {
|
std::cerr << "[PBFParser] way not parsed" << std::endl;
|
||||||
luabind::call_function<int>(
|
|
||||||
myLuaState,
|
|
||||||
"way_function",
|
|
||||||
boost::ref(way),
|
|
||||||
way.path.size()
|
|
||||||
);
|
|
||||||
if(!externalMemory->wayFunction(way)) {
|
|
||||||
std::cerr << "[PBFParser] way not parsed" << std::endl;
|
|
||||||
}
|
|
||||||
} catch (const luabind::error &er) {
|
|
||||||
std::cerr << er.what() << std::endl;
|
|
||||||
lua_State* Ler=er.state();
|
|
||||||
report_errors(Ler, -1);
|
|
||||||
} catch (std::exception & e) {
|
|
||||||
ERR(e.what());
|
|
||||||
} catch (...) {
|
|
||||||
ERR("Unknown error occurred during XML way parsing!");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if ( xmlStrEqual( currentName, ( const xmlChar* ) "relation" ) == 1 ) {
|
if ( xmlStrEqual( currentName, ( const xmlChar* ) "relation" ) == 1 ) {
|
||||||
_RawRestrictionContainer r = _ReadXMLRestriction();
|
_RawRestrictionContainer r = _ReadXMLRestriction();
|
||||||
@ -138,7 +76,7 @@ bool XMLParser::Parse() {
|
|||||||
|
|
||||||
_RawRestrictionContainer XMLParser::_ReadXMLRestriction() {
|
_RawRestrictionContainer XMLParser::_ReadXMLRestriction() {
|
||||||
_RawRestrictionContainer restriction;
|
_RawRestrictionContainer restriction;
|
||||||
std::string exception_of_restriction_tag;
|
std::string exception_string;
|
||||||
|
|
||||||
if ( xmlTextReaderIsEmptyElement( inputReader ) != 1 ) {
|
if ( xmlTextReaderIsEmptyElement( inputReader ) != 1 ) {
|
||||||
const int depth = xmlTextReaderDepth( inputReader );while ( xmlTextReaderRead( inputReader ) == 1 ) {
|
const int depth = xmlTextReaderDepth( inputReader );while ( xmlTextReaderRead( inputReader ) == 1 ) {
|
||||||
@ -168,7 +106,7 @@ _RawRestrictionContainer XMLParser::_ReadXMLRestriction() {
|
|||||||
restriction.restriction.flags.isOnly = true;
|
restriction.restriction.flags.isOnly = true;
|
||||||
}
|
}
|
||||||
if ( xmlStrEqual(k, (const xmlChar *) "except") ) {
|
if ( xmlStrEqual(k, (const xmlChar *) "except") ) {
|
||||||
exception_of_restriction_tag = (const char*) value;
|
exception_string = (const char*) value;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -204,19 +142,8 @@ _RawRestrictionContainer XMLParser::_ReadXMLRestriction() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//Check if restriction shall be ignored
|
if( ShouldIgnoreRestriction(exception_string) )
|
||||||
if( "" != exception_of_restriction_tag ) {
|
restriction.fromWay = UINT_MAX; //workaround to ignore the restriction
|
||||||
//Be warned, this is quadratic work here, but we assume that
|
|
||||||
//only a few exceptions are actually defined.
|
|
||||||
std::vector<std::string> tokenized_exception_tags_of_restriction;
|
|
||||||
boost::algorithm::split_regex(tokenized_exception_tags_of_restriction, exception_of_restriction_tag, boost::regex("[;][ ]*"));
|
|
||||||
BOOST_FOREACH(std::string & str, tokenized_exception_tags_of_restriction) {
|
|
||||||
if(restriction_exceptions_vector.end() != std::find(restriction_exceptions_vector.begin(), restriction_exceptions_vector.end(), str)) {
|
|
||||||
restriction.fromWay = UINT_MAX; //workaround to ignore the restriction
|
|
||||||
break; //BOOST_FOREACH
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return restriction;
|
return restriction;
|
||||||
}
|
}
|
||||||
|
@ -25,28 +25,18 @@
|
|||||||
|
|
||||||
#include "../typedefs.h"
|
#include "../typedefs.h"
|
||||||
#include "BaseParser.h"
|
#include "BaseParser.h"
|
||||||
#include "ExtractorCallbacks.h"
|
|
||||||
#include "ScriptingEnvironment.h"
|
|
||||||
|
|
||||||
class XMLParser : public BaseParser<ExtractorCallbacks, _Node, _RawRestrictionContainer, ExtractionWay> {
|
class XMLParser : public BaseParser {
|
||||||
public:
|
public:
|
||||||
XMLParser(const char * filename);
|
XMLParser(ExtractorCallbacks* em, ScriptingEnvironment& se, const char* filename);
|
||||||
virtual ~XMLParser();
|
bool ReadHeader();
|
||||||
void RegisterCallbacks(ExtractorCallbacks * em);
|
|
||||||
void RegisterScriptingEnvironment(ScriptingEnvironment & _se);
|
|
||||||
bool Init();
|
|
||||||
bool Parse();
|
bool Parse();
|
||||||
|
|
||||||
private:
|
private:
|
||||||
_RawRestrictionContainer _ReadXMLRestriction();
|
_RawRestrictionContainer _ReadXMLRestriction();
|
||||||
ExtractionWay _ReadXMLWay();
|
ExtractionWay _ReadXMLWay();
|
||||||
ImportNode _ReadXMLNode( );
|
ImportNode _ReadXMLNode();
|
||||||
/* Input Reader */
|
|
||||||
xmlTextReaderPtr inputReader;
|
xmlTextReaderPtr inputReader;
|
||||||
ExtractorCallbacks * externalMemory;
|
|
||||||
lua_State *myLuaState;
|
|
||||||
|
|
||||||
std::vector<std::string> restriction_exceptions_vector;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
#endif /* XMLPARSER_H_ */
|
#endif /* XMLPARSER_H_ */
|
||||||
|
@ -94,20 +94,19 @@ int main (int argc, char *argv[]) {
|
|||||||
|
|
||||||
stringMap[""] = 0;
|
stringMap[""] = 0;
|
||||||
extractCallBacks = new ExtractorCallbacks(&externalMemory, &stringMap);
|
extractCallBacks = new ExtractorCallbacks(&externalMemory, &stringMap);
|
||||||
BaseParser<ExtractorCallbacks, _Node, _RawRestrictionContainer, ExtractionWay> * parser;
|
BaseParser* parser;
|
||||||
if(isPBF) {
|
if(isPBF) {
|
||||||
parser = new PBFParser(argv[1]);
|
parser = new PBFParser(extractCallBacks, scriptingEnvironment, argv[1]);
|
||||||
} else {
|
} else {
|
||||||
parser = new XMLParser(argv[1]);
|
parser = new XMLParser(extractCallBacks, scriptingEnvironment, argv[1]);
|
||||||
}
|
}
|
||||||
parser->RegisterCallbacks(extractCallBacks);
|
|
||||||
parser->RegisterScriptingEnvironment(scriptingEnvironment);
|
if(!parser->ReadHeader())
|
||||||
|
|
||||||
if(!parser->Init())
|
|
||||||
ERR("Parser not initialized!");
|
ERR("Parser not initialized!");
|
||||||
|
INFO("Parsing in progress..");
|
||||||
double time = get_timestamp();
|
double time = get_timestamp();
|
||||||
parser->Parse();
|
parser->Parse();
|
||||||
INFO("parsing finished after " << get_timestamp() - time << " seconds");
|
INFO("Parsing finished after " << get_timestamp() - time << " seconds");
|
||||||
|
|
||||||
externalMemory.PrepareData(outputFileName, restrictionsFileName, amountOfRAM);
|
externalMemory.PrepareData(outputFileName, restrictionsFileName, amountOfRAM);
|
||||||
|
|
||||||
|
@ -205,14 +205,15 @@ def write_timestamp
|
|||||||
end
|
end
|
||||||
|
|
||||||
def reprocess
|
def reprocess
|
||||||
|
use_pbf = true
|
||||||
Dir.chdir TEST_FOLDER do
|
Dir.chdir TEST_FOLDER do
|
||||||
write_osm
|
write_osm
|
||||||
write_timestamp
|
write_timestamp
|
||||||
convert_osm_to_pbf
|
convert_osm_to_pbf if use_pbf
|
||||||
unless extracted?
|
unless extracted?
|
||||||
log_preprocess_info
|
log_preprocess_info
|
||||||
log "== Extracting #{@osm_file}.osm...", :preprocess
|
log "== Extracting #{@osm_file}.osm...", :preprocess
|
||||||
unless system "../osrm-extract #{@osm_file}.osm.pbf 1>>#{PREPROCESS_LOG_FILE} 2>>#{PREPROCESS_LOG_FILE} #{PROFILES_PATH}/#{@profile}.lua"
|
unless system "../osrm-extract #{@osm_file}.osm#{'.pbf' if use_pbf} 1>>#{PREPROCESS_LOG_FILE} 2>>#{PREPROCESS_LOG_FILE} #{PROFILES_PATH}/#{@profile}.lua"
|
||||||
log "*** Exited with code #{$?.exitstatus}.", :preprocess
|
log "*** Exited with code #{$?.exitstatus}.", :preprocess
|
||||||
raise ExtractError.new $?.exitstatus, "osrm-extract exited with code #{$?.exitstatus}."
|
raise ExtractError.new $?.exitstatus, "osrm-extract exited with code #{$?.exitstatus}."
|
||||||
end
|
end
|
||||||
|
Loading…
Reference in New Issue
Block a user