Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Cleaning of several regressions in the parsing code.

Browse Source
This commit is contained in:
DennisOSRM
2013-02-27 17:36:44 +01:00
parent af5f2f85da
commit 53af4ee39f
11 changed files with 297 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Extractor/BaseParser.cpp
+6 -3
View File
@@ -21,7 +21,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "BaseParser.h"
BaseParser::BaseParser(ExtractorCallbacks* ec, ScriptingEnvironment& se) :
externalMemory(ec), scriptingEnvironment(se), luaState(NULL), use_turn_restrictions(true) {
extractor_callbacks(ec), scriptingEnvironment(se), luaState(NULL), use_turn_restrictions(true) {
luaState = se.getLuaStateForThreadID(0);
ReadUseRestrictionsSetting();
ReadRestrictionExceptions();
@@ -73,7 +73,7 @@ void BaseParser::report_errors(lua_State *L, const int status) const {
void BaseParser::ParseNodeInLua(ImportNode& n, lua_State* localLuaState) {
try {
luabind::call_function<int>( localLuaState, "node_function", boost::ref(n) );
luabind::call_function<void>( localLuaState, "node_function", boost::ref(n) );
} catch (const luabind::error &er) {
lua_State* Ler=er.state();
report_errors(Ler, -1);
@@ -82,8 +82,11 @@ void BaseParser::ParseNodeInLua(ImportNode& n, lua_State* localLuaState) {
}
void BaseParser::ParseWayInLua(ExtractionWay& w, lua_State* localLuaState) {
if(2 > w.path.size()) {
return;
}
try {
luabind::call_function<int>( localLuaState, "way_function", boost::ref(w), w.path.size() );
luabind::call_function<void>( localLuaState, "way_function", boost::ref(w) );
} catch (const luabind::error &er) {
lua_State* Ler=er.state();
report_errors(Ler, -1);
Extractor/BaseParser.h
+1 -1
View File
@@ -48,7 +48,7 @@ protected:
virtual void ReadRestrictionExceptions();
virtual bool ShouldIgnoreRestriction(const std::string& except_tag_string) const;
ExtractorCallbacks* externalMemory;
ExtractorCallbacks* extractor_callbacks;
ScriptingEnvironment& scriptingEnvironment;
lua_State* luaState;
std::vector<std::string> restriction_exceptions;
Extractor/ExtractorCallbacks.cpp
+27 -32
View File
@@ -48,58 +48,54 @@ ExtractorCallbacks::ExtractorCallbacks(ExtractionContainers * ext, StringMap * s
stringMap = strMap;
}
ExtractorCallbacks::~ExtractorCallbacks() {
}
ExtractorCallbacks::~ExtractorCallbacks() { }
/** warning: caller needs to take care of synchronization! */
bool ExtractorCallbacks::nodeFunction(_Node &n) {
if(n.lat <= 85*100000 && n.lat >= -85*100000)
void ExtractorCallbacks::nodeFunction(const _Node &n) {
if(n.lat <= 85*100000 && n.lat >= -85*100000) {
externalMemory->allNodes.push_back(n);
return true;
}
}
bool ExtractorCallbacks::restrictionFunction(_RawRestrictionContainer &r) {
bool ExtractorCallbacks::restrictionFunction(const _RawRestrictionContainer &r) {
externalMemory->restrictionsVector.push_back(r);
return true;
}
/** warning: caller needs to take care of synchronization! */
bool ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) {
if ( parsed_way.speed > 0 || (0 < parsed_way.duration)) { //Only true if the way is specified by the speed profile
if(parsed_way.id == UINT_MAX){
WARN("found bogus way with id: " << parsed_way.id << " of size " << parsed_way.path.size());
return true;
}
//Get the unique identifier for the street name
const StringMap::const_iterator string_map_iterator = stringMap->find(parsed_way.name);
if(string_map_iterator == stringMap->end()) {
parsed_way.nameID = externalMemory->nameVector.size();
externalMemory->nameVector.push_back(parsed_way.name);
stringMap->insert(StringMap::value_type(parsed_way.name, parsed_way.nameID));
} else {
parsed_way.nameID = string_map_iterator->second;
void ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) {
if((0 < parsed_way.speed) || (0 < parsed_way.duration)) { //Only true if the way is specified by the speed profile
if(UINT_MAX == parsed_way.id){
DEBUG("found bogus way with id: " << parsed_way.id << " of size " << parsed_way.path.size());
return;
}
if(parsed_way.duration > 0) {
if(0 < parsed_way.duration) {
//TODO: iterate all way segments and set duration corresponding to the length of each segment
parsed_way.speed = parsed_way.duration/(parsed_way.path.size()-1);
}
if(fabs(-1. - parsed_way.speed) < FLT_EPSILON){
WARN("found way with bogus speed, id: " << parsed_way.id);
return true;
}
if(parsed_way.id == UINT_MAX) {
WARN("found way with unknown type: " << parsed_way.id);
return true;
if(FLT_EPSILON >= fabs(-1. - parsed_way.speed)){
DEBUG("found way with bogus speed, id: " << parsed_way.id);
return;
}
if ( parsed_way.direction == ExtractionWay::opposite ){
//Get the unique identifier for the street name
const StringMap::const_iterator string_map_iterator = stringMap->find(parsed_way.name);
if(stringMap->end() == string_map_iterator) {
parsed_way.nameID = externalMemory->nameVector.size();
externalMemory->nameVector.push_back(parsed_way.name);
stringMap->insert(std::make_pair(parsed_way.name, parsed_way.nameID));
} else {
parsed_way.nameID = string_map_iterator->second;
}
if(ExtractionWay::opposite == parsed_way.direction) {
std::reverse( parsed_way.path.begin(), parsed_way.path.end() );
parsed_way.direction = ExtractionWay::oneway;
}
bool split_bidirectional_edge = (parsed_way.backward_speed > 0) && (parsed_way.speed != parsed_way.backward_speed);
const bool split_bidirectional_edge = (parsed_way.backward_speed > 0) && (parsed_way.speed != parsed_way.backward_speed);
for(std::vector< NodeID >::size_type n = 0; n < parsed_way.path.size()-1; ++n) {
externalMemory->allEdges.push_back(
@@ -122,7 +118,7 @@ bool ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) {
//The following information is needed to identify start and end segments of restrictions
externalMemory->wayStartEndVector.push_back(_WayIDStartAndEndEdge(parsed_way.id, parsed_way.path[0], parsed_way.path[1], parsed_way.path[parsed_way.path.size()-2], parsed_way.path.back()));
if ( split_bidirectional_edge) { //Only true if the way should be split
if(split_bidirectional_edge) { //Only true if the way should be split
std::reverse( parsed_way.path.begin(), parsed_way.path.end() );
for(std::vector< NodeID >::size_type n = 0; n < parsed_way.path.size()-1; ++n) {
externalMemory->allEdges.push_back(
@@ -143,5 +139,4 @@ bool ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) {
externalMemory->wayStartEndVector.push_back(_WayIDStartAndEndEdge(parsed_way.id, parsed_way.path[0], parsed_way.path[1], parsed_way.path[parsed_way.path.size()-2], parsed_way.path.back()));
}
}
return true;
}
Extractor/ExtractorCallbacks.h
+3 -3
View File
@@ -45,12 +45,12 @@ public:
~ExtractorCallbacks();
/** warning: caller needs to take care of synchronization! */
bool nodeFunction(_Node &n);
void nodeFunction(const _Node &n);
bool restrictionFunction(_RawRestrictionContainer &r);
bool restrictionFunction(const _RawRestrictionContainer &r);
/** warning: caller needs to take care of synchronization! */
bool wayFunction(ExtractionWay &w);
void wayFunction(ExtractionWay &w);
};
Extractor/PBFParser.cpp
+30 -29
View File
@@ -160,14 +160,16 @@ inline void PBFParser::parseDenseNode(_ThreadData * threadData) {
int m_lastDenseLongitude = 0;
ImportNode n;
std::vector<ImportNode> nodesToParse;
for(int i = 0, idSize = dense.id_size(); i < idSize; ++i) {
std::vector<ImportNode> extracted_nodes_vector;
const int number_of_nodes = dense.id_size();
extracted_nodes_vector.reserve(number_of_nodes);
for(int i = 0; i < number_of_nodes; ++i) {
n.Clear();
m_lastDenseID += dense.id( i );
m_lastDenseLatitude += dense.lat( i );
m_lastDenseLongitude += dense.lon( i );
n.id = m_lastDenseID;
n.lat = 100000*( ( double ) m_lastDenseLatitude * threadData->PBFprimitiveBlock.granularity() +threadData-> PBFprimitiveBlock.lat_offset() ) / NANO;
n.lat = 100000*( ( double ) m_lastDenseLatitude * threadData->PBFprimitiveBlock.granularity() + threadData->PBFprimitiveBlock.lat_offset() ) / NANO;
n.lon = 100000*( ( double ) m_lastDenseLongitude * threadData->PBFprimitiveBlock.granularity() + threadData->PBFprimitiveBlock.lon_offset() ) / NANO;
while (denseTagIndex < dense.keys_vals_size()) {
const int tagValue = dense.keys_vals( denseTagIndex );
@@ -181,20 +183,18 @@ inline void PBFParser::parseDenseNode(_ThreadData * threadData) {
n.keyVals.Add(key, value);
denseTagIndex += 2;
}
nodesToParse.push_back(n);
extracted_nodes_vector.push_back(n);
}
unsigned endi_nodes = nodesToParse.size();
#pragma omp parallel for schedule ( guided )
for(unsigned i = 0; i < endi_nodes; ++i) {
ImportNode &n = nodesToParse[i];
ParseNodeInLua( n, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) );
}
for(int i = 0; i < number_of_nodes; ++i) {
ImportNode &n = extracted_nodes_vector[i];
ParseNodeInLua( n, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) );
}
BOOST_FOREACH(ImportNode &n, nodesToParse) {
if(!externalMemory->nodeFunction(n))
std::cerr << "[PBFParser] dense node not parsed" << std::endl;
}
BOOST_FOREACH(ImportNode &n, extracted_nodes_vector) {
extractor_callbacks->nodeFunction(n);
}
}
inline void PBFParser::parseNode(_ThreadData * ) {
@@ -234,7 +234,7 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
}
if( isRestriction && ShouldIgnoreRestriction(except_tag_string) ) {
isRestriction = false;
continue;
}
if(isRestriction) {
@@ -284,7 +284,7 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
break;
}
}
if(!externalMemory->restrictionFunction(currentRestrictionContainer)) {
if(!extractor_callbacks->restrictionFunction(currentRestrictionContainer)) {
std::cerr << "[PBFParser] relation not parsed" << std::endl;
}
}
@@ -293,37 +293,38 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
inline void PBFParser::parseWay(_ThreadData * threadData) {
ExtractionWay w;
std::vector<ExtractionWay> waysToParse(threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).ways_size());
for(int i = 0, ways_size = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).ways_size(); i < ways_size; ++i) {
std::vector<ExtractionWay> waysToParse;
const int number_of_ways = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).ways_size();
waysToParse.reserve(number_of_ways);
for(int i = 0; i < number_of_ways; ++i) {
w.Clear();
const OSMPBF::Way& inputWay = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).ways( i );
w.id = inputWay.id();
unsigned pathNode(0);
for(int i = 0; i < inputWay.refs_size(); ++i) {
const int number_of_referenced_nodes = inputWay.refs_size();
for(int i = 0; i < number_of_referenced_nodes; ++i) {
pathNode += inputWay.refs(i);
w.path.push_back(pathNode);
}
assert(inputWay.keys_size() == inputWay.vals_size());
for(int i = 0; i < inputWay.keys_size(); ++i) {
const int number_of_keys = inputWay.keys_size();
for(int i = 0; i < number_of_keys; ++i) {
const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(inputWay.keys(i));
const std::string & val = threadData->PBFprimitiveBlock.stringtable().s(inputWay.vals(i));
w.keyVals.Add(key, val);
}
waysToParse.push_back(w);
}
unsigned endi_ways = waysToParse.size();
#pragma omp parallel for schedule ( guided )
for(unsigned i = 0; i < endi_ways; ++i) {
ExtractionWay & w = waysToParse[i];
ParseWayInLua( w, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) );
}
for(int i = 0; i < number_of_ways; ++i) {
ExtractionWay & w = waysToParse[i];
ParseWayInLua( w, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) );
}
BOOST_FOREACH(ExtractionWay & w, waysToParse) {
if(!externalMemory->wayFunction(w))
std::cerr << "[PBFParser] way not parsed" << std::endl;
}
BOOST_FOREACH(ExtractionWay & w, waysToParse) {
extractor_callbacks->wayFunction(w);
}
}
inline void PBFParser::loadGroup(_ThreadData * threadData) {
Extractor/XMLParser.cpp
+7 -6
View File
@@ -52,22 +52,23 @@ bool XMLParser::Parse() {
if ( xmlStrEqual( currentName, ( const xmlChar* ) "node" ) == 1 ) {
ImportNode n = _ReadXMLNode();
ParseNodeInLua( n, luaState );
if(!externalMemory->nodeFunction(n))
std::cerr << "[XMLParser] dense node not parsed" << std::endl;
extractor_callbacks->nodeFunction(n);
// if(!extractor_callbacks->nodeFunction(n))
// std::cerr << "[XMLParser] dense node not parsed" << std::endl;
}
if ( xmlStrEqual( currentName, ( const xmlChar* ) "way" ) == 1 ) {
ExtractionWay way = _ReadXMLWay( );
ParseWayInLua( way, luaState );
if(!externalMemory->wayFunction(way))
std::cerr << "[PBFParser] way not parsed" << std::endl;
extractor_callbacks->wayFunction(way);
// if(!extractor_callbacks->wayFunction(way))
// std::cerr << "[PBFParser] way not parsed" << std::endl;
}
if( use_turn_restrictions ) {
if ( xmlStrEqual( currentName, ( const xmlChar* ) "relation" ) == 1 ) {
_RawRestrictionContainer r = _ReadXMLRestriction();
if(r.fromWay != UINT_MAX) {
if(!externalMemory->restrictionFunction(r)) {
if(!extractor_callbacks->restrictionFunction(r)) {
std::cerr << "[XMLParser] restriction not parsed" << std::endl;
}
}