Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Cleaning of several regressions in the parsing code.

Browse Source
This commit is contained in:
DennisOSRM 2013-02-27 17:36:44 +01:00
parent af5f2f85da
commit 53af4ee39f
11 changed files with 297 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Extractor/BaseParser.cpp
View File

@ -21,7 +21,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "BaseParser.h"
BaseParser::BaseParser(ExtractorCallbacks* ec, ScriptingEnvironment& se) :
externalMemory(ec), scriptingEnvironment(se), luaState(NULL), use_turn_restrictions(true) {
extractor_callbacks(ec), scriptingEnvironment(se), luaState(NULL), use_turn_restrictions(true) {
luaState = se.getLuaStateForThreadID(0);
ReadUseRestrictionsSetting();
ReadRestrictionExceptions();
@ -73,7 +73,7 @@ void BaseParser::report_errors(lua_State *L, const int status) const {
void BaseParser::ParseNodeInLua(ImportNode& n, lua_State* localLuaState) {
try {
luabind::call_function<int>( localLuaState, "node_function", boost::ref(n) );
luabind::call_function<void>( localLuaState, "node_function", boost::ref(n) );
} catch (const luabind::error &er) {
lua_State* Ler=er.state();
report_errors(Ler, -1);
@ -82,8 +82,11 @@ void BaseParser::ParseNodeInLua(ImportNode& n, lua_State* localLuaState) {
}
void BaseParser::ParseWayInLua(ExtractionWay& w, lua_State* localLuaState) {
if(2 > w.path.size()) {
return;
}
try {
luabind::call_function<int>( localLuaState, "way_function", boost::ref(w), w.path.size() );
luabind::call_function<void>( localLuaState, "way_function", boost::ref(w) );
} catch (const luabind::error &er) {
lua_State* Ler=er.state();
report_errors(Ler, -1);

2
Extractor/BaseParser.h
View File

@ -48,7 +48,7 @@ protected:
virtual void ReadRestrictionExceptions();
virtual bool ShouldIgnoreRestriction(const std::string& except_tag_string) const;
ExtractorCallbacks* externalMemory;
ExtractorCallbacks* extractor_callbacks;
ScriptingEnvironment& scriptingEnvironment;
lua_State* luaState;
std::vector<std::string> restriction_exceptions;

59
Extractor/ExtractorCallbacks.cpp
View File

@ -48,58 +48,54 @@ ExtractorCallbacks::ExtractorCallbacks(ExtractionContainers * ext, StringMap * s
stringMap = strMap;
}
ExtractorCallbacks::~ExtractorCallbacks() {
}
ExtractorCallbacks::~ExtractorCallbacks() { }
/** warning: caller needs to take care of synchronization! */
bool ExtractorCallbacks::nodeFunction(_Node &n) {
if(n.lat <= 85*100000 && n.lat >= -85*100000)
void ExtractorCallbacks::nodeFunction(const _Node &n) {
if(n.lat <= 85*100000 && n.lat >= -85*100000) {
externalMemory->allNodes.push_back(n);
return true;
}
}
bool ExtractorCallbacks::restrictionFunction(_RawRestrictionContainer &r) {
bool ExtractorCallbacks::restrictionFunction(const _RawRestrictionContainer &r) {
externalMemory->restrictionsVector.push_back(r);
return true;
}
/** warning: caller needs to take care of synchronization! */
bool ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) {
if ( parsed_way.speed > 0 || (0 < parsed_way.duration)) { //Only true if the way is specified by the speed profile
if(parsed_way.id == UINT_MAX){
WARN("found bogus way with id: " << parsed_way.id << " of size " << parsed_way.path.size());
return true;
}
//Get the unique identifier for the street name
const StringMap::const_iterator string_map_iterator = stringMap->find(parsed_way.name);
if(string_map_iterator == stringMap->end()) {
parsed_way.nameID = externalMemory->nameVector.size();
externalMemory->nameVector.push_back(parsed_way.name);
stringMap->insert(StringMap::value_type(parsed_way.name, parsed_way.nameID));
} else {
parsed_way.nameID = string_map_iterator->second;
void ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) {
if((0 < parsed_way.speed) || (0 < parsed_way.duration)) { //Only true if the way is specified by the speed profile
if(UINT_MAX == parsed_way.id){
DEBUG("found bogus way with id: " << parsed_way.id << " of size " << parsed_way.path.size());
return;
}
if(parsed_way.duration > 0) {
if(0 < parsed_way.duration) {
//TODO: iterate all way segments and set duration corresponding to the length of each segment
parsed_way.speed = parsed_way.duration/(parsed_way.path.size()-1);
}
if(fabs(-1. - parsed_way.speed) < FLT_EPSILON){
WARN("found way with bogus speed, id: " << parsed_way.id);
return true;
}
if(parsed_way.id == UINT_MAX) {
WARN("found way with unknown type: " << parsed_way.id);
return true;
if(FLT_EPSILON >= fabs(-1. - parsed_way.speed)){
DEBUG("found way with bogus speed, id: " << parsed_way.id);
return;
}
if ( parsed_way.direction == ExtractionWay::opposite ){
//Get the unique identifier for the street name
const StringMap::const_iterator string_map_iterator = stringMap->find(parsed_way.name);
if(stringMap->end() == string_map_iterator) {
parsed_way.nameID = externalMemory->nameVector.size();
externalMemory->nameVector.push_back(parsed_way.name);
stringMap->insert(std::make_pair(parsed_way.name, parsed_way.nameID));
} else {
parsed_way.nameID = string_map_iterator->second;
}
if(ExtractionWay::opposite == parsed_way.direction) {
std::reverse( parsed_way.path.begin(), parsed_way.path.end() );
parsed_way.direction = ExtractionWay::oneway;
}
bool split_bidirectional_edge = (parsed_way.backward_speed > 0) && (parsed_way.speed != parsed_way.backward_speed);
const bool split_bidirectional_edge = (parsed_way.backward_speed > 0) && (parsed_way.speed != parsed_way.backward_speed);
for(std::vector< NodeID >::size_type n = 0; n < parsed_way.path.size()-1; ++n) {
externalMemory->allEdges.push_back(
@ -122,7 +118,7 @@ bool ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) {
//The following information is needed to identify start and end segments of restrictions
externalMemory->wayStartEndVector.push_back(_WayIDStartAndEndEdge(parsed_way.id, parsed_way.path[0], parsed_way.path[1], parsed_way.path[parsed_way.path.size()-2], parsed_way.path.back()));
if ( split_bidirectional_edge) { //Only true if the way should be split
if(split_bidirectional_edge) { //Only true if the way should be split
std::reverse( parsed_way.path.begin(), parsed_way.path.end() );
for(std::vector< NodeID >::size_type n = 0; n < parsed_way.path.size()-1; ++n) {
externalMemory->allEdges.push_back(
@ -143,5 +139,4 @@ bool ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) {
externalMemory->wayStartEndVector.push_back(_WayIDStartAndEndEdge(parsed_way.id, parsed_way.path[0], parsed_way.path[1], parsed_way.path[parsed_way.path.size()-2], parsed_way.path.back()));
}
}
return true;
}

6
Extractor/ExtractorCallbacks.h
View File

@ -45,12 +45,12 @@ public:
~ExtractorCallbacks();
/** warning: caller needs to take care of synchronization! */
bool nodeFunction(_Node &n);
void nodeFunction(const _Node &n);
bool restrictionFunction(_RawRestrictionContainer &r);
bool restrictionFunction(const _RawRestrictionContainer &r);
/** warning: caller needs to take care of synchronization! */
bool wayFunction(ExtractionWay &w);
void wayFunction(ExtractionWay &w);
};

59
Extractor/PBFParser.cpp
View File

@ -160,14 +160,16 @@ inline void PBFParser::parseDenseNode(_ThreadData * threadData) {
int m_lastDenseLongitude = 0;
ImportNode n;
std::vector<ImportNode> nodesToParse;
for(int i = 0, idSize = dense.id_size(); i < idSize; ++i) {
std::vector<ImportNode> extracted_nodes_vector;
const int number_of_nodes = dense.id_size();
extracted_nodes_vector.reserve(number_of_nodes);
for(int i = 0; i < number_of_nodes; ++i) {
n.Clear();
m_lastDenseID += dense.id( i );
m_lastDenseLatitude += dense.lat( i );
m_lastDenseLongitude += dense.lon( i );
n.id = m_lastDenseID;
n.lat = 100000*( ( double ) m_lastDenseLatitude * threadData->PBFprimitiveBlock.granularity() +threadData-> PBFprimitiveBlock.lat_offset() ) / NANO;
n.lat = 100000*( ( double ) m_lastDenseLatitude * threadData->PBFprimitiveBlock.granularity() + threadData->PBFprimitiveBlock.lat_offset() ) / NANO;
n.lon = 100000*( ( double ) m_lastDenseLongitude * threadData->PBFprimitiveBlock.granularity() + threadData->PBFprimitiveBlock.lon_offset() ) / NANO;
while (denseTagIndex < dense.keys_vals_size()) {
const int tagValue = dense.keys_vals( denseTagIndex );
@ -181,20 +183,18 @@ inline void PBFParser::parseDenseNode(_ThreadData * threadData) {
n.keyVals.Add(key, value);
denseTagIndex += 2;
}
nodesToParse.push_back(n);
extracted_nodes_vector.push_back(n);
}
unsigned endi_nodes = nodesToParse.size();
#pragma omp parallel for schedule ( guided )
for(unsigned i = 0; i < endi_nodes; ++i) {
ImportNode &n = nodesToParse[i];
ParseNodeInLua( n, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) );
}
for(int i = 0; i < number_of_nodes; ++i) {
ImportNode &n = extracted_nodes_vector[i];
ParseNodeInLua( n, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) );
}
BOOST_FOREACH(ImportNode &n, nodesToParse) {
if(!externalMemory->nodeFunction(n))
std::cerr << "[PBFParser] dense node not parsed" << std::endl;
}
BOOST_FOREACH(ImportNode &n, extracted_nodes_vector) {
extractor_callbacks->nodeFunction(n);
}
}
inline void PBFParser::parseNode(_ThreadData * ) {
@ -234,7 +234,7 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
}
if( isRestriction && ShouldIgnoreRestriction(except_tag_string) ) {
isRestriction = false;
continue;
}
if(isRestriction) {
@ -284,7 +284,7 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
break;
}
}
if(!externalMemory->restrictionFunction(currentRestrictionContainer)) {
if(!extractor_callbacks->restrictionFunction(currentRestrictionContainer)) {
std::cerr << "[PBFParser] relation not parsed" << std::endl;
}
}
@ -293,37 +293,38 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
inline void PBFParser::parseWay(_ThreadData * threadData) {
ExtractionWay w;
std::vector<ExtractionWay> waysToParse(threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).ways_size());
for(int i = 0, ways_size = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).ways_size(); i < ways_size; ++i) {
std::vector<ExtractionWay> waysToParse;
const int number_of_ways = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).ways_size();
waysToParse.reserve(number_of_ways);
for(int i = 0; i < number_of_ways; ++i) {
w.Clear();
const OSMPBF::Way& inputWay = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).ways( i );
w.id = inputWay.id();
unsigned pathNode(0);
for(int i = 0; i < inputWay.refs_size(); ++i) {
const int number_of_referenced_nodes = inputWay.refs_size();
for(int i = 0; i < number_of_referenced_nodes; ++i) {
pathNode += inputWay.refs(i);
w.path.push_back(pathNode);
}
assert(inputWay.keys_size() == inputWay.vals_size());
for(int i = 0; i < inputWay.keys_size(); ++i) {
const int number_of_keys = inputWay.keys_size();
for(int i = 0; i < number_of_keys; ++i) {
const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(inputWay.keys(i));
const std::string & val = threadData->PBFprimitiveBlock.stringtable().s(inputWay.vals(i));
w.keyVals.Add(key, val);
}
waysToParse.push_back(w);
}
unsigned endi_ways = waysToParse.size();
#pragma omp parallel for schedule ( guided )
for(unsigned i = 0; i < endi_ways; ++i) {
ExtractionWay & w = waysToParse[i];
ParseWayInLua( w, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) );
}
for(int i = 0; i < number_of_ways; ++i) {
ExtractionWay & w = waysToParse[i];
ParseWayInLua( w, scriptingEnvironment.getLuaStateForThreadID(omp_get_thread_num()) );
}
BOOST_FOREACH(ExtractionWay & w, waysToParse) {
if(!externalMemory->wayFunction(w))
std::cerr << "[PBFParser] way not parsed" << std::endl;
}
BOOST_FOREACH(ExtractionWay & w, waysToParse) {
extractor_callbacks->wayFunction(w);
}
}
inline void PBFParser::loadGroup(_ThreadData * threadData) {

13
Extractor/XMLParser.cpp
View File

@ -52,22 +52,23 @@ bool XMLParser::Parse() {
if ( xmlStrEqual( currentName, ( const xmlChar* ) "node" ) == 1 ) {
ImportNode n = _ReadXMLNode();
ParseNodeInLua( n, luaState );
if(!externalMemory->nodeFunction(n))
std::cerr << "[XMLParser] dense node not parsed" << std::endl;
extractor_callbacks->nodeFunction(n);
// if(!extractor_callbacks->nodeFunction(n))
// std::cerr << "[XMLParser] dense node not parsed" << std::endl;
}
if ( xmlStrEqual( currentName, ( const xmlChar* ) "way" ) == 1 ) {
ExtractionWay way = _ReadXMLWay( );
ParseWayInLua( way, luaState );
if(!externalMemory->wayFunction(way))
std::cerr << "[PBFParser] way not parsed" << std::endl;
extractor_callbacks->wayFunction(way);
// if(!extractor_callbacks->wayFunction(way))
// std::cerr << "[PBFParser] way not parsed" << std::endl;
}
if( use_turn_restrictions ) {
if ( xmlStrEqual( currentName, ( const xmlChar* ) "relation" ) == 1 ) {
_RawRestrictionContainer r = _ReadXMLRestriction();
if(r.fromWay != UINT_MAX) {
if(!externalMemory->restrictionFunction(r)) {
if(!extractor_callbacks->restrictionFunction(r)) {
std::cerr << "[XMLParser] restriction not parsed" << std::endl;
}
}

1
profile.lua
View File

@ -1 +0,0 @@
profiles/car.lua

219
profile.lua Normal file
View File

@ -0,0 +1,219 @@
-- Begin of globals
require("lib/access")
barrier_whitelist = { ["cattle_grid"] = true, ["border_control"] = true, ["toll_booth"] = true, ["sally_port"] = true, ["gate"] = true, ["no"] = true}
access_tag_whitelist = { ["yes"] = true, ["motorcar"] = true, ["motor_vehicle"] = true, ["vehicle"] = true, ["permissive"] = true, ["designated"] = true }
access_tag_blacklist = { ["no"] = true, ["private"] = true, ["agricultural"] = true, ["forestry"] = true }
access_tag_restricted = { ["destination"] = true, ["delivery"] = true }
access_tags = { "motorcar", "motor_vehicle", "vehicle" }
access_tags_hierachy = { "motorcar", "motor_vehicle", "vehicle", "access" }
service_tag_restricted = { ["parking_aisle"] = true }
ignore_in_grid = { ["ferry"] = true }
restriction_exception_tags = { "motorcar", "motor_vehicle", "vehicle" }
speed_profile = {
["motorway"] = 90,
["motorway_link"] = 75,
["trunk"] = 85,
["trunk_link"] = 70,
["primary"] = 65,
["primary_link"] = 60,
["secondary"] = 55,
["secondary_link"] = 50,
["tertiary"] = 40,
["tertiary_link"] = 30,
["unclassified"] = 25,
["residential"] = 25,
["living_street"] = 10,
["service"] = 15,
-- ["track"] = 5,
["ferry"] = 5,
["shuttle_train"] = 10,
["default"] = 50
}
take_minimum_of_speeds = false
obey_oneway = true
obey_bollards = true
use_restrictions = true
ignore_areas = true -- future feature
traffic_signal_penalty = 2
u_turn_penalty = 20
-- End of globals
function get_exceptions(vector)
for i,v in ipairs(restriction_exception_tags) do
vector:Add(v)
end
end
local function parse_maxspeed(source)
if source == nil then
return 0
end
local n = tonumber(source:match("%d*"))
if n == nil then
n = 0
end
if string.match(source, "mph") or string.match(source, "mp/h") then
n = (n*1609)/1000;
end
return math.abs(n)
end
function node_function (node)
local barrier = node.tags:Find ("barrier")
local access = Access.find_access_tag(node, access_tags_hierachy)
local traffic_signal = node.tags:Find("highway")
--flag node if it carries a traffic light
if traffic_signal == "traffic_signals" then
node.traffic_light = true;
end
-- parse access and barrier tags
if access and access ~= "" then
if access_tag_blacklist[access] then
node.bollard = true
end
elseif barrier and barrier ~= "" then
if barrier_whitelist[barrier] then
return
else
node.bollard = true
end
end
return 1
end
function way_function (way)
-- First, get the properties of each way that we come across
local highway = way.tags:Find("highway")
local name = way.tags:Find("name")
local ref = way.tags:Find("ref")
local junction = way.tags:Find("junction")
local route = way.tags:Find("route")
local maxspeed = parse_maxspeed(way.tags:Find ( "maxspeed") )
local maxspeed_forward = tonumber(way.tags:Find( "maxspeed:forward"))
local maxspeed_backward = tonumber(way.tags:Find( "maxspeed:backward"))
local barrier = way.tags:Find("barrier")
local oneway = way.tags:Find("oneway")
local cycleway = way.tags:Find("cycleway")
local duration = way.tags:Find("duration")
local service = way.tags:Find("service")
local area = way.tags:Find("area")
local access = Access.find_access_tag(way, access_tags_hierachy)
-- Second, parse the way according to these properties
if ignore_areas and ("yes" == area) then
return 0
end
-- Check if we are allowed to access the way
if access_tag_blacklist[access] then
return 0
end
-- Set the name that will be used for instructions
if "" ~= ref then
way.name = ref
elseif "" ~= name then
way.name = name
-- else
-- way.name = highway -- if no name exists, use way type
end
if "roundabout" == junction then
way.roundabout = true;
end
-- Handling ferries and piers
if (speed_profile[route] ~= nil and speed_profile[route] > 0) then
if durationIsValid(duration) then
way.duration = math.max( parseDuration(duration), 1 );
end
way.direction = Way.bidirectional
if speed_profile[route] ~= nil then
highway = route;
end
if tonumber(way.duration) < 0 then
way.speed = speed_profile[highway]
end
end
-- Set the avg speed on the way if it is accessible by road class
if (speed_profile[highway] ~= nil and way.speed == -1 ) then
if maxspeed > speed_profile[highway] then
way.speed = maxspeed
else
if 0 == maxspeed then
maxspeed = math.huge
end
way.speed = math.min(speed_profile[highway], maxspeed)
end
end
-- Set the avg speed on ways that are marked accessible
if "" ~= highway and access_tag_whitelist[access] and way.speed == -1 then
if 0 == maxspeed then
maxspeed = math.huge
end
way.speed = math.min(speed_profile["default"], maxspeed)
end
-- Set access restriction flag if access is allowed under certain restrictions only
if access ~= "" and access_tag_restricted[access] then
way.is_access_restricted = true
end
-- Set access restriction flag if service is allowed under certain restrictions only
if service ~= "" and service_tag_restricted[service] then
way.is_access_restricted = true
end
-- Set direction according to tags on way
if obey_oneway then
if oneway == "no" or oneway == "0" or oneway == "false" then
way.direction = Way.bidirectional
elseif oneway == "-1" then
way.direction = Way.opposite
elseif oneway == "yes" or oneway == "1" or oneway == "true" or junction == "roundabout" or highway == "motorway_link" or highway == "motorway" then
way.direction = Way.oneway
else
way.direction = Way.bidirectional
end
else
way.direction = Way.bidirectional
end
-- Override speed settings if explicit forward/backward maxspeeds are given
if maxspeed_forward ~= nil and maxspeed_forward > 0 then
if Way.bidirectional == way.direction then
way.backward_speed = way.speed
end
way.speed = maxspeed_forward
end
if maxspeed_backward ~= nil and maxspeed_backward > 0 then
way.backward_speed = maxspeed_backward
end
-- Override general direction settings of there is a specific one for our mode of travel
if ignore_in_grid[highway] ~= nil and ignore_in_grid[highway] then
way.ignore_in_grid = true
end
way.type = 1
return 1
end
-- These are wrappers to parse vectors of nodes and ways and thus to speed up any tracing JIT
function node_vector_function(vector)
for v in vector.nodes do
node_function(v)
end
end

7
profiles/bicycle.lua
View File

@ -109,12 +109,7 @@ function node_function (node)
return 1
end
function way_function (way, numberOfNodesInWay)
-- A way must have two nodes or more
if(numberOfNodesInWay < 2) then
return 0;
end
function way_function (way)
-- initial routability check, filters out buildings, boundaries, etc
local highway = way.tags:Find("highway")
local route = way.tags:Find("route")

8
profiles/car.lua
View File

@ -89,13 +89,7 @@ function node_function (node)
end
function way_function (way, numberOfNodesInWay)
-- A way must have two nodes or more
if(numberOfNodesInWay < 2) then
return 0;
end
function way_function (way)
-- First, get the properties of each way that we come across
local highway = way.tags:Find("highway")
local name = way.tags:Find("name")

7
profiles/foot.lua
View File

@ -75,13 +75,8 @@ function node_function (node)
return 1
end
function way_function (way, numberOfNodesInWay)
function way_function (way)
-- A way must have two nodes or more
if(numberOfNodesInWay < 2) then
return 0;
end
-- First, get the properties of each way that we come across
local highway = way.tags:Find("highway")
local name = way.tags:Find("name")

7
profiles/testbot.lua
View File

@ -46,12 +46,7 @@ function node_function (node)
return 1
end
function way_function (way, numberOfNodesInWay)
-- A way must have two nodes or more
if(numberOfNodesInWay < 2) then
return 0;
end
function way_function (way)
local highway = way.tags:Find("highway")
local name = way.tags:Find("name")
local oneway = way.tags:Find("oneway")