Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Removal of v8 as it sucked big time during integration and first

Browse Source 
(partially) working parsing with LUA as the scripting engine.
This commit is contained in:
DennisOSRM 2012-08-29 18:33:18 +02:00
parent 4c58674393
commit 1412c5db42
14 changed files with 901 additions and 872 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
DataStructures/ImportNode.h
View File

@ -22,6 +22,8 @@ or see http://www.gnu.org/licenses/agpl.txt.
#define IMPORTNODE_H_ #define IMPORTNODE_H_
#include "NodeCoords.h" #include "NodeCoords.h"
#include "../DataStructures/HashTable.h"
struct _Node : NodeInfo{ struct _Node : NodeInfo{
_Node(int _lat, int _lon, unsigned int _id, bool _bollard, bool _trafficLight) : NodeInfo(_lat, _lon, _id), bollard(_bollard), trafficLight(_trafficLight) {} _Node(int _lat, int _lon, unsigned int _id, bool _bollard, bool _trafficLight) : NodeInfo(_lat, _lon, _id), bollard(_bollard), trafficLight(_trafficLight) {}
@ -38,7 +40,10 @@ struct _Node : NodeInfo{
} }
bool bollard; bool bollard;
bool trafficLight; bool trafficLight;
};
struct ImportNode : public _Node {
HashTable<std::string, std::string> keyVals;
}; };
#endif /* IMPORTNODE_H_ */ #endif /* IMPORTNODE_H_ */

5
Extractor/BaseParser.h
View File

@ -21,12 +21,15 @@ or see http://www.gnu.org/licenses/agpl.txt.
#ifndef BASEPARSER_H_ #ifndef BASEPARSER_H_
#define BASEPARSER_H_ #define BASEPARSER_H_
#include <luabind/luabind.hpp>
template<typename NodeT, typename RestrictionT, typename WayT> template<typename NodeT, typename RestrictionT, typename WayT>
class BaseParser { class BaseParser {
public: public:
virtual ~BaseParser() {} virtual ~BaseParser() {}
virtual bool Init() = 0; virtual bool Init() = 0;
virtual bool RegisterCallbacks(bool (*nodeCallbackPointer)(NodeT), bool (*restrictionCallbackPointer)(RestrictionT), bool (*wayCallbackPointer)(WayT), bool (*addressCallbackPointer)(NodeT, HashTable<std::string, std::string>&)) = 0; virtual bool RegisterCallbacks(bool (*nodeCallbackPointer)(NodeT), bool (*restrictionCallbackPointer)(RestrictionT), bool (*wayCallbackPointer)(WayT)) = 0;
virtual void RegisterLUAState(lua_State *myLuaState) = 0;
virtual bool Parse() = 0; virtual bool Parse() = 0;
}; };

317
Extractor/ExtractionContainers.cpp Normal file
View File

@ -0,0 +1,317 @@
/*
open source routing machine
Copyright (C) Dennis Luxen, others 2010
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU AFFERO General Public License as published by
the Free Software Foundation; either version 3 of the License, or
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
or see http://www.gnu.org/licenses/agpl.txt.
*/
#include "ExtractionContainers.h"
void ExtractionContainers::PrepareData(const Settings & settings, const std::string & outputFileName, const std::string restrictionsFileName, const unsigned amountOfRAM) {
try {
unsigned usedNodeCounter = 0;
unsigned usedEdgeCounter = 0;
double time = get_timestamp();
// INFO("raw no. of names: " << nameVector.size());
// INFO("raw no. of nodes: " << allNodes.size());
// INFO("no. of used nodes: " << usedNodeIDs.size());
// INFO("raw no. of edges: " << allEdges.size());
// INFO("raw no. of ways: " << wayStartEndVector.size());
// INFO("raw no. of addresses: " << adressVector.size());
// INFO("raw no. of restrictions: " << restrictionsVector.size());
cout << "[extractor] parsing finished after " << get_timestamp() - time << " seconds" << endl;
time = get_timestamp();
boost::uint64_t memory_to_use = static_cast<boost::uint64_t>(amountOfRAM) * 1024 * 1024 * 1024;
cout << "[extractor] Sorting used nodes ... " << flush;
stxxl::sort(usedNodeIDs.begin(), usedNodeIDs.end(), Cmp(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Erasing duplicate nodes ... " << flush;
stxxl::vector<NodeID>::iterator NewEnd = unique ( usedNodeIDs.begin(),usedNodeIDs.end() ) ;
usedNodeIDs.resize ( NewEnd - usedNodeIDs.begin() );
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Sorting all nodes ... " << flush;
stxxl::sort(allNodes.begin(), allNodes.end(), CmpNodeByID(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Sorting used ways ... " << flush;
stxxl::sort(wayStartEndVector.begin(), wayStartEndVector.end(), CmpWayByID(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
cout << "[extractor] Sorting restrctns. by from... " << flush;
stxxl::sort(restrictionsVector.begin(), restrictionsVector.end(), CmpRestrictionContainerByFrom(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
cout << "[extractor] Fixing restriction starts ... " << flush;
STXXLRestrictionsVector::iterator restrictionsIT = restrictionsVector.begin();
STXXLWayIDStartEndVector::iterator wayStartAndEndEdgeIT = wayStartEndVector.begin();
while(wayStartAndEndEdgeIT != wayStartEndVector.end() && restrictionsIT != restrictionsVector.end()) {
if(wayStartAndEndEdgeIT->wayID < restrictionsIT->fromWay){
++wayStartAndEndEdgeIT;
continue;
}
if(wayStartAndEndEdgeIT->wayID > restrictionsIT->fromWay) {
++restrictionsIT;
continue;
}
assert(wayStartAndEndEdgeIT->wayID == restrictionsIT->fromWay);
NodeID viaNode = restrictionsIT->restriction.viaNode;
if(wayStartAndEndEdgeIT->firstStart == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstTarget;
} else if(wayStartAndEndEdgeIT->firstTarget == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstStart;
} else if(wayStartAndEndEdgeIT->lastStart == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastTarget;
} else if(wayStartAndEndEdgeIT->lastTarget == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastStart;
}
++restrictionsIT;
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Sorting restrctns. by to ... " << flush;
stxxl::sort(restrictionsVector.begin(), restrictionsVector.end(), CmpRestrictionContainerByTo(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
unsigned usableRestrictionsCounter(0);
cout << "[extractor] Fixing restriction ends ... " << flush;
restrictionsIT = restrictionsVector.begin();
wayStartAndEndEdgeIT = wayStartEndVector.begin();
while(wayStartAndEndEdgeIT != wayStartEndVector.end() && restrictionsIT != restrictionsVector.end()) {
if(wayStartAndEndEdgeIT->wayID < restrictionsIT->toWay){
++wayStartAndEndEdgeIT;
continue;
}
if(wayStartAndEndEdgeIT->wayID > restrictionsIT->toWay) {
++restrictionsIT;
continue;
}
NodeID viaNode = restrictionsIT->restriction.viaNode;
if(wayStartAndEndEdgeIT->lastStart == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastTarget;
} else if(wayStartAndEndEdgeIT->lastTarget == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastStart;
} else if(wayStartAndEndEdgeIT->firstStart == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstTarget;
} else if(wayStartAndEndEdgeIT->firstTarget == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstStart;
}
if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) {
++usableRestrictionsCounter;
}
++restrictionsIT;
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
INFO("usable restrictions: " << usableRestrictionsCounter );
//serialize restrictions
ofstream restrictionsOutstream;
restrictionsOutstream.open(restrictionsFileName.c_str(), ios::binary);
restrictionsOutstream.write((char*)&usableRestrictionsCounter, sizeof(unsigned));
for(restrictionsIT = restrictionsVector.begin(); restrictionsIT != restrictionsVector.end(); ++restrictionsIT) {
if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) {
restrictionsOutstream.write((char *)&(restrictionsIT->restriction), sizeof(_Restriction));
}
}
restrictionsOutstream.close();
ofstream fout;
fout.open(outputFileName.c_str(), ios::binary);
fout.write((char*)&usedNodeCounter, sizeof(unsigned));
time = get_timestamp();
cout << "[extractor] Confirming/Writing used nodes ... " << flush;
STXXLNodeVector::iterator nodesIT = allNodes.begin();
STXXLNodeIDVector::iterator usedNodeIDsIT = usedNodeIDs.begin();
while(usedNodeIDsIT != usedNodeIDs.end() && nodesIT != allNodes.end()) {
if(*usedNodeIDsIT < nodesIT->id){
++usedNodeIDsIT;
continue;
}
if(*usedNodeIDsIT > nodesIT->id) {
++nodesIT;
continue;
}
if(*usedNodeIDsIT == nodesIT->id) {
if(!settings.obeyBollards && nodesIT->bollard)
nodesIT->bollard = false;
fout.write((char*)&(*nodesIT), sizeof(_Node));
++usedNodeCounter;
++usedNodeIDsIT;
++nodesIT;
}
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
cout << "[extractor] setting number of nodes ... " << flush;
ios::pos_type positionInFile = fout.tellp();
fout.seekp(ios::beg);
fout.write((char*)&usedNodeCounter, sizeof(unsigned));
fout.seekp(positionInFile);
cout << "ok" << endl;
time = get_timestamp();
// Sort edges by start.
cout << "[extractor] Sorting edges by start ... " << flush;
stxxl::sort(allEdges.begin(), allEdges.end(), CmpEdgeByStartID(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Setting start coords ... " << flush;
fout.write((char*)&usedEdgeCounter, sizeof(unsigned));
// Traverse list of edges and nodes in parallel and set start coord
nodesIT = allNodes.begin();
STXXLEdgeVector::iterator edgeIT = allEdges.begin();
while(edgeIT != allEdges.end() && nodesIT != allNodes.end()) {
if(edgeIT->start < nodesIT->id){
++edgeIT;
continue;
}
if(edgeIT->start > nodesIT->id) {
nodesIT++;
continue;
}
if(edgeIT->start == nodesIT->id) {
edgeIT->startCoord.lat = nodesIT->lat;
edgeIT->startCoord.lon = nodesIT->lon;
++edgeIT;
}
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
// Sort Edges by target
cout << "[extractor] Sorting edges by target ... " << flush;
stxxl::sort(allEdges.begin(), allEdges.end(), CmpEdgeByTargetID(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Setting target coords ... " << flush;
// Traverse list of edges and nodes in parallel and set target coord
nodesIT = allNodes.begin();
edgeIT = allEdges.begin();
while(edgeIT != allEdges.end() && nodesIT != allNodes.end()) {
if(edgeIT->target < nodesIT->id){
++edgeIT;
continue;
}
if(edgeIT->target > nodesIT->id) {
++nodesIT;
continue;
}
if(edgeIT->target == nodesIT->id) {
if(edgeIT->startCoord.lat != INT_MIN && edgeIT->startCoord.lon != INT_MIN) {
edgeIT->targetCoord.lat = nodesIT->lat;
edgeIT->targetCoord.lon = nodesIT->lon;
double distance = ApproximateDistance(edgeIT->startCoord.lat, edgeIT->startCoord.lon, nodesIT->lat, nodesIT->lon);
assert(edgeIT->speed != -1);
double weight = ( distance * 10. ) / (edgeIT->speed / 3.6);
int intWeight = std::max(1, (int)std::floor((edgeIT->isDurationSet ? edgeIT->speed : weight)+.5) );
int intDist = std::max(1, (int)distance);
short zero = 0;
short one = 1;
fout.write((char*)&edgeIT->start, sizeof(unsigned));
fout.write((char*)&edgeIT->target, sizeof(unsigned));
fout.write((char*)&intDist, sizeof(int));
switch(edgeIT->direction) {
case _Way::notSure:
fout.write((char*)&zero, sizeof(short));
break;
case _Way::oneway:
fout.write((char*)&one, sizeof(short));
break;
case _Way::bidirectional:
fout.write((char*)&zero, sizeof(short));
break;
case _Way::opposite:
fout.write((char*)&one, sizeof(short));
break;
default:
cerr << "[error] edge with no direction: " << edgeIT->direction << endl;
assert(false);
break;
}
fout.write((char*)&intWeight, sizeof(int));
assert(edgeIT->type >= 0);
fout.write((char*)&edgeIT->type, sizeof(short));
fout.write((char*)&edgeIT->nameID, sizeof(unsigned));
fout.write((char*)&edgeIT->isRoundabout, sizeof(bool));
fout.write((char*)&edgeIT->ignoreInGrid, sizeof(bool));
fout.write((char*)&edgeIT->isAccessRestricted, sizeof(bool));
}
++usedEdgeCounter;
++edgeIT;
}
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
cout << "[extractor] setting number of edges ... " << flush;
fout.seekp(positionInFile);
fout.write((char*)&usedEdgeCounter, sizeof(unsigned));
fout.close();
cout << "ok" << endl;
time = get_timestamp();
cout << "[extractor] writing street name index ... " << flush;
std::string nameOutFileName = (outputFileName + ".names");
ofstream nameOutFile(nameOutFileName.c_str(), ios::binary);
unsigned sizeOfNameIndex = nameVector.size();
nameOutFile.write((char *)&(sizeOfNameIndex), sizeof(unsigned));
BOOST_FOREACH(string str, nameVector) {
unsigned lengthOfRawString = strlen(str.c_str());
nameOutFile.write((char *)&(lengthOfRawString), sizeof(unsigned));
nameOutFile.write(str.c_str(), lengthOfRawString);
}
nameOutFile.close();
cout << "ok, after " << get_timestamp() - time << "s" << endl;
// time = get_timestamp();
// cout << "[extractor] writing address list ... " << flush;
//
// adressFileName.append(".address");
// ofstream addressOutFile(adressFileName.c_str());
// for(STXXLAddressVector::iterator it = adressVector.begin(); it != adressVector.end(); it++) {
// addressOutFile << it->node.id << "|" << it->node.lat << "|" << it->node.lon << "|" << it->city << "|" << it->street << "|" << it->housenumber << "|" << it->state << "|" << it->country << "\n";
// }
// addressOutFile.close();
// cout << "ok, after " << get_timestamp() - time << "s" << endl;
INFO("Processed " << usedNodeCounter << " nodes and " << usedEdgeCounter << " edges");
} catch ( const exception& e ) {
cerr << "Caught Execption:" << e.what() << endl;
}
}

63
Extractor/ExtractionContainers.h Normal file
View File

@ -0,0 +1,63 @@
/*
open source routing machine
Copyright (C) Dennis Luxen, others 2010
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU AFFERO General Public License as published by
the Free Software Foundation; either version 3 of the License, or
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
or see http://www.gnu.org/licenses/agpl.txt.
*/
#ifndef EXTRACTIONCONTAINERS_H_
#define EXTRACTIONCONTAINERS_H_
#include <boost/foreach.hpp>
#include <stxxl.h>
#include "ExtractorStructs.h"
#include "../DataStructures/Util.h"
class ExtractionContainers {
public:
typedef stxxl::vector<NodeID> STXXLNodeIDVector;
typedef stxxl::vector<_Node> STXXLNodeVector;
typedef stxxl::vector<_Edge> STXXLEdgeVector;
typedef stxxl::vector<_Address> STXXLAddressVector;
typedef stxxl::vector<std::string> STXXLStringVector;
typedef stxxl::vector<_RawRestrictionContainer> STXXLRestrictionsVector;
typedef stxxl::vector<_WayIDStartAndEndEdge> STXXLWayIDStartEndVector;
ExtractionContainers() { nameVector.push_back(""); }
virtual ~ExtractionContainers() {
usedNodeIDs.clear();
allNodes.clear();
allEdges.clear();
adressVector.clear();
nameVector.clear();
restrictionsVector.clear();
wayStartEndVector.clear();
}
void PrepareData(const Settings & settings, const std::string & outputFileName, const std::string restrictionsFileName, const unsigned amountOfRAM);
STXXLNodeIDVector usedNodeIDs;
STXXLNodeVector allNodes;
STXXLEdgeVector allEdges;
STXXLAddressVector adressVector;
STXXLStringVector nameVector;
STXXLRestrictionsVector restrictionsVector;
STXXLWayIDStartEndVector wayStartEndVector;
};
#endif /* EXTRACTIONCONTAINERS_H_ */

370
Extractor/ExtractorCallBacks.h
View File

@ -1,370 +0,0 @@
/*
open source routing machine
Copyright (C) Dennis Luxen, others 2010
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU AFFERO General Public License as published by
the Free Software Foundation; either version 3 of the License, or
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
or see http://www.gnu.org/licenses/agpl.txt.
*/
#ifndef EXTRACTORCALLBACKS_H_
#define EXTRACTORCALLBACKS_H_
#include <string>
#include <vector>
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/regex.hpp>
#include <boost/regex.hpp>
#include <stxxl.h>
#include "ExtractorStructs.h"
#include "V8Helper.h"
typedef stxxl::vector<NodeID> STXXLNodeIDVector;
typedef stxxl::vector<_Node> STXXLNodeVector;
typedef stxxl::vector<_Edge> STXXLEdgeVector;
typedef stxxl::vector<_Address> STXXLAddressVector;
typedef stxxl::vector<std::string> STXXLStringVector;
typedef stxxl::vector<_RawRestrictionContainer> STXXLRestrictionsVector;
typedef stxxl::vector<_WayIDStartAndEndEdge> STXXLWayIDStartEndVector;
struct STXXLContainers {
STXXLNodeIDVector usedNodeIDs;
STXXLNodeVector allNodes;
STXXLEdgeVector allEdges;
STXXLAddressVector adressVector;
STXXLStringVector nameVector;
STXXLRestrictionsVector restrictionsVector;
STXXLWayIDStartEndVector wayStartEndVector;
STXXLContainers() {
nameVector.push_back("");
}
~STXXLContainers() {
usedNodeIDs.clear();
allNodes.clear();
allEdges.clear();
adressVector.clear();
nameVector.clear();
restrictionsVector.clear();
wayStartEndVector.clear();
}
};
class ExtractorCallbacks{
private:
Settings settings;
StringMap * stringMap;
STXXLContainers * externalMemory;
struct DurationContainer {
int hours;
int minutes;
};
bool checkForValidTiming(const std::string &s, DurationContainer & duration) const {
boost::regex e ("((\\d|\\d\\d):)*(\\d|\\d\\d)",boost::regex_constants::icase|boost::regex_constants::perl);
std::vector< std::string > result;
boost::algorithm::split_regex( result, s, boost::regex( ":" ) ) ;
bool matched = regex_match(s, e);
if(matched) {
duration.hours = (result.size()== 2) ? atoi(result[0].c_str()) : 0;
duration.minutes = (result.size()== 2) ? atoi(result[1].c_str()) : atoi(result[0].c_str());
}
return matched;
}
inline int parseMaxspeed(std::string input) const { //call-by-value on purpose.
boost::algorithm::to_lower(input);
int n = atoi(input.c_str());
if (input.find("mph") != std::string::npos || input.find("mp/h") != std::string::npos) {
n = (n*1609)/1000;
}
return n;
}
/** V8 JavaScript Engine **/
v8::HandleScope handle_scope;
v8::Handle<v8::ObjectTemplate> global_templ;
v8::Persistent<v8::Context> context;
//forbid default c'tor
ExtractorCallbacks() {externalMemory = NULL; stringMap = NULL; }
public:
explicit ExtractorCallbacks(STXXLContainers * ext, Settings set, StringMap * strMap) {
externalMemory = ext;
settings = set;
stringMap = strMap;
global_templ = v8::ObjectTemplate::New();
//register global print function
global_templ->Set(v8::String::New("print"), v8::FunctionTemplate::New(V8Helper::PrintToConsole));
//register global version function
global_templ->Set(v8::String::New("version"), v8::FunctionTemplate::New(V8Helper::Version));
// v8::Persistent<v8::Object> script_core;
context = v8::Context::New(0, global_templ);
//Enter the created context for compiling
v8::Context::Scope context_scope = v8::Context::Scope(context);
//todo: open speedprofile.js
std::string text = "print('Starting V8 Scripting Engine '+version());";
v8::Handle<v8::String> source = v8::String::New(text.c_str());
v8::TryCatch try_catch;
v8::Handle<v8::Script> script = v8::Script::Compile(source);
if (script.IsEmpty()) {
// Print errors that happened during compilation.
V8Helper::ReportException(&try_catch);
} else {
v8::Handle<v8::Value> result = script->Run();
if (result.IsEmpty()) {
assert(try_catch.HasCaught());
// Print errors that happened during execution.
V8Helper::ReportException(&try_catch);
} else {
assert(!try_catch.HasCaught());
if (!result->IsUndefined()) {
// If all went well and the result wasn't undefined then print
// the returned value.
v8::String::Utf8Value str(result);
std::cout << V8Helper::ToCString(str) << std::endl;
}
}
}
}
~ExtractorCallbacks() {
context.Dispose();
}
/** warning: caller needs to take care of synchronization! */
inline bool adressFunction(_Node n, HashTable<std::string, std::string> &keyVals) {
/*
std::string housenumber(keyVals.Find("addr:housenumber"));
std::string housename(keyVals.Find("addr:housename"));
std::string street(keyVals.Find("addr:street"));
std::string state(keyVals.Find("addr:state"));
std::string country(keyVals.Find("addr:country"));
std::string postcode(keyVals.Find("addr:postcode"));
std::string city(keyVals.Find("addr:city"));
if(housenumber != "" || housename != "" || street != "") {
if(housenumber == "")
housenumber = housename;
addressVector->push_back(_Address(n, housenumber, street, state, country, postcode, city));
}
*/
return true;
}
/** warning: caller needs to take care of synchronization! */
inline bool nodeFunction(_Node &n) {
externalMemory->allNodes.push_back(n);
return true;
}
inline bool restrictionFunction(_RawRestrictionContainer &r) {
externalMemory->restrictionsVector.push_back(r);
return true;
}
/** warning: caller needs to take care of synchronization! */
inline bool wayFunction(_Way &w) {
//Get the properties of the way.
std::string highway( w.keyVals.Find("highway") );
std::string name( w.keyVals.Find("name") );
std::string ref( w.keyVals.Find("ref"));
std::string junction( w.keyVals.Find("junction") );
std::string route( w.keyVals.Find("route") );
int maxspeed( parseMaxspeed(w.keyVals.Find("maxspeed")) );
std::string man_made( w.keyVals.Find("man_made") );
std::string barrier( w.keyVals.Find("barrier") );
std::string oneway( w.keyVals.Find("oneway"));
std::string cycleway( w.keyVals.Find("cycleway"));
std::string duration ( w.keyVals.Find("duration"));
std::string service (w.keyVals.Find("service"));
std::string area(w.keyVals.Find("area"));
if("yes" == area && settings.ignoreAreas)
return true;
//Save the name of the way if it has one, ref has precedence over name tag.
if ( 0 < ref.length() )
w.name = ref;
else
if ( 0 < name.length() )
w.name = name;
if(junction == "roundabout") {
w.roundabout = true;
}
//Is the route tag listed as usable way in the profile?
if(settings[route] > 0 || settings[man_made] > 0) {
w.useful = true;
DurationContainer dc;
if(checkForValidTiming(duration, dc)){
w.speed = (600*(dc.hours*60+dc.minutes))/std::max((unsigned)(w.path.size()-1),1u);
w.isDurationSet = true;
} else {
w.speed = settings[route];
}
w.direction = _Way::bidirectional;
if(0 < settings[route])
highway = route;
else if (0 < settings[man_made]) {
highway = man_made;
}
}
//determine the access value
std::string access;
std::string onewayClass;
std::string accessTag;
BOOST_FOREACH(std::string & s, settings.accessTags) {
access = std::string(w.keyVals.Find(s));
if(0 < access.size()) {
accessTag = s;
onewayClass = std::string(w.keyVals.Find("oneway:"+access));
break;
}
}
if(0 < access.size()) {
// handle ways with default access = no
if(settings.accessForbiddenDefault.find(access) != settings.accessForbiddenDefault.end()) {
access = std::string("no");
}
}
//Is the highway tag listed as usable way?
if(0 < settings[highway] || "yes" == access || "designated" == access) {
if(!w.isDurationSet) {
if(0 < settings[highway]) {
if(0 < maxspeed)
if(settings.takeMinimumOfSpeeds)
w.speed = std::min(settings[highway], maxspeed);
else
w.speed = maxspeed;
else
w.speed = settings[highway];
} else {
if(0 < maxspeed)
if(settings.takeMinimumOfSpeeds)
w.speed = std::min(settings.defaultSpeed, maxspeed);
else w.speed = maxspeed;
else
w.speed = settings.defaultSpeed;
highway = "default";
}
}
w.useful = true;
//Okay, do we have access to that way?
if(0 < access.size()) { //fastest way to check for non-empty string
//If access is forbidden, we don't want to route there.
if(settings.accessForbiddenKeys.find(access) != settings.accessForbiddenKeys.end()) {
w.access = false;
}
if(settings.accessRestrictionKeys.find(access) != settings.accessRestrictionKeys.end()) {
w.isAccessRestricted = true;
}
}
if(0 < service.size()) {
if(settings.accessRestrictedService.find(service) != settings.accessRestrictedService.end()) {
w.isAccessRestricted = true;
}
}
if("no" == access) {
return true;
}
if( settings.obeyOneways ) {
if( onewayClass == "yes" || onewayClass == "1" || onewayClass == "true" ) {
w.direction = _Way::oneway;
} else if( onewayClass == "no" || onewayClass == "0" || onewayClass == "false" ) {
w.direction = _Way::bidirectional;
} else if( onewayClass == "-1" ) {
w.direction = _Way::opposite;
} else if( oneway == "no" || oneway == "0" || oneway == "false" ) {
w.direction = _Way::bidirectional;
} else if( accessTag == "bicycle" && (cycleway == "opposite" || cycleway == "opposite_track" || cycleway == "opposite_lane") ) {
w.direction = _Way::bidirectional;
} else if( oneway == "-1") {
w.direction = _Way::opposite;
} else if( oneway == "yes" || oneway == "1" || oneway == "true" || junction == "roundabout" || highway == "motorway_link" || highway == "motorway" ) {
w.direction = _Way::oneway;
} else {
w.direction = _Way::bidirectional;
}
} else {
w.direction = _Way::bidirectional;
}
}
if ( w.useful && w.access && (1 < w.path.size()) ) { //Only true if the way is specified by the speed profile
w.type = settings.GetHighwayTypeID(highway);
if(0 > w.type) {
ERR("Resolved highway " << highway << " to " << w.type);
}
//Get the unique identifier for the street name
const StringMap::const_iterator strit = stringMap->find(w.name);
if(strit == stringMap->end()) {
w.nameID = externalMemory->nameVector.size();
externalMemory->nameVector.push_back(w.name);
stringMap->insert(StringMap::value_type(w.name, w.nameID));
} else {
w.nameID = strit->second;
}
if(-1 == w.speed){
WARN("found way with bogus speed, id: " << w.id);
return true;
}
if(w.id == UINT_MAX) {
WARN("found way with unknown type: " << w.id);
return true;
}
if ( w.direction == _Way::opposite ){
std::reverse( w.path.begin(), w.path.end() );
}
for(vector< NodeID >::size_type n = 0; n < w.path.size()-1; ++n) {
externalMemory->allEdges.push_back(_Edge(w.path[n], w.path[n+1], w.type, w.direction, w.speed, w.nameID, w.roundabout, highway == settings.excludeFromGrid || "pier" == highway, w.isDurationSet, w.isAccessRestricted));
externalMemory->usedNodeIDs.push_back(w.path[n]);
}
externalMemory->usedNodeIDs.push_back(w.path.back());
//The following information is needed to identify start and end segments of restrictions
externalMemory->wayStartEndVector.push_back(_WayIDStartAndEndEdge(w.id, w.path[0], w.path[1], w.path[w.path.size()-2], w.path[w.path.size()-1]));
}
return true;
}
};
#endif /* EXTRACTORCALLBACKS_H_ */

268
Extractor/ExtractorCallbacks.cpp Normal file
View File

@ -0,0 +1,268 @@
/*
open source routing machine
Copyright (C) Dennis Luxen, others 2010
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU AFFERO General Public License as published by
the Free Software Foundation; either version 3 of the License, or
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
or see http://www.gnu.org/licenses/agpl.txt.
*/
/*
open source routing machine
Copyright (C) Dennis Luxen, others 2010
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU AFFERO General Public License as published by
the Free Software Foundation; either version 3 of the License, or
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
or see http://www.gnu.org/licenses/agpl.txt.
*/
#include "ExtractorCallbacks.h"
bool ExtractorCallbacks::checkForValidTiming(const std::string &s, DurationContainer & duration) {
boost::regex e ("((\\d|\\d\\d):)*(\\d|\\d\\d)",boost::regex_constants::icase|boost::regex_constants::perl);
std::vector< std::string > result;
boost::algorithm::split_regex( result, s, boost::regex( ":" ) ) ;
bool matched = regex_match(s, e);
if(matched) {
duration.hours = (result.size()== 2) ? atoi(result[0].c_str()) : 0;
duration.minutes = (result.size()== 2) ? atoi(result[1].c_str()) : atoi(result[0].c_str());
}
return matched;
}
inline int ExtractorCallbacks::parseMaxspeed(std::string input) const { //call-by-value on purpose.
boost::algorithm::to_lower(input);
int n = atoi(input.c_str());
if (input.find("mph") != std::string::npos || input.find("mp/h") != std::string::npos) {
n = (n*1609)/1000;
}
return n;
}
ExtractorCallbacks::ExtractorCallbacks() {externalMemory = NULL; stringMap = NULL; }
ExtractorCallbacks::ExtractorCallbacks(ExtractionContainers * ext, Settings set, StringMap * strMap) {
externalMemory = ext;
settings = set;
stringMap = strMap;
}
ExtractorCallbacks::~ExtractorCallbacks() {
}
/** warning: caller needs to take care of synchronization! */
bool ExtractorCallbacks::nodeFunction(_Node &n) {
externalMemory->allNodes.push_back(n);
return true;
}
bool ExtractorCallbacks::restrictionFunction(_RawRestrictionContainer &r) {
externalMemory->restrictionsVector.push_back(r);
return true;
}
/** warning: caller needs to take care of synchronization! */
bool ExtractorCallbacks::wayFunction(_Way &w) {
//Get the properties of the way.
std::string highway( w.keyVals.Find("highway") );
std::string name( w.keyVals.Find("name") );
std::string ref( w.keyVals.Find("ref"));
std::string junction( w.keyVals.Find("junction") );
std::string route( w.keyVals.Find("route") );
int maxspeed( parseMaxspeed(w.keyVals.Find("maxspeed")) );
std::string man_made( w.keyVals.Find("man_made") );
std::string barrier( w.keyVals.Find("barrier") );
std::string oneway( w.keyVals.Find("oneway"));
std::string cycleway( w.keyVals.Find("cycleway"));
std::string duration ( w.keyVals.Find("duration"));
std::string service (w.keyVals.Find("service"));
std::string area(w.keyVals.Find("area"));
if("yes" == area && settings.ignoreAreas)
return true;
//Save the name of the way if it has one, ref has precedence over name tag.
if ( 0 < ref.length() )
w.name = ref;
else
if ( 0 < name.length() )
w.name = name;
if(junction == "roundabout") {
w.roundabout = true;
}
//Is the route tag listed as usable way in the profile?
if(settings[route] > 0 || settings[man_made] > 0) {
w.useful = true;
DurationContainer dc;
if(checkForValidTiming(duration, dc)){
w.speed = (600*(dc.hours*60+dc.minutes))/std::max((unsigned)(w.path.size()-1),1u);
w.isDurationSet = true;
} else {
w.speed = settings[route];
}
w.direction = _Way::bidirectional;
if(0 < settings[route])
highway = route;
else if (0 < settings[man_made]) {
highway = man_made;
}
}
//determine the access value
std::string access;
std::string onewayClass;
std::string accessTag;
BOOST_FOREACH(std::string & s, settings.accessTags) {
access = std::string(w.keyVals.Find(s));
if(0 < access.size()) {
accessTag = s;
onewayClass = std::string(w.keyVals.Find("oneway:"+access));
break;
}
}
if(0 < access.size()) {
// handle ways with default access = no
if(settings.accessForbiddenDefault.find(access) != settings.accessForbiddenDefault.end()) {
access = std::string("no");
}
}
//Is the highway tag listed as usable way?
if(0 < settings[highway] || "yes" == access || "designated" == access) {
if(!w.isDurationSet) {
if(0 < settings[highway]) {
if(0 < maxspeed)
if(settings.takeMinimumOfSpeeds)
w.speed = std::min(settings[highway], maxspeed);
else
w.speed = maxspeed;
else
w.speed = settings[highway];
} else {
if(0 < maxspeed)
if(settings.takeMinimumOfSpeeds)
w.speed = std::min(settings.defaultSpeed, maxspeed);
else w.speed = maxspeed;
else
w.speed = settings.defaultSpeed;
highway = "default";
}
}
w.useful = true;
//Okay, do we have access to that way?
if(0 < access.size()) { //fastest way to check for non-empty string
//If access is forbidden, we don't want to route there.
if(settings.accessForbiddenKeys.find(access) != settings.accessForbiddenKeys.end()) {
w.access = false;
}
if(settings.accessRestrictionKeys.find(access) != settings.accessRestrictionKeys.end()) {
w.isAccessRestricted = true;
}
}
if(0 < service.size()) {
if(settings.accessRestrictedService.find(service) != settings.accessRestrictedService.end()) {
w.isAccessRestricted = true;
}
}
if("no" == access) {
return true;
}
if( settings.obeyOneways ) {
if( onewayClass == "yes" || onewayClass == "1" || onewayClass == "true" ) {
w.direction = _Way::oneway;
} else if( onewayClass == "no" || onewayClass == "0" || onewayClass == "false" ) {
w.direction = _Way::bidirectional;
} else if( onewayClass == "-1" ) {
w.direction = _Way::opposite;
} else if( oneway == "no" || oneway == "0" || oneway == "false" ) {
w.direction = _Way::bidirectional;
} else if( accessTag == "bicycle" && (cycleway == "opposite" || cycleway == "opposite_track" || cycleway == "opposite_lane") ) {
w.direction = _Way::bidirectional;
} else if( oneway == "-1") {
w.direction = _Way::opposite;
} else if( oneway == "yes" || oneway == "1" || oneway == "true" || junction == "roundabout" || highway == "motorway_link" || highway == "motorway" ) {
w.direction = _Way::oneway;
} else {
w.direction = _Way::bidirectional;
}
} else {
w.direction = _Way::bidirectional;
}
}
if ( w.useful && w.access && (1 < w.path.size()) ) { //Only true if the way is specified by the speed profile
w.type = settings.GetHighwayTypeID(highway);
if(0 > w.type) {
ERR("Resolved highway " << highway << " to " << w.type);
}
//Get the unique identifier for the street name
const StringMap::const_iterator strit = stringMap->find(w.name);
if(strit == stringMap->end()) {
w.nameID = externalMemory->nameVector.size();
externalMemory->nameVector.push_back(w.name);
stringMap->insert(StringMap::value_type(w.name, w.nameID));
} else {
w.nameID = strit->second;
}
if(-1 == w.speed){
WARN("found way with bogus speed, id: " << w.id);
return true;
}
if(w.id == UINT_MAX) {
WARN("found way with unknown type: " << w.id);
return true;
}
if ( w.direction == _Way::opposite ){
std::reverse( w.path.begin(), w.path.end() );
}
for(vector< NodeID >::size_type n = 0; n < w.path.size()-1; ++n) {
externalMemory->allEdges.push_back(_Edge(w.path[n], w.path[n+1], w.type, w.direction, w.speed, w.nameID, w.roundabout, highway == settings.excludeFromGrid || "pier" == highway, w.isDurationSet, w.isAccessRestricted));
externalMemory->usedNodeIDs.push_back(w.path[n]);
}
externalMemory->usedNodeIDs.push_back(w.path.back());
//The following information is needed to identify start and end segments of restrictions
externalMemory->wayStartEndVector.push_back(_WayIDStartAndEndEdge(w.id, w.path[0], w.path[1], w.path[w.path.size()-2], w.path[w.path.size()-1]));
}
return true;
}

63
Extractor/ExtractorCallbacks.h Normal file
View File

@ -0,0 +1,63 @@
/*
open source routing machine
Copyright (C) Dennis Luxen, others 2010
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU AFFERO General Public License as published by
the Free Software Foundation; either version 3 of the License, or
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
or see http://www.gnu.org/licenses/agpl.txt.
*/
#ifndef EXTRACTORCALLBACKS_H_
#define EXTRACTORCALLBACKS_H_
#include <string>
#include <vector>
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/regex.hpp>
#include <boost/regex.hpp>
#include "ExtractionContainers.h"
#include "ExtractorStructs.h"
class ExtractorCallbacks{
private:
Settings settings;
StringMap * stringMap;
ExtractionContainers * externalMemory;
struct DurationContainer {
int hours;
int minutes;
};
bool checkForValidTiming(const std::string &s, DurationContainer & duration);
inline int parseMaxspeed(std::string input) const;
ExtractorCallbacks();
public:
explicit ExtractorCallbacks(ExtractionContainers * ext, Settings set, StringMap * strMap);
~ExtractorCallbacks();
/** warning: caller needs to take care of synchronization! */
bool nodeFunction(_Node &n);
bool restrictionFunction(_RawRestrictionContainer &r);
/** warning: caller needs to take care of synchronization! */
bool wayFunction(_Way &w);
};
#endif /* EXTRACTORCALLBACKS_H_ */

33
Extractor/LuaUtil.h Normal file
View File

@ -0,0 +1,33 @@
/*
open source routing machine
Copyright (C) Dennis Luxen, others 2010
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU AFFERO General Public License as published by
the Free Software Foundation; either version 3 of the License, or
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
or see http://www.gnu.org/licenses/agpl.txt.
*/
#ifndef LUAUTIL_H_
#define LUAUTIL_H_
#include <iostream>
template<typename T>
void LUA_print(T number) {
std::cout << "[LUA] " << number << std::endl;
}
#endif /* LUAUTIL_H_ */

73
Extractor/PBFParser.h
View File

@ -22,6 +22,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
#define PBFPARSER_H_ #define PBFPARSER_H_
#include <zlib.h> #include <zlib.h>
#include <boost/ref.hpp>
#include <boost/shared_ptr.hpp> #include <boost/shared_ptr.hpp>
#include <osmpbf/fileformat.pb.h> #include <osmpbf/fileformat.pb.h>
@ -36,6 +37,8 @@ or see http://www.gnu.org/licenses/agpl.txt.
class PBFParser : public BaseParser<_Node, _RawRestrictionContainer, _Way> { class PBFParser : public BaseParser<_Node, _RawRestrictionContainer, _Way> {
typedef BaseParser<_Node, _RawRestrictionContainer, _Way> super;
enum EntityType { enum EntityType {
TypeNode = 1, TypeNode = 1,
TypeWay = 2, TypeWay = 2,
@ -63,7 +66,7 @@ class PBFParser : public BaseParser<_Node, _RawRestrictionContainer, _Way> {
}; };
public: public:
PBFParser(const char * fileName) { PBFParser(const char * fileName) : myLuaState(NULL) {
GOOGLE_PROTOBUF_VERIFY_VERSION; GOOGLE_PROTOBUF_VERIFY_VERSION;
//TODO: What is the bottleneck here? Filling the queue or reading the stuff from disk? //TODO: What is the bottleneck here? Filling the queue or reading the stuff from disk?
threadDataQueue.reset( new ConcurrentQueue<_ThreadData*>(2500) ); /* Max 2500 items in queue, hardcoded. */ threadDataQueue.reset( new ConcurrentQueue<_ThreadData*>(2500) ); /* Max 2500 items in queue, hardcoded. */
@ -77,18 +80,22 @@ public:
groupCount = 0; groupCount = 0;
//Dummy initialization //Dummy initialization
wayCallback = NULL; nodeCallback = NULL; wayCallback = NULL;
addressCallback = NULL; restrictionCallback = NULL; nodeCallback = NULL;
restrictionCallback = NULL;
} }
bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way),bool (*addressCallbackPointer)(_Node, HashTable<std::string, std::string>&) ) { bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way) ) {
nodeCallback = *nodeCallbackPointer; nodeCallback = *nodeCallbackPointer;
wayCallback = *wayCallbackPointer; wayCallback = *wayCallbackPointer;
restrictionCallback = *restrictionCallbackPointer; restrictionCallback = *restrictionCallbackPointer;
addressCallback = *addressCallbackPointer;
return true; return true;
} }
void RegisterLUAState(lua_State *ml) {
myLuaState = ml;
}
~PBFParser() { ~PBFParser() {
if(input.is_open()) if(input.is_open())
input.close(); input.close();
@ -188,7 +195,7 @@ public:
// Start the read and parse threads // Start the read and parse threads
boost::thread readThread(boost::bind(&PBFParser::ReadData, this)); boost::thread readThread(boost::bind(&PBFParser::ReadData, this));
boost::thread parseThread(boost::bind(&PBFParser::ParseData, this)); boost::thread parseThread(boost::bind(&PBFParser::ParseData, this));
// Wait for the threads to finish // Wait for the threads to finish
readThread.join(); readThread.join();
parseThread.join(); parseThread.join();
@ -206,11 +213,11 @@ private:
int m_lastDenseLongitude = 0; int m_lastDenseLongitude = 0;
for(int i = 0; i < dense.id_size(); i++) { for(int i = 0; i < dense.id_size(); i++) {
HashTable<std::string, std::string> keyVals;
m_lastDenseID += dense.id( i ); m_lastDenseID += dense.id( i );
m_lastDenseLatitude += dense.lat( i ); m_lastDenseLatitude += dense.lat( i );
m_lastDenseLongitude += dense.lon( i ); m_lastDenseLongitude += dense.lon( i );
_Node n; ImportNode n;
n.id = m_lastDenseID; n.id = m_lastDenseID;
n.lat = 100000*( ( double ) m_lastDenseLatitude * threadData->PBFprimitiveBlock.granularity() +threadData-> PBFprimitiveBlock.lat_offset() ) / NANO; n.lat = 100000*( ( double ) m_lastDenseLatitude * threadData->PBFprimitiveBlock.granularity() +threadData-> PBFprimitiveBlock.lat_offset() ) / NANO;
n.lon = 100000*( ( double ) m_lastDenseLongitude * threadData->PBFprimitiveBlock.granularity() + threadData->PBFprimitiveBlock.lon_offset() ) / NANO; n.lon = 100000*( ( double ) m_lastDenseLongitude * threadData->PBFprimitiveBlock.granularity() + threadData->PBFprimitiveBlock.lon_offset() ) / NANO;
@ -223,22 +230,35 @@ private:
int keyValue = dense.keys_vals ( denseTagIndex+1 ); int keyValue = dense.keys_vals ( denseTagIndex+1 );
std::string key = threadData->PBFprimitiveBlock.stringtable().s(tagValue).data(); std::string key = threadData->PBFprimitiveBlock.stringtable().s(tagValue).data();
std::string value = threadData->PBFprimitiveBlock.stringtable().s(keyValue).data(); std::string value = threadData->PBFprimitiveBlock.stringtable().s(keyValue).data();
keyVals.Add(key, value); n.keyVals.Add(key, value);
denseTagIndex += 2; denseTagIndex += 2;
} }
std::string barrierValue = keyVals.Find("barrier");
std::string access = keyVals.Find("access");
if(access != "yes" && 0 < barrierValue.length() && "cattle_grid" != barrierValue && "border_control" != barrierValue && "toll_booth" != barrierValue && "no" != barrierValue)
n.bollard = true;
if("traffic_signals" == keyVals.Find("highway")) int ret = -1;
n.trafficLight = true; try {
ret = luabind::call_function<int>(
myLuaState,
"node_function",
boost::ref(n)
);
if(!(*nodeCallback)(n))
std::cerr << "[PBFParser] dense node not parsed" << std::endl;
} catch (const luabind::error &er) {
cerr << er.what() << endl;
lua_State* Ler=er.state();
report_errors(Ler, -1);
}
catch (...) {
cerr<<"Unknown error!"<<endl;
}
}
}
if(!(*addressCallback)(n, keyVals)) void report_errors(lua_State *L, int status)
std::cerr << "[PBFParser] adress not parsed" << std::endl; {
if ( status!=0 ) {
if(!(*nodeCallback)(n)) std::cerr << "-- " << lua_tostring(L, -1) << std::endl;
std::cerr << "[PBFParser] dense node not parsed" << std::endl; lua_pop(L, 1); // remove error message
} }
} }
@ -311,11 +331,11 @@ private:
break; break;
} }
} }
// if(UINT_MAX != currentRestriction.viaNode) { // if(UINT_MAX != currentRestriction.viaNode) {
// cout << "restr from " << currentRestriction.from << " via "; // cout << "restr from " << currentRestriction.from << " via ";
// cout << "node " << currentRestriction.viaNode; // cout << "node " << currentRestriction.viaNode;
// cout << " to " << currentRestriction.to << endl; // cout << " to " << currentRestriction.to << endl;
// } // }
if(!(*restrictionCallback)(currentRestrictionContainer)) if(!(*restrictionCallback)(currentRestrictionContainer))
std::cerr << "[PBFParser] relation not parsed" << std::endl; std::cerr << "[PBFParser] relation not parsed" << std::endl;
} }
@ -537,12 +557,13 @@ private:
bool (*nodeCallback)(_Node); bool (*nodeCallback)(_Node);
bool (*wayCallback)(_Way); bool (*wayCallback)(_Way);
bool (*restrictionCallback)(_RawRestrictionContainer); bool (*restrictionCallback)(_RawRestrictionContainer);
bool (*addressCallback)(_Node, HashTable<std::string, std::string>&);
/* the input stream to parse */ /* the input stream to parse */
std::fstream input; std::fstream input;
/* ThreadData Queue */ /* ThreadData Queue */
boost::shared_ptr<ConcurrentQueue < _ThreadData* > > threadDataQueue; boost::shared_ptr<ConcurrentQueue < _ThreadData* > > threadDataQueue;
lua_State *myLuaState;
}; };
#endif /* PBFPARSER_H_ */ #endif /* PBFPARSER_H_ */

161
Extractor/V8Helper.h
View File

@ -1,161 +0,0 @@
/*
open source routing machine
Copyright (C) Dennis Luxen, others 2010
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU AFFERO General Public License as published by
the Free Software Foundation; either version 3 of the License, or
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
or see http://www.gnu.org/licenses/agpl.txt.
*/
#ifndef V8HELPER_H_
#define V8HELPER_H_
#include <iostream>
#include <v8.h>
namespace V8Helper {
//Provides an output mechanism to the V8 Engine
static v8::Handle<v8::Value> PrintToConsole (const v8::Arguments& args){
std::cout << "[JS] : " << *v8::String::AsciiValue(args[0]) << std::endl;
return v8::Undefined();
}
//Returns the version of the V8 Engine
v8::Handle<v8::Value> Version(const v8::Arguments& args) {
return v8::String::New(v8::V8::GetVersion());
}
//Most of the stuff below is from http://blog.owned.co.za provided with an MIT License
template<class T>
v8::Handle<v8::Object> WrapClass(T* y) {
// Handle scope for temporary handles,
v8::HandleScope handle_scope;
v8::Persistent<v8::ObjectTemplate> class_template_;
v8::Handle<v8::ObjectTemplate> raw_template = v8::ObjectTemplate::New();
//The raw template is the ObjectTemplate (that can be exposed to script too)
//but is maintained internally.
raw_template->SetInternalFieldCount(1);
//Create the actual template object,
class_template_ = v8::Persistent<v8::ObjectTemplate>::New(raw_template);
//Create the new handle to return, and set its template type
v8::Handle<v8::Object> result = class_template_->NewInstance();
v8::Handle<v8::External> class_ptr = v8::External::New(static_cast<T*>(y));
//Point the 0 index Field to the c++ pointer for unwrapping later
result->SetInternalField(0, class_ptr);
//Return the value, releasing the other handles on its way.
return handle_scope.Close(result);
}
template<class T>
v8::Handle<v8::Object> ExposeClass(v8::Persistent<v8::Context> context, T* y, v8::Handle<v8::Value> exposedName, v8::PropertyAttribute props) {
v8::HandleScope handle_scope;
v8::Handle<v8::Object> obj = WrapClass<T>(y);
context->Global()->Set(exposedName, obj, props);
return handle_scope.Close(obj);
}
template<class T>
T* UnwrapClass(v8::Handle<v8::Object> obj) {
v8::Handle<v8::External> field = v8::Handle<v8::External>::Cast(obj->GetInternalField(0));
void* ptr = field->Value();
return static_cast<T*>(ptr);
}
void Expose(v8::Handle<v8::Object> intoObject, v8::Handle<v8::Value> namev8String, v8::InvocationCallback funcID) {
v8::HandleScope handle_scope;
v8::Handle<v8::FunctionTemplate> thisFunc = v8::FunctionTemplate::New(funcID);
intoObject->Set(namev8String, thisFunc->GetFunction());
}
template<typename WrappedType, typename PropertyType, PropertyType WrappedType::*MemVar>
v8::Handle<v8::Value> GetMember(v8::Local<v8::String> property, const v8::AccessorInfo &info) {
v8::Local<v8::Object> self = info.Holder();
v8::Local<v8::External> wrap = v8::Local<v8::External>::Cast(self->GetInternalField(0));
void* ptr = wrap->Value();
int value = static_cast<WrappedType*>(ptr)->*MemVar;
return PropertyType::New(value);
}
template <typename WrappedType, typename CPPPropertyType, CPPPropertyType WrappedType::*MemVar>
void SetIntMember(v8::Local<v8::String> property, v8::Local<v8::Value> value, const v8::AccessorInfo& info) {
v8::Local<v8::Object> self = info.Holder();
v8::Local<v8::External> wrap = v8::Local<v8::External>::Cast(self->GetInternalField(0));
void * ptr = wrap->Value();
static_cast<WrappedType*>(ptr)->*MemVar = value->Int32Value();
}
template<typename WrappedType, typename JSPropertyType, typename CPPProperyType, CPPProperyType WrappedType::*MemVar>
v8::Handle<v8::Value> GetInt(v8::Local<v8::String> property, const v8::AccessorInfo &info) {
v8::Local<v8::Object> self = info.Holder();
v8::Local<v8::External> wrap = v8::Local<v8::External>::Cast(self->GetInternalField(0));
void* ptr = wrap->Value();
CPPProperyType value = static_cast<WrappedType*>(ptr)->member1;
return JSPropertyType::New(value);
}
// Extracts a C string from a V8 Utf8Value.
const char* ToCString(const v8::String::Utf8Value& value) {
return *value ? *value : "<string conversion failed>";
}
void ReportException(v8::TryCatch* try_catch) {
v8::HandleScope handle_scope;
v8::String::Utf8Value exception(try_catch->Exception());
const char* exception_string = ToCString(exception);
v8::Handle<v8::Message> message = try_catch->Message();
if (message.IsEmpty()) {
// V8 didn't provide any extra information about this error; just
// print the exception.
std::cout << exception_string << std::endl;
} else {
// Print (filename):(line number): (message).
v8::String::Utf8Value filename(message->GetScriptResourceName());
int linenum = message->GetLineNumber();
std::cout << ToCString(filename) << ":" << linenum << ": " << exception_string << std::endl;
// Print line of source code.
v8::String::Utf8Value sourceline(message->GetSourceLine());
std::cout << ToCString(sourceline) << std::endl;
// Print wavy underline (GetUnderline is deprecated).
int start = message->GetStartColumn();
for (int i = 0; i < start; ++i) {
std::cout << " ";
}
for (int i = start, end = message->GetEndColumn(); i < end; ++i) {
std::cout << "^";
}
std::cout << std::endl;
v8::String::Utf8Value stack_trace(try_catch->StackTrace());
if (stack_trace.length()) {
std::cout << ToCString(stack_trace) << std::endl;
}
}
}
}
#endif /* V8HELPER_H_ */

10
Extractor/XMLParser.h
View File

@ -31,18 +31,23 @@ or see http://www.gnu.org/licenses/agpl.txt.
class XMLParser : public BaseParser<_Node, _RawRestrictionContainer, _Way> { class XMLParser : public BaseParser<_Node, _RawRestrictionContainer, _Way> {
public: public:
XMLParser(const char * filename) : nodeCallback(NULL), wayCallback(NULL), restrictionCallback(NULL){ XMLParser(const char * filename) : nodeCallback(NULL), wayCallback(NULL), restrictionCallback(NULL), myLuaState(NULL){
WARN("Parsing plain .osm/.osm.bz2 is deprecated. Switch to .pbf"); WARN("Parsing plain .osm/.osm.bz2 is deprecated. Switch to .pbf");
inputReader = inputReaderFactory(filename); inputReader = inputReaderFactory(filename);
} }
virtual ~XMLParser() {} virtual ~XMLParser() {}
bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way), bool (*addressCallbackPointer)(_Node, HashTable<std::string, std::string>&) ) { bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way)) {
nodeCallback = *nodeCallbackPointer; nodeCallback = *nodeCallbackPointer;
wayCallback = *wayCallbackPointer; wayCallback = *wayCallbackPointer;
restrictionCallback = *restrictionCallbackPointer; restrictionCallback = *restrictionCallbackPointer;
return true; return true;
} }
void RegisterLUAState(lua_State *ml) {
myLuaState = ml;
}
bool Init() { bool Init() {
return (xmlTextReaderRead( inputReader ) == 1); return (xmlTextReaderRead( inputReader ) == 1);
} }
@ -293,6 +298,7 @@ private:
bool (*wayCallback)(_Way); bool (*wayCallback)(_Way);
bool (*restrictionCallback)(_RawRestrictionContainer); bool (*restrictionCallback)(_RawRestrictionContainer);
lua_State *myLuaState;
}; };
#endif /* XMLPARSER_H_ */ #endif /* XMLPARSER_H_ */

7
SConstruct
View File

@ -118,6 +118,8 @@ else:
if GetOption('buildconfiguration') != 'debug' and GetOption('nomarch') == None and sys.platform != 'darwin': if GetOption('buildconfiguration') != 'debug' and GetOption('nomarch') == None and sys.platform != 'darwin':
env.Append(CCFLAGS = ['-march=native']) env.Append(CCFLAGS = ['-march=native'])
env.ParseConfig('pkg-config --cflags --libs luabind')
if not conf.CheckHeader('omp.h'): if not conf.CheckHeader('omp.h'):
print "Compiler does not support OpenMP. Exiting" print "Compiler does not support OpenMP. Exiting"
Exit(-1) Exit(-1)
@ -148,9 +150,6 @@ if not (env.Detect('protoc')):
if not conf.CheckLibWithHeader('stxxl', 'stxxl.h', 'CXX'): if not conf.CheckLibWithHeader('stxxl', 'stxxl.h', 'CXX'):
print "stxxl library not found. Exiting" print "stxxl library not found. Exiting"
Exit(-1) Exit(-1)
if not conf.CheckLibWithHeader('v8', 'v8.h', 'CXX'):
print "v8 library not found. Exiting"
Exit(-1)
if not conf.CheckLibWithHeader('xml2', 'libxml/xmlreader.h', 'CXX'): if not conf.CheckLibWithHeader('xml2', 'libxml/xmlreader.h', 'CXX'):
print "libxml2 library or header not found. Exiting" print "libxml2 library or header not found. Exiting"
Exit(-1) Exit(-1)
@ -270,7 +269,7 @@ if sys.platform != 'darwin':
env.Append(CCFLAGS = ['-fopenmp']) env.Append(CCFLAGS = ['-fopenmp'])
env.Append(LINKFLAGS = ['-fopenmp']) env.Append(LINKFLAGS = ['-fopenmp'])
env.Program(target = 'osrm-extract', source = ["extractor.cpp", Glob('Util/*.cpp')]) env.Program(target = 'osrm-extract', source = ["extractor.cpp", Glob('Util/*.cpp'), Glob('Extractor/*.cpp')])
env.Program(target = 'osrm-prepare', source = ["createHierarchy.cpp", Glob('Contractor/*.cpp'), Glob('Util/SRTMLookup/*.cpp'), Glob('Algorithms/*.cpp')]) env.Program(target = 'osrm-prepare', source = ["createHierarchy.cpp", Glob('Contractor/*.cpp'), Glob('Util/SRTMLookup/*.cpp'), Glob('Algorithms/*.cpp')])
env.Program(target = 'osrm-routed', source = ["routed.cpp", 'Descriptors/DescriptionFactory.cpp', Glob('ThirdParty/*.cc'), Glob('Server/DataStructures/*.cpp')], CCFLAGS = env['CCFLAGS'] + ['-DROUTED']) env.Program(target = 'osrm-routed', source = ["routed.cpp", 'Descriptors/DescriptionFactory.cpp', Glob('ThirdParty/*.cc'), Glob('Server/DataStructures/*.cpp')], CCFLAGS = env['CCFLAGS'] + ['-DROUTED'])
env = conf.Finish() env = conf.Finish()

371
extractor.cpp
View File

@ -35,17 +35,24 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include <string> #include <string>
#include <vector> #include <vector>
extern "C" {
#include <lua.h>
#include <lauxlib.h>
#include <lualib.h>
}
#include <luabind/luabind.hpp>
#include <libxml/xmlreader.h> #include <libxml/xmlreader.h>
#include <boost/foreach.hpp> #include <boost/foreach.hpp>
#include <boost/property_tree/ptree.hpp> #include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/ini_parser.hpp> #include <boost/property_tree/ini_parser.hpp>
#include <unistd.h> #include <unistd.h>
#include <stxxl.h>
#include "typedefs.h" #include "typedefs.h"
#include "DataStructures/InputReaderFactory.h" #include "DataStructures/InputReaderFactory.h"
#include "Extractor/ExtractorCallBacks.h" #include "Extractor/ExtractorCallbacks.h"
#include "Extractor/ExtractorStructs.h" #include "Extractor/ExtractorStructs.h"
#include "Extractor/LuaUtil.h"
#include "Extractor/PBFParser.h" #include "Extractor/PBFParser.h"
#include "Extractor/XMLParser.h" #include "Extractor/XMLParser.h"
#include "Util/BaseConfiguration.h" #include "Util/BaseConfiguration.h"
@ -53,22 +60,19 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "Util/MachineInfo.h" #include "Util/MachineInfo.h"
#include "Util/StringUtil.h" #include "Util/StringUtil.h"
#include "Extractor/ExtractionContainers.h"
using namespace std; using namespace std;
typedef BaseConfiguration ExtractorConfiguration; typedef BaseConfiguration ExtractorConfiguration;
unsigned globalRestrictionCounter = 0; unsigned globalRestrictionCounter = 0;
ExtractorCallbacks * extractCallBacks; ExtractorCallbacks * extractCallBacks;
//
bool nodeFunction(_Node n); bool nodeFunction(_Node n);
bool adressFunction(_Node n, HashTable<string, string> & keyVals);
bool restrictionFunction(_RawRestrictionContainer r); bool restrictionFunction(_RawRestrictionContainer r);
bool wayFunction(_Way w); bool wayFunction(_Way w);
template<class ClassT>
bool removeIfUnused(ClassT n) { return (false == n.used); }
int main (int argc, char *argv[]) { int main (int argc, char *argv[]) {
if(argc < 2) { if(argc < 2) {
@ -193,6 +197,52 @@ int main (int argc, char *argv[]) {
ERR("caught: " << e.what() ); ERR("caught: " << e.what() );
} }
/*** Setup Scripting Environment ***/
// Create a new lua state
lua_State *myLuaState = luaL_newstate();
// Connect LuaBind to this lua state
luabind::open(myLuaState);
// Add our function to the state's global scope
luabind::module(myLuaState) [
luabind::def("print", LUA_print<std::string>)
];
if(0 != luaL_dostring(
myLuaState,
"print('Initializing LUA engine')\n"
)) {
ERR(lua_tostring(myLuaState,-1)<< " occured in scripting block");
}
luabind::module(myLuaState) [
luabind::class_<HashTable<std::string, std::string> >("keyVals")
.def("Add", &HashTable<std::string, std::string>::Add)
.def("Find", &HashTable<std::string, std::string>::Find)
];
luabind::module(myLuaState) [
luabind::class_<ImportNode>("Node")
.def(luabind::constructor<>())
.def_readwrite("lat", &ImportNode::lat)
.def_readwrite("lon", &ImportNode::lon)
.def_readwrite("id", &ImportNode::id)
.def_readwrite("bollard", &ImportNode::bollard)
.def_readwrite("traffic_light", &ImportNode::trafficLight)
.def_readwrite("tags", &ImportNode::keyVals)
];
// Now call our function in a lua script
if(0 != luaL_dofile(myLuaState, "profile.lua")) {
ERR(lua_tostring(myLuaState,-1)<< " occured in scripting block");
}
//open string library;
luaopen_string(myLuaState);
/*** End of Scripting Environment Setup; ***/
unsigned amountOfRAM = 1; unsigned amountOfRAM = 1;
unsigned installedRAM = GetPhysicalmemory(); unsigned installedRAM = GetPhysicalmemory();
if(installedRAM < 2048264) { if(installedRAM < 2048264) {
@ -207,11 +257,7 @@ int main (int argc, char *argv[]) {
} }
StringMap stringMap; StringMap stringMap;
STXXLContainers externalMemory; ExtractionContainers externalMemory;
unsigned usedNodeCounter = 0;
unsigned usedEdgeCounter = 0;
double time = get_timestamp();
stringMap[""] = 0; stringMap[""] = 0;
extractCallBacks = new ExtractorCallbacks(&externalMemory, settings, &stringMap); extractCallBacks = new ExtractorCallbacks(&externalMemory, settings, &stringMap);
@ -221,301 +267,15 @@ int main (int argc, char *argv[]) {
} else { } else {
parser = new XMLParser(argv[1]); parser = new XMLParser(argv[1]);
} }
parser->RegisterCallbacks(&nodeFunction, &restrictionFunction, &wayFunction, &adressFunction); parser->RegisterCallbacks(&nodeFunction, &restrictionFunction, &wayFunction);
parser->RegisterLUAState(myLuaState);
if(!parser->Init()) if(!parser->Init())
INFO("Parser not initialized!"); INFO("Parser not initialized!");
parser->Parse(); parser->Parse();
try { externalMemory.PrepareData(settings, outputFileName, restrictionsFileName, amountOfRAM);
// INFO("raw no. of names: " << externalMemory.nameVector.size());
// INFO("raw no. of nodes: " << externalMemory.allNodes.size());
// INFO("no. of used nodes: " << externalMemory.usedNodeIDs.size());
// INFO("raw no. of edges: " << externalMemory.allEdges.size());
// INFO("raw no. of ways: " << externalMemory.wayStartEndVector.size());
// INFO("raw no. of addresses: " << externalMemory.adressVector.size());
// INFO("raw no. of restrictions: " << externalMemory.restrictionsVector.size());
cout << "[extractor] parsing finished after " << get_timestamp() - time << " seconds" << endl;
time = get_timestamp();
boost::uint64_t memory_to_use = static_cast<boost::uint64_t>(amountOfRAM) * 1024 * 1024 * 1024;
cout << "[extractor] Sorting used nodes ... " << flush;
stxxl::sort(externalMemory.usedNodeIDs.begin(), externalMemory.usedNodeIDs.end(), Cmp(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Erasing duplicate nodes ... " << flush;
stxxl::vector<NodeID>::iterator NewEnd = unique ( externalMemory.usedNodeIDs.begin(),externalMemory.usedNodeIDs.end() ) ;
externalMemory.usedNodeIDs.resize ( NewEnd - externalMemory.usedNodeIDs.begin() );
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Sorting all nodes ... " << flush;
stxxl::sort(externalMemory.allNodes.begin(), externalMemory.allNodes.end(), CmpNodeByID(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Sorting used ways ... " << flush;
stxxl::sort(externalMemory.wayStartEndVector.begin(), externalMemory.wayStartEndVector.end(), CmpWayByID(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
cout << "[extractor] Sorting restrctns. by from... " << flush;
stxxl::sort(externalMemory.restrictionsVector.begin(), externalMemory.restrictionsVector.end(), CmpRestrictionContainerByFrom(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
cout << "[extractor] Fixing restriction starts ... " << flush;
STXXLRestrictionsVector::iterator restrictionsIT = externalMemory.restrictionsVector.begin();
STXXLWayIDStartEndVector::iterator wayStartAndEndEdgeIT = externalMemory.wayStartEndVector.begin();
while(wayStartAndEndEdgeIT != externalMemory.wayStartEndVector.end() && restrictionsIT != externalMemory.restrictionsVector.end()) {
if(wayStartAndEndEdgeIT->wayID < restrictionsIT->fromWay){
++wayStartAndEndEdgeIT;
continue;
}
if(wayStartAndEndEdgeIT->wayID > restrictionsIT->fromWay) {
++restrictionsIT;
continue;
}
assert(wayStartAndEndEdgeIT->wayID == restrictionsIT->fromWay);
NodeID viaNode = restrictionsIT->restriction.viaNode;
if(wayStartAndEndEdgeIT->firstStart == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstTarget;
} else if(wayStartAndEndEdgeIT->firstTarget == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstStart;
} else if(wayStartAndEndEdgeIT->lastStart == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastTarget;
} else if(wayStartAndEndEdgeIT->lastTarget == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastStart;
}
++restrictionsIT;
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Sorting restrctns. by to ... " << flush;
stxxl::sort(externalMemory.restrictionsVector.begin(), externalMemory.restrictionsVector.end(), CmpRestrictionContainerByTo(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
unsigned usableRestrictionsCounter(0);
cout << "[extractor] Fixing restriction ends ... " << flush;
restrictionsIT = externalMemory.restrictionsVector.begin();
wayStartAndEndEdgeIT = externalMemory.wayStartEndVector.begin();
while(wayStartAndEndEdgeIT != externalMemory.wayStartEndVector.end() && restrictionsIT != externalMemory.restrictionsVector.end()) {
if(wayStartAndEndEdgeIT->wayID < restrictionsIT->toWay){
++wayStartAndEndEdgeIT;
continue;
}
if(wayStartAndEndEdgeIT->wayID > restrictionsIT->toWay) {
++restrictionsIT;
continue;
}
NodeID viaNode = restrictionsIT->restriction.viaNode;
if(wayStartAndEndEdgeIT->lastStart == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastTarget;
} else if(wayStartAndEndEdgeIT->lastTarget == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastStart;
} else if(wayStartAndEndEdgeIT->firstStart == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstTarget;
} else if(wayStartAndEndEdgeIT->firstTarget == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstStart;
}
if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) {
++usableRestrictionsCounter;
}
++restrictionsIT;
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
INFO("usable restrictions: " << usableRestrictionsCounter );
//serialize restrictions
ofstream restrictionsOutstream;
restrictionsOutstream.open(restrictionsFileName.c_str(), ios::binary);
restrictionsOutstream.write((char*)&usableRestrictionsCounter, sizeof(unsigned));
for(restrictionsIT = externalMemory.restrictionsVector.begin(); restrictionsIT != externalMemory.restrictionsVector.end(); ++restrictionsIT) {
if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) {
restrictionsOutstream.write((char *)&(restrictionsIT->restriction), sizeof(_Restriction));
}
}
restrictionsOutstream.close();
ofstream fout;
fout.open(outputFileName.c_str(), ios::binary);
fout.write((char*)&usedNodeCounter, sizeof(unsigned));
time = get_timestamp();
cout << "[extractor] Confirming/Writing used nodes ... " << flush;
STXXLNodeVector::iterator nodesIT = externalMemory.allNodes.begin();
STXXLNodeIDVector::iterator usedNodeIDsIT = externalMemory.usedNodeIDs.begin();
while(usedNodeIDsIT != externalMemory.usedNodeIDs.end() && nodesIT != externalMemory.allNodes.end()) {
if(*usedNodeIDsIT < nodesIT->id){
++usedNodeIDsIT;
continue;
}
if(*usedNodeIDsIT > nodesIT->id) {
++nodesIT;
continue;
}
if(*usedNodeIDsIT == nodesIT->id) {
if(!settings.obeyBollards && nodesIT->bollard)
nodesIT->bollard = false;
fout.write((char*)&(*nodesIT), sizeof(_Node));
++usedNodeCounter;
++usedNodeIDsIT;
++nodesIT;
}
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
cout << "[extractor] setting number of nodes ... " << flush;
ios::pos_type positionInFile = fout.tellp();
fout.seekp(ios::beg);
fout.write((char*)&usedNodeCounter, sizeof(unsigned));
fout.seekp(positionInFile);
cout << "ok" << endl;
time = get_timestamp();
// Sort edges by start.
cout << "[extractor] Sorting edges by start ... " << flush;
stxxl::sort(externalMemory.allEdges.begin(), externalMemory.allEdges.end(), CmpEdgeByStartID(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Setting start coords ... " << flush;
fout.write((char*)&usedEdgeCounter, sizeof(unsigned));
// Traverse list of edges and nodes in parallel and set start coord
nodesIT = externalMemory.allNodes.begin();
STXXLEdgeVector::iterator edgeIT = externalMemory.allEdges.begin();
while(edgeIT != externalMemory.allEdges.end() && nodesIT != externalMemory.allNodes.end()) {
if(edgeIT->start < nodesIT->id){
++edgeIT;
continue;
}
if(edgeIT->start > nodesIT->id) {
nodesIT++;
continue;
}
if(edgeIT->start == nodesIT->id) {
edgeIT->startCoord.lat = nodesIT->lat;
edgeIT->startCoord.lon = nodesIT->lon;
++edgeIT;
}
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
// Sort Edges by target
cout << "[extractor] Sorting edges by target ... " << flush;
stxxl::sort(externalMemory.allEdges.begin(), externalMemory.allEdges.end(), CmpEdgeByTargetID(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Setting target coords ... " << flush;
// Traverse list of edges and nodes in parallel and set target coord
nodesIT = externalMemory.allNodes.begin();
edgeIT = externalMemory.allEdges.begin();
while(edgeIT != externalMemory.allEdges.end() && nodesIT != externalMemory.allNodes.end()) {
if(edgeIT->target < nodesIT->id){
++edgeIT;
continue;
}
if(edgeIT->target > nodesIT->id) {
++nodesIT;
continue;
}
if(edgeIT->target == nodesIT->id) {
if(edgeIT->startCoord.lat != INT_MIN && edgeIT->startCoord.lon != INT_MIN) {
edgeIT->targetCoord.lat = nodesIT->lat;
edgeIT->targetCoord.lon = nodesIT->lon;
double distance = ApproximateDistance(edgeIT->startCoord.lat, edgeIT->startCoord.lon, nodesIT->lat, nodesIT->lon);
assert(edgeIT->speed != -1);
double weight = ( distance * 10. ) / (edgeIT->speed / 3.6);
int intWeight = std::max(1, (int)std::floor((edgeIT->isDurationSet ? edgeIT->speed : weight)+.5) );
int intDist = std::max(1, (int)distance);
short zero = 0;
short one = 1;
fout.write((char*)&edgeIT->start, sizeof(unsigned));
fout.write((char*)&edgeIT->target, sizeof(unsigned));
fout.write((char*)&intDist, sizeof(int));
switch(edgeIT->direction) {
case _Way::notSure:
fout.write((char*)&zero, sizeof(short));
break;
case _Way::oneway:
fout.write((char*)&one, sizeof(short));
break;
case _Way::bidirectional:
fout.write((char*)&zero, sizeof(short));
break;
case _Way::opposite:
fout.write((char*)&one, sizeof(short));
break;
default:
cerr << "[error] edge with no direction: " << edgeIT->direction << endl;
assert(false);
break;
}
fout.write((char*)&intWeight, sizeof(int));
assert(edgeIT->type >= 0);
fout.write((char*)&edgeIT->type, sizeof(short));
fout.write((char*)&edgeIT->nameID, sizeof(unsigned));
fout.write((char*)&edgeIT->isRoundabout, sizeof(bool));
fout.write((char*)&edgeIT->ignoreInGrid, sizeof(bool));
fout.write((char*)&edgeIT->isAccessRestricted, sizeof(bool));
}
++usedEdgeCounter;
++edgeIT;
}
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
cout << "[extractor] setting number of edges ... " << flush;
fout.seekp(positionInFile);
fout.write((char*)&usedEdgeCounter, sizeof(unsigned));
fout.close();
cout << "ok" << endl;
time = get_timestamp();
cout << "[extractor] writing street name index ... " << flush;
std::string nameOutFileName = (outputFileName + ".names");
ofstream nameOutFile(nameOutFileName.c_str(), ios::binary);
unsigned sizeOfNameIndex = externalMemory.nameVector.size();
nameOutFile.write((char *)&(sizeOfNameIndex), sizeof(unsigned));
BOOST_FOREACH(string str, externalMemory.nameVector) {
unsigned lengthOfRawString = strlen(str.c_str());
nameOutFile.write((char *)&(lengthOfRawString), sizeof(unsigned));
nameOutFile.write(str.c_str(), lengthOfRawString);
}
nameOutFile.close();
cout << "ok, after " << get_timestamp() - time << "s" << endl;
// time = get_timestamp();
// cout << "[extractor] writing address list ... " << flush;
//
// adressFileName.append(".address");
// ofstream addressOutFile(adressFileName.c_str());
// for(STXXLAddressVector::iterator it = adressVector.begin(); it != adressVector.end(); it++) {
// addressOutFile << it->node.id << "|" << it->node.lat << "|" << it->node.lon << "|" << it->city << "|" << it->street << "|" << it->housenumber << "|" << it->state << "|" << it->country << "\n";
// }
// addressOutFile.close();
// cout << "ok, after " << get_timestamp() - time << "s" << endl;
} catch ( const exception& e ) {
cerr << "Caught Execption:" << e.what() << endl;
return false;
}
double endTime = (get_timestamp() - startupTime);
INFO("Processed " << (usedNodeCounter)/(endTime) << " nodes/sec and " << usedEdgeCounter/endTime << " edges/sec");
stringMap.clear(); stringMap.clear();
delete parser; delete parser;
delete extractCallBacks; delete extractCallBacks;
@ -530,11 +290,6 @@ bool nodeFunction(_Node n) {
return true; return true;
} }
bool adressFunction(_Node n, HashTable<string, string> & keyVals){
extractCallBacks->adressFunction(n, keyVals);
return true;
}
bool restrictionFunction(_RawRestrictionContainer r) { bool restrictionFunction(_RawRestrictionContainer r) {
extractCallBacks->restrictionFunction(r); extractCallBacks->restrictionFunction(r);
++globalRestrictionCounter; ++globalRestrictionCounter;

27
profile.lua Normal file
View File

@ -0,0 +1,27 @@
bollards_whitelist = { [""] = true, ["cattle_grid"] = true, ["border_control"] = true, ["toll_booth"] = true, ["no"] = true}
access_whitelist = { ["yes"] = true, ["motorcar"] = true, ["permissive"] = true }
access_blacklist = { ["no"] = true, ["private"] = true, ["agricultural"] = true, ["forestery"] = true }
function node_function (node)
local barrier = node.tags:Find("barrier")
local access = node.tags:Find("access")
local traffic_signal = node.tags:Find("highway")
if traffic_signal == "traffic_signals" then
node.traffic_light = true;
end
if access_blacklist[barrier] then
node.bollard = true;
end
if not bollards_whitelist[barrier] then
node.bollard = true;
end
return 1
end
function way_function (way)
return 1
end