From 1412c5db42afedc4c9c19835bf3c7fa7b16084cc Mon Sep 17 00:00:00 2001 From: DennisOSRM Date: Wed, 29 Aug 2012 18:33:18 +0200 Subject: [PATCH] Removal of v8 as it sucked big time during integration and first (partially) working parsing with LUA as the scripting engine. --- DataStructures/ImportNode.h | 5 + Extractor/BaseParser.h | 5 +- Extractor/ExtractionContainers.cpp | 317 ++++++++++++++++++++++++ Extractor/ExtractionContainers.h | 63 +++++ Extractor/ExtractorCallBacks.h | 370 ---------------------------- Extractor/ExtractorCallbacks.cpp | 268 +++++++++++++++++++++ Extractor/ExtractorCallbacks.h | 63 +++++ Extractor/LuaUtil.h | 33 +++ Extractor/PBFParser.h | 73 ++++-- Extractor/V8Helper.h | 161 ------------- Extractor/XMLParser.h | 10 +- SConstruct | 7 +- extractor.cpp | 371 +++++------------------------ profile.lua | 27 +++ 14 files changed, 901 insertions(+), 872 deletions(-) create mode 100644 Extractor/ExtractionContainers.cpp create mode 100644 Extractor/ExtractionContainers.h delete mode 100644 Extractor/ExtractorCallBacks.h create mode 100644 Extractor/ExtractorCallbacks.cpp create mode 100644 Extractor/ExtractorCallbacks.h create mode 100644 Extractor/LuaUtil.h delete mode 100644 Extractor/V8Helper.h create mode 100644 profile.lua diff --git a/DataStructures/ImportNode.h b/DataStructures/ImportNode.h index 5f34d691c..a51929850 100644 --- a/DataStructures/ImportNode.h +++ b/DataStructures/ImportNode.h @@ -22,6 +22,8 @@ or see http://www.gnu.org/licenses/agpl.txt. #define IMPORTNODE_H_ #include "NodeCoords.h" +#include "../DataStructures/HashTable.h" + struct _Node : NodeInfo{ _Node(int _lat, int _lon, unsigned int _id, bool _bollard, bool _trafficLight) : NodeInfo(_lat, _lon, _id), bollard(_bollard), trafficLight(_trafficLight) {} @@ -38,7 +40,10 @@ struct _Node : NodeInfo{ } bool bollard; bool trafficLight; +}; +struct ImportNode : public _Node { + HashTable keyVals; }; #endif /* IMPORTNODE_H_ */ diff --git a/Extractor/BaseParser.h b/Extractor/BaseParser.h index d3cb226aa..d0dcbff27 100644 --- a/Extractor/BaseParser.h +++ b/Extractor/BaseParser.h @@ -21,12 +21,15 @@ or see http://www.gnu.org/licenses/agpl.txt. #ifndef BASEPARSER_H_ #define BASEPARSER_H_ +#include + template class BaseParser { public: virtual ~BaseParser() {} virtual bool Init() = 0; - virtual bool RegisterCallbacks(bool (*nodeCallbackPointer)(NodeT), bool (*restrictionCallbackPointer)(RestrictionT), bool (*wayCallbackPointer)(WayT), bool (*addressCallbackPointer)(NodeT, HashTable&)) = 0; + virtual bool RegisterCallbacks(bool (*nodeCallbackPointer)(NodeT), bool (*restrictionCallbackPointer)(RestrictionT), bool (*wayCallbackPointer)(WayT)) = 0; + virtual void RegisterLUAState(lua_State *myLuaState) = 0; virtual bool Parse() = 0; }; diff --git a/Extractor/ExtractionContainers.cpp b/Extractor/ExtractionContainers.cpp new file mode 100644 index 000000000..bd96e406c --- /dev/null +++ b/Extractor/ExtractionContainers.cpp @@ -0,0 +1,317 @@ +/* + open source routing machine + Copyright (C) Dennis Luxen, others 2010 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU AFFERO General Public License as published by + the Free Software Foundation; either version 3 of the License, or + any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + or see http://www.gnu.org/licenses/agpl.txt. + */ + +#include "ExtractionContainers.h" + +void ExtractionContainers::PrepareData(const Settings & settings, const std::string & outputFileName, const std::string restrictionsFileName, const unsigned amountOfRAM) { + try { + unsigned usedNodeCounter = 0; + unsigned usedEdgeCounter = 0; + double time = get_timestamp(); + // INFO("raw no. of names: " << nameVector.size()); + // INFO("raw no. of nodes: " << allNodes.size()); + // INFO("no. of used nodes: " << usedNodeIDs.size()); + // INFO("raw no. of edges: " << allEdges.size()); + // INFO("raw no. of ways: " << wayStartEndVector.size()); + // INFO("raw no. of addresses: " << adressVector.size()); + // INFO("raw no. of restrictions: " << restrictionsVector.size()); + + cout << "[extractor] parsing finished after " << get_timestamp() - time << " seconds" << endl; + time = get_timestamp(); + boost::uint64_t memory_to_use = static_cast(amountOfRAM) * 1024 * 1024 * 1024; + + cout << "[extractor] Sorting used nodes ... " << flush; + stxxl::sort(usedNodeIDs.begin(), usedNodeIDs.end(), Cmp(), memory_to_use); + cout << "ok, after " << get_timestamp() - time << "s" << endl; + + time = get_timestamp(); + cout << "[extractor] Erasing duplicate nodes ... " << flush; + stxxl::vector::iterator NewEnd = unique ( usedNodeIDs.begin(),usedNodeIDs.end() ) ; + usedNodeIDs.resize ( NewEnd - usedNodeIDs.begin() ); + cout << "ok, after " << get_timestamp() - time << "s" << endl; + time = get_timestamp(); + + cout << "[extractor] Sorting all nodes ... " << flush; + stxxl::sort(allNodes.begin(), allNodes.end(), CmpNodeByID(), memory_to_use); + cout << "ok, after " << get_timestamp() - time << "s" << endl; + time = get_timestamp(); + + cout << "[extractor] Sorting used ways ... " << flush; + stxxl::sort(wayStartEndVector.begin(), wayStartEndVector.end(), CmpWayByID(), memory_to_use); + cout << "ok, after " << get_timestamp() - time << "s" << endl; + + cout << "[extractor] Sorting restrctns. by from... " << flush; + stxxl::sort(restrictionsVector.begin(), restrictionsVector.end(), CmpRestrictionContainerByFrom(), memory_to_use); + cout << "ok, after " << get_timestamp() - time << "s" << endl; + + cout << "[extractor] Fixing restriction starts ... " << flush; + STXXLRestrictionsVector::iterator restrictionsIT = restrictionsVector.begin(); + STXXLWayIDStartEndVector::iterator wayStartAndEndEdgeIT = wayStartEndVector.begin(); + + while(wayStartAndEndEdgeIT != wayStartEndVector.end() && restrictionsIT != restrictionsVector.end()) { + if(wayStartAndEndEdgeIT->wayID < restrictionsIT->fromWay){ + ++wayStartAndEndEdgeIT; + continue; + } + if(wayStartAndEndEdgeIT->wayID > restrictionsIT->fromWay) { + ++restrictionsIT; + continue; + } + assert(wayStartAndEndEdgeIT->wayID == restrictionsIT->fromWay); + NodeID viaNode = restrictionsIT->restriction.viaNode; + + if(wayStartAndEndEdgeIT->firstStart == viaNode) { + restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstTarget; + } else if(wayStartAndEndEdgeIT->firstTarget == viaNode) { + restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstStart; + } else if(wayStartAndEndEdgeIT->lastStart == viaNode) { + restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastTarget; + } else if(wayStartAndEndEdgeIT->lastTarget == viaNode) { + restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastStart; + } + ++restrictionsIT; + } + + cout << "ok, after " << get_timestamp() - time << "s" << endl; + time = get_timestamp(); + + cout << "[extractor] Sorting restrctns. by to ... " << flush; + stxxl::sort(restrictionsVector.begin(), restrictionsVector.end(), CmpRestrictionContainerByTo(), memory_to_use); + cout << "ok, after " << get_timestamp() - time << "s" << endl; + + time = get_timestamp(); + unsigned usableRestrictionsCounter(0); + cout << "[extractor] Fixing restriction ends ... " << flush; + restrictionsIT = restrictionsVector.begin(); + wayStartAndEndEdgeIT = wayStartEndVector.begin(); + while(wayStartAndEndEdgeIT != wayStartEndVector.end() && restrictionsIT != restrictionsVector.end()) { + if(wayStartAndEndEdgeIT->wayID < restrictionsIT->toWay){ + ++wayStartAndEndEdgeIT; + continue; + } + if(wayStartAndEndEdgeIT->wayID > restrictionsIT->toWay) { + ++restrictionsIT; + continue; + } + NodeID viaNode = restrictionsIT->restriction.viaNode; + if(wayStartAndEndEdgeIT->lastStart == viaNode) { + restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastTarget; + } else if(wayStartAndEndEdgeIT->lastTarget == viaNode) { + restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastStart; + } else if(wayStartAndEndEdgeIT->firstStart == viaNode) { + restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstTarget; + } else if(wayStartAndEndEdgeIT->firstTarget == viaNode) { + restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstStart; + } + + if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) { + ++usableRestrictionsCounter; + } + ++restrictionsIT; + } + cout << "ok, after " << get_timestamp() - time << "s" << endl; + INFO("usable restrictions: " << usableRestrictionsCounter ); + //serialize restrictions + ofstream restrictionsOutstream; + restrictionsOutstream.open(restrictionsFileName.c_str(), ios::binary); + restrictionsOutstream.write((char*)&usableRestrictionsCounter, sizeof(unsigned)); + for(restrictionsIT = restrictionsVector.begin(); restrictionsIT != restrictionsVector.end(); ++restrictionsIT) { + if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) { + restrictionsOutstream.write((char *)&(restrictionsIT->restriction), sizeof(_Restriction)); + } + } + restrictionsOutstream.close(); + + ofstream fout; + fout.open(outputFileName.c_str(), ios::binary); + fout.write((char*)&usedNodeCounter, sizeof(unsigned)); + time = get_timestamp(); + cout << "[extractor] Confirming/Writing used nodes ... " << flush; + + STXXLNodeVector::iterator nodesIT = allNodes.begin(); + STXXLNodeIDVector::iterator usedNodeIDsIT = usedNodeIDs.begin(); + while(usedNodeIDsIT != usedNodeIDs.end() && nodesIT != allNodes.end()) { + if(*usedNodeIDsIT < nodesIT->id){ + ++usedNodeIDsIT; + continue; + } + if(*usedNodeIDsIT > nodesIT->id) { + ++nodesIT; + continue; + } + if(*usedNodeIDsIT == nodesIT->id) { + if(!settings.obeyBollards && nodesIT->bollard) + nodesIT->bollard = false; + fout.write((char*)&(*nodesIT), sizeof(_Node)); + ++usedNodeCounter; + ++usedNodeIDsIT; + ++nodesIT; + } + } + + cout << "ok, after " << get_timestamp() - time << "s" << endl; + + cout << "[extractor] setting number of nodes ... " << flush; + ios::pos_type positionInFile = fout.tellp(); + fout.seekp(ios::beg); + fout.write((char*)&usedNodeCounter, sizeof(unsigned)); + fout.seekp(positionInFile); + + cout << "ok" << endl; + time = get_timestamp(); + + // Sort edges by start. + cout << "[extractor] Sorting edges by start ... " << flush; + stxxl::sort(allEdges.begin(), allEdges.end(), CmpEdgeByStartID(), memory_to_use); + cout << "ok, after " << get_timestamp() - time << "s" << endl; + time = get_timestamp(); + + cout << "[extractor] Setting start coords ... " << flush; + fout.write((char*)&usedEdgeCounter, sizeof(unsigned)); + // Traverse list of edges and nodes in parallel and set start coord + nodesIT = allNodes.begin(); + STXXLEdgeVector::iterator edgeIT = allEdges.begin(); + while(edgeIT != allEdges.end() && nodesIT != allNodes.end()) { + if(edgeIT->start < nodesIT->id){ + ++edgeIT; + continue; + } + if(edgeIT->start > nodesIT->id) { + nodesIT++; + continue; + } + if(edgeIT->start == nodesIT->id) { + edgeIT->startCoord.lat = nodesIT->lat; + edgeIT->startCoord.lon = nodesIT->lon; + ++edgeIT; + } + } + cout << "ok, after " << get_timestamp() - time << "s" << endl; + time = get_timestamp(); + + // Sort Edges by target + cout << "[extractor] Sorting edges by target ... " << flush; + stxxl::sort(allEdges.begin(), allEdges.end(), CmpEdgeByTargetID(), memory_to_use); + cout << "ok, after " << get_timestamp() - time << "s" << endl; + time = get_timestamp(); + + cout << "[extractor] Setting target coords ... " << flush; + // Traverse list of edges and nodes in parallel and set target coord + nodesIT = allNodes.begin(); + edgeIT = allEdges.begin(); + + while(edgeIT != allEdges.end() && nodesIT != allNodes.end()) { + if(edgeIT->target < nodesIT->id){ + ++edgeIT; + continue; + } + if(edgeIT->target > nodesIT->id) { + ++nodesIT; + continue; + } + if(edgeIT->target == nodesIT->id) { + if(edgeIT->startCoord.lat != INT_MIN && edgeIT->startCoord.lon != INT_MIN) { + edgeIT->targetCoord.lat = nodesIT->lat; + edgeIT->targetCoord.lon = nodesIT->lon; + + double distance = ApproximateDistance(edgeIT->startCoord.lat, edgeIT->startCoord.lon, nodesIT->lat, nodesIT->lon); + assert(edgeIT->speed != -1); + double weight = ( distance * 10. ) / (edgeIT->speed / 3.6); + int intWeight = std::max(1, (int)std::floor((edgeIT->isDurationSet ? edgeIT->speed : weight)+.5) ); + int intDist = std::max(1, (int)distance); + short zero = 0; + short one = 1; + + fout.write((char*)&edgeIT->start, sizeof(unsigned)); + fout.write((char*)&edgeIT->target, sizeof(unsigned)); + fout.write((char*)&intDist, sizeof(int)); + switch(edgeIT->direction) { + case _Way::notSure: + fout.write((char*)&zero, sizeof(short)); + break; + case _Way::oneway: + fout.write((char*)&one, sizeof(short)); + break; + case _Way::bidirectional: + fout.write((char*)&zero, sizeof(short)); + + break; + case _Way::opposite: + fout.write((char*)&one, sizeof(short)); + break; + default: + cerr << "[error] edge with no direction: " << edgeIT->direction << endl; + assert(false); + break; + } + fout.write((char*)&intWeight, sizeof(int)); + assert(edgeIT->type >= 0); + fout.write((char*)&edgeIT->type, sizeof(short)); + fout.write((char*)&edgeIT->nameID, sizeof(unsigned)); + fout.write((char*)&edgeIT->isRoundabout, sizeof(bool)); + fout.write((char*)&edgeIT->ignoreInGrid, sizeof(bool)); + fout.write((char*)&edgeIT->isAccessRestricted, sizeof(bool)); + } + ++usedEdgeCounter; + ++edgeIT; + } + } + cout << "ok, after " << get_timestamp() - time << "s" << endl; + cout << "[extractor] setting number of edges ... " << flush; + + fout.seekp(positionInFile); + fout.write((char*)&usedEdgeCounter, sizeof(unsigned)); + fout.close(); + cout << "ok" << endl; + time = get_timestamp(); + cout << "[extractor] writing street name index ... " << flush; + std::string nameOutFileName = (outputFileName + ".names"); + ofstream nameOutFile(nameOutFileName.c_str(), ios::binary); + unsigned sizeOfNameIndex = nameVector.size(); + nameOutFile.write((char *)&(sizeOfNameIndex), sizeof(unsigned)); + + BOOST_FOREACH(string str, nameVector) { + unsigned lengthOfRawString = strlen(str.c_str()); + nameOutFile.write((char *)&(lengthOfRawString), sizeof(unsigned)); + nameOutFile.write(str.c_str(), lengthOfRawString); + } + + nameOutFile.close(); + cout << "ok, after " << get_timestamp() - time << "s" << endl; + + // time = get_timestamp(); + // cout << "[extractor] writing address list ... " << flush; + // + // adressFileName.append(".address"); + // ofstream addressOutFile(adressFileName.c_str()); + // for(STXXLAddressVector::iterator it = adressVector.begin(); it != adressVector.end(); it++) { + // addressOutFile << it->node.id << "|" << it->node.lat << "|" << it->node.lon << "|" << it->city << "|" << it->street << "|" << it->housenumber << "|" << it->state << "|" << it->country << "\n"; + // } + // addressOutFile.close(); + // cout << "ok, after " << get_timestamp() - time << "s" << endl; + + INFO("Processed " << usedNodeCounter << " nodes and " << usedEdgeCounter << " edges"); + + + } catch ( const exception& e ) { + cerr << "Caught Execption:" << e.what() << endl; + } +} + diff --git a/Extractor/ExtractionContainers.h b/Extractor/ExtractionContainers.h new file mode 100644 index 000000000..62e4a9717 --- /dev/null +++ b/Extractor/ExtractionContainers.h @@ -0,0 +1,63 @@ +/* + open source routing machine + Copyright (C) Dennis Luxen, others 2010 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU AFFERO General Public License as published by + the Free Software Foundation; either version 3 of the License, or + any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + or see http://www.gnu.org/licenses/agpl.txt. + */ + +#ifndef EXTRACTIONCONTAINERS_H_ +#define EXTRACTIONCONTAINERS_H_ + +#include +#include + +#include "ExtractorStructs.h" +#include "../DataStructures/Util.h" + +class ExtractionContainers { +public: + typedef stxxl::vector STXXLNodeIDVector; + typedef stxxl::vector<_Node> STXXLNodeVector; + typedef stxxl::vector<_Edge> STXXLEdgeVector; + typedef stxxl::vector<_Address> STXXLAddressVector; + typedef stxxl::vector STXXLStringVector; + typedef stxxl::vector<_RawRestrictionContainer> STXXLRestrictionsVector; + typedef stxxl::vector<_WayIDStartAndEndEdge> STXXLWayIDStartEndVector; + + ExtractionContainers() { nameVector.push_back(""); } + virtual ~ExtractionContainers() { + usedNodeIDs.clear(); + allNodes.clear(); + allEdges.clear(); + adressVector.clear(); + nameVector.clear(); + restrictionsVector.clear(); + wayStartEndVector.clear(); + } + + void PrepareData(const Settings & settings, const std::string & outputFileName, const std::string restrictionsFileName, const unsigned amountOfRAM); + + STXXLNodeIDVector usedNodeIDs; + STXXLNodeVector allNodes; + STXXLEdgeVector allEdges; + STXXLAddressVector adressVector; + STXXLStringVector nameVector; + STXXLRestrictionsVector restrictionsVector; + STXXLWayIDStartEndVector wayStartEndVector; + +}; + +#endif /* EXTRACTIONCONTAINERS_H_ */ diff --git a/Extractor/ExtractorCallBacks.h b/Extractor/ExtractorCallBacks.h deleted file mode 100644 index 86fb56348..000000000 --- a/Extractor/ExtractorCallBacks.h +++ /dev/null @@ -1,370 +0,0 @@ -/* - open source routing machine - Copyright (C) Dennis Luxen, others 2010 - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU AFFERO General Public License as published by -the Free Software Foundation; either version 3 of the License, or -any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -or see http://www.gnu.org/licenses/agpl.txt. - */ - -#ifndef EXTRACTORCALLBACKS_H_ -#define EXTRACTORCALLBACKS_H_ - -#include -#include - -#include -#include -#include - -#include -#include "ExtractorStructs.h" - -#include "V8Helper.h" - -typedef stxxl::vector STXXLNodeIDVector; -typedef stxxl::vector<_Node> STXXLNodeVector; -typedef stxxl::vector<_Edge> STXXLEdgeVector; -typedef stxxl::vector<_Address> STXXLAddressVector; -typedef stxxl::vector STXXLStringVector; -typedef stxxl::vector<_RawRestrictionContainer> STXXLRestrictionsVector; -typedef stxxl::vector<_WayIDStartAndEndEdge> STXXLWayIDStartEndVector; - - -struct STXXLContainers { - STXXLNodeIDVector usedNodeIDs; - STXXLNodeVector allNodes; - STXXLEdgeVector allEdges; - STXXLAddressVector adressVector; - STXXLStringVector nameVector; - STXXLRestrictionsVector restrictionsVector; - STXXLWayIDStartEndVector wayStartEndVector; - - STXXLContainers() { - nameVector.push_back(""); - } - - ~STXXLContainers() { - usedNodeIDs.clear(); - allNodes.clear(); - allEdges.clear(); - adressVector.clear(); - nameVector.clear(); - restrictionsVector.clear(); - wayStartEndVector.clear(); - } -}; - -class ExtractorCallbacks{ -private: - Settings settings; - StringMap * stringMap; - STXXLContainers * externalMemory; - - struct DurationContainer { - int hours; - int minutes; - }; - - bool checkForValidTiming(const std::string &s, DurationContainer & duration) const { - boost::regex e ("((\\d|\\d\\d):)*(\\d|\\d\\d)",boost::regex_constants::icase|boost::regex_constants::perl); - - std::vector< std::string > result; - boost::algorithm::split_regex( result, s, boost::regex( ":" ) ) ; - bool matched = regex_match(s, e); - if(matched) { - duration.hours = (result.size()== 2) ? atoi(result[0].c_str()) : 0; - duration.minutes = (result.size()== 2) ? atoi(result[1].c_str()) : atoi(result[0].c_str()); - } - return matched; - } - - inline int parseMaxspeed(std::string input) const { //call-by-value on purpose. - boost::algorithm::to_lower(input); - int n = atoi(input.c_str()); - if (input.find("mph") != std::string::npos || input.find("mp/h") != std::string::npos) { - n = (n*1609)/1000; - } - return n; - } - - /** V8 JavaScript Engine **/ - v8::HandleScope handle_scope; - v8::Handle global_templ; - v8::Persistent context; - - //forbid default c'tor - ExtractorCallbacks() {externalMemory = NULL; stringMap = NULL; } -public: - explicit ExtractorCallbacks(STXXLContainers * ext, Settings set, StringMap * strMap) { - externalMemory = ext; - settings = set; - stringMap = strMap; - - global_templ = v8::ObjectTemplate::New(); - - - //register global print function - global_templ->Set(v8::String::New("print"), v8::FunctionTemplate::New(V8Helper::PrintToConsole)); - - //register global version function - global_templ->Set(v8::String::New("version"), v8::FunctionTemplate::New(V8Helper::Version)); - -// v8::Persistent script_core; - - context = v8::Context::New(0, global_templ); - //Enter the created context for compiling - v8::Context::Scope context_scope = v8::Context::Scope(context); - - //todo: open speedprofile.js - std::string text = "print('Starting V8 Scripting Engine '+version());"; - v8::Handle source = v8::String::New(text.c_str()); - v8::TryCatch try_catch; - v8::Handle script = v8::Script::Compile(source); - if (script.IsEmpty()) { - // Print errors that happened during compilation. - V8Helper::ReportException(&try_catch); - } else { - v8::Handle result = script->Run(); - if (result.IsEmpty()) { - assert(try_catch.HasCaught()); - // Print errors that happened during execution. - V8Helper::ReportException(&try_catch); - } else { - assert(!try_catch.HasCaught()); - if (!result->IsUndefined()) { - // If all went well and the result wasn't undefined then print - // the returned value. - v8::String::Utf8Value str(result); - std::cout << V8Helper::ToCString(str) << std::endl; - } - } - } - } - - ~ExtractorCallbacks() { - context.Dispose(); - } - - /** warning: caller needs to take care of synchronization! */ - inline bool adressFunction(_Node n, HashTable &keyVals) { - /* - std::string housenumber(keyVals.Find("addr:housenumber")); - std::string housename(keyVals.Find("addr:housename")); - std::string street(keyVals.Find("addr:street")); - std::string state(keyVals.Find("addr:state")); - std::string country(keyVals.Find("addr:country")); - std::string postcode(keyVals.Find("addr:postcode")); - std::string city(keyVals.Find("addr:city")); - - if(housenumber != "" || housename != "" || street != "") { - if(housenumber == "") - housenumber = housename; - addressVector->push_back(_Address(n, housenumber, street, state, country, postcode, city)); - } - */ - return true; - } - - /** warning: caller needs to take care of synchronization! */ - inline bool nodeFunction(_Node &n) { - externalMemory->allNodes.push_back(n); - return true; - } - - inline bool restrictionFunction(_RawRestrictionContainer &r) { - externalMemory->restrictionsVector.push_back(r); - return true; - } - - /** warning: caller needs to take care of synchronization! */ - inline bool wayFunction(_Way &w) { - - //Get the properties of the way. - std::string highway( w.keyVals.Find("highway") ); - std::string name( w.keyVals.Find("name") ); - std::string ref( w.keyVals.Find("ref")); - std::string junction( w.keyVals.Find("junction") ); - std::string route( w.keyVals.Find("route") ); - int maxspeed( parseMaxspeed(w.keyVals.Find("maxspeed")) ); - std::string man_made( w.keyVals.Find("man_made") ); - std::string barrier( w.keyVals.Find("barrier") ); - std::string oneway( w.keyVals.Find("oneway")); - std::string cycleway( w.keyVals.Find("cycleway")); - std::string duration ( w.keyVals.Find("duration")); - std::string service (w.keyVals.Find("service")); - std::string area(w.keyVals.Find("area")); - - if("yes" == area && settings.ignoreAreas) - return true; - - //Save the name of the way if it has one, ref has precedence over name tag. - if ( 0 < ref.length() ) - w.name = ref; - else - if ( 0 < name.length() ) - w.name = name; - - if(junction == "roundabout") { - w.roundabout = true; - } - - //Is the route tag listed as usable way in the profile? - if(settings[route] > 0 || settings[man_made] > 0) { - w.useful = true; - DurationContainer dc; - if(checkForValidTiming(duration, dc)){ - w.speed = (600*(dc.hours*60+dc.minutes))/std::max((unsigned)(w.path.size()-1),1u); - w.isDurationSet = true; - } else { - w.speed = settings[route]; - } - w.direction = _Way::bidirectional; - if(0 < settings[route]) - highway = route; - else if (0 < settings[man_made]) { - highway = man_made; - } - } - - //determine the access value - std::string access; - std::string onewayClass; - std::string accessTag; - BOOST_FOREACH(std::string & s, settings.accessTags) { - access = std::string(w.keyVals.Find(s)); - if(0 < access.size()) { - accessTag = s; - onewayClass = std::string(w.keyVals.Find("oneway:"+access)); - break; - } - } - - if(0 < access.size()) { - // handle ways with default access = no - if(settings.accessForbiddenDefault.find(access) != settings.accessForbiddenDefault.end()) { - access = std::string("no"); - } - } - - //Is the highway tag listed as usable way? - if(0 < settings[highway] || "yes" == access || "designated" == access) { - if(!w.isDurationSet) { - if(0 < settings[highway]) { - if(0 < maxspeed) - if(settings.takeMinimumOfSpeeds) - w.speed = std::min(settings[highway], maxspeed); - else - w.speed = maxspeed; - else - w.speed = settings[highway]; - } else { - if(0 < maxspeed) - if(settings.takeMinimumOfSpeeds) - w.speed = std::min(settings.defaultSpeed, maxspeed); - else w.speed = maxspeed; - else - w.speed = settings.defaultSpeed; - highway = "default"; - } - } - w.useful = true; - - //Okay, do we have access to that way? - if(0 < access.size()) { //fastest way to check for non-empty string - //If access is forbidden, we don't want to route there. - if(settings.accessForbiddenKeys.find(access) != settings.accessForbiddenKeys.end()) { - w.access = false; - } - if(settings.accessRestrictionKeys.find(access) != settings.accessRestrictionKeys.end()) { - w.isAccessRestricted = true; - } - } - - if(0 < service.size()) { - if(settings.accessRestrictedService.find(service) != settings.accessRestrictedService.end()) { - w.isAccessRestricted = true; - } - } - - if("no" == access) { - return true; - } - - if( settings.obeyOneways ) { - if( onewayClass == "yes" || onewayClass == "1" || onewayClass == "true" ) { - w.direction = _Way::oneway; - } else if( onewayClass == "no" || onewayClass == "0" || onewayClass == "false" ) { - w.direction = _Way::bidirectional; - } else if( onewayClass == "-1" ) { - w.direction = _Way::opposite; - } else if( oneway == "no" || oneway == "0" || oneway == "false" ) { - w.direction = _Way::bidirectional; - } else if( accessTag == "bicycle" && (cycleway == "opposite" || cycleway == "opposite_track" || cycleway == "opposite_lane") ) { - w.direction = _Way::bidirectional; - } else if( oneway == "-1") { - w.direction = _Way::opposite; - } else if( oneway == "yes" || oneway == "1" || oneway == "true" || junction == "roundabout" || highway == "motorway_link" || highway == "motorway" ) { - w.direction = _Way::oneway; - } else { - w.direction = _Way::bidirectional; - } - } else { - w.direction = _Way::bidirectional; - } - } - - if ( w.useful && w.access && (1 < w.path.size()) ) { //Only true if the way is specified by the speed profile - w.type = settings.GetHighwayTypeID(highway); - if(0 > w.type) { - ERR("Resolved highway " << highway << " to " << w.type); - } - - //Get the unique identifier for the street name - const StringMap::const_iterator strit = stringMap->find(w.name); - if(strit == stringMap->end()) { - w.nameID = externalMemory->nameVector.size(); - externalMemory->nameVector.push_back(w.name); - stringMap->insert(StringMap::value_type(w.name, w.nameID)); - } else { - w.nameID = strit->second; - } - - if(-1 == w.speed){ - WARN("found way with bogus speed, id: " << w.id); - return true; - } - if(w.id == UINT_MAX) { - WARN("found way with unknown type: " << w.id); - return true; - } - - if ( w.direction == _Way::opposite ){ - std::reverse( w.path.begin(), w.path.end() ); - } - - for(vector< NodeID >::size_type n = 0; n < w.path.size()-1; ++n) { - externalMemory->allEdges.push_back(_Edge(w.path[n], w.path[n+1], w.type, w.direction, w.speed, w.nameID, w.roundabout, highway == settings.excludeFromGrid || "pier" == highway, w.isDurationSet, w.isAccessRestricted)); - externalMemory->usedNodeIDs.push_back(w.path[n]); - } - externalMemory->usedNodeIDs.push_back(w.path.back()); - - //The following information is needed to identify start and end segments of restrictions - externalMemory->wayStartEndVector.push_back(_WayIDStartAndEndEdge(w.id, w.path[0], w.path[1], w.path[w.path.size()-2], w.path[w.path.size()-1])); - } - return true; - } -}; - -#endif /* EXTRACTORCALLBACKS_H_ */ diff --git a/Extractor/ExtractorCallbacks.cpp b/Extractor/ExtractorCallbacks.cpp new file mode 100644 index 000000000..3c522bbcc --- /dev/null +++ b/Extractor/ExtractorCallbacks.cpp @@ -0,0 +1,268 @@ +/* + open source routing machine + Copyright (C) Dennis Luxen, others 2010 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU AFFERO General Public License as published by +the Free Software Foundation; either version 3 of the License, or +any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +or see http://www.gnu.org/licenses/agpl.txt. + */ + +/* + open source routing machine + Copyright (C) Dennis Luxen, others 2010 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU AFFERO General Public License as published by +the Free Software Foundation; either version 3 of the License, or +any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +or see http://www.gnu.org/licenses/agpl.txt. + */ + + +#include "ExtractorCallbacks.h" + +bool ExtractorCallbacks::checkForValidTiming(const std::string &s, DurationContainer & duration) { + boost::regex e ("((\\d|\\d\\d):)*(\\d|\\d\\d)",boost::regex_constants::icase|boost::regex_constants::perl); + + std::vector< std::string > result; + boost::algorithm::split_regex( result, s, boost::regex( ":" ) ) ; + bool matched = regex_match(s, e); + if(matched) { + duration.hours = (result.size()== 2) ? atoi(result[0].c_str()) : 0; + duration.minutes = (result.size()== 2) ? atoi(result[1].c_str()) : atoi(result[0].c_str()); + } + return matched; +} + +inline int ExtractorCallbacks::parseMaxspeed(std::string input) const { //call-by-value on purpose. + boost::algorithm::to_lower(input); + int n = atoi(input.c_str()); + if (input.find("mph") != std::string::npos || input.find("mp/h") != std::string::npos) { + n = (n*1609)/1000; + } + return n; +} + +ExtractorCallbacks::ExtractorCallbacks() {externalMemory = NULL; stringMap = NULL; } +ExtractorCallbacks::ExtractorCallbacks(ExtractionContainers * ext, Settings set, StringMap * strMap) { + externalMemory = ext; + settings = set; + stringMap = strMap; +} + +ExtractorCallbacks::~ExtractorCallbacks() { +} + + +/** warning: caller needs to take care of synchronization! */ +bool ExtractorCallbacks::nodeFunction(_Node &n) { + externalMemory->allNodes.push_back(n); + return true; +} + +bool ExtractorCallbacks::restrictionFunction(_RawRestrictionContainer &r) { + externalMemory->restrictionsVector.push_back(r); + return true; +} + +/** warning: caller needs to take care of synchronization! */ +bool ExtractorCallbacks::wayFunction(_Way &w) { + + //Get the properties of the way. + std::string highway( w.keyVals.Find("highway") ); + std::string name( w.keyVals.Find("name") ); + std::string ref( w.keyVals.Find("ref")); + std::string junction( w.keyVals.Find("junction") ); + std::string route( w.keyVals.Find("route") ); + int maxspeed( parseMaxspeed(w.keyVals.Find("maxspeed")) ); + std::string man_made( w.keyVals.Find("man_made") ); + std::string barrier( w.keyVals.Find("barrier") ); + std::string oneway( w.keyVals.Find("oneway")); + std::string cycleway( w.keyVals.Find("cycleway")); + std::string duration ( w.keyVals.Find("duration")); + std::string service (w.keyVals.Find("service")); + std::string area(w.keyVals.Find("area")); + + if("yes" == area && settings.ignoreAreas) + return true; + + //Save the name of the way if it has one, ref has precedence over name tag. + if ( 0 < ref.length() ) + w.name = ref; + else + if ( 0 < name.length() ) + w.name = name; + + if(junction == "roundabout") { + w.roundabout = true; + } + + //Is the route tag listed as usable way in the profile? + if(settings[route] > 0 || settings[man_made] > 0) { + w.useful = true; + DurationContainer dc; + if(checkForValidTiming(duration, dc)){ + w.speed = (600*(dc.hours*60+dc.minutes))/std::max((unsigned)(w.path.size()-1),1u); + w.isDurationSet = true; + } else { + w.speed = settings[route]; + } + w.direction = _Way::bidirectional; + if(0 < settings[route]) + highway = route; + else if (0 < settings[man_made]) { + highway = man_made; + } + } + + //determine the access value + std::string access; + std::string onewayClass; + std::string accessTag; + BOOST_FOREACH(std::string & s, settings.accessTags) { + access = std::string(w.keyVals.Find(s)); + if(0 < access.size()) { + accessTag = s; + onewayClass = std::string(w.keyVals.Find("oneway:"+access)); + break; + } + } + + if(0 < access.size()) { + // handle ways with default access = no + if(settings.accessForbiddenDefault.find(access) != settings.accessForbiddenDefault.end()) { + access = std::string("no"); + } + } + + //Is the highway tag listed as usable way? + if(0 < settings[highway] || "yes" == access || "designated" == access) { + if(!w.isDurationSet) { + if(0 < settings[highway]) { + if(0 < maxspeed) + if(settings.takeMinimumOfSpeeds) + w.speed = std::min(settings[highway], maxspeed); + else + w.speed = maxspeed; + else + w.speed = settings[highway]; + } else { + if(0 < maxspeed) + if(settings.takeMinimumOfSpeeds) + w.speed = std::min(settings.defaultSpeed, maxspeed); + else w.speed = maxspeed; + else + w.speed = settings.defaultSpeed; + highway = "default"; + } + } + w.useful = true; + + //Okay, do we have access to that way? + if(0 < access.size()) { //fastest way to check for non-empty string + //If access is forbidden, we don't want to route there. + if(settings.accessForbiddenKeys.find(access) != settings.accessForbiddenKeys.end()) { + w.access = false; + } + if(settings.accessRestrictionKeys.find(access) != settings.accessRestrictionKeys.end()) { + w.isAccessRestricted = true; + } + } + + if(0 < service.size()) { + if(settings.accessRestrictedService.find(service) != settings.accessRestrictedService.end()) { + w.isAccessRestricted = true; + } + } + + if("no" == access) { + return true; + } + + if( settings.obeyOneways ) { + if( onewayClass == "yes" || onewayClass == "1" || onewayClass == "true" ) { + w.direction = _Way::oneway; + } else if( onewayClass == "no" || onewayClass == "0" || onewayClass == "false" ) { + w.direction = _Way::bidirectional; + } else if( onewayClass == "-1" ) { + w.direction = _Way::opposite; + } else if( oneway == "no" || oneway == "0" || oneway == "false" ) { + w.direction = _Way::bidirectional; + } else if( accessTag == "bicycle" && (cycleway == "opposite" || cycleway == "opposite_track" || cycleway == "opposite_lane") ) { + w.direction = _Way::bidirectional; + } else if( oneway == "-1") { + w.direction = _Way::opposite; + } else if( oneway == "yes" || oneway == "1" || oneway == "true" || junction == "roundabout" || highway == "motorway_link" || highway == "motorway" ) { + w.direction = _Way::oneway; + } else { + w.direction = _Way::bidirectional; + } + } else { + w.direction = _Way::bidirectional; + } + } + + if ( w.useful && w.access && (1 < w.path.size()) ) { //Only true if the way is specified by the speed profile + w.type = settings.GetHighwayTypeID(highway); + if(0 > w.type) { + ERR("Resolved highway " << highway << " to " << w.type); + } + + //Get the unique identifier for the street name + const StringMap::const_iterator strit = stringMap->find(w.name); + if(strit == stringMap->end()) { + w.nameID = externalMemory->nameVector.size(); + externalMemory->nameVector.push_back(w.name); + stringMap->insert(StringMap::value_type(w.name, w.nameID)); + } else { + w.nameID = strit->second; + } + + if(-1 == w.speed){ + WARN("found way with bogus speed, id: " << w.id); + return true; + } + if(w.id == UINT_MAX) { + WARN("found way with unknown type: " << w.id); + return true; + } + + if ( w.direction == _Way::opposite ){ + std::reverse( w.path.begin(), w.path.end() ); + } + + for(vector< NodeID >::size_type n = 0; n < w.path.size()-1; ++n) { + externalMemory->allEdges.push_back(_Edge(w.path[n], w.path[n+1], w.type, w.direction, w.speed, w.nameID, w.roundabout, highway == settings.excludeFromGrid || "pier" == highway, w.isDurationSet, w.isAccessRestricted)); + externalMemory->usedNodeIDs.push_back(w.path[n]); + } + externalMemory->usedNodeIDs.push_back(w.path.back()); + + //The following information is needed to identify start and end segments of restrictions + externalMemory->wayStartEndVector.push_back(_WayIDStartAndEndEdge(w.id, w.path[0], w.path[1], w.path[w.path.size()-2], w.path[w.path.size()-1])); + } + return true; +} + + + + diff --git a/Extractor/ExtractorCallbacks.h b/Extractor/ExtractorCallbacks.h new file mode 100644 index 000000000..917e89529 --- /dev/null +++ b/Extractor/ExtractorCallbacks.h @@ -0,0 +1,63 @@ +/* + open source routing machine + Copyright (C) Dennis Luxen, others 2010 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU AFFERO General Public License as published by +the Free Software Foundation; either version 3 of the License, or +any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +or see http://www.gnu.org/licenses/agpl.txt. + */ + +#ifndef EXTRACTORCALLBACKS_H_ +#define EXTRACTORCALLBACKS_H_ + +#include +#include + +#include +#include +#include + +#include "ExtractionContainers.h" +#include "ExtractorStructs.h" + +class ExtractorCallbacks{ +private: + Settings settings; + StringMap * stringMap; + ExtractionContainers * externalMemory; + + struct DurationContainer { + int hours; + int minutes; + }; + + bool checkForValidTiming(const std::string &s, DurationContainer & duration); + inline int parseMaxspeed(std::string input) const; + ExtractorCallbacks(); +public: + explicit ExtractorCallbacks(ExtractionContainers * ext, Settings set, StringMap * strMap); + + ~ExtractorCallbacks(); + + /** warning: caller needs to take care of synchronization! */ + bool nodeFunction(_Node &n); + + bool restrictionFunction(_RawRestrictionContainer &r); + + /** warning: caller needs to take care of synchronization! */ + bool wayFunction(_Way &w); + +}; + +#endif /* EXTRACTORCALLBACKS_H_ */ diff --git a/Extractor/LuaUtil.h b/Extractor/LuaUtil.h new file mode 100644 index 000000000..a1060164a --- /dev/null +++ b/Extractor/LuaUtil.h @@ -0,0 +1,33 @@ +/* + open source routing machine + Copyright (C) Dennis Luxen, others 2010 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU AFFERO General Public License as published by +the Free Software Foundation; either version 3 of the License, or +any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +or see http://www.gnu.org/licenses/agpl.txt. + */ + + + +#ifndef LUAUTIL_H_ +#define LUAUTIL_H_ + +#include + +template +void LUA_print(T number) { + std::cout << "[LUA] " << number << std::endl; +} + +#endif /* LUAUTIL_H_ */ diff --git a/Extractor/PBFParser.h b/Extractor/PBFParser.h index 29f5661d3..372e8d817 100644 --- a/Extractor/PBFParser.h +++ b/Extractor/PBFParser.h @@ -22,6 +22,7 @@ or see http://www.gnu.org/licenses/agpl.txt. #define PBFPARSER_H_ #include +#include #include #include @@ -36,6 +37,8 @@ or see http://www.gnu.org/licenses/agpl.txt. class PBFParser : public BaseParser<_Node, _RawRestrictionContainer, _Way> { + typedef BaseParser<_Node, _RawRestrictionContainer, _Way> super; + enum EntityType { TypeNode = 1, TypeWay = 2, @@ -63,7 +66,7 @@ class PBFParser : public BaseParser<_Node, _RawRestrictionContainer, _Way> { }; public: - PBFParser(const char * fileName) { + PBFParser(const char * fileName) : myLuaState(NULL) { GOOGLE_PROTOBUF_VERIFY_VERSION; //TODO: What is the bottleneck here? Filling the queue or reading the stuff from disk? threadDataQueue.reset( new ConcurrentQueue<_ThreadData*>(2500) ); /* Max 2500 items in queue, hardcoded. */ @@ -77,18 +80,22 @@ public: groupCount = 0; //Dummy initialization - wayCallback = NULL; nodeCallback = NULL; - addressCallback = NULL; restrictionCallback = NULL; + wayCallback = NULL; + nodeCallback = NULL; + restrictionCallback = NULL; } - bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way),bool (*addressCallbackPointer)(_Node, HashTable&) ) { + bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way) ) { nodeCallback = *nodeCallbackPointer; wayCallback = *wayCallbackPointer; restrictionCallback = *restrictionCallbackPointer; - addressCallback = *addressCallbackPointer; return true; } + void RegisterLUAState(lua_State *ml) { + myLuaState = ml; + } + ~PBFParser() { if(input.is_open()) input.close(); @@ -188,7 +195,7 @@ public: // Start the read and parse threads boost::thread readThread(boost::bind(&PBFParser::ReadData, this)); boost::thread parseThread(boost::bind(&PBFParser::ParseData, this)); - + // Wait for the threads to finish readThread.join(); parseThread.join(); @@ -206,11 +213,11 @@ private: int m_lastDenseLongitude = 0; for(int i = 0; i < dense.id_size(); i++) { - HashTable keyVals; + m_lastDenseID += dense.id( i ); m_lastDenseLatitude += dense.lat( i ); m_lastDenseLongitude += dense.lon( i ); - _Node n; + ImportNode n; n.id = m_lastDenseID; n.lat = 100000*( ( double ) m_lastDenseLatitude * threadData->PBFprimitiveBlock.granularity() +threadData-> PBFprimitiveBlock.lat_offset() ) / NANO; n.lon = 100000*( ( double ) m_lastDenseLongitude * threadData->PBFprimitiveBlock.granularity() + threadData->PBFprimitiveBlock.lon_offset() ) / NANO; @@ -223,22 +230,35 @@ private: int keyValue = dense.keys_vals ( denseTagIndex+1 ); std::string key = threadData->PBFprimitiveBlock.stringtable().s(tagValue).data(); std::string value = threadData->PBFprimitiveBlock.stringtable().s(keyValue).data(); - keyVals.Add(key, value); + n.keyVals.Add(key, value); denseTagIndex += 2; } - std::string barrierValue = keyVals.Find("barrier"); - std::string access = keyVals.Find("access"); - if(access != "yes" && 0 < barrierValue.length() && "cattle_grid" != barrierValue && "border_control" != barrierValue && "toll_booth" != barrierValue && "no" != barrierValue) - n.bollard = true; - if("traffic_signals" == keyVals.Find("highway")) - n.trafficLight = true; + int ret = -1; + try { + ret = luabind::call_function( + myLuaState, + "node_function", + boost::ref(n) + ); + if(!(*nodeCallback)(n)) + std::cerr << "[PBFParser] dense node not parsed" << std::endl; + } catch (const luabind::error &er) { + cerr << er.what() << endl; + lua_State* Ler=er.state(); + report_errors(Ler, -1); + } + catch (...) { + cerr<<"Unknown error!"<&); /* the input stream to parse */ std::fstream input; /* ThreadData Queue */ boost::shared_ptr > threadDataQueue; + + lua_State *myLuaState; }; #endif /* PBFPARSER_H_ */ diff --git a/Extractor/V8Helper.h b/Extractor/V8Helper.h deleted file mode 100644 index 46fc4ee3e..000000000 --- a/Extractor/V8Helper.h +++ /dev/null @@ -1,161 +0,0 @@ -/* - open source routing machine - Copyright (C) Dennis Luxen, others 2010 - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU AFFERO General Public License as published by -the Free Software Foundation; either version 3 of the License, or -any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -or see http://www.gnu.org/licenses/agpl.txt. - */ - -#ifndef V8HELPER_H_ -#define V8HELPER_H_ - -#include - -#include - -namespace V8Helper { -//Provides an output mechanism to the V8 Engine -static v8::Handle PrintToConsole (const v8::Arguments& args){ - std::cout << "[JS] : " << *v8::String::AsciiValue(args[0]) << std::endl; - return v8::Undefined(); -} - -//Returns the version of the V8 Engine -v8::Handle Version(const v8::Arguments& args) { - return v8::String::New(v8::V8::GetVersion()); -} - -//Most of the stuff below is from http://blog.owned.co.za provided with an MIT License - -template -v8::Handle WrapClass(T* y) { - // Handle scope for temporary handles, - v8::HandleScope handle_scope; - v8::Persistent class_template_; - - v8::Handle raw_template = v8::ObjectTemplate::New(); - - //The raw template is the ObjectTemplate (that can be exposed to script too) - //but is maintained internally. - raw_template->SetInternalFieldCount(1); - - //Create the actual template object, - class_template_ = v8::Persistent::New(raw_template); - - //Create the new handle to return, and set its template type - v8::Handle result = class_template_->NewInstance(); - v8::Handle class_ptr = v8::External::New(static_cast(y)); - - //Point the 0 index Field to the c++ pointer for unwrapping later - result->SetInternalField(0, class_ptr); - - //Return the value, releasing the other handles on its way. - return handle_scope.Close(result); -} - - -template -v8::Handle ExposeClass(v8::Persistent context, T* y, v8::Handle exposedName, v8::PropertyAttribute props) { - v8::HandleScope handle_scope; - - v8::Handle obj = WrapClass(y); - context->Global()->Set(exposedName, obj, props); - - return handle_scope.Close(obj); -} - -template -T* UnwrapClass(v8::Handle obj) { - v8::Handle field = v8::Handle::Cast(obj->GetInternalField(0)); - void* ptr = field->Value(); - return static_cast(ptr); -} - -void Expose(v8::Handle intoObject, v8::Handle namev8String, v8::InvocationCallback funcID) { - v8::HandleScope handle_scope; - - v8::Handle thisFunc = v8::FunctionTemplate::New(funcID); - intoObject->Set(namev8String, thisFunc->GetFunction()); -} - -template -v8::Handle GetMember(v8::Local property, const v8::AccessorInfo &info) { - v8::Local self = info.Holder(); - v8::Local wrap = v8::Local::Cast(self->GetInternalField(0)); - void* ptr = wrap->Value(); - int value = static_cast(ptr)->*MemVar; - return PropertyType::New(value); -} - - -template -void SetIntMember(v8::Local property, v8::Local value, const v8::AccessorInfo& info) { - v8::Local self = info.Holder(); - v8::Local wrap = v8::Local::Cast(self->GetInternalField(0)); - void * ptr = wrap->Value(); - static_cast(ptr)->*MemVar = value->Int32Value(); -} - -template -v8::Handle GetInt(v8::Local property, const v8::AccessorInfo &info) { - v8::Local self = info.Holder(); - v8::Local wrap = v8::Local::Cast(self->GetInternalField(0)); - void* ptr = wrap->Value(); - CPPProperyType value = static_cast(ptr)->member1; - return JSPropertyType::New(value); -} - -// Extracts a C string from a V8 Utf8Value. -const char* ToCString(const v8::String::Utf8Value& value) { - return *value ? *value : ""; -} - -void ReportException(v8::TryCatch* try_catch) { - v8::HandleScope handle_scope; - v8::String::Utf8Value exception(try_catch->Exception()); - const char* exception_string = ToCString(exception); - v8::Handle message = try_catch->Message(); - if (message.IsEmpty()) { - // V8 didn't provide any extra information about this error; just - // print the exception. - std::cout << exception_string << std::endl; - } else { - // Print (filename):(line number): (message). - v8::String::Utf8Value filename(message->GetScriptResourceName()); - int linenum = message->GetLineNumber(); - std::cout << ToCString(filename) << ":" << linenum << ": " << exception_string << std::endl; - // Print line of source code. - v8::String::Utf8Value sourceline(message->GetSourceLine()); - std::cout << ToCString(sourceline) << std::endl; - - // Print wavy underline (GetUnderline is deprecated). - int start = message->GetStartColumn(); - for (int i = 0; i < start; ++i) { - std::cout << " "; - } - - for (int i = start, end = message->GetEndColumn(); i < end; ++i) { - std::cout << "^"; - } - std::cout << std::endl; - v8::String::Utf8Value stack_trace(try_catch->StackTrace()); - if (stack_trace.length()) { - std::cout << ToCString(stack_trace) << std::endl; - } - } -} -} - -#endif /* V8HELPER_H_ */ diff --git a/Extractor/XMLParser.h b/Extractor/XMLParser.h index 9a5538086..d378b2f8f 100644 --- a/Extractor/XMLParser.h +++ b/Extractor/XMLParser.h @@ -31,18 +31,23 @@ or see http://www.gnu.org/licenses/agpl.txt. class XMLParser : public BaseParser<_Node, _RawRestrictionContainer, _Way> { public: - XMLParser(const char * filename) : nodeCallback(NULL), wayCallback(NULL), restrictionCallback(NULL){ + XMLParser(const char * filename) : nodeCallback(NULL), wayCallback(NULL), restrictionCallback(NULL), myLuaState(NULL){ WARN("Parsing plain .osm/.osm.bz2 is deprecated. Switch to .pbf"); inputReader = inputReaderFactory(filename); } virtual ~XMLParser() {} - bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way), bool (*addressCallbackPointer)(_Node, HashTable&) ) { + bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way)) { nodeCallback = *nodeCallbackPointer; wayCallback = *wayCallbackPointer; restrictionCallback = *restrictionCallbackPointer; return true; } + + void RegisterLUAState(lua_State *ml) { + myLuaState = ml; + } + bool Init() { return (xmlTextReaderRead( inputReader ) == 1); } @@ -293,6 +298,7 @@ private: bool (*wayCallback)(_Way); bool (*restrictionCallback)(_RawRestrictionContainer); + lua_State *myLuaState; }; #endif /* XMLPARSER_H_ */ diff --git a/SConstruct b/SConstruct index 0194d4a4b..4d13255a2 100644 --- a/SConstruct +++ b/SConstruct @@ -118,6 +118,8 @@ else: if GetOption('buildconfiguration') != 'debug' and GetOption('nomarch') == None and sys.platform != 'darwin': env.Append(CCFLAGS = ['-march=native']) +env.ParseConfig('pkg-config --cflags --libs luabind') + if not conf.CheckHeader('omp.h'): print "Compiler does not support OpenMP. Exiting" Exit(-1) @@ -148,9 +150,6 @@ if not (env.Detect('protoc')): if not conf.CheckLibWithHeader('stxxl', 'stxxl.h', 'CXX'): print "stxxl library not found. Exiting" Exit(-1) -if not conf.CheckLibWithHeader('v8', 'v8.h', 'CXX'): - print "v8 library not found. Exiting" - Exit(-1) if not conf.CheckLibWithHeader('xml2', 'libxml/xmlreader.h', 'CXX'): print "libxml2 library or header not found. Exiting" Exit(-1) @@ -270,7 +269,7 @@ if sys.platform != 'darwin': env.Append(CCFLAGS = ['-fopenmp']) env.Append(LINKFLAGS = ['-fopenmp']) -env.Program(target = 'osrm-extract', source = ["extractor.cpp", Glob('Util/*.cpp')]) +env.Program(target = 'osrm-extract', source = ["extractor.cpp", Glob('Util/*.cpp'), Glob('Extractor/*.cpp')]) env.Program(target = 'osrm-prepare', source = ["createHierarchy.cpp", Glob('Contractor/*.cpp'), Glob('Util/SRTMLookup/*.cpp'), Glob('Algorithms/*.cpp')]) env.Program(target = 'osrm-routed', source = ["routed.cpp", 'Descriptors/DescriptionFactory.cpp', Glob('ThirdParty/*.cc'), Glob('Server/DataStructures/*.cpp')], CCFLAGS = env['CCFLAGS'] + ['-DROUTED']) env = conf.Finish() diff --git a/extractor.cpp b/extractor.cpp index eef253867..406d627d7 100644 --- a/extractor.cpp +++ b/extractor.cpp @@ -35,17 +35,24 @@ or see http://www.gnu.org/licenses/agpl.txt. #include #include +extern "C" { +#include +#include +#include +} +#include + #include #include #include #include #include -#include #include "typedefs.h" #include "DataStructures/InputReaderFactory.h" -#include "Extractor/ExtractorCallBacks.h" +#include "Extractor/ExtractorCallbacks.h" #include "Extractor/ExtractorStructs.h" +#include "Extractor/LuaUtil.h" #include "Extractor/PBFParser.h" #include "Extractor/XMLParser.h" #include "Util/BaseConfiguration.h" @@ -53,22 +60,19 @@ or see http://www.gnu.org/licenses/agpl.txt. #include "Util/MachineInfo.h" #include "Util/StringUtil.h" +#include "Extractor/ExtractionContainers.h" + using namespace std; typedef BaseConfiguration ExtractorConfiguration; unsigned globalRestrictionCounter = 0; ExtractorCallbacks * extractCallBacks; - +// bool nodeFunction(_Node n); -bool adressFunction(_Node n, HashTable & keyVals); bool restrictionFunction(_RawRestrictionContainer r); bool wayFunction(_Way w); -template -bool removeIfUnused(ClassT n) { return (false == n.used); } - - int main (int argc, char *argv[]) { if(argc < 2) { @@ -193,6 +197,52 @@ int main (int argc, char *argv[]) { ERR("caught: " << e.what() ); } + /*** Setup Scripting Environment ***/ + + // Create a new lua state + lua_State *myLuaState = luaL_newstate(); + + // Connect LuaBind to this lua state + luabind::open(myLuaState); + + // Add our function to the state's global scope + luabind::module(myLuaState) [ + luabind::def("print", LUA_print) + ]; + + if(0 != luaL_dostring( + myLuaState, + "print('Initializing LUA engine')\n" + )) { + ERR(lua_tostring(myLuaState,-1)<< " occured in scripting block"); + } + + luabind::module(myLuaState) [ + luabind::class_ >("keyVals") + .def("Add", &HashTable::Add) + .def("Find", &HashTable::Find) + ]; + + luabind::module(myLuaState) [ + luabind::class_("Node") + .def(luabind::constructor<>()) + .def_readwrite("lat", &ImportNode::lat) + .def_readwrite("lon", &ImportNode::lon) + .def_readwrite("id", &ImportNode::id) + .def_readwrite("bollard", &ImportNode::bollard) + .def_readwrite("traffic_light", &ImportNode::trafficLight) + .def_readwrite("tags", &ImportNode::keyVals) + ]; + // Now call our function in a lua script + if(0 != luaL_dofile(myLuaState, "profile.lua")) { + ERR(lua_tostring(myLuaState,-1)<< " occured in scripting block"); + } + + //open string library; + luaopen_string(myLuaState); + + /*** End of Scripting Environment Setup; ***/ + unsigned amountOfRAM = 1; unsigned installedRAM = GetPhysicalmemory(); if(installedRAM < 2048264) { @@ -207,11 +257,7 @@ int main (int argc, char *argv[]) { } StringMap stringMap; - STXXLContainers externalMemory; - - unsigned usedNodeCounter = 0; - unsigned usedEdgeCounter = 0; - double time = get_timestamp(); + ExtractionContainers externalMemory; stringMap[""] = 0; extractCallBacks = new ExtractorCallbacks(&externalMemory, settings, &stringMap); @@ -221,301 +267,15 @@ int main (int argc, char *argv[]) { } else { parser = new XMLParser(argv[1]); } - parser->RegisterCallbacks(&nodeFunction, &restrictionFunction, &wayFunction, &adressFunction); + parser->RegisterCallbacks(&nodeFunction, &restrictionFunction, &wayFunction); + parser->RegisterLUAState(myLuaState); + if(!parser->Init()) INFO("Parser not initialized!"); parser->Parse(); - try { - // INFO("raw no. of names: " << externalMemory.nameVector.size()); - // INFO("raw no. of nodes: " << externalMemory.allNodes.size()); - // INFO("no. of used nodes: " << externalMemory.usedNodeIDs.size()); - // INFO("raw no. of edges: " << externalMemory.allEdges.size()); - // INFO("raw no. of ways: " << externalMemory.wayStartEndVector.size()); - // INFO("raw no. of addresses: " << externalMemory.adressVector.size()); - // INFO("raw no. of restrictions: " << externalMemory.restrictionsVector.size()); + externalMemory.PrepareData(settings, outputFileName, restrictionsFileName, amountOfRAM); - cout << "[extractor] parsing finished after " << get_timestamp() - time << " seconds" << endl; - time = get_timestamp(); - boost::uint64_t memory_to_use = static_cast(amountOfRAM) * 1024 * 1024 * 1024; - - cout << "[extractor] Sorting used nodes ... " << flush; - stxxl::sort(externalMemory.usedNodeIDs.begin(), externalMemory.usedNodeIDs.end(), Cmp(), memory_to_use); - cout << "ok, after " << get_timestamp() - time << "s" << endl; - - time = get_timestamp(); - cout << "[extractor] Erasing duplicate nodes ... " << flush; - stxxl::vector::iterator NewEnd = unique ( externalMemory.usedNodeIDs.begin(),externalMemory.usedNodeIDs.end() ) ; - externalMemory.usedNodeIDs.resize ( NewEnd - externalMemory.usedNodeIDs.begin() ); - cout << "ok, after " << get_timestamp() - time << "s" << endl; - time = get_timestamp(); - - cout << "[extractor] Sorting all nodes ... " << flush; - stxxl::sort(externalMemory.allNodes.begin(), externalMemory.allNodes.end(), CmpNodeByID(), memory_to_use); - cout << "ok, after " << get_timestamp() - time << "s" << endl; - time = get_timestamp(); - - cout << "[extractor] Sorting used ways ... " << flush; - stxxl::sort(externalMemory.wayStartEndVector.begin(), externalMemory.wayStartEndVector.end(), CmpWayByID(), memory_to_use); - cout << "ok, after " << get_timestamp() - time << "s" << endl; - - cout << "[extractor] Sorting restrctns. by from... " << flush; - stxxl::sort(externalMemory.restrictionsVector.begin(), externalMemory.restrictionsVector.end(), CmpRestrictionContainerByFrom(), memory_to_use); - cout << "ok, after " << get_timestamp() - time << "s" << endl; - - cout << "[extractor] Fixing restriction starts ... " << flush; - STXXLRestrictionsVector::iterator restrictionsIT = externalMemory.restrictionsVector.begin(); - STXXLWayIDStartEndVector::iterator wayStartAndEndEdgeIT = externalMemory.wayStartEndVector.begin(); - - while(wayStartAndEndEdgeIT != externalMemory.wayStartEndVector.end() && restrictionsIT != externalMemory.restrictionsVector.end()) { - if(wayStartAndEndEdgeIT->wayID < restrictionsIT->fromWay){ - ++wayStartAndEndEdgeIT; - continue; - } - if(wayStartAndEndEdgeIT->wayID > restrictionsIT->fromWay) { - ++restrictionsIT; - continue; - } - assert(wayStartAndEndEdgeIT->wayID == restrictionsIT->fromWay); - NodeID viaNode = restrictionsIT->restriction.viaNode; - - if(wayStartAndEndEdgeIT->firstStart == viaNode) { - restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstTarget; - } else if(wayStartAndEndEdgeIT->firstTarget == viaNode) { - restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstStart; - } else if(wayStartAndEndEdgeIT->lastStart == viaNode) { - restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastTarget; - } else if(wayStartAndEndEdgeIT->lastTarget == viaNode) { - restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastStart; - } - ++restrictionsIT; - } - - cout << "ok, after " << get_timestamp() - time << "s" << endl; - time = get_timestamp(); - - cout << "[extractor] Sorting restrctns. by to ... " << flush; - stxxl::sort(externalMemory.restrictionsVector.begin(), externalMemory.restrictionsVector.end(), CmpRestrictionContainerByTo(), memory_to_use); - cout << "ok, after " << get_timestamp() - time << "s" << endl; - - time = get_timestamp(); - unsigned usableRestrictionsCounter(0); - cout << "[extractor] Fixing restriction ends ... " << flush; - restrictionsIT = externalMemory.restrictionsVector.begin(); - wayStartAndEndEdgeIT = externalMemory.wayStartEndVector.begin(); - while(wayStartAndEndEdgeIT != externalMemory.wayStartEndVector.end() && restrictionsIT != externalMemory.restrictionsVector.end()) { - if(wayStartAndEndEdgeIT->wayID < restrictionsIT->toWay){ - ++wayStartAndEndEdgeIT; - continue; - } - if(wayStartAndEndEdgeIT->wayID > restrictionsIT->toWay) { - ++restrictionsIT; - continue; - } - NodeID viaNode = restrictionsIT->restriction.viaNode; - if(wayStartAndEndEdgeIT->lastStart == viaNode) { - restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastTarget; - } else if(wayStartAndEndEdgeIT->lastTarget == viaNode) { - restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastStart; - } else if(wayStartAndEndEdgeIT->firstStart == viaNode) { - restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstTarget; - } else if(wayStartAndEndEdgeIT->firstTarget == viaNode) { - restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstStart; - } - - if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) { - ++usableRestrictionsCounter; - } - ++restrictionsIT; - } - cout << "ok, after " << get_timestamp() - time << "s" << endl; - INFO("usable restrictions: " << usableRestrictionsCounter ); - //serialize restrictions - ofstream restrictionsOutstream; - restrictionsOutstream.open(restrictionsFileName.c_str(), ios::binary); - restrictionsOutstream.write((char*)&usableRestrictionsCounter, sizeof(unsigned)); - for(restrictionsIT = externalMemory.restrictionsVector.begin(); restrictionsIT != externalMemory.restrictionsVector.end(); ++restrictionsIT) { - if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) { - restrictionsOutstream.write((char *)&(restrictionsIT->restriction), sizeof(_Restriction)); - } - } - restrictionsOutstream.close(); - - ofstream fout; - fout.open(outputFileName.c_str(), ios::binary); - fout.write((char*)&usedNodeCounter, sizeof(unsigned)); - time = get_timestamp(); - cout << "[extractor] Confirming/Writing used nodes ... " << flush; - - STXXLNodeVector::iterator nodesIT = externalMemory.allNodes.begin(); - STXXLNodeIDVector::iterator usedNodeIDsIT = externalMemory.usedNodeIDs.begin(); - while(usedNodeIDsIT != externalMemory.usedNodeIDs.end() && nodesIT != externalMemory.allNodes.end()) { - if(*usedNodeIDsIT < nodesIT->id){ - ++usedNodeIDsIT; - continue; - } - if(*usedNodeIDsIT > nodesIT->id) { - ++nodesIT; - continue; - } - if(*usedNodeIDsIT == nodesIT->id) { - if(!settings.obeyBollards && nodesIT->bollard) - nodesIT->bollard = false; - fout.write((char*)&(*nodesIT), sizeof(_Node)); - ++usedNodeCounter; - ++usedNodeIDsIT; - ++nodesIT; - } - } - - cout << "ok, after " << get_timestamp() - time << "s" << endl; - - cout << "[extractor] setting number of nodes ... " << flush; - ios::pos_type positionInFile = fout.tellp(); - fout.seekp(ios::beg); - fout.write((char*)&usedNodeCounter, sizeof(unsigned)); - fout.seekp(positionInFile); - - cout << "ok" << endl; - time = get_timestamp(); - - // Sort edges by start. - cout << "[extractor] Sorting edges by start ... " << flush; - stxxl::sort(externalMemory.allEdges.begin(), externalMemory.allEdges.end(), CmpEdgeByStartID(), memory_to_use); - cout << "ok, after " << get_timestamp() - time << "s" << endl; - time = get_timestamp(); - - cout << "[extractor] Setting start coords ... " << flush; - fout.write((char*)&usedEdgeCounter, sizeof(unsigned)); - // Traverse list of edges and nodes in parallel and set start coord - nodesIT = externalMemory.allNodes.begin(); - STXXLEdgeVector::iterator edgeIT = externalMemory.allEdges.begin(); - while(edgeIT != externalMemory.allEdges.end() && nodesIT != externalMemory.allNodes.end()) { - if(edgeIT->start < nodesIT->id){ - ++edgeIT; - continue; - } - if(edgeIT->start > nodesIT->id) { - nodesIT++; - continue; - } - if(edgeIT->start == nodesIT->id) { - edgeIT->startCoord.lat = nodesIT->lat; - edgeIT->startCoord.lon = nodesIT->lon; - ++edgeIT; - } - } - cout << "ok, after " << get_timestamp() - time << "s" << endl; - time = get_timestamp(); - - // Sort Edges by target - cout << "[extractor] Sorting edges by target ... " << flush; - stxxl::sort(externalMemory.allEdges.begin(), externalMemory.allEdges.end(), CmpEdgeByTargetID(), memory_to_use); - cout << "ok, after " << get_timestamp() - time << "s" << endl; - time = get_timestamp(); - - cout << "[extractor] Setting target coords ... " << flush; - // Traverse list of edges and nodes in parallel and set target coord - nodesIT = externalMemory.allNodes.begin(); - edgeIT = externalMemory.allEdges.begin(); - - while(edgeIT != externalMemory.allEdges.end() && nodesIT != externalMemory.allNodes.end()) { - if(edgeIT->target < nodesIT->id){ - ++edgeIT; - continue; - } - if(edgeIT->target > nodesIT->id) { - ++nodesIT; - continue; - } - if(edgeIT->target == nodesIT->id) { - if(edgeIT->startCoord.lat != INT_MIN && edgeIT->startCoord.lon != INT_MIN) { - edgeIT->targetCoord.lat = nodesIT->lat; - edgeIT->targetCoord.lon = nodesIT->lon; - - double distance = ApproximateDistance(edgeIT->startCoord.lat, edgeIT->startCoord.lon, nodesIT->lat, nodesIT->lon); - assert(edgeIT->speed != -1); - double weight = ( distance * 10. ) / (edgeIT->speed / 3.6); - int intWeight = std::max(1, (int)std::floor((edgeIT->isDurationSet ? edgeIT->speed : weight)+.5) ); - int intDist = std::max(1, (int)distance); - short zero = 0; - short one = 1; - - fout.write((char*)&edgeIT->start, sizeof(unsigned)); - fout.write((char*)&edgeIT->target, sizeof(unsigned)); - fout.write((char*)&intDist, sizeof(int)); - switch(edgeIT->direction) { - case _Way::notSure: - fout.write((char*)&zero, sizeof(short)); - break; - case _Way::oneway: - fout.write((char*)&one, sizeof(short)); - break; - case _Way::bidirectional: - fout.write((char*)&zero, sizeof(short)); - - break; - case _Way::opposite: - fout.write((char*)&one, sizeof(short)); - break; - default: - cerr << "[error] edge with no direction: " << edgeIT->direction << endl; - assert(false); - break; - } - fout.write((char*)&intWeight, sizeof(int)); - assert(edgeIT->type >= 0); - fout.write((char*)&edgeIT->type, sizeof(short)); - fout.write((char*)&edgeIT->nameID, sizeof(unsigned)); - fout.write((char*)&edgeIT->isRoundabout, sizeof(bool)); - fout.write((char*)&edgeIT->ignoreInGrid, sizeof(bool)); - fout.write((char*)&edgeIT->isAccessRestricted, sizeof(bool)); - } - ++usedEdgeCounter; - ++edgeIT; - } - } - cout << "ok, after " << get_timestamp() - time << "s" << endl; - cout << "[extractor] setting number of edges ... " << flush; - - fout.seekp(positionInFile); - fout.write((char*)&usedEdgeCounter, sizeof(unsigned)); - fout.close(); - cout << "ok" << endl; - time = get_timestamp(); - cout << "[extractor] writing street name index ... " << flush; - std::string nameOutFileName = (outputFileName + ".names"); - ofstream nameOutFile(nameOutFileName.c_str(), ios::binary); - unsigned sizeOfNameIndex = externalMemory.nameVector.size(); - nameOutFile.write((char *)&(sizeOfNameIndex), sizeof(unsigned)); - - BOOST_FOREACH(string str, externalMemory.nameVector) { - unsigned lengthOfRawString = strlen(str.c_str()); - nameOutFile.write((char *)&(lengthOfRawString), sizeof(unsigned)); - nameOutFile.write(str.c_str(), lengthOfRawString); - } - - nameOutFile.close(); - cout << "ok, after " << get_timestamp() - time << "s" << endl; - - // time = get_timestamp(); - // cout << "[extractor] writing address list ... " << flush; - // - // adressFileName.append(".address"); - // ofstream addressOutFile(adressFileName.c_str()); - // for(STXXLAddressVector::iterator it = adressVector.begin(); it != adressVector.end(); it++) { - // addressOutFile << it->node.id << "|" << it->node.lat << "|" << it->node.lon << "|" << it->city << "|" << it->street << "|" << it->housenumber << "|" << it->state << "|" << it->country << "\n"; - // } - // addressOutFile.close(); - // cout << "ok, after " << get_timestamp() - time << "s" << endl; - - } catch ( const exception& e ) { - cerr << "Caught Execption:" << e.what() << endl; - return false; - } - - double endTime = (get_timestamp() - startupTime); - INFO("Processed " << (usedNodeCounter)/(endTime) << " nodes/sec and " << usedEdgeCounter/endTime << " edges/sec"); stringMap.clear(); delete parser; delete extractCallBacks; @@ -530,11 +290,6 @@ bool nodeFunction(_Node n) { return true; } -bool adressFunction(_Node n, HashTable & keyVals){ - extractCallBacks->adressFunction(n, keyVals); - return true; -} - bool restrictionFunction(_RawRestrictionContainer r) { extractCallBacks->restrictionFunction(r); ++globalRestrictionCounter; diff --git a/profile.lua b/profile.lua new file mode 100644 index 000000000..946ba8f0c --- /dev/null +++ b/profile.lua @@ -0,0 +1,27 @@ +bollards_whitelist = { [""] = true, ["cattle_grid"] = true, ["border_control"] = true, ["toll_booth"] = true, ["no"] = true} +access_whitelist = { ["yes"] = true, ["motorcar"] = true, ["permissive"] = true } +access_blacklist = { ["no"] = true, ["private"] = true, ["agricultural"] = true, ["forestery"] = true } + +function node_function (node) + local barrier = node.tags:Find("barrier") + local access = node.tags:Find("access") + local traffic_signal = node.tags:Find("highway") + + if traffic_signal == "traffic_signals" then + node.traffic_light = true; + end + + if access_blacklist[barrier] then + node.bollard = true; + end + + if not bollards_whitelist[barrier] then + node.bollard = true; + end + return 1 +end + +function way_function (way) + + return 1 +end