diff --git a/DataStructures/BaseParser.h b/DataStructures/BaseParser.h index 76b92bac3..8856afffe 100644 --- a/DataStructures/BaseParser.h +++ b/DataStructures/BaseParser.h @@ -21,12 +21,12 @@ or see http://www.gnu.org/licenses/agpl.txt. #ifndef BASEPARSER_H_ #define BASEPARSER_H_ -template +template class BaseParser { public: virtual ~BaseParser() {} virtual bool Init() = 0; - virtual bool RegisterCallbacks(bool (*nodeCallbackPointer)(NodeT), bool (*relationCallbackPointer)(RelationT), bool (*wayCallbackPointer)(WayT), bool (*addressCallbackPointer)(NodeT, HashTable)) = 0; + virtual bool RegisterCallbacks(bool (*nodeCallbackPointer)(NodeT), bool (*restrictionCallbackPointer)(RestrictionT), bool (*wayCallbackPointer)(WayT), bool (*addressCallbackPointer)(NodeT, HashTable)) = 0; virtual bool Parse() = 0; private: }; diff --git a/DataStructures/ExtractorCallBacks.h b/DataStructures/ExtractorCallBacks.h index b72ab4a1a..a6bc281a6 100644 --- a/DataStructures/ExtractorCallBacks.h +++ b/DataStructures/ExtractorCallBacks.h @@ -30,28 +30,31 @@ typedef stxxl::vector<_Node> STXXLNodeVector; typedef stxxl::vector<_Edge> STXXLEdgeVector; typedef stxxl::vector<_Address> STXXLAddressVector; typedef stxxl::vector STXXLStringVector; +typedef stxxl::vector<_RawRestrictionContainer> STXXLRestrictionsVector; +typedef stxxl::vector<_WayIDStartAndEndEdge> STXXLWayIDStartEndVector; +struct STXXLContainers { + STXXLNodeIDVector usedNodeIDs; + STXXLNodeVector allNodes; + STXXLEdgeVector allEdges; + STXXLAddressVector adressVector; + STXXLStringVector nameVector; + STXXLRestrictionsVector restrictionsVector; + STXXLWayIDStartEndVector wayStartEndVector; +}; class ExtractorCallbacks{ private: static const unsigned MAX_LOCAL_VECTOR_SIZE = 100; - STXXLNodeVector * allNodes; - STXXLNodeIDVector * usedNodeIDs; - STXXLEdgeVector * allEdges; - STXXLStringVector * nameVector; - STXXLAddressVector * addressVector; Settings settings; StringMap * stringMap; + STXXLContainers * externalMemory; public: - ExtractorCallbacks(STXXLNodeVector * aNodes, STXXLNodeIDVector * uNodes, STXXLEdgeVector * aEdges, STXXLStringVector * nVector, STXXLAddressVector * adrVector, Settings s, StringMap * strMap){ - allNodes = aNodes; - usedNodeIDs = uNodes; - allEdges = aEdges; - nameVector = nVector; - addressVector = adrVector; - settings = s; + ExtractorCallbacks(STXXLContainers * ext, Settings set, StringMap * strMap){ + externalMemory = ext; + settings = set; stringMap = strMap; } @@ -80,12 +83,12 @@ public: /** warning: caller needs to take care of synchronization! */ bool nodeFunction(_Node &n) { - allNodes->push_back(n); + externalMemory->allNodes.push_back(n); return true; } - bool relationFunction(_Relation &r) { - //do nothing; + bool restrictionFunction(_RawRestrictionContainer &r) { + externalMemory->restrictionsVector.push_back(r); return true; } @@ -178,18 +181,17 @@ public: w.access = false; } - if ( w.useful && w.access && w.path.size() ) { + if ( w.useful && w.access && w.path.size() > 1 ) { StringMap::iterator strit = stringMap->find(w.name); - if(strit == stringMap->end()) - { - w.nameID = nameVector->size(); - nameVector->push_back(w.name); + if(strit == stringMap->end()) { + w.nameID = externalMemory->nameVector.size(); + externalMemory->nameVector.push_back(w.name); stringMap->insert(std::make_pair(w.name, w.nameID) ); } else { w.nameID = strit->second; } for ( unsigned i = 0; i < w.path.size(); ++i ) { - usedNodeIDs->push_back(w.path[i]); + externalMemory->usedNodeIDs.push_back(w.path[i]); } if ( w.direction == _Way::opposite ){ @@ -197,7 +199,7 @@ public: } vector< NodeID > & path = w.path; assert(w.type > -1 || w.maximumSpeed != -1); - assert(path.size()>0); + assert(path.size()>1); if(w.maximumSpeed == -1) w.maximumSpeed = settings.speedProfile.speed[w.type]; @@ -209,8 +211,10 @@ public: e.direction = w.direction; e.speed = w.maximumSpeed; e.nameID = w.nameID; - allEdges->push_back(e); + externalMemory->allEdges.push_back(e); } + assert(w.id != UINT_MAX); + externalMemory->wayStartEndVector.push_back(_WayIDStartAndEndEdge(w.id, path[0], path[1], path[path.size()-2], path[path.size()-1])); } return true; } diff --git a/DataStructures/ExtractorStructs.h b/DataStructures/ExtractorStructs.h index 377507e63..1c11ed764 100644 --- a/DataStructures/ExtractorStructs.h +++ b/DataStructures/ExtractorStructs.h @@ -82,7 +82,8 @@ ostream & operator<<(ostream & out, const _Coordinate & c){ } struct _Way { - _Way() { + _Way() : id(UINT_MAX) { + direction = _Way::notSure; maximumSpeed = -1; type = -1; @@ -125,9 +126,11 @@ struct _Address { }; struct _Relation { + _Relation() : type(unknown){} enum { - unknown = 0, ferry + unknown = 0, ferry, turnRestriction } type; + HashTable keyVals; }; struct _Edge { @@ -153,13 +156,107 @@ struct _Edge { }; +struct _Restriction { + NodeID viaNode; + NodeID fromNode; + NodeID toNode; + struct bits { //mostly unused + char isOnly:1; + char unused1:1; + char unused2:1; + char unused3:1; + char unused4:1; + char unused5:1; + char unused6:1; + char unused7:1; + } flags; + + + _Restriction(NodeID vn) : viaNode(vn), fromNode(UINT_MAX), toNode(UINT_MAX) { } + _Restriction(bool isOnly = false) : viaNode(UINT_MAX), fromNode(UINT_MAX), toNode(UINT_MAX) { + flags.isOnly = isOnly; + } +}; + +struct _RawRestrictionContainer { + _Restriction restriction; + EdgeID fromWay; + EdgeID toWay; + unsigned viaWay; + + _RawRestrictionContainer(EdgeID f, EdgeID t, NodeID vn, unsigned vw) : fromWay(f), toWay(t), viaWay(vw) { restriction.viaNode = vn;} + _RawRestrictionContainer(bool isOnly = false) : fromWay(UINT_MAX), toWay(UINT_MAX), viaWay(UINT_MAX) { restriction.flags.isOnly = isOnly;} + + static _RawRestrictionContainer min_value() { + return _RawRestrictionContainer(numeric_limits::min(), numeric_limits::min(), numeric_limits::min(), numeric_limits::min()); + } + static _RawRestrictionContainer max_value() { + return _RawRestrictionContainer(numeric_limits::max(), numeric_limits::max(), numeric_limits::max(), numeric_limits::max()); + } +}; + + +struct CmpRestrictionByFrom: public std::binary_function<_RawRestrictionContainer, _RawRestrictionContainer, bool> { + typedef _RawRestrictionContainer value_type; + bool operator () (const _RawRestrictionContainer & a, const _RawRestrictionContainer & b) const { + return a.fromWay < b.fromWay; + } + value_type max_value() { + return _RawRestrictionContainer::max_value(); + } + value_type min_value() { + return _RawRestrictionContainer::min_value(); + } +}; + +struct CmpRestrictionByTo: public std::binary_function<_RawRestrictionContainer, _RawRestrictionContainer, bool> { + typedef _RawRestrictionContainer value_type; + bool operator () (const _RawRestrictionContainer & a, const _RawRestrictionContainer & b) const { + return a.toWay < b.toWay; + } + value_type max_value() { + return _RawRestrictionContainer::max_value(); + } + value_type min_value() { + return _RawRestrictionContainer::min_value(); + } +}; + +struct _WayIDStartAndEndEdge { + unsigned wayID; + NodeID firstStart; + NodeID firstTarget; + NodeID lastStart; + NodeID lastTarget; + _WayIDStartAndEndEdge() : wayID(UINT_MAX), firstStart(UINT_MAX), firstTarget(UINT_MAX), lastStart(UINT_MAX), lastTarget(UINT_MAX) {} + _WayIDStartAndEndEdge(unsigned w, NodeID fs, NodeID ft, NodeID ls, NodeID lt) : wayID(w), firstStart(fs), firstTarget(ft), lastStart(ls), lastTarget(lt) {} + + static _WayIDStartAndEndEdge min_value() { + return _WayIDStartAndEndEdge(numeric_limits::min(), numeric_limits::min(), numeric_limits::min(), numeric_limits::min(), numeric_limits::min()); + } + static _WayIDStartAndEndEdge max_value() { + return _WayIDStartAndEndEdge(numeric_limits::max(), numeric_limits::max(), numeric_limits::max(), numeric_limits::max(), numeric_limits::max()); + } +}; + +struct CmpWayStartAndEnd : public std::binary_function<_WayIDStartAndEndEdge, _WayIDStartAndEndEdge, bool> { + typedef _WayIDStartAndEndEdge value_type; + bool operator () (const _WayIDStartAndEndEdge & a, const _WayIDStartAndEndEdge & b) const { + return a.wayID < b.wayID; + } + value_type max_value() { + return _WayIDStartAndEndEdge::max_value(); + } + value_type min_value() { + return _WayIDStartAndEndEdge::min_value(); + } +}; + struct Settings { struct SpeedProfile { vector< double > speed; vector< string > names; } speedProfile; - // vector accessList; - // int trafficLightPenalty; int indexInAccessListOf( const string & key) { for(unsigned i = 0; i< speedProfile.names.size(); i++) { if(speedProfile.names[i] == key) diff --git a/DataStructures/PBFParser.h b/DataStructures/PBFParser.h index 517f76521..aaf6b66d7 100644 --- a/DataStructures/PBFParser.h +++ b/DataStructures/PBFParser.h @@ -32,7 +32,7 @@ or see http://www.gnu.org/licenses/agpl.txt. #include "ExtractorStructs.h" -class PBFParser : public BaseParser<_Node, _Relation, _Way> { +class PBFParser : public BaseParser<_Node, _RawRestrictionContainer, _Way> { enum EntityType { TypeNode = 1, @@ -74,10 +74,10 @@ public: groupCount = 0; } - bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*relationCallbackPointer)(_Relation), bool (*wayCallbackPointer)(_Way),bool (*addressCallbackPointer)(_Node, HashTable) ) { + bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way),bool (*addressCallbackPointer)(_Node, HashTable) ) { nodeCallback = *nodeCallbackPointer; wayCallback = *wayCallbackPointer; - relationCallback = *relationCallbackPointer; + restrictionCallback = *restrictionCallbackPointer; addressCallback = *addressCallbackPointer; return true; } @@ -226,12 +226,76 @@ private: void parseRelation(_ThreadData * threadData) { const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ); for(int i = 0; i < group.relations_size(); i++ ) { - _Relation r; - r.type = _Relation::unknown; + const OSMPBF::Relation& inputRelation = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).relations(i); + bool isRestriction = false; + bool isOnlyRestriction = false; + for(int k = 0; k < inputRelation.keys_size(); k++) { + const std::string key = threadData->PBFprimitiveBlock.stringtable().s(inputRelation.keys(k)); + const std::string val = threadData->PBFprimitiveBlock.stringtable().s(inputRelation.vals(k)); + if ("type" == key && "restriction" == val) { + isRestriction = true; + } + if ("restriction" == key) { + if(val.find("only_") == 0) + isOnlyRestriction = true; + } + + } + if(isRestriction) { + long long lastRef = 0; + _RawRestrictionContainer currentRestrictionContainer(isOnlyRestriction); + for(int rolesIndex = 0; rolesIndex < inputRelation.roles_sid_size(); rolesIndex++) { + string role(threadData->PBFprimitiveBlock.stringtable().s( inputRelation.roles_sid( rolesIndex ) ).data()); + lastRef += inputRelation.memids(rolesIndex); + + if(false == ("from" == role || "to" == role || "via" == role)) { + continue; + } + + switch(inputRelation.types(rolesIndex)) { + case 0: //node + if("from" == role || "to" == role) //Only via should be a node + continue; + assert("via" == role); + if(UINT_MAX != currentRestrictionContainer.viaWay) + currentRestrictionContainer.viaWay = UINT_MAX; + assert(UINT_MAX == currentRestrictionContainer.viaWay); + currentRestrictionContainer.restriction.viaNode = lastRef; + break; + case 1: //way + assert("from" == role || "to" == role || "via" == role); + if("from" == role) { + currentRestrictionContainer.fromWay = lastRef; + } + if ("to" == role) { + currentRestrictionContainer.toWay = lastRef; + } + if ("via" == role) { + assert(currentRestrictionContainer.restriction.toNode == UINT_MAX); + currentRestrictionContainer.viaWay = lastRef; + } + break; + case 2: //relation, not used. relations relating to relations are evil. + continue; + assert(false); + break; + + default: //should not happen + cout << "unknown"; + assert(false); + break; + } + } +// if(UINT_MAX != currentRestriction.viaNode) { +// cout << "restr from " << currentRestriction.from << " via "; +// cout << "node " << currentRestriction.viaNode; +// cout << " to " << currentRestriction.to << endl; +// } #pragma omp critical - { - if(!(*relationCallback)(r)) - std::cerr << "[PBFParser] relation not parsed" << std::endl; + { + if(!(*restrictionCallback)(currentRestrictionContainer)) + std::cerr << "[PBFParser] relation not parsed" << std::endl; + } } } } @@ -442,7 +506,7 @@ private: /* Function pointer for nodes */ bool (*nodeCallback)(_Node); bool (*wayCallback)(_Way); - bool (*relationCallback)(_Relation); + bool (*restrictionCallback)(_RawRestrictionContainer); bool (*addressCallback)(_Node, HashTable); /* the input stream to parse */ std::fstream input; diff --git a/DataStructures/XMLParser.h b/DataStructures/XMLParser.h index 156fe322e..a65e4edf9 100644 --- a/DataStructures/XMLParser.h +++ b/DataStructures/XMLParser.h @@ -29,17 +29,18 @@ or see http://www.gnu.org/licenses/agpl.txt. #include "ExtractorStructs.h" #include "InputReaderFactory.h" -class XMLParser : public BaseParser<_Node, _Relation, _Way> { +class XMLParser : public BaseParser<_Node, _RawRestrictionContainer, _Way> { public: XMLParser(const char * filename) { + WARN("Parsing plain .osm/.osm.bz2 is deprecated. Switch to .pbf"); inputReader = inputReaderFactory(filename); } ~XMLParser() {} - bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*relationCallbackPointer)(_Relation), bool (*wayCallbackPointer)(_Way), bool (*addressCallbackPointer)(_Node, HashTable) ) { + bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way), bool (*addressCallbackPointer)(_Node, HashTable) ) { nodeCallback = *nodeCallbackPointer; wayCallback = *wayCallbackPointer; - relationCallback = *relationCallbackPointer; + restrictionCallback = *restrictionCallbackPointer; return true; } bool Init() { @@ -73,8 +74,7 @@ public: if ( xmlStrEqual( currentName, ( const xmlChar* ) "relation" ) == 1 ) { _Relation r; r.type = _Relation::unknown; - if(!(*relationCallback)(r)) - std::cerr << "[XMLParser] relation not parsed" << std::endl; + //todo: parse relation } xmlFree( currentName ); } @@ -210,7 +210,7 @@ private: /* Function pointer for nodes */ bool (*nodeCallback)(_Node); bool (*wayCallback)(_Way); - bool (*relationCallback)(_Relation); + bool (*restrictionCallback)(_RawRestrictionContainer); }; diff --git a/extractor.cpp b/extractor.cpp index 4767b559a..fb0ca5fba 100644 --- a/extractor.cpp +++ b/extractor.cpp @@ -50,12 +50,12 @@ or see http://www.gnu.org/licenses/agpl.txt. typedef BaseConfiguration ExtractorConfiguration; -unsigned globalRelationCounter = 0; +unsigned globalRestrictionCounter = 0; ExtractorCallbacks * extractCallBacks; bool nodeFunction(_Node n); -bool adressFunction(_Node n, HashTable keyVals); -bool relationFunction(_Relation r); +bool adressFunction(_Node n, HashTable keyVals); +bool restrictionFunction(_RawRestrictionContainer r); bool wayFunction(_Way w); template @@ -63,14 +63,15 @@ bool removeIfUnused(ClassT n) { return (false == n.used); } int main (int argc, char *argv[]) { if(argc <= 1) { - std::cerr << "usage: " << endl << argv[0] << " " << std::endl; + cerr << "usage: " << endl << argv[0] << " " << endl; exit(-1); } - std::cout << "[extractor] extracting data from input file " << argv[1] << std::endl; + cout << "[extractor] extracting data from input file " << argv[1] << endl; bool isPBF = false; - std::string outputFileName(argv[1]); - std::string::size_type pos = outputFileName.find(".osm.bz2"); + string outputFileName(argv[1]); + string restrictionsFileName(argv[1]); + string::size_type pos = outputFileName.find(".osm.bz2"); if(pos==string::npos) { pos = outputFileName.find(".osm.pbf"); if(pos!=string::npos) { @@ -79,36 +80,34 @@ int main (int argc, char *argv[]) { } if(pos!=string::npos) { outputFileName.replace(pos, 8, ".osrm"); + restrictionsFileName.replace(pos, 8, ".osrm.restrictions"); } else { pos=outputFileName.find(".osm"); if(pos!=string::npos) { outputFileName.replace(pos, 5, ".osrm"); + restrictionsFileName.replace(pos, 5, ".osrm.restrictions"); } else { outputFileName.append(".osrm"); + restrictionsFileName.append(".osrm.restrictions"); } } - std::string adressFileName(outputFileName); - - + string adressFileName(outputFileName); unsigned amountOfRAM = 1; unsigned installedRAM = GetPhysicalmemory(); - if(installedRAM < 2048264) { - std::cout << "[Warning] Machine has less than 2GB RAM." << std::endl; + if(installedRAM < 2048264) { + cout << "[Warning] Machine has less than 2GB RAM." << endl; } if(testDataFile("extractor.ini")) { ExtractorConfiguration extractorConfig("extractor.ini"); unsigned memoryAmountFromFile = atoi(extractorConfig.GetParameter("Memory").c_str()); if( memoryAmountFromFile != 0 && memoryAmountFromFile <= installedRAM/(1024*1024*1024)) amountOfRAM = memoryAmountFromFile; - std::cout << "[extractor] using " << amountOfRAM << " GB of RAM for buffers" << std::endl; + cout << "[extractor] using " << amountOfRAM << " GB of RAM for buffers" << endl; } - STXXLNodeIDVector usedNodeIDs; - STXXLNodeVector allNodes; - STXXLEdgeVector allEdges; - STXXLAddressVector adressVector; - STXXLStringVector nameVector; + STXXLContainers externalMemory; + unsigned usedNodeCounter = 0; unsigned usedEdgeCounter = 0; @@ -120,60 +119,148 @@ int main (int argc, char *argv[]) { double time = get_timestamp(); stringMap->set_empty_key(GetRandomString()); - stringMap->insert(std::make_pair("", 0)); - extractCallBacks = new ExtractorCallbacks(&allNodes, &usedNodeIDs, &allEdges, &nameVector, &adressVector, settings, stringMap); + stringMap->insert(make_pair("", 0)); + extractCallBacks = new ExtractorCallbacks(&externalMemory, settings, stringMap); - BaseParser<_Node, _Relation, _Way> * parser; + BaseParser<_Node, _RawRestrictionContainer, _Way> * parser; if(isPBF) { parser = new PBFParser(argv[1]); } else { parser = new XMLParser(argv[1]); } - parser->RegisterCallbacks(&nodeFunction, &relationFunction, &wayFunction, &adressFunction); + parser->RegisterCallbacks(&nodeFunction, &restrictionFunction, &wayFunction, &adressFunction); if(parser->Init()) { parser->Parse(); } else { - std::cerr << "[error] parser not initialized!" << std::endl; + cerr << "[error] parser not initialized!" << endl; exit(-1); } delete parser; try { - // std::cout << "[info] raw no. of names: " << nameVector.size() << std::endl; - // std::cout << "[info] raw no. of nodes: " << allNodes.size() << std::endl; - // std::cout << "[info] no. of used nodes: " << usedNodeIDs.size() << std::endl; - // std::cout << "[info] raw no. of edges: " << allEdges.size() << std::endl; - // std::cout << "[info] raw no. of relations: " << globalRelationCounter << std::endl; - // std::cout << "[info] raw no. of addresses: " << adressVector.size() << std::endl; +// INFO("raw no. of names: " << externalMemory.nameVector.size()); +// INFO("raw no. of nodes: " << externalMemory.allNodes.size()); +// INFO("no. of used nodes: " << externalMemory.usedNodeIDs.size()); +// INFO("raw no. of edges: " << externalMemory.allEdges.size()); +// INFO("raw no. of ways: " << externalMemory.wayStartEndVector.size()); +// INFO("raw no. of addresses: " << externalMemory.adressVector.size()); +// INFO("raw no. of restrictions: " << externalMemory.restrictionsVector.size()); - std::cout << "[extractor] parsing finished after " << get_timestamp() - time << "seconds" << std::endl; + cout << "[extractor] parsing finished after " << get_timestamp() - time << "seconds" << endl; time = get_timestamp(); unsigned memory_to_use = amountOfRAM * 1024 * 1024 * 1024; - std::cout << "[extractor] Sorting used nodes ... " << std::flush; - stxxl::sort(usedNodeIDs.begin(), usedNodeIDs.end(), Cmp(), memory_to_use); - std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; + cout << "[extractor] Sorting used nodes ... " << flush; + stxxl::sort(externalMemory.usedNodeIDs.begin(), externalMemory.usedNodeIDs.end(), Cmp(), memory_to_use); + cout << "ok, after " << get_timestamp() - time << "s" << endl; time = get_timestamp(); - std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush; - stxxl::vector::iterator NewEnd = unique ( usedNodeIDs.begin(),usedNodeIDs.end() ) ; - usedNodeIDs.resize ( NewEnd - usedNodeIDs.begin() ); + cout << "[extractor] Erasing duplicate nodes ... " << flush; + stxxl::vector::iterator NewEnd = unique ( externalMemory.usedNodeIDs.begin(),externalMemory.usedNodeIDs.end() ) ; + externalMemory.usedNodeIDs.resize ( NewEnd - externalMemory.usedNodeIDs.begin() ); cout << "ok, after " << get_timestamp() - time << "s" << endl; time = get_timestamp(); - std::cout << "[extractor] Sorting all nodes ... " << std::flush; - stxxl::sort(allNodes.begin(), allNodes.end(), CmpNodeByID(), memory_to_use); - std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; + cout << "[extractor] Sorting all nodes ... " << flush; + stxxl::sort(externalMemory.allNodes.begin(), externalMemory.allNodes.end(), CmpNodeByID(), memory_to_use); + cout << "ok, after " << get_timestamp() - time << "s" << endl; time = get_timestamp(); - std::ofstream fout; - fout.open(outputFileName.c_str(), std::ios::binary); - fout.write((char*)&usedNodeCounter, sizeof(unsigned)); + cout << "[extractor] Sorting used ways ... " << flush; + stxxl::sort(externalMemory.wayStartEndVector.begin(), externalMemory.wayStartEndVector.end(), CmpWayStartAndEnd(), memory_to_use); + cout << "ok, after " << get_timestamp() - time << "s" << endl; - std::cout << "[extractor] Confirming used nodes ... " << std::flush; - STXXLNodeVector::iterator nodesIT = allNodes.begin(); - STXXLNodeIDVector::iterator usedNodeIDsIT = usedNodeIDs.begin(); - while(usedNodeIDsIT != usedNodeIDs.end() && nodesIT != allNodes.end()) { + cout << "[extractor] Sorting restrctns. by from... " << flush; + stxxl::sort(externalMemory.restrictionsVector.begin(), externalMemory.restrictionsVector.end(), CmpRestrictionByFrom(), memory_to_use); + cout << "ok, after " << get_timestamp() - time << "s" << endl; + + cout << "[extractor] Fixing restriction starts ... " << flush; + STXXLRestrictionsVector::iterator restrictionsIT = externalMemory.restrictionsVector.begin(); + STXXLWayIDStartEndVector::iterator wayStartAndEndEdgeIT = externalMemory.wayStartEndVector.begin(); + + while(wayStartAndEndEdgeIT != externalMemory.wayStartEndVector.end() && restrictionsIT != externalMemory.restrictionsVector.end()) { + if(wayStartAndEndEdgeIT->wayID < restrictionsIT->fromWay){ + wayStartAndEndEdgeIT++; + continue; + } + if(wayStartAndEndEdgeIT->wayID > restrictionsIT->fromWay) { + restrictionsIT++; + continue; + } + assert(wayStartAndEndEdgeIT->wayID == restrictionsIT->fromWay); + NodeID viaNode = restrictionsIT->restriction.viaNode; + + if(wayStartAndEndEdgeIT->firstStart == viaNode) { + restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstTarget; + } else if(wayStartAndEndEdgeIT->firstTarget == viaNode) { + restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstStart; + } else if(wayStartAndEndEdgeIT->lastStart == viaNode) { + restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastTarget; + } else if(wayStartAndEndEdgeIT->lastTarget == viaNode) { + restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastStart; + } + restrictionsIT++; + } + + cout << "ok, after " << get_timestamp() - time << "s" << endl; + time = get_timestamp(); + + cout << "[extractor] Sorting restrctns. by to ... " << flush; + stxxl::sort(externalMemory.restrictionsVector.begin(), externalMemory.restrictionsVector.end(), CmpRestrictionByTo(), memory_to_use); + cout << "ok, after " << get_timestamp() - time << "s" << endl; + + time = get_timestamp(); + unsigned usableRestrictionsCounter(0); + cout << "[extractor] Fixing restriction ends ... " << flush; + restrictionsIT = externalMemory.restrictionsVector.begin(); + wayStartAndEndEdgeIT = externalMemory.wayStartEndVector.begin(); + while(wayStartAndEndEdgeIT != externalMemory.wayStartEndVector.end() && + restrictionsIT != externalMemory.restrictionsVector.end()) { + if(wayStartAndEndEdgeIT->wayID < restrictionsIT->toWay){ + wayStartAndEndEdgeIT++; + continue; + } + if(wayStartAndEndEdgeIT->wayID > restrictionsIT->toWay) { + restrictionsIT++; + continue; + } + NodeID viaNode = restrictionsIT->restriction.viaNode; + if(wayStartAndEndEdgeIT->lastStart == viaNode) { + restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastTarget; + } else if(wayStartAndEndEdgeIT->lastTarget == viaNode) { + restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastStart; + } else if(wayStartAndEndEdgeIT->firstStart == viaNode) { + restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstTarget; + } else if(wayStartAndEndEdgeIT->firstTarget == viaNode) { + restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstStart; + } + + if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) { + usableRestrictionsCounter++; + } + restrictionsIT++; + } + + cout << "ok, after " << get_timestamp() - time << "s" << endl; + //todo: serialize restrictions + ofstream restrictionsOutstream; + restrictionsOutstream.open(restrictionsFileName.c_str(), ios::binary); + restrictionsOutstream.write((char*)&usableRestrictionsCounter, sizeof(unsigned)); + for(restrictionsIT = externalMemory.restrictionsVector.begin(); restrictionsIT != externalMemory.restrictionsVector.end(); restrictionsIT++) { + if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) { + restrictionsOutstream.write((char *)&(restrictionsIT->restriction), sizeof(_Restriction)); + } + } + restrictionsOutstream.close(); + + ofstream fout; + fout.open(outputFileName.c_str(), ios::binary); + fout.write((char*)&usedNodeCounter, sizeof(unsigned)); + time = get_timestamp(); + cout << "[extractor] Confirming used nodes ... " << flush; + STXXLNodeVector::iterator nodesIT = externalMemory.allNodes.begin(); + STXXLNodeIDVector::iterator usedNodeIDsIT = externalMemory.usedNodeIDs.begin(); + while(usedNodeIDsIT != externalMemory.usedNodeIDs.end() && nodesIT != externalMemory.allNodes.end()) { if(*usedNodeIDsIT < nodesIT->id){ usedNodeIDsIT++; continue; @@ -191,30 +278,31 @@ int main (int argc, char *argv[]) { nodesIT++; } } - std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; + + cout << "ok, after " << get_timestamp() - time << "s" << endl; time = get_timestamp(); - std::cout << "[extractor] setting number of nodes ... " << std::flush; - std::ios::pos_type positionInFile = fout.tellp(); - fout.seekp(std::ios::beg); + cout << "[extractor] setting number of nodes ... " << flush; + ios::pos_type positionInFile = fout.tellp(); + fout.seekp(ios::beg); fout.write((char*)&usedNodeCounter, sizeof(unsigned)); fout.seekp(positionInFile); - std::cout << "ok" << std::endl; + cout << "ok" << endl; time = get_timestamp(); // Sort edges by start. - std::cout << "[extractor] Sorting edges by start ... " << std::flush; - stxxl::sort(allEdges.begin(), allEdges.end(), CmpEdgeByStartID(), memory_to_use); - std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; + cout << "[extractor] Sorting edges by start ... " << flush; + stxxl::sort(externalMemory.allEdges.begin(), externalMemory.allEdges.end(), CmpEdgeByStartID(), memory_to_use); + cout << "ok, after " << get_timestamp() - time << "s" << endl; time = get_timestamp(); - std::cout << "[extractor] Setting start coords ... " << std::flush; + cout << "[extractor] Setting start coords ... " << flush; fout.write((char*)&usedEdgeCounter, sizeof(unsigned)); // Traverse list of edges and nodes in parallel and set start coord - nodesIT = allNodes.begin(); - STXXLEdgeVector::iterator edgeIT = allEdges.begin(); - while(edgeIT != allEdges.end() && nodesIT != allNodes.end()) { + nodesIT = externalMemory.allNodes.begin(); + STXXLEdgeVector::iterator edgeIT = externalMemory.allEdges.begin(); + while(edgeIT != externalMemory.allEdges.end() && nodesIT != externalMemory.allNodes.end()) { if(edgeIT->start < nodesIT->id){ edgeIT++; continue; @@ -229,20 +317,20 @@ int main (int argc, char *argv[]) { edgeIT++; } } - std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; + cout << "ok, after " << get_timestamp() - time << "s" << endl; time = get_timestamp(); // Sort Edges by target - std::cout << "[extractor] Sorting edges by target ... " << std::flush; - stxxl::sort(allEdges.begin(), allEdges.end(), CmpEdgeByTargetID(), memory_to_use); - std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; + cout << "[extractor] Sorting edges by target ... " << flush; + stxxl::sort(externalMemory.allEdges.begin(), externalMemory.allEdges.end(), CmpEdgeByTargetID(), memory_to_use); + cout << "ok, after " << get_timestamp() - time << "s" << endl; time = get_timestamp(); - std::cout << "[extractor] Setting target coords ... " << std::flush; + cout << "[extractor] Setting target coords ... " << flush; // Traverse list of edges and nodes in parallel and set target coord - nodesIT = allNodes.begin(); - edgeIT = allEdges.begin(); - while(edgeIT != allEdges.end() && nodesIT != allNodes.end()) { + nodesIT = externalMemory.allNodes.begin(); + edgeIT = externalMemory.allEdges.begin(); + while(edgeIT != externalMemory.allEdges.end() && nodesIT != externalMemory.allNodes.end()) { if(edgeIT->target < nodesIT->id){ edgeIT++; continue; @@ -285,7 +373,7 @@ int main (int argc, char *argv[]) { fout.write((char*)&one, sizeof(short)); break; default: - std::cerr << "[error] edge with no direction: " << edgeIT->direction << std::endl; + cerr << "[error] edge with no direction: " << edgeIT->direction << endl; assert(false); break; } @@ -298,25 +386,25 @@ int main (int argc, char *argv[]) { edgeIT++; } } - std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; + cout << "ok, after " << get_timestamp() - time << "s" << endl; time = get_timestamp(); - std::cout << "[extractor] setting number of edges ... " << std::flush; + cout << "[extractor] setting number of edges ... " << flush; fout.seekp(positionInFile); fout.write((char*)&usedEdgeCounter, sizeof(unsigned)); fout.close(); - std::cout << "ok" << std::endl; + cout << "ok" << endl; time = get_timestamp(); - std::cout << "[extractor] writing street name index ... " << std::flush; - std::vector * nameIndex = new std::vector(nameVector.size()+1, 0); + cout << "[extractor] writing street name index ... " << flush; + vector * nameIndex = new vector(externalMemory.nameVector.size()+1, 0); outputFileName.append(".names"); - std::ofstream nameOutFile(outputFileName.c_str(), std::ios::binary); + ofstream nameOutFile(outputFileName.c_str(), ios::binary); unsigned sizeOfNameIndex = nameIndex->size(); nameOutFile.write((char *)&(sizeOfNameIndex), sizeof(unsigned)); - for(STXXLStringVector::iterator it = nameVector.begin(); it != nameVector.end(); it++) { + for(STXXLStringVector::iterator it = externalMemory.nameVector.begin(); it != externalMemory.nameVector.end(); it++) { unsigned lengthOfRawString = strlen(it->c_str()); nameOutFile.write((char *)&(lengthOfRawString), sizeof(unsigned)); nameOutFile.write(it->c_str(), lengthOfRawString); @@ -324,26 +412,26 @@ int main (int argc, char *argv[]) { nameOutFile.close(); delete nameIndex; - std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; + cout << "ok, after " << get_timestamp() - time << "s" << endl; // time = get_timestamp(); - // std::cout << "[extractor] writing address list ... " << std::flush; + // cout << "[extractor] writing address list ... " << flush; // // adressFileName.append(".address"); - // std::ofstream addressOutFile(adressFileName.c_str()); + // ofstream addressOutFile(adressFileName.c_str()); // for(STXXLAddressVector::iterator it = adressVector.begin(); it != adressVector.end(); it++) { // addressOutFile << it->node.id << "|" << it->node.lat << "|" << it->node.lon << "|" << it->city << "|" << it->street << "|" << it->housenumber << "|" << it->state << "|" << it->country << "\n"; // } // addressOutFile.close(); - // std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; + // cout << "ok, after " << get_timestamp() - time << "s" << endl; - } catch ( const std::exception& e ) { - std::cerr << "Caught Execption:" << e.what() << std::endl; + } catch ( const exception& e ) { + cerr << "Caught Execption:" << e.what() << endl; return false; } delete extractCallBacks; - std::cout << "[extractor] finished." << std::endl; + cout << "[extractor] finished." << endl; return 0; } @@ -352,13 +440,14 @@ bool nodeFunction(_Node n) { return true; } -bool adressFunction(_Node n, HashTable keyVals){ +bool adressFunction(_Node n, HashTable keyVals){ extractCallBacks->adressFunction(n, keyVals); return true; } -bool relationFunction(_Relation r) { - globalRelationCounter++; +bool restrictionFunction(_RawRestrictionContainer r) { + extractCallBacks->restrictionFunction(r); + globalRestrictionCounter++; return true; } bool wayFunction(_Way w) {