Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Extracting additional features from graph

Browse Source
This commit is contained in:
Dennis Luxen 2011-07-06 12:33:41 +00:00
parent 4883c6c197
commit a37528362d
6 changed files with 382 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
DataStructures/BaseParser.h
View File

@ -21,12 +21,12 @@ or see http://www.gnu.org/licenses/agpl.txt.
#ifndef BASEPARSER_H_ #ifndef BASEPARSER_H_
#define BASEPARSER_H_ #define BASEPARSER_H_
template<typename NodeT, typename RelationT, typename WayT> template<typename NodeT, typename RestrictionT, typename WayT>
class BaseParser { class BaseParser {
public: public:
virtual ~BaseParser() {} virtual ~BaseParser() {}
virtual bool Init() = 0; virtual bool Init() = 0;
virtual bool RegisterCallbacks(bool (*nodeCallbackPointer)(NodeT), bool (*relationCallbackPointer)(RelationT), bool (*wayCallbackPointer)(WayT), bool (*addressCallbackPointer)(NodeT, HashTable<std::string, std::string>)) = 0; virtual bool RegisterCallbacks(bool (*nodeCallbackPointer)(NodeT), bool (*restrictionCallbackPointer)(RestrictionT), bool (*wayCallbackPointer)(WayT), bool (*addressCallbackPointer)(NodeT, HashTable<std::string, std::string>)) = 0;
virtual bool Parse() = 0; virtual bool Parse() = 0;
private: private:
}; };

50
DataStructures/ExtractorCallBacks.h
View File

@ -30,28 +30,31 @@ typedef stxxl::vector<_Node> STXXLNodeVector;
typedef stxxl::vector<_Edge> STXXLEdgeVector; typedef stxxl::vector<_Edge> STXXLEdgeVector;
typedef stxxl::vector<_Address> STXXLAddressVector; typedef stxxl::vector<_Address> STXXLAddressVector;
typedef stxxl::vector<string> STXXLStringVector; typedef stxxl::vector<string> STXXLStringVector;
typedef stxxl::vector<_RawRestrictionContainer> STXXLRestrictionsVector;
typedef stxxl::vector<_WayIDStartAndEndEdge> STXXLWayIDStartEndVector;
struct STXXLContainers {
STXXLNodeIDVector usedNodeIDs;
STXXLNodeVector allNodes;
STXXLEdgeVector allEdges;
STXXLAddressVector adressVector;
STXXLStringVector nameVector;
STXXLRestrictionsVector restrictionsVector;
STXXLWayIDStartEndVector wayStartEndVector;
};
class ExtractorCallbacks{ class ExtractorCallbacks{
private: private:
static const unsigned MAX_LOCAL_VECTOR_SIZE = 100; static const unsigned MAX_LOCAL_VECTOR_SIZE = 100;
STXXLNodeVector * allNodes;
STXXLNodeIDVector * usedNodeIDs;
STXXLEdgeVector * allEdges;
STXXLStringVector * nameVector;
STXXLAddressVector * addressVector;
Settings settings; Settings settings;
StringMap * stringMap; StringMap * stringMap;
STXXLContainers * externalMemory;
public: public:
ExtractorCallbacks(STXXLNodeVector * aNodes, STXXLNodeIDVector * uNodes, STXXLEdgeVector * aEdges, STXXLStringVector * nVector, STXXLAddressVector * adrVector, Settings s, StringMap * strMap){ ExtractorCallbacks(STXXLContainers * ext, Settings set, StringMap * strMap){
allNodes = aNodes; externalMemory = ext;
usedNodeIDs = uNodes; settings = set;
allEdges = aEdges;
nameVector = nVector;
addressVector = adrVector;
settings = s;
stringMap = strMap; stringMap = strMap;
} }
@ -80,12 +83,12 @@ public:
/** warning: caller needs to take care of synchronization! */ /** warning: caller needs to take care of synchronization! */
bool nodeFunction(_Node &n) { bool nodeFunction(_Node &n) {
allNodes->push_back(n); externalMemory->allNodes.push_back(n);
return true; return true;
} }
bool relationFunction(_Relation &r) { bool restrictionFunction(_RawRestrictionContainer &r) {
//do nothing; externalMemory->restrictionsVector.push_back(r);
return true; return true;
} }
@ -178,18 +181,17 @@ public:
w.access = false; w.access = false;
} }
if ( w.useful && w.access && w.path.size() ) { if ( w.useful && w.access && w.path.size() > 1 ) {
StringMap::iterator strit = stringMap->find(w.name); StringMap::iterator strit = stringMap->find(w.name);
if(strit == stringMap->end()) if(strit == stringMap->end()) {
{ w.nameID = externalMemory->nameVector.size();
w.nameID = nameVector->size(); externalMemory->nameVector.push_back(w.name);
nameVector->push_back(w.name);
stringMap->insert(std::make_pair(w.name, w.nameID) ); stringMap->insert(std::make_pair(w.name, w.nameID) );
} else { } else {
w.nameID = strit->second; w.nameID = strit->second;
} }
for ( unsigned i = 0; i < w.path.size(); ++i ) { for ( unsigned i = 0; i < w.path.size(); ++i ) {
usedNodeIDs->push_back(w.path[i]); externalMemory->usedNodeIDs.push_back(w.path[i]);
} }
if ( w.direction == _Way::opposite ){ if ( w.direction == _Way::opposite ){
@ -197,7 +199,7 @@ public:
} }
vector< NodeID > & path = w.path; vector< NodeID > & path = w.path;
assert(w.type > -1 || w.maximumSpeed != -1); assert(w.type > -1 || w.maximumSpeed != -1);
assert(path.size()>0); assert(path.size()>1);
if(w.maximumSpeed == -1) if(w.maximumSpeed == -1)
w.maximumSpeed = settings.speedProfile.speed[w.type]; w.maximumSpeed = settings.speedProfile.speed[w.type];
@ -209,8 +211,10 @@ public:
e.direction = w.direction; e.direction = w.direction;
e.speed = w.maximumSpeed; e.speed = w.maximumSpeed;
e.nameID = w.nameID; e.nameID = w.nameID;
allEdges->push_back(e); externalMemory->allEdges.push_back(e);
} }
assert(w.id != UINT_MAX);
externalMemory->wayStartEndVector.push_back(_WayIDStartAndEndEdge(w.id, path[0], path[1], path[path.size()-2], path[path.size()-1]));
} }
return true; return true;
} }

105
DataStructures/ExtractorStructs.h
View File

@ -82,7 +82,8 @@ ostream & operator<<(ostream & out, const _Coordinate & c){
} }
struct _Way { struct _Way {
_Way() { _Way() : id(UINT_MAX) {
direction = _Way::notSure; direction = _Way::notSure;
maximumSpeed = -1; maximumSpeed = -1;
type = -1; type = -1;
@ -125,9 +126,11 @@ struct _Address {
}; };
struct _Relation { struct _Relation {
_Relation() : type(unknown){}
enum { enum {
unknown = 0, ferry unknown = 0, ferry, turnRestriction
} type; } type;
HashTable<std::string, std::string> keyVals;
}; };
struct _Edge { struct _Edge {
@ -153,13 +156,107 @@ struct _Edge {
}; };
struct _Restriction {
NodeID viaNode;
NodeID fromNode;
NodeID toNode;
struct bits { //mostly unused
char isOnly:1;
char unused1:1;
char unused2:1;
char unused3:1;
char unused4:1;
char unused5:1;
char unused6:1;
char unused7:1;
} flags;
_Restriction(NodeID vn) : viaNode(vn), fromNode(UINT_MAX), toNode(UINT_MAX) { }
_Restriction(bool isOnly = false) : viaNode(UINT_MAX), fromNode(UINT_MAX), toNode(UINT_MAX) {
flags.isOnly = isOnly;
}
};
struct _RawRestrictionContainer {
_Restriction restriction;
EdgeID fromWay;
EdgeID toWay;
unsigned viaWay;
_RawRestrictionContainer(EdgeID f, EdgeID t, NodeID vn, unsigned vw) : fromWay(f), toWay(t), viaWay(vw) { restriction.viaNode = vn;}
_RawRestrictionContainer(bool isOnly = false) : fromWay(UINT_MAX), toWay(UINT_MAX), viaWay(UINT_MAX) { restriction.flags.isOnly = isOnly;}
static _RawRestrictionContainer min_value() {
return _RawRestrictionContainer(numeric_limits<unsigned>::min(), numeric_limits<unsigned>::min(), numeric_limits<unsigned>::min(), numeric_limits<unsigned>::min());
}
static _RawRestrictionContainer max_value() {
return _RawRestrictionContainer(numeric_limits<unsigned>::max(), numeric_limits<unsigned>::max(), numeric_limits<unsigned>::max(), numeric_limits<unsigned>::max());
}
};
struct CmpRestrictionByFrom: public std::binary_function<_RawRestrictionContainer, _RawRestrictionContainer, bool> {
typedef _RawRestrictionContainer value_type;
bool operator () (const _RawRestrictionContainer & a, const _RawRestrictionContainer & b) const {
return a.fromWay < b.fromWay;
}
value_type max_value() {
return _RawRestrictionContainer::max_value();
}
value_type min_value() {
return _RawRestrictionContainer::min_value();
}
};
struct CmpRestrictionByTo: public std::binary_function<_RawRestrictionContainer, _RawRestrictionContainer, bool> {
typedef _RawRestrictionContainer value_type;
bool operator () (const _RawRestrictionContainer & a, const _RawRestrictionContainer & b) const {
return a.toWay < b.toWay;
}
value_type max_value() {
return _RawRestrictionContainer::max_value();
}
value_type min_value() {
return _RawRestrictionContainer::min_value();
}
};
struct _WayIDStartAndEndEdge {
unsigned wayID;
NodeID firstStart;
NodeID firstTarget;
NodeID lastStart;
NodeID lastTarget;
_WayIDStartAndEndEdge() : wayID(UINT_MAX), firstStart(UINT_MAX), firstTarget(UINT_MAX), lastStart(UINT_MAX), lastTarget(UINT_MAX) {}
_WayIDStartAndEndEdge(unsigned w, NodeID fs, NodeID ft, NodeID ls, NodeID lt) : wayID(w), firstStart(fs), firstTarget(ft), lastStart(ls), lastTarget(lt) {}
static _WayIDStartAndEndEdge min_value() {
return _WayIDStartAndEndEdge(numeric_limits<unsigned>::min(), numeric_limits<unsigned>::min(), numeric_limits<unsigned>::min(), numeric_limits<unsigned>::min(), numeric_limits<unsigned>::min());
}
static _WayIDStartAndEndEdge max_value() {
return _WayIDStartAndEndEdge(numeric_limits<unsigned>::max(), numeric_limits<unsigned>::max(), numeric_limits<unsigned>::max(), numeric_limits<unsigned>::max(), numeric_limits<unsigned>::max());
}
};
struct CmpWayStartAndEnd : public std::binary_function<_WayIDStartAndEndEdge, _WayIDStartAndEndEdge, bool> {
typedef _WayIDStartAndEndEdge value_type;
bool operator () (const _WayIDStartAndEndEdge & a, const _WayIDStartAndEndEdge & b) const {
return a.wayID < b.wayID;
}
value_type max_value() {
return _WayIDStartAndEndEdge::max_value();
}
value_type min_value() {
return _WayIDStartAndEndEdge::min_value();
}
};
struct Settings { struct Settings {
struct SpeedProfile { struct SpeedProfile {
vector< double > speed; vector< double > speed;
vector< string > names; vector< string > names;
} speedProfile; } speedProfile;
// vector<string> accessList;
// int trafficLightPenalty;
int indexInAccessListOf( const string & key) { int indexInAccessListOf( const string & key) {
for(unsigned i = 0; i< speedProfile.names.size(); i++) { for(unsigned i = 0; i< speedProfile.names.size(); i++) {
if(speedProfile.names[i] == key) if(speedProfile.names[i] == key)

82
DataStructures/PBFParser.h
View File

@ -32,7 +32,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "ExtractorStructs.h" #include "ExtractorStructs.h"
class PBFParser : public BaseParser<_Node, _Relation, _Way> { class PBFParser : public BaseParser<_Node, _RawRestrictionContainer, _Way> {
enum EntityType { enum EntityType {
TypeNode = 1, TypeNode = 1,
@ -74,10 +74,10 @@ public:
groupCount = 0; groupCount = 0;
} }
bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*relationCallbackPointer)(_Relation), bool (*wayCallbackPointer)(_Way),bool (*addressCallbackPointer)(_Node, HashTable<std::string, std::string>) ) { bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way),bool (*addressCallbackPointer)(_Node, HashTable<std::string, std::string>) ) {
nodeCallback = *nodeCallbackPointer; nodeCallback = *nodeCallbackPointer;
wayCallback = *wayCallbackPointer; wayCallback = *wayCallbackPointer;
relationCallback = *relationCallbackPointer; restrictionCallback = *restrictionCallbackPointer;
addressCallback = *addressCallbackPointer; addressCallback = *addressCallbackPointer;
return true; return true;
} }
@ -226,12 +226,76 @@ private:
void parseRelation(_ThreadData * threadData) { void parseRelation(_ThreadData * threadData) {
const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ); const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID );
for(int i = 0; i < group.relations_size(); i++ ) { for(int i = 0; i < group.relations_size(); i++ ) {
_Relation r; const OSMPBF::Relation& inputRelation = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).relations(i);
r.type = _Relation::unknown; bool isRestriction = false;
bool isOnlyRestriction = false;
for(int k = 0; k < inputRelation.keys_size(); k++) {
const std::string key = threadData->PBFprimitiveBlock.stringtable().s(inputRelation.keys(k));
const std::string val = threadData->PBFprimitiveBlock.stringtable().s(inputRelation.vals(k));
if ("type" == key && "restriction" == val) {
isRestriction = true;
}
if ("restriction" == key) {
if(val.find("only_") == 0)
isOnlyRestriction = true;
}
}
if(isRestriction) {
long long lastRef = 0;
_RawRestrictionContainer currentRestrictionContainer(isOnlyRestriction);
for(int rolesIndex = 0; rolesIndex < inputRelation.roles_sid_size(); rolesIndex++) {
string role(threadData->PBFprimitiveBlock.stringtable().s( inputRelation.roles_sid( rolesIndex ) ).data());
lastRef += inputRelation.memids(rolesIndex);
if(false == ("from" == role || "to" == role || "via" == role)) {
continue;
}
switch(inputRelation.types(rolesIndex)) {
case 0: //node
if("from" == role || "to" == role) //Only via should be a node
continue;
assert("via" == role);
if(UINT_MAX != currentRestrictionContainer.viaWay)
currentRestrictionContainer.viaWay = UINT_MAX;
assert(UINT_MAX == currentRestrictionContainer.viaWay);
currentRestrictionContainer.restriction.viaNode = lastRef;
break;
case 1: //way
assert("from" == role || "to" == role || "via" == role);
if("from" == role) {
currentRestrictionContainer.fromWay = lastRef;
}
if ("to" == role) {
currentRestrictionContainer.toWay = lastRef;
}
if ("via" == role) {
assert(currentRestrictionContainer.restriction.toNode == UINT_MAX);
currentRestrictionContainer.viaWay = lastRef;
}
break;
case 2: //relation, not used. relations relating to relations are evil.
continue;
assert(false);
break;
default: //should not happen
cout << "unknown";
assert(false);
break;
}
}
// if(UINT_MAX != currentRestriction.viaNode) {
// cout << "restr from " << currentRestriction.from << " via ";
// cout << "node " << currentRestriction.viaNode;
// cout << " to " << currentRestriction.to << endl;
// }
#pragma omp critical #pragma omp critical
{ {
if(!(*relationCallback)(r)) if(!(*restrictionCallback)(currentRestrictionContainer))
std::cerr << "[PBFParser] relation not parsed" << std::endl; std::cerr << "[PBFParser] relation not parsed" << std::endl;
}
} }
} }
} }
@ -442,7 +506,7 @@ private:
/* Function pointer for nodes */ /* Function pointer for nodes */
bool (*nodeCallback)(_Node); bool (*nodeCallback)(_Node);
bool (*wayCallback)(_Way); bool (*wayCallback)(_Way);
bool (*relationCallback)(_Relation); bool (*restrictionCallback)(_RawRestrictionContainer);
bool (*addressCallback)(_Node, HashTable<std::string, std::string>); bool (*addressCallback)(_Node, HashTable<std::string, std::string>);
/* the input stream to parse */ /* the input stream to parse */
std::fstream input; std::fstream input;

12
DataStructures/XMLParser.h
View File

@ -29,17 +29,18 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "ExtractorStructs.h" #include "ExtractorStructs.h"
#include "InputReaderFactory.h" #include "InputReaderFactory.h"
class XMLParser : public BaseParser<_Node, _Relation, _Way> { class XMLParser : public BaseParser<_Node, _RawRestrictionContainer, _Way> {
public: public:
XMLParser(const char * filename) { XMLParser(const char * filename) {
WARN("Parsing plain .osm/.osm.bz2 is deprecated. Switch to .pbf");
inputReader = inputReaderFactory(filename); inputReader = inputReaderFactory(filename);
} }
~XMLParser() {} ~XMLParser() {}
bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*relationCallbackPointer)(_Relation), bool (*wayCallbackPointer)(_Way), bool (*addressCallbackPointer)(_Node, HashTable<std::string, std::string>) ) { bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way), bool (*addressCallbackPointer)(_Node, HashTable<std::string, std::string>) ) {
nodeCallback = *nodeCallbackPointer; nodeCallback = *nodeCallbackPointer;
wayCallback = *wayCallbackPointer; wayCallback = *wayCallbackPointer;
relationCallback = *relationCallbackPointer; restrictionCallback = *restrictionCallbackPointer;
return true; return true;
} }
bool Init() { bool Init() {
@ -73,8 +74,7 @@ public:
if ( xmlStrEqual( currentName, ( const xmlChar* ) "relation" ) == 1 ) { if ( xmlStrEqual( currentName, ( const xmlChar* ) "relation" ) == 1 ) {
_Relation r; _Relation r;
r.type = _Relation::unknown; r.type = _Relation::unknown;
if(!(*relationCallback)(r)) //todo: parse relation
std::cerr << "[XMLParser] relation not parsed" << std::endl;
} }
xmlFree( currentName ); xmlFree( currentName );
} }
@ -210,7 +210,7 @@ private:
/* Function pointer for nodes */ /* Function pointer for nodes */
bool (*nodeCallback)(_Node); bool (*nodeCallback)(_Node);
bool (*wayCallback)(_Way); bool (*wayCallback)(_Way);
bool (*relationCallback)(_Relation); bool (*restrictionCallback)(_RawRestrictionContainer);
}; };

257
extractor.cpp
View File

@ -50,12 +50,12 @@ or see http://www.gnu.org/licenses/agpl.txt.
typedef BaseConfiguration ExtractorConfiguration; typedef BaseConfiguration ExtractorConfiguration;
unsigned globalRelationCounter = 0; unsigned globalRestrictionCounter = 0;
ExtractorCallbacks * extractCallBacks; ExtractorCallbacks * extractCallBacks;
bool nodeFunction(_Node n); bool nodeFunction(_Node n);
bool adressFunction(_Node n, HashTable<std::string, std::string> keyVals); bool adressFunction(_Node n, HashTable<string, string> keyVals);
bool relationFunction(_Relation r); bool restrictionFunction(_RawRestrictionContainer r);
bool wayFunction(_Way w); bool wayFunction(_Way w);
template<class ClassT> template<class ClassT>
@ -63,14 +63,15 @@ bool removeIfUnused(ClassT n) { return (false == n.used); }
int main (int argc, char *argv[]) { int main (int argc, char *argv[]) {
if(argc <= 1) { if(argc <= 1) {
std::cerr << "usage: " << endl << argv[0] << " <file.osm/.osm.bz2/.osm.pbf>" << std::endl; cerr << "usage: " << endl << argv[0] << " <file.osm/.osm.bz2/.osm.pbf>" << endl;
exit(-1); exit(-1);
} }
std::cout << "[extractor] extracting data from input file " << argv[1] << std::endl; cout << "[extractor] extracting data from input file " << argv[1] << endl;
bool isPBF = false; bool isPBF = false;
std::string outputFileName(argv[1]); string outputFileName(argv[1]);
std::string::size_type pos = outputFileName.find(".osm.bz2"); string restrictionsFileName(argv[1]);
string::size_type pos = outputFileName.find(".osm.bz2");
if(pos==string::npos) { if(pos==string::npos) {
pos = outputFileName.find(".osm.pbf"); pos = outputFileName.find(".osm.pbf");
if(pos!=string::npos) { if(pos!=string::npos) {
@ -79,36 +80,34 @@ int main (int argc, char *argv[]) {
} }
if(pos!=string::npos) { if(pos!=string::npos) {
outputFileName.replace(pos, 8, ".osrm"); outputFileName.replace(pos, 8, ".osrm");
restrictionsFileName.replace(pos, 8, ".osrm.restrictions");
} else { } else {
pos=outputFileName.find(".osm"); pos=outputFileName.find(".osm");
if(pos!=string::npos) { if(pos!=string::npos) {
outputFileName.replace(pos, 5, ".osrm"); outputFileName.replace(pos, 5, ".osrm");
restrictionsFileName.replace(pos, 5, ".osrm.restrictions");
} else { } else {
outputFileName.append(".osrm"); outputFileName.append(".osrm");
restrictionsFileName.append(".osrm.restrictions");
} }
} }
std::string adressFileName(outputFileName); string adressFileName(outputFileName);
unsigned amountOfRAM = 1; unsigned amountOfRAM = 1;
unsigned installedRAM = GetPhysicalmemory(); unsigned installedRAM = GetPhysicalmemory();
if(installedRAM < 2048264) { if(installedRAM < 2048264) {
std::cout << "[Warning] Machine has less than 2GB RAM." << std::endl; cout << "[Warning] Machine has less than 2GB RAM." << endl;
} }
if(testDataFile("extractor.ini")) { if(testDataFile("extractor.ini")) {
ExtractorConfiguration extractorConfig("extractor.ini"); ExtractorConfiguration extractorConfig("extractor.ini");
unsigned memoryAmountFromFile = atoi(extractorConfig.GetParameter("Memory").c_str()); unsigned memoryAmountFromFile = atoi(extractorConfig.GetParameter("Memory").c_str());
if( memoryAmountFromFile != 0 && memoryAmountFromFile <= installedRAM/(1024*1024*1024)) if( memoryAmountFromFile != 0 && memoryAmountFromFile <= installedRAM/(1024*1024*1024))
amountOfRAM = memoryAmountFromFile; amountOfRAM = memoryAmountFromFile;
std::cout << "[extractor] using " << amountOfRAM << " GB of RAM for buffers" << std::endl; cout << "[extractor] using " << amountOfRAM << " GB of RAM for buffers" << endl;
} }
STXXLNodeIDVector usedNodeIDs; STXXLContainers externalMemory;
STXXLNodeVector allNodes;
STXXLEdgeVector allEdges;
STXXLAddressVector adressVector;
STXXLStringVector nameVector;
unsigned usedNodeCounter = 0; unsigned usedNodeCounter = 0;
unsigned usedEdgeCounter = 0; unsigned usedEdgeCounter = 0;
@ -120,60 +119,148 @@ int main (int argc, char *argv[]) {
double time = get_timestamp(); double time = get_timestamp();
stringMap->set_empty_key(GetRandomString()); stringMap->set_empty_key(GetRandomString());
stringMap->insert(std::make_pair("", 0)); stringMap->insert(make_pair("", 0));
extractCallBacks = new ExtractorCallbacks(&allNodes, &usedNodeIDs, &allEdges, &nameVector, &adressVector, settings, stringMap); extractCallBacks = new ExtractorCallbacks(&externalMemory, settings, stringMap);
BaseParser<_Node, _Relation, _Way> * parser; BaseParser<_Node, _RawRestrictionContainer, _Way> * parser;
if(isPBF) { if(isPBF) {
parser = new PBFParser(argv[1]); parser = new PBFParser(argv[1]);
} else { } else {
parser = new XMLParser(argv[1]); parser = new XMLParser(argv[1]);
} }
parser->RegisterCallbacks(&nodeFunction, &relationFunction, &wayFunction, &adressFunction); parser->RegisterCallbacks(&nodeFunction, &restrictionFunction, &wayFunction, &adressFunction);
if(parser->Init()) { if(parser->Init()) {
parser->Parse(); parser->Parse();
} else { } else {
std::cerr << "[error] parser not initialized!" << std::endl; cerr << "[error] parser not initialized!" << endl;
exit(-1); exit(-1);
} }
delete parser; delete parser;
try { try {
// std::cout << "[info] raw no. of names: " << nameVector.size() << std::endl; // INFO("raw no. of names: " << externalMemory.nameVector.size());
// std::cout << "[info] raw no. of nodes: " << allNodes.size() << std::endl; // INFO("raw no. of nodes: " << externalMemory.allNodes.size());
// std::cout << "[info] no. of used nodes: " << usedNodeIDs.size() << std::endl; // INFO("no. of used nodes: " << externalMemory.usedNodeIDs.size());
// std::cout << "[info] raw no. of edges: " << allEdges.size() << std::endl; // INFO("raw no. of edges: " << externalMemory.allEdges.size());
// std::cout << "[info] raw no. of relations: " << globalRelationCounter << std::endl; // INFO("raw no. of ways: " << externalMemory.wayStartEndVector.size());
// std::cout << "[info] raw no. of addresses: " << adressVector.size() << std::endl; // INFO("raw no. of addresses: " << externalMemory.adressVector.size());
// INFO("raw no. of restrictions: " << externalMemory.restrictionsVector.size());
std::cout << "[extractor] parsing finished after " << get_timestamp() - time << "seconds" << std::endl; cout << "[extractor] parsing finished after " << get_timestamp() - time << "seconds" << endl;
time = get_timestamp(); time = get_timestamp();
unsigned memory_to_use = amountOfRAM * 1024 * 1024 * 1024; unsigned memory_to_use = amountOfRAM * 1024 * 1024 * 1024;
std::cout << "[extractor] Sorting used nodes ... " << std::flush; cout << "[extractor] Sorting used nodes ... " << flush;
stxxl::sort(usedNodeIDs.begin(), usedNodeIDs.end(), Cmp(), memory_to_use); stxxl::sort(externalMemory.usedNodeIDs.begin(), externalMemory.usedNodeIDs.end(), Cmp(), memory_to_use);
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp(); time = get_timestamp();
std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush; cout << "[extractor] Erasing duplicate nodes ... " << flush;
stxxl::vector<NodeID>::iterator NewEnd = unique ( usedNodeIDs.begin(),usedNodeIDs.end() ) ; stxxl::vector<NodeID>::iterator NewEnd = unique ( externalMemory.usedNodeIDs.begin(),externalMemory.usedNodeIDs.end() ) ;
usedNodeIDs.resize ( NewEnd - usedNodeIDs.begin() ); externalMemory.usedNodeIDs.resize ( NewEnd - externalMemory.usedNodeIDs.begin() );
cout << "ok, after " << get_timestamp() - time << "s" << endl; cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp(); time = get_timestamp();
std::cout << "[extractor] Sorting all nodes ... " << std::flush; cout << "[extractor] Sorting all nodes ... " << flush;
stxxl::sort(allNodes.begin(), allNodes.end(), CmpNodeByID(), memory_to_use); stxxl::sort(externalMemory.allNodes.begin(), externalMemory.allNodes.end(), CmpNodeByID(), memory_to_use);
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp(); time = get_timestamp();
std::ofstream fout; cout << "[extractor] Sorting used ways ... " << flush;
fout.open(outputFileName.c_str(), std::ios::binary); stxxl::sort(externalMemory.wayStartEndVector.begin(), externalMemory.wayStartEndVector.end(), CmpWayStartAndEnd(), memory_to_use);
fout.write((char*)&usedNodeCounter, sizeof(unsigned)); cout << "ok, after " << get_timestamp() - time << "s" << endl;
std::cout << "[extractor] Confirming used nodes ... " << std::flush; cout << "[extractor] Sorting restrctns. by from... " << flush;
STXXLNodeVector::iterator nodesIT = allNodes.begin(); stxxl::sort(externalMemory.restrictionsVector.begin(), externalMemory.restrictionsVector.end(), CmpRestrictionByFrom(), memory_to_use);
STXXLNodeIDVector::iterator usedNodeIDsIT = usedNodeIDs.begin(); cout << "ok, after " << get_timestamp() - time << "s" << endl;
while(usedNodeIDsIT != usedNodeIDs.end() && nodesIT != allNodes.end()) {
cout << "[extractor] Fixing restriction starts ... " << flush;
STXXLRestrictionsVector::iterator restrictionsIT = externalMemory.restrictionsVector.begin();
STXXLWayIDStartEndVector::iterator wayStartAndEndEdgeIT = externalMemory.wayStartEndVector.begin();
while(wayStartAndEndEdgeIT != externalMemory.wayStartEndVector.end() && restrictionsIT != externalMemory.restrictionsVector.end()) {
if(wayStartAndEndEdgeIT->wayID < restrictionsIT->fromWay){
wayStartAndEndEdgeIT++;
continue;
}
if(wayStartAndEndEdgeIT->wayID > restrictionsIT->fromWay) {
restrictionsIT++;
continue;
}
assert(wayStartAndEndEdgeIT->wayID == restrictionsIT->fromWay);
NodeID viaNode = restrictionsIT->restriction.viaNode;
if(wayStartAndEndEdgeIT->firstStart == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstTarget;
} else if(wayStartAndEndEdgeIT->firstTarget == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstStart;
} else if(wayStartAndEndEdgeIT->lastStart == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastTarget;
} else if(wayStartAndEndEdgeIT->lastTarget == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastStart;
}
restrictionsIT++;
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Sorting restrctns. by to ... " << flush;
stxxl::sort(externalMemory.restrictionsVector.begin(), externalMemory.restrictionsVector.end(), CmpRestrictionByTo(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
unsigned usableRestrictionsCounter(0);
cout << "[extractor] Fixing restriction ends ... " << flush;
restrictionsIT = externalMemory.restrictionsVector.begin();
wayStartAndEndEdgeIT = externalMemory.wayStartEndVector.begin();
while(wayStartAndEndEdgeIT != externalMemory.wayStartEndVector.end() &&
restrictionsIT != externalMemory.restrictionsVector.end()) {
if(wayStartAndEndEdgeIT->wayID < restrictionsIT->toWay){
wayStartAndEndEdgeIT++;
continue;
}
if(wayStartAndEndEdgeIT->wayID > restrictionsIT->toWay) {
restrictionsIT++;
continue;
}
NodeID viaNode = restrictionsIT->restriction.viaNode;
if(wayStartAndEndEdgeIT->lastStart == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastTarget;
} else if(wayStartAndEndEdgeIT->lastTarget == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastStart;
} else if(wayStartAndEndEdgeIT->firstStart == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstTarget;
} else if(wayStartAndEndEdgeIT->firstTarget == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstStart;
}
if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) {
usableRestrictionsCounter++;
}
restrictionsIT++;
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
//todo: serialize restrictions
ofstream restrictionsOutstream;
restrictionsOutstream.open(restrictionsFileName.c_str(), ios::binary);
restrictionsOutstream.write((char*)&usableRestrictionsCounter, sizeof(unsigned));
for(restrictionsIT = externalMemory.restrictionsVector.begin(); restrictionsIT != externalMemory.restrictionsVector.end(); restrictionsIT++) {
if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) {
restrictionsOutstream.write((char *)&(restrictionsIT->restriction), sizeof(_Restriction));
}
}
restrictionsOutstream.close();
ofstream fout;
fout.open(outputFileName.c_str(), ios::binary);
fout.write((char*)&usedNodeCounter, sizeof(unsigned));
time = get_timestamp();
cout << "[extractor] Confirming used nodes ... " << flush;
STXXLNodeVector::iterator nodesIT = externalMemory.allNodes.begin();
STXXLNodeIDVector::iterator usedNodeIDsIT = externalMemory.usedNodeIDs.begin();
while(usedNodeIDsIT != externalMemory.usedNodeIDs.end() && nodesIT != externalMemory.allNodes.end()) {
if(*usedNodeIDsIT < nodesIT->id){ if(*usedNodeIDsIT < nodesIT->id){
usedNodeIDsIT++; usedNodeIDsIT++;
continue; continue;
@ -191,30 +278,31 @@ int main (int argc, char *argv[]) {
nodesIT++; nodesIT++;
} }
} }
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp(); time = get_timestamp();
std::cout << "[extractor] setting number of nodes ... " << std::flush; cout << "[extractor] setting number of nodes ... " << flush;
std::ios::pos_type positionInFile = fout.tellp(); ios::pos_type positionInFile = fout.tellp();
fout.seekp(std::ios::beg); fout.seekp(ios::beg);
fout.write((char*)&usedNodeCounter, sizeof(unsigned)); fout.write((char*)&usedNodeCounter, sizeof(unsigned));
fout.seekp(positionInFile); fout.seekp(positionInFile);
std::cout << "ok" << std::endl; cout << "ok" << endl;
time = get_timestamp(); time = get_timestamp();
// Sort edges by start. // Sort edges by start.
std::cout << "[extractor] Sorting edges by start ... " << std::flush; cout << "[extractor] Sorting edges by start ... " << flush;
stxxl::sort(allEdges.begin(), allEdges.end(), CmpEdgeByStartID(), memory_to_use); stxxl::sort(externalMemory.allEdges.begin(), externalMemory.allEdges.end(), CmpEdgeByStartID(), memory_to_use);
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp(); time = get_timestamp();
std::cout << "[extractor] Setting start coords ... " << std::flush; cout << "[extractor] Setting start coords ... " << flush;
fout.write((char*)&usedEdgeCounter, sizeof(unsigned)); fout.write((char*)&usedEdgeCounter, sizeof(unsigned));
// Traverse list of edges and nodes in parallel and set start coord // Traverse list of edges and nodes in parallel and set start coord
nodesIT = allNodes.begin(); nodesIT = externalMemory.allNodes.begin();
STXXLEdgeVector::iterator edgeIT = allEdges.begin(); STXXLEdgeVector::iterator edgeIT = externalMemory.allEdges.begin();
while(edgeIT != allEdges.end() && nodesIT != allNodes.end()) { while(edgeIT != externalMemory.allEdges.end() && nodesIT != externalMemory.allNodes.end()) {
if(edgeIT->start < nodesIT->id){ if(edgeIT->start < nodesIT->id){
edgeIT++; edgeIT++;
continue; continue;
@ -229,20 +317,20 @@ int main (int argc, char *argv[]) {
edgeIT++; edgeIT++;
} }
} }
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp(); time = get_timestamp();
// Sort Edges by target // Sort Edges by target
std::cout << "[extractor] Sorting edges by target ... " << std::flush; cout << "[extractor] Sorting edges by target ... " << flush;
stxxl::sort(allEdges.begin(), allEdges.end(), CmpEdgeByTargetID(), memory_to_use); stxxl::sort(externalMemory.allEdges.begin(), externalMemory.allEdges.end(), CmpEdgeByTargetID(), memory_to_use);
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp(); time = get_timestamp();
std::cout << "[extractor] Setting target coords ... " << std::flush; cout << "[extractor] Setting target coords ... " << flush;
// Traverse list of edges and nodes in parallel and set target coord // Traverse list of edges and nodes in parallel and set target coord
nodesIT = allNodes.begin(); nodesIT = externalMemory.allNodes.begin();
edgeIT = allEdges.begin(); edgeIT = externalMemory.allEdges.begin();
while(edgeIT != allEdges.end() && nodesIT != allNodes.end()) { while(edgeIT != externalMemory.allEdges.end() && nodesIT != externalMemory.allNodes.end()) {
if(edgeIT->target < nodesIT->id){ if(edgeIT->target < nodesIT->id){
edgeIT++; edgeIT++;
continue; continue;
@ -285,7 +373,7 @@ int main (int argc, char *argv[]) {
fout.write((char*)&one, sizeof(short)); fout.write((char*)&one, sizeof(short));
break; break;
default: default:
std::cerr << "[error] edge with no direction: " << edgeIT->direction << std::endl; cerr << "[error] edge with no direction: " << edgeIT->direction << endl;
assert(false); assert(false);
break; break;
} }
@ -298,25 +386,25 @@ int main (int argc, char *argv[]) {
edgeIT++; edgeIT++;
} }
} }
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp(); time = get_timestamp();
std::cout << "[extractor] setting number of edges ... " << std::flush; cout << "[extractor] setting number of edges ... " << flush;
fout.seekp(positionInFile); fout.seekp(positionInFile);
fout.write((char*)&usedEdgeCounter, sizeof(unsigned)); fout.write((char*)&usedEdgeCounter, sizeof(unsigned));
fout.close(); fout.close();
std::cout << "ok" << std::endl; cout << "ok" << endl;
time = get_timestamp(); time = get_timestamp();
std::cout << "[extractor] writing street name index ... " << std::flush; cout << "[extractor] writing street name index ... " << flush;
std::vector<unsigned> * nameIndex = new std::vector<unsigned>(nameVector.size()+1, 0); vector<unsigned> * nameIndex = new vector<unsigned>(externalMemory.nameVector.size()+1, 0);
outputFileName.append(".names"); outputFileName.append(".names");
std::ofstream nameOutFile(outputFileName.c_str(), std::ios::binary); ofstream nameOutFile(outputFileName.c_str(), ios::binary);
unsigned sizeOfNameIndex = nameIndex->size(); unsigned sizeOfNameIndex = nameIndex->size();
nameOutFile.write((char *)&(sizeOfNameIndex), sizeof(unsigned)); nameOutFile.write((char *)&(sizeOfNameIndex), sizeof(unsigned));
for(STXXLStringVector::iterator it = nameVector.begin(); it != nameVector.end(); it++) { for(STXXLStringVector::iterator it = externalMemory.nameVector.begin(); it != externalMemory.nameVector.end(); it++) {
unsigned lengthOfRawString = strlen(it->c_str()); unsigned lengthOfRawString = strlen(it->c_str());
nameOutFile.write((char *)&(lengthOfRawString), sizeof(unsigned)); nameOutFile.write((char *)&(lengthOfRawString), sizeof(unsigned));
nameOutFile.write(it->c_str(), lengthOfRawString); nameOutFile.write(it->c_str(), lengthOfRawString);
@ -324,26 +412,26 @@ int main (int argc, char *argv[]) {
nameOutFile.close(); nameOutFile.close();
delete nameIndex; delete nameIndex;
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; cout << "ok, after " << get_timestamp() - time << "s" << endl;
// time = get_timestamp(); // time = get_timestamp();
// std::cout << "[extractor] writing address list ... " << std::flush; // cout << "[extractor] writing address list ... " << flush;
// //
// adressFileName.append(".address"); // adressFileName.append(".address");
// std::ofstream addressOutFile(adressFileName.c_str()); // ofstream addressOutFile(adressFileName.c_str());
// for(STXXLAddressVector::iterator it = adressVector.begin(); it != adressVector.end(); it++) { // for(STXXLAddressVector::iterator it = adressVector.begin(); it != adressVector.end(); it++) {
// addressOutFile << it->node.id << "|" << it->node.lat << "|" << it->node.lon << "|" << it->city << "|" << it->street << "|" << it->housenumber << "|" << it->state << "|" << it->country << "\n"; // addressOutFile << it->node.id << "|" << it->node.lat << "|" << it->node.lon << "|" << it->city << "|" << it->street << "|" << it->housenumber << "|" << it->state << "|" << it->country << "\n";
// } // }
// addressOutFile.close(); // addressOutFile.close();
// std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; // cout << "ok, after " << get_timestamp() - time << "s" << endl;
} catch ( const std::exception& e ) { } catch ( const exception& e ) {
std::cerr << "Caught Execption:" << e.what() << std::endl; cerr << "Caught Execption:" << e.what() << endl;
return false; return false;
} }
delete extractCallBacks; delete extractCallBacks;
std::cout << "[extractor] finished." << std::endl; cout << "[extractor] finished." << endl;
return 0; return 0;
} }
@ -352,13 +440,14 @@ bool nodeFunction(_Node n) {
return true; return true;
} }
bool adressFunction(_Node n, HashTable<std::string, std::string> keyVals){ bool adressFunction(_Node n, HashTable<string, string> keyVals){
extractCallBacks->adressFunction(n, keyVals); extractCallBacks->adressFunction(n, keyVals);
return true; return true;
} }
bool relationFunction(_Relation r) { bool restrictionFunction(_RawRestrictionContainer r) {
globalRelationCounter++; extractCallBacks->restrictionFunction(r);
globalRestrictionCounter++;
return true; return true;
} }
bool wayFunction(_Way w) { bool wayFunction(_Way w) {