Merge remote-tracking branch 'dennis/master'

AUTHORS.TXT

@@ -8,3 +8,4 @@ Bharath Vissapragada
 Pascal Neis
 Sasa Ivetic
 Emil Tin
+Henning Moll

Contractor/EdgeBasedGraphFactory.cpp

@@ -146,9 +146,9 @@ void EdgeBasedGraphFactory::Run() {
                         }
                     }
                     ++secondRestrictionIterator;
-                } while(u == secondRestrictionIterator->fromNode);
+                } while(secondRestrictionIterator != inputRestrictions.end() && u == secondRestrictionIterator->fromNode);
             }
-            if(_nodeBasedGraph->EndEdges(v) == _nodeBasedGraph->BeginEdges(v) + 1 && _nodeBasedGraph->GetEdgeData(e1).type != INT_MAX) {
+            if(_nodeBasedGraph->EndEdges(v) == _nodeBasedGraph->BeginEdges(v) + 1 && _nodeBasedGraph->GetEdgeData(e1).type != SHRT_MAX) {
                 EdgeBasedNode currentNode;
                 currentNode.nameID = _nodeBasedGraph->GetEdgeData(e1).nameID;
                 currentNode.lat1 = inputNodeInfoList[u].lat;
@@ -194,7 +194,7 @@ void EdgeBasedGraphFactory::Run() {
                                 }
                             }
                             ++secondRestrictionIterator;
-                        } while(u == secondRestrictionIterator->fromNode);
+                        } while(secondRestrictionIterator != inputRestrictions.end() && u == secondRestrictionIterator->fromNode);
                     }
 
                     if( !isTurnRestricted || (isOnlyAllowed && w == onlyToNode) ) { //only add an edge if turn is not prohibited

DataStructures/BaseParser.h

@@ -26,7 +26,7 @@ class BaseParser {
 public:
     virtual ~BaseParser() {}
     virtual bool Init() = 0;
-    virtual bool RegisterCallbacks(bool (*nodeCallbackPointer)(NodeT), bool (*restrictionCallbackPointer)(RestrictionT), bool (*wayCallbackPointer)(WayT), bool (*addressCallbackPointer)(NodeT, HashTable<std::string, std::string>)) = 0;
+    virtual bool RegisterCallbacks(bool (*nodeCallbackPointer)(NodeT), bool (*restrictionCallbackPointer)(RestrictionT), bool (*wayCallbackPointer)(WayT), bool (*addressCallbackPointer)(NodeT, HashTable<std::string, std::string>&)) = 0;
     virtual bool Parse() = 0;
 };
 

DataStructures/ExtractorCallBacks.h

@@ -29,7 +29,7 @@ typedef stxxl::vector<NodeID> STXXLNodeIDVector;
 typedef stxxl::vector<_Node> STXXLNodeVector;
 typedef stxxl::vector<_Edge> STXXLEdgeVector;
 typedef stxxl::vector<_Address> STXXLAddressVector;
-typedef stxxl::vector<string> STXXLStringVector;
+typedef stxxl::vector<std::string> STXXLStringVector;
 typedef stxxl::vector<_RawRestrictionContainer> STXXLRestrictionsVector;
 typedef stxxl::vector<_WayIDStartAndEndEdge> STXXLWayIDStartEndVector;
 
@@ -42,6 +42,20 @@ struct STXXLContainers {
     STXXLStringVector           nameVector;
     STXXLRestrictionsVector     restrictionsVector;
     STXXLWayIDStartEndVector    wayStartEndVector;
+
+    STXXLContainers() {
+        nameVector.push_back("");
+    }
+
+    ~STXXLContainers() {
+        usedNodeIDs.clear();
+        allNodes.clear();
+        allEdges.clear();
+        adressVector.clear();
+        nameVector.clear();
+        restrictionsVector.clear();
+        wayStartEndVector.clear();
+    }
 };
 
 class ExtractorCallbacks{
@@ -51,7 +65,7 @@ private:
     STXXLContainers * externalMemory;
 
 public:
-    ExtractorCallbacks(STXXLContainers * ext, Settings set, StringMap * strMap){
+    ExtractorCallbacks(STXXLContainers * ext, Settings set, StringMap * strMap) {
         externalMemory = ext;
         settings = set;
         stringMap = strMap;
@@ -163,7 +177,6 @@ public:
                     w.direction = _Way::bidirectional;
                 } else if( oneway == "-1") {
                     w.direction  = _Way::opposite;
-                    std::reverse( w.path.begin(), w.path.end() );
                 }
                 else if( oneway == "yes" || oneway == "1" || oneway == "true" || onewayClass == "yes" || onewayClass == "1" || onewayClass == "true" || junction == "roundabout" || highway == "motorway_link" || highway == "motorway" ) {
                     w.direction = _Way::oneway;

DataStructures/NNGrid.h

@@ -484,7 +484,7 @@ private:
         assert( x<=1.0 && x >= 0);
         assert( y<=1.0 && y >= 0);
 
-        unsigned line = 1073741824.0*y;
+        unsigned line = (32768 * (32768-1))*y;
         line = line - (line % 32768);
         assert(line % 32768 == 0);
         unsigned column = 32768.*x;

DataStructures/PBFParser.h

@@ -22,6 +22,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
 #define PBFPARSER_H_
 
 #include <zlib.h>
+#include <boost/shared_ptr.hpp>
 
 #include "BaseParser.h"
 
@@ -61,9 +62,9 @@ class PBFParser : public BaseParser<_Node, _RawRestrictionContainer, _Way> {
     };
 
 public:
-    PBFParser(const char * fileName) 
+    PBFParser(const char * fileName) { /* Max 25 items in queue */
-        : threadDataQueue( new ConcurrentQueue<_ThreadData*>(25) ) { /* Max 25 items in queue */
         GOOGLE_PROTOBUF_VERIFY_VERSION;
+        threadDataQueue.reset( new ConcurrentQueue<_ThreadData*>(25) );
         input.open(fileName, std::ios::in | std::ios::binary);
 
         if (!input) {
@@ -78,7 +79,7 @@ public:
         addressCallback = NULL;     restrictionCallback = NULL;
     }
 
-    bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way),bool (*addressCallbackPointer)(_Node, HashTable<std::string, std::string>) ) {
+    bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way),bool (*addressCallbackPointer)(_Node, HashTable<std::string, std::string>&) ) {
         nodeCallback = *nodeCallbackPointer;
         wayCallback = *wayCallbackPointer;
         restrictionCallback = *restrictionCallbackPointer;
@@ -95,8 +96,6 @@ public:
         while (threadDataQueue->try_pop(td)) {
             delete td;
         }
-        delete threadDataQueue;
-
         google::protobuf::ShutdownProtobufLibrary();
 
 #ifndef NDEBUG
@@ -145,8 +144,10 @@ public:
 
             if (keepRunning)
                 threadDataQueue->push(threadData);
-            else
+            else {
                 threadDataQueue->push(NULL); // No more data to read, parse stops when NULL encountered
+                delete threadData;
+            }
         } while(keepRunning);
     }
 
@@ -219,13 +220,6 @@ private:
                 int keyValue = dense.keys_vals ( denseTagIndex+1 );
                 std::string key = threadData->PBFprimitiveBlock.stringtable().s(tagValue).data();
                 std::string value = threadData->PBFprimitiveBlock.stringtable().s(keyValue).data();
-
-                if("barrier" == key && "bollard" == value) {
-                	n.bollard = true;
-                }
-                if("highway" == key && "traffic_signals" == value) {
-                	n.trafficLight = true;
-                }
                 keyVals.Add(key, value);
                 denseTagIndex += 2;
             }
@@ -364,7 +358,7 @@ private:
     }
 
     void loadBlock(_ThreadData * threadData) {
-        blockCount++;
+        ++blockCount;
         threadData->currentGroupID = 0;
         threadData->currentEntityID = 0;
     }
@@ -386,14 +380,14 @@ private:
         if ( size > MAX_BLOB_HEADER_SIZE || size < 0 ) {
             return false;
         }
-        char *data = (char*)malloc(size);
+        char *data = new char[size];
         stream.read(data, size*sizeof(data[0]));
 
         if ( !(threadData->PBFBlobHeader).ParseFromArray( data, size ) ){
-            free(data);
+            delete[] data;
             return false;
         }
-        free(data);
+        delete[] data;
         return true;
     }
 
@@ -493,25 +487,26 @@ private:
             return false;
         }
 
-        if ( !readPBFBlobHeader(stream, threadData) )
+        if ( !readPBFBlobHeader(stream, threadData) ){
-            return false;
-
-        if ( threadData->PBFBlobHeader.type() != "OSMData" ) {
-            std::cerr << "[error] invalid block type, found" << threadData->PBFBlobHeader.type().data() << "instead of OSMData" << std::endl;
             return false;
         }
 
-        if ( !readBlob(stream, threadData) )
+        if ( threadData->PBFBlobHeader.type() != "OSMData" ) {
             return false;
+        }
+
+        if ( !readBlob(stream, threadData) ) {
+            return false;
+        }
 
         if ( !threadData->PBFprimitiveBlock.ParseFromArray( &(threadData->charBuffer[0]), threadData-> charBuffer.size() ) ) {
-            std::cerr << "[error] failed to parse PrimitiveBlock" << std::endl;
+            ERR("failed to parse PrimitiveBlock");
             return false;
         }
         return true;
     }
 
-    static Endianness getMachineEndianness() {
+    Endianness getMachineEndianness() const {
         int i(1);
         char *p = (char *) &i;
         if (p[0] == 1)
@@ -531,12 +526,12 @@ private:
     bool (*nodeCallback)(_Node);
     bool (*wayCallback)(_Way);
     bool (*restrictionCallback)(_RawRestrictionContainer);
-    bool (*addressCallback)(_Node, HashTable<std::string, std::string>);
+    bool (*addressCallback)(_Node, HashTable<std::string, std::string>&);
     /* the input stream to parse */
     std::fstream input;
 
     /* ThreadData Queue */
-    ConcurrentQueue < _ThreadData* >* threadDataQueue;
+    boost::shared_ptr<ConcurrentQueue < _ThreadData* > > threadDataQueue;
 };
 
 #endif /* PBFPARSER_H_ */

DataStructures/XMLParser.h

@@ -37,7 +37,7 @@ public:
     }
     virtual ~XMLParser() {}
 
-    bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way), bool (*addressCallbackPointer)(_Node, HashTable<std::string, std::string>) ) {
+    bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way), bool (*addressCallbackPointer)(_Node, HashTable<std::string, std::string>&) ) {
         nodeCallback = *nodeCallbackPointer;
         wayCallback = *wayCallbackPointer;
         restrictionCallback = *restrictionCallbackPointer;

README.TXT

@@ -3,13 +3,18 @@ Compilation
 
 Compiling the source code is easy. If you are running a decent linux 
 installing dependencies and running make should suffice. Make sure the following
-dependencies are installed:
+dependencies are installed (for libraries choose the -dev packages):
 
  - Boost		1.41+
  - g++			4.2+
  - libxml2		2.7+
  - scons 		2.10+
  - stxxl		1.3.1+
+ - libprotobuf  2.3.0+ (also protobuf-compiler)
+ - libbz2       any
+ - libzip       any
+ - libmagic++   (from ImageMagick)
+  
 
 Building the binaries is done by using scons. It should check for required
 libraries and header files and report missing ones.The Scons script accepts
@@ -72,7 +77,12 @@ preprocessing runs in three steps, all done by seperate programs.
 
 'osrm-extract file.osm' extracts the road network of an osm file. This is
 necessary, because the osm data is not made to support fast routing out of the
-box. The output of the step is a file called 'file.osrm'
+box. The output of the step is a file called 'file.osrm'.
+
+'osrm-extract' makes heavy use of STXXL memory management; STXXL will create
+a temporary file in /var/tmp. If you would prefer space allocated elsewhere,
+create a file named '.stxxl' in the current directory and specify the location
+and size of the virtual disk there, e.g. 'disk=./stxxl,20480,syscall'.
 
 'osrm-prepare file.osrm file.restrictions' preprocesses the road network and 
 computes additional information that is exploited later to speed up the path 
@@ -83,9 +93,9 @@ information.
 
 'osrm-routed' starts the server on TCP Port 5000. The
 server communicates over http and can be queried by any browser or http-capable
-command line tool. The server responds with KML-formatted output.Assume the 
+command line tool. The server responds with KML-formatted output. Assume the 
 server is installed on machine localhost and a map containing the Netherlands 
-has been installed. Computing a route from Amsterdam to Den Haag can be done by
+has been installed. Computing a route from Amsterdam to The Hague can be done by
 calling 
         http://localhost:5000/route&52.370197&4.890444&52.048167&4.3175
 which gives a shortest (fastest) route between the two points. To locate a

SConstruct

@@ -98,6 +98,9 @@ if not conf.CheckLibWithHeader('bz2', 'bzlib.h', 'CXX'):
 if not conf.CheckLibWithHeader('libzip', 'zip.h', 'CXX'):
 	print "Zip library not found. Exiting"
 	Exit(-1)
+if not conf.CheckLibWithHeader('pthread', 'pthread.h', 'CXX'):
+        print "pthread not found. Exiting"
+        Exit(-1)
 if not conf.CheckLibWithHeader('protobuf', 'google/protobuf/descriptor.h', 'CXX'):
 	print "Google Protobuffer library not found. Exiting"
 	Exit(-1)

Util/GraphLoader.h

@@ -364,7 +364,7 @@ template<typename NodeT, typename EdgeT>
 unsigned readHSGRFromStream(istream &in, vector<NodeT>& nodeList, vector<EdgeT> & edgeList) {
     unsigned numberOfNodes = 0;
     in.read((char*) & numberOfNodes, sizeof(unsigned));
-    nodeList.resize(numberOfNodes);
+    nodeList.resize(numberOfNodes + 1);
     NodeT currentNode;
     for(unsigned nodeCounter = 0; nodeCounter < numberOfNodes; ++nodeCounter ) {
         in.read((char*) &currentNode, sizeof(NodeT));

createHierarchy.cpp

@@ -155,6 +155,7 @@ int main (int argc, char *argv[]) {
 
     std::vector< ContractionCleanup::Edge > contractedEdges;
     contractor->GetEdges( contractedEdges );
+    delete contractor;
 
     ContractionCleanup * cleanup = new ContractionCleanup(n, contractedEdges);
     contractedEdges.clear();
@@ -183,7 +184,7 @@ int main (int argc, char *argv[]) {
     numberOfNodes+=1;
 
     std::vector< StaticGraph<EdgeData>::_StrNode > _nodes;
-    _nodes.resize( numberOfNodes);
+    _nodes.resize( numberOfNodes + 1 );
 
     StaticGraph<EdgeData>::EdgeIterator edge = 0;
     StaticGraph<EdgeData>::EdgeIterator position = 0;

extractor.cpp

@@ -60,7 +60,7 @@ unsigned globalRestrictionCounter = 0;
 ExtractorCallbacks * extractCallBacks;
 
 bool nodeFunction(_Node n);
-bool adressFunction(_Node n, HashTable<string, string> keyVals);
+bool adressFunction(_Node n, HashTable<string, string> & keyVals);
 bool restrictionFunction(_RawRestrictionContainer r);
 bool wayFunction(_Way w);
 
@@ -69,8 +69,16 @@ bool removeIfUnused(ClassT n) { return (false == n.used); }
 
 
 int main (int argc, char *argv[]) {
+
     GUARANTEE((argc > 1) ,"usage: \n" << argv[0] << " <file.osm/.osm.bz2/.osm.pbf>");
 
+    //Check if another instance of stxxl is already running or if there is a general problem
+    try {
+        stxxl::vector<unsigned> testForRunningInstance;
+    } catch(std::exception & e) {
+        ERR("Could not instantiate STXXL layer." << std::endl << e.what());
+    }
+
     INFO("extracting data from input file " << argv[1]);
     bool isPBF(false);
     std::string outputFileName(argv[1]);
@@ -182,10 +190,9 @@ int main (int argc, char *argv[]) {
         parser = new XMLParser(argv[1]);
     }
     parser->RegisterCallbacks(&nodeFunction, &restrictionFunction, &wayFunction, &adressFunction);
-    GUARANTEE(parser->Init(), "Parser not initialized!");
+    if(!parser->Init())
+        INFO("Parser not initialized!");
     parser->Parse();
-    DELETE(parser);
-    stringMap.clear();
 
     try {
         //        INFO("raw no. of names:        " << externalMemory.nameVector.size());
@@ -446,10 +453,9 @@ int main (int argc, char *argv[]) {
         cout << "ok" << endl;
         time = get_timestamp();
         cout << "[extractor] writing street name index ... " << flush;
-        vector<unsigned> * nameIndex = new vector<unsigned>(externalMemory.nameVector.size()+1, 0);
         outputFileName.append(".names");
         ofstream nameOutFile(outputFileName.c_str(), ios::binary);
-        unsigned sizeOfNameIndex = nameIndex->size();
+        unsigned sizeOfNameIndex = externalMemory.nameVector.size();
         nameOutFile.write((char *)&(sizeOfNameIndex), sizeof(unsigned));
 
         BOOST_FOREACH(string str, externalMemory.nameVector) {
@@ -459,7 +465,6 @@ int main (int argc, char *argv[]) {
         }
 
         nameOutFile.close();
-        delete nameIndex;
         cout << "ok, after " << get_timestamp() - time << "s" << endl;
 
         //        time = get_timestamp();
@@ -477,7 +482,9 @@ int main (int argc, char *argv[]) {
         cerr <<  "Caught Execption:" << e.what() << endl;
         return false;
     }
+    DELETE(parser);
 
+    stringMap.clear();
     delete extractCallBacks;
     cout << "[extractor] finished." << endl;
     return 0;
@@ -488,7 +495,7 @@ bool nodeFunction(_Node n) {
     return true;
 }
 
-bool adressFunction(_Node n, HashTable<string, string> keyVals){
+bool adressFunction(_Node n, HashTable<string, string> & keyVals){
     extractCallBacks->adressFunction(n, keyVals);
     return true;
 }