extractLargeNetwork has fault-tolerance now.

extractNetwork has been deprecated.

DataStructures/extractorStructs.h

@@ -73,11 +73,12 @@ struct _Way {
     short type;
 };
 
-struct _Stats {
+struct _Edge {
-	NodeID numberOfNodes;
+    NodeID start;
-	NodeID numberOfEdges;
+    NodeID target;
-	NodeID numberOfWays;
+    short type;
-	NodeID numberOfMaxspeed;
+    short direction;
+    double speed;
 };
 
 struct Settings {
@@ -115,7 +116,7 @@ struct Cmp : public std::binary_function<NodeID, NodeID, bool>
     }
 };
 
-_Way _ReadXMLWay( xmlTextReaderPtr& inputReader, Settings& settings, _Stats& stats ) {
+_Way _ReadXMLWay( xmlTextReaderPtr& inputReader, Settings& settings ) {
     _Way way;
     way.direction = _Way::notSure;
     way.maximumSpeed = -1;
@@ -188,27 +189,42 @@ _Way _ReadXMLWay( xmlTextReaderPtr& inputReader, Settings& settings, _Stats& sta
                         xmlStrPrintf( buffer, 100, ( const xmlChar* ) "%.lf", maxspeed );
                         if ( xmlStrEqual( value, buffer ) == 1 ) {
                             way.maximumSpeed = maxspeed;
-							stats.numberOfMaxspeed++;
                         } else {
                             xmlStrPrintf( buffer, 100, ( const xmlChar* ) "%.lf kmh", maxspeed );
                             if ( xmlStrEqual( value, buffer ) == 1 ) {
                                 way.maximumSpeed = maxspeed;
-								stats.numberOfMaxspeed++;
                             } else {
                                 xmlStrPrintf( buffer, 100, ( const xmlChar* ) "%.lfkmh", maxspeed );
                                 if ( xmlStrEqual( value, buffer ) == 1 ) {
                                     way.maximumSpeed = maxspeed;
-									stats.numberOfMaxspeed++;
                                 } else {
                                     xmlStrPrintf( buffer, 100, ( const xmlChar* ) "%.lf km/h", maxspeed );
                                     if ( xmlStrEqual( value, buffer ) == 1 ) {
                                         way.maximumSpeed = maxspeed;
-										stats.numberOfMaxspeed++;
                                     } else {
                                         xmlStrPrintf( buffer, 100, ( const xmlChar* ) "%.lfkm/h", maxspeed );
                                         if ( xmlStrEqual( value, buffer ) == 1 ) {
                                             way.maximumSpeed = maxspeed;
-											stats.numberOfMaxspeed++;
+                                        } else {
+                                            xmlStrPrintf( buffer, 100, ( const xmlChar* ) "%.lf mph", maxspeed );
+                                            if ( xmlStrEqual( value, buffer ) == 1 ) {
+                                                way.maximumSpeed = maxspeed;
+                                            } else {
+                                                xmlStrPrintf( buffer, 100, ( const xmlChar* ) "%.lfmph", maxspeed );
+                                                if ( xmlStrEqual( value, buffer ) == 1 ) {
+                                                    way.maximumSpeed = maxspeed;
+                                                } else {
+                                                    xmlStrPrintf( buffer, 100, ( const xmlChar* ) "%.lf mp/h", maxspeed );
+                                                    if ( xmlStrEqual( value, buffer ) == 1 ) {
+                                                        way.maximumSpeed = maxspeed;
+                                                    } else {
+                                                        xmlStrPrintf( buffer, 100, ( const xmlChar* ) "%.lfmp/h", maxspeed );
+                                                        if ( xmlStrEqual( value, buffer ) == 1 ) {
+                                                            way.maximumSpeed = maxspeed;
+                                                        }
+                                                    }
+                                                }
+                                            }
                                         }
                                     }
                                 }

extractLargeNetwork.cpp

@@ -37,11 +37,17 @@ or see http://www.gnu.org/licenses/agpl.txt.
 
 using namespace std;
 typedef google::dense_hash_map<NodeID, _Node> NodeMap;
+typedef stxxl::vector<NodeID> STXXLNodeIDVector;
+typedef stxxl::vector<_Node> STXXLNodeVector;
+typedef stxxl::vector<_Edge> STXXLEdgeVector;
 
-_Stats stats;
 Settings settings;
 vector<NodeID> SignalNodes;
-
+STXXLNodeIDVector usedNodes;
+STXXLNodeVector allNodes;
+STXXLNodeVector confirmedNodes;
+STXXLEdgeVector allEdges;
+STXXLEdgeVector confirmedEdges;
 NodeMap * nodeMap = new NodeMap();
 
 int main (int argc, char *argv[])
@@ -57,9 +63,6 @@ int main (int argc, char *argv[])
     settings.speedProfile.speed.insert(settings.speedProfile.speed.begin(), speeds, speeds+13);
 
     xmlTextReaderPtr inputReader = xmlNewTextReaderFilename( argv[1] );
-	ofstream allNodeFile("_allnodes", ios::binary);
-	ofstream usedNodesFile("_usednodes", ios::binary);
-	ofstream wayFile("_ways", ios::binary);
     nodeMap->set_empty_key(UINT_MAX);
     try {
         while ( xmlTextReaderRead( inputReader ) == 1 ) {
@@ -74,27 +77,23 @@ int main (int argc, char *argv[])
                 continue;
 
             if ( xmlStrEqual( currentName, ( const xmlChar* ) "node" ) == 1 ) {
-				stats.numberOfNodes++;
                 _Node node = _ReadXMLNode( inputReader );
-				allNodeFile.write((char *)&node, sizeof(node));// << node.id << node.lat << node.lon << node.trafficSignal << endl;
+                allNodes.push_back(node);
-
                 if ( node.trafficSignal )
                     SignalNodes.push_back( node.id );
 
             }
             else if ( xmlStrEqual( currentName, ( const xmlChar* ) "way" ) == 1 ) {
-				stats.numberOfWays++;
+                _Way way = _ReadXMLWay( inputReader, settings );
-				_Way way = _ReadXMLWay( inputReader, settings, stats );
 
                 if ( way.usefull && way.access && way.path.size() ) {
                     for ( unsigned i = 0; i < way.path.size(); ++i ) {
-						usedNodesFile.write((char *)&way.path[i], sizeof(NodeID));
+                        usedNodes.push_back(way.path[i]);
                     }
 
                     if ( way.direction == _Way::opposite )
                         std::reverse( way.path.begin(), way.path.end() );
 
-					stats.numberOfEdges += ( int ) way.path.size() - 1;
                     {
                         vector< NodeID > & path = way.path;
                         double speed = way.maximumSpeed;
@@ -103,29 +102,23 @@ int main (int argc, char *argv[])
 
                         for(vector< NodeID >::size_type n = 0; n < path.size()-1; n++)
                         {
-							//serialize edge (path[n], path[n+1])
+                            _Edge e;
-							wayFile.write((char*)&way.path[n], sizeof(NodeID));
+                            e.start = way.path[n];
-							wayFile.write((char*)&way.path[n+1], sizeof(NodeID));
+                            e.target = way.path[n+1];
-							wayFile.write((char*)&way.type, sizeof(short));
+                            e.type = way.type;
-							wayFile.write((char*)&way.direction, sizeof(short));
+                            e.direction = way.direction;
-							wayFile.write((char*)&way.maximumSpeed, sizeof(double));
+                            e.speed = way.maximumSpeed;
+                            allEdges.push_back(e);
                         }
                     }
                 }
             }
             xmlFree( currentName );
         }
-		allNodeFile.close();
-		usedNodesFile.close();
-		wayFile.close();
         cout << "ok, after " << get_timestamp() - time << "s" << endl;
         time = get_timestamp();
         unsigned memory_to_use = 1024 * 1024 * 1024;
 
-		stxxl::syscall_file f("_usednodes", stxxl::file::DIRECT | stxxl::file::RDWR);
-		typedef stxxl::vector<NodeID> usedNodesVectorType;
-
-		usedNodesVectorType usedNodes( &f);
         cout << "Sorting used nodes ..." << flush;
         stxxl::sort(usedNodes.begin(), usedNodes.end(), Cmp(), memory_to_use);
         cout << "ok, after " << get_timestamp() - time << "s" << endl;
@@ -136,24 +129,11 @@ int main (int argc, char *argv[])
         cout << "ok, after " << get_timestamp() - time << "s" << endl;
         time = get_timestamp();
 
-		stxxl::syscall_file fallnodes("_allnodes", stxxl::file::DIRECT | stxxl::file::RDWR);
-		typedef stxxl::vector< _Node > second_vector_type;
-
-		second_vector_type van(&fallnodes);
         cout << "Sorting all nodes ..." << flush;
-		stxxl::ksort(van.begin(), van.end(), memory_to_use);
+        stxxl::ksort(allNodes.begin(), allNodes.end(), memory_to_use);
         cout << "ok, after " << get_timestamp() - time << "s" << endl;
         time = get_timestamp();
 
-		cout << endl << "Statistics: " << endl;
-		cout << "All Nodes: " << stats.numberOfNodes << endl;
-		cout << "Used Nodes: " << nodeMap->size() << endl;
-		cout << "Number of Ways: " << stats.numberOfWays << endl;
-		cout << "Edges in graph: " << stats.numberOfEdges << endl;
-		cout << "Number of ways with maxspeed information: " << stats.numberOfMaxspeed << endl;
-		cout << "Number of nodes with traffic lights: " << SignalNodes.size() << endl;
-		cout << "finished loading data" << endl;
-		cout << "calculated edge weights and writing to disk ..." << flush;
         string name(argv[1]);
         int pos=name.find(".osm"); // pos=9
         if(pos!=string::npos)
@@ -165,64 +145,84 @@ int main (int argc, char *argv[])
 
         ofstream fout;
         fout.open(name.c_str());
-		ifstream inall("_allnodes", ios::binary);
+        //        ifstream inway("_ways", ios::binary);
-		ifstream inuse("_usednodes", ios::binary);
-		ifstream inway("_ways", ios::binary);
 
         cout << "Writing used nodes ..." << flush;
-		fout << usedNodes.size() << endl;
         NodeID counter = 0;
-		for(usedNodesVectorType::iterator it = usedNodes.begin(); it!=usedNodes.end(); it++)
+        NodeID notfound = 0;
+        STXXLNodeVector::iterator nvit = allNodes.begin();
+        STXXLNodeIDVector::iterator niit = usedNodes.begin();
+        while(niit != usedNodes.end() && nvit != allNodes.end())
         {
-			NodeID currentNodeID = *it;
+            if(*niit < nvit->id){
-			_Node current_Node;
+                niit++;
-			inall.read((char *)&current_Node, sizeof(_Node));
+                continue;
-			while(currentNodeID!=current_Node.id)
+            }
+            if(*niit > nvit->id)
             {
-				inall.read((char *)&current_Node, sizeof(_Node));
+                nvit++;
+                continue;
             }
-			fout << current_Node.id<< " " << current_Node.lon << " " << current_Node.lat << "\n";
+            if(*niit == nvit->id)
-			nodeMap->insert(std::make_pair(current_Node.id, current_Node));
+            {
-			counter++;
+                confirmedNodes.push_back(*nvit);
+                nodeMap->insert(std::make_pair(nvit->id, *nvit));
+                niit++;
+                nvit++;
             }
+        }
+        fout << confirmedNodes.size() << endl;
+        for(STXXLNodeVector::iterator ut = confirmedNodes.begin(); ut != confirmedNodes.end(); ut++)
+        {
+            fout << ut->id<< " " << ut->lon << " " << ut->lat << "\n";
+        }
+
         cout << "ok, after " << get_timestamp() - time << "s" << endl;
         time = get_timestamp();
 
-		cout << "writing used ways ..." << endl;
+        cout << "confirming used ways ..." << endl;
-		NodeID start, target;
+        for(STXXLEdgeVector::iterator eit = allEdges.begin(); eit != allEdges.end(); eit++)
-		short type, direction;
-		double maximumSpeed;
-		fout << stats.numberOfEdges << "\n";
-		while(!inway.eof())
         {
+            assert(eit->type > -1 || eit->speed != -1);
 
-			inway.read((char*)&start, sizeof(NodeID));
+            NodeMap::iterator startit = nodeMap->find(eit->start);
-			inway.read((char*)&target, sizeof(NodeID));
-			inway.read((char*)&type, sizeof(short));
-			inway.read((char*)&direction, sizeof(short));
-			inway.read((char*)&maximumSpeed, sizeof(double));
-			assert(type > -1 || maximumSpeed != -1);
-
-			NodeMap::iterator startit = nodeMap->find(start);
             if(startit == nodeMap->end())
             {
-				cerr << "Node " << start << " missing albeit referenced in way. Edge skipped" << endl;
                 continue;
             }
-			NodeMap::iterator targetit = nodeMap->find(target);
+            NodeMap::iterator targetit = nodeMap->find(eit->target);
 
             if(targetit == nodeMap->end())
             {
-				cerr << "Node << " << target << "missing albeit reference in a way. Edge skipped" << endl;
                 continue;
             }
+            confirmedEdges.push_back(*eit);
+        }
+        fout << confirmedEdges.size() << "\n";
+        cout << "ok, after " << get_timestamp() - time << "s" << endl;
+        time = get_timestamp();
 
+        cout << "writing confirmed ways ..." << endl;
+
+        for(STXXLEdgeVector::iterator eit = confirmedEdges.begin(); eit != confirmedEdges.end(); eit++)
+        {
+            NodeMap::iterator startit = nodeMap->find(eit->start);
+            if(startit == nodeMap->end())
+            {
+                continue;
+            }
+            NodeMap::iterator targetit = nodeMap->find(eit->target);
+
+            if(targetit == nodeMap->end())
+            {
+                continue;
+            }
             double distance = ApproximateDistance(startit->second.lat, startit->second.lon, targetit->second.lat, targetit->second.lon);
-			if(maximumSpeed == -1)
+            if(eit->speed == -1)
-				maximumSpeed = settings.speedProfile.speed[type];
+                eit->speed = settings.speedProfile.speed[eit->type];
-			double weight = ( distance * 10. ) / (maximumSpeed / 3.6);
+            double weight = ( distance * 10. ) / (eit->speed / 3.6);
             double intWeight = max(1, (int) weight);
-			switch(direction)
+            switch(eit->direction)
             {
             case _Way::notSure:
                 fout << startit->first << " " << targetit->first << " " << max(1, (int)distance) << " " << 0 << " " << intWeight << "\n";
@@ -241,7 +241,6 @@ int main (int argc, char *argv[])
                 break;
             }
         }
-		inway.close();
         fout.close();
         cout << "ok, after " << get_timestamp() - time << "s" << endl;
         time = get_timestamp();
@@ -249,12 +248,20 @@ int main (int argc, char *argv[])
         cerr <<  "Caught Execption:" << e.what() << endl;
         return false;
     }
-	SignalNodes.clear();
-	xmlFreeTextReader(inputReader);
 
-	remove("_allnodes");
+    cout << endl << "Statistics:" << endl;
-	remove("_usednodes");
+    cout << "-----------" << endl;
-	remove("_ways");
+    cout << "Usable Nodes: " << confirmedNodes.size() << endl;
-	return true;
+    cout << "Usable Ways : " << confirmedEdges.size() << endl;
+
+    SignalNodes.clear();
+    usedNodes.clear();
+    allNodes.clear();
+    confirmedNodes.clear();
+    allEdges.clear();
+    confirmedEdges.clear();
+
+    xmlFreeTextReader(inputReader);
+    return 0;
 }
 

extractNetwork.cpp

@@ -18,180 +18,13 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 or see http://www.gnu.org/licenses/agpl.txt.
  */
 
-#include <cassert>
-#include <climits>
-#include <cstdlib>
 #include <iostream>
-#include <string>
-#include <sstream>
-#include <vector>
-#include <libxml/xmlreader.h>
-#include <google/sparse_hash_map>
 
-#include "typedefs.h"
-#include "DataStructures/extractorStructs.h"
 using namespace std;
-typedef google::dense_hash_map<NodeID, _Node> NodeMap;
-
-_Stats stats;
-Settings settings;
-NodeMap AllNodes;
-
-vector<NodeID> SignalNodes;
-vector<NodeID> UsedNodes;
-vector<_Way> UsedWays;
 
 int main (int argc, char *argv[])
 {
-    if(argc <= 1)
+    cerr << argv[0] << " is deprecated. Use extractLargeNetwork" << endl;
-    {
+    return -1;
-        cerr << "usage: " << endl << argv[0] << " <file.osm>" << endl;
-        exit(-1);
-    }
-    cout << "reading input file. This may take some time ..." << flush;
-
-    settings.speedProfile.names.insert(settings.speedProfile.names.begin(), names, names+13);
-    settings.speedProfile.speed.insert(settings.speedProfile.speed.begin(), speeds, speeds+13);
-
-    AllNodes.set_empty_key(UINT_MAX);
-    xmlTextReaderPtr inputReader = xmlNewTextReaderFilename( argv[1] );
-    ofstream nodeFile("_nodes", ios::binary);
-    ofstream wayFile("_ways", ios::binary);
-    try {
-        while ( xmlTextReaderRead( inputReader ) == 1 ) {
-            const int type = xmlTextReaderNodeType( inputReader );
-
-            //1 is Element
-            if ( type != 1 )
-                continue;
-
-            xmlChar* currentName = xmlTextReaderName( inputReader );
-            if ( currentName == NULL )
-                continue;
-
-            if ( xmlStrEqual( currentName, ( const xmlChar* ) "node" ) == 1 ) {
-                stats.numberOfNodes++;
-                _Node node = _ReadXMLNode( inputReader );
-                AllNodes.insert(make_pair(node.id, node) );
-
-                if ( node.trafficSignal )
-                    SignalNodes.push_back( node.id );
-
-            }
-            else if ( xmlStrEqual( currentName, ( const xmlChar* ) "way" ) == 1 ) {
-                stats.numberOfWays++;
-                _Way way = _ReadXMLWay( inputReader, settings, stats );
-
-                if ( way.usefull && way.access && way.path.size() ) {
-                    for ( unsigned i = 0; i < way.path.size(); ++i ) {
-                        UsedNodes.push_back( way.path[i] );
-                    }
-
-                    if ( way.direction == _Way::opposite )
-                        std::reverse( way.path.begin(), way.path.end() );
-
-                    stats.numberOfEdges += ( int ) way.path.size() - 1;
-
-                    UsedWays.push_back(way);
-                }
-            }
-            xmlFree( currentName );
-        }
-        sort(UsedNodes.begin(), UsedNodes.end());
-        UsedNodes.erase(unique(UsedNodes.begin(), UsedNodes.end()), UsedNodes.end() );
-        sort(SignalNodes.begin(), SignalNodes.end());
-        SignalNodes.erase(unique(SignalNodes.begin(), SignalNodes.end()), SignalNodes.end() );
-        cout << "ok" << endl;
-        cout << endl << "Statistics: " << endl;
-        cout << "All Nodes: " << stats.numberOfNodes << endl;
-        cout << "Used Nodes: " << UsedNodes.size() << endl;
-        cout << "Number of Ways: " << stats.numberOfWays << endl;
-        cout << "Edges in graph: " << stats.numberOfEdges << endl;
-        cout << "Number of ways with maxspeed information: " << stats.numberOfMaxspeed << endl;
-        cout << "Number of nodes with traffic lights: " << SignalNodes.size() << endl;
-        cout << "finished loading data" << endl;
-        cout << "calculated edge weights and writing to disk ..." << flush;
-        string name(argv[1]);
-        int pos=name.find(".osm"); // pos=9
-        if(pos!=string::npos)
-        {
-            //replace
-            name.replace(pos, 5, ".osrm");
-        } else {
-            name.append(".osrm");
-        }
-
-        ofstream fout;
-        fout.open(name.c_str());
-
-        fout << UsedNodes.size() << endl;
-        for(vector<NodeID>::size_type i = 0; i < UsedNodes.size(); i++)
-        {
-            NodeMap::iterator it = AllNodes.find(UsedNodes[i]);
-            assert(it!=AllNodes.end());
-            fout << UsedNodes[i] << " " << it->second.lon << " " << it->second.lat << "\n";
-        }
-        fout << flush;
-        UsedNodes.clear();
-        fout << stats.numberOfEdges << endl;
-        for(vector<_Way>::size_type i = 0; i < UsedWays.size(); i++)
-        {
-            vector< NodeID > & path = UsedWays[i].path;
-            double speed = UsedWays[i].maximumSpeed;
-            assert(UsedWays[i].type > -1 || UsedWays[i].maximumSpeed != -1);
-            assert(path.size()>0);
-
-            for(vector< NodeID >::size_type n = 0; n < path.size()-1; n++)
-            {
-                //insert path[n], path[n+1]
-                NodeMap::iterator startit = AllNodes.find(path[n]);
-                if(startit == AllNodes.end())
-                {
-                    cerr << "Node " << path[n] << " missing albeit referenced in way. Edge skipped" << endl;
-                    continue;
-                }
-                NodeMap::iterator targetit = AllNodes.find(path[n+1]);
-
-                if(targetit == AllNodes.end())
-                {
-                    cerr << "Node << " << path[n+1] << "missing albeit reference in a way. Edge skipped" << endl;
-                    continue;
-                }
-                double distance = ApproximateDistance(startit->second.lat, startit->second.lon, targetit->second.lat, targetit->second.lon);
-                if(speed == -1)
-                    speed = settings.speedProfile.speed[UsedWays[i].type];
-                double weight = ( distance * 10. ) / (speed / 3.6);
-                double intWeight = max(1, (int) weight);
-                switch(UsedWays[i].direction)
-                {
-                case _Way::notSure:
-                    fout << startit->first << " " << targetit->first << " " << max(1, (int)distance) << " " << 0 << " " << intWeight << "\n";
-                    break;
-                case _Way::oneway:
-                    fout << startit->first << " " << targetit->first << " " << max(1, (int)distance) << " " << 1 << " " << intWeight << "\n";
-                    break;
-                case _Way::bidirectional:
-                    fout << startit->first << " " << targetit->first << " " << max(1, (int)distance) << " " << 0 << " " << intWeight << "\n";
-                    break;
-                case _Way::opposite:
-                    fout << startit->first << " " << targetit->first << " " << max(1, (int)distance) << " " << 1 << " " << intWeight << "\n";
-                    break;
-                default:
-                    assert(false);
-                    break;
-                }
-            }
-        }
-        fout.close();
-        cout << "ok" << endl;
-    } catch ( const std::exception& e ) {
-        cerr <<  "Caught Execption:" << e.what() << endl;
-        return false;
-    }
-    AllNodes.clear();
-    SignalNodes.clear();
-    UsedWays.clear();
-    xmlFreeTextReader(inputReader);
-    return true;
 }