extraction is now able to handle bzip2 compressed input files

DataStructures/InputReaderFactory.h

@@ -0,0 +1,84 @@
+/*
+    open source routing machine
+    Copyright (C) Dennis Luxen, others 2010
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU AFFERO General Public License as published by
+the Free Software Foundation; either version 3 of the License, or
+any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+or see http://www.gnu.org/licenses/agpl.txt.
+ */
+
+#ifndef BZ2INPUTREADER_H
+#define BZ2INPUTREADER_H
+
+#include <stdio.h>
+#include <string.h>
+#include <libxml/xmlreader.h>
+
+#include <bzlib.h>
+
+struct Context {
+    FILE* file;
+    BZFILE* bz2;
+    bool error;
+};
+
+int readFromBz2Stream( void* pointer, char* buffer, int len )
+{
+    Context* context = (Context*) pointer;
+    if ( len == 0 || context->error )
+        return 0;
+
+    int error = 0;
+    int read = BZ2_bzRead( &error, context->bz2, buffer, len );
+    if ( error == BZ_OK )
+        return read;
+
+    context->error = true;
+    if ( error == BZ_STREAM_END )
+        return read;
+    return 0;
+}
+
+int closeBz2Stream( void *pointer )
+{
+    Context* context = (Context*) pointer;
+    BZ2_bzclose( context->bz2 );
+    fclose( context->file );
+    delete context;
+    return 0;
+}
+
+xmlTextReaderPtr inputReaderFactory( const char* name )
+{
+    std::string inputName(name);
+
+    if(inputName.find(".osm.bz2")!=string::npos)
+    {
+        Context* context = new Context;
+        context->error = false;
+        context->file = fopen( name, "r" );
+        int error;
+        context->bz2 = BZ2_bzReadOpen( &error, context->file, 0, 0, NULL, 0 );
+        if ( context->bz2 == NULL || context->file == NULL ) {
+            delete context;
+            return NULL;
+        }
+
+        return xmlReaderForIO( readFromBz2Stream, closeBz2Stream, (void*) context, NULL, NULL, 0 );
+    } else {
+        return xmlNewTextReaderFilename(name);
+    }
+}
+
+#endif // BZ2INPUTREADER_H

DataStructures/extractorStructs.h

@@ -21,6 +21,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
 #ifndef EXTRACTORSTRUCTS_H_
 #define EXTRACTORSTRUCTS_H_
 
+#include <climits>
 #include <cmath>
 
 /*     Default Speed Profile:
@@ -62,6 +63,12 @@ struct _Node : NodeInfo{
 	}
 };
 
+struct _Coordinate {
+	int lat;
+	int lon;
+	_Coordinate () : lat(INT_MIN), lon(INT_MIN) {};
+};
+
 struct _Way {
 	std::vector< NodeID > path;
 	enum {
@@ -74,11 +81,16 @@ struct _Way {
 };
 
 struct _Edge {
+	_Edge() {};
+	_Edge(NodeID s, NodeID t) : start(s), target(t) { }
 	NodeID start;
 	NodeID target;
 	short type;
 	short direction;
 	double speed;
+
+	_Coordinate startCoord;
+	_Coordinate targetCoord;
 };
 
 struct Settings {
@@ -101,7 +113,7 @@ struct Settings {
 
 struct Cmp : public std::binary_function<NodeID, NodeID, bool>
 {
-    typedef unsigned value_type;
+	typedef NodeID value_type;
 	bool operator ()  (const NodeID & a, const NodeID & b) const
 	{
 		return a < b;
@@ -116,6 +128,40 @@ struct Cmp : public std::binary_function<NodeID, NodeID, bool>
 	}
 };
 
+struct CompareEdgeByStart : public std::binary_function<_Edge, _Edge, bool>
+{
+	typedef _Edge value_type;
+	bool operator ()  (const _Edge & a, const _Edge & b) const
+	{
+		return a.start < b.start;
+	}
+	value_type max_value()
+	{
+		return _Edge(UINT_MAX, UINT_MAX);
+	}
+	value_type min_value()
+	{
+		return _Edge(0, 0);
+	}
+};
+
+struct CompareEdgeByTarget : public std::binary_function<_Edge, _Edge, bool>
+{
+	typedef _Edge value_type;
+	bool operator ()  (const _Edge & a, const _Edge & b) const
+	{
+		return a.target < b.target;
+	}
+	value_type max_value()
+	{
+		return _Edge(UINT_MAX, UINT_MAX);
+	}
+	value_type min_value()
+	{
+		return _Edge(0, 0);
+	}
+};
+
 _Way _ReadXMLWay( xmlTextReaderPtr& inputReader, Settings& settings ) {
 	_Way way;
 	way.direction = _Way::notSure;

Docs/3rdparty.txt

@@ -4,3 +4,4 @@ Scons		1.3+
 Boost		1.37+
 sparsehash 	1.4+
 stxxl		1.2.1+
+libz2-dev	1.0.5+

SConstruct

@@ -53,6 +53,9 @@ if not conf.CheckCXXHeader('google/sparse_hash_map'):
 if not conf.CheckCXXHeader('boost/asio.hpp'):
 	print "boost/asio.hpp not found. Exiting"
 	Exit(-1)
+if not conf.CheckLibWithHeader('bz2', 'bzlib.h', 'CXX'):
+	print "bz2 library not found. Exiting"
+	Exit(-1)
 if not conf.CheckLib('boost_thread'):
 	if not conf.CheckLib('boost_thread-mt'):
 		print "boost thread library not found. Exiting"
@@ -87,8 +90,8 @@ env.Append(LINKFLAGS = ' -fopenmp')
 env.Program("extractNetwork.cpp")
 env.Program("extractLargeNetwork.cpp")	
 env.Program("createHierarchy.cpp")
-env.Append(CCFLAGS = ' -lboost_regex -lboost_iostreams -lboost_system')
-env.Append(LINKFLAGS = ' -lboost_regex -lboost_iostreams -lboost_system')
+env.Append(CCFLAGS = ' -lboost_regex -lboost_iostreams -lboost_system -lbz2')
+env.Append(LINKFLAGS = ' -lboost_regex -lboost_iostreams -lboost_system -lbz2')
 env.Program("routed.cpp")
 env = conf.Finish()
 

extractLargeNetwork.cpp

@@ -36,6 +36,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
 #include <stxxl.h>
 
 #include "typedefs.h"
+#include "DataStructures/InputReaderFactory.h"
 #include "DataStructures/extractorStructs.h"
 
 using namespace std;
@@ -61,11 +62,12 @@ int main (int argc, char *argv[])
         exit(-1);
     }
     cout << "reading input file. This may take some time ..." << flush;
+    xmlTextReaderPtr inputReader = inputReaderFactory(argv[1]);
+
     double time = get_timestamp();
     settings.speedProfile.names.insert(settings.speedProfile.names.begin(), names, names+13);
     settings.speedProfile.speed.insert(settings.speedProfile.speed.begin(), speeds, speeds+13);
 
-	xmlTextReaderPtr inputReader = xmlNewTextReaderFilename( argv[1] );
     nodeMap->set_empty_key(UINT_MAX);
     try {
         while ( xmlTextReaderRead( inputReader ) == 1 ) {
@@ -118,6 +120,9 @@ int main (int argc, char *argv[])
             }
             xmlFree( currentName );
         }
+        cout << "raw no. of nodes: " << allNodes.size() << endl;
+        cout << "raw no. of edges: " << allEdges.size() << endl;
+
         cout << "ok, after " << get_timestamp() - time << "s" << endl;
         time = get_timestamp();
         unsigned memory_to_use = 1024 * 1024 * 1024;
@@ -138,17 +143,23 @@ int main (int argc, char *argv[])
         time = get_timestamp();
 
         string name(argv[1]);
-		int pos=name.find(".osm"); // pos=9
+        int pos;
+        pos = name.find(".osm.bz2");
+        if(pos!=string::npos)
+        {
+            name.replace(pos, 8, ".osrm");
+        } else {
+            pos=name.find(".osm");
             if(pos!=string::npos)
             {
                 name.replace(pos, 5, ".osrm");
             } else {
                 name.append(".osrm");
             }
+        }
 
         ofstream fout;
         fout.open(name.c_str());
-		//        ifstream inway("_ways", ios::binary);
 
         cout << "Confirming used nodes ..." << flush;
         NodeID counter = 0;
@@ -268,7 +279,7 @@ int main (int argc, char *argv[])
     allEdges.clear();
     confirmedEdges.clear();
 
-	xmlFreeTextReader(inputReader);
+    //    xmlFreeTextReader(inputReader);
     return 0;
 }