From ad77d6cfecd7700d2a51cfcd59fc02632c40fb2f Mon Sep 17 00:00:00 2001 From: DennisOSRM Date: Thu, 15 Dec 2011 17:48:00 +0100 Subject: [PATCH] graph is built in-place, so memory peak usage is halved. --- DataStructures/StaticGraph.h | 32 ++++++++------- Plugins/ObjectForPluginStruct.h | 12 ++++-- Util/GraphLoader.h | 25 ++++++++++++ createHierarchy.cpp | 71 ++++++++++++++++++++++++++++----- 4 files changed, 112 insertions(+), 28 deletions(-) diff --git a/DataStructures/StaticGraph.h b/DataStructures/StaticGraph.h index cb37590e5..62c8295c2 100644 --- a/DataStructures/StaticGraph.h +++ b/DataStructures/StaticGraph.h @@ -49,12 +49,18 @@ public: } }; + struct _StrNode { + //index of the first edge + EdgeIterator firstEdge; + }; + + struct _StrEdge { + NodeID target; + EdgeDataT data; + }; + StaticGraph( const int nodes, std::vector< InputEdge > &graph ) { -#ifdef _GLIBCXX_PARALLEL - __gnu_parallel::sort( graph.begin(), graph.end() ); -#else std::sort( graph.begin(), graph.end() ); -#endif _numNodes = nodes; _numEdges = ( EdgeIterator ) graph.size(); _nodes.resize( _numNodes + 1); @@ -79,6 +85,14 @@ public: } } + StaticGraph( std::vector<_StrNode> & nodes, std::vector<_StrEdge> & edges) { + _numNodes = nodes.size(); + _numEdges = edges.size(); + + _nodes.swap(nodes); + _edges.swap(edges); + } + unsigned GetNumberOfNodes() const { return _numNodes; } @@ -142,16 +156,6 @@ public: private: - struct _StrNode { - //index of the first edge - EdgeIterator firstEdge; - }; - - struct _StrEdge { - NodeID target; - EdgeDataT data; - }; - NodeIterator _numNodes; EdgeIterator _numEdges; diff --git a/Plugins/ObjectForPluginStruct.h b/Plugins/ObjectForPluginStruct.h index 945ae7fc8..10657e14e 100644 --- a/Plugins/ObjectForPluginStruct.h +++ b/Plugins/ObjectForPluginStruct.h @@ -41,10 +41,14 @@ struct ObjectsForQueryStruct { std::cout << "[objects] loading query data structures ..." << std::flush; ifstream hsgrInStream(hsgrPath.c_str(), ios::binary); //Deserialize road network graph - std::vector< InputEdge> edgeList; - const int n = readHSGRFromStream(hsgrInStream, edgeList); - graph = new QueryGraph(n, edgeList); - std::vector< InputEdge >().swap( edgeList ); //free memory + std::vector< QueryGraph::_StrNode> nodeList; + std::vector< QueryGraph::_StrEdge> edgeList; + const int n = readHSGRFromStream(hsgrInStream, nodeList, edgeList); + graph = new QueryGraph(nodeList, edgeList); + INFO("Graph has " << graph->GetNumberOfNodes() << " nodes"); + INFO("Graph has " << graph->GetNumberOfEdges() << " edges"); + + //Init nearest neighbor data structure ifstream nodesInStream(nodesPath.c_str(), ios::binary); diff --git a/Util/GraphLoader.h b/Util/GraphLoader.h index 30074d577..54bd56e25 100644 --- a/Util/GraphLoader.h +++ b/Util/GraphLoader.h @@ -354,6 +354,31 @@ NodeID readDDSGGraphFromStream(istream &in, vector& edgeList, vector +unsigned readHSGRFromStream(istream &in, vector& nodeList, vector & edgeList) { + unsigned numberOfNodes = 0; + in.read((char*) & numberOfNodes, sizeof(unsigned)); + INFO("Loading " << numberOfNodes << " nodes"); + nodeList.resize(numberOfNodes); + NodeT currentNode; + for(unsigned nodeCounter = 0; nodeCounter < numberOfNodes; ++nodeCounter ) { + in.read((char*) ¤tNode, sizeof(NodeT)); + nodeList[nodeCounter] = currentNode; + } + + unsigned numberOfEdges = 0; + in.read((char*) &numberOfEdges, sizeof(unsigned)); + INFO("Loading " << numberOfEdges << " edges"); + edgeList.resize(numberOfEdges); + EdgeT currentEdge; + for(unsigned edgeCounter = 0; edgeCounter < numberOfEdges; ++edgeCounter) { + in.read((char*) ¤tEdge, sizeof(EdgeT)); + edgeList[edgeCounter] = currentEdge; + } + + return numberOfNodes; +} + template unsigned readHSGRFromStream(istream &in, vector & edgeList) { NodeID numberOfNodes = 0; diff --git a/createHierarchy.cpp b/createHierarchy.cpp index bf5a2dba1..fc6bbacb8 100644 --- a/createHierarchy.cpp +++ b/createHierarchy.cpp @@ -90,11 +90,11 @@ int main (int argc, char *argv[]) { INFO("preprocessing data from input file " << argv[1] << " using STL " #ifdef _GLIBCXX_PARALLEL - "parallel (GCC)" + "parallel (GCC)" #else - "serial" + "serial" #endif - " mode"); + " mode"); ifstream in; in.open (argv[1], ifstream::in | ifstream::binary); if (!in.is_open()) { @@ -160,18 +160,69 @@ int main (int argc, char *argv[]) { cleanup->GetData(cleanedEdgeList); DELETE( cleanup ); - INFO("Serializing edges "); + INFO("Building Node Array"); + // std::sort( graph.begin(), graph.end() ); + sort(cleanedEdgeList.begin(), cleanedEdgeList.end()); + unsigned numberOfNodes = 0; + unsigned numberOfEdges = cleanedEdgeList.size(); + INFO("Serializing compacted graph"); ofstream edgeOutFile(edgeOut, ios::binary); - Percent p(cleanedEdgeList.size()); + BOOST_FOREACH(InputEdge & edge, cleanedEdgeList) { - p.printIncrement(); - edgeOutFile.write((char *)&(edge.data), sizeof(EdgeData)); - edgeOutFile.write((char *)&(edge.source), sizeof(NodeID)); - edgeOutFile.write((char *)&(edge.target), sizeof(NodeID)); + if(edge.source > numberOfNodes) { + numberOfNodes = edge.source; + } + if(edge.target > numberOfNodes) { + numberOfNodes = edge.target; + } } + // _numNodes = nodes; + // _numEdges = ( EdgeIterator ) graph.size(); + numberOfNodes+=1; + + std::vector< StaticGraph::_StrNode > _nodes; +// std::vector< StaticGraph::_StrEdge > _edges; + _nodes.resize( numberOfNodes); + + StaticGraph::EdgeIterator edge = 0; + StaticGraph::EdgeIterator position = 0; + for ( StaticGraph::NodeIterator node = 0; node <= numberOfNodes; ++node ) { + StaticGraph::EdgeIterator lastEdge = edge; + while ( edge < numberOfEdges && cleanedEdgeList[edge].source == node ) + ++edge; + _nodes[node].firstEdge = position; //=edge + position += edge - lastEdge; //remove + } + //Serialize numberOfNodes, nodes + edgeOutFile.write((char*) &numberOfNodes, sizeof(unsigned)); + edgeOutFile.write((char*) &_nodes[0], sizeof(StaticGraph::_StrNode)*(numberOfNodes)); + + + //Serialize number of Edges + edgeOutFile.write((char*) &position, sizeof(unsigned)); + + edge = 0; + int counter = 0; + StaticGraph::_StrEdge currentEdge; + for ( StaticGraph::NodeIterator node = 0; node < numberOfNodes; ++node ) { + for ( StaticGraph::EdgeIterator i = _nodes[node].firstEdge, e = _nodes[node+1].firstEdge; i != e; ++i ) { + + currentEdge.target = cleanedEdgeList[edge].target; + currentEdge.data = cleanedEdgeList[edge].data; + if(currentEdge.data.distance <= 0) { + INFO("Edge: " << i << ",source: " << cleanedEdgeList[edge].source << ", target: " << cleanedEdgeList[edge].target << ", dist: " << currentEdge.data.distance); + ERR("Failed at edges of node " << node << " of " << numberOfNodes); + } + //Serialize edges + edgeOutFile.write((char*) ¤tEdge, sizeof(StaticGraph::_StrEdge)); + ++edge; + ++counter; + } + } + INFO("Written " << counter << " edges, expected " << position); edgeOutFile.close(); cleanedEdgeList.clear(); - + _nodes.clear(); INFO("finished preprocessing"); return 0; }