From 666371099e1563c49c11137319c52d374a9ca82f Mon Sep 17 00:00:00 2001 From: DennisOSRM Date: Wed, 23 May 2012 21:18:38 +0200 Subject: [PATCH] Independent set tie-breaker in O(1) space and time --- Contractor/Contractor.h | 106 +++++++++++++++++++-------- Contractor/EdgeBasedGraphFactory.cpp | 13 ++-- Contractor/EdgeBasedGraphFactory.h | 5 +- DataStructures/ImportEdge.h | 2 +- DataStructures/NNGrid.h | 23 ++---- createHierarchy.cpp | 12 +-- 6 files changed, 93 insertions(+), 68 deletions(-) diff --git a/Contractor/Contractor.h b/Contractor/Contractor.h index b9c58b762..3dec197e3 100644 --- a/Contractor/Contractor.h +++ b/Contractor/Contractor.h @@ -31,7 +31,7 @@ or see http://www.gnu.org/licenses/agpl.txt. #include - +#include "../DataStructures/DeallocatingVector.h" #include "../DataStructures/DynamicGraph.h" #include "../DataStructures/Percent.h" #include "../DataStructures/BinaryHeap.h" @@ -77,8 +77,7 @@ private: struct _PriorityData { int depth; - NodeID bias; - _PriorityData() : depth(0), bias(0) { } + _PriorityData() : depth(0) { } }; struct _ContractionInformation { @@ -95,17 +94,39 @@ private: } }; + class XORFastHash { + std::vector hashLookups; + public: + XORFastHash() { + hashLookups.resize(1 << 16); + for(unsigned i = 0; i < (1 << 16); ++i) + hashLookups[i] = i; + std::random_shuffle(hashLookups.begin(), hashLookups.end()); + } + unsigned short operator()(const unsigned originalValue) const { + + unsigned short msb = (((originalValue-1) >> 16) & 0xffff); + unsigned short lsb = ((originalValue-1) & 0xffff); + return hashLookups[lsb] ^ msb; + } + }; + public: template Contractor( int nodes, ContainerT& inputEdges) { - std::vector< _ContractorEdge > edges; - edges.reserve( 2 * inputEdges.size() ); - BOOST_FOREACH(typename ContainerT::value_type & currentEdge, inputEdges) { + DeallocatingVector< _ContractorEdge > edges; + + typename ContainerT::deallocation_iterator diter = inputEdges.dbegin(); + typename ContainerT::deallocation_iterator dend = inputEdges.dend(); + + + //BOOST_FOREACH(typename ContainerT::value_type & currentEdge, inputEdges) { + for(; diter!=dend; ++diter) { _ContractorEdge edge; - edge.source = currentEdge.source(); - edge.target = currentEdge.target(); - edge.data = _ContractorEdgeData( (std::max)((int)currentEdge.weight(), 1 ), 1, currentEdge.id()/*, currentEdge.getNameIDOfTurnTarget(), currentEdge.turnInstruction()*/, false, currentEdge.isForward(), currentEdge.isBackward()); + edge.source = diter->source(); + edge.target = diter->target(); + edge.data = _ContractorEdgeData( (std::max)((int)diter->weight(), 1 ), 1, diter->id()/*, currentEdge.getNameIDOfTurnTarget(), currentEdge.turnInstruction()*/, false, diter->isForward(), diter->isBackward()); assert( edge.data.distance > 0 ); #ifndef NDEBUG @@ -116,12 +137,12 @@ public: #endif edges.push_back( edge ); std::swap( edge.source, edge.target ); - edge.data.forward = currentEdge.isBackward(); - edge.data.backward = currentEdge.isForward(); + edge.data.forward = diter->isBackward(); + edge.data.backward = diter->isForward(); edges.push_back( edge ); } //clear input vector and trim the current set of edges with the well-known swap trick - ContainerT().swap( inputEdges ); + inputEdges.clear(); sort( edges.begin(), edges.end() ); NodeID edge = 0; @@ -169,7 +190,7 @@ public: } } std::cout << "merged " << edges.size() - edge << " edges out of " << edges.size() << std::endl; - edges.resize( edge ); +// edges.resize( edge ); _graph.reset( new _DynamicGraph( nodes, edges ) ); edges.clear(); @@ -213,16 +234,13 @@ public: NodeID numberOfContractedNodes = 0; std::vector< std::pair< NodeID, bool > > remainingNodes( numberOfNodes ); - std::vector< double > nodePriority( numberOfNodes ); + std::vector< float > nodePriority( numberOfNodes ); std::vector< _PriorityData > nodeData( numberOfNodes ); //initialize the variables #pragma omp parallel for schedule ( guided ) for ( int x = 0; x < ( int ) numberOfNodes; ++x ) remainingNodes[x].first = x; - std::random_shuffle( remainingNodes.begin(), remainingNodes.end() ); - for ( int x = 0; x < ( int ) numberOfNodes; ++x ) - nodeData[remainingNodes[x].first].bias = x; std::cout << "initializing elimination PQ ..." << std::flush; #pragma omp parallel @@ -238,7 +256,7 @@ public: bool flushedContractor = false; while ( numberOfContractedNodes < numberOfNodes ) { if(!flushedContractor && (numberOfContractedNodes > (numberOfNodes*0.65) ) ){ - std::vector<_ContractorEdge> newSetOfEdges; //this one is not explicitely cleared since it goes out of scope anywa + DeallocatingVector<_ContractorEdge> newSetOfEdges; //this one is not explicitely cleared since it goes out of scope anywa std::cout << " [flush " << numberOfContractedNodes << " nodes] " << std::flush; //Delete old heap data to free memory that we need for the coming operations @@ -249,7 +267,7 @@ public: //Create new priority array - std::vector newNodePriority(remainingNodes.size()); + std::vector newNodePriority(remainingNodes.size()); //this map gives the old IDs from the new ones, necessary to get a consistent graph at the end of contraction oldNodeIDFromNewNodeIDMap.resize(remainingNodes.size()); //this map gives the new IDs from the old ones, necessary to remap targets from the remaining graph @@ -309,7 +327,7 @@ public: //Replace old priorities array by new one nodePriority.swap(newNodePriority); //Delete old nodePriority vector - std::vector().swap(newNodePriority); + std::vector().swap(newNodePriority); //old Graph is removed _graph.reset(); @@ -318,6 +336,7 @@ public: //int nodes, const ContainerT &graph _graph.reset( new _DynamicGraph(remainingNodes.size(), newSetOfEdges)); + newSetOfEdges.clear(); flushedContractor = true; //INFO: MAKE SURE THIS IS THE LAST OPERATION OF THE FLUSH! @@ -325,7 +344,6 @@ public: for ( unsigned threadNum = 0; threadNum < maxThreads; ++threadNum ) { threadData.push_back( new _ThreadData( _graph->GetNumberOfNodes() ) ); } - } const int last = ( int ) remainingNodes.size(); @@ -417,8 +435,8 @@ public: // // avgdegree /= std::max((unsigned)1,(unsigned)remainingNodes.size() ); // quaddegree /= std::max((unsigned)1,(unsigned)remainingNodes.size() ); - - // INFO("rest: " << remainingNodes.size() << ", max: " << maxdegree << ", min: " << mindegree << ", avg: " << avgdegree << ", quad: " << quaddegree); +// +// INFO("rest: " << remainingNodes.size() << ", max: " << maxdegree << ", min: " << mindegree << ", avg: " << avgdegree << ", quad: " << quaddegree); p.printStatus(numberOfContractedNodes); } @@ -429,11 +447,14 @@ public: } template< class Edge > - void GetEdges( std::vector< Edge >& edges ) { + void GetEdges( DeallocatingVector< Edge >& edges ) { + Percent p (_graph->GetNumberOfNodes()); + INFO("Getting edges of minimized graph"); NodeID numberOfNodes = _graph->GetNumberOfNodes(); if(oldNodeIDFromNewNodeIDMap.size()) { for ( NodeID node = 0; node < numberOfNodes; ++node ) { - for ( _DynamicGraph::EdgeIterator edge = _graph->BeginEdges( node ), endEdges = _graph->EndEdges( node ); edge < endEdges; edge++ ) { + p.printStatus(node); + for ( _DynamicGraph::EdgeIterator edge = _graph->BeginEdges( node ), endEdges = _graph->EndEdges( node ); edge < endEdges; ++edge ) { const NodeID target = _graph->GetTarget( edge ); const _DynamicGraph::EdgeData& data = _graph->GetEdgeData( edge ); Edge newEdge; @@ -456,7 +477,7 @@ public: } } } - INFO("Renumbered remaining edges, freeing space"); + INFO("Renumbered edges of minimized graph, freeing space"); _graph.reset(); std::vector().swap(oldNodeIDFromNewNodeIDMap); INFO("Loading temporary edges"); @@ -468,7 +489,7 @@ public: //loads edges of graph before renumbering, no need for further numbering action. NodeID start; NodeID target; - edges.reserve(edges.size()+numberOfTemporaryEdges); + //edges.reserve(edges.size()+numberOfTemporaryEdges); _DynamicGraph::EdgeData data; for(unsigned i = 0; i < numberOfTemporaryEdges; ++i) { temporaryEdgeStorage.read((char*)&start, sizeof(NodeID)); @@ -666,7 +687,7 @@ private: } } - bool _UpdateNeighbours( std::vector< double > & priorities, std::vector< _PriorityData > & nodeData, _ThreadData* const data, NodeID node) { + bool _UpdateNeighbours( std::vector< float > & priorities, std::vector< _PriorityData > & nodeData, _ThreadData* const data, NodeID node) { std::vector< NodeID >& neighbours = data->neighbours; neighbours.clear(); @@ -691,7 +712,7 @@ private: return true; } - bool _IsIndependent( const std::vector< double >& priorities, const std::vector< _PriorityData >& nodeData, _ThreadData* const data, NodeID node ) { + bool _IsIndependent( const std::vector< float >& priorities, const std::vector< _PriorityData >& nodeData, _ThreadData* const data, NodeID node ) { const double priority = priorities[node]; std::vector< NodeID >& neighbours = data->neighbours; @@ -699,14 +720,17 @@ private: for ( _DynamicGraph::EdgeIterator e = _graph->BeginEdges( node ) ; e < _graph->EndEdges( node ) ; ++e ) { const NodeID target = _graph->GetTarget( e ); + if(node==target) + continue; const double targetPriority = priorities[target]; assert( targetPriority >= 0 ); //found a neighbour with lower priority? if ( priority > targetPriority ) return false; //tie breaking - if ( priority == targetPriority && nodeData[node].bias < nodeData[target].bias ) + if ( priority == targetPriority && bias(node, target) ) { return false; + } neighbours.push_back( target ); } @@ -719,26 +743,44 @@ private: for ( _DynamicGraph::EdgeIterator e = _graph->BeginEdges( u ) ; e < _graph->EndEdges( u ) ; ++e ) { const NodeID target = _graph->GetTarget( e ); + if(node==target) + continue; const double targetPriority = priorities[target]; assert( targetPriority >= 0 ); //found a neighbour with lower priority? - if ( priority > targetPriority ) + if ( priority > targetPriority) return false; //tie breaking - if ( priority == targetPriority && nodeData[node].bias < nodeData[target].bias ) + if ( priority == targetPriority && bias(node, target) ) { return false; + } } } return true; } + inline bool bias(const NodeID a, const NodeID b) { + unsigned short hasha = fastHash(a); + unsigned short hashb = fastHash(b); + + //Decision with branching +// if(hasha != hashb) +// return hasha < hashb; +// return a < b; + + //Decision without branching + return (hasha < hashb)+((hasha==hashb)*(a _graph; std::vector<_DynamicGraph::InputEdge> contractedEdges; std::string temporaryEdgeStorageFilename; std::vector oldNodeIDFromNewNodeIDMap; long initialFilePosition; + + XORFastHash fastHash; }; #endif // CONTRACTOR_H_INCLUDED diff --git a/Contractor/EdgeBasedGraphFactory.cpp b/Contractor/EdgeBasedGraphFactory.cpp index 3c4bafad7..bf8c16b7d 100644 --- a/Contractor/EdgeBasedGraphFactory.cpp +++ b/Contractor/EdgeBasedGraphFactory.cpp @@ -85,7 +85,7 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory(int nodes, std::vector edges; + DeallocatingVector< _NodeBasedEdge > edges; // edges.reserve( 2 * inputEdges.size() ); for ( std::vector< NodeBasedEdge >::const_iterator i = inputEdges.begin(); i != inputEdges.end(); ++i ) { @@ -123,13 +123,13 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory(int nodes, std::vector().swap(inputEdges); - std::vector<_NodeBasedEdge>(edges).swap(edges); + //std::vector<_NodeBasedEdge>(edges).swap(edges); std::sort( edges.begin(), edges.end() ); _nodeBasedGraph.reset(new _NodeBasedDynamicGraph( nodes, edges )); } -void EdgeBasedGraphFactory::GetEdgeBasedEdges(std::vector< EdgeBasedEdge >& outputEdgeList ) { +void EdgeBasedGraphFactory::GetEdgeBasedEdges(DeallocatingVector< EdgeBasedEdge >& outputEdgeList ) { GUARANTEE(0 == outputEdgeList.size(), "Vector passed to EdgeBasedGraphFactory::GetEdgeBasedEdges(..) is not empty"); GUARANTEE(0 != edgeBasedEdges.size(), "No edges in edge based graph"); @@ -255,9 +255,6 @@ void EdgeBasedGraphFactory::Run(const char * originalEdgeDataFilename) { //distance += heightPenalty; //distance += ComputeTurnPenalty(u, v, w); assert(edgeData1.edgeBasedNodeID != edgeData2.edgeBasedNodeID); - if(edgeBasedEdges.size() == edgeBasedEdges.capacity()-3) { - edgeBasedEdges.reserve(edgeBasedEdges.size()*1.2); - } if(originalEdgeData.size() == originalEdgeData.capacity()-3) { originalEdgeData.reserve(originalEdgeData.size()*1.2); } @@ -290,7 +287,9 @@ void EdgeBasedGraphFactory::Run(const char * originalEdgeDataFilename) { INFO("Removing duplicate nodes (if any)"); edgeBasedNodes.erase( std::unique(edgeBasedNodes.begin(), edgeBasedNodes.end()), edgeBasedNodes.end() ); INFO("Applying vector self-swap trick to free up memory"); - edgeBasedNodes.swap(edgeBasedNodes); + INFO("size: " << edgeBasedNodes.size() << ", cap: " << edgeBasedNodes.capacity()); + std::vector(edgeBasedNodes).swap(edgeBasedNodes); + INFO("size: " << edgeBasedNodes.size() << ", cap: " << edgeBasedNodes.capacity()); INFO("Node-based graph contains " << nodeBasedEdgeCounter << " edges"); INFO("Edge-based graph contains " << edgeBasedEdges.size() << " edges, blowup is " << (double)edgeBasedEdges.size()/(double)nodeBasedEdgeCounter); INFO("Edge-based graph skipped " << numberOfSkippedTurns << " turns, defined by " << numberOfTurnRestrictions << " restrictions."); diff --git a/Contractor/EdgeBasedGraphFactory.h b/Contractor/EdgeBasedGraphFactory.h index 762ca35b7..18fd2c976 100644 --- a/Contractor/EdgeBasedGraphFactory.h +++ b/Contractor/EdgeBasedGraphFactory.h @@ -34,6 +34,7 @@ #include #include "../typedefs.h" +#include "../DataStructures/DeallocatingVector.h" #include "../DataStructures/DynamicGraph.h" #include "../DataStructures/ExtractorStructs.h" #include "../DataStructures/HashTable.h" @@ -103,7 +104,7 @@ private: RestrictionMap _restrictionMap; - std::vector edgeBasedEdges; + DeallocatingVector edgeBasedEdges; std::vector edgeBasedNodes; std::vector originalEdgeData; std::vector inputNodeInfoList; @@ -127,7 +128,7 @@ public: explicit EdgeBasedGraphFactory(int nodes, std::vector & inputEdges, std::vector & _bollardNodes, std::vector & trafficLights, std::vector<_Restriction> & inputRestrictions, std::vector & nI, boost::property_tree::ptree speedProfile, std::string & srtm); void Run(const char * originalEdgeDataFilename); - void GetEdgeBasedEdges( std::vector< EdgeBasedEdge >& edges ); + void GetEdgeBasedEdges( DeallocatingVector< EdgeBasedEdge >& edges ); void GetEdgeBasedNodes( std::vector< EdgeBasedNode> & nodes); void GetOriginalEdgeData( std::vector< OriginalEdgeData> & originalEdgeData); short AnalyzeTurn(const NodeID u, const NodeID v, const NodeID w) const; diff --git a/DataStructures/ImportEdge.h b/DataStructures/ImportEdge.h index 61190d056..a90cd474d 100644 --- a/DataStructures/ImportEdge.h +++ b/DataStructures/ImportEdge.h @@ -103,7 +103,7 @@ public: /** Default constructor. target and weight are set to 0.*/ EdgeBasedEdge() : - _source(0), _target(0), _edgeID(0)/*, _nameID1(0)*/, _weight(0), _forward(0), _backward(0)/*, _turnInstruction(0)*/ { assert(false); } //shall not be used. + _source(0), _target(0), _edgeID(0), _weight(0), _forward(false), _backward(false) { } explicit EdgeBasedEdge(NodeID s, NodeID t, NodeID v, EdgeWeight w, bool f, bool b) : _source(s), _target(t), _edgeID(v), _weight(w), _forward(f), _backward(b){} diff --git a/DataStructures/NNGrid.h b/DataStructures/NNGrid.h index c46498ac7..04f1e3f3b 100644 --- a/DataStructures/NNGrid.h +++ b/DataStructures/NNGrid.h @@ -42,25 +42,14 @@ or see http://www.gnu.org/licenses/agpl.txt. #include "ExtractorStructs.h" #include "GridEdge.h" -#include "LRUCache.h" #include "Percent.h" #include "PhantomNodes.h" #include "Util.h" #include "StaticGraph.h" #include "../Algorithms/Bresenham.h" -static const unsigned MAX_CACHE_ELEMENTS = 1000; - namespace NNGrid{ -//struct IdenticalHashFunction { -//public: -// inline unsigned operator ()(const unsigned value) const { -// return value; -// } -//}; - - static boost::thread_specific_ptr localStream; template @@ -99,7 +88,7 @@ public: } template - void ConstructGrid(std::vector & edgeList, char * ramIndexOut, char * fileIndexOut) { + inline void ConstructGrid(std::vector & edgeList, char * ramIndexOut, char * fileIndexOut) { //TODO: Implement this using STXXL-Streams #ifndef ROUTED Percent p(edgeList.size()); @@ -141,7 +130,7 @@ public: entriesInFileWithRAMSameIndex.push_back(gridEntry); } cellMap.clear(); - BuildCellIndexToFileIndexMap(indexInRamTable, cellMap); + BuildCellIndexToFileIndexMap(indexInRamTable, cellMap); /*unsigned numberOfBytesInCell = */FillCell(entriesInFileWithRAMSameIndex, lastPositionInIndexFile, cellMap); ramIndexTable[indexInRamTable] = lastPositionInIndexFile; entriesInFileWithRAMSameIndex.clear(); @@ -324,7 +313,7 @@ private: return (std::fabs(d1 - d2) < 0.0001); } - unsigned FillCell(std::vector& entriesWithSameRAMIndex, const unsigned long fileOffset, boost::unordered_map< unsigned, unsigned > & cellMap ) { + inline unsigned FillCell(std::vector& entriesWithSameRAMIndex, const unsigned long fileOffset, boost::unordered_map< unsigned, unsigned > & cellMap ) { std::vector tmpBuffer(32*32*4096,0); unsigned long indexIntoTmpBuffer = 0; unsigned numberOfWrittenBytes = 0; @@ -376,7 +365,7 @@ private: return numberOfWrittenBytes; } - unsigned FlushEntriesWithSameFileIndexToBuffer( std::vector &vectorWithSameFileIndex, std::vector & tmpBuffer, const unsigned long index) { + inline unsigned FlushEntriesWithSameFileIndexToBuffer( std::vector &vectorWithSameFileIndex, std::vector & tmpBuffer, const unsigned long index) const { sort( vectorWithSameFileIndex.begin(), vectorWithSameFileIndex.end() ); vectorWithSameFileIndex.erase(unique(vectorWithSameFileIndex.begin(), vectorWithSameFileIndex.end()), vectorWithSameFileIndex.end()); const unsigned lengthOfBucket = vectorWithSameFileIndex.size(); @@ -472,7 +461,7 @@ private: localStream->read((char *)&result[currentSizeOfResult], lengthOfBucket*sizeof(_GridEdge)); } - void AddEdge(const _GridEdge & edge) { + inline void AddEdge(const _GridEdge & edge) { #ifndef ROUTED std::vector indexList; GetListOfIndexesForEdgeAndGridSize(edge.startCoord, edge.targetCoord, indexList); @@ -521,7 +510,7 @@ private: return (p-x)*(p-x) + (q-y)*(q-y); } - void GetListOfIndexesForEdgeAndGridSize(const _Coordinate& start, const _Coordinate& target, std::vector &indexList) const { + inline void GetListOfIndexesForEdgeAndGridSize(const _Coordinate& start, const _Coordinate& target, std::vector &indexList) const { double lat1 = start.lat/100000.; double lon1 = start.lon/100000.; diff --git a/createHierarchy.cpp b/createHierarchy.cpp index 8f268b38d..e1d8c92c5 100644 --- a/createHierarchy.cpp +++ b/createHierarchy.cpp @@ -43,6 +43,7 @@ or see http://www.gnu.org/licenses/agpl.txt. #include "Contractor/Contractor.h" #include "Contractor/EdgeBasedGraphFactory.h" #include "DataStructures/BinaryHeap.h" +#include "DataStructures/DeallocatingVector.h" #include "DataStructures/ExtractorStructs.h" #include "DataStructures/NNGrid.h" #include "DataStructures/QueryEdge.h" @@ -125,21 +126,14 @@ int main (int argc, char *argv[]) { INFO("Generating edge-expanded graph representation"); EdgeBasedGraphFactory * edgeBasedGraphFactory = new EdgeBasedGraphFactory (nodeBasedNodeNumber, edgeList, bollardNodes, trafficLightNodes, inputRestrictions, internalToExternalNodeMapping, speedProfile, SRTM_ROOT); std::vector().swap(edgeList); - edgeBasedGraphFactory->Run(edgeOut); std::vector<_Restriction>().swap(inputRestrictions); std::vector().swap(bollardNodes); std::vector().swap(trafficLightNodes); NodeID edgeBasedNodeNumber = edgeBasedGraphFactory->GetNumberOfNodes(); - std::vector edgeBasedEdgeList; + DeallocatingVector edgeBasedEdgeList; edgeBasedGraphFactory->GetEdgeBasedEdges(edgeBasedEdgeList); -// stxxl::vector externalEdgeBasedEdgeList; -// BOOST_FOREACH(EdgeBasedEdge & edge, edgeBasedEdgeList) { -// externalEdgeBasedEdgeList.push_back(edge); -// } -// std::vector().swap(edgeBasedEdgeList); - /*** * Writing info on original (node-based) nodes */ @@ -191,7 +185,7 @@ int main (int argc, char *argv[]) { contractor->Run(); INFO("Contraction took " << get_timestamp() - contractionStartedTimestamp << " sec"); - std::vector< QueryEdge > contractedEdgeList; + DeallocatingVector< QueryEdge > contractedEdgeList; contractor->GetEdges( contractedEdgeList ); delete contractor;