Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

First working implementation of Contractor flush at 75% of contracted

Browse Source 
nodes
This commit is contained in:
DennisOSRM 2012-04-04 09:55:43 +02:00
parent a5c5f4c02c
commit 964c0425bf
2 changed files with 124 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

136
Contractor/Contractor.h
View File

@ -20,17 +20,15 @@ or see http://www.gnu.org/licenses/agpl.txt.
#ifndef CONTRACTOR_H_INCLUDED
#define CONTRACTOR_H_INCLUDED
#ifdef _GLIBCXX_PARALLEL
#include <parallel/algorithm>
#else
#include <algorithm>
#endif
#include <boost/shared_ptr.hpp>
#include "../DataStructures/DynamicGraph.h"
#include "../DataStructures/Percent.h"
#include "../DataStructures/BinaryHeap.h"
#include "../Util/OpenMPReplacement.h"
#include "../Util/StringUtil.h"
#include <ctime>
#include <vector>
#include <queue>
@ -45,7 +43,7 @@ private:
_EdgeBasedContractorEdgeData() :
distance(0), originalEdges(0), via(0), nameID(0), turnInstruction(0), shortcut(0), forward(0), backward(0) {}
_EdgeBasedContractorEdgeData( unsigned _distance, unsigned _originalEdges, unsigned _via, unsigned _nameID, short _turnInstruction, bool _shortcut, bool _forward, bool _backward) :
distance(_distance), originalEdges(_originalEdges), via(_via), nameID(_nameID), turnInstruction(_turnInstruction), shortcut(_shortcut), forward(_forward), backward(_backward) {}
distance(_distance), originalEdges(_originalEdges), via(_via), nameID(_nameID), turnInstruction(_turnInstruction), shortcut(_shortcut), forward(_forward), backward(_backward), originalViaNodeID(false) {}
unsigned distance;
unsigned originalEdges;
unsigned via;
@ -54,6 +52,7 @@ private:
bool shortcut:1;
bool forward:1;
bool backward:1;
bool originalViaNodeID:1;
} data;
struct _HeapData {
@ -122,12 +121,7 @@ public:
}
//remove data from memory
std::vector< InputEdge >().swap( inputEdges );
#ifdef _GLIBCXX_PARALLEL
__gnu_parallel::sort( edges.begin(), edges.end() );
#else
sort( edges.begin(), edges.end() );
#endif
NodeID edge = 0;
for ( NodeID i = 0; i < edges.size(); ) {
const NodeID source = edges[i].source;
@ -194,9 +188,17 @@ public:
// INFO(" ->(" << highestNode << "," << _graph->GetTarget(i) << "); via: " << _graph->GetEdgeData(i).via);
// }
//Create temporary file
GetTemporaryFileName(temporaryEdgeStorageFilename);
INFO("75% Storage at " << temporaryEdgeStorageFilename);
}
~Contractor() { }
~Contractor() {
//Delete temporary file
// remove(temporaryEdgeStorageFilename.c_str());
}
void Run() {
const NodeID numberOfNodes = _graph->GetNumberOfNodes();
@ -235,9 +237,8 @@ public:
bool flushedContractor = false;
while ( numberOfContractedNodes < numberOfNodes ) {
if(!flushedContractor && (numberOfContractedNodes > (numberOfNodes*0.75) ) ){
if(!flushedContractor && (numberOfContractedNodes > (numberOfNodes*0.5) ) ){
INFO("Flushing memory after " << numberOfContractedNodes << " nodes");
//TODO:
//Delete old heap data to free memory that we need for the coming operations
for ( unsigned threadNum = 0; threadNum < maxThreads; threadNum++ ) {
@ -249,15 +250,15 @@ public:
//Create new priority array
std::vector<double> newNodePriority(remainingNodes.size());
//this map gives the old IDs from the new ones, necessary to get a consistent graph at the end of contraction
std::vector<NodeID> oldNodeIDMap(remainingNodes.size());
oldNodeIDFromNewNodeIDMap.resize(remainingNodes.size());
//this map gives the new IDs from the old ones, necessary to remap targets from the remaining graph
std::vector<NodeID> newNodeIDMap(numberOfNodes, UINT_MAX);
std::vector<NodeID> newNodeIDFromOldNodeIDMap(numberOfNodes, UINT_MAX);
//build forward and backward renumbering map and remap ids in remainingNodes and Priorities.
for(unsigned newNodeID = 0; newNodeID < remainingNodes.size(); ++newNodeID) {
//create renumbering maps in both directions
oldNodeIDMap[newNodeID] = remainingNodes[newNodeID].first;
newNodeIDMap[remainingNodes[newNodeID].first] = newNodeID;
oldNodeIDFromNewNodeIDMap[newNodeID] = remainingNodes[newNodeID].first;
newNodeIDFromOldNodeIDMap[remainingNodes[newNodeID].first] = newNodeID;
newNodePriority[newNodeID] = nodePriority[remainingNodes[newNodeID].first];
remainingNodes[newNodeID].first = newNodeID;
}
@ -265,15 +266,46 @@ public:
//create new _DynamicGraph, goes out of scope after the renumbering
boost::shared_ptr<_DynamicGraph> _newGraph ( new _DynamicGraph(remainingNodes.size()) );
//walk over all remainingNodes
for(unsigned i = 0; i < remainingNodes.size(); ++i) {
//TODO:Wenn Knoten kontrahiert: speichern seiner Kanten in vector mit
//TODO:Wenn Knoten nicht kontrahiert: Einfügen der Kanten in neuen DynamicGraph und
// dabei Source, Target umnummerieren und Priorität in neuem Prioritäten-Array speichern.
//Write dummy number of edges to temporary file
std::ofstream temporaryEdgeStorage(temporaryEdgeStorageFilename.c_str(), std::ios::binary);
initialFilePosition = temporaryEdgeStorage.tellp();
unsigned numberOfTemporaryEdges = 0;
temporaryEdgeStorage.write((char*)&numberOfTemporaryEdges, sizeof(unsigned));
//walk over all nodes
for(unsigned i = 0; i < _graph->GetNumberOfNodes(); ++i) {
//INFO("Restructuring node " << i << "|" << _graph->GetNumberOfNodes());
const NodeID start = i;
//UINT_MAX indicates that node is already contracted
for(_DynamicGraph::EdgeIterator currentEdge = _graph->BeginEdges(start); currentEdge < _graph->EndEdges(start); ++currentEdge) {
_DynamicGraph::EdgeData & data = _graph->GetEdgeData(currentEdge);
const NodeID target = _graph->GetTarget(currentEdge);
if(UINT_MAX == newNodeIDFromOldNodeIDMap[i] ){
//Save edges of this node w/o renumbering.
temporaryEdgeStorage.write((char*)&start, sizeof(NodeID));
temporaryEdgeStorage.write((char*)&target, sizeof(NodeID));
temporaryEdgeStorage.write((char*)&data, sizeof(_DynamicGraph::EdgeData));
++numberOfTemporaryEdges;
}else {
//node is not yet contracted.
//add (renumbered) outgoing edges to new DynamicGraph.
data.originalViaNodeID = true;
if(newNodeIDFromOldNodeIDMap[start] == UINT_MAX)
ERR("Could not resolve start");
if(UINT_MAX == newNodeIDFromOldNodeIDMap[target])
ERR("Could not resolve target");
_newGraph->InsertEdge(newNodeIDFromOldNodeIDMap[start], newNodeIDFromOldNodeIDMap[target], data );
}
//
}
}
//Note the number of temporarily stored edges
temporaryEdgeStorage.seekp(initialFilePosition);
temporaryEdgeStorage.write((char*)&numberOfTemporaryEdges, sizeof(unsigned));
temporaryEdgeStorage.close();
INFO("Flushed " << numberOfTemporaryEdges << " to disk");
//Delete map from old NodeIDs to new ones.
std::vector<NodeID>().swap(newNodeIDMap);
std::vector<NodeID>().swap(newNodeIDFromOldNodeIDMap);
//Replace old priorities array by new one
nodePriority.swap(newNodePriority);
@ -281,15 +313,14 @@ public:
std::vector<double>().swap(newNodePriority);
//Alten Graphen löschen und neuen Graphen speichern.
//ThreadData mit Heaps passender Größe reinitialisieren.
//reinitialize heaps and ThreadData objects with appropriate size
for ( unsigned threadNum = 0; threadNum < maxThreads; ++threadNum ) {
threadData.push_back( new _ThreadData( remainingNodes.size() ) );
threadData.push_back( new _ThreadData( _newGraph->GetNumberOfNodes() ) );
}
//TODO: Check if old Graph is removed
//old Graph is removed
_graph.swap(_newGraph);
// numberOfContractedNodes = 0;
flushedContractor = true;
}
@ -399,9 +430,49 @@ public:
for ( NodeID node = 0; node < numberOfNodes; ++node ) {
for ( _DynamicGraph::EdgeIterator edge = _graph->BeginEdges( node ), endEdges = _graph->EndEdges( node ); edge < endEdges; edge++ ) {
const NodeID target = _graph->GetTarget( edge );
const _EdgeBasedContractorEdgeData& data = _graph->GetEdgeData( edge );
const _DynamicGraph::EdgeData& data = _graph->GetEdgeData( edge );
Edge newEdge;
newEdge.source = node;
newEdge.source = oldNodeIDFromNewNodeIDMap[node];
newEdge.target = oldNodeIDFromNewNodeIDMap[target];
if(UINT_MAX == newEdge.source)
ERR("Could not resolve start");
if(UINT_MAX == newEdge.target)
ERR("Could not resolve target");
assert(UINT_MAX != newEdge.source);
assert(UINT_MAX != newEdge.target);
newEdge.data.distance = data.distance;
newEdge.data.shortcut = data.shortcut;
if(!data.originalViaNodeID)
newEdge.data.via = oldNodeIDFromNewNodeIDMap[data.via];
else
newEdge.data.via = data.via;
if(newEdge.data.via == UINT_MAX)
ERR("could not resolve via node");
newEdge.data.nameID = data.nameID;
newEdge.data.turnInstruction = data.turnInstruction;
newEdge.data.forward = data.forward;
newEdge.data.backward = data.backward;
edges.push_back( newEdge );
}
}
std::ifstream temporaryEdgeStorage(temporaryEdgeStorageFilename.c_str(), std::ios::binary);
//Also get the edges from temporary storage
unsigned numberOfTemporaryEdges = 0;
temporaryEdgeStorage.read((char*)&numberOfTemporaryEdges, sizeof(unsigned));
INFO("loading " << numberOfTemporaryEdges << " back from disk");
//loads edges of graph before renumbering, no need for further numbering action.
NodeID start;
NodeID target;
_DynamicGraph::EdgeData data;
for(unsigned i = 0; i < numberOfTemporaryEdges; ++i) {
temporaryEdgeStorage.read((char*)&start, sizeof(NodeID));
temporaryEdgeStorage.read((char*)&target, sizeof(NodeID));
temporaryEdgeStorage.read((char*)&data, sizeof(_DynamicGraph::EdgeData));
Edge newEdge;
newEdge.source = start;
newEdge.target = target;
newEdge.data.distance = data.distance;
newEdge.data.shortcut = data.shortcut;
@ -412,7 +483,7 @@ public:
newEdge.data.backward = data.backward;
edges.push_back( newEdge );
}
}
temporaryEdgeStorage.close();
}
private:
@ -664,6 +735,9 @@ private:
boost::shared_ptr<_DynamicGraph> _graph;
std::vector<_DynamicGraph::InputEdge> contractedEdges;
std::string temporaryEdgeStorageFilename;
std::vector<NodeID> oldNodeIDFromNewNodeIDMap;
long initialFilePosition;
};
#endif // CONTRACTOR_H_INCLUDED

17
Util/StringUtil.h
View File

@ -21,6 +21,9 @@ or see http://www.gnu.org/licenses/agpl.txt.
#ifndef STRINGUTIL_H_
#define STRINGUTIL_H_
#include <cstdlib>
#include <string>
// precision: position after decimal point
// length: maximum number of digits including comma and decimals
template< int length, int precision >
@ -143,4 +146,18 @@ inline std::string HTMLDeEntitize( std::string result) {
inline bool StringStartsWith(std::string & input, std::string & prefix) {
return (input.find(prefix) == 0);
}
/*
* Function returns a 'random' filename in temporary directors.
* May not be platform independent.
*/
inline void GetTemporaryFileName(std::string & filename) {
char buffer[L_tmpnam];
char * retPointer = tmpnam (buffer);
if(0 == retPointer)
ERR("Could not create temporary file name");
filename = buffer;
}
#endif /* STRINGUTIL_H_ */