Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Merge pull request #213 from DennisOSRM/feature/FlushContractor

Browse Source 
Feature/flush contractor
This commit is contained in:
Project OSRM 2012-04-05 05:06:03 -07:00
commit e412366bb2
7 changed files with 169 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored Normal file
View File

@ -0,0 +1,3 @@
[submodule "Docs"]
path = Docs
url = https://github.com/DennisSchiefer/Project-OSRM-Web.git

153
Contractor/Contractor.h
View File

@ -20,17 +20,15 @@ or see http://www.gnu.org/licenses/agpl.txt.
#ifndef CONTRACTOR_H_INCLUDED
#define CONTRACTOR_H_INCLUDED
#ifdef _GLIBCXX_PARALLEL
#include <parallel/algorithm>
#else
#include <algorithm>
#endif
#include <boost/shared_ptr.hpp>
#include "../DataStructures/DynamicGraph.h"
#include "../DataStructures/Percent.h"
#include "../DataStructures/BinaryHeap.h"
#include "../Util/OpenMPReplacement.h"
#include "../Util/StringUtil.h"
#include <ctime>
#include <vector>
#include <queue>
@ -45,7 +43,7 @@ private:
_EdgeBasedContractorEdgeData() :
distance(0), originalEdges(0), via(0), nameID(0), turnInstruction(0), shortcut(0), forward(0), backward(0) {}
_EdgeBasedContractorEdgeData( unsigned _distance, unsigned _originalEdges, unsigned _via, unsigned _nameID, short _turnInstruction, bool _shortcut, bool _forward, bool _backward) :
distance(_distance), originalEdges(_originalEdges), via(_via), nameID(_nameID), turnInstruction(_turnInstruction), shortcut(_shortcut), forward(_forward), backward(_backward) {}
distance(_distance), originalEdges(_originalEdges), via(_via), nameID(_nameID), turnInstruction(_turnInstruction), shortcut(_shortcut), forward(_forward), backward(_backward), originalViaNodeID(false) {}
unsigned distance;
unsigned originalEdges;
unsigned via;
@ -54,6 +52,7 @@ private:
bool shortcut:1;
bool forward:1;
bool backward:1;
bool originalViaNodeID:1;
} data;
struct _HeapData {
@ -122,12 +121,7 @@ public:
}
//remove data from memory
std::vector< InputEdge >().swap( inputEdges );
#ifdef _GLIBCXX_PARALLEL
__gnu_parallel::sort( edges.begin(), edges.end() );
#else
sort( edges.begin(), edges.end() );
#endif
NodeID edge = 0;
for ( NodeID i = 0; i < edges.size(); ) {
const NodeID source = edges[i].source;
@ -194,9 +188,16 @@ public:
// INFO(" ->(" << highestNode << "," << _graph->GetTarget(i) << "); via: " << _graph->GetEdgeData(i).via);
// }
//Create temporary file
GetTemporaryFileName(temporaryEdgeStorageFilename);
}
~Contractor() { }
~Contractor() {
//Delete temporary file
remove(temporaryEdgeStorageFilename.c_str());
}
void Run() {
const NodeID numberOfNodes = _graph->GetNumberOfNodes();
@ -233,7 +234,93 @@ public:
}
std::cout << "ok" << std::endl << "preprocessing ..." << std::flush;
bool flushedContractor = false;
while ( numberOfContractedNodes < numberOfNodes ) {
if(!flushedContractor && (numberOfContractedNodes > (numberOfNodes*0.75) ) ){
INFO("Flushing memory after " << numberOfContractedNodes << " nodes");
//Delete old heap data to free memory that we need for the coming operations
for ( unsigned threadNum = 0; threadNum < maxThreads; threadNum++ ) {
delete threadData[threadNum];
}
threadData.clear();
//Create new priority array
std::vector<double> newNodePriority(remainingNodes.size());
//this map gives the old IDs from the new ones, necessary to get a consistent graph at the end of contraction
oldNodeIDFromNewNodeIDMap.resize(remainingNodes.size());
//this map gives the new IDs from the old ones, necessary to remap targets from the remaining graph
std::vector<NodeID> newNodeIDFromOldNodeIDMap(numberOfNodes, UINT_MAX);
//build forward and backward renumbering map and remap ids in remainingNodes and Priorities.
for(unsigned newNodeID = 0; newNodeID < remainingNodes.size(); ++newNodeID) {
//create renumbering maps in both directions
oldNodeIDFromNewNodeIDMap[newNodeID] = remainingNodes[newNodeID].first;
newNodeIDFromOldNodeIDMap[remainingNodes[newNodeID].first] = newNodeID;
newNodePriority[newNodeID] = nodePriority[remainingNodes[newNodeID].first];
remainingNodes[newNodeID].first = newNodeID;
}
//create new _DynamicGraph, goes out of scope after the renumbering
boost::shared_ptr<_DynamicGraph> _newGraph ( new _DynamicGraph(remainingNodes.size()) );
//Write dummy number of edges to temporary file
std::ofstream temporaryEdgeStorage(temporaryEdgeStorageFilename.c_str(), std::ios::binary);
initialFilePosition = temporaryEdgeStorage.tellp();
unsigned numberOfTemporaryEdges = 0;
temporaryEdgeStorage.write((char*)&numberOfTemporaryEdges, sizeof(unsigned));
//walk over all nodes
for(unsigned i = 0; i < _graph->GetNumberOfNodes(); ++i) {
//INFO("Restructuring node " << i << "|" << _graph->GetNumberOfNodes());
const NodeID start = i;
//UINT_MAX indicates that node is already contracted
for(_DynamicGraph::EdgeIterator currentEdge = _graph->BeginEdges(start); currentEdge < _graph->EndEdges(start); ++currentEdge) {
_DynamicGraph::EdgeData & data = _graph->GetEdgeData(currentEdge);
const NodeID target = _graph->GetTarget(currentEdge);
if(UINT_MAX == newNodeIDFromOldNodeIDMap[i] ){
//Save edges of this node w/o renumbering.
temporaryEdgeStorage.write((char*)&start, sizeof(NodeID));
temporaryEdgeStorage.write((char*)&target, sizeof(NodeID));
temporaryEdgeStorage.write((char*)&data, sizeof(_DynamicGraph::EdgeData));
++numberOfTemporaryEdges;
}else {
//node is not yet contracted.
//add (renumbered) outgoing edges to new DynamicGraph.
data.originalViaNodeID = true;
assert(UINT_MAX != newNodeIDFromOldNodeIDMap[start] );
assert(UINT_MAX != newNodeIDFromOldNodeIDMap[target]);
_newGraph->InsertEdge(newNodeIDFromOldNodeIDMap[start], newNodeIDFromOldNodeIDMap[target], data );
}
}
}
//Note the number of temporarily stored edges
temporaryEdgeStorage.seekp(initialFilePosition);
temporaryEdgeStorage.write((char*)&numberOfTemporaryEdges, sizeof(unsigned));
temporaryEdgeStorage.close();
// INFO("Flushed " << numberOfTemporaryEdges << " edges to disk");
//Delete map from old NodeIDs to new ones.
std::vector<NodeID>().swap(newNodeIDFromOldNodeIDMap);
//Replace old priorities array by new one
nodePriority.swap(newNodePriority);
//Delete old nodePriority vector
std::vector<double>().swap(newNodePriority);
//Alten Graphen löschen und neuen Graphen speichern.
//reinitialize heaps and ThreadData objects with appropriate size
for ( unsigned threadNum = 0; threadNum < maxThreads; ++threadNum ) {
threadData.push_back( new _ThreadData( _newGraph->GetNumberOfNodes() ) );
}
//old Graph is removed
_graph.swap(_newGraph);
flushedContractor = true;
}
const int last = ( int ) remainingNodes.size();
#pragma omp parallel
{
@ -340,9 +427,43 @@ public:
for ( NodeID node = 0; node < numberOfNodes; ++node ) {
for ( _DynamicGraph::EdgeIterator edge = _graph->BeginEdges( node ), endEdges = _graph->EndEdges( node ); edge < endEdges; edge++ ) {
const NodeID target = _graph->GetTarget( edge );
const _EdgeBasedContractorEdgeData& data = _graph->GetEdgeData( edge );
const _DynamicGraph::EdgeData& data = _graph->GetEdgeData( edge );
Edge newEdge;
newEdge.source = node;
newEdge.source = oldNodeIDFromNewNodeIDMap[node];
newEdge.target = oldNodeIDFromNewNodeIDMap[target];
assert(UINT_MAX != newEdge.source);
assert(UINT_MAX != newEdge.target);
newEdge.data.distance = data.distance;
newEdge.data.shortcut = data.shortcut;
if(!data.originalViaNodeID)
newEdge.data.via = oldNodeIDFromNewNodeIDMap[data.via];
else
newEdge.data.via = data.via;
assert(newEdge.data.via != UINT_MAX);
newEdge.data.nameID = data.nameID;
newEdge.data.turnInstruction = data.turnInstruction;
newEdge.data.forward = data.forward;
newEdge.data.backward = data.backward;
edges.push_back( newEdge );
}
}
std::ifstream temporaryEdgeStorage(temporaryEdgeStorageFilename.c_str(), std::ios::binary);
//Also get the edges from temporary storage
unsigned numberOfTemporaryEdges = 0;
temporaryEdgeStorage.read((char*)&numberOfTemporaryEdges, sizeof(unsigned));
//loads edges of graph before renumbering, no need for further numbering action.
NodeID start;
NodeID target;
_DynamicGraph::EdgeData data;
for(unsigned i = 0; i < numberOfTemporaryEdges; ++i) {
temporaryEdgeStorage.read((char*)&start, sizeof(NodeID));
temporaryEdgeStorage.read((char*)&target, sizeof(NodeID));
temporaryEdgeStorage.read((char*)&data, sizeof(_DynamicGraph::EdgeData));
Edge newEdge;
newEdge.source = start;
newEdge.target = target;
newEdge.data.distance = data.distance;
newEdge.data.shortcut = data.shortcut;
@ -353,7 +474,7 @@ public:
newEdge.data.backward = data.backward;
edges.push_back( newEdge );
}
}
temporaryEdgeStorage.close();
}
private:
@ -604,6 +725,10 @@ private:
}
boost::shared_ptr<_DynamicGraph> _graph;
std::vector<_DynamicGraph::InputEdge> contractedEdges;
std::string temporaryEdgeStorageFilename;
std::vector<NodeID> oldNodeIDFromNewNodeIDMap;
long initialFilePosition;
};
#endif // CONTRACTOR_H_INCLUDED

10
Contractor/EdgeBasedGraphFactory.cpp
View File

@ -77,13 +77,15 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory(int nodes, std::vector<NodeBasedEdg
// INFO("traffic signal penalty: " << trafficSignalPenalty << ", U-Turn penalty: " << uturnPenalty << ", takeMinimumOfSpeeds=" << (takeMinimumOfSpeeds ? "yes" : "no"));
BOOST_FOREACH(NodeID id, bn)
BOOST_FOREACH(NodeID id, bn) {
_barrierNodes[id] = true;
BOOST_FOREACH(NodeID id, tl)
}
BOOST_FOREACH(NodeID id, tl) {
_trafficLights[id] = true;
}
std::vector< _NodeBasedEdge > edges;
edges.reserve( 2 * inputEdges.size() );
// edges.reserve( 2 * inputEdges.size() );
for ( std::vector< NodeBasedEdge >::const_iterator i = inputEdges.begin(); i != inputEdges.end(); ++i ) {
_NodeBasedEdge edge;
@ -191,7 +193,7 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(
void EdgeBasedGraphFactory::Run() {
INFO("Generating edge based representation of input data");
edgeBasedNodes.reserve(_nodeBasedGraph->GetNumberOfEdges());
// edgeBasedNodes.reserve(_nodeBasedGraph->GetNumberOfEdges());
Percent p(_nodeBasedGraph->GetNumberOfNodes());
int numberOfSkippedTurns(0);
int nodeBasedEdgeCounter(0);

1
Docs Submodule

@ -0,0 +1 @@
Subproject commit bab2887f8f7b17d6346e5f18d0464a66d8ae4e98

44
Docs/FAQ.txt
View File

@ -1,44 +0,0 @@
FAQ
---
Q: What platforms does OSMR run on?
A: Virtually any Unix-like platform with g++ installed. It has been developed
under Linux and tested on MacOS X 10.6. It should run under Windows as well
though the code will need some adjustments.
Q: What is the workflow to get the engine up and running
A: Road network extraction->Preprocessing->Startup
Q: What does OSRM stand for?
A: It is an abbreviation for Open Source Routing Machine.
Q: What does HSGR stand for?
A: It is an abbreviation for Hierarchy Search GRaph.
Q: What is the .nodes file?
A: It is a map that translates between internal and external Node IDs. Remember
that external NodeIDs can be arbitrary and non-contigous. Internally the
nodes are numbered from 0 to n-1.
Q: The routing engine crashes with a seg-fault
A: Check the startup parameters.
Q: Something about the route is odd. I know that there is a better path
A: Most probably it is missing data in the OSM file.
Q: I work for this company that would like to use the code, but we are hesistant
because of the license.
A: Contact me. Probably, we can work something out.
Q: How fast is this thing?
A: Good question. Here is a number. The engine was able to handle more than
2800 requests per Minute on the German road network with the travel time
metric on a Core 2 Duo. This also includes transfer of data across a
switched 100MBit/s LAN. So, I guess it's fair to say it's fast.
Q: What is the difference between extractNetwork and extractLargeNetwork?
A: extractNetwork does all of its magic in RAM, while extractLargeNetwork uses the
stxxl library to store the data in external memory. Use the first one for small
networks, but beware of running out of RAM. Swapping is the kiss of death. For
larger networks use extractLargeNetwork. It will be somewhat slower on the
smaller files, but way faster with large ones like the planet file.

6
Docs/KnownBugs.txt
View File

@ -1,6 +0,0 @@
- extractorLargeNetwork finishes with:
terminate called after throwing an instance of 'stxxl::resource_error'
what(): Error in function void stxxl::mutex::lock() pthread_mutex_lock(&_mutex) Invalid argument
This ist a known, albeit odd behavior that is caused by a bug in stxxl. It can be ignored. Upgrade
to version 1.3.1 of stxxl.

17
Util/StringUtil.h
View File

@ -21,6 +21,9 @@ or see http://www.gnu.org/licenses/agpl.txt.
#ifndef STRINGUTIL_H_
#define STRINGUTIL_H_
#include <cstdlib>
#include <string>
// precision: position after decimal point
// length: maximum number of digits including comma and decimals
template< int length, int precision >
@ -143,4 +146,18 @@ inline std::string HTMLDeEntitize( std::string result) {
inline bool StringStartsWith(std::string & input, std::string & prefix) {
return (input.find(prefix) == 0);
}
/*
* Function returns a 'random' filename in temporary directors.
* May not be platform independent.
*/
inline void GetTemporaryFileName(std::string & filename) {
char buffer[L_tmpnam];
char * retPointer = tmpnam (buffer);
if(0 == retPointer)
ERR("Could not create temporary file name");
filename = buffer;
}
#endif /* STRINGUTIL_H_ */