From 4f73b4725a1de29703c8ec8a2ad8a8a21681e6eb Mon Sep 17 00:00:00 2001 From: DennisOSRM Date: Fri, 4 May 2012 16:17:46 +0200 Subject: [PATCH 1/6] Chopping off another 1.6ms from every nearest neighbor query. --- DataStructures/NNGrid.h | 67 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/DataStructures/NNGrid.h b/DataStructures/NNGrid.h index 90dd1817b..7c1521a42 100644 --- a/DataStructures/NNGrid.h +++ b/DataStructures/NNGrid.h @@ -158,17 +158,20 @@ public: } bool FindPhantomNodeForCoordinate( const _Coordinate & location, PhantomNode & resultNode) { +// double time1 = get_timestamp(); bool foundNode = false; _Coordinate startCoord(100000*(lat2y(static_cast(location.lat)/100000.)), location.lon); /** search for point on edge close to source */ unsigned fileIndex = GetFileIndexForLatLon(startCoord.lat, startCoord.lon); std::vector<_GridEdge> candidates; - boost::unordered_map< unsigned, unsigned, IdenticalHashFunction > cellMap; +// boost::unordered_map< unsigned, unsigned, IdenticalHashFunction > cellMap; for(int j = -32768; (j < (32768+1)) && (fileIndex != UINT_MAX); j+=32768) { for(int i = -1; i < 2; ++i){ - GetContentsOfFileBucket(fileIndex+i+j, candidates, cellMap); + GetContentsOfFileBucketEnumerated(fileIndex+i+j, candidates/*, cellMap*/); } } +// double time2 = get_timestamp(); +// INFO("NN-Lookup in " << 1000*(time2-time1) << "ms"); _GridEdge smallestEdge; _Coordinate tmp; double dist = numeric_limits::max(); @@ -222,6 +225,7 @@ public: resultNode.ratio = ratio; // INFO("New weight1: " << resultNode.weight1 << ", new weight2: " << resultNode.weight2 << ", ratio: " << ratio); // INFO("selected node: " << resultNode.edgeBasedNode << ", bidirected: " << (resultNode.isBidirected() ? "yes" : "no") << "\n--"); +// INFO("NN-Lookup in " << 1000*(time2-time1) << "ms"); return foundNode; } @@ -282,6 +286,24 @@ public: private: + inline unsigned GetCellIndexFromRAMAndFileIndex(const unsigned ramIndex, const unsigned fileIndex) const { + unsigned lineBase = ramIndex/1024; + lineBase = lineBase*32*32768; + unsigned columnBase = ramIndex%1024; + columnBase=columnBase*32; + for (int i = 0;i < 32;++i) { + for (int j = 0;j < 32;++j) { + const unsigned localFileIndex = lineBase + i * 32768 + columnBase + j; + if(localFileIndex == fileIndex) { + unsigned cellIndex = i * 32 + j; + return cellIndex; + } + } + } + return UINT_MAX; + } + + inline void BuildCellIndexToFileIndexMap(const unsigned ramIndex, boost::unordered_map& cellMap){ unsigned lineBase = ramIndex/1024; lineBase = lineBase*32*32768; @@ -389,6 +411,44 @@ private: return counter; } + inline void GetContentsOfFileBucketEnumerated(const unsigned fileIndex, std::vector<_GridEdge>& result) const { + unsigned ramIndex = GetRAMIndexFromFileIndex(fileIndex); + unsigned long startIndexInFile = ramIndexTable[ramIndex]; + if(startIndexInFile == ULONG_MAX) { + return; + } + unsigned enumeratedIndex = GetCellIndexFromRAMAndFileIndex(ramIndex, fileIndex); + + + //todo: move to thread specific pointer + unsigned long cellIndex[32*32]; + + if(!localStream.get() || !localStream->is_open()) { + localStream.reset(new std::ifstream(iif.c_str(), std::ios::in | std::ios::binary)); + } + if(!localStream->good()) { + localStream->clear(std::ios::goodbit); + DEBUG("Resetting stale filestream"); + } + + localStream->seekg(startIndexInFile); + //todo: only read the single necessary cell index + localStream->read((char*) cellIndex, 32*32*sizeof(unsigned long)); + + if(cellIndex[enumeratedIndex] == ULONG_MAX) { + return; + } + const unsigned long position = cellIndex[enumeratedIndex] + 32*32*sizeof(unsigned long) ; + + unsigned lengthOfBucket; + unsigned currentSizeOfResult = result.size(); + localStream->seekg(position); + localStream->read((char *)&(lengthOfBucket), sizeof(unsigned)); + result.resize(currentSizeOfResult+lengthOfBucket); + localStream->read((char *)&result[currentSizeOfResult], lengthOfBucket*sizeof(_GridEdge)); + } + + inline void GetContentsOfFileBucket(const unsigned fileIndex, std::vector<_GridEdge>& result, boost::unordered_map< unsigned, unsigned, IdenticalHashFunction > & cellMap) { unsigned ramIndex = GetRAMIndexFromFileIndex(fileIndex); unsigned long startIndexInFile = ramIndexTable[ramIndex]; @@ -415,6 +475,7 @@ private: return; } const unsigned long position = cellIndex[cellMap[fileIndex]] + 32*32*sizeof(unsigned long) ; + unsigned lengthOfBucket; unsigned currentSizeOfResult = result.size(); localStream->seekg(position); @@ -514,7 +575,7 @@ private: return fileIndex; } - inline unsigned GetRAMIndexFromFileIndex(const int fileIndex) { + inline unsigned GetRAMIndexFromFileIndex(const int fileIndex) const { unsigned fileLine = fileIndex / 32768; fileLine = fileLine / 32; fileLine = fileLine * 1024; From f5da6a0e5f5ef850e62b6f24782eade05fcea361 Mon Sep 17 00:00:00 2001 From: Emil Tin Date: Sat, 5 May 2012 16:12:58 +0200 Subject: [PATCH 2/6] use .osrm.edges file in Rake task and cucumber tests --- Rakefile | 1 + features/step_definitions/processing.rb | 1 + features/support/config.rb | 1 + 3 files changed, 3 insertions(+) diff --git a/Rakefile b/Rakefile index 0c5fb768a..2e7a44d61 100644 --- a/Rakefile +++ b/Rakefile @@ -64,6 +64,7 @@ def write_server_ini osm_file hsgrData=#{DATA_FOLDER}/#{osm_file}.osrm.hsgr nodesData=#{DATA_FOLDER}/#{osm_file}.osrm.nodes + edgesData=#{DATA_FOLDER}/#{osm_file}.osrm.edges ramIndex=#{DATA_FOLDER}/#{osm_file}.osrm.ramIndex fileIndex=#{DATA_FOLDER}/#{osm_file}.osrm.fileIndex namesData=#{DATA_FOLDER}/#{osm_file}.osrm.names diff --git a/features/step_definitions/processing.rb b/features/step_definitions/processing.rb index bd98e9be3..323604e1d 100644 --- a/features/step_definitions/processing.rb +++ b/features/step_definitions/processing.rb @@ -58,6 +58,7 @@ Given /^the preprocessed files for "([^"]*)" are present and up to date$/ do |ar File.exists?("#{area}.osrm.restrictions").should == true File.exists?("#{area}.osrm.hsgr").should == true File.exists?("#{area}.osrm.nodes").should == true + File.exists?("#{area}.osrm.edges").should == true File.exists?("#{area}.osrm.ramIndex").should == true File.exists?("#{area}.osrm.fileIndex").should == true end diff --git a/features/support/config.rb b/features/support/config.rb index 80db02b12..10ed87b28 100644 --- a/features/support/config.rb +++ b/features/support/config.rb @@ -32,6 +32,7 @@ Port = 5000 hsgrData=#{@osm_file}.osrm.hsgr nodesData=#{@osm_file}.osrm.nodes +edgesData=#{@osm_file}.osrm.edges ramIndex=#{@osm_file}.osrm.ramIndex fileIndex=#{@osm_file}.osrm.fileIndex namesData=#{@osm_file}.osrm.names From 580a760aa1aee6ac7dd59c6f98da76d68edbaa5e Mon Sep 17 00:00:00 2001 From: Emil Tin Date: Sat, 5 May 2012 17:05:55 +0200 Subject: [PATCH 3/6] speedprofiles specifically for testing --- features/support/config.rb | 2 +- features/time.feature | 6 +----- test/speedprofiles/bicycle.ini | 10 ++++++++++ test/speedprofiles/car.ini | 23 +++++++++++++++++++++++ test/speedprofiles/foot.ini | 28 ++++++++++++++++++++++++++++ 5 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 test/speedprofiles/bicycle.ini create mode 100644 test/speedprofiles/car.ini create mode 100644 test/speedprofiles/foot.ini diff --git a/features/support/config.rb b/features/support/config.rb index 80db02b12..e6cd22672 100644 --- a/features/support/config.rb +++ b/features/support/config.rb @@ -10,7 +10,7 @@ end def read_speedprofile profile @speedprofile = {} @speedprofile_str = nil - s = File.read "speedprofiles/#{profile}.ini" + s = File.read "test/speedprofiles/#{profile}.ini" s.scan /(.*)=(.*)/ do |option| @speedprofile[option[0].strip] = option[1].strip end diff --git a/features/time.feature b/features/time.feature index 564033db2..e2da7fc57 100644 --- a/features/time.feature +++ b/features/time.feature @@ -6,9 +6,6 @@ Feature: Estimation of travel time Background: Use specific speeds Given the speedprofile "bicycle" - And the speedprofile settings - | primary | 15 | - | footway | 5 | Scenario: Basic travel time, 1m scale Given a grid size of 1 meters @@ -178,5 +175,4 @@ Feature: Estimation of travel time | 1 | 4 | ab | 72s | | 4 | 3 | ab | 24s | | 4 | 2 | ab | 48s | - | 4 | 1 | ab | 72s | - + | 4 | 1 | ab | 72s | \ No newline at end of file diff --git a/test/speedprofiles/bicycle.ini b/test/speedprofiles/bicycle.ini new file mode 100644 index 000000000..622ebe427 --- /dev/null +++ b/test/speedprofiles/bicycle.ini @@ -0,0 +1,10 @@ +[bicycle] + accessTag = bicycle + obeyOneways = yes + obeyBollards = no + useRestrictions = yes + takeMinimumOfSpeeds = yes + defaultSpeed = 15 + primary = 15 + cycleway = 15 + footway = 5 \ No newline at end of file diff --git a/test/speedprofiles/car.ini b/test/speedprofiles/car.ini new file mode 100644 index 000000000..b2ce3c4c0 --- /dev/null +++ b/test/speedprofiles/car.ini @@ -0,0 +1,23 @@ +[car] + accessTag = motorcar + defaultSpeed = 50 + obeyOneways = yes + useRestrictions = yes + barrier = bollard + + motorway = 100 + motorway_link = 90 + trunk = 90 + trunk_link = 70 + primary = 70 + primary_link = 60 + secondary = 60 + secondary_link = 50 + tertiary = 50 + tertiary_link = 40 + road = 40 + residential = 40 + unclassified = 30 + service = 20 + living_street = 10 + diff --git a/test/speedprofiles/foot.ini b/test/speedprofiles/foot.ini new file mode 100644 index 000000000..656c82ad5 --- /dev/null +++ b/test/speedprofiles/foot.ini @@ -0,0 +1,28 @@ +[foot] + accessTag = foot + defaultSpeed = 5 + obeyOneways = no + useRestrictions = no + obeyBollards = no + + primary = 5 + primary_link = 5 + secondary = 5 + secondary_link = 5 + tertiary = 5 + residential = 5 + unclassified = 5 + living_street = 5 + road = 5 + service = 5 + track = 5 + path = 5 + cycleway = 5 + footway = 5 + pedestrian = 5 + pier = 5 + steps = 2 + + ferry = 5 + + excludeFromGrid = ferry From 5b247a3699430bb6b6128ca641dbe0a828ea4f41 Mon Sep 17 00:00:00 2001 From: DennisOSRM Date: Tue, 8 May 2012 11:12:53 +0200 Subject: [PATCH 4/6] Removing unnecessary over-allocation --- Contractor/Contractor.h | 1 + 1 file changed, 1 insertion(+) diff --git a/Contractor/Contractor.h b/Contractor/Contractor.h index 941fb65f6..8dbaad5cc 100644 --- a/Contractor/Contractor.h +++ b/Contractor/Contractor.h @@ -171,6 +171,7 @@ public: std::cout << "merged " << edges.size() - edge << " edges out of " << edges.size() << std::endl; edges.resize( edge ); + _graph.reset(); _graph.reset( new _DynamicGraph( nodes, edges ) ); INFO("Finished building dynamic graph"); edges.clear(); From ac6ba24861c3dbbaf7dee82ed6c26e9e4d0df7ba Mon Sep 17 00:00:00 2001 From: DennisOSRM Date: Tue, 8 May 2012 11:14:48 +0200 Subject: [PATCH 5/6] Fixes issue #253 --- routed.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routed.cpp b/routed.cpp index 7a5bad8c4..65ae56344 100644 --- a/routed.cpp +++ b/routed.cpp @@ -66,7 +66,7 @@ BOOL WINAPI console_ctrl_handler(DWORD ctrl_type) int main (int argc, char *argv[]) { #ifdef __linux__ if(!mlockall(MCL_CURRENT | MCL_FUTURE)) - ERR("Process " << argv[0] << "could not be locked to RAM"); + WARN("Process " << argv[0] << "could not be locked to RAM"); #endif #ifndef _WIN32 From f0188c428fcc0acd6cf67ff0d8b1271a63b0dd28 Mon Sep 17 00:00:00 2001 From: DennisOSRM Date: Tue, 8 May 2012 19:46:01 +0200 Subject: [PATCH 6/6] Removing some debug output --- Contractor/Contractor.h | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/Contractor/Contractor.h b/Contractor/Contractor.h index 8dbaad5cc..52a83f90f 100644 --- a/Contractor/Contractor.h +++ b/Contractor/Contractor.h @@ -171,7 +171,6 @@ public: std::cout << "merged " << edges.size() - edge << " edges out of " << edges.size() << std::endl; edges.resize( edge ); - _graph.reset(); _graph.reset( new _DynamicGraph( nodes, edges ) ); INFO("Finished building dynamic graph"); edges.clear(); @@ -240,7 +239,8 @@ public: bool flushedContractor = false; while ( numberOfContractedNodes < numberOfNodes ) { if(!flushedContractor && (numberOfContractedNodes > (numberOfNodes*0.75) ) ){ - INFO("Flushing memory after " << numberOfContractedNodes << " nodes"); + std::vector<_ContractorEdge> newSetOfEdges; //this one is not explicitely cleared since it goes out of scope anywa + std::cout << " [flush " << numberOfContractedNodes << " nodes] " << std::flush; //Delete old heap data to free memory that we need for the coming operations for ( unsigned threadNum = 0; threadNum < maxThreads; threadNum++ ) { @@ -265,9 +265,6 @@ public: remainingNodes[newNodeID].first = newNodeID; } - //create new _DynamicGraph, goes out of scope after the renumbering - boost::shared_ptr<_DynamicGraph> _newGraph ( new _DynamicGraph(remainingNodes.size()) ); - //Write dummy number of edges to temporary file std::ofstream temporaryEdgeStorage(temporaryEdgeStorageFilename.c_str(), std::ios::binary); initialFilePosition = temporaryEdgeStorage.tellp(); @@ -278,7 +275,6 @@ public: for(unsigned i = 0; i < _graph->GetNumberOfNodes(); ++i) { //INFO("Restructuring node " << i << "|" << _graph->GetNumberOfNodes()); const NodeID start = i; - //UINT_MAX indicates that node is already contracted for(_DynamicGraph::EdgeIterator currentEdge = _graph->BeginEdges(start); currentEdge < _graph->EndEdges(start); ++currentEdge) { _DynamicGraph::EdgeData & data = _graph->GetEdgeData(currentEdge); const NodeID target = _graph->GetTarget(currentEdge); @@ -291,10 +287,15 @@ public: }else { //node is not yet contracted. //add (renumbered) outgoing edges to new DynamicGraph. - data.originalViaNodeID = true; + _ContractorEdge newEdge; + newEdge.source = newNodeIDFromOldNodeIDMap[start]; + newEdge.target = newNodeIDFromOldNodeIDMap[target]; + newEdge.data = data; + newEdge.data.originalViaNodeID = true; assert(UINT_MAX != newNodeIDFromOldNodeIDMap[start] ); assert(UINT_MAX != newNodeIDFromOldNodeIDMap[target]); - _newGraph->InsertEdge(newNodeIDFromOldNodeIDMap[start], newNodeIDFromOldNodeIDMap[target], data ); +// _newGraph->InsertEdge(newNodeIDFromOldNodeIDMap[start], newNodeIDFromOldNodeIDMap[target], data ); + newSetOfEdges.push_back(newEdge); } } } @@ -311,17 +312,22 @@ public: nodePriority.swap(newNodePriority); //Delete old nodePriority vector std::vector().swap(newNodePriority); - //Alten Graphen löschen und neuen Graphen speichern. + //old Graph is removed + _graph.reset(); + //create new graph + std::sort(newSetOfEdges.begin(), newSetOfEdges.end()); + + //int nodes, const ContainerT &graph + _graph.reset( new _DynamicGraph(remainingNodes.size(), newSetOfEdges)); + flushedContractor = true; + + //INFO: MAKE SURE THIS IS THE LAST OPERATION OF THE FLUSH! //reinitialize heaps and ThreadData objects with appropriate size for ( unsigned threadNum = 0; threadNum < maxThreads; ++threadNum ) { - threadData.push_back( new _ThreadData( _newGraph->GetNumberOfNodes() ) ); + threadData.push_back( new _ThreadData( _graph->GetNumberOfNodes() ) ); } - //old Graph is removed - _graph.swap(_newGraph); - - flushedContractor = true; } const int last = ( int ) remainingNodes.size();