Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Parameterize amount of RAM for extractLargenetwork. Fixes ticket 21.

Browse Source
This commit is contained in:
Dennis Luxen 2011-03-24 15:06:49 +00:00
parent 417f02d30a
commit 5f711c5e03
3 changed files with 24 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
DataStructures/BinaryHeap.h
View File

@ -120,7 +120,7 @@ struct _SimpleHeapData {
_SimpleHeapData( NodeID p ) : parent(p) { } _SimpleHeapData( NodeID p ) : parent(p) { }
}; };
template < typename NodeID, typename Key, typename Weight, typename Data, typename IndexStorage = SparseTableStorage > template < typename NodeID, typename Key, typename Weight, typename Data, typename IndexStorage = ArrayStorage<NodeID, NodeID> >
class BinaryHeap { class BinaryHeap {
private: private:
BinaryHeap( const BinaryHeap& right ); BinaryHeap( const BinaryHeap& right );

24
extractLargeNetwork.cpp
View File

@ -35,6 +35,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include <libxml/xmlreader.h> #include <libxml/xmlreader.h>
#include <google/sparse_hash_map> #include <google/sparse_hash_map>
#include <unistd.h>
#include <stxxl.h> #include <stxxl.h>
#include "typedefs.h" #include "typedefs.h"
@ -43,6 +44,10 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "DataStructures/ExtractorStructs.h" #include "DataStructures/ExtractorStructs.h"
#include "DataStructures/PBFParser.h" #include "DataStructures/PBFParser.h"
#include "DataStructures/XMLParser.h" #include "DataStructures/XMLParser.h"
#include "Util/BaseConfiguration.h"
#include "Util/InputFileUtil.h"
typedef BaseConfiguration ExtractorConfiguration;
unsigned globalRelationCounter = 0; unsigned globalRelationCounter = 0;
ExtractorCallbacks * extractCallBacks; ExtractorCallbacks * extractCallBacks;
@ -83,6 +88,21 @@ int main (int argc, char *argv[]) {
} }
std::string adressFileName(outputFileName); std::string adressFileName(outputFileName);
unsigned amountOfRAM = 1;
unsigned installedRAM = (sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE));
if(installedRAM < 2097422336) {
std::cout << "[Warning] Machine has less than 2GB RAM." << std::endl;
}
if(testDataFile("extractor.ini")) {
ExtractorConfiguration extractorConfig("extractor.ini");
unsigned memoryAmountFromFile = atoi(extractorConfig.GetParameter("Memory").c_str());
if( memoryAmountFromFile != 0 && memoryAmountFromFile <= installedRAM/(1024*1024*1024))
amountOfRAM = memoryAmountFromFile;
std::cout << "[extractor] using " << amountOfRAM << " GB of RAM for buffers" << std::endl;
}
STXXLNodeIDVector usedNodeIDs; STXXLNodeIDVector usedNodeIDs;
STXXLNodeVector allNodes; STXXLNodeVector allNodes;
STXXLEdgeVector allEdges; STXXLEdgeVector allEdges;
@ -127,7 +147,7 @@ int main (int argc, char *argv[]) {
std::cout << "[extractor] parsing finished after " << get_timestamp() - time << "seconds" << std::endl; std::cout << "[extractor] parsing finished after " << get_timestamp() - time << "seconds" << std::endl;
time = get_timestamp(); time = get_timestamp();
unsigned memory_to_use = 1024 * 1024 * 1024; unsigned memory_to_use = amountOfRAM * 1024 * 1024 * 1024;
std::cout << "[extractor] Sorting used nodes ... " << std::flush; std::cout << "[extractor] Sorting used nodes ... " << std::flush;
stxxl::sort(usedNodeIDs.begin(), usedNodeIDs.end(), Cmp(), memory_to_use); stxxl::sort(usedNodeIDs.begin(), usedNodeIDs.end(), Cmp(), memory_to_use);
@ -165,7 +185,7 @@ int main (int argc, char *argv[]) {
fout.write((char*)&(nodesIT->id), sizeof(unsigned)); fout.write((char*)&(nodesIT->id), sizeof(unsigned));
fout.write((char*)&(nodesIT->lon), sizeof(int)); fout.write((char*)&(nodesIT->lon), sizeof(int));
fout.write((char*)&(nodesIT->lat), sizeof(int)); fout.write((char*)&(nodesIT->lat), sizeof(int));
// std::cout << "serializing: " << nodesIT->id << ", lat: " << nodesIT->lat << ", lon: " << nodesIT->lon << std::endl; // std::cout << "serializing: " << nodesIT->id << ", lat: " << nodesIT->lat << ", lon: " << nodesIT->lon << std::endl;
usedNodeCounter++; usedNodeCounter++;
usedNodeIDsIT++; usedNodeIDsIT++;
nodesIT++; nodesIT++;

1
extractor.ini Normal file
View File

@ -0,0 +1 @@
Memory = 2