Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Fixing data type issue that prevented large files on windows. See issue

Browse Source 
#55
This commit is contained in:
DennisOSRM 2012-12-29 14:14:01 +01:00
parent b869184c10
commit 943c15927a
4 changed files with 89 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Contractor/Contractor.h
View File

@ -274,7 +274,7 @@ public:
TemporaryStorage & tempStorage = TemporaryStorage::GetInstance(); TemporaryStorage & tempStorage = TemporaryStorage::GetInstance();
//Write dummy number of edges to temporary file //Write dummy number of edges to temporary file
// std::ofstream temporaryEdgeStorage(temporaryEdgeStorageFilename.c_str(), std::ios::binary); // std::ofstream temporaryEdgeStorage(temporaryEdgeStorageFilename.c_str(), std::ios::binary);
long initialFilePosition = tempStorage.tell(temporaryStorageSlotID); uint64_t initialFilePosition = tempStorage.tell(temporaryStorageSlotID);
unsigned numberOfTemporaryEdges = 0; unsigned numberOfTemporaryEdges = 0;
tempStorage.writeToSlot(temporaryStorageSlotID, (char*)&numberOfTemporaryEdges, sizeof(unsigned)); tempStorage.writeToSlot(temporaryStorageSlotID, (char*)&numberOfTemporaryEdges, sizeof(unsigned));

62
DataStructures/NNGrid.h
View File

@ -21,10 +21,12 @@ or see http://www.gnu.org/licenses/agpl.txt.
#ifndef NNGRID_H_ #ifndef NNGRID_H_
#define NNGRID_H_ #define NNGRID_H_
#include <algorithm>
#include <cassert> #include <cassert>
#include <cfloat> #include <cfloat>
#include <cmath> #include <cmath>
#include <cstring>
#include <algorithm>
#include <fstream> #include <fstream>
#include <limits> #include <limits>
#include <vector> #include <vector>
@ -58,7 +60,7 @@ template<bool WriteAccess = false>
class NNGrid { class NNGrid {
public: public:
NNGrid() /*: cellCache(500), fileCache(500)*/ { NNGrid() /*: cellCache(500), fileCache(500)*/ {
ramIndexTable.resize((1024*1024), ULONG_MAX); ramIndexTable.resize((1024*1024), std::numeric_limits<uint64_t>::max());
} }
NNGrid(const char* rif, const char* _i) { NNGrid(const char* rif, const char* _i) {
@ -66,7 +68,7 @@ public:
ERR("Not available in Write mode"); ERR("Not available in Write mode");
} }
iif = std::string(_i); iif = std::string(_i);
ramIndexTable.resize((1024*1024), ULONG_MAX); ramIndexTable.resize((1024*1024), std::numeric_limits<uint64_t>::max());
ramInFile.open(rif, std::ios::in | std::ios::binary); ramInFile.open(rif, std::ios::in | std::ios::binary);
if(!ramInFile) { ERR(rif << " not found"); } if(!ramInFile) { ERR(rif << " not found"); }
@ -87,7 +89,7 @@ public:
void OpenIndexFiles() { void OpenIndexFiles() {
assert(ramInFile.is_open()); assert(ramInFile.is_open());
ramInFile.read(static_cast<char*>(static_cast<void*>(&ramIndexTable[0]) ), sizeof(unsigned long)*1024*1024); ramInFile.read(static_cast<char*>(static_cast<void*>(&ramIndexTable[0]) ), sizeof(uint64_t)*1024*1024);
ramInFile.close(); ramInFile.close();
} }
@ -114,8 +116,8 @@ public:
INFO("finished sorting after " << (get_timestamp() - timestamp) << "s"); INFO("finished sorting after " << (get_timestamp() - timestamp) << "s");
std::vector<GridEntry> entriesInFileWithRAMSameIndex; std::vector<GridEntry> entriesInFileWithRAMSameIndex;
unsigned indexInRamTable = entries.begin()->ramIndex; unsigned indexInRamTable = entries.begin()->ramIndex;
unsigned long lastPositionInIndexFile = 0; uint64_t lastPositionInIndexFile = 0;
cout << "writing data ..." << flush; std::cout << "writing data ..." << std::flush;
p.reinit(entries.size()); p.reinit(entries.size());
boost::unordered_map< unsigned, unsigned > cellMap(1024); boost::unordered_map< unsigned, unsigned > cellMap(1024);
BOOST_FOREACH(GridEntry & gridEntry, entries) { BOOST_FOREACH(GridEntry & gridEntry, entries) {
@ -143,9 +145,9 @@ public:
indexOutFile.close(); indexOutFile.close();
//Serialize RAM Index //Serialize RAM Index
ofstream ramFile(ramIndexOut, std::ios::out | std::ios::binary | std::ios::trunc); std::ofstream ramFile(ramIndexOut, std::ios::out | std::ios::binary | std::ios::trunc);
//write 4 MB of index Table in RAM //write 4 MB of index Table in RAM
ramFile.write((char *)&ramIndexTable[0], sizeof(unsigned long)*1024*1024 ); ramFile.write((char *)&ramIndexTable[0], sizeof(uint64_t)*1024*1024 );
//close ram index file //close ram index file
ramFile.close(); ramFile.close();
} }
@ -174,7 +176,7 @@ public:
// INFO("looked up " << candidates.size()); // INFO("looked up " << candidates.size());
_GridEdge smallestEdge; _GridEdge smallestEdge;
_Coordinate tmp, edgeStartCoord, edgeEndCoord; _Coordinate tmp, edgeStartCoord, edgeEndCoord;
double dist = numeric_limits<double>::max(); double dist = std::numeric_limits<double>::max();
double r, tmpDist; double r, tmpDist;
BOOST_FOREACH(_GridEdge candidate, candidates) { BOOST_FOREACH(_GridEdge candidate, candidates) {
@ -314,13 +316,13 @@ private:
return (std::fabs(d1 - d2) < FLT_EPSILON); return (std::fabs(d1 - d2) < FLT_EPSILON);
} }
inline unsigned FillCell(std::vector<GridEntry>& entriesWithSameRAMIndex, const unsigned long fileOffset, boost::unordered_map< unsigned, unsigned > & cellMap ) { inline unsigned FillCell(std::vector<GridEntry>& entriesWithSameRAMIndex, const uint64_t fileOffset, boost::unordered_map< unsigned, unsigned > & cellMap ) {
std::vector<char> tmpBuffer(32*32*4096,0); std::vector<char> tmpBuffer(32*32*4096,0);
unsigned long indexIntoTmpBuffer = 0; uint64_t indexIntoTmpBuffer = 0;
unsigned numberOfWrittenBytes = 0; unsigned numberOfWrittenBytes = 0;
assert(indexOutFile.is_open()); assert(indexOutFile.is_open());
std::vector<unsigned long> cellIndex(32*32,ULONG_MAX); std::vector<uint64_t> cellIndex(32*32,std::numeric_limits<uint64_t>::max());
for(unsigned i = 0; i < entriesWithSameRAMIndex.size() -1; ++i) { for(unsigned i = 0; i < entriesWithSameRAMIndex.size() -1; ++i) {
assert(entriesWithSameRAMIndex[i].ramIndex== entriesWithSameRAMIndex[i+1].ramIndex); assert(entriesWithSameRAMIndex[i].ramIndex== entriesWithSameRAMIndex[i+1].ramIndex);
@ -356,8 +358,8 @@ private:
indexIntoTmpBuffer += FlushEntriesWithSameFileIndexToBuffer(entriesWithSameFileIndex, tmpBuffer, indexIntoTmpBuffer); indexIntoTmpBuffer += FlushEntriesWithSameFileIndexToBuffer(entriesWithSameFileIndex, tmpBuffer, indexIntoTmpBuffer);
assert(entriesWithSameFileIndex.size() == 0); assert(entriesWithSameFileIndex.size() == 0);
indexOutFile.write(static_cast<char*>(static_cast<void*>(&cellIndex[0])),32*32*sizeof(unsigned long)); indexOutFile.write(static_cast<char*>(static_cast<void*>(&cellIndex[0])),32*32*sizeof(uint64_t));
numberOfWrittenBytes += 32*32*sizeof(unsigned long); numberOfWrittenBytes += 32*32*sizeof(uint64_t);
//write contents of tmpbuffer to disk //write contents of tmpbuffer to disk
indexOutFile.write(&tmpBuffer[0], indexIntoTmpBuffer*sizeof(char)); indexOutFile.write(&tmpBuffer[0], indexIntoTmpBuffer*sizeof(char));
@ -366,7 +368,7 @@ private:
return numberOfWrittenBytes; return numberOfWrittenBytes;
} }
inline unsigned FlushEntriesWithSameFileIndexToBuffer( std::vector<GridEntry> &vectorWithSameFileIndex, std::vector<char> & tmpBuffer, const unsigned long index) const { inline unsigned FlushEntriesWithSameFileIndexToBuffer( std::vector<GridEntry> &vectorWithSameFileIndex, std::vector<char> & tmpBuffer, const uint64_t index) const {
sort( vectorWithSameFileIndex.begin(), vectorWithSameFileIndex.end() ); sort( vectorWithSameFileIndex.begin(), vectorWithSameFileIndex.end() );
vectorWithSameFileIndex.erase(unique(vectorWithSameFileIndex.begin(), vectorWithSameFileIndex.end()), vectorWithSameFileIndex.end()); vectorWithSameFileIndex.erase(unique(vectorWithSameFileIndex.begin(), vectorWithSameFileIndex.end()), vectorWithSameFileIndex.end());
const unsigned lengthOfBucket = vectorWithSameFileIndex.size(); const unsigned lengthOfBucket = vectorWithSameFileIndex.size();
@ -394,8 +396,8 @@ private:
inline void GetContentsOfFileBucketEnumerated(const unsigned fileIndex, std::vector<_GridEdge>& result) const { inline void GetContentsOfFileBucketEnumerated(const unsigned fileIndex, std::vector<_GridEdge>& result) const {
unsigned ramIndex = GetRAMIndexFromFileIndex(fileIndex); unsigned ramIndex = GetRAMIndexFromFileIndex(fileIndex);
unsigned long startIndexInFile = ramIndexTable[ramIndex]; uint64_t startIndexInFile = ramIndexTable[ramIndex];
if(startIndexInFile == ULONG_MAX) { if(startIndexInFile == std::numeric_limits<uint64_t>::max()) {
return; return;
} }
unsigned enumeratedIndex = GetCellIndexFromRAMAndFileIndex(ramIndex, fileIndex); unsigned enumeratedIndex = GetCellIndexFromRAMAndFileIndex(ramIndex, fileIndex);
@ -409,14 +411,14 @@ private:
} }
//only read the single necessary cell index //only read the single necessary cell index
localStream->seekg(startIndexInFile+(enumeratedIndex*sizeof(unsigned long))); localStream->seekg(startIndexInFile+(enumeratedIndex*sizeof(uint64_t)));
unsigned long fetchedIndex = 0; uint64_t fetchedIndex = 0;
localStream->read(static_cast<char*>( static_cast<void*>(&fetchedIndex)), sizeof(unsigned long)); localStream->read(static_cast<char*>( static_cast<void*>(&fetchedIndex)), sizeof(uint64_t));
if(fetchedIndex == ULONG_MAX) { if(fetchedIndex == std::numeric_limits<uint64_t>::max()) {
return; return;
} }
const unsigned long position = fetchedIndex + 32*32*sizeof(unsigned long) ; const uint64_t position = fetchedIndex + 32*32*sizeof(uint64_t) ;
unsigned lengthOfBucket; unsigned lengthOfBucket;
unsigned currentSizeOfResult = result.size(); unsigned currentSizeOfResult = result.size();
@ -428,12 +430,12 @@ private:
inline void GetContentsOfFileBucket(const unsigned fileIndex, std::vector<_GridEdge>& result, boost::unordered_map< unsigned, unsigned> & cellMap) { inline void GetContentsOfFileBucket(const unsigned fileIndex, std::vector<_GridEdge>& result, boost::unordered_map< unsigned, unsigned> & cellMap) {
unsigned ramIndex = GetRAMIndexFromFileIndex(fileIndex); unsigned ramIndex = GetRAMIndexFromFileIndex(fileIndex);
unsigned long startIndexInFile = ramIndexTable[ramIndex]; uint64_t startIndexInFile = ramIndexTable[ramIndex];
if(startIndexInFile == ULONG_MAX) { if(startIndexInFile == std::numeric_limits<uint64_t>::max()) {
return; return;
} }
unsigned long cellIndex[32*32]; uint64_t cellIndex[32*32];
cellMap.clear(); cellMap.clear();
BuildCellIndexToFileIndexMap(ramIndex, cellMap); BuildCellIndexToFileIndexMap(ramIndex, cellMap);
@ -446,12 +448,12 @@ private:
} }
localStream->seekg(startIndexInFile); localStream->seekg(startIndexInFile);
localStream->read(static_cast<char*>(static_cast<void*>( cellIndex)), 32*32*sizeof(unsigned long)); localStream->read(static_cast<char*>(static_cast<void*>( cellIndex)), 32*32*sizeof(uint64_t));
assert(cellMap.find(fileIndex) != cellMap.end()); assert(cellMap.find(fileIndex) != cellMap.end());
if(cellIndex[cellMap[fileIndex]] == ULONG_MAX) { if(cellIndex[cellMap[fileIndex]] == std::numeric_limits<uint64_t>::max()) {
return; return;
} }
const unsigned long position = cellIndex[cellMap[fileIndex]] + 32*32*sizeof(unsigned long) ; const uint64_t position = cellIndex[cellMap[fileIndex]] + 32*32*sizeof(uint64_t) ;
unsigned lengthOfBucket; unsigned lengthOfBucket;
unsigned currentSizeOfResult = result.size(); unsigned currentSizeOfResult = result.size();
@ -574,14 +576,14 @@ private:
return ramIndex; return ramIndex;
} }
const static unsigned long END_OF_BUCKET_DELIMITER = UINT_MAX; const static uint64_t END_OF_BUCKET_DELIMITER = boost::integer_traits<uint64_t>::const_max;
std::ofstream indexOutFile; std::ofstream indexOutFile;
std::ifstream ramInFile; std::ifstream ramInFile;
#ifndef ROUTED #ifndef ROUTED
stxxl::vector<GridEntry> entries; stxxl::vector<GridEntry> entries;
#endif #endif
std::vector<unsigned long> ramIndexTable; //8 MB for first level index in RAM std::vector<uint64_t> ramIndexTable; //8 MB for first level index in RAM
std::string iif; std::string iif;
// LRUCache<int,std::vector<unsigned> > cellCache; // LRUCache<int,std::vector<unsigned> > cellCache;
// LRUCache<int,std::vector<_Edge> > fileCache; // LRUCache<int,std::vector<_Edge> > fileCache;

90
Extractor/ExtractionContainers.cpp
View File

@ -27,31 +27,31 @@ void ExtractionContainers::PrepareData(const std::string & outputFileName, const
double time = get_timestamp(); double time = get_timestamp();
boost::uint64_t memory_to_use = static_cast<boost::uint64_t>(amountOfRAM) * 1024 * 1024 * 1024; boost::uint64_t memory_to_use = static_cast<boost::uint64_t>(amountOfRAM) * 1024 * 1024 * 1024;
cout << "[extractor] Sorting used nodes ... " << flush; std::cout << "[extractor] Sorting used nodes ... " << std::flush;
stxxl::sort(usedNodeIDs.begin(), usedNodeIDs.end(), Cmp(), memory_to_use); stxxl::sort(usedNodeIDs.begin(), usedNodeIDs.end(), Cmp(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl; std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
time = get_timestamp(); time = get_timestamp();
cout << "[extractor] Erasing duplicate nodes ... " << flush; std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush;
stxxl::vector<NodeID>::iterator NewEnd = unique ( usedNodeIDs.begin(),usedNodeIDs.end() ) ; stxxl::vector<NodeID>::iterator NewEnd = std::unique ( usedNodeIDs.begin(),usedNodeIDs.end() ) ;
usedNodeIDs.resize ( NewEnd - usedNodeIDs.begin() ); usedNodeIDs.resize ( NewEnd - usedNodeIDs.begin() );
cout << "ok, after " << get_timestamp() - time << "s" << endl; std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
time = get_timestamp(); time = get_timestamp();
cout << "[extractor] Sorting all nodes ... " << flush; std::cout << "[extractor] Sorting all nodes ... " << std::flush;
stxxl::sort(allNodes.begin(), allNodes.end(), CmpNodeByID(), memory_to_use); stxxl::sort(allNodes.begin(), allNodes.end(), CmpNodeByID(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl; std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
time = get_timestamp(); time = get_timestamp();
cout << "[extractor] Sorting used ways ... " << flush; std::cout << "[extractor] Sorting used ways ... " << std::flush;
stxxl::sort(wayStartEndVector.begin(), wayStartEndVector.end(), CmpWayByID(), memory_to_use); stxxl::sort(wayStartEndVector.begin(), wayStartEndVector.end(), CmpWayByID(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl; std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
cout << "[extractor] Sorting restrctns. by from... " << flush; std::cout << "[extractor] Sorting restrctns. by from... " << std::flush;
stxxl::sort(restrictionsVector.begin(), restrictionsVector.end(), CmpRestrictionContainerByFrom(), memory_to_use); stxxl::sort(restrictionsVector.begin(), restrictionsVector.end(), CmpRestrictionContainerByFrom(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl; std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
cout << "[extractor] Fixing restriction starts ... " << flush; std::cout << "[extractor] Fixing restriction starts ... " << std::flush;
STXXLRestrictionsVector::iterator restrictionsIT = restrictionsVector.begin(); STXXLRestrictionsVector::iterator restrictionsIT = restrictionsVector.begin();
STXXLWayIDStartEndVector::iterator wayStartAndEndEdgeIT = wayStartEndVector.begin(); STXXLWayIDStartEndVector::iterator wayStartAndEndEdgeIT = wayStartEndVector.begin();
@ -79,16 +79,16 @@ void ExtractionContainers::PrepareData(const std::string & outputFileName, const
++restrictionsIT; ++restrictionsIT;
} }
cout << "ok, after " << get_timestamp() - time << "s" << endl; std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
time = get_timestamp(); time = get_timestamp();
cout << "[extractor] Sorting restrctns. by to ... " << flush; std::cout << "[extractor] Sorting restrctns. by to ... " << std::flush;
stxxl::sort(restrictionsVector.begin(), restrictionsVector.end(), CmpRestrictionContainerByTo(), memory_to_use); stxxl::sort(restrictionsVector.begin(), restrictionsVector.end(), CmpRestrictionContainerByTo(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl; std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
time = get_timestamp(); time = get_timestamp();
unsigned usableRestrictionsCounter(0); unsigned usableRestrictionsCounter(0);
cout << "[extractor] Fixing restriction ends ... " << flush; std::cout << "[extractor] Fixing restriction ends ... " << std::flush;
restrictionsIT = restrictionsVector.begin(); restrictionsIT = restrictionsVector.begin();
wayStartAndEndEdgeIT = wayStartEndVector.begin(); wayStartAndEndEdgeIT = wayStartEndVector.begin();
while(wayStartAndEndEdgeIT != wayStartEndVector.end() && restrictionsIT != restrictionsVector.end()) { while(wayStartAndEndEdgeIT != wayStartEndVector.end() && restrictionsIT != restrictionsVector.end()) {
@ -116,11 +116,11 @@ void ExtractionContainers::PrepareData(const std::string & outputFileName, const
} }
++restrictionsIT; ++restrictionsIT;
} }
cout << "ok, after " << get_timestamp() - time << "s" << endl; std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
INFO("usable restrictions: " << usableRestrictionsCounter ); INFO("usable restrictions: " << usableRestrictionsCounter );
//serialize restrictions //serialize restrictions
ofstream restrictionsOutstream; std::ofstream restrictionsOutstream;
restrictionsOutstream.open(restrictionsFileName.c_str(), ios::binary); restrictionsOutstream.open(restrictionsFileName.c_str(), std::ios::binary);
restrictionsOutstream.write((char*)&usableRestrictionsCounter, sizeof(unsigned)); restrictionsOutstream.write((char*)&usableRestrictionsCounter, sizeof(unsigned));
for(restrictionsIT = restrictionsVector.begin(); restrictionsIT != restrictionsVector.end(); ++restrictionsIT) { for(restrictionsIT = restrictionsVector.begin(); restrictionsIT != restrictionsVector.end(); ++restrictionsIT) {
if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) { if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) {
@ -129,11 +129,11 @@ void ExtractionContainers::PrepareData(const std::string & outputFileName, const
} }
restrictionsOutstream.close(); restrictionsOutstream.close();
ofstream fout; std::ofstream fout;
fout.open(outputFileName.c_str(), ios::binary); fout.open(outputFileName.c_str(), std::ios::binary);
fout.write((char*)&usedNodeCounter, sizeof(unsigned)); fout.write((char*)&usedNodeCounter, sizeof(unsigned));
time = get_timestamp(); time = get_timestamp();
cout << "[extractor] Confirming/Writing used nodes ... " << flush; std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush;
STXXLNodeVector::iterator nodesIT = allNodes.begin(); STXXLNodeVector::iterator nodesIT = allNodes.begin();
STXXLNodeIDVector::iterator usedNodeIDsIT = usedNodeIDs.begin(); STXXLNodeIDVector::iterator usedNodeIDsIT = usedNodeIDs.begin();
@ -154,24 +154,24 @@ void ExtractionContainers::PrepareData(const std::string & outputFileName, const
} }
} }
cout << "ok, after " << get_timestamp() - time << "s" << endl; std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
cout << "[extractor] setting number of nodes ... " << flush; std::cout << "[extractor] setting number of nodes ... " << std::flush;
ios::pos_type positionInFile = fout.tellp(); std::ios::pos_type positionInFile = fout.tellp();
fout.seekp(ios::beg); fout.seekp(std::ios::beg);
fout.write((char*)&usedNodeCounter, sizeof(unsigned)); fout.write((char*)&usedNodeCounter, sizeof(unsigned));
fout.seekp(positionInFile); fout.seekp(positionInFile);
cout << "ok" << endl; std::cout << "ok" << std::endl;
time = get_timestamp(); time = get_timestamp();
// Sort edges by start. // Sort edges by start.
cout << "[extractor] Sorting edges by start ... " << flush; std::cout << "[extractor] Sorting edges by start ... " << std::flush;
stxxl::sort(allEdges.begin(), allEdges.end(), CmpEdgeByStartID(), memory_to_use); stxxl::sort(allEdges.begin(), allEdges.end(), CmpEdgeByStartID(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl; std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
time = get_timestamp(); time = get_timestamp();
cout << "[extractor] Setting start coords ... " << flush; std::cout << "[extractor] Setting start coords ... " << std::flush;
fout.write((char*)&usedEdgeCounter, sizeof(unsigned)); fout.write((char*)&usedEdgeCounter, sizeof(unsigned));
// Traverse list of edges and nodes in parallel and set start coord // Traverse list of edges and nodes in parallel and set start coord
nodesIT = allNodes.begin(); nodesIT = allNodes.begin();
@ -191,16 +191,16 @@ void ExtractionContainers::PrepareData(const std::string & outputFileName, const
++edgeIT; ++edgeIT;
} }
} }
cout << "ok, after " << get_timestamp() - time << "s" << endl; std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
time = get_timestamp(); time = get_timestamp();
// Sort Edges by target // Sort Edges by target
cout << "[extractor] Sorting edges by target ... " << flush; std::cout << "[extractor] Sorting edges by target ... " << std::flush;
stxxl::sort(allEdges.begin(), allEdges.end(), CmpEdgeByTargetID(), memory_to_use); stxxl::sort(allEdges.begin(), allEdges.end(), CmpEdgeByTargetID(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl; std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
time = get_timestamp(); time = get_timestamp();
cout << "[extractor] Setting target coords ... " << flush; std::cout << "[extractor] Setting target coords ... " << std::flush;
// Traverse list of edges and nodes in parallel and set target coord // Traverse list of edges and nodes in parallel and set target coord
nodesIT = allNodes.begin(); nodesIT = allNodes.begin();
edgeIT = allEdges.begin(); edgeIT = allEdges.begin();
@ -245,8 +245,8 @@ void ExtractionContainers::PrepareData(const std::string & outputFileName, const
fout.write((char*)&one, sizeof(short)); fout.write((char*)&one, sizeof(short));
break; break;
default: default:
cerr << "[error] edge with no direction: " << edgeIT->direction << endl; std::cerr << "[error] edge with no direction: " << edgeIT->direction << std::endl;
assert(false); assert(false);
break; break;
} }
fout.write((char*)&intWeight, sizeof(int)); fout.write((char*)&intWeight, sizeof(int));
@ -261,28 +261,28 @@ void ExtractionContainers::PrepareData(const std::string & outputFileName, const
++edgeIT; ++edgeIT;
} }
} }
cout << "ok, after " << get_timestamp() - time << "s" << endl; std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
cout << "[extractor] setting number of edges ... " << flush; std::cout << "[extractor] setting number of edges ... " << std::flush;
fout.seekp(positionInFile); fout.seekp(positionInFile);
fout.write((char*)&usedEdgeCounter, sizeof(unsigned)); fout.write((char*)&usedEdgeCounter, sizeof(unsigned));
fout.close(); fout.close();
cout << "ok" << endl; std::cout << "ok" << std::endl;
time = get_timestamp(); time = get_timestamp();
cout << "[extractor] writing street name index ... " << flush; std::cout << "[extractor] writing street name index ... " << std::flush;
std::string nameOutFileName = (outputFileName + ".names"); std::string nameOutFileName = (outputFileName + ".names");
ofstream nameOutFile(nameOutFileName.c_str(), ios::binary); std::ofstream nameOutFile(nameOutFileName.c_str(), std::ios::binary);
unsigned sizeOfNameIndex = nameVector.size(); unsigned sizeOfNameIndex = nameVector.size();
nameOutFile.write((char *)&(sizeOfNameIndex), sizeof(unsigned)); nameOutFile.write((char *)&(sizeOfNameIndex), sizeof(unsigned));
BOOST_FOREACH(string str, nameVector) { BOOST_FOREACH(const std::string & str, nameVector) {
unsigned lengthOfRawString = strlen(str.c_str()); unsigned lengthOfRawString = strlen(str.c_str());
nameOutFile.write((char *)&(lengthOfRawString), sizeof(unsigned)); nameOutFile.write((char *)&(lengthOfRawString), sizeof(unsigned));
nameOutFile.write(str.c_str(), lengthOfRawString); nameOutFile.write(str.c_str(), lengthOfRawString);
} }
nameOutFile.close(); nameOutFile.close();
cout << "ok, after " << get_timestamp() - time << "s" << endl; std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
// time = get_timestamp(); // time = get_timestamp();
// cout << "[extractor] writing address list ... " << flush; // cout << "[extractor] writing address list ... " << flush;
@ -298,8 +298,8 @@ void ExtractionContainers::PrepareData(const std::string & outputFileName, const
INFO("Processed " << usedNodeCounter << " nodes and " << usedEdgeCounter << " edges"); INFO("Processed " << usedNodeCounter << " nodes and " << usedEdgeCounter << " edges");
} catch ( const exception& e ) { } catch ( const std::exception& e ) {
cerr << "Caught Execption:" << e.what() << endl; std::cerr << "Caught Execption:" << e.what() << std::endl;
} }
} }

17
typedefs.h
View File

@ -21,13 +21,18 @@ or see http://www.gnu.org/licenses/agpl.txt.
#ifndef TYPEDEFS_H_ #ifndef TYPEDEFS_H_
#define TYPEDEFS_H_ #define TYPEDEFS_H_
#include <cmath>
#include <climits>
#include <cstdlib>
// To fix long and long long woes
#include <boost/integer.hpp>
#include <boost/integer_traits.hpp>
#ifdef __APPLE__ #ifdef __APPLE__
#include <signal.h> #include <signal.h>
#endif #endif
#include <cmath>
#include <climits>
#include <cstdlib>
#include <iostream> #include <iostream>
using namespace std; using namespace std;
@ -53,7 +58,7 @@ using namespace std;
#define M_PI 3.14159265358979323846 #define M_PI 3.14159265358979323846
#endif #endif
//Necessary workaround for Windows as VS doesn't implement C99 // Necessary workaround for Windows as VS doesn't implement C99
#ifdef _MSC_VER #ifdef _MSC_VER
template<typename digitT> template<typename digitT>
digitT round(digitT x) { digitT round(digitT x) {
@ -66,7 +71,7 @@ typedef unsigned int NodeID;
typedef unsigned int EdgeID; typedef unsigned int EdgeID;
typedef unsigned int EdgeWeight; typedef unsigned int EdgeWeight;
static const NodeID SPECIAL_NODEID = UINT_MAX; static const NodeID SPECIAL_NODEID = boost::integer_traits<uint32_t>::const_max;
static const EdgeID SPECIAL_EDGEID = UINT_MAX; static const EdgeID SPECIAL_EDGEID = boost::integer_traits<uint32_t>::const_max;
#endif /* TYPEDEFS_H_ */ #endif /* TYPEDEFS_H_ */