Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

implements #801

Browse Source
This commit is contained in:
Dennis Luxen 2013-11-27 23:24:00 +01:00
parent 186859fcf0
commit 5fe79764de
3 changed files with 133 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
Contractor/Contractor.h
View File

@ -214,14 +214,14 @@ public:
//Create temporary file
// GetTemporaryFileName(temporaryEdgeStorageFilename);
temporaryStorageSlotID = TemporaryStorage::GetInstance().allocateSlot();
temporaryStorageSlotID = TemporaryStorage::GetInstance().AllocateSlot();
std::cout << "contractor finished initalization" << std::endl;
}
~Contractor() {
//Delete temporary file
// remove(temporaryEdgeStorageFilename.c_str());
TemporaryStorage::GetInstance().deallocateSlot(temporaryStorageSlotID);
TemporaryStorage::GetInstance().DeallocateSlot(temporaryStorageSlotID);
}
void Run() {
@ -287,9 +287,9 @@ public:
TemporaryStorage & tempStorage = TemporaryStorage::GetInstance();
//Write dummy number of edges to temporary file
// std::ofstream temporaryEdgeStorage(temporaryEdgeStorageFilename.c_str(), std::ios::binary);
uint64_t initialFilePosition = tempStorage.tell(temporaryStorageSlotID);
uint64_t initialFilePosition = tempStorage.Tell(temporaryStorageSlotID);
unsigned numberOfTemporaryEdges = 0;
tempStorage.writeToSlot(temporaryStorageSlotID, (char*)&numberOfTemporaryEdges, sizeof(unsigned));
tempStorage.WriteToSlot(temporaryStorageSlotID, (char*)&numberOfTemporaryEdges, sizeof(unsigned));
//walk over all nodes
for(unsigned i = 0; i < _graph->GetNumberOfNodes(); ++i) {
@ -299,11 +299,11 @@ public:
const NodeID target = _graph->GetTarget(currentEdge);
if(UINT_MAX == newNodeIDFromOldNodeIDMap[i] ){
//Save edges of this node w/o renumbering.
tempStorage.writeToSlot(temporaryStorageSlotID, (char*)&start, sizeof(NodeID));
tempStorage.writeToSlot(temporaryStorageSlotID, (char*)&target, sizeof(NodeID));
tempStorage.writeToSlot(temporaryStorageSlotID, (char*)&data, sizeof(_DynamicGraph::EdgeData));
tempStorage.WriteToSlot(temporaryStorageSlotID, (char*)&start, sizeof(NodeID));
tempStorage.WriteToSlot(temporaryStorageSlotID, (char*)&target, sizeof(NodeID));
tempStorage.WriteToSlot(temporaryStorageSlotID, (char*)&data, sizeof(_DynamicGraph::EdgeData));
++numberOfTemporaryEdges;
}else {
} else {
//node is not yet contracted.
//add (renumbered) outgoing edges to new DynamicGraph.
_ContractorEdge newEdge;
@ -324,8 +324,8 @@ public:
}
}
//Note the number of temporarily stored edges
tempStorage.seek(temporaryStorageSlotID, initialFilePosition);
tempStorage.writeToSlot(temporaryStorageSlotID, (char*)&numberOfTemporaryEdges, sizeof(unsigned));
tempStorage.Seek(temporaryStorageSlotID, initialFilePosition);
tempStorage.WriteToSlot(temporaryStorageSlotID, (char*)&numberOfTemporaryEdges, sizeof(unsigned));
//Delete map from old NodeIDs to new ones.
std::vector<NodeID>().swap(newNodeIDFromOldNodeIDMap);
@ -498,16 +498,16 @@ public:
TemporaryStorage & tempStorage = TemporaryStorage::GetInstance();
//Also get the edges from temporary storage
unsigned numberOfTemporaryEdges = 0;
tempStorage.readFromSlot(temporaryStorageSlotID, (char*)&numberOfTemporaryEdges, sizeof(unsigned));
tempStorage.ReadFromSlot(temporaryStorageSlotID, (char*)&numberOfTemporaryEdges, sizeof(unsigned));
//loads edges of graph before renumbering, no need for further numbering action.
NodeID start;
NodeID target;
//edges.reserve(edges.size()+numberOfTemporaryEdges);
_DynamicGraph::EdgeData data;
for(unsigned i = 0; i < numberOfTemporaryEdges; ++i) {
tempStorage.readFromSlot(temporaryStorageSlotID, (char*)&start, sizeof(NodeID));
tempStorage.readFromSlot(temporaryStorageSlotID, (char*)&target, sizeof(NodeID));
tempStorage.readFromSlot(temporaryStorageSlotID, (char*)&data, sizeof(_DynamicGraph::EdgeData));
tempStorage.ReadFromSlot(temporaryStorageSlotID, (char*)&start, sizeof(NodeID));
tempStorage.ReadFromSlot(temporaryStorageSlotID, (char*)&target, sizeof(NodeID));
tempStorage.ReadFromSlot(temporaryStorageSlotID, (char*)&data, sizeof(_DynamicGraph::EdgeData));
Edge newEdge;
newEdge.source = start;
newEdge.target = target;
@ -518,7 +518,7 @@ public:
newEdge.data.backward = data.backward;
edges.push_back( newEdge );
}
tempStorage.deallocateSlot(temporaryStorageSlotID);
tempStorage.DeallocateSlot(temporaryStorageSlotID);
}
private:

120
Contractor/TemporaryStorage.cpp
View File

@ -28,113 +28,137 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "TemporaryStorage.h"
TemporaryStorage::TemporaryStorage() {
tempDirectory = boost::filesystem::temp_directory_path();
temp_directory = boost::filesystem::temp_directory_path();
}
TemporaryStorage & TemporaryStorage::GetInstance(){
static TemporaryStorage runningInstance;
return runningInstance;
static TemporaryStorage static_instance;
return static_instance;
}
TemporaryStorage::~TemporaryStorage() {
removeAll();
RemoveAll();
mutex.unlock();
}
void TemporaryStorage::removeAll() {
void TemporaryStorage::RemoveAll() {
boost::mutex::scoped_lock lock(mutex);
for(unsigned slot_id = 0; slot_id < vectorOfStreamDatas.size(); ++slot_id) {
deallocateSlot(slot_id);
for(unsigned slot_id = 0; slot_id < stream_data_list.size(); ++slot_id) {
DeallocateSlot(slot_id);
}
vectorOfStreamDatas.clear();
stream_data_list.clear();
}
int TemporaryStorage::allocateSlot() {
int TemporaryStorage::AllocateSlot() {
boost::mutex::scoped_lock lock(mutex);
try {
vectorOfStreamDatas.push_back(StreamData());
//SimpleLogger().Write() << "created new temporary file: " << vectorOfStreamDatas.back().pathToTemporaryFile;
stream_data_list.push_back(StreamData());
} catch(boost::filesystem::filesystem_error & e) {
abort(e);
Abort(e);
}
return vectorOfStreamDatas.size() - 1;
CheckIfTemporaryDeviceFull();
return stream_data_list.size() - 1;
}
void TemporaryStorage::deallocateSlot(int slotID) {
void TemporaryStorage::DeallocateSlot(const int slot_id) {
try {
StreamData & data = vectorOfStreamDatas[slotID];
StreamData & data = stream_data_list[slot_id];
boost::mutex::scoped_lock lock(*data.readWriteMutex);
if(!boost::filesystem::exists(data.pathToTemporaryFile)) {
if(!boost::filesystem::exists(data.temp_path)) {
return;
}
if(data.streamToTemporaryFile->is_open()) {
data.streamToTemporaryFile->close();
if(data.temp_file->is_open()) {
data.temp_file->close();
}
boost::filesystem::remove(data.pathToTemporaryFile);
boost::filesystem::remove(data.temp_path);
} catch(boost::filesystem::filesystem_error & e) {
abort(e);
Abort(e);
}
}
void TemporaryStorage::writeToSlot(int slotID, char * pointer, std::streamsize size) {
void TemporaryStorage::WriteToSlot(
const int slot_id,
char * pointer,
std::streamsize size
) {
try {
StreamData & data = vectorOfStreamDatas[slotID];
StreamData & data = stream_data_list[slot_id];
boost::mutex::scoped_lock lock(*data.readWriteMutex);
BOOST_ASSERT_MSG(
data.writeMode,
data.write_mode,
"Writing after first read is not allowed"
);
data.streamToTemporaryFile->write(pointer, size);
data.temp_file->write(pointer, size);
CheckIfTemporaryDeviceFull();
} catch(boost::filesystem::filesystem_error & e) {
abort(e);
Abort(e);
}
}
void TemporaryStorage::readFromSlot(int slotID, char * pointer, std::streamsize size) {
void TemporaryStorage::ReadFromSlot(
const int slot_id,
char * pointer,
std::streamsize size
) {
try {
StreamData & data = vectorOfStreamDatas[slotID];
StreamData & data = stream_data_list[slot_id];
boost::mutex::scoped_lock lock(*data.readWriteMutex);
if(data.writeMode) {
data.writeMode = false;
data.streamToTemporaryFile->seekg(0, data.streamToTemporaryFile->beg);
if(data.write_mode) {
data.write_mode = false;
data.temp_file->seekg(0, data.temp_file->beg);
}
data.streamToTemporaryFile->read(pointer, size);
data.temp_file->read(pointer, size);
} catch(boost::filesystem::filesystem_error & e) {
abort(e);
Abort(e);
}
}
unsigned TemporaryStorage::getFreeBytesOnTemporaryDevice() {
boost::filesystem::space_info tempSpaceInfo;
uint64_t TemporaryStorage::GetFreeBytesOnTemporaryDevice() {
uint64_t value = -1;
try {
tempSpaceInfo = boost::filesystem::space(tempDirectory);
boost::filesystem::path p = boost::filesystem::temp_directory_path();
boost::filesystem::space_info s = boost::filesystem::space( p );
value = s.free;
} catch(boost::filesystem::filesystem_error & e) {
abort(e);
Abort(e);
}
return tempSpaceInfo.available;
return value;
}
boost::filesystem::fstream::pos_type TemporaryStorage::tell(int slotID) {
void TemporaryStorage::CheckIfTemporaryDeviceFull() {
boost::filesystem::path p = boost::filesystem::temp_directory_path();
boost::filesystem::space_info s = boost::filesystem::space( p );
if(1024*1024 > s.free) {
throw OSRMException("temporary device is full");
}
}
boost::filesystem::fstream::pos_type TemporaryStorage::Tell(int slot_id) {
boost::filesystem::fstream::pos_type position;
try {
StreamData & data = vectorOfStreamDatas[slotID];
StreamData & data = stream_data_list[slot_id];
boost::mutex::scoped_lock lock(*data.readWriteMutex);
position = data.streamToTemporaryFile->tellp();
position = data.temp_file->tellp();
} catch(boost::filesystem::filesystem_error & e) {
abort(e);
}
Abort(e);
}
return position;
}
void TemporaryStorage::abort(boost::filesystem::filesystem_error& ) {
removeAll();
void TemporaryStorage::Abort(boost::filesystem::filesystem_error& e) {
RemoveAll();
throw OSRMException(e.what());
}
void TemporaryStorage::seek(int slotID, boost::filesystem::fstream::pos_type position) {
void TemporaryStorage::Seek(
const int slot_id,
const boost::filesystem::fstream::pos_type position
) {
try {
StreamData & data = vectorOfStreamDatas[slotID];
StreamData & data = stream_data_list[slot_id];
boost::mutex::scoped_lock lock(*data.readWriteMutex);
data.streamToTemporaryFile->seekg(position);
data.temp_file->seekg(position);
} catch(boost::filesystem::filesystem_error & e) {
abort(e);
Abort(e);
}
}

100
Contractor/TemporaryStorage.h
View File

@ -28,46 +28,23 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef TEMPORARYSTORAGE_H_
#define TEMPORARYSTORAGE_H_
#include <vector>
#include <fstream>
#include <boost/assert.hpp>
#include <boost/foreach.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/thread/mutex.hpp>
#include "../Util/BoostFileSystemFix.h"
#include "../Util/OSRMException.h"
#include "../Util/SimpleLogger.h"
#include "../typedefs.h"
//This is one big workaround for latest boost renaming woes.
#include <boost/assert.hpp>
#include <boost/foreach.hpp>
#include <boost/integer.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/make_shared.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/thread/mutex.hpp>
#if BOOST_FILESYSTEM_VERSION < 3
#warning Boost Installation with Filesystem3 missing, activating workaround
#include <cstdio>
namespace boost {
namespace filesystem {
inline path temp_directory_path() {
char * buffer;
buffer = tmpnam (NULL);
#include <vector>
#include <fstream>
return path(buffer);
}
inline path unique_path(const path&) {
return temp_directory_path();
}
}
}
#endif
#ifndef BOOST_FILESYSTEM_VERSION
#define BOOST_FILESYSTEM_VERSION 3
#endif
/**
* This class implements a singleton file storage for temporary data.
* temporary slots can be accessed by other objects through an int
@ -77,48 +54,63 @@ inline path unique_path(const path&) {
* -> Data is written in first phase and reread in second.
*/
static boost::filesystem::path tempDirectory;
static boost::filesystem::path temp_directory;
static std::string TemporaryFilePattern("OSRM-%%%%-%%%%-%%%%");
class TemporaryStorage {
public:
static TemporaryStorage & GetInstance();
virtual ~TemporaryStorage();
int allocateSlot();
void deallocateSlot(int slotID);
void writeToSlot(int slotID, char * pointer, std::streamsize size);
void readFromSlot(int slotID, char * pointer, std::streamsize size);
int AllocateSlot();
void DeallocateSlot(const int slot_id);
void WriteToSlot(const int slot_id, char * pointer, std::streamsize size);
void ReadFromSlot(const int slot_id, char * pointer, std::streamsize size);
//returns the number of free bytes
unsigned getFreeBytesOnTemporaryDevice();
boost::filesystem::fstream::pos_type tell(int slotID);
void seek(int slotID, boost::filesystem::fstream::pos_type);
void removeAll();
uint64_t GetFreeBytesOnTemporaryDevice();
boost::filesystem::fstream::pos_type Tell(int slot_id);
void Seek(const int slot_id, boost::filesystem::fstream::pos_type);
void RemoveAll();
private:
TemporaryStorage();
TemporaryStorage(TemporaryStorage const &){};
TemporaryStorage& operator=(TemporaryStorage const &) {
TemporaryStorage & operator=(TemporaryStorage const &) {
return *this;
}
void abort(boost::filesystem::filesystem_error& e);
void Abort(boost::filesystem::filesystem_error& e);
void CheckIfTemporaryDeviceFull();
struct StreamData {
bool writeMode;
boost::filesystem::path pathToTemporaryFile;
boost::shared_ptr<boost::filesystem::fstream> streamToTemporaryFile;
bool write_mode;
boost::filesystem::path temp_path;
boost::shared_ptr<boost::filesystem::fstream> temp_file;
boost::shared_ptr<boost::mutex> readWriteMutex;
StreamData() :
writeMode(true),
pathToTemporaryFile (boost::filesystem::unique_path(tempDirectory.append(TemporaryFilePattern.begin(), TemporaryFilePattern.end()))),
streamToTemporaryFile(new boost::filesystem::fstream(pathToTemporaryFile, std::ios::in | std::ios::out | std::ios::trunc | std::ios::binary)),
readWriteMutex(new boost::mutex)
write_mode(true),
temp_path(
boost::filesystem::unique_path(
temp_directory.append(
TemporaryFilePattern.begin(),
TemporaryFilePattern.end()
)
)
),
temp_file(
new boost::filesystem::fstream(
temp_path,
std::ios::in|std::ios::out|std::ios::trunc|std::ios::binary
)
),
readWriteMutex(boost::make_shared<boost::mutex>())
{
if(streamToTemporaryFile->fail()) {
if(temp_file->fail()) {
throw OSRMException("temporary file could not be created");
}
}
};
//vector of file streams that is used to store temporary data
std::vector<StreamData> vectorOfStreamDatas;
std::vector<StreamData> stream_data_list;
boost::mutex mutex;
};