implements #801
This commit is contained in:
Dennis Luxen
parent
cb86d3d829
commit
4911601b24
@ -214,14 +214,14 @@ public:
|
||||
//Create temporary file
|
||||
|
||||
// GetTemporaryFileName(temporaryEdgeStorageFilename);
|
||||
temporaryStorageSlotID = TemporaryStorage::GetInstance().allocateSlot();
|
||||
temporaryStorageSlotID = TemporaryStorage::GetInstance().AllocateSlot();
|
||||
std::cout << "contractor finished initalization" << std::endl;
|
||||
}
|
||||
|
||||
~Contractor() {
|
||||
//Delete temporary file
|
||||
// remove(temporaryEdgeStorageFilename.c_str());
|
||||
TemporaryStorage::GetInstance().deallocateSlot(temporaryStorageSlotID);
|
||||
TemporaryStorage::GetInstance().DeallocateSlot(temporaryStorageSlotID);
|
||||
}
|
||||
|
||||
void Run() {
|
||||
@ -287,9 +287,9 @@ public:
|
||||
TemporaryStorage & tempStorage = TemporaryStorage::GetInstance();
|
||||
//Write dummy number of edges to temporary file
|
||||
// std::ofstream temporaryEdgeStorage(temporaryEdgeStorageFilename.c_str(), std::ios::binary);
|
||||
uint64_t initialFilePosition = tempStorage.tell(temporaryStorageSlotID);
|
||||
uint64_t initialFilePosition = tempStorage.Tell(temporaryStorageSlotID);
|
||||
unsigned numberOfTemporaryEdges = 0;
|
||||
tempStorage.writeToSlot(temporaryStorageSlotID, (char*)&numberOfTemporaryEdges, sizeof(unsigned));
|
||||
tempStorage.WriteToSlot(temporaryStorageSlotID, (char*)&numberOfTemporaryEdges, sizeof(unsigned));
|
||||
|
||||
//walk over all nodes
|
||||
for(unsigned i = 0; i < _graph->GetNumberOfNodes(); ++i) {
|
||||
@ -299,9 +299,9 @@ public:
|
||||
const NodeID target = _graph->GetTarget(currentEdge);
|
||||
if(UINT_MAX == newNodeIDFromOldNodeIDMap[i] ){
|
||||
//Save edges of this node w/o renumbering.
|
||||
tempStorage.writeToSlot(temporaryStorageSlotID, (char*)&start, sizeof(NodeID));
|
||||
tempStorage.writeToSlot(temporaryStorageSlotID, (char*)&target, sizeof(NodeID));
|
||||
tempStorage.writeToSlot(temporaryStorageSlotID, (char*)&data, sizeof(_DynamicGraph::EdgeData));
|
||||
tempStorage.WriteToSlot(temporaryStorageSlotID, (char*)&start, sizeof(NodeID));
|
||||
tempStorage.WriteToSlot(temporaryStorageSlotID, (char*)&target, sizeof(NodeID));
|
||||
tempStorage.WriteToSlot(temporaryStorageSlotID, (char*)&data, sizeof(_DynamicGraph::EdgeData));
|
||||
++numberOfTemporaryEdges;
|
||||
} else {
|
||||
//node is not yet contracted.
|
||||
@ -324,8 +324,8 @@ public:
|
||||
}
|
||||
}
|
||||
//Note the number of temporarily stored edges
|
||||
tempStorage.seek(temporaryStorageSlotID, initialFilePosition);
|
||||
tempStorage.writeToSlot(temporaryStorageSlotID, (char*)&numberOfTemporaryEdges, sizeof(unsigned));
|
||||
tempStorage.Seek(temporaryStorageSlotID, initialFilePosition);
|
||||
tempStorage.WriteToSlot(temporaryStorageSlotID, (char*)&numberOfTemporaryEdges, sizeof(unsigned));
|
||||
|
||||
//Delete map from old NodeIDs to new ones.
|
||||
std::vector<NodeID>().swap(newNodeIDFromOldNodeIDMap);
|
||||
@ -498,16 +498,16 @@ public:
|
||||
TemporaryStorage & tempStorage = TemporaryStorage::GetInstance();
|
||||
//Also get the edges from temporary storage
|
||||
unsigned numberOfTemporaryEdges = 0;
|
||||
tempStorage.readFromSlot(temporaryStorageSlotID, (char*)&numberOfTemporaryEdges, sizeof(unsigned));
|
||||
tempStorage.ReadFromSlot(temporaryStorageSlotID, (char*)&numberOfTemporaryEdges, sizeof(unsigned));
|
||||
//loads edges of graph before renumbering, no need for further numbering action.
|
||||
NodeID start;
|
||||
NodeID target;
|
||||
//edges.reserve(edges.size()+numberOfTemporaryEdges);
|
||||
_DynamicGraph::EdgeData data;
|
||||
for(unsigned i = 0; i < numberOfTemporaryEdges; ++i) {
|
||||
tempStorage.readFromSlot(temporaryStorageSlotID, (char*)&start, sizeof(NodeID));
|
||||
tempStorage.readFromSlot(temporaryStorageSlotID, (char*)&target, sizeof(NodeID));
|
||||
tempStorage.readFromSlot(temporaryStorageSlotID, (char*)&data, sizeof(_DynamicGraph::EdgeData));
|
||||
tempStorage.ReadFromSlot(temporaryStorageSlotID, (char*)&start, sizeof(NodeID));
|
||||
tempStorage.ReadFromSlot(temporaryStorageSlotID, (char*)&target, sizeof(NodeID));
|
||||
tempStorage.ReadFromSlot(temporaryStorageSlotID, (char*)&data, sizeof(_DynamicGraph::EdgeData));
|
||||
Edge newEdge;
|
||||
newEdge.source = start;
|
||||
newEdge.target = target;
|
||||
@ -518,7 +518,7 @@ public:
|
||||
newEdge.data.backward = data.backward;
|
||||
edges.push_back( newEdge );
|
||||
}
|
||||
tempStorage.deallocateSlot(temporaryStorageSlotID);
|
||||
tempStorage.DeallocateSlot(temporaryStorageSlotID);
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
@ -28,113 +28,137 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#include "TemporaryStorage.h"
|
||||
|
||||
TemporaryStorage::TemporaryStorage() {
|
||||
tempDirectory = boost::filesystem::temp_directory_path();
|
||||
temp_directory = boost::filesystem::temp_directory_path();
|
||||
}
|
||||
|
||||
TemporaryStorage & TemporaryStorage::GetInstance(){
|
||||
static TemporaryStorage runningInstance;
|
||||
return runningInstance;
|
||||
static TemporaryStorage static_instance;
|
||||
return static_instance;
|
||||
}
|
||||
|
||||
TemporaryStorage::~TemporaryStorage() {
|
||||
removeAll();
|
||||
RemoveAll();
|
||||
mutex.unlock();
|
||||
}
|
||||
|
||||
void TemporaryStorage::removeAll() {
|
||||
void TemporaryStorage::RemoveAll() {
|
||||
boost::mutex::scoped_lock lock(mutex);
|
||||
for(unsigned slot_id = 0; slot_id < vectorOfStreamDatas.size(); ++slot_id) {
|
||||
deallocateSlot(slot_id);
|
||||
for(unsigned slot_id = 0; slot_id < stream_data_list.size(); ++slot_id) {
|
||||
DeallocateSlot(slot_id);
|
||||
}
|
||||
vectorOfStreamDatas.clear();
|
||||
stream_data_list.clear();
|
||||
}
|
||||
|
||||
int TemporaryStorage::allocateSlot() {
|
||||
int TemporaryStorage::AllocateSlot() {
|
||||
boost::mutex::scoped_lock lock(mutex);
|
||||
try {
|
||||
vectorOfStreamDatas.push_back(StreamData());
|
||||
//SimpleLogger().Write() << "created new temporary file: " << vectorOfStreamDatas.back().pathToTemporaryFile;
|
||||
stream_data_list.push_back(StreamData());
|
||||
} catch(boost::filesystem::filesystem_error & e) {
|
||||
abort(e);
|
||||
Abort(e);
|
||||
}
|
||||
return vectorOfStreamDatas.size() - 1;
|
||||
CheckIfTemporaryDeviceFull();
|
||||
return stream_data_list.size() - 1;
|
||||
}
|
||||
|
||||
void TemporaryStorage::deallocateSlot(int slotID) {
|
||||
void TemporaryStorage::DeallocateSlot(const int slot_id) {
|
||||
try {
|
||||
StreamData & data = vectorOfStreamDatas[slotID];
|
||||
StreamData & data = stream_data_list[slot_id];
|
||||
boost::mutex::scoped_lock lock(*data.readWriteMutex);
|
||||
if(!boost::filesystem::exists(data.pathToTemporaryFile)) {
|
||||
if(!boost::filesystem::exists(data.temp_path)) {
|
||||
return;
|
||||
}
|
||||
if(data.streamToTemporaryFile->is_open()) {
|
||||
data.streamToTemporaryFile->close();
|
||||
if(data.temp_file->is_open()) {
|
||||
data.temp_file->close();
|
||||
}
|
||||
|
||||
boost::filesystem::remove(data.pathToTemporaryFile);
|
||||
boost::filesystem::remove(data.temp_path);
|
||||
} catch(boost::filesystem::filesystem_error & e) {
|
||||
abort(e);
|
||||
Abort(e);
|
||||
}
|
||||
}
|
||||
|
||||
void TemporaryStorage::writeToSlot(int slotID, char * pointer, std::streamsize size) {
|
||||
void TemporaryStorage::WriteToSlot(
|
||||
const int slot_id,
|
||||
char * pointer,
|
||||
std::streamsize size
|
||||
) {
|
||||
try {
|
||||
StreamData & data = vectorOfStreamDatas[slotID];
|
||||
StreamData & data = stream_data_list[slot_id];
|
||||
boost::mutex::scoped_lock lock(*data.readWriteMutex);
|
||||
BOOST_ASSERT_MSG(
|
||||
data.writeMode,
|
||||
data.write_mode,
|
||||
"Writing after first read is not allowed"
|
||||
);
|
||||
data.streamToTemporaryFile->write(pointer, size);
|
||||
data.temp_file->write(pointer, size);
|
||||
CheckIfTemporaryDeviceFull();
|
||||
} catch(boost::filesystem::filesystem_error & e) {
|
||||
abort(e);
|
||||
Abort(e);
|
||||
}
|
||||
}
|
||||
void TemporaryStorage::readFromSlot(int slotID, char * pointer, std::streamsize size) {
|
||||
void TemporaryStorage::ReadFromSlot(
|
||||
const int slot_id,
|
||||
char * pointer,
|
||||
std::streamsize size
|
||||
) {
|
||||
try {
|
||||
StreamData & data = vectorOfStreamDatas[slotID];
|
||||
StreamData & data = stream_data_list[slot_id];
|
||||
boost::mutex::scoped_lock lock(*data.readWriteMutex);
|
||||
if(data.writeMode) {
|
||||
data.writeMode = false;
|
||||
data.streamToTemporaryFile->seekg(0, data.streamToTemporaryFile->beg);
|
||||
if(data.write_mode) {
|
||||
data.write_mode = false;
|
||||
data.temp_file->seekg(0, data.temp_file->beg);
|
||||
}
|
||||
data.streamToTemporaryFile->read(pointer, size);
|
||||
data.temp_file->read(pointer, size);
|
||||
} catch(boost::filesystem::filesystem_error & e) {
|
||||
abort(e);
|
||||
Abort(e);
|
||||
}
|
||||
}
|
||||
|
||||
unsigned TemporaryStorage::getFreeBytesOnTemporaryDevice() {
|
||||
boost::filesystem::space_info tempSpaceInfo;
|
||||
uint64_t TemporaryStorage::GetFreeBytesOnTemporaryDevice() {
|
||||
uint64_t value = -1;
|
||||
try {
|
||||
tempSpaceInfo = boost::filesystem::space(tempDirectory);
|
||||
boost::filesystem::path p = boost::filesystem::temp_directory_path();
|
||||
boost::filesystem::space_info s = boost::filesystem::space( p );
|
||||
value = s.free;
|
||||
} catch(boost::filesystem::filesystem_error & e) {
|
||||
abort(e);
|
||||
Abort(e);
|
||||
}
|
||||
return tempSpaceInfo.available;
|
||||
return value;
|
||||
}
|
||||
|
||||
boost::filesystem::fstream::pos_type TemporaryStorage::tell(int slotID) {
|
||||
void TemporaryStorage::CheckIfTemporaryDeviceFull() {
|
||||
boost::filesystem::path p = boost::filesystem::temp_directory_path();
|
||||
boost::filesystem::space_info s = boost::filesystem::space( p );
|
||||
if(1024*1024 > s.free) {
|
||||
throw OSRMException("temporary device is full");
|
||||
}
|
||||
}
|
||||
|
||||
boost::filesystem::fstream::pos_type TemporaryStorage::Tell(int slot_id) {
|
||||
boost::filesystem::fstream::pos_type position;
|
||||
try {
|
||||
StreamData & data = vectorOfStreamDatas[slotID];
|
||||
StreamData & data = stream_data_list[slot_id];
|
||||
boost::mutex::scoped_lock lock(*data.readWriteMutex);
|
||||
position = data.streamToTemporaryFile->tellp();
|
||||
position = data.temp_file->tellp();
|
||||
} catch(boost::filesystem::filesystem_error & e) {
|
||||
abort(e);
|
||||
Abort(e);
|
||||
}
|
||||
return position;
|
||||
}
|
||||
|
||||
void TemporaryStorage::abort(boost::filesystem::filesystem_error& ) {
|
||||
removeAll();
|
||||
void TemporaryStorage::Abort(boost::filesystem::filesystem_error& e) {
|
||||
RemoveAll();
|
||||
throw OSRMException(e.what());
|
||||
}
|
||||
|
||||
void TemporaryStorage::seek(int slotID, boost::filesystem::fstream::pos_type position) {
|
||||
void TemporaryStorage::Seek(
|
||||
const int slot_id,
|
||||
const boost::filesystem::fstream::pos_type position
|
||||
) {
|
||||
try {
|
||||
StreamData & data = vectorOfStreamDatas[slotID];
|
||||
StreamData & data = stream_data_list[slot_id];
|
||||
boost::mutex::scoped_lock lock(*data.readWriteMutex);
|
||||
data.streamToTemporaryFile->seekg(position);
|
||||
data.temp_file->seekg(position);
|
||||
} catch(boost::filesystem::filesystem_error & e) {
|
||||
abort(e);
|
||||
Abort(e);
|
||||
}
|
||||
}
|
||||
|
||||
@ -28,46 +28,23 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#ifndef TEMPORARYSTORAGE_H_
|
||||
#define TEMPORARYSTORAGE_H_
|
||||
|
||||
#include <vector>
|
||||
#include <fstream>
|
||||
|
||||
#include <boost/assert.hpp>
|
||||
#include <boost/foreach.hpp>
|
||||
#include <boost/filesystem.hpp>
|
||||
#include <boost/filesystem/fstream.hpp>
|
||||
#include <boost/shared_ptr.hpp>
|
||||
#include <boost/thread/mutex.hpp>
|
||||
|
||||
#include "../Util/BoostFileSystemFix.h"
|
||||
#include "../Util/OSRMException.h"
|
||||
#include "../Util/SimpleLogger.h"
|
||||
#include "../typedefs.h"
|
||||
|
||||
//This is one big workaround for latest boost renaming woes.
|
||||
#include <boost/assert.hpp>
|
||||
#include <boost/foreach.hpp>
|
||||
#include <boost/integer.hpp>
|
||||
#include <boost/filesystem.hpp>
|
||||
#include <boost/filesystem/fstream.hpp>
|
||||
#include <boost/make_shared.hpp>
|
||||
#include <boost/shared_ptr.hpp>
|
||||
#include <boost/thread/mutex.hpp>
|
||||
|
||||
#if BOOST_FILESYSTEM_VERSION < 3
|
||||
#warning Boost Installation with Filesystem3 missing, activating workaround
|
||||
#include <cstdio>
|
||||
namespace boost {
|
||||
namespace filesystem {
|
||||
inline path temp_directory_path() {
|
||||
char * buffer;
|
||||
buffer = tmpnam (NULL);
|
||||
#include <vector>
|
||||
#include <fstream>
|
||||
|
||||
return path(buffer);
|
||||
}
|
||||
|
||||
inline path unique_path(const path&) {
|
||||
return temp_directory_path();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#ifndef BOOST_FILESYSTEM_VERSION
|
||||
#define BOOST_FILESYSTEM_VERSION 3
|
||||
#endif
|
||||
/**
|
||||
* This class implements a singleton file storage for temporary data.
|
||||
* temporary slots can be accessed by other objects through an int
|
||||
@ -77,48 +54,63 @@ inline path unique_path(const path&) {
|
||||
* -> Data is written in first phase and reread in second.
|
||||
*/
|
||||
|
||||
static boost::filesystem::path tempDirectory;
|
||||
static boost::filesystem::path temp_directory;
|
||||
static std::string TemporaryFilePattern("OSRM-%%%%-%%%%-%%%%");
|
||||
class TemporaryStorage {
|
||||
public:
|
||||
static TemporaryStorage & GetInstance();
|
||||
virtual ~TemporaryStorage();
|
||||
|
||||
int allocateSlot();
|
||||
void deallocateSlot(int slotID);
|
||||
void writeToSlot(int slotID, char * pointer, std::streamsize size);
|
||||
void readFromSlot(int slotID, char * pointer, std::streamsize size);
|
||||
int AllocateSlot();
|
||||
void DeallocateSlot(const int slot_id);
|
||||
void WriteToSlot(const int slot_id, char * pointer, std::streamsize size);
|
||||
void ReadFromSlot(const int slot_id, char * pointer, std::streamsize size);
|
||||
//returns the number of free bytes
|
||||
unsigned getFreeBytesOnTemporaryDevice();
|
||||
boost::filesystem::fstream::pos_type tell(int slotID);
|
||||
void seek(int slotID, boost::filesystem::fstream::pos_type);
|
||||
void removeAll();
|
||||
uint64_t GetFreeBytesOnTemporaryDevice();
|
||||
boost::filesystem::fstream::pos_type Tell(int slot_id);
|
||||
void Seek(const int slot_id, boost::filesystem::fstream::pos_type);
|
||||
void RemoveAll();
|
||||
private:
|
||||
TemporaryStorage();
|
||||
TemporaryStorage(TemporaryStorage const &){};
|
||||
|
||||
TemporaryStorage & operator=(TemporaryStorage const &) {
|
||||
return *this;
|
||||
}
|
||||
void abort(boost::filesystem::filesystem_error& e);
|
||||
|
||||
void Abort(boost::filesystem::filesystem_error& e);
|
||||
void CheckIfTemporaryDeviceFull();
|
||||
|
||||
struct StreamData {
|
||||
bool writeMode;
|
||||
boost::filesystem::path pathToTemporaryFile;
|
||||
boost::shared_ptr<boost::filesystem::fstream> streamToTemporaryFile;
|
||||
bool write_mode;
|
||||
boost::filesystem::path temp_path;
|
||||
boost::shared_ptr<boost::filesystem::fstream> temp_file;
|
||||
boost::shared_ptr<boost::mutex> readWriteMutex;
|
||||
StreamData() :
|
||||
writeMode(true),
|
||||
pathToTemporaryFile (boost::filesystem::unique_path(tempDirectory.append(TemporaryFilePattern.begin(), TemporaryFilePattern.end()))),
|
||||
streamToTemporaryFile(new boost::filesystem::fstream(pathToTemporaryFile, std::ios::in | std::ios::out | std::ios::trunc | std::ios::binary)),
|
||||
readWriteMutex(new boost::mutex)
|
||||
write_mode(true),
|
||||
temp_path(
|
||||
boost::filesystem::unique_path(
|
||||
temp_directory.append(
|
||||
TemporaryFilePattern.begin(),
|
||||
TemporaryFilePattern.end()
|
||||
)
|
||||
)
|
||||
),
|
||||
temp_file(
|
||||
new boost::filesystem::fstream(
|
||||
temp_path,
|
||||
std::ios::in|std::ios::out|std::ios::trunc|std::ios::binary
|
||||
)
|
||||
),
|
||||
readWriteMutex(boost::make_shared<boost::mutex>())
|
||||
{
|
||||
if(streamToTemporaryFile->fail()) {
|
||||
if(temp_file->fail()) {
|
||||
throw OSRMException("temporary file could not be created");
|
||||
}
|
||||
}
|
||||
};
|
||||
//vector of file streams that is used to store temporary data
|
||||
std::vector<StreamData> vectorOfStreamDatas;
|
||||
std::vector<StreamData> stream_data_list;
|
||||
boost::mutex mutex;
|
||||
};
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user