Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

use stxxl as external data store instead of hand-rolled code in /tmp

Browse Source
This commit is contained in:
Dennis Luxen 2014-07-28 15:58:50 +02:00
parent 4990e544cf
commit 2bebed44ff
5 changed files with 20 additions and 304 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
Contractor/Contractor.h
View File

@ -28,20 +28,23 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef CONTRACTOR_H #ifndef CONTRACTOR_H
#define CONTRACTOR_H #define CONTRACTOR_H
#include "TemporaryStorage.h"
#include "../DataStructures/BinaryHeap.h" #include "../DataStructures/BinaryHeap.h"
#include "../DataStructures/DeallocatingVector.h" #include "../DataStructures/DeallocatingVector.h"
#include "../DataStructures/DynamicGraph.h" #include "../DataStructures/DynamicGraph.h"
#include "../DataStructures/Percent.h" #include "../DataStructures/Percent.h"
#include "../DataStructures/QueryEdge.h"
#include "../DataStructures/XORFastHash.h" #include "../DataStructures/XORFastHash.h"
#include "../DataStructures/XORFastHashStorage.h" #include "../DataStructures/XORFastHashStorage.h"
#include "../Util/SimpleLogger.h" #include "../Util/SimpleLogger.h"
#include "../Util/StringUtil.h" #include "../Util/StringUtil.h"
#include "../Util/TimingUtil.h" #include "../Util/TimingUtil.h"
#include "../typedefs.h"
#include <boost/assert.hpp> #include <boost/assert.hpp>
#include <boost/range/irange.hpp> #include <boost/range/irange.hpp>
#include <stxxl/vector>
#include <tbb/enumerable_thread_specific.h> #include <tbb/enumerable_thread_specific.h>
#include <tbb/parallel_for.h> #include <tbb/parallel_for.h>
#include <tbb/parallel_sort.h> #include <tbb/parallel_sort.h>
@ -157,7 +160,6 @@ class Contractor
{ {
std::vector<ContractorEdge> edges; std::vector<ContractorEdge> edges;
edges.reserve(input_edge_list.size() * 2); edges.reserve(input_edge_list.size() * 2);
temp_edge_counter = 0;
const auto dend = input_edge_list.dend(); const auto dend = input_edge_list.dend();
for (auto diter = input_edge_list.dbegin(); diter != dend; ++diter) for (auto diter = input_edge_list.dbegin(); diter != dend; ++diter)
@ -279,13 +281,10 @@ class Contractor
// << "); via: " << contractor_graph->GetEdgeData(i).via; // << "); via: " << contractor_graph->GetEdgeData(i).via;
// } // }
// Create temporary file
edge_storage_slot = TemporaryStorage::GetInstance().AllocateSlot();
std::cout << "contractor finished initalization" << std::endl; std::cout << "contractor finished initalization" << std::endl;
} }
~Contractor() { TemporaryStorage::GetInstance().DeallocateSlot(edge_storage_slot); } ~Contractor() { }
void Run() void Run()
{ {
@ -370,7 +369,6 @@ class Contractor
node_priorities[remaining_nodes[new_node_id].id]; node_priorities[remaining_nodes[new_node_id].id];
remaining_nodes[new_node_id].id = new_node_id; remaining_nodes[new_node_id].id = new_node_id;
} }
TemporaryStorage &temporary_storage = TemporaryStorage::GetInstance();
// walk over all nodes // walk over all nodes
for (const auto i : boost::irange(0u, (unsigned)contractor_graph->GetNumberOfNodes())) for (const auto i : boost::irange(0u, (unsigned)contractor_graph->GetNumberOfNodes()))
{ {
@ -382,15 +380,7 @@ class Contractor
const NodeID target = contractor_graph->GetTarget(current_edge); const NodeID target = contractor_graph->GetTarget(current_edge);
if (SPECIAL_NODEID == new_node_id_from_orig_id_map[i]) if (SPECIAL_NODEID == new_node_id_from_orig_id_map[i])
{ {
// Save edges of this node w/o renumbering. external_edge_list.push_back({source, target, data});
temporary_storage.WriteToSlot(
edge_storage_slot, (char *)&source, sizeof(NodeID));
temporary_storage.WriteToSlot(
edge_storage_slot, (char *)&target, sizeof(NodeID));
temporary_storage.WriteToSlot(edge_storage_slot,
(char *)&data,
sizeof(ContractorGraph::EdgeData));
++temp_edge_counter;
} }
else else
{ {
@ -612,22 +602,8 @@ class Contractor
orig_node_id_to_new_id_map.shrink_to_fit(); orig_node_id_to_new_id_map.shrink_to_fit();
BOOST_ASSERT(0 == orig_node_id_to_new_id_map.capacity()); BOOST_ASSERT(0 == orig_node_id_to_new_id_map.capacity());
TemporaryStorage &temporary_storage = TemporaryStorage::GetInstance();
// loads edges of graph before renumbering, no need for further numbering action. edges.append(external_edge_list.begin(), external_edge_list.end());
NodeID source;
NodeID target;
ContractorGraph::EdgeData data;
for (uint64_t i = 0; i < temp_edge_counter; ++i)
{
temporary_storage.ReadFromSlot(edge_storage_slot, (char *)&source, sizeof(NodeID));
temporary_storage.ReadFromSlot(edge_storage_slot, (char *)&target, sizeof(NodeID));
temporary_storage.ReadFromSlot(
edge_storage_slot, (char *)&data, sizeof(ContractorGraph::EdgeData));
edges.emplace_back(source, target, data);
}
temporary_storage.DeallocateSlot(edge_storage_slot);
} }
private: private:
@ -989,8 +965,7 @@ class Contractor
std::shared_ptr<ContractorGraph> contractor_graph; std::shared_ptr<ContractorGraph> contractor_graph;
std::vector<ContractorGraph::InputEdge> contracted_edge_list; std::vector<ContractorGraph::InputEdge> contracted_edge_list;
std::size_t edge_storage_slot; stxxl::vector<QueryEdge> external_edge_list;
uint64_t temp_edge_counter;
std::vector<NodeID> orig_node_id_to_new_id_map; std::vector<NodeID> orig_node_id_to_new_id_map;
XORFastHash fast_hash; XORFastHash fast_hash;
}; };

1
Contractor/Prepare.cpp
View File

@ -44,6 +44,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "../typedefs.h" #include "../typedefs.h"
#include <boost/filesystem/fstream.hpp> #include <boost/filesystem/fstream.hpp>
#include <boost/program_options.hpp>
#include <boost/range.hpp> #include <boost/range.hpp>
#include <tbb/task_scheduler_init.h> #include <tbb/task_scheduler_init.h>

175
Contractor/TemporaryStorage.cpp
View File

@ -1,175 +0,0 @@
/*
Copyright (c) 2013, Project OSRM, Dennis Luxen, others
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this list
of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "TemporaryStorage.h"
#include <boost/range/irange.hpp>
StreamData::StreamData()
: write_mode(true),
temp_path(boost::filesystem::unique_path(temp_directory / TemporaryFilePattern)),
temp_file(new boost::filesystem::fstream(
temp_path, std::ios::in | std::ios::out | std::ios::trunc | std::ios::binary)),
readWriteMutex(std::make_shared<boost::mutex>())
{
if (temp_file->fail())
{
throw OSRMException("temporary file could not be created");
}
}
TemporaryStorage::TemporaryStorage() { temp_directory = boost::filesystem::temp_directory_path(); }
TemporaryStorage &TemporaryStorage::GetInstance()
{
static TemporaryStorage static_instance;
return static_instance;
}
TemporaryStorage::~TemporaryStorage() { RemoveAll(); }
void TemporaryStorage::RemoveAll()
{
boost::mutex::scoped_lock lock(mutex);
for (const auto slot_id : boost::irange((std::size_t)0, stream_data_list.size()))
{
DeallocateSlot(slot_id);
}
stream_data_list.clear();
}
std::size_t TemporaryStorage::AllocateSlot()
{
boost::mutex::scoped_lock lock(mutex);
try { stream_data_list.push_back(StreamData()); }
catch (boost::filesystem::filesystem_error &e) { Abort(e); }
CheckIfTemporaryDeviceFull();
return stream_data_list.size() - 1;
}
void TemporaryStorage::DeallocateSlot(const std::size_t slot_id)
{
try
{
StreamData &data = stream_data_list[slot_id];
boost::mutex::scoped_lock lock(*data.readWriteMutex);
if (!boost::filesystem::exists(data.temp_path))
{
return;
}
if (data.temp_file->is_open())
{
data.temp_file->close();
}
boost::filesystem::remove(data.temp_path);
}
catch (boost::filesystem::filesystem_error &e) { Abort(e); }
}
void TemporaryStorage::WriteToSlot(const std::size_t slot_id, char *pointer, const std::size_t size)
{
try
{
StreamData &data = stream_data_list[slot_id];
BOOST_ASSERT(data.write_mode);
boost::mutex::scoped_lock lock(*data.readWriteMutex);
BOOST_ASSERT_MSG(data.write_mode, "Writing after first read is not allowed");
if (1073741824 < data.buffer.size())
{
data.temp_file->write(&data.buffer[0], data.buffer.size());
// data.temp_file->write(pointer, size);
data.buffer.clear();
CheckIfTemporaryDeviceFull();
}
data.buffer.insert(data.buffer.end(), pointer, pointer + size);
}
catch (boost::filesystem::filesystem_error &e) { Abort(e); }
}
void TemporaryStorage::ReadFromSlot(const std::size_t slot_id, char *pointer, const std::size_t size)
{
try
{
StreamData &data = stream_data_list[slot_id];
boost::mutex::scoped_lock lock(*data.readWriteMutex);
if (data.write_mode)
{
data.write_mode = false;
data.temp_file->write(&data.buffer[0], data.buffer.size());
data.buffer.clear();
data.temp_file->seekg(data.temp_file->beg);
BOOST_ASSERT(data.temp_file->beg == data.temp_file->tellg());
}
BOOST_ASSERT(!data.write_mode);
data.temp_file->read(pointer, size);
}
catch (boost::filesystem::filesystem_error &error) { Abort(error); }
}
uint64_t TemporaryStorage::GetFreeBytesOnTemporaryDevice()
{
uint64_t value = -1;
try
{
boost::filesystem::path path = boost::filesystem::temp_directory_path();
boost::filesystem::space_info space_info = boost::filesystem::space(path);
value = space_info.free;
}
catch (boost::filesystem::filesystem_error &error) { Abort(error); }
return value;
}
void TemporaryStorage::CheckIfTemporaryDeviceFull()
{
boost::filesystem::path path = boost::filesystem::temp_directory_path();
boost::filesystem::space_info space_info = boost::filesystem::space(path);
if ((1024 * 1024) > space_info.free)
{
throw OSRMException("temporary device is full");
}
}
boost::filesystem::fstream::pos_type TemporaryStorage::Tell(const int slot_id)
{
boost::filesystem::fstream::pos_type position;
try
{
StreamData &data = stream_data_list[slot_id];
boost::mutex::scoped_lock lock(*data.readWriteMutex);
position = data.temp_file->tellp();
}
catch (boost::filesystem::filesystem_error &e) { Abort(e); }
return position;
}
void TemporaryStorage::Abort(const boost::filesystem::filesystem_error &error)
{
RemoveAll();
throw OSRMException(error.what());
}

96
Contractor/TemporaryStorage.h
View File

@ -1,96 +0,0 @@
/*
Copyright (c) 2013, Project OSRM, Dennis Luxen, others
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this list
of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef TEMPORARYSTORAGE_H_
#define TEMPORARYSTORAGE_H_
#include "../Util/BoostFileSystemFix.h"
#include "../Util/OSRMException.h"
#include "../Util/SimpleLogger.h"
#include "../typedefs.h"
#include <boost/assert.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/thread/mutex.hpp>
#include <cstdint>
#include <vector>
#include <fstream>
#include <memory>
struct StreamData
{
bool write_mode;
boost::filesystem::path temp_path;
std::shared_ptr<boost::filesystem::fstream> temp_file;
std::shared_ptr<boost::mutex> readWriteMutex;
std::vector<char> buffer;
StreamData();
};
// This class implements a singleton file storage for temporary data.
// temporary slots can be accessed by other objects through an int
// On deallocation every slot gets deallocated
//
// Access is sequential, which means, that there is no random access
// -> Data is written in first phase and reread in second.
static boost::filesystem::path temp_directory;
static std::string TemporaryFilePattern("OSRM-%%%%-%%%%-%%%%");
class TemporaryStorage
{
public:
static TemporaryStorage &GetInstance();
virtual ~TemporaryStorage();
std::size_t AllocateSlot();
void DeallocateSlot(const std::size_t slot_id);
void WriteToSlot(const std::size_t slot_id, char *pointer, const std::size_t size);
void ReadFromSlot(const std::size_t slot_id, char *pointer, const std::size_t size);
// returns the number of free bytes
uint64_t GetFreeBytesOnTemporaryDevice();
boost::filesystem::fstream::pos_type Tell(const int slot_id);
void RemoveAll();
private:
TemporaryStorage();
TemporaryStorage(TemporaryStorage const &) {};
TemporaryStorage &operator=(TemporaryStorage const &) { return *this; }
void Abort(const boost::filesystem::filesystem_error &e);
void CheckIfTemporaryDeviceFull();
// vector of file streams that is used to store temporary data
boost::mutex mutex;
std::vector<StreamData> stream_data_list;
};
#endif /* TEMPORARYSTORAGE_H_ */

11
DataStructures/DeallocatingVector.h
View File

@ -296,6 +296,17 @@ class DeallocatingVector
const std::size_t _index = current_size % ELEMENTS_PER_BLOCK; const std::size_t _index = current_size % ELEMENTS_PER_BLOCK;
return (bucket_list[_bucket][_index]); return (bucket_list[_bucket][_index]);
} }
template<class InputIterator>
const inline void append(InputIterator first, const InputIterator last)
{
InputIterator position = first;
while (position != last)
{
push_back(*position);
++position;
}
}
}; };
#endif /* DEALLOCATINGVECTOR_H_ */ #endif /* DEALLOCATINGVECTOR_H_ */