Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Make fstream non-static and StaticRTree thread-specific instead

Browse Source
This commit is contained in:
Patrick Niklaus 2014-06-27 16:25:55 +02:00
parent e776a51c73
commit 3c4feecda0
4 changed files with 69 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
DataStructures/StaticRTree.h
View File

@ -64,7 +64,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
const static uint32_t RTREE_BRANCHING_FACTOR = 64; const static uint32_t RTREE_BRANCHING_FACTOR = 64;
const static uint32_t RTREE_LEAF_NODE_SIZE = 1024; const static uint32_t RTREE_LEAF_NODE_SIZE = 1024;
static boost::thread_specific_ptr<boost::filesystem::ifstream> thread_local_rtree_stream;
// Implements a static, i.e. packed, R-tree // Implements a static, i.e. packed, R-tree
template <class EdgeDataT, template <class EdgeDataT,
@ -304,6 +303,7 @@ class StaticRTree
uint64_t m_element_count; uint64_t m_element_count;
const std::string m_leaf_node_filename; const std::string m_leaf_node_filename;
std::shared_ptr<CoordinateListT> m_coordinate_list; std::shared_ptr<CoordinateListT> m_coordinate_list;
boost::filesystem::ifstream leaves_stream;
public: public:
StaticRTree() = delete; StaticRTree() = delete;
@ -507,9 +507,8 @@ class StaticRTree
throw OSRMException("mem index file is empty"); throw OSRMException("mem index file is empty");
} }
boost::filesystem::ifstream leaf_node_file(leaf_file, std::ios::binary); leaves_stream.open(leaf_file, std::ios::binary);
leaf_node_file.read((char *)&m_element_count, sizeof(uint64_t)); leaves_stream.read((char *)&m_element_count, sizeof(uint64_t));
leaf_node_file.close();
// SimpleLogger().Write() << tree_size << " nodes in search tree"; // SimpleLogger().Write() << tree_size << " nodes in search tree";
// SimpleLogger().Write() << m_element_count << " elements in leafs"; // SimpleLogger().Write() << m_element_count << " elements in leafs";
@ -532,14 +531,8 @@ class StaticRTree
throw OSRMException("mem index file is empty"); throw OSRMException("mem index file is empty");
} }
boost::filesystem::ifstream leaf_node_file(leaf_file, std::ios::binary); leaves_stream.open(leaf_file, std::ios::binary);
leaf_node_file.read((char *)&m_element_count, sizeof(uint64_t)); leaves_stream.read((char *)&m_element_count, sizeof(uint64_t));
leaf_node_file.close();
if (thread_local_rtree_stream.get())
{
thread_local_rtree_stream->close();
}
// SimpleLogger().Write() << tree_size << " nodes in search tree"; // SimpleLogger().Write() << tree_size << " nodes in search tree";
// SimpleLogger().Write() << m_element_count << " elements in leafs"; // SimpleLogger().Write() << m_element_count << " elements in leafs";
@ -988,22 +981,21 @@ class StaticRTree
inline void LoadLeafFromDisk(const uint32_t leaf_id, LeafNode &result_node) inline void LoadLeafFromDisk(const uint32_t leaf_id, LeafNode &result_node)
{ {
if (!thread_local_rtree_stream.get() || !thread_local_rtree_stream->is_open()) if (!leaves_stream.is_open())
{ {
thread_local_rtree_stream.reset(new boost::filesystem::ifstream( leaves_stream.open(m_leaf_node_filename, std::ios::in | std::ios::binary);
m_leaf_node_filename, std::ios::in | std::ios::binary));
} }
if (!thread_local_rtree_stream->good()) if (!leaves_stream.good())
{ {
thread_local_rtree_stream->clear(std::ios::goodbit); leaves_stream.clear(std::ios::goodbit);
SimpleLogger().Write(logDEBUG) << "Resetting stale filestream"; SimpleLogger().Write(logDEBUG) << "Resetting stale filestream";
} }
const uint64_t seek_pos = sizeof(uint64_t) + leaf_id * sizeof(LeafNode); const uint64_t seek_pos = sizeof(uint64_t) + leaf_id * sizeof(LeafNode);
thread_local_rtree_stream->seekg(seek_pos); leaves_stream.seekg(seek_pos);
BOOST_ASSERT_MSG(thread_local_rtree_stream->good(), BOOST_ASSERT_MSG(leaves_stream.good(),
"Seeking to position in leaf file failed."); "Seeking to position in leaf file failed.");
thread_local_rtree_stream->read((char *)&result_node, sizeof(LeafNode)); leaves_stream.read((char *)&result_node, sizeof(LeafNode));
BOOST_ASSERT_MSG(thread_local_rtree_stream->good(), "Reading from leaf file failed."); BOOST_ASSERT_MSG(leaves_stream.good(), "Reading from leaf file failed.");
} }
inline bool EdgesAreEquivalent(const FixedPointCoordinate &a, inline bool EdgesAreEquivalent(const FixedPointCoordinate &a,

6
Server/DataStructures/BaseDataFacade.h
View File

@ -95,16 +95,16 @@ template <class EdgeDataT> class BaseDataFacade
virtual bool LocateClosestEndPointForCoordinate(const FixedPointCoordinate &input_coordinate, virtual bool LocateClosestEndPointForCoordinate(const FixedPointCoordinate &input_coordinate,
FixedPointCoordinate &result, FixedPointCoordinate &result,
const unsigned zoom_level = 18) const = 0; const unsigned zoom_level = 18) = 0;
virtual bool FindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate, virtual bool FindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate,
PhantomNode &resulting_phantom_node, PhantomNode &resulting_phantom_node,
const unsigned zoom_level) const = 0; const unsigned zoom_level) = 0;
virtual bool IncrementalFindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate, virtual bool IncrementalFindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate,
std::vector<PhantomNode> &resulting_phantom_node_vector, std::vector<PhantomNode> &resulting_phantom_node_vector,
const unsigned zoom_level, const unsigned zoom_level,
const unsigned number_of_results) const = 0; const unsigned number_of_results) = 0;
virtual unsigned GetCheckSum() const = 0; virtual unsigned GetCheckSum() const = 0;

38
Server/DataStructures/InternalDataFacade.h
View File

@ -71,8 +71,10 @@ template <class EdgeDataT> class InternalDataFacade : public BaseDataFacade<Edge
ShM<unsigned, false>::vector m_geometry_indices; ShM<unsigned, false>::vector m_geometry_indices;
ShM<unsigned, false>::vector m_geometry_list; ShM<unsigned, false>::vector m_geometry_list;
std::shared_ptr<StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, false>::vector, false>> boost::thread_specific_ptr<StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, false>::vector, false>>
m_static_rtree; m_static_rtree;
boost::filesystem::path ram_index_path;
boost::filesystem::path file_index_path;
RangeTable<16, false> m_name_table; RangeTable<16, false> m_name_table;
void LoadTimestamp(const boost::filesystem::path &timestamp_path) void LoadTimestamp(const boost::filesystem::path &timestamp_path)
@ -192,13 +194,13 @@ template <class EdgeDataT> class InternalDataFacade : public BaseDataFacade<Edge
geometry_stream.close(); geometry_stream.close();
} }
void LoadRTree(const boost::filesystem::path &ram_index_path, void LoadRTree()
const boost::filesystem::path &file_index_path)
{ {
BOOST_ASSERT_MSG(!m_coordinate_list->empty(), "coordinates must be loaded before r-tree"); BOOST_ASSERT_MSG(!m_coordinate_list->empty(), "coordinates must be loaded before r-tree");
m_static_rtree = std::make_shared<StaticRTree<RTreeLeaf>>( m_static_rtree.reset(
ram_index_path, file_index_path, m_coordinate_list); new StaticRTree<RTreeLeaf>(ram_index_path, file_index_path, m_coordinate_list)
);
} }
void LoadStreetNames(const boost::filesystem::path &names_file) void LoadStreetNames(const boost::filesystem::path &names_file)
@ -266,10 +268,10 @@ template <class EdgeDataT> class InternalDataFacade : public BaseDataFacade<Edge
const boost::filesystem::path &timestamp_path = paths_iterator->second; const boost::filesystem::path &timestamp_path = paths_iterator->second;
paths_iterator = server_paths.find("ramindex"); paths_iterator = server_paths.find("ramindex");
BOOST_ASSERT(server_paths.end() != paths_iterator); BOOST_ASSERT(server_paths.end() != paths_iterator);
const boost::filesystem::path &ram_index_path = paths_iterator->second; ram_index_path = paths_iterator->second;
paths_iterator = server_paths.find("fileindex"); paths_iterator = server_paths.find("fileindex");
BOOST_ASSERT(server_paths.end() != paths_iterator); BOOST_ASSERT(server_paths.end() != paths_iterator);
const boost::filesystem::path &file_index_path = paths_iterator->second; file_index_path = paths_iterator->second;
paths_iterator = server_paths.find("nodesdata"); paths_iterator = server_paths.find("nodesdata");
BOOST_ASSERT(server_paths.end() != paths_iterator); BOOST_ASSERT(server_paths.end() != paths_iterator);
const boost::filesystem::path &nodes_data_path = paths_iterator->second; const boost::filesystem::path &nodes_data_path = paths_iterator->second;
@ -297,7 +299,6 @@ template <class EdgeDataT> class InternalDataFacade : public BaseDataFacade<Edge
SimpleLogger().Write() << "loading r-tree"; SimpleLogger().Write() << "loading r-tree";
AssertPathExists(ram_index_path); AssertPathExists(ram_index_path);
AssertPathExists(file_index_path); AssertPathExists(file_index_path);
LoadRTree(ram_index_path, file_index_path);
SimpleLogger().Write() << "loading timestamp"; SimpleLogger().Write() << "loading timestamp";
LoadTimestamp(timestamp_path); LoadTimestamp(timestamp_path);
SimpleLogger().Write() << "loading street names"; SimpleLogger().Write() << "loading street names";
@ -358,16 +359,26 @@ template <class EdgeDataT> class InternalDataFacade : public BaseDataFacade<Edge
bool LocateClosestEndPointForCoordinate(const FixedPointCoordinate &input_coordinate, bool LocateClosestEndPointForCoordinate(const FixedPointCoordinate &input_coordinate,
FixedPointCoordinate &result, FixedPointCoordinate &result,
const unsigned zoom_level = 18) const const unsigned zoom_level = 18)
{ {
if (!m_static_rtree.get())
{
LoadRTree();
}
return m_static_rtree->LocateClosestEndPointForCoordinate( return m_static_rtree->LocateClosestEndPointForCoordinate(
input_coordinate, result, zoom_level); input_coordinate, result, zoom_level);
} }
bool FindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate, bool FindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate,
PhantomNode &resulting_phantom_node, PhantomNode &resulting_phantom_node,
const unsigned zoom_level) const const unsigned zoom_level)
{ {
if (!m_static_rtree.get())
{
LoadRTree();
}
return m_static_rtree->FindPhantomNodeForCoordinate( return m_static_rtree->FindPhantomNodeForCoordinate(
input_coordinate, resulting_phantom_node, zoom_level); input_coordinate, resulting_phantom_node, zoom_level);
} }
@ -376,8 +387,13 @@ template <class EdgeDataT> class InternalDataFacade : public BaseDataFacade<Edge
IncrementalFindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate, IncrementalFindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate,
std::vector<PhantomNode> &resulting_phantom_node_vector, std::vector<PhantomNode> &resulting_phantom_node_vector,
const unsigned zoom_level, const unsigned zoom_level,
const unsigned number_of_results) const const unsigned number_of_results)
{ {
if (!m_static_rtree.get())
{
LoadRTree();
}
return m_static_rtree->IncrementalFindPhantomNodeForCoordinate( return m_static_rtree->IncrementalFindPhantomNodeForCoordinate(
input_coordinate, resulting_phantom_node_vector, zoom_level, number_of_results); input_coordinate, resulting_phantom_node_vector, zoom_level, number_of_results);
} }

36
Server/DataStructures/SharedDataFacade.h
View File

@ -83,8 +83,9 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
ShM<unsigned, true>::vector m_geometry_indices; ShM<unsigned, true>::vector m_geometry_indices;
ShM<unsigned, true>::vector m_geometry_list; ShM<unsigned, true>::vector m_geometry_list;
std::shared_ptr<StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>> boost::thread_specific_ptr<StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>>
m_static_rtree; m_static_rtree;
boost::filesystem::path file_index_path;
std::shared_ptr<RangeTable<16, true>> m_name_table; std::shared_ptr<RangeTable<16, true>> m_name_table;
@ -105,18 +106,19 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
m_timestamp.begin()); m_timestamp.begin());
} }
void LoadRTree(const boost::filesystem::path &file_index_path) void LoadRTree()
{ {
BOOST_ASSERT_MSG(!m_coordinate_list->empty(), "coordinates must be loaded before r-tree"); BOOST_ASSERT_MSG(!m_coordinate_list->empty(), "coordinates must be loaded before r-tree");
RTreeNode *tree_ptr = RTreeNode *tree_ptr =
data_layout->GetBlockPtr<RTreeNode>(shared_memory, SharedDataLayout::R_SEARCH_TREE); data_layout->GetBlockPtr<RTreeNode>(shared_memory, SharedDataLayout::R_SEARCH_TREE);
m_static_rtree = m_static_rtree.reset(
std::make_shared<StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>>( new StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>(
tree_ptr, tree_ptr,
data_layout->num_entries[SharedDataLayout::R_SEARCH_TREE], data_layout->num_entries[SharedDataLayout::R_SEARCH_TREE],
file_index_path, file_index_path,
m_coordinate_list); m_coordinate_list)
);
} }
void LoadGraph() void LoadGraph()
@ -248,7 +250,7 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
const char *file_index_ptr = const char *file_index_ptr =
data_layout->GetBlockPtr<char>(shared_memory, SharedDataLayout::FILE_INDEX_PATH); data_layout->GetBlockPtr<char>(shared_memory, SharedDataLayout::FILE_INDEX_PATH);
boost::filesystem::path file_index_path(file_index_ptr); file_index_path = boost::filesystem::path(file_index_ptr);
if (!boost::filesystem::exists(file_index_path)) if (!boost::filesystem::exists(file_index_path))
{ {
SimpleLogger().Write(logDEBUG) << "Leaf file name " << file_index_path.string(); SimpleLogger().Write(logDEBUG) << "Leaf file name " << file_index_path.string();
@ -260,7 +262,6 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
LoadChecksum(); LoadChecksum();
LoadNodeAndEdgeInformation(); LoadNodeAndEdgeInformation();
LoadGeometries(); LoadGeometries();
LoadRTree(file_index_path);
LoadTimestamp(); LoadTimestamp();
LoadViaNodeList(); LoadViaNodeList();
LoadNames(); LoadNames();
@ -349,16 +350,26 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
bool LocateClosestEndPointForCoordinate(const FixedPointCoordinate &input_coordinate, bool LocateClosestEndPointForCoordinate(const FixedPointCoordinate &input_coordinate,
FixedPointCoordinate &result, FixedPointCoordinate &result,
const unsigned zoom_level = 18) const const unsigned zoom_level = 18)
{ {
if (!m_static_rtree.get())
{
LoadRTree();
}
return m_static_rtree->LocateClosestEndPointForCoordinate( return m_static_rtree->LocateClosestEndPointForCoordinate(
input_coordinate, result, zoom_level); input_coordinate, result, zoom_level);
} }
bool FindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate, bool FindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate,
PhantomNode &resulting_phantom_node, PhantomNode &resulting_phantom_node,
const unsigned zoom_level) const const unsigned zoom_level)
{ {
if (!m_static_rtree.get())
{
LoadRTree();
}
return m_static_rtree->FindPhantomNodeForCoordinate( return m_static_rtree->FindPhantomNodeForCoordinate(
input_coordinate, resulting_phantom_node, zoom_level); input_coordinate, resulting_phantom_node, zoom_level);
} }
@ -367,8 +378,13 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
IncrementalFindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate, IncrementalFindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate,
std::vector<PhantomNode> &resulting_phantom_node_vector, std::vector<PhantomNode> &resulting_phantom_node_vector,
const unsigned zoom_level, const unsigned zoom_level,
const unsigned number_of_results) const const unsigned number_of_results)
{ {
if (!m_static_rtree.get())
{
LoadRTree();
}
return m_static_rtree->IncrementalFindPhantomNodeForCoordinate( return m_static_rtree->IncrementalFindPhantomNodeForCoordinate(
input_coordinate, resulting_phantom_node_vector, zoom_level, number_of_results); input_coordinate, resulting_phantom_node_vector, zoom_level, number_of_results);
} }