Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Initial support for SharedDataFacade

Browse Source 
SharedDataLayout was refactored to include canary values at the
boundaries of each memory block. This makes it easy to detect overruns
and block-size mismatches between osrm-datastore and the
SharedDataFacade.
This commit is contained in:
Patrick Niklaus 2014-06-05 17:18:28 +02:00
parent 7a7d0c09d9
commit 807f1d7c1c
7 changed files with 320 additions and 338 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
DataStructures/RangeTable.h
View File

@ -1,6 +1,9 @@
#ifndef __RANGE_TABLE_H__ #ifndef __RANGE_TABLE_H__
#define __RANGE_TABLE_H__ #define __RANGE_TABLE_H__
#include <boost/range/irange.hpp>
#include <fstream>
#include <vector> #include <vector>
#if defined(__GNUC__) && defined(__SSE2__) #if defined(__GNUC__) && defined(__SSE2__)
@ -8,6 +11,9 @@
#include <xmmintrin.h> #include <xmmintrin.h>
#endif #endif
#include "SharedMemoryFactory.h"
#include "SharedMemoryVectorWrapper.h"
/* /*
* These pre-declarations are needed because parsing C++ is hard * These pre-declarations are needed because parsing C++ is hard
* and otherwise the compiler gets confused. * and otherwise the compiler gets confused.
@ -21,9 +27,6 @@ std::ostream& operator<<(std::ostream &out, const RangeTable<BLOCK_SIZE, USE_SHA
template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY> template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
std::istream& operator>>(std::istream &in, RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table); std::istream& operator>>(std::istream &in, RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table);
#include "SharedMemoryFactory.h"
#include "SharedMemoryVectorWrapper.h"
/** /**
* Stores adjacent ranges in a compressed format. * Stores adjacent ranges in a compressed format.
* *
@ -49,6 +52,7 @@ public:
typedef typename ShM<BlockT, USE_SHARED_MEMORY>::vector BlockContainerT; typedef typename ShM<BlockT, USE_SHARED_MEMORY>::vector BlockContainerT;
typedef typename ShM<unsigned, USE_SHARED_MEMORY>::vector OffsetContainerT; typedef typename ShM<unsigned, USE_SHARED_MEMORY>::vector OffsetContainerT;
typedef decltype(boost::irange(0u,0u)) RangeT;
friend std::ostream& operator<< <>(std::ostream &out, const RangeTable &table); friend std::ostream& operator<< <>(std::ostream &out, const RangeTable &table);
friend std::istream& operator>> <>(std::istream &in, RangeTable &table); friend std::istream& operator>> <>(std::istream &in, RangeTable &table);
@ -56,7 +60,8 @@ public:
RangeTable() {} RangeTable() {}
// for loading from shared memory // for loading from shared memory
explicit RangeTable(OffsetContainerT& external_offsets, BlockContainerT& external_blocks) explicit RangeTable(OffsetContainerT& external_offsets, BlockContainerT& external_blocks, unsigned sum_lengths)
: sum_lengths(sum_lengths)
{ {
block_offsets.swap(external_offsets); block_offsets.swap(external_offsets);
diff_blocks.swap(external_blocks); diff_blocks.swap(external_blocks);
@ -134,7 +139,7 @@ public:
sum_lengths = lengths_prefix_sum; sum_lengths = lengths_prefix_sum;
} }
inline void GetRange(const unsigned id, unsigned& begin_idx, unsigned& end_idx) const inline RangeT GetRange(const unsigned id) const
{ {
BOOST_ASSERT(id < block_offsets.size() + diff_blocks.size() * BLOCK_SIZE); BOOST_ASSERT(id < block_offsets.size() + diff_blocks.size() * BLOCK_SIZE);
// internal_idx 0 is implicitly stored in block_offsets[block_idx] // internal_idx 0 is implicitly stored in block_offsets[block_idx]
@ -143,6 +148,8 @@ public:
BOOST_ASSERT(block_idx < diff_blocks.size()); BOOST_ASSERT(block_idx < diff_blocks.size());
unsigned begin_idx = 0;
unsigned end_idx = 0;
begin_idx = block_offsets[block_idx]; begin_idx = block_offsets[block_idx];
const BlockT& block = diff_blocks[block_idx]; const BlockT& block = diff_blocks[block_idx];
if (internal_idx > 0) if (internal_idx > 0)
@ -163,6 +170,9 @@ public:
} }
BOOST_ASSERT(begin_idx < sum_lengths && end_idx <= sum_lengths); BOOST_ASSERT(begin_idx < sum_lengths && end_idx <= sum_lengths);
BOOST_ASSERT(begin_idx <= end_idx);
return boost::irange(begin_idx, end_idx);
} }
private: private:

2
DataStructures/StaticRTree.h
View File

@ -766,7 +766,9 @@ class StaticRTree
} }
const uint64_t seek_pos = sizeof(uint64_t) + leaf_id * sizeof(LeafNode); const uint64_t seek_pos = sizeof(uint64_t) + leaf_id * sizeof(LeafNode);
thread_local_rtree_stream->seekg(seek_pos); thread_local_rtree_stream->seekg(seek_pos);
BOOST_ASSERT_MSG(thread_local_rtree_stream->good(), "Seeking to position in leaf file failed.");
thread_local_rtree_stream->read((char *)&result_node, sizeof(LeafNode)); thread_local_rtree_stream->read((char *)&result_node, sizeof(LeafNode));
BOOST_ASSERT_MSG(thread_local_rtree_stream->good(), "Reading from leaf file failed.");
} }
inline bool EdgesAreEquivalent(const FixedPointCoordinate &a, inline bool EdgesAreEquivalent(const FixedPointCoordinate &a,

2
Extractor/ExtractionContainers.cpp
View File

@ -30,6 +30,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "../Util/OSRMException.h" #include "../Util/OSRMException.h"
#include "../Util/SimpleLogger.h" #include "../Util/SimpleLogger.h"
#include "../Util/TimingUtil.h" #include "../Util/TimingUtil.h"
#include "../DataStructures/RangeTable.h"
#include <boost/assert.hpp> #include <boost/assert.hpp>
#include <boost/filesystem.hpp> #include <boost/filesystem.hpp>
@ -64,6 +65,7 @@ void ExtractionContainers::PrepareData(const std::string &output_file_name,
{ {
unsigned number_of_used_nodes = 0; unsigned number_of_used_nodes = 0;
unsigned number_of_used_edges = 0; unsigned number_of_used_edges = 0;
std::cout << "[extractor] Sorting used nodes ... " << std::flush; std::cout << "[extractor] Sorting used nodes ... " << std::flush;
TIMER_START(sorting_used_nodes); TIMER_START(sorting_used_nodes);
stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), stxxl_memory); stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), stxxl_memory);

18
Server/DataStructures/InternalDataFacade.h
View File

@ -381,18 +381,16 @@ template <class EdgeDataT> class InternalDataFacade : public BaseDataFacade<Edge
result = ""; result = "";
return; return;
} }
unsigned begin_index; auto range = m_name_table.GetRange(name_id);
unsigned end_index;
m_name_table.GetRange(name_id, begin_index, end_index);
BOOST_ASSERT_MSG(begin_index < m_names_char_list.size(), "begin index of name too high");
BOOST_ASSERT_MSG(end_index < m_names_char_list.size(), "end index of name too high");
BOOST_ASSERT_MSG(begin_index <= end_index, "string ends before begin");
result.clear(); result.clear();
result.resize(end_index - begin_index); if (range.begin() != range.end())
std::copy(m_names_char_list.begin() + begin_index, {
m_names_char_list.begin() + end_index, result.resize(range.back() - range.front());
result.begin()); std::copy(m_names_char_list.begin() + range.front(),
m_names_char_list.begin() + range.back(),
result.begin());
}
} }
virtual unsigned GetGeometryIndexForEdgeID(const unsigned id) const virtual unsigned GetGeometryIndexForEdgeID(const unsigned id) const

120
Server/DataStructures/SharedDataFacade.h
View File

@ -33,6 +33,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "BaseDataFacade.h" #include "BaseDataFacade.h"
#include "SharedDataType.h" #include "SharedDataType.h"
#include "../../DataStructures/RangeTable.h"
#include "../../DataStructures/StaticGraph.h" #include "../../DataStructures/StaticGraph.h"
#include "../../DataStructures/StaticRTree.h" #include "../../DataStructures/StaticRTree.h"
#include "../../Util/BoostFileSystemFix.h" #include "../../Util/BoostFileSystemFix.h"
@ -51,6 +52,7 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
typedef StaticGraph<EdgeData, true> QueryGraph; typedef StaticGraph<EdgeData, true> QueryGraph;
typedef typename StaticGraph<EdgeData, true>::NodeArrayEntry GraphNode; typedef typename StaticGraph<EdgeData, true>::NodeArrayEntry GraphNode;
typedef typename StaticGraph<EdgeData, true>::EdgeArrayEntry GraphEdge; typedef typename StaticGraph<EdgeData, true>::EdgeArrayEntry GraphEdge;
typedef typename RangeTable<16, true>::BlockT NameIndexBlock;
typedef typename QueryGraph::InputEdge InputEdge; typedef typename QueryGraph::InputEdge InputEdge;
typedef typename super::RTreeLeaf RTreeLeaf; typedef typename super::RTreeLeaf RTreeLeaf;
typedef typename StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>::TreeNode typedef typename StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>::TreeNode
@ -84,43 +86,51 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
std::shared_ptr<StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>> std::shared_ptr<StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>>
m_static_rtree; m_static_rtree;
std::shared_ptr<RangeTable<16, true>> m_name_table;
void LoadChecksum() void LoadChecksum()
{ {
m_check_sum = data_layout->checksum; m_check_sum = *data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::HSGR_CHECKSUM);
SimpleLogger().Write() << "set checksum: " << m_check_sum; SimpleLogger().Write() << "set checksum: " << m_check_sum;
} }
void LoadTimestamp() void LoadTimestamp()
{ {
char *timestamp_ptr = shared_memory + data_layout->GetTimeStampOffset(); char *timestamp_ptr = data_layout->GetBlockPtr<char>(shared_memory, SharedDataLayout::TIMESTAMP);
m_timestamp.resize(data_layout->timestamp_length); m_timestamp.resize(data_layout->GetBlockSize(SharedDataLayout::TIMESTAMP));
std::copy( std::copy(
timestamp_ptr, timestamp_ptr + data_layout->timestamp_length, m_timestamp.begin()); timestamp_ptr,
timestamp_ptr + data_layout->GetBlockSize(SharedDataLayout::TIMESTAMP),
m_timestamp.begin());
} }
void LoadRTree(const boost::filesystem::path &file_index_path) void LoadRTree(const boost::filesystem::path &file_index_path)
{ {
BOOST_ASSERT_MSG(!m_coordinate_list->empty(), "coordinates must be loaded before r-tree"); BOOST_ASSERT_MSG(!m_coordinate_list->empty(), "coordinates must be loaded before r-tree");
RTreeNode *tree_ptr = (RTreeNode *)(shared_memory + data_layout->GetRSearchTreeOffset()); RTreeNode *tree_ptr = data_layout->GetBlockPtr<RTreeNode>(shared_memory, SharedDataLayout::R_SEARCH_TREE);
m_static_rtree = m_static_rtree =
std::make_shared<StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>>( std::make_shared<StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>>(
tree_ptr, data_layout->r_search_tree_size, file_index_path, m_coordinate_list); tree_ptr,
data_layout->num_entries[SharedDataLayout::R_SEARCH_TREE],
file_index_path,
m_coordinate_list);
} }
void LoadGraph() void LoadGraph()
{ {
m_number_of_nodes = data_layout->graph_node_list_size; m_number_of_nodes = data_layout->num_entries[SharedDataLayout::GRAPH_NODE_LIST];
GraphNode *graph_nodes_ptr = GraphNode *graph_nodes_ptr =
(GraphNode *)(shared_memory + data_layout->GetGraphNodeListOffset()); data_layout->GetBlockPtr<GraphNode>(shared_memory, SharedDataLayout::GRAPH_NODE_LIST);
GraphEdge *graph_edges_ptr = GraphEdge *graph_edges_ptr =
(GraphEdge *)(shared_memory + data_layout->GetGraphEdgeListOffset()); data_layout->GetBlockPtr<GraphEdge>(shared_memory, SharedDataLayout::GRAPH_EDGE_LIST);
typename ShM<GraphNode, true>::vector node_list(graph_nodes_ptr, typename ShM<GraphNode, true>::vector node_list(graph_nodes_ptr,
data_layout->graph_node_list_size); data_layout->num_entries[SharedDataLayout::GRAPH_NODE_LIST]);
typename ShM<GraphEdge, true>::vector edge_list(graph_edges_ptr, typename ShM<GraphEdge, true>::vector edge_list(graph_edges_ptr,
data_layout->graph_edge_list_size); data_layout->num_entries[SharedDataLayout::GRAPH_EDGE_LIST]);
m_query_graph.reset(new QueryGraph(node_list, edge_list)); m_query_graph.reset(new QueryGraph(node_list, edge_list));
} }
@ -128,63 +138,62 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
{ {
FixedPointCoordinate *coordinate_list_ptr = FixedPointCoordinate *coordinate_list_ptr =
(FixedPointCoordinate *)(shared_memory + data_layout->GetCoordinateListOffset()); data_layout->GetBlockPtr<FixedPointCoordinate>(shared_memory, SharedDataLayout::COORDINATE_LIST);
m_coordinate_list = std::make_shared<ShM<FixedPointCoordinate, true>::vector>( m_coordinate_list = std::make_shared<ShM<FixedPointCoordinate, true>::vector>(
coordinate_list_ptr, data_layout->coordinate_list_size); coordinate_list_ptr, data_layout->num_entries[SharedDataLayout::COORDINATE_LIST]);
TurnInstruction *turn_instruction_list_ptr = TurnInstruction *turn_instruction_list_ptr =
(TurnInstruction *)(shared_memory + data_layout->GetTurnInstructionListOffset()); data_layout->GetBlockPtr<TurnInstruction>(shared_memory, SharedDataLayout::TURN_INSTRUCTION);
typename ShM<TurnInstruction, true>::vector turn_instruction_list( typename ShM<TurnInstruction, true>::vector turn_instruction_list(
turn_instruction_list_ptr, data_layout->turn_instruction_list_size); turn_instruction_list_ptr, data_layout->num_entries[SharedDataLayout::TURN_INSTRUCTION]);
m_turn_instruction_list.swap(turn_instruction_list); m_turn_instruction_list.swap(turn_instruction_list);
unsigned *name_id_list_ptr = unsigned *name_id_list_ptr = data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::NAME_ID_LIST);
(unsigned *)(shared_memory + data_layout->GetNameIDListOffset());
typename ShM<unsigned, true>::vector name_id_list(name_id_list_ptr, typename ShM<unsigned, true>::vector name_id_list(name_id_list_ptr,
data_layout->name_id_list_size); data_layout->num_entries[SharedDataLayout::NAME_ID_LIST]);
m_name_ID_list.swap(name_id_list); m_name_ID_list.swap(name_id_list);
} }
void LoadViaNodeList() void LoadViaNodeList()
{ {
NodeID *via_node_list_ptr = (NodeID *)(shared_memory + data_layout->GetViaNodeListOffset()); NodeID *via_node_list_ptr = data_layout->GetBlockPtr<NodeID>(shared_memory, SharedDataLayout::VIA_NODE_LIST);
typename ShM<NodeID, true>::vector via_node_list(via_node_list_ptr, typename ShM<NodeID, true>::vector via_node_list(via_node_list_ptr,
data_layout->via_node_list_size); data_layout->num_entries[SharedDataLayout::VIA_NODE_LIST]);
m_via_node_list.swap(via_node_list); m_via_node_list.swap(via_node_list);
} }
void LoadNames() void LoadNames()
{ {
unsigned *street_names_index_ptr = unsigned *offsets_ptr = data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::NAME_OFFSETS);
(unsigned *)(shared_memory + data_layout->GetNameIndexOffset()); NameIndexBlock *blocks_ptr = data_layout->GetBlockPtr<NameIndexBlock>(shared_memory, SharedDataLayout::NAME_BLOCKS);
typename ShM<unsigned, true>::vector name_begin_indices(street_names_index_ptr, typename ShM<unsigned, true>::vector name_offsets(offsets_ptr,
data_layout->name_index_list_size); data_layout->num_entries[SharedDataLayout::NAME_OFFSETS]);
m_name_begin_indices.swap(name_begin_indices); typename ShM<NameIndexBlock, true>::vector name_blocks(blocks_ptr,
data_layout->num_entries[SharedDataLayout::NAME_BLOCKS]);
char *names_list_ptr = (char *)(shared_memory + data_layout->GetNameListOffset()); char *names_list_ptr = data_layout->GetBlockPtr<char>(shared_memory, SharedDataLayout::NAME_CHAR_LIST);
typename ShM<char, true>::vector names_char_list(names_list_ptr, typename ShM<char, true>::vector names_char_list(names_list_ptr,
data_layout->name_char_list_size); data_layout->num_entries[SharedDataLayout::NAME_CHAR_LIST]);
m_name_table = std::make_shared<RangeTable<16, true>>(name_offsets, name_blocks, names_char_list.size());
m_names_char_list.swap(names_char_list); m_names_char_list.swap(names_char_list);
} }
void LoadGeometries() void LoadGeometries()
{ {
unsigned *geometries_compressed_ptr = unsigned *geometries_compressed_ptr = data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::GEOMETRIES_INDICATORS);
(unsigned *)(shared_memory + data_layout->GetGeometriesIndicatorOffset());
typename ShM<bool, true>::vector egde_is_compressed(geometries_compressed_ptr, typename ShM<bool, true>::vector egde_is_compressed(geometries_compressed_ptr,
data_layout->geometries_indicators); data_layout->num_entries[SharedDataLayout::GEOMETRIES_INDICATORS]);
m_egde_is_compressed.swap(egde_is_compressed); m_egde_is_compressed.swap(egde_is_compressed);
unsigned *geometries_index_ptr = unsigned *geometries_index_ptr = data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::GEOMETRIES_INDEX);
(unsigned *)(shared_memory + data_layout->GetGeometriesIndexListOffset());
typename ShM<unsigned, true>::vector geometry_begin_indices( typename ShM<unsigned, true>::vector geometry_begin_indices(
geometries_index_ptr, data_layout->geometries_index_list_size); geometries_index_ptr, data_layout->num_entries[SharedDataLayout::GEOMETRIES_INDEX]);
m_geometry_indices.swap(geometry_begin_indices); m_geometry_indices.swap(geometry_begin_indices);
unsigned *geometries_list_ptr = unsigned *geometries_list_ptr = data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::GEOMETRIES_LIST);
(unsigned *)(shared_memory + data_layout->GetGeometryListOffset());
typename ShM<unsigned, true>::vector geometry_list(geometries_list_ptr, typename ShM<unsigned, true>::vector geometry_list(geometries_list_ptr,
data_layout->geometries_list_size); data_layout->num_entries[SharedDataLayout::GEOMETRIES_LIST]);
m_geometry_list.swap(geometry_list); m_geometry_list.swap(geometry_list);
} }
@ -219,21 +228,28 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
m_layout_memory.reset(SharedMemoryFactory::Get(CURRENT_LAYOUT)); m_layout_memory.reset(SharedMemoryFactory::Get(CURRENT_LAYOUT));
data_layout = (SharedDataLayout *)(m_layout_memory->Ptr()); data_layout = (SharedDataLayout *)(m_layout_memory->Ptr());
boost::filesystem::path ram_index_path(data_layout->ram_index_file_name);
if (!boost::filesystem::exists(ram_index_path))
{
throw OSRMException("no leaf index file given. "
"Is any data loaded into shared memory?");
}
m_large_memory.reset(SharedMemoryFactory::Get(CURRENT_DATA)); m_large_memory.reset(SharedMemoryFactory::Get(CURRENT_DATA));
shared_memory = (char *)(m_large_memory->Ptr()); shared_memory = (char *)(m_large_memory->Ptr());
std::ofstream out("debug.bin");
out.write(shared_memory, data_layout->GetSizeOfLayout());
out.close();
const char* file_index_ptr = data_layout->GetBlockPtr<char>(shared_memory, SharedDataLayout::FILE_INDEX_PATH);
boost::filesystem::path file_index_path(file_index_ptr);
if (!boost::filesystem::exists(file_index_path))
{
SimpleLogger().Write(logDEBUG) << "Leaf file name " << file_index_path.string();
throw OSRMException("Could not load leaf index file."
"Is any data loaded into shared memory?");
}
LoadGraph(); LoadGraph();
LoadChecksum(); LoadChecksum();
LoadNodeAndEdgeInformation(); LoadNodeAndEdgeInformation();
LoadGeometries(); LoadGeometries();
LoadRTree(ram_index_path); LoadRTree(file_index_path);
LoadTimestamp(); LoadTimestamp();
LoadViaNodeList(); LoadViaNodeList();
LoadNames(); LoadNames();
@ -339,18 +355,16 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
result = ""; result = "";
return; return;
} }
BOOST_ASSERT_MSG(name_id < m_name_begin_indices.size(), "name id too high"); auto range = m_name_table->GetRange(name_id);
const unsigned begin_index = m_name_begin_indices[name_id];
const unsigned end_index = m_name_begin_indices[name_id + 1];
BOOST_ASSERT_MSG(begin_index <= m_names_char_list.size(), "begin index of name too high");
BOOST_ASSERT_MSG(end_index <= m_names_char_list.size(), "end index of name too high");
BOOST_ASSERT_MSG(begin_index <= end_index, "string ends before begin");
result.clear(); result.clear();
result.resize(end_index - begin_index); if (range.begin() != range.end())
std::copy(m_names_char_list.begin() + begin_index, {
m_names_char_list.begin() + end_index, result.resize(range.back() - range.front());
result.begin()); std::copy(m_names_char_list.begin() + range.front(),
m_names_char_list.begin() + range.back(),
result.begin());
}
} }
std::string GetTimestamp() const { return m_timestamp; } std::string GetTimestamp() const { return m_timestamp; }

317
Server/DataStructures/SharedDataType.h
View File

@ -28,230 +28,145 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef SHARED_DATA_TYPE_H_ #ifndef SHARED_DATA_TYPE_H_
#define SHARED_DATA_TYPE_H_ #define SHARED_DATA_TYPE_H_
#include "BaseDataFacade.h" #include "../../Util/SimpleLogger.h"
#include "../../DataStructures/QueryEdge.h"
#include "../../DataStructures/StaticGraph.h"
#include "../../DataStructures/StaticRTree.h"
#include "../../DataStructures/TurnInstructions.h"
#include "../../typedefs.h"
#include <osrm/Coordinate.h>
#include <cstdint> #include <cstdint>
typedef BaseDataFacade<QueryEdge::EdgeData>::RTreeLeaf RTreeLeaf; #include <array>
typedef StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>::TreeNode RTreeNode;
typedef StaticGraph<QueryEdge::EdgeData> QueryGraph; // Added at the start and end of each block as sanity check
constexpr char CANARY[] = "OSRM";
struct SharedDataLayout struct SharedDataLayout
{ {
uint64_t name_index_list_size; enum BlockID {
uint64_t name_char_list_size; NAME_OFFSETS = 0,
uint64_t name_id_list_size; NAME_BLOCKS,
uint64_t via_node_list_size; NAME_CHAR_LIST,
uint64_t graph_node_list_size; NAME_ID_LIST,
uint64_t graph_edge_list_size; VIA_NODE_LIST,
uint64_t coordinate_list_size; GRAPH_NODE_LIST,
uint64_t turn_instruction_list_size; GRAPH_EDGE_LIST,
uint64_t r_search_tree_size; COORDINATE_LIST,
uint64_t geometries_index_list_size; TURN_INSTRUCTION,
uint64_t geometries_list_size; R_SEARCH_TREE,
uint64_t geometries_indicators; GEOMETRIES_INDEX,
GEOMETRIES_LIST,
GEOMETRIES_INDICATORS,
HSGR_CHECKSUM,
TIMESTAMP,
FILE_INDEX_PATH,
NUM_BLOCKS
};
unsigned checksum; std::array<uint64_t, NUM_BLOCKS> num_entries;
unsigned timestamp_length; std::array<uint64_t, NUM_BLOCKS> entry_size;
char ram_index_file_name[1024];
SharedDataLayout() SharedDataLayout()
: name_index_list_size(0), name_char_list_size(0), name_id_list_size(0), : num_entries()
via_node_list_size(0), graph_node_list_size(0), graph_edge_list_size(0), , entry_size()
coordinate_list_size(0), turn_instruction_list_size(0), r_search_tree_size(0),
geometries_index_list_size(0), geometries_list_size(0), geometries_indicators(0),
checksum(0), timestamp_length(0)
{ {
ram_index_file_name[0] = '\0';
} }
void PrintInformation() const void PrintInformation() const
{ {
SimpleLogger().Write(logDEBUG) << "-"; SimpleLogger().Write(logDEBUG) << "-";
SimpleLogger().Write(logDEBUG) << "name_index_list_size: " << name_index_list_size; SimpleLogger().Write(logDEBUG) << "name_offsets_size: " << num_entries[NAME_OFFSETS];
SimpleLogger().Write(logDEBUG) << "name_char_list_size: " << name_char_list_size; SimpleLogger().Write(logDEBUG) << "name_blocks_size: " << num_entries[NAME_BLOCKS];
SimpleLogger().Write(logDEBUG) << "name_id_list_size: " << name_id_list_size; SimpleLogger().Write(logDEBUG) << "name_char_list_size: " << num_entries[NAME_CHAR_LIST];
SimpleLogger().Write(logDEBUG) << "via_node_list_size: " << via_node_list_size; SimpleLogger().Write(logDEBUG) << "name_id_list_size: " << num_entries[NAME_ID_LIST];
SimpleLogger().Write(logDEBUG) << "graph_node_list_size: " << graph_node_list_size; SimpleLogger().Write(logDEBUG) << "via_node_list_size: " << num_entries[VIA_NODE_LIST];
SimpleLogger().Write(logDEBUG) << "graph_edge_list_size: " << graph_edge_list_size; SimpleLogger().Write(logDEBUG) << "graph_node_list_size: " << num_entries[GRAPH_NODE_LIST];
SimpleLogger().Write(logDEBUG) << "timestamp_length: " << timestamp_length; SimpleLogger().Write(logDEBUG) << "graph_edge_list_size: " << num_entries[GRAPH_EDGE_LIST];
SimpleLogger().Write(logDEBUG) << "coordinate_list_size: " << coordinate_list_size; SimpleLogger().Write(logDEBUG) << "timestamp_length: " << num_entries[TIMESTAMP];
SimpleLogger().Write(logDEBUG) SimpleLogger().Write(logDEBUG) << "coordinate_list_size: " << num_entries[COORDINATE_LIST];
<< "turn_instruction_list_size: " << turn_instruction_list_size; SimpleLogger().Write(logDEBUG) << "turn_instruction_list_size: " << num_entries[TURN_INSTRUCTION];
SimpleLogger().Write(logDEBUG) << "r_search_tree_size: " << r_search_tree_size; SimpleLogger().Write(logDEBUG) << "r_search_tree_size: " << num_entries[R_SEARCH_TREE];
SimpleLogger().Write(logDEBUG) << "geometries_indicators: " << geometries_indicators SimpleLogger().Write(logDEBUG) << "geometries_indicators: " << num_entries[GEOMETRIES_INDICATORS]
<< "/" << ((geometries_indicators / 8) + 1); << "/" << ((num_entries[GEOMETRIES_INDICATORS] / 8) + 1);
SimpleLogger().Write(logDEBUG) SimpleLogger().Write(logDEBUG) << "geometries_index_list_size: " << num_entries[GEOMETRIES_INDEX];
<< "geometries_index_list_size: " << geometries_index_list_size; SimpleLogger().Write(logDEBUG) << "geometries_list_size: " << num_entries[GEOMETRIES_LIST];
SimpleLogger().Write(logDEBUG) << "geometries_list_size: " << geometries_list_size; SimpleLogger().Write(logDEBUG) << "sizeof(checksum): " << entry_size[HSGR_CHECKSUM];
SimpleLogger().Write(logDEBUG) << "checksum: " << checksum;
SimpleLogger().Write(logDEBUG) << "sizeof(checksum): " << sizeof(checksum); SimpleLogger().Write(logDEBUG) << "NAME_OFFSETS " << ": " << GetBlockSize(NAME_OFFSETS );
SimpleLogger().Write(logDEBUG) << "ram index file name: " << ram_index_file_name; SimpleLogger().Write(logDEBUG) << "NAME_BLOCKS " << ": " << GetBlockSize(NAME_BLOCKS );
SimpleLogger().Write(logDEBUG) << "NAME_CHAR_LIST " << ": " << GetBlockSize(NAME_CHAR_LIST );
SimpleLogger().Write(logDEBUG) << "NAME_ID_LIST " << ": " << GetBlockSize(NAME_ID_LIST );
SimpleLogger().Write(logDEBUG) << "VIA_NODE_LIST " << ": " << GetBlockSize(VIA_NODE_LIST );
SimpleLogger().Write(logDEBUG) << "GRAPH_NODE_LIST " << ": " << GetBlockSize(GRAPH_NODE_LIST );
SimpleLogger().Write(logDEBUG) << "GRAPH_EDGE_LIST " << ": " << GetBlockSize(GRAPH_EDGE_LIST );
SimpleLogger().Write(logDEBUG) << "COORDINATE_LIST " << ": " << GetBlockSize(COORDINATE_LIST );
SimpleLogger().Write(logDEBUG) << "TURN_INSTRUCTION " << ": " << GetBlockSize(TURN_INSTRUCTION );
SimpleLogger().Write(logDEBUG) << "R_SEARCH_TREE " << ": " << GetBlockSize(R_SEARCH_TREE );
SimpleLogger().Write(logDEBUG) << "GEOMETRIES_INDEX " << ": " << GetBlockSize(GEOMETRIES_INDEX );
SimpleLogger().Write(logDEBUG) << "GEOMETRIES_LIST " << ": " << GetBlockSize(GEOMETRIES_LIST );
SimpleLogger().Write(logDEBUG) << "GEOMETRIES_INDICATORS" << ": " << GetBlockSize(GEOMETRIES_INDICATORS);
SimpleLogger().Write(logDEBUG) << "HSGR_CHECKSUM " << ": " << GetBlockSize(HSGR_CHECKSUM );
SimpleLogger().Write(logDEBUG) << "TIMESTAMP " << ": " << GetBlockSize(TIMESTAMP );
SimpleLogger().Write(logDEBUG) << "FILE_INDEX_PATH " << ": " << GetBlockSize(FILE_INDEX_PATH );
} }
uint64_t GetSizeOfLayout() const template<typename T>
inline void SetBlockSize(BlockID bid, uint64_t entries)
{ {
uint64_t result = num_entries[bid] = entries;
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) + entry_size[bid] = sizeof(T);
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) + }
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) + inline uint64_t GetBlockSize(BlockID bid) const
(timestamp_length * sizeof(char)) + {
(coordinate_list_size * sizeof(FixedPointCoordinate)) + // special encoding
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) + if (bid == GEOMETRIES_INDICATORS)
(r_search_tree_size * sizeof(RTreeNode)) + {
(geometries_indicators / 32 + 1) * sizeof(unsigned) + return (num_entries[GEOMETRIES_INDICATORS]/32 + 1) * entry_size[GEOMETRIES_INDICATORS];
(geometries_index_list_size * sizeof(unsigned)) + }
(geometries_list_size * sizeof(unsigned)) + sizeof(checksum) + 1024 * sizeof(char);
return num_entries[bid] * entry_size[bid];
}
inline uint64_t GetSizeOfLayout() const
{
return GetBlockOffset(NUM_BLOCKS) + NUM_BLOCKS*2*sizeof(CANARY);
}
inline uint64_t GetBlockOffset(BlockID bid) const
{
uint64_t result = sizeof(CANARY);
for (unsigned i = 0; i < bid; i++)
{
result += GetBlockSize((BlockID) i) + 2*sizeof(CANARY);
}
return result; return result;
} }
uint64_t GetNameIndexOffset() const { return 0; } template<typename T, bool WRITE_CANARY=false>
uint64_t GetNameListOffset() const inline T* GetBlockPtr(char* shared_memory, BlockID bid)
{ {
uint64_t result = (name_index_list_size * sizeof(unsigned)); T* ptr = (T*)(shared_memory + GetBlockOffset(bid));
return result; if (WRITE_CANARY)
} {
uint64_t GetNameIDListOffset() const char* start_canary_ptr = shared_memory + GetBlockOffset(bid) - sizeof(CANARY);
{ char* end_canary_ptr = shared_memory + GetBlockOffset(bid) + GetBlockSize(bid);
uint64_t result = std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)); std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
return result; }
} else
uint64_t GetViaNodeListOffset() const {
{ char* start_canary_ptr = shared_memory + GetBlockOffset(bid) - sizeof(CANARY);
uint64_t result = (name_index_list_size * sizeof(unsigned)) + char* end_canary_ptr = shared_memory + GetBlockOffset(bid) + GetBlockSize(bid);
(name_char_list_size * sizeof(char)) + bool start_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
(name_id_list_size * sizeof(unsigned)); bool end_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
return result; if (!start_canary_alive)
} {
uint64_t GetGraphNodeListOffset() const throw OSRMException("Start canary of block corrupted.");
{ }
uint64_t result = if (!end_canary_alive)
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) + {
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)); throw OSRMException("End canary of block corrupted.");
return result; }
} }
uint64_t GetGraphEdgeListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry));
return result;
}
uint64_t GetTimeStampOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry));
return result;
}
uint64_t GetCoordinateListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char));
return result;
}
uint64_t GetTurnInstructionListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate));
return result;
}
uint64_t GetRSearchTreeOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass));
return result;
}
uint64_t GetGeometriesIndicatorOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode));
return result;
}
uint64_t GetGeometriesIndexListOffset() const return ptr;
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode)) +
(geometries_indicators / 32 + 1) * sizeof(unsigned);
return result;
}
uint64_t GetGeometryListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode)) +
(geometries_indicators / 32 + 1) * sizeof(unsigned) +
(geometries_index_list_size * sizeof(unsigned));
return result;
}
uint64_t GetChecksumOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode)) +
(geometries_indicators / 32 + 1) * sizeof(unsigned) +
(geometries_index_list_size * sizeof(unsigned)) +
(geometries_list_size * sizeof(unsigned));
return result;
} }
}; };

179
datastore.cpp
View File

@ -26,7 +26,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
#include "DataStructures/OriginalEdgeData.h" #include "DataStructures/OriginalEdgeData.h"
#include "DataStructures/RangeTable.h"
#include "DataStructures/QueryEdge.h"
#include "DataStructures/SharedMemoryFactory.h" #include "DataStructures/SharedMemoryFactory.h"
#include "DataStructures/SharedMemoryVectorWrapper.h"
#include "DataStructures/StaticGraph.h"
#include "DataStructures/StaticRTree.h"
#include "DataStructures/TurnInstructions.h"
#include "Server/DataStructures/BaseDataFacade.h"
#include "Server/DataStructures/SharedDataType.h" #include "Server/DataStructures/SharedDataType.h"
#include "Server/DataStructures/SharedBarriers.h" #include "Server/DataStructures/SharedBarriers.h"
#include "Util/BoostFileSystemFix.h" #include "Util/BoostFileSystemFix.h"
@ -35,6 +42,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "Util/FingerPrint.h" #include "Util/FingerPrint.h"
#include "typedefs.h" #include "typedefs.h"
#include <osrm/Coordinate.h>
typedef BaseDataFacade<QueryEdge::EdgeData>::RTreeLeaf RTreeLeaf;
typedef StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>::TreeNode RTreeNode;
typedef StaticGraph<QueryEdge::EdgeData> QueryGraph;
#ifdef __linux__ #ifdef __linux__
#include <sys/mman.h> #include <sys/mman.h>
#endif #endif
@ -161,7 +174,7 @@ int main(const int argc, const char *argv[])
BOOST_ASSERT(!paths_iterator->second.empty()); BOOST_ASSERT(!paths_iterator->second.empty());
const boost::filesystem::path index_file_path_absolute = const boost::filesystem::path index_file_path_absolute =
boost::filesystem::portable_canonical(paths_iterator->second); boost::filesystem::portable_canonical(paths_iterator->second);
const std::string &file_index_file_name = index_file_path_absolute.string(); const std::string &file_index_path = index_file_path_absolute.string();
paths_iterator = server_paths.find("nodesdata"); paths_iterator = server_paths.find("nodesdata");
BOOST_ASSERT(server_paths.end() != paths_iterator); BOOST_ASSERT(server_paths.end() != paths_iterator);
BOOST_ASSERT(!paths_iterator->second.empty()); BOOST_ASSERT(!paths_iterator->second.empty());
@ -201,37 +214,34 @@ int main(const int argc, const char *argv[])
SharedDataLayout *shared_layout_ptr = static_cast<SharedDataLayout *>(layout_memory->Ptr()); SharedDataLayout *shared_layout_ptr = static_cast<SharedDataLayout *>(layout_memory->Ptr());
shared_layout_ptr = new (layout_memory->Ptr()) SharedDataLayout(); shared_layout_ptr = new (layout_memory->Ptr()) SharedDataLayout();
std::copy(file_index_file_name.begin(), shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::FILE_INDEX_PATH, file_index_path.length() + 1);
(file_index_file_name.length() <= 1024 ? file_index_file_name.end()
: file_index_file_name.begin() + 1023),
shared_layout_ptr->ram_index_file_name);
// add zero termination
unsigned end_of_string_index = std::min((std::size_t)1023, file_index_file_name.length());
shared_layout_ptr->ram_index_file_name[end_of_string_index] = '\0';
// collect number of elements to store in shared memory object // collect number of elements to store in shared memory object
SimpleLogger().Write() << "load names from: " << names_data_path; SimpleLogger().Write() << "load names from: " << names_data_path;
// number of entries in name index // number of entries in name index
boost::filesystem::ifstream name_stream(names_data_path, std::ios::binary); boost::filesystem::ifstream name_stream(names_data_path, std::ios::binary);
unsigned name_index_size = 0; unsigned name_blocks = 0;
name_stream.read((char *)&name_index_size, sizeof(unsigned)); name_stream.read((char *)&name_blocks, sizeof(unsigned));
shared_layout_ptr->name_index_list_size = name_index_size; shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::NAME_OFFSETS, name_blocks);
SimpleLogger().Write() << "size: " << name_index_size; shared_layout_ptr->SetBlockSize<typename RangeTable<16, true>::BlockT>(SharedDataLayout::NAME_BLOCKS, name_blocks);
BOOST_ASSERT_MSG(0 != shared_layout_ptr->name_index_list_size, "name file broken"); SimpleLogger().Write() << "name offsets size: " << name_blocks;
BOOST_ASSERT_MSG(0 != name_blocks, "name file broken");
unsigned number_of_chars = 0; unsigned number_of_chars = 0;
name_stream.read((char *)&number_of_chars, sizeof(unsigned)); name_stream.read((char *)&number_of_chars, sizeof(unsigned));
shared_layout_ptr->name_char_list_size = number_of_chars; shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::NAME_CHAR_LIST, number_of_chars);
// Loading information for original edges // Loading information for original edges
boost::filesystem::ifstream edges_input_stream(edges_data_path, std::ios::binary); boost::filesystem::ifstream edges_input_stream(edges_data_path, std::ios::binary);
unsigned number_of_original_edges = 0; unsigned number_of_original_edges = 0;
edges_input_stream.read((char *)&number_of_original_edges, sizeof(unsigned)); edges_input_stream.read((char *)&number_of_original_edges, sizeof(unsigned));
shared_layout_ptr->via_node_list_size = number_of_original_edges; // note: settings this all to the same size is correct, we extract them from the same struct
shared_layout_ptr->name_id_list_size = number_of_original_edges; shared_layout_ptr->SetBlockSize<NodeID>(SharedDataLayout::VIA_NODE_LIST, number_of_original_edges);
shared_layout_ptr->turn_instruction_list_size = number_of_original_edges; shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::NAME_ID_LIST, number_of_original_edges);
shared_layout_ptr->geometries_indicators = number_of_original_edges; shared_layout_ptr->SetBlockSize<TurnInstruction>(SharedDataLayout::TURN_INSTRUCTION, number_of_original_edges);
// note: there are 32 geometry indicators in one unsigned block
shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::GEOMETRIES_INDICATORS, number_of_original_edges);
boost::filesystem::ifstream hsgr_input_stream(hsgr_path, std::ios::binary); boost::filesystem::ifstream hsgr_input_stream(hsgr_path, std::ios::binary);
@ -250,26 +260,26 @@ int main(const int argc, const char *argv[])
// load checksum // load checksum
unsigned checksum = 0; unsigned checksum = 0;
hsgr_input_stream.read((char *)&checksum, sizeof(unsigned)); hsgr_input_stream.read((char *)&checksum, sizeof(unsigned));
shared_layout_ptr->checksum = checksum; shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::HSGR_CHECKSUM, 1);
// load graph node size // load graph node size
unsigned number_of_graph_nodes = 0; unsigned number_of_graph_nodes = 0;
hsgr_input_stream.read((char *)&number_of_graph_nodes, sizeof(unsigned)); hsgr_input_stream.read((char *)&number_of_graph_nodes, sizeof(unsigned));
BOOST_ASSERT_MSG((0 != number_of_graph_nodes), "number of nodes is zero"); BOOST_ASSERT_MSG((0 != number_of_graph_nodes), "number of nodes is zero");
shared_layout_ptr->graph_node_list_size = number_of_graph_nodes; shared_layout_ptr->SetBlockSize<QueryGraph::NodeArrayEntry>(SharedDataLayout::GRAPH_NODE_LIST, number_of_graph_nodes);
// load graph edge size // load graph edge size
unsigned number_of_graph_edges = 0; unsigned number_of_graph_edges = 0;
hsgr_input_stream.read((char *)&number_of_graph_edges, sizeof(unsigned)); hsgr_input_stream.read((char *)&number_of_graph_edges, sizeof(unsigned));
BOOST_ASSERT_MSG(0 != number_of_graph_edges, "number of graph edges is zero"); BOOST_ASSERT_MSG(0 != number_of_graph_edges, "number of graph edges is zero");
shared_layout_ptr->graph_edge_list_size = number_of_graph_edges; shared_layout_ptr->SetBlockSize<QueryGraph::EdgeArrayEntry>(SharedDataLayout::GRAPH_EDGE_LIST, number_of_graph_edges);
// load rsearch tree size // load rsearch tree size
boost::filesystem::ifstream tree_node_file(ram_index_path, std::ios::binary); boost::filesystem::ifstream tree_node_file(ram_index_path, std::ios::binary);
uint32_t tree_size = 0; uint32_t tree_size = 0;
tree_node_file.read((char *)&tree_size, sizeof(uint32_t)); tree_node_file.read((char *)&tree_size, sizeof(uint32_t));
shared_layout_ptr->r_search_tree_size = tree_size; shared_layout_ptr->SetBlockSize<RTreeNode>(SharedDataLayout::R_SEARCH_TREE, tree_size);
// load timestamp size // load timestamp size
std::string m_timestamp; std::string m_timestamp;
@ -295,13 +305,13 @@ int main(const int argc, const char *argv[])
{ {
m_timestamp.resize(25); m_timestamp.resize(25);
} }
shared_layout_ptr->timestamp_length = m_timestamp.length(); shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::TIMESTAMP, m_timestamp.length());
// load coordinate size // load coordinate size
boost::filesystem::ifstream nodes_input_stream(nodes_data_path, std::ios::binary); boost::filesystem::ifstream nodes_input_stream(nodes_data_path, std::ios::binary);
unsigned coordinate_list_size = 0; unsigned coordinate_list_size = 0;
nodes_input_stream.read((char *)&coordinate_list_size, sizeof(unsigned)); nodes_input_stream.read((char *)&coordinate_list_size, sizeof(unsigned));
shared_layout_ptr->coordinate_list_size = coordinate_list_size; shared_layout_ptr->SetBlockSize<FixedPointCoordinate>(SharedDataLayout::COORDINATE_LIST, coordinate_list_size);
// load geometries sizes // load geometries sizes
std::ifstream geometry_input_stream(geometries_data_path.string().c_str(), std::ios::binary); std::ifstream geometry_input_stream(geometries_data_path.string().c_str(), std::ios::binary);
@ -309,11 +319,11 @@ int main(const int argc, const char *argv[])
unsigned number_of_compressed_geometries = 0; unsigned number_of_compressed_geometries = 0;
geometry_input_stream.read((char *)&number_of_geometries_indices, sizeof(unsigned)); geometry_input_stream.read((char *)&number_of_geometries_indices, sizeof(unsigned));
shared_layout_ptr->geometries_index_list_size = number_of_geometries_indices; shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::GEOMETRIES_INDEX, number_of_geometries_indices);
boost::iostreams::seek( boost::iostreams::seek(
geometry_input_stream, number_of_geometries_indices * sizeof(unsigned), BOOST_IOS::cur); geometry_input_stream, number_of_geometries_indices * sizeof(unsigned), BOOST_IOS::cur);
geometry_input_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned)); geometry_input_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned));
shared_layout_ptr->geometries_list_size = number_of_compressed_geometries; shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::GEOMETRIES_LIST, number_of_compressed_geometries);
// allocate shared memory block // allocate shared memory block
SimpleLogger().Write() << "allocating shared memory of " SimpleLogger().Write() << "allocating shared memory of "
@ -323,35 +333,70 @@ int main(const int argc, const char *argv[])
char *shared_memory_ptr = static_cast<char *>(shared_memory->Ptr()); char *shared_memory_ptr = static_cast<char *>(shared_memory->Ptr());
// read actual data into shared memory object // // read actual data into shared memory object //
// hsgr checksum
unsigned* checksum_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr,
SharedDataLayout::HSGR_CHECKSUM);
*checksum_ptr = checksum;
// ram index file name
char* file_index_path_ptr = shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr,
SharedDataLayout::FILE_INDEX_PATH);
// make sure we have 0 ending
std::fill(file_index_path_ptr,
file_index_path_ptr +
shared_layout_ptr->GetBlockSize(SharedDataLayout::FILE_INDEX_PATH),
0);
std::copy(file_index_path.begin(), file_index_path.end(), file_index_path_ptr);
// Loading street names // Loading street names
unsigned *name_index_ptr = unsigned *name_offsets_ptr =
(unsigned *)(shared_memory_ptr + shared_layout_ptr->GetNameIndexOffset()); shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr,
if (shared_layout_ptr->name_index_list_size > 0) SharedDataLayout::NAME_OFFSETS);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_OFFSETS) > 0)
{ {
name_stream.read((char *)name_index_ptr, name_stream.read((char *)name_offsets_ptr,
shared_layout_ptr->name_index_list_size * sizeof(unsigned)); shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_OFFSETS));
} }
char *name_char_ptr = shared_memory_ptr + shared_layout_ptr->GetNameListOffset(); unsigned *name_blocks_ptr =
if (shared_layout_ptr->name_char_list_size > 0) shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr,
SharedDataLayout::NAME_BLOCKS);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_BLOCKS) > 0)
{ {
name_stream.read(name_char_ptr, shared_layout_ptr->name_char_list_size * sizeof(char)); name_stream.read((char *)name_blocks_ptr,
shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_BLOCKS));
} }
char *name_char_ptr =
shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr,
SharedDataLayout::NAME_CHAR_LIST);
unsigned temp_length;
name_stream.read((char*) &temp_length, sizeof(unsigned));
BOOST_ASSERT_MSG(temp_length == shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST),
"Name file corrupted!");
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST) > 0)
{
name_stream.read(name_char_ptr, shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST));
}
name_stream.close(); name_stream.close();
// load original edge information // load original edge information
NodeID *via_node_ptr = NodeID *via_node_ptr = shared_layout_ptr->GetBlockPtr<NodeID, true>(shared_memory_ptr,
(NodeID *)(shared_memory_ptr + shared_layout_ptr->GetViaNodeListOffset()); SharedDataLayout::VIA_NODE_LIST);
unsigned *name_id_ptr = unsigned *name_id_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr,
(unsigned *)(shared_memory_ptr + shared_layout_ptr->GetNameIDListOffset()); SharedDataLayout::NAME_ID_LIST);
TurnInstruction *turn_instructions_ptr = TurnInstruction *turn_instructions_ptr =
(TurnInstruction *)(shared_memory_ptr + shared_layout_ptr->GetBlockPtr<TurnInstruction, true>(shared_memory_ptr,
shared_layout_ptr->GetTurnInstructionListOffset()); SharedDataLayout::TURN_INSTRUCTION);
unsigned *geometries_indicator_ptr = unsigned *geometries_indicator_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr,
(unsigned *)(shared_memory_ptr + shared_layout_ptr->GetGeometriesIndicatorOffset()); SharedDataLayout::GEOMETRIES_INDICATORS);
OriginalEdgeData current_edge_data; OriginalEdgeData current_edge_data;
for (unsigned i = 0; i < number_of_original_edges; ++i) for (unsigned i = 0; i < number_of_original_edges; ++i)
@ -382,33 +427,32 @@ int main(const int argc, const char *argv[])
// load compressed geometry // load compressed geometry
unsigned temporary_value; unsigned temporary_value;
unsigned *geometries_index_ptr = unsigned *geometries_index_ptr =
(unsigned *)(shared_memory_ptr + shared_layout_ptr->GetGeometriesIndexListOffset()); shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr, SharedDataLayout::GEOMETRIES_INDEX);
geometry_input_stream.seekg(0, geometry_input_stream.beg); geometry_input_stream.seekg(0, geometry_input_stream.beg);
geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned)); geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned));
BOOST_ASSERT(temporary_value == shared_layout_ptr->geometries_index_list_size); BOOST_ASSERT(temporary_value == shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_INDEX]);
if (shared_layout_ptr->geometries_index_list_size > 0)
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_INDEX) > 0)
{ {
geometry_input_stream.read((char *)geometries_index_ptr, geometry_input_stream.read((char *)geometries_index_ptr,
shared_layout_ptr->geometries_index_list_size * shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_INDEX));
sizeof(unsigned));
} }
unsigned *geometries_list_ptr = unsigned *geometries_list_ptr =
(unsigned *)(shared_memory_ptr + shared_layout_ptr->GetGeometryListOffset()); shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr, SharedDataLayout::GEOMETRIES_LIST);
geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned)); geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned));
BOOST_ASSERT(temporary_value == shared_layout_ptr->geometries_list_size); BOOST_ASSERT(temporary_value == shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_LIST]);
if (shared_layout_ptr->geometries_list_size > 0) if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_LIST) > 0)
{ {
geometry_input_stream.read((char *)geometries_list_ptr, geometry_input_stream.read((char *)geometries_list_ptr,
shared_layout_ptr->geometries_list_size * sizeof(unsigned)); shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_LIST));
} }
// Loading list of coordinates // Loading list of coordinates
FixedPointCoordinate *coordinates_ptr = FixedPointCoordinate *coordinates_ptr =
(FixedPointCoordinate *)(shared_memory_ptr + shared_layout_ptr->GetBlockPtr<FixedPointCoordinate, true>(shared_memory_ptr,
shared_layout_ptr->GetCoordinateListOffset()); SharedDataLayout::COORDINATE_LIST);
NodeInfo current_node; NodeInfo current_node;
for (unsigned i = 0; i < coordinate_list_size; ++i) for (unsigned i = 0; i < coordinate_list_size; ++i)
@ -419,40 +463,37 @@ int main(const int argc, const char *argv[])
nodes_input_stream.close(); nodes_input_stream.close();
// store timestamp // store timestamp
char *timestamp_ptr = char *timestamp_ptr = shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr, SharedDataLayout::TIMESTAMP);
static_cast<char *>(shared_memory_ptr + shared_layout_ptr->GetTimeStampOffset());
std::copy(m_timestamp.c_str(), m_timestamp.c_str() + m_timestamp.length(), timestamp_ptr); std::copy(m_timestamp.c_str(), m_timestamp.c_str() + m_timestamp.length(), timestamp_ptr);
// store search tree portion of rtree // store search tree portion of rtree
char *rtree_ptr = char *rtree_ptr = shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr, SharedDataLayout::R_SEARCH_TREE);
static_cast<char *>(shared_memory_ptr + shared_layout_ptr->GetRSearchTreeOffset());
if (tree_size > 0) if (tree_size > 0)
{ {
tree_node_file.read(rtree_ptr, sizeof(RTreeNode) * tree_size); tree_node_file.read(rtree_ptr, sizeof(RTreeNode) * tree_size);
} }
tree_node_file.close(); tree_node_file.close();
// load the nodes of the search graph // load the nodes of the search graph
QueryGraph::NodeArrayEntry *graph_node_list_ptr = QueryGraph::NodeArrayEntry *graph_node_list_ptr =
(QueryGraph::NodeArrayEntry *)(shared_memory_ptr + shared_layout_ptr->GetBlockPtr<QueryGraph::NodeArrayEntry, true>(shared_memory_ptr,
shared_layout_ptr->GetGraphNodeListOffset()); SharedDataLayout::GRAPH_NODE_LIST);
if (shared_layout_ptr->graph_node_list_size > 0) if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_NODE_LIST) > 0)
{ {
hsgr_input_stream.read((char *)graph_node_list_ptr, hsgr_input_stream.read((char *)graph_node_list_ptr,
shared_layout_ptr->graph_node_list_size * shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_NODE_LIST));
sizeof(QueryGraph::NodeArrayEntry));
} }
// load the edges of the search graph // load the edges of the search graph
QueryGraph::EdgeArrayEntry *graph_edge_list_ptr = QueryGraph::EdgeArrayEntry *graph_edge_list_ptr =
(QueryGraph::EdgeArrayEntry *)(shared_memory_ptr + shared_layout_ptr->GetBlockPtr<QueryGraph::EdgeArrayEntry, true>(shared_memory_ptr,
shared_layout_ptr->GetGraphEdgeListOffset()); SharedDataLayout::GRAPH_EDGE_LIST);
if (shared_layout_ptr->graph_edge_list_size > 0) if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST) > 0)
{ {
hsgr_input_stream.read((char *)graph_edge_list_ptr, hsgr_input_stream.read((char *)graph_edge_list_ptr,
shared_layout_ptr->graph_edge_list_size * shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST));
sizeof(QueryGraph::EdgeArrayEntry));
} }
hsgr_input_stream.close(); hsgr_input_stream.close();