Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

street name file is now more canonical

Browse Source
This commit is contained in:
Dennis Luxen
2013-09-26 18:19:51 +02:00
parent e9d93ae210
commit 333aba8be6
7 changed files with 294 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Server/DataStructures/InternalDataFacade.h
+25 -16
View File
@@ -47,10 +47,10 @@ private:
InternalDataFacade() { }
unsigned m_check_sum;
unsigned m_number_of_nodes;
QueryGraph * m_query_graph;
std::string m_timestamp;
unsigned m_check_sum;
unsigned m_number_of_nodes;
QueryGraph * m_query_graph;
std::string m_timestamp;
ShM<FixedPointCoordinate, false>::vector m_coordinate_list;
ShM<NodeID, false>::vector m_via_node_list;
@@ -59,7 +59,7 @@ private:
ShM<char, false>::vector m_names_char_list;
ShM<unsigned, false>::vector m_name_begin_indices;
StaticRTree<RTreeLeaf, false> * m_static_rtree;
StaticRTree<RTreeLeaf, false> * m_static_rtree;
void LoadTimestamp(const boost::filesystem::path & timestamp_path) {
@@ -168,19 +168,28 @@ private:
const boost::filesystem::path & names_file
) {
boost::filesystem::ifstream name_stream(names_file, std::ios::binary);
unsigned size = 0;
name_stream.read((char *)&size, sizeof(unsigned));
BOOST_ASSERT_MSG(0 != size, "name file broken");
unsigned number_of_names = 0;
unsigned number_of_chars = 0;
name_stream.read((char *)&number_of_names, sizeof(unsigned));
name_stream.read((char *)&number_of_chars, sizeof(unsigned));
BOOST_ASSERT_MSG(0 != number_of_names, "name file broken");
BOOST_ASSERT_MSG(0 != number_of_chars, "name file broken");
m_name_begin_indices.resize(size);
name_stream.read((char*)&m_name_begin_indices[0], size*sizeof(unsigned));
name_stream.read((char *)&size, sizeof(unsigned));
BOOST_ASSERT_MSG(0 != size, "name file broken");
m_names_char_list.resize(size+1); //+1 is sentinel/dummy element
name_stream.read((char *)&m_names_char_list[0], size*sizeof(char));
BOOST_ASSERT_MSG(0 != m_names_char_list.size(), "could not load any names");
m_name_begin_indices.resize(number_of_names);
name_stream.read(
(char*)&m_name_begin_indices[0],
number_of_names*sizeof(unsigned)
);
m_names_char_list.resize(number_of_chars+1); //+1 gives sentinel element
name_stream.read(
(char *)&m_names_char_list[0],
number_of_chars*sizeof(char)
);
BOOST_ASSERT_MSG(
0 != m_names_char_list.size(),
"could not load any names"
);
name_stream.close();
}
public:
Server/DataStructures/SharedDataFacade.h
+59 -55
View File
@@ -24,6 +24,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
//implements all data storage when shared memory is _NOT_ used
#include "BaseDataFacade.h"
#include "SharedDataType.h"
#include "../../DataStructures/StaticGraph.h"
#include "../../DataStructures/StaticRTree.h"
@@ -46,6 +47,9 @@ private:
typedef typename super::RTreeLeaf RTreeLeaf;
typedef typename StaticRTree<RTreeLeaf, true>::TreeNode RTreeNode;
SharedDataLayout * data_layout;
char * shared_memory;
unsigned m_check_sum;
unsigned m_number_of_nodes;
QueryGraph * m_query_graph;
@@ -62,46 +66,46 @@ private:
SharedDataFacade() { }
void LoadTimestamp() {
uint64_t timestamp_size = *static_cast<uint64_t *>(
SharedMemoryFactory::Get(TIMESTAMP_SIZE)->Ptr()
char * timestamp_ptr = shared_memory + data_layout->GetTimeStampOffset();
m_timestamp.resize(data_layout->timestamp_length);
std::copy(
timestamp_ptr,
timestamp_ptr+data_layout->timestamp_length,
m_timestamp.begin()
);
timestamp_size = std::min(timestamp_size, (uint64_t)25);
SharedMemory * search_tree = SharedMemoryFactory::Get(TIMESTAMP);
char * tree_ptr = static_cast<char *>( search_tree->Ptr() );
m_timestamp.resize(timestamp_size);
std::copy(tree_ptr, tree_ptr+timestamp_size, m_timestamp.begin());
}
void LoadRTree(
const boost::filesystem::path & file_index_path
) {
uint64_t tree_size = *static_cast<uint64_t *>(
SharedMemoryFactory::Get(R_SEARCH_TREE_SIZE)->Ptr()
RTreeNode * tree_ptr = (RTreeNode *)(
shared_memory + data_layout->GetRSearchTreeOffset()
);
SharedMemory * search_tree = SharedMemoryFactory::Get(R_SEARCH_TREE);
RTreeNode * tree_ptr = static_cast<RTreeNode *>( search_tree->Ptr() );
m_static_rtree = new StaticRTree<RTreeLeaf, true>(
tree_ptr,
tree_size,
data_layout->r_search_tree_size,
file_index_path
);
}
void LoadGraph() {
m_number_of_nodes = *static_cast<uint64_t *>(
SharedMemoryFactory::Get(GRAPH_NODE_LIST_SIZE)->Ptr()
m_number_of_nodes = data_layout->graph_node_list_size;
GraphNode * graph_nodes_ptr = (GraphNode *)(
shared_memory + data_layout->GetGraphNodeListOffset()
);
SharedMemory * graph_nodes = SharedMemoryFactory::Get(GRAPH_NODE_LIST);
GraphNode * graph_nodes_ptr = static_cast<GraphNode *>( graph_nodes->Ptr() );
uint64_t number_of_edges = *static_cast<uint64_t *>(
SharedMemoryFactory::Get(GRAPH_EDGE_LIST_SIZE)->Ptr()
GraphEdge * graph_edges_ptr = (GraphEdge *)(
shared_memory + data_layout->GetGraphEdgeListOffsett()
);
SharedMemory * graph_edges = SharedMemoryFactory::Get(GRAPH_EDGE_LIST);
GraphEdge * graph_edges_ptr = static_cast<GraphEdge *>( graph_edges->Ptr() );
typename ShM<GraphNode, true>::vector node_list(graph_nodes_ptr, m_number_of_nodes);
typename ShM<GraphEdge, true>::vector edge_list(graph_edges_ptr, number_of_edges);
typename ShM<GraphNode, true>::vector node_list(
graph_nodes_ptr,
data_layout->graph_node_list_size
);
typename ShM<GraphEdge, true>::vector edge_list(
graph_edges_ptr,
data_layout->graph_edge_list_size
);
m_query_graph = new QueryGraph(
node_list ,
edge_list
@@ -110,70 +114,62 @@ private:
}
void LoadNodeAndEdgeInformation() {
uint64_t number_of_coordinates = *static_cast<uint64_t *>(
SharedMemoryFactory::Get(COORDINATE_LIST_SIZE)->Ptr()
);
FixedPointCoordinate * coordinate_list_ptr = static_cast<FixedPointCoordinate *>(
SharedMemoryFactory::Get(COORDINATE_LIST)->Ptr()
FixedPointCoordinate * coordinate_list_ptr = (FixedPointCoordinate *)(
shared_memory + data_layout->GetCoordinateListOffset()
);
typename ShM<FixedPointCoordinate, true>::vector coordinate_list(
coordinate_list_ptr,
number_of_coordinates
data_layout->coordinate_list_size
);
m_coordinate_list.swap( coordinate_list );
uint64_t number_of_turn_instructions = *static_cast<uint64_t *>(
SharedMemoryFactory::Get(TURN_INSTRUCTION_LIST_SIZE)->Ptr()
TurnInstruction * turn_instruction_list_ptr = (TurnInstruction *)(
shared_memory + data_layout->GetTurnInstructionListOffset()
);
TurnInstruction * turn_instruction_list_ptr = static_cast<TurnInstruction *>(
SharedMemoryFactory::Get(TURN_INSTRUCTION_LIST)->Ptr()
);
typename ShM<TurnInstruction, true>::vector turn_instruction_list(
turn_instruction_list_ptr,
number_of_turn_instructions
data_layout->turn_instruction_list_size
);
m_turn_instruction_list.swap(turn_instruction_list);
unsigned * name_id_list_ptr = (unsigned *)(
shared_memory + data_layout->GetNameIDListOffset()
);
typename ShM<unsigned, true>::vector name_id_list(
name_id_list_ptr,
data_layout->name_id_list_size
);
m_name_ID_list.swap(name_id_list);
}
void LoadViaNodeList() {
uint64_t number_of_via_nodes = * static_cast<uint64_t *> (
SharedMemoryFactory::Get(VIA_NODE_LIST_SIZE)->Ptr()
);
NodeID * via_node_list_ptr = static_cast<NodeID *>(
SharedMemoryFactory::Get(VIA_NODE_LIST)->Ptr()
NodeID * via_node_list_ptr = (NodeID *)(
shared_memory + data_layout->GetViaNodeListOffset()
);
typename ShM<NodeID, true>::vector via_node_list(
via_node_list_ptr,
number_of_via_nodes
data_layout->via_node_list_size
);
m_via_node_list.swap(via_node_list);
}
void LoadNames() {
uint64_t street_names_index_size = * static_cast<uint64_t *> (
SharedMemoryFactory::Get(NAME_INDEX_SIZE)->Ptr()
);
unsigned * street_names_index_ptr = static_cast<unsigned *>(
SharedMemoryFactory::Get(NAMES_INDEX)->Ptr()
unsigned * street_names_index_ptr = (unsigned *)(
shared_memory + data_layout->GetNameIndexOffset()
);
typename ShM<unsigned, true>::vector name_begin_indices(
street_names_index_ptr,
street_names_index_size
data_layout->names_index_size
);
m_name_begin_indices.swap(m_name_begin_indices);
uint64_t names_list_size = * static_cast<uint64_t *>(
SharedMemoryFactory::Get(NAMES_LIST_SIZE)->Ptr()
);
char * names_list_ptr = static_cast<char *>(
SharedMemoryFactory::Get(NAMES_LIST)->Ptr()
char * names_list_ptr = (char *)(
shared_memory + data_layout->GetNameListOffset()
);
typename ShM<char, true>::vector names_char_list(
names_list_ptr,
names_list_size
data_layout->names_list_size
);
m_names_char_list.swap(names_char_list);
}
@@ -194,6 +190,14 @@ public:
base_path
);
data_layout = (SharedDataLayout *)(
SharedMemoryFactory::Get(LAYOUT_1)->Ptr()
);
shared_memory = (char *)(
SharedMemoryFactory::Get(DATA_1)->Ptr()
);
//load data
SimpleLogger().Write() << "loading graph data";
LoadGraph();
Server/DataStructures/SharedDataType.h
+190 -21
View File
@@ -21,28 +21,197 @@ or see http://www.gnu.org/licenses/agpl.txt.
#ifndef SHARED_DATA_TYPE_H_
#define SHARED_DATA_TYPE_H_
#include "BaseDataFacade.h"
#include "../../DataStructures/Coordinate.h"
#include "../../DataStructures/QueryEdge.h"
#include "../../DataStructures/StaticGraph.h"
#include "../../DataStructures/StaticRTree.h"
#include "../../DataStructures/TurnInstructions.h"
#include "../../typedefs.h"
#include <boost/integer.hpp>
typedef BaseDataFacade<QueryEdge::EdgeData>::RTreeLeaf RTreeLeaf;
typedef StaticRTree<RTreeLeaf, true>::TreeNode RTreeNode;
typedef StaticGraph<QueryEdge::EdgeData> QueryGraph;
struct SharedDataLayout {
uint64_t names_index_size;
uint64_t names_list_size;
uint64_t name_id_list_size;
uint64_t via_node_list_size;
uint64_t graph_node_list_size;
uint64_t graph_edge_list_size;
uint64_t coordinate_list_size;
uint64_t turn_instruction_list_size;
uint64_t r_search_tree_size;
unsigned checksum;
unsigned timestamp_length;
SharedDataLayout() :
names_index_size(0),
names_list_size(0),
name_id_list_size(0),
via_node_list_size(0),
graph_node_list_size(0),
graph_edge_list_size(0),
coordinate_list_size(0),
turn_instruction_list_size(0),
r_search_tree_size(0),
checksum(0),
timestamp_length(0)
{ }
uint64_t GetSizeOfLayout() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) ) +
(via_node_list_size * sizeof(NodeID) ) +
(graph_node_list_size * sizeof(QueryGraph::_StrNode)) +
(graph_edge_list_size * sizeof(QueryGraph::_StrEdge)) +
(timestamp_length * sizeof(char) ) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructions) ) +
(r_search_tree_size * sizeof(RTreeNode) ) +
sizeof(checksum);
return result;
}
uint64_t GetNameIndexOffset() const {
return 0;
}
uint64_t GetNameListOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) );
return result;
}
uint64_t GetNameIDListOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) );
return result;
}
uint64_t GetViaNodeListOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) );
return result;
}
uint64_t GetGraphNodeListOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) ) +
(via_node_list_size * sizeof(NodeID) );
return result;
}
uint64_t GetGraphEdgeListOffsett() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) ) +
(via_node_list_size * sizeof(NodeID) ) +
(graph_node_list_size * sizeof(QueryGraph::_StrNode)) ;
return result;
}
uint64_t GetTimeStampOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) ) +
(via_node_list_size * sizeof(NodeID) ) +
(graph_node_list_size * sizeof(QueryGraph::_StrNode)) +
(graph_edge_list_size * sizeof(QueryGraph::_StrEdge));
return result;
}
uint64_t GetCoordinateListOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) ) +
(via_node_list_size * sizeof(NodeID) ) +
(graph_node_list_size * sizeof(QueryGraph::_StrNode)) +
(graph_edge_list_size * sizeof(QueryGraph::_StrEdge)) +
(timestamp_length * sizeof(char) );
return result;
}
uint64_t GetTurnInstructionListOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) ) +
(via_node_list_size * sizeof(NodeID) ) +
(graph_node_list_size * sizeof(QueryGraph::_StrNode)) +
(graph_edge_list_size * sizeof(QueryGraph::_StrEdge)) +
(timestamp_length * sizeof(char) ) +
(coordinate_list_size * sizeof(FixedPointCoordinate));
return result;
}
uint64_t GetRSearchTreeOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) ) +
(via_node_list_size * sizeof(NodeID) ) +
(graph_node_list_size * sizeof(QueryGraph::_StrNode)) +
(graph_edge_list_size * sizeof(QueryGraph::_StrEdge)) +
(timestamp_length * sizeof(char) ) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructions) );
return result;
}
uint64_t GetChecksumOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) ) +
(via_node_list_size * sizeof(NodeID) ) +
(graph_node_list_size * sizeof(QueryGraph::_StrNode)) +
(graph_edge_list_size * sizeof(QueryGraph::_StrEdge)) +
(timestamp_length * sizeof(char) ) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructions) ) +
(r_search_tree_size * sizeof(RTreeNode) );
return result;
}
};
enum SharedDataType {
NAMES_INDEX = 0,
NAME_INDEX_SIZE,
NAMES_LIST,
NAMES_LIST_SIZE,
NAME_ID_LIST,
NAME_ID_LIST_SIZE,
VIA_NODE_LIST,
VIA_NODE_LIST_SIZE,
GRAPH_NODE_LIST,
GRAPH_NODE_LIST_SIZE,
GRAPH_EDGE_LIST,
GRAPH_EDGE_LIST_SIZE,
CHECK_SUM,
TIMESTAMP,
TIMESTAMP_SIZE,
COORDINATE_LIST,
COORDINATE_LIST_SIZE,
TURN_INSTRUCTION_LIST,
TURN_INSTRUCTION_LIST_SIZE,
R_SEARCH_TREE,
R_SEARCH_TREE_SIZE
// NAMES_INDEX = 0,
// NAME_INDEX_SIZE,
// NAMES_LIST,
// NAMES_LIST_SIZE,
// NAME_ID_LIST,
// NAME_ID_LIST_SIZE,
// VIA_NODE_LIST,
// VIA_NODE_LIST_SIZE,
// GRAPH_NODE_LIST,
// GRAPH_NODE_LIST_SIZE,
// GRAPH_EDGE_LIST,
// GRAPH_EDGE_LIST_SIZE,
// CHECK_SUM,
// TIMESTAMP,
// TIMESTAMP_SIZE,
// COORDINATE_LIST,
// COORDINATE_LIST_SIZE,
// TURN_INSTRUCTION_LIST,
// TURN_INSTRUCTION_LIST_SIZE,
// R_SEARCH_TREE,
// R_SEARCH_TREE_SIZE
LAYOUT_1,
DATA_1,
LAYOUT_2,
DATA_2,
LAYOUT_3,
DATA_3,
LAYOUT_4,
DATA_5
};
#endif /* SHARED_DATA_TYPE_H_ */