street name file is now more canonical

This commit is contained in:
Dennis Luxen 2013-09-26 18:19:51 +02:00
parent f76361a345
commit b87a98bbda
7 changed files with 294 additions and 102 deletions

View File

@ -37,9 +37,7 @@ struct OSRMLockFile {
boost::filesystem::path operator()() {
boost::filesystem::path temp_dir =
boost::filesystem::temp_directory_path();
// SimpleLogger().Write(logDEBUG) << "creating lock file in " << temp_dir;
boost::filesystem::path lock_file = temp_dir / "osrm.lock";
// SimpleLogger().Write(logDEBUG) << "locking at " << lock_file;
return lock_file;
}
};
@ -66,7 +64,6 @@ class SharedMemory : boost::noncopyable {
boost::interprocess::xsi_shared_memory::remove(m_shmid);
}
}
};
public:

View File

@ -106,12 +106,12 @@ public:
std::size_t size() const { return m_size; }
DataT & operator[](const int index) {
DataT & operator[](const unsigned index) {
BOOST_ASSERT_MSG(index < m_size, "invalid size");
return m_ptr[index];
}
const DataT & operator[](const int index) const {
const DataT & operator[](const unsigned index) const {
BOOST_ASSERT_MSG(index < m_size, "invalid size");
return m_ptr[index];
}

View File

@ -36,7 +36,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "SharedMemoryFactory.h"
#include "SharedMemoryVectorWrapper.h"
#include "../Server/DataStructures/SharedDataType.h"
#include "../Util/OSRMException.h"
#include "../Util/SimpleLogger.h"
#include "../Util/TimingUtil.h"

View File

@ -297,8 +297,21 @@ void ExtractionContainers::PrepareData(const std::string & output_file_name, con
std::ios::binary
);
const unsigned number_of_ways = name_list.size()+1;
name_file_stream.write((char *)&(number_of_ways), sizeof(unsigned));
//write number of names
const unsigned number_of_names = name_list.size()+1;
name_file_stream.write((char *)&(number_of_names), sizeof(unsigned));
//compute total number of chars
unsigned total_number_of_chars = 0;
BOOST_FOREACH(const std::string & str, name_list) {
total_number_of_chars += strlen(str.c_str());
}
//write total number of chars
name_file_stream.write(
(char *)&(total_number_of_chars),
sizeof(unsigned)
);
//write prefixe sums
unsigned name_lengths_prefix_sum = 0;
BOOST_FOREACH(const std::string & str, name_list) {
name_file_stream.write(
@ -307,12 +320,13 @@ void ExtractionContainers::PrepareData(const std::string & output_file_name, con
);
name_lengths_prefix_sum += strlen(str.c_str());
}
//duplicate on purpose!
name_file_stream.write(
(char *)&(name_lengths_prefix_sum),
sizeof(unsigned)
);
//duplicate on purpose!
name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
//write all chars consecutively
BOOST_FOREACH(const std::string & str, name_list) {
const unsigned lengthOfRawString = strlen(str.c_str());
name_file_stream.write(str.c_str(), lengthOfRawString);

View File

@ -168,19 +168,28 @@ private:
const boost::filesystem::path & names_file
) {
boost::filesystem::ifstream name_stream(names_file, std::ios::binary);
unsigned size = 0;
name_stream.read((char *)&size, sizeof(unsigned));
BOOST_ASSERT_MSG(0 != size, "name file broken");
unsigned number_of_names = 0;
unsigned number_of_chars = 0;
name_stream.read((char *)&number_of_names, sizeof(unsigned));
name_stream.read((char *)&number_of_chars, sizeof(unsigned));
BOOST_ASSERT_MSG(0 != number_of_names, "name file broken");
BOOST_ASSERT_MSG(0 != number_of_chars, "name file broken");
m_name_begin_indices.resize(size);
name_stream.read((char*)&m_name_begin_indices[0], size*sizeof(unsigned));
name_stream.read((char *)&size, sizeof(unsigned));
BOOST_ASSERT_MSG(0 != size, "name file broken");
m_names_char_list.resize(size+1); //+1 is sentinel/dummy element
name_stream.read((char *)&m_names_char_list[0], size*sizeof(char));
BOOST_ASSERT_MSG(0 != m_names_char_list.size(), "could not load any names");
m_name_begin_indices.resize(number_of_names);
name_stream.read(
(char*)&m_name_begin_indices[0],
number_of_names*sizeof(unsigned)
);
m_names_char_list.resize(number_of_chars+1); //+1 gives sentinel element
name_stream.read(
(char *)&m_names_char_list[0],
number_of_chars*sizeof(char)
);
BOOST_ASSERT_MSG(
0 != m_names_char_list.size(),
"could not load any names"
);
name_stream.close();
}
public:

View File

@ -24,6 +24,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
//implements all data storage when shared memory is _NOT_ used
#include "BaseDataFacade.h"
#include "SharedDataType.h"
#include "../../DataStructures/StaticGraph.h"
#include "../../DataStructures/StaticRTree.h"
@ -46,6 +47,9 @@ private:
typedef typename super::RTreeLeaf RTreeLeaf;
typedef typename StaticRTree<RTreeLeaf, true>::TreeNode RTreeNode;
SharedDataLayout * data_layout;
char * shared_memory;
unsigned m_check_sum;
unsigned m_number_of_nodes;
QueryGraph * m_query_graph;
@ -62,46 +66,46 @@ private:
SharedDataFacade() { }
void LoadTimestamp() {
uint64_t timestamp_size = *static_cast<uint64_t *>(
SharedMemoryFactory::Get(TIMESTAMP_SIZE)->Ptr()
char * timestamp_ptr = shared_memory + data_layout->GetTimeStampOffset();
m_timestamp.resize(data_layout->timestamp_length);
std::copy(
timestamp_ptr,
timestamp_ptr+data_layout->timestamp_length,
m_timestamp.begin()
);
timestamp_size = std::min(timestamp_size, (uint64_t)25);
SharedMemory * search_tree = SharedMemoryFactory::Get(TIMESTAMP);
char * tree_ptr = static_cast<char *>( search_tree->Ptr() );
m_timestamp.resize(timestamp_size);
std::copy(tree_ptr, tree_ptr+timestamp_size, m_timestamp.begin());
}
void LoadRTree(
const boost::filesystem::path & file_index_path
) {
uint64_t tree_size = *static_cast<uint64_t *>(
SharedMemoryFactory::Get(R_SEARCH_TREE_SIZE)->Ptr()
RTreeNode * tree_ptr = (RTreeNode *)(
shared_memory + data_layout->GetRSearchTreeOffset()
);
SharedMemory * search_tree = SharedMemoryFactory::Get(R_SEARCH_TREE);
RTreeNode * tree_ptr = static_cast<RTreeNode *>( search_tree->Ptr() );
m_static_rtree = new StaticRTree<RTreeLeaf, true>(
tree_ptr,
tree_size,
data_layout->r_search_tree_size,
file_index_path
);
}
void LoadGraph() {
m_number_of_nodes = *static_cast<uint64_t *>(
SharedMemoryFactory::Get(GRAPH_NODE_LIST_SIZE)->Ptr()
m_number_of_nodes = data_layout->graph_node_list_size;
GraphNode * graph_nodes_ptr = (GraphNode *)(
shared_memory + data_layout->GetGraphNodeListOffset()
);
SharedMemory * graph_nodes = SharedMemoryFactory::Get(GRAPH_NODE_LIST);
GraphNode * graph_nodes_ptr = static_cast<GraphNode *>( graph_nodes->Ptr() );
uint64_t number_of_edges = *static_cast<uint64_t *>(
SharedMemoryFactory::Get(GRAPH_EDGE_LIST_SIZE)->Ptr()
GraphEdge * graph_edges_ptr = (GraphEdge *)(
shared_memory + data_layout->GetGraphEdgeListOffsett()
);
SharedMemory * graph_edges = SharedMemoryFactory::Get(GRAPH_EDGE_LIST);
GraphEdge * graph_edges_ptr = static_cast<GraphEdge *>( graph_edges->Ptr() );
typename ShM<GraphNode, true>::vector node_list(graph_nodes_ptr, m_number_of_nodes);
typename ShM<GraphEdge, true>::vector edge_list(graph_edges_ptr, number_of_edges);
typename ShM<GraphNode, true>::vector node_list(
graph_nodes_ptr,
data_layout->graph_node_list_size
);
typename ShM<GraphEdge, true>::vector edge_list(
graph_edges_ptr,
data_layout->graph_edge_list_size
);
m_query_graph = new QueryGraph(
node_list ,
edge_list
@ -110,70 +114,62 @@ private:
}
void LoadNodeAndEdgeInformation() {
uint64_t number_of_coordinates = *static_cast<uint64_t *>(
SharedMemoryFactory::Get(COORDINATE_LIST_SIZE)->Ptr()
);
FixedPointCoordinate * coordinate_list_ptr = static_cast<FixedPointCoordinate *>(
SharedMemoryFactory::Get(COORDINATE_LIST)->Ptr()
FixedPointCoordinate * coordinate_list_ptr = (FixedPointCoordinate *)(
shared_memory + data_layout->GetCoordinateListOffset()
);
typename ShM<FixedPointCoordinate, true>::vector coordinate_list(
coordinate_list_ptr,
number_of_coordinates
data_layout->coordinate_list_size
);
m_coordinate_list.swap( coordinate_list );
uint64_t number_of_turn_instructions = *static_cast<uint64_t *>(
SharedMemoryFactory::Get(TURN_INSTRUCTION_LIST_SIZE)->Ptr()
TurnInstruction * turn_instruction_list_ptr = (TurnInstruction *)(
shared_memory + data_layout->GetTurnInstructionListOffset()
);
TurnInstruction * turn_instruction_list_ptr = static_cast<TurnInstruction *>(
SharedMemoryFactory::Get(TURN_INSTRUCTION_LIST)->Ptr()
);
typename ShM<TurnInstruction, true>::vector turn_instruction_list(
turn_instruction_list_ptr,
number_of_turn_instructions
data_layout->turn_instruction_list_size
);
m_turn_instruction_list.swap(turn_instruction_list);
unsigned * name_id_list_ptr = (unsigned *)(
shared_memory + data_layout->GetNameIDListOffset()
);
typename ShM<unsigned, true>::vector name_id_list(
name_id_list_ptr,
data_layout->name_id_list_size
);
m_name_ID_list.swap(name_id_list);
}
void LoadViaNodeList() {
uint64_t number_of_via_nodes = * static_cast<uint64_t *> (
SharedMemoryFactory::Get(VIA_NODE_LIST_SIZE)->Ptr()
);
NodeID * via_node_list_ptr = static_cast<NodeID *>(
SharedMemoryFactory::Get(VIA_NODE_LIST)->Ptr()
NodeID * via_node_list_ptr = (NodeID *)(
shared_memory + data_layout->GetViaNodeListOffset()
);
typename ShM<NodeID, true>::vector via_node_list(
via_node_list_ptr,
number_of_via_nodes
data_layout->via_node_list_size
);
m_via_node_list.swap(via_node_list);
}
void LoadNames() {
uint64_t street_names_index_size = * static_cast<uint64_t *> (
SharedMemoryFactory::Get(NAME_INDEX_SIZE)->Ptr()
);
unsigned * street_names_index_ptr = static_cast<unsigned *>(
SharedMemoryFactory::Get(NAMES_INDEX)->Ptr()
unsigned * street_names_index_ptr = (unsigned *)(
shared_memory + data_layout->GetNameIndexOffset()
);
typename ShM<unsigned, true>::vector name_begin_indices(
street_names_index_ptr,
street_names_index_size
data_layout->names_index_size
);
m_name_begin_indices.swap(m_name_begin_indices);
uint64_t names_list_size = * static_cast<uint64_t *>(
SharedMemoryFactory::Get(NAMES_LIST_SIZE)->Ptr()
);
char * names_list_ptr = static_cast<char *>(
SharedMemoryFactory::Get(NAMES_LIST)->Ptr()
char * names_list_ptr = (char *)(
shared_memory + data_layout->GetNameListOffset()
);
typename ShM<char, true>::vector names_char_list(
names_list_ptr,
names_list_size
data_layout->names_list_size
);
m_names_char_list.swap(names_char_list);
}
@ -194,6 +190,14 @@ public:
base_path
);
data_layout = (SharedDataLayout *)(
SharedMemoryFactory::Get(LAYOUT_1)->Ptr()
);
shared_memory = (char *)(
SharedMemoryFactory::Get(DATA_1)->Ptr()
);
//load data
SimpleLogger().Write() << "loading graph data";
LoadGraph();

View File

@ -21,28 +21,197 @@ or see http://www.gnu.org/licenses/agpl.txt.
#ifndef SHARED_DATA_TYPE_H_
#define SHARED_DATA_TYPE_H_
#include "BaseDataFacade.h"
#include "../../DataStructures/Coordinate.h"
#include "../../DataStructures/QueryEdge.h"
#include "../../DataStructures/StaticGraph.h"
#include "../../DataStructures/StaticRTree.h"
#include "../../DataStructures/TurnInstructions.h"
#include "../../typedefs.h"
#include <boost/integer.hpp>
typedef BaseDataFacade<QueryEdge::EdgeData>::RTreeLeaf RTreeLeaf;
typedef StaticRTree<RTreeLeaf, true>::TreeNode RTreeNode;
typedef StaticGraph<QueryEdge::EdgeData> QueryGraph;
struct SharedDataLayout {
uint64_t names_index_size;
uint64_t names_list_size;
uint64_t name_id_list_size;
uint64_t via_node_list_size;
uint64_t graph_node_list_size;
uint64_t graph_edge_list_size;
uint64_t coordinate_list_size;
uint64_t turn_instruction_list_size;
uint64_t r_search_tree_size;
unsigned checksum;
unsigned timestamp_length;
SharedDataLayout() :
names_index_size(0),
names_list_size(0),
name_id_list_size(0),
via_node_list_size(0),
graph_node_list_size(0),
graph_edge_list_size(0),
coordinate_list_size(0),
turn_instruction_list_size(0),
r_search_tree_size(0),
checksum(0),
timestamp_length(0)
{ }
uint64_t GetSizeOfLayout() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) ) +
(via_node_list_size * sizeof(NodeID) ) +
(graph_node_list_size * sizeof(QueryGraph::_StrNode)) +
(graph_edge_list_size * sizeof(QueryGraph::_StrEdge)) +
(timestamp_length * sizeof(char) ) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructions) ) +
(r_search_tree_size * sizeof(RTreeNode) ) +
sizeof(checksum);
return result;
}
uint64_t GetNameIndexOffset() const {
return 0;
}
uint64_t GetNameListOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) );
return result;
}
uint64_t GetNameIDListOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) );
return result;
}
uint64_t GetViaNodeListOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) );
return result;
}
uint64_t GetGraphNodeListOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) ) +
(via_node_list_size * sizeof(NodeID) );
return result;
}
uint64_t GetGraphEdgeListOffsett() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) ) +
(via_node_list_size * sizeof(NodeID) ) +
(graph_node_list_size * sizeof(QueryGraph::_StrNode)) ;
return result;
}
uint64_t GetTimeStampOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) ) +
(via_node_list_size * sizeof(NodeID) ) +
(graph_node_list_size * sizeof(QueryGraph::_StrNode)) +
(graph_edge_list_size * sizeof(QueryGraph::_StrEdge));
return result;
}
uint64_t GetCoordinateListOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) ) +
(via_node_list_size * sizeof(NodeID) ) +
(graph_node_list_size * sizeof(QueryGraph::_StrNode)) +
(graph_edge_list_size * sizeof(QueryGraph::_StrEdge)) +
(timestamp_length * sizeof(char) );
return result;
}
uint64_t GetTurnInstructionListOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) ) +
(via_node_list_size * sizeof(NodeID) ) +
(graph_node_list_size * sizeof(QueryGraph::_StrNode)) +
(graph_edge_list_size * sizeof(QueryGraph::_StrEdge)) +
(timestamp_length * sizeof(char) ) +
(coordinate_list_size * sizeof(FixedPointCoordinate));
return result;
}
uint64_t GetRSearchTreeOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) ) +
(via_node_list_size * sizeof(NodeID) ) +
(graph_node_list_size * sizeof(QueryGraph::_StrNode)) +
(graph_edge_list_size * sizeof(QueryGraph::_StrEdge)) +
(timestamp_length * sizeof(char) ) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructions) );
return result;
}
uint64_t GetChecksumOffset() const {
uint64_t result =
(names_index_size * sizeof(unsigned) ) +
(names_list_size * sizeof(char) ) +
(name_id_list_size * sizeof(unsigned) ) +
(via_node_list_size * sizeof(NodeID) ) +
(graph_node_list_size * sizeof(QueryGraph::_StrNode)) +
(graph_edge_list_size * sizeof(QueryGraph::_StrEdge)) +
(timestamp_length * sizeof(char) ) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructions) ) +
(r_search_tree_size * sizeof(RTreeNode) );
return result;
}
};
enum SharedDataType {
NAMES_INDEX = 0,
NAME_INDEX_SIZE,
NAMES_LIST,
NAMES_LIST_SIZE,
NAME_ID_LIST,
NAME_ID_LIST_SIZE,
VIA_NODE_LIST,
VIA_NODE_LIST_SIZE,
GRAPH_NODE_LIST,
GRAPH_NODE_LIST_SIZE,
GRAPH_EDGE_LIST,
GRAPH_EDGE_LIST_SIZE,
CHECK_SUM,
TIMESTAMP,
TIMESTAMP_SIZE,
COORDINATE_LIST,
COORDINATE_LIST_SIZE,
TURN_INSTRUCTION_LIST,
TURN_INSTRUCTION_LIST_SIZE,
R_SEARCH_TREE,
R_SEARCH_TREE_SIZE
// NAMES_INDEX = 0,
// NAME_INDEX_SIZE,
// NAMES_LIST,
// NAMES_LIST_SIZE,
// NAME_ID_LIST,
// NAME_ID_LIST_SIZE,
// VIA_NODE_LIST,
// VIA_NODE_LIST_SIZE,
// GRAPH_NODE_LIST,
// GRAPH_NODE_LIST_SIZE,
// GRAPH_EDGE_LIST,
// GRAPH_EDGE_LIST_SIZE,
// CHECK_SUM,
// TIMESTAMP,
// TIMESTAMP_SIZE,
// COORDINATE_LIST,
// COORDINATE_LIST_SIZE,
// TURN_INSTRUCTION_LIST,
// TURN_INSTRUCTION_LIST_SIZE,
// R_SEARCH_TREE,
// R_SEARCH_TREE_SIZE
LAYOUT_1,
DATA_1,
LAYOUT_2,
DATA_2,
LAYOUT_3,
DATA_3,
LAYOUT_4,
DATA_5
};
#endif /* SHARED_DATA_TYPE_H_ */