Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Also support loading core information into shared memory

Browse Source
This commit is contained in:
Patrick Niklaus 2015-08-19 11:44:49 +02:00
parent 48d1a5ec5d
commit 92956f2b45
6 changed files with 103 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
contractor/contractor.hpp
View File

@ -549,12 +549,19 @@ class Contractor
if (remaining_nodes.size() > 2)
{
// TODO: for small cores a sorted array of core ids might also work good
for (const auto& node : remaining_nodes)
{
auto orig_id = orig_node_id_from_new_node_id_map[node.id];
is_core_node[orig_id] = true;
}
}
else
{
// in this case we don't need core markers since we fully contracted
// the graph
is_core_node.clear();
}
SimpleLogger().Write() << "[core] " << remaining_nodes.size() << " nodes " << contractor_graph->GetNumberOfEdges() << " edges." << std::endl;

44
datastore.cpp
View File

@ -164,6 +164,10 @@ int main(const int argc, const char *argv[])
{
throw osrm::exception("no geometry file found");
}
if (server_paths.find("core") == server_paths.end())
{
throw osrm::exception("no core file found");
}
ServerPaths::const_iterator paths_iterator = server_paths.find("hsgrdata");
BOOST_ASSERT(server_paths.end() != paths_iterator);
@ -199,6 +203,10 @@ int main(const int argc, const char *argv[])
BOOST_ASSERT(server_paths.end() != paths_iterator);
BOOST_ASSERT(!paths_iterator->second.empty());
const boost::filesystem::path &geometries_data_path = paths_iterator->second;
paths_iterator = server_paths.find("core");
BOOST_ASSERT(server_paths.end() != paths_iterator);
BOOST_ASSERT(!paths_iterator->second.empty());
const boost::filesystem::path &core_marker_path = paths_iterator->second;
// determine segment to use
bool segment2_in_use = SharedMemory::RegionExists(LAYOUT_2);
@ -329,6 +337,13 @@ int main(const int argc, const char *argv[])
}
shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::TIMESTAMP, m_timestamp.length());
// load core marker size
boost::filesystem::ifstream core_marker_file(core_marker_path, std::ios::binary);
uint32_t number_of_core_markers = 0;
core_marker_file.read((char *)&number_of_core_markers, sizeof(uint32_t));
shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::CORE_MARKER, number_of_core_markers);
// load coordinate size
boost::filesystem::ifstream nodes_input_stream(nodes_data_path, std::ios::binary);
unsigned coordinate_list_size = 0;
@ -509,6 +524,35 @@ int main(const int argc, const char *argv[])
}
tree_node_file.close();
// load core markers
std::vector<char> unpacked_core_markers(number_of_core_markers);
core_marker_file.read((char *)unpacked_core_markers.data(), sizeof(char)*number_of_core_markers);
unsigned *core_marker_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(
shared_memory_ptr, SharedDataLayout::CORE_MARKER);
for (auto i = 0u; i < number_of_core_markers; ++i)
{
BOOST_ASSERT(unpacked_core_markers[i] == 0 || unpacked_core_markers[i] == 1);
if (unpacked_core_markers[i] == 1)
{
const unsigned bucket = i / 32;
const unsigned offset = i % 32;
const unsigned value = [&]
{
unsigned return_value = 0;
if (0 != offset)
{
return_value = core_marker_ptr[bucket];
}
return return_value;
}();
core_marker_ptr[bucket] = (value | (1 << offset));
}
}
// load the nodes of the search graph
QueryGraph::NodeArrayEntry *graph_node_list_ptr =
shared_layout_ptr->GetBlockPtr<QueryGraph::NodeArrayEntry, true>(

13
server/data_structures/internal_datafacade.hpp
View File

@ -181,6 +181,12 @@ template <class EdgeDataT> class InternalDataFacade final : public BaseDataFacad
std::vector<char> unpacked_core_markers(number_of_markers);
core_stream.read((char *)unpacked_core_markers.data(), sizeof(char)*number_of_markers);
// in this case we have nothing to do
if (number_of_markers <= 0)
{
return;
}
m_is_core_node.resize(number_of_markers);
for (auto i = 0u; i < number_of_markers; ++i)
{
@ -493,9 +499,16 @@ template <class EdgeDataT> class InternalDataFacade final : public BaseDataFacad
}
virtual bool IsCoreNode(const NodeID id) const override final
{
if (m_is_core_node.size() > 0)
{
return m_is_core_node[id];
}
else
{
return false;
}
}
virtual void GetUncompressedGeometry(const unsigned id,
std::vector<unsigned> &result_nodes) const override final

22
server/data_structures/shared_datafacade.hpp
View File

@ -194,6 +194,21 @@ template <class EdgeDataT> class SharedDataFacade final : public BaseDataFacade<
m_names_char_list.swap(names_char_list);
}
void LoadCoreInformation()
{
if (data_layout->num_entries[SharedDataLayout::CORE_MARKER] <= 0)
{
return;
}
unsigned *core_marker_ptr = data_layout->GetBlockPtr<unsigned>(
shared_memory, SharedDataLayout::CORE_MARKER);
typename ShM<bool, true>::vector is_core_node(
core_marker_ptr,
data_layout->num_entries[SharedDataLayout::CORE_MARKER]);
m_is_core_node.swap(is_core_node);
}
void LoadGeometries()
{
unsigned *geometries_compressed_ptr = data_layout->GetBlockPtr<unsigned>(
@ -269,6 +284,7 @@ template <class EdgeDataT> class SharedDataFacade final : public BaseDataFacade<
LoadTimestamp();
LoadViaNodeList();
LoadNames();
LoadCoreInformation();
data_layout->PrintInformation();
@ -450,7 +466,11 @@ template <class EdgeDataT> class SharedDataFacade final : public BaseDataFacade<
bool IsCoreNode(const NodeID id) const override final
{
//return m_is_core_node[id];
if (m_is_core_node.size() > 0)
{
return m_is_core_node.at(id);
}
return false;
}

45
server/data_structures/shared_datatype.hpp
View File

@ -62,6 +62,7 @@ struct SharedDataLayout
HSGR_CHECKSUM,
TIMESTAMP,
FILE_INDEX_PATH,
CORE_MARKER,
NUM_BLOCKS
};
@ -72,40 +73,6 @@ struct SharedDataLayout
void PrintInformation() const
{
SimpleLogger().Write(logDEBUG) << "-";
SimpleLogger().Write(logDEBUG)
<< "name_offsets_size: " << num_entries[NAME_OFFSETS];
SimpleLogger().Write(logDEBUG)
<< "name_blocks_size: " << num_entries[NAME_BLOCKS];
SimpleLogger().Write(logDEBUG)
<< "name_char_list_size: " << num_entries[NAME_CHAR_LIST];
SimpleLogger().Write(logDEBUG)
<< "name_id_list_size: " << num_entries[NAME_ID_LIST];
SimpleLogger().Write(logDEBUG)
<< "via_node_list_size: " << num_entries[VIA_NODE_LIST];
SimpleLogger().Write(logDEBUG)
<< "graph_node_list_size: " << num_entries[GRAPH_NODE_LIST];
SimpleLogger().Write(logDEBUG)
<< "graph_edge_list_size: " << num_entries[GRAPH_EDGE_LIST];
SimpleLogger().Write(logDEBUG) << "timestamp_length: " << num_entries[TIMESTAMP];
SimpleLogger().Write(logDEBUG)
<< "coordinate_list_size: " << num_entries[COORDINATE_LIST];
SimpleLogger().Write(logDEBUG)
<< "turn_instruction_list_size: " << num_entries[TURN_INSTRUCTION];
SimpleLogger().Write(logDEBUG)
<< "travel_mode_list_size: " << num_entries[TRAVEL_MODE];
SimpleLogger().Write(logDEBUG)
<< "r_search_tree_size: " << num_entries[R_SEARCH_TREE];
SimpleLogger().Write(logDEBUG)
<< "geometries_indicators: " << num_entries[GEOMETRIES_INDICATORS] << "/"
<< ((num_entries[GEOMETRIES_INDICATORS] / 8) + 1);
SimpleLogger().Write(logDEBUG)
<< "geometries_index_list_size: " << num_entries[GEOMETRIES_INDEX];
SimpleLogger().Write(logDEBUG)
<< "geometries_list_size: " << num_entries[GEOMETRIES_LIST];
SimpleLogger().Write(logDEBUG)
<< "sizeof(checksum): " << entry_size[HSGR_CHECKSUM];
SimpleLogger().Write(logDEBUG) << "NAME_OFFSETS "
<< ": " << GetBlockSize(NAME_OFFSETS);
SimpleLogger().Write(logDEBUG) << "NAME_BLOCKS "
@ -140,6 +107,8 @@ struct SharedDataLayout
<< ": " << GetBlockSize(TIMESTAMP);
SimpleLogger().Write(logDEBUG) << "FILE_INDEX_PATH "
<< ": " << GetBlockSize(FILE_INDEX_PATH);
SimpleLogger().Write(logDEBUG) << "CORE_MARKER "
<< ": " << GetBlockSize(CORE_MARKER);
}
template <typename T> inline void SetBlockSize(BlockID bid, uint64_t entries)
@ -150,11 +119,11 @@ struct SharedDataLayout
inline uint64_t GetBlockSize(BlockID bid) const
{
// special encoding
if (bid == GEOMETRIES_INDICATORS)
// special bit encoding
if (bid == GEOMETRIES_INDICATORS || bid == CORE_MARKER)
{
return (num_entries[GEOMETRIES_INDICATORS] / 32 + 1) *
entry_size[GEOMETRIES_INDICATORS];
return (num_entries[bid] / 32 + 1) *
entry_size[bid];
}
return num_entries[bid] * entry_size[bid];

10
util/datastore_options.hpp
View File

@ -69,6 +69,8 @@ bool GenerateDataStoreOptions(const int argc, const char *argv[], ServerPaths &p
".ramIndex file")(
"fileindex", boost::program_options::value<boost::filesystem::path>(&paths["fileindex"]),
".fileIndex file")(
"core", boost::program_options::value<boost::filesystem::path>(&paths["core"]),
".core file")(
"namesdata", boost::program_options::value<boost::filesystem::path>(&paths["namesdata"]),
".names file")("timestamp",
boost::program_options::value<boost::filesystem::path>(&paths["timestamp"]),
@ -130,6 +132,8 @@ bool GenerateDataStoreOptions(const int argc, const char *argv[], ServerPaths &p
!paths.find("ramindex")->second.string().empty()) ||
(paths.find("fileindex") != paths.end() &&
!paths.find("fileindex")->second.string().empty()) ||
(paths.find("core") != paths.end() &&
!paths.find("core")->second.string().empty()) ||
(paths.find("timestamp") != paths.end() &&
!paths.find("timestamp")->second.string().empty());
@ -199,6 +203,12 @@ bool GenerateDataStoreOptions(const int argc, const char *argv[], ServerPaths &p
path_iterator->second = base_string + ".fileIndex";
}
path_iterator = paths.find("core");
if (path_iterator != paths.end())
{
path_iterator->second = base_string + ".core";
}
path_iterator = paths.find("namesdata");
if (path_iterator != paths.end())
{