Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

refactor loading code of .datasource_indexes and .datasource_names files and move to io.hpp

Browse Source
This commit is contained in:
Huyen Chau Nguyen 2016-10-19 18:52:57 -07:00
parent 7b97e1035f
commit 69a60686dc
3 changed files with 89 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
include/engine/datafacade/internal_datafacade.hpp
View File

@ -292,14 +292,12 @@ class InternalDataFacade final : public BaseDataFacade
}
BOOST_ASSERT(datasources_stream);
std::uint64_t number_of_datasources = 0;
datasources_stream.read(reinterpret_cast<char *>(&number_of_datasources),
sizeof(number_of_datasources));
auto number_of_datasources = storage::io::readDatasourceIndexesSize(datasources_stream);
if (number_of_datasources > 0)
{
m_datasource_list.resize(number_of_datasources);
datasources_stream.read(reinterpret_cast<char *>(&(m_datasource_list[0])),
number_of_datasources * sizeof(uint8_t));
storage::io::readDatasourceIndexes(
datasources_stream, &m_datasource_list.front(), number_of_datasources);
}
boost::filesystem::ifstream datasourcenames_stream(datasource_names_file, std::ios::binary);
@ -309,10 +307,16 @@ class InternalDataFacade final : public BaseDataFacade
" for reading!");
}
BOOST_ASSERT(datasourcenames_stream);
std::string name;
while (std::getline(datasourcenames_stream, name))
auto datasource_names_data = storage::io::readDatasourceNamesData(datasourcenames_stream);
m_datasource_names.resize(datasource_names_data.lengths.size());
for (std::uint32_t i = 0; i < datasource_names_data.lengths.size(); ++i)
{
m_datasource_names.push_back(std::move(name));
auto name_begin =
datasource_names_data.names.begin() + datasource_names_data.offsets[i];
auto name_end = datasource_names_data.names.begin() + datasource_names_data.offsets[i] +
datasource_names_data.lengths[i];
m_datasource_names[i] = std::move(std::string(name_begin, name_end));
}
}

51
include/storage/io.hpp
View File

@ -98,6 +98,7 @@ inline std::uint32_t readEdgesSize(boost::filesystem::ifstream &edges_input_stre
// Reads edge data from .edge files which includes its
// geometry, name ID, turn instruction, lane data ID, travel mode, entry class ID
// Needs to be called after readEdgesSize() to get the correct offset in the stream
template <typename GeometryIDT,
typename NameIDT,
typename TurnInstructionT,
@ -141,6 +142,7 @@ inline std::uint32_t readNodesSize(boost::filesystem::ifstream &nodes_input_stre
}
// Reads coordinates and OSM node IDs from .nodes files
// Needs to be called after readNodesSize() to get the correct offset in the stream
template <typename CoordinateT, typename OSMNodeIDVectorT>
void readNodesData(boost::filesystem::ifstream &nodes_input_stream,
CoordinateT coordinate_list[],
@ -156,6 +158,55 @@ void readNodesData(boost::filesystem::ifstream &nodes_input_stream,
BOOST_ASSERT(coordinate_list[i].IsValid());
}
}
// Returns the number of indexes in a .datasource_indexes file
// TODO: The original code used uint_64t to store the number of indexes. Can someone confirm that
// this makes sense?
inline std::uint64_t
readDatasourceIndexesSize(boost::filesystem::ifstream &datasource_indexes_input_stream)
{
std::uint64_t number_of_datasource_indexes;
datasource_indexes_input_stream.read(reinterpret_cast<char *>(&number_of_datasource_indexes),
sizeof(std::uint64_t));
return number_of_datasource_indexes;
}
// Reads datasource_indexes
// Needs to be called after readDatasourceIndexesSize() to get the correct offset in the stream
inline void readDatasourceIndexes(boost::filesystem::ifstream &datasource_indexes_input_stream,
uint8_t datasource_buffer[],
std::uint32_t number_of_datasource_indexes)
{
datasource_indexes_input_stream.read(reinterpret_cast<char *>(datasource_buffer),
number_of_datasource_indexes * sizeof(std::uint8_t));
}
// Reads datasource names out of .datasource_names files and metadata such as
// the length and offset of each name
struct DatasourceNamesData
{
std::vector<char> names;
std::vector<std::uint32_t> offsets;
std::vector<std::uint32_t> lengths;
};
inline DatasourceNamesData
readDatasourceNamesData(boost::filesystem::ifstream &datasource_names_input_stream)
{
DatasourceNamesData datasource_names_data;
if (datasource_names_input_stream)
{
std::string name;
while (std::getline(datasource_names_input_stream, name))
{
datasource_names_data.offsets.push_back(datasource_names_data.names.size());
datasource_names_data.lengths.push_back(name.size());
std::copy(name.c_str(),
name.c_str() + name.size(),
std::back_inserter(datasource_names_data.names));
}
}
return datasource_names_data;
}
}
}
}

59
src/storage/storage.cpp
View File

@ -1,3 +1,4 @@
#include "storage/storage.hpp"
#include "contractor/query_edge.hpp"
#include "extractor/compressed_edge_container.hpp"
#include "extractor/guidance/turn_instruction.hpp"
@ -9,7 +10,6 @@
#include "storage/shared_barriers.hpp"
#include "storage/shared_datatype.hpp"
#include "storage/shared_memory.hpp"
#include "storage/storage.hpp"
#include "engine/datafacade/datafacade_base.hpp"
#include "util/coordinate.hpp"
#include "util/exception.hpp"
@ -346,15 +346,15 @@ Storage::ReturnCode Storage::Run(int max_wait)
std::uint64_t number_of_compressed_datasources = 0;
if (geometry_datasource_input_stream)
{
geometry_datasource_input_stream.read(
reinterpret_cast<char *>(&number_of_compressed_datasources),
sizeof(number_of_compressed_datasources));
number_of_compressed_datasources =
io::readDatasourceIndexesSize(geometry_datasource_input_stream);
}
shared_layout_ptr->SetBlockSize<uint8_t>(SharedDataLayout::DATASOURCES_LIST,
number_of_compressed_datasources);
// Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't
// exist
boost::filesystem::ifstream datasource_names_input_stream(config.datasource_names_path,
std::ios::binary);
if (!datasource_names_input_stream)
@ -362,30 +362,22 @@ Storage::ReturnCode Storage::Run(int max_wait)
throw util::exception("Could not open " + config.datasource_names_path.string() +
" for reading.");
}
std::vector<char> m_datasource_name_data;
std::vector<std::size_t> m_datasource_name_offsets;
std::vector<std::size_t> m_datasource_name_lengths;
io::DatasourceNamesData datasource_names_data;
if (datasource_names_input_stream)
{
std::string name;
while (std::getline(datasource_names_input_stream, name))
{
m_datasource_name_offsets.push_back(m_datasource_name_data.size());
std::copy(name.c_str(),
name.c_str() + name.size(),
std::back_inserter(m_datasource_name_data));
m_datasource_name_lengths.push_back(name.size());
}
datasource_names_data = io::readDatasourceNamesData(datasource_names_input_stream);
}
shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::DATASOURCE_NAME_DATA,
m_datasource_name_data.size());
shared_layout_ptr->SetBlockSize<std::size_t>(SharedDataLayout::DATASOURCE_NAME_OFFSETS,
m_datasource_name_offsets.size());
shared_layout_ptr->SetBlockSize<std::size_t>(SharedDataLayout::DATASOURCE_NAME_LENGTHS,
m_datasource_name_lengths.size());
datasource_names_data.names.size());
shared_layout_ptr->SetBlockSize<std::uint32_t>(SharedDataLayout::DATASOURCE_NAME_OFFSETS,
datasource_names_data.offsets.size());
shared_layout_ptr->SetBlockSize<std::uint32_t>(SharedDataLayout::DATASOURCE_NAME_LENGTHS,
datasource_names_data.lengths.size());
boost::filesystem::ifstream intersection_stream(config.intersection_class_path,
std::ios::binary);
if (!static_cast<bool>(intersection_stream))
throw util::exception("Could not open " + config.intersection_class_path.string() +
" for reading.");
@ -643,35 +635,37 @@ Storage::ReturnCode Storage::Run(int max_wait)
shared_memory_ptr, SharedDataLayout::DATASOURCES_LIST);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCES_LIST) > 0)
{
geometry_datasource_input_stream.read(
reinterpret_cast<char *>(datasources_list_ptr),
shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCES_LIST));
io::readDatasourceIndexes(geometry_datasource_input_stream,
datasources_list_ptr,
number_of_compressed_datasources);
}
// load datasource name information (if it exists)
char *datasource_name_data_ptr = shared_layout_ptr->GetBlockPtr<char, true>(
shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_DATA);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_DATA) > 0)
{
std::copy(
m_datasource_name_data.begin(), m_datasource_name_data.end(), datasource_name_data_ptr);
std::copy(datasource_names_data.names.begin(),
datasource_names_data.names.end(),
datasource_name_data_ptr);
}
auto datasource_name_offsets_ptr = shared_layout_ptr->GetBlockPtr<std::size_t, true>(
auto datasource_name_offsets_ptr = shared_layout_ptr->GetBlockPtr<std::uint32_t, true>(
shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_OFFSETS);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_OFFSETS) > 0)
{
std::copy(m_datasource_name_offsets.begin(),
m_datasource_name_offsets.end(),
std::copy(datasource_names_data.offsets.begin(),
datasource_names_data.offsets.end(),
datasource_name_offsets_ptr);
}
auto datasource_name_lengths_ptr = shared_layout_ptr->GetBlockPtr<std::size_t, true>(
auto datasource_name_lengths_ptr = shared_layout_ptr->GetBlockPtr<std::uint32_t, true>(
shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_LENGTHS);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_LENGTHS) > 0)
{
std::copy(m_datasource_name_lengths.begin(),
m_datasource_name_lengths.end(),
std::copy(datasource_names_data.lengths.begin(),
datasource_names_data.lengths.end(),
datasource_name_lengths_ptr);
}
@ -803,7 +797,6 @@ Storage::ReturnCode Storage::Run(int max_wait)
static_cast<SharedDataTimestamp *>(data_type_memory->Ptr());
{
boost::interprocess::scoped_lock<boost::interprocess::named_upgradable_mutex>
current_regions_exclusive_lock;