Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Added indexed array data type with variable and fixed group blocks

Browse Source
This commit is contained in:
Michael Krasnyk
2017-01-19 15:14:30 +01:00
committed by Patrick Niklaus
parent cedeb15ade
commit 6e1c4bfecd
15 changed files with 781 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/extractor/extraction_containers.cpp
+4 -37
View File
@@ -3,13 +3,13 @@
#include "extractor/extraction_way.hpp"
#include "util/coordinate_calculation.hpp"
#include "util/range_table.hpp"
#include "util/exception.hpp"
#include "util/exception_utils.hpp"
#include "util/fingerprint.hpp"
#include "util/io.hpp"
#include "util/log.hpp"
#include "util/name_table.hpp"
#include "util/timing_util.hpp"
#include <boost/assert.hpp>
@@ -176,43 +176,10 @@ void ExtractionContainers::WriteCharData(const std::string &file_name)
util::UnbufferedLog log;
log << "writing street name index ... ";
TIMER_START(write_index);
boost::filesystem::ofstream file_stream(file_name, std::ios::binary);
boost::filesystem::ofstream file(file_name, std::ios::binary);
// transforms in-place name offsets to name lengths
BOOST_ASSERT(!name_offsets.empty());
for (auto curr = name_offsets.begin(), next = name_offsets.begin() + 1;
next != name_offsets.end();
++curr, ++next)
{
*curr = *next - *curr;
}
// removes the total length sentinel
unsigned total_length = name_offsets.back();
name_offsets.pop_back();
// builds and writes the index
util::RangeTable<> index_range(name_offsets);
file_stream << index_range;
file_stream.write((char *)&total_length, sizeof(unsigned));
// write all chars consecutively
char write_buffer[WRITE_BLOCK_BUFFER_SIZE];
unsigned buffer_len = 0;
for (const auto c : name_char_data)
{
write_buffer[buffer_len++] = c;
if (buffer_len >= WRITE_BLOCK_BUFFER_SIZE)
{
file_stream.write(write_buffer, WRITE_BLOCK_BUFFER_SIZE);
buffer_len = 0;
}
}
file_stream.write(write_buffer, buffer_len);
const util::NameTable::IndexedData indexed_data;
indexed_data.write(file, name_offsets.begin(), name_offsets.end(), name_char_data.begin());
TIMER_STOP(write_index);
log << "ok, after " << TIMER_SEC(write_index) << "s";
src/extractor/extractor_callbacks.cpp
+12 -18
View File
@@ -269,40 +269,34 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti
const auto road_classification = parsed_way.road_classification;
const constexpr std::size_t MAX_STRING_LENGTH = 255u;
// Get the unique identifier for the street name, destination, and ref
const auto name_iterator = string_map.find(
MapKey(parsed_way.name, parsed_way.destinations, parsed_way.ref, parsed_way.pronunciation));
auto name_id = EMPTY_NAMEID;
NameID name_id = EMPTY_NAMEID;
if (string_map.end() == name_iterator)
{
const auto name_length = std::min(MAX_STRING_LENGTH, parsed_way.name.size());
const auto destinations_length =
std::min(MAX_STRING_LENGTH, parsed_way.destinations.size());
const auto pronunciation_length =
std::min(MAX_STRING_LENGTH, parsed_way.pronunciation.size());
const auto ref_length = std::min(MAX_STRING_LENGTH, parsed_way.ref.size());
// name_offsets already has an offset of a new name, take the offset index as the name id
// name_offsets has a sentinel element with the total name data size
// take the sentinels index as the name id of the new name data pack
// (name [name_id], destination [+1], pronunciation [+2], ref [+3])
name_id = external_memory.name_offsets.size() - 1;
std::copy(parsed_way.name.c_str(),
parsed_way.name.c_str() + name_length,
std::copy(parsed_way.name.begin(),
parsed_way.name.end(),
std::back_inserter(external_memory.name_char_data));
external_memory.name_offsets.push_back(external_memory.name_char_data.size());
std::copy(parsed_way.destinations.c_str(),
parsed_way.destinations.c_str() + destinations_length,
std::copy(parsed_way.destinations.begin(),
parsed_way.destinations.end(),
std::back_inserter(external_memory.name_char_data));
external_memory.name_offsets.push_back(external_memory.name_char_data.size());
std::copy(parsed_way.pronunciation.c_str(),
parsed_way.pronunciation.c_str() + pronunciation_length,
std::copy(parsed_way.pronunciation.begin(),
parsed_way.pronunciation.end(),
std::back_inserter(external_memory.name_char_data));
external_memory.name_offsets.push_back(external_memory.name_char_data.size());
std::copy(parsed_way.ref.c_str(),
parsed_way.ref.c_str() + ref_length,
std::copy(parsed_way.ref.begin(),
parsed_way.ref.end(),
std::back_inserter(external_memory.name_char_data));
external_memory.name_offsets.push_back(external_memory.name_char_data.size());
src/extractor/guidance/turn_handler.cpp
+1 -1
View File
@@ -574,7 +574,7 @@ TurnHandler::findForkCandidatesByGeometry(Intersection &intersection) const
//
//
// left left
// / \
// / \ 
// /____ right \ ______ right
// | |
// | |
src/storage/storage.cpp
+5 -36
View File
@@ -194,19 +194,10 @@ void Storage::PopulateLayout(DataLayout &layout)
}
{
// collect number of elements to store in shared memory object
util::Log() << "load names from: " << config.names_data_path;
// number of entries in name index
io::FileReader name_file(config.names_data_path, io::FileReader::HasNoFingerprint);
const auto name_blocks = name_file.ReadElementCount32();
layout.SetBlockSize<unsigned>(DataLayout::NAME_OFFSETS, name_blocks);
layout.SetBlockSize<typename util::RangeTable<16, true>::BlockT>(DataLayout::NAME_BLOCKS,
name_blocks);
BOOST_ASSERT_MSG(0 != name_blocks, "name file broken");
const auto number_of_chars = name_file.ReadElementCount32();
layout.SetBlockSize<char>(DataLayout::NAME_CHAR_LIST, number_of_chars);
layout.SetBlockSize<char>(DataLayout::NAME_CHAR_DATA, name_file.GetSize());
}
{
@@ -451,35 +442,13 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Name data
{
io::FileReader name_file(config.names_data_path, io::FileReader::HasNoFingerprint);
const auto name_blocks_count = name_file.ReadElementCount32();
name_file.Skip<std::uint32_t>(1); // name_char_list_count
BOOST_ASSERT(name_blocks_count * sizeof(unsigned) ==
layout.GetBlockSize(DataLayout::NAME_OFFSETS));
BOOST_ASSERT(name_blocks_count * sizeof(typename util::RangeTable<16, true>::BlockT) ==
layout.GetBlockSize(DataLayout::NAME_BLOCKS));
// Loading street names
const auto name_offsets_ptr =
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::NAME_OFFSETS);
name_file.ReadInto(name_offsets_ptr, name_blocks_count);
const auto name_blocks_ptr =
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::NAME_BLOCKS);
name_file.ReadInto(reinterpret_cast<char *>(name_blocks_ptr),
layout.GetBlockSize(DataLayout::NAME_BLOCKS));
// The file format contains the element count a second time. Don't know why,
// but we need to read it here to progress the file pointer to the correct spot
const auto temp_count = name_file.ReadElementCount32();
std::size_t name_file_size = name_file.GetSize();
BOOST_ASSERT(name_file_size == layout.GetBlockSize(DataLayout::NAME_CHAR_DATA));
const auto name_char_ptr =
layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::NAME_CHAR_LIST);
layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::NAME_CHAR_DATA);
BOOST_ASSERT_MSG(temp_count == layout.GetBlockSize(DataLayout::NAME_CHAR_LIST),
"Name file corrupted!");
name_file.ReadInto(name_char_ptr, temp_count);
name_file.ReadInto<char>(name_char_ptr, name_file_size);
}
// Turn lane data
src/util/name_table.cpp
+31 -34
View File
@@ -1,64 +1,58 @@
#include "util/name_table.hpp"
#include "storage/io.hpp"
#include "util/exception.hpp"
#include "util/log.hpp"
#include <algorithm>
#include <iterator>
#include <limits>
#include <boost/filesystem/fstream.hpp>
namespace osrm
{
namespace util
{
NameTable::NameTable(const std::string &filename)
NameTable::NameTable(const std::string &file_name)
{
storage::io::FileReader name_stream_file_reader(filename,
storage::io::FileReader::HasNoFingerprint);
using FileReader = storage::io::FileReader;
m_name_table.ReadARangeTable(name_stream_file_reader);
FileReader name_stream_file_reader(file_name, FileReader::HasNoFingerprint);
const auto file_size = name_stream_file_reader.GetSize();
const auto number_of_chars = name_stream_file_reader.ReadElementCount32();
m_buffer = BufferType(static_cast<ValueType *>(::operator new(file_size)),
[](void *ptr) { ::operator delete(ptr); });
name_stream_file_reader.ReadInto<char>(m_buffer.get(), file_size);
m_name_table.reset(m_buffer.get(), m_buffer.get() + file_size);
m_names_char_list.resize(number_of_chars + 1); //+1 gives sentinel element
m_names_char_list.back() = 0;
if (number_of_chars > 0)
{
name_stream_file_reader.ReadInto(&m_names_char_list[0], number_of_chars);
}
else
if (m_name_table.empty())
{
util::Log() << "list of street names is empty in construction of name table from: \""
<< filename << "\"";
<< file_name << "\"";
}
}
void NameTable::reset(ValueType *begin, ValueType *end)
{
m_buffer.reset();
m_name_table.reset(begin, end);
}
StringView NameTable::GetNameForID(const NameID id) const
{
if (std::numeric_limits<NameID>::max() == id)
{
if (id == INVALID_NAMEID)
return {};
}
auto range = m_name_table.GetRange(id);
return m_name_table.at(id);
}
if (range.begin() == range.end())
{
StringView NameTable::GetDestinationsForID(const NameID id) const
{
if (id == INVALID_NAMEID)
return {};
}
auto first = begin(m_names_char_list) + range.front();
auto last = begin(m_names_char_list) + range.back() + 1;
const std::size_t len = last - first;
return StringView{&*first, len};
return m_name_table.at(id + 1);
}
StringView NameTable::GetRefForID(const NameID id) const
{
if (id == INVALID_NAMEID)
return {};
// Way string data is stored in blocks based on `id` as follows:
//
// | name | destination | pronunciation | ref |
@@ -71,11 +65,14 @@ StringView NameTable::GetRefForID(const NameID id) const
// Offset 0 is name, 1 is destination, 2 is pronunciation, 3 is ref.
// See datafacades and extractor callbacks for details.
const constexpr auto OFFSET_REF = 3u;
return GetNameForID(id + OFFSET_REF);
return m_name_table.at(id + OFFSET_REF);
}
StringView NameTable::GetPronunciationForID(const NameID id) const
{
if (id == INVALID_NAMEID)
return {};
// Way string data is stored in blocks based on `id` as follows:
//
// | name | destination | pronunciation | ref |
@@ -88,7 +85,7 @@ StringView NameTable::GetPronunciationForID(const NameID id) const
// Offset 0 is name, 1 is destination, 2 is pronunciation, 3 is ref.
// See datafacades and extractor callbacks for details.
const constexpr auto OFFSET_PRONUNCIATION = 2u;
return GetNameForID(id + OFFSET_PRONUNCIATION);
return m_name_table.at(id + OFFSET_PRONUNCIATION);
}
} // namespace util