Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Use FileWriter for better error handling when writing name data.

Browse Source
This commit is contained in:
Daniel Patterson 2017-05-27 22:38:46 -07:00 committed by Patrick Niklaus
parent 382a5cebab
commit 7eab227ab1
2 changed files with 26 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
include/util/indexed_data.hpp
View File

@ -1,6 +1,8 @@
#ifndef OSRM_INDEXED_DATA_HPP #ifndef OSRM_INDEXED_DATA_HPP
#define OSRM_INDEXED_DATA_HPP #define OSRM_INDEXED_DATA_HPP
#include "storage/io.hpp"
#include "util/exception.hpp" #include "util/exception.hpp"
#include "util/string_view.hpp" #include "util/string_view.hpp"
@ -9,7 +11,6 @@
#include <array> #include <array>
#include <iterator> #include <iterator>
#include <limits> #include <limits>
#include <ostream>
#include <string> #include <string>
#include <type_traits> #include <type_traits>
@ -85,7 +86,7 @@ template <int N, typename T = std::string> struct VariableGroupBlock
/// prefix length. /// prefix length.
/// Returns the block prefix length. /// Returns the block prefix length.
template <typename Offset, typename OffsetIterator> template <typename Offset, typename OffsetIterator>
Offset WriteBlockReference(std::ostream &out, Offset WriteBlockReference(storage::io::FileWriter &out,
Offset data_offset, Offset data_offset,
OffsetIterator first, OffsetIterator first,
OffsetIterator last) const OffsetIterator last) const
@ -105,7 +106,7 @@ template <int N, typename T = std::string> struct VariableGroupBlock
prefix_length += byte_length; prefix_length += byte_length;
} }
out.write((const char *)&refernce, sizeof(refernce)); out.WriteOne(refernce);
return prefix_length; return prefix_length;
} }
@ -118,7 +119,8 @@ template <int N, typename T = std::string> struct VariableGroupBlock
/// [first..last] is an inclusive range of block data. /// [first..last] is an inclusive range of block data.
/// The length of the last item in the block is not stored. /// The length of the last item in the block is not stored.
template <typename OffsetIterator> template <typename OffsetIterator>
void WriteBlockPrefix(std::ostream &out, OffsetIterator first, OffsetIterator last) const void
WriteBlockPrefix(storage::io::FileWriter &out, OffsetIterator first, OffsetIterator last) const
{ {
for (OffsetIterator curr = first, next = std::next(first); curr != last; ++curr, ++next) for (OffsetIterator curr = first, next = std::next(first); curr != last; ++curr, ++next)
{ {
@ -127,7 +129,9 @@ template <int N, typename T = std::string> struct VariableGroupBlock
if (byte_length == 0) if (byte_length == 0)
continue; continue;
out.write((const char *)&data_length, byte_length); // Here, we're only writing a few bytes from the 4-byte std::uint32_t,
// so we need to cast to (char *)
out.WriteFrom((const char *)&data_length, byte_length);
} }
} }
@ -175,20 +179,23 @@ template <int N, typename T = std::string> struct FixedGroupBlock
/// Write a block reference {offset}, where offset is a global block offset /// Write a block reference {offset}, where offset is a global block offset
/// Returns the fixed block prefix length. /// Returns the fixed block prefix length.
template <typename Offset, typename OffsetIterator> template <typename Offset, typename OffsetIterator>
Offset Offset WriteBlockReference(storage::io::FileWriter &out,
WriteBlockReference(std::ostream &out, Offset data_offset, OffsetIterator, OffsetIterator) const Offset data_offset,
OffsetIterator,
OffsetIterator) const
{ {
BOOST_ASSERT(data_offset <= std::numeric_limits<decltype(BlockReference::offset)>::max()); BOOST_ASSERT(data_offset <= std::numeric_limits<decltype(BlockReference::offset)>::max());
BlockReference refernce{static_cast<decltype(BlockReference::offset)>(data_offset)}; BlockReference refernce{static_cast<decltype(BlockReference::offset)>(data_offset)};
out.write((const char *)&refernce, sizeof(refernce)); out.WriteOne(refernce);
return BLOCK_SIZE; return BLOCK_SIZE;
} }
/// Write a fixed length block prefix. /// Write a fixed length block prefix.
template <typename OffsetIterator> template <typename OffsetIterator>
void WriteBlockPrefix(std::ostream &out, OffsetIterator first, OffsetIterator last) const void
WriteBlockPrefix(storage::io::FileWriter &out, OffsetIterator first, OffsetIterator last) const
{ {
std::uint32_t index = 0; std::uint32_t index = 0;
std::array<ValueType, BLOCK_SIZE> block_prefix; std::array<ValueType, BLOCK_SIZE> block_prefix;
@ -200,7 +207,7 @@ template <int N, typename T = std::string> struct FixedGroupBlock
block_prefix[index++] = static_cast<ValueType>(data_length); block_prefix[index++] = static_cast<ValueType>(data_length);
} }
out.write((const char *)block_prefix.data(), block_prefix.size()); out.WriteFrom(block_prefix.data(), block_prefix.size());
} }
/// Advances the range to an item stored in the referenced block. /// Advances the range to an item stored in the referenced block.
@ -244,8 +251,10 @@ template <typename GroupBlock> struct IndexedData
bool empty() const { return blocks_number == 0; } bool empty() const { return blocks_number == 0; }
template <typename OffsetIterator, typename DataIterator> template <typename OffsetIterator, typename DataIterator>
void void write(storage::io::FileWriter &out,
write(std::ostream &out, OffsetIterator first, OffsetIterator last, DataIterator data) const OffsetIterator first,
OffsetIterator last,
DataIterator data) const
{ {
static_assert(sizeof(typename DataIterator::value_type) == 1, "data basic type must char"); static_assert(sizeof(typename DataIterator::value_type) == 1, "data basic type must char");
@ -259,7 +268,7 @@ template <typename GroupBlock> struct IndexedData
const BlocksNumberType number_of_blocks = const BlocksNumberType number_of_blocks =
number_of_elements == 0 ? 0 number_of_elements == 0 ? 0
: 1 + (std::distance(first, sentinel) - 1) / (BLOCK_SIZE + 1); : 1 + (std::distance(first, sentinel) - 1) / (BLOCK_SIZE + 1);
out.write((const char *)&number_of_blocks, sizeof(number_of_blocks)); out.WriteOne(number_of_blocks);
// Write block references and compute the total data size that includes prefix and data // Write block references and compute the total data size that includes prefix and data
const GroupBlock block; const GroupBlock block;
@ -273,7 +282,7 @@ template <typename GroupBlock> struct IndexedData
} }
// Write the total data size // Write the total data size
out.write((const char *)&data_size, sizeof(data_size)); out.WriteOne(data_size);
// Write data blocks that are (prefix, data) // Write data blocks that are (prefix, data)
for (OffsetIterator curr = first, next = first; next != sentinel; curr = next) for (OffsetIterator curr = first, next = first; next != sentinel; curr = next)
@ -281,7 +290,8 @@ template <typename GroupBlock> struct IndexedData
std::advance(next, std::min<diff_type>(BLOCK_SIZE, std::distance(next, sentinel))); std::advance(next, std::min<diff_type>(BLOCK_SIZE, std::distance(next, sentinel)));
block.WriteBlockPrefix(out, curr, next); block.WriteBlockPrefix(out, curr, next);
std::advance(next, std::min<diff_type>(1, std::distance(next, sentinel))); std::advance(next, std::min<diff_type>(1, std::distance(next, sentinel)));
std::copy(data + *curr, data + *next, std::ostream_iterator<unsigned char>(out)); std::for_each(
data + *curr, data + *next, [&out](const auto &element) { out.WriteOne(element); });
} }
} }

6
src/extractor/extraction_containers.cpp
View File

@ -174,11 +174,7 @@ void ExtractionContainers::WriteCharData(const std::string &file_name)
util::UnbufferedLog log; util::UnbufferedLog log;
log << "writing street name index ... "; log << "writing street name index ... ";
TIMER_START(write_index); TIMER_START(write_index);
boost::filesystem::ofstream file(file_name, std::ios::binary); storage::io::FileWriter file(file_name, storage::io::FileWriter::GenerateFingerprint);
// TODO wrap this in a FileWriter
const auto fingerprint = util::FingerPrint::GetValid();
file.write(reinterpret_cast<const char *>(&fingerprint), sizeof(util::FingerPrint));
const util::NameTable::IndexedData indexed_data; const util::NameTable::IndexedData indexed_data;
indexed_data.write(file, name_offsets.begin(), name_offsets.end(), name_char_data.begin()); indexed_data.write(file, name_offsets.begin(), name_offsets.end(), name_char_data.begin());