Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Normalize file writes

Browse Source
This commit is contained in:
Pepijn Schoen
2017-03-04 11:52:40 +01:00
committed by Patrick Niklaus
parent e705ff16e3
commit c7fc36a61b
11 changed files with 179 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/extractor/edge_based_graph_factory.hpp
+3 -1
View File
@@ -21,6 +21,8 @@
#include "util/node_based_graph.hpp"
#include "util/typedefs.hpp"
#include "storage/io.hpp"
#include <algorithm>
#include <cstddef>
#include <cstdint>
@@ -196,7 +198,7 @@ class EdgeBasedGraphFactory
boost::optional<Mapping> InsertEdgeBasedNode(const NodeID u, const NodeID v);
void FlushVectorToStream(std::ofstream &edge_data_file,
void FlushVectorToStream(storage::io::FileWriter &edge_data_file,
std::vector<OriginalEdgeData> &original_edge_data_vector) const;
std::size_t restricted_turns_counter;
include/extractor/extraction_containers.hpp
+4 -2
View File
@@ -8,6 +8,8 @@
#include "extractor/restriction.hpp"
#include "extractor/scripting_environment.hpp"
#include "storage/io.hpp"
#include <cstdint>
#include <stxxl/vector>
#include <unordered_map>
@@ -37,9 +39,9 @@ class ExtractionContainers
void PrepareRestrictions();
void PrepareEdges(ScriptingEnvironment &scripting_environment);
void WriteNodes(std::ofstream &file_out_stream) const;
void WriteNodes(storage::io::FileWriter &file_out) const;
void WriteRestrictions(const std::string &restrictions_file_name) const;
void WriteEdges(std::ofstream &file_out_stream) const;
void WriteEdges(storage::io::FileWriter &file_out) const;
void WriteCharData(const std::string &file_name);
public:
include/storage/io.hpp
+1 -1
View File
@@ -284,7 +284,7 @@ class FileWriter
}
}
template <typename T> void WriteFrom(const T &target) { WriteFrom(&target, 1); }
template <typename T> void WriteFrom(const T &src) { WriteFrom(&src, 1); }
template <typename T> void WriteOne(const T tmp) { WriteFrom(tmp); }
include/util/io.hpp
-111
View File
@@ -22,90 +22,6 @@ namespace osrm
namespace util
{
inline bool writeFingerprint(std::ostream &stream)
{
const auto fingerprint = FingerPrint::GetValid();
stream.write(reinterpret_cast<const char *>(&fingerprint), sizeof(fingerprint));
return static_cast<bool>(stream);
}
template <typename simple_type>
bool serializeVector(std::ostream &stream, const std::vector<simple_type> &data)
{
std::uint64_t count = data.size();
stream.write(reinterpret_cast<const char *>(&count), sizeof(count));
if (!data.empty())
stream.write(reinterpret_cast<const char *>(&data[0]), sizeof(simple_type) * count);
return static_cast<bool>(stream);
}
template <typename simple_type>
bool serializeVector(const std::string &filename, const std::vector<simple_type> &data)
{
std::ofstream stream(filename, std::ios::binary);
writeFingerprint(stream);
return serializeVector(stream, data);
}
// serializes a vector of vectors into an adjacency array (creates a copy of the data internally)
template <typename simple_type>
bool serializeVectorIntoAdjacencyArray(const std::string &filename,
const std::vector<std::vector<simple_type>> &data)
{
storage::io::FileWriter file(filename, storage::io::FileWriter::HasNoFingerprint);
std::vector<std::uint32_t> offsets;
offsets.reserve(data.size() + 1);
std::uint64_t current_offset = 0;
offsets.push_back(current_offset);
for (auto const &vec : data)
{
current_offset += vec.size();
offsets.push_back(boost::numeric_cast<std::uint32_t>(current_offset));
}
std::vector<simple_type> all_data;
all_data.reserve(offsets.back());
for (auto const &vec : data)
all_data.insert(all_data.end(), vec.begin(), vec.end());
file.SerializeVector(offsets);
file.SerializeVector(all_data);
return true;
}
template <typename simple_type, std::size_t WRITE_BLOCK_BUFFER_SIZE = 1024>
bool serializeVector(std::ofstream &out_stream, const stxxl::vector<simple_type> &data)
{
const std::uint64_t size = data.size();
out_stream.write(reinterpret_cast<const char *>(&size), sizeof(size));
simple_type write_buffer[WRITE_BLOCK_BUFFER_SIZE];
std::size_t buffer_len = 0;
for (const auto entry : data)
{
write_buffer[buffer_len++] = entry;
if (buffer_len >= WRITE_BLOCK_BUFFER_SIZE)
{
out_stream.write(reinterpret_cast<const char *>(write_buffer),
WRITE_BLOCK_BUFFER_SIZE * sizeof(simple_type));
buffer_len = 0;
}
}
// write remaining entries
if (buffer_len > 0)
out_stream.write(reinterpret_cast<const char *>(write_buffer),
buffer_len * sizeof(simple_type));
return static_cast<bool>(out_stream);
}
template <typename simple_type>
void deserializeAdjacencyArray(const std::string &filename,
std::vector<std::uint32_t> &offsets,
@@ -124,33 +40,6 @@ void deserializeAdjacencyArray(const std::string &filename,
SOURCE_REF);
}
inline bool serializeFlags(const boost::filesystem::path &path, const std::vector<bool> &flags)
{
// TODO this should be replaced with a FILE-based write using error checking
std::ofstream flag_stream(path.string(), std::ios::binary);
writeFingerprint(flag_stream);
std::uint32_t number_of_bits = flags.size();
flag_stream.write(reinterpret_cast<const char *>(&number_of_bits), sizeof(number_of_bits));
// putting bits in ints
std::uint32_t chunk = 0;
std::size_t chunk_count = 0;
for (std::size_t bit_nr = 0; bit_nr < number_of_bits;)
{
std::bitset<32> chunk_bitset;
for (std::size_t chunk_bit = 0; chunk_bit < 32 && bit_nr < number_of_bits;
++chunk_bit, ++bit_nr)
chunk_bitset[chunk_bit] = flags[bit_nr];
chunk = chunk_bitset.to_ulong();
++chunk_count;
flag_stream.write(reinterpret_cast<const char *>(&chunk), sizeof(chunk));
}
Log() << "Wrote " << number_of_bits << " bits in " << chunk_count << " chunks (Flags).";
return static_cast<bool>(flag_stream);
}
} // namespace util
} // namespace osrm
include/util/range_table.hpp
+13 -1
View File
@@ -139,7 +139,19 @@ template <unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY> class RangeTable
sum_lengths = lengths_prefix_sum;
}
void ReadARangeTable(osrm::storage::io::FileReader &filereader)
void Write(osrm::storage::io::FileWriter &filewriter)
{
unsigned number_of_blocks = diff_blocks.size();
filewriter.WriteElementCount32(number_of_blocks);
filewriter.WriteOne(sum_lengths);
filewriter.WriteFrom(block_offsets.data(), number_of_blocks);
filewriter.WriteFrom(diff_blocks.data(), number_of_blocks);
}
void Read(osrm::storage::io::FileReader &filereader)
{
unsigned number_of_blocks = filereader.ReadElementCount32();
// read total length
include/util/static_rtree.hpp
+5 -3
View File
@@ -331,12 +331,14 @@ class StaticRTree
});
// open tree file
boost::filesystem::ofstream tree_node_file(tree_node_filename, std::ios::binary);
storage::io::FileWriter tree_node_file(tree_node_filename,
storage::io::FileWriter::HasNoFingerprint);
std::uint64_t size_of_tree = m_search_tree.size();
BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty");
tree_node_file.write((char *)&size_of_tree, sizeof(size_of_tree));
tree_node_file.write((char *)&m_search_tree[0], sizeof(TreeNode) * size_of_tree);
tree_node_file.WriteOne(size_of_tree);
tree_node_file.WriteFrom(&m_search_tree[0], size_of_tree);
MapLeafNodesFile(leaf_node_filename);
}