Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Normalize file writes

Browse Source
This commit is contained in:
Pepijn Schoen 2017-03-04 11:52:40 +01:00 committed by Patrick Niklaus
parent e705ff16e3
commit c7fc36a61b
11 changed files with 179 additions and 338 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/extractor/edge_based_graph_factory.hpp
View File

@ -21,6 +21,8 @@
#include "util/node_based_graph.hpp" #include "util/node_based_graph.hpp"
#include "util/typedefs.hpp" #include "util/typedefs.hpp"
#include "storage/io.hpp"
#include <algorithm> #include <algorithm>
#include <cstddef> #include <cstddef>
#include <cstdint> #include <cstdint>
@ -196,7 +198,7 @@ class EdgeBasedGraphFactory
boost::optional<Mapping> InsertEdgeBasedNode(const NodeID u, const NodeID v); boost::optional<Mapping> InsertEdgeBasedNode(const NodeID u, const NodeID v);
void FlushVectorToStream(std::ofstream &edge_data_file, void FlushVectorToStream(storage::io::FileWriter &edge_data_file,
std::vector<OriginalEdgeData> &original_edge_data_vector) const; std::vector<OriginalEdgeData> &original_edge_data_vector) const;
std::size_t restricted_turns_counter; std::size_t restricted_turns_counter;

6
include/extractor/extraction_containers.hpp
View File

@ -8,6 +8,8 @@
#include "extractor/restriction.hpp" #include "extractor/restriction.hpp"
#include "extractor/scripting_environment.hpp" #include "extractor/scripting_environment.hpp"
#include "storage/io.hpp"
#include <cstdint> #include <cstdint>
#include <stxxl/vector> #include <stxxl/vector>
#include <unordered_map> #include <unordered_map>
@ -37,9 +39,9 @@ class ExtractionContainers
void PrepareRestrictions(); void PrepareRestrictions();
void PrepareEdges(ScriptingEnvironment &scripting_environment); void PrepareEdges(ScriptingEnvironment &scripting_environment);
void WriteNodes(std::ofstream &file_out_stream) const; void WriteNodes(storage::io::FileWriter &file_out) const;
void WriteRestrictions(const std::string &restrictions_file_name) const; void WriteRestrictions(const std::string &restrictions_file_name) const;
void WriteEdges(std::ofstream &file_out_stream) const; void WriteEdges(storage::io::FileWriter &file_out) const;
void WriteCharData(const std::string &file_name); void WriteCharData(const std::string &file_name);
public: public:

2
include/storage/io.hpp
View File

@ -284,7 +284,7 @@ class FileWriter
} }
} }
template <typename T> void WriteFrom(const T &target) { WriteFrom(&target, 1); } template <typename T> void WriteFrom(const T &src) { WriteFrom(&src, 1); }
template <typename T> void WriteOne(const T tmp) { WriteFrom(tmp); } template <typename T> void WriteOne(const T tmp) { WriteFrom(tmp); }

111
include/util/io.hpp
View File

@ -22,90 +22,6 @@ namespace osrm
namespace util namespace util
{ {
inline bool writeFingerprint(std::ostream &stream)
{
const auto fingerprint = FingerPrint::GetValid();
stream.write(reinterpret_cast<const char *>(&fingerprint), sizeof(fingerprint));
return static_cast<bool>(stream);
}
template <typename simple_type>
bool serializeVector(std::ostream &stream, const std::vector<simple_type> &data)
{
std::uint64_t count = data.size();
stream.write(reinterpret_cast<const char *>(&count), sizeof(count));
if (!data.empty())
stream.write(reinterpret_cast<const char *>(&data[0]), sizeof(simple_type) * count);
return static_cast<bool>(stream);
}
template <typename simple_type>
bool serializeVector(const std::string &filename, const std::vector<simple_type> &data)
{
std::ofstream stream(filename, std::ios::binary);
writeFingerprint(stream);
return serializeVector(stream, data);
}
// serializes a vector of vectors into an adjacency array (creates a copy of the data internally)
template <typename simple_type>
bool serializeVectorIntoAdjacencyArray(const std::string &filename,
const std::vector<std::vector<simple_type>> &data)
{
storage::io::FileWriter file(filename, storage::io::FileWriter::HasNoFingerprint);
std::vector<std::uint32_t> offsets;
offsets.reserve(data.size() + 1);
std::uint64_t current_offset = 0;
offsets.push_back(current_offset);
for (auto const &vec : data)
{
current_offset += vec.size();
offsets.push_back(boost::numeric_cast<std::uint32_t>(current_offset));
}
std::vector<simple_type> all_data;
all_data.reserve(offsets.back());
for (auto const &vec : data)
all_data.insert(all_data.end(), vec.begin(), vec.end());
file.SerializeVector(offsets);
file.SerializeVector(all_data);
return true;
}
template <typename simple_type, std::size_t WRITE_BLOCK_BUFFER_SIZE = 1024>
bool serializeVector(std::ofstream &out_stream, const stxxl::vector<simple_type> &data)
{
const std::uint64_t size = data.size();
out_stream.write(reinterpret_cast<const char *>(&size), sizeof(size));
simple_type write_buffer[WRITE_BLOCK_BUFFER_SIZE];
std::size_t buffer_len = 0;
for (const auto entry : data)
{
write_buffer[buffer_len++] = entry;
if (buffer_len >= WRITE_BLOCK_BUFFER_SIZE)
{
out_stream.write(reinterpret_cast<const char *>(write_buffer),
WRITE_BLOCK_BUFFER_SIZE * sizeof(simple_type));
buffer_len = 0;
}
}
// write remaining entries
if (buffer_len > 0)
out_stream.write(reinterpret_cast<const char *>(write_buffer),
buffer_len * sizeof(simple_type));
return static_cast<bool>(out_stream);
}
template <typename simple_type> template <typename simple_type>
void deserializeAdjacencyArray(const std::string &filename, void deserializeAdjacencyArray(const std::string &filename,
std::vector<std::uint32_t> &offsets, std::vector<std::uint32_t> &offsets,
@ -124,33 +40,6 @@ void deserializeAdjacencyArray(const std::string &filename,
SOURCE_REF); SOURCE_REF);
} }
inline bool serializeFlags(const boost::filesystem::path &path, const std::vector<bool> &flags)
{
// TODO this should be replaced with a FILE-based write using error checking
std::ofstream flag_stream(path.string(), std::ios::binary);
writeFingerprint(flag_stream);
std::uint32_t number_of_bits = flags.size();
flag_stream.write(reinterpret_cast<const char *>(&number_of_bits), sizeof(number_of_bits));
// putting bits in ints
std::uint32_t chunk = 0;
std::size_t chunk_count = 0;
for (std::size_t bit_nr = 0; bit_nr < number_of_bits;)
{
std::bitset<32> chunk_bitset;
for (std::size_t chunk_bit = 0; chunk_bit < 32 && bit_nr < number_of_bits;
++chunk_bit, ++bit_nr)
chunk_bitset[chunk_bit] = flags[bit_nr];
chunk = chunk_bitset.to_ulong();
++chunk_count;
flag_stream.write(reinterpret_cast<const char *>(&chunk), sizeof(chunk));
}
Log() << "Wrote " << number_of_bits << " bits in " << chunk_count << " chunks (Flags).";
return static_cast<bool>(flag_stream);
}
} // namespace util } // namespace util
} // namespace osrm } // namespace osrm

14
include/util/range_table.hpp
View File

@ -139,7 +139,19 @@ template <unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY> class RangeTable
sum_lengths = lengths_prefix_sum; sum_lengths = lengths_prefix_sum;
} }
void ReadARangeTable(osrm::storage::io::FileReader &filereader) void Write(osrm::storage::io::FileWriter &filewriter)
{
unsigned number_of_blocks = diff_blocks.size();
filewriter.WriteElementCount32(number_of_blocks);
filewriter.WriteOne(sum_lengths);
filewriter.WriteFrom(block_offsets.data(), number_of_blocks);
filewriter.WriteFrom(diff_blocks.data(), number_of_blocks);
}
void Read(osrm::storage::io::FileReader &filereader)
{ {
unsigned number_of_blocks = filereader.ReadElementCount32(); unsigned number_of_blocks = filereader.ReadElementCount32();
// read total length // read total length

8
include/util/static_rtree.hpp
View File

@ -331,12 +331,14 @@ class StaticRTree
}); });
// open tree file // open tree file
boost::filesystem::ofstream tree_node_file(tree_node_filename, std::ios::binary); storage::io::FileWriter tree_node_file(tree_node_filename,
storage::io::FileWriter::HasNoFingerprint);
std::uint64_t size_of_tree = m_search_tree.size(); std::uint64_t size_of_tree = m_search_tree.size();
BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty"); BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty");
tree_node_file.write((char *)&size_of_tree, sizeof(size_of_tree));
tree_node_file.write((char *)&m_search_tree[0], sizeof(TreeNode) * size_of_tree); tree_node_file.WriteOne(size_of_tree);
tree_node_file.WriteFrom(&m_search_tree[0], size_of_tree);
MapLeafNodesFile(leaf_node_filename); MapLeafNodesFile(leaf_node_filename);
} }

90
src/contractor/contractor.cpp
View File

@ -713,47 +713,27 @@ Contractor::LoadEdgeExpandedGraph(const ContractorConfig &config,
return; return;
// Now save out the updated compressed geometries // Now save out the updated compressed geometries
std::ofstream geometry_stream(config.geometry_path, std::ios::binary); storage::io::FileWriter geometry_file(config.geometry_path,
if (!geometry_stream) storage::io::FileWriter::HasNoFingerprint);
{
const std::string message{"Failed to open " + config.geometry_path + " for writing"};
throw util::exception(message + SOURCE_REF);
}
const unsigned number_of_indices = geometry_indices.size(); const unsigned number_of_indices = geometry_indices.size();
const unsigned number_of_compressed_geometries = geometry_node_list.size(); const unsigned number_of_compressed_geometries = geometry_node_list.size();
geometry_stream.write(reinterpret_cast<const char *>(&number_of_indices), sizeof(unsigned));
geometry_stream.write(reinterpret_cast<char *>(&(geometry_indices[0])), geometry_file.WriteOne(number_of_indices);
number_of_indices * sizeof(unsigned)); geometry_file.WriteFrom(geometry_indices.data(), number_of_indices);
geometry_stream.write(reinterpret_cast<const char *>(&number_of_compressed_geometries),
sizeof(unsigned)); geometry_file.WriteOne(number_of_compressed_geometries);
geometry_stream.write(reinterpret_cast<char *>(&(geometry_node_list[0])), geometry_file.WriteFrom(geometry_node_list.data(), number_of_compressed_geometries);
number_of_compressed_geometries * sizeof(NodeID)); geometry_file.WriteFrom(geometry_fwd_weight_list.data(), number_of_compressed_geometries);
geometry_stream.write(reinterpret_cast<char *>(&(geometry_fwd_weight_list[0])), geometry_file.WriteFrom(geometry_rev_weight_list.data(), number_of_compressed_geometries);
number_of_compressed_geometries * sizeof(EdgeWeight)); geometry_file.WriteFrom(geometry_fwd_duration_list.data(), number_of_compressed_geometries);
geometry_stream.write(reinterpret_cast<char *>(&(geometry_rev_weight_list[0])), geometry_file.WriteFrom(geometry_rev_duration_list.data(), number_of_compressed_geometries);
number_of_compressed_geometries * sizeof(EdgeWeight));
geometry_stream.write(reinterpret_cast<char *>(&(geometry_fwd_duration_list[0])),
number_of_compressed_geometries * sizeof(EdgeWeight));
geometry_stream.write(reinterpret_cast<char *>(&(geometry_rev_duration_list[0])),
number_of_compressed_geometries * sizeof(EdgeWeight));
}; };
const auto save_datasource_indexes = [&] { const auto save_datasource_indexes = [&] {
std::ofstream datasource_stream(config.datasource_indexes_path, std::ios::binary); storage::io::FileWriter datasource_file(config.datasource_indexes_path,
if (!datasource_stream) storage::io::FileWriter::HasNoFingerprint);
{ datasource_file.SerializeVector(geometry_datasource);
const std::string message{"Failed to open " + config.datasource_indexes_path +
" for writing"};
throw util::exception(message + SOURCE_REF);
}
std::uint64_t number_of_datasource_entries = geometry_datasource.size();
datasource_stream.write(reinterpret_cast<const char *>(&number_of_datasource_entries),
sizeof(number_of_datasource_entries));
if (number_of_datasource_entries > 0)
{
datasource_stream.write(reinterpret_cast<char *>(&(geometry_datasource[0])),
number_of_datasource_entries * sizeof(uint8_t));
}
}; };
const auto save_datastore_names = [&] { const auto save_datastore_names = [&] {
@ -951,11 +931,10 @@ void Contractor::WriteNodeLevels(std::vector<float> &&in_node_levels) const
{ {
std::vector<float> node_levels(std::move(in_node_levels)); std::vector<float> node_levels(std::move(in_node_levels));
boost::filesystem::ofstream order_output_stream(config.level_output_path, std::ios::binary); storage::io::FileWriter node_level_file(config.level_output_path,
storage::io::FileWriter::HasNoFingerprint);
unsigned level_size = node_levels.size(); node_level_file.SerializeVector(node_levels);
order_output_stream.write((char *)&level_size, sizeof(unsigned));
order_output_stream.write((char *)node_levels.data(), sizeof(float) * node_levels.size());
} }
void Contractor::WriteCoreNodeMarker(std::vector<bool> &&in_is_core_node) const void Contractor::WriteCoreNodeMarker(std::vector<bool> &&in_is_core_node) const
@ -967,12 +946,12 @@ void Contractor::WriteCoreNodeMarker(std::vector<bool> &&in_is_core_node) const
unpacked_bool_flags[i] = is_core_node[i] ? 1 : 0; unpacked_bool_flags[i] = is_core_node[i] ? 1 : 0;
} }
boost::filesystem::ofstream core_marker_output_stream(config.core_output_path, storage::io::FileWriter core_marker_output_file(config.core_output_path,
std::ios::binary); storage::io::FileWriter::HasNoFingerprint);
unsigned size = unpacked_bool_flags.size();
core_marker_output_stream.write((char *)&size, sizeof(unsigned)); const std::size_t count = unpacked_bool_flags.size();
core_marker_output_stream.write((char *)unpacked_bool_flags.data(), core_marker_output_file.WriteElementCount32(count);
sizeof(char) * unpacked_bool_flags.size()); core_marker_output_file.WriteFrom(unpacked_bool_flags.data(), count);
} }
std::size_t std::size_t
@ -984,9 +963,9 @@ Contractor::WriteContractedGraph(unsigned max_node_id,
const std::uint64_t contracted_edge_count = contracted_edge_list.size(); const std::uint64_t contracted_edge_count = contracted_edge_list.size();
util::Log() << "Serializing compacted graph of " << contracted_edge_count << " edges"; util::Log() << "Serializing compacted graph of " << contracted_edge_count << " edges";
const util::FingerPrint fingerprint = util::FingerPrint::GetValid(); storage::io::FileWriter hsgr_output_file(config.graph_output_path,
boost::filesystem::ofstream hsgr_output_stream(config.graph_output_path, std::ios::binary); storage::io::FileWriter::GenerateFingerprint);
hsgr_output_stream.write((char *)&fingerprint, sizeof(util::FingerPrint));
const NodeID max_used_node_id = [&contracted_edge_list] { const NodeID max_used_node_id = [&contracted_edge_list] {
NodeID tmp_max = 0; NodeID tmp_max = 0;
for (const QueryEdge &edge : contracted_edge_list) for (const QueryEdge &edge : contracted_edge_list)
@ -1038,17 +1017,15 @@ Contractor::WriteContractedGraph(unsigned max_node_id,
const std::uint64_t node_array_size = node_array.size(); const std::uint64_t node_array_size = node_array.size();
// serialize crc32, aka checksum // serialize crc32, aka checksum
hsgr_output_stream.write((char *)&edges_crc32, sizeof(unsigned)); hsgr_output_file.WriteOne(edges_crc32);
// serialize number of nodes // serialize number of nodes
hsgr_output_stream.write((char *)&node_array_size, sizeof(std::uint64_t)); hsgr_output_file.WriteOne(node_array_size);
// serialize number of edges // serialize number of edges
hsgr_output_stream.write((char *)&contracted_edge_count, sizeof(std::uint64_t)); hsgr_output_file.WriteOne(contracted_edge_count);
// serialize all nodes // serialize all nodes
if (node_array_size > 0) if (node_array_size > 0)
{ {
hsgr_output_stream.write((char *)&node_array[0], hsgr_output_file.WriteFrom(node_array.data(), node_array_size);
sizeof(util::StaticGraph<EdgeData>::NodeArrayEntry) *
node_array_size);
} }
// serialize all edges // serialize all edges
@ -1082,8 +1059,7 @@ Contractor::WriteContractedGraph(unsigned max_node_id,
throw util::exception("Edge weight is <= 0" + SOURCE_REF); throw util::exception("Edge weight is <= 0" + SOURCE_REF);
} }
#endif #endif
hsgr_output_stream.write((char *)&current_edge, hsgr_output_file.WriteOne(current_edge);
sizeof(util::StaticGraph<EdgeData>::EdgeArrayEntry));
++number_of_used_edges; ++number_of_used_edges;
} }

89
src/extractor/edge_based_graph_factory.cpp
View File

@ -5,6 +5,9 @@
#include "extractor/node_based_graph_to_edge_based_graph_mapping_writer.hpp" #include "extractor/node_based_graph_to_edge_based_graph_mapping_writer.hpp"
#include "extractor/scripting_environment.hpp" #include "extractor/scripting_environment.hpp"
#include "extractor/suffix_table.hpp" #include "extractor/suffix_table.hpp"
#include "storage/io.hpp"
#include "util/bearing.hpp" #include "util/bearing.hpp"
#include "util/coordinate.hpp" #include "util/coordinate.hpp"
#include "util/coordinate_calculation.hpp" #include "util/coordinate_calculation.hpp"
@ -20,7 +23,6 @@
#include <algorithm> #include <algorithm>
#include <cmath> #include <cmath>
#include <fstream>
#include <iomanip> #include <iomanip>
#include <limits> #include <limits>
#include <sstream> #include <sstream>
@ -178,14 +180,16 @@ EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeID nod
} }
void EdgeBasedGraphFactory::FlushVectorToStream( void EdgeBasedGraphFactory::FlushVectorToStream(
std::ofstream &edge_data_file, std::vector<OriginalEdgeData> &original_edge_data_vector) const storage::io::FileWriter &edge_data_file,
std::vector<OriginalEdgeData> &original_edge_data_vector) const
{ {
if (original_edge_data_vector.empty()) if (original_edge_data_vector.empty())
{ {
return; return;
} }
edge_data_file.write((char *)&(original_edge_data_vector[0]),
original_edge_data_vector.size() * sizeof(OriginalEdgeData)); edge_data_file.WriteFrom(original_edge_data_vector.data(), original_edge_data_vector.size());
original_edge_data_vector.clear(); original_edge_data_vector.clear();
} }
@ -340,20 +344,17 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
skipped_uturns_counter = 0; skipped_uturns_counter = 0;
skipped_barrier_turns_counter = 0; skipped_barrier_turns_counter = 0;
std::ofstream edge_data_file(original_edge_data_filename.c_str(), std::ios::binary); storage::io::FileWriter edge_data_file(original_edge_data_filename,
std::ofstream edge_segment_file; storage::io::FileWriter::HasNoFingerprint);
std::ofstream turn_penalties_index_file;
if (generate_edge_lookup) storage::io::FileWriter edge_segment_file(edge_segment_lookup_filename,
{ storage::io::FileWriter::HasNoFingerprint);
edge_segment_file.open(edge_segment_lookup_filename.c_str(), std::ios::binary);
turn_penalties_index_file.open(turn_penalties_index_filename.c_str(), std::ios::binary); storage::io::FileWriter turn_penalties_index_file(turn_penalties_index_filename,
} storage::io::FileWriter::HasNoFingerprint);
// Writes a dummy value at the front that is updated later with the total length // Writes a dummy value at the front that is updated later with the total length
const std::uint64_t length_prefix_empty_space{0}; edge_data_file.WriteElementCount64(0);
edge_data_file.write(reinterpret_cast<const char *>(&length_prefix_empty_space),
sizeof(length_prefix_empty_space));
std::vector<OriginalEdgeData> original_edge_data_vector; std::vector<OriginalEdgeData> original_edge_data_vector;
original_edge_data_vector.reserve(1024 * 1024); original_edge_data_vector.reserve(1024 * 1024);
@ -607,8 +608,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
lookup::SegmentHeaderBlock header = {node_count, first_node.node_id}; lookup::SegmentHeaderBlock header = {node_count, first_node.node_id};
edge_segment_file.write(reinterpret_cast<const char *>(&header), edge_segment_file.WriteOne(header);
sizeof(header));
for (auto target_node : node_based_edges) for (auto target_node : node_based_edges)
{ {
@ -622,8 +622,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
target_node.weight, target_node.weight,
target_node.duration}; target_node.duration};
edge_segment_file.write(reinterpret_cast<const char *>(&nodeblock), edge_segment_file.WriteOne(nodeblock);
sizeof(nodeblock));
previous = target_node.node_id; previous = target_node.node_id;
} }
@ -654,12 +653,8 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
lookup::TurnIndexBlock turn_index_block = { lookup::TurnIndexBlock turn_index_block = {
from_node.node_id, via_node.node_id, to_node.node_id}; from_node.node_id, via_node.node_id, to_node.node_id};
BOOST_ASSERT(turn_penalties_index_file.tellp() /
(sizeof(turn_index_block)) == turn_penalties_index_file.WriteOne(turn_index_block);
turn_id);
turn_penalties_index_file.write(
reinterpret_cast<const char *>(&turn_index_block),
sizeof(turn_index_block));
} }
} }
} }
@ -667,59 +662,53 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
} }
// write weight penalties per turn // write weight penalties per turn
std::ofstream turn_weight_penalties_file(turn_weight_penalties_filename.c_str(), storage::io::FileWriter turn_weight_penalties_file(turn_weight_penalties_filename,
std::ios::binary); storage::io::FileWriter::HasNoFingerprint);
lookup::TurnPenaltiesHeader turn_weight_penalties_header{turn_weight_penalties.size()}; lookup::TurnPenaltiesHeader turn_weight_penalties_header{turn_weight_penalties.size()};
turn_weight_penalties_file.write(reinterpret_cast<const char *>(&turn_weight_penalties_header), turn_weight_penalties_file.WriteOne(turn_weight_penalties_header);
sizeof(turn_weight_penalties_header)); turn_weight_penalties_file.WriteFrom(turn_weight_penalties.data(),
turn_weight_penalties_file.write(reinterpret_cast<const char *>(turn_weight_penalties.data()),
sizeof(decltype(turn_weight_penalties)::value_type) *
turn_weight_penalties.size()); turn_weight_penalties.size());
// write duration penalties per turn if we need them // write duration penalties per turn if we need them
BOOST_ASSERT(!profile_properties.fallback_to_duration || turn_duration_penalties.size() == 0); BOOST_ASSERT(!profile_properties.fallback_to_duration || turn_duration_penalties.size() == 0);
std::ofstream turn_duration_penalties_file(turn_duration_penalties_filename.c_str(),
std::ios::binary); storage::io::FileWriter turn_duration_penalties_file(turn_duration_penalties_filename,
storage::io::FileWriter::HasNoFingerprint);
lookup::TurnPenaltiesHeader turn_duration_penalties_header{turn_duration_penalties.size()}; lookup::TurnPenaltiesHeader turn_duration_penalties_header{turn_duration_penalties.size()};
turn_duration_penalties_file.write( turn_duration_penalties_file.WriteOne(turn_duration_penalties_header);
reinterpret_cast<const char *>(&turn_duration_penalties_header),
sizeof(turn_duration_penalties_header));
if (!profile_properties.fallback_to_duration) if (!profile_properties.fallback_to_duration)
{ {
BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size()); BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size());
turn_duration_penalties_file.write( turn_duration_penalties_file.WriteFrom(turn_duration_penalties.data(),
reinterpret_cast<const char *>(turn_duration_penalties.data()), turn_duration_penalties.size());
sizeof(decltype(turn_duration_penalties)::value_type) * turn_duration_penalties.size());
} }
util::Log() << "Created " << entry_class_hash.size() << " entry classes and " util::Log() << "Created " << entry_class_hash.size() << " entry classes and "
<< bearing_class_hash.size() << " Bearing Classes"; << bearing_class_hash.size() << " Bearing Classes";
util::Log() << "Writing Turn Lane Data to File..."; util::Log() << "Writing Turn Lane Data to File...";
std::ofstream turn_lane_data_file(turn_lane_data_filename.c_str(), std::ios::binary);
storage::io::FileWriter turn_lane_data_file(turn_lane_data_filename,
storage::io::FileWriter::HasNoFingerprint);
std::vector<util::guidance::LaneTupleIdPair> lane_data(lane_data_map.size()); std::vector<util::guidance::LaneTupleIdPair> lane_data(lane_data_map.size());
// extract lane data sorted by ID // extract lane data sorted by ID
for (auto itr : lane_data_map) for (auto itr : lane_data_map)
lane_data[itr.second] = itr.first; lane_data[itr.second] = itr.first;
std::uint64_t size = lane_data.size(); turn_lane_data_file.SerializeVector(lane_data);
turn_lane_data_file.write(reinterpret_cast<const char *>(&size), sizeof(size));
if (!lane_data.empty())
turn_lane_data_file.write(reinterpret_cast<const char *>(&lane_data[0]),
sizeof(util::guidance::LaneTupleIdPair) * lane_data.size());
util::Log() << "done."; util::Log() << "done.";
FlushVectorToStream(edge_data_file, original_edge_data_vector); FlushVectorToStream(edge_data_file, original_edge_data_vector);
// Finally jump back to the empty space at the beginning and write length prefix // Finally jump back to the empty space at the beginning and write length prefix
edge_data_file.seekp(std::ios::beg); edge_data_file.SkipToBeginning();
const auto length_prefix = boost::numeric_cast<std::uint64_t>(original_edges_counter); const auto length_prefix = boost::numeric_cast<std::uint64_t>(original_edges_counter);
static_assert(sizeof(length_prefix_empty_space) == sizeof(length_prefix), "type mismatch"); edge_data_file.WriteElementCount64(length_prefix);
edge_data_file.write(reinterpret_cast<const char *>(&length_prefix), sizeof(length_prefix));
util::Log() << "Generated " << m_edge_based_node_list.size() << " edge based nodes"; util::Log() << "Generated " << m_edge_based_node_list.size() << " edge based nodes";
util::Log() << "Node-based graph contains " << node_based_edge_counter << " edges"; util::Log() << "Node-based graph contains " << node_based_edge_counter << " edges";

72
src/extractor/extraction_containers.cpp
View File

@ -12,6 +12,8 @@
#include "util/name_table.hpp" #include "util/name_table.hpp"
#include "util/timing_util.hpp" #include "util/timing_util.hpp"
#include "storage/io.hpp"
#include <boost/assert.hpp> #include <boost/assert.hpp>
#include <boost/filesystem.hpp> #include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp> #include <boost/filesystem/fstream.hpp>
@ -152,17 +154,15 @@ void ExtractionContainers::PrepareData(ScriptingEnvironment &scripting_environme
const std::string &restrictions_file_name, const std::string &restrictions_file_name,
const std::string &name_file_name) const std::string &name_file_name)
{ {
std::ofstream file_out_stream; storage::io::FileWriter file_out(output_file_name,
file_out_stream.open(output_file_name.c_str(), std::ios::binary); storage::io::FileWriter::GenerateFingerprint);
const util::FingerPrint fingerprint = util::FingerPrint::GetValid();
file_out_stream.write((char *)&fingerprint, sizeof(util::FingerPrint));
FlushVectors(); FlushVectors();
PrepareNodes(); PrepareNodes();
WriteNodes(file_out_stream); WriteNodes(file_out);
PrepareEdges(scripting_environment); PrepareEdges(scripting_environment);
WriteEdges(file_out_stream); WriteEdges(file_out);
PrepareRestrictions(); PrepareRestrictions();
WriteRestrictions(restrictions_file_name); WriteRestrictions(restrictions_file_name);
@ -542,22 +542,16 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm
} }
} }
void ExtractionContainers::WriteEdges(std::ofstream &file_out_stream) const void ExtractionContainers::WriteEdges(storage::io::FileWriter &file_out) const
{ {
std::vector<NodeBasedEdge> normal_edges;
std::size_t start_position = 0; normal_edges.reserve(all_edges_list.size());
std::uint64_t used_edges_counter = 0;
std::uint32_t used_edges_counter_buffer = 0;
{ {
util::UnbufferedLog log; util::UnbufferedLog log;
log << "Writing used edges ... " << std::flush; log << "Writing used edges ... " << std::flush;
TIMER_START(write_edges); TIMER_START(write_edges);
// Traverse list of edges and nodes in parallel and set target coord // Traverse list of edges and nodes in parallel and set target coord
start_position = file_out_stream.tellp();
file_out_stream.write((char *)&used_edges_counter_buffer,
sizeof(used_edges_counter_buffer));
for (const auto &edge : all_edges_list) for (const auto &edge : all_edges_list)
{ {
if (edge.result.source == SPECIAL_NODEID || edge.result.target == SPECIAL_NODEID) if (edge.result.source == SPECIAL_NODEID || edge.result.target == SPECIAL_NODEID)
@ -567,41 +561,30 @@ void ExtractionContainers::WriteEdges(std::ofstream &file_out_stream) const
// IMPORTANT: here, we're using slicing to only write the data from the base // IMPORTANT: here, we're using slicing to only write the data from the base
// class of NodeBasedEdgeWithOSM // class of NodeBasedEdgeWithOSM
NodeBasedEdge tmp = edge.result; normal_edges.push_back(edge.result);
file_out_stream.write((char *)&tmp, sizeof(NodeBasedEdge));
used_edges_counter++;
} }
if (used_edges_counter > std::numeric_limits<unsigned>::max()) if (normal_edges.size() > std::numeric_limits<uint32_t>::max())
{ {
throw util::exception("There are too many edges, OSRM only supports 2^32" + SOURCE_REF); throw util::exception("There are too many edges, OSRM only supports 2^32" + SOURCE_REF);
} }
file_out.WriteElementCount32(normal_edges.size());
file_out.WriteFrom(normal_edges.data(), normal_edges.size());
TIMER_STOP(write_edges); TIMER_STOP(write_edges);
log << "ok, after " << TIMER_SEC(write_edges) << "s"; log << "ok, after " << TIMER_SEC(write_edges) << "s";
log << "Processed " << normal_edges.size() << " edges";
} }
{
util::UnbufferedLog log;
log << "setting number of edges ... " << std::flush;
used_edges_counter_buffer = boost::numeric_cast<std::uint32_t>(used_edges_counter);
file_out_stream.seekp(start_position);
file_out_stream.write((char *)&used_edges_counter_buffer,
sizeof(used_edges_counter_buffer));
log << "ok";
}
util::Log() << "Processed " << used_edges_counter << " edges";
} }
void ExtractionContainers::WriteNodes(std::ofstream &file_out_stream) const void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const
{ {
{ {
// write dummy value, will be overwritten later // write dummy value, will be overwritten later
util::UnbufferedLog log; util::UnbufferedLog log;
log << "setting number of nodes ... " << std::flush; log << "setting number of nodes ... " << std::flush;
file_out_stream.write((char *)&max_internal_node_id, sizeof(unsigned)); file_out.WriteElementCount32(max_internal_node_id);
log << "ok"; log << "ok";
} }
@ -629,7 +612,7 @@ void ExtractionContainers::WriteNodes(std::ofstream &file_out_stream) const
} }
BOOST_ASSERT(*node_id_iterator == node_iterator->node_id); BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode)); file_out.WriteOne((*node_iterator));
++node_id_iterator; ++node_id_iterator;
++node_iterator; ++node_iterator;
@ -644,13 +627,11 @@ void ExtractionContainers::WriteNodes(std::ofstream &file_out_stream) const
void ExtractionContainers::WriteRestrictions(const std::string &path) const void ExtractionContainers::WriteRestrictions(const std::string &path) const
{ {
// serialize restrictions // serialize restrictions
std::ofstream restrictions_out_stream;
unsigned written_restriction_count = 0; unsigned written_restriction_count = 0;
restrictions_out_stream.open(path.c_str(), std::ios::binary); storage::io::FileWriter restrictions_out_file(path,
const util::FingerPrint fingerprint = util::FingerPrint::GetValid(); storage::io::FileWriter::GenerateFingerprint);
restrictions_out_stream.write((char *)&fingerprint, sizeof(util::FingerPrint));
const auto count_position = restrictions_out_stream.tellp(); restrictions_out_file.WriteElementCount32(written_restriction_count);
restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned));
for (const auto &restriction_container : restrictions_list) for (const auto &restriction_container : restrictions_list)
{ {
@ -658,13 +639,12 @@ void ExtractionContainers::WriteRestrictions(const std::string &path) const
SPECIAL_NODEID != restriction_container.restriction.via.node && SPECIAL_NODEID != restriction_container.restriction.via.node &&
SPECIAL_NODEID != restriction_container.restriction.to.node) SPECIAL_NODEID != restriction_container.restriction.to.node)
{ {
restrictions_out_stream.write((char *)&(restriction_container.restriction), restrictions_out_file.WriteOne(restriction_container.restriction);
sizeof(TurnRestriction));
++written_restriction_count; ++written_restriction_count;
} }
} }
restrictions_out_stream.seekp(count_position); restrictions_out_file.SkipToBeginning();
restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned)); restrictions_out_file.WriteElementCount32(written_restriction_count);
util::Log() << "usable restrictions: " << written_restriction_count; util::Log() << "usable restrictions: " << written_restriction_count;
} }

87
src/extractor/extractor.cpp
View File

@ -46,7 +46,6 @@
#include <atomic> #include <atomic>
#include <bitset> #include <bitset>
#include <chrono> #include <chrono>
#include <fstream>
#include <future> #include <future>
#include <iostream> #include <iostream>
#include <iterator> #include <iterator>
@ -167,8 +166,10 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
} }
util::Log() << "timestamp: " << timestamp; util::Log() << "timestamp: " << timestamp;
boost::filesystem::ofstream timestamp_out(config.timestamp_file_name); storage::io::FileWriter timestamp_file(config.timestamp_file_name,
timestamp_out.write(timestamp.c_str(), timestamp.length()); storage::io::FileWriter::HasNoFingerprint);
timestamp_file.WriteFrom(timestamp.c_str(), timestamp.length());
// initialize vectors holding parsed objects // initialize vectors holding parsed objects
tbb::concurrent_vector<std::pair<std::size_t, ExtractionNode>> resulting_nodes; tbb::concurrent_vector<std::pair<std::size_t, ExtractionNode>> resulting_nodes;
@ -277,7 +278,11 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
util::Log() << "Saving edge-based node weights to file."; util::Log() << "Saving edge-based node weights to file.";
TIMER_START(timer_write_node_weights); TIMER_START(timer_write_node_weights);
util::serializeVector(config.edge_based_node_weights_output_path, edge_based_node_weights);
storage::io::FileWriter edge_file(config.edge_based_node_weights_output_path,
storage::io::FileWriter::GenerateFingerprint);
edge_file.SerializeVector(edge_based_node_weights);
TIMER_STOP(timer_write_node_weights); TIMER_STOP(timer_write_node_weights);
util::Log() << "Done writing. (" << TIMER_SEC(timer_write_node_weights) << ")"; util::Log() << "Done writing. (" << TIMER_SEC(timer_write_node_weights) << ")";
@ -314,13 +319,9 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
void Extractor::WriteProfileProperties(const std::string &output_path, void Extractor::WriteProfileProperties(const std::string &output_path,
const ProfileProperties &properties) const const ProfileProperties &properties) const
{ {
boost::filesystem::ofstream out_stream(output_path); storage::io::FileWriter file(output_path, storage::io::FileWriter::HasNoFingerprint);
if (!out_stream)
{
throw util::exception("Could not open " + output_path + " for writing." + SOURCE_REF);
}
out_stream.write(reinterpret_cast<const char *>(&properties), sizeof(properties)); file.WriteOne(properties);
} }
void Extractor::FindComponents(unsigned max_edge_id, void Extractor::FindComponents(unsigned max_edge_id,
@ -535,14 +536,10 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment,
*/ */
void Extractor::WriteNodeMapping(const std::vector<QueryNode> &internal_to_external_node_map) void Extractor::WriteNodeMapping(const std::vector<QueryNode> &internal_to_external_node_map)
{ {
boost::filesystem::ofstream node_stream(config.node_output_path, std::ios::binary); storage::io::FileWriter node_file(config.node_output_path,
const std::uint64_t size_of_mapping = internal_to_external_node_map.size(); storage::io::FileWriter::HasNoFingerprint);
node_stream.write((char *)&size_of_mapping, sizeof(std::uint64_t));
if (size_of_mapping > 0) node_file.SerializeVector(internal_to_external_node_map);
{
node_stream.write((char *)internal_to_external_node_map.data(),
size_of_mapping * sizeof(QueryNode));
}
} }
/** /**
@ -597,22 +594,19 @@ void Extractor::WriteEdgeBasedGraph(
EdgeID const max_edge_id, EdgeID const max_edge_id,
util::DeallocatingVector<EdgeBasedEdge> const &edge_based_edge_list) util::DeallocatingVector<EdgeBasedEdge> const &edge_based_edge_list)
{ {
storage::io::FileWriter file(output_file_filename,
std::ofstream file_out_stream; storage::io::FileWriter::GenerateFingerprint);
file_out_stream.open(output_file_filename.c_str(), std::ios::binary);
const util::FingerPrint fingerprint = util::FingerPrint::GetValid();
file_out_stream.write((char *)&fingerprint, sizeof(util::FingerPrint));
util::Log() << "Writing edge-based-graph edges ... " << std::flush; util::Log() << "Writing edge-based-graph edges ... " << std::flush;
TIMER_START(write_edges); TIMER_START(write_edges);
std::uint64_t number_of_used_edges = edge_based_edge_list.size(); std::uint64_t number_of_used_edges = edge_based_edge_list.size();
file_out_stream.write((char *)&number_of_used_edges, sizeof(number_of_used_edges)); file.WriteElementCount64(number_of_used_edges);
file_out_stream.write((char *)&max_edge_id, sizeof(max_edge_id)); file.WriteOne(max_edge_id);
for (const auto &edge : edge_based_edge_list) for (const auto &edge : edge_based_edge_list)
{ {
file_out_stream.write((char *)&edge, sizeof(EdgeBasedEdge)); file.WriteOne(edge);
} }
TIMER_STOP(write_edges); TIMER_STOP(write_edges);
@ -627,17 +621,11 @@ void Extractor::WriteIntersectionClassificationData(
const std::vector<util::guidance::BearingClass> &bearing_classes, const std::vector<util::guidance::BearingClass> &bearing_classes,
const std::vector<util::guidance::EntryClass> &entry_classes) const const std::vector<util::guidance::EntryClass> &entry_classes) const
{ {
std::ofstream file_out_stream(output_file_name.c_str(), std::ios::binary); storage::io::FileWriter file(output_file_name, storage::io::FileWriter::GenerateFingerprint);
if (!file_out_stream)
{
util::Log(logERROR) << "Failed to open " << output_file_name << " for writing";
return;
}
util::Log() << "Writing Intersection Classification Data"; util::Log() << "Writing Intersection Classification Data";
TIMER_START(write_edges); TIMER_START(write_edges);
util::writeFingerprint(file_out_stream); file.SerializeVector(node_based_intersection_classes);
util::serializeVector(file_out_stream, node_based_intersection_classes);
// create range table for vectors: // create range table for vectors:
std::vector<unsigned> bearing_counts; std::vector<unsigned> bearing_counts;
@ -651,22 +639,18 @@ void Extractor::WriteIntersectionClassificationData(
} }
util::RangeTable<> bearing_class_range_table(bearing_counts); util::RangeTable<> bearing_class_range_table(bearing_counts);
file_out_stream << bearing_class_range_table; bearing_class_range_table.Write(file);
file.WriteOne(total_bearings);
file_out_stream.write(reinterpret_cast<const char *>(&total_bearings), sizeof(total_bearings));
for (const auto &bearing_class : bearing_classes) for (const auto &bearing_class : bearing_classes)
{ {
const auto &bearings = bearing_class.getAvailableBearings(); const auto &bearings = bearing_class.getAvailableBearings();
file_out_stream.write(reinterpret_cast<const char *>(&bearings[0]), file.WriteFrom(bearings.data(), bearings.size());
sizeof(bearings[0]) * bearings.size());
} }
if (!static_cast<bool>(file_out_stream)) file.SerializeVector(entry_classes);
{
throw util::exception("Failed to write to " + output_file_name + "." + SOURCE_REF);
}
util::serializeVector(file_out_stream, entry_classes);
TIMER_STOP(write_edges); TIMER_STOP(write_edges);
util::Log() << "ok, after " << TIMER_SEC(write_edges) << "s for " util::Log() << "ok, after " << TIMER_SEC(write_edges) << "s for "
<< node_based_intersection_classes.size() << " Indices into " << node_based_intersection_classes.size() << " Indices into "
@ -684,20 +668,9 @@ void Extractor::WriteTurnLaneData(const std::string &turn_lane_file) const
util::Log() << "Writing turn lane masks..."; util::Log() << "Writing turn lane masks...";
TIMER_START(turn_lane_timer); TIMER_START(turn_lane_timer);
std::ofstream ofs(turn_lane_file, std::ios::binary); storage::io::FileWriter file(turn_lane_file, storage::io::FileWriter::HasNoFingerprint);
if (!ofs) file.SerializeVector(turn_lane_offsets);
throw osrm::util::exception("Failed to open " + turn_lane_file + " for writing." + file.SerializeVector(turn_lane_masks);
SOURCE_REF);
if (!util::serializeVector(ofs, turn_lane_offsets))
{
throw util::exception("Error while writing to " + turn_lane_file + SOURCE_REF);
}
if (!util::serializeVector(ofs, turn_lane_masks))
{
throw util::exception("Error while writing to " + turn_lane_file + SOURCE_REF);
}
TIMER_STOP(turn_lane_timer); TIMER_STOP(turn_lane_timer);
util::Log() << "done (" << TIMER_SEC(turn_lane_timer) << ")"; util::Log() << "done (" << TIMER_SEC(turn_lane_timer) << ")";

34
unit_tests/util/io.cpp
View File

@ -1,6 +1,6 @@
#include "util/io.hpp"
#include "storage/io.hpp" #include "storage/io.hpp"
#include "util/exception.hpp" #include "util/exception.hpp"
#include "util/fingerprint.hpp"
#include "util/typedefs.hpp" #include "util/typedefs.hpp"
#include "util/version.hpp" #include "util/version.hpp"
@ -26,7 +26,11 @@ BOOST_AUTO_TEST_CASE(io_data)
std::vector<int> data_in(53), data_out; std::vector<int> data_in(53), data_out;
std::iota(begin(data_in), end(data_in), 0); std::iota(begin(data_in), end(data_in), 0);
osrm::util::serializeVector(IO_TMP_FILE, data_in); {
osrm::storage::io::FileWriter outfile(IO_TMP_FILE,
osrm::storage::io::FileWriter::GenerateFingerprint);
outfile.SerializeVector(data_in);
}
osrm::storage::io::FileReader infile(IO_TMP_FILE, osrm::storage::io::FileReader infile(IO_TMP_FILE,
osrm::storage::io::FileReader::VerifyFingerprint); osrm::storage::io::FileReader::VerifyFingerprint);
@ -58,7 +62,11 @@ BOOST_AUTO_TEST_CASE(file_too_small)
std::vector<int> v(53); std::vector<int> v(53);
std::iota(begin(v), end(v), 0); std::iota(begin(v), end(v), 0);
osrm::util::serializeVector(IO_TOO_SMALL_FILE, v); {
osrm::storage::io::FileWriter outfile(
IO_TOO_SMALL_FILE, osrm::storage::io::FileWriter::GenerateFingerprint);
outfile.SerializeVector(v);
}
std::fstream f(IO_TOO_SMALL_FILE); std::fstream f(IO_TOO_SMALL_FILE);
f.seekp(sizeof(osrm::util::FingerPrint), std::ios_base::beg); f.seekp(sizeof(osrm::util::FingerPrint), std::ios_base::beg);
@ -88,12 +96,12 @@ BOOST_AUTO_TEST_CASE(io_corrupt_fingerprint)
{ {
std::vector<int> v(153); std::vector<int> v(153);
std::iota(begin(v), end(v), 0); std::iota(begin(v), end(v), 0);
osrm::util::serializeVector(IO_CORRUPT_FINGERPRINT_FILE, v);
std::fstream f(IO_CORRUPT_FINGERPRINT_FILE); osrm::storage::io::FileWriter outfile(IO_CORRUPT_FINGERPRINT_FILE,
f.seekp(0, std::ios_base::beg); osrm::storage::io::FileWriter::HasNoFingerprint);
std::uint64_t garbage = 0xDEADBEEFCAFEFACE;
f.write(reinterpret_cast<char *>(&garbage), sizeof(garbage)); outfile.WriteOne(0xDEADBEEFCAFEFACE);
outfile.SerializeVector(v);
} }
try try
@ -115,7 +123,15 @@ BOOST_AUTO_TEST_CASE(io_incompatible_fingerprint)
{ {
std::vector<int> v(153); std::vector<int> v(153);
std::iota(begin(v), end(v), 0); std::iota(begin(v), end(v), 0);
osrm::util::serializeVector(IO_INCOMPATIBLE_FINGERPRINT_FILE, v);
{
osrm::storage::io::FileWriter outfile(IO_INCOMPATIBLE_FINGERPRINT_FILE,
osrm::storage::io::FileWriter::HasNoFingerprint);
const auto fingerprint = osrm::util::FingerPrint::GetValid();
outfile.WriteOne(fingerprint);
outfile.SerializeVector(v);
}
std::fstream f(IO_INCOMPATIBLE_FINGERPRINT_FILE); std::fstream f(IO_INCOMPATIBLE_FINGERPRINT_FILE);
f.seekp(5, std::ios_base::beg); // Seek past `OSRN` and Major version byte f.seekp(5, std::ios_base::beg); // Seek past `OSRN` and Major version byte