Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Normalize file writes

Browse Source
This commit is contained in:
Pepijn Schoen 2017-03-04 11:52:40 +01:00 committed by Patrick Niklaus
parent e705ff16e3
commit c7fc36a61b
11 changed files with 179 additions and 338 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/extractor/edge_based_graph_factory.hpp
View File

@ -21,6 +21,8 @@
#include "util/node_based_graph.hpp"
#include "util/typedefs.hpp"
#include "storage/io.hpp"
#include <algorithm>
#include <cstddef>
#include <cstdint>
@ -196,7 +198,7 @@ class EdgeBasedGraphFactory
boost::optional<Mapping> InsertEdgeBasedNode(const NodeID u, const NodeID v);
void FlushVectorToStream(std::ofstream &edge_data_file,
void FlushVectorToStream(storage::io::FileWriter &edge_data_file,
std::vector<OriginalEdgeData> &original_edge_data_vector) const;
std::size_t restricted_turns_counter;

6
include/extractor/extraction_containers.hpp
View File

@ -8,6 +8,8 @@
#include "extractor/restriction.hpp"
#include "extractor/scripting_environment.hpp"
#include "storage/io.hpp"
#include <cstdint>
#include <stxxl/vector>
#include <unordered_map>
@ -37,9 +39,9 @@ class ExtractionContainers
void PrepareRestrictions();
void PrepareEdges(ScriptingEnvironment &scripting_environment);
void WriteNodes(std::ofstream &file_out_stream) const;
void WriteNodes(storage::io::FileWriter &file_out) const;
void WriteRestrictions(const std::string &restrictions_file_name) const;
void WriteEdges(std::ofstream &file_out_stream) const;
void WriteEdges(storage::io::FileWriter &file_out) const;
void WriteCharData(const std::string &file_name);
public:

2
include/storage/io.hpp
View File

@ -284,7 +284,7 @@ class FileWriter
}
}
template <typename T> void WriteFrom(const T &target) { WriteFrom(&target, 1); }
template <typename T> void WriteFrom(const T &src) { WriteFrom(&src, 1); }
template <typename T> void WriteOne(const T tmp) { WriteFrom(tmp); }

111
include/util/io.hpp
View File

@ -22,90 +22,6 @@ namespace osrm
namespace util
{
inline bool writeFingerprint(std::ostream &stream)
{
const auto fingerprint = FingerPrint::GetValid();
stream.write(reinterpret_cast<const char *>(&fingerprint), sizeof(fingerprint));
return static_cast<bool>(stream);
}
template <typename simple_type>
bool serializeVector(std::ostream &stream, const std::vector<simple_type> &data)
{
std::uint64_t count = data.size();
stream.write(reinterpret_cast<const char *>(&count), sizeof(count));
if (!data.empty())
stream.write(reinterpret_cast<const char *>(&data[0]), sizeof(simple_type) * count);
return static_cast<bool>(stream);
}
template <typename simple_type>
bool serializeVector(const std::string &filename, const std::vector<simple_type> &data)
{
std::ofstream stream(filename, std::ios::binary);
writeFingerprint(stream);
return serializeVector(stream, data);
}
// serializes a vector of vectors into an adjacency array (creates a copy of the data internally)
template <typename simple_type>
bool serializeVectorIntoAdjacencyArray(const std::string &filename,
const std::vector<std::vector<simple_type>> &data)
{
storage::io::FileWriter file(filename, storage::io::FileWriter::HasNoFingerprint);
std::vector<std::uint32_t> offsets;
offsets.reserve(data.size() + 1);
std::uint64_t current_offset = 0;
offsets.push_back(current_offset);
for (auto const &vec : data)
{
current_offset += vec.size();
offsets.push_back(boost::numeric_cast<std::uint32_t>(current_offset));
}
std::vector<simple_type> all_data;
all_data.reserve(offsets.back());
for (auto const &vec : data)
all_data.insert(all_data.end(), vec.begin(), vec.end());
file.SerializeVector(offsets);
file.SerializeVector(all_data);
return true;
}
template <typename simple_type, std::size_t WRITE_BLOCK_BUFFER_SIZE = 1024>
bool serializeVector(std::ofstream &out_stream, const stxxl::vector<simple_type> &data)
{
const std::uint64_t size = data.size();
out_stream.write(reinterpret_cast<const char *>(&size), sizeof(size));
simple_type write_buffer[WRITE_BLOCK_BUFFER_SIZE];
std::size_t buffer_len = 0;
for (const auto entry : data)
{
write_buffer[buffer_len++] = entry;
if (buffer_len >= WRITE_BLOCK_BUFFER_SIZE)
{
out_stream.write(reinterpret_cast<const char *>(write_buffer),
WRITE_BLOCK_BUFFER_SIZE * sizeof(simple_type));
buffer_len = 0;
}
}
// write remaining entries
if (buffer_len > 0)
out_stream.write(reinterpret_cast<const char *>(write_buffer),
buffer_len * sizeof(simple_type));
return static_cast<bool>(out_stream);
}
template <typename simple_type>
void deserializeAdjacencyArray(const std::string &filename,
std::vector<std::uint32_t> &offsets,
@ -124,33 +40,6 @@ void deserializeAdjacencyArray(const std::string &filename,
SOURCE_REF);
}
inline bool serializeFlags(const boost::filesystem::path &path, const std::vector<bool> &flags)
{
// TODO this should be replaced with a FILE-based write using error checking
std::ofstream flag_stream(path.string(), std::ios::binary);
writeFingerprint(flag_stream);
std::uint32_t number_of_bits = flags.size();
flag_stream.write(reinterpret_cast<const char *>(&number_of_bits), sizeof(number_of_bits));
// putting bits in ints
std::uint32_t chunk = 0;
std::size_t chunk_count = 0;
for (std::size_t bit_nr = 0; bit_nr < number_of_bits;)
{
std::bitset<32> chunk_bitset;
for (std::size_t chunk_bit = 0; chunk_bit < 32 && bit_nr < number_of_bits;
++chunk_bit, ++bit_nr)
chunk_bitset[chunk_bit] = flags[bit_nr];
chunk = chunk_bitset.to_ulong();
++chunk_count;
flag_stream.write(reinterpret_cast<const char *>(&chunk), sizeof(chunk));
}
Log() << "Wrote " << number_of_bits << " bits in " << chunk_count << " chunks (Flags).";
return static_cast<bool>(flag_stream);
}
} // namespace util
} // namespace osrm

14
include/util/range_table.hpp
View File

@ -139,7 +139,19 @@ template <unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY> class RangeTable
sum_lengths = lengths_prefix_sum;
}
void ReadARangeTable(osrm::storage::io::FileReader &filereader)
void Write(osrm::storage::io::FileWriter &filewriter)
{
unsigned number_of_blocks = diff_blocks.size();
filewriter.WriteElementCount32(number_of_blocks);
filewriter.WriteOne(sum_lengths);
filewriter.WriteFrom(block_offsets.data(), number_of_blocks);
filewriter.WriteFrom(diff_blocks.data(), number_of_blocks);
}
void Read(osrm::storage::io::FileReader &filereader)
{
unsigned number_of_blocks = filereader.ReadElementCount32();
// read total length

8
include/util/static_rtree.hpp
View File

@ -331,12 +331,14 @@ class StaticRTree
});
// open tree file
boost::filesystem::ofstream tree_node_file(tree_node_filename, std::ios::binary);
storage::io::FileWriter tree_node_file(tree_node_filename,
storage::io::FileWriter::HasNoFingerprint);
std::uint64_t size_of_tree = m_search_tree.size();
BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty");
tree_node_file.write((char *)&size_of_tree, sizeof(size_of_tree));
tree_node_file.write((char *)&m_search_tree[0], sizeof(TreeNode) * size_of_tree);
tree_node_file.WriteOne(size_of_tree);
tree_node_file.WriteFrom(&m_search_tree[0], size_of_tree);
MapLeafNodesFile(leaf_node_filename);
}

90
src/contractor/contractor.cpp
View File

@ -713,47 +713,27 @@ Contractor::LoadEdgeExpandedGraph(const ContractorConfig &config,
return;
// Now save out the updated compressed geometries
std::ofstream geometry_stream(config.geometry_path, std::ios::binary);
if (!geometry_stream)
{
const std::string message{"Failed to open " + config.geometry_path + " for writing"};
throw util::exception(message + SOURCE_REF);
}
storage::io::FileWriter geometry_file(config.geometry_path,
storage::io::FileWriter::HasNoFingerprint);
const unsigned number_of_indices = geometry_indices.size();
const unsigned number_of_compressed_geometries = geometry_node_list.size();
geometry_stream.write(reinterpret_cast<const char *>(&number_of_indices), sizeof(unsigned));
geometry_stream.write(reinterpret_cast<char *>(&(geometry_indices[0])),
number_of_indices * sizeof(unsigned));
geometry_stream.write(reinterpret_cast<const char *>(&number_of_compressed_geometries),
sizeof(unsigned));
geometry_stream.write(reinterpret_cast<char *>(&(geometry_node_list[0])),
number_of_compressed_geometries * sizeof(NodeID));
geometry_stream.write(reinterpret_cast<char *>(&(geometry_fwd_weight_list[0])),
number_of_compressed_geometries * sizeof(EdgeWeight));
geometry_stream.write(reinterpret_cast<char *>(&(geometry_rev_weight_list[0])),
number_of_compressed_geometries * sizeof(EdgeWeight));
geometry_stream.write(reinterpret_cast<char *>(&(geometry_fwd_duration_list[0])),
number_of_compressed_geometries * sizeof(EdgeWeight));
geometry_stream.write(reinterpret_cast<char *>(&(geometry_rev_duration_list[0])),
number_of_compressed_geometries * sizeof(EdgeWeight));
geometry_file.WriteOne(number_of_indices);
geometry_file.WriteFrom(geometry_indices.data(), number_of_indices);
geometry_file.WriteOne(number_of_compressed_geometries);
geometry_file.WriteFrom(geometry_node_list.data(), number_of_compressed_geometries);
geometry_file.WriteFrom(geometry_fwd_weight_list.data(), number_of_compressed_geometries);
geometry_file.WriteFrom(geometry_rev_weight_list.data(), number_of_compressed_geometries);
geometry_file.WriteFrom(geometry_fwd_duration_list.data(), number_of_compressed_geometries);
geometry_file.WriteFrom(geometry_rev_duration_list.data(), number_of_compressed_geometries);
};
const auto save_datasource_indexes = [&] {
std::ofstream datasource_stream(config.datasource_indexes_path, std::ios::binary);
if (!datasource_stream)
{
const std::string message{"Failed to open " + config.datasource_indexes_path +
" for writing"};
throw util::exception(message + SOURCE_REF);
}
std::uint64_t number_of_datasource_entries = geometry_datasource.size();
datasource_stream.write(reinterpret_cast<const char *>(&number_of_datasource_entries),
sizeof(number_of_datasource_entries));
if (number_of_datasource_entries > 0)
{
datasource_stream.write(reinterpret_cast<char *>(&(geometry_datasource[0])),
number_of_datasource_entries * sizeof(uint8_t));
}
storage::io::FileWriter datasource_file(config.datasource_indexes_path,
storage::io::FileWriter::HasNoFingerprint);
datasource_file.SerializeVector(geometry_datasource);
};
const auto save_datastore_names = [&] {
@ -951,11 +931,10 @@ void Contractor::WriteNodeLevels(std::vector<float> &&in_node_levels) const
{
std::vector<float> node_levels(std::move(in_node_levels));
boost::filesystem::ofstream order_output_stream(config.level_output_path, std::ios::binary);
storage::io::FileWriter node_level_file(config.level_output_path,
storage::io::FileWriter::HasNoFingerprint);
unsigned level_size = node_levels.size();
order_output_stream.write((char *)&level_size, sizeof(unsigned));
order_output_stream.write((char *)node_levels.data(), sizeof(float) * node_levels.size());
node_level_file.SerializeVector(node_levels);
}
void Contractor::WriteCoreNodeMarker(std::vector<bool> &&in_is_core_node) const
@ -967,12 +946,12 @@ void Contractor::WriteCoreNodeMarker(std::vector<bool> &&in_is_core_node) const
unpacked_bool_flags[i] = is_core_node[i] ? 1 : 0;
}
boost::filesystem::ofstream core_marker_output_stream(config.core_output_path,
std::ios::binary);
unsigned size = unpacked_bool_flags.size();
core_marker_output_stream.write((char *)&size, sizeof(unsigned));
core_marker_output_stream.write((char *)unpacked_bool_flags.data(),
sizeof(char) * unpacked_bool_flags.size());
storage::io::FileWriter core_marker_output_file(config.core_output_path,
storage::io::FileWriter::HasNoFingerprint);
const std::size_t count = unpacked_bool_flags.size();
core_marker_output_file.WriteElementCount32(count);
core_marker_output_file.WriteFrom(unpacked_bool_flags.data(), count);
}
std::size_t
@ -984,9 +963,9 @@ Contractor::WriteContractedGraph(unsigned max_node_id,
const std::uint64_t contracted_edge_count = contracted_edge_list.size();
util::Log() << "Serializing compacted graph of " << contracted_edge_count << " edges";
const util::FingerPrint fingerprint = util::FingerPrint::GetValid();
boost::filesystem::ofstream hsgr_output_stream(config.graph_output_path, std::ios::binary);
hsgr_output_stream.write((char *)&fingerprint, sizeof(util::FingerPrint));
storage::io::FileWriter hsgr_output_file(config.graph_output_path,
storage::io::FileWriter::GenerateFingerprint);
const NodeID max_used_node_id = [&contracted_edge_list] {
NodeID tmp_max = 0;
for (const QueryEdge &edge : contracted_edge_list)
@ -1038,17 +1017,15 @@ Contractor::WriteContractedGraph(unsigned max_node_id,
const std::uint64_t node_array_size = node_array.size();
// serialize crc32, aka checksum
hsgr_output_stream.write((char *)&edges_crc32, sizeof(unsigned));
hsgr_output_file.WriteOne(edges_crc32);
// serialize number of nodes
hsgr_output_stream.write((char *)&node_array_size, sizeof(std::uint64_t));
hsgr_output_file.WriteOne(node_array_size);
// serialize number of edges
hsgr_output_stream.write((char *)&contracted_edge_count, sizeof(std::uint64_t));
hsgr_output_file.WriteOne(contracted_edge_count);
// serialize all nodes
if (node_array_size > 0)
{
hsgr_output_stream.write((char *)&node_array[0],
sizeof(util::StaticGraph<EdgeData>::NodeArrayEntry) *
node_array_size);
hsgr_output_file.WriteFrom(node_array.data(), node_array_size);
}
// serialize all edges
@ -1082,8 +1059,7 @@ Contractor::WriteContractedGraph(unsigned max_node_id,
throw util::exception("Edge weight is <= 0" + SOURCE_REF);
}
#endif
hsgr_output_stream.write((char *)&current_edge,
sizeof(util::StaticGraph<EdgeData>::EdgeArrayEntry));
hsgr_output_file.WriteOne(current_edge);
++number_of_used_edges;
}

89
src/extractor/edge_based_graph_factory.cpp
View File

@ -5,6 +5,9 @@
#include "extractor/node_based_graph_to_edge_based_graph_mapping_writer.hpp"
#include "extractor/scripting_environment.hpp"
#include "extractor/suffix_table.hpp"
#include "storage/io.hpp"
#include "util/bearing.hpp"
#include "util/coordinate.hpp"
#include "util/coordinate_calculation.hpp"
@ -20,7 +23,6 @@
#include <algorithm>
#include <cmath>
#include <fstream>
#include <iomanip>
#include <limits>
#include <sstream>
@ -178,14 +180,16 @@ EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeID nod
}
void EdgeBasedGraphFactory::FlushVectorToStream(
std::ofstream &edge_data_file, std::vector<OriginalEdgeData> &original_edge_data_vector) const
storage::io::FileWriter &edge_data_file,
std::vector<OriginalEdgeData> &original_edge_data_vector) const
{
if (original_edge_data_vector.empty())
{
return;
}
edge_data_file.write((char *)&(original_edge_data_vector[0]),
original_edge_data_vector.size() * sizeof(OriginalEdgeData));
edge_data_file.WriteFrom(original_edge_data_vector.data(), original_edge_data_vector.size());
original_edge_data_vector.clear();
}
@ -340,20 +344,17 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
skipped_uturns_counter = 0;
skipped_barrier_turns_counter = 0;
std::ofstream edge_data_file(original_edge_data_filename.c_str(), std::ios::binary);
std::ofstream edge_segment_file;
std::ofstream turn_penalties_index_file;
storage::io::FileWriter edge_data_file(original_edge_data_filename,
storage::io::FileWriter::HasNoFingerprint);
if (generate_edge_lookup)
{
edge_segment_file.open(edge_segment_lookup_filename.c_str(), std::ios::binary);
turn_penalties_index_file.open(turn_penalties_index_filename.c_str(), std::ios::binary);
}
storage::io::FileWriter edge_segment_file(edge_segment_lookup_filename,
storage::io::FileWriter::HasNoFingerprint);
storage::io::FileWriter turn_penalties_index_file(turn_penalties_index_filename,
storage::io::FileWriter::HasNoFingerprint);
// Writes a dummy value at the front that is updated later with the total length
const std::uint64_t length_prefix_empty_space{0};
edge_data_file.write(reinterpret_cast<const char *>(&length_prefix_empty_space),
sizeof(length_prefix_empty_space));
edge_data_file.WriteElementCount64(0);
std::vector<OriginalEdgeData> original_edge_data_vector;
original_edge_data_vector.reserve(1024 * 1024);
@ -607,8 +608,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
lookup::SegmentHeaderBlock header = {node_count, first_node.node_id};
edge_segment_file.write(reinterpret_cast<const char *>(&header),
sizeof(header));
edge_segment_file.WriteOne(header);
for (auto target_node : node_based_edges)
{
@ -622,8 +622,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
target_node.weight,
target_node.duration};
edge_segment_file.write(reinterpret_cast<const char *>(&nodeblock),
sizeof(nodeblock));
edge_segment_file.WriteOne(nodeblock);
previous = target_node.node_id;
}
@ -654,12 +653,8 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
lookup::TurnIndexBlock turn_index_block = {
from_node.node_id, via_node.node_id, to_node.node_id};
BOOST_ASSERT(turn_penalties_index_file.tellp() /
(sizeof(turn_index_block)) ==
turn_id);
turn_penalties_index_file.write(
reinterpret_cast<const char *>(&turn_index_block),
sizeof(turn_index_block));
turn_penalties_index_file.WriteOne(turn_index_block);
}
}
}
@ -667,59 +662,53 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
}
// write weight penalties per turn
std::ofstream turn_weight_penalties_file(turn_weight_penalties_filename.c_str(),
std::ios::binary);
storage::io::FileWriter turn_weight_penalties_file(turn_weight_penalties_filename,
storage::io::FileWriter::HasNoFingerprint);
lookup::TurnPenaltiesHeader turn_weight_penalties_header{turn_weight_penalties.size()};
turn_weight_penalties_file.write(reinterpret_cast<const char *>(&turn_weight_penalties_header),
sizeof(turn_weight_penalties_header));
turn_weight_penalties_file.write(reinterpret_cast<const char *>(turn_weight_penalties.data()),
sizeof(decltype(turn_weight_penalties)::value_type) *
turn_weight_penalties_file.WriteOne(turn_weight_penalties_header);
turn_weight_penalties_file.WriteFrom(turn_weight_penalties.data(),
turn_weight_penalties.size());
// write duration penalties per turn if we need them
BOOST_ASSERT(!profile_properties.fallback_to_duration || turn_duration_penalties.size() == 0);
std::ofstream turn_duration_penalties_file(turn_duration_penalties_filename.c_str(),
std::ios::binary);
storage::io::FileWriter turn_duration_penalties_file(turn_duration_penalties_filename,
storage::io::FileWriter::HasNoFingerprint);
lookup::TurnPenaltiesHeader turn_duration_penalties_header{turn_duration_penalties.size()};
turn_duration_penalties_file.write(
reinterpret_cast<const char *>(&turn_duration_penalties_header),
sizeof(turn_duration_penalties_header));
turn_duration_penalties_file.WriteOne(turn_duration_penalties_header);
if (!profile_properties.fallback_to_duration)
{
BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size());
turn_duration_penalties_file.write(
reinterpret_cast<const char *>(turn_duration_penalties.data()),
sizeof(decltype(turn_duration_penalties)::value_type) * turn_duration_penalties.size());
turn_duration_penalties_file.WriteFrom(turn_duration_penalties.data(),
turn_duration_penalties.size());
}
util::Log() << "Created " << entry_class_hash.size() << " entry classes and "
<< bearing_class_hash.size() << " Bearing Classes";
util::Log() << "Writing Turn Lane Data to File...";
std::ofstream turn_lane_data_file(turn_lane_data_filename.c_str(), std::ios::binary);
storage::io::FileWriter turn_lane_data_file(turn_lane_data_filename,
storage::io::FileWriter::HasNoFingerprint);
std::vector<util::guidance::LaneTupleIdPair> lane_data(lane_data_map.size());
// extract lane data sorted by ID
for (auto itr : lane_data_map)
lane_data[itr.second] = itr.first;
std::uint64_t size = lane_data.size();
turn_lane_data_file.write(reinterpret_cast<const char *>(&size), sizeof(size));
if (!lane_data.empty())
turn_lane_data_file.write(reinterpret_cast<const char *>(&lane_data[0]),
sizeof(util::guidance::LaneTupleIdPair) * lane_data.size());
turn_lane_data_file.SerializeVector(lane_data);
util::Log() << "done.";
FlushVectorToStream(edge_data_file, original_edge_data_vector);
// Finally jump back to the empty space at the beginning and write length prefix
edge_data_file.seekp(std::ios::beg);
edge_data_file.SkipToBeginning();
const auto length_prefix = boost::numeric_cast<std::uint64_t>(original_edges_counter);
static_assert(sizeof(length_prefix_empty_space) == sizeof(length_prefix), "type mismatch");
edge_data_file.write(reinterpret_cast<const char *>(&length_prefix), sizeof(length_prefix));
edge_data_file.WriteElementCount64(length_prefix);
util::Log() << "Generated " << m_edge_based_node_list.size() << " edge based nodes";
util::Log() << "Node-based graph contains " << node_based_edge_counter << " edges";

72
src/extractor/extraction_containers.cpp
View File

@ -12,6 +12,8 @@
#include "util/name_table.hpp"
#include "util/timing_util.hpp"
#include "storage/io.hpp"
#include <boost/assert.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
@ -152,17 +154,15 @@ void ExtractionContainers::PrepareData(ScriptingEnvironment &scripting_environme
const std::string &restrictions_file_name,
const std::string &name_file_name)
{
std::ofstream file_out_stream;
file_out_stream.open(output_file_name.c_str(), std::ios::binary);
const util::FingerPrint fingerprint = util::FingerPrint::GetValid();
file_out_stream.write((char *)&fingerprint, sizeof(util::FingerPrint));
storage::io::FileWriter file_out(output_file_name,
storage::io::FileWriter::GenerateFingerprint);
FlushVectors();
PrepareNodes();
WriteNodes(file_out_stream);
WriteNodes(file_out);
PrepareEdges(scripting_environment);
WriteEdges(file_out_stream);
WriteEdges(file_out);
PrepareRestrictions();
WriteRestrictions(restrictions_file_name);
@ -542,22 +542,16 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm
}
}
void ExtractionContainers::WriteEdges(std::ofstream &file_out_stream) const
void ExtractionContainers::WriteEdges(storage::io::FileWriter &file_out) const
{
std::size_t start_position = 0;
std::uint64_t used_edges_counter = 0;
std::uint32_t used_edges_counter_buffer = 0;
std::vector<NodeBasedEdge> normal_edges;
normal_edges.reserve(all_edges_list.size());
{
util::UnbufferedLog log;
log << "Writing used edges ... " << std::flush;
TIMER_START(write_edges);
// Traverse list of edges and nodes in parallel and set target coord
start_position = file_out_stream.tellp();
file_out_stream.write((char *)&used_edges_counter_buffer,
sizeof(used_edges_counter_buffer));
for (const auto &edge : all_edges_list)
{
if (edge.result.source == SPECIAL_NODEID || edge.result.target == SPECIAL_NODEID)
@ -567,41 +561,30 @@ void ExtractionContainers::WriteEdges(std::ofstream &file_out_stream) const
// IMPORTANT: here, we're using slicing to only write the data from the base
// class of NodeBasedEdgeWithOSM
NodeBasedEdge tmp = edge.result;
file_out_stream.write((char *)&tmp, sizeof(NodeBasedEdge));
used_edges_counter++;
normal_edges.push_back(edge.result);
}
if (used_edges_counter > std::numeric_limits<unsigned>::max())
if (normal_edges.size() > std::numeric_limits<uint32_t>::max())
{
throw util::exception("There are too many edges, OSRM only supports 2^32" + SOURCE_REF);
}
file_out.WriteElementCount32(normal_edges.size());
file_out.WriteFrom(normal_edges.data(), normal_edges.size());
TIMER_STOP(write_edges);
log << "ok, after " << TIMER_SEC(write_edges) << "s";
log << "Processed " << normal_edges.size() << " edges";
}
{
util::UnbufferedLog log;
log << "setting number of edges ... " << std::flush;
used_edges_counter_buffer = boost::numeric_cast<std::uint32_t>(used_edges_counter);
file_out_stream.seekp(start_position);
file_out_stream.write((char *)&used_edges_counter_buffer,
sizeof(used_edges_counter_buffer));
log << "ok";
}
util::Log() << "Processed " << used_edges_counter << " edges";
}
void ExtractionContainers::WriteNodes(std::ofstream &file_out_stream) const
void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const
{
{
// write dummy value, will be overwritten later
util::UnbufferedLog log;
log << "setting number of nodes ... " << std::flush;
file_out_stream.write((char *)&max_internal_node_id, sizeof(unsigned));
file_out.WriteElementCount32(max_internal_node_id);
log << "ok";
}
@ -629,7 +612,7 @@ void ExtractionContainers::WriteNodes(std::ofstream &file_out_stream) const
}
BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode));
file_out.WriteOne((*node_iterator));
++node_id_iterator;
++node_iterator;
@ -644,13 +627,11 @@ void ExtractionContainers::WriteNodes(std::ofstream &file_out_stream) const
void ExtractionContainers::WriteRestrictions(const std::string &path) const
{
// serialize restrictions
std::ofstream restrictions_out_stream;
unsigned written_restriction_count = 0;
restrictions_out_stream.open(path.c_str(), std::ios::binary);
const util::FingerPrint fingerprint = util::FingerPrint::GetValid();
restrictions_out_stream.write((char *)&fingerprint, sizeof(util::FingerPrint));
const auto count_position = restrictions_out_stream.tellp();
restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned));
storage::io::FileWriter restrictions_out_file(path,
storage::io::FileWriter::GenerateFingerprint);
restrictions_out_file.WriteElementCount32(written_restriction_count);
for (const auto &restriction_container : restrictions_list)
{
@ -658,13 +639,12 @@ void ExtractionContainers::WriteRestrictions(const std::string &path) const
SPECIAL_NODEID != restriction_container.restriction.via.node &&
SPECIAL_NODEID != restriction_container.restriction.to.node)
{
restrictions_out_stream.write((char *)&(restriction_container.restriction),
sizeof(TurnRestriction));
restrictions_out_file.WriteOne(restriction_container.restriction);
++written_restriction_count;
}
}
restrictions_out_stream.seekp(count_position);
restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned));
restrictions_out_file.SkipToBeginning();
restrictions_out_file.WriteElementCount32(written_restriction_count);
util::Log() << "usable restrictions: " << written_restriction_count;
}

87
src/extractor/extractor.cpp
View File

@ -46,7 +46,6 @@
#include <atomic>
#include <bitset>
#include <chrono>
#include <fstream>
#include <future>
#include <iostream>
#include <iterator>
@ -167,8 +166,10 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
}
util::Log() << "timestamp: " << timestamp;
boost::filesystem::ofstream timestamp_out(config.timestamp_file_name);
timestamp_out.write(timestamp.c_str(), timestamp.length());
storage::io::FileWriter timestamp_file(config.timestamp_file_name,
storage::io::FileWriter::HasNoFingerprint);
timestamp_file.WriteFrom(timestamp.c_str(), timestamp.length());
// initialize vectors holding parsed objects
tbb::concurrent_vector<std::pair<std::size_t, ExtractionNode>> resulting_nodes;
@ -277,7 +278,11 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
util::Log() << "Saving edge-based node weights to file.";
TIMER_START(timer_write_node_weights);
util::serializeVector(config.edge_based_node_weights_output_path, edge_based_node_weights);
storage::io::FileWriter edge_file(config.edge_based_node_weights_output_path,
storage::io::FileWriter::GenerateFingerprint);
edge_file.SerializeVector(edge_based_node_weights);
TIMER_STOP(timer_write_node_weights);
util::Log() << "Done writing. (" << TIMER_SEC(timer_write_node_weights) << ")";
@ -314,13 +319,9 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
void Extractor::WriteProfileProperties(const std::string &output_path,
const ProfileProperties &properties) const
{
boost::filesystem::ofstream out_stream(output_path);
if (!out_stream)
{
throw util::exception("Could not open " + output_path + " for writing." + SOURCE_REF);
}
storage::io::FileWriter file(output_path, storage::io::FileWriter::HasNoFingerprint);
out_stream.write(reinterpret_cast<const char *>(&properties), sizeof(properties));
file.WriteOne(properties);
}
void Extractor::FindComponents(unsigned max_edge_id,
@ -535,14 +536,10 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment,
*/
void Extractor::WriteNodeMapping(const std::vector<QueryNode> &internal_to_external_node_map)
{
boost::filesystem::ofstream node_stream(config.node_output_path, std::ios::binary);
const std::uint64_t size_of_mapping = internal_to_external_node_map.size();
node_stream.write((char *)&size_of_mapping, sizeof(std::uint64_t));
if (size_of_mapping > 0)
{
node_stream.write((char *)internal_to_external_node_map.data(),
size_of_mapping * sizeof(QueryNode));
}
storage::io::FileWriter node_file(config.node_output_path,
storage::io::FileWriter::HasNoFingerprint);
node_file.SerializeVector(internal_to_external_node_map);
}
/**
@ -597,22 +594,19 @@ void Extractor::WriteEdgeBasedGraph(
EdgeID const max_edge_id,
util::DeallocatingVector<EdgeBasedEdge> const &edge_based_edge_list)
{
std::ofstream file_out_stream;
file_out_stream.open(output_file_filename.c_str(), std::ios::binary);
const util::FingerPrint fingerprint = util::FingerPrint::GetValid();
file_out_stream.write((char *)&fingerprint, sizeof(util::FingerPrint));
storage::io::FileWriter file(output_file_filename,
storage::io::FileWriter::GenerateFingerprint);
util::Log() << "Writing edge-based-graph edges ... " << std::flush;
TIMER_START(write_edges);
std::uint64_t number_of_used_edges = edge_based_edge_list.size();
file_out_stream.write((char *)&number_of_used_edges, sizeof(number_of_used_edges));
file_out_stream.write((char *)&max_edge_id, sizeof(max_edge_id));
file.WriteElementCount64(number_of_used_edges);
file.WriteOne(max_edge_id);
for (const auto &edge : edge_based_edge_list)
{
file_out_stream.write((char *)&edge, sizeof(EdgeBasedEdge));
file.WriteOne(edge);
}
TIMER_STOP(write_edges);
@ -627,17 +621,11 @@ void Extractor::WriteIntersectionClassificationData(
const std::vector<util::guidance::BearingClass> &bearing_classes,
const std::vector<util::guidance::EntryClass> &entry_classes) const
{
std::ofstream file_out_stream(output_file_name.c_str(), std::ios::binary);
if (!file_out_stream)
{
util::Log(logERROR) << "Failed to open " << output_file_name << " for writing";
return;
}
storage::io::FileWriter file(output_file_name, storage::io::FileWriter::GenerateFingerprint);
util::Log() << "Writing Intersection Classification Data";
TIMER_START(write_edges);
util::writeFingerprint(file_out_stream);
util::serializeVector(file_out_stream, node_based_intersection_classes);
file.SerializeVector(node_based_intersection_classes);
// create range table for vectors:
std::vector<unsigned> bearing_counts;
@ -651,22 +639,18 @@ void Extractor::WriteIntersectionClassificationData(
}
util::RangeTable<> bearing_class_range_table(bearing_counts);
file_out_stream << bearing_class_range_table;
bearing_class_range_table.Write(file);
file.WriteOne(total_bearings);
file_out_stream.write(reinterpret_cast<const char *>(&total_bearings), sizeof(total_bearings));
for (const auto &bearing_class : bearing_classes)
{
const auto &bearings = bearing_class.getAvailableBearings();
file_out_stream.write(reinterpret_cast<const char *>(&bearings[0]),
sizeof(bearings[0]) * bearings.size());
file.WriteFrom(bearings.data(), bearings.size());
}
if (!static_cast<bool>(file_out_stream))
{
throw util::exception("Failed to write to " + output_file_name + "." + SOURCE_REF);
}
file.SerializeVector(entry_classes);
util::serializeVector(file_out_stream, entry_classes);
TIMER_STOP(write_edges);
util::Log() << "ok, after " << TIMER_SEC(write_edges) << "s for "
<< node_based_intersection_classes.size() << " Indices into "
@ -684,20 +668,9 @@ void Extractor::WriteTurnLaneData(const std::string &turn_lane_file) const
util::Log() << "Writing turn lane masks...";
TIMER_START(turn_lane_timer);
std::ofstream ofs(turn_lane_file, std::ios::binary);
if (!ofs)
throw osrm::util::exception("Failed to open " + turn_lane_file + " for writing." +
SOURCE_REF);
if (!util::serializeVector(ofs, turn_lane_offsets))
{
throw util::exception("Error while writing to " + turn_lane_file + SOURCE_REF);
}
if (!util::serializeVector(ofs, turn_lane_masks))
{
throw util::exception("Error while writing to " + turn_lane_file + SOURCE_REF);
}
storage::io::FileWriter file(turn_lane_file, storage::io::FileWriter::HasNoFingerprint);
file.SerializeVector(turn_lane_offsets);
file.SerializeVector(turn_lane_masks);
TIMER_STOP(turn_lane_timer);
util::Log() << "done (" << TIMER_SEC(turn_lane_timer) << ")";

34
unit_tests/util/io.cpp
View File

@ -1,6 +1,6 @@
#include "util/io.hpp"
#include "storage/io.hpp"
#include "util/exception.hpp"
#include "util/fingerprint.hpp"
#include "util/typedefs.hpp"
#include "util/version.hpp"
@ -26,7 +26,11 @@ BOOST_AUTO_TEST_CASE(io_data)
std::vector<int> data_in(53), data_out;
std::iota(begin(data_in), end(data_in), 0);
osrm::util::serializeVector(IO_TMP_FILE, data_in);
{
osrm::storage::io::FileWriter outfile(IO_TMP_FILE,
osrm::storage::io::FileWriter::GenerateFingerprint);
outfile.SerializeVector(data_in);
}
osrm::storage::io::FileReader infile(IO_TMP_FILE,
osrm::storage::io::FileReader::VerifyFingerprint);
@ -58,7 +62,11 @@ BOOST_AUTO_TEST_CASE(file_too_small)
std::vector<int> v(53);
std::iota(begin(v), end(v), 0);
osrm::util::serializeVector(IO_TOO_SMALL_FILE, v);
{
osrm::storage::io::FileWriter outfile(
IO_TOO_SMALL_FILE, osrm::storage::io::FileWriter::GenerateFingerprint);
outfile.SerializeVector(v);
}
std::fstream f(IO_TOO_SMALL_FILE);
f.seekp(sizeof(osrm::util::FingerPrint), std::ios_base::beg);
@ -88,12 +96,12 @@ BOOST_AUTO_TEST_CASE(io_corrupt_fingerprint)
{
std::vector<int> v(153);
std::iota(begin(v), end(v), 0);
osrm::util::serializeVector(IO_CORRUPT_FINGERPRINT_FILE, v);
std::fstream f(IO_CORRUPT_FINGERPRINT_FILE);
f.seekp(0, std::ios_base::beg);
std::uint64_t garbage = 0xDEADBEEFCAFEFACE;
f.write(reinterpret_cast<char *>(&garbage), sizeof(garbage));
osrm::storage::io::FileWriter outfile(IO_CORRUPT_FINGERPRINT_FILE,
osrm::storage::io::FileWriter::HasNoFingerprint);
outfile.WriteOne(0xDEADBEEFCAFEFACE);
outfile.SerializeVector(v);
}
try
@ -115,7 +123,15 @@ BOOST_AUTO_TEST_CASE(io_incompatible_fingerprint)
{
std::vector<int> v(153);
std::iota(begin(v), end(v), 0);
osrm::util::serializeVector(IO_INCOMPATIBLE_FINGERPRINT_FILE, v);
{
osrm::storage::io::FileWriter outfile(IO_INCOMPATIBLE_FINGERPRINT_FILE,
osrm::storage::io::FileWriter::HasNoFingerprint);
const auto fingerprint = osrm::util::FingerPrint::GetValid();
outfile.WriteOne(fingerprint);
outfile.SerializeVector(v);
}
std::fstream f(IO_INCOMPATIBLE_FINGERPRINT_FILE);
f.seekp(5, std::ios_base::beg); // Seek past `OSRN` and Major version byte