From 3b0b896501e07163f0253b57e74c1ca3b3ff8d93 Mon Sep 17 00:00:00 2001 From: Kajari Ghosh Date: Tue, 21 Mar 2017 16:43:48 -0400 Subject: [PATCH] throw errors after every write stream (#3843) --- CHANGELOG.md | 9 +++- include/util/indexed_data.hpp | 34 ++++++++++++ include/util/io.hpp | 29 +++++++++++ include/util/range_table.hpp | 23 +++++++++ include/util/static_rtree.hpp | 18 +++++++ src/contractor/contractor.cpp | 48 ++++++++++++++++- src/extractor/compressed_edge_container.cpp | 9 ++++ src/extractor/edge_based_graph_factory.cpp | 57 +++++++++++++++++++++ src/extractor/extraction_containers.cpp | 34 ++++++++++++ src/extractor/extractor.cpp | 31 +++++++++++ 10 files changed, 289 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09c04015c..e9ffdd7fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,16 @@ +# 5.6.4 + - Changes from 5.6.3 + - Bugfixes + - Fix #3838 throws errors if write streams have failed + # 5.6.3 - - Changes from 5.6.0 + - Changes from 5.6.2 - Bugfixes - #3790 Fix incorrect speed values in tile plugin # 5.6.2 - - Changes from 5.6.0 + - Changes from 5.6.1 - Bugfixes - Fix incorrect forward datasources getter in facade - Fix include `access=private` non-car roads in the car profile diff --git a/include/util/indexed_data.hpp b/include/util/indexed_data.hpp index abb869a10..dd1582286 100644 --- a/include/util/indexed_data.hpp +++ b/include/util/indexed_data.hpp @@ -2,6 +2,7 @@ #define OSRM_INDEXED_DATA_HPP #include "util/exception.hpp" +#include "util/exception_utils.hpp" #include "util/string_view.hpp" #include @@ -107,6 +108,11 @@ template struct VariableGroupBlock out.write((const char *)&refernce, sizeof(refernce)); + if (!out) + { + throw util::exception(std::string("Error writing block reference: ") + SOURCE_REF); + } + return prefix_length; } @@ -128,6 +134,11 @@ template struct VariableGroupBlock continue; out.write((const char *)&data_length, byte_length); + + if (!out) + { + throw util::exception(std::string("Error writing block prefix: ") + SOURCE_REF); + } } } @@ -183,6 +194,12 @@ template struct FixedGroupBlock BlockReference refernce{static_cast(data_offset)}; out.write((const char *)&refernce, sizeof(refernce)); + if (!out) + { + throw util::exception(std::string("Error writing group block reference: ") + + SOURCE_REF); + } + return BLOCK_SIZE; } @@ -201,6 +218,12 @@ template struct FixedGroupBlock block_prefix[index++] = static_cast(data_length); } out.write((const char *)block_prefix.data(), block_prefix.size()); + + if (!out) + { + throw util::exception(std::string("Error writing a fixed length block prefix: ") + + SOURCE_REF); + } } /// Advances the range to an item stored in the referenced block. @@ -261,6 +284,11 @@ template struct IndexedData : 1 + (std::distance(first, sentinel) - 1) / (BLOCK_SIZE + 1); out.write((const char *)&number_of_blocks, sizeof(number_of_blocks)); + if (!out) + { + throw util::exception(std::string("Error writing indexed data: ") + SOURCE_REF); + } + // Write block references and compute the total data size that includes prefix and data const GroupBlock block; DataSizeType data_size = 0; @@ -275,6 +303,12 @@ template struct IndexedData // Write the total data size out.write((const char *)&data_size, sizeof(data_size)); + if (!out) + { + throw util::exception(std::string("Error writing the total indexed data size: ") + + SOURCE_REF); + } + // Write data blocks that are (prefix, data) for (OffsetIterator curr = first, next = first; next != sentinel; curr = next) { diff --git a/include/util/io.hpp b/include/util/io.hpp index be307ea20..259481167 100644 --- a/include/util/io.hpp +++ b/include/util/io.hpp @@ -15,6 +15,7 @@ #include #include "storage/io.hpp" +#include "util/exception_utils.hpp" #include "util/fingerprint.hpp" namespace osrm @@ -26,6 +27,10 @@ inline bool writeFingerprint(std::ostream &stream) { const auto fingerprint = FingerPrint::GetValid(); stream.write(reinterpret_cast(&fingerprint), sizeof(fingerprint)); + if (!stream) + { + throw util::exception(std::string("Error writing fingerprint: ") + SOURCE_REF); + } return static_cast(stream); } @@ -36,6 +41,10 @@ bool serializeVector(std::ostream &stream, const std::vector &data) stream.write(reinterpret_cast(&count), sizeof(count)); if (!data.empty()) stream.write(reinterpret_cast(&data[0]), sizeof(simple_type) * count); + if (!stream) + { + throw util::exception(std::string("Error serializing vector: ") + SOURCE_REF); + } return static_cast(stream); } @@ -82,6 +91,11 @@ bool serializeVector(std::ofstream &out_stream, const stxxl::vector { const std::uint64_t size = data.size(); out_stream.write(reinterpret_cast(&size), sizeof(size)); + if (!out_stream) + { + throw util::exception(std::string("Error serializing vector. Writing size step: ") + + SOURCE_REF); + } simple_type write_buffer[WRITE_BLOCK_BUFFER_SIZE]; std::size_t buffer_len = 0; @@ -102,6 +116,11 @@ bool serializeVector(std::ofstream &out_stream, const stxxl::vector if (buffer_len > 0) out_stream.write(reinterpret_cast(write_buffer), buffer_len * sizeof(simple_type)); + if (!out_stream) + { + throw util::exception(std::string("Error serializing vector. Writing data step: ") + + SOURCE_REF); + } return static_cast(out_stream); } @@ -133,6 +152,11 @@ inline bool serializeFlags(const boost::filesystem::path &path, const std::vecto std::uint32_t number_of_bits = flags.size(); flag_stream.write(reinterpret_cast(&number_of_bits), sizeof(number_of_bits)); + if (!flag_stream) + { + throw util::exception(std::string("Error on serializing flags. Write size step: ") + + SOURCE_REF); + } // putting bits in ints std::uint32_t chunk = 0; std::size_t chunk_count = 0; @@ -147,6 +171,11 @@ inline bool serializeFlags(const boost::filesystem::path &path, const std::vecto ++chunk_count; flag_stream.write(reinterpret_cast(&chunk), sizeof(chunk)); } + if (!flag_stream) + { + throw util::exception(std::string("Error on serializing flags. Write data step: ") + + SOURCE_REF); + } Log() << "Wrote " << number_of_bits << " bits in " << chunk_count << " chunks (Flags)."; return static_cast(flag_stream); } diff --git a/include/util/range_table.hpp b/include/util/range_table.hpp index 0d95bbbc1..ce5096ff0 100644 --- a/include/util/range_table.hpp +++ b/include/util/range_table.hpp @@ -2,6 +2,7 @@ #define RANGE_TABLE_HPP #include "storage/io.hpp" +#include "util/exception_utils.hpp" #include "util/integer_range.hpp" #include "util/shared_memory_vector_wrapper.hpp" @@ -221,12 +222,34 @@ std::ostream &operator<<(std::ostream &out, const RangeTable(&(geometry_rev_duration_list[0])), number_of_compressed_geometries * sizeof(EdgeWeight)); + if (!geometry_stream) + { + throw util::exception( + std::string( + "Error in contractor, updating compressed geometries in `.geometries` file: ") + + config.geometry_path); + } + }; const auto save_datasource_indexes = [&] { @@ -754,6 +762,12 @@ Contractor::LoadEdgeExpandedGraph(const ContractorConfig &config, datasource_stream.write(reinterpret_cast(&(geometry_datasource[0])), number_of_datasource_entries * sizeof(uint8_t)); } + if (!datasource_stream) + { + throw util::exception(std::string("Error in contractor, saving datasource indices in " + "`.datasource_indexes` file: ") + + SOURCE_REF); + } }; const auto save_datastore_names = [&] { @@ -956,6 +970,13 @@ void Contractor::WriteNodeLevels(std::vector &&in_node_levels) const unsigned level_size = node_levels.size(); order_output_stream.write((char *)&level_size, sizeof(unsigned)); order_output_stream.write((char *)node_levels.data(), sizeof(float) * node_levels.size()); + + if (!order_output_stream) + { + throw util::exception( + std::string("Error in contractor, writing node levels to `.level` file: ") + + SOURCE_REF); + } } void Contractor::WriteCoreNodeMarker(std::vector &&in_is_core_node) const @@ -973,6 +994,13 @@ void Contractor::WriteCoreNodeMarker(std::vector &&in_is_core_node) const core_marker_output_stream.write((char *)&size, sizeof(unsigned)); core_marker_output_stream.write((char *)unpacked_bool_flags.data(), sizeof(char) * unpacked_bool_flags.size()); + + if (!core_marker_output_stream) + { + throw util::exception( + std::string("Error in contractor, writing code node markers to `.core` file: ") + + SOURCE_REF); + } } std::size_t @@ -987,6 +1015,12 @@ Contractor::WriteContractedGraph(unsigned max_node_id, const util::FingerPrint fingerprint = util::FingerPrint::GetValid(); boost::filesystem::ofstream hsgr_output_stream(config.graph_output_path, std::ios::binary); hsgr_output_stream.write((char *)&fingerprint, sizeof(util::FingerPrint)); + if (!hsgr_output_stream) + { + throw util::exception( + std::string("Error in contractor, writing contracted graph to `.hsgr` file: ") + + SOURCE_REF); + } const NodeID max_used_node_id = [&contracted_edge_list] { NodeID tmp_max = 0; for (const QueryEdge &edge : contracted_edge_list) @@ -1050,6 +1084,12 @@ Contractor::WriteContractedGraph(unsigned max_node_id, sizeof(util::StaticGraph::NodeArrayEntry) * node_array_size); } + if (!hsgr_output_stream) + { + throw util::exception( + std::string("Error in contractor, serializing graph node array in `.hsgr` file: ") + + SOURCE_REF); + } // serialize all edges util::Log() << "Building edge array"; @@ -1058,7 +1098,7 @@ Contractor::WriteContractedGraph(unsigned max_node_id, util::StaticGraph::EdgeArrayEntry current_edge; for (const auto edge : util::irange(0UL, contracted_edge_list.size())) { - // some self-loops are required for oneway handling. Need to assertthat we only keep these + // some self-loops are required for oneway handling. Need to assert that we only keep these // (TODO) // no eigen loops // BOOST_ASSERT(contracted_edge_list[edge].source != contracted_edge_list[edge].target || @@ -1084,6 +1124,12 @@ Contractor::WriteContractedGraph(unsigned max_node_id, #endif hsgr_output_stream.write((char *)¤t_edge, sizeof(util::StaticGraph::EdgeArrayEntry)); + if (!hsgr_output_stream) + { + throw util::exception( + std::string("Error in contractor, serializing graph edge array in `.hsgr` file: ") + + SOURCE_REF); + } ++number_of_used_edges; } diff --git a/src/extractor/compressed_edge_container.cpp b/src/extractor/compressed_edge_container.cpp index b34ab90e7..013adec64 100644 --- a/src/extractor/compressed_edge_container.cpp +++ b/src/extractor/compressed_edge_container.cpp @@ -1,4 +1,6 @@ #include "extractor/compressed_edge_container.hpp" +#include "util/exception.hpp" +#include "util/exception_utils.hpp" #include "util/log.hpp" #include @@ -109,6 +111,13 @@ void CompressedEdgeContainer::SerializeInternalVector(const std::string &path) c // write compressed geometry reverse durations geometry_out_stream.write((char *)(m_compressed_geometry_rev_durations.data()), sizeof(EdgeWeight) * m_compressed_geometry_rev_durations.size()); + + if (!geometry_out_stream) + { + throw util::exception(std::string("Error in CompressedEdgeContainer, writing compressed " + "geometry to `.geometry` file: ") + + SOURCE_REF); + } } // Adds info for a compressed edge to the container. edge_id_2 diff --git a/src/extractor/edge_based_graph_factory.cpp b/src/extractor/edge_based_graph_factory.cpp index ab32f4e7f..fd7738e52 100644 --- a/src/extractor/edge_based_graph_factory.cpp +++ b/src/extractor/edge_based_graph_factory.cpp @@ -4,6 +4,7 @@ #include "util/coordinate.hpp" #include "util/coordinate_calculation.hpp" #include "util/exception.hpp" +#include "util/exception_utils.hpp" #include "util/guidance/turn_bearing.hpp" #include "util/integer_range.hpp" #include "util/log.hpp" @@ -183,6 +184,15 @@ void EdgeBasedGraphFactory::FlushVectorToStream( } edge_data_file.write((char *)&(original_edge_data_vector[0]), original_edge_data_vector.size() * sizeof(OriginalEdgeData)); + + if (!edge_data_file) + { + throw util::exception( + std::string( + "Error in EdgeBasedGraphFactory, flushing vector to stream for `.ebg` file: ") + + SOURCE_REF); + } + original_edge_data_vector.clear(); } @@ -339,6 +349,13 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( edge_data_file.write(reinterpret_cast(&length_prefix_empty_space), sizeof(length_prefix_empty_space)); + if (!edge_data_file) + { + throw util::exception( + std::string("Error writing dummy value to edge based graph in `.ebg` file: ") + + SOURCE_REF); + } + std::vector original_edge_data_vector; original_edge_data_vector.reserve(1024 * 1024); @@ -608,9 +625,18 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( edge_segment_file.write(reinterpret_cast(&nodeblock), sizeof(nodeblock)); + previous = target_node.node_id; } + if (!edge_segment_file) + { + throw util::exception( + std::string("Error in generating edge lookups and writing to " + "`.edge_segment_lookup` file: ") + + SOURCE_REF); + } + // We also now write out the mapping between the edge-expanded edges and the // original nodes. Since each edge represents a possible maneuver, external // programs can use this to quickly perform updates to edge weights in order @@ -644,6 +670,14 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( turn_penalties_index_file.write( reinterpret_cast(&turn_index_block), sizeof(turn_index_block)); + + if (!turn_penalties_index_file) + { + throw util::exception( + std::string("Error writing data for turn penalties to " + "`.turn_penalties_index` file ") + + SOURCE_REF); + } } } } @@ -659,6 +693,12 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( turn_weight_penalties_file.write(reinterpret_cast(turn_weight_penalties.data()), sizeof(decltype(turn_weight_penalties)::value_type) * turn_weight_penalties.size()); + if (!turn_weight_penalties_file) + { + throw util::exception( + std::string("Error writing turn weight penalties to `.turn_weight_penalties` file ") + + SOURCE_REF); + } // write duration penalties per turn if we need them BOOST_ASSERT(!profile_properties.fallback_to_duration || turn_duration_penalties.size() == 0); @@ -668,12 +708,20 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( turn_duration_penalties_file.write( reinterpret_cast(&turn_duration_penalties_header), sizeof(turn_duration_penalties_header)); + if (!profile_properties.fallback_to_duration) { BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size()); turn_duration_penalties_file.write( reinterpret_cast(turn_duration_penalties.data()), sizeof(decltype(turn_duration_penalties)::value_type) * turn_duration_penalties.size()); + if (!turn_duration_penalties_file) + { + throw util::exception( + std::string( + "Error writing turn duration penalties to `.turn_duration_penalties` file ") + + SOURCE_REF); + } } util::Log() << "Created " << entry_class_hash.size() << " entry classes and " @@ -693,6 +741,11 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( turn_lane_data_file.write(reinterpret_cast(&lane_data[0]), sizeof(util::guidance::LaneTupleIdPair) * lane_data.size()); + if (!turn_lane_data_file) + { + throw util::exception(std::string("Error writing turn lane data file: ") + SOURCE_REF); + } + util::Log() << "done."; FlushVectorToStream(edge_data_file, original_edge_data_vector); @@ -704,6 +757,10 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( static_assert(sizeof(length_prefix_empty_space) == sizeof(length_prefix), "type mismatch"); edge_data_file.write(reinterpret_cast(&length_prefix), sizeof(length_prefix)); + if (!edge_data_file) + { + throw util::exception(std::string("Error writing data to `.ebg`") + SOURCE_REF); + } util::Log() << "Generated " << m_edge_based_node_list.size() << " edge based nodes"; util::Log() << "Node-based graph contains " << node_based_edge_counter << " edges"; diff --git a/src/extractor/extraction_containers.cpp b/src/extractor/extraction_containers.cpp index 6676c695a..21e14c725 100644 --- a/src/extractor/extraction_containers.cpp +++ b/src/extractor/extraction_containers.cpp @@ -158,6 +158,13 @@ void ExtractionContainers::PrepareData(ScriptingEnvironment &scripting_environme file_out_stream.open(output_file_name.c_str(), std::ios::binary); const util::FingerPrint fingerprint = util::FingerPrint::GetValid(); file_out_stream.write((char *)&fingerprint, sizeof(util::FingerPrint)); + if (!file_out_stream) + { + throw util::exception( + std::string( + "Error writing fingerprint during extraction containers data preparation: ") + + SOURCE_REF); + } FlushVectors(); @@ -571,9 +578,16 @@ void ExtractionContainers::WriteEdges(std::ofstream &file_out_stream) const // class of NodeBasedEdgeWithOSM NodeBasedEdge tmp = edge.result; file_out_stream.write((char *)&tmp, sizeof(NodeBasedEdge)); + used_edges_counter++; } + if (!file_out_stream) + { + throw util::exception(std::string("Error writing edges in extraction containers: ") + + SOURCE_REF); + } + if (used_edges_counter > std::numeric_limits::max()) { throw util::exception("There are too many edges, OSRM only supports 2^32" + SOURCE_REF); @@ -591,6 +605,13 @@ void ExtractionContainers::WriteEdges(std::ofstream &file_out_stream) const file_out_stream.seekp(start_position); file_out_stream.write((char *)&used_edges_counter_buffer, sizeof(used_edges_counter_buffer)); + + if (!file_out_stream) + { + throw util::exception( + std::string("Error writing number of used edges in extraction containers: ") + + SOURCE_REF); + } log << "ok"; } @@ -632,6 +653,12 @@ void ExtractionContainers::WriteNodes(std::ofstream &file_out_stream) const BOOST_ASSERT(*node_id_iterator == node_iterator->node_id); file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode)); + if (!file_out_stream) + { + throw util::exception( + std::string("Error writing size of edges buffer in extraction containers: ") + + SOURCE_REF); + } ++node_id_iterator; ++node_iterator; @@ -651,6 +678,7 @@ void ExtractionContainers::WriteRestrictions(const std::string &path) const restrictions_out_stream.open(path.c_str(), std::ios::binary); const util::FingerPrint fingerprint = util::FingerPrint::GetValid(); restrictions_out_stream.write((char *)&fingerprint, sizeof(util::FingerPrint)); + const auto count_position = restrictions_out_stream.tellp(); restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned)); @@ -665,8 +693,14 @@ void ExtractionContainers::WriteRestrictions(const std::string &path) const ++written_restriction_count; } } + restrictions_out_stream.seekp(count_position); restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned)); + if (!restrictions_out_stream) + { + throw util::exception(std::string("Error writing restrictions in extraction containers: ") + + SOURCE_REF); + } util::Log() << "usable restrictions: " << written_restriction_count; } diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index 4eeed9f9e..80edb4639 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -166,6 +166,12 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) boost::filesystem::ofstream timestamp_out(config.timestamp_file_name); timestamp_out.write(timestamp.c_str(), timestamp.length()); + if (!timestamp_out) + { + throw util::exception(std::string("Error writing timestamp in extractor: ") + + SOURCE_REF); + } + // initialize vectors holding parsed objects tbb::concurrent_vector> resulting_nodes; tbb::concurrent_vector> resulting_ways; @@ -317,6 +323,12 @@ void Extractor::WriteProfileProperties(const std::string &output_path, } out_stream.write(reinterpret_cast(&properties), sizeof(properties)); + + if (!out_stream) + { + throw util::exception(std::string("Error writing profile properties in extractor: ") + + SOURCE_REF); + } } void Extractor::FindComponents(unsigned max_edge_id, @@ -536,6 +548,11 @@ void Extractor::WriteNodeMapping(const std::vector &internal_to_exter node_stream.write((char *)internal_to_external_node_map.data(), size_of_mapping * sizeof(QueryNode)); } + if (!node_stream) + { + throw util::exception(std::string("Error writing node mapping in extractor: ") + + SOURCE_REF); + } } /** @@ -608,6 +625,12 @@ void Extractor::WriteEdgeBasedGraph( file_out_stream.write((char *)&edge, sizeof(EdgeBasedEdge)); } + if (!file_out_stream) + { + throw util::exception(std::string("Error writing edge based graph in extractor: ") + + SOURCE_REF); + } + TIMER_STOP(write_edges); util::Log() << "ok, after " << TIMER_SEC(write_edges) << "s"; @@ -647,6 +670,14 @@ void Extractor::WriteIntersectionClassificationData( file_out_stream << bearing_class_range_table; file_out_stream.write(reinterpret_cast(&total_bearings), sizeof(total_bearings)); + + if (!file_out_stream) + { + throw util::exception( + std::string("Error in writing interestction classification data in extractor: ") + + SOURCE_REF); + } + for (const auto &bearing_class : bearing_classes) { const auto &bearings = bearing_class.getAvailableBearings();