Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Use an enum type for the 'should read fingerprint' flag, rather than a mysterious boolean

Browse Source 
Fix tests.
This commit is contained in:
Daniel Patterson 2016-11-15 14:49:28 -08:00
parent 7b1131b982
commit 53ef2e2955
7 changed files with 58 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
include/storage/io.hpp
View File

@ -31,19 +31,24 @@ class FileReader
boost::filesystem::ifstream input_stream; boost::filesystem::ifstream input_stream;
public: public:
FileReader(const std::string &filename, const bool check_fingerprint = false) enum FingerprintFlag
: FileReader(boost::filesystem::path(filename), check_fingerprint) {
VerifyFingerprint,
HasNoFingerprint
};
FileReader(const std::string &filename, const FingerprintFlag flag)
: FileReader(boost::filesystem::path(filename), flag)
{ {
} }
FileReader(const boost::filesystem::path &filename_, const bool check_fingerprint = false) FileReader(const boost::filesystem::path &filename_, const FingerprintFlag flag)
: filename(filename_.string()) : filename(filename_.string())
{ {
input_stream.open(filename_, std::ios::binary); input_stream.open(filename_, std::ios::binary);
if (!input_stream) if (!input_stream)
throw util::exception("Error opening " + filename + ":" + std::strerror(errno)); throw util::exception("Error opening " + filename + ":" + std::strerror(errno));
if (check_fingerprint && !ReadAndCheckFingerprint()) if (flag == VerifyFingerprint && !ReadAndCheckFingerprint())
{ {
throw util::exception("Fingerprint mismatch in " + filename); throw util::exception("Fingerprint mismatch in " + filename);
} }

1
include/storage/shared_datatype.hpp
View File

@ -7,7 +7,6 @@
#include <cstdint> #include <cstdint>
#include <array> #include <array>
#include <iostream>
namespace osrm namespace osrm
{ {

20
include/util/io.hpp
View File

@ -30,12 +30,8 @@ inline bool writeFingerprint(std::ostream &stream)
} }
template <typename simple_type> template <typename simple_type>
bool serializeVector(const std::string &filename, const std::vector<simple_type> &data) bool serializeVector(std::ostream &stream, const std::vector<simple_type> &data)
{ {
std::ofstream stream(filename, std::ios::binary);
writeFingerprint(stream);
std::uint64_t count = data.size(); std::uint64_t count = data.size();
stream.write(reinterpret_cast<const char *>(&count), sizeof(count)); stream.write(reinterpret_cast<const char *>(&count), sizeof(count));
if (!data.empty()) if (!data.empty())
@ -44,13 +40,13 @@ bool serializeVector(const std::string &filename, const std::vector<simple_type>
} }
template <typename simple_type> template <typename simple_type>
bool serializeVector(std::ostream &stream, const std::vector<simple_type> &data) bool serializeVector(const std::string &filename, const std::vector<simple_type> &data)
{ {
std::uint64_t count = data.size(); std::ofstream stream(filename, std::ios::binary);
stream.write(reinterpret_cast<const char *>(&count), sizeof(count));
if (!data.empty()) writeFingerprint(stream);
stream.write(reinterpret_cast<const char *>(&data[0]), sizeof(simple_type) * count);
return static_cast<bool>(stream); return serializeVector(stream, data);
} }
// serializes a vector of vectors into an adjacency array (creates a copy of the data internally) // serializes a vector of vectors into an adjacency array (creates a copy of the data internally)
@ -116,7 +112,7 @@ void deserializeAdjacencyArray(const std::string &filename,
std::vector<std::uint32_t> &offsets, std::vector<std::uint32_t> &offsets,
std::vector<simple_type> &data) std::vector<simple_type> &data)
{ {
storage::io::FileReader file(filename); storage::io::FileReader file(filename, storage::io::FileReader::HasNoFingerprint);
file.DeserializeVector(offsets); file.DeserializeVector(offsets);
file.DeserializeVector(data); file.DeserializeVector(data);

3
include/util/static_rtree.hpp
View File

@ -346,7 +346,8 @@ class StaticRTree
const CoordinateListT &coordinate_list) const CoordinateListT &coordinate_list)
: m_coordinate_list(coordinate_list) : m_coordinate_list(coordinate_list)
{ {
storage::io::FileReader tree_node_file(node_file); storage::io::FileReader tree_node_file(node_file,
storage::io::FileReader::HasNoFingerprint);
const auto tree_size = tree_node_file.ReadElementCount64(); const auto tree_size = tree_node_file.ReadElementCount64();

3
src/contractor/contractor.cpp
View File

@ -167,7 +167,8 @@ int Contractor::Run()
std::string node_file_name = config.osrm_input_path.string() + ".enw"; std::string node_file_name = config.osrm_input_path.string() + ".enw";
{ {
storage::io::FileReader node_file(node_file_name, true); storage::io::FileReader node_file(node_file_name,
storage::io::FileReader::VerifyFingerprint);
node_file.DeserializeVector(node_weights); node_file.DeserializeVector(node_weights);
} }
util::SimpleLogger().Write() << "Done reading node weights."; util::SimpleLogger().Write() << "Done reading node weights.";

60
src/storage/storage.cpp
View File

@ -261,7 +261,7 @@ void Storage::PopulateLayout(DataLayout &layout)
// collect number of elements to store in shared memory object // collect number of elements to store in shared memory object
util::SimpleLogger().Write() << "load names from: " << config.names_data_path; util::SimpleLogger().Write() << "load names from: " << config.names_data_path;
// number of entries in name index // number of entries in name index
io::FileReader name_file(config.names_data_path); io::FileReader name_file(config.names_data_path, io::FileReader::HasNoFingerprint);
const auto name_blocks = name_file.ReadElementCount32(); const auto name_blocks = name_file.ReadElementCount32();
layout.SetBlockSize<unsigned>(DataLayout::NAME_OFFSETS, name_blocks); layout.SetBlockSize<unsigned>(DataLayout::NAME_OFFSETS, name_blocks);
@ -287,7 +287,7 @@ void Storage::PopulateLayout(DataLayout &layout)
// Loading information for original edges // Loading information for original edges
{ {
io::FileReader edges_file(config.edges_data_path); io::FileReader edges_file(config.edges_data_path, io::FileReader::HasNoFingerprint);
const auto number_of_original_edges = edges_file.ReadElementCount64(); const auto number_of_original_edges = edges_file.ReadElementCount64();
// note: settings this all to the same size is correct, we extract them from the same struct // note: settings this all to the same size is correct, we extract them from the same struct
@ -306,7 +306,7 @@ void Storage::PopulateLayout(DataLayout &layout)
} }
{ {
io::FileReader hsgr_file(config.hsgr_data_path); io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::HasNoFingerprint);
const auto hsgr_header = io::readHSGRHeader(hsgr_file); const auto hsgr_header = io::readHSGRHeader(hsgr_file);
layout.SetBlockSize<unsigned>(DataLayout::HSGR_CHECKSUM, 1); layout.SetBlockSize<unsigned>(DataLayout::HSGR_CHECKSUM, 1);
@ -318,7 +318,7 @@ void Storage::PopulateLayout(DataLayout &layout)
// load rsearch tree size // load rsearch tree size
{ {
io::FileReader tree_node_file(config.ram_index_path); io::FileReader tree_node_file(config.ram_index_path, io::FileReader::HasNoFingerprint);
const auto tree_size = tree_node_file.ReadElementCount64(); const auto tree_size = tree_node_file.ReadElementCount64();
layout.SetBlockSize<RTreeNode>(DataLayout::R_SEARCH_TREE, tree_size); layout.SetBlockSize<RTreeNode>(DataLayout::R_SEARCH_TREE, tree_size);
@ -332,21 +332,21 @@ void Storage::PopulateLayout(DataLayout &layout)
// read timestampsize // read timestampsize
{ {
io::FileReader timestamp_file(config.timestamp_path); io::FileReader timestamp_file(config.timestamp_path, io::FileReader::HasNoFingerprint);
const auto timestamp_size = timestamp_file.Size(); const auto timestamp_size = timestamp_file.Size();
layout.SetBlockSize<char>(DataLayout::TIMESTAMP, timestamp_size); layout.SetBlockSize<char>(DataLayout::TIMESTAMP, timestamp_size);
} }
// load core marker size // load core marker size
{ {
io::FileReader core_marker_file(config.core_data_path); io::FileReader core_marker_file(config.core_data_path, io::FileReader::HasNoFingerprint);
const auto number_of_core_markers = core_marker_file.ReadElementCount32(); const auto number_of_core_markers = core_marker_file.ReadElementCount32();
layout.SetBlockSize<unsigned>(DataLayout::CORE_MARKER, number_of_core_markers); layout.SetBlockSize<unsigned>(DataLayout::CORE_MARKER, number_of_core_markers);
} }
// load coordinate size // load coordinate size
{ {
io::FileReader node_file(config.nodes_data_path); io::FileReader node_file(config.nodes_data_path, io::FileReader::HasNoFingerprint);
const auto coordinate_list_size = node_file.ReadElementCount64(); const auto coordinate_list_size = node_file.ReadElementCount64();
layout.SetBlockSize<util::Coordinate>(DataLayout::COORDINATE_LIST, coordinate_list_size); layout.SetBlockSize<util::Coordinate>(DataLayout::COORDINATE_LIST, coordinate_list_size);
// we'll read a list of OSM node IDs from the same data, so set the block size for the same // we'll read a list of OSM node IDs from the same data, so set the block size for the same
@ -358,7 +358,7 @@ void Storage::PopulateLayout(DataLayout &layout)
// load geometries sizes // load geometries sizes
{ {
io::FileReader geometry_file(config.geometries_path); io::FileReader geometry_file(config.geometries_path, io::FileReader::HasNoFingerprint);
const auto number_of_geometries_indices = geometry_file.ReadElementCount32(); const auto number_of_geometries_indices = geometry_file.ReadElementCount32();
layout.SetBlockSize<unsigned>(DataLayout::GEOMETRIES_INDEX, number_of_geometries_indices); layout.SetBlockSize<unsigned>(DataLayout::GEOMETRIES_INDEX, number_of_geometries_indices);
@ -377,7 +377,8 @@ void Storage::PopulateLayout(DataLayout &layout)
// load datasource sizes. This file is optional, and it's non-fatal if it doesn't // load datasource sizes. This file is optional, and it's non-fatal if it doesn't
// exist. // exist.
{ {
io::FileReader geometry_datasource_file(config.datasource_indexes_path); io::FileReader geometry_datasource_file(config.datasource_indexes_path,
io::FileReader::HasNoFingerprint);
const auto number_of_compressed_datasources = geometry_datasource_file.ReadElementCount64(); const auto number_of_compressed_datasources = geometry_datasource_file.ReadElementCount64();
layout.SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST, layout.SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST,
number_of_compressed_datasources); number_of_compressed_datasources);
@ -386,7 +387,8 @@ void Storage::PopulateLayout(DataLayout &layout)
// Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't // Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't
// exist // exist
{ {
io::FileReader datasource_names_file(config.datasource_names_path); io::FileReader datasource_names_file(config.datasource_names_path,
io::FileReader::HasNoFingerprint);
const io::DatasourceNamesData datasource_names_data = const io::DatasourceNamesData datasource_names_data =
io::readDatasourceNames(datasource_names_file); io::readDatasourceNames(datasource_names_file);
@ -400,7 +402,8 @@ void Storage::PopulateLayout(DataLayout &layout)
} }
{ {
io::FileReader intersection_file(config.intersection_class_path, true); io::FileReader intersection_file(config.intersection_class_path,
io::FileReader::VerifyFingerprint);
std::vector<BearingClassID> bearing_class_id_table; std::vector<BearingClassID> bearing_class_id_table;
intersection_file.DeserializeVector(bearing_class_id_table); intersection_file.DeserializeVector(bearing_class_id_table);
@ -435,7 +438,7 @@ void Storage::PopulateLayout(DataLayout &layout)
{ {
// Loading turn lane data // Loading turn lane data
io::FileReader lane_data_file(config.turn_lane_data_path); io::FileReader lane_data_file(config.turn_lane_data_path, io::FileReader::HasNoFingerprint);
const auto lane_tuple_count = lane_data_file.ReadElementCount64(); const auto lane_tuple_count = lane_data_file.ReadElementCount64();
layout.SetBlockSize<util::guidance::LaneTupleIdPair>(DataLayout::TURN_LANE_DATA, layout.SetBlockSize<util::guidance::LaneTupleIdPair>(DataLayout::TURN_LANE_DATA,
lane_tuple_count); lane_tuple_count);
@ -450,7 +453,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Load the HSGR file // Load the HSGR file
{ {
io::FileReader hsgr_file(config.hsgr_data_path); io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::HasNoFingerprint);
auto hsgr_header = io::readHSGRHeader(hsgr_file); auto hsgr_header = io::readHSGRHeader(hsgr_file);
unsigned *checksum_ptr = unsigned *checksum_ptr =
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM); layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
@ -491,9 +494,9 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Name data // Name data
{ {
io::FileReader name_file(config.names_data_path); io::FileReader name_file(config.names_data_path, io::FileReader::HasNoFingerprint);
const auto name_blocks_count = name_file.ReadElementCount32(); const auto name_blocks_count = name_file.ReadElementCount32();
const auto name_char_list_count = name_file.ReadElementCount32(); name_file.Skip<std::uint32_t>(1); // name_char_list_count
using NameRangeTable = util::RangeTable<16, true>; using NameRangeTable = util::RangeTable<16, true>;
@ -528,7 +531,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Turn lane data // Turn lane data
{ {
io::FileReader lane_data_file(config.turn_lane_data_path); io::FileReader lane_data_file(config.turn_lane_data_path, io::FileReader::HasNoFingerprint);
const auto lane_tuple_count = lane_data_file.ReadElementCount64(); const auto lane_tuple_count = lane_data_file.ReadElementCount64();
@ -579,7 +582,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Load original edge data // Load original edge data
{ {
io::FileReader edges_input_file(config.edges_data_path); io::FileReader edges_input_file(config.edges_data_path, io::FileReader::HasNoFingerprint);
const auto number_of_original_edges = edges_input_file.ReadElementCount64(); const auto number_of_original_edges = edges_input_file.ReadElementCount64();
@ -620,7 +623,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// load compressed geometry // load compressed geometry
{ {
io::FileReader geometry_input_file(config.geometries_path); io::FileReader geometry_input_file(config.geometries_path,
io::FileReader::HasNoFingerprint);
const auto geometry_index_count = geometry_input_file.ReadElementCount32(); const auto geometry_index_count = geometry_input_file.ReadElementCount32();
const auto geometries_index_ptr = const auto geometries_index_ptr =
@ -649,7 +653,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
} }
{ {
io::FileReader geometry_datasource_file(config.datasource_indexes_path); io::FileReader geometry_datasource_file(config.datasource_indexes_path,
io::FileReader::HasNoFingerprint);
const auto number_of_compressed_datasources = geometry_datasource_file.ReadElementCount64(); const auto number_of_compressed_datasources = geometry_datasource_file.ReadElementCount64();
// load datasource information (if it exists) // load datasource information (if it exists)
@ -664,7 +669,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
{ {
/* Load names */ /* Load names */
io::FileReader datasource_names_file(config.datasource_names_path); io::FileReader datasource_names_file(config.datasource_names_path,
io::FileReader::HasNoFingerprint);
const auto datasource_names_data = io::readDatasourceNames(datasource_names_file); const auto datasource_names_data = io::readDatasourceNames(datasource_names_file);
@ -711,7 +717,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Loading list of coordinates // Loading list of coordinates
{ {
io::FileReader nodes_file(config.nodes_data_path); io::FileReader nodes_file(config.nodes_data_path, io::FileReader::HasNoFingerprint);
nodes_file.Skip<std::uint64_t>(1); // node_count nodes_file.Skip<std::uint64_t>(1); // node_count
const auto coordinates_ptr = const auto coordinates_ptr =
layout.GetBlockPtr<util::Coordinate, true>(memory_ptr, DataLayout::COORDINATE_LIST); layout.GetBlockPtr<util::Coordinate, true>(memory_ptr, DataLayout::COORDINATE_LIST);
@ -729,7 +735,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// store timestamp // store timestamp
{ {
io::FileReader timestamp_file(config.timestamp_path); io::FileReader timestamp_file(config.timestamp_path, io::FileReader::HasNoFingerprint);
const auto timestamp_size = timestamp_file.Size(); const auto timestamp_size = timestamp_file.Size();
const auto timestamp_ptr = const auto timestamp_ptr =
@ -740,7 +746,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// store search tree portion of rtree // store search tree portion of rtree
{ {
io::FileReader tree_node_file(config.ram_index_path); io::FileReader tree_node_file(config.ram_index_path, io::FileReader::HasNoFingerprint);
// perform this read so that we're at the right stream position for the next // perform this read so that we're at the right stream position for the next
// read. // read.
tree_node_file.Skip<std::uint64_t>(1); tree_node_file.Skip<std::uint64_t>(1);
@ -751,7 +757,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
} }
{ {
io::FileReader core_marker_file(config.core_data_path); io::FileReader core_marker_file(config.core_data_path, io::FileReader::HasNoFingerprint);
const auto number_of_core_markers = core_marker_file.ReadElementCount32(); const auto number_of_core_markers = core_marker_file.ReadElementCount32();
// load core markers // load core markers
@ -785,7 +791,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// load profile properties // load profile properties
{ {
io::FileReader profile_properties_file(config.properties_path); io::FileReader profile_properties_file(config.properties_path,
io::FileReader::HasNoFingerprint);
const auto profile_properties_ptr = layout.GetBlockPtr<extractor::ProfileProperties, true>( const auto profile_properties_ptr = layout.GetBlockPtr<extractor::ProfileProperties, true>(
memory_ptr, DataLayout::PROPERTIES); memory_ptr, DataLayout::PROPERTIES);
profile_properties_file.ReadInto(profile_properties_ptr, profile_properties_file.ReadInto(profile_properties_ptr,
@ -794,7 +801,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Load intersection data // Load intersection data
{ {
io::FileReader intersection_file(config.intersection_class_path, true); io::FileReader intersection_file(config.intersection_class_path,
io::FileReader::VerifyFingerprint);
std::vector<BearingClassID> bearing_class_id_table; std::vector<BearingClassID> bearing_class_id_table;
intersection_file.DeserializeVector(bearing_class_id_table); intersection_file.DeserializeVector(bearing_class_id_table);

5
unit_tests/util/io.cpp
View File

@ -20,8 +20,9 @@ BOOST_AUTO_TEST_CASE(io_data)
osrm::util::serializeVector(IO_TMP_FILE, data_in); osrm::util::serializeVector(IO_TMP_FILE, data_in);
osrm::storage::io::FileReader f(IO_TMP_FILE); osrm::storage::io::FileReader infile(IO_TMP_FILE,
f.DeserializeVector(data_out); osrm::storage::io::FileReader::VerifyFingerprint);
infile.DeserializeVector(data_out);
BOOST_REQUIRE_EQUAL(data_in.size(), data_out.size()); BOOST_REQUIRE_EQUAL(data_in.size(), data_out.size());
BOOST_CHECK_EQUAL_COLLECTIONS(data_out.begin(), data_out.end(), data_in.begin(), data_in.end()); BOOST_CHECK_EQUAL_COLLECTIONS(data_out.begin(), data_out.end(), data_in.begin(), data_in.end());