diff --git a/CHANGELOG.md b/CHANGELOG.md index 58d2d79ed..440469e2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ - Removed the `./profile.lua -> ./profiles/car.lua` symlink. Use specific profiles from the `profiles` directory. - Infrastructure - Disabled link-time optimized (LTO) builds by default. Enable by passing `-DENABLE_LTO=ON` to `cmake` if you need the performance and know what you are doing. + - File handling + - Datafile versioning is now based on OSRM semver values, rather than source code checksums. + Datafiles are compatible between patch levels, but incompatible between minor version or higher bumps. # 5.5.1 - Changes from 5.5.0 diff --git a/CMakeLists.txt b/CMakeLists.txt index f1219f6ae..8a1c4f8e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,13 +107,6 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/include/) include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/sol2/) include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/variant/include) -add_custom_target(FingerPrintConfigure ALL ${CMAKE_COMMAND} - "-DOUTPUT_DIR=${CMAKE_CURRENT_BINARY_DIR}" - "-DSOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}" - -P "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FingerPrint-Config.cmake" - COMMENT "Configuring revision fingerprint" - VERBATIM) - set(BOOST_COMPONENTS date_time chrono filesystem iostreams program_options regex system thread unit_test_framework) configure_file( @@ -134,7 +127,6 @@ add_library(STORAGE OBJECT ${StorageGlob}) add_library(ENGINE OBJECT ${EngineGlob}) add_library(SERVER OBJECT ${ServerGlob}) -add_dependencies(UTIL FingerPrintConfigure) set_target_properties(UTIL PROPERTIES LINKER_LANGUAGE CXX) add_executable(osrm-extract src/tools/extract.cpp) diff --git a/cmake/FingerPrint-Config.cmake b/cmake/FingerPrint-Config.cmake deleted file mode 100644 index 552659a8e..000000000 --- a/cmake/FingerPrint-Config.cmake +++ /dev/null @@ -1,24 +0,0 @@ -set(OLDFILE ${OUTPUT_DIR}/include/util/fingerprint_impl.hpp) -set(NEWFILE ${OLDFILE}.tmp) -set(INFILE ${SOURCE_DIR}/include/util/fingerprint_impl.hpp.in) -file(MD5 ${SOURCE_DIR}/src/tools/contract.cpp MD5PREPARE) -file(MD5 ${SOURCE_DIR}/include/util/static_rtree.hpp MD5RTREE) -file(MD5 ${SOURCE_DIR}/include/util/graph_loader.hpp MD5GRAPH) -file(MD5 ${SOURCE_DIR}/src/storage/storage.cpp MD5OBJECTS) - -CONFIGURE_FILE(${INFILE} ${NEWFILE}) - -file(MD5 ${NEWFILE} MD5NEW) - -if (EXISTS ${OLDFILE}) - file(MD5 ${OLDFILE} MD5OLD) - if(NOT ${MD5NEW} STREQUAL ${MD5OLD}) - file(REMOVE_RECURSE ${OLDFILE}) - file(RENAME ${NEWFILE} ${OLDFILE}) - else() - file(REMOVE_RECURSE ${NEWFILE}) - message(STATUS "Fingerprint unchanged, not regenerating") - endif() -else() - file(RENAME ${NEWFILE} ${OLDFILE}) -endif() diff --git a/include/storage/io.hpp b/include/storage/io.hpp index 5c05bf435..35a781dac 100644 --- a/include/storage/io.hpp +++ b/include/storage/io.hpp @@ -5,6 +5,7 @@ #include "util/exception_utils.hpp" #include "util/fingerprint.hpp" #include "util/log.hpp" +#include "util/version.hpp" #include #include @@ -117,12 +118,31 @@ class FileReader bool ReadAndCheckFingerprint() { - auto fingerprint = ReadOne(); - const auto valid = util::FingerPrint::GetValid(); - // compare the compilation state stored in the fingerprint - return valid.IsMagicNumberOK(fingerprint) && valid.TestContractor(fingerprint) && - valid.TestGraphUtil(fingerprint) && valid.TestRTree(fingerprint) && - valid.TestQueryObjects(fingerprint); + auto loaded_fingerprint = ReadOne(); + const auto expected_fingerprint = util::FingerPrint::GetValid(); + + if (!loaded_fingerprint.IsValid()) + { + util::Log(logERROR) << "Fingerprint magic number or checksum is invalid in " + << filepath.string(); + return false; + } + + if (!expected_fingerprint.IsDataCompatible(loaded_fingerprint)) + { + util::Log(logERROR) << filepath.string() + << " is not compatible with this version of OSRM"; + + util::Log(logERROR) << "It was prepared with OSRM " + << loaded_fingerprint.GetMajorVersion() << "." + << loaded_fingerprint.GetMinorVersion() << "." + << loaded_fingerprint.GetPatchVersion() << " but you are running " + << OSRM_VERSION; + util::Log(logERROR) << "Data is only compatible between minor releases."; + return false; + } + + return true; } std::size_t Size() diff --git a/include/storage/serialization.hpp b/include/storage/serialization.hpp index 41bbee41e..086708ea1 100644 --- a/include/storage/serialization.hpp +++ b/include/storage/serialization.hpp @@ -40,13 +40,6 @@ struct HSGRHeader // file and returns them in a HSGRHeader struct inline HSGRHeader readHSGRHeader(io::FileReader &input_file) { - const util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid(); - const auto fingerprint_loaded = input_file.ReadOne(); - if (!fingerprint_loaded.TestGraphUtil(fingerprint_valid)) - { - util::Log(logWARNING) << ".hsgr was prepared with different build.\n" - "Reprocess to get rid of this warning."; - } HSGRHeader header; input_file.ReadInto(header.checksum); diff --git a/include/util/fingerprint.hpp b/include/util/fingerprint.hpp index edeb52d43..aaf89131d 100644 --- a/include/util/fingerprint.hpp +++ b/include/util/fingerprint.hpp @@ -1,7 +1,9 @@ #ifndef FINGERPRINT_H #define FINGERPRINT_H +#include #include +#include #include namespace osrm @@ -14,25 +16,25 @@ class FingerPrint { public: static FingerPrint GetValid(); - const boost::uuids::uuid &GetFingerPrint() const; - bool IsMagicNumberOK(const FingerPrint &other) const; - bool TestGraphUtil(const FingerPrint &other) const; - bool TestContractor(const FingerPrint &other) const; - bool TestRTree(const FingerPrint &other) const; - bool TestQueryObjects(const FingerPrint &other) const; + + bool IsValid() const; + bool IsDataCompatible(const FingerPrint &other) const; + + int GetMajorVersion() const; + int GetMinorVersion() const; + int GetPatchVersion() const; private: - unsigned magic_number; - char md5_prepare[33]; - char md5_tree[33]; - char md5_graph[33]; - char md5_objects[33]; - - // initialize to {6ba7b810-9dad-11d1-80b4-00c04fd430c8} - boost::uuids::uuid named_uuid; + std::uint8_t CalculateChecksum() const; + // Here using std::array so that == can be used to conveniently compare contents + std::array magic_number; + std::uint8_t major_version; + std::uint8_t minor_version; + std::uint8_t patch_version; + std::uint8_t checksum; // CRC8 of the previous bytes to ensure the fingerprint is not damaged }; -static_assert(sizeof(FingerPrint) == 152, "FingerPrint has unexpected size"); +static_assert(sizeof(FingerPrint) == 8, "FingerPrint has unexpected size"); static_assert(std::is_trivial::value, "FingerPrint needs to be trivial."); } } diff --git a/include/util/fingerprint_impl.hpp.in b/include/util/fingerprint_impl.hpp.in deleted file mode 100644 index aaca9e06d..000000000 --- a/include/util/fingerprint_impl.hpp.in +++ /dev/null @@ -1,95 +0,0 @@ -#include "util/fingerprint.hpp" -#include "util/exception.hpp" -#include "util/exception_utils.hpp" - -#include -#include - -#include - -#include -#include - -#cmakedefine MD5PREPARE "${MD5PREPARE}" -#cmakedefine MD5RTREE "${MD5RTREE}" -#cmakedefine MD5GRAPH "${MD5GRAPH}" -#cmakedefine MD5OBJECTS "${MD5OBJECTS}" - -namespace osrm -{ -namespace util -{ - -FingerPrint FingerPrint::GetValid() -{ - FingerPrint fingerprint; - - fingerprint.magic_number = 1297240911; - fingerprint.md5_prepare[32] = fingerprint.md5_tree[32] = fingerprint.md5_graph[32] = fingerprint.md5_objects[32] = '\0'; - - // 6ba7b810-9dad-11d1-80b4-00c04fd430c8 is a Well Known UUID representing the DNS - // namespace. Its use here indicates that we own this part of the UUID space - // in the DNS realm. *Usually*, we would then generate a UUID based on "something.project-osrm.org", - // but this whole class is a hack. Anyway, named_uuid needs to be initialized to something - // before it can be used, and this is as good as anything else for these purposes. - fingerprint.named_uuid = boost::uuids::string_generator()( "{6ba7b810-9dad-11d1-80b4-00c04fd430c8}"); - - boost::uuids::name_generator gen(fingerprint.named_uuid); - std::string temp_string; - - std::memcpy(fingerprint.md5_prepare, MD5PREPARE, 32); - temp_string += fingerprint.md5_prepare; - std::memcpy(fingerprint.md5_tree, MD5RTREE, 32); - temp_string += fingerprint.md5_tree; - std::memcpy(fingerprint.md5_graph, MD5GRAPH, 32); - temp_string += fingerprint.md5_graph; - std::memcpy(fingerprint.md5_objects, MD5OBJECTS, 32); - temp_string += fingerprint.md5_objects; - - fingerprint.named_uuid = gen(temp_string); - - return fingerprint; -} - -const boost::uuids::uuid &FingerPrint::GetFingerPrint() const { return named_uuid; } - -bool FingerPrint::IsMagicNumberOK(const FingerPrint& other) const { return other.magic_number == magic_number; } - -bool FingerPrint::TestGraphUtil(const FingerPrint &other) const -{ - if (!IsMagicNumberOK(other)) - { - throw exception(std::string("hsgr input file misses magic number. Check or reprocess the file") + SOURCE_REF); - } - return std::equal(md5_graph, md5_graph + 32, other.md5_graph); -} - -bool FingerPrint::TestContractor(const FingerPrint &other) const -{ - if (!IsMagicNumberOK(other)) - { - throw exception(std::string("osrm input file misses magic number. Check or reprocess the file") + SOURCE_REF); - } - return std::equal(md5_prepare, md5_prepare + 32, other.md5_prepare); -} - -bool FingerPrint::TestRTree(const FingerPrint &other) const -{ - if (!IsMagicNumberOK(other)) - { - throw exception(std::string("r-tree input file misses magic number. Check or reprocess the file") + SOURCE_REF); - } - return std::equal(md5_tree, md5_tree + 32, other.md5_tree); -} - -bool FingerPrint::TestQueryObjects(const FingerPrint &other) const -{ - if (!IsMagicNumberOK(other)) - { - throw exception(std::string("missing magic number. Check or reprocess the file") + SOURCE_REF); - } - return std::equal(md5_objects, md5_objects + 32, other.md5_objects); -} - -} -} diff --git a/include/util/version.hpp.in b/include/util/version.hpp.in index cb97dca12..7f830bb3b 100644 --- a/include/util/version.hpp.in +++ b/include/util/version.hpp.in @@ -1,10 +1,12 @@ #ifndef VERSION_HPP #define VERSION_HPP -#define OSRM_VERSION_MAJOR "@OSRM_VERSION_MAJOR@" -#define OSRM_VERSION_MINOR "@OSRM_VERSION_MINOR@" -#define OSRM_VERSION_PATCH "@OSRM_VERSION_PATCH@" +#define OSRM_VERSION_MAJOR @OSRM_VERSION_MAJOR@ +#define OSRM_VERSION_MINOR @OSRM_VERSION_MINOR@ +#define OSRM_VERSION_PATCH @OSRM_VERSION_PATCH@ -#define OSRM_VERSION "v" OSRM_VERSION_MAJOR "." OSRM_VERSION_MINOR "." OSRM_VERSION_PATCH +#define OSRM_VERSION__(A,B,C) "v" #A "." #B "." #C +#define OSRM_VERSION_(A,B,C) OSRM_VERSION__(A,B,C) +#define OSRM_VERSION OSRM_VERSION_(OSRM_VERSION_MAJOR, OSRM_VERSION_MINOR, OSRM_VERSION_PATCH) #endif // VERSION_HPP diff --git a/src/contractor/contractor.cpp b/src/contractor/contractor.cpp index ce482c0e0..6ba3f2bb8 100644 --- a/src/contractor/contractor.cpp +++ b/src/contractor/contractor.cpp @@ -584,8 +584,25 @@ EdgeID Contractor::LoadEdgeExpandedGraph( const EdgeBasedGraphHeader graph_header = *(reinterpret_cast(edge_based_graph_region.get_address())); - const util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid(); - graph_header.fingerprint.TestContractor(fingerprint_valid); + const util::FingerPrint expected_fingerprint = util::FingerPrint::GetValid(); + if (!graph_header.fingerprint.IsValid()) + { + util::Log(logERROR) << edge_based_graph_filename << " does not have a valid fingerprint"; + throw util::exception("Invalid fingerprint"); + } + + if (!expected_fingerprint.IsDataCompatible(graph_header.fingerprint)) + { + util::Log(logERROR) << edge_based_graph_filename + << " is not compatible with this version of OSRM."; + util::Log(logERROR) << "It was prepared with OSRM " + << graph_header.fingerprint.GetMajorVersion() << "." + << graph_header.fingerprint.GetMinorVersion() << "." + << graph_header.fingerprint.GetPatchVersion() << " but you are running " + << OSRM_VERSION; + util::Log(logERROR) << "Data is only compatible between minor releases."; + throw util::exception("Incompatible file version" + SOURCE_REF); + } edge_based_edge_list.resize(graph_header.number_of_edges); util::Log() << "Reading " << graph_header.number_of_edges << " edges from the edge based graph"; diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 17d9f681a..4662eb6c3 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -290,7 +290,7 @@ void Storage::PopulateLayout(DataLayout &layout) } { - io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::HasNoFingerprint); + io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::VerifyFingerprint); const auto hsgr_header = serialization::readHSGRHeader(hsgr_file); layout.SetBlockSize(DataLayout::HSGR_CHECKSUM, 1); @@ -437,7 +437,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // Load the HSGR file { - io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::HasNoFingerprint); + io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::VerifyFingerprint); auto hsgr_header = serialization::readHSGRHeader(hsgr_file); unsigned *checksum_ptr = layout.GetBlockPtr(memory_ptr, DataLayout::HSGR_CHECKSUM); diff --git a/src/util/fingerprint.cpp b/src/util/fingerprint.cpp index 2764c7bab..d5e8e7ede 100644 --- a/src/util/fingerprint.cpp +++ b/src/util/fingerprint.cpp @@ -1,2 +1,95 @@ #include "util/fingerprint.hpp" -#include "util/fingerprint_impl.hpp" +#include "util/exception.hpp" +#include "util/exception_utils.hpp" +#include "util/version.hpp" + +#include +#include + +#include + +#include +#include + +namespace osrm +{ +namespace util +{ + +/** + * Constructs a valid fingerprint for the current (running) version of OSRM. + * This can be compared to one read from a file to determine whether the + * current code is compatible with the file being read. + */ +FingerPrint FingerPrint::GetValid() +{ + FingerPrint fingerprint; + + // 4 chars, 'O','S','R','N' - note the N instead of M, v1 of the fingerprint + // used M, so we add one and use N to indicate the newer fingerprint magic number. + // Bump this value if the fingerprint format ever changes. + fingerprint.magic_number = {{'O', 'S', 'R', 'N'}}; + fingerprint.major_version = OSRM_VERSION_MAJOR; + fingerprint.minor_version = OSRM_VERSION_MINOR; + fingerprint.patch_version = OSRM_VERSION_PATCH; + fingerprint.checksum = fingerprint.CalculateChecksum(); + + return fingerprint; +} + +int FingerPrint::GetMajorVersion() const { return major_version; } +int FingerPrint::GetMinorVersion() const { return minor_version; } +int FingerPrint::GetPatchVersion() const { return patch_version; } + +/** + * Calculates the CRC8 of the FingerPrint struct, using all bytes except the + * final `checksum` field, which should be last in the struct (this function + * checks that it is) + */ +std::uint8_t FingerPrint::CalculateChecksum() const +{ + // Verify that the checksum is a single byte (because we're returning an 8 bit checksum) + // This assumes that a byte == 8 bits, which is mostly true these days unless you're doing + // something really weird + static_assert(sizeof(checksum) == 1, "Checksum needs to be a single byte"); + const constexpr int CRC_BITS = 8; + + // This constant comes from + // https://en.wikipedia.org/wiki/Polynomial_representations_of_cyclic_redundancy_checks + // CRC-8-CCITT normal polynomial value. + const constexpr int CRC_POLYNOMIAL = 0x07; + boost::crc_optimal crc8; + + // Verify that the checksum is the last field, because we're going to CRC all the bytes + // leading up to it + static_assert(offsetof(FingerPrint, checksum) == sizeof(FingerPrint) - sizeof(checksum), + "Checksum must be the final field in the Fingerprint struct"); + + // Calculate checksum of all bytes except the checksum byte, which is at the end. + crc8.process_bytes(this, sizeof(FingerPrint) - sizeof(checksum)); + + return crc8.checksum(); +} + +/** + * Verifies that the fingerprint has the expected magic number, and the checksum is correct. + */ +bool FingerPrint::IsValid() const +{ + // Note: == on std::array compares contents, which is what we want here. + return magic_number == GetValid().magic_number && checksum == CalculateChecksum(); +} + +/** + * Determines whether two fingerprints are data compatible. + * Our compatibility rules say that we maintain data compatibility for all PATCH versions. + * A difference in either the MAJOR or MINOR version fields means the data is considered + * incompatible. + */ +bool FingerPrint::IsDataCompatible(const FingerPrint &other) const +{ + return IsValid() && other.major_version == major_version && + other.minor_version == minor_version; +} +} +} diff --git a/unit_tests/util/io.cpp b/unit_tests/util/io.cpp index 16d6944fa..75e72d07b 100644 --- a/unit_tests/util/io.cpp +++ b/unit_tests/util/io.cpp @@ -2,6 +2,7 @@ #include "storage/io.hpp" #include "util/exception.hpp" #include "util/typedefs.hpp" +#include "util/version.hpp" #include #include @@ -14,6 +15,8 @@ const static std::string IO_TMP_FILE = "test_io.tmp"; const static std::string IO_NONEXISTENT_FILE = "non_existent_test_io.tmp"; const static std::string IO_TOO_SMALL_FILE = "file_too_small_test_io.tmp"; const static std::string IO_CORRUPT_FINGERPRINT_FILE = "corrupt_fingerprint_file_test_io.tmp"; +const static std::string IO_INCOMPATIBLE_FINGERPRINT_FILE = + "incompatible_fingerprint_file_test_io.tmp"; const static std::string IO_TEXT_FILE = "plain_text_file.tmp"; BOOST_AUTO_TEST_SUITE(osrm_io) @@ -57,16 +60,18 @@ BOOST_AUTO_TEST_CASE(file_too_small) osrm::util::serializeVector(IO_TOO_SMALL_FILE, v); - std::ofstream f(IO_TOO_SMALL_FILE); - f.seekp(0, std::ios_base::beg); - std::uint64_t garbage = 0xDEADBEEFCAFEFACE; - f.write(reinterpret_cast(&garbage), sizeof(garbage)); + std::fstream f(IO_TOO_SMALL_FILE); + f.seekp(sizeof(osrm::util::FingerPrint), std::ios_base::beg); + std::uint64_t badcount = 100; + f.write(reinterpret_cast(&badcount), sizeof(badcount)); } try { osrm::storage::io::FileReader infile(IO_TOO_SMALL_FILE, osrm::storage::io::FileReader::VerifyFingerprint); + std::vector buffer; + infile.DeserializeVector(buffer); BOOST_REQUIRE_MESSAGE(false, "Should not get here"); } catch (const osrm::util::exception &e) @@ -105,6 +110,33 @@ BOOST_AUTO_TEST_CASE(io_corrupt_fingerprint) } } +BOOST_AUTO_TEST_CASE(io_incompatible_fingerprint) +{ + { + std::vector v(153); + std::iota(begin(v), end(v), 0); + osrm::util::serializeVector(IO_INCOMPATIBLE_FINGERPRINT_FILE, v); + + std::fstream f(IO_INCOMPATIBLE_FINGERPRINT_FILE); + f.seekp(5, std::ios_base::beg); // Seek past `OSRN` and Major version byte + std::uint8_t incompatibleminor = static_cast(OSRM_VERSION_MAJOR) + 1; + f.write(reinterpret_cast(&incompatibleminor), sizeof(incompatibleminor)); + } + + try + { + osrm::storage::io::FileReader infile(IO_INCOMPATIBLE_FINGERPRINT_FILE, + osrm::storage::io::FileReader::VerifyFingerprint); + BOOST_REQUIRE_MESSAGE(false, "Should not get here"); + } + catch (const osrm::util::exception &e) + { + const std::string expected("Fingerprint mismatch in " + IO_INCOMPATIBLE_FINGERPRINT_FILE); + const std::string got(e.what()); + BOOST_REQUIRE(std::equal(expected.begin(), expected.end(), got.begin())); + } +} + BOOST_AUTO_TEST_CASE(io_read_lines) { {