Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Replace fingerprint with semver-based scheme. (#3467)

Browse Source
This commit is contained in:
Daniel Patterson 2017-01-06 13:45:08 -08:00 committed by GitHub
parent c01ea2ea3e
commit f7e8581a1b
12 changed files with 203 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -8,6 +8,9 @@
- Removed the `./profile.lua -> ./profiles/car.lua` symlink. Use specific profiles from the `profiles` directory.
- Infrastructure
- Disabled link-time optimized (LTO) builds by default. Enable by passing `-DENABLE_LTO=ON` to `cmake` if you need the performance and know what you are doing.
- File handling
- Datafile versioning is now based on OSRM semver values, rather than source code checksums.
Datafiles are compatible between patch levels, but incompatible between minor version or higher bumps.
# 5.5.1
- Changes from 5.5.0

8
CMakeLists.txt
View File

@ -107,13 +107,6 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/include/)
include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/sol2/)
include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/variant/include)
add_custom_target(FingerPrintConfigure ALL ${CMAKE_COMMAND}
"-DOUTPUT_DIR=${CMAKE_CURRENT_BINARY_DIR}"
"-DSOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}"
-P "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FingerPrint-Config.cmake"
COMMENT "Configuring revision fingerprint"
VERBATIM)
set(BOOST_COMPONENTS date_time chrono filesystem iostreams program_options regex system thread unit_test_framework)
configure_file(
@ -134,7 +127,6 @@ add_library(STORAGE OBJECT ${StorageGlob})
add_library(ENGINE OBJECT ${EngineGlob})
add_library(SERVER OBJECT ${ServerGlob})
add_dependencies(UTIL FingerPrintConfigure)
set_target_properties(UTIL PROPERTIES LINKER_LANGUAGE CXX)
add_executable(osrm-extract src/tools/extract.cpp)

24
cmake/FingerPrint-Config.cmake
View File

@ -1,24 +0,0 @@
set(OLDFILE ${OUTPUT_DIR}/include/util/fingerprint_impl.hpp)
set(NEWFILE ${OLDFILE}.tmp)
set(INFILE ${SOURCE_DIR}/include/util/fingerprint_impl.hpp.in)
file(MD5 ${SOURCE_DIR}/src/tools/contract.cpp MD5PREPARE)
file(MD5 ${SOURCE_DIR}/include/util/static_rtree.hpp MD5RTREE)
file(MD5 ${SOURCE_DIR}/include/util/graph_loader.hpp MD5GRAPH)
file(MD5 ${SOURCE_DIR}/src/storage/storage.cpp MD5OBJECTS)
CONFIGURE_FILE(${INFILE} ${NEWFILE})
file(MD5 ${NEWFILE} MD5NEW)
if (EXISTS ${OLDFILE})
file(MD5 ${OLDFILE} MD5OLD)
if(NOT ${MD5NEW} STREQUAL ${MD5OLD})
file(REMOVE_RECURSE ${OLDFILE})
file(RENAME ${NEWFILE} ${OLDFILE})
else()
file(REMOVE_RECURSE ${NEWFILE})
message(STATUS "Fingerprint unchanged, not regenerating")
endif()
else()
file(RENAME ${NEWFILE} ${OLDFILE})
endif()

32
include/storage/io.hpp
View File

@ -5,6 +5,7 @@
#include "util/exception_utils.hpp"
#include "util/fingerprint.hpp"
#include "util/log.hpp"
#include "util/version.hpp"
#include <boost/filesystem/fstream.hpp>
#include <boost/iostreams/seek.hpp>
@ -117,12 +118,31 @@ class FileReader
bool ReadAndCheckFingerprint()
{
auto fingerprint = ReadOne<util::FingerPrint>();
const auto valid = util::FingerPrint::GetValid();
// compare the compilation state stored in the fingerprint
return valid.IsMagicNumberOK(fingerprint) && valid.TestContractor(fingerprint) &&
valid.TestGraphUtil(fingerprint) && valid.TestRTree(fingerprint) &&
valid.TestQueryObjects(fingerprint);
auto loaded_fingerprint = ReadOne<util::FingerPrint>();
const auto expected_fingerprint = util::FingerPrint::GetValid();
if (!loaded_fingerprint.IsValid())
{
util::Log(logERROR) << "Fingerprint magic number or checksum is invalid in "
<< filepath.string();
return false;
}
if (!expected_fingerprint.IsDataCompatible(loaded_fingerprint))
{
util::Log(logERROR) << filepath.string()
<< " is not compatible with this version of OSRM";
util::Log(logERROR) << "It was prepared with OSRM "
<< loaded_fingerprint.GetMajorVersion() << "."
<< loaded_fingerprint.GetMinorVersion() << "."
<< loaded_fingerprint.GetPatchVersion() << " but you are running "
<< OSRM_VERSION;
util::Log(logERROR) << "Data is only compatible between minor releases.";
return false;
}
return true;
}
std::size_t Size()

7
include/storage/serialization.hpp
View File

@ -40,13 +40,6 @@ struct HSGRHeader
// file and returns them in a HSGRHeader struct
inline HSGRHeader readHSGRHeader(io::FileReader &input_file)
{
const util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid();
const auto fingerprint_loaded = input_file.ReadOne<util::FingerPrint>();
if (!fingerprint_loaded.TestGraphUtil(fingerprint_valid))
{
util::Log(logWARNING) << ".hsgr was prepared with different build.\n"
"Reprocess to get rid of this warning.";
}
HSGRHeader header;
input_file.ReadInto(header.checksum);

32
include/util/fingerprint.hpp
View File

@ -1,7 +1,9 @@
#ifndef FINGERPRINT_H
#define FINGERPRINT_H
#include <array>
#include <boost/uuid/uuid.hpp>
#include <cstdint>
#include <type_traits>
namespace osrm
@ -14,25 +16,25 @@ class FingerPrint
{
public:
static FingerPrint GetValid();
const boost::uuids::uuid &GetFingerPrint() const;
bool IsMagicNumberOK(const FingerPrint &other) const;
bool TestGraphUtil(const FingerPrint &other) const;
bool TestContractor(const FingerPrint &other) const;
bool TestRTree(const FingerPrint &other) const;
bool TestQueryObjects(const FingerPrint &other) const;
bool IsValid() const;
bool IsDataCompatible(const FingerPrint &other) const;
int GetMajorVersion() const;
int GetMinorVersion() const;
int GetPatchVersion() const;
private:
unsigned magic_number;
char md5_prepare[33];
char md5_tree[33];
char md5_graph[33];
char md5_objects[33];
// initialize to {6ba7b810-9dad-11d1-80b4-00c04fd430c8}
boost::uuids::uuid named_uuid;
std::uint8_t CalculateChecksum() const;
// Here using std::array so that == can be used to conveniently compare contents
std::array<std::uint8_t, 4> magic_number;
std::uint8_t major_version;
std::uint8_t minor_version;
std::uint8_t patch_version;
std::uint8_t checksum; // CRC8 of the previous bytes to ensure the fingerprint is not damaged
};
static_assert(sizeof(FingerPrint) == 152, "FingerPrint has unexpected size");
static_assert(sizeof(FingerPrint) == 8, "FingerPrint has unexpected size");
static_assert(std::is_trivial<FingerPrint>::value, "FingerPrint needs to be trivial.");
}
}

95
include/util/fingerprint_impl.hpp.in
View File

@ -1,95 +0,0 @@
#include "util/fingerprint.hpp"
#include "util/exception.hpp"
#include "util/exception_utils.hpp"
#include <boost/uuid/name_generator.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include <cstring>
#include <algorithm>
#include <string>
#cmakedefine MD5PREPARE "${MD5PREPARE}"
#cmakedefine MD5RTREE "${MD5RTREE}"
#cmakedefine MD5GRAPH "${MD5GRAPH}"
#cmakedefine MD5OBJECTS "${MD5OBJECTS}"
namespace osrm
{
namespace util
{
FingerPrint FingerPrint::GetValid()
{
FingerPrint fingerprint;
fingerprint.magic_number = 1297240911;
fingerprint.md5_prepare[32] = fingerprint.md5_tree[32] = fingerprint.md5_graph[32] = fingerprint.md5_objects[32] = '\0';
// 6ba7b810-9dad-11d1-80b4-00c04fd430c8 is a Well Known UUID representing the DNS
// namespace. Its use here indicates that we own this part of the UUID space
// in the DNS realm. *Usually*, we would then generate a UUID based on "something.project-osrm.org",
// but this whole class is a hack. Anyway, named_uuid needs to be initialized to something
// before it can be used, and this is as good as anything else for these purposes.
fingerprint.named_uuid = boost::uuids::string_generator()( "{6ba7b810-9dad-11d1-80b4-00c04fd430c8}");
boost::uuids::name_generator gen(fingerprint.named_uuid);
std::string temp_string;
std::memcpy(fingerprint.md5_prepare, MD5PREPARE, 32);
temp_string += fingerprint.md5_prepare;
std::memcpy(fingerprint.md5_tree, MD5RTREE, 32);
temp_string += fingerprint.md5_tree;
std::memcpy(fingerprint.md5_graph, MD5GRAPH, 32);
temp_string += fingerprint.md5_graph;
std::memcpy(fingerprint.md5_objects, MD5OBJECTS, 32);
temp_string += fingerprint.md5_objects;
fingerprint.named_uuid = gen(temp_string);
return fingerprint;
}
const boost::uuids::uuid &FingerPrint::GetFingerPrint() const { return named_uuid; }
bool FingerPrint::IsMagicNumberOK(const FingerPrint& other) const { return other.magic_number == magic_number; }
bool FingerPrint::TestGraphUtil(const FingerPrint &other) const
{
if (!IsMagicNumberOK(other))
{
throw exception(std::string("hsgr input file misses magic number. Check or reprocess the file") + SOURCE_REF);
}
return std::equal(md5_graph, md5_graph + 32, other.md5_graph);
}
bool FingerPrint::TestContractor(const FingerPrint &other) const
{
if (!IsMagicNumberOK(other))
{
throw exception(std::string("osrm input file misses magic number. Check or reprocess the file") + SOURCE_REF);
}
return std::equal(md5_prepare, md5_prepare + 32, other.md5_prepare);
}
bool FingerPrint::TestRTree(const FingerPrint &other) const
{
if (!IsMagicNumberOK(other))
{
throw exception(std::string("r-tree input file misses magic number. Check or reprocess the file") + SOURCE_REF);
}
return std::equal(md5_tree, md5_tree + 32, other.md5_tree);
}
bool FingerPrint::TestQueryObjects(const FingerPrint &other) const
{
if (!IsMagicNumberOK(other))
{
throw exception(std::string("missing magic number. Check or reprocess the file") + SOURCE_REF);
}
return std::equal(md5_objects, md5_objects + 32, other.md5_objects);
}
}
}

10
include/util/version.hpp.in
View File

@ -1,10 +1,12 @@
#ifndef VERSION_HPP
#define VERSION_HPP
#define OSRM_VERSION_MAJOR "@OSRM_VERSION_MAJOR@"
#define OSRM_VERSION_MINOR "@OSRM_VERSION_MINOR@"
#define OSRM_VERSION_PATCH "@OSRM_VERSION_PATCH@"
#define OSRM_VERSION_MAJOR @OSRM_VERSION_MAJOR@
#define OSRM_VERSION_MINOR @OSRM_VERSION_MINOR@
#define OSRM_VERSION_PATCH @OSRM_VERSION_PATCH@
#define OSRM_VERSION "v" OSRM_VERSION_MAJOR "." OSRM_VERSION_MINOR "." OSRM_VERSION_PATCH
#define OSRM_VERSION__(A,B,C) "v" #A "." #B "." #C
#define OSRM_VERSION_(A,B,C) OSRM_VERSION__(A,B,C)
#define OSRM_VERSION OSRM_VERSION_(OSRM_VERSION_MAJOR, OSRM_VERSION_MINOR, OSRM_VERSION_PATCH)
#endif // VERSION_HPP

21
src/contractor/contractor.cpp
View File

@ -584,8 +584,25 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
const EdgeBasedGraphHeader graph_header =
*(reinterpret_cast<const EdgeBasedGraphHeader *>(edge_based_graph_region.get_address()));
const util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid();
graph_header.fingerprint.TestContractor(fingerprint_valid);
const util::FingerPrint expected_fingerprint = util::FingerPrint::GetValid();
if (!graph_header.fingerprint.IsValid())
{
util::Log(logERROR) << edge_based_graph_filename << " does not have a valid fingerprint";
throw util::exception("Invalid fingerprint");
}
if (!expected_fingerprint.IsDataCompatible(graph_header.fingerprint))
{
util::Log(logERROR) << edge_based_graph_filename
<< " is not compatible with this version of OSRM.";
util::Log(logERROR) << "It was prepared with OSRM "
<< graph_header.fingerprint.GetMajorVersion() << "."
<< graph_header.fingerprint.GetMinorVersion() << "."
<< graph_header.fingerprint.GetPatchVersion() << " but you are running "
<< OSRM_VERSION;
util::Log(logERROR) << "Data is only compatible between minor releases.";
throw util::exception("Incompatible file version" + SOURCE_REF);
}
edge_based_edge_list.resize(graph_header.number_of_edges);
util::Log() << "Reading " << graph_header.number_of_edges << " edges from the edge based graph";

4
src/storage/storage.cpp
View File

@ -290,7 +290,7 @@ void Storage::PopulateLayout(DataLayout &layout)
}
{
io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::HasNoFingerprint);
io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::VerifyFingerprint);
const auto hsgr_header = serialization::readHSGRHeader(hsgr_file);
layout.SetBlockSize<unsigned>(DataLayout::HSGR_CHECKSUM, 1);
@ -437,7 +437,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Load the HSGR file
{
io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::HasNoFingerprint);
io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::VerifyFingerprint);
auto hsgr_header = serialization::readHSGRHeader(hsgr_file);
unsigned *checksum_ptr =
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);

95
src/util/fingerprint.cpp
View File

@ -1,2 +1,95 @@
#include "util/fingerprint.hpp"
#include "util/fingerprint_impl.hpp"
#include "util/exception.hpp"
#include "util/exception_utils.hpp"
#include "util/version.hpp"
#include <boost/assert.hpp>
#include <boost/crc.hpp>
#include <cstring>
#include <algorithm>
#include <string>
namespace osrm
{
namespace util
{
/**
* Constructs a valid fingerprint for the current (running) version of OSRM.
* This can be compared to one read from a file to determine whether the
* current code is compatible with the file being read.
*/
FingerPrint FingerPrint::GetValid()
{
FingerPrint fingerprint;
// 4 chars, 'O','S','R','N' - note the N instead of M, v1 of the fingerprint
// used M, so we add one and use N to indicate the newer fingerprint magic number.
// Bump this value if the fingerprint format ever changes.
fingerprint.magic_number = {{'O', 'S', 'R', 'N'}};
fingerprint.major_version = OSRM_VERSION_MAJOR;
fingerprint.minor_version = OSRM_VERSION_MINOR;
fingerprint.patch_version = OSRM_VERSION_PATCH;
fingerprint.checksum = fingerprint.CalculateChecksum();
return fingerprint;
}
int FingerPrint::GetMajorVersion() const { return major_version; }
int FingerPrint::GetMinorVersion() const { return minor_version; }
int FingerPrint::GetPatchVersion() const { return patch_version; }
/**
* Calculates the CRC8 of the FingerPrint struct, using all bytes except the
* final `checksum` field, which should be last in the struct (this function
* checks that it is)
*/
std::uint8_t FingerPrint::CalculateChecksum() const
{
// Verify that the checksum is a single byte (because we're returning an 8 bit checksum)
// This assumes that a byte == 8 bits, which is mostly true these days unless you're doing
// something really weird
static_assert(sizeof(checksum) == 1, "Checksum needs to be a single byte");
const constexpr int CRC_BITS = 8;
// This constant comes from
// https://en.wikipedia.org/wiki/Polynomial_representations_of_cyclic_redundancy_checks
// CRC-8-CCITT normal polynomial value.
const constexpr int CRC_POLYNOMIAL = 0x07;
boost::crc_optimal<CRC_BITS, CRC_POLYNOMIAL> crc8;
// Verify that the checksum is the last field, because we're going to CRC all the bytes
// leading up to it
static_assert(offsetof(FingerPrint, checksum) == sizeof(FingerPrint) - sizeof(checksum),
"Checksum must be the final field in the Fingerprint struct");
// Calculate checksum of all bytes except the checksum byte, which is at the end.
crc8.process_bytes(this, sizeof(FingerPrint) - sizeof(checksum));
return crc8.checksum();
}
/**
* Verifies that the fingerprint has the expected magic number, and the checksum is correct.
*/
bool FingerPrint::IsValid() const
{
// Note: == on std::array compares contents, which is what we want here.
return magic_number == GetValid().magic_number && checksum == CalculateChecksum();
}
/**
* Determines whether two fingerprints are data compatible.
* Our compatibility rules say that we maintain data compatibility for all PATCH versions.
* A difference in either the MAJOR or MINOR version fields means the data is considered
* incompatible.
*/
bool FingerPrint::IsDataCompatible(const FingerPrint &other) const
{
return IsValid() && other.major_version == major_version &&
other.minor_version == minor_version;
}
}
}

40
unit_tests/util/io.cpp
View File

@ -2,6 +2,7 @@
#include "storage/io.hpp"
#include "util/exception.hpp"
#include "util/typedefs.hpp"
#include "util/version.hpp"
#include <boost/test/test_case_template.hpp>
#include <boost/test/unit_test.hpp>
@ -14,6 +15,8 @@ const static std::string IO_TMP_FILE = "test_io.tmp";
const static std::string IO_NONEXISTENT_FILE = "non_existent_test_io.tmp";
const static std::string IO_TOO_SMALL_FILE = "file_too_small_test_io.tmp";
const static std::string IO_CORRUPT_FINGERPRINT_FILE = "corrupt_fingerprint_file_test_io.tmp";
const static std::string IO_INCOMPATIBLE_FINGERPRINT_FILE =
"incompatible_fingerprint_file_test_io.tmp";
const static std::string IO_TEXT_FILE = "plain_text_file.tmp";
BOOST_AUTO_TEST_SUITE(osrm_io)
@ -57,16 +60,18 @@ BOOST_AUTO_TEST_CASE(file_too_small)
osrm::util::serializeVector(IO_TOO_SMALL_FILE, v);
std::ofstream f(IO_TOO_SMALL_FILE);
f.seekp(0, std::ios_base::beg);
std::uint64_t garbage = 0xDEADBEEFCAFEFACE;
f.write(reinterpret_cast<char *>(&garbage), sizeof(garbage));
std::fstream f(IO_TOO_SMALL_FILE);
f.seekp(sizeof(osrm::util::FingerPrint), std::ios_base::beg);
std::uint64_t badcount = 100;
f.write(reinterpret_cast<char *>(&badcount), sizeof(badcount));
}
try
{
osrm::storage::io::FileReader infile(IO_TOO_SMALL_FILE,
osrm::storage::io::FileReader::VerifyFingerprint);
std::vector<int> buffer;
infile.DeserializeVector(buffer);
BOOST_REQUIRE_MESSAGE(false, "Should not get here");
}
catch (const osrm::util::exception &e)
@ -105,6 +110,33 @@ BOOST_AUTO_TEST_CASE(io_corrupt_fingerprint)
}
}
BOOST_AUTO_TEST_CASE(io_incompatible_fingerprint)
{
{
std::vector<int> v(153);
std::iota(begin(v), end(v), 0);
osrm::util::serializeVector(IO_INCOMPATIBLE_FINGERPRINT_FILE, v);
std::fstream f(IO_INCOMPATIBLE_FINGERPRINT_FILE);
f.seekp(5, std::ios_base::beg); // Seek past `OSRN` and Major version byte
std::uint8_t incompatibleminor = static_cast<std::uint8_t>(OSRM_VERSION_MAJOR) + 1;
f.write(reinterpret_cast<char *>(&incompatibleminor), sizeof(incompatibleminor));
}
try
{
osrm::storage::io::FileReader infile(IO_INCOMPATIBLE_FINGERPRINT_FILE,
osrm::storage::io::FileReader::VerifyFingerprint);
BOOST_REQUIRE_MESSAGE(false, "Should not get here");
}
catch (const osrm::util::exception &e)
{
const std::string expected("Fingerprint mismatch in " + IO_INCOMPATIBLE_FINGERPRINT_FILE);
const std::string got(e.what());
BOOST_REQUIRE(std::equal(expected.begin(), expected.end(), got.begin()));
}
}
BOOST_AUTO_TEST_CASE(io_read_lines)
{
{