From 41d9aeb5b5a4a2f7e45da313ce446c02763a97d6 Mon Sep 17 00:00:00 2001 From: Lev Dragunov Date: Tue, 15 Jan 2019 13:18:20 +0300 Subject: [PATCH] Data timestamp version in response --- CHANGELOG.md | 1 + docs/http.md | 9 ++++++++- include/engine/api/route_api.hpp | 1 + .../contiguous_internalmem_datafacade.hpp | 15 ++++++++++++++ include/engine/datafacade/datafacade_base.hpp | 2 ++ include/extractor/extractor_config.hpp | 1 + include/extractor/files.hpp | 20 +++++++++++++++++++ include/storage/serialization.hpp | 12 +++++++++++ include/storage/storage_config.hpp | 1 + include/storage/view_factory.hpp | 5 +++++ include/util/typedefs.hpp | 1 + src/extractor/extractor.cpp | 7 +++++++ src/storage/storage.cpp | 7 +++++++ unit_tests/engine/offline_facade.cpp | 1 + unit_tests/library/route.cpp | 1 + unit_tests/mocks/mock_datafacade.hpp | 2 +- 16 files changed, 84 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d51ad2837..05b54c4bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ - Changes from 5.21.0 - Features: - ADDED: new waypoints parameter to the `route` plugin, enabling silent waypoints [#5345](https://github.com/Project-OSRM/osrm-backend/pull/5345) + - ADDED: data timestamp information in the response. [#5115](https://github.com/Project-OSRM/osrm-backend/issues/5115) # 5.21.0 - Changes from 5.20.0 diff --git a/docs/http.md b/docs/http.md index bd47fcc7e..f3ad7c463 100644 --- a/docs/http.md +++ b/docs/http.md @@ -70,6 +70,8 @@ curl 'http://router.project-osrm.org/route/v1/driving/polyline(ofp_Ik_vpAilAyu@t ### Responses +#### Code + Every response object has a `code` property containing one of the strings below or a service dependent code: | Type | Description | @@ -87,12 +89,17 @@ Every response object has a `code` property containing one of the strings below - `message` is a **optional** human-readable error message. All other status types are service dependent. - In case of an error the HTTP status code will be `400`. Otherwise the HTTP status code will be `200` and `code` will be `Ok`. +#### Data version + +Every response object has a `data_version` propetry containing timestamp from the original OpenStreetMap file. May be `n/a` if original OSM file has not `osmosis_replication_timestamp` section. + #### Example response ```json { "code": "Ok", -"message": "Everything worked" +"message": "Everything worked", +"data_version": "2017-11-17T21:43:02Z" } ``` diff --git a/include/engine/api/route_api.hpp b/include/engine/api/route_api.hpp index d5736cbfe..e5f537e0e 100644 --- a/include/engine/api/route_api.hpp +++ b/include/engine/api/route_api.hpp @@ -68,6 +68,7 @@ class RouteAPI : public BaseAPI response.values["waypoints"] = BaseAPI::MakeWaypoints(all_start_end_points); response.values["routes"] = std::move(jsRoutes); response.values["code"] = "Ok"; + response.values["data_version"] = facade.GetTimestamp(); } protected: diff --git a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp index 7aabc704e..66cfad502 100644 --- a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp +++ b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp @@ -137,6 +137,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade extractor::Datasources *m_datasources; std::uint32_t m_check_sum; + DataTimestamp m_data_timestamp; util::vector_view m_coordinate_list; extractor::PackedOSMIDsView m_osmnodeid_list; util::vector_view m_lane_description_offsets; @@ -183,6 +184,8 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade m_check_sum = *index.GetBlockPtr("/common/connectivity_checksum"); + m_data_timestamp = *index.GetBlockPtr("/common/timestamp"); + std::tie(m_coordinate_list, m_osmnodeid_list) = make_nbn_data_view(index, "/common/nbn_data"); @@ -432,6 +435,18 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade std::uint32_t GetCheckSum() const override final { return m_check_sum; } + std::string GetTimestamp() const override final + { + if (!m_data_timestamp) + { + return "n/a"; + } + time_t ts(m_data_timestamp); + char buf[21]; // sizeof "2019-01-01T01:01:01Z\n"]; + strftime(buf, 21, "%FT%TZ", gmtime(&ts)); + return std::string(buf, 20); + } + GeometryID GetGeometryIndex(const NodeID id) const override final { return edge_based_node_data.GetGeometryID(id); diff --git a/include/engine/datafacade/datafacade_base.hpp b/include/engine/datafacade/datafacade_base.hpp index 23b2bdae7..157dbb1ca 100644 --- a/include/engine/datafacade/datafacade_base.hpp +++ b/include/engine/datafacade/datafacade_base.hpp @@ -74,6 +74,8 @@ class BaseDataFacade virtual std::uint32_t GetCheckSum() const = 0; + virtual std::string GetTimestamp() const = 0; + // node and edge information access virtual util::Coordinate GetCoordinateOfNode(const NodeID id) const = 0; diff --git a/include/extractor/extractor_config.hpp b/include/extractor/extractor_config.hpp index 1bd305821..2b6ae064e 100644 --- a/include/extractor/extractor_config.hpp +++ b/include/extractor/extractor_config.hpp @@ -55,6 +55,7 @@ struct ExtractorConfig final : storage::IOConfig ".osrm.geometry", ".osrm.nbg_nodes", ".osrm.ebg_nodes", + ".osrm.timestamp", ".osrm.edges", ".osrm.ebg", ".osrm.ramIndex", diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index 0807e5c56..493c54765 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -308,6 +308,26 @@ inline void writeTurnLaneData(const boost::filesystem::path &path, storage::serialization::write(writer, "/common/turn_lanes/data", turn_lane_data); } +// reads .osrm.timestamp +template +inline void readTimestamp(const boost::filesystem::path &path, TimestampDataT ×tamp) +{ + const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; + storage::tar::FileReader reader{path, fingerprint}; + + storage::serialization::read(reader, "/common/timestamp", timestamp); +} + +// writes .osrm.timestamp +template +inline void writeTimestamp(const boost::filesystem::path &path, const TimestampDataT ×tamp) +{ + const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; + storage::tar::FileWriter writer{path, fingerprint}; + + storage::serialization::write(writer, "/common/timestamp", timestamp); +} + // reads .osrm.maneuver_overrides template inline void readManeuverOverrides(const boost::filesystem::path &path, diff --git a/include/storage/serialization.hpp b/include/storage/serialization.hpp index 97cba94ef..ee50b361e 100644 --- a/include/storage/serialization.hpp +++ b/include/storage/serialization.hpp @@ -201,6 +201,18 @@ inline void read(tar::FileReader &reader, const std::string &name, std::string & reader.ReadInto(name, const_cast(data.data()), count); } +inline void write(tar::FileWriter &writer, const std::string &name, const std::uint64_t &data) +{ + writer.WriteElementCount64(name, data); + writer.WriteFrom(name, &data, 1); +} + +inline void read(tar::FileReader &reader, const std::string &name, std::uint64_t &data) +{ + data = reader.ReadElementCount64(name); + reader.ReadInto(name, &data, 1); +} + template inline void read(tar::FileReader &reader, const std::string &name, std::vector &data) { diff --git a/include/storage/storage_config.hpp b/include/storage/storage_config.hpp index 4d133f087..83b51a0bd 100644 --- a/include/storage/storage_config.hpp +++ b/include/storage/storage_config.hpp @@ -58,6 +58,7 @@ struct StorageConfig final : IOConfig ".osrm.turn_duration_penalties", ".osrm.datasource_names", ".osrm.names", + ".osrm.timestamp", ".osrm.properties", ".osrm.icd", ".osrm.maneuver_overrides"}, diff --git a/include/storage/view_factory.hpp b/include/storage/view_factory.hpp index b04e3026e..659c4786c 100644 --- a/include/storage/view_factory.hpp +++ b/include/storage/view_factory.hpp @@ -272,6 +272,11 @@ inline auto make_partition_view(const SharedDataIndex &index, const std::string level_data_ptr, std::move(partition), std::move(cell_to_children)}; } +inline auto make_timestamp_view(const SharedDataIndex &index, const std::string &name) +{ + return index.template GetBlockPtr(name); +} + inline auto make_cell_storage_view(const SharedDataIndex &index, const std::string &name) { auto source_boundary = make_vector_view(index, name + "/source_boundary"); diff --git a/include/util/typedefs.hpp b/include/util/typedefs.hpp index d647a7db7..540645200 100644 --- a/include/util/typedefs.hpp +++ b/include/util/typedefs.hpp @@ -79,6 +79,7 @@ using EdgeDistance = float; using SegmentWeight = std::uint32_t; using SegmentDuration = std::uint32_t; using TurnPenalty = std::int16_t; // turn penalty in 100ms units +using DataTimestamp = std::uint64_t; static const std::size_t INVALID_INDEX = std::numeric_limits::max(); diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index abd0bab93..1de4c8889 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -51,6 +51,7 @@ #include #include #include +#include #include #include @@ -425,11 +426,17 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, // write .timestamp data file std::string timestamp = header.get("osmosis_replication_timestamp"); + osmium::Timestamp ts; if (timestamp.empty()) { timestamp = "n/a"; } + else + { + ts = osmium::Timestamp(timestamp); + } util::Log() << "timestamp: " << timestamp; + files::writeTimestamp(config.GetPath(".osrm.timestamp").string(), DataTimestamp(ts)); } // Extraction containers and restriction parser diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 503b1c10e..832f7b0d7 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -302,6 +302,7 @@ std::vector> Storage::GetStaticFiles() {REQUIRED, config.GetPath(".osrm.ebg_nodes")}, {REQUIRED, config.GetPath(".osrm.tls")}, {REQUIRED, config.GetPath(".osrm.tld")}, + {REQUIRED, config.GetPath(".osrm.timestamp")}, {REQUIRED, config.GetPath(".osrm.maneuver_overrides")}, {REQUIRED, config.GetPath(".osrm.edges")}, {REQUIRED, config.GetPath(".osrm.names")}, @@ -401,6 +402,12 @@ void Storage::PopulateStaticData(const SharedDataIndex &index) extractor::files::readNames(config.GetPath(".osrm.names"), name_table); } + // Timestamp mark + { + auto timestamp = make_timestamp_view(index, "/common/timestamp"); + extractor::files::readTimestamp(config.GetPath(".osrm.timestamp"), *timestamp); + } + // Turn lane data { auto turn_lane_data = make_lane_data_view(index, "/common/turn_lanes"); diff --git a/unit_tests/engine/offline_facade.cpp b/unit_tests/engine/offline_facade.cpp index bede4eabe..001140f98 100644 --- a/unit_tests/engine/offline_facade.cpp +++ b/unit_tests/engine/offline_facade.cpp @@ -341,6 +341,7 @@ class ContiguousInternalMemoryDataFacade StringView GetDestinationsForID(const NameID /*id*/) const override { return StringView{}; } StringView GetExitsForID(const NameID /*id*/) const override { return StringView{}; } bool GetContinueStraightDefault() const override { return false; } + std::string GetTimestamp() const override { return "n/a"; } double GetMapMatchingMaxSpeed() const override { return 0; } const char *GetWeightName() const override { return ""; } unsigned GetWeightPrecision() const override { return 0; } diff --git a/unit_tests/library/route.cpp b/unit_tests/library/route.cpp index 6a6faad96..7bb148745 100644 --- a/unit_tests/library/route.cpp +++ b/unit_tests/library/route.cpp @@ -47,6 +47,7 @@ BOOST_AUTO_TEST_CASE(test_route_same_coordinates_fixture) json::Object reference{ {{"code", "Ok"}, + {"data_version", "2016-03-05T00:26:02Z"}, {"waypoints", json::Array{{json::Object{{{"name", "Boulevard du Larvotto"}, {"location", location}, diff --git a/unit_tests/mocks/mock_datafacade.hpp b/unit_tests/mocks/mock_datafacade.hpp index e68a1a306..07ff051bb 100644 --- a/unit_tests/mocks/mock_datafacade.hpp +++ b/unit_tests/mocks/mock_datafacade.hpp @@ -10,7 +10,6 @@ #include "extractor/turn_lane_types.hpp" #include "guidance/turn_bearing.hpp" #include "guidance/turn_instruction.hpp" -#include "guidance/turn_instruction.hpp" #include "engine/algorithm.hpp" #include "engine/datafacade/algorithm_datafacade.hpp" @@ -54,6 +53,7 @@ class MockBaseDataFacade : public engine::datafacade::BaseDataFacade { return 0; } + std::string GetTimestamp() const override { return "n/a"; } NodeForwardRange GetUncompressedForwardGeometry(const EdgeID /* id */) const override { static NodeID data[] = {0, 1, 2, 3};