Merge branch 'master' of https://github.com/Project-OSRM/osrm-backend into master

CHANGELOG.md

@@ -1,11 +1,18 @@
 # Unreleased
-  - Changes from 5.21.0
+  - Changes from 5.23.0
+    - Misc:
+      - CHANGED: Unify `.osrm.turn_penalites_index` dump processing same with `.osrm.turn_weight_penalties` and `.osrm.turn_duration_penalties` [#5868](https://github.com/Project-OSRM/osrm-backend/pull/5868)
+    - Infrastructure
+      - CHANGED: Bundled protozero updated to v1.7.0. [#5858](https://github.com/Project-OSRM/osrm-backend/pull/5858)
+    - Windows:
+      - FIXED: Fix bit-shift overflow in MLD partition step. [#5878](https://github.com/Project-OSRM/osrm-backend/pull/5878)
+
+
+# 5.23.0
+  - Changes from 5.22.0
     - Build:
-      - ADDED: optionally build Node `lts` and `latest` bindings [#5347](https://github.com/Project-OSRM/osrm-backend/pull/5347)
       - FIXED: pessimistic calls to std::move [#5560](https://github.com/Project-OSRM/osrm-backend/pull/5561)
     - Features:
-      - ADDED: new waypoints parameter to the `route` plugin, enabling silent waypoints [#5345](https://github.com/Project-OSRM/osrm-backend/pull/5345)
-      - ADDED: data timestamp information in the response (saved in new file `.osrm.timestamp`). [#5115](https://github.com/Project-OSRM/osrm-backend/issues/5115)
       - ADDED: new API parameter - `snapping=any|default` to allow snapping to previously unsnappable edges [#5361](https://github.com/Project-OSRM/osrm-backend/pull/5361)
       - ADDED: keepalive support to the osrm-routed HTTP server [#5518](https://github.com/Project-OSRM/osrm-backend/pull/5518)
       - ADDED: flatbuffers output format support [#5513](https://github.com/Project-OSRM/osrm-backend/pull/5513)
@@ -17,12 +24,25 @@
       - CHANGED: default car weight was reduced to 2000 kg. [#5371](https://github.com/Project-OSRM/osrm-backend/pull/5371)
       - CHANGED: default car height was reduced to 2 meters. [#5389](https://github.com/Project-OSRM/osrm-backend/pull/5389)
       - FIXED: treat `bicycle=use_sidepath` as no access on the tagged way. [#5622](https://github.com/Project-OSRM/osrm-backend/pull/5622)
+      - FIXED: fix table result when source and destination on same one-way segment. [#5828](https://github.com/Project-OSRM/osrm-backend/pull/5828)
+      - FIXED: fix occasional segfault when swapping data with osrm-datastore and using `exclude=` [#5844](https://github.com/Project-OSRM/osrm-backend/pull/5844)
+      - FIXED: fix crash in MLD alternative search if source or target are invalid [#5851](https://github.com/Project-OSRM/osrm-backend/pull/5851)
     - Misc:
       - CHANGED: Reduce memory usage for raster source handling. [#5572](https://github.com/Project-OSRM/osrm-backend/pull/5572)
       - CHANGED: Add cmake option `ENABLE_DEBUG_LOGGING` to control whether output debug logging. [#3427](https://github.com/Project-OSRM/osrm-backend/issues/3427)
       - CHANGED: updated extent of Hong Kong as left hand drive country. [#5535](https://github.com/Project-OSRM/osrm-backend/issues/5535)
+      - FIXED: corrected error message when failing to snap input coordinates [#5846](https://github.com/Project-OSRM/osrm-backend/pull/5846)
     - Infrastructure
       - REMOVED: STXXL support removed as STXXL became abandonware. [#5760](https://github.com/Project-OSRM/osrm-backend/pull/5760)
+
+# 5.22.0
+  - Changes from 5.21.0
+    - Build:
+      - ADDED: optionally build Node `lts` and `latest` bindings [#5347](https://github.com/Project-OSRM/osrm-backend/pull/5347)
+    - Features:
+      - ADDED: new waypoints parameter to the `route` plugin, enabling silent waypoints [#5345](https://github.com/Project-OSRM/osrm-backend/pull/5345)
+      - ADDED: data timestamp information in the response (saved in new file `.osrm.timestamp`). [#5115](https://github.com/Project-OSRM/osrm-backend/issues/5115)
+
 # 5.21.0
   - Changes from 5.20.0
     - Features:

data/driving_side.geojson

@@ -4,7 +4,7 @@
     {
       "type": "Feature",
       "properties": {
-        
+        "driving_side": "left"
       },
       "geometry": {
         "type": "Polygon",
@@ -301,7 +301,7 @@
     {
       "type": "Feature",
       "properties": {
-        
+        "driving_side": "left"
       },
       "geometry": {
         "type": "Polygon",
@@ -334,7 +334,7 @@
     {
       "type": "Feature",
       "properties": {
-        
+        "driving_side": "left"
       },
       "geometry": {
         "type": "Polygon",
@@ -783,7 +783,7 @@
     {
       "type": "Feature",
       "properties": {
-        
+        "driving_side": "left"
       },
       "geometry": {
         "type": "Polygon",
@@ -1528,7 +1528,7 @@
     {
       "type": "Feature",
       "properties": {
-        
+        "driving_side": "left"
       },
       "geometry": {
         "type": "Polygon",
@@ -1557,7 +1557,7 @@
     {
       "type": "Feature",
       "properties": {
-        
+        "driving_side": "left"
       },
       "geometry": {
         "type": "Polygon",
@@ -1586,7 +1586,7 @@
     {
       "type": "Feature",
       "properties": {
-        
+        "driving_side": "left"
       },
       "geometry": {
         "type": "Polygon",
@@ -1623,7 +1623,7 @@
     {
       "type": "Feature",
       "properties": {
-        
+        "driving_side": "left"
       },
       "geometry": {
         "type": "Polygon",
@@ -3040,7 +3040,7 @@
     {
       "type": "Feature",
       "properties": {
-        
+        "driving_side": "left"
       },
       "geometry": {
         "type": "Polygon",
@@ -3081,7 +3081,7 @@
     {
       "type": "Feature",
       "properties": {
-        
+        "driving_side": "left"
       },
       "geometry": {
         "type": "Polygon",
@@ -3126,7 +3126,7 @@
     {
       "type": "Feature",
       "properties": {
-        
+        "driving_side": "left"
       },
       "geometry": {
         "type": "Polygon",

features/testbot/oneway_phantom.feature

@@ -0,0 +1,85 @@
+@routing @testbot @oneway
+Feature: Handle multiple phantom nodes in one-way segment
+
+# Check we handle routes where source and destination are
+# phantom nodes on the same one-way segment.
+# See: https://github.com/Project-OSRM/osrm-backend/issues/5788
+
+    Background:
+    Given the profile "testbot"
+
+    Scenario: One-way segment with adjacent phantom nodes
+    Given the node map
+        """
+        d  c
+
+        a12b
+        """
+
+    And the ways
+       | nodes | oneway |
+       | ab    | yes    |
+       | bc    | no     |
+       | cd    | no     |
+       | da    | no     |
+
+    When I route I should get
+       | from | to | route          | time      | distance |
+       | 1    | 2  | ab,ab          | 5s  +-0.1 | 50m  ~1% |
+       | 1    | c  | ab,bc,bc       | 30s +-0.1 | 300m ~1% |
+       | 2    | 1  | ab,bc,cd,da,ab | 65s +-0.1 | 650m ~1% |
+       | 2    | c  | ab,bc,bc       | 25s +-0.1 | 250m ~1% |
+       | c    | 1  | cd,da,ab       | 40s +-0.1 | 400m ~1% |
+       | c    | 2  | cd,da,ab       | 45s +-0.1 | 450m ~1% |
+
+    When I request a travel time matrix I should get
+       |   | 1        | 2        | c         |
+       | 1 | 0        | 5  +-0.1 | 30 +-0.1  |
+       | 2 | 65 +-0.1 | 0        | 25 +-0.1  |
+       | c | 40 +-0.1 | 45 +-0.1 | 0         |
+
+    When I request a travel time matrix I should get
+       |   | 1  | 2       | c        |
+       | 1 | 0  | 5 +-0.1 | 30 +-0.1 |
+
+    When I request a travel time matrix I should get
+       |   | 1        | 2  | c        |
+       | 2 | 65 +-0.1 | 0  | 25 +-0.1 |
+
+    When I request a travel time matrix I should get
+       |   | 1        |
+       | 1 | 0        |
+       | 2 | 65 +-0.1 |
+       | c | 40 +-0.1 |
+
+    When I request a travel time matrix I should get
+       |   | 2        |
+       | 1 | 5  +-0.1 |
+       | 2 | 0        |
+       | c | 45 +-0.1 |
+
+    When I request a travel distance matrix I should get
+       |   | 1       | 2       | c       |
+       | 1 | 0       | 50  ~1% | 300 ~1% |
+       | 2 | 650 ~1% | 0       | 250 ~1% |
+       | c | 400 ~1% | 450 ~1% | 0       |
+
+    When I request a travel distance matrix I should get
+       |   | 1   | 2      | c       |
+       | 1 | 0   | 50 ~1% | 300 ~1% |
+
+    When I request a travel distance matrix I should get
+       |   | 1       | 2   | c       |
+       | 2 | 650 ~1% | 0   | 250 ~1% |
+
+    When I request a travel distance matrix I should get
+       |   | 1       |
+       | 1 | 0       |
+       | 2 | 650 ~1% |
+       | c | 400 ~1% |
+
+    When I request a travel distance matrix I should get
+       |   | 2       |
+       | 1 | 50  ~1% |
+       | 2 | 0       |
+       | c | 450 ~1% |

features/testbot/zero-speed-updates.feature

@@ -93,6 +93,31 @@ Feature: Check zero speed updates
           |    1 |  2 | NoRoute |
 
 
+    Scenario: Routing with alternatives on restricted way
+        Given the node map
+            """
+            a-1-b-2-c
+            """
+
+        And the ways
+            | nodes | oneway |
+            | abc   | no     |
+        And the contract extra arguments "--segment-speed-file {speeds_file}"
+        And the customize extra arguments "--segment-speed-file {speeds_file}"
+        And the speed file
+            """
+            1,2,0
+            2,1,0
+            """
+        And the query options
+            | alternatives | true |
+
+
+        When I route I should get
+          | from | to | code    | alternative |
+          |    1 |  2 | NoRoute |             |
+
+
     Scenario: Routing on restricted oneway
         Given the node map
             """

include/engine/data_watchdog.hpp

@@ -56,12 +56,15 @@ class DataWatchdogImpl<AlgorithmT, datafacade::ContiguousInternalMemoryDataFacad
             static_region = *static_shared_region;
             updatable_region = *updatable_shared_region;
 
+            {
+                boost::unique_lock<boost::shared_mutex> swap_lock(factory_mutex);
                 facade_factory =
                     DataFacadeFactory<datafacade::ContiguousInternalMemoryDataFacade, AlgorithmT>(
                         std::make_shared<datafacade::SharedMemoryAllocator>(
                             std::vector<storage::SharedRegionRegister::ShmKey>{
                                 static_region.shm_key, updatable_region.shm_key}));
             }
+        }
 
         watcher = std::thread(&DataWatchdogImpl::Run, this);
     }
@@ -75,10 +78,14 @@ class DataWatchdogImpl<AlgorithmT, datafacade::ContiguousInternalMemoryDataFacad
 
     std::shared_ptr<const Facade> Get(const api::BaseParameters &params) const
     {
+        // make sure facade_factory stays stable while we call Get()
+        boost::shared_lock<boost::shared_mutex> swap_lock(factory_mutex);
         return facade_factory.Get(params);
     }
     std::shared_ptr<const Facade> Get(const api::TileParameters &params) const
     {
+        // make sure facade_factory stays stable while we call Get()
+        boost::shared_lock<boost::shared_mutex> swap_lock(factory_mutex);
         return facade_factory.Get(params);
     }
 
@@ -111,16 +118,20 @@ class DataWatchdogImpl<AlgorithmT, datafacade::ContiguousInternalMemoryDataFacad
                         << (int)updatable_region.shm_key << " with timestamps "
                         << static_region.timestamp << " and " << updatable_region.timestamp;
 
+            {
+                boost::unique_lock<boost::shared_mutex> swap_lock(factory_mutex);
                 facade_factory =
                     DataFacadeFactory<datafacade::ContiguousInternalMemoryDataFacade, AlgorithmT>(
                         std::make_shared<datafacade::SharedMemoryAllocator>(
                             std::vector<storage::SharedRegionRegister::ShmKey>{
                                 static_region.shm_key, updatable_region.shm_key}));
             }
+        }
 
         util::Log() << "DataWatchdog thread stopped";
     }
 
+    mutable boost::shared_mutex factory_mutex;
     const std::string dataset_name;
     storage::SharedMonitor<storage::SharedRegionRegister> barrier;
     std::thread watcher;

include/engine/plugins/plugin_base.hpp

@@ -371,6 +371,22 @@ class BasePlugin
         }
         return phantom_node_pairs;
     }
+
+    std::string MissingPhantomErrorMessage(const std::vector<PhantomNodePair> &phantom_nodes,
+                                           const std::vector<util::Coordinate> &coordinates) const
+    {
+        BOOST_ASSERT(phantom_nodes.size() < coordinates.size());
+        auto mismatch = std::mismatch(phantom_nodes.begin(),
+                                      phantom_nodes.end(),
+                                      coordinates.begin(),
+                                      coordinates.end(),
+                                      [](const auto &phantom_node, const auto &coordinate) {
+                                          return phantom_node.first.input_location == coordinate;
+                                      });
+        std::size_t missing_index = std::distance(phantom_nodes.begin(), mismatch.first);
+        return std::string("Could not find a matching segment for coordinate ") +
+               std::to_string(missing_index);
+    }
 };
 }
 }

include/extractor/files.hpp

@@ -399,6 +399,28 @@ inline void readTurnDurationPenalty(const boost::filesystem::path &path, TurnPen
     storage::serialization::read(reader, "/common/turn_penalty/duration", turn_penalty);
 }
 
+// writes .osrm.turn_penalties_index
+template <typename TurnIndexT>
+inline void writeTurnPenaltiesIndex(const boost::filesystem::path &path,
+                                    const TurnIndexT &turn_penalties_index)
+{
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
+
+    storage::serialization::write(writer, "/extractor/turn_index", turn_penalties_index);
+}
+
+// read .osrm.turn_penalties_index
+template <typename TurnIndexT>
+inline void readTurnPenaltiesIndex(const boost::filesystem::path &path,
+                                   TurnIndexT &turn_penalties_index)
+{
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
+
+    storage::serialization::read(reader, "/extractor/turn_index", turn_penalties_index);
+}
+
 // writes .osrm.restrictions
 template <typename ConditionalRestrictionsT>
 inline void writeConditionalRestrictions(const boost::filesystem::path &path,

include/partitioner/multi_level_partition.hpp

@@ -212,10 +212,10 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
                                 // create mask that has `bits` ones at its LSBs.
                                 // 000011
                                 BOOST_ASSERT(offset < NUM_PARTITION_BITS);
-                                PartitionID mask = (1UL << offset) - 1UL;
+                                PartitionID mask = (1ULL << offset) - 1ULL;
                                 // 001111
                                 BOOST_ASSERT(next_offset < NUM_PARTITION_BITS);
-                                PartitionID next_mask = (1UL << next_offset) - 1UL;
+                                PartitionID next_mask = (1ULL << next_offset) - 1ULL;
                                 // 001100
                                 masks[lidx++] = next_mask ^ mask;
                             });

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "osrm",
-  "version": "5.22.0-customsnapping.2",
+  "version": "5.24.0-unreleased",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "osrm",
-  "version": "5.22.0-customsnapping.2",
+  "version": "5.24.0-unreleased",
   "private": false,
   "description": "The Open Source Routing Machine is a high performance routing engine written in C++14 designed to run on OpenStreetMap data.",
   "dependencies": {

src/engine/plugins/table.cpp

@@ -75,10 +75,8 @@ Status TablePlugin::HandleRequest(const RoutingAlgorithmsInterface &algorithms,
 
     if (phantom_nodes.size() != params.coordinates.size())
     {
-        return Error("NoSegment",
-                     std::string("Could not find a matching segment for coordinate ") +
-                         std::to_string(phantom_nodes.size()),
-                     result);
+        return Error(
+            "NoSegment", MissingPhantomErrorMessage(phantom_nodes, params.coordinates), result);
     }
 
     auto snapped_phantoms = SnapPhantomNodes(phantom_nodes);

src/engine/plugins/trip.cpp

@@ -199,8 +199,7 @@ Status TripPlugin::HandleRequest(const RoutingAlgorithmsInterface &algorithms,
     if (phantom_node_pairs.size() != number_of_locations)
     {
         return Error("NoSegment",
-                     std::string("Could not find a matching segment for coordinate ") +
-                         std::to_string(phantom_node_pairs.size()),
+                     MissingPhantomErrorMessage(phantom_node_pairs, parameters.coordinates),
                      result);
     }
     BOOST_ASSERT(phantom_node_pairs.size() == number_of_locations);

src/engine/plugins/viaroute.cpp

@@ -90,8 +90,7 @@ Status ViaRoutePlugin::HandleRequest(const RoutingAlgorithmsInterface &algorithm
     if (phantom_node_pairs.size() != route_parameters.coordinates.size())
     {
         return Error("NoSegment",
-                     std::string("Could not find a matching segment for coordinate ") +
-                         std::to_string(phantom_node_pairs.size()),
+                     MissingPhantomErrorMessage(phantom_node_pairs, route_parameters.coordinates),
                      result);
     }
     BOOST_ASSERT(phantom_node_pairs.size() == route_parameters.coordinates.size());

src/engine/routing_algorithms/alternative_path_mld.cpp

@@ -663,6 +663,10 @@ makeCandidateVias(SearchEngineData<Algorithm> &search_engine_data,
     Heap &reverse_heap = *search_engine_data.reverse_heap_1;
 
     insertNodesInHeaps(forward_heap, reverse_heap, phantom_node_pair);
+    if (forward_heap.Empty() || reverse_heap.Empty())
+    {
+        return {};
+    }
 
     // The single via node in the shortest paths s,via and via,t sub-paths and
     // the weight for the shortest path s,t we return and compare alternatives to.

src/engine/routing_algorithms/many_to_many_mld.cpp

@@ -36,6 +36,59 @@ inline LevelID getNodeQueryLevel(const MultiLevelPartition &partition,
     return node_level;
 }
 
+template <bool DIRECTION>
+void relaxBorderEdges(const DataFacade<mld::Algorithm> &facade,
+                      const NodeID node,
+                      const EdgeWeight weight,
+                      const EdgeDuration duration,
+                      const EdgeDistance distance,
+                      typename SearchEngineData<mld::Algorithm>::ManyToManyQueryHeap &query_heap,
+                      LevelID level)
+{
+    for (const auto edge : facade.GetBorderEdgeRange(level, node))
+    {
+        const auto &data = facade.GetEdgeData(edge);
+        if ((DIRECTION == FORWARD_DIRECTION) ? facade.IsForwardEdge(edge)
+                                             : facade.IsBackwardEdge(edge))
+        {
+            const NodeID to = facade.GetTarget(edge);
+            if (facade.ExcludeNode(to))
+            {
+                continue;
+            }
+
+            const auto turn_id = data.turn_id;
+            const auto node_id = DIRECTION == FORWARD_DIRECTION ? node : facade.GetTarget(edge);
+            const auto node_weight = facade.GetNodeWeight(node_id);
+            const auto node_duration = facade.GetNodeDuration(node_id);
+            const auto node_distance = facade.GetNodeDistance(node_id);
+            const auto turn_weight = node_weight + facade.GetWeightPenaltyForEdgeID(turn_id);
+            const auto turn_duration = node_duration + facade.GetDurationPenaltyForEdgeID(turn_id);
+
+            BOOST_ASSERT_MSG(node_weight + turn_weight > 0, "edge weight is invalid");
+            const auto to_weight = weight + turn_weight;
+            const auto to_duration = duration + turn_duration;
+            const auto to_distance = distance + node_distance;
+
+            // New Node discovered -> Add to Heap + Node Info Storage
+            if (!query_heap.WasInserted(to))
+            {
+                query_heap.Insert(to, to_weight, {node, false, to_duration, to_distance});
+            }
+            // Found a shorter Path -> Update weight and set new parent
+            else if (std::tie(to_weight, to_duration, to_distance, node) <
+                     std::tie(query_heap.GetKey(to),
+                              query_heap.GetData(to).duration,
+                              query_heap.GetData(to).distance,
+                              query_heap.GetData(to).parent))
+            {
+                query_heap.GetData(to) = {node, false, to_duration, to_distance};
+                query_heap.DecreaseKey(to, to_weight);
+            }
+        }
+    }
+}
+
 template <bool DIRECTION, typename... Args>
 void relaxOutgoingEdges(const DataFacade<mld::Algorithm> &facade,
                         const NodeID node,
@@ -140,48 +193,7 @@ void relaxOutgoingEdges(const DataFacade<mld::Algorithm> &facade,
         }
     }
 
-    for (const auto edge : facade.GetBorderEdgeRange(level, node))
-    {
-        const auto &data = facade.GetEdgeData(edge);
-        if ((DIRECTION == FORWARD_DIRECTION) ? facade.IsForwardEdge(edge)
-                                             : facade.IsBackwardEdge(edge))
-        {
-            const NodeID to = facade.GetTarget(edge);
-            if (facade.ExcludeNode(to))
-            {
-                continue;
-            }
-
-            const auto turn_id = data.turn_id;
-            const auto node_id = DIRECTION == FORWARD_DIRECTION ? node : facade.GetTarget(edge);
-            const auto node_weight = facade.GetNodeWeight(node_id);
-            const auto node_duration = facade.GetNodeDuration(node_id);
-            const auto node_distance = facade.GetNodeDistance(node_id);
-            const auto turn_weight = node_weight + facade.GetWeightPenaltyForEdgeID(turn_id);
-            const auto turn_duration = node_duration + facade.GetDurationPenaltyForEdgeID(turn_id);
-
-            BOOST_ASSERT_MSG(node_weight + turn_weight > 0, "edge weight is invalid");
-            const auto to_weight = weight + turn_weight;
-            const auto to_duration = duration + turn_duration;
-            const auto to_distance = distance + node_distance;
-
-            // New Node discovered -> Add to Heap + Node Info Storage
-            if (!query_heap.WasInserted(to))
-            {
-                query_heap.Insert(to, to_weight, {node, false, to_duration, to_distance});
-            }
-            // Found a shorter Path -> Update weight and set new parent
-            else if (std::tie(to_weight, to_duration, to_distance, node) <
-                     std::tie(query_heap.GetKey(to),
-                              query_heap.GetData(to).duration,
-                              query_heap.GetData(to).distance,
-                              query_heap.GetData(to).parent))
-            {
-                query_heap.GetData(to) = {node, false, to_duration, to_distance};
-                query_heap.DecreaseKey(to, to_weight);
-            }
-        }
-    }
+    relaxBorderEdges<DIRECTION>(facade, node, weight, duration, distance, query_heap, level);
 }
 
 //
@@ -297,37 +309,19 @@ oneToManySearch(SearchEngineData<Algorithm> &engine_working_data,
                            EdgeWeight initial_weight,
                            EdgeDuration initial_duration,
                            EdgeDistance initial_distance) {
-
-        // Update single node paths
+        if (target_nodes_index.count(node))
+        {
+            // Source and target on the same edge node. If target is not reachable directly via
+            // the node (e.g destination is before source on oneway segment) we want to allow
+            // node to be visited later in the search along a reachable path.
+            // Therefore, we manually run first step of search without marking node as visited.
             update_values(node, initial_weight, initial_duration, initial_distance);
-
+            relaxBorderEdges<DIRECTION>(
+                facade, node, initial_weight, initial_duration, initial_distance, query_heap, 0);
+        }
+        else
+        {
             query_heap.Insert(node, initial_weight, {node, initial_duration, initial_distance});
-
-        // Place adjacent nodes into heap
-        for (auto edge : facade.GetAdjacentEdgeRange(node))
-        {
-            const auto &data = facade.GetEdgeData(edge);
-            const auto to = facade.GetTarget(edge);
-
-            if (facade.ExcludeNode(to))
-            {
-                continue;
-            }
-
-            if ((DIRECTION == FORWARD_DIRECTION ? facade.IsForwardEdge(edge)
-                                                : facade.IsBackwardEdge(edge)) &&
-                !query_heap.WasInserted(to))
-            {
-                const auto turn_id = data.turn_id;
-                const auto node_id = DIRECTION == FORWARD_DIRECTION ? node : to;
-                const auto edge_weight = initial_weight + facade.GetNodeWeight(node_id) +
-                                         facade.GetWeightPenaltyForEdgeID(turn_id);
-                const auto edge_duration = initial_duration + facade.GetNodeDuration(node_id) +
-                                           facade.GetDurationPenaltyForEdgeID(turn_id);
-                const auto edge_distance = initial_distance + facade.GetNodeDistance(node_id);
-
-                query_heap.Insert(to, edge_weight, {node, edge_duration, edge_distance});
-            }
         }
     };
 

src/extractor/edge_based_graph_factory.cpp

@@ -53,9 +53,6 @@ template <> struct hash<std::pair<NodeID, NodeID>>
 };
 }
 
-// Buffer size of turn_indexes_write_buffer to reduce number of write(v) syscals
-const constexpr int TURN_INDEX_WRITE_BUFFER_SIZE = 1000;
-
 namespace osrm
 {
 namespace extractor
@@ -449,10 +446,6 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
 
     std::size_t node_based_edge_counter = 0;
 
-    storage::tar::FileWriter turn_penalties_index_file(
-        turn_penalties_index_filename, storage::tar::FileWriter::GenerateFingerprint);
-    turn_penalties_index_file.WriteFrom("/extractor/turn_index", (char *)nullptr, 0);
-
     SuffixTable street_name_suffix_table(scripting_environment);
     const auto &turn_lanes_data = transformTurnLaneMapIntoArrays(lane_description_map);
     intersection::MergableRoadDetector mergable_road_detector(m_node_based_graph,
@@ -468,6 +461,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
     // FIXME these need to be tuned in pre-allocated size
     std::vector<TurnPenalty> turn_weight_penalties;
     std::vector<TurnPenalty> turn_duration_penalties;
+    std::vector<lookup::TurnIndexBlock> turn_penalties_index;
 
     // Now, renumber all our maneuver overrides to use edge-based-nodes
     std::vector<StorageManeuverOverride> storage_maneuver_overrides;
@@ -487,14 +481,6 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
     {
         const NodeID node_count = m_node_based_graph.GetNumberOfNodes();
 
-        // Because we write TurnIndexBlock data as we go, we'll
-        // buffer them into groups of 1000 to reduce the syscall
-        // count by 1000x.  This doesn't need much memory, but
-        // greatly reduces the syscall overhead of writing lots
-        // of small objects
-        std::vector<lookup::TurnIndexBlock> turn_indexes_write_buffer;
-        turn_indexes_write_buffer.reserve(TURN_INDEX_WRITE_BUFFER_SIZE);
-
         // This struct is the buffered output of the `processor_stage`.  This data is
         // appended to the various output arrays/files by the `output_stage`.
         // same as IntersectionData, but grouped with edge to allow sorting after creating.
@@ -510,7 +496,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
             m_edge_based_edge_list.push_back(edge_with_data.edge);
             turn_weight_penalties.push_back(edge_with_data.turn_weight_penalty);
             turn_duration_penalties.push_back(edge_with_data.turn_duration_penalty);
-            turn_indexes_write_buffer.push_back(edge_with_data.turn_index);
+            turn_penalties_index.push_back(edge_with_data.turn_index);
         };
 
         struct EdgesPipelineBuffer
@@ -1054,15 +1040,6 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
                 // NOTE: potential overflow here if we hit 2^32 routable edges
                 BOOST_ASSERT(m_edge_based_edge_list.size() <= std::numeric_limits<NodeID>::max());
 
-                // Buffer writes to reduce syscall count
-                if (turn_indexes_write_buffer.size() >= TURN_INDEX_WRITE_BUFFER_SIZE)
-                {
-                    turn_penalties_index_file.ContinueFrom("/extractor/turn_index",
-                                                           turn_indexes_write_buffer.data(),
-                                                           turn_indexes_write_buffer.size());
-                    turn_indexes_write_buffer.clear();
-                }
-
                 // Copy via-way restrictions delayed data
                 delayed_data.insert(
                     delayed_data.end(), buffer->delayed_data.begin(), buffer->delayed_data.end());
@@ -1118,15 +1095,6 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
 
             storage_maneuver_overrides.push_back(storage_override);
         }
-
-        // Flush the turn_indexes_write_buffer if it's not empty
-        if (!turn_indexes_write_buffer.empty())
-        {
-            turn_penalties_index_file.ContinueFrom("/extractor/turn_index",
-                                                   turn_indexes_write_buffer.data(),
-                                                   turn_indexes_write_buffer.size());
-            turn_indexes_write_buffer.clear();
-        }
     }
     {
         util::Log() << "Sorting and writing " << storage_maneuver_overrides.size()
@@ -1162,9 +1130,11 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
                                                    indexed_conditionals);
 
     // write weight penalties per turn
-    BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size());
+    BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size() &&
+                 turn_weight_penalties.size() == turn_penalties_index.size());
     files::writeTurnWeightPenalty(turn_weight_penalties_filename, turn_weight_penalties);
     files::writeTurnDurationPenalty(turn_duration_penalties_filename, turn_duration_penalties);
+    files::writeTurnPenaltiesIndex(turn_penalties_index_filename, turn_penalties_index);
 
     util::Log() << "Generated " << m_edge_based_node_segments.size() << " edge based node segments";
     util::Log() << "Node-based graph contains " << node_based_edge_counter << " edges";

src/extractor/extractor.cpp

@@ -1,5 +1,6 @@
 #include "extractor/extractor.hpp"
 
+#include "extractor/compressed_edge_container.hpp"
 #include "extractor/compressed_node_based_graph_edge.hpp"
 #include "extractor/edge_based_edge.hpp"
 #include "extractor/extraction_containers.hpp"
@@ -11,47 +12,34 @@
 #include "extractor/maneuver_override_relation_parser.hpp"
 #include "extractor/name_table.hpp"
 #include "extractor/node_based_graph_factory.hpp"
-#include "extractor/raster_source.hpp"
 #include "extractor/restriction_filter.hpp"
+#include "extractor/restriction_index.hpp"
 #include "extractor/restriction_parser.hpp"
 #include "extractor/scripting_environment.hpp"
+#include "extractor/tarjan_scc.hpp"
+#include "extractor/way_restriction_map.hpp"
 
 #include "guidance/files.hpp"
 #include "guidance/guidance_processing.hpp"
 #include "guidance/segregated_intersection_classification.hpp"
 #include "guidance/turn_data_container.hpp"
 
-#include "storage/io.hpp"
-
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
 #include "util/integer_range.hpp"
 #include "util/log.hpp"
-#include "util/range_table.hpp"
-#include "util/timing_util.hpp"
-
-#include "extractor/compressed_edge_container.hpp"
-#include "extractor/restriction_index.hpp"
-#include "extractor/way_restriction_map.hpp"
 #include "util/static_graph.hpp"
 #include "util/static_rtree.hpp"
+#include "util/timing_util.hpp"
 
 // Keep debug include to make sure the debug header is in sync with types.
 #include "util/debug.hpp"
 
-#include "extractor/tarjan_scc.hpp"
-
 #include <boost/assert.hpp>
-#include <boost/filesystem.hpp>
-#include <boost/filesystem/fstream.hpp>
-#include <boost/iterator/function_input_iterator.hpp>
-#include <boost/optional/optional.hpp>
-#include <boost/scope_exit.hpp>
 
 #include <osmium/handler/node_locations_for_ways.hpp>
 #include <osmium/index/map/flex_mem.hpp>
 #include <osmium/io/any_input.hpp>
-#include <osmium/osm/timestamp.hpp>
 #include <osmium/thread/pool.hpp>
 #include <osmium/visitor.hpp>
 
@@ -62,15 +50,11 @@
 #endif
 #include <tbb/pipeline.h>
 
-#include <cstdlib>
-
 #include <algorithm>
 #include <atomic>
 #include <bitset>
 #include <chrono>
-#include <future>
 #include <iostream>
-#include <iterator>
 #include <memory>
 #include <thread>
 #include <tuple>
@@ -193,7 +177,7 @@ std::vector<CompressedNodeBasedGraphEdge> toEdgeList(const util::NodeBasedDynami
 
     return edges;
 }
-}
+} // namespace
 
 /**
  * TODO: Refactor this function into smaller functions for better readability.
@@ -288,16 +272,6 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
     //
     // Luckily node based node ids still coincide with the coordinate array.
     // That's the reason we can only here write out the final compressed node based graph.
-
-    // Dumps to file asynchronously and makes sure we wait for its completion.
-    std::future<void> compressed_node_based_graph_writing;
-
-    BOOST_SCOPE_EXIT_ALL(&)
-    {
-        if (compressed_node_based_graph_writing.valid())
-            compressed_node_based_graph_writing.wait();
-    };
-
     files::writeCompressedNodeBasedGraph(config.GetPath(".osrm.cnbg").string(),
                                          toEdgeList(node_based_graph));
 
@@ -519,7 +493,6 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
     // OSM elements Lua parser
     tbb::filter_t<SharedBuffer, ParsedBuffer> buffer_transformer(
         tbb::filter::parallel, [&](const SharedBuffer buffer) {
-
             ParsedBuffer parsed_buffer;
             parsed_buffer.buffer = buffer;
             scripting_environment.ProcessElements(*buffer,
@@ -540,7 +513,6 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
     unsigned number_of_maneuver_overrides = 0;
     tbb::filter_t<ParsedBuffer, void> buffer_storage(
         tbb::filter::serial_in_order, [&](const ParsedBuffer &parsed_buffer) {
-
             number_of_nodes += parsed_buffer.resulting_nodes.size();
             // put parsed objects thru extractor callbacks
             for (const auto &result : parsed_buffer.resulting_nodes)
@@ -564,7 +536,6 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
             {
                 extractor_callbacks->ProcessManeuverOverride(result);
             }
-
         });
 
     tbb::filter_t<SharedBuffer, std::shared_ptr<ExtractionRelationContainer>> buffer_relation_cache(
@@ -606,7 +577,6 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
     tbb::filter_t<std::shared_ptr<ExtractionRelationContainer>, void> buffer_storage_relation(
         tbb::filter::serial_in_order,
         [&](const std::shared_ptr<ExtractionRelationContainer> parsed_relations) {
-
             number_of_relations += parsed_relations->GetRelationsNum();
             relations.Merge(std::move(*parsed_relations));
         });

src/guidance/segregated_intersection_classification.cpp

@@ -226,7 +226,7 @@ std::unordered_set<EdgeID> findSegregatedNodes(const extractor::NodeBasedGraphFa
     auto const collect_edge_info_fn = [&](auto const &edges1, NodeID node2) {
         std::vector<EdgeInfo> info;
 
-        for (auto const &e : edges1)
+        for (auto e : edges1)
         {
             NodeID const target = graph.GetTarget(e);
             if (target == node2)

src/updater/updater.cpp

@@ -432,6 +432,10 @@ updateTurnPenalties(const UpdaterConfig &config,
 {
     const auto weight_multiplier = profile_properties.GetWeightMultiplier();
 
+    // [NOTE] turn_index_blocks could be simply loaded by `files::readTurnPenaltiesIndex()`,
+    //      however, we leave the below mmap to keep compatiblity.
+    //      Use `files::readTurnPenaltiesIndex()` instead once the compatiblity is not that
+    //      important.
     // Mapped file pointer for turn indices
     boost::iostreams::mapped_file_source turn_index_region;
     const extractor::lookup::TurnIndexBlock *turn_index_blocks;

src/util/timezones.cpp

@@ -154,7 +154,7 @@ boost::optional<struct tm> Timezoner::operator()(const point_t &point) const
 {
     std::vector<rtree_t::value_type> result;
     rtree.query(boost::geometry::index::intersects(point), std::back_inserter(result));
-    for (const auto v : result)
+    for (const auto &v : result)
     {
         const auto index = v.second;
         if (boost::geometry::within(point, local_times[index].first))

third_party/protozero/.clang-tidy

@@ -1,14 +1,33 @@
 ---
-Checks: '*,-cert-dcl21-cpp,-cert-err60-cpp,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-cppcoreguidelines-pro-type-reinterpret-cast,-fuchsia-*,-google-runtime-references,-hicpp-no-array-decay'
+Checks: '*,-bugprone-signed-char-misuse,-cert-dcl21-cpp,-cert-err58-cpp,-cert-err60-cpp,-cppcoreguidelines-avoid-c-arrays,-cppcoreguidelines-avoid-magic-numbers,-cppcoreguidelines-macro-usage,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-cppcoreguidelines-pro-type-reinterpret-cast,-fuchsia-*,-google-runtime-references,-hicpp-avoid-c-arrays,-hicpp-no-array-decay,-hicpp-vararg,-modernize-avoid-c-arrays,-modernize-use-trailing-return-type,-readability-implicit-bool-conversion,-readability-magic-numbers'
 #
 #  Disabled checks:
 #
+#  bugprone-signed-char-misuse
+#    Lots of warnings in varint.hpp otherwise.
+#
 #  cert-dcl21-cpp
 #    It is unclear whether this is still a good recommendation in modern C++.
 #
+#  cert-err58-cpp
+#    Due to the Catch2 test framework.
+#
 #  cert-err60-cpp
 #    Reports std::runtime_error as broken which we can't do anything about.
 #
+#  cppcoreguidelines-avoid-c-arrays
+#  hicpp-avoid-c-arrays
+#  modernize-avoid-c-arrays
+#    Makes sense for some array, but especially for char arrays using
+#    std::array isn't a good solution.
+#
+#  cppcoreguidelines-avoid-magic-numbers
+#  readability-magic-numbers
+#    Good idea, but it goes too far to force this everywhere.
+#
+#  cppcoreguidelines-macro-usage
+#    There are cases where macros are simply needed.
+#
 #  cppcoreguidelines-pro-bounds-array-to-pointer-decay
 #    Limited use and many false positives including for all asserts.
 #
@@ -28,6 +47,12 @@ Checks: '*,-cert-dcl21-cpp,-cert-err60-cpp,-cppcoreguidelines-pro-bounds-pointer
 #  hicpp-no-array-decay
 #    Limited use and many false positives including for all asserts.
 #
+#  modernize-use-trailing-return-type
+#    We are not quite that modern.
+#
+#  readability-implicit-bool-conversion
+#    Not necessarily more readable.
+#
 WarningsAsErrors: '*'
 HeaderFilterRegex: '\/include\/'
 AnalyzeTemporaryDtors: false

third_party/protozero/.travis.yml

@@ -6,10 +6,6 @@
 
 language: generic
 
-sudo: false
-
-dist: trusty
-
 #-----------------------------------------------------------------------------
 
 # Save common build configurations as shortcuts, so we can reference them later.
@@ -17,147 +13,265 @@ addons_shortcuts:
   addons_clang35: &clang35
     apt:
       sources: [ 'ubuntu-toolchain-r-test', 'llvm-toolchain-trusty-3.5' ]
-      packages: [ 'libprotobuf-dev','protobuf-compiler', 'clang-3.5' ]
+      packages: [ 'libprotobuf-dev', 'protobuf-compiler', 'clang-3.5' ]
   addons_clang38: &clang38
     apt:
-      sources: [ 'ubuntu-toolchain-r-test', 'llvm-toolchain-trusty-3.8' ]
-      packages: [ 'libprotobuf-dev','protobuf-compiler', 'clang-3.8' ]
+      packages: [ 'libprotobuf-dev', 'protobuf-compiler', 'clang-3.8' ]
   addons_clang39: &clang39
     apt:
-      sources: [ 'ubuntu-toolchain-r-test', 'llvm-toolchain-trusty-3.9' ]
-      packages: [ 'libprotobuf-dev','protobuf-compiler', 'clang-3.9' ]
+      packages: [ 'libprotobuf-dev', 'protobuf-compiler', 'clang-3.9' ]
   addons_clang40: &clang40
     apt:
-      sources: [ 'ubuntu-toolchain-r-test', 'llvm-toolchain-trusty-4.0' ]
-      packages: [ 'libprotobuf-dev','protobuf-compiler', 'clang-4.0' ]
+      sources: [ 'ubuntu-toolchain-r-test' ]
+      packages: [ 'libprotobuf-dev', 'protobuf-compiler', 'clang-4.0' ]
   addons_clang50: &clang50
     apt:
-      sources: [ 'ubuntu-toolchain-r-test', 'llvm-toolchain-trusty-5.0' ]
-      packages: [ 'libprotobuf-dev','protobuf-compiler', 'clang-5.0', 'clang-tidy-5.0' ]
-  addons_gcc47: &gcc47
+      sources: [ 'ubuntu-toolchain-r-test' ]
+      packages: [ 'libprotobuf-dev', 'protobuf-compiler', 'clang-5.0' ]
+  addons_clang60: &clang60
     apt:
       sources: [ 'ubuntu-toolchain-r-test' ]
-      packages: [ 'libprotobuf-dev','protobuf-compiler', 'g++-4.7', 'gcc-4.7' ]
+      packages: [ 'libprotobuf-dev', 'protobuf-compiler', 'clang-6.0' ]
+  addons_clang7: &clang7
+    apt:
+      packages: [ 'libprotobuf-dev', 'protobuf-compiler', 'clang-7' ]
+  addons_clang8: &clang8
+    apt:
+      packages: [ 'libprotobuf-dev', 'protobuf-compiler', 'clang-8' ]
+  addons_clang9: &clang9
+    apt:
+      packages: [ 'libprotobuf-dev', 'protobuf-compiler', 'clang-9', 'clang-tidy-9' ]
   addons_gcc48: &gcc48
     apt:
       sources: [ 'ubuntu-toolchain-r-test' ]
-      packages: [ 'libprotobuf-dev','protobuf-compiler', 'g++-4.8', 'gcc-4.8' ]
+      packages: [ 'libprotobuf-dev', 'protobuf-compiler', 'g++-4.8', 'gcc-4.8' ]
   addons_gcc49: &gcc49
     apt:
       sources: [ 'ubuntu-toolchain-r-test' ]
-      packages: [ 'libprotobuf-dev','protobuf-compiler', 'g++-4.9', 'gcc-4.9' ]
+      packages: [ 'libprotobuf-dev', 'protobuf-compiler', 'g++-4.9', 'gcc-4.9' ]
   addons_gcc5: &gcc5
     apt:
       sources: [ 'ubuntu-toolchain-r-test' ]
-      packages: [ 'libprotobuf-dev','protobuf-compiler', 'g++-5', 'gcc-5' ]
+      packages: [ 'libprotobuf-dev', 'protobuf-compiler', 'g++-5', 'gcc-5' ]
   addons_gcc6: &gcc6
     apt:
       sources: [ 'ubuntu-toolchain-r-test' ]
-      packages: [ 'libprotobuf-dev','protobuf-compiler', 'g++-6', 'gcc-6' ]
+      packages: [ 'libprotobuf-dev', 'protobuf-compiler', 'g++-6', 'gcc-6' ]
+  addons_gcc7: &gcc7
+    apt:
+      packages: [ 'libprotobuf-dev', 'protobuf-compiler' ]
+  addons_gcc8: &gcc8
+    apt:
+      packages: [ 'libprotobuf-dev', 'protobuf-compiler', 'g++-8', 'gcc-8' ]
 
 #-----------------------------------------------------------------------------
 
-matrix:
+jobs:
   include:
     - os: linux
+      dist: trusty
       compiler: "clang-3.5"
-      env: BUILD='Debug' CC=clang-3.5 CXX=clang++-3.5
+      env: BUILD=Debug CC=clang-3.5 CXX=clang++-3.5
       addons: *clang35
+
     - os: linux
+      dist: xenial
       compiler: "clang-3.8"
-      env: BUILD='Debug' CC=clang-3.8 CXX=clang++-3.8
+      env: BUILD=Debug CC=clang-3.8 CXX=clang++-3.8
       addons: *clang38
+
     - os: linux
+      dist: xenial
       compiler: "clang-3.9"
-      env: BUILD='Debug' CC=clang-3.9 CXX=clang++-3.9
+      env: BUILD=Debug CC=clang-3.9 CXX=clang++-3.9
       addons: *clang39
+
     - os: linux
+      dist: xenial
       compiler: "clang-4.0"
-      env: BUILD='Debug' CC=clang-4.0 CXX=clang++-4.0
+      env: BUILD=Debug CC=clang-4.0 CXX=clang++-4.0
       addons: *clang40
+
     - os: linux
+      dist: xenial
       compiler: "clang-5.0"
-      env: BUILD='Debug' CC=clang-5.0 CXX=clang++-5.0
-           CLANG_TIDY=clang-tidy-5.0
+      env: BUILD=Debug CC=clang-5.0 CXX=clang++-5.0
       addons: *clang50
+
     - os: linux
-      compiler: "clang-5.0"
-      env: BUILD='Release' CC=clang-5.0 CXX=clang++-5.0
-      addons: *clang50
+      dist: xenial
+      compiler: "clang-6.0"
+      env: BUILD=Debug CC=clang-6.0 CXX=clang++-6.0
+      addons: *clang60
+
     - os: linux
-      compiler: "clang-5.0"
-      env: BUILD='Debug' CC=clang-5.0 CXX=clang++-5.0
+      dist: bionic
+      compiler: "clang-7"
+      env: BUILD=Debug CC=clang-7 CXX=clang++-7
+      addons: *clang7
+
+    - os: linux
+      dist: bionic
+      compiler: "clang-8"
+      env: BUILD=Debug CC=clang-8 CXX=clang++-8
+      addons: *clang8
+
+    - os: linux
+      dist: bionic
+      compiler: "clang-9"
+      env: BUILD=Debug CC=clang-9 CXX=clang++-9
+           CLANG_TIDY=clang-tidy-9
+      addons: *clang9
+
+    - os: linux
+      dist: bionic
+      compiler: "clang-9"
+      env: BUILD=Debug CC=clang-9 CXX=clang++-9
            CXXFLAGS="-fsanitize=address,undefined,integer -fno-sanitize-recover=all -fno-omit-frame-pointer"
            LDFLAGS="-fsanitize=address,undefined,integer"
-      # LSAN doesn't work on container-based system
-      sudo: required
-      addons: *clang50
+      addons: *clang9
+
     - os: linux
-      compiler: "gcc-4.7"
-      env: BUILD='Debug' CC=gcc-4.7 CXX=g++-4.7
-      addons: *gcc47
+      dist: bionic
+      compiler: "clang-9"
+      env: BUILD=Release CC=clang-9 CXX=clang++-9
+      addons: *clang9
+
     - os: linux
+      arch: arm64
+      dist: bionic
+      compiler: "clang-9"
+      env: BUILD=Debug CC=clang-9 CXX=clang++-9
+           CXXFLAGS="-fsanitize=address,undefined,integer -fno-sanitize-recover=all -fno-omit-frame-pointer"
+           LDFLAGS="-fsanitize=address,undefined,integer"
+      addons: *clang9
+
+    - os: linux
+      arch: ppc64le
+      dist: bionic
+      compiler: "clang-9"
+      env: BUILD=Debug CC=clang-9 CXX=clang++-9
+           CXXFLAGS="-fsanitize=address,undefined,integer -fno-sanitize-recover=all -fno-omit-frame-pointer"
+           LDFLAGS="-fsanitize=address,undefined,integer"
+      addons: *clang9
+
+    - os: linux
+      arch: s390x
+      dist: bionic
+      compiler: "clang-9"
+      env: BUILD=Debug CC=clang-9 CXX=clang++-9
+           CXXFLAGS="-fsanitize=address,undefined,integer -fno-sanitize-recover=all -fno-omit-frame-pointer"
+           LDFLAGS="-fsanitize=address,undefined,integer"
+      addons: *clang9
+
+    - os: linux
+      dist: trusty
       compiler: "gcc-4.8"
-      env: BUILD='Debug' CC=gcc-4.8 CXX=g++-4.8
+      env: BUILD=Debug CC=gcc-4.8 CXX=g++-4.8
       addons: *gcc48
+
     - os: linux
+      dist: trusty
       compiler: "gcc-4.9"
-      env: BUILD='Debug' CC=gcc-4.9 CXX=g++-4.9
-           COVERAGE=gcov-4.9
-           CXXFLAGS="--coverage" LDFLAGS="--coverage"
+      env: BUILD=Debug CC=gcc-4.9 CXX=g++-4.9
       addons: *gcc49
+
     - os: linux
+      dist: trusty
       compiler: "gcc-5"
-      env: BUILD='Debug' CC=gcc-5 CXX=g++-5
+      env: BUILD=Debug CC=gcc-5 CXX=g++-5
            CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0"
       addons: *gcc5
+
     - os: linux
+      dist: xenial
       compiler: "gcc-5"
-      env: BUILD='Debug' CC=gcc-5 CXX=g++-5
+      env: BUILD=Debug CC=gcc-5 CXX=g++-5
            CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=1"
       addons: *gcc5
+
     - os: linux
+      dist: xenial
       compiler: "gcc-6"
-      env: BUILD='Debug' CC=gcc-6 CXX=g++-6
+      env: BUILD=Debug CC=gcc-6 CXX=g++-6
       addons: *gcc6
+
     - os: linux
-      compiler: "gcc-6"
-      env: BUILD='Debug' CC=gcc-6 CXX=g++-6
-           PROTOZERO_DATA_VIEW=std::experimental::string_view
-      addons: *gcc6
+      dist: bionic
+      compiler: "gcc-7"
+      env: BUILD=Debug CC=gcc-7 CXX=g++-7
+      addons: *gcc7
+
     - os: linux
-      compiler: "gcc-6"
-      env: BUILD='Release' CC=gcc-6 CXX=g++-6
-      addons: *gcc6
+      dist: bionic
+      compiler: "gcc-8"
+      env: BUILD=Debug CC=gcc-8 CXX=g++-8
+      addons: *gcc8
+
+    - os: linux
+      dist: bionic
+      compiler: "gcc-8"
+      env: BUILD=Debug CC=gcc-8 CXX=g++-8
+           COVERAGE=gcov-8
+           CXXFLAGS="--coverage" LDFLAGS="--coverage"
+      addons: *gcc8
+
+    - os: linux
+      dist: bionic
+      compiler: "gcc-8"
+      env: BUILD=Debug CC=gcc-8 CXX=g++-8
+           PROTOZERO_DATA_VIEW=std::string_view
+      addons: *gcc8
+
+    - os: linux
+      dist: bionic
+      compiler: "gcc-8"
+      env: BUILD=Release CC=gcc-8 CXX=g++-8
+      addons: *gcc8
+
+    - os: linux
+      arch: arm64
+      dist: bionic
+      compiler: "gcc-8"
+      env: BUILD=Debug CC=gcc-8 CXX=g++-8
+      addons: *gcc8
+
+    - os: linux
+      arch: ppc64le
+      dist: bionic
+      compiler: "gcc-8"
+      env: BUILD=Debug CC=gcc-8 CXX=g++-8
+      addons: *gcc8
+
+    - os: linux
+      arch: s390x
+      dist: bionic
+      compiler: "gcc-8"
+      env: BUILD=Debug CC=gcc-8 CXX=g++-8
+      addons: *gcc8
+
     - os: osx
-      osx_image: xcode6.4
+      osx_image: xcode9.4
       compiler: clang
-      env: BUILD='Debug'
+      env: BUILD=Debug
+
     - os: osx
-      osx_image: xcode7.3
+      osx_image: xcode10.3
       compiler: clang
-      env: BUILD='Debug'
+      env: BUILD=Debug
+
     - os: osx
-      osx_image: xcode8.3
+      osx_image: xcode11.4
       compiler: clang
-      env: BUILD='Debug'
+      env: BUILD=Debug
+
     - os: osx
-      osx_image: xcode9.1
+      osx_image: xcode11.4
       compiler: clang
-      env: BUILD='Debug'
-    - os: osx
-      osx_image: xcode9.1
-      compiler: clang
-      env: BUILD='Release'
+      env: BUILD=Release
 
 #-----------------------------------------------------------------------------
 
-install:
-  - if [[ $(uname -s) == 'Darwin' ]]; then
-      brew update;
-      brew install protobuf;
-    fi
-
 script:
   - mkdir build
   - cd build

third_party/protozero/CHANGELOG.md

@@ -5,7 +5,6 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 This project adheres to [Semantic Versioning](http://semver.org/).
 
-
 ## [unreleased] -
 
 ### Added
@@ -15,6 +14,87 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Fixed
 
 
+## [1.7.0] - 2020-06-08
+
+### Added
+
+- Support for buffer types other that `std::string`. `pbf_writer` is now
+  just a typedef for `basic_pbf_writer<std::string>`. Other buffer types
+  can be used with `basic_pbf_writer`. See `doc/advanced.md` for details.
+
+### Changed
+
+- Switched to *catch2* for testing.
+- Some minor tweaks.
+
+### Fixed
+
+- Removed some undefined behaviour.
+
+
+## [1.6.8] - 2019-08-15
+
+### Changed
+
+- Various code cleanups due to clang-tidy warnings.
+
+### Fixed
+
+- Made `data_view::compare` noexcept.
+
+
+## [1.6.7] - 2018-02-21
+
+### Fixed
+
+- Signed-unsigned comparison on 32 bit systems.
+
+
+## [1.6.6] - 2018-02-20
+
+### Fixed
+
+- Fixed several place with possible undefined behaviour.
+
+
+## [1.6.5] - 2018-02-05
+
+### Fixed
+
+- Avoid UB: Do not calculate pointer outside array bounds.
+- Specify proto2 syntax in .proto files to appease protoc.
+
+
+## [1.6.4] - 2018-11-08
+
+### Added
+
+- Add function `data()` to get the not yet read data from a `pbf_reader`.
+- New `add_packed_fixed()` template function for `pbf_writer`.
+- New `length_of_varint()` helper function calculates how long a varint
+  would be for a specified value.
+
+### Changed
+
+- More consistent implementation of operators as free friend functions.
+
+### Fixed
+
+- Fixed some zigzag encoding tests on MSVC.
+- Add extra cast so we do an xor with unsigned ints.
+- No more bitwise operations on signed integers in varint decoder.
+- No more bitwise operations on signed integers in zigzag encoder/decoder.
+
+
+## [1.6.3] - 2018-07-17
+
+### Changed
+
+- Moved `byteswap_inplace` functions from detail into protozero namespace.
+  They can be useful outsize protozero.
+- More asserts and unit tests and small cleanups.
+
+
 ## [1.6.2] - 2018-03-09
 
 ### Changed
@@ -301,7 +381,14 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 - Make pbf reader and writer code endianess-aware.
 
 
-[unreleased]: https://github.com/osmcode/libosmium/compare/v1.6.2...HEAD
+[unreleased]: https://github.com/osmcode/libosmium/compare/v1.7.0...HEAD
+[1.7.0]: https://github.com/osmcode/libosmium/compare/v1.6.8...v1.7.0
+[1.6.8]: https://github.com/osmcode/libosmium/compare/v1.6.7...v1.6.8
+[1.6.7]: https://github.com/osmcode/libosmium/compare/v1.6.6...v1.6.7
+[1.6.6]: https://github.com/osmcode/libosmium/compare/v1.6.5...v1.6.6
+[1.6.5]: https://github.com/osmcode/libosmium/compare/v1.6.4...v1.6.5
+[1.6.4]: https://github.com/osmcode/libosmium/compare/v1.6.3...v1.6.4
+[1.6.3]: https://github.com/osmcode/libosmium/compare/v1.6.2...v1.6.3
 [1.6.2]: https://github.com/osmcode/libosmium/compare/v1.6.1...v1.6.2
 [1.6.1]: https://github.com/osmcode/libosmium/compare/v1.6.0...v1.6.1
 [1.6.0]: https://github.com/osmcode/libosmium/compare/v1.5.3...v1.6.0

third_party/protozero/CMakeLists.txt

@@ -13,8 +13,8 @@ cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
 project(protozero)
 
 set(PROTOZERO_VERSION_MAJOR 1)
-set(PROTOZERO_VERSION_MINOR 6)
-set(PROTOZERO_VERSION_PATCH 2)
+set(PROTOZERO_VERSION_MINOR 7)
+set(PROTOZERO_VERSION_PATCH 0)
 
 set(PROTOZERO_VERSION
     "${PROTOZERO_VERSION_MAJOR}.${PROTOZERO_VERSION_MINOR}.${PROTOZERO_VERSION_PATCH}")
@@ -26,7 +26,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 option(WERROR "Add -Werror flag to build (turns warnings into errors)" ON)
 
 if(MSVC)
-    add_definitions(-std=c++11 /W3)
+    add_definitions(/W3)
     add_definitions(-D_CRT_SECURE_NO_WARNINGS -D_SCL_SECURE_NO_WARNINGS)
 else()
     add_definitions(-std=c++11 -Wall -Wextra -pedantic -Wsign-compare -Wunused-parameter -Wno-float-equal -Wno-covered-switch-default)
@@ -58,7 +58,7 @@ find_package(Protobuf)
 #
 #-----------------------------------------------------------------------------
 message(STATUS "Looking for clang-tidy")
-find_program(CLANG_TIDY NAMES clang-tidy clang-tidy-6.0 clang-tidy-5.0)
+find_program(CLANG_TIDY NAMES clang-tidy clang-tidy-10 clang-tidy-9 clang-tidy-8 clang-tidy-7 clang-tidy-6.0 clang-tidy-5.0)
 
 if(CLANG_TIDY)
     message(STATUS "Looking for clang-tidy - found ${CLANG_TIDY}")
@@ -66,7 +66,8 @@ if(CLANG_TIDY)
         ${CLANG_TIDY}
         -p ${CMAKE_BINARY_DIR}
         ${CMAKE_SOURCE_DIR}/test/*.cpp
-        ${CMAKE_SOURCE_DIR}/test/t/*/*.cpp
+        ${CMAKE_SOURCE_DIR}/test/t/*/reader_test_cases.cpp
+        ${CMAKE_SOURCE_DIR}/test/t/*/writer_test_cases.cpp
         ${CMAKE_SOURCE_DIR}/test/unit/*.cpp
         ${CMAKE_SOURCE_DIR}/tools/*.cpp
     )

third_party/protozero/README.md

@@ -56,9 +56,9 @@ You have to have a working knowledge of how
 * Read the [upgrading instructions](UPGRADING.md) if you are upgrading from
   an older version of Protozero.
 
-The build process will also build the Doxygen-based reference documentation
-if you have [Doxygen](http://www.stack.nl/~dimitri/doxygen/) installed. Then
-open `doc/html/index.html` in your browser to read it.
+The build process will also build the Doxygen-based reference documentation if
+you have Doxygen installed. Then open `doc/html/index.html` in your browser to
+read it.
 
 
 ## Endianness

third_party/protozero/UPGRADING.md

@@ -13,6 +13,11 @@ macro `PROTOZERO_STRICT_API` in which case Protozero will compile without the
 code used for backwards compatibilty. You will then get compile errors for
 older API usages.
 
+## Upgrading from *v1.6* to *v1.7*
+
+* The `pbf_writer` class is now a typedef for `basic_pbf_writer<std::string>`
+  If you have forward declared it in your code, it might have to change.
+
 ## Upgrading from *v1.5* to *v1.6.0*
 
 * The `data_view` class moved from `types.hpp` into its own header file

third_party/protozero/appveyor.yml

@@ -24,6 +24,9 @@ environment:
     autocrlf: false
   - config: RelWithDebInfo
     autocrlf: false
+  - config: Debug
+    autocrlf: false
+    platform: x86
 
 #-----------------------------------------------------------------------------
 
@@ -36,22 +39,32 @@ init:
 # halts: "msys2-runtime and catgets are in conflict. Remove catgets?"
 # See also: https://github.com/Alexpux/MSYS2-packages/issues/1141
 install:
-  - if [%config%]==[MSYS2] (
-          C:\msys64\usr\bin\pacman --noconfirm --sync --refresh --refresh --sysupgrade --sysupgrade --ask=20
-       && C:\msys64\usr\bin\pacman -Rc --noconfirm mingw-w64-x86_64-gcc-libs
+  - if "%config%"=="MSYS2" (
+      set "PATH=C:\msys64\mingw64\bin;C:\msys64\usr\bin;%PATH%" &&
+      pacman --noconfirm --sync --refresh --refresh --sysupgrade --sysupgrade --ask=20 &&
+      pacman -Rc --noconfirm mingw-w64-x86_64-gcc-libs &&
+      pacman -S --needed --noconfirm mingw-w64-x86_64-gcc mingw-w64-x86_64-cmake mingw-w64-x86_64-doxygen mingw-w64-x86_64-protobuf
     )
 
 build_script:
-  - if [%config%]==[MSYS2] (
-        build-msys2.bat
+  - cd c:\projects\protozero
+  - mkdir build
+  - cd build
+  - if "%platform%"=="x64" (
+      set vcvarsall_arg=amd64
     ) else (
-        build-appveyor.bat
+      set vcvarsall_arg=x86
+    )
+  - if "%config%"=="MSYS2" (
+      cmake .. -LA -G "MSYS Makefiles" &&
+      make VERBOSE=1
+    ) else (
+      "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall" %vcvarsall_arg% &&
+      cmake .. -LA -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=%config% &&
+      nmake VERBOSE=1
     )
 
-# remove garbage VS messages
-# http://help.appveyor.com/discussions/problems/4569-the-target-_convertpdbfiles-listed-in-a-beforetargets-attribute-at-c-does-not-exist-in-the-project-and-will-be-ignored
-before_build:
-  - del "C:\Program Files (x86)\MSBuild\14.0\Microsoft.Common.targets\ImportAfter\Xamarin.Common.targets"
-
+test_script:
+  - ctest --output-on-failure
 
 #-----------------------------------------------------------------------------

third_party/protozero/build-appveyor.bat

@@ -1,65 +0,0 @@
-@ECHO OFF
-SETLOCAL
-SET EL=0
-
-ECHO ~~~~~~ %~f0 ~~~~~~
-
-::show all available env vars
-SET
-ECHO cmake on AppVeyor
-cmake -version
-
-ECHO activating VS cmd prompt && CALL "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64
-IF %ERRORLEVEL% NEQ 0 GOTO ERROR
-
-SET protobuf_sdk=protozero-dep-protobuf-2.6.1.7z
-IF EXIST %protobuf_sdk% (ECHO protobuf already downloaded) ELSE (ECHO downloading protobuf ... && powershell Invoke-WebRequest https://mapbox.s3.amazonaws.com/windows-builds/windows-build-deps/$env:protobuf_sdk -OutFile $pwd\$env:protobuf_sdk)
-IF %ERRORLEVEL% NEQ 0 GOTO ERROR
-IF EXIST deps\protobuf (ECHO protobuf already extracted) ELSE (CALL 7z x -y %protobuf_sdk% | %windir%\system32\FIND "ing archive")
-IF %ERRORLEVEL% NEQ 0 GOTO ERROR
-SET PATH=%~dp0deps\protobuf;%PATH%
-
-IF EXIST build ECHO deleting build dir... && RD /Q /S build
-IF %ERRORLEVEL% NEQ 0 GOTO ERROR
-
-MKDIR build
-IF %ERRORLEVEL% NEQ 0 GOTO ERROR
-
-CD build
-ECHO config^: %config%
-
-::This will produce lots of LNK4099 warnings which can be ignored.
-::Unfortunately they can't be disabled, see
-::http://stackoverflow.com/questions/661606/visual-c-how-to-disable-specific-linker-warnings
-SET CMAKE_CMD=cmake .. ^
--LA -G "Visual Studio 14 Win64"
-
-ECHO calling^: %CMAKE_CMD%
-%CMAKE_CMD%
-IF %ERRORLEVEL% NEQ 0 GOTO ERROR
-
-SET avlogger=
-IF /I "%APPVEYOR%"=="True" SET avlogger=/logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll"
-
-msbuild protozero.sln ^
-/p:Configuration=%config% ^
-/toolsversion:14.0 ^
-/p:Platform=x64 ^
-/p:PlatformToolset=v140 %avlogger%
-IF %ERRORLEVEL% NEQ 0 GOTO ERROR
-
-ctest --output-on-failure ^
--C %config% ^
-IF %ERRORLEVEL% NEQ 0 GOTO ERROR
-
-GOTO DONE
-
-:ERROR
-ECHO ~~~~~~ ERROR %~f0 ~~~~~~
-SET EL=%ERRORLEVEL%
-
-:DONE
-IF %EL% NEQ 0 ECHO. && ECHO !!! ERRORLEVEL^: %EL% !!! && ECHO.
-ECHO ~~~~~~ DONE %~f0 ~~~~~~
-
-EXIT /b %EL%

third_party/protozero/build-local.bat

@@ -1,29 +0,0 @@
-@ECHO OFF
-SETLOCAL
-SET EL=0
-
-ECHO =========== %~f0 ===========
-
-SET VERBOSITY_MSBUILD=diagnostic
-IF NOT "%1"=="" SET VERBOSITY_MSBUILD=%1
-SET platform=x64
-SET configuration=Release
-CALL build-appveyor.bat %VERBOSITY_MSBUILD%
-IF %ERRORLEVEL% NEQ 0 GOTO ERROR
-
-SET platform=x86
-SET configuration=Debug
-CALL build-appveyor.bat %VERBOSITY_MSBUILD%
-IF %ERRORLEVEL% NEQ 0 GOTO ERROR
-
-GOTO DONE
-
-:ERROR
-ECHO =========== ERROR %~f0 ===========
-ECHO ERRORLEVEL^: %ERRORLEVEL%
-SET EL=%ERRORLEVEL%
-
-:DONE
-ECHO =========== DONE %~f0 ===========
-
-EXIT /b %EL%

third_party/protozero/build-msys2.bat

@@ -1,18 +0,0 @@
-echo "Adding MSYS2 to path..."
-SET "PATH=C:\msys64\mingw64\bin;C:\msys64\usr\bin;%PATH%"
-echo %PATH%
-
-echo "Installing MSYS2 packages..."
-bash -lc "pacman -S --needed --noconfirm mingw-w64-x86_64-gcc mingw-w64-x86_64-cmake mingw-w64-x86_64-doxygen mingw-w64-x86_64-protobuf"
-
-echo "Generating makefiles"
-mkdir build
-cd build
-cmake .. -LA -G "MSYS Makefiles"
-
-echo "Building"
-make VERBOSE=1
-
-echo "Testing"
-ctest --output-on-failure
-

third_party/protozero/doc/Doxyfile.in

@@ -2046,7 +2046,7 @@ EXTERNAL_PAGES         = YES
 # interpreter (i.e. the result of 'which perl').
 # The default file (with absolute path) is: /usr/bin/perl.
 
-PERL_PATH              = /usr/bin/perl
+#PERL_PATH              = /usr/bin/perl
 
 #---------------------------------------------------------------------------
 # Configuration options related to the dot tool
@@ -2068,7 +2068,7 @@ CLASS_DIAGRAMS         = YES
 # the mscgen tool resides. If left empty the tool is assumed to be found in the
 # default search path.
 
-MSCGEN_PATH            =
+#MSCGEN_PATH            =
 
 # You can include diagrams made with dia in doxygen documentation. Doxygen will
 # then run dia to produce the diagram and insert it in the documentation. The

third_party/protozero/doc/advanced.md

@@ -269,3 +269,58 @@ still considerably cheaper than decoding the varints. You have to benchmark
 your use case to see whether the `reserve()` (or whatever you are using the
 `size()` for) is worth it.
 
+
+## Using a different buffer class than std::string
+
+Normally you are using the `pbf_writer` or `pbf_builder` classes which use a
+`std::string` that you supply as their buffer for building the actual protocol
+buffers message into. But you can use a different buffer implementation
+instead. This might be useful if you want to use a fixed-size buffer for
+instance.
+
+The `pbf_writer` and `pbf_builder` classes are actually only aliases for the
+`basic_pbf_writer` and `basic_pbf_builder` template classes:
+
+```cpp
+using pbf_writer = basic_pbf_writer<std::string>;
+
+template <typename T>
+using pbf_builder = basic_pbf_builder<std::string, T>;
+```
+
+If you want to use a different buffer type, use the `basic_*` form of the
+class and use the buffer class as template parameter. When instantiating the
+`basic_pbf_writer` or `basic_pbf_builder`, the only parameter to the
+constructor must always be a reference to an object of the buffer class.
+
+```cpp
+some_buffer_class buffer;
+basic_pbf_writer<some_buffer_class> writer{buffer};
+```
+
+For this to work you must supply template specializations for some static
+functions in the `protozero::buffer_customization` struct, see
+`buffer_tmpl.hpp` for details.
+
+Protozero already supports two buffer types:
+* `std::string` (to use include `protozero/buffer_string.hpp`)
+* `std::vector<char>` (to use include `protozero/buffer_vector.hpp`)
+
+There is a class `protozero::fixed_size_buffer_adaptor` you can use as adaptor
+for any fixed-sized buffer you might have. Include `protozero/buffer_fixed.hpp`
+to use it:
+
+```cpp
+#include <protozero/buffer_fixed.hpp>
+
+your_buffer_class some_buffer;
+protozero::fixed_size_buffer_adaptor buffer_adaptor{some_buffer.data(), some_buffer.size()};
+basic_pbf_writer<protozero::fixed_size_buffer_adaptor> writer{buffer_adaptor};
+```
+
+The buffer adaptor can be initialized with any container if it supports the
+`data()` and `size()` member functions:
+
+```cpp
+protozero::fixed_size_buffer_adaptor buffer_adaptor{some_buffer};
+```

third_party/protozero/include/protozero/basic_pbf_builder.hpp

@@ -0,0 +1,266 @@
+#ifndef PROTOZERO_BASIC_PBF_BUILDER_HPP
+#define PROTOZERO_BASIC_PBF_BUILDER_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file basic_pbf_builder.hpp
+ *
+ * @brief Contains the basic_pbf_builder template class.
+ */
+
+#include "basic_pbf_writer.hpp"
+#include "types.hpp"
+
+#include <type_traits>
+
+namespace protozero {
+
+/**
+ * The basic_pbf_builder is used to write PBF formatted messages into a buffer.
+ * It is based on the basic_pbf_writer class and has all the same methods. The
+ * difference is that while the pbf_writer class takes an integer tag,
+ * this template class takes a tag of the template type T. The idea is that
+ * T will be an enumeration value and this helps reduce the possibility of
+ * programming errors.
+ *
+ * Almost all methods in this class can throw an std::bad_alloc exception if
+ * the underlying buffer class wants to resize.
+ *
+ * Read the tutorial to understand how this class is used. In most cases you
+ * want to use the pbf_builder class which uses a std::string as buffer type.
+ */
+template <typename TBuffer, typename T>
+class basic_pbf_builder : public basic_pbf_writer<TBuffer> {
+
+    static_assert(std::is_same<pbf_tag_type, typename std::underlying_type<T>::type>::value,
+                  "T must be enum with underlying type protozero::pbf_tag_type");
+
+public:
+
+    /// The type of messages this class will build.
+    using enum_type = T;
+
+    basic_pbf_builder() = default;
+
+    /**
+     * Create a builder using the given string as a data store. The object
+     * stores a reference to that string and adds all data to it. The string
+     * doesn't have to be empty. The pbf_message object will just append data.
+     */
+    explicit basic_pbf_builder(TBuffer& data) noexcept :
+        basic_pbf_writer<TBuffer>{data} {
+    }
+
+    /**
+     * Construct a pbf_builder for a submessage from the pbf_message or
+     * pbf_writer of the parent message.
+     *
+     * @param parent_writer The parent pbf_message or pbf_writer
+     * @param tag Tag of the field that will be written
+     */
+    template <typename P>
+    basic_pbf_builder(basic_pbf_writer<TBuffer>& parent_writer, P tag) noexcept :
+        basic_pbf_writer<TBuffer>{parent_writer, pbf_tag_type(tag)} {
+    }
+
+/// @cond INTERNAL
+#define PROTOZERO_WRITER_WRAP_ADD_SCALAR(name, type) \
+    void add_##name(T tag, type value) { \
+        basic_pbf_writer<TBuffer>::add_##name(pbf_tag_type(tag), value); \
+    }
+
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(bool, bool)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(enum, int32_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(int32, int32_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint32, int32_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint32, uint32_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(int64, int64_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint64, int64_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint64, uint64_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed32, uint32_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed32, int32_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed64, uint64_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed64, int64_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(float, float)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(double, double)
+
+#undef PROTOZERO_WRITER_WRAP_ADD_SCALAR
+/// @endcond
+
+    /**
+     * Add "bytes" field to data.
+     *
+     * @param tag Tag of the field
+     * @param value Pointer to value to be written
+     * @param size Number of bytes to be written
+     */
+    void add_bytes(T tag, const char* value, std::size_t size) {
+        basic_pbf_writer<TBuffer>::add_bytes(pbf_tag_type(tag), value, size);
+    }
+
+    /**
+     * Add "bytes" field to data.
+     *
+     * @param tag Tag of the field
+     * @param value Value to be written
+     */
+    void add_bytes(T tag, const data_view& value) {
+        basic_pbf_writer<TBuffer>::add_bytes(pbf_tag_type(tag), value);
+    }
+
+    /**
+     * Add "bytes" field to data.
+     *
+     * @param tag Tag of the field
+     * @param value Value to be written
+     */
+    void add_bytes(T tag, const std::string& value) {
+        basic_pbf_writer<TBuffer>::add_bytes(pbf_tag_type(tag), value);
+    }
+
+    /**
+     * Add "bytes" field to data. Bytes from the value are written until
+     * a null byte is encountered. The null byte is not added.
+     *
+     * @param tag Tag of the field
+     * @param value Pointer to zero-delimited value to be written
+     */
+    void add_bytes(T tag, const char* value) {
+        basic_pbf_writer<TBuffer>::add_bytes(pbf_tag_type(tag), value);
+    }
+
+    /**
+     * Add "bytes" field to data using vectored input. All the data in the
+     * 2nd and further arguments is "concatenated" with only a single copy
+     * into the final buffer.
+     *
+     * This will work with objects of any type supporting the data() and
+     * size() methods like std::string or protozero::data_view.
+     *
+     * Example:
+     * @code
+     * std::string data1 = "abc";
+     * std::string data2 = "xyz";
+     * builder.add_bytes_vectored(1, data1, data2);
+     * @endcode
+     *
+     * @tparam Ts List of types supporting data() and size() methods.
+     * @param tag Tag of the field
+     * @param values List of objects of types Ts with data to be appended.
+     */
+    template <typename... Ts>
+    void add_bytes_vectored(T tag, Ts&&... values) {
+        basic_pbf_writer<TBuffer>::add_bytes_vectored(pbf_tag_type(tag), std::forward<Ts>(values)...);
+    }
+
+    /**
+     * Add "string" field to data.
+     *
+     * @param tag Tag of the field
+     * @param value Pointer to value to be written
+     * @param size Number of bytes to be written
+     */
+    void add_string(T tag, const char* value, std::size_t size) {
+        basic_pbf_writer<TBuffer>::add_string(pbf_tag_type(tag), value, size);
+    }
+
+    /**
+     * Add "string" field to data.
+     *
+     * @param tag Tag of the field
+     * @param value Value to be written
+     */
+    void add_string(T tag, const data_view& value) {
+        basic_pbf_writer<TBuffer>::add_string(pbf_tag_type(tag), value);
+    }
+
+    /**
+     * Add "string" field to data.
+     *
+     * @param tag Tag of the field
+     * @param value Value to be written
+     */
+    void add_string(T tag, const std::string& value) {
+        basic_pbf_writer<TBuffer>::add_string(pbf_tag_type(tag), value);
+    }
+
+    /**
+     * Add "string" field to data. Bytes from the value are written until
+     * a null byte is encountered. The null byte is not added.
+     *
+     * @param tag Tag of the field
+     * @param value Pointer to value to be written
+     */
+    void add_string(T tag, const char* value) {
+        basic_pbf_writer<TBuffer>::add_string(pbf_tag_type(tag), value);
+    }
+
+    /**
+     * Add "message" field to data.
+     *
+     * @param tag Tag of the field
+     * @param value Pointer to message to be written
+     * @param size Length of the message
+     */
+    void add_message(T tag, const char* value, std::size_t size) {
+        basic_pbf_writer<TBuffer>::add_message(pbf_tag_type(tag), value, size);
+    }
+
+    /**
+     * Add "message" field to data.
+     *
+     * @param tag Tag of the field
+     * @param value Value to be written. The value must be a complete message.
+     */
+    void add_message(T tag, const data_view& value) {
+        basic_pbf_writer<TBuffer>::add_message(pbf_tag_type(tag), value);
+    }
+
+    /**
+     * Add "message" field to data.
+     *
+     * @param tag Tag of the field
+     * @param value Value to be written. The value must be a complete message.
+     */
+    void add_message(T tag, const std::string& value) {
+        basic_pbf_writer<TBuffer>::add_message(pbf_tag_type(tag), value);
+    }
+
+/// @cond INTERNAL
+#define PROTOZERO_WRITER_WRAP_ADD_PACKED(name) \
+    template <typename InputIterator> \
+    void add_packed_##name(T tag, InputIterator first, InputIterator last) { \
+        basic_pbf_writer<TBuffer>::add_packed_##name(pbf_tag_type(tag), first, last); \
+    }
+
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(bool)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(enum)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(int32)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(sint32)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(uint32)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(int64)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(sint64)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(uint64)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed32)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed32)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed64)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed64)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(float)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(double)
+
+#undef PROTOZERO_WRITER_WRAP_ADD_PACKED
+/// @endcond
+
+}; // class basic_pbf_builder
+
+} // end namespace protozero
+
+#endif // PROTOZERO_BASIC_PBF_BUILDER_HPP

third_party/protozero/include/protozero/buffer_fixed.hpp

@@ -0,0 +1,222 @@
+#ifndef PROTOZERO_BUFFER_FIXED_HPP
+#define PROTOZERO_BUFFER_FIXED_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file buffer_fixed.hpp
+ *
+ * @brief Contains the fixed_size_buffer_adaptor class.
+ */
+
+#include "buffer_tmpl.hpp"
+#include "config.hpp"
+
+#include <algorithm>
+#include <cstddef>
+#include <iterator>
+#include <stdexcept>
+
+namespace protozero {
+
+/**
+ * This class can be used instead of std::string if you want to create a
+ * vector tile in a fixed-size buffer. Any operation that needs more space
+ * than is available will fail with a std::length_error exception.
+ */
+class fixed_size_buffer_adaptor {
+
+    char* m_data;
+    std::size_t m_capacity;
+    std::size_t m_size = 0;
+
+public:
+
+    /// @cond usual container typedefs not documented
+
+    using size_type = std::size_t;
+
+    using value_type = char;
+    using reference = value_type&;
+    using const_reference = const value_type&;
+    using pointer = value_type*;
+    using const_pointer = const value_type*;
+
+    using iterator = pointer;
+    using const_iterator = const_pointer;
+
+    /// @endcond
+
+    /**
+     * Constructor.
+     *
+     * @param data Pointer to some memory allocated for the buffer.
+     * @param capacity Number of bytes available.
+     */
+    fixed_size_buffer_adaptor(char* data, std::size_t capacity) noexcept :
+        m_data(data),
+        m_capacity(capacity) {
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param container Some container class supporting the member functions
+     *        data() and size().
+     */
+    template <typename T>
+    explicit fixed_size_buffer_adaptor(T& container) :
+        m_data(container.data()),
+        m_capacity(container.size()) {
+    }
+
+    /// Returns a pointer to the data in the buffer.
+    const char* data() const noexcept {
+        return m_data;
+    }
+
+    /// Returns a pointer to the data in the buffer.
+    char* data() noexcept {
+        return m_data;
+    }
+
+    /// The capacity this buffer was created with.
+    std::size_t capacity() const noexcept {
+        return m_capacity;
+    }
+
+    /// The number of bytes used in the buffer. Always <= capacity().
+    std::size_t size() const noexcept {
+        return m_size;
+    }
+
+    /// Return iterator to beginning of data.
+    char* begin() noexcept {
+        return m_data;
+    }
+
+    /// Return iterator to beginning of data.
+    const char* begin() const noexcept {
+        return m_data;
+    }
+
+    /// Return iterator to beginning of data.
+    const char* cbegin() const noexcept {
+        return m_data;
+    }
+
+    /// Return iterator to end of data.
+    char* end() noexcept {
+        return m_data + m_size;
+    }
+
+    /// Return iterator to end of data.
+    const char* end() const noexcept {
+        return m_data + m_size;
+    }
+
+    /// Return iterator to end of data.
+    const char* cend() const noexcept {
+        return m_data + m_size;
+    }
+
+/// @cond INTERNAL
+
+    // Do not rely on anything beyond this point
+
+    void append(const char* data, std::size_t count) {
+        if (m_size + count > m_capacity) {
+            throw std::length_error{"fixed size data store exhausted"};
+        }
+        std::copy_n(data, count, m_data + m_size);
+        m_size += count;
+    }
+
+    void append_zeros(std::size_t count) {
+        if (m_size + count > m_capacity) {
+            throw std::length_error{"fixed size data store exhausted"};
+        }
+        std::fill_n(m_data + m_size, count, '\0');
+        m_size += count;
+    }
+
+    void resize(std::size_t size) {
+        protozero_assert(size < m_size);
+        if (size > m_capacity) {
+            throw std::length_error{"fixed size data store exhausted"};
+        }
+        m_size = size;
+    }
+
+    void erase_range(std::size_t from, std::size_t to) {
+        protozero_assert(from <= m_size);
+        protozero_assert(to <= m_size);
+        protozero_assert(from < to);
+        std::copy(m_data + to, m_data + m_size, m_data + from);
+        m_size -= (to - from);
+    }
+
+    char* at_pos(std::size_t pos) {
+        protozero_assert(pos <= m_size);
+        return m_data + pos;
+    }
+
+    void push_back(char ch) {
+        if (m_size >= m_capacity) {
+            throw std::length_error{"fixed size data store exhausted"};
+        }
+        m_data[m_size++] = ch;
+    }
+/// @endcond
+
+}; // class fixed_size_buffer_adaptor
+
+/// @cond INTERNAL
+template <>
+struct buffer_customization<fixed_size_buffer_adaptor> {
+
+    static std::size_t size(const fixed_size_buffer_adaptor* buffer) noexcept {
+        return buffer->size();
+    }
+
+    static void append(fixed_size_buffer_adaptor* buffer, const char* data, std::size_t count) {
+        buffer->append(data, count);
+    }
+
+    static void append_zeros(fixed_size_buffer_adaptor* buffer, std::size_t count) {
+        buffer->append_zeros(count);
+    }
+
+    static void resize(fixed_size_buffer_adaptor* buffer, std::size_t size) {
+        buffer->resize(size);
+    }
+
+    static void reserve_additional(fixed_size_buffer_adaptor* /*buffer*/, std::size_t /*size*/) {
+        /* nothing to be done for fixed-size buffers */
+    }
+
+    static void erase_range(fixed_size_buffer_adaptor* buffer, std::size_t from, std::size_t to) {
+        buffer->erase_range(from, to);
+    }
+
+    static char* at_pos(fixed_size_buffer_adaptor* buffer, std::size_t pos) {
+        return buffer->at_pos(pos);
+    }
+
+    static void push_back(fixed_size_buffer_adaptor* buffer, char ch) {
+        buffer->push_back(ch);
+    }
+
+};
+/// @endcond
+
+} // namespace protozero
+
+#endif // PROTOZERO_BUFFER_FIXED_HPP

third_party/protozero/include/protozero/buffer_string.hpp

@@ -0,0 +1,76 @@
+#ifndef PROTOZERO_BUFFER_STRING_HPP
+#define PROTOZERO_BUFFER_STRING_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file buffer_string.hpp
+ *
+ * @brief Contains the customization points for buffer implementation based
+ *        on std::string
+ */
+
+#include "buffer_tmpl.hpp"
+
+#include <cstddef>
+#include <iterator>
+#include <string>
+
+namespace protozero {
+
+// Implementation of buffer customizations points for std::string
+
+/// @cond INTERNAL
+template <>
+struct buffer_customization<std::string> {
+
+    static std::size_t size(const std::string* buffer) noexcept {
+        return buffer->size();
+    }
+
+    static void append(std::string* buffer, const char* data, std::size_t count) {
+        buffer->append(data, count);
+    }
+
+    static void append_zeros(std::string* buffer, std::size_t count) {
+        buffer->append(count, '\0');
+    }
+
+    static void resize(std::string* buffer, std::size_t size) {
+        protozero_assert(size < buffer->size());
+        buffer->resize(size);
+    }
+
+    static void reserve_additional(std::string* buffer, std::size_t size) {
+        buffer->reserve(buffer->size() + size);
+    }
+
+    static void erase_range(std::string* buffer, std::size_t from, std::size_t to) {
+        protozero_assert(from <= buffer->size());
+        protozero_assert(to <= buffer->size());
+        protozero_assert(from <= to);
+        buffer->erase(std::next(buffer->begin(), from), std::next(buffer->begin(), to));
+    }
+
+    static char* at_pos(std::string* buffer, std::size_t pos) {
+        protozero_assert(pos <= buffer->size());
+        return (&*buffer->begin()) + pos;
+    }
+
+    static void push_back(std::string* buffer, char ch) {
+        buffer->push_back(ch);
+    }
+
+};
+/// @endcond
+
+} // namespace protozero
+
+#endif // PROTOZERO_BUFFER_STRING_HPP

third_party/protozero/include/protozero/buffer_tmpl.hpp

@@ -0,0 +1,113 @@
+#ifndef PROTOZERO_BUFFER_TMPL_HPP
+#define PROTOZERO_BUFFER_TMPL_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file buffer_tmpl.hpp
+ *
+ * @brief Contains the customization points for buffer implementations.
+ */
+
+#include <cstddef>
+#include <iterator>
+#include <string>
+
+namespace protozero {
+
+// Implementation of buffer customizations points for std::string
+
+/// @cond INTERNAL
+template <typename T>
+struct buffer_customization {
+
+    /**
+     * Get the number of bytes currently used in the buffer.
+     *
+     * @param buffer Pointer to the buffer.
+     * @returns number of bytes used in the buffer.
+     */
+    static std::size_t size(const std::string* buffer);
+
+    /**
+     * Append count bytes from data to the buffer.
+     *
+     * @param buffer Pointer to the buffer.
+     * @param data Pointer to the data.
+     * @param count Number of bytes to be added to the buffer.
+     */
+    static void append(std::string* buffer, const char* data, std::size_t count);
+
+    /**
+     * Append count zero bytes to the buffer.
+     *
+     * @param buffer Pointer to the buffer.
+     * @param count Number of bytes to be added to the buffer.
+     */
+    static void append_zeros(std::string* buffer, std::size_t count);
+
+    /**
+     * Shrink the buffer to the specified size. The new size will always be
+     * smaller than the current size.
+     *
+     * @param buffer Pointer to the buffer.
+     * @param size New size of the buffer.
+     *
+     * @pre size < current size of buffer
+     */
+    static void resize(std::string* buffer, std::size_t size);
+
+    /**
+     * Reserve an additional size bytes for use in the buffer. This is used for
+     * variable-sized buffers to tell the buffer implementation that soon more
+     * memory will be used. The implementation can ignore this.
+     *
+     * @param buffer Pointer to the buffer.
+     * @param size Number of bytes to reserve.
+     */
+    static void reserve_additional(std::string* buffer, std::size_t size);
+
+    /**
+     * Delete data from the buffer. This must move back the data after the
+     * part being deleted and resize the buffer accordingly.
+     *
+     * @param buffer Pointer to the buffer.
+     * @param from Offset into the buffer where we want to erase from.
+     * @param to Offset into the buffer one past the last byte we want to erase.
+     *
+     * @pre from, to <= size of the buffer, from < to
+     */
+    static void erase_range(std::string* buffer, std::size_t from, std::size_t to);
+
+    /**
+     * Return a pointer to the memory at the specified position in the buffer.
+     *
+     * @param buffer Pointer to the buffer.
+     * @param pos The position in the buffer.
+     * @returns pointer to the memory in the buffer at the specified position.
+     *
+     * @pre pos <= size of the buffer
+     */
+    static char* at_pos(std::string* buffer, std::size_t pos);
+
+    /**
+     * Add a char to the buffer incrementing the number of chars in the buffer.
+     *
+     * @param buffer Pointer to the buffer.
+     * @param ch The character to add.
+     */
+    static void push_back(std::string* buffer, char ch);
+
+};
+/// @endcond
+
+} // namespace protozero
+
+#endif // PROTOZERO_BUFFER_TMPL_HPP

third_party/protozero/include/protozero/buffer_vector.hpp

@@ -0,0 +1,76 @@
+#ifndef PROTOZERO_BUFFER_VECTOR_HPP
+#define PROTOZERO_BUFFER_VECTOR_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file buffer_vector.hpp
+ *
+ * @brief Contains the customization points for buffer implementation based
+ *        on std::vector<char>
+ */
+
+#include "buffer_tmpl.hpp"
+
+#include <cstddef>
+#include <iterator>
+#include <vector>
+
+namespace protozero {
+
+// Implementation of buffer customizations points for std::vector<char>
+
+/// @cond INTERNAL
+template <>
+struct buffer_customization<std::vector<char>> {
+
+    static std::size_t size(const std::vector<char>* buffer) noexcept {
+        return buffer->size();
+    }
+
+    static void append(std::vector<char>* buffer, const char* data, std::size_t count) {
+        buffer->insert(buffer->end(), data, data + count);
+    }
+
+    static void append_zeros(std::vector<char>* buffer, std::size_t count) {
+        buffer->insert(buffer->end(), count, '\0');
+    }
+
+    static void resize(std::vector<char>* buffer, std::size_t size) {
+        protozero_assert(size < buffer->size());
+        buffer->resize(size);
+    }
+
+    static void reserve_additional(std::vector<char>* buffer, std::size_t size) {
+        buffer->reserve(buffer->size() + size);
+    }
+
+    static void erase_range(std::vector<char>* buffer, std::size_t from, std::size_t to) {
+        protozero_assert(from <= buffer->size());
+        protozero_assert(to <= buffer->size());
+        protozero_assert(from <= to);
+        buffer->erase(std::next(buffer->begin(), from), std::next(buffer->begin(), to));
+    }
+
+    static char* at_pos(std::vector<char>* buffer, std::size_t pos) {
+        protozero_assert(pos <= buffer->size());
+        return (&*buffer->begin()) + pos;
+    }
+
+    static void push_back(std::vector<char>* buffer, char ch) {
+        buffer->push_back(ch);
+    }
+
+};
+/// @endcond
+
+} // namespace protozero
+
+#endif // PROTOZERO_BUFFER_VECTOR_HPP

third_party/protozero/include/protozero/byteswap.hpp

@@ -16,7 +16,7 @@ documentation.
  * @brief Contains functions to swap bytes in values (for different endianness).
  */
 
-#include <protozero/config.hpp>
+#include "config.hpp"
 
 #include <cstdint>
 
@@ -27,10 +27,10 @@ inline uint32_t byteswap_impl(uint32_t value) noexcept {
 #ifdef PROTOZERO_USE_BUILTIN_BSWAP
     return __builtin_bswap32(value);
 #else
-    return ((value & 0xff000000) >> 24) |
-           ((value & 0x00ff0000) >>  8) |
-           ((value & 0x0000ff00) <<  8) |
-           ((value & 0x000000ff) << 24);
+    return ((value & 0xff000000U) >> 24U) |
+           ((value & 0x00ff0000U) >>  8U) |
+           ((value & 0x0000ff00U) <<  8U) |
+           ((value & 0x000000ffU) << 24U);
 #endif
 }
 
@@ -38,46 +38,62 @@ inline uint64_t byteswap_impl(uint64_t value) noexcept {
 #ifdef PROTOZERO_USE_BUILTIN_BSWAP
     return __builtin_bswap64(value);
 #else
-    return ((value & 0xff00000000000000ULL) >> 56) |
-           ((value & 0x00ff000000000000ULL) >> 40) |
-           ((value & 0x0000ff0000000000ULL) >> 24) |
-           ((value & 0x000000ff00000000ULL) >>  8) |
-           ((value & 0x00000000ff000000ULL) <<  8) |
-           ((value & 0x0000000000ff0000ULL) << 24) |
-           ((value & 0x000000000000ff00ULL) << 40) |
-           ((value & 0x00000000000000ffULL) << 56);
+    return ((value & 0xff00000000000000ULL) >> 56U) |
+           ((value & 0x00ff000000000000ULL) >> 40U) |
+           ((value & 0x0000ff0000000000ULL) >> 24U) |
+           ((value & 0x000000ff00000000ULL) >>  8U) |
+           ((value & 0x00000000ff000000ULL) <<  8U) |
+           ((value & 0x0000000000ff0000ULL) << 24U) |
+           ((value & 0x000000000000ff00ULL) << 40U) |
+           ((value & 0x00000000000000ffULL) << 56U);
 #endif
 }
 
+} // end namespace detail
+
+/// byteswap the data pointed to by ptr in-place.
 inline void byteswap_inplace(uint32_t* ptr) noexcept {
-    *ptr = byteswap_impl(*ptr);
+    *ptr = detail::byteswap_impl(*ptr);
 }
 
+/// byteswap the data pointed to by ptr in-place.
 inline void byteswap_inplace(uint64_t* ptr) noexcept {
-    *ptr = byteswap_impl(*ptr);
+    *ptr = detail::byteswap_impl(*ptr);
 }
 
+/// byteswap the data pointed to by ptr in-place.
 inline void byteswap_inplace(int32_t* ptr) noexcept {
-    auto bptr = reinterpret_cast<uint32_t*>(ptr);
-    *bptr = byteswap_impl(*bptr);
+    auto* bptr = reinterpret_cast<uint32_t*>(ptr);
+    *bptr = detail::byteswap_impl(*bptr);
 }
 
+/// byteswap the data pointed to by ptr in-place.
 inline void byteswap_inplace(int64_t* ptr) noexcept {
-    auto bptr = reinterpret_cast<uint64_t*>(ptr);
-    *bptr = byteswap_impl(*bptr);
+    auto* bptr = reinterpret_cast<uint64_t*>(ptr);
+    *bptr = detail::byteswap_impl(*bptr);
 }
 
+/// byteswap the data pointed to by ptr in-place.
 inline void byteswap_inplace(float* ptr) noexcept {
-    auto bptr = reinterpret_cast<uint32_t*>(ptr);
-    *bptr = byteswap_impl(*bptr);
+    auto* bptr = reinterpret_cast<uint32_t*>(ptr);
+    *bptr = detail::byteswap_impl(*bptr);
 }
 
+/// byteswap the data pointed to by ptr in-place.
 inline void byteswap_inplace(double* ptr) noexcept {
-    auto bptr = reinterpret_cast<uint64_t*>(ptr);
-    *bptr = byteswap_impl(*bptr);
+    auto* bptr = reinterpret_cast<uint64_t*>(ptr);
+    *bptr = detail::byteswap_impl(*bptr);
 }
 
+namespace detail {
+
+    // Added for backwards compatibility with any code that might use this
+    // function (even if it shouldn't have). Will be removed in a later
+    // version of protozero.
+    using ::protozero::byteswap_inplace;
+
 } // end namespace detail
+
 } // end namespace protozero
 
 #endif // PROTOZERO_BYTESWAP_HPP

third_party/protozero/include/protozero/data_view.hpp

@@ -16,7 +16,7 @@ documentation.
  * @brief Contains the implementation of the data_view class.
  */
 
-#include <protozero/config.hpp>
+#include "config.hpp"
 
 #include <algorithm>
 #include <cstddef>
@@ -55,8 +55,8 @@ public:
      * @param length Length of the data.
      */
     constexpr data_view(const char* ptr, std::size_t length) noexcept
-        : m_data(ptr),
-          m_size(length) {
+        : m_data{ptr},
+          m_size{length} {
     }
 
     /**
@@ -65,8 +65,8 @@ public:
      * @param str String with the data.
      */
     data_view(const std::string& str) noexcept // NOLINT(google-explicit-constructor, hicpp-explicit-conversions)
-        : m_data(str.data()),
-          m_size(str.size()) {
+        : m_data{str.data()},
+          m_size{str.size()} {
     }
 
     /**
@@ -75,8 +75,8 @@ public:
      * @param ptr Pointer to the data.
      */
     data_view(const char* ptr) noexcept // NOLINT(google-explicit-constructor, hicpp-explicit-conversions)
-        : m_data(ptr),
-          m_size(std::strlen(ptr)) {
+        : m_data{ptr},
+          m_size{std::strlen(ptr)} {
     }
 
     /**
@@ -141,8 +141,8 @@ public:
      *
      * @pre Must not be default constructed data_view.
      */
-    int compare(data_view other) const {
-        protozero_assert(m_data && other.m_data);
+    int compare(data_view other) const noexcept {
+        assert(m_data && other.m_data);
         const int cmp = std::memcmp(data(), other.data(),
                                     std::min(size(), other.size()));
         if (cmp == 0) {

third_party/protozero/include/protozero/iterators.hpp

@@ -16,8 +16,8 @@ documentation.
  * @brief Contains the iterators for access to packed repeated fields.
  */
 
-#include <protozero/config.hpp>
-#include <protozero/varint.hpp>
+#include "config.hpp"
+#include "varint.hpp"
 
 #if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN
 # include <protozero/byteswap.hpp>
@@ -56,7 +56,7 @@ public:
      * Default constructor. Create empty iterator_range.
      */
     constexpr iterator_range() :
-        P(iterator{}, iterator{}) {
+        P{iterator{}, iterator{}} {
     }
 
     /**
@@ -66,8 +66,8 @@ public:
      * @param last_iterator Iterator to end of range.
      */
     constexpr iterator_range(iterator&& first_iterator, iterator&& last_iterator) :
-        P(std::forward<iterator>(first_iterator),
-          std::forward<iterator>(last_iterator)) {
+        P{std::forward<iterator>(first_iterator),
+          std::forward<iterator>(last_iterator)} {
     }
 
     /// Return iterator to beginning of range.
@@ -164,6 +164,8 @@ class const_fixed_iterator {
 
 public:
 
+    /// @cond usual iterator functions not documented
+
     using iterator_category = std::random_access_iterator_tag;
     using value_type        = T;
     using difference_type   = std::ptrdiff_t;
@@ -173,7 +175,7 @@ public:
     const_fixed_iterator() noexcept = default;
 
     explicit const_fixed_iterator(const char* data) noexcept :
-        m_data(data) {
+        m_data{data} {
     }
 
     const_fixed_iterator(const const_fixed_iterator&) noexcept = default;
@@ -184,11 +186,11 @@ public:
 
     ~const_fixed_iterator() noexcept = default;
 
-    value_type operator*() const {
+    value_type operator*() const noexcept {
         value_type result;
         std::memcpy(&result, m_data, sizeof(value_type));
 #if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN
-        detail::byteswap_inplace(&result);
+        byteswap_inplace(&result);
 #endif
         return result;
     }
@@ -204,14 +206,6 @@ public:
         return tmp;
     }
 
-    bool operator==(const_fixed_iterator rhs) const noexcept {
-        return m_data == rhs.m_data;
-    }
-
-    bool operator!=(const_fixed_iterator rhs) const noexcept {
-        return !(*this == rhs);
-    }
-
     const_fixed_iterator& operator--() noexcept {
         m_data -= sizeof(value_type);
         return *this;
@@ -223,6 +217,14 @@ public:
         return tmp;
     }
 
+    friend bool operator==(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
+        return lhs.m_data == rhs.m_data;
+    }
+
+    friend bool operator!=(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
+        return !(lhs == rhs);
+    }
+
     friend bool operator<(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
         return lhs.m_data < rhs.m_data;
     }
@@ -237,7 +239,6 @@ public:
 
     friend bool operator>=(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
         return !(lhs < rhs);
-
     }
 
     const_fixed_iterator& operator+=(difference_type val) noexcept {
@@ -276,6 +277,8 @@ public:
         return *(*this + n);
     }
 
+    /// @endcond
+
 }; // class const_fixed_iterator
 
 /**
@@ -288,13 +291,15 @@ class const_varint_iterator {
 protected:
 
     /// Pointer to current iterator position
-    const char* m_data = nullptr;
+    const char* m_data = nullptr; // NOLINT(misc-non-private-member-variables-in-classes, cppcoreguidelines-non-private-member-variables-in-classes,-warnings-as-errors)
 
     /// Pointer to end iterator position
-    const char* m_end = nullptr;
+    const char* m_end = nullptr; // NOLINT(misc-non-private-member-variables-in-classes, cppcoreguidelines-non-private-member-variables-in-classes,-warnings-as-errors)
 
 public:
 
+    /// @cond usual iterator functions not documented
+
     using iterator_category = std::forward_iterator_tag;
     using value_type        = T;
     using difference_type   = std::ptrdiff_t;
@@ -302,19 +307,24 @@ public:
     using reference         = value_type&;
 
     static difference_type distance(const_varint_iterator begin, const_varint_iterator end) noexcept {
+        // The "distance" between default initialized const_varint_iterator's
+        // is always 0.
+        if (!begin.m_data) {
+            return 0;
+        }
         // We know that each varint contains exactly one byte with the most
         // significant bit not set. We can use this to quickly figure out
         // how many varints there are without actually decoding the varints.
         return std::count_if(begin.m_data, end.m_data, [](char c) noexcept {
-            return (static_cast<unsigned char>(c) & 0x80u) == 0;
+            return (static_cast<unsigned char>(c) & 0x80U) == 0;
         });
     }
 
     const_varint_iterator() noexcept = default;
 
     const_varint_iterator(const char* data, const char* end) noexcept :
-        m_data(data),
-        m_end(end) {
+        m_data{data},
+        m_end{end} {
     }
 
     const_varint_iterator(const const_varint_iterator&) noexcept = default;
@@ -326,16 +336,19 @@ public:
     ~const_varint_iterator() noexcept = default;
 
     value_type operator*() const {
+        protozero_assert(m_data);
         const char* d = m_data; // will be thrown away
         return static_cast<value_type>(decode_varint(&d, m_end));
     }
 
     const_varint_iterator& operator++() {
+        protozero_assert(m_data);
         skip_varint(&m_data, m_end);
         return *this;
     }
 
     const_varint_iterator operator++(int) {
+        protozero_assert(m_data);
         const const_varint_iterator tmp{*this};
         ++(*this);
         return tmp;
@@ -349,6 +362,8 @@ public:
         return !(*this == rhs);
     }
 
+    /// @endcond
+
 }; // class const_varint_iterator
 
 /**
@@ -360,6 +375,8 @@ class const_svarint_iterator : public const_varint_iterator<T> {
 
 public:
 
+    /// @cond usual iterator functions not documented
+
     using iterator_category = std::forward_iterator_tag;
     using value_type        = T;
     using difference_type   = std::ptrdiff_t;
@@ -367,11 +384,11 @@ public:
     using reference         = value_type&;
 
     const_svarint_iterator() noexcept :
-        const_varint_iterator<T>() {
+        const_varint_iterator<T>{} {
     }
 
     const_svarint_iterator(const char* data, const char* end) noexcept :
-        const_varint_iterator<T>(data, end) {
+        const_varint_iterator<T>{data, end} {
     }
 
     const_svarint_iterator(const const_svarint_iterator&) = default;
@@ -383,21 +400,26 @@ public:
     ~const_svarint_iterator() = default;
 
     value_type operator*() const {
+        protozero_assert(this->m_data);
         const char* d = this->m_data; // will be thrown away
         return static_cast<value_type>(decode_zigzag64(decode_varint(&d, this->m_end)));
     }
 
     const_svarint_iterator& operator++() {
+        protozero_assert(this->m_data);
         skip_varint(&this->m_data, this->m_end);
         return *this;
     }
 
     const_svarint_iterator operator++(int) {
+        protozero_assert(this->m_data);
         const const_svarint_iterator tmp{*this};
         ++(*this);
         return tmp;
     }
 
+    /// @endcond
+
 }; // class const_svarint_iterator
 
 } // end namespace protozero
@@ -408,6 +430,8 @@ namespace std {
     // functions can't be partially specialized, we have to do this for
     // every value_type we are using.
 
+    /// @cond individual overloads do not need to be documented
+
     template <>
     inline typename protozero::const_varint_iterator<int32_t>::difference_type
     distance<protozero::const_varint_iterator<int32_t>>(protozero::const_varint_iterator<int32_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name)
@@ -450,6 +474,8 @@ namespace std {
         return protozero::const_svarint_iterator<int64_t>::distance(first, last);
     }
 
+    /// @endcond
+
 } // end namespace std
 
 #endif // PROTOZERO_ITERATORS_HPP

third_party/protozero/include/protozero/pbf_builder.hpp

@@ -16,249 +16,16 @@ documentation.
  * @brief Contains the pbf_builder template class.
  */
 
-#include <protozero/pbf_writer.hpp>
-#include <protozero/types.hpp>
+#include "basic_pbf_builder.hpp"
+#include "pbf_writer.hpp"
 
-#include <type_traits>
+#include <string>
 
 namespace protozero {
 
-/**
- * The pbf_builder is used to write PBF formatted messages into a buffer. It
- * is based on the pbf_writer class and has all the same methods. The
- * difference is that while the pbf_writer class takes an integer tag,
- * this template class takes a tag of the template type T. The idea is that
- * T will be an enumeration value and this helps reduce the possibility of
- * programming errors.
- *
- * Almost all methods in this class can throw an std::bad_alloc exception if
- * the std::string used as a buffer wants to resize.
- *
- * Read the tutorial to understand how this class is used.
- */
+/// Specialization of basic_pbf_builder using std::string as buffer type.
 template <typename T>
-class pbf_builder : public pbf_writer {
-
-    static_assert(std::is_same<pbf_tag_type, typename std::underlying_type<T>::type>::value,
-                  "T must be enum with underlying type protozero::pbf_tag_type");
-
-public:
-
-    /// The type of messages this class will build.
-    using enum_type = T;
-
-    pbf_builder() = default;
-
-    /**
-     * Create a builder using the given string as a data store. The object
-     * stores a reference to that string and adds all data to it. The string
-     * doesn't have to be empty. The pbf_message object will just append data.
-     */
-    explicit pbf_builder(std::string& data) noexcept :
-        pbf_writer(data) {
-    }
-
-    /**
-     * Construct a pbf_builder for a submessage from the pbf_message or
-     * pbf_writer of the parent message.
-     *
-     * @param parent_writer The parent pbf_message or pbf_writer
-     * @param tag Tag of the field that will be written
-     */
-    template <typename P>
-    pbf_builder(pbf_writer& parent_writer, P tag) noexcept :
-        pbf_writer(parent_writer, pbf_tag_type(tag)) {
-    }
-
-/// @cond INTERNAL
-#define PROTOZERO_WRITER_WRAP_ADD_SCALAR(name, type) \
-    void add_##name(T tag, type value) { \
-        pbf_writer::add_##name(pbf_tag_type(tag), value); \
-    }
-
-    PROTOZERO_WRITER_WRAP_ADD_SCALAR(bool, bool)
-    PROTOZERO_WRITER_WRAP_ADD_SCALAR(enum, int32_t)
-    PROTOZERO_WRITER_WRAP_ADD_SCALAR(int32, int32_t)
-    PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint32, int32_t)
-    PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint32, uint32_t)
-    PROTOZERO_WRITER_WRAP_ADD_SCALAR(int64, int64_t)
-    PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint64, int64_t)
-    PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint64, uint64_t)
-    PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed32, uint32_t)
-    PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed32, int32_t)
-    PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed64, uint64_t)
-    PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed64, int64_t)
-    PROTOZERO_WRITER_WRAP_ADD_SCALAR(float, float)
-    PROTOZERO_WRITER_WRAP_ADD_SCALAR(double, double)
-
-#undef PROTOZERO_WRITER_WRAP_ADD_SCALAR
-/// @endcond
-
-    /**
-     * Add "bytes" field to data.
-     *
-     * @param tag Tag of the field
-     * @param value Pointer to value to be written
-     * @param size Number of bytes to be written
-     */
-    void add_bytes(T tag, const char* value, std::size_t size) {
-        pbf_writer::add_bytes(pbf_tag_type(tag), value, size);
-    }
-
-    /**
-     * Add "bytes" field to data.
-     *
-     * @param tag Tag of the field
-     * @param value Value to be written
-     */
-    void add_bytes(T tag, const data_view& value) {
-        pbf_writer::add_bytes(pbf_tag_type(tag), value);
-    }
-
-    /**
-     * Add "bytes" field to data.
-     *
-     * @param tag Tag of the field
-     * @param value Value to be written
-     */
-    void add_bytes(T tag, const std::string& value) {
-        pbf_writer::add_bytes(pbf_tag_type(tag), value);
-    }
-
-    /**
-     * Add "bytes" field to data. Bytes from the value are written until
-     * a null byte is encountered. The null byte is not added.
-     *
-     * @param tag Tag of the field
-     * @param value Pointer to zero-delimited value to be written
-     */
-    void add_bytes(T tag, const char* value) {
-        pbf_writer::add_bytes(pbf_tag_type(tag), value);
-    }
-
-    /**
-     * Add "bytes" field to data using vectored input. All the data in the
-     * 2nd and further arguments is "concatenated" with only a single copy
-     * into the final buffer.
-     *
-     * This will work with objects of any type supporting the data() and
-     * size() methods like std::string or protozero::data_view.
-     *
-     * Example:
-     * @code
-     * std::string data1 = "abc";
-     * std::string data2 = "xyz";
-     * builder.add_bytes_vectored(1, data1, data2);
-     * @endcode
-     *
-     * @tparam Ts List of types supporting data() and size() methods.
-     * @param tag Tag of the field
-     * @param values List of objects of types Ts with data to be appended.
-     */
-    template <typename... Ts>
-    void add_bytes_vectored(T tag, Ts&&... values) {
-        pbf_writer::add_bytes_vectored(pbf_tag_type(tag), std::forward<Ts>(values)...);
-    }
-
-    /**
-     * Add "string" field to data.
-     *
-     * @param tag Tag of the field
-     * @param value Pointer to value to be written
-     * @param size Number of bytes to be written
-     */
-    void add_string(T tag, const char* value, std::size_t size) {
-        pbf_writer::add_string(pbf_tag_type(tag), value, size);
-    }
-
-    /**
-     * Add "string" field to data.
-     *
-     * @param tag Tag of the field
-     * @param value Value to be written
-     */
-    void add_string(T tag, const data_view& value) {
-        pbf_writer::add_string(pbf_tag_type(tag), value);
-    }
-
-    /**
-     * Add "string" field to data.
-     *
-     * @param tag Tag of the field
-     * @param value Value to be written
-     */
-    void add_string(T tag, const std::string& value) {
-        pbf_writer::add_string(pbf_tag_type(tag), value);
-    }
-
-    /**
-     * Add "string" field to data. Bytes from the value are written until
-     * a null byte is encountered. The null byte is not added.
-     *
-     * @param tag Tag of the field
-     * @param value Pointer to value to be written
-     */
-    void add_string(T tag, const char* value) {
-        pbf_writer::add_string(pbf_tag_type(tag), value);
-    }
-
-    /**
-     * Add "message" field to data.
-     *
-     * @param tag Tag of the field
-     * @param value Pointer to message to be written
-     * @param size Length of the message
-     */
-    void add_message(T tag, const char* value, std::size_t size) {
-        pbf_writer::add_message(pbf_tag_type(tag), value, size);
-    }
-
-    /**
-     * Add "message" field to data.
-     *
-     * @param tag Tag of the field
-     * @param value Value to be written. The value must be a complete message.
-     */
-    void add_message(T tag, const data_view& value) {
-        pbf_writer::add_message(pbf_tag_type(tag), value);
-    }
-
-    /**
-     * Add "message" field to data.
-     *
-     * @param tag Tag of the field
-     * @param value Value to be written. The value must be a complete message.
-     */
-    void add_message(T tag, const std::string& value) {
-        pbf_writer::add_message(pbf_tag_type(tag), value);
-    }
-
-/// @cond INTERNAL
-#define PROTOZERO_WRITER_WRAP_ADD_PACKED(name) \
-    template <typename InputIterator> \
-    void add_packed_##name(T tag, InputIterator first, InputIterator last) { \
-        pbf_writer::add_packed_##name(pbf_tag_type(tag), first, last); \
-    }
-
-    PROTOZERO_WRITER_WRAP_ADD_PACKED(bool)
-    PROTOZERO_WRITER_WRAP_ADD_PACKED(enum)
-    PROTOZERO_WRITER_WRAP_ADD_PACKED(int32)
-    PROTOZERO_WRITER_WRAP_ADD_PACKED(sint32)
-    PROTOZERO_WRITER_WRAP_ADD_PACKED(uint32)
-    PROTOZERO_WRITER_WRAP_ADD_PACKED(int64)
-    PROTOZERO_WRITER_WRAP_ADD_PACKED(sint64)
-    PROTOZERO_WRITER_WRAP_ADD_PACKED(uint64)
-    PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed32)
-    PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed32)
-    PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed64)
-    PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed64)
-    PROTOZERO_WRITER_WRAP_ADD_PACKED(float)
-    PROTOZERO_WRITER_WRAP_ADD_PACKED(double)
-
-#undef PROTOZERO_WRITER_WRAP_ADD_PACKED
-/// @endcond
-
-}; // class pbf_builder
+using pbf_builder = basic_pbf_builder<std::string, T>;
 
 } // end namespace protozero
 

third_party/protozero/include/protozero/pbf_message.hpp

@@ -16,8 +16,8 @@ documentation.
  * @brief Contains the pbf_message template class.
  */
 
-#include <protozero/pbf_reader.hpp>
-#include <protozero/types.hpp>
+#include "pbf_reader.hpp"
+#include "types.hpp"
 
 #include <type_traits>
 
@@ -78,7 +78,7 @@ public:
      */
     template <typename... Args>
     pbf_message(Args&&... args) noexcept : // NOLINT(google-explicit-constructor, hicpp-explicit-conversions)
-        pbf_reader(std::forward<Args>(args)...) {
+        pbf_reader{std::forward<Args>(args)...} {
     }
 
     /**

third_party/protozero/include/protozero/pbf_reader.hpp

@@ -16,12 +16,12 @@ documentation.
  * @brief Contains the pbf_reader class.
  */
 
-#include <protozero/config.hpp>
-#include <protozero/data_view.hpp>
-#include <protozero/exception.hpp>
-#include <protozero/iterators.hpp>
-#include <protozero/types.hpp>
-#include <protozero/varint.hpp>
+#include "config.hpp"
+#include "data_view.hpp"
+#include "exception.hpp"
+#include "iterators.hpp"
+#include "types.hpp"
+#include "varint.hpp"
 
 #if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN
 # include <protozero/byteswap.hpp>
@@ -80,7 +80,7 @@ class pbf_reader {
         skip_bytes(sizeof(T));
         std::memcpy(&result, data, sizeof(T));
 #if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN
-        detail::byteswap_inplace(&result);
+        byteswap_inplace(&result);
 #endif
         return result;
     }
@@ -98,7 +98,8 @@ class pbf_reader {
 
     template <typename T>
     T get_varint() {
-        return static_cast<T>(decode_varint(&m_data, m_end));
+        const auto val = static_cast<T>(decode_varint(&m_data, m_end));
+        return val;
     }
 
     template <typename T>
@@ -112,7 +113,7 @@ class pbf_reader {
     }
 
     void skip_bytes(pbf_length_type len) {
-        if (m_data + len > m_end) {
+        if (m_end - m_data < static_cast<ptrdiff_t>(len)) {
             throw end_of_buffer_exception{};
         }
         m_data += len;
@@ -151,8 +152,8 @@ public:
      * @post There is no current field.
      */
     explicit pbf_reader(const data_view& view) noexcept
-        : m_data(view.data()),
-          m_end(view.data() + view.size()) {
+        : m_data{view.data()},
+          m_end{view.data() + view.size()} {
     }
 
     /**
@@ -166,8 +167,8 @@ public:
      * @post There is no current field.
      */
     pbf_reader(const char* data, std::size_t size) noexcept
-        : m_data(data),
-          m_end(data + size) {
+        : m_data{data},
+          m_end{data + size} {
     }
 
 #ifndef PROTOZERO_STRICT_API
@@ -183,8 +184,8 @@ public:
      * @deprecated Use one of the other constructors.
      */
     explicit pbf_reader(const std::pair<const char*, std::size_t>& data) noexcept
-        : m_data(data.first),
-          m_end(data.first + data.second) {
+        : m_data{data.first},
+          m_end{data.first + data.second} {
     }
 #endif
 
@@ -199,8 +200,8 @@ public:
      * @post There is no current field.
      */
     explicit pbf_reader(const std::string& data) noexcept
-        : m_data(data.data()),
-          m_end(data.data() + data.size()) {
+        : m_data{data.data()},
+          m_end{data.data() + data.size()} {
     }
 
     /**
@@ -242,7 +243,14 @@ public:
      * read.
      */
     operator bool() const noexcept { // NOLINT(google-explicit-constructor, hicpp-explicit-conversions)
-        return m_data < m_end;
+        return m_data != m_end;
+    }
+
+    /**
+     * Get a view of the not yet read data.
+     */
+    data_view data() const noexcept {
+        return {m_data, static_cast<std::size_t>(m_end - m_data)};
     }
 
     /**
@@ -279,7 +287,7 @@ public:
         }
 
         const auto value = get_varint<uint32_t>();
-        m_tag = pbf_tag_type(value >> 3u);
+        m_tag = pbf_tag_type(value >> 3U);
 
         // tags 0 and 19000 to 19999 are not allowed as per
         // https://developers.google.com/protocol-buffers/docs/proto#assigning-tags
@@ -287,7 +295,7 @@ public:
             throw invalid_tag_exception{};
         }
 
-        m_wire_type = pbf_wire_type(value & 0x07u);
+        m_wire_type = pbf_wire_type(value & 0x07U);
         switch (m_wire_type) {
             case pbf_wire_type::varint:
             case pbf_wire_type::fixed64:
@@ -486,9 +494,9 @@ public:
     bool get_bool() {
         protozero_assert(tag() != 0 && "call next() before accessing field value");
         protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
-        const auto data = m_data;
+        const bool result = m_data[0] != 0;
         skip_varint(&m_data, m_end);
-        return data[0] != 0;
+        return result;
     }
 
     /**

third_party/protozero/include/protozero/types.hpp

@@ -16,7 +16,7 @@ documentation.
  * @brief Contains the declaration of low-level types used in the pbf format.
  */
 
-#include <protozero/config.hpp>
+#include "config.hpp"
 
 #include <algorithm>
 #include <cstddef>
@@ -53,7 +53,7 @@ enum class pbf_wire_type : uint32_t {
  */
 template <typename T>
 constexpr inline uint32_t tag_and_type(T tag, pbf_wire_type wire_type) noexcept {
-    return (static_cast<uint32_t>(static_cast<pbf_tag_type>(tag)) << 3u) | static_cast<uint32_t>(wire_type);
+    return (static_cast<uint32_t>(static_cast<pbf_tag_type>(tag)) << 3U) | static_cast<uint32_t>(wire_type);
 }
 
 /**

third_party/protozero/include/protozero/varint.hpp

@@ -16,7 +16,8 @@ documentation.
  * @brief Contains low-level varint and zigzag encoding and decoding functions.
  */
 
-#include <protozero/exception.hpp>
+#include "buffer_tmpl.hpp"
+#include "exception.hpp"
 
 #include <cstdint>
 
@@ -31,30 +32,30 @@ namespace detail {
 
     // from https://github.com/facebook/folly/blob/master/folly/Varint.h
     inline uint64_t decode_varint_impl(const char** data, const char* end) {
-        const auto begin = reinterpret_cast<const int8_t*>(*data);
-        const auto iend = reinterpret_cast<const int8_t*>(end);
+        const auto* begin = reinterpret_cast<const int8_t*>(*data);
+        const auto* iend = reinterpret_cast<const int8_t*>(end);
         const int8_t* p = begin;
         uint64_t val = 0;
 
         if (iend - begin >= max_varint_length) {  // fast path
             do {
-                int64_t b;
-                b = *p++; val  = uint64_t((b & 0x7fu)       ); if (b >= 0) { break; }
-                b = *p++; val |= uint64_t((b & 0x7fu) <<  7u); if (b >= 0) { break; }
-                b = *p++; val |= uint64_t((b & 0x7fu) << 14u); if (b >= 0) { break; }
-                b = *p++; val |= uint64_t((b & 0x7fu) << 21u); if (b >= 0) { break; }
-                b = *p++; val |= uint64_t((b & 0x7fu) << 28u); if (b >= 0) { break; }
-                b = *p++; val |= uint64_t((b & 0x7fu) << 35u); if (b >= 0) { break; }
-                b = *p++; val |= uint64_t((b & 0x7fu) << 42u); if (b >= 0) { break; }
-                b = *p++; val |= uint64_t((b & 0x7fu) << 49u); if (b >= 0) { break; }
-                b = *p++; val |= uint64_t((b & 0x7fu) << 56u); if (b >= 0) { break; }
-                b = *p++; val |= uint64_t((b & 0x01u) << 63u); if (b >= 0) { break; }
+                int64_t b = *p++;
+                          val  = ((uint64_t(b) & 0x7fU)       ); if (b >= 0) { break; }
+                b = *p++; val |= ((uint64_t(b) & 0x7fU) <<  7U); if (b >= 0) { break; }
+                b = *p++; val |= ((uint64_t(b) & 0x7fU) << 14U); if (b >= 0) { break; }
+                b = *p++; val |= ((uint64_t(b) & 0x7fU) << 21U); if (b >= 0) { break; }
+                b = *p++; val |= ((uint64_t(b) & 0x7fU) << 28U); if (b >= 0) { break; }
+                b = *p++; val |= ((uint64_t(b) & 0x7fU) << 35U); if (b >= 0) { break; }
+                b = *p++; val |= ((uint64_t(b) & 0x7fU) << 42U); if (b >= 0) { break; }
+                b = *p++; val |= ((uint64_t(b) & 0x7fU) << 49U); if (b >= 0) { break; }
+                b = *p++; val |= ((uint64_t(b) & 0x7fU) << 56U); if (b >= 0) { break; }
+                b = *p++; val |= ((uint64_t(b) & 0x01U) << 63U); if (b >= 0) { break; }
                 throw varint_too_long_exception{};
             } while (false);
         } else {
             unsigned int shift = 0;
             while (p != iend && *p < 0) {
-                val |= uint64_t(*p++ & 0x7fu) << shift;
+                val |= (uint64_t(*p++) & 0x7fU) << shift;
                 shift += 7;
             }
             if (p == iend) {
@@ -88,7 +89,7 @@ namespace detail {
  */
 inline uint64_t decode_varint(const char** data, const char* end) {
     // If this is a one-byte varint, decode it here.
-    if (end != *data && ((**data & 0x80u) == 0)) {
+    if (end != *data && ((static_cast<uint64_t>(**data) & 0x80U) == 0)) {
         const auto val = static_cast<uint64_t>(**data);
         ++(*data);
         return val;
@@ -110,15 +111,15 @@ inline uint64_t decode_varint(const char** data, const char* end) {
  *         before the end of the varint.
  */
 inline void skip_varint(const char** data, const char* end) {
-    const auto begin = reinterpret_cast<const int8_t*>(*data);
-    const auto iend = reinterpret_cast<const int8_t*>(end);
+    const auto* begin = reinterpret_cast<const int8_t*>(*data);
+    const auto* iend = reinterpret_cast<const int8_t*>(end);
     const int8_t* p = begin;
 
     while (p != iend && *p < 0) {
         ++p;
     }
 
-    if (p >= begin + max_varint_length) {
+    if (p - begin >= max_varint_length) {
         throw varint_too_long_exception{};
     }
 
@@ -140,17 +141,73 @@ inline void skip_varint(const char** data, const char* end) {
  * @param value The integer that will be encoded.
  * @returns the number of bytes written
  * @throws Any exception thrown by increment or dereference operator on data.
+ * @deprecated Use add_varint_to_buffer() instead.
  */
 template <typename T>
 inline int write_varint(T data, uint64_t value) {
     int n = 1;
 
-    while (value >= 0x80u) {
-        *data++ = char((value & 0x7fu) | 0x80u);
-        value >>= 7u;
+    while (value >= 0x80U) {
+        *data++ = char((value & 0x7fU) | 0x80U);
+        value >>= 7U;
+        ++n;
+    }
+    *data = char(value);
+
+    return n;
+}
+
+/**
+ * Varint encode a 64 bit integer.
+ *
+ * @tparam TBuffer A buffer type.
+ * @param buffer Output buffer the varint will be written to.
+ * @param value The integer that will be encoded.
+ * @returns the number of bytes written
+ * @throws Any exception thrown by calling the buffer_push_back() function.
+ */
+template <typename TBuffer>
+inline void add_varint_to_buffer(TBuffer* buffer, uint64_t value) {
+    while (value >= 0x80U) {
+        buffer_customization<TBuffer>::push_back(buffer, char((value & 0x7fU) | 0x80U));
+        value >>= 7U;
+    }
+    buffer_customization<TBuffer>::push_back(buffer, char(value));
+}
+
+/**
+ * Varint encode a 64 bit integer.
+ *
+ * @param data Where to add the varint. There must be enough space available!
+ * @param value The integer that will be encoded.
+ * @returns the number of bytes written
+ */
+inline int add_varint_to_buffer(char* data, uint64_t value) noexcept {
+    int n = 1;
+
+    while (value >= 0x80U) {
+        *data++ = char((value & 0x7fU) | 0x80U);
+        value >>= 7U;
+        ++n;
+    }
+    *data = char(value);
+
+    return n;
+}
+
+/**
+ * Get the length of the varint the specified value would produce.
+ *
+ * @param value The integer to be encoded.
+ * @returns the number of bytes the varint would have if we created it.
+ */
+inline int length_of_varint(uint64_t value) noexcept {
+    int n = 1;
+
+    while (value >= 0x80U) {
+        value >>= 7U;
         ++n;
     }
-    *data++ = char(value);
 
     return n;
 }
@@ -159,28 +216,28 @@ inline int write_varint(T data, uint64_t value) {
  * ZigZag encodes a 32 bit integer.
  */
 inline constexpr uint32_t encode_zigzag32(int32_t value) noexcept {
-    return (static_cast<uint32_t>(value) << 1u) ^ (static_cast<uint32_t>(value >> 31u));
+    return (static_cast<uint32_t>(value) << 1U) ^ static_cast<uint32_t>(-static_cast<int32_t>(static_cast<uint32_t>(value) >> 31U));
 }
 
 /**
  * ZigZag encodes a 64 bit integer.
  */
 inline constexpr uint64_t encode_zigzag64(int64_t value) noexcept {
-    return (static_cast<uint64_t>(value) << 1u) ^ (static_cast<uint64_t>(value >> 63u));
+    return (static_cast<uint64_t>(value) << 1U) ^ static_cast<uint64_t>(-static_cast<int64_t>(static_cast<uint64_t>(value) >> 63U));
 }
 
 /**
  * Decodes a 32 bit ZigZag-encoded integer.
  */
 inline constexpr int32_t decode_zigzag32(uint32_t value) noexcept {
-    return static_cast<int32_t>(value >> 1u) ^ -static_cast<int32_t>(value & 1u);
+    return static_cast<int32_t>((value >> 1U) ^ static_cast<uint32_t>(-static_cast<int32_t>(value & 1U)));
 }
 
 /**
  * Decodes a 64 bit ZigZag-encoded integer.
  */
 inline constexpr int64_t decode_zigzag64(uint64_t value) noexcept {
-    return static_cast<int64_t>(value >> 1u) ^ -static_cast<int64_t>(value & 1u);
+    return static_cast<int64_t>((value >> 1U) ^ static_cast<uint64_t>(-static_cast<int64_t>(value & 1U)));
 }
 
 } // end namespace protozero

third_party/protozero/include/protozero/version.hpp

@@ -20,15 +20,15 @@ documentation.
 #define PROTOZERO_VERSION_MAJOR 1
 
 /// The minor version number
-#define PROTOZERO_VERSION_MINOR 6
+#define PROTOZERO_VERSION_MINOR 7
 
 /// The patch number
-#define PROTOZERO_VERSION_PATCH 2
+#define PROTOZERO_VERSION_PATCH 0
 
 /// The complete version number
 #define PROTOZERO_VERSION_CODE (PROTOZERO_VERSION_MAJOR * 10000 + PROTOZERO_VERSION_MINOR * 100 + PROTOZERO_VERSION_PATCH)
 
 /// Version number as string
-#define PROTOZERO_VERSION_STRING "1.6.2"
+#define PROTOZERO_VERSION_STRING "1.7.0"
 
 #endif // PROTOZERO_VERSION_HPP

third_party/protozero/test/create_pbf_test_data.sh

@@ -8,9 +8,16 @@
 #  If called without a test case it will iterate over all test cases generating
 #  all data.
 #
+#  This program should be called with the "test" directory as current directory.
+#
 
 set -e
 
+if [ -z "$CXX" ]; then
+    echo "Please set CXX before running this script"
+    exit 1
+fi
+
 if [ -z "$1" ]; then
     for dir in t/*; do
         $0 $dir

third_party/protozero/test/include/buffer.hpp

@@ -0,0 +1,162 @@
+#ifndef BUFFER_HPP
+#define BUFFER_HPP
+
+#include "test.hpp"
+
+#include <protozero/buffer_fixed.hpp>
+#include <protozero/buffer_string.hpp>
+#include <protozero/buffer_vector.hpp>
+
+// This "simulates" an externally defined buffer type to make sure our
+// buffer adaptor functions do the right thing.
+namespace test_external {
+    class ext_buffer : public std::string {
+    };
+} // namespace test_external
+
+namespace protozero {
+
+    template <>
+    struct buffer_customization<test_external::ext_buffer> {
+
+        static std::size_t size(const test_external::ext_buffer* buffer) noexcept {
+            return buffer->size();
+        }
+
+        static void append(test_external::ext_buffer* buffer, const char* data, std::size_t count) {
+            buffer->append(data, count);
+        }
+
+        static void append_zeros(test_external::ext_buffer* buffer, std::size_t count) {
+            buffer->append(count, '\0');
+        }
+
+        static void resize(test_external::ext_buffer* buffer, std::size_t size) {
+            protozero_assert(size < buffer->size());
+            buffer->resize(size);
+        }
+
+        static void reserve_additional(test_external::ext_buffer* buffer, std::size_t size) {
+            buffer->reserve(buffer->size() + size);
+        }
+
+        static void erase_range(test_external::ext_buffer* buffer, std::size_t from, std::size_t to) {
+            protozero_assert(from <= buffer->size());
+            protozero_assert(to <= buffer->size());
+            protozero_assert(from <= to);
+            buffer->erase(std::next(buffer->begin(), from), std::next(buffer->begin(), to));
+        }
+
+        static char* at_pos(test_external::ext_buffer* buffer, std::size_t pos) {
+            protozero_assert(pos <= buffer->size());
+            return (&*buffer->begin()) + pos;
+        }
+
+        static void push_back(test_external::ext_buffer* buffer, char ch) {
+            buffer->push_back(ch);
+        }
+
+    };
+
+} // namespace protozero
+
+// The following structs are used in many tests using TEMPLATE_TEST_CASE() to
+// test the different buffer types:
+//
+// 1. Dynamically sized buffer based on std::string.
+// 2. Dynamically sized buffer based on std::vector<char>.
+// 3. Statically sized buffer based on std::array<char, N>.
+// 4. Externally defined buffer.
+
+class buffer_test_string {
+
+    std::string m_buffer;
+
+public:
+
+    using type = std::string;
+    using writer_type = protozero::pbf_writer; // == protozero::basic_pbf_writer<type>;
+
+    type& buffer() noexcept {
+        return m_buffer;
+    }
+
+    const char *data() const noexcept {
+        return m_buffer.data();
+    }
+
+    std::size_t size() const noexcept {
+        return m_buffer.size();
+    }
+}; // class buffer_test_string
+
+class buffer_test_vector {
+
+    std::vector<char> m_buffer;
+
+public:
+
+    using type = std::vector<char>;
+    using writer_type = protozero::basic_pbf_writer<type>;
+
+    type& buffer() noexcept {
+        return m_buffer;
+    }
+
+    const char *data() const noexcept {
+        return m_buffer.data();
+    }
+
+    std::size_t size() const noexcept {
+        return m_buffer.size();
+    }
+}; // class buffer_test_vector
+
+class buffer_test_array {
+
+public:
+    using type = protozero::fixed_size_buffer_adaptor;
+    using writer_type = protozero::basic_pbf_writer<type>;
+
+    type& buffer() noexcept {
+        return adaptor;
+    }
+
+    const char *data() const noexcept {
+        return adaptor.data();
+    }
+
+    std::size_t size() const noexcept {
+        return adaptor.size();
+    }
+
+private:
+
+    std::array<char, 1024> m_buffer = {{0}};
+    type adaptor{m_buffer};
+
+}; // class buffer_test_array
+
+class buffer_test_external {
+
+    test_external::ext_buffer m_buffer;
+
+public:
+
+    using type = test_external::ext_buffer;
+    using writer_type = protozero::basic_pbf_writer<type>;
+
+    type& buffer() noexcept {
+        return m_buffer;
+    }
+
+    const char *data() const noexcept {
+        return m_buffer.data();
+    }
+
+    std::size_t size() const noexcept {
+        return m_buffer.size();
+    }
+}; // class buffer_test_external
+
+#endif // BUFFER_HPP

third_party/protozero/test/include/packed_access.hpp

@@ -1,5 +1,8 @@
 // NOLINT(llvm-header-guard)
 
+#include <array>
+#include <sstream>
+
 #define PBF_TYPE_NAME PROTOZERO_TEST_STRING(PBF_TYPE)
 #define GET_TYPE PROTOZERO_TEST_CONCAT(get_packed_, PBF_TYPE)
 #define ADD_TYPE PROTOZERO_TEST_CONCAT(add_packed_, PBF_TYPE)
@@ -92,7 +95,7 @@ TEST_CASE("read repeated packed field: " PBF_TYPE_NAME) {
             for (std::string::size_type i = 1; i < abuffer.size() - n; ++i) {
                 protozero::pbf_reader item{abuffer.data() + n, i};
                 REQUIRE(item.next());
-                REQUIRE_THROWS_AS(item.GET_TYPE(), const protozero::end_of_buffer_exception&);
+                REQUIRE_THROWS_AS(item.GET_TYPE(), protozero::end_of_buffer_exception);
             }
         }
 
@@ -105,21 +108,27 @@ TEST_CASE("write repeated packed field: " PBF_TYPE_NAME) {
     protozero::pbf_writer pw{buffer};
 
     SECTION("empty") {
-        cpp_type data[] = { 17 };
+        std::array<cpp_type, 1> data = {{ 17 }};
         pw.ADD_TYPE(1, std::begin(data), std::begin(data) /* !!!! */);
 
         REQUIRE(buffer == load_data("repeated_packed_" PBF_TYPE_NAME "/data-empty"));
     }
 
     SECTION("one") {
-        cpp_type data[] = { 17 };
+        std::array<cpp_type, 1> data = {{ 17 }};
         pw.ADD_TYPE(1, std::begin(data), std::end(data));
 
         REQUIRE(buffer == load_data("repeated_packed_" PBF_TYPE_NAME "/data-one"));
     }
 
     SECTION("many") {
-        cpp_type data[] = {
+        std::array<cpp_type,
+#if PBF_TYPE_IS_SIGNED
+                   8
+#else
+                   5
+#endif
+        > data = {{
                17
             , 200
             ,   0
@@ -130,7 +139,7 @@ TEST_CASE("write repeated packed field: " PBF_TYPE_NAME) {
             ,  -1
             ,std::numeric_limits<cpp_type>::min()
 #endif
-        };
+        }};
         pw.ADD_TYPE(1, std::begin(data), std::end(data));
 
         REQUIRE(buffer == load_data("repeated_packed_" PBF_TYPE_NAME "/data-many"));
@@ -246,9 +255,9 @@ TEST_CASE("write from different types of iterators: " PBF_TYPE_NAME) {
 
     SECTION("from uint16_t") {
 #if PBF_TYPE_IS_SIGNED
-        const  int16_t data[] = { 1, 4, 9, 16, 25 };
+        const std::array< int16_t, 5> data = {{ 1, 4, 9, 16, 25 }};
 #else
-        const uint16_t data[] = { 1, 4, 9, 16, 25 };
+        const std::array<uint16_t, 5> data = {{ 1, 4, 9, 16, 25 }};
 #endif
 
         pw.ADD_TYPE(1, std::begin(data), std::end(data));
@@ -290,7 +299,7 @@ TEST_CASE("write from different types of iterators: " PBF_TYPE_NAME) {
     REQUIRE(std::distance(it_range.begin(), it_range.end()) == 0);
     REQUIRE(it_range.size() == 0); // NOLINT(readability-container-size-empty)
 
-    REQUIRE_THROWS_AS(it_range.front(), const assert_error&);
-    REQUIRE_THROWS_AS(it_range.drop_front(), const assert_error&);
+    REQUIRE_THROWS_AS(it_range.front(), assert_error);
+    REQUIRE_THROWS_AS(it_range.drop_front(), assert_error);
 }
 

third_party/protozero/test/include/scalar_access.hpp

@@ -88,7 +88,7 @@ TEST_CASE("read field: " PBF_TYPE_NAME) {
         for (std::string::size_type i = 1; i < buffer.size(); ++i) {
             protozero::pbf_reader item{buffer.data(), i};
             REQUIRE(item.next());
-            REQUIRE_THROWS_AS(item.GET_TYPE(), const protozero::end_of_buffer_exception&);
+            REQUIRE_THROWS_AS(item.GET_TYPE(), protozero::end_of_buffer_exception);
         }
     }
 }

third_party/protozero/test/include/test.hpp

@@ -3,6 +3,9 @@
 
 #include <catch.hpp>
 
+#include <array>
+#include <vector>
+
 #include <stdexcept>
 // Define protozero_assert() to throw this error. This allows the tests to
 // check that the assert fails.

third_party/protozero/test/reader_tests.cpp

@@ -1,5 +1,7 @@
 
+#include <cstdlib>
 #include <fstream>
+#include <iterator>
 #include <stdexcept>
 #include <string>
 

third_party/protozero/test/t/alignment/reader_test_cases.cpp

@@ -47,7 +47,7 @@ TEST_CASE("check alignment issues for fixed32 field") {
             for (std::string::size_type i = 1; i < abuffer.size() - n; ++i) {
                 protozero::pbf_reader item{abuffer.data() + n, i};
                 REQUIRE(item.next());
-                REQUIRE_THROWS_AS(item.get_fixed32(), const protozero::end_of_buffer_exception&);
+                REQUIRE_THROWS_AS(item.get_fixed32(), protozero::end_of_buffer_exception);
             }
         }
 
@@ -55,7 +55,7 @@ TEST_CASE("check alignment issues for fixed32 field") {
             abuffer.append(load_data("fixed32/data-zero"));
             protozero::pbf_reader item{abuffer.data() + n, abuffer.size() - n};
 
-            REQUIRE_THROWS_AS(item.get_fixed32(), const assert_error&);
+            REQUIRE_THROWS_AS(item.get_fixed32(), assert_error);
             REQUIRE(item.next());
             REQUIRE(item.get_fixed32() == 0UL);
             REQUIRE_THROWS(item.get_fixed32());
@@ -66,7 +66,7 @@ TEST_CASE("check alignment issues for fixed32 field") {
             abuffer.append(load_data("fixed32/data-zero"));
             protozero::pbf_reader item{abuffer.data() + n, abuffer.size() - n};
 
-            REQUIRE_THROWS_AS(item.skip(), const assert_error&);
+            REQUIRE_THROWS_AS(item.skip(), assert_error);
             REQUIRE(item.next());
             item.skip();
             REQUIRE_THROWS(item.skip());
@@ -114,7 +114,7 @@ TEST_CASE("check alignment issues for fixed64 field") {
             for (std::string::size_type i = 1; i < abuffer.size() - n; ++i) {
                 protozero::pbf_reader item{abuffer.data() + n, i};
                 REQUIRE(item.next());
-                REQUIRE_THROWS_AS(item.get_fixed64(), const protozero::end_of_buffer_exception&);
+                REQUIRE_THROWS_AS(item.get_fixed64(), protozero::end_of_buffer_exception);
             }
         }
     }

third_party/protozero/test/t/bool/bool_testcase.proto

@@ -1,3 +1,4 @@
+syntax = "proto2";
 
 option optimize_for = LITE_RUNTIME;
 

third_party/protozero/test/t/bool/reader_test_cases.cpp

@@ -126,7 +126,7 @@ TEST_CASE("write bool field using moved pbf_builder") {
 
     protozero::pbf_builder<TestBoolean::Test> pw{std::move(pw2)};
     REQUIRE(pw.valid());
-    REQUIRE_FALSE(pw2.valid()); // NOLINT(hicpp-invalid-access-moved, bugprone-use-after-move)
+    REQUIRE_FALSE(pw2.valid()); // NOLINT(hicpp-invalid-access-moved, bugprone-use-after-move, clang-analyzer-cplusplus.Move)
 
     SECTION("false") {
         pw.add_bool(TestBoolean::Test::required_bool_b, false);

third_party/protozero/test/t/bool/testcase.cpp

@@ -1,8 +1,9 @@
 
 #include <testcase.hpp>
+
 #include "testcase.pb.h"
 
-int main(int c, char *argv[]) {
+int main() {
     TestBoolean::Test msg;
 
     msg.set_b(0);

third_party/protozero/test/t/bool/writer_test_cases.cpp

@@ -1,21 +1,22 @@
 
 #include <string>
 
-#include <test.hpp> // IWYU pragma: keep
+#include <buffer.hpp>
 
 #include "t/bool/bool_testcase.pb.h"
 
-TEST_CASE("write bool field and check with libprotobuf") {
+TEMPLATE_TEST_CASE("write bool field and check with libprotobuf", "",
+    buffer_test_string, buffer_test_vector, buffer_test_array, buffer_test_external) {
 
-    std::string buffer;
-    protozero::pbf_writer pw{buffer};
+    TestType buffer;
+    typename TestType::writer_type pw{buffer.buffer()};
 
     TestBoolean::Test msg;
 
     SECTION("false") {
         pw.add_bool(1, false);
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE_FALSE(msg.b());
     }
@@ -23,7 +24,7 @@ TEST_CASE("write bool field and check with libprotobuf") {
     SECTION("true") {
         pw.add_bool(1, true);
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.b());
     }

third_party/protozero/test/t/bytes/bytes_testcase.proto

@@ -1,3 +1,4 @@
+syntax = "proto2";
 
 option optimize_for = LITE_RUNTIME;
 

third_party/protozero/test/t/bytes/reader_test_cases.cpp

@@ -51,7 +51,7 @@ TEST_CASE("read bytes field: end of buffer") {
     for (std::string::size_type i = 1; i < buffer.size(); ++i) {
         protozero::pbf_reader item{buffer.data(), i};
         REQUIRE(item.next());
-        REQUIRE_THROWS_AS(item.get_bytes(), const protozero::end_of_buffer_exception&);
+        REQUIRE_THROWS_AS(item.get_bytes(), protozero::end_of_buffer_exception);
     }
 }
 

third_party/protozero/test/t/bytes/testcase.cpp

@@ -1,8 +1,9 @@
 
 #include <testcase.hpp>
+
 #include "testcase.pb.h"
 
-int main(int c, char *argv[]) {
+int main() {
     TestBytes::Test msg;
 
     msg.set_s("");

third_party/protozero/test/t/bytes/writer_test_cases.cpp

@@ -1,19 +1,20 @@
 
-#include <test.hpp>
+#include <buffer.hpp>
 
 #include "t/bytes/bytes_testcase.pb.h"
 
-TEST_CASE("write bytes field and check with libprotobuf") {
+TEMPLATE_TEST_CASE("write bytes field and check with libprotobuf", "",
+    buffer_test_string, buffer_test_vector, buffer_test_array, buffer_test_external) {
 
-    std::string buffer;
-    protozero::pbf_writer pw{buffer};
+    TestType buffer;
+    typename TestType::writer_type pw{buffer.buffer()};
 
     TestBytes::Test msg;
 
     SECTION("empty") {
         pw.add_string(1, "");
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.s().empty());
     }
@@ -21,7 +22,7 @@ TEST_CASE("write bytes field and check with libprotobuf") {
     SECTION("one") {
         pw.add_string(1, "x");
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.s() == "x");
     }
@@ -29,7 +30,7 @@ TEST_CASE("write bytes field and check with libprotobuf") {
     SECTION("string") {
         pw.add_string(1, "foobar");
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.s() == "foobar");
     }
@@ -42,7 +43,7 @@ TEST_CASE("write bytes field and check with libprotobuf") {
 
         pw.add_string(1, data);
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.s().size() == 3);
         REQUIRE(msg.s()[1] == char(2));

third_party/protozero/test/t/complex/reader_test_cases.cpp

@@ -1,6 +1,12 @@
 
 #include <test.hpp>
 
+#include <protozero/buffer_fixed.hpp>
+
+#include <algorithm>
+#include <array>
+#include <numeric>
+
 namespace TestComplex {
 
 enum class Test : protozero::pbf_tag_type {
@@ -120,11 +126,7 @@ TEST_CASE("read complex data using pbf_reader: all") {
             }
             case 7: {
                 const auto pi = item.get_packed_sint32();
-                int32_t sum = 0;
-                for (auto val : pi) {
-                    sum += val;
-                }
-                REQUIRE(sum == 5);
+                REQUIRE(std::accumulate(pi.cbegin(), pi.cend(), 0) == 5);
                 break;
             }
             case 8: {
@@ -265,11 +267,7 @@ TEST_CASE("read complex data using pbf_message: all") {
             }
             case TestComplex::Test::packed_sint32_d: {
                 const auto pi = item.get_packed_sint32();
-                int32_t sum = 0;
-                for (auto val : pi) {
-                    sum += val;
-                }
-                REQUIRE(sum == 5);
+                REQUIRE(std::accumulate(pi.cbegin(), pi.cend(), 0) == 5);
                 break;
             }
             case TestComplex::Test::optional_string_s: {
@@ -416,7 +414,7 @@ TEST_CASE("write complex data using pbf_writer: all") {
     pw.add_uint32(4, 66);
     pw.add_uint32(4, 66);
 
-    const int32_t d[] = { -17, 22 };
+    const std::array<int32_t, 2> d = {{ -17, 22 }};
     pw.add_packed_sint32(7, std::begin(d), std::end(d));
 
     pw.add_int64(3, 555555555);
@@ -453,15 +451,7 @@ TEST_CASE("write complex data using pbf_writer: all") {
             }
             case 7: {
                 const auto pi = item.get_packed_sint32();
-                int32_t sum = 0;
-                for (auto val : pi) {
-                    sum += val;
-                }
-                REQUIRE(sum == 5);
-                break;
-            }
-            case 8: {
-                REQUIRE(item.get_string() == "optionalstring");
+                REQUIRE(std::accumulate(pi.cbegin(), pi.cend(), 0) == 5);
                 break;
             }
             default: {
@@ -577,7 +567,7 @@ TEST_CASE("write complex data using pbf_builder: all") {
     pw.add_uint32(TestComplex::Test::repeated_uint32_u, 66);
     pw.add_uint32(TestComplex::Test::repeated_uint32_u, 66);
 
-    const int32_t d[] = { -17, 22 };
+    const std::array<int32_t, 2> d = {{ -17, 22 }};
     pw.add_packed_sint32(TestComplex::Test::packed_sint32_d, std::begin(d), std::end(d));
 
     pw.add_int64(TestComplex::Test::optional_int64_j, 555555555);
@@ -614,15 +604,7 @@ TEST_CASE("write complex data using pbf_builder: all") {
             }
             case 7: {
                 const auto pi = item.get_packed_sint32();
-                int32_t sum = 0;
-                for (auto val : pi) {
-                    sum += val;
-                }
-                REQUIRE(sum == 5);
-                break;
-            }
-            case 8: {
-                REQUIRE(item.get_string() == "optionalstring");
+                REQUIRE(std::accumulate(pi.cbegin(), pi.cend(), 0) == 5);
                 break;
             }
             default: {
@@ -684,3 +666,77 @@ TEST_CASE("write complex with subwriter using pbf_builder") {
     check_message(buffer_test);
 }
 
+TEST_CASE("write complex data using basic_pbf_writer<fixed_size_buffer_adaptor>: all") {
+    std::string data;
+    data.resize(10240);
+    protozero::fixed_size_buffer_adaptor buffer{&*data.begin(), data.size()};
+    protozero::basic_pbf_writer<protozero::fixed_size_buffer_adaptor> pw{buffer};
+    pw.add_fixed32(1, 12345678);
+
+    std::string sdata;
+    sdata.resize(10240);
+    protozero::fixed_size_buffer_adaptor submessage{&*sdata.begin(), sdata.size()};
+    protozero::basic_pbf_writer<protozero::fixed_size_buffer_adaptor> pws{submessage};
+    pws.add_string(1, "foobar");
+    pw.add_message(5, submessage.data(), submessage.size());
+
+    pw.add_uint32(4, 22);
+    pw.add_uint32(4, 44);
+    pw.add_int64(2, -9876543);
+    pw.add_uint32(4, 44);
+    pw.add_uint32(4, 66);
+    pw.add_uint32(4, 66);
+
+    const std::array<int32_t, 2> d = {{ -17, 22 }};
+    pw.add_packed_sint32(7, std::begin(d), std::end(d));
+
+    pw.add_int64(3, 555555555);
+
+    protozero::pbf_reader item{buffer.data(), buffer.size()};
+
+    int number_of_u = 0;
+    while (item.next()) {
+        switch (item.tag()) {
+            case 1: {
+                REQUIRE(item.get_fixed32() == 12345678L);
+                break;
+            }
+            case 2: {
+                REQUIRE(true);
+                item.skip();
+                break;
+            }
+            case 3: {
+                REQUIRE(item.get_int64() == 555555555LL);
+                break;
+            }
+            case 4: {
+                item.skip();
+                ++number_of_u;
+                break;
+            }
+            case 5: {
+                protozero::pbf_reader subitem = item.get_message();
+                REQUIRE(subitem.next());
+                REQUIRE(subitem.get_string() == "foobar");
+                REQUIRE_FALSE(subitem.next());
+                break;
+            }
+            case 7: {
+                const auto pi = item.get_packed_sint32();
+                REQUIRE(std::accumulate(pi.cbegin(), pi.cend(), 0) == 5);
+                break;
+            }
+            case 8: {
+                REQUIRE(item.get_string() == "optionalstring");
+                break;
+            }
+            default: {
+                REQUIRE(false); // should not be here
+                break;
+            }
+        }
+    }
+    REQUIRE(number_of_u == 5);
+}
+

third_party/protozero/test/t/complex/testcase.cpp

@@ -1,8 +1,9 @@
 
 #include <testcase.hpp>
+
 #include "testcase.pb.h"
 
-int main(int c, char *argv[]) {
+int main() {
     TestComplex::Test msg;
 
     msg.set_f(12345678);

third_party/protozero/test/t/complex/testcase.proto

@@ -1,3 +1,4 @@
+syntax = "proto2";
 
 option optimize_for = LITE_RUNTIME;
 

third_party/protozero/test/t/double/double_testcase.proto

@@ -1,3 +1,4 @@
+syntax = "proto2";
 
 option optimize_for = LITE_RUNTIME;
 

third_party/protozero/test/t/double/reader_test_cases.cpp

@@ -45,7 +45,7 @@ TEST_CASE("read double field") {
             for (std::string::size_type i = 1; i < abuffer.size() - n; ++i) {
                 protozero::pbf_reader item{abuffer.data() + n, i};
                 REQUIRE(item.next());
-                REQUIRE_THROWS_AS(item.get_double(), const protozero::end_of_buffer_exception&);
+                REQUIRE_THROWS_AS(item.get_double(), protozero::end_of_buffer_exception);
             }
         }
     }

third_party/protozero/test/t/double/testcase.cpp

@@ -1,8 +1,9 @@
 
 #include <testcase.hpp>
+
 #include "testcase.pb.h"
 
-int main(int c, char *argv[]) {
+int main() {
     TestDouble::Test msg;
 
     msg.set_x(0.0);

third_party/protozero/test/t/double/writer_test_cases.cpp

@@ -1,19 +1,20 @@
 
-#include <test.hpp>
+#include <buffer.hpp>
 
 #include "t/double/double_testcase.pb.h"
 
-TEST_CASE("write double field and check with libprotobuf") {
+TEMPLATE_TEST_CASE("write double field and check with libprotobuf", "",
+    buffer_test_string, buffer_test_vector, buffer_test_array, buffer_test_external) {
 
-    std::string buffer;
-    protozero::pbf_writer pw{buffer};
+    TestType buffer;
+    typename TestType::writer_type pw{buffer.buffer()};
 
     TestDouble::Test msg;
 
     SECTION("zero") {
         pw.add_double(1, 0.0);
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.x() == Approx(0.0));
     }
@@ -21,7 +22,7 @@ TEST_CASE("write double field and check with libprotobuf") {
     SECTION("positive") {
         pw.add_double(1, 4.893);
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.x() == Approx(4.893));
     }
@@ -29,7 +30,7 @@ TEST_CASE("write double field and check with libprotobuf") {
     SECTION("negative") {
         pw.add_double(1, -9232.33);
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.x() == Approx(-9232.33));
     }

third_party/protozero/test/t/enum/enum_testcase.proto

@@ -1,3 +1,4 @@
+syntax = "proto2";
 
 option optimize_for = LITE_RUNTIME;
 

third_party/protozero/test/t/enum/testcase.cpp

@@ -1,8 +1,9 @@
 
 #include <testcase.hpp>
+
 #include "testcase.pb.h"
 
-int main(int c, char *argv[]) {
+int main() {
     TestEnum::Test msg;
 
     msg.set_color(TestEnum::BLACK);

third_party/protozero/test/t/enum/writer_test_cases.cpp

@@ -1,19 +1,20 @@
 
-#include <test.hpp>
+#include <buffer.hpp>
 
 #include "t/enum/enum_testcase.pb.h"
 
-TEST_CASE("write enum field and check with libprotobuf") {
+TEMPLATE_TEST_CASE("write enum field and check with libprotobuf", "",
+    buffer_test_string, buffer_test_vector, buffer_test_array, buffer_test_external) {
 
-    std::string buffer;
-    protozero::pbf_writer pw{buffer};
+    TestType buffer;
+    typename TestType::writer_type pw{buffer.buffer()};
 
     TestEnum::Test msg;
 
     SECTION("zero") {
         pw.add_enum(1, 0L);
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.color() == TestEnum::Color::BLACK);
     }
@@ -21,7 +22,7 @@ TEST_CASE("write enum field and check with libprotobuf") {
     SECTION("positive") {
         pw.add_enum(1, 3L);
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.color() == TestEnum::Color::BLUE);
     }
@@ -29,7 +30,7 @@ TEST_CASE("write enum field and check with libprotobuf") {
     SECTION("negative") {
         pw.add_enum(1, -1L);
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.color() == TestEnum::Color::NEG);
     }
@@ -37,7 +38,7 @@ TEST_CASE("write enum field and check with libprotobuf") {
     SECTION("max") {
         pw.add_enum(1, std::numeric_limits<int32_t>::max() - 1);
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.color() == TestEnum::Color::MAX);
     }
@@ -45,7 +46,7 @@ TEST_CASE("write enum field and check with libprotobuf") {
     SECTION("min") {
         pw.add_enum(1, std::numeric_limits<int32_t>::min() + 1);
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.color() == TestEnum::Color::MIN);
     }

third_party/protozero/test/t/fixed32/fixed32_testcase.proto

@@ -1,3 +1,4 @@
+syntax = "proto2";
 
 option optimize_for = LITE_RUNTIME;
 

third_party/protozero/test/t/fixed32/testcase.cpp

@@ -1,8 +1,9 @@
 
 #include <testcase.hpp>
+
 #include "testcase.pb.h"
 
-int main(int c, char *argv[]) {
+int main() {
     TestFixed32::Test msg;
 
     msg.set_i(0);

third_party/protozero/test/t/fixed32/writer_test_cases.cpp

@@ -1,19 +1,20 @@
 
-#include <test.hpp>
+#include <buffer.hpp>
 
 #include "t/fixed32/fixed32_testcase.pb.h"
 
-TEST_CASE("write fixed32 field and check with libprotobuf") {
+TEMPLATE_TEST_CASE("write fixed32 field and check with libprotobuf", "",
+    buffer_test_string, buffer_test_vector, buffer_test_array, buffer_test_external) {
 
-    std::string buffer;
-    protozero::pbf_writer pw{buffer};
+    TestType buffer;
+    typename TestType::writer_type pw{buffer.buffer()};
 
     TestFixed32::Test msg;
 
     SECTION("zero") {
         pw.add_fixed32(1, 0);
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.i() == 0);
     }
@@ -21,7 +22,7 @@ TEST_CASE("write fixed32 field and check with libprotobuf") {
     SECTION("max") {
         pw.add_fixed32(1, std::numeric_limits<uint32_t>::max());
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.i() == std::numeric_limits<uint32_t>::max());
     }
@@ -29,7 +30,7 @@ TEST_CASE("write fixed32 field and check with libprotobuf") {
     SECTION("min") {
         pw.add_fixed32(1, std::numeric_limits<uint32_t>::min());
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.i() == std::numeric_limits<uint32_t>::min());
     }

third_party/protozero/test/t/fixed64/testcase.cpp

@@ -1,8 +1,9 @@
 
 #include <testcase.hpp>
+
 #include "testcase.pb.h"
 
-int main(int c, char *argv[]) {
+int main() {
     TestFixed64::Test msg;
 
     msg.set_i(0);

third_party/protozero/test/t/fixed64/testcase.proto

@@ -1,3 +1,4 @@
+syntax = "proto2";
 
 option optimize_for = LITE_RUNTIME;
 

third_party/protozero/test/t/float/reader_test_cases.cpp

@@ -46,7 +46,7 @@ TEST_CASE("read float field") {
             for (std::string::size_type i = 1; i < abuffer.size() - n; ++i) {
                 protozero::pbf_reader item{abuffer.data() + n, i};
                 REQUIRE(item.next());
-                REQUIRE_THROWS_AS(item.get_float(), const protozero::end_of_buffer_exception&);
+                REQUIRE_THROWS_AS(item.get_float(), protozero::end_of_buffer_exception);
             }
         }
     }
@@ -57,17 +57,17 @@ TEST_CASE("write float field") {
     protozero::pbf_writer pw{buffer};
 
     SECTION("zero") {
-        pw.add_float(1, 0.0f);
+        pw.add_float(1, 0.0F);
         REQUIRE(buffer == load_data("float/data-zero"));
     }
 
     SECTION("positive") {
-        pw.add_float(1, 5.34f);
+        pw.add_float(1, 5.34F);
         REQUIRE(buffer == load_data("float/data-pos"));
     }
 
     SECTION("negative") {
-        pw.add_float(1, -1.71f);
+        pw.add_float(1, -1.71F);
         REQUIRE(buffer == load_data("float/data-neg"));
     }
 }

third_party/protozero/test/t/float/testcase.cpp

@@ -1,8 +1,9 @@
 
 #include <testcase.hpp>
+
 #include "testcase.pb.h"
 
-int main(int c, char *argv[]) {
+int main() {
     TestFloat::Test msg;
 
     msg.set_x(0.0);

third_party/protozero/test/t/float/testcase.proto

@@ -1,3 +1,4 @@
+syntax = "proto2";
 
 option optimize_for = LITE_RUNTIME;
 

third_party/protozero/test/t/int32/int32_testcase.proto

@@ -1,3 +1,4 @@
+syntax = "proto2";
 
 option optimize_for = LITE_RUNTIME;
 

third_party/protozero/test/t/int32/testcase.cpp

@@ -1,8 +1,9 @@
 
 #include <testcase.hpp>
+
 #include "testcase.pb.h"
 
-int main(int c, char *argv[]) {
+int main() {
     TestInt32::Test msg;
 
     msg.set_i(0);

third_party/protozero/test/t/int32/writer_test_cases.cpp

@@ -1,19 +1,20 @@
 
-#include <test.hpp>
+#include <buffer.hpp>
 
 #include "t/int32/int32_testcase.pb.h"
 
-TEST_CASE("write int32 field and check with libprotobuf") {
+TEMPLATE_TEST_CASE("write int32 field and check with libprotobuf", "",
+    buffer_test_string, buffer_test_vector, buffer_test_array, buffer_test_external) {
 
-    std::string buffer;
-    protozero::pbf_writer pw{buffer};
+    TestType buffer;
+    typename TestType::writer_type pw{buffer.buffer()};
 
     TestInt32::Test msg;
 
     SECTION("zero") {
         pw.add_int32(1, 0L);
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.i() == 0L);
     }
@@ -21,7 +22,7 @@ TEST_CASE("write int32 field and check with libprotobuf") {
     SECTION("positive") {
         pw.add_int32(1, 1L);
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.i() == 1L);
     }
@@ -29,7 +30,7 @@ TEST_CASE("write int32 field and check with libprotobuf") {
     SECTION("negative") {
         pw.add_int32(1, -1L);
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.i() == -1L);
     }
@@ -37,7 +38,7 @@ TEST_CASE("write int32 field and check with libprotobuf") {
     SECTION("max") {
         pw.add_int32(1, std::numeric_limits<int32_t>::max());
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.i() == std::numeric_limits<int32_t>::max());
     }
@@ -45,7 +46,7 @@ TEST_CASE("write int32 field and check with libprotobuf") {
     SECTION("min") {
         pw.add_int32(1, std::numeric_limits<int32_t>::min());
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.i() == std::numeric_limits<int32_t>::min());
     }

third_party/protozero/test/t/int64/testcase.cpp

@@ -1,8 +1,9 @@
 
 #include <testcase.hpp>
+
 #include "testcase.pb.h"
 
-int main(int c, char *argv[]) {
+int main() {
     TestInt64::Test msg;
 
     msg.set_i(0);

third_party/protozero/test/t/int64/testcase.proto

@@ -1,3 +1,4 @@
+syntax = "proto2";
 
 option optimize_for = LITE_RUNTIME;
 

third_party/protozero/test/t/message/message_testcase.proto

@@ -1,3 +1,4 @@
+syntax = "proto2";
 
 option optimize_for = LITE_RUNTIME;
 

third_party/protozero/test/t/message/reader_test_cases.cpp

@@ -6,10 +6,16 @@ TEST_CASE("read message field: string") {
 
     protozero::pbf_reader item{buffer};
 
+    REQUIRE(item.data().data() == buffer.data());
+    REQUIRE(item.data().size() == buffer.size());
+
     REQUIRE(item.next());
     protozero::pbf_reader subitem{item.get_message()};
     REQUIRE_FALSE(item.next());
 
+    REQUIRE(item.data().data() == buffer.data() + buffer.size());
+    REQUIRE(item.data().empty());
+
     REQUIRE(subitem.next());
     REQUIRE(subitem.get_string() == "foobar");
     REQUIRE_FALSE(subitem.next());
@@ -21,7 +27,7 @@ TEST_CASE("read message field: end of buffer") {
     for (std::string::size_type i = 1; i < buffer.size(); ++i) {
         protozero::pbf_reader item{buffer.data(), i};
         REQUIRE(item.next());
-        REQUIRE_THROWS_AS(item.get_string(), const protozero::end_of_buffer_exception&);
+        REQUIRE_THROWS_AS(item.get_string(), protozero::end_of_buffer_exception);
     }
 }
 

third_party/protozero/test/t/message/testcase.cpp

@@ -1,8 +1,9 @@
 
 #include <testcase.hpp>
+
 #include "testcase.pb.h"
 
-int main(int c, char *argv[]) {
+int main() {
     TestMessage::Test msg;
 
     TestMessage::Sub* submsg = msg.mutable_submessage();

third_party/protozero/test/t/message/writer_test_cases.cpp

@@ -1,28 +1,29 @@
 
-#include <test.hpp>
+#include <buffer.hpp>
 
 #include "t/message/message_testcase.pb.h"
 
-TEST_CASE("write message field and check with libprotobuf") {
+TEMPLATE_TEST_CASE("write message field and check with libprotobuf", "",
+    buffer_test_string, buffer_test_vector, buffer_test_array, buffer_test_external) {
 
-    std::string buffer_test;
-    protozero::pbf_writer pbf_test{buffer_test};
+    TestType buffer;
+    typename TestType::writer_type pw{buffer.buffer()};
 
     SECTION("string") {
         std::string buffer_submessage;
         protozero::pbf_writer pbf_submessage{buffer_submessage};
         pbf_submessage.add_string(1, "foobar");
 
-        pbf_test.add_message(1, buffer_submessage);
+        pw.add_message(1, buffer_submessage);
     }
 
     SECTION("string with subwriter") {
-        protozero::pbf_writer pbf_submessage{pbf_test, 1};
+        typename TestType::writer_type pbf_submessage{pw, 1};
         pbf_submessage.add_string(1, "foobar");
     }
 
     TestMessage::Test msg;
-    msg.ParseFromString(buffer_test);
+    msg.ParseFromArray(buffer.data(), buffer.size());
     REQUIRE(msg.submessage().s() == "foobar");
 
 }

third_party/protozero/test/t/nested/nested_testcase.proto

@@ -1,3 +1,4 @@
+syntax = "proto2";
 
 option optimize_for = LITE_RUNTIME;
 

third_party/protozero/test/t/nested/testcase.cpp

@@ -1,8 +1,9 @@
 
 #include <testcase.hpp>
+
 #include "testcase.pb.h"
 
-int main(int c, char *argv[]) {
+int main() {
     TestNested::Test msg;
     msg.set_i(77);
 

third_party/protozero/test/t/nested/writer_test_cases.cpp

@@ -1,12 +1,13 @@
 
-#include <test.hpp>
+#include <buffer.hpp>
 
 #include "t/nested/nested_testcase.pb.h"
 
-TEST_CASE("write nested message fields and check with libprotobuf") {
+TEMPLATE_TEST_CASE("write nested message fields and check with libprotobuf", "",
+    buffer_test_string, buffer_test_vector, buffer_test_array, buffer_test_external) {
 
-    std::string buffer_test;
-    protozero::pbf_writer pbf_test{buffer_test};
+    TestType buffer;
+    typename TestType::writer_type pw{buffer.buffer()};
 
     SECTION("string") {
         std::string buffer_subsub;
@@ -19,23 +20,23 @@ TEST_CASE("write nested message fields and check with libprotobuf") {
         pbf_sub.add_string(1, buffer_subsub);
         pbf_sub.add_int32(2, 88);
 
-        pbf_test.add_message(1, buffer_sub);
+        pw.add_message(1, buffer_sub);
     }
 
     SECTION("with subwriter") {
-        protozero::pbf_writer pbf_sub{pbf_test, 1};
+        typename TestType::writer_type pbf_sub{pw, 1};
         {
-            protozero::pbf_writer pbf_subsub(pbf_sub, 1);
+            typename TestType::writer_type pbf_subsub(pbf_sub, 1);
             pbf_subsub.add_string(1, "foobar");
             pbf_subsub.add_int32(2, 99);
         }
         pbf_sub.add_int32(2, 88);
     }
 
-    pbf_test.add_int32(2, 77);
+    pw.add_int32(2, 77);
 
     TestNested::Test msg;
-    msg.ParseFromString(buffer_test);
+    msg.ParseFromArray(buffer.data(), buffer.size());
 
     REQUIRE(msg.i() == 77);
     REQUIRE(msg.sub().i() == 88);

third_party/protozero/test/t/repeated/reader_test_cases.cpp

@@ -48,7 +48,7 @@ TEST_CASE("read repeated fields: end of buffer") {
     for (std::string::size_type i = 1; i < buffer.size(); ++i) {
         protozero::pbf_reader item{buffer.data(), i};
         REQUIRE(item.next());
-        REQUIRE_THROWS_AS(item.get_int32(), const protozero::end_of_buffer_exception&);
+        REQUIRE_THROWS_AS(item.get_int32(), protozero::end_of_buffer_exception);
     }
 }
 

third_party/protozero/test/t/repeated/repeated_testcase.proto

@@ -1,3 +1,4 @@
+syntax = "proto2";
 
 option optimize_for = LITE_RUNTIME;
 

third_party/protozero/test/t/repeated/testcase.cpp

@@ -1,8 +1,9 @@
 
 #include <testcase.hpp>
+
 #include "testcase.pb.h"
 
-int main(int c, char *argv[]) {
+int main() {
     TestRepeated::Test msg;
 
     write_to_file(msg, "data-empty.pbf");

third_party/protozero/test/t/repeated/writer_test_cases.cpp

@@ -1,19 +1,20 @@
 
-#include <test.hpp>
+#include <buffer.hpp>
 
 #include "t/repeated/repeated_testcase.pb.h"
 
-TEST_CASE("write repeated fields and check with libprotobuf") {
+TEMPLATE_TEST_CASE("write repeated fields and check with libprotobuf", "",
+    buffer_test_string, buffer_test_vector, buffer_test_array, buffer_test_external) {
 
-    std::string buffer;
-    protozero::pbf_writer pw{buffer};
+    TestType buffer;
+    typename TestType::writer_type pw{buffer.buffer()};
 
     TestRepeated::Test msg;
 
     SECTION("one") {
         pw.add_int32(1, 0L);
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.i().size() == 1);
         REQUIRE(msg.i(0) == 0L);
@@ -26,7 +27,7 @@ TEST_CASE("write repeated fields and check with libprotobuf") {
         pw.add_int32(1, std::numeric_limits<int32_t>::max());
         pw.add_int32(1, std::numeric_limits<int32_t>::min());
 
-        msg.ParseFromString(buffer);
+        msg.ParseFromArray(buffer.data(), buffer.size());
 
         REQUIRE(msg.i().size() == 5);
         REQUIRE(msg.i(0) == 0L);

third_party/protozero/test/t/repeated_packed_bool/reader_test_cases.cpp

@@ -1,6 +1,8 @@
 
 #include <test.hpp>
 
+#include <array>
+
 TEST_CASE("read repeated packed bool field: empty") {
     const std::string buffer = load_data("repeated_packed_bool/data-empty");
 
@@ -51,7 +53,7 @@ TEST_CASE("read repeated packed bool field: end of buffer") {
     for (std::string::size_type i = 1; i < buffer.size(); ++i) {
         protozero::pbf_reader item{buffer.data(), i};
         REQUIRE(item.next());
-        REQUIRE_THROWS_AS(item.get_packed_bool(), const protozero::end_of_buffer_exception&);
+        REQUIRE_THROWS_AS(item.get_packed_bool(), protozero::end_of_buffer_exception);
     }
 }
 
@@ -60,21 +62,21 @@ TEST_CASE("write repeated packed bool field") {
     protozero::pbf_writer pw{buffer};
 
     SECTION("empty") {
-        const bool data[] = { true };
+        const std::array<bool, 1> data = {{ true }};
         pw.add_packed_bool(1, std::begin(data), std::begin(data) /* !!!! */);
 
         REQUIRE(buffer == load_data("repeated_packed_bool/data-empty"));
     }
 
     SECTION("one") {
-        const bool data[] = { true };
+        const std::array<bool, 1> data = {{ true }};
         pw.add_packed_bool(1, std::begin(data), std::end(data));
 
         REQUIRE(buffer == load_data("repeated_packed_bool/data-one"));
     }
 
     SECTION("many") {
-        const bool data[] = { true, true, false, true };
+        const std::array<bool, 4> data = {{ true, true, false, true }};
         pw.add_packed_bool(1, std::begin(data), std::end(data));
 
         REQUIRE(buffer == load_data("repeated_packed_bool/data-many"));

third_party/protozero/test/t/repeated_packed_bool/testcase.cpp

@@ -1,8 +1,9 @@
 
 #include <testcase.hpp>
+
 #include "testcase.pb.h"
 
-int main(int c, char *argv[]) {
+int main() {
     TestRepeatedPackedBool::Test msg;
 
     write_to_file(msg, "data-empty.pbf");