Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Improve performance of JSON rendering (#6380)

Browse Source
This commit is contained in:
Siarhei Fedartsou 2022-10-03 21:43:51 +02:00 committed by GitHub
parent 41fd947ebd
commit 9a4b4648f4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 317 additions and 711 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/osrm-backend.yml vendored
View File

@ -715,6 +715,7 @@ jobs:
pushd ${OSRM_BUILD_DIR}
make --jobs=${JOBS} benchmarks
./src/benchmarks/alias-bench
./src/benchmarks/json-render-bench ../src/benchmarks/portugal_to_korea.json
./src/benchmarks/match-bench ../test/data/ch/monaco.osrm
./src/benchmarks/packedvector-bench
./src/benchmarks/rtree-bench ../test/data/monaco.osrm.ramIndex ../test/data/monaco.osrm.fileIndex ../test/data/monaco.osrm.nbg_nodes

1
CHANGELOG.md
View File

@ -9,6 +9,7 @@
- NodeJS:
- FIXED: Support `skip_waypoints` in Node bindings [#6060](https://github.com/Project-OSRM/osrm-backend/pull/6060)
- Misc:
- CHANGED: Improve performance of JSON rendering. Fix undefined behaviour in JSON numbers formatting. [#6380](https://github.com/Project-OSRM/osrm-backend/pull/6380)
- ADDED: Add timestamps for logs. [#6375](https://github.com/Project-OSRM/osrm-backend/pull/6375)
- CHANGED: Improve performance of map matching via getPathDistance optimization. [#6378](https://github.com/Project-OSRM/osrm-backend/pull/6378)
- CHANGED: Optimize RestrictionParser performance. [#6344](https://github.com/Project-OSRM/osrm-backend/pull/6344)

2
features/testbot/annotations.feature
View File

@ -115,4 +115,4 @@ Feature: Annotations
When I route I should get
| from | to | route | a:speed | a:distance | a:duration | a:nodes |
| a | c | abc,abc | 10:10 | 249.987619:299.962882 | 25:30 | 1:2:3 |
| a | c | abc,abc | 10:10 | 249.987618946:299.962882039 | 25:30 | 1:2:3 |

4
features/testbot/matching.feature
View File

@ -810,6 +810,6 @@ Feature: Basic Map Matching
# These should have the same weights/duration in either direction
When I match I should get
| trace | geometry | a:distance | a:duration | a:weight | duration |
| 2345 | 1.00018,1,1.000314,1 | 14.914666 | 1.4 | 1.4 | 1.4 |
| 4321 | 1.00027,1,1.000135,1 | 15.02597 | 1.5 | 1.5 | 1.5 |
| 2345 | 1.00018,1,1.000314,1 | 14.914666491 | 1.4 | 1.4 | 1.4 |
| 4321 | 1.00027,1,1.000135,1 | 15.025969972 | 1.5 | 1.5 | 1.5 |

4
features/testbot/snap_intersection.feature
View File

@ -570,7 +570,7 @@ Feature: Snapping at intersections
| a,f,k | ac,cf,cf,fj,kj,kj | 132.8s | 132.8 |
| k,f | ik,fi,fi | 54.3s | 54.3 |
| f,a | ef,ae,ae | 66.6s | 66.6 |
| k,f,a | kj,fj,fj,ef,ae,ae | 141.4s | 141.4 |
| k,f,a | kj,fj,fj,ef,ae,ae | 141.399999999s | 141.399999999 |
When I request a travel time matrix I should get
| | a | f | k |
@ -626,4 +626,4 @@ Feature: Snapping at intersections
| a,f,k | ad,df,df,fj,kj,kj | 105.6s | 105.6 |
| k,f | ik,fi,fi | 54.3s | 54.3 |
| f,a | ef,ae,ae | 66.6s | 66.6 |
| k,f,a | ik,fi,fi,ef,ae,ae | 120.9s | 120.9 |
| k,f,a | ik,fi,fi,ef,ae,ae | 120.899999999s | 120.899999999 |

8
features/testbot/weight.feature
View File

@ -53,10 +53,10 @@ Feature: Weight tests
When I route I should get
| waypoints | route | distances | weights | times | a:distance | a:duration | a:weight | a:speed |
| s,t | abc,abc | 20m,0m | 2,0 | 2s,0s | 20.034627 | 2 | 2 | 10 |
| t,s | abc,abc | 20m,0m | 2,0 | 2s,0s | 20.034627 | 2 | 2 | 10 |
| s,e | abc,abc | 40m,0m | 3.9,0 | 3.9s,0s | 29.940636:10.017313 | 3:0.9 | 3:0.9 | 10:11.1 |
| e,s | abc,abc | 40m,0m | 3.9,0 | 3.9s,0s | 10.017313:29.940636 | 0.9:3 | 0.9:3 | 11.1:10 |
| s,t | abc,abc | 20m,0m | 2,0 | 2s,0s | 20.034626629 | 2 | 2 | 10 |
| t,s | abc,abc | 20m,0m | 2,0 | 2s,0s | 20.034626629 | 2 | 2 | 10 |
| s,e | abc,abc | 40m,0m | 3.9,0 | 3.9s,0s | 29.940636463:10.017313314 | 3:0.9 | 3:0.9 | 10:11.1 |
| e,s | abc,abc | 40m,0m | 3.9,0 | 3.9s,0s | 10.017313314:29.940636463 | 0.9:3 | 0.9:3 | 11.1:10 |
Scenario: Step weights -- way_function: fail if no weight or weight_per_meter property

5
fuzz/escape_json.cc
View File

@ -5,13 +5,14 @@
#include <iterator>
#include <string>
using osrm::util::escape_JSON;
using osrm::util::EscapeJSONString;
extern "C" int LLVMFuzzerTestOneInput(const unsigned char *data, unsigned long size)
{
const std::string in(reinterpret_cast<const char *>(data), size);
const auto escaped = escape_JSON(in);
std::string escaped;
EscapeJSONString(in, escaped);
escape(escaped.data());
return 0;

4
include/nodejs/json_v8_renderer.hpp
View File

@ -60,8 +60,8 @@ struct V8Renderer
inline void renderToV8(v8::Local<v8::Value> &out, const osrm::json::Object &object)
{
osrm::json::Value value = object;
mapbox::util::apply_visitor(V8Renderer(out), value);
V8Renderer renderer(out);
renderer(object);
}
} // namespace node_osrm

49
include/util/cast.hpp
View File

@ -1,49 +0,0 @@
#ifndef CAST_HPP
#define CAST_HPP
#include <iomanip>
#include <sstream>
#include <string>
#include <type_traits>
#include <boost/algorithm/string/classification.hpp>
#include <boost/algorithm/string/trim.hpp>
namespace osrm
{
namespace util
{
namespace cast
{
template <typename Enumeration>
inline auto enum_to_underlying(Enumeration const value) ->
typename std::underlying_type<Enumeration>::type
{
return static_cast<typename std::underlying_type<Enumeration>::type>(value);
}
template <typename T, int Precision = 6> inline std::string to_string_with_precision(const T x)
{
static_assert(std::is_arithmetic<T>::value, "integral or floating point type required");
std::ostringstream out;
out << std::fixed << std::setprecision(Precision) << x;
auto rv = out.str();
// Javascript has no separation of float / int, digits without a '.' are integral typed
// X.Y.0 -> X.Y
// X.0 -> X
boost::trim_right_if(rv, boost::is_any_of("0"));
boost::trim_right_if(rv, boost::is_any_of("."));
// Note:
// - assumes the locale to use '.' as digit separator
// - this is not identical to: trim_right_if(rv, is_any_of('0 .'))
return rv;
}
} // namespace cast
} // namespace util
} // namespace osrm
#endif // CAST_HPP

520
include/util/ieee754.hpp
View File

@ -1,520 +0,0 @@
#ifndef IEEE754_HPP
#define IEEE754_HPP
/**
Copyright (C) 2014 Milo Yip
Imported from:
https://github.com/miloyip/dtoa-benchmark/blob/c4020c62754950d38a1aaaed2975b05b441d1e7d/src/milo/dtoa_milo.h
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
**/
#include <assert.h>
#include <math.h>
#include <memory.h>
#if defined(_MSC_VER)
#include "rapidjson/msinttypes/stdint.h"
#include <intrin.h>
#else
#include <stdint.h>
#endif
#define UINT64_C2(h, l) ((static_cast<uint64_t>(h) << 32) | static_cast<uint64_t>(l))
namespace osrm
{
namespace util
{
namespace ieee754
{
struct DiyFp
{
DiyFp() {}
DiyFp(uint64_t f, int e) : f(f), e(e) {}
DiyFp(double d)
{
union
{
double d;
uint64_t u64;
} u = {d};
int biased_e = (u.u64 & kDpExponentMask) >> kDpSignificandSize;
uint64_t significand = (u.u64 & kDpSignificandMask);
if (biased_e != 0)
{
f = significand + kDpHiddenBit;
e = biased_e - kDpExponentBias;
}
else
{
f = significand;
e = kDpMinExponent + 1;
}
}
DiyFp operator-(const DiyFp &rhs) const
{
assert(e == rhs.e);
assert(f >= rhs.f);
return DiyFp(f - rhs.f, e);
}
DiyFp operator*(const DiyFp &rhs) const
{
#if defined(_MSC_VER) && defined(_M_AMD64)
uint64_t h;
uint64_t l = _umul128(f, rhs.f, &h);
if (l & (uint64_t(1) << 63)) // rounding
h++;
return DiyFp(h, e + rhs.e + 64);
#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__)
__extension__ using uint128 = unsigned __int128;
uint128 p = static_cast<uint128>(f) * static_cast<uint128>(rhs.f);
uint64_t h = p >> 64;
uint64_t l = static_cast<uint64_t>(p);
if (l & (uint64_t(1) << 63)) // rounding
h++;
return DiyFp(h, e + rhs.e + 64);
#else
const uint64_t M32 = 0xFFFFFFFF;
const uint64_t a = f >> 32;
const uint64_t b = f & M32;
const uint64_t c = rhs.f >> 32;
const uint64_t d = rhs.f & M32;
const uint64_t ac = a * c;
const uint64_t bc = b * c;
const uint64_t ad = a * d;
const uint64_t bd = b * d;
uint64_t tmp = (bd >> 32) + (ad & M32) + (bc & M32);
tmp += 1U << 31; /// mult_round
return DiyFp(ac + (ad >> 32) + (bc >> 32) + (tmp >> 32), e + rhs.e + 64);
#endif
}
DiyFp Normalize() const
{
#if defined(_MSC_VER) && defined(_M_AMD64)
unsigned long index;
_BitScanReverse64(&index, f);
return DiyFp(f << (63 - index), e - (63 - index));
#elif defined(__GNUC__)
int s = __builtin_clzll(f);
return DiyFp(f << s, e - s);
#else
DiyFp res = *this;
while (!(res.f & kDpHiddenBit))
{
res.f <<= 1;
res.e--;
}
res.f <<= (kDiySignificandSize - kDpSignificandSize - 1);
res.e = res.e - (kDiySignificandSize - kDpSignificandSize - 1);
return res;
#endif
}
DiyFp NormalizeBoundary() const
{
#if defined(_MSC_VER) && defined(_M_AMD64)
unsigned long index;
_BitScanReverse64(&index, f);
return DiyFp(f << (63 - index), e - (63 - index));
#else
DiyFp res = *this;
while (!(res.f & (kDpHiddenBit << 1)))
{
res.f <<= 1;
res.e--;
}
res.f <<= (kDiySignificandSize - kDpSignificandSize - 2);
res.e = res.e - (kDiySignificandSize - kDpSignificandSize - 2);
return res;
#endif
}
void NormalizedBoundaries(DiyFp *minus, DiyFp *plus) const
{
DiyFp pl = DiyFp((f << 1) + 1, e - 1).NormalizeBoundary();
DiyFp mi = (f == kDpHiddenBit) ? DiyFp((f << 2) - 1, e - 2) : DiyFp((f << 1) - 1, e - 1);
mi.f <<= mi.e - pl.e;
mi.e = pl.e;
*plus = pl;
*minus = mi;
}
static const int kDiySignificandSize = 64;
static const int kDpSignificandSize = 52;
static const int kDpExponentBias = 0x3FF + kDpSignificandSize;
static const int kDpMinExponent = -kDpExponentBias;
static const uint64_t kDpExponentMask = UINT64_C2(0x7FF00000, 0x00000000);
static const uint64_t kDpSignificandMask = UINT64_C2(0x000FFFFF, 0xFFFFFFFF);
static const uint64_t kDpHiddenBit = UINT64_C2(0x00100000, 0x00000000);
uint64_t f;
int e;
};
inline DiyFp GetCachedPower(int e, int *K)
{
// 10^-348, 10^-340, ..., 10^340
static const uint64_t kCachedPowers_F[] = {
UINT64_C2(0xfa8fd5a0, 0x081c0288), UINT64_C2(0xbaaee17f, 0xa23ebf76),
UINT64_C2(0x8b16fb20, 0x3055ac76), UINT64_C2(0xcf42894a, 0x5dce35ea),
UINT64_C2(0x9a6bb0aa, 0x55653b2d), UINT64_C2(0xe61acf03, 0x3d1a45df),
UINT64_C2(0xab70fe17, 0xc79ac6ca), UINT64_C2(0xff77b1fc, 0xbebcdc4f),
UINT64_C2(0xbe5691ef, 0x416bd60c), UINT64_C2(0x8dd01fad, 0x907ffc3c),
UINT64_C2(0xd3515c28, 0x31559a83), UINT64_C2(0x9d71ac8f, 0xada6c9b5),
UINT64_C2(0xea9c2277, 0x23ee8bcb), UINT64_C2(0xaecc4991, 0x4078536d),
UINT64_C2(0x823c1279, 0x5db6ce57), UINT64_C2(0xc2109436, 0x4dfb5637),
UINT64_C2(0x9096ea6f, 0x3848984f), UINT64_C2(0xd77485cb, 0x25823ac7),
UINT64_C2(0xa086cfcd, 0x97bf97f4), UINT64_C2(0xef340a98, 0x172aace5),
UINT64_C2(0xb23867fb, 0x2a35b28e), UINT64_C2(0x84c8d4df, 0xd2c63f3b),
UINT64_C2(0xc5dd4427, 0x1ad3cdba), UINT64_C2(0x936b9fce, 0xbb25c996),
UINT64_C2(0xdbac6c24, 0x7d62a584), UINT64_C2(0xa3ab6658, 0x0d5fdaf6),
UINT64_C2(0xf3e2f893, 0xdec3f126), UINT64_C2(0xb5b5ada8, 0xaaff80b8),
UINT64_C2(0x87625f05, 0x6c7c4a8b), UINT64_C2(0xc9bcff60, 0x34c13053),
UINT64_C2(0x964e858c, 0x91ba2655), UINT64_C2(0xdff97724, 0x70297ebd),
UINT64_C2(0xa6dfbd9f, 0xb8e5b88f), UINT64_C2(0xf8a95fcf, 0x88747d94),
UINT64_C2(0xb9447093, 0x8fa89bcf), UINT64_C2(0x8a08f0f8, 0xbf0f156b),
UINT64_C2(0xcdb02555, 0x653131b6), UINT64_C2(0x993fe2c6, 0xd07b7fac),
UINT64_C2(0xe45c10c4, 0x2a2b3b06), UINT64_C2(0xaa242499, 0x697392d3),
UINT64_C2(0xfd87b5f2, 0x8300ca0e), UINT64_C2(0xbce50864, 0x92111aeb),
UINT64_C2(0x8cbccc09, 0x6f5088cc), UINT64_C2(0xd1b71758, 0xe219652c),
UINT64_C2(0x9c400000, 0x00000000), UINT64_C2(0xe8d4a510, 0x00000000),
UINT64_C2(0xad78ebc5, 0xac620000), UINT64_C2(0x813f3978, 0xf8940984),
UINT64_C2(0xc097ce7b, 0xc90715b3), UINT64_C2(0x8f7e32ce, 0x7bea5c70),
UINT64_C2(0xd5d238a4, 0xabe98068), UINT64_C2(0x9f4f2726, 0x179a2245),
UINT64_C2(0xed63a231, 0xd4c4fb27), UINT64_C2(0xb0de6538, 0x8cc8ada8),
UINT64_C2(0x83c7088e, 0x1aab65db), UINT64_C2(0xc45d1df9, 0x42711d9a),
UINT64_C2(0x924d692c, 0xa61be758), UINT64_C2(0xda01ee64, 0x1a708dea),
UINT64_C2(0xa26da399, 0x9aef774a), UINT64_C2(0xf209787b, 0xb47d6b85),
UINT64_C2(0xb454e4a1, 0x79dd1877), UINT64_C2(0x865b8692, 0x5b9bc5c2),
UINT64_C2(0xc83553c5, 0xc8965d3d), UINT64_C2(0x952ab45c, 0xfa97a0b3),
UINT64_C2(0xde469fbd, 0x99a05fe3), UINT64_C2(0xa59bc234, 0xdb398c25),
UINT64_C2(0xf6c69a72, 0xa3989f5c), UINT64_C2(0xb7dcbf53, 0x54e9bece),
UINT64_C2(0x88fcf317, 0xf22241e2), UINT64_C2(0xcc20ce9b, 0xd35c78a5),
UINT64_C2(0x98165af3, 0x7b2153df), UINT64_C2(0xe2a0b5dc, 0x971f303a),
UINT64_C2(0xa8d9d153, 0x5ce3b396), UINT64_C2(0xfb9b7cd9, 0xa4a7443c),
UINT64_C2(0xbb764c4c, 0xa7a44410), UINT64_C2(0x8bab8eef, 0xb6409c1a),
UINT64_C2(0xd01fef10, 0xa657842c), UINT64_C2(0x9b10a4e5, 0xe9913129),
UINT64_C2(0xe7109bfb, 0xa19c0c9d), UINT64_C2(0xac2820d9, 0x623bf429),
UINT64_C2(0x80444b5e, 0x7aa7cf85), UINT64_C2(0xbf21e440, 0x03acdd2d),
UINT64_C2(0x8e679c2f, 0x5e44ff8f), UINT64_C2(0xd433179d, 0x9c8cb841),
UINT64_C2(0x9e19db92, 0xb4e31ba9), UINT64_C2(0xeb96bf6e, 0xbadf77d9),
UINT64_C2(0xaf87023b, 0x9bf0ee6b)};
static const int16_t kCachedPowers_E[] = {
-1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, -954, -927, -901,
-874, -847, -821, -794, -768, -741, -715, -688, -661, -635, -608, -582, -555,
-529, -502, -475, -449, -422, -396, -369, -343, -316, -289, -263, -236, -210,
-183, -157, -130, -103, -77, -50, -24, 3, 30, 56, 83, 109, 136,
162, 189, 216, 242, 269, 295, 322, 348, 375, 402, 428, 455, 481,
508, 534, 561, 588, 614, 641, 667, 694, 720, 747, 774, 800, 827,
853, 880, 907, 933, 960, 986, 1013, 1039, 1066};
// int k = static_cast<int>(ceil((-61 - e) * 0.30102999566398114)) + 374;
double dk =
(-61 - e) * 0.30102999566398114 + 347; // dk must be positive, so can do ceiling in positive
int k = static_cast<int>(dk);
if (k != dk)
k++;
unsigned index = static_cast<unsigned>((k >> 3) + 1);
*K = -(-348 + static_cast<int>(index << 3)); // decimal exponent no need lookup table
assert(index < sizeof(kCachedPowers_F) / sizeof(kCachedPowers_F[0]));
return DiyFp(kCachedPowers_F[index], kCachedPowers_E[index]);
}
inline void
GrisuRound(char *buffer, int len, uint64_t delta, uint64_t rest, uint64_t ten_kappa, uint64_t wp_w)
{
while (rest < wp_w && delta - rest >= ten_kappa &&
(rest + ten_kappa < wp_w || /// closer
wp_w - rest > rest + ten_kappa - wp_w))
{
buffer[len - 1]--;
rest += ten_kappa;
}
}
inline unsigned CountDecimalDigit32(uint32_t n)
{
// Simple pure C++ implementation was faster than __builtin_clz version in this situation.
if (n < 10)
return 1;
if (n < 100)
return 2;
if (n < 1000)
return 3;
if (n < 10000)
return 4;
if (n < 100000)
return 5;
if (n < 1000000)
return 6;
if (n < 10000000)
return 7;
if (n < 100000000)
return 8;
if (n < 1000000000)
return 9;
return 10;
}
inline void
DigitGen(const DiyFp &W, const DiyFp &Mp, uint64_t delta, char *buffer, int *len, int *K)
{
static const uint32_t kPow10[] = {
1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000};
const DiyFp one(uint64_t(1) << -Mp.e, Mp.e);
const DiyFp wp_w = Mp - W;
uint32_t p1 = static_cast<uint32_t>(Mp.f >> -one.e);
uint64_t p2 = Mp.f & (one.f - 1);
int kappa = static_cast<int>(CountDecimalDigit32(p1));
*len = 0;
while (kappa > 0)
{
uint32_t d;
switch (kappa)
{
case 10:
d = p1 / 1000000000;
p1 %= 1000000000;
break;
case 9:
d = p1 / 100000000;
p1 %= 100000000;
break;
case 8:
d = p1 / 10000000;
p1 %= 10000000;
break;
case 7:
d = p1 / 1000000;
p1 %= 1000000;
break;
case 6:
d = p1 / 100000;
p1 %= 100000;
break;
case 5:
d = p1 / 10000;
p1 %= 10000;
break;
case 4:
d = p1 / 1000;
p1 %= 1000;
break;
case 3:
d = p1 / 100;
p1 %= 100;
break;
case 2:
d = p1 / 10;
p1 %= 10;
break;
case 1:
d = p1;
p1 = 0;
break;
default:
#if defined(_MSC_VER)
__assume(0);
#elif __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5)
__builtin_unreachable();
#else
d = 0;
#endif
}
if (d || *len)
buffer[(*len)++] = '0' + static_cast<char>(d);
kappa--;
uint64_t tmp = (static_cast<uint64_t>(p1) << -one.e) + p2;
if (tmp <= delta)
{
*K += kappa;
GrisuRound(
buffer, *len, delta, tmp, static_cast<uint64_t>(kPow10[kappa]) << -one.e, wp_w.f);
return;
}
}
// kappa = 0
for (;;)
{
p2 *= 10;
delta *= 10;
char d = static_cast<char>(p2 >> -one.e);
if (d || *len)
buffer[(*len)++] = '0' + d;
p2 &= one.f - 1;
kappa--;
if (p2 < delta)
{
*K += kappa;
GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * kPow10[-kappa]);
return;
}
}
}
inline void Grisu2(double value, char *buffer, int *length, int *K)
{
const DiyFp v(value);
DiyFp w_m, w_p;
v.NormalizedBoundaries(&w_m, &w_p);
const DiyFp c_mk = GetCachedPower(w_p.e, K);
const DiyFp W = v.Normalize() * c_mk;
DiyFp Wp = w_p * c_mk;
DiyFp Wm = w_m * c_mk;
Wm.f++;
Wp.f--;
DigitGen(W, Wp, Wp.f - Wm.f, buffer, length, K);
}
inline const char *GetDigitsLut()
{
static const char cDigitsLut[200] = {
'0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', '0', '6', '0', '7', '0',
'8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', '1', '5', '1', '6',
'1', '7', '1', '8', '1', '9', '2', '0', '2', '1', '2', '2', '2', '3', '2', '4', '2',
'5', '2', '6', '2', '7', '2', '8', '2', '9', '3', '0', '3', '1', '3', '2', '3', '3',
'3', '4', '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1', '4',
'2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', '5', '0',
'5', '1', '5', '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5',
'9', '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6', '7',
'6', '8', '6', '9', '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7',
'6', '7', '7', '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', '8', '4',
'8', '5', '8', '6', '8', '7', '8', '8', '8', '9', '9', '0', '9', '1', '9', '2', '9',
'3', '9', '4', '9', '5', '9', '6', '9', '7', '9', '8', '9', '9'};
return cDigitsLut;
}
inline void WriteExponent(int K, char *buffer)
{
if (K < 0)
{
*buffer++ = '-';
K = -K;
}
if (K >= 100)
{
*buffer++ = '0' + static_cast<char>(K / 100);
K %= 100;
const char *d = GetDigitsLut() + K * 2;
*buffer++ = d[0];
*buffer++ = d[1];
}
else if (K >= 10)
{
const char *d = GetDigitsLut() + K * 2;
*buffer++ = d[0];
*buffer++ = d[1];
}
else
*buffer++ = '0' + static_cast<char>(K);
*buffer = '\0';
}
inline void Prettify(char *buffer, int length, int k)
{
const int kk = length + k; // 10^(kk-1) <= v < 10^kk
if (length <= kk && kk <= 21)
{
// 1234e7 -> 12340000000
for (int i = length; i < kk; i++)
buffer[i] = '0';
buffer[kk] = '.';
buffer[kk + 1] = '0';
buffer[kk + 2] = '\0';
}
else if (0 < kk && kk <= 21)
{
// 1234e-2 -> 12.34
memmove(&buffer[kk + 1], &buffer[kk], length - kk);
buffer[kk] = '.';
buffer[length + 1] = '\0';
}
else if (-6 < kk && kk <= 0)
{
// 1234e-6 -> 0.001234
const int offset = 2 - kk;
memmove(&buffer[offset], &buffer[0], length);
buffer[0] = '0';
buffer[1] = '.';
for (int i = 2; i < offset; i++)
buffer[i] = '0';
buffer[length + offset] = '\0';
}
else if (length == 1)
{
// 1e30
buffer[1] = 'e';
WriteExponent(kk - 1, &buffer[2]);
}
else
{
// 1234e30 -> 1.234e33
memmove(&buffer[2], &buffer[1], length - 1);
buffer[1] = '.';
buffer[length + 1] = 'e';
WriteExponent(kk - 1, &buffer[0 + length + 2]);
}
}
inline void dtoa_milo(double value, char *buffer)
{
// Not handling NaN and inf
assert(!isnan(value));
assert(!isinf(value));
if (value == 0)
{
buffer[0] = '0';
buffer[1] = '.';
buffer[2] = '0';
buffer[3] = '\0';
}
else
{
if (value < 0)
{
*buffer++ = '-';
value = -value;
}
int length, K;
Grisu2(value, buffer, &length, &K);
Prettify(buffer, length, K);
}
}
} // namespace ieee754
} // namespace util
} // namespace osrm
#endif // IEEE754_HPP

218
include/util/json_renderer.hpp
View File

@ -4,17 +4,18 @@
#ifndef JSON_RENDERER_HPP
#define JSON_RENDERER_HPP
#include "util/cast.hpp"
#include "util/ieee754.hpp"
#include "util/string_util.hpp"
#include "osrm/json_container.hpp"
#include <algorithm>
#include <iterator>
#include <ostream>
#include <string>
#include <vector>
#include <fmt/compile.h>
namespace osrm
{
namespace util
@ -22,173 +23,146 @@ namespace util
namespace json
{
namespace
template <typename Out> struct Renderer
{
constexpr int MAX_FLOAT_STRING_LENGTH = 256;
explicit Renderer(Out &_out) : out(_out) {}
void operator()(const String &string)
{
write('"');
// here we assume that vast majority of strings don't need to be escaped,
// so we check it first and escape only if needed
if (RequiresJSONStringEscaping(string.value))
{
std::string escaped;
// just a guess that 16 bytes for escaped characters will be enough to avoid
// reallocations
escaped.reserve(string.value.size() + 16);
EscapeJSONString(string.value, escaped);
write(escaped);
}
else
{
write(string.value);
}
write('"');
}
struct Renderer
void operator()(const Number &number)
{
explicit Renderer(std::ostream &_out) : out(_out) {}
// `fmt::memory_buffer` stores first 500 bytes in the object itself(i.e. on stack in this
// case) and then grows using heap if needed
fmt::memory_buffer buffer;
fmt::format_to(std::back_inserter(buffer), FMT_COMPILE("{}"), number.value);
void operator()(const String &string) const
// Truncate to 10 decimal places
size_t decimalpos = std::find(buffer.begin(), buffer.end(), '.') - buffer.begin();
if (buffer.size() > (decimalpos + 10))
{
out << "\"";
out << escape_JSON(string.value);
out << "\"";
buffer.resize(decimalpos + 10);
}
void operator()(const Number &number) const
{
char buffer[MAX_FLOAT_STRING_LENGTH] = {'\0'};
ieee754::dtoa_milo(number.value, buffer);
// Trucate to 10 decimal places
int pos = 0;
int decimalpos = 0;
while (decimalpos == 0 && pos < MAX_FLOAT_STRING_LENGTH && buffer[pos] != 0)
{
if (buffer[pos] == '.')
{
decimalpos = pos;
break;
}
++pos;
}
while (pos < MAX_FLOAT_STRING_LENGTH && buffer[pos] != 0)
{
if (pos - decimalpos == 10)
{
buffer[pos] = '\0';
break;
}
++pos;
}
out << buffer;
write(buffer.data(), buffer.size());
}
void operator()(const Object &object) const
void operator()(const Object &object)
{
out << "{";
write('{');
for (auto it = object.values.begin(), end = object.values.end(); it != end;)
{
out << "\"" << it->first << "\":";
write('\"');
write(it->first);
write<>("\":");
mapbox::util::apply_visitor(Renderer(out), it->second);
if (++it != end)
{
out << ",";
write(',');
}
}
out << "}";
write('}');
}
void operator()(const Array &array) const
void operator()(const Array &array)
{
out << "[";
write('[');
for (auto it = array.values.cbegin(), end = array.values.cend(); it != end;)
{
mapbox::util::apply_visitor(Renderer(out), *it);
if (++it != end)
{
out << ",";
write(',');
}
}
out << "]";
write(']');
}
void operator()(const True &) const { out << "true"; }
void operator()(const True &) { write<>("true"); }
void operator()(const False &) const { out << "false"; }
void operator()(const False &) { write<>("false"); }
void operator()(const Null &) const { out << "null"; }
void operator()(const Null &) { write<>("null"); }
private:
std::ostream &out;
};
void write(const std::string &str);
void write(const char *str, size_t size);
void write(char ch);
struct ArrayRenderer
template <size_t StrLength> void write(const char (&str)[StrLength])
{
explicit ArrayRenderer(std::vector<char> &_out) : out(_out) {}
void operator()(const String &string) const
{
out.push_back('\"');
const auto string_to_insert = escape_JSON(string.value);
out.insert(std::end(out), std::begin(string_to_insert), std::end(string_to_insert));
out.push_back('\"');
}
void operator()(const Number &number) const
{
const std::string number_string = cast::to_string_with_precision(number.value);
out.insert(out.end(), number_string.begin(), number_string.end());
}
void operator()(const Object &object) const
{
out.push_back('{');
for (auto it = object.values.begin(), end = object.values.end(); it != end;)
{
out.push_back('\"');
out.insert(out.end(), it->first.begin(), it->first.end());
out.push_back('\"');
out.push_back(':');
mapbox::util::apply_visitor(ArrayRenderer(out), it->second);
if (++it != end)
{
out.push_back(',');
}
}
out.push_back('}');
}
void operator()(const Array &array) const
{
out.push_back('[');
for (auto it = array.values.cbegin(), end = array.values.cend(); it != end;)
{
mapbox::util::apply_visitor(ArrayRenderer(out), *it);
if (++it != end)
{
out.push_back(',');
}
}
out.push_back(']');
}
void operator()(const True &) const
{
const std::string temp("true");
out.insert(out.end(), temp.begin(), temp.end());
}
void operator()(const False &) const
{
const std::string temp("false");
out.insert(out.end(), temp.begin(), temp.end());
}
void operator()(const Null &) const
{
const std::string temp("null");
out.insert(out.end(), temp.begin(), temp.end());
write(str, StrLength - 1);
}
private:
std::vector<char> &out;
Out &out;
};
template <> void Renderer<std::vector<char>>::write(const std::string &str)
{
out.insert(out.end(), str.begin(), str.end());
}
template <> void Renderer<std::vector<char>>::write(const char *str, size_t size)
{
out.insert(out.end(), str, str + size);
}
template <> void Renderer<std::vector<char>>::write(char ch) { out.push_back(ch); }
template <> void Renderer<std::ostream>::write(const std::string &str) { out << str; }
template <> void Renderer<std::ostream>::write(const char *str, size_t size)
{
out.write(str, size);
}
template <> void Renderer<std::ostream>::write(char ch) { out << ch; }
template <> void Renderer<std::string>::write(const std::string &str) { out += str; }
template <> void Renderer<std::string>::write(const char *str, size_t size)
{
out.append(str, size);
}
template <> void Renderer<std::string>::write(char ch) { out += ch; }
inline void render(std::ostream &out, const Object &object)
{
Value value = object;
mapbox::util::apply_visitor(Renderer(out), value);
Renderer renderer(out);
renderer(object);
}
inline void render(std::string &out, const Object &object)
{
Renderer renderer(out);
renderer(object);
}
inline void render(std::vector<char> &out, const Object &object)
{
Value value = object;
mapbox::util::apply_visitor(ArrayRenderer(out), value);
Renderer renderer(out);
renderer(object);
}
} // namespace json

28
include/util/string_util.hpp
View File

@ -58,11 +58,30 @@ template <int length, int precision> char *printInt(char *buffer, int value)
return buffer;
}
inline std::string escape_JSON(const std::string &input)
inline bool RequiresJSONStringEscaping(const std::string &string)
{
for (const char letter : string)
{
switch (letter)
{
case '\\':
case '"':
case '/':
case '\b':
case '\f':
case '\n':
case '\r':
case '\t':
return true;
default:
continue;
}
}
return false;
}
inline void EscapeJSONString(const std::string &input, std::string &output)
{
// escape and skip reallocations if possible
std::string output;
output.reserve(input.size() + 4); // +4 assumes two backslashes on avg
for (const char letter : input)
{
switch (letter)
@ -96,7 +115,6 @@ inline std::string escape_JSON(const std::string &input)
break;
}
}
return output;
}
inline std::size_t URIDecode(const std::string &input, std::string &output)

15
src/benchmarks/CMakeLists.txt
View File

@ -30,6 +30,18 @@ target_link_libraries(match-bench
${TBB_LIBRARIES}
${MAYBE_SHAPEFILE})
add_executable(json-render-bench
EXCLUDE_FROM_ALL
json_render.cpp
$<TARGET_OBJECTS:UTIL>)
target_link_libraries(json-render-bench
osrm
${BOOST_BASE_LIBRARIES}
${CMAKE_THREAD_LIBS_INIT}
${TBB_LIBRARIES}
${MAYBE_SHAPEFILE})
add_executable(alias-bench
EXCLUDE_FROM_ALL
${AliasBenchmarkSources}
@ -41,6 +53,8 @@ target_link_libraries(alias-bench
${TBB_LIBRARIES}
${MAYBE_SHAPEFILE})
add_executable(packedvector-bench
EXCLUDE_FROM_ALL
${PackedVectorBenchmarkSources}
@ -58,4 +72,5 @@ add_custom_target(benchmarks
rtree-bench
packedvector-bench
match-bench
json-render-bench
alias-bench)

128
src/benchmarks/json_render.cpp Normal file
View File

@ -0,0 +1,128 @@
#include "osrm/json_container.hpp"
#include "util/json_container.hpp"
#include "util/json_renderer.hpp"
#include "util/timing_util.hpp"
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <rapidjson/document.h>
#include <sstream>
#include <stdexcept>
using namespace osrm;
namespace
{
void convert(const rapidjson::Value &value, json::Value &result)
{
if (value.IsString())
{
result = json::String{value.GetString()};
}
else if (value.IsNumber())
{
result = json::Number{value.GetDouble()};
}
else if (value.IsObject())
{
json::Object object;
for (auto itr = value.MemberBegin(); itr != value.MemberEnd(); ++itr)
{
json::Value member;
convert(itr->value, member);
object.values.emplace(itr->name.GetString(), std::move(member));
}
result = std::move(object);
}
else if (value.IsArray())
{
json::Array array;
for (auto itr = value.Begin(); itr != value.End(); ++itr)
{
json::Value member;
convert(*itr, member);
array.values.push_back(std::move(member));
}
result = std::move(array);
}
else if (value.IsBool())
{
if (value.GetBool())
{
result = json::True{};
}
else
{
result = json::False{};
}
}
else if (value.IsNull())
{
result = json::Null{};
}
else
{
throw std::runtime_error("unknown type");
}
}
json::Object load(const char *filename)
{
// load file to std string
std::ifstream file(filename);
std::stringstream buffer;
buffer << file.rdbuf();
std::string json = buffer.str();
// load rapidjson document
rapidjson::Document document;
document.Parse(json.c_str());
if (document.HasParseError())
{
throw std::runtime_error("Failed to parse JSON");
}
json::Value result;
convert(document, result);
return result.get<json::Object>();
}
} // namespace
int main(int argc, char **argv)
{
if (argc < 2)
{
std::cerr << "Usage: " << argv[0] << " file.json\n";
return EXIT_FAILURE;
}
const auto obj = load(argv[1]);
TIMER_START(string);
std::string out_str;
json::render(out_str, obj);
TIMER_STOP(string);
std::cout << "String: " << TIMER_MSEC(string) << "ms" << std::endl;
TIMER_START(stringstream);
std::stringstream ss;
json::render(ss, obj);
std::string out_ss_str{ss.str()};
TIMER_STOP(stringstream);
std::cout << "Stringstream: " << TIMER_MSEC(stringstream) << "ms" << std::endl;
TIMER_START(vector);
std::vector<char> out_vec;
json::render(out_vec, obj);
TIMER_STOP(vector);
std::cout << "Vector: " << TIMER_MSEC(vector) << "ms" << std::endl;
if (std::string{out_vec.begin(), out_vec.end()} != out_str || out_str != out_ss_str)
{
throw std::logic_error("Vector/stringstream/string results are not equal");
}
return EXIT_SUCCESS;
}

1
src/benchmarks/portugal_to_korea.json Normal file
View File

File diff suppressed because one or more lines are too long

8
src/nodejs/node_osrm.cpp
View File

@ -167,13 +167,13 @@ inline void async(const Nan::FunctionCallbackInfo<v8::Value> &info,
ParseResult(status, json_result);
if (pluginParams.renderToBuffer)
{
std::ostringstream buf;
osrm::util::json::render(buf, json_result);
result = buf.str();
std::string json_string;
osrm::util::json::render(json_string, json_result);
result = std::move(json_string);
}
else
{
result = json_result;
result = std::move(json_result);
}
}
break;

30
unit_tests/util/json_render.cpp Normal file
View File

@ -0,0 +1,30 @@
#include "util/json_container.hpp"
#include "util/json_renderer.hpp"
#include <boost/test/unit_test.hpp>
#include <iostream>
BOOST_AUTO_TEST_SUITE(json_renderer)
using namespace osrm::util::json;
BOOST_AUTO_TEST_CASE(number_truncating)
{
std::string str;
Renderer<std::string> renderer(str);
// this number would have more than 10 decimals if not truncated
renderer(Number{42.9995999594999399299});
BOOST_CHECK_EQUAL(str, "42.999599959");
}
BOOST_AUTO_TEST_CASE(integer)
{
std::string str;
Renderer<std::string> renderer(str);
renderer(Number{42.0});
BOOST_CHECK_EQUAL(str, "42");
}
BOOST_AUTO_TEST_SUITE_END()

10
unit_tests/util/string_util.cpp
View File

@ -12,13 +12,19 @@ using namespace osrm::util;
BOOST_AUTO_TEST_CASE(json_escaping)
{
std::string input{"\b\\"};
std::string output{escape_JSON(input)};
std::string output;
EscapeJSONString(input, output);
BOOST_CHECK(RequiresJSONStringEscaping(input));
BOOST_CHECK_EQUAL(output, "\\b\\\\");
input = "Aleja \"Solidarnosci\"";
output = escape_JSON(input);
output.clear();
EscapeJSONString(input, output);
BOOST_CHECK(RequiresJSONStringEscaping(input));
BOOST_CHECK_EQUAL(output, "Aleja \\\"Solidarnosci\\\"");
BOOST_CHECK(!RequiresJSONStringEscaping("Aleja Solidarnosci"));
}
BOOST_AUTO_TEST_CASE(print_int)