Bump flatbuffers to v24.3.25 version (#6968)

2024-07-10 20:44:36 +02:00
parent 7436835244
commit 825132eec7
737 changed files with 50923 additions and 99849 deletions
@@ -1,3 +1,9 @@
+local compat = require("flatbuffers.compat")
+-- locals for slightly faster access
+local string_pack = compat.string_pack
+local string_unpack = compat.string_unpack
+
+
 local m = {} -- the module table

 local mt = {} -- the module metatable
@@ -44,7 +50,10 @@ function mt:Slice(startPos, endPos)
        -- updated the startPos based on the size of the
        -- value
        while startPos < endPos do
-            local v = d[startPos] or '/0'
+            local v = d[startPos]
+            if not v or v == "" then
+                v = '/0'
+            end
            table.insert(b, v)
            startPos = startPos + #v
        end
@@ -104,19 +113,15 @@ function mt:Set(value, position)
    self.data[position] = value
 end

-- locals for slightly faster access
-local sunpack = string.unpack
-local spack = string.pack
-
 -- Pack the data into a binary representation
 function m.Pack(fmt, ...)
-    return spack(fmt, ...)
+    return string_pack(fmt, ...)
 end

 -- Unpack the data from a binary representation in
 -- a Lua value
 function m.Unpack(fmt, s, pos)
-    return sunpack(fmt, s.str, pos + 1)
+    return string_unpack(fmt, s.str, pos + 1)
 end

 -- Return the binary array module
@@ -1,6 +1,7 @@
 local N = require("flatbuffers.numTypes")
 local ba = require("flatbuffers.binaryarray")
 local compat = require("flatbuffers.compat")
+local string_unpack = compat.string_unpack

 local m = {}

@@ -29,12 +30,12 @@ local getAlignSize = compat.GetAlignSize

 local function vtableEqual(a, objectStart, b)
    UOffsetT:EnforceNumber(objectStart)
-    if (#a * VOffsetT.bytewidth) ~= #b then
+    if (#a * 2) ~= #b then
        return false
    end

    for i, elem in ipairs(a) do
-        local x = string.unpack(VOffsetT.packFmt, b, 1 + (i - 1) * VOffsetT.bytewidth)
+        local x = string_unpack(VOffsetT.packFmt, b, 1 + (i - 1) * 2)
        if x ~= 0 or elem ~= 0 then
            local y = objectStart - elem
            if x ~= y then
@@ -60,6 +61,23 @@ function m.New(initialSize)
    return o
 end

+-- Clears the builder and resets the state. It does not actually clear the backing binary array, it just reuses it as
+-- needed. This is a performant way to use the builder for multiple constructions without the overhead of multiple
+-- builder allocations.
+function mt:Clear()
+    self.finished = false
+    self.nested = false
+    self.minalign = 1
+    self.currentVTable = nil
+    self.objectEnd = nil
+    self.head = self.bytes.size -- place the head at the end of the binary array
+
+    -- clear vtables instead of making a new table
+    local vtable = self.vtables
+    local vtableCount = #vtable
+    for i=1,vtableCount do vtable[i] = nil end
+end
+
 function mt:Output(full)
    assert(self.finished, "Builder Not Finished")
    if full then
@@ -100,11 +118,11 @@ function mt:WriteVtable()
    while i >= 1 do

        local vt2Offset = self.vtables[i]
-        local vt2Start = #self.bytes - vt2Offset
+        local vt2Start = self.bytes.size - vt2Offset
        local vt2lenstr = self.bytes:Slice(vt2Start, vt2Start+1)
-        local vt2Len = string.unpack(VOffsetT.packFmt, vt2lenstr, 1)
+        local vt2Len = string_unpack(VOffsetT.packFmt, vt2lenstr, 1)

-        local metadata = VtableMetadataFields * VOffsetT.bytewidth
+        local metadata = VtableMetadataFields * 2
        local vt2End = vt2Start + vt2Len
        local vt2 = self.bytes:Slice(vt2Start+metadata,vt2End)

@@ -133,15 +151,15 @@ function mt:WriteVtable()
        self:PrependVOffsetT(objectSize)

        local vBytes = #self.currentVTable + VtableMetadataFields
-        vBytes = vBytes * VOffsetT.bytewidth
+        vBytes = vBytes * 2
        self:PrependVOffsetT(vBytes)

-        local objectStart = #self.bytes - objectOffset
+        local objectStart = self.bytes.size - objectOffset
        self.bytes:Set(SOffsetT:Pack(self:Offset() - objectOffset),objectStart)

        table.insert(self.vtables, self:Offset())
    else
-        local objectStart = #self.bytes - objectOffset
+        local objectStart = self.bytes.size - objectOffset
        self.head = objectStart
        self.bytes:Set(SOffsetT:Pack(exisitingVTable - objectOffset),self.head)
    end
@@ -157,7 +175,7 @@ function mt:EndObject()
 end

 local function growByteBuffer(self, desiredSize)
-    local s = #self.bytes
+    local s = self.bytes.size
    assert(s < MAX_BUFFER_SIZE, "Flat Buffers cannot grow buffer beyond 2 gigabytes")
    local newsize = s
    repeat
@@ -173,7 +191,7 @@ function mt:Head()
 end

 function mt:Offset()
-   return #self.bytes - self.head
+   return self.bytes.size - self.head
 end

 function mt:Pad(n)
@@ -192,15 +210,15 @@ function mt:Prep(size, additionalBytes)

    local h = self.head

-    local k = #self.bytes - h + additionalBytes
-    local alignsize = ((~k) + 1) & (size - 1) -- getAlignSize(k, size)
+    local k = self.bytes.size - h + additionalBytes
+    local alignsize = getAlignSize(k, size)

    local desiredSize = alignsize + size + additionalBytes

    while self.head < desiredSize do
-        local oldBufSize = #self.bytes
+        local oldBufSize = self.bytes.size
        growByteBuffer(self, desiredSize)
-        local updatedHead = self.head + #self.bytes - oldBufSize
+        local updatedHead = self.head + self.bytes.size - oldBufSize
        self.head = updatedHead
    end

@@ -208,17 +226,17 @@ function mt:Prep(size, additionalBytes)
 end

 function mt:PrependSOffsetTRelative(off)
-    self:Prep(SOffsetT.bytewidth, 0)
+    self:Prep(4, 0)
    assert(off <= self:Offset(), "Offset arithmetic error")
-    local off2 = self:Offset() - off + SOffsetT.bytewidth
+    local off2 = self:Offset() - off + 4
    self:Place(off2, SOffsetT)
 end

 function mt:PrependUOffsetTRelative(off)
-    self:Prep(UOffsetT.bytewidth, 0)
+    self:Prep(4, 0)
    local soffset = self:Offset()
    if off <= soffset then
-        local off2 = soffset - off + UOffsetT.bytewidth
+        local off2 = soffset - off + 4
        self:Place(off2, UOffsetT)
    else
        error("Offset arithmetic error")
@@ -228,8 +246,9 @@ end
 function mt:StartVector(elemSize, numElements, alignment)
    assert(not self.nested)
    self.nested = true
-    self:Prep(Uint32.bytewidth, elemSize * numElements)
-    self:Prep(alignment, elemSize * numElements)
+    local elementSize = elemSize * numElements
+    self:Prep(4, elementSize) -- Uint32 length
+    self:Prep(alignment, elementSize)
    return self:Offset()
 end

@@ -246,7 +265,7 @@ function mt:CreateString(s)

    assert(type(s) == "string")

-    self:Prep(UOffsetT.bytewidth, (#s + 1)*Uint8.bytewidth)
+    self:Prep(4, #s + 1)
    self:Place(0, Uint8)

    local l = #s
@@ -254,20 +273,21 @@ function mt:CreateString(s)

    self.bytes:Set(s, self.head, self.head + l)

-    return self:EndVector(#s)
+    return self:EndVector(l)
 end

 function mt:CreateByteVector(x)
    assert(not self.nested)
    self.nested = true
-    self:Prep(UOffsetT.bytewidth, #x*Uint8.bytewidth)

    local l = #x
+    self:Prep(4, l)
+
    self.head = self.head - l

    self.bytes:Set(x, self.head, self.head + l)

-    return self:EndVector(#x)
+    return self:EndVector(l)
 end

 function mt:Slot(slotnum)
@@ -278,15 +298,10 @@ end

 local function finish(self, rootTable, sizePrefix)
    UOffsetT:EnforceNumber(rootTable)
-    local prepSize = UOffsetT.bytewidth
-    if sizePrefix then
-        prepSize = prepSize + Int32.bytewidth
-    end
-
-    self:Prep(self.minalign, prepSize)
+    self:Prep(self.minalign, sizePrefix and 8 or 4)
    self:PrependUOffsetTRelative(rootTable)
    if sizePrefix then
-        local size = #self.bytes - self.head
+        local size = self.bytes.size - self.head
        Int32:EnforceNumber(size)
        self:PrependInt32(size)
    end
@@ -308,8 +323,9 @@ function mt:Prepend(flags, off)
 end

 function mt:PrependSlot(flags, o, x, d)
-    flags:EnforceNumber(x)
-    flags:EnforceNumber(d)
+    flags:EnforceNumbers(x,d)
+--    flags:EnforceNumber(x)
+--    flags:EnforceNumber(d)
    if x ~= d then
        self:Prepend(flags, x)
        self:Slot(o)
@@ -1,17 +1,15 @@
-local m = {}
-
-local getAlignSize
-if _VERSION == "Lua 5.3" then
-    getAlignSize = function(k, size)
-            return ((~k) + 1) & (size - 1)
-        end    
-else
-    getAlignSize = function(self, size, additionalBytes)        
-        local alignsize = bit32.bnot(#self.bytes-self:Head() + additionalBytes) + 1
-        return bit32.band(alignsize,(size - 1))
-    end
-end
-    
-m.GetAlignSize = getAlignSize
-
-return m
+local compats = {
+    ["Lua 5.1"] = function()  
+        -- Check if Lua JIT is installed first
+        local ok = pcall(require, "jit")
+        if not ok then
+            return require("flatbuffers.compat_5_1")
+        else
+            return require("flatbuffers.compat_luajit")
+        end
+    end,
+    ["Lua 5.2"] = function() return require("flatbuffers.compat_5_1") end,
+    ["Lua 5.3"] = function() return require("flatbuffers.compat_5_3") end,
+    ["Lua 5.4"] = function() return require("flatbuffers.compat_5_3") end,
+}
+return assert(compats[_VERSION], "Unsupported Lua Version: ".._VERSION)()
@@ -0,0 +1,21 @@
+local m = {}
+local ok, bit = pcall(require, "bit32")
+assert(ok, "The Bit32 library must be installed")
+assert(pcall(require, "compat53"), "The Compat 5.3 library must be installed")
+
+m.GetAlignSize = function(k, size)
+    return bit.band(bit.bnot(k) + 1,(size - 1))
+end
+
+if not table.unpack then
+    table.unpack = unpack
+end
+
+if not table.pack then
+    table.pack = pack
+end
+
+m.string_pack = string.pack
+m.string_unpack = string.unpack
+
+return m
@@ -0,0 +1,14 @@
+-- We need to put it into a separate file to avoid syntax error like `unexpected symbol near '~'`
+local m = {}
+
+
+m.GetAlignSize = function(k, size)
+    return ((~k) + 1) & (size - 1)
+end
+
+
+m.string_pack = string.pack
+m.string_unpack = string.unpack
+
+
+return m
@@ -0,0 +1,213 @@
+local bit = require("bit")
+local ffi = require("ffi")
+local band = bit.band
+local bnot = bit.bnot
+
+
+local m = {}
+local Uint8Bound = 256 -- bound is the max uintN + 1
+local Uint16Bound = 65536
+local Uint32Bound = 4294967296
+
+if not table.unpack then
+    table.unpack = unpack
+end
+
+if not table.pack then
+    table.pack = pack
+end
+
+m.GetAlignSize = function(k, size)
+    return band((bnot(k) + 1), (size - 1))
+end
+
+
+local function pack_I1(n)
+    return string.char(n)
+end
+local function pack_i1(n)
+    if n < 0 then
+        n = Uint8Bound + n
+    end
+    return pack_I1(n)
+end
+
+local function unpack_I1(n, pos)
+    return string.byte(n, pos)
+end
+local function unpack_i1(n, pos)
+    local res = unpack_I1(n, pos)
+    if res >= Uint8Bound / 2 then
+        return res - Uint8Bound
+    end
+    return res
+end
+
+local b2 = ffi.new("unsigned char[2]")
+local function pack_I2(n)
+    for i = 0, 1 do
+        b2[i] = bit.band(n, 255)
+        n = bit.rshift(n, 8)
+    end
+    return ffi.string(b2, 2)
+end
+local function pack_i2(n)
+    if n < 0 then
+        n = Uint16Bound + n
+    end
+    return pack_I2(n)
+end
+
+local function unpack_I2(n, pos)
+    local a, b = string.byte(n, pos, pos + 1)
+    return b * Uint8Bound + a
+end
+local function unpack_i2(n, pos)
+    local res = unpack_I2(n, pos)
+    if res >= Uint16Bound / 2 then
+        return res - Uint16Bound
+    end
+    return res
+end
+
+local b4 = ffi.new("unsigned char[4]")
+local function pack_I4(n)
+    for i = 0, 3 do
+        b4[i] = bit.band(n, 255)
+        n = bit.rshift(n, 8)
+    end
+    return ffi.string(b4, 4)
+end
+local function pack_i4(n)
+    if n < 0 then
+        n = Uint32Bound + n
+    end
+    return pack_I4(n)
+end
+
+local function unpack_I4(n, pos)
+    local a, b, c, d = string.byte(n, pos, pos + 3)
+    return Uint8Bound * (Uint8Bound * ((Uint8Bound * d) + c) + b) + a
+end
+local function unpack_i4(n, pos)
+    local res = unpack_I4(n, pos)
+    if res >= Uint32Bound / 2 then
+        return res - Uint32Bound
+    end
+    return res
+end
+
+local b8 = ffi.new("unsigned char[8]")
+local function pack_I8(n)
+    n = ffi.cast("unsigned long long", n)
+    local hi = math.floor(tonumber(n / Uint32Bound))
+    local li = n % Uint32Bound
+    for i = 0, 3 do
+        b8[i] = bit.band(li, 255)
+        li = bit.rshift(li, 8)
+    end
+    for i = 4, 7 do
+        b8[i] = bit.band(hi, 255)
+        hi = bit.rshift(hi, 8)
+    end
+    return ffi.string(b8, 8)
+end
+local function pack_i8(n)
+    n = ffi.cast("signed long long", n)
+    return pack_I8(n)
+end
+
+local function unpack_I8(n, pos)
+    local a, b, c, d = string.byte(n, pos, pos + 3)
+    local li = Uint8Bound * (Uint8Bound * ((Uint8Bound * d) + c) + b) + a
+    local a, b, c, d = string.byte(n, pos + 4, pos + 7)
+    local hi = Uint8Bound * (Uint8Bound * ((Uint8Bound * d) + c) + b) + a
+    return ffi.cast("unsigned long long", hi) * Uint32Bound + li
+end
+local function unpack_i8(n, pos)
+    local res = unpack_I8(n, pos)
+    return ffi.cast("signed long long", res)
+end
+
+local bf = ffi.new("float[1]")
+local function pack_f(n)
+    bf[0] = n
+    return ffi.string(bf, 4)
+end
+
+local function unpack_f(n, pos)
+    ffi.copy(bf, ffi.cast("char *", n) + pos - 1, 4)
+    return tonumber(bf[0])
+end
+
+local bd = ffi.new("double[1]")
+local function pack_d(n)
+    bd[0] = n
+    return ffi.string(bd, 8)
+end
+
+local function unpack_d(n, pos)
+    ffi.copy(bd, ffi.cast("char *", n) + pos - 1, 8)
+    return tonumber(bd[0])
+end
+
+
+m.string_pack = function(fmt, i, ...)
+    if fmt == "<I1" then
+        return pack_I1(i)
+    elseif fmt == "<I2" then
+        return pack_I2(i)
+    elseif fmt == "<I4" then
+        return pack_I4(i)
+    elseif fmt == "<I8" then
+        return pack_I8(i)
+    elseif fmt == "<i1" then
+        return pack_i1(i)
+    elseif fmt == "<i2" then
+        return pack_i2(i)
+    elseif fmt == "<i4" then
+        return pack_i4(i)
+    elseif fmt == "<i8" then
+        return pack_i8(i)
+    elseif fmt == "<f" then
+        return pack_f(i)
+    elseif fmt == "<d" then
+        return pack_d(i)
+    else
+        error(string.format("FIXME: support fmt %s", fmt))
+    end
+end
+
+
+m.string_unpack = function(fmt, s, pos)
+    if not pos then
+        pos = 1
+    end
+
+    if fmt == "<I1" then
+        return unpack_I1(s, pos)
+    elseif fmt == "<I2" then
+        return unpack_I2(s, pos)
+    elseif fmt == "<I4" then
+        return unpack_I4(s, pos)
+    elseif fmt == "<I8" then
+        return unpack_I8(s, pos)
+    elseif fmt == "<i1" then
+        return unpack_i1(s, pos)
+    elseif fmt == "<i2" then
+        return unpack_i2(s, pos)
+    elseif fmt == "<i4" then
+        return unpack_i4(s, pos)
+    elseif fmt == "<i8" then
+        return unpack_i8(s, pos)
+    elseif fmt == "<f" then
+        return unpack_f(s, pos)
+    elseif fmt == "<d" then
+        return unpack_d(s, pos)
+    else
+        error(string.format("FIXME: support fmt %s", fmt))
+    end
+end
+
+
+return m
@@ -34,6 +34,20 @@ function type_mt:EnforceNumber(n)
    error("Number is not in the valid range") 
 end

+function type_mt:EnforceNumbers(a,b)
+   -- duplicate code since the overhead of function calls
+    -- for such a popular method is time consuming
+    if not self.min_value and not self.max_value then
+        return
+    end
+
+    if self.min_value <= a and a <= self.max_value and self.min_value <= b and b <= self.max_value then
+        return
+    end
+
+    error("Number is not in the valid range")
+end
+
 function type_mt:EnforceNumberAndPack(n)
    return bpack(self.packFmt, n)    
 end
@@ -58,7 +72,8 @@ local bool_mt =
    Unpack = function(self, buf, pos) return buf[pos] == "1" end,
    ValidNumber = function(self, n) return true end, -- anything is a valid boolean in Lua
    EnforceNumber = function(self, n) end, -- anything is a valid boolean in Lua
-    EnforceNumberAndPack = function(self, n) return self:Pack(value) end,
+    EnforceNumbers = function(self, a, b) end, -- anything is a valid boolean in Lua
+    EnforceNumberAndPack = function(self, n) return self:Pack(n) end,
 }

 local uint8_mt = 
@@ -1,3 +1,7 @@
+local compat = require("flatbuffers.compat")
+local string_unpack = compat.string_unpack
+
+
 local m = {}
 local mt = {}

@@ -6,69 +10,93 @@ local mt_name = "flatbuffers.view.mt"
 local N = require("flatbuffers.numTypes")
 local binaryarray = require("flatbuffers.binaryarray")

+local function enforceOffset(off)
+    if off < 0 or off > 42949672951 then
+        error("Offset is not valid")
+    end
+end
+
+local function unPackUoffset(bytes, off)
+    return string_unpack("<I4", bytes.str, off + 1)
+end
+
+local function unPackVoffset(bytes, off)
+    return string_unpack("<I2", bytes.str, off + 1)
+end
+
 function m.New(buf, pos)
-    N.UOffsetT:EnforceNumber(pos)
-    
+    enforceOffset(pos)
    -- need to convert from a string buffer into
    -- a binary array

    local o = {
        bytes = type(buf) == "string" and binaryarray.New(buf) or buf,
-        pos = pos
+        pos = pos,
    }
    setmetatable(o, {__index = mt, __metatable = mt_name})
    return o
 end

 function mt:Offset(vtableOffset)
-    local vtable = self.pos - self:Get(N.SOffsetT, self.pos)
-    local vtableEnd = self:Get(N.VOffsetT, vtable)
-    if vtableOffset < vtableEnd then
-        return self:Get(N.VOffsetT, vtable + vtableOffset)
+    local vtable = self.vtable
+    if not vtable then
+        vtable = self.pos - self:Get(N.SOffsetT, self.pos)
+        self.vtable = vtable
+        self.vtableEnd = self:Get(N.VOffsetT, vtable)
+    end
+    if vtableOffset < self.vtableEnd then
+        return unPackVoffset(self.bytes, vtable + vtableOffset)
    end
    return 0
 end

 function mt:Indirect(off)
-    N.UOffsetT:EnforceNumber(off)
-    return off + N.UOffsetT:Unpack(self.bytes, off)
+    enforceOffset(off)
+    return off + unPackUoffset(self.bytes, off)
 end

 function mt:String(off)
-    N.UOffsetT:EnforceNumber(off)
-    off = off + N.UOffsetT:Unpack(self.bytes, off)
-    local start = off + N.UOffsetT.bytewidth
-    local length = N.UOffsetT:Unpack(self.bytes, off)
+    enforceOffset(off)
+    off = off + unPackUoffset(self.bytes, off)
+    local start = off + 4
+    local length = unPackUoffset(self.bytes, off)
    return self.bytes:Slice(start, start+length)
 end

 function mt:VectorLen(off)
-    N.UOffsetT:EnforceNumber(off)
+    enforceOffset(off)
    off = off + self.pos
-    off = off + N.UOffsetT:Unpack(self.bytes, off)
-    return N.UOffsetT:Unpack(self.bytes, off)
+    off = off + unPackUoffset(self.bytes, off)
+    return unPackUoffset(self.bytes, off)
 end

 function mt:Vector(off)
-    N.UOffsetT:EnforceNumber(off)
-    
+    enforceOffset(off)
    off = off + self.pos
-    local x = off + self:Get(N.UOffsetT, off)
-    x = x + N.UOffsetT.bytewidth
-    return x
+    return off + self:Get(N.UOffsetT, off) + 4
+end
+
+function mt:VectorAsString(off, start, stop)
+    local o = self:Offset(off)
+    if o ~= 0 then
+        start = start or 1
+        stop = stop or self:VectorLen(o)
+        local a = self:Vector(o) + start - 1
+        return self.bytes:Slice(a, a + stop - start + 1)
+    end
+    return nil
 end

 function mt:Union(t2, off)
    assert(getmetatable(t2) == mt_name)
-    N.UOffsetT:EnforceNumber(off)
-    
+    enforceOffset(off)
    off = off + self.pos
    t2.pos = off + self:Get(N.UOffsetT, off)
    t2.bytes = self.bytes
 end

 function mt:Get(flags, off)
-    N.UOffsetT:EnforceNumber(off)
+    enforceOffset(off)
    return flags:Unpack(self.bytes, off)
 end

@@ -85,8 +113,7 @@ function mt:GetSlot(slot, d, validatorFlags)
 end

 function mt:GetVOffsetTSlot(slot, d)
-    N.VOffsetT:EnforceNumber(slot)
-    N.VOffsetT:EnforceNumber(d)
+    N.VOffsetT:EnforceNumbers(slot, d)
    local off = self:Offset(slot)
    if off == 0 then
        return d