在Android中使用FlatBuffers（中篇）

提示：扫一扫查出行
【扫一扫了解最新限行尾号】
复制提示

本文来自网易云社区。

FlatBuffers、Protobuf及JSON对比测试

FlatBuffers相对于Protobuf的表现又如何呢？这里我们用数据说话，对比一下FlatBuffers格式、JSON格式与Protobuf的表现。测试同样用fastjson作为JSON的编码解码工具。

测试用的数据结构所有的数据结构，Protobuf相关的测试代码，及JSON的测试代码同在Android中使用Protocol Buffers 一文所述，FlatBuffers的测试代码如上面看到的 AddressBookFlatBuffers 。

通过如下的这段代码来执行测试：

    private class ProtoTestTask extends AsyncTask<Void, Void, Void> {

        private static final int BUFFER_LEN = 8192;

        private void compress(InputStream is, OutputStream os)

                throws Exception {

            GZIPOutputStream gos = new GZIPOutputStream(os);

            int count;

            byte data[] = new byte[BUFFER_LEN];

            while ((count = is.read(data, 0, BUFFER_LEN)) != -1) {

                gos.write(data, 0, count);

            }

            gos.finish();

            gos.close();

        }

        private int getCompressedDataLength(byte[] data) {

            ByteArrayInputStream bais =new ByteArrayInputStream(data);

            ByteArrayOutputStream baos = new ByteArrayOutputStream();      try {

                compress(bais, baos);

            } catch (Exception e) {

            }

            return baos.toByteArray().length;

        }

        private void dumpDataLengthInfo(byte[] protobufData, String jsonData, byte[] flatbufData) {

            int compressedProtobufLength = getCompressedDataLength(protobufData);

            int compressedJSONLength = getCompressedDataLength(jsonData.getBytes());

            int compressedFlatbufLength = getCompressedDataLength(flatbufData);

            Log.i(TAG, String.format("%-120s", "Data length"));

            Log.i(TAG, String.format("%-20s%-20s%-20s%-20s%-20s%-20s", "Protobuf", "Protobuf (GZIP)",

                    "JSON", "JSON (GZIP)", "Flatbuf", "Flatbuf (GZIP)"));

            Log.i(TAG, String.format("%-20s%-20s%-20s%-20s%-20s%-20s",

                    String.valueOf(protobufData.length), compressedProtobufLength,

                    String.valueOf(jsonData.getBytes().length), compressedJSONLength,

                    String.valueOf(flatbufData.length), compressedFlatbufLength));

        }

        private void doEncodeTest(String[] names, int times) {

            long startTime = System.nanoTime();

            byte[] protobufData = AddressBookProtobuf.encodeTest(names, times);

            long protobufTime = System.nanoTime();

            protobufTime = protobufTime - startTime;

            startTime = System.nanoTime();

            String jsonData = AddressBookJson.encodeTest(names, times);

            long jsonTime = System.nanoTime();

            jsonTime = jsonTime - startTime;

            startTime = System.nanoTime();

            byte[] flatbufData = AddressBookFlatBuffers.encodeTest(names, times);

            long flatbufTime = System.nanoTime();

            flatbufTime = flatbufTime - startTime;

            dumpDataLengthInfo(protobufData, jsonData, flatbufData);

            Log.i(TAG, String.format("%-20s%-20s%-20s%-20s", "Encode Times", String.valueOf(times),

                    "Names Length", String.valueOf(names.length)));

            Log.i(TAG, String.format("%-20s%-20s%-20s%-20s%-20s%-20s",

                    "ProtobufTime", String.valueOf(protobufTime),

                    "JsonTime", String.valueOf(jsonTime),

                    "FlatbufTime", String.valueOf(flatbufTime)));

        }

        private void doEncodeTest10(int times) {

            doEncodeTest(TestUtils.sTestNames10, times);

        }

        private void doEncodeTest50(int times) {

            doEncodeTest(TestUtils.sTestNames50, times);

        }

        private void doEncodeTest100(int times) {

            doEncodeTest(TestUtils.sTestNames100, times);

        }

        private void doEncodeTest(int times) {

            doEncodeTest10(times);

            doEncodeTest50(times);

            doEncodeTest100(times);

        }

        private void doDecodeTest(String[] names, int times) {

            byte[] protobufBytes = AddressBookProtobuf.encodeTest(names);

            ByteArrayInputStream bais = new ByteArrayInputStream(protobufBytes);

            long startTime = System.nanoTime();

            AddressBookProtobuf.decodeTest(bais, times);

            long protobufTime = System.nanoTime();

            protobufTime = protobufTime - startTime;

            String jsonStr = AddressBookJson.encodeTest(names);

            startTime = System.nanoTime();

            AddressBookJson.decodeTest(jsonStr, times);

            long jsonTime = System.nanoTime();

            jsonTime = jsonTime - startTime;

            byte[] flatbufData = AddressBookFlatBuffers.encodeTest(names);

            startTime = System.nanoTime();

            AddressBookFlatBuffers.decodeTest(flatbufData, times);

            long flatbufTime = System.nanoTime();

            flatbufTime = flatbufTime - startTime;

            Log.i(TAG, String.format("%-20s%-20s%-20s%-20s", "Decode Times", String.valueOf(times),

                    "Names Length", String.valueOf(names.length)));

            Log.i(TAG, String.format("%-20s%-20s%-20s%-20s%-20s%-20s",

                    "ProtobufTime", String.valueOf(protobufTime),

                    "JsonTime", String.valueOf(jsonTime),

                    "FlatbufTime", String.valueOf(flatbufTime)));

        }

        private void doDecodeTest10(int times) {

            doDecodeTest(TestUtils.sTestNames10, times);

        }

        private void doDecodeTest50(int times) {

            doDecodeTest(TestUtils.sTestNames50, times);

        }

        private void doDecodeTest100(int times) {

            doDecodeTest(TestUtils.sTestNames100, times);

        }

        private void doDecodeTest(int times) {

            doDecodeTest10(times);

            doDecodeTest50(times);

            doDecodeTest100(times);

        }

        @Override

        protected Void doInBackground(Void... params) {

            TestUtils.initTest();

            doEncodeTest(5000);

            doDecodeTest(5000);

            return null;

        }

        @Override

        protected void onPostExecute(Void aVoid) {

            super.onPostExecute(aVoid);

        }

    }

这里我们执行3组编码测试及3组解码测试。对于编码测试，第一组的单个数据中包含10个Person，第二组的包含50个，第三组的包含100个，然后对每个数据分别执行5000次的编码操作。

对于解码测试，三组中单个数据同样包含10个Person、50个及100个，然后对每个数据分别执行5000次的解码码操作。

在Galaxy Nexus的Android 4.4.4 CM平台上执行上述测试，最终得到如下结果：

编码后数据长度对比 (Bytes)

Person个数	Protobuf	Protobuf（GZIP）	JSON	JSON（GZIP）	Flatbuf	Flatbuf（GZIP)
10	860	288	1703	343	1532	513
50	4300	986	8463	1048	7452	1814
100	8600	1841	16913	1918	14852	3416

相同的数据，经过编码，在压缩前JSON的数据最长，FlatBuffers的数据长度与JSON的短大概10 ％，而Protobuf的数据长度则大概只有JSON的一半。而在用GZIP压缩后，Protobuf的数据长度与JSON的接近，FlatBuffers的数据长度则接近两者的两倍。

编码性能对比 (S)

Person个数	Protobuf	JSON	FlatBuffers
10	6.000	8.952	12.464
50	26.847	45.782	56.752
100	50.602	73.688	108.426

编码性能Protobuf相对于JSON有较大幅度的提高，而FlatBuffers则有较大幅度的降低。

解码性能对比 (S)

Person个数	Protobuf	JSON	FlatBuffers
10	0.255	10.766	0.014
50	0.245	51.134	0.014
100	0.323	101.070	0.006

解码性能方面，Protobuf相对于JSON，有着惊人的提升。Protobuf的解码时间几乎不随着数据长度的增长而有太大的增长，而JSON则随着数据长度的增加，解码所需要的时间也越来越长。而FlatBuffers则由于无需解码，在性能方面相对于前两者更有着非常大的提升。

FlatBuffers 编码原理

FlatBuffers的Java库只提供了如下的4个类：

./com/google/flatbuffers/Constants.java

./com/google/flatbuffers/FlatBufferBuilder.java

./com/google/flatbuffers/Struct.java

./com/google/flatbuffers/Table.java

Constants 类定义FlatBuffers中可用的基本原始数据类型的长度：

public class Constants {

    // Java doesn't seem to have these.

    /** The number of bytes in an `byte`. */

    static final int SIZEOF_BYTE = 1;

    /** The number of bytes in a `short`. */

    static final int SIZEOF_SHORT = 2;

    /** The number of bytes in an `int`. */

    static final int SIZEOF_INT = 4;

    /** The number of bytes in an `float`. */

    static final int SIZEOF_FLOAT = 4;

    /** The number of bytes in an `long`. */

    static final int SIZEOF_LONG = 8;

    /** The number of bytes in an `double`. */

    static final int SIZEOF_DOUBLE = 8;

    /** The number of bytes in a file identifier. */

    static final int FILE_IDENTIFIER_LENGTH = 4;

}

FlatBufferBuilder 用于FlatBuffers编码，它会将我们的结构化数据序列化为字节数组。我们借助于 FlatBufferBuilder 在 ByteBuffer 中放置基本数据类型的数据、数组、字符串及对象。ByteBuffer 用于处理字节序，在序列化时，它将数据按适当的字节序进行序列化，在发序列化时，它将多个字节转换为适当的数据类型。在 .fbs 文件中定义的 table 和 struct，为它们生成的Java 类会继承 Table 和 Struct 。

在反序列化时，输入的ByteBuffer数据被当作字节数组，Table提供了针对字节数组的操作，生成的Java类负责对这些数据进行解释。对于FlatBuffers编码的数据，无需进行解码，只需进行解释。在编译 .fbs 文件时，每个字段在这段数据中的位置将被确定。每个字段的类型及长度将被硬编码进生成的Java类。

Struct 类的代码也比较简洁：

package com.google.flatbuffers;

import java.nio.ByteBuffer;

/// @cond FLATBUFFERS_INTERNAL

/**

 * All structs in the generated code derive from this class, and add their own accessors.

 */

public class Struct {

  /** Used to hold the position of the `bb` buffer. */

  protected int bb_pos;

  /** The underlying ByteBuffer to hold the data of the Struct. */

  protected ByteBuffer bb;

}

整体的结构如下图：

在序列化结构化数据时，我们首先需要创建一个 FlatBufferBuilder ，在这个对象的创建过程中会分配或从调用者那里获取 ByteBuffer ，序列化的数据将保存在这个 ByteBuffer 中：

   /**

    * Start with a buffer of size `initial_size`, then grow as required.

    *

    * @param initial_size The initial size of the internal buffer to use.

    */

    public FlatBufferBuilder(int initial_size) {

        if (initial_size <= 0) initial_size = 1;

        space = initial_size;

        bb = newByteBuffer(initial_size);

    }

   /**

    * Start with a buffer of 1KiB, then grow as required.

    */

    public FlatBufferBuilder() {

        this(1024);

    }

    /**

     * Alternative constructor allowing reuse of {@link ByteBuffer}s.  The builder

     * can still grow the buffer as necessary.  User classes should make sure

     * to call {@link #dataBuffer()} to obtain the resulting encoded message.

     *

     * @param existing_bb The byte buffer to reuse.

     */

    public FlatBufferBuilder(ByteBuffer existing_bb) {

        init(existing_bb);

    }

    /**

     * Alternative initializer that allows reusing this object on an existing

     * `ByteBuffer`. This method resets the builder's internal state, but keeps

     * objects that have been allocated for temporary storage.

     *

     * @param existing_bb The byte buffer to reuse.

     * @return Returns `this`.

     */

    public FlatBufferBuilder init(ByteBuffer existing_bb){

        bb = existing_bb;

        bb.clear();

        bb.order(ByteOrder.LITTLE_ENDIAN);

        minalign = 1;

        space = bb.capacity();

        vtable_in_use = 0;

        nested = false;

        finished = false;

        object_start = 0;

        num_vtables = 0;

        vector_num_elems = 0;

        return this;

    }

    static ByteBuffer newByteBuffer(int capacity) {

        ByteBuffer newbb = ByteBuffer.allocate(capacity);

        newbb.order(ByteOrder.LITTLE_ENDIAN);

        return newbb;

    }

下面我们更详细地分析基本数据类型数据、数组及对象的序列化过程。ByteBuffer 为小尾端的。

FlatBuffers编码基本数据类型

FlatBuffer 的基本数据类型主要包括如下这些：

Boolean

Byte

Short

Int

Long

Float

Double

FlatBufferBuilder 提供了三组方法用于操作这些数据：

    public void putBoolean(boolean x);

    public void putByte   (byte    x);

    public void putShort  (short   x);

    public void putInt    (int     x);

    public void putLong   (long    x);

    public void putFloat  (float   x);

    public void putDouble (double  x);

    public void addBoolean(boolean x);

    public void addByte   (byte    x);

    public void addShort  (short   x);

    public void addInt    (int     x);

    public void addLong   (long    x);

    public void addFloat  (float   x);

    public void addDouble (double  x);

    public void addBoolean(int o, boolean x, boolean d);

    public void addByte(int o, byte x, int d);

    public void addShort(int o, short x, int d);

    public void addInt    (int o, int     x, int     d);

    public void addLong   (int o, long    x, long    d);

    public void addFloat  (int o, float   x, double  d);

    public void addDouble (int o, double  x, double  d);

putXXX 那一组，直接地将一个数据放入 ByteBuffer 中，它们的实现基本如下面这样：

    public void putBoolean(boolean x) {

        bb.put(space -= Constants.SIZEOF_BYTE, (byte) (x ? 1 : 0));

    }

    public void putByte(byte x) {

        bb.put(space -= Constants.SIZEOF_BYTE, x);

    }

    public void putShort(short x) {

        bb.putShort(space -= Constants.SIZEOF_SHORT, x);

    }

Boolean值会被先转为byte类型再放入 ByteBuffer。另外一点值得注意的是，数据是从 ByteBuffer 的结尾处开始放置的，space用于记录最近放入的数据的位置及剩余的空间。

addXXX(XXX x) 那一组在放入数据之前会先做对齐处理，并在需要时扩展 ByteBuffer 的容量：

    static ByteBuffer growByteBuffer(ByteBuffer bb) {

        int old_buf_size = bb.capacity();

        if ((old_buf_size & 0xC0000000) != 0)  // Ensure we don't grow beyond what fits in an int.

            throw new AssertionError("FlatBuffers: cannot grow buffer beyond 2 gigabytes.");

        int new_buf_size = old_buf_size << 1;

        bb.position(0);

        ByteBuffer nbb = newByteBuffer(new_buf_size);

        nbb.position(new_buf_size - old_buf_size);

        nbb.put(bb);

        return nbb;

    }

   public void pad(int byte_size) {

       for (int i = 0; i < byte_size; i++) bb.put(--space, (byte) 0);

   }

    public void prep(int size, int additional_bytes) {

        // Track the biggest thing we've ever aligned to.

        if (size > minalign) minalign = size;

        // Find the amount of alignment needed such that `size` is properly

        // aligned after `additional_bytes`

        int align_size = ((~(bb.capacity() - space + additional_bytes)) + 1) & (size - 1);

        // Reallocate the buffer if needed.

        while (space < align_size + size + additional_bytes) {

            int old_buf_size = bb.capacity();

            bb = growByteBuffer(bb);

            space += bb.capacity() - old_buf_size;

        }

        pad(align_size);

    }

    public void addBoolean(boolean x) {

        prep(Constants.SIZEOF_BYTE, 0);

        putBoolean(x);

    }

    public void addInt(int x) {

        prep(Constants.SIZEOF_INT, 0);

        putInt(x);

    }

对齐是数据存放的起始位置相对于ByteBuffer的结束位置的对齐，additional bytes被认为是不需要对齐的，且在必要的时候会在ByteBuffer可用空间的结尾处填充值为0的字节。在扩展 ByteBuffer 的空间时，老的ByteBuffer被放在新ByteBuffer的结尾处。

addXXX(int o, XXX x, YYY y) 这一组方法在放入数据之后，会将 vtable 中对应位置的值更新为最近放入的数据的offset。

    public void addShort(int o, short x, int d) {

        if (force_defaults || x != d) {

            addShort(x);

            slot(o);

        }

    }

    public void slot(int voffset) {

        vtable[voffset] = offset();

    }

后面我们在分析编码对象时再来详细地了解vtable。

基本上，在我们的应用程序代码中不要直接调用这些方法，它们主要在构造对象时用于存储对象的基本数据类型字段。