Skip to content

Commit 7ff77d6

Browse files
alecthomasalecthomas
committed
General purpose binary encoder/decoder.
0 parents  commit 7ff77d6

3 files changed

Lines changed: 359 additions & 0 deletions

File tree

‎README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Compact binary encoding for Go
2+
3+
The Go standard library package
4+
[encoding/binary](http://golang.org/pkg/encoding/binary/) provides
5+
encoding/decoding of *fixed-size* Go values or slices of same. This package
6+
extends support to arbitrary, variable-sized values by prefixing these values
7+
with their varint-encoded size, recursively. It expects the encoded type and
8+
decoded type to match exactly and makes no attempt to reconcile or check for
9+
any differences.

‎binary.go‎

Lines changed: 243 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,243 @@
1+
package binary
2+
3+
import (
4+
"bufio"
5+
"bytes"
6+
"encoding"
7+
"encoding/binary"
8+
"errors"
9+
"fmt"
10+
"io"
11+
"reflect"
12+
)
13+
14+
var (
15+
DefaultEndian = binary.LittleEndian
16+
)
17+
18+
func Marshal(v interface{}) ([]byte, error) {
19+
b := &bytes.Buffer{}
20+
if err := NewEncoder(b).Encode(v); err != nil {
21+
return nil, err
22+
}
23+
return b.Bytes(), nil
24+
}
25+
26+
func Unmarshal(b []byte, v interface{}) error {
27+
return NewDecoder(bytes.NewReader(b)).Decode(v)
28+
}
29+
30+
type Encoder struct {
31+
w io.Writer
32+
}
33+
34+
func NewEncoder(w io.Writer) *Encoder {
35+
return &Encoder{w}
36+
}
37+
38+
func (e *Encoder) writeVarint(v int) error {
39+
b := make([]byte, 8)
40+
l := binary.PutUvarint(b, uint64(v))
41+
b = b[:l]
42+
_, err := e.w.Write(b)
43+
return err
44+
}
45+
46+
func (b *Encoder) Encode(v interface{}) (err error) {
47+
switch cv := v.(type) {
48+
case encoding.BinaryMarshaler:
49+
buf, err := cv.MarshalBinary()
50+
if err != nil {
51+
return err
52+
}
53+
if err = b.writeVarint(len(buf)); err != nil {
54+
return err
55+
}
56+
_, err = b.w.Write(buf)
57+
58+
case []byte: // fast-path byte arrays
59+
if err = b.writeVarint(len(cv)); err != nil {
60+
return
61+
}
62+
_, err = b.w.Write(cv)
63+
64+
case string:
65+
if err = b.writeVarint(len(cv)); err != nil {
66+
return
67+
}
68+
_, err = b.w.Write([]byte(cv))
69+
70+
case bool:
71+
var out byte
72+
if cv {
73+
out = 1
74+
}
75+
err = binary.Write(b.w, DefaultEndian, out)
76+
77+
case int:
78+
err = binary.Write(b.w, DefaultEndian, int64(cv))
79+
80+
case uint:
81+
err = binary.Write(b.w, DefaultEndian, int64(cv))
82+
83+
case int8, uint8, int16, uint16, int32, uint32, int64, uint64, float32,
84+
float64, complex64, complex128:
85+
err = binary.Write(b.w, DefaultEndian, v)
86+
87+
default:
88+
rv := reflect.Indirect(reflect.ValueOf(v))
89+
t := rv.Type()
90+
switch t.Kind() {
91+
case reflect.Array, reflect.Slice:
92+
l := rv.Len()
93+
if err = b.writeVarint(l); err != nil {
94+
return
95+
}
96+
for i := 0; i < l; i++ {
97+
if err = b.Encode(rv.Index(i).Interface()); err != nil {
98+
return
99+
}
100+
}
101+
102+
case reflect.Struct:
103+
l := rv.NumField()
104+
for i := 0; i < l; i++ {
105+
if v := rv.Field(i); v.CanSet() && t.Field(i).Name != "_" {
106+
if err = b.Encode(v.Interface()); err != nil {
107+
return
108+
}
109+
}
110+
}
111+
112+
case reflect.Map:
113+
l := rv.Len()
114+
if err = b.writeVarint(l); err != nil {
115+
return
116+
}
117+
for _, key := range rv.MapKeys() {
118+
value := rv.MapIndex(key)
119+
if err = b.Encode(key.Interface()); err != nil {
120+
return err
121+
}
122+
if err = b.Encode(value.Interface()); err != nil {
123+
return err
124+
}
125+
}
126+
127+
default:
128+
return errors.New("unsupported type " + t.String())
129+
}
130+
}
131+
return
132+
}
133+
134+
type Decoder struct {
135+
r *bufio.Reader
136+
}
137+
138+
func NewDecoder(r io.Reader) *Decoder {
139+
return &Decoder{bufio.NewReader(r)}
140+
}
141+
142+
func (d *Decoder) Decode(v interface{}) (err error) {
143+
switch cv := v.(type) {
144+
case *string:
145+
var l uint64
146+
if l, err = binary.ReadUvarint(d.r); err != nil {
147+
return
148+
}
149+
buf := make([]byte, l)
150+
_, err = d.r.Read(buf)
151+
*cv = string(buf)
152+
153+
case *bool:
154+
var out byte
155+
err = binary.Read(d.r, DefaultEndian, &out)
156+
*cv = out != 0
157+
158+
case *int:
159+
var out int64
160+
err = binary.Read(d.r, DefaultEndian, &out)
161+
*cv = int(out)
162+
163+
case *uint:
164+
var out uint64
165+
err = binary.Read(d.r, DefaultEndian, &out)
166+
*cv = uint(out)
167+
168+
case *int8, *uint8, *int16, *uint16, *int32, *uint32, *int64, *uint64, *float32,
169+
*float64, *complex64, *complex128:
170+
err = binary.Read(d.r, DefaultEndian, v)
171+
172+
default:
173+
// Check if the type implements the encoding.BinaryUnmarshaler interface, and use it if so.
174+
if i, ok := v.(encoding.BinaryUnmarshaler); ok {
175+
var l uint64
176+
if l, err = binary.ReadUvarint(d.r); err != nil {
177+
return
178+
}
179+
buf := make([]byte, l)
180+
_, err = d.r.Read(buf)
181+
return i.UnmarshalBinary(buf)
182+
}
183+
184+
// Otherwise, use reflection.
185+
rv := reflect.Indirect(reflect.ValueOf(v))
186+
if !rv.CanAddr() {
187+
return errors.New("can only Decode to pointer type")
188+
}
189+
t := rv.Type()
190+
191+
switch t.Kind() {
192+
case reflect.Array, reflect.Slice:
193+
var l uint64
194+
if l, err = binary.ReadUvarint(d.r); err != nil {
195+
return
196+
}
197+
if t.Kind() == reflect.Slice {
198+
rv.Set(reflect.MakeSlice(t, int(l), int(l)))
199+
} else if int(l) != t.Len() {
200+
return fmt.Errorf("encoded size %d != real size %d", l, t.Len())
201+
}
202+
for i := 0; i < int(l); i++ {
203+
if err = d.Decode(rv.Index(i).Addr().Interface()); err != nil {
204+
return
205+
}
206+
}
207+
208+
case reflect.Struct:
209+
l := rv.NumField()
210+
for i := 0; i < l; i++ {
211+
if v := rv.Field(i); v.CanSet() && t.Field(i).Name != "_" {
212+
if err = d.Decode(v.Addr().Interface()); err != nil {
213+
return
214+
}
215+
}
216+
}
217+
218+
case reflect.Map:
219+
var l uint64
220+
if l, err = binary.ReadUvarint(d.r); err != nil {
221+
return
222+
}
223+
kt := t.Key()
224+
vt := t.Elem()
225+
rv.Set(reflect.MakeMap(t))
226+
for i := 0; i < int(l); i++ {
227+
kv := reflect.Indirect(reflect.New(kt))
228+
if err = d.Decode(kv.Addr().Interface()); err != nil {
229+
return
230+
}
231+
vv := reflect.Indirect(reflect.New(vt))
232+
if err = d.Decode(vv.Addr().Interface()); err != nil {
233+
return
234+
}
235+
rv.SetMapIndex(kv, vv)
236+
}
237+
238+
default:
239+
return errors.New("unsupported type " + t.String())
240+
}
241+
}
242+
return
243+
}

‎binary_test.go‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
package binary
2+
3+
import (
4+
"errors"
5+
"testing"
6+
"time"
7+
8+
"github.com/stretchrcom/testify/assert"
9+
)
10+
11+
type s0 struct {
12+
A string
13+
B string
14+
C int16
15+
}
16+
17+
var (
18+
s0v = &s0{"A", "B", 1}
19+
s0b = []byte{0x1, 0x41, 0x1, 0x42, 0x1, 0x0}
20+
)
21+
22+
func TestBinaryEncodeStruct(t *testing.T) {
23+
b, err := Marshal(s0v)
24+
assert.NoError(t, err)
25+
assert.Equal(t, s0b, b)
26+
}
27+
28+
func TestBinaryDecodeStruct(t *testing.T) {
29+
s := &s0{}
30+
err := Unmarshal(s0b, s)
31+
assert.NoError(t, err)
32+
assert.Equal(t, s0v, s)
33+
}
34+
35+
func TestBinaryDecodeToValueErrors(t *testing.T) {
36+
b := []byte{1, 0, 0, 0}
37+
var v uint32
38+
err := Unmarshal(b, v)
39+
assert.Error(t, err)
40+
err = Unmarshal(b, &v)
41+
assert.NoError(t, err)
42+
assert.Equal(t, uint32(1), v)
43+
}
44+
45+
type s1 struct {
46+
Name string
47+
BirthDay time.Time
48+
Phone string
49+
Siblings int
50+
Spouse bool
51+
Money float64
52+
Tags map[string]string
53+
Aliases []string
54+
}
55+
56+
var (
57+
s1v = &s1{
58+
Name: "Bob Smith",
59+
BirthDay: time.Date(2013, 1, 2, 3, 4, 5, 6, time.UTC),
60+
Phone: "5551234567",
61+
Siblings: 2,
62+
Spouse: false,
63+
Money: 100.0,
64+
Tags: map[string]string{"key": "value"},
65+
Aliases: []string{"Bobby", "Robert"},
66+
}
67+
68+
svb = []byte{0x9, 0x42, 0x6f, 0x62, 0x20, 0x53, 0x6d, 0x69, 0x74, 0x68, 0xf,
69+
0x1, 0x0, 0x0, 0x0, 0xe, 0xc8, 0x75, 0x9a, 0xa5, 0x0, 0x0, 0x0, 0x6, 0xff,
70+
0xff, 0xa, 0x35, 0x35, 0x35, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x2,
71+
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x59,
72+
0x40, 0x1, 0x3, 0x6b, 0x65, 0x79, 0x5, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2, 0x5,
73+
0x42, 0x6f, 0x62, 0x62, 0x79, 0x6, 0x52, 0x6f, 0x62, 0x65, 0x72, 0x74}
74+
)
75+
76+
func TestBinaryEncodeComplex(t *testing.T) {
77+
b, err := Marshal(s1v)
78+
assert.NoError(t, err)
79+
assert.Equal(t, svb, b)
80+
s := &s1{}
81+
err = Unmarshal(svb, s)
82+
assert.NoError(t, err)
83+
assert.Equal(t, s1v, s)
84+
}
85+
86+
type s2 struct {
87+
b []byte
88+
}
89+
90+
func (s *s2) UnmarshalBinary(data []byte) error {
91+
if len(data) != 1 {
92+
return errors.New("expected data to be length 1")
93+
}
94+
s.b = data
95+
return nil
96+
}
97+
98+
func (s *s2) MarshalBinary() (data []byte, err error) {
99+
return s.b, nil
100+
}
101+
102+
func TestBinaryMarshalUnMarshaler(t *testing.T) {
103+
s2v := &s2{[]byte{0x13}}
104+
b, err := Marshal(s2v)
105+
assert.NoError(t, err)
106+
assert.Equal(t, []byte{0x1, 0x13}, b)
107+
}

0 commit comments

Comments
 (0)