diff --git a/README.md b/README.md index 4f36996..14b9577 100644 --- a/README.md +++ b/README.md @@ -436,6 +436,9 @@ In addition to the default JSON encoding, `zerolog` can produce binary logs usin go build -tags binary_log . ``` +To Decode binary encoded log files you can use any CBOR decoder. One has been tested to work +with zerolog library is CSD(https://github.com/toravir/csd/). + ## Benchmarks All operations are allocation free (those numbers *include* JSON encoding): diff --git a/array.go b/array.go index 9555369..66c2a4f 100644 --- a/array.go +++ b/array.go @@ -1,6 +1,7 @@ package zerolog import ( + "net" "sync" "time" ) @@ -174,3 +175,21 @@ func (a *Array) Interface(i interface{}) *Array { a.buf = appendInterface(appendArrayDelim(a.buf), i) return a } + +// IPAddr adds IPv4 or IPv6 address to the array +func (a *Array) IPAddr(ip net.IP) *Array { + a.buf = appendIPAddr(appendArrayDelim(a.buf), ip) + return a +} + +// IPPrefix adds IPv4 or IPv6 Prefix (IP + mask) to the array +func (a *Array) IPPrefix(pfx net.IPNet) *Array { + a.buf = appendIPPrefix(appendArrayDelim(a.buf), pfx) + return a +} + +// MACAddr adds a MAC (Ethernet) address to the array +func (a *Array) MACAddr(ha net.HardwareAddr) *Array { + a.buf = appendMACAddr(appendArrayDelim(a.buf), ha) + return a +} diff --git a/array_test.go b/array_test.go index 952a579..19bac56 100644 --- a/array_test.go +++ b/array_test.go @@ -1,6 +1,7 @@ package zerolog import ( + "net" "testing" "time" ) @@ -24,8 +25,9 @@ func TestArray(t *testing.T) { Bytes([]byte("b")). Hex([]byte{0x1f}). Time(time.Time{}). + IPAddr(net.IP{192, 168, 0, 10}). Dur(0) - want := `[true,1,2,3,4,5,6,7,8,9,10,11,12,"a","b","1f","0001-01-01T00:00:00Z",0]` + want := `[true,1,2,3,4,5,6,7,8,9,10,11,12,"a","b","1f","0001-01-01T00:00:00Z","192.168.0.10",0]` if got := decodeObjectToStr(a.write([]byte{})); got != want { t.Errorf("Array.write()\ngot: %s\nwant: %s", got, want) } diff --git a/context.go b/context.go index 25ef7cf..1f92dae 100644 --- a/context.go +++ b/context.go @@ -2,6 +2,7 @@ package zerolog import ( "io/ioutil" + "net" "time" ) @@ -330,3 +331,21 @@ func (c Context) Caller() Context { c.l = c.l.Hook(ch) return c } + +// IPAddr adds IPv4 or IPv6 Address to the context +func (c Context) IPAddr(key string, ip net.IP) Context { + c.l.context = appendIPAddr(appendKey(c.l.context, key), ip) + return c +} + +// IPPrefix adds IPv4 or IPv6 Prefix (address and mask) to the context +func (c Context) IPPrefix(key string, pfx net.IPNet) Context { + c.l.context = appendIPPrefix(appendKey(c.l.context, key), pfx) + return c +} + +// MACAddr adds MAC address to the context +func (c Context) MACAddr(key string, ha net.HardwareAddr) Context { + c.l.context = appendMACAddr(appendKey(c.l.context, key), ha) + return c +} diff --git a/encoder_cbor.go b/encoder_cbor.go index 6d001ac..aa72013 100644 --- a/encoder_cbor.go +++ b/encoder_cbor.go @@ -5,6 +5,7 @@ package zerolog // This file contains bindings to do binary encoding. import ( + "net" "time" "github.com/rs/zerolog/internal/cbor" @@ -211,6 +212,18 @@ func decodeObjectToStr(in []byte) string { return cbor.DecodeObjectToStr(in) } +func appendIPAddr(dst []byte, ip net.IP) []byte { + return cbor.AppendIPAddr(dst, ip) +} + +func appendIPPrefix(dst []byte, pfx net.IPNet) []byte { + return cbor.AppendIPPrefix(dst, pfx) +} + +func appendMACAddr(dst []byte, ha net.HardwareAddr) []byte { + return cbor.AppendMACAddr(dst, ha) +} + // decodeIfBinaryToBytes - converts a binary formatted log msg to a // JSON formatted Bytes Log message. func decodeIfBinaryToBytes(in []byte) []byte { diff --git a/encoder_json.go b/encoder_json.go index 1e5db80..df15c7a 100644 --- a/encoder_json.go +++ b/encoder_json.go @@ -6,6 +6,7 @@ package zerolog // JSON encoded byte stream. import ( + "net" "strconv" "time" @@ -211,6 +212,18 @@ func decodeIfBinaryToBytes(in []byte) []byte { return in } -func appendHex(in []byte, val []byte) []byte { - return json.AppendHex(in, val) +func appendIPAddr(dst []byte, ip net.IP) []byte { + return json.AppendIPAddr(dst, ip) +} + +func appendIPPrefix(dst []byte, pfx net.IPNet) []byte { + return json.AppendIPPrefix(dst, pfx) +} + +func appendMACAddr(dst []byte, ha net.HardwareAddr) []byte { + return json.AppendMACAddr(dst, ha) +} + +func appendHex(in []byte, val []byte) []byte { + return json.AppendHex(in, val) } diff --git a/event.go b/event.go index 05ee4d8..dffaf3e 100644 --- a/event.go +++ b/event.go @@ -2,6 +2,7 @@ package zerolog import ( "fmt" + "net" "os" "runtime" "strconv" @@ -597,3 +598,30 @@ func (e *Event) caller(skip int) *Event { e.buf = appendString(appendKey(e.buf, CallerFieldName), file+":"+strconv.Itoa(line)) return e } + +// IPAddr adds IPv4 or IPv6 Address to the event +func (e *Event) IPAddr(key string, ip net.IP) *Event { + if e == nil { + return e + } + e.buf = appendIPAddr(appendKey(e.buf, key), ip) + return e +} + +// IPPrefix adds IPv4 or IPv6 Prefix (address and mask) to the event +func (e *Event) IPPrefix(key string, pfx net.IPNet) *Event { + if e == nil { + return e + } + e.buf = appendIPPrefix(appendKey(e.buf, key), pfx) + return e +} + +// MACAddr adds MAC address to the event +func (e *Event) MACAddr(key string, ha net.HardwareAddr) *Event { + if e == nil { + return e + } + e.buf = appendMACAddr(appendKey(e.buf, key), ha) + return e +} diff --git a/fields.go b/fields.go index 95e83ef..9d13f0c 100644 --- a/fields.go +++ b/fields.go @@ -1,6 +1,7 @@ package zerolog import ( + "net" "sort" "time" ) @@ -118,6 +119,12 @@ func appendFields(dst []byte, fields map[string]interface{}) []byte { dst = appendDurations(dst, val, DurationFieldUnit, DurationFieldInteger) case nil: dst = appendNil(dst) + case net.IP: + dst = appendIPAddr(dst, val) + case net.IPNet: + dst = appendIPPrefix(dst, val) + case net.HardwareAddr: + dst = appendMACAddr(dst, val) default: dst = appendInterface(dst, val) } diff --git a/internal/cbor/README.md b/internal/cbor/README.md index ff71754..92c2e8c 100644 --- a/internal/cbor/README.md +++ b/internal/cbor/README.md @@ -1,74 +1,56 @@ -Reference: - CBOR Encoding is described in RFC7049 https://tools.ietf.org/html/rfc7049 +## Reference: + CBOR Encoding is described in [RFC7049](https://tools.ietf.org/html/rfc7049) + +## Comparison of JSON vs CBOR + +Two main areas of reduction are: + +1. CPU usage to write a log msg +2. Size (in bytes) of log messages. -Tests and benchmark: - +CPU Usage savings are below: ``` -sprint @ cbor>go test -v -benchmem -bench=. -=== RUN TestDecodeInteger ---- PASS: TestDecodeInteger (0.00s) -=== RUN TestDecodeString ---- PASS: TestDecodeString (0.00s) -=== RUN TestDecodeArray ---- PASS: TestDecodeArray (0.00s) -=== RUN TestDecodeMap ---- PASS: TestDecodeMap (0.00s) -=== RUN TestDecodeBool ---- PASS: TestDecodeBool (0.00s) -=== RUN TestDecodeFloat ---- PASS: TestDecodeFloat (0.00s) -=== RUN TestDecodeTimestamp ---- PASS: TestDecodeTimestamp (0.00s) -=== RUN TestDecodeCbor2Json ---- PASS: TestDecodeCbor2Json (0.00s) -=== RUN TestAppendString ---- PASS: TestAppendString (0.00s) -=== RUN TestAppendBytes ---- PASS: TestAppendBytes (0.00s) -=== RUN TestAppendTimeNow ---- PASS: TestAppendTimeNow (0.00s) -=== RUN TestAppendTimePastPresentInteger ---- PASS: TestAppendTimePastPresentInteger (0.00s) -=== RUN TestAppendTimePastPresentFloat ---- PASS: TestAppendTimePastPresentFloat (0.00s) -=== RUN TestAppendNull ---- PASS: TestAppendNull (0.00s) -=== RUN TestAppendBool ---- PASS: TestAppendBool (0.00s) -=== RUN TestAppendBoolArray ---- PASS: TestAppendBoolArray (0.00s) -=== RUN TestAppendInt ---- PASS: TestAppendInt (0.00s) -=== RUN TestAppendIntArray ---- PASS: TestAppendIntArray (0.00s) -=== RUN TestAppendFloat32 ---- PASS: TestAppendFloat32 (0.00s) -goos: linux -goarch: amd64 -pkg: github.com/toravir/zerolog/internal/cbor -BenchmarkAppendString/MultiBytesLast-4 30000000 43.3 ns/op 0 B/op 0 allocs/op -BenchmarkAppendString/NoEncoding-4 30000000 48.2 ns/op 0 B/op 0 allocs/op -BenchmarkAppendString/EncodingFirst-4 30000000 48.2 ns/op 0 B/op 0 allocs/op -BenchmarkAppendString/EncodingMiddle-4 30000000 41.7 ns/op 0 B/op 0 allocs/op -BenchmarkAppendString/EncodingLast-4 30000000 51.8 ns/op 0 B/op 0 allocs/op -BenchmarkAppendString/MultiBytesFirst-4 50000000 38.0 ns/op 0 B/op 0 allocs/op -BenchmarkAppendString/MultiBytesMiddle-4 50000000 38.0 ns/op 0 B/op 0 allocs/op -BenchmarkAppendTime/Integer-4 50000000 39.6 ns/op 0 B/op 0 allocs/op -BenchmarkAppendTime/Float-4 30000000 56.1 ns/op 0 B/op 0 allocs/op -BenchmarkAppendInt/uint8-4 50000000 29.1 ns/op 0 B/op 0 allocs/op -BenchmarkAppendInt/uint16-4 50000000 30.3 ns/op 0 B/op 0 allocs/op -BenchmarkAppendInt/uint32-4 50000000 37.1 ns/op 0 B/op 0 allocs/op -BenchmarkAppendInt/int8-4 100000000 21.5 ns/op 0 B/op 0 allocs/op -BenchmarkAppendInt/int16-4 50000000 25.8 ns/op 0 B/op 0 allocs/op -BenchmarkAppendInt/int32-4 50000000 26.7 ns/op 0 B/op 0 allocs/op -BenchmarkAppendInt/int-Positive-4 100000000 21.5 ns/op 0 B/op 0 allocs/op -BenchmarkAppendInt/int-Negative-4 100000000 20.7 ns/op 0 B/op 0 allocs/op -BenchmarkAppendInt/uint64-4 50000000 36.7 ns/op 0 B/op 0 allocs/op -BenchmarkAppendInt/int64-4 30000000 39.6 ns/op 0 B/op 0 allocs/op -BenchmarkAppendFloat/Float32-4 50000000 23.9 ns/op 0 B/op 0 allocs/op -BenchmarkAppendFloat/Float64-4 50000000 32.8 ns/op 0 B/op 0 allocs/op -PASS -ok github.com/toravir/zerolog/internal/cbor 34.969s -sprint @ cbor> +name JSON time/op CBOR time/op delta +Info-32 15.3ns ± 1% 11.7ns ± 3% -23.78% (p=0.000 n=9+10) +ContextFields-32 16.2ns ± 2% 12.3ns ± 3% -23.97% (p=0.000 n=9+9) +ContextAppend-32 6.70ns ± 0% 6.20ns ± 0% -7.44% (p=0.000 n=9+9) +LogFields-32 66.4ns ± 0% 24.6ns ± 2% -62.89% (p=0.000 n=10+9) +LogArrayObject-32 911ns ±11% 768ns ± 6% -15.64% (p=0.000 n=10+10) +LogFieldType/Floats-32 70.3ns ± 2% 29.5ns ± 1% -57.98% (p=0.000 n=10+10) +LogFieldType/Err-32 14.0ns ± 3% 12.1ns ± 8% -13.20% (p=0.000 n=8+10) +LogFieldType/Dur-32 17.2ns ± 2% 13.1ns ± 1% -24.27% (p=0.000 n=10+9) +LogFieldType/Object-32 54.3ns ±11% 52.3ns ± 7% ~ (p=0.239 n=10+10) +LogFieldType/Ints-32 20.3ns ± 2% 15.1ns ± 2% -25.50% (p=0.000 n=9+10) +LogFieldType/Interfaces-32 642ns ±11% 621ns ± 9% ~ (p=0.118 n=10+10) +LogFieldType/Interface(Objects)-32 635ns ±13% 632ns ± 9% ~ (p=0.592 n=10+10) +LogFieldType/Times-32 294ns ± 0% 27ns ± 1% -90.71% (p=0.000 n=10+9) +LogFieldType/Durs-32 121ns ± 0% 33ns ± 2% -72.44% (p=0.000 n=9+9) +LogFieldType/Interface(Object)-32 56.6ns ± 8% 52.3ns ± 8% -7.54% (p=0.007 n=10+10) +LogFieldType/Errs-32 17.8ns ± 3% 16.1ns ± 2% -9.71% (p=0.000 n=10+9) +LogFieldType/Time-32 40.5ns ± 1% 12.7ns ± 6% -68.66% (p=0.000 n=8+9) +LogFieldType/Bool-32 12.0ns ± 5% 10.2ns ± 2% -15.18% (p=0.000 n=10+8) +LogFieldType/Bools-32 17.2ns ± 2% 12.6ns ± 4% -26.63% (p=0.000 n=10+10) +LogFieldType/Int-32 12.3ns ± 2% 11.2ns ± 4% -9.27% (p=0.000 n=9+10) +LogFieldType/Float-32 16.7ns ± 1% 12.6ns ± 2% -24.42% (p=0.000 n=7+9) +LogFieldType/Str-32 12.7ns ± 7% 11.3ns ± 7% -10.88% (p=0.000 n=10+9) +LogFieldType/Strs-32 20.3ns ± 3% 18.2ns ± 3% -10.25% (p=0.000 n=9+10) +LogFieldType/Interface-32 183ns ±12% 175ns ± 9% ~ (p=0.078 n=10+10) ``` + +Log message size savings is greatly dependent on the number and type of fields in the log message. +Assuming this log message (with an Integer, timestamp and string, in addition to level). + +`{"level":"error","Fault":41650,"time":"2018-04-01T15:18:19-07:00","message":"Some Message"}` + +Two measurements were done for the log file sizes - one without any compression, second +using [compress/zlib](https://golang.org/pkg/compress/zlib/). + +Results for 10,000 log messages: + +| Log Format | Plain File Size (in KB) | Compressed File Size (in KB) | +| :--- | :---: | :---: | +| JSON | 920 | 28 | +| CBOR | 550 | 28 | + +The example used to calculate the above data is available in [Examples](examples). diff --git a/internal/cbor/cbor.go b/internal/cbor/cbor.go index fe216ce..969f591 100644 --- a/internal/cbor/cbor.go +++ b/internal/cbor/cbor.go @@ -7,25 +7,34 @@ import "time" const ( majorOffset = 5 additionalMax = 23 - //Non Values + + // Non Values. additionalTypeBoolFalse byte = 20 additionalTypeBoolTrue byte = 21 additionalTypeNull byte = 22 - //Integer (+ve and -ve) Sub-types + + // Integer (+ve and -ve) Sub-types. additionalTypeIntUint8 byte = 24 additionalTypeIntUint16 byte = 25 additionalTypeIntUint32 byte = 26 additionalTypeIntUint64 byte = 27 - //Float Sub-types + + // Float Sub-types. additionalTypeFloat16 byte = 25 additionalTypeFloat32 byte = 26 additionalTypeFloat64 byte = 27 additionalTypeBreak byte = 31 - //Tag Sub-types - additionalTypeTimestamp byte = 01 - additionalTypeEmbeddedJSON byte = 31 - additionalTypeTagHexString uint16 = 262 - //Unspecified number of elements + + // Tag Sub-types. + additionalTypeTimestamp byte = 01 + + // Extended Tags - from https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml + additionalTypeTagNetworkAddr uint16 = 260 + additionalTypeTagNetworkPrefix uint16 = 261 + additionalTypeEmbeddedJSON uint16 = 262 + additionalTypeTagHexString uint16 = 263 + + // Unspecified number of elements. additionalTypeInfiniteCount byte = 31 ) const ( diff --git a/internal/cbor/decode_stream.go b/internal/cbor/decode_stream.go new file mode 100644 index 0000000..f1ff2a1 --- /dev/null +++ b/internal/cbor/decode_stream.go @@ -0,0 +1,610 @@ +package cbor + +// This file contains code to decode a stream of CBOR Data into JSON. + +import ( + "bufio" + "bytes" + "fmt" + "io" + "math" + "net" + "runtime" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +var decodeTimeZone *time.Location + +const hexTable = "0123456789abcdef" + +func readNBytes(src *bufio.Reader, n int) []byte { + ret := make([]byte, n) + for i := 0; i < n; i++ { + ch, e := src.ReadByte() + if e != nil { + panic(fmt.Errorf("Tried to Read %d Bytes.. But hit end of file", n)) + } + ret[i] = ch + } + return ret +} + +func readByte(src *bufio.Reader) byte { + b, e := src.ReadByte() + if e != nil { + panic(fmt.Errorf("Tried to Read 1 Byte.. But hit end of file")) + } + return b +} + +func decodeIntAdditonalType(src *bufio.Reader, minor byte) int64 { + val := int64(0) + if minor <= 23 { + val = int64(minor) + } else { + bytesToRead := 0 + switch minor { + case additionalTypeIntUint8: + bytesToRead = 1 + case additionalTypeIntUint16: + bytesToRead = 2 + case additionalTypeIntUint32: + bytesToRead = 4 + case additionalTypeIntUint64: + bytesToRead = 8 + default: + panic(fmt.Errorf("Invalid Additional Type: %d in decodeInteger (expected <28)", minor)) + } + pb := readNBytes(src, bytesToRead) + for i := 0; i < bytesToRead; i++ { + val = val * 256 + val += int64(pb[i]) + } + } + return val +} + +func decodeInteger(src *bufio.Reader) int64 { + pb := readByte(src) + major := pb & maskOutAdditionalType + minor := pb & maskOutMajorType + if major != majorTypeUnsignedInt && major != majorTypeNegativeInt { + panic(fmt.Errorf("Major type is: %d in decodeInteger!! (expected 0 or 1)", major)) + } + val := decodeIntAdditonalType(src, minor) + if major == 0 { + return val + } + return (-1 - val) +} + +func decodeFloat(src *bufio.Reader) (float64, int) { + pb := readByte(src) + major := pb & maskOutAdditionalType + minor := pb & maskOutMajorType + if major != majorTypeSimpleAndFloat { + panic(fmt.Errorf("Incorrect Major type is: %d in decodeFloat", major)) + } + + switch minor { + case additionalTypeFloat16: + panic(fmt.Errorf("float16 is not suppported in decodeFloat")) + + case additionalTypeFloat32: + pb := readNBytes(src, 4) + switch string(pb) { + case float32Nan: + return math.NaN(), 4 + case float32PosInfinity: + return math.Inf(0), 4 + case float32NegInfinity: + return math.Inf(-1), 4 + } + n := uint32(0) + for i := 0; i < 4; i++ { + n = n * 256 + n += uint32(pb[i]) + } + val := math.Float32frombits(n) + return float64(val), 4 + case additionalTypeFloat64: + pb := readNBytes(src, 8) + switch string(pb) { + case float64Nan: + return math.NaN(), 8 + case float64PosInfinity: + return math.Inf(0), 8 + case float64NegInfinity: + return math.Inf(-1), 8 + } + n := uint64(0) + for i := 0; i < 8; i++ { + n = n * 256 + n += uint64(pb[i]) + } + val := math.Float64frombits(n) + return val, 8 + } + panic(fmt.Errorf("Invalid Additional Type: %d in decodeFloat", minor)) +} + +func decodeStringComplex(dst []byte, s string, pos uint) []byte { + i := int(pos) + const hex = "0123456789abcdef" + start := 0 + + for i < len(s) { + b := s[i] + if b >= utf8.RuneSelf { + r, size := utf8.DecodeRuneInString(s[i:]) + if r == utf8.RuneError && size == 1 { + // In case of error, first append previous simple characters to + // the byte slice if any and append a replacement character code + // in place of the invalid sequence. + if start < i { + dst = append(dst, s[start:i]...) + } + dst = append(dst, `\ufffd`...) + i += size + start = i + continue + } + i += size + continue + } + if b >= 0x20 && b <= 0x7e && b != '\\' && b != '"' { + i++ + continue + } + // We encountered a character that needs to be encoded. + // Let's append the previous simple characters to the byte slice + // and switch our operation to read and encode the remainder + // characters byte-by-byte. + if start < i { + dst = append(dst, s[start:i]...) + } + switch b { + case '"', '\\': + dst = append(dst, '\\', b) + case '\b': + dst = append(dst, '\\', 'b') + case '\f': + dst = append(dst, '\\', 'f') + case '\n': + dst = append(dst, '\\', 'n') + case '\r': + dst = append(dst, '\\', 'r') + case '\t': + dst = append(dst, '\\', 't') + default: + dst = append(dst, '\\', 'u', '0', '0', hex[b>>4], hex[b&0xF]) + } + i++ + start = i + } + if start < len(s) { + dst = append(dst, s[start:]...) + } + return dst +} + +func decodeString(src *bufio.Reader, noQuotes bool) []byte { + pb := readByte(src) + major := pb & maskOutAdditionalType + minor := pb & maskOutMajorType + if major != majorTypeByteString { + panic(fmt.Errorf("Major type is: %d in decodeString", major)) + } + result := []byte{} + if !noQuotes { + result = append(result, '"') + } + length := decodeIntAdditonalType(src, minor) + len := int(length) + pbs := readNBytes(src, len) + result = append(result, pbs...) + if noQuotes { + return result + } + return append(result, '"') +} + +func decodeUTF8String(src *bufio.Reader) []byte { + pb := readByte(src) + major := pb & maskOutAdditionalType + minor := pb & maskOutMajorType + if major != majorTypeUtf8String { + panic(fmt.Errorf("Major type is: %d in decodeUTF8String", major)) + } + result := []byte{'"'} + length := decodeIntAdditonalType(src, minor) + len := int(length) + pbs := readNBytes(src, len) + + for i := 0; i < len; i++ { + // Check if the character needs encoding. Control characters, slashes, + // and the double quote need json encoding. Bytes above the ascii + // boundary needs utf8 encoding. + if pbs[i] < 0x20 || pbs[i] > 0x7e || pbs[i] == '\\' || pbs[i] == '"' { + // We encountered a character that needs to be encoded. Switch + // to complex version of the algorithm. + dst := []byte{'"'} + dst = decodeStringComplex(dst, string(pbs), uint(i)) + return append(dst, '"') + } + } + // The string has no need for encoding an therefore is directly + // appended to the byte slice. + result = append(result, pbs...) + return append(result, '"') +} + +func array2Json(src *bufio.Reader, dst io.Writer) { + dst.Write([]byte{'['}) + pb := readByte(src) + major := pb & maskOutAdditionalType + minor := pb & maskOutMajorType + if major != majorTypeArray { + panic(fmt.Errorf("Major type is: %d in array2Json", major)) + } + len := 0 + unSpecifiedCount := false + if minor == additionalTypeInfiniteCount { + unSpecifiedCount = true + } else { + length := decodeIntAdditonalType(src, minor) + len = int(length) + } + for i := 0; unSpecifiedCount || i < len; i++ { + if unSpecifiedCount { + pb, e := src.Peek(1) + if e != nil { + panic(e) + } + if pb[0] == byte(majorTypeSimpleAndFloat|additionalTypeBreak) { + readByte(src) + break + } + } + cbor2JsonOneObject(src, dst) + if unSpecifiedCount { + pb, e := src.Peek(1) + if e != nil { + panic(e) + } + if pb[0] == byte(majorTypeSimpleAndFloat|additionalTypeBreak) { + readByte(src) + break + } + dst.Write([]byte{','}) + } else if i+1 < len { + dst.Write([]byte{','}) + } + } + dst.Write([]byte{']'}) +} + +func map2Json(src *bufio.Reader, dst io.Writer) { + pb := readByte(src) + major := pb & maskOutAdditionalType + minor := pb & maskOutMajorType + if major != majorTypeMap { + panic(fmt.Errorf("Major type is: %d in map2Json", major)) + } + len := 0 + unSpecifiedCount := false + if minor == additionalTypeInfiniteCount { + unSpecifiedCount = true + } else { + length := decodeIntAdditonalType(src, minor) + len = int(length) + } + dst.Write([]byte{'{'}) + for i := 0; unSpecifiedCount || i < len; i++ { + if unSpecifiedCount { + pb, e := src.Peek(1) + if e != nil { + panic(e) + } + if pb[0] == byte(majorTypeSimpleAndFloat|additionalTypeBreak) { + readByte(src) + break + } + } + cbor2JsonOneObject(src, dst) + if i%2 == 0 { + // Even position values are keys. + dst.Write([]byte{':'}) + } else { + if unSpecifiedCount { + pb, e := src.Peek(1) + if e != nil { + panic(e) + } + if pb[0] == byte(majorTypeSimpleAndFloat|additionalTypeBreak) { + readByte(src) + break + } + dst.Write([]byte{','}) + } else if i+1 < len { + dst.Write([]byte{','}) + } + } + } + dst.Write([]byte{'}'}) +} + +func decodeTagData(src *bufio.Reader) []byte { + pb := readByte(src) + major := pb & maskOutAdditionalType + minor := pb & maskOutMajorType + if major != majorTypeTags { + panic(fmt.Errorf("Major type is: %d in decodeTagData", major)) + } + switch minor { + case additionalTypeTimestamp: + return decodeTimeStamp(src) + + // Tag value is larger than 256 (so uint16). + case additionalTypeIntUint16: + val := decodeIntAdditonalType(src, minor) + + switch uint16(val) { + case additionalTypeEmbeddedJSON: + pb := readByte(src) + dataMajor := pb & maskOutAdditionalType + if dataMajor != majorTypeByteString { + panic(fmt.Errorf("Unsupported embedded Type: %d in decodeEmbeddedJSON", dataMajor)) + } + src.UnreadByte() + return decodeString(src, true) + + case additionalTypeTagNetworkAddr: + octets := decodeString(src, true) + ss := []byte{'"'} + switch len(octets) { + case 6: // MAC address. + ha := net.HardwareAddr(octets) + ss = append(append(ss, ha.String()...), '"') + case 4: // IPv4 address. + fallthrough + case 16: // IPv6 address. + ip := net.IP(octets) + ss = append(append(ss, ip.String()...), '"') + default: + panic(fmt.Errorf("Unexpected Network Address length: %d (expected 4,6,16)", len(octets))) + } + return ss + + case additionalTypeTagNetworkPrefix: + pb := readByte(src) + if pb != byte(majorTypeMap|0x1) { + panic(fmt.Errorf("IP Prefix is NOT of MAP of 1 elements as expected")) + } + octets := decodeString(src, true) + val := decodeInteger(src) + ip := net.IP(octets) + var mask net.IPMask + pfxLen := int(val) + if len(octets) == 4 { + mask = net.CIDRMask(pfxLen, 32) + } else { + mask = net.CIDRMask(pfxLen, 128) + } + ipPfx := net.IPNet{IP: ip, Mask: mask} + ss := []byte{'"'} + ss = append(append(ss, ipPfx.String()...), '"') + return ss + + case additionalTypeTagHexString: + octets := decodeString(src, true) + ss := []byte{'"'} + for _, v := range octets { + ss = append(ss, hexTable[v>>4], hexTable[v&0x0f]) + } + return append(ss, '"') + + default: + panic(fmt.Errorf("Unsupported Additional Tag Type: %d in decodeTagData", val)) + } + } + panic(fmt.Errorf("Unsupported Additional Type: %d in decodeTagData", minor)) +} + +func decodeTimeStamp(src *bufio.Reader) []byte { + pb := readByte(src) + src.UnreadByte() + tsMajor := pb & maskOutAdditionalType + if tsMajor == majorTypeUnsignedInt || tsMajor == majorTypeNegativeInt { + n := decodeInteger(src) + t := time.Unix(n, 0) + if decodeTimeZone != nil { + t = t.In(decodeTimeZone) + } else { + t = t.In(time.UTC) + } + tsb := []byte{} + tsb = append(tsb, '"') + tsb = t.AppendFormat(tsb, IntegerTimeFieldFormat) + tsb = append(tsb, '"') + return tsb + } else if tsMajor == majorTypeSimpleAndFloat { + n, _ := decodeFloat(src) + secs := int64(n) + n -= float64(secs) + n *= float64(1e9) + t := time.Unix(secs, int64(n)) + if decodeTimeZone != nil { + t = t.In(decodeTimeZone) + } else { + t = t.In(time.UTC) + } + tsb := []byte{} + tsb = append(tsb, '"') + tsb = t.AppendFormat(tsb, NanoTimeFieldFormat) + tsb = append(tsb, '"') + return tsb + } + panic(fmt.Errorf("TS format is neigther int nor float: %d", tsMajor)) +} + +func decodeSimpleFloat(src *bufio.Reader) []byte { + pb := readByte(src) + major := pb & maskOutAdditionalType + minor := pb & maskOutMajorType + if major != majorTypeSimpleAndFloat { + panic(fmt.Errorf("Major type is: %d in decodeSimpleFloat", major)) + } + switch minor { + case additionalTypeBoolTrue: + return []byte("true") + case additionalTypeBoolFalse: + return []byte("false") + case additionalTypeNull: + return []byte("null") + case additionalTypeFloat16: + fallthrough + case additionalTypeFloat32: + fallthrough + case additionalTypeFloat64: + src.UnreadByte() + v, bc := decodeFloat(src) + ba := []byte{} + switch { + case math.IsNaN(v): + return []byte("\"NaN\"") + case math.IsInf(v, 1): + return []byte("\"+Inf\"") + case math.IsInf(v, -1): + return []byte("\"-Inf\"") + } + if bc == 5 { + ba = strconv.AppendFloat(ba, v, 'f', -1, 32) + } else { + ba = strconv.AppendFloat(ba, v, 'f', -1, 64) + } + return ba + default: + panic(fmt.Errorf("Invalid Additional Type: %d in decodeSimpleFloat", minor)) + } +} + +func cbor2JsonOneObject(src *bufio.Reader, dst io.Writer) { + pb, e := src.Peek(1) + if e != nil { + panic(e) + } + major := (pb[0] & maskOutAdditionalType) + + switch major { + case majorTypeUnsignedInt: + fallthrough + case majorTypeNegativeInt: + n := decodeInteger(src) + dst.Write([]byte(strconv.Itoa(int(n)))) + + case majorTypeByteString: + s := decodeString(src, false) + dst.Write(s) + + case majorTypeUtf8String: + s := decodeUTF8String(src) + dst.Write(s) + + case majorTypeArray: + array2Json(src, dst) + + case majorTypeMap: + map2Json(src, dst) + + case majorTypeTags: + s := decodeTagData(src) + dst.Write(s) + + case majorTypeSimpleAndFloat: + s := decodeSimpleFloat(src) + dst.Write(s) + } +} + +func moreBytesToRead(src *bufio.Reader) bool { + _, e := src.ReadByte() + if e == nil { + src.UnreadByte() + return true + } + return false +} + +// Cbor2JsonManyObjects decodes all the CBOR Objects read from src +// reader. It keeps on decoding until reader returns EOF (error when reading). +// Decoded string is written to the dst. At the end of every CBOR Object +// newline is written to the output stream. +// +// Returns error (if any) that was encountered during decode. +// The child functions will generate a panic when error is encountered and +// this function will recover non-runtime Errors and return the reason as error. +func Cbor2JsonManyObjects(src io.Reader, dst io.Writer) (err error) { + defer func() { + if r := recover(); r != nil { + if _, ok := r.(runtime.Error); ok { + panic(r) + } + err = r.(error) + } + }() + bufRdr := bufio.NewReader(src) + for moreBytesToRead(bufRdr) { + cbor2JsonOneObject(bufRdr, dst) + dst.Write([]byte("\n")) + } + return nil +} + +// Detect if the bytes to be printed is Binary or not. +func binaryFmt(p []byte) bool { + if len(p) > 0 && p[0] > 0x7F { + return true + } + return false +} + +func getReader(str string) *bufio.Reader { + return bufio.NewReader(strings.NewReader(str)) +} + +// DecodeIfBinaryToString converts a binary formatted log msg to a +// JSON formatted String Log message - suitable for printing to Console/Syslog. +func DecodeIfBinaryToString(in []byte) string { + if binaryFmt(in) { + var b bytes.Buffer + Cbor2JsonManyObjects(strings.NewReader(string(in)), &b) + return b.String() + } + return string(in) +} + +// DecodeObjectToStr checks if the input is a binary format, if so, +// it will decode a single Object and return the decoded string. +func DecodeObjectToStr(in []byte) string { + if binaryFmt(in) { + var b bytes.Buffer + cbor2JsonOneObject(getReader(string(in)), &b) + return b.String() + } + return string(in) +} + +// DecodeIfBinaryToBytes checks if the input is a binary format, if so, +// it will decode all Objects and return the decoded string as byte array. +func DecodeIfBinaryToBytes(in []byte) []byte { + if binaryFmt(in) { + var b bytes.Buffer + Cbor2JsonManyObjects(bytes.NewReader(in), &b) + return b.Bytes() + } + return in +} diff --git a/internal/cbor/decoder.go b/internal/cbor/decoder.go deleted file mode 100644 index 71e8648..0000000 --- a/internal/cbor/decoder.go +++ /dev/null @@ -1,548 +0,0 @@ -package cbor - -import ( - "bytes" - "fmt" - "io" - "math" - "strconv" - "time" - "unicode/utf8" -) - -var decodeTimeZone *time.Location - -const hexTable = "0123456789abcdef" - -func decodeIntAdditonalType(src []byte, minor byte) (int64, uint, error) { - val := int64(0) - bytesRead := 0 - if minor <= 23 { - val = int64(minor) - bytesRead = 0 - } else { - switch minor { - case additionalTypeIntUint8: - bytesRead = 1 - case additionalTypeIntUint16: - bytesRead = 2 - case additionalTypeIntUint32: - bytesRead = 4 - case additionalTypeIntUint64: - bytesRead = 8 - default: - return 0, 0, fmt.Errorf("Invalid Additional Type: %d in decodeInteger (expected <28)", minor) - } - for i := 0; i < bytesRead; i++ { - val = val * 256 - val += int64(src[i]) - } - } - return val, uint(bytesRead), nil -} - -func decodeInteger(src []byte) (int64, uint, error) { - major := src[0] & maskOutAdditionalType - minor := src[0] & maskOutMajorType - if major != majorTypeUnsignedInt && major != majorTypeNegativeInt { - return 0, 0, fmt.Errorf("Major type is: %d in decodeInteger!! (expected 0 or 1)", major) - } - val, bytesRead, err := decodeIntAdditonalType(src[1:], minor) - if err != nil { - return 0, 0, err - } - if major == 0 { - return val, 1 + bytesRead, nil - } - return (-1 - val), 1 + bytesRead, nil -} - -func decodeFloat(src []byte) (float64, uint, error) { - major := (src[0] & maskOutAdditionalType) - minor := src[0] & maskOutMajorType - if major != majorTypeSimpleAndFloat { - return 0, 0, fmt.Errorf("Incorrect Major type is: %d in decodeFloat", major) - } - - switch minor { - case additionalTypeFloat16: - return 0, 0, fmt.Errorf("float16 is not suppported in decodeFloat") - case additionalTypeFloat32: - switch string(src[1:5]) { - case float32Nan: - return math.NaN(), 5, nil - case float32PosInfinity: - return math.Inf(0), 5, nil - case float32NegInfinity: - return math.Inf(-1), 5, nil - } - n := uint32(0) - for i := 0; i < 4; i++ { - n = n * 256 - n += uint32(src[i+1]) - } - val := math.Float32frombits(n) - return float64(val), 5, nil - case additionalTypeFloat64: - switch string(src[1:9]) { - case float64Nan: - return math.NaN(), 9, nil - case float64PosInfinity: - return math.Inf(0), 9, nil - case float64NegInfinity: - return math.Inf(-1), 9, nil - } - n := uint64(0) - for i := 0; i < 8; i++ { - n = n * 256 - n += uint64(src[i+1]) - } - val := math.Float64frombits(n) - return val, 9, nil - } - return 0, 0, fmt.Errorf("Invalid Additional Type: %d in decodeFloat", minor) -} - -func decodeStringComplex(dst []byte, s string, pos uint) []byte { - i := int(pos) - const hex = "0123456789abcdef" - start := 0 - - for i < len(s) { - b := s[i] - if b >= utf8.RuneSelf { - r, size := utf8.DecodeRuneInString(s[i:]) - if r == utf8.RuneError && size == 1 { - // In case of error, first append previous simple characters to - // the byte slice if any and append a replacement character code - // in place of the invalid sequence. - if start < i { - dst = append(dst, s[start:i]...) - } - dst = append(dst, `\ufffd`...) - i += size - start = i - continue - } - i += size - continue - } - if b >= 0x20 && b <= 0x7e && b != '\\' && b != '"' { - i++ - continue - } - // We encountered a character that needs to be encoded. - // Let's append the previous simple characters to the byte slice - // and switch our operation to read and encode the remainder - // characters byte-by-byte. - if start < i { - dst = append(dst, s[start:i]...) - } - switch b { - case '"', '\\': - dst = append(dst, '\\', b) - case '\b': - dst = append(dst, '\\', 'b') - case '\f': - dst = append(dst, '\\', 'f') - case '\n': - dst = append(dst, '\\', 'n') - case '\r': - dst = append(dst, '\\', 'r') - case '\t': - dst = append(dst, '\\', 't') - default: - dst = append(dst, '\\', 'u', '0', '0', hex[b>>4], hex[b&0xF]) - } - i++ - start = i - } - if start < len(s) { - dst = append(dst, s[start:]...) - } - return dst -} - -func decodeString(src []byte, noQuotes bool) ([]byte, uint, error) { - major := src[0] & maskOutAdditionalType - minor := src[0] & maskOutMajorType - if major != majorTypeByteString { - return []byte{}, 0, fmt.Errorf("Major type is: %d in decodeString", major) - } - result := []byte{'"'} - if noQuotes { - result = []byte{} - } - length, bytesRead, err := decodeIntAdditonalType(src[1:], minor) - if err != nil { - return []byte{}, 0, err - } - bytesRead++ - st := bytesRead - len := uint(length) - bytesRead += len - - result = append(result, src[st:st+len]...) - if noQuotes { - return result, bytesRead, nil - } - return append(result, '"'), bytesRead, nil -} - -func decodeUTF8String(src []byte) ([]byte, uint, error) { - major := src[0] & maskOutAdditionalType - minor := src[0] & maskOutMajorType - if major != majorTypeUtf8String { - return []byte{}, 0, fmt.Errorf("Major type is: %d in decodeUTF8String", major) - } - result := []byte{'"'} - length, bytesRead, err := decodeIntAdditonalType(src[1:], minor) - if err != nil { - return []byte{}, 0, err - } - bytesRead++ - st := bytesRead - len := uint(length) - bytesRead += len - - for i := st; i < bytesRead; i++ { - // Check if the character needs encoding. Control characters, slashes, - // and the double quote need json encoding. Bytes above the ascii - // boundary needs utf8 encoding. - if src[i] < 0x20 || src[i] > 0x7e || src[i] == '\\' || src[i] == '"' { - // We encountered a character that needs to be encoded. Switch - // to complex version of the algorithm. - dst := []byte{'"'} - dst = decodeStringComplex(dst, string(src[st:st+len]), i-st) - return append(dst, '"'), bytesRead, nil - } - } - // The string has no need for encoding an therefore is directly - // appended to the byte slice. - result = append(result, src[st:st+len]...) - return append(result, '"'), bytesRead, nil -} - -func array2Json(src []byte, dst io.Writer) (uint, error) { - dst.Write([]byte{'['}) - major := (src[0] & maskOutAdditionalType) - minor := src[0] & maskOutMajorType - if major != majorTypeArray { - return 0, fmt.Errorf("Major type is: %d in array2Json", major) - } - len := 0 - bytesRead := uint(0) - unSpecifiedCount := false - if minor == additionalTypeInfiniteCount { - unSpecifiedCount = true - bytesRead = 1 - } else { - var length int64 - var err error - length, bytesRead, err = decodeIntAdditonalType(src[1:], minor) - if err != nil { - fmt.Println("Error!!!") - return 0, err - } - len = int(length) - bytesRead++ - } - curPos := bytesRead - for i := 0; unSpecifiedCount || i < len; i++ { - bc, err := Cbor2JsonOneObject(src[curPos:], dst) - if err != nil { - if src[curPos] == byte(majorTypeSimpleAndFloat|additionalTypeBreak) { - bytesRead++ - break - } - return 0, err - } - curPos += bc - bytesRead += bc - if unSpecifiedCount { - if src[curPos] == byte(majorTypeSimpleAndFloat|additionalTypeBreak) { - bytesRead++ - break - } - dst.Write([]byte{','}) - } else if i+1 < len { - dst.Write([]byte{','}) - } - } - dst.Write([]byte{']'}) - return bytesRead, nil -} - -func map2Json(src []byte, dst io.Writer) (uint, error) { - major := (src[0] & maskOutAdditionalType) - minor := src[0] & maskOutMajorType - if major != majorTypeMap { - return 0, fmt.Errorf("Major type is: %d in map2Json", major) - } - len := 0 - bytesRead := uint(0) - unSpecifiedCount := false - if minor == additionalTypeInfiniteCount { - unSpecifiedCount = true - bytesRead = 1 - } else { - var length int64 - var err error - length, bytesRead, err = decodeIntAdditonalType(src[1:], minor) - if err != nil { - fmt.Println("Error!!!") - return 0, err - } - len = int(length) - bytesRead++ - } - if len%2 == 1 { - return 0, fmt.Errorf("Invalid Length of map %d - has to be even", len) - } - dst.Write([]byte{'{'}) - curPos := bytesRead - for i := 0; unSpecifiedCount || i < len; i++ { - bc, err := Cbor2JsonOneObject(src[curPos:], dst) - if err != nil { - //We hit the BREAK - if src[curPos] == byte(majorTypeSimpleAndFloat|additionalTypeBreak) { - bytesRead++ - break - } - return 0, err - } - curPos += bc - bytesRead += bc - if i%2 == 0 { - //Even position values are keys - dst.Write([]byte{':'}) - } else { - if unSpecifiedCount { - if src[curPos] == byte(majorTypeSimpleAndFloat|additionalTypeBreak) { - bytesRead++ - break - } - dst.Write([]byte{','}) - } else if i+1 < len { - dst.Write([]byte{','}) - } - } - } - dst.Write([]byte{'}'}) - return bytesRead, nil -} - -func decodeTagData(src []byte) ([]byte, uint, error) { - major := (src[0] & maskOutAdditionalType) - minor := src[0] & maskOutMajorType - if major != majorTypeTags { - return nil, 0, fmt.Errorf("Major type is: %d in decodeTagData", major) - } - if minor == additionalTypeTimestamp { - tsMajor := src[1] & maskOutAdditionalType - if tsMajor == majorTypeUnsignedInt || tsMajor == majorTypeNegativeInt { - n, bc, err := decodeInteger(src[1:]) - if err != nil { - return []byte{}, 0, err - } - t := time.Unix(n, 0) - if decodeTimeZone != nil { - t = t.In(decodeTimeZone) - } else { - t = t.In(time.UTC) - } - tsb := []byte{} - tsb = append(tsb, '"') - tsb = t.AppendFormat(tsb, IntegerTimeFieldFormat) - tsb = append(tsb, '"') - return tsb, 1 + bc, nil - } else if tsMajor == majorTypeSimpleAndFloat { - n, bc, err := decodeFloat(src[1:]) - if err != nil { - return []byte{}, 0, err - } - secs := int64(n) - n -= float64(secs) - n *= float64(1e9) - t := time.Unix(secs, int64(n)) - if decodeTimeZone != nil { - t = t.In(decodeTimeZone) - } else { - t = t.In(time.UTC) - } - tsb := []byte{} - tsb = append(tsb, '"') - tsb = t.AppendFormat(tsb, NanoTimeFieldFormat) - tsb = append(tsb, '"') - return tsb, 1 + bc, nil - } else { - return nil, 0, fmt.Errorf("TS format is neigther int nor float: %d", tsMajor) - } - } else if minor == additionalTypeEmbeddedJSON { - dataMajor := src[1] & maskOutAdditionalType - if dataMajor == majorTypeByteString { - emb, bc, err := decodeString(src[1:], true) - if err != nil { - return nil, 0, err - } - return emb, 1 + bc, nil - } - return nil, 0, fmt.Errorf("Unsupported embedded Type: %d in decodeEmbeddedJSON", dataMajor) - } else if minor == additionalTypeIntUint16 { - val,_,_ := decodeIntAdditonalType(src[1:], minor) - if uint16(val) == additionalTypeTagHexString { - emb, bc, _ := decodeString(src[3:], true) - dst := []byte{'"'} - for _, v := range emb { - dst = append(dst, hexTable[v>>4], hexTable[v&0x0f]) - } - return append(dst, '"'), 3+bc, nil - } - } - return nil, 0, fmt.Errorf("Unsupported Additional Type: %d in decodeTagData", minor) -} - -func decodeSimpleFloat(src []byte) ([]byte, uint, error) { - major := (src[0] & maskOutAdditionalType) - minor := src[0] & maskOutMajorType - if major != majorTypeSimpleAndFloat { - return nil, 0, fmt.Errorf("Major type is: %d in decodeSimpleFloat", major) - } - switch minor { - case additionalTypeBoolTrue: - return []byte("true"), 1, nil - case additionalTypeBoolFalse: - return []byte("false"), 1, nil - case additionalTypeNull: - return []byte("null"), 1, nil - - case additionalTypeFloat16: - fallthrough - case additionalTypeFloat32: - fallthrough - case additionalTypeFloat64: - v, bc, err := decodeFloat(src) - if err != nil { - return nil, 0, err - } - ba := []byte{} - switch { - case math.IsNaN(v): - return []byte("\"NaN\""), bc, nil - case math.IsInf(v, 1): - return []byte("\"+Inf\""), bc, nil - case math.IsInf(v, -1): - return []byte("\"-Inf\""), bc, nil - } - if bc == 5 { - ba = strconv.AppendFloat(ba, v, 'f', -1, 32) - } else { - ba = strconv.AppendFloat(ba, v, 'f', -1, 64) - } - return ba, bc, nil - default: - return nil, 0, fmt.Errorf("Invalid Additional Type: %d in decodeSimpleFloat", minor) - } -} - -// Cbor2JsonOneObject takes in byte array and decodes ONE CBOR Object -// usually a MAP. Use this when only ONE CBOR object needs decoding. -// Decoded string is written to the dst. -// Returns the bytes decoded and if any error was encountered. -func Cbor2JsonOneObject(src []byte, dst io.Writer) (uint, error) { - var err error - major := (src[0] & maskOutAdditionalType) - bc := uint(0) - var s []byte - switch major { - case majorTypeUnsignedInt: - fallthrough - case majorTypeNegativeInt: - var n int64 - n, bc, err = decodeInteger(src) - dst.Write([]byte(strconv.Itoa(int(n)))) - - case majorTypeByteString: - s, bc, err = decodeString(src, false) - dst.Write(s) - - case majorTypeUtf8String: - s, bc, err = decodeUTF8String(src) - dst.Write(s) - - case majorTypeArray: - bc, err = array2Json(src, dst) - - case majorTypeMap: - bc, err = map2Json(src, dst) - - case majorTypeTags: - s, bc, err = decodeTagData(src) - dst.Write(s) - - case majorTypeSimpleAndFloat: - s, bc, err = decodeSimpleFloat(src) - dst.Write(s) - } - return bc, err -} - -// Cbor2JsonManyObjects decodes all the CBOR Objects present in the -// source byte array. It keeps on decoding until it runs out of bytes. -// Decoded string is written to the dst. At the end of every CBOR Object -// newline is written to the output stream. -// Returns the number of bytes decoded and if any error was encountered. -func Cbor2JsonManyObjects(src []byte, dst io.Writer) (uint, error) { - curPos := uint(0) - totalBytes := uint(len(src)) - for curPos < totalBytes { - bc, err := Cbor2JsonOneObject(src[curPos:], dst) - if err != nil { - return curPos, err - } - dst.Write([]byte("\n")) - curPos += bc - } - return curPos, nil -} - -// Detect if the bytes to be printed is Binary or not. -func binaryFmt(p []byte) bool { - if len(p) > 0 && p[0] > 0x7F { - return true - } - return false -} - -// DecodeIfBinaryToString converts a binary formatted log msg to a -// JSON formatted String Log message - suitable for printing to Console/Syslog. -func DecodeIfBinaryToString(in []byte) string { - if binaryFmt(in) { - var b bytes.Buffer - Cbor2JsonManyObjects(in, &b) - return b.String() - } - return string(in) -} - -// DecodeObjectToStr checks if the input is a binary format, if so, -// it will decode a single Object and return the decoded string. -func DecodeObjectToStr(in []byte) string { - if binaryFmt(in) { - var b bytes.Buffer - Cbor2JsonOneObject(in, &b) - return b.String() - } - return string(in) -} - -// DecodeIfBinaryToBytes checks if the input is a binary format, if so, -// it will decode all Objects and return the decoded string as byte array. -func DecodeIfBinaryToBytes(in []byte) []byte { - if binaryFmt(in) { - var b bytes.Buffer - Cbor2JsonManyObjects(in, &b) - return b.Bytes() - } - return in -} diff --git a/internal/cbor/decoder_test.go b/internal/cbor/decoder_test.go index b17ba11..812e27f 100644 --- a/internal/cbor/decoder_test.go +++ b/internal/cbor/decoder_test.go @@ -9,8 +9,8 @@ import ( func TestDecodeInteger(t *testing.T) { for _, tc := range integerTestCases { - gotv, gotc, err := decodeInteger([]byte(tc.binary)) - if gotv != int64(tc.val) || int(gotc) != len(tc.binary) || err != nil { + gotv := decodeInteger(getReader(tc.binary)) + if gotv != int64(tc.val) { t.Errorf("decodeInteger(0x%s)=0x%d, want: 0x%d", hex.EncodeToString([]byte(tc.binary)), gotv, tc.val) } @@ -19,10 +19,7 @@ func TestDecodeInteger(t *testing.T) { func TestDecodeString(t *testing.T) { for _, tt := range encodeStringTests { - got, _, err := decodeUTF8String([]byte(tt.binary)) - if err != nil { - t.Errorf("Got Error for the case: %s", hex.EncodeToString([]byte(tt.binary))) - } + got := decodeUTF8String(getReader(tt.binary)) if string(got) != "\""+tt.json+"\"" { t.Errorf("DecodeString(0x%s)=%s, want:\"%s\"\n", hex.EncodeToString([]byte(tt.binary)), string(got), hex.EncodeToString([]byte(tt.json))) @@ -33,10 +30,7 @@ func TestDecodeString(t *testing.T) { func TestDecodeArray(t *testing.T) { for _, tc := range integerArrayTestCases { buf := bytes.NewBuffer([]byte{}) - _, err := array2Json([]byte(tc.binary), buf) - if err != nil { - panic(err) - } + array2Json(getReader(tc.binary), buf) if buf.String() != tc.json { t.Errorf("array2Json(0x%s)=%s, want: %s", hex.EncodeToString([]byte(tc.binary)), buf.String(), tc.json) } @@ -54,20 +48,14 @@ func TestDecodeArray(t *testing.T) { } for _, tc := range infiniteArrayTestCases { buf := bytes.NewBuffer([]byte{}) - _, err := array2Json([]byte(tc.in), buf) - if err != nil { - panic(err) - } + array2Json(getReader(tc.in), buf) if buf.String() != tc.out { t.Errorf("array2Json(0x%s)=%s, want: %s", hex.EncodeToString([]byte(tc.out)), buf.String(), tc.out) } } for _, tc := range booleanArrayTestCases { buf := bytes.NewBuffer([]byte{}) - _, err := array2Json([]byte(tc.binary), buf) - if err != nil { - t.Errorf("array2Json(0x%s) errored out: %s", hex.EncodeToString([]byte(tc.binary)), err.Error()) - } + array2Json(getReader(tc.binary), buf) if buf.String() != tc.json { t.Errorf("array2Json(0x%s)=%s, want: %s", hex.EncodeToString([]byte(tc.binary)), buf.String(), tc.json) } @@ -94,20 +82,14 @@ var mapDecodeTestCases = []struct { func TestDecodeMap(t *testing.T) { for _, tc := range mapDecodeTestCases { buf := bytes.NewBuffer([]byte{}) - _, err := map2Json(tc.bin, buf) - if err != nil { - t.Errorf("map2Json(0x%s) returned error", err) - } + map2Json(getReader(string(tc.bin)), buf) if buf.String() != tc.json { t.Errorf("map2Json(0x%s)=%s, want: %s", hex.EncodeToString(tc.bin), buf.String(), tc.json) } } for _, tc := range infiniteMapDecodeTestCases { buf := bytes.NewBuffer([]byte{}) - _, err := map2Json(tc.bin, buf) - if err != nil { - t.Errorf("map2Json(0x%s) returned error", err) - } + map2Json(getReader(string(tc.bin)), buf) if buf.String() != tc.json { t.Errorf("map2Json(0x%s)=%s, want: %s", hex.EncodeToString(tc.bin), buf.String(), tc.json) } @@ -116,10 +98,7 @@ func TestDecodeMap(t *testing.T) { func TestDecodeBool(t *testing.T) { for _, tc := range booleanTestCases { - got, _, err := decodeSimpleFloat([]byte(tc.binary)) - if err != nil { - t.Errorf("decodeSimpleFloat(0x%s) errored %s", hex.EncodeToString([]byte(tc.binary)), err.Error()) - } + got := decodeSimpleFloat(getReader(tc.binary)) if string(got) != tc.json { t.Errorf("decodeSimpleFloat(0x%s)=%s, want:%s", hex.EncodeToString([]byte(tc.binary)), string(got), tc.json) } @@ -128,10 +107,7 @@ func TestDecodeBool(t *testing.T) { func TestDecodeFloat(t *testing.T) { for _, tc := range float32TestCases { - got, _, err := decodeFloat([]byte(tc.binary)) - if err != nil { - t.Errorf("decodeFloat(0x%s) returned error: %s", hex.EncodeToString([]byte(tc.binary)), err.Error()) - } + got, _ := decodeFloat(getReader(tc.binary)) if got != float64(tc.val) { t.Errorf("decodeFloat(0x%s)=%f, want:%f", hex.EncodeToString([]byte(tc.binary)), got, tc.val) } @@ -141,19 +117,13 @@ func TestDecodeFloat(t *testing.T) { func TestDecodeTimestamp(t *testing.T) { decodeTimeZone, _ = time.LoadLocation("UTC") for _, tc := range timeIntegerTestcases { - tm, _, err := decodeTagData([]byte(tc.binary)) - if err != nil { - t.Errorf("decodeTagData(0x%s) returned error: %s", hex.EncodeToString([]byte(tc.binary)), err.Error()) - } + tm := decodeTagData(getReader(tc.binary)) if string(tm) != "\""+tc.rfcStr+"\"" { t.Errorf("decodeFloat(0x%s)=%s, want:%s", hex.EncodeToString([]byte(tc.binary)), tm, tc.rfcStr) } } for _, tc := range timeFloatTestcases { - tm, _, err := decodeTagData([]byte(tc.out)) - if err != nil { - t.Errorf("decodeTagData(0x%s) returned error: %s", hex.EncodeToString([]byte(tc.out)), err.Error()) - } + tm := decodeTagData(getReader(tc.out)) //Since we convert to float and back - it may be slightly off - so //we cannot check for exact equality instead, we'll check it is //very close to each other Less than a Microsecond (lets not yet do nanosec) @@ -166,6 +136,33 @@ func TestDecodeTimestamp(t *testing.T) { } } +func TestDecodeNetworkAddr(t *testing.T) { + for _, tc := range ipAddrTestCases { + d1 := decodeTagData(getReader(tc.binary)) + if string(d1) != tc.text { + t.Errorf("decodeNetworkAddr(0x%s)=%s, want:%s", hex.EncodeToString([]byte(tc.binary)), d1, tc.text) + } + } +} + +func TestDecodeMACAddr(t *testing.T) { + for _, tc := range macAddrTestCases { + d1 := decodeTagData(getReader(tc.binary)) + if string(d1) != tc.text { + t.Errorf("decodeNetworkAddr(0x%s)=%s, want:%s", hex.EncodeToString([]byte(tc.binary)), d1, tc.text) + } + } +} + +func TestDecodeIPPrefix(t *testing.T) { + for _, tc := range IPPrefixTestCases { + d1 := decodeTagData(getReader(tc.binary)) + if string(d1) != tc.text { + t.Errorf("decodeIPPrefix(0x%s)=%s, want:%s", hex.EncodeToString([]byte(tc.binary)), d1, tc.text) + } + } +} + var compositeCborTestCases = []struct { binary []byte json string @@ -177,12 +174,32 @@ var compositeCborTestCases = []struct { func TestDecodeCbor2Json(t *testing.T) { for _, tc := range compositeCborTestCases { buf := bytes.NewBuffer([]byte{}) - _, err := Cbor2JsonManyObjects(tc.binary, buf) - if err != nil { - t.Errorf("cbor2JsonManyObjects(0x%s) returned error", err) - } - if buf.String() != tc.json { - t.Errorf("cbor2JsonManyObjects(0x%s)=%s, want: %s", hex.EncodeToString(tc.binary), buf.String(), tc.json) + err := Cbor2JsonManyObjects(getReader(string(tc.binary)), buf) + if buf.String() != tc.json || err != nil { + t.Errorf("cbor2JsonManyObjects(0x%s)=%s, want: %s, err:%s", hex.EncodeToString(tc.binary), buf.String(), tc.json, err.Error()) + } + } +} + +var negativeCborTestCases = []struct { + binary []byte + errStr string +}{ + {[]byte("\xb9\x64IETF\x20\x65Array\x9f\x20\x00\x18\xc8\x14"), "Tried to Read 18 Bytes.. But hit end of file"}, + {[]byte("\xbf\x64IETF\x20\x65Array\x9f\x20\x00\x18\xc8\x14"), "EOF"}, + {[]byte("\xbf\x14IETF\x20\x65Array\x9f\x20\x00\x18\xc8\x14"), "Tried to Read 40736 Bytes.. But hit end of file"}, + {[]byte("\xbf\x64IETF"), "EOF"}, + {[]byte("\xbf\x64IETF\x20\x65Array\x9f\x20\x00\x18\xc8\xff\xff\xff"), "Invalid Additional Type: 31 in decodeSimpleFloat"}, + {[]byte("\xbf\x64IETF\x20\x65Array"), "EOF"}, + {[]byte("\xbf\x64"), "Tried to Read 4 Bytes.. But hit end of file"}, +} + +func TestDecodeNegativeCbor2Json(t *testing.T) { + for _, tc := range negativeCborTestCases { + buf := bytes.NewBuffer([]byte{}) + err := Cbor2JsonManyObjects(getReader(string(tc.binary)), buf) + if err == nil || err.Error() != tc.errStr { + t.Errorf("Expected error got:%s, want:%s", err, tc.errStr) } } } diff --git a/internal/cbor/examples/genLog.go b/internal/cbor/examples/genLog.go new file mode 100644 index 0000000..43a2000 --- /dev/null +++ b/internal/cbor/examples/genLog.go @@ -0,0 +1,55 @@ +package main + +import ( + "compress/zlib" + "flag" + "io" + "log" + "os" + "time" + + "github.com/rs/zerolog" +) + +func writeLog(fname string, count int, useCompress bool) { + opFile := os.Stdout + if fname != "" { + fil, _ := os.Create(fname) + opFile = fil + defer func() { + if err := fil.Close(); err != nil { + log.Fatal(err) + } + }() + } + + var f io.WriteCloser = opFile + if useCompress { + f = zlib.NewWriter(f) + defer func() { + if err := f.Close(); err != nil { + log.Fatal(err) + } + }() + + } + + zerolog.TimestampFunc = func() time.Time { return time.Now().Round(time.Second) } + log := zerolog.New(f).With(). + Timestamp(). + Logger() + for i := 0; i < count; i++ { + log.Error(). + Int("Fault", 41650+i).Msg("Some Message") + } +} + +func main() { + outFile := flag.String("out", "", "Output File to which logs will be written to (WILL overwrite if already present).") + numLogs := flag.Int("num", 10, "Number of log messages to generate.") + doCompress := flag.Bool("compress", false, "Enable inline compressed writer") + + flag.Parse() + + writeLog(*outFile, *numLogs, *doCompress) +} diff --git a/internal/cbor/examples/makefile b/internal/cbor/examples/makefile new file mode 100644 index 0000000..d28075d --- /dev/null +++ b/internal/cbor/examples/makefile @@ -0,0 +1,10 @@ +all: genLogJSON genLogCBOR + +genLogJSON: genLog.go + go build -o genLogJSON genLog.go + +genLogCBOR: genLog.go + go build -tags binary_log -o genLogCBOR genLog.go + +clean: + rm -f genLogJSON genLogCBOR diff --git a/internal/cbor/string.go b/internal/cbor/string.go index b90edac..8b3ab96 100644 --- a/internal/cbor/string.go +++ b/internal/cbor/string.go @@ -48,8 +48,13 @@ func AppendBytes(dst, s []byte) []byte { func AppendEmbeddedJSON(dst, s []byte) []byte { major := majorTypeTags minor := additionalTypeEmbeddedJSON - dst = append(dst, byte(major|minor)) + // Append the TAG to indicate this is Embedded JSON. + dst = append(dst, byte(major|additionalTypeIntUint16)) + dst = append(dst, byte(minor>>8)) + dst = append(dst, byte(minor&0xff)) + + // Append the JSON Object as Byte String. major = majorTypeByteString l := len(s) diff --git a/internal/cbor/types.go b/internal/cbor/types.go index b983ed7..9b1216d 100644 --- a/internal/cbor/types.go +++ b/internal/cbor/types.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "math" + "net" ) // AppendNull inserts a 'Nil' object into the dst byte array. @@ -430,9 +431,40 @@ func AppendArrayDelim(dst []byte) []byte { return dst } -func AppendHex (dst []byte, val []byte) []byte { - dst = append(dst, byte(majorTypeTags|additionalTypeIntUint16)) - dst = append(dst, byte(additionalTypeTagHexString>>8)) - dst = append(dst, byte(additionalTypeTagHexString&0xff)) - return AppendBytes(dst, val) +// AppendIPAddr encodes and inserts an IP Address (IPv4 or IPv6). +func AppendIPAddr(dst []byte, ip net.IP) []byte { + dst = append(dst, byte(majorTypeTags|additionalTypeIntUint16)) + dst = append(dst, byte(additionalTypeTagNetworkAddr>>8)) + dst = append(dst, byte(additionalTypeTagNetworkAddr&0xff)) + return AppendBytes(dst, ip) +} + +// AppendIPPrefix encodes and inserts an IP Address Prefix (Address + Mask Length). +func AppendIPPrefix(dst []byte, pfx net.IPNet) []byte { + dst = append(dst, byte(majorTypeTags|additionalTypeIntUint16)) + dst = append(dst, byte(additionalTypeTagNetworkPrefix>>8)) + dst = append(dst, byte(additionalTypeTagNetworkPrefix&0xff)) + + // Prefix is a tuple (aka MAP of 1 pair of elements) - + // first element is prefix, second is mask length. + dst = append(dst, byte(majorTypeMap|0x1)) + dst = AppendBytes(dst, pfx.IP) + maskLen, _ := pfx.Mask.Size() + return AppendUint8(dst, uint8(maskLen)) +} + +// AppendMACAddr encodes and inserts an Hardware (MAC) address. +func AppendMACAddr(dst []byte, ha net.HardwareAddr) []byte { + dst = append(dst, byte(majorTypeTags|additionalTypeIntUint16)) + dst = append(dst, byte(additionalTypeTagNetworkAddr>>8)) + dst = append(dst, byte(additionalTypeTagNetworkAddr&0xff)) + return AppendBytes(dst, ha) +} + +// AppendHex adds a TAG and inserts a hex bytes as a string. +func AppendHex(dst []byte, val []byte) []byte { + dst = append(dst, byte(majorTypeTags|additionalTypeIntUint16)) + dst = append(dst, byte(additionalTypeTagHexString>>8)) + dst = append(dst, byte(additionalTypeTagHexString&0xff)) + return AppendBytes(dst, val) } diff --git a/internal/cbor/types_test.go b/internal/cbor/types_test.go index 010e2a0..b3a1094 100644 --- a/internal/cbor/types_test.go +++ b/internal/cbor/types_test.go @@ -2,6 +2,7 @@ package cbor import ( "encoding/hex" + "net" "testing" ) @@ -182,6 +183,72 @@ func TestAppendFloat32(t *testing.T) { } } +var ipAddrTestCases = []struct { + ipaddr net.IP + text string // ASCII representation of ipaddr + binary string // CBOR representation of ipaddr +}{ + {net.IP{10, 0, 0, 1}, "\"10.0.0.1\"", "\xd9\x01\x04\x44\x0a\x00\x00\x01"}, + {net.IP{0x20, 0x01, 0x0d, 0xb8, 0x85, 0xa3, 0x0, 0x0, 0x0, 0x0, 0x8a, 0x2e, 0x03, 0x70, 0x73, 0x34}, + "\"2001:db8:85a3::8a2e:370:7334\"", + "\xd9\x01\x04\x50\x20\x01\x0d\xb8\x85\xa3\x00\x00\x00\x00\x8a\x2e\x03\x70\x73\x34"}, +} + +func TestAppendNetworkAddr(t *testing.T) { + for _, tc := range ipAddrTestCases { + s := AppendIPAddr([]byte{}, tc.ipaddr) + got := string(s) + if got != tc.binary { + t.Errorf("AppendIPAddr(%s)=0x%s, want: 0x%s", + tc.ipaddr, hex.EncodeToString(s), + hex.EncodeToString([]byte(tc.binary))) + } + } +} + +var macAddrTestCases = []struct { + macaddr net.HardwareAddr + text string // ASCII representation of macaddr + binary string // CBOR representation of macaddr +}{ + {net.HardwareAddr{0x12, 0x34, 0x56, 0x78, 0x90, 0xab}, "\"12:34:56:78:90:ab\"", "\xd9\x01\x04\x46\x12\x34\x56\x78\x90\xab"}, + {net.HardwareAddr{0x20, 0x01, 0x0d, 0xb8, 0x85, 0xa3}, "\"20:01:0d:b8:85:a3\"", "\xd9\x01\x04\x46\x20\x01\x0d\xb8\x85\xa3"}, +} + +func TestAppendMacAddr(t *testing.T) { + for _, tc := range macAddrTestCases { + s := AppendMACAddr([]byte{}, tc.macaddr) + got := string(s) + if got != tc.binary { + t.Errorf("AppendMACAddr(%s)=0x%s, want: 0x%s", + tc.macaddr.String(), hex.EncodeToString(s), + hex.EncodeToString([]byte(tc.binary))) + } + } +} + +var IPPrefixTestCases = []struct { + pfx net.IPNet + text string // ASCII representation of pfx + binary string // CBOR representation of pfx +}{ + {net.IPNet{IP: net.IP{0, 0, 0, 0}, Mask: net.CIDRMask(0, 32)}, "\"0.0.0.0/0\"", "\xd9\x01\x05\xa1\x44\x00\x00\x00\x00\x00"}, + {net.IPNet{IP: net.IP{192, 168, 0, 100}, Mask: net.CIDRMask(24, 32)}, "\"192.168.0.100/24\"", + "\xd9\x01\x05\xa1\x44\xc0\xa8\x00\x64\x18\x18"}, +} + +func TestAppendIPPrefix(t *testing.T) { + for _, tc := range IPPrefixTestCases { + s := AppendIPPrefix([]byte{}, tc.pfx) + got := string(s) + if got != tc.binary { + t.Errorf("AppendIPPrefix(%s)=0x%s, want: 0x%s", + tc.pfx.String(), hex.EncodeToString(s), + hex.EncodeToString([]byte(tc.binary))) + } + } +} + func BenchmarkAppendInt(b *testing.B) { type st struct { sz byte diff --git a/internal/json/types.go b/internal/json/types.go index 927d1c8..1beff1f 100644 --- a/internal/json/types.go +++ b/internal/json/types.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "math" + "net" "strconv" ) @@ -333,6 +334,8 @@ func AppendInterface(dst []byte, i interface{}) []byte { return append(dst, marshaled...) } +// AppendObjectData takes in an object that is already in a byte array +// and adds it to the dst. func AppendObjectData(dst []byte, o []byte) []byte { // Two conditions we want to put a ',' between existing content and // new content: @@ -345,3 +348,19 @@ func AppendObjectData(dst []byte, o []byte) []byte { } return append(dst, o...) } + +// AppendIPAddr adds IPv4 or IPv6 address to dst. +func AppendIPAddr(dst []byte, ip net.IP) []byte { + return AppendString(dst, ip.String()) +} + +// AppendIPPrefix adds IPv4 or IPv6 Prefix (address & mask) to dst. +func AppendIPPrefix(dst []byte, pfx net.IPNet) []byte { + return AppendString(dst, pfx.String()) + +} + +// AppendMACAddr adds MAC address to dst. +func AppendMACAddr(dst []byte, ha net.HardwareAddr) []byte { + return AppendString(dst, ha.String()) +} diff --git a/internal/json/types_test.go b/internal/json/types_test.go index d90dac1..719649d 100644 --- a/internal/json/types_test.go +++ b/internal/json/types_test.go @@ -2,6 +2,7 @@ package json import ( "math" + "net" "reflect" "testing" ) @@ -61,3 +62,106 @@ func TestAppendType(t *testing.T) { }) } } + +func Test_appendMAC(t *testing.T) { + MACtests := []struct { + input string + want []byte + }{ + {"01:23:45:67:89:ab", []byte(`"01:23:45:67:89:ab"`)}, + {"cd:ef:11:22:33:44", []byte(`"cd:ef:11:22:33:44"`)}, + } + for _, tt := range MACtests { + t.Run("MAC", func(t *testing.T) { + ha, _ := net.ParseMAC(tt.input) + if got := AppendMACAddr([]byte{}, ha); !reflect.DeepEqual(got, tt.want) { + t.Errorf("appendMACAddr() = %s, want %s", got, tt.want) + } + }) + } +} + +func Test_appendIP(t *testing.T) { + IPv4tests := []struct { + input net.IP + want []byte + }{ + {net.IP{0, 0, 0, 0}, []byte(`"0.0.0.0"`)}, + {net.IP{192, 0, 2, 200}, []byte(`"192.0.2.200"`)}, + } + + for _, tt := range IPv4tests { + t.Run("IPv4", func(t *testing.T) { + if got := AppendIPAddr([]byte{}, tt.input); !reflect.DeepEqual(got, tt.want) { + t.Errorf("appendIPAddr() = %s, want %s", got, tt.want) + } + }) + } + IPv6tests := []struct { + input net.IP + want []byte + }{ + {net.IPv6zero, []byte(`"::"`)}, + {net.IPv6linklocalallnodes, []byte(`"ff02::1"`)}, + {net.IP{0x20, 0x01, 0x0d, 0xb8, 0x85, 0xa3, 0x00, 0x00, 0x00, 0x00, 0x8a, 0x2e, 0x03, 0x70, 0x73, 0x34}, []byte(`"2001:db8:85a3::8a2e:370:7334"`)}, + } + for _, tt := range IPv6tests { + t.Run("IPv6", func(t *testing.T) { + if got := AppendIPAddr([]byte{}, tt.input); !reflect.DeepEqual(got, tt.want) { + t.Errorf("appendIPAddr() = %s, want %s", got, tt.want) + } + }) + } +} + +func Test_appendIPPrefix(t *testing.T) { + IPv4Prefixtests := []struct { + input net.IPNet + want []byte + }{ + {net.IPNet{IP: net.IP{0, 0, 0, 0}, Mask: net.IPv4Mask(0, 0, 0, 0)}, []byte(`"0.0.0.0/0"`)}, + {net.IPNet{IP: net.IP{192, 0, 2, 200}, Mask: net.IPv4Mask(255, 255, 255, 0)}, []byte(`"192.0.2.200/24"`)}, + } + for _, tt := range IPv4Prefixtests { + t.Run("IPv4", func(t *testing.T) { + if got := AppendIPPrefix([]byte{}, tt.input); !reflect.DeepEqual(got, tt.want) { + t.Errorf("appendIPPrefix() = %s, want %s", got, tt.want) + } + }) + } + IPv6Prefixtests := []struct { + input net.IPNet + want []byte + }{ + {net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, 128)}, []byte(`"::/0"`)}, + {net.IPNet{IP: net.IPv6linklocalallnodes, Mask: net.CIDRMask(128, 128)}, []byte(`"ff02::1/128"`)}, + {net.IPNet{IP: net.IP{0x20, 0x01, 0x0d, 0xb8, 0x85, 0xa3, 0x00, 0x00, 0x00, 0x00, 0x8a, 0x2e, 0x03, 0x70, 0x73, 0x34}, + Mask: net.CIDRMask(64, 128)}, + []byte(`"2001:db8:85a3::8a2e:370:7334/64"`)}, + } + for _, tt := range IPv6Prefixtests { + t.Run("IPv6", func(t *testing.T) { + if got := AppendIPPrefix([]byte{}, tt.input); !reflect.DeepEqual(got, tt.want) { + t.Errorf("appendIPPrefix() = %s, want %s", got, tt.want) + } + }) + } +} + +func Test_appendMac(t *testing.T) { + MACtests := []struct { + input net.HardwareAddr + want []byte + }{ + {net.HardwareAddr{0x12, 0x34, 0x56, 0x78, 0x90, 0xab}, []byte(`"12:34:56:78:90:ab"`)}, + {net.HardwareAddr{0x12, 0x34, 0x00, 0x00, 0x90, 0xab}, []byte(`"12:34:00:00:90:ab"`)}, + } + + for _, tt := range MACtests { + t.Run("MAC", func(t *testing.T) { + if got := AppendMACAddr([]byte{}, tt.input); !reflect.DeepEqual(got, tt.want) { + t.Errorf("appendMAC() = %s, want %s", got, tt.want) + } + }) + } +} diff --git a/log_example_test.go b/log_example_test.go index 9a1b344..5d541b3 100644 --- a/log_example_test.go +++ b/log_example_test.go @@ -5,6 +5,7 @@ package zerolog_test import ( "errors" stdlog "log" + "net" "os" "time" @@ -395,3 +396,36 @@ func ExampleContext_Durs() { // Output: {"foo":"bar","durs":[10000,20000],"message":"hello world"} } + +func ExampleContext_IPAddr() { + hostIP := net.IP{192, 168, 0, 100} + log := zerolog.New(os.Stdout).With(). + IPAddr("HostIP", hostIP). + Logger() + + log.Log().Msg("hello world") + + // Output: {"HostIP":"192.168.0.100","message":"hello world"} +} + +func ExampleContext_IPPrefix() { + route := net.IPNet{IP: net.IP{192, 168, 0, 0}, Mask: net.CIDRMask(24, 32)} + log := zerolog.New(os.Stdout).With(). + IPPrefix("Route", route). + Logger() + + log.Log().Msg("hello world") + + // Output: {"Route":"192.168.0.0/24","message":"hello world"} +} + +func ExampleContext_MacAddr() { + mac := net.HardwareAddr{0x00, 0x14, 0x22, 0x01, 0x23, 0x45} + log := zerolog.New(os.Stdout).With(). + MACAddr("hostMAC", mac). + Logger() + + log.Log().Msg("hello world") + + // Output: {"hostMAC":"00:14:22:01:23:45","message":"hello world"} +} diff --git a/log_test.go b/log_test.go index 339f459..ebe716e 100644 --- a/log_test.go +++ b/log_test.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "fmt" + "net" "reflect" "runtime" "testing" @@ -127,10 +128,11 @@ func TestFieldsMap(t *testing.T) { "uint64": uint64(10), "float32": float32(11), "float64": float64(12), + "ipv6": net.IP{0x20, 0x01, 0x0d, 0xb8, 0x85, 0xa3, 0x00, 0x00, 0x00, 0x00, 0x8a, 0x2e, 0x03, 0x70, 0x73, 0x34}, "dur": 1 * time.Second, "time": time.Time{}, }).Msg("") - if got, want := decodeIfBinaryToString(out.Bytes()), `{"bool":true,"bytes":"bar","dur":1000,"error":"some error","float32":11,"float64":12,"int":1,"int16":3,"int32":4,"int64":5,"int8":2,"nil":null,"string":"foo","time":"0001-01-01T00:00:00Z","uint":6,"uint16":8,"uint32":9,"uint64":10,"uint8":7}`+"\n"; got != want { + if got, want := decodeIfBinaryToString(out.Bytes()), `{"bool":true,"bytes":"bar","dur":1000,"error":"some error","float32":11,"float64":12,"int":1,"int16":3,"int32":4,"int64":5,"int8":2,"ipv6":"2001:db8:85a3::8a2e:370:7334","nil":null,"string":"foo","time":"0001-01-01T00:00:00Z","uint":6,"uint16":8,"uint32":9,"uint64":10,"uint8":7}`+"\n"; got != want { t.Errorf("invalid log output:\ngot: %v\nwant: %v", got, want) } } @@ -186,13 +188,17 @@ func TestFields(t *testing.T) { Uint16("uint16", 8). Uint32("uint32", 9). Uint64("uint64", 10). + IPAddr("IPv4", net.IP{192, 168, 0, 100}). + IPAddr("IPv6", net.IP{0x20, 0x01, 0x0d, 0xb8, 0x85, 0xa3, 0x00, 0x00, 0x00, 0x00, 0x8a, 0x2e, 0x03, 0x70, 0x73, 0x34}). + MACAddr("Mac", net.HardwareAddr{0x00, 0x14, 0x22, 0x01, 0x23, 0x45}). + IPPrefix("Prefix", net.IPNet{IP: net.IP{192, 168, 0, 100}, Mask: net.CIDRMask(24, 32)}). Float32("float32", 11). Float64("float64", 12). Dur("dur", 1*time.Second). Time("time", time.Time{}). TimeDiff("diff", now, now.Add(-10*time.Second)). Msg("") - if got, want := decodeIfBinaryToString(out.Bytes()), `{"caller":"`+caller+`","string":"foo","bytes":"bar","hex":"12ef","json":{"some":"json"},"error":"some error","bool":true,"int":1,"int8":2,"int16":3,"int32":4,"int64":5,"uint":6,"uint8":7,"uint16":8,"uint32":9,"uint64":10,"float32":11,"float64":12,"dur":1000,"time":"0001-01-01T00:00:00Z","diff":10000}`+"\n"; got != want { + if got, want := decodeIfBinaryToString(out.Bytes()), `{"caller":"`+caller+`","string":"foo","bytes":"bar","hex":"12ef","json":{"some":"json"},"error":"some error","bool":true,"int":1,"int8":2,"int16":3,"int32":4,"int64":5,"uint":6,"uint8":7,"uint16":8,"uint32":9,"uint64":10,"IPv4":"192.168.0.100","IPv6":"2001:db8:85a3::8a2e:370:7334","Mac":"00:14:22:01:23:45","Prefix":"192.168.0.100/24","float32":11,"float64":12,"dur":1000,"time":"0001-01-01T00:00:00Z","diff":10000}`+"\n"; got != want { t.Errorf("invalid log output:\ngot: %v\nwant: %v", got, want) } }