initial commit

author: Luke Shumaker <lukeshu@lukeshu.com> 2020-01-26 09:05:07 -0500
committer: Luke Shumaker <lukeshu@lukeshu.com> 2020-01-26 09:05:07 -0500
commit: a5cba20c1e3b0956737327ef9214fd2cbc221add (patch)
tree: fc19e2d97243449676fc5d2a5a6f57ad356010e6
6 files changed, 311 insertions, 0 deletions
diff --git a/go.mod b/go.mod
new file mode 100644
index 0000000..2989deb
--- /dev/null
+++ b/go.mod
@@ -0,0 +1,5 @@
+module git.lukeshu.com/go/librrd
+
+go 1.13
+
+require github.com/stretchr/testify v1.4.0
diff --git a/go.sum b/go.sum
new file mode 100644
index 0000000..e863f51
--- /dev/null
+++ b/go.sum
@@ -0,0 +1,10 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/rrdformat/errors_binary.go b/rrdformat/errors_binary.go
new file mode 100644
index 0000000..7329927
--- /dev/null
+++ b/rrdformat/errors_binary.go
@@ -0,0 +1,75 @@
+package rrdformat
+
+import (
+	"fmt"
+	"io"
+)
+
+type BinaryError struct {
+	msg    string
+	ctxPos int
+	ctxDat []byte
+	ctxEOF bool
+}
+
+func newBinError(msg string, ctxFile []byte, ctxStart, ctxLen int) error {
+	if ctxStart+ctxLen > len(ctxFile) {
+		ctxLen = len(ctxFile) - ctxStart
+	}
+	return BinaryError{
+		msg:    msg,
+		ctxPos: ctxStart,
+		ctxDat: ctxFile[ctxStart : ctxStart+ctxLen],
+		ctxEOF: ctxStart+ctxLen == len(ctxFile),
+	}
+}
+
+func (e BinaryError) Error() string {
+	return "invalid RRD: " + e.msg
+}
+
+var cAsciiEscapes = map[byte]byte{
+	0x00: '0',
+	0x07: 'a',
+	0x08: 'b',
+	0x09: 't',
+	0x0A: 'n',
+	0x0B: 'v',
+	0x0C: 'f',
+	0x0D: 'r',
+}
+
+func (e BinaryError) Format(s fmt.State, verb rune) {
+	switch verb {
+	case 'v':
+		io.WriteString(s, e.Error())
+		if s.Flag('+') {
+			fmt.Fprintf(s, "\n\tat byte %d:", e.ctxPos)
+			io.WriteString(s, "\n\t\tascii:")
+			for _, byte := range e.ctxDat {
+				if ' ' <= byte && byte <= '~' {
+					fmt.Fprintf(s, "  %c", byte)
+				} else if c, ok := cAsciiEscapes[byte]; ok {
+					fmt.Fprintf(s, " \\%c", c)
+				} else {
+					io.WriteString(s, " ??")
+				}
+			}
+			if e.ctxEOF {
+				io.WriteString(s, " <EOF>")
+			}
+			io.WriteString(s, "\n\t\thex  :")
+			for _, byte := range e.ctxDat {
+				fmt.Fprintf(s, " %02x", byte)
+			}
+			if e.ctxEOF {
+				io.WriteString(s, " <EOF>")
+			}
+			io.WriteString(s, "\n")
+		}
+	case 's':
+		io.WriteString(s, e.Error())
+	case 'q':
+		fmt.Fprintf(s, "%q", e.Error())
+	}
+}
diff --git a/rrdformat/errors_binary_test.go b/rrdformat/errors_binary_test.go
new file mode 100644
index 0000000..f420a1b
--- /dev/null
+++ b/rrdformat/errors_binary_test.go
@@ -0,0 +1,34 @@
+package rrdformat
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBinaryError(t *testing.T) {
+	assert := assert.New(t)
+
+	bad404 := []byte(`<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"`)
+	err := newBinError("not an RRD file: wrong magic number", bad404, 0, 4)
+	assert.Equal(err.Error(), `invalid RRD: not an RRD file: wrong magic number`)
+	assert.Equal(fmt.Sprintf("%v", err), `invalid RRD: not an RRD file: wrong magic number`)
+	assert.Equal(fmt.Sprintf("%q", err), `"invalid RRD: not an RRD file: wrong magic number"`)
+	assert.Equal(fmt.Sprintf("%+v", err), `invalid RRD: not an RRD file: wrong magic number
+	at byte 0:
+		ascii:  <  !  D  O
+		hex  : 3c 21 44 4f
+`)
+
+	badShort := []byte{'R'}
+	err = newBinError("not an RRD file: wrong magic number", badShort, 0, 4)
+	assert.Equal(err.Error(), `invalid RRD: not an RRD file: wrong magic number`)
+	assert.Equal(fmt.Sprintf("%v", err), `invalid RRD: not an RRD file: wrong magic number`)
+	assert.Equal(fmt.Sprintf("%q", err), `"invalid RRD: not an RRD file: wrong magic number"`)
+	assert.Equal(fmt.Sprintf("%+v", err), `invalid RRD: not an RRD file: wrong magic number
+	at byte 0:
+		ascii:  R <EOF>
+		hex  : 52 <EOF>
+`)
+}
diff --git a/rrdformat/format.go b/rrdformat/format.go
new file mode 100644
index 0000000..0233a5c
--- /dev/null
+++ b/rrdformat/format.go
@@ -0,0 +1,173 @@
+package rrdformat
+
+import (
+	"bytes"
+	"encoding"
+	"encoding/binary"
+	"encoding/xml"
+	"math"
+)
+
+type Unival uint64
+
+func (u Unival) AsUint64() uint64   { return uint64(u) }
+func (u Unival) AsFloat64() float64 { return math.Float64frombits(uint64(u)) }
+
+const XMLNS = "https://oss.oetiker.ch/rrdtool/rrdtool-dump.xml"
+
+// rrdtool:
+//   rrd_format.h: stat_head_t -- static header of the database
+//
+// javascriptRRD:
+//   rrdFile.js: RRDHeader
+type Header struct {
+	// identification section
+	Cookie  []byte // 4 bytes
+	Version []byte // 4 bytes (eh, we let the \0-terminator decided how long)
+	// padding
+	FloatCookie []byte // 8 bytes
+
+	Bytes
+
+	// structure definition
+	DSCnt      uint64 // how many different DS provide input to the RRD
+	RRACnt     uint64 // how many RRAs will be maintained in the RRD
+	PDPStep    uint64 // PDP interval in seconds
+	Parameters [10]Unival
+
+	// sniffed metadata
+	ByteOrder   binary.ByteOrder
+	FloatWidth  int
+	FloatAlign  int
+	IntWidth    int
+	IntAlign    int
+	UnivalWidth int
+	UnivalAlign int
+}
+
+func (h *Header) UnmarshalBinary(data []byte) error {
+	// magic number cookie
+	if !bytes.HasPrefix(data, []byte("RRD\x00")) {
+		return newBinError("not an RRD file: wrong magic number", data, 0, 4)
+	}
+	h.Cookie = data[0:4]
+
+	// version string
+	null := bytes.IndexByte(data[4:], 0)
+	if null < 0 {
+		return newBinError("no null-terminator on version string", data, 4, 5)
+	}
+	null += 4
+	h.Version = data[4:null]
+	switch string(h.Version) {
+	case "0003":
+	case "0004":
+	case "0005":
+	default:
+	}
+
+	// float cookie
+	//
+	// Assume IEEE 754 doubles.  C doesn't assume 754 doubles, but anything that doesn't use 754 doubles is exotic
+	// enough that I'm OK saying "you're going to need to use `rrdtool dump`".  This lets us assume that:
+	//  - a 'double' is 8 bytes wide
+
+	//  - the value will be exactly equal, and we don't need to worry about weird rounding.
+	h.FloatWidth = 8
+	magicFloat := float64(8.642135e130)
+	floatAddrPacked := null + 1
+	floatAddr32 := ((floatAddrPacked + 3) / 4) * 4
+	floatAddr64 := ((floatAddrPacked + 7) / 8) * 8
+	var restOffset int
+	switch {
+	case len(data) < floatAddr32+h.FloatWidth:
+		return newBinError("unexpected end of file", data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked)
+	case math.Float64frombits(binary.LittleEndian.Uint64(data[floatAddr32:])) == magicFloat:
+		h.FloatCookie = data[floatAddr32 : floatAddr32+h.FloatWidth]
+		h.ByteOrder = binary.LittleEndian
+		h.FloatAlign = 4
+		restOffset = floatAddr32 + h.FloatWidth
+	case math.Float64frombits(binary.BigEndian.Uint64(data[floatAddr32:])) == magicFloat:
+		h.FloatCookie = data[floatAddr32 : floatAddr32+h.FloatWidth]
+		h.ByteOrder = binary.BigEndian
+		h.FloatAlign = 4
+		restOffset = floatAddr32 + h.FloatWidth
+	case len(data) < floatAddr64+h.FloatWidth:
+		return newBinError("unexpected end of file", data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked)
+	case math.Float64frombits(binary.LittleEndian.Uint64(data[floatAddr64:])) == magicFloat:
+		h.FloatCookie = data[floatAddr64 : floatAddr64+h.FloatWidth]
+		h.ByteOrder = binary.LittleEndian
+		h.FloatAlign = 8
+		restOffset = floatAddr64 + h.FloatWidth
+	case math.Float64frombits(binary.BigEndian.Uint64(data[floatAddr64:])) == magicFloat:
+		h.FloatCookie = data[floatAddr64 : floatAddr64+h.FloatWidth]
+		h.ByteOrder = binary.BigEndian
+		h.FloatAlign = 8
+		restOffset = floatAddr64 + h.FloatWidth
+	default:
+		return newBinError("failed to sniff byte-order and float-alignment",
+			data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked)
+	}
+
+	switch h.FloatAlign {
+	case 4:
+		// Assume that if floats are only 32-bit aligned, then everything is 32-bit
+		h.IntWidth = 4
+		h.IntAlign = 4
+	case 8:
+		// If floats are 64-bit aligned, then this might be all-in on 64-bit, or it might 32-bit ints.
+
+		// (The following heuristic is borrowed from javascriptRRD, and adjusted to also work with big-endian.)
+		//
+		// The next 2 things after the float_cookie are ds_cnt and rra_cnt (both 'unsigned long'--which may be
+		// either 32 or 64 bit).  We'll inspect the bytes a bit to guess how long a long is.
+		//
+		// By assuming
+		//  1. ds_cnt <= math.MaxUint32
+		//  2l. rra_cnt > 0 (relevant if little-endian)
+		//  2b. ds_cnt > 0 (relevant if big-endian)
+		// we can inspect the 4 bytes (marked "big" or "little" below) that are either
+		//  - the most significant bits of 64-bit ds_cnt, or
+		//  - the entirety of 32-bit rra_cnt (if littlen-endian) or 32-bit ds_cnt (if big-endian)
+		// If we see that those 4 bytes are all 0, then we assume that it's part of a 64-bit ds_cnt.
+		//
+		//       |           |           |           |           |           |           |    big    |  little   |
+		//       |00|01|02|03|04|05|06|07|08|09|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|
+		//  32   | R| R| D|\0| 0| 0| 0| 3|\0|                    |<----doublecookie----->|<--ds_cnt->|<-rra_cnt->|
+		//  64le | R| R| D|\0| 0| 0| 0| 3|\0|                    |<----doublecookie----->|<1111----ds_cnt---0000>|
+		//  64be | R| R| D|\0| 0| 0| 0| 3|\0|                    |<----doublecookie----->|<0000----ds_cnt---1111>|
+		if len(data) < restOffset+8 {
+			return newBinError("unexpected end of file", data, restOffset, 8)
+		}
+		offset := map[binary.ByteOrder]int{
+			binary.BigEndian:    restOffset,     // 24 in the above diagram
+			binary.LittleEndian: restOffset + 4, // 28 in the above diagram
+		}[h.ByteOrder]
+		if h.ByteOrder.Uint32(data[offset:]) == 0 {
+			h.IntWidth = 8
+			h.IntAlign = 8
+		} else {
+			h.IntWidth = 4
+			h.IntAlign = 4
+		}
+	}
+
+	return nil
+}
+
+func (h *Header) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	if err := e.EncodeElement(h.Version, xml.StartElement{Name: xml.Name{Local: "version", Space: XMLNS}}); err != nil {
+		return err
+	}
+	if err := e.EncodeElement(h.PDPStep, xml.StartElement{Name: xml.Name{Local: "step", Space: XMLNS}}); err != nil {
+		return err
+	}
+	return nil
+}
+
+//var _ encoding.BinaryMarshaler = &Header{}
+var _ encoding.BinaryUnmarshaler = &Header{}
+
+var _ xml.Marshaler = &Header{}
+
+//var _ xml.Unmarshaler = &Header{}
diff --git a/rrdformat/format_test.go b/rrdformat/format_test.go
new file mode 100644
index 0000000..e2c1aa0
--- /dev/null
+++ b/rrdformat/format_test.go
@@ -0,0 +1,14 @@
+package rrdformat
+
+import (
+	"encoding/xml"
+	"testing"
+)
+
+func TestXML(t *testing.T) {
+	out, err := xml.Marshal(&Header{
+		Version: []byte("0003"),
+		PDPStep: 300,
+	})
+	t.Log(string(out), err)
+}
author	Luke Shumaker <lukeshu@lukeshu.com>	2020-01-26 09:05:07 -0500
committer	Luke Shumaker <lukeshu@lukeshu.com>	2020-01-26 09:05:07 -0500
commit	a5cba20c1e3b0956737327ef9214fd2cbc221add (patch)
tree	fc19e2d97243449676fc5d2a5a6f57ad356010e6