ahhh

7 files changed, 260 insertions, 207 deletions

diff --git a/backend.go b/backend.go
index fc67d82..72242a1 100644
--- a/backend.go
+++ b/backend.go
@@ -2,6 +2,7 @@ package libfastimport
 
 import (
 	"bufio"
+	"fmt"
 	"io"
 
 	"git.lukeshu.com/go/libfastimport/textproto"
@@ -15,6 +16,8 @@ type Backend struct {
 	fiw *textproto.FIWriter
 	cbr *textproto.CatBlobReader
 
+	inCommit bool
+
 	err   error
 	onErr func(error) error
 }
@@ -45,12 +48,27 @@ func NewBackend(fastImport io.WriteCloser, catBlob io.Reader, onErr func(error)
 	return ret
 }
 
+// will panic if Cmd is a type that may only be used in a commit but
+// we aren't in a commit.
 func (b *Backend) Do(cmd Cmd) error {
 	if b.err == nil {
 		return b.err
 	}
 
-	err := cmd.fiWriteCmd(b.fiw)
+	switch cmd.fiCmdClass() {
+	case cmdClassCommand:
+		_, b.inCommit = cmd.(CmdCommit)
+	case cmdClassCommit:
+		if !b.inCommit {
+			panic(fmt.Errorf("Cannot issue commit sub-command outside of a commit: %[1]T(%#[1]v)", cmd))
+		}
+	case cmdClassComment:
+		/* do nothing */
+	default:
+		panic(fmt.Errorf("invalid cmdClass: %d", cmd.fiCmdClass()))
+	}
+
+	err := cmd.fiCmdWrite(b.fiw)
 	if err != nil {
 		return b.onErr(err)
 	}
@@ -59,7 +77,7 @@ func (b *Backend) Do(cmd Cmd) error {
 		return b.onErr(err)
 	}
 
-	if _, ok := cmd.(CmdDone); ok {
+	if _, isDone := cmd.(CmdDone); isDone {
 		return b.onErr(nil)
 	}
 
diff --git a/cmd.go b/cmd.go
new file mode 100644
index 0000000..7ba40e0
--- /dev/null
+++ b/cmd.go
@@ -0,0 +1,19 @@
+package libfastimport
+
+import (
+	"git.lukeshu.com/go/libfastimport/textproto"
+)
+
+type cmdClass int
+
+const (
+	cmdClassCommand cmdClass = 1 // may be a top-level command
+	cmdClassCommit  cmdClass = 2 // may be used within in a commit
+
+	cmdClassComment cmdClass = cmdClassCommand | cmdClassCommit
+)
+
+type Cmd interface {
+	fiCmdWrite(*textproto.FIWriter) error
+	fiCmdClass() cmdClass
+}
diff --git a/commands.go b/cmd_command.go
index 7e7c4f3..a1d1a41 100644
--- a/commands.go
+++ b/cmd_command.go
@@ -1,15 +1,9 @@
 package libfastimport
 
 import (
-	"strconv"
-
 	"git.lukeshu.com/go/libfastimport/textproto"
 )
 
-type Cmd interface {
-	fiWriteCmd(*textproto.FIWriter) error
-}
-
 type CmdCommit struct {
 	Ref       string
 	Mark      int // optional; < 1 for non-use
@@ -18,10 +12,10 @@ type CmdCommit struct {
 	Msg       string
 	From      string
 	Merge     []string
-	Tree      []FileAction
 }
 
-func (c CmdCommit) fiWriteCmd(fiw *textproto.FIWriter) error {
+func (c CmdCommit) fiCmdClass() cmdClass { return cmdClassCommand }
+func (c CmdCommit) fiCmdWrite(fiw *textproto.FIWriter) error {
 	ez := &ezfiw{fiw: fiw}
 
 	ez.WriteLine("commit", c.Ref)
@@ -40,18 +34,7 @@ func (c CmdCommit) fiWriteCmd(fiw *textproto.FIWriter) error {
 		ez.WriteLine("merge", merge)
 	}
 
-	if ez.err != nil {
-		return ez.err
-	}
-
-	for _, action := range c.Tree {
-		err := action.fiWriteFA(fiw)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
+	return ez.err
 }
 
 type CmdTag struct {
@@ -61,7 +44,8 @@ type CmdTag struct {
 	Data      string
 }
 
-func (c CmdTag) fiWriteCmd(fiw *textproto.FIWriter) error {
+func (c CmdTag) fiCmdClass() cmdClass { return cmdClassCommand }
+func (c CmdTag) fiCmdWrite(fiw *textproto.FIWriter) error {
 	ez := &ezfiw{fiw: fiw}
 
 	ez.WriteLine("tag", c.RefName)
@@ -77,7 +61,8 @@ type CmdReset struct {
 	CommitIsh string // optional
 }
 
-func (c CmdReset) fiWriteCmd(fiw *textproto.FIWriter) error {
+func (c CmdReset) fiCmdClass() cmdClass { return cmdClassCommand }
+func (c CmdReset) fiCmdWrite(fiw *textproto.FIWriter) error {
 	ez := &ezfiw{fiw: fiw}
 
 	ez.WriteLine("reset", c.RefName)
@@ -93,7 +78,8 @@ type CmdBlob struct {
 	Data string
 }
 
-func (c CmdBlob) fiWriteCmd(fiw *textproto.FIWriter) error {
+func (c CmdBlob) fiCmdClass() cmdClass { return cmdClassCommand }
+func (c CmdBlob) fiCmdWrite(fiw *textproto.FIWriter) error {
 	ez := &ezfiw{fiw: fiw}
 
 	ez.WriteLine("blob")
@@ -107,7 +93,8 @@ func (c CmdBlob) fiWriteCmd(fiw *textproto.FIWriter) error {
 
 type CmdCheckpoint struct{}
 
-func (c CmdCheckpoint) fiWriteCmd(fiw *textproto.FIWriter) error {
+func (c CmdCheckpoint) fiCmdClass() cmdClass { return cmdClassCommand }
+func (c CmdCheckpoint) fiCmdWrite(fiw *textproto.FIWriter) error {
 	return fiw.WriteLine("checkpoint")
 }
 
@@ -115,42 +102,18 @@ type CmdProgress struct {
 	Str string
 }
 
-func (c CmdProgress) fiWriteCmd(fiw *textproto.FIWriter) error {
+func (c CmdProgress) fiCmdClass() cmdClass { return cmdClassCommand }
+func (c CmdProgress) fiCmdWrite(fiw *textproto.FIWriter) error {
 	return fiw.WriteLine("progress", c.Str)
 }
 
-type CmdGetMark struct {
-	Mark int
-}
-
-func (c CmdGetMark) fiWriteCmd(fiw *textproto.FIWriter) error {
-	return fiw.WriteLine("get-mark", ":"+strconv.Itoa(c.Mark))
-}
-
-type CmdCatBlob struct {
-	DataRef string
-}
-
-func (c CmdCatBlob) fiWriteCmd(fiw *textproto.FIWriter) error {
-	return fiw.WriteLine("cat-blob", c.DataRef)
-}
-
-// See FileLs for using ls inside of a commit
-type CmdLs struct {
-	DataRef string
-	Path    textproto.Path
-}
-
-func (c CmdLs) fiWriteCmd(fiw *textproto.FIWriter) error {
-	return fiw.WriteLine("ls", c.DataRef, c.Path)
-}
-
 type CmdFeature struct {
 	Feature  string
 	Argument string
 }
 
-func (c CmdFeature) fiWriteCmd(fiw *textproto.FIWriter) error {
+func (c CmdFeature) fiCmdClass() cmdClass { return cmdClassCommand }
+func (c CmdFeature) fiCmdWrite(fiw *textproto.FIWriter) error {
 	if c.Argument != "" {
 		return fiw.WriteLine("feature", c.Feature+"="+c.Argument)
 	} else {
@@ -162,20 +125,14 @@ type CmdOption struct {
 	Option string
 }
 
-func (c CmdOption) fiWriteCmd(fiw *textproto.FIWriter) error {
+func (c CmdOption) fiCmdClass() cmdClass { return cmdClassCommand }
+func (c CmdOption) fiCmdWrite(fiw *textproto.FIWriter) error {
 	return fiw.WriteLine("option", c.Option)
 }
 
 type CmdDone struct{}
 
-func (c CmdDone) fiWriteCmd(fiw *textproto.FIWriter) error {
+func (c CmdDone) fiCmdClass() cmdClass { return cmdClassCommand }
+func (c CmdDone) fiCmdWrite(fiw *textproto.FIWriter) error {
 	return fiw.WriteLine("done")
 }
-
-type CmdComment struct {
-	Comment string
-}
-
-func (c CmdComment) fiWriteCmd(fiw *textproto.FIWriter) error {
-	return fiw.WriteLine("#" + c.Comment)
-}
diff --git a/cmd_comment.go b/cmd_comment.go
new file mode 100644
index 0000000..d783ecc
--- /dev/null
+++ b/cmd_comment.go
@@ -0,0 +1,54 @@
+package libfastimport
+
+import (
+	"strconv"
+
+	"git.lukeshu.com/go/libfastimport/textproto"
+)
+
+type CmdComment struct {
+	Comment string
+}
+
+func (c CmdComment) fiCmdClass() cmdClass { return cmdClassComment }
+func (c CmdComment) fiCmdWrite(fiw *textproto.FIWriter) error {
+	return fiw.WriteLine("#" + c.Comment)
+}
+
+type CmdGetMark struct {
+	Mark int
+}
+
+func (c CmdGetMark) fiCmdClass() cmdClass { return cmdClassComment }
+func (c CmdGetMark) fiCmdWrite(fiw *textproto.FIWriter) error {
+	return fiw.WriteLine("get-mark", ":"+strconv.Itoa(c.Mark))
+}
+
+type CmdCatBlob struct {
+	DataRef string
+}
+
+func (c CmdCatBlob) fiCmdClass() cmdClass { return cmdClassComment }
+func (c CmdCatBlob) fiCmdWrite(fiw *textproto.FIWriter) error {
+	return fiw.WriteLine("cat-blob", c.DataRef)
+}
+
+type CmdLs struct {
+	DataRef string // optional if inside of a commit
+	Path    textproto.Path
+}
+
+func (c CmdLs) fiCmdClass() cmdClass {
+	if c.DataRef == "" {
+		return cmdClassCommit
+	}
+	return cmdClassComment
+}
+func (c CmdLs) fiCmdWrite(fiw *textproto.FIWriter) error {
+	if c.DataRef == "" {
+		return fiw.WriteLine("ls", c.Path)
+	} else {
+		return fiw.WriteLine("ls", c.DataRef, c.Path)
+	}
+}
+
diff --git a/fileactions.go b/cmd_commit.go
index 910fe5a..5c882f8 100644
--- a/fileactions.go
+++ b/cmd_commit.go
@@ -4,17 +4,14 @@ import (
 	"git.lukeshu.com/go/libfastimport/textproto"
 )
 
-type FileAction interface {
-	fiWriteFA(*textproto.FIWriter) error
-}
-
 type FileModify struct {
 	Mode    textproto.Mode
 	Path    textproto.Path
 	DataRef string
 }
 
-func (o FileModify) fiWriteFA(fiw *textproto.FIWriter) error {
+func (o FileModify) fiCmdClass() cmdClass { return cmdClassCommit }
+func (o FileModify) fiCmdWrite(fiw *textproto.FIWriter) error {
 	return fiw.WriteLine("M", o.Mode, o.DataRef, o.Path)
 }
 
@@ -24,7 +21,8 @@ type FileModifyInline struct {
 	Data string
 }
 
-func (o FileModifyInline) fiWriteFA(fiw *textproto.FIWriter) error {
+func (o FileModifyInline) fiCmdClass() cmdClass { return cmdClassCommit }
+func (o FileModifyInline) fiCmdWrite(fiw *textproto.FIWriter) error {
 	ez := &ezfiw{fiw: fiw}
 	ez.WriteLine("M", o.Mode, "inline", o.Path)
 	ez.WriteData(o.Data)
@@ -35,7 +33,8 @@ type FileDelete struct {
 	Path textproto.Path
 }
 
-func (o FileDelete) fiWriteFA(fiw *textproto.FIWriter) error {
+func (o FileDelete) fiCmdClass() cmdClass { return cmdClassCommit }
+func (o FileDelete) fiCmdWrite(fiw *textproto.FIWriter) error {
 	return fiw.WriteLine("D", o.Path)
 }
 
@@ -44,7 +43,8 @@ type FileCopy struct {
 	Dst textproto.Path
 }
 
-func (o FileCopy) fiWriteFA(fiw *textproto.FIWriter) error {
+func (o FileCopy) fiCmdClass() cmdClass { return cmdClassCommit }
+func (o FileCopy) fiCmdWrite(fiw *textproto.FIWriter) error {
 	return fiw.WriteLine("C", o.Src, o.Dst)
 }
 
@@ -53,13 +53,15 @@ type FileRename struct {
 	Dst string
 }
 
-func (o FileRename) fiWriteFA(fiw *textproto.FIWriter) error {
+func (o FileRename) fiCmdClass() cmdClass { return cmdClassCommit }
+func (o FileRename) fiCmdWrite(fiw *textproto.FIWriter) error {
 	return fiw.WriteLine("R", o.Src, o.Dst)
 }
 
 type FileDeleteAll struct{}
 
-func (o FileDeleteAll) fiWriteFA(fiw *textproto.FIWriter) error {
+func (o FileDeleteAll) fiCmdClass() cmdClass { return cmdClassCommit }
+func (o FileDeleteAll) fiCmdWrite(fiw *textproto.FIWriter) error {
 	return fiw.WriteLine("deleteall")
 }
 
@@ -68,7 +70,8 @@ type NoteModify struct {
 	DataRef   string
 }
 
-func (o NoteModify) fiWriteFA(fiw *textproto.FIWriter) error {
+func (o NoteModify) fiCmdClass() cmdClass { return cmdClassCommit }
+func (o NoteModify) fiCmdWrite(fiw *textproto.FIWriter) error {
 	return fiw.WriteLine("N", o.DataRef, o.CommitIsh)
 }
 
@@ -77,18 +80,10 @@ type NoteModifyInline struct {
 	Data      string
 }
 
-func (o NoteModifyInline) fiWriteFA(fiw *textproto.FIWriter) error {
+func (o NoteModifyInline) fiCmdClass() cmdClass { return cmdClassCommit }
+func (o NoteModifyInline) fiCmdWrite(fiw *textproto.FIWriter) error {
 	ez := &ezfiw{fiw: fiw}
 	ez.WriteLine("N", "inline", o.CommitIsh)
 	ez.WriteData(o.Data)
 	return ez.err
 }
-
-// See CmdLs for using ls outside of a commit
-type FileLs struct {
-	Path textproto.Path
-}
-
-func (o FileLs) fiWriteFA(fiw *textproto.FIWriter) error {
-	return fiw.WriteLine("ls", o.Path)
-}
diff --git a/frontend.go b/frontend.go
index 2150a35..604e7af 100644
--- a/frontend.go
+++ b/frontend.go
@@ -17,16 +17,6 @@ func (e UnsupportedCommand) Error() string {
 	return "Unsupported command: " + string(e)
 }
 
-func trimLinePrefix(line string, prefix string) string {
-	if !strings.HasPrefix(line, prefix) {
-		panic("line didn't have prefix")
-	}
-	if !strings.HasSuffix(line, "\n") {
-		panic("line didn't have prefix")
-	}
-	return strings.TrimSuffix(strings.TrimPrefix(line, prefix), "\n")
-}
-
 // A Frontend is something that produces a fast-import stream; the
 // Frontend object provides methods for reading from it.
 type Frontend struct {
@@ -34,6 +24,8 @@ type Frontend struct {
 	cbw *textproto.CatBlobWriter
 	w   *bufio.Writer
 
+	inCommit bool
+
 	cmd chan Cmd
 	err error
 }
@@ -85,38 +77,6 @@ func (f *Frontend) nextLine() (line string, err error) {
 	}
 }
 
-func parse_data(line string) (data string, err error) {
-	nl := strings.IndexByte(line, '\n')
-	if nl < 0 {
-		return "", fmt.Errorf("data: expected newline: %v", data)
-	}
-	head := line[:nl+1]
-	rest := line[nl+1:]
-	if !strings.HasPrefix(head, "data ") {
-		return "", fmt.Errorf("data: could not parse: %v", data)
-	}
-	if strings.HasPrefix(head, "data <<") {
-		// Delimited format
-		delim := trimLinePrefix(head, "data <<")
-		suffix := "\n" + delim + "\n"
-		if !strings.HasSuffix(rest, suffix) {
-			return "", fmt.Errorf("data: did not find suffix: %v", suffix)
-		}
-		data = strings.TrimSuffix(rest, suffix)
-	} else {
-		// Exact byte count format
-		size, err := strconv.Atoi(trimLinePrefix(head, "data "))
-		if err != nil {
-			return "", err
-		}
-		if size != len(rest) {
-			panic("FIReader should not have let this happen")
-		}
-		data = rest
-	}
-	return
-}
-
 func (f *Frontend) parse() error {
 	line, err := f.nextLine()
 	if err != nil {
@@ -229,84 +189,71 @@ func (f *Frontend) parse() error {
 					return err
 				}
 			}
-			for {
-				switch {
-				case strings.HasPrefix(line, "M "):
-					str := trimLinePrefix(line, "M ")
-					sp1 := strings.IndexByte(str, ' ')
-					sp2 := strings.IndexByte(str, ' ')
-					if sp1 < 0 || sp2 < 0 {
-						return fmt.Errorf("commit: malformed modify command: %v", line)
-					}
-					nMode, err := strconv.ParseUint(str[:sp1], 8, 18)
-					if err != nil {
-						return err
-					}
-					ref := str[sp1+1 : sp2]
-					path := textproto.PathUnescape(str[sp2+1:])
-					if ref == "inline" {
-						line, err = f.nextLine()
-						if err != nil {
-							return err
-						}
-						if !strings.HasPrefix(line, "data ") {
-							return fmt.Errorf("commit: modify: expected data command: %v", line)
-						}
-						data, err := parse_data(line)
-						if err != nil {
-							return err
-						}
-						c.Tree = append(c.Tree, FileModifyInline{Mode: textproto.Mode(nMode), Path: path, Data: data})
-					} else {
-						c.Tree = append(c.Tree, FileModify{Mode: textproto.Mode(nMode), Path: path, DataRef: ref})
-					}
-				case strings.HasPrefix(line, "D "):
-					c.Tree = append(c.Tree, FileDelete{Path: textproto.PathUnescape(trimLinePrefix(line, "D "))})
-				case strings.HasPrefix(line, "C "):
-					// BUG(lukeshu): TODO: commit C not implemented
-					panic("TODO: commit C not implemented")
-				case strings.HasPrefix(line, "R "):
-					// BUG(lukeshu): TODO: commit R not implemented
-					panic("TODO: commit R not implemented")
-				case strings.HasPrefix(line, "N "):
-					str := trimLinePrefix(line, "N ")
-					sp := strings.IndexByte(str, ' ')
-					if sp < 0 {
-						return fmt.Errorf("commit: malformed notemodify command: %v", line)
-					}
-					ref := str[:sp]
-					commitish := str[sp+1:]
-					if ref == "inline" {
-						line, err = f.nextLine()
-						if err != nil {
-							return err
-						}
-						if !strings.HasPrefix(line, "data ") {
-							return fmt.Errorf("commit: notemodify: expected data command: %v", line)
-						}
-						data, err := parse_data(line)
-						if err != nil {
-							return err
-						}
-						c.Tree = append(c.Tree, NoteModifyInline{CommitIsh: commitish, Data: data})
-					} else {
-						c.Tree = append(c.Tree, NoteModify{CommitIsh: commitish, DataRef: ref})
-					}
-				case strings.HasPrefix(line, "ls "):
-					// BUG(lukeshu): TODO: in-commit ls not implemented
-					panic("TODO: in-commit ls not implemented")
-				case line == "deleteall\n":
-					c.Tree = append(c.Tree, FileDeleteAll{})
-				default:
-					break
+			f.cmd <- c
+		case strings.HasPrefix(line, "M "):
+			fmt.Printf("line: %q\n", line)
+			str := trimLinePrefix(line, "M ")
+			sp1 := strings.IndexByte(str, ' ')
+			sp2 := strings.IndexByte(str[sp1+1:], ' ')
+			if sp1 < 0 || sp2 < 0 {
+				return fmt.Errorf("commit: malformed modify command: %v", line)
+			}
+			nMode, err := strconv.ParseUint(str[:sp1], 8, 18)
+			if err != nil {
+				return err
+			}
+			ref := str[sp1+1 : sp2]
+			path := textproto.PathUnescape(str[sp2+1:])
+			if ref == "inline" {
+				line, err = f.nextLine()
+				if err != nil {
+					return err
+				}
+				if !strings.HasPrefix(line, "data ") {
+					return fmt.Errorf("commit: modify: expected data command: %v", line)
+				}
+				data, err := parse_data(line)
+				if err != nil {
+					return err
 				}
+				f.cmd <- FileModifyInline{Mode: textproto.Mode(nMode), Path: path, Data: data}
+			} else {
+				f.cmd <- FileModify{Mode: textproto.Mode(nMode), Path: path, DataRef: ref}
+			}
+		case strings.HasPrefix(line, "D "):
+			f.cmd <- FileDelete{Path: textproto.PathUnescape(trimLinePrefix(line, "D "))}
+		case strings.HasPrefix(line, "C "):
+			// BUG(lukeshu): TODO: commit C not implemented
+			panic("TODO: commit C not implemented")
+		case strings.HasPrefix(line, "R "):
+			// BUG(lukeshu): TODO: commit R not implemented
+			panic("TODO: commit R not implemented")
+		case strings.HasPrefix(line, "N "):
+			str := trimLinePrefix(line, "N ")
+			sp := strings.IndexByte(str, ' ')
+			if sp < 0 {
+				return fmt.Errorf("commit: malformed notemodify command: %v", line)
+			}
+			ref := str[:sp]
+			commitish := str[sp+1:]
+			if ref == "inline" {
 				line, err = f.nextLine()
 				if err != nil {
 					return err
 				}
+				if !strings.HasPrefix(line, "data ") {
+					return fmt.Errorf("commit: notemodify: expected data command: %v", line)
+				}
+				data, err := parse_data(line)
+				if err != nil {
+					return err
+				}
+				f.cmd <- NoteModifyInline{CommitIsh: commitish, Data: data}
+			} else {
+				f.cmd <- NoteModify{CommitIsh: commitish, DataRef: ref}
 			}
-			f.cmd <- c
-			continue
+		case line == "deleteall\n":
+			f.cmd <- FileDeleteAll{}
 		case strings.HasPrefix(line, "feature "):
 			// 'feature' SP <feature> ('=' <argument>)? LF
 			str := trimLinePrefix(line, "feature ")
@@ -323,16 +270,17 @@ func (f *Frontend) parse() error {
 			}
 		case strings.HasPrefix(line, "ls "):
 			// 'ls' SP <dataref> SP <path> LF
-			sp1 := strings.IndexByte(line, ' ')
-			sp2 := strings.IndexByte(line[sp1+1:], ' ')
-			lf := strings.IndexByte(line[sp2+1:], '\n')
-			if sp1 < 0 || sp2 < 0 || lf < 0 {
-				return fmt.Errorf("ls: outside of a commit both <dataref> and <path> are required: %v", line)
+			str := trimLinePrefix(line, "ls ")
+			sp := -1
+			if !strings.HasPrefix(str, "\"") {
+				sp = strings.IndexByte(line, ' ')
 			}
-			f.cmd <- CmdLs{
-				DataRef: line[sp1+1 : sp2],
-				Path:    textproto.PathUnescape(line[sp2+1 : lf]),
+			c := CmdLs{}
+			c.Path = textproto.PathUnescape(str[sp+1:])
+			if sp >= 0 {
+				c.DataRef = str[:sp]
 			}
+			f.cmd <- c
 		case strings.HasPrefix(line, "option "):
 			// 'option' SP <option> LF
 			f.cmd <- CmdOption{Option: trimLinePrefix(line, "option ")}
@@ -406,6 +354,19 @@ func (f *Frontend) parse() error {
 func (f *Frontend) ReadCmd() (Cmd, error) {
 	cmd, ok := <-f.cmd
 	if ok {
+		switch cmd.fiCmdClass() {
+		case cmdClassCommand:
+			_, f.inCommit = cmd.(CmdCommit)
+		case cmdClassCommit:
+			if !f.inCommit {
+				// BUG(lukeshu): idk what to do here
+				panic("oops")
+			}
+		case cmdClassComment:
+			/* do nothing */
+		default:
+			panic(fmt.Errorf("invalid cmdClass: %d", cmd.fiCmdClass()))
+		}
 		return cmd, nil
 	}
 	return nil, f.err
diff --git a/util.go b/util.go
new file mode 100644
index 0000000..dbb7d70
--- /dev/null
+++ b/util.go
@@ -0,0 +1,49 @@
+package libfastimport
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+func trimLinePrefix(line string, prefix string) string {
+	if !strings.HasPrefix(line, prefix) {
+		panic("line didn't have prefix")
+	}
+	if !strings.HasSuffix(line, "\n") {
+		panic("line didn't have prefix")
+	}
+	return strings.TrimSuffix(strings.TrimPrefix(line, prefix), "\n")
+}
+
+func parse_data(line string) (data string, err error) {
+	nl := strings.IndexByte(line, '\n')
+	if nl < 0 {
+		return "", fmt.Errorf("data: expected newline: %v", data)
+	}
+	head := line[:nl+1]
+	rest := line[nl+1:]
+	if !strings.HasPrefix(head, "data ") {
+		return "", fmt.Errorf("data: could not parse: %v", data)
+	}
+	if strings.HasPrefix(head, "data <<") {
+		// Delimited format
+		delim := trimLinePrefix(head, "data <<")
+		suffix := "\n" + delim + "\n"
+		if !strings.HasSuffix(rest, suffix) {
+			return "", fmt.Errorf("data: did not find suffix: %v", suffix)
+		}
+		data = strings.TrimSuffix(rest, suffix)
+	} else {
+		// Exact byte count format
+		size, err := strconv.Atoi(trimLinePrefix(head, "data "))
+		if err != nil {
+			return "", err
+		}
+		if size != len(rest) {
+			panic("FIReader should not have let this happen")
+		}
+		data = rest
+	}
+	return
+}