From d5737a0e71b17a97b82ae68e49acc41a08fcc0ad Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 26 Dec 2022 22:02:13 -0700 Subject: cmd/btrfs-rec: Add a utility function for reading JSON files with progress --- cmd/btrfs-rec/inspect_lstrees.go | 12 ++--- cmd/btrfs-rec/inspect_rebuildmappings.go | 3 +- cmd/btrfs-rec/inspect_rebuildnodes.go | 3 +- cmd/btrfs-rec/inspect_scandevices.go | 14 ------ cmd/btrfs-rec/main.go | 7 +-- cmd/btrfs-rec/util.go | 83 ++++++++++++++++++++++++++++++++ lib/textui/log.go | 4 ++ 7 files changed, 97 insertions(+), 29 deletions(-) create mode 100644 cmd/btrfs-rec/util.go diff --git a/cmd/btrfs-rec/inspect_lstrees.go b/cmd/btrfs-rec/inspect_lstrees.go index 7f59eaa..e92c544 100644 --- a/cmd/btrfs-rec/inspect_lstrees.go +++ b/cmd/btrfs-rec/inspect_lstrees.go @@ -5,7 +5,6 @@ package main import ( - "encoding/json" "os" "strconv" "text/tabwriter" @@ -36,15 +35,14 @@ func init() { Args: cliutil.WrapPositionalArgs(cobra.NoArgs), }, RunE: func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error { - var scanResults map[btrfsvol.DeviceID]btrfsinspect.ScanOneDeviceResult + ctx := cmd.Context() + var scanResults btrfsinspect.ScanDevicesResult if scandevicesFilename != "" { - scanResultsBytes, err := os.ReadFile(scandevicesFilename) + var err error + scanResults, err = readJSONFile[btrfsinspect.ScanDevicesResult](ctx, scandevicesFilename) if err != nil { return err } - if err := json.Unmarshal(scanResultsBytes, &scanResults); err != nil { - return err - } } var treeErrCnt int @@ -65,7 +63,7 @@ func init() { table.Flush() } visitedNodes := make(containers.Set[btrfsvol.LogicalAddr]) - btrfsutil.WalkAllTrees(cmd.Context(), fs, btrfsutil.WalkAllTreesHandler{ + btrfsutil.WalkAllTrees(ctx, fs, btrfsutil.WalkAllTreesHandler{ PreTree: func(name string, treeID btrfsprim.ObjID) { treeErrCnt = 0 treeItemCnt = make(map[btrfsitem.Type]int) diff --git a/cmd/btrfs-rec/inspect_rebuildmappings.go b/cmd/btrfs-rec/inspect_rebuildmappings.go index 54535ec..da7d12e 100644 --- a/cmd/btrfs-rec/inspect_rebuildmappings.go +++ b/cmd/btrfs-rec/inspect_rebuildmappings.go @@ -15,6 +15,7 @@ import ( "github.com/spf13/cobra" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect" "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildmappings" ) @@ -37,7 +38,7 @@ func init() { ctx := cmd.Context() dlog.Infof(ctx, "Reading %q...", args[0]) - scanResults, err := readScanResults(args[0]) + scanResults, err := readJSONFile[btrfsinspect.ScanDevicesResult](ctx, args[0]) if err != nil { return err } diff --git a/cmd/btrfs-rec/inspect_rebuildnodes.go b/cmd/btrfs-rec/inspect_rebuildnodes.go index 5f6d9b5..0f3d60e 100644 --- a/cmd/btrfs-rec/inspect_rebuildnodes.go +++ b/cmd/btrfs-rec/inspect_rebuildnodes.go @@ -17,6 +17,7 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsprim" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect" "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildnodes" "git.lukeshu.com/btrfs-progs-ng/lib/containers" ) @@ -31,7 +32,7 @@ func init() { ctx := cmd.Context() dlog.Infof(ctx, "Reading %q...", args[0]) - nodeScanResults, err := readScanResults(args[0]) + nodeScanResults, err := readJSONFile[btrfsinspect.ScanDevicesResult](ctx, args[0]) if err != nil { return err } diff --git a/cmd/btrfs-rec/inspect_scandevices.go b/cmd/btrfs-rec/inspect_scandevices.go index 5c8b2b0..7235e45 100644 --- a/cmd/btrfs-rec/inspect_scandevices.go +++ b/cmd/btrfs-rec/inspect_scandevices.go @@ -58,17 +58,3 @@ func writeScanResults(w io.Writer, results btrfsinspect.ScanDevicesResult) (err CompactIfUnder: 16, }, results) } - -func readScanResults(filename string) (btrfsinspect.ScanDevicesResult, error) { - fh, err := os.Open(filename) - if err != nil { - return nil, err - } - var scanResults btrfsinspect.ScanDevicesResult - buf := bufio.NewReader(fh) - if err := lowmemjson.DecodeThenEOF(buf, &scanResults); err != nil { - return nil, err - } - _ = fh.Close() - return scanResults, nil -} diff --git a/cmd/btrfs-rec/main.go b/cmd/btrfs-rec/main.go index 3a00544..87e8696 100644 --- a/cmd/btrfs-rec/main.go +++ b/cmd/btrfs-rec/main.go @@ -6,7 +6,6 @@ package main import ( "context" - "encoding/json" "os" "github.com/datawire/dlib/dgroup" @@ -122,14 +121,10 @@ func main() { }() if mappingsFlag != "" { - bs, err := os.ReadFile(mappingsFlag) + mappingsJSON, err := readJSONFile[[]btrfsvol.Mapping](ctx, mappingsFlag) if err != nil { return err } - var mappingsJSON []btrfsvol.Mapping - if err := json.Unmarshal(bs, &mappingsJSON); err != nil { - return err - } for _, mapping := range mappingsJSON { if err := fs.LV.AddMapping(mapping); err != nil { return err diff --git a/cmd/btrfs-rec/util.go b/cmd/btrfs-rec/util.go new file mode 100644 index 0000000..adfe97e --- /dev/null +++ b/cmd/btrfs-rec/util.go @@ -0,0 +1,83 @@ +// Copyright (C) 2022 Luke Shumaker +// +// SPDX-License-Identifier: GPL-2.0-or-later + +package main + +import ( + "bufio" + "context" + "io" + "os" + "time" + + "git.lukeshu.com/go/lowmemjson" + "github.com/datawire/dlib/dlog" + + "git.lukeshu.com/btrfs-progs-ng/lib/textui" +) + +type runeScanner struct { + progress textui.Portion[int64] + progressWriter *textui.Progress[textui.Portion[int64]] + unreadCnt uint64 + reader *bufio.Reader + closer io.Closer +} + +func newRuneScanner(ctx context.Context, fh *os.File) (*runeScanner, error) { + fi, err := fh.Stat() + if err != nil { + return nil, err + } + ret := &runeScanner{ + progress: textui.Portion[int64]{ + D: fi.Size(), + }, + progressWriter: textui.NewProgress[textui.Portion[int64]](ctx, dlog.LogLevelInfo, 1*time.Second), + reader: bufio.NewReader(fh), + closer: fh, + } + return ret, nil +} + +func (rs *runeScanner) ReadRune() (r rune, size int, err error) { + r, size, err = rs.reader.ReadRune() + if rs.unreadCnt > 0 { + rs.unreadCnt-- + } else { + rs.progress.N += int64(size) + rs.progressWriter.Set(rs.progress) + } + return +} + +func (rs *runeScanner) UnreadRune() error { + rs.unreadCnt++ + return rs.reader.UnreadRune() +} + +func (rs *runeScanner) Close() error { + rs.progressWriter.Done() + return rs.closer.Close() +} + +func readJSONFile[T any](ctx context.Context, filename string) (T, error) { + fh, err := os.Open(filename) + if err != nil { + var zero T + return zero, err + } + buf, err := newRuneScanner(dlog.WithField(ctx, "btrfs.read-json-file", filename), fh) + if err != nil { + var zero T + return zero, err + } + var ret T + if err := lowmemjson.DecodeThenEOF(buf, &ret); err != nil { + var zero T + return zero, err + } + _ = buf.Close() + return ret, nil +} diff --git a/lib/textui/log.go b/lib/textui/log.go index 4421074..e94a24f 100644 --- a/lib/textui/log.go +++ b/lib/textui/log.go @@ -288,6 +288,8 @@ func fieldOrd(key string) int { case "btrfsinspect.rebuild-mappings.substep": return -1 + case "btrfs.read-json-file": + return -1 default: return 1 } @@ -303,6 +305,8 @@ func fieldName(key string) string { return strings.TrimPrefix(key, "btrfsinspect.scandevices.") case strings.HasPrefix(key, "btrfsinspect.rebuild-mappings."): return strings.TrimPrefix(key, "btrfsinspect.rebuild-mappings.") + case strings.HasPrefix(key, "btrfs."): + return strings.TrimPrefix(key, "btrfs.") default: return key } -- cgit v1.1-4-g5e80