cli: wip: Rethink the "scan" CLI commands

6 files changed, 39 insertions, 309 deletions

diff --git a/cmd/btrfs-rec/inspect_dumpsums.go b/cmd/btrfs-rec/inspect_dumpsums.go
deleted file mode 100644
index 28ae7ef..0000000
--- a/cmd/btrfs-rec/inspect_dumpsums.go
+++ /dev/null
@@ -1,35 +0,0 @@
-// Copyright (C) 2022  Luke Shumaker <lukeshu@lukeshu.com>
-//
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-package main
-
-import (
-	"os"
-
-	"github.com/datawire/dlib/dlog"
-	"github.com/datawire/ocibuild/pkg/cliutil"
-	"github.com/spf13/cobra"
-
-	"git.lukeshu.com/btrfs-progs-ng/lib/btrfs"
-	"git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildmappings"
-)
-
-func init() {
-	inspectors = append(inspectors, subcommand{
-		Command: cobra.Command{
-			Use:   "dump-sums",
-			Short: "Dump a buncha checksums as JSON",
-			Args:  cliutil.WrapPositionalArgs(cobra.NoArgs),
-		},
-		RunE: func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error {
-			ctx := cmd.Context()
-			sums, err := rebuildmappings.SumEverything(ctx, fs)
-			if err != nil {
-				return err
-			}
-			dlog.Info(ctx, "Writing sums as gob to stdout...")
-			return rebuildmappings.WriteAllSums(os.Stdout, sums)
-		},
-	})
-}
diff --git a/cmd/btrfs-rec/inspect_scanfornodes.go b/cmd/btrfs-rec/inspect_scandevices.go
index 13f7760..f5caadb 100644
--- a/cmd/btrfs-rec/inspect_scanfornodes.go
+++ b/cmd/btrfs-rec/inspect_scandevices.go
@@ -5,11 +5,12 @@
 package main
 
 import (
+	"bufio"
 	"context"
-	"encoding/json"
 	"os"
 	"sync"
 
+	"git.lukeshu.com/go/lowmemjson"
 	"github.com/datawire/dlib/dgroup"
 	"github.com/datawire/dlib/dlog"
 	"github.com/datawire/ocibuild/pkg/cliutil"
@@ -23,23 +24,15 @@ import (
 func init() {
 	inspectors = append(inspectors, subcommand{
 		Command: cobra.Command{
-			Use:   "scan-for-nodes",
-			Short: "Scan devices for (potentially lost) nodes",
-			Long: "" +
-				"The found information is printed as JSON on stdout, and can\n" +
-				"be read by `btrfs-rec inspect rebuild-mappings`.\n" +
-				"\n" +
-				"This information is mostly useful for rebuilding a broken\n" +
-				"chunk/dev-extent/blockgroup trees, but can also have some\n" +
-				"minimal utility in repairing other trees.\n" +
-				"\n" +
-				"This is very similar the initial scan done by\n" +
-				"`btrfs rescue chunk-recover`.  Like `btrfs rescue chunk-recover`,\n" +
-				"this is likely probably slow because it reads the entirety of\n" +
-				"each device.",
+			Use:  "scandevices",
 			Args: cliutil.WrapPositionalArgs(cobra.NoArgs),
 		},
-		RunE: func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error {
+		RunE: func(fs *btrfs.FS, cmd *cobra.Command, _ []string) (err error) {
+			maybeSetErr := func(_err error) {
+				if err == nil && _err != nil {
+					err = _err
+				}
+			}
 			ctx := cmd.Context()
 
 			var resultsMu sync.Mutex
@@ -66,7 +59,14 @@ func init() {
 			}
 
 			dlog.Info(ctx, "Writing scan results to stdout...")
-			return json.NewEncoder(os.Stdout).Encode(results)
+			buffer := bufio.NewWriter(os.Stdout)
+			defer func() {
+				maybeSetErr(buffer.Flush())
+			}()
+			return lowmemjson.Encode(&lowmemjson.ReEncoder{
+				Out:    buffer,
+				Indent: "\t",
+			}, results)
 		},
 	})
 }
diff --git a/cmd/btrfs-rec/inspect_scanforextents.go b/cmd/btrfs-rec/inspect_scanforextents.go
deleted file mode 100644
index 9b5ada4..0000000
--- a/cmd/btrfs-rec/inspect_scanforextents.go
+++ /dev/null
@@ -1,55 +0,0 @@
-// Copyright (C) 2022  Luke Shumaker <lukeshu@lukeshu.com>
-//
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-package main
-
-import (
-	"os"
-	"runtime"
-
-	"github.com/datawire/dlib/dlog"
-	"github.com/datawire/ocibuild/pkg/cliutil"
-	"github.com/spf13/cobra"
-
-	"git.lukeshu.com/btrfs-progs-ng/lib/btrfs"
-	"git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildmappings"
-)
-
-func init() {
-	inspectors = append(inspectors, subcommand{
-		Command: cobra.Command{
-			Use:  "scan-for-extents NODESCAN.json DUMPSUMS.gob",
-			Args: cliutil.WrapPositionalArgs(cobra.ExactArgs(2)),
-		},
-		RunE: func(fs *btrfs.FS, cmd *cobra.Command, args []string) error {
-			ctx := cmd.Context()
-
-			dlog.Infof(ctx, "Reading %q...", args[0])
-			bgs, err := rebuildmappings.ReadNodeScanResults(fs, args[0])
-			if err != nil {
-				return err
-			}
-			runtime.GC()
-			dlog.Infof(ctx, "... done reading %q", args[0])
-
-			dlog.Infof(ctx, "Reading %q...", args[1])
-			sums, err := rebuildmappings.ReadAllSums(args[1])
-			if err != nil {
-				return err
-			}
-			dlog.Infof(ctx, "... done reading %q", args[1])
-
-			if err := rebuildmappings.ScanForExtents(ctx, fs, bgs, sums); err != nil {
-				return err
-			}
-
-			dlog.Infof(ctx, "Writing reconstructed mappings to stdout...")
-			if err := writeMappingsJSON(os.Stdout, fs); err != nil {
-				return err
-			}
-
-			return nil
-		},
-	})
-}
diff --git a/lib/btrfsprogs/btrfsinspect/rebuildmappings/allsums.go b/lib/btrfsprogs/btrfsinspect/rebuildmappings/allsums.go
index 43d09fe..1a8855b 100644
--- a/lib/btrfsprogs/btrfsinspect/rebuildmappings/allsums.go
+++ b/lib/btrfsprogs/btrfsinspect/rebuildmappings/allsums.go
@@ -6,23 +6,10 @@ package rebuildmappings
 
 import (
 	"context"
-	"encoding/gob"
-	"io"
 	"math"
-	"os"
-	"runtime"
-	"strings"
-	"sync"
 
-	"github.com/datawire/dlib/dgroup"
-	"github.com/datawire/dlib/dlog"
-
-	"git.lukeshu.com/btrfs-progs-ng/lib/btrfs"
-	"git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem"
-	"git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfssum"
 	"git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol"
 	"git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect"
-	"git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsutil"
 )
 
 type AllSums struct {
@@ -67,149 +54,3 @@ func (as AllSums) WalkLogical(ctx context.Context, fn func(btrfsvol.LogicalAddr,
 	}
 	return nil
 }
-
-// Read/Write AllSums ////////////////////////////////////////////////
-
-func ReadAllSums(filename string) (AllSums, error) {
-	fh, err := os.Open(filename)
-	if err != nil {
-		return AllSums{}, err
-	}
-	defer fh.Close()
-	var val AllSums
-	if err := gob.NewDecoder(fh).Decode(&val); err != nil {
-		return AllSums{}, err
-	}
-	return val, nil
-}
-
-func WriteAllSums(w io.Writer, sums AllSums) error {
-	return gob.NewEncoder(w).Encode(sums)
-}
-
-func SumEverything(ctx context.Context, fs *btrfs.FS) (AllSums, error) {
-	var ret AllSums
-
-	// ChecksumSize
-	var alg btrfssum.CSumType
-	var csumSize int
-	if err := func() error {
-		sb, err := fs.Superblock()
-		if err != nil {
-			return err
-		}
-		alg = sb.ChecksumType
-		csumSize = alg.Size()
-		return nil
-	}(); err != nil {
-		return ret, err
-	}
-
-	// Logical
-	dlog.Info(ctx, "Walking CSUM_TREE...")
-	func() {
-		var curAddr btrfsvol.LogicalAddr
-		var curSums strings.Builder
-		btrfsutil.NewBrokenTrees(ctx, fs).TreeWalk(ctx, btrfs.CSUM_TREE_OBJECTID,
-			func(err *btrfs.TreeError) {
-				dlog.Error(ctx, err)
-			},
-			btrfs.TreeWalkHandler{
-				Item: func(path btrfs.TreePath, item btrfs.Item) error {
-					if item.Key.ItemType != btrfsitem.EXTENT_CSUM_KEY {
-						return nil
-					}
-					body := item.Body.(btrfsitem.ExtentCSum)
-
-					for i, sum := range body.Sums {
-						laddr := btrfsvol.LogicalAddr(item.Key.Offset) + (btrfsvol.LogicalAddr(i) * btrfsitem.CSumBlockSize)
-						if laddr != curAddr+(btrfsvol.LogicalAddr(curSums.Len()/csumSize)*btrfsitem.CSumBlockSize) {
-							if curSums.Len() > 0 {
-								ret.Logical = append(ret.Logical, btrfsinspect.SumRun[btrfsvol.LogicalAddr]{
-									ChecksumSize: csumSize,
-									Addr:         curAddr,
-									Sums:         curSums.String(),
-								})
-							}
-							curAddr = laddr
-							curSums.Reset()
-						}
-						curSums.Write(sum[:csumSize])
-					}
-					return nil
-				},
-			},
-		)
-		if curSums.Len() > 0 {
-			ret.Logical = append(ret.Logical, btrfsinspect.SumRun[btrfsvol.LogicalAddr]{
-				ChecksumSize: csumSize,
-				Addr:         curAddr,
-				Sums:         curSums.String(),
-			})
-		}
-	}()
-	if err := ctx.Err(); err != nil {
-		return ret, err
-	}
-	dlog.Info(ctx, "... done walking")
-	runtime.GC()
-	dlog.Info(ctx, "... GC'd")
-
-	// Physical
-	dlog.Info(ctx, "Summing devices...")
-	if err := func() error {
-		devs := fs.LV.PhysicalVolumes()
-
-		var mu sync.Mutex
-		ret.Physical = make(map[btrfsvol.DeviceID]btrfsinspect.SumRun[btrfsvol.PhysicalAddr], len(devs))
-
-		grp := dgroup.NewGroup(ctx, dgroup.GroupConfig{})
-		for devID, dev := range devs {
-			devID, dev := devID, dev
-			grp.Go(dev.Name(), func(ctx context.Context) error {
-				devSize := dev.Size()
-				numSums := int(devSize / btrfsitem.CSumBlockSize)
-				sums := make([]byte, numSums*csumSize)
-				lastPct := -1
-				progress := func(curSum int) {
-					pct := int(100 * float64(curSum) / float64(numSums))
-					if pct != lastPct || curSum == numSums {
-						dlog.Infof(ctx, "... dev[%q] summed %v%%",
-							dev.Name(), pct)
-						lastPct = pct
-					}
-				}
-				for i := 0; i < numSums; i++ {
-					if err := ctx.Err(); err != nil {
-						return err
-					}
-					progress(i)
-					sum, err := btrfsutil.ChecksumPhysical(dev, alg, btrfsvol.PhysicalAddr(i*btrfsitem.CSumBlockSize))
-					if err != nil {
-						return err
-					}
-					copy(sums[i*csumSize:], sum[:csumSize])
-				}
-				progress(numSums)
-				sumsStr := string(sums)
-				mu.Lock()
-				ret.Physical[devID] = btrfsinspect.SumRun[btrfsvol.PhysicalAddr]{
-					ChecksumSize: csumSize,
-					Addr:         0,
-					Sums:         sumsStr,
-				}
-				mu.Unlock()
-				return nil
-			})
-		}
-		return grp.Wait()
-	}(); err != nil {
-		return ret, err
-	}
-	dlog.Info(ctx, "... done summing devices")
-	runtime.GC()
-	dlog.Info(ctx, "... GC'd")
-
-	// Return
-	return ret, nil
-}
diff --git a/lib/btrfsprogs/btrfsinspect/rebuildmappings/blockgroups.go b/lib/btrfsprogs/btrfsinspect/rebuildmappings/blockgroups.go
index 2621c44..a98bf3f 100644
--- a/lib/btrfsprogs/btrfsinspect/rebuildmappings/blockgroups.go
+++ b/lib/btrfsprogs/btrfsinspect/rebuildmappings/blockgroups.go
@@ -5,9 +5,7 @@
 package rebuildmappings
 
 import (
-	"encoding/json"
 	"fmt"
-	"os"
 	"sort"
 
 	"git.lukeshu.com/btrfs-progs-ng/lib/btrfs"
@@ -22,25 +20,6 @@ type BlockGroup struct {
 	Flags btrfsvol.BlockGroupFlags
 }
 
-func ReadNodeScanResults(fs *btrfs.FS, filename string) (map[btrfsvol.LogicalAddr]BlockGroup, error) {
-	scanResultsBytes, err := os.ReadFile(filename)
-	if err != nil {
-		return nil, err
-	}
-
-	var scanResults btrfsinspect.ScanDevicesResult
-	if err := json.Unmarshal(scanResultsBytes, &scanResults); err != nil {
-		return nil, err
-	}
-
-	bgTree, err := ReduceScanResults(fs, scanResults)
-	if err != nil {
-		return nil, err
-	}
-
-	return bgTree, nil
-}
-
 func ReduceScanResults(fs *btrfs.FS, scanResults btrfsinspect.ScanDevicesResult) (map[btrfsvol.LogicalAddr]BlockGroup, error) {
 	// Reduce
 	bgSet := make(map[BlockGroup]struct{})
diff --git a/scripts/main.sh b/scripts/main.sh
index 11ce901..cc03e9f 100755
--- a/scripts/main.sh
+++ b/scripts/main.sh
@@ -17,27 +17,27 @@ go build ./cmd/btrfs-rec
 mkdir -p "$b.gen"
 { set +x; } &>/dev/null
 
-gen $b.gen/0.scan-for-nodes.json \
+gen $b.gen/0.scandevices.json \
     ./btrfs-rec --pv=$b.img \
-    inspect scan-for-nodes
-gen $b.gen/1.mappings.json \
-    ./btrfs-rec --pv=$b.img \
-    inspect rebuild-mappings $b.gen/0.scan-for-nodes.json
-gen $b.gen/2.csums.gob \
-    ./btrfs-rec --pv=$b.img --mappings=$b.gen/1.mappings.json \
-    inspect dump-sums
-# gen $b.gen/3.dbg.txt \
+    inspect scandevices
+# gen $b.gen/1.mappings.json \
+#     ./btrfs-rec --pv=$b.img \
+#     inspect rebuild-mappings $b.gen/0.scan-for-nodes.json
+# gen $b.gen/2.csums.gob \
+#     ./btrfs-rec --pv=$b.img --mappings=$b.gen/1.mappings.json \
+#     inspect dump-sums
+# # gen $b.gen/3.dbg.txt \
+# #     ./btrfs-rec --pv=$b.img --mappings=$b.gen/1.mappings.json \
+# #     inspect dbg $b.gen/2.csums.gob
+# gen $b.gen/3.mappings.json \
 #     ./btrfs-rec --pv=$b.img --mappings=$b.gen/1.mappings.json \
-#     inspect dbg $b.gen/2.csums.gob
-gen $b.gen/3.mappings.json \
-    ./btrfs-rec --pv=$b.img --mappings=$b.gen/1.mappings.json \
-    inspect scan-for-extents $b.gen/0.scan-for-nodes.json $b.gen/2.csums.gob
-gen $b.gen/4.ls-files.txt \
-    ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \
-    inspect ls-files
-gen $b.gen/4.ls-trees.txt \
-    ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \
-    inspect ls-trees --nodescan=$b.gen/0.scan-for-nodes.json
-gen $b.gen/4.nodes.json \
-    ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \
-    inspect rebuild-nodes $b.gen/0.scan-for-nodes.json
+#     inspect scan-for-extents $b.gen/0.scan-for-nodes.json $b.gen/2.csums.gob
+# gen $b.gen/4.ls-files.txt \
+#     ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \
+#     inspect ls-files
+# gen $b.gen/4.ls-trees.txt \
+#     ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \
+#     inspect ls-trees --nodescan=$b.gen/0.scan-for-nodes.json
+# gen $b.gen/4.nodes.json \
+#     ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \
+#     inspect rebuild-nodes $b.gen/0.scan-for-nodes.json