From 286db83409b56de7b0ac3a74709018c01de43f44 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 9 Aug 2022 16:01:20 -0400 Subject: cli: wip: Rethink the "scan" CLI commands --- cmd/btrfs-rec/inspect_dumpsums.go | 35 ----- cmd/btrfs-rec/inspect_scandevices.go | 72 ++++++++++ cmd/btrfs-rec/inspect_scanforextents.go | 55 ------- cmd/btrfs-rec/inspect_scanfornodes.go | 72 ---------- .../btrfsinspect/rebuildmappings/allsums.go | 159 --------------------- .../btrfsinspect/rebuildmappings/blockgroups.go | 21 --- scripts/main.sh | 44 +++--- 7 files changed, 94 insertions(+), 364 deletions(-) delete mode 100644 cmd/btrfs-rec/inspect_dumpsums.go create mode 100644 cmd/btrfs-rec/inspect_scandevices.go delete mode 100644 cmd/btrfs-rec/inspect_scanforextents.go delete mode 100644 cmd/btrfs-rec/inspect_scanfornodes.go diff --git a/cmd/btrfs-rec/inspect_dumpsums.go b/cmd/btrfs-rec/inspect_dumpsums.go deleted file mode 100644 index 28ae7ef..0000000 --- a/cmd/btrfs-rec/inspect_dumpsums.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (C) 2022 Luke Shumaker -// -// SPDX-License-Identifier: GPL-2.0-or-later - -package main - -import ( - "os" - - "github.com/datawire/dlib/dlog" - "github.com/datawire/ocibuild/pkg/cliutil" - "github.com/spf13/cobra" - - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildmappings" -) - -func init() { - inspectors = append(inspectors, subcommand{ - Command: cobra.Command{ - Use: "dump-sums", - Short: "Dump a buncha checksums as JSON", - Args: cliutil.WrapPositionalArgs(cobra.NoArgs), - }, - RunE: func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error { - ctx := cmd.Context() - sums, err := rebuildmappings.SumEverything(ctx, fs) - if err != nil { - return err - } - dlog.Info(ctx, "Writing sums as gob to stdout...") - return rebuildmappings.WriteAllSums(os.Stdout, sums) - }, - }) -} diff --git a/cmd/btrfs-rec/inspect_scandevices.go b/cmd/btrfs-rec/inspect_scandevices.go new file mode 100644 index 0000000..f5caadb --- /dev/null +++ b/cmd/btrfs-rec/inspect_scandevices.go @@ -0,0 +1,72 @@ +// Copyright (C) 2022 Luke Shumaker +// +// SPDX-License-Identifier: GPL-2.0-or-later + +package main + +import ( + "bufio" + "context" + "os" + "sync" + + "git.lukeshu.com/go/lowmemjson" + "github.com/datawire/dlib/dgroup" + "github.com/datawire/dlib/dlog" + "github.com/datawire/ocibuild/pkg/cliutil" + "github.com/spf13/cobra" + + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect" +) + +func init() { + inspectors = append(inspectors, subcommand{ + Command: cobra.Command{ + Use: "scandevices", + Args: cliutil.WrapPositionalArgs(cobra.NoArgs), + }, + RunE: func(fs *btrfs.FS, cmd *cobra.Command, _ []string) (err error) { + maybeSetErr := func(_err error) { + if err == nil && _err != nil { + err = _err + } + } + ctx := cmd.Context() + + var resultsMu sync.Mutex + results := make(map[btrfsvol.DeviceID]btrfsinspect.ScanOneDeviceResult) + grp := dgroup.NewGroup(ctx, dgroup.GroupConfig{}) + for _, dev := range fs.LV.PhysicalVolumes() { + dev := dev + grp.Go(dev.Name(), func(ctx context.Context) error { + superblock, err := dev.Superblock() + if err != nil { + return err + } + dlog.Infof(ctx, "dev[%q] Scanning for unreachable nodes...", dev.Name()) + devResult, err := btrfsinspect.ScanOneDevice(ctx, dev, *superblock) + dlog.Infof(ctx, "dev[%q] Finished scanning", dev.Name()) + resultsMu.Lock() + results[superblock.DevItem.DevID] = devResult + resultsMu.Unlock() + return err + }) + } + if err := grp.Wait(); err != nil { + return err + } + + dlog.Info(ctx, "Writing scan results to stdout...") + buffer := bufio.NewWriter(os.Stdout) + defer func() { + maybeSetErr(buffer.Flush()) + }() + return lowmemjson.Encode(&lowmemjson.ReEncoder{ + Out: buffer, + Indent: "\t", + }, results) + }, + }) +} diff --git a/cmd/btrfs-rec/inspect_scanforextents.go b/cmd/btrfs-rec/inspect_scanforextents.go deleted file mode 100644 index 9b5ada4..0000000 --- a/cmd/btrfs-rec/inspect_scanforextents.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (C) 2022 Luke Shumaker -// -// SPDX-License-Identifier: GPL-2.0-or-later - -package main - -import ( - "os" - "runtime" - - "github.com/datawire/dlib/dlog" - "github.com/datawire/ocibuild/pkg/cliutil" - "github.com/spf13/cobra" - - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildmappings" -) - -func init() { - inspectors = append(inspectors, subcommand{ - Command: cobra.Command{ - Use: "scan-for-extents NODESCAN.json DUMPSUMS.gob", - Args: cliutil.WrapPositionalArgs(cobra.ExactArgs(2)), - }, - RunE: func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - - dlog.Infof(ctx, "Reading %q...", args[0]) - bgs, err := rebuildmappings.ReadNodeScanResults(fs, args[0]) - if err != nil { - return err - } - runtime.GC() - dlog.Infof(ctx, "... done reading %q", args[0]) - - dlog.Infof(ctx, "Reading %q...", args[1]) - sums, err := rebuildmappings.ReadAllSums(args[1]) - if err != nil { - return err - } - dlog.Infof(ctx, "... done reading %q", args[1]) - - if err := rebuildmappings.ScanForExtents(ctx, fs, bgs, sums); err != nil { - return err - } - - dlog.Infof(ctx, "Writing reconstructed mappings to stdout...") - if err := writeMappingsJSON(os.Stdout, fs); err != nil { - return err - } - - return nil - }, - }) -} diff --git a/cmd/btrfs-rec/inspect_scanfornodes.go b/cmd/btrfs-rec/inspect_scanfornodes.go deleted file mode 100644 index 13f7760..0000000 --- a/cmd/btrfs-rec/inspect_scanfornodes.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (C) 2022 Luke Shumaker -// -// SPDX-License-Identifier: GPL-2.0-or-later - -package main - -import ( - "context" - "encoding/json" - "os" - "sync" - - "github.com/datawire/dlib/dgroup" - "github.com/datawire/dlib/dlog" - "github.com/datawire/ocibuild/pkg/cliutil" - "github.com/spf13/cobra" - - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect" -) - -func init() { - inspectors = append(inspectors, subcommand{ - Command: cobra.Command{ - Use: "scan-for-nodes", - Short: "Scan devices for (potentially lost) nodes", - Long: "" + - "The found information is printed as JSON on stdout, and can\n" + - "be read by `btrfs-rec inspect rebuild-mappings`.\n" + - "\n" + - "This information is mostly useful for rebuilding a broken\n" + - "chunk/dev-extent/blockgroup trees, but can also have some\n" + - "minimal utility in repairing other trees.\n" + - "\n" + - "This is very similar the initial scan done by\n" + - "`btrfs rescue chunk-recover`. Like `btrfs rescue chunk-recover`,\n" + - "this is likely probably slow because it reads the entirety of\n" + - "each device.", - Args: cliutil.WrapPositionalArgs(cobra.NoArgs), - }, - RunE: func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error { - ctx := cmd.Context() - - var resultsMu sync.Mutex - results := make(map[btrfsvol.DeviceID]btrfsinspect.ScanOneDeviceResult) - grp := dgroup.NewGroup(ctx, dgroup.GroupConfig{}) - for _, dev := range fs.LV.PhysicalVolumes() { - dev := dev - grp.Go(dev.Name(), func(ctx context.Context) error { - superblock, err := dev.Superblock() - if err != nil { - return err - } - dlog.Infof(ctx, "dev[%q] Scanning for unreachable nodes...", dev.Name()) - devResult, err := btrfsinspect.ScanOneDevice(ctx, dev, *superblock) - dlog.Infof(ctx, "dev[%q] Finished scanning", dev.Name()) - resultsMu.Lock() - results[superblock.DevItem.DevID] = devResult - resultsMu.Unlock() - return err - }) - } - if err := grp.Wait(); err != nil { - return err - } - - dlog.Info(ctx, "Writing scan results to stdout...") - return json.NewEncoder(os.Stdout).Encode(results) - }, - }) -} diff --git a/lib/btrfsprogs/btrfsinspect/rebuildmappings/allsums.go b/lib/btrfsprogs/btrfsinspect/rebuildmappings/allsums.go index 43d09fe..1a8855b 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildmappings/allsums.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildmappings/allsums.go @@ -6,23 +6,10 @@ package rebuildmappings import ( "context" - "encoding/gob" - "io" "math" - "os" - "runtime" - "strings" - "sync" - "github.com/datawire/dlib/dgroup" - "github.com/datawire/dlib/dlog" - - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfssum" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsutil" ) type AllSums struct { @@ -67,149 +54,3 @@ func (as AllSums) WalkLogical(ctx context.Context, fn func(btrfsvol.LogicalAddr, } return nil } - -// Read/Write AllSums //////////////////////////////////////////////// - -func ReadAllSums(filename string) (AllSums, error) { - fh, err := os.Open(filename) - if err != nil { - return AllSums{}, err - } - defer fh.Close() - var val AllSums - if err := gob.NewDecoder(fh).Decode(&val); err != nil { - return AllSums{}, err - } - return val, nil -} - -func WriteAllSums(w io.Writer, sums AllSums) error { - return gob.NewEncoder(w).Encode(sums) -} - -func SumEverything(ctx context.Context, fs *btrfs.FS) (AllSums, error) { - var ret AllSums - - // ChecksumSize - var alg btrfssum.CSumType - var csumSize int - if err := func() error { - sb, err := fs.Superblock() - if err != nil { - return err - } - alg = sb.ChecksumType - csumSize = alg.Size() - return nil - }(); err != nil { - return ret, err - } - - // Logical - dlog.Info(ctx, "Walking CSUM_TREE...") - func() { - var curAddr btrfsvol.LogicalAddr - var curSums strings.Builder - btrfsutil.NewBrokenTrees(ctx, fs).TreeWalk(ctx, btrfs.CSUM_TREE_OBJECTID, - func(err *btrfs.TreeError) { - dlog.Error(ctx, err) - }, - btrfs.TreeWalkHandler{ - Item: func(path btrfs.TreePath, item btrfs.Item) error { - if item.Key.ItemType != btrfsitem.EXTENT_CSUM_KEY { - return nil - } - body := item.Body.(btrfsitem.ExtentCSum) - - for i, sum := range body.Sums { - laddr := btrfsvol.LogicalAddr(item.Key.Offset) + (btrfsvol.LogicalAddr(i) * btrfsitem.CSumBlockSize) - if laddr != curAddr+(btrfsvol.LogicalAddr(curSums.Len()/csumSize)*btrfsitem.CSumBlockSize) { - if curSums.Len() > 0 { - ret.Logical = append(ret.Logical, btrfsinspect.SumRun[btrfsvol.LogicalAddr]{ - ChecksumSize: csumSize, - Addr: curAddr, - Sums: curSums.String(), - }) - } - curAddr = laddr - curSums.Reset() - } - curSums.Write(sum[:csumSize]) - } - return nil - }, - }, - ) - if curSums.Len() > 0 { - ret.Logical = append(ret.Logical, btrfsinspect.SumRun[btrfsvol.LogicalAddr]{ - ChecksumSize: csumSize, - Addr: curAddr, - Sums: curSums.String(), - }) - } - }() - if err := ctx.Err(); err != nil { - return ret, err - } - dlog.Info(ctx, "... done walking") - runtime.GC() - dlog.Info(ctx, "... GC'd") - - // Physical - dlog.Info(ctx, "Summing devices...") - if err := func() error { - devs := fs.LV.PhysicalVolumes() - - var mu sync.Mutex - ret.Physical = make(map[btrfsvol.DeviceID]btrfsinspect.SumRun[btrfsvol.PhysicalAddr], len(devs)) - - grp := dgroup.NewGroup(ctx, dgroup.GroupConfig{}) - for devID, dev := range devs { - devID, dev := devID, dev - grp.Go(dev.Name(), func(ctx context.Context) error { - devSize := dev.Size() - numSums := int(devSize / btrfsitem.CSumBlockSize) - sums := make([]byte, numSums*csumSize) - lastPct := -1 - progress := func(curSum int) { - pct := int(100 * float64(curSum) / float64(numSums)) - if pct != lastPct || curSum == numSums { - dlog.Infof(ctx, "... dev[%q] summed %v%%", - dev.Name(), pct) - lastPct = pct - } - } - for i := 0; i < numSums; i++ { - if err := ctx.Err(); err != nil { - return err - } - progress(i) - sum, err := btrfsutil.ChecksumPhysical(dev, alg, btrfsvol.PhysicalAddr(i*btrfsitem.CSumBlockSize)) - if err != nil { - return err - } - copy(sums[i*csumSize:], sum[:csumSize]) - } - progress(numSums) - sumsStr := string(sums) - mu.Lock() - ret.Physical[devID] = btrfsinspect.SumRun[btrfsvol.PhysicalAddr]{ - ChecksumSize: csumSize, - Addr: 0, - Sums: sumsStr, - } - mu.Unlock() - return nil - }) - } - return grp.Wait() - }(); err != nil { - return ret, err - } - dlog.Info(ctx, "... done summing devices") - runtime.GC() - dlog.Info(ctx, "... GC'd") - - // Return - return ret, nil -} diff --git a/lib/btrfsprogs/btrfsinspect/rebuildmappings/blockgroups.go b/lib/btrfsprogs/btrfsinspect/rebuildmappings/blockgroups.go index 2621c44..a98bf3f 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildmappings/blockgroups.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildmappings/blockgroups.go @@ -5,9 +5,7 @@ package rebuildmappings import ( - "encoding/json" "fmt" - "os" "sort" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" @@ -22,25 +20,6 @@ type BlockGroup struct { Flags btrfsvol.BlockGroupFlags } -func ReadNodeScanResults(fs *btrfs.FS, filename string) (map[btrfsvol.LogicalAddr]BlockGroup, error) { - scanResultsBytes, err := os.ReadFile(filename) - if err != nil { - return nil, err - } - - var scanResults btrfsinspect.ScanDevicesResult - if err := json.Unmarshal(scanResultsBytes, &scanResults); err != nil { - return nil, err - } - - bgTree, err := ReduceScanResults(fs, scanResults) - if err != nil { - return nil, err - } - - return bgTree, nil -} - func ReduceScanResults(fs *btrfs.FS, scanResults btrfsinspect.ScanDevicesResult) (map[btrfsvol.LogicalAddr]BlockGroup, error) { // Reduce bgSet := make(map[BlockGroup]struct{}) diff --git a/scripts/main.sh b/scripts/main.sh index 11ce901..cc03e9f 100755 --- a/scripts/main.sh +++ b/scripts/main.sh @@ -17,27 +17,27 @@ go build ./cmd/btrfs-rec mkdir -p "$b.gen" { set +x; } &>/dev/null -gen $b.gen/0.scan-for-nodes.json \ +gen $b.gen/0.scandevices.json \ ./btrfs-rec --pv=$b.img \ - inspect scan-for-nodes -gen $b.gen/1.mappings.json \ - ./btrfs-rec --pv=$b.img \ - inspect rebuild-mappings $b.gen/0.scan-for-nodes.json -gen $b.gen/2.csums.gob \ - ./btrfs-rec --pv=$b.img --mappings=$b.gen/1.mappings.json \ - inspect dump-sums -# gen $b.gen/3.dbg.txt \ + inspect scandevices +# gen $b.gen/1.mappings.json \ +# ./btrfs-rec --pv=$b.img \ +# inspect rebuild-mappings $b.gen/0.scan-for-nodes.json +# gen $b.gen/2.csums.gob \ +# ./btrfs-rec --pv=$b.img --mappings=$b.gen/1.mappings.json \ +# inspect dump-sums +# # gen $b.gen/3.dbg.txt \ +# # ./btrfs-rec --pv=$b.img --mappings=$b.gen/1.mappings.json \ +# # inspect dbg $b.gen/2.csums.gob +# gen $b.gen/3.mappings.json \ # ./btrfs-rec --pv=$b.img --mappings=$b.gen/1.mappings.json \ -# inspect dbg $b.gen/2.csums.gob -gen $b.gen/3.mappings.json \ - ./btrfs-rec --pv=$b.img --mappings=$b.gen/1.mappings.json \ - inspect scan-for-extents $b.gen/0.scan-for-nodes.json $b.gen/2.csums.gob -gen $b.gen/4.ls-files.txt \ - ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \ - inspect ls-files -gen $b.gen/4.ls-trees.txt \ - ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \ - inspect ls-trees --nodescan=$b.gen/0.scan-for-nodes.json -gen $b.gen/4.nodes.json \ - ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \ - inspect rebuild-nodes $b.gen/0.scan-for-nodes.json +# inspect scan-for-extents $b.gen/0.scan-for-nodes.json $b.gen/2.csums.gob +# gen $b.gen/4.ls-files.txt \ +# ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \ +# inspect ls-files +# gen $b.gen/4.ls-trees.txt \ +# ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \ +# inspect ls-trees --nodescan=$b.gen/0.scan-for-nodes.json +# gen $b.gen/4.nodes.json \ +# ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \ +# inspect rebuild-nodes $b.gen/0.scan-for-nodes.json -- cgit v1.2.3-2-g168b