From d9cb7963948cfb1d705c35653f5237a5e8fee3f3 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 15 Jul 2022 11:11:10 -0600 Subject: down to just 2 rebuilt errors! ERRO[0248] ... dev["../scratch/dump-zero.img"] error: could not pair blockgroup laddr=0x0000000000500000 (size=0x0000000000800000 flags=METADATA|single) with a mapping THREAD=/main ERRO[0248] ... dev["../scratch/dump-zero.img"] error: adding flags from blockgroup: (0xc00022a050).AddMapping: member devext has locked size=0x0000000000800000, but union would have size=0x0000000040000000 THREAD=/main --- cmd/btrfs-rec/inspect_rebuildmappings.go | 40 ++++++++----- cmd/btrfs-rec/inspect_scanforextents.go | 99 ++++++++++++++++++++++---------- scripts/main.sh | 4 +- 3 files changed, 99 insertions(+), 44 deletions(-) diff --git a/cmd/btrfs-rec/inspect_rebuildmappings.go b/cmd/btrfs-rec/inspect_rebuildmappings.go index c32bf53..ab767a9 100644 --- a/cmd/btrfs-rec/inspect_rebuildmappings.go +++ b/cmd/btrfs-rec/inspect_rebuildmappings.go @@ -60,21 +60,35 @@ func init() { } dlog.Infof(ctx, "Writing reconstructed mappings to stdout...") - mappings := fs.LV.Mappings() - _, _ = io.WriteString(os.Stdout, "[\n") - for i, mapping := range mappings { - suffix := "," - if i == len(mappings)-1 { - suffix = "" - } - bs, err := json.Marshal(mapping) - if err != nil { - return err - } - fmt.Printf(" %s%s\n", bs, suffix) + if err := writeMappingsJSON(os.Stdout, fs); err != nil { + return err } - _, _ = io.WriteString(os.Stdout, "]\n") + return nil }, }) } + +func writeMappingsJSON(w io.Writer, fs *btrfs.FS) error { + mappings := fs.LV.Mappings() + if _, err := io.WriteString(w, "[\n"); err != nil { + return err + } + for i, mapping := range mappings { + suffix := "," + if i == len(mappings)-1 { + suffix = "" + } + bs, err := json.Marshal(mapping) + if err != nil { + return err + } + if _, err := fmt.Printf(" %s%s\n", bs, suffix); err != nil { + return err + } + } + if _, err := io.WriteString(w, "]\n"); err != nil { + return err + } + return nil +} diff --git a/cmd/btrfs-rec/inspect_scanforextents.go b/cmd/btrfs-rec/inspect_scanforextents.go index 98f7129..d9280ac 100644 --- a/cmd/btrfs-rec/inspect_scanforextents.go +++ b/cmd/btrfs-rec/inspect_scanforextents.go @@ -5,9 +5,7 @@ package main import ( - "bufio" "context" - "encoding/json" "os" "runtime" "sort" @@ -23,7 +21,6 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsutil" - "git.lukeshu.com/btrfs-progs-ng/lib/maps" ) const csumBlockSize = 4 * 1024 @@ -43,6 +40,14 @@ func init() { sum2laddrs := listUnmappedCheckummedExtents(ctx, fs) dlog.Info(ctx, "... done reading checksum tree") + dlog.Info(ctx, "Pruning duplicate sums...") + for sum, addrs := range sum2laddrs { + if len(addrs) != 1 { + delete(sum2laddrs, sum) + } + } + dlog.Info(ctx, "... done pruning") + devs := fs.LV.PhysicalVolumes() gaps := listPhysicalGaps(fs) @@ -66,29 +71,14 @@ func init() { return err } - dlog.Info(ctx, "Writing scan results to stdout...") - out := bufio.NewWriter(os.Stdout) - _, _ = out.WriteString("{") - for i, sum := range maps.SortedKeys(sum2laddrs) { - _, _ = out.WriteString("\n ") - - kBytes, _ := json.Marshal(sum) - _, _ = out.Write(kBytes) + dlog.Info(ctx, "Rebuilding mappings from results...") + rebuildMappings(ctx, fs, devs, sum2laddrs, sum2paddrs) + dlog.Info(ctx, "... done rebuilding mappings") - _, _ = out.WriteString(": ") - - vBytes, _ := json.Marshal(map[string]interface{}{ - "laddrs": sum2laddrs[sum], - "paddrs": sum2paddrs[sum], - }) - _, _ = out.Write(vBytes) - - if i != len(sum2laddrs)-1 { - _, _ = out.WriteString(",") - } + dlog.Infof(ctx, "Writing reconstructed mappings to stdout...") + if err := writeMappingsJSON(os.Stdout, fs); err != nil { + return err } - _, _ = out.WriteString("\n}") - out.Flush() return nil }, @@ -167,11 +157,18 @@ func scanOneDev[T any](ctx context.Context, dev *btrfs.Device, gaps []physicalGa return nil, err } - devSize := dev.Size() + var totalBlocks int64 + for _, gap := range gaps { + for paddr := roundUp(gap.Beg, csumBlockSize); paddr+csumBlockSize <= gap.End; paddr += csumBlockSize { + totalBlocks++ + } + } + lastProgress := -1 - progress := func(pos btrfsvol.PhysicalAddr) { - pct := int(100 * float64(pos) / float64(devSize)) - if pct != lastProgress || pos == devSize { + var curBlock int64 + progress := func() { + pct := int(100 * float64(curBlock) / float64(totalBlocks)) + if pct != lastProgress || curBlock == totalBlocks { dlog.Infof(ctx, "... dev[%q] scanned %v%%", dev.Name(), pct) lastProgress = pct @@ -187,7 +184,8 @@ func scanOneDev[T any](ctx context.Context, dev *btrfs.Device, gaps []physicalGa devSum2paddrs := make(map[shortSum][]btrfsvol.QualifiedPhysicalAddr) for _, gap := range gaps { for paddr := roundUp(gap.Beg, csumBlockSize); paddr+csumBlockSize <= gap.End; paddr += csumBlockSize { - progress(paddr) + progress() + curBlock++ if err := ctx.Err(); err != nil { return nil, err } @@ -211,6 +209,47 @@ func scanOneDev[T any](ctx context.Context, dev *btrfs.Device, gaps []physicalGa }) } } - progress(devSize) + progress() return devSum2paddrs, nil } + +func rebuildMappings(ctx context.Context, fs *btrfs.FS, + devs map[btrfsvol.DeviceID]*btrfs.Device, + sum2laddrs map[shortSum][]btrfsvol.LogicalAddr, + sum2paddrs map[shortSum][]btrfsvol.QualifiedPhysicalAddr) { + + totalPairs := len(sum2paddrs) + lastProgress := -1 + var donePairs int + progress := func() { + pct := int(100 * float64(donePairs) / float64(totalPairs)) + if pct != lastProgress || donePairs == totalPairs { + dlog.Infof(ctx, "... rebuilt %v%% (%v/%v)", pct, donePairs, totalPairs) + lastProgress = pct + if pct%5 == 0 { + runtime.GC() + } + } + } + + for sum, paddrs := range sum2paddrs { + progress() + if len(paddrs) == 1 { + mapping := btrfsvol.Mapping{ + LAddr: sum2laddrs[sum][0], + PAddr: paddrs[0], + Size: csumBlockSize, + } + if err := fs.LV.AddMapping(mapping); err != nil { + dlog.Errorf(ctx, "... dev[%q] error: adding chunk: %v", + devs[paddrs[0].Dev].Name(), err) + } + } else { + delete(sum2paddrs, sum) + delete(sum2laddrs, sum) + } + donePairs++ + + } + progress() +} diff --git a/scripts/main.sh b/scripts/main.sh index ce51142..146d4a3 100755 --- a/scripts/main.sh +++ b/scripts/main.sh @@ -10,7 +10,9 @@ if ! test -s ../scratch/dump-zero.mappings.0.json; then 2> >(tee >&2 ../scratch/dump-zero.mappings.0.log) fi time ./btrfs-rec --pv=../scratch/dump-zero.img --mappings=../scratch/dump-zero.mappings.0.json inspect scan-for-extents \ - > ../scratch/dump-zero.scan-for-extents.json + > ../scratch/dump-zero.mappings.1.json +time ./btrfs-rec --pv=../scratch/dump-zero.img --mappings=../scratch/dump-zero.mappings.1.json inspect rebuild-mappings ../scratch/dump-zero.scan-for-nodes.json \ + > ../scratch/dump-zero.mappings.2.json #time ./btrfs-rec --pv=../scratch/dump-zero.img --mappings=../scratch/dump-zero.mappings.0.json inspect ls-files \ # &> ../scratch/dump-zero.ls-files.txt -- cgit v1.2.3-2-g168b