diff options
author | Luke Shumaker <lukeshu@lukeshu.com> | 2023-04-05 07:49:02 -0600 |
---|---|---|
committer | Luke Shumaker <lukeshu@lukeshu.com> | 2023-04-05 07:49:02 -0600 |
commit | 68eb7a16b9759646619a7d9dec2b62fa9d0c30cf (patch) | |
tree | 8bb4b70337a299f1dacb3c2858210d4bf6bd4b04 | |
parent | b0f290078d531d2dcb5d34e809b0711ce9b6491e (diff) | |
parent | d7dd6dfd7aeb40f06ff4fbe7f906d8feed64b95f (diff) |
Merge branch 'lukeshu/misc'
-rw-r--r-- | cmd/btrfs-rec/inspect/rebuildmappings/process.go | 2 | ||||
-rw-r--r-- | cmd/btrfs-rec/inspect/rebuildtrees/rebuild.go | 28 | ||||
-rw-r--r-- | cmd/btrfs-rec/inspect/rebuildtrees/rebuild_wantcb.go | 61 | ||||
-rw-r--r-- | cmd/btrfs-rec/inspect/rebuildtrees/scan.go | 30 | ||||
-rw-r--r-- | cmd/btrfs-rec/inspect_lstrees.go | 23 | ||||
-rw-r--r-- | cmd/btrfs-rec/inspect_rebuildtrees.go | 24 | ||||
-rw-r--r-- | cmd/btrfs-rec/main.go | 63 | ||||
-rw-r--r-- | lib/btrfs/btrfsvol/lvm.go | 9 | ||||
-rw-r--r-- | lib/btrfsutil/graph.go | 99 | ||||
-rw-r--r-- | lib/btrfsutil/listnodes.go | 24 | ||||
-rw-r--r-- | lib/btrfsutil/old_rebuilt_forrest.go | 10 | ||||
-rw-r--r-- | lib/btrfsutil/rebuilt_forrest.go | 40 | ||||
-rw-r--r-- | lib/btrfsutil/rebuilt_tree.go | 134 | ||||
-rw-r--r-- | lib/btrfsutil/scan.go | 8 | ||||
-rw-r--r-- | lib/containers/set.go | 17 | ||||
-rw-r--r-- | lib/maps/maputil.go | 17 | ||||
-rw-r--r-- | lib/textui/log.go | 20 | ||||
-rwxr-xr-x | scripts/main.sh | 16 | ||||
-rwxr-xr-x | scripts/mount.sh | 1 |
19 files changed, 409 insertions, 217 deletions
diff --git a/cmd/btrfs-rec/inspect/rebuildmappings/process.go b/cmd/btrfs-rec/inspect/rebuildmappings/process.go index 7a49cc6..2e694b5 100644 --- a/cmd/btrfs-rec/inspect/rebuildmappings/process.go +++ b/cmd/btrfs-rec/inspect/rebuildmappings/process.go @@ -38,7 +38,7 @@ func RebuildMappings(ctx context.Context, fs *btrfs.FS, scanResults ScanDevicesR devIDs := maps.SortedKeys(scanResults) devices := fs.LV.PhysicalVolumes() for _, devID := range devIDs { - if _, ok := devices[devID]; !ok { + if !maps.HasKey(devices, devID) { return fmt.Errorf("device ID %v mentioned in scan results is not part of the filesystem", devID) } devResults := scanResults[devID] diff --git a/cmd/btrfs-rec/inspect/rebuildtrees/rebuild.go b/cmd/btrfs-rec/inspect/rebuildtrees/rebuild.go index 0d25ac3..427070a 100644 --- a/cmd/btrfs-rec/inspect/rebuildtrees/rebuild.go +++ b/cmd/btrfs-rec/inspect/rebuildtrees/rebuild.go @@ -198,11 +198,13 @@ func (o *rebuilder) processAddedItemQueue(ctx context.Context) error { ctx := dlog.WithField(ctx, "btrfs.inspect.rebuild-trees.rebuild.settle.item", key) tree := o.rebuilt.RebuiltTree(ctx, key.TreeID) - incPtr, ok := tree.RebuiltItems(ctx).Load(key.Key) + incPtr, ok := tree.RebuiltAcquireItems(ctx).Load(key.Key) + tree.RebuiltReleaseItems() if !ok { panic(fmt.Errorf("should not happen: failed to load already-added item: %v", key)) } - excPtr, ok := tree.RebuiltPotentialItems(ctx).Load(key.Key) + excPtr, ok := tree.RebuiltAcquirePotentialItems(ctx).Load(key.Key) + tree.RebuiltReleasePotentialItems() if ok && tree.RebuiltShouldReplace(incPtr.Node, excPtr.Node) { wantKey := wantWithTree{ TreeID: key.TreeID, @@ -517,24 +519,12 @@ func (o *rebuilder) resolveTreeAugments(ctx context.Context, treeID btrfsprim.Ob //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// func (queue *treeAugmentQueue) has(wantKey want) bool { - if queue != nil { - if queue.zero != nil { - if _, ok := queue.zero[wantKey]; ok { - return true - } - } - if queue.single != nil { - if _, ok := queue.single[wantKey]; ok { - return true - } - } - if queue.multi != nil { - if _, ok := queue.multi[wantKey]; ok { - return true - } - } + if queue == nil { + return false } - return false + return (queue.zero != nil && maps.HasKey(queue.zero, wantKey)) || + (queue.single != nil && maps.HasKey(queue.single, wantKey)) || + (queue.multi != nil && maps.HasKey(queue.multi, wantKey)) } func (queue *treeAugmentQueue) store(wantKey want, choices containers.Set[btrfsvol.LogicalAddr]) { diff --git a/cmd/btrfs-rec/inspect/rebuildtrees/rebuild_wantcb.go b/cmd/btrfs-rec/inspect/rebuildtrees/rebuild_wantcb.go index eff2a83..0526e93 100644 --- a/cmd/btrfs-rec/inspect/rebuildtrees/rebuild_wantcb.go +++ b/cmd/btrfs-rec/inspect/rebuildtrees/rebuild_wantcb.go @@ -43,18 +43,22 @@ func (o graphCallbacks) Want(ctx context.Context, reason string, treeID btrfspri } func (o *rebuilder) _want(ctx context.Context, wantKey wantWithTree) (key btrfsprim.Key, ok bool) { - if o.rebuilt.RebuiltTree(ctx, wantKey.TreeID) == nil { + tree := o.rebuilt.RebuiltTree(ctx, wantKey.TreeID) + if tree == nil { o.enqueueRetry(wantKey.TreeID) return btrfsprim.Key{}, false } + tgt := wantKey.Key.Key() + // check if we already have it - tgt := wantKey.Key.Key() - if key, _, ok := o.rebuilt.RebuiltTree(ctx, wantKey.TreeID).RebuiltItems(ctx).Search(func(key btrfsprim.Key, _ btrfsutil.ItemPtr) int { + key, _, ok = tree.RebuiltAcquireItems(ctx).Search(func(key btrfsprim.Key, _ btrfsutil.ItemPtr) int { key.Offset = 0 return tgt.Compare(key) - }); ok { + }) + tree.RebuiltReleaseItems() + if ok { return key, true } @@ -64,15 +68,16 @@ func (o *rebuilder) _want(ctx context.Context, wantKey wantWithTree) (key btrfsp return btrfsprim.Key{}, false } wants := make(containers.Set[btrfsvol.LogicalAddr]) - o.rebuilt.RebuiltTree(ctx, wantKey.TreeID).RebuiltPotentialItems(ctx).Subrange( + tree.RebuiltAcquirePotentialItems(ctx).Subrange( func(k btrfsprim.Key, _ btrfsutil.ItemPtr) int { k.Offset = 0 return tgt.Compare(k) }, func(_ btrfsprim.Key, v btrfsutil.ItemPtr) bool { - wants.InsertFrom(o.rebuilt.RebuiltTree(ctx, wantKey.TreeID).RebuiltLeafToRoots(ctx, v.Node)) + wants.InsertFrom(tree.RebuiltLeafToRoots(ctx, v.Node)) return true }) + tree.RebuiltReleasePotentialItems() o.wantAugment(ctx, wantKey, wants) return btrfsprim.Key{}, false } @@ -93,15 +98,19 @@ func (o graphCallbacks) WantOff(ctx context.Context, reason string, treeID btrfs } func (o *rebuilder) _wantOff(ctx context.Context, wantKey wantWithTree) (ok bool) { - if o.rebuilt.RebuiltTree(ctx, wantKey.TreeID) == nil { + tree := o.rebuilt.RebuiltTree(ctx, wantKey.TreeID) + if tree == nil { o.enqueueRetry(wantKey.TreeID) return false } + tgt := wantKey.Key.Key() + // check if we already have it - tgt := wantKey.Key.Key() - if _, ok := o.rebuilt.RebuiltTree(ctx, wantKey.TreeID).RebuiltItems(ctx).Load(tgt); ok { + _, ok = tree.RebuiltAcquireItems(ctx).Load(tgt) + tree.RebuiltReleaseItems() + if ok { return true } @@ -111,12 +120,13 @@ func (o *rebuilder) _wantOff(ctx context.Context, wantKey wantWithTree) (ok bool return false } wants := make(containers.Set[btrfsvol.LogicalAddr]) - o.rebuilt.RebuiltTree(ctx, wantKey.TreeID).RebuiltPotentialItems(ctx).Subrange( + tree.RebuiltAcquirePotentialItems(ctx).Subrange( func(k btrfsprim.Key, _ btrfsutil.ItemPtr) int { return tgt.Compare(k) }, func(_ btrfsprim.Key, v btrfsutil.ItemPtr) bool { - wants.InsertFrom(o.rebuilt.RebuiltTree(ctx, wantKey.TreeID).RebuiltLeafToRoots(ctx, v.Node)) + wants.InsertFrom(tree.RebuiltLeafToRoots(ctx, v.Node)) return true }) + tree.RebuiltReleasePotentialItems() o.wantAugment(ctx, wantKey, wants) return false } @@ -134,16 +144,18 @@ func (o graphCallbacks) WantDirIndex(ctx context.Context, reason string, treeID } ctx = withWant(ctx, logFieldItemWant, reason, wantKey) - if o.rebuilt.RebuiltTree(ctx, treeID) == nil { + tree := o.rebuilt.RebuiltTree(ctx, treeID) + if tree == nil { o.enqueueRetry(treeID) return } + tgt := wantKey.Key.Key() + // check if we already have it - tgt := wantKey.Key.Key() found := false - o.rebuilt.RebuiltTree(ctx, treeID).RebuiltItems(ctx).Subrange( + tree.RebuiltAcquireItems(ctx).Subrange( func(key btrfsprim.Key, _ btrfsutil.ItemPtr) int { key.Offset = 0 return tgt.Compare(key) @@ -154,6 +166,7 @@ func (o graphCallbacks) WantDirIndex(ctx context.Context, reason string, treeID } return !found }) + tree.RebuiltReleaseItems() if found { return } @@ -164,17 +177,18 @@ func (o graphCallbacks) WantDirIndex(ctx context.Context, reason string, treeID return } wants := make(containers.Set[btrfsvol.LogicalAddr]) - o.rebuilt.RebuiltTree(ctx, treeID).RebuiltPotentialItems(ctx).Subrange( + tree.RebuiltAcquirePotentialItems(ctx).Subrange( func(key btrfsprim.Key, _ btrfsutil.ItemPtr) int { key.Offset = 0 return tgt.Compare(key) }, func(_ btrfsprim.Key, ptr btrfsutil.ItemPtr) bool { if itemName, ok := o.scan.Names[ptr]; ok && bytes.Equal(itemName, name) { - wants.InsertFrom(o.rebuilt.RebuiltTree(ctx, treeID).RebuiltLeafToRoots(ctx, ptr.Node)) + wants.InsertFrom(tree.RebuiltLeafToRoots(ctx, ptr.Node)) } return true }) + tree.RebuiltReleasePotentialItems() o.wantAugment(ctx, wantKey, wants) } @@ -259,7 +273,8 @@ func (o graphCallbacks) _wantRange( ctx = withWant(ctx, logFieldItemWant, reason, wantKey) wantKey.Key.OffsetType = offsetRange - if o.rebuilt.RebuiltTree(ctx, treeID) == nil { + tree := o.rebuilt.RebuiltTree(ctx, treeID) + if tree == nil { o.enqueueRetry(treeID) return } @@ -275,7 +290,7 @@ func (o graphCallbacks) _wantRange( }) o._walkRange( ctx, - o.rebuilt.RebuiltTree(ctx, treeID).RebuiltItems(ctx), + tree.RebuiltAcquireItems(ctx), treeID, objID, typ, beg, end, func(runKey btrfsprim.Key, _ btrfsutil.ItemPtr, runBeg, runEnd uint64) { var overlappingGaps []*containers.RBNode[gap] @@ -315,12 +330,13 @@ func (o graphCallbacks) _wantRange( }) } }) + tree.RebuiltReleaseItems() // Step 2: Fill each gap. if gaps.Len() == 0 { return } - potentialItems := o.rebuilt.RebuiltTree(ctx, treeID).RebuiltPotentialItems(ctx) + potentialItems := tree.RebuiltAcquirePotentialItems(ctx) gaps.Range(func(rbNode *containers.RBNode[gap]) bool { gap := rbNode.Value last := gap.Beg @@ -340,7 +356,7 @@ func (o graphCallbacks) _wantRange( wantKey.Key.OffsetLow = gap.Beg wantKey.Key.OffsetHigh = gap.End wantCtx := withWant(ctx, logFieldItemWant, reason, wantKey) - o.wantAugment(wantCtx, wantKey, o.rebuilt.RebuiltTree(wantCtx, treeID).RebuiltLeafToRoots(wantCtx, v.Node)) + o.wantAugment(wantCtx, wantKey, tree.RebuiltLeafToRoots(wantCtx, v.Node)) last = runEnd }) if last < gap.End { @@ -352,6 +368,7 @@ func (o graphCallbacks) _wantRange( } return true }) + tree.RebuiltReleasePotentialItems() } // WantCSum implements btrfscheck.GraphCallbacks. @@ -372,7 +389,9 @@ func (o graphCallbacks) WantCSum(ctx context.Context, reason string, inodeTree, o.enqueueRetry(inodeTree) return } - inodePtr, ok := o.rebuilt.RebuiltTree(inodeCtx, inodeTree).RebuiltItems(inodeCtx).Load(inodeWant.Key.Key()) + tree := o.rebuilt.RebuiltTree(inodeCtx, inodeTree) + inodePtr, ok := tree.RebuiltAcquireItems(inodeCtx).Load(inodeWant.Key.Key()) + tree.RebuiltReleaseItems() if !ok { panic(fmt.Errorf("should not happen: could not load key: %v", inodeWant)) } diff --git a/cmd/btrfs-rec/inspect/rebuildtrees/scan.go b/cmd/btrfs-rec/inspect/rebuildtrees/scan.go index a6f9c7a..f266dab 100644 --- a/cmd/btrfs-rec/inspect/rebuildtrees/scan.go +++ b/cmd/btrfs-rec/inspect/rebuildtrees/scan.go @@ -41,31 +41,35 @@ type ScanDevicesResult struct { Sizes map[btrfsutil.ItemPtr]SizeAndErr // EXTENT_CSUM and EXTENT_DATA } -func ScanDevices(ctx context.Context, fs *btrfs.FS, nodeList []btrfsvol.LogicalAddr) (ScanDevicesResult, error) { +func ScanDevices(_ctx context.Context, fs *btrfs.FS, nodeList []btrfsvol.LogicalAddr) (ScanDevicesResult, error) { + // read-superblock ///////////////////////////////////////////////////////////// + ctx := dlog.WithField(_ctx, "btrfs.inspect.rebuild-trees.read.substep", "read-superblock") dlog.Info(ctx, "Reading superblock...") sb, err := fs.Superblock() if err != nil { return ScanDevicesResult{}, err } - dlog.Infof(ctx, "Reading node data from FS...") - - var stats textui.Portion[int] - stats.D = len(nodeList) - progressWriter := textui.NewProgress[textui.Portion[int]]( - dlog.WithField(ctx, "btrfs.inspect.rebuild-trees.read.substep", "read-nodes"), - dlog.LogLevelInfo, textui.Tunable(1*time.Second)) - + // read-roots ////////////////////////////////////////////////////////////////// + ctx = dlog.WithField(_ctx, "btrfs.inspect.rebuild-trees.read.substep", "read-roots") ret := ScanDevicesResult{ Superblock: *sb, Graph: btrfsutil.NewGraph(ctx, *sb), - Flags: make(map[btrfsutil.ItemPtr]FlagsAndErr), Names: make(map[btrfsutil.ItemPtr][]byte), Sizes: make(map[btrfsutil.ItemPtr]SizeAndErr), } + // read-nodes ////////////////////////////////////////////////////////////////// + ctx = dlog.WithField(_ctx, "btrfs.inspect.rebuild-trees.read.substep", "read-nodes") + dlog.Infof(ctx, "Reading node data from FS...") + var stats textui.Portion[int] + stats.D = len(nodeList) + progressWriter := textui.NewProgress[textui.Portion[int]]( + ctx, + dlog.LogLevelInfo, + textui.Tunable(1*time.Second)) progressWriter.Set(stats) for _, laddr := range nodeList { if err := ctx.Err(); err != nil { @@ -78,11 +82,8 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, nodeList []btrfsvol.LogicalA fs.ReleaseNode(node) return ScanDevicesResult{}, err } - ret.insertNode(node) - fs.ReleaseNode(node) - stats.N++ progressWriter.Set(stats) } @@ -92,7 +93,8 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, nodeList []btrfsvol.LogicalA progressWriter.Done() dlog.Info(ctx, "... done reading node data") - ctx = dlog.WithField(ctx, "btrfs.inspect.rebuild-trees.read.substep", "check") + // check /////////////////////////////////////////////////////////////////////// + ctx = dlog.WithField(_ctx, "btrfs.inspect.rebuild-trees.read.substep", "check") if err := ret.Graph.FinalCheck(ctx, fs); err != nil { return ScanDevicesResult{}, err } diff --git a/cmd/btrfs-rec/inspect_lstrees.go b/cmd/btrfs-rec/inspect_lstrees.go index 24622b4..decb75c 100644 --- a/cmd/btrfs-rec/inspect_lstrees.go +++ b/cmd/btrfs-rec/inspect_lstrees.go @@ -25,28 +25,16 @@ import ( ) func init() { - var nodeListFilename string - cmd := &cobra.Command{ + inspectors.AddCommand(&cobra.Command{ Use: "ls-trees", Short: "A brief view what types of items are in each tree", Long: "" + "If no --node-list is given, then a slow sector-by-sector scan " + "will be used to find all lost+found nodes.", Args: cliutil.WrapPositionalArgs(cobra.NoArgs), - RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error { + RunE: runWithReadableFSAndNodeList(func(fs btrfs.ReadableFS, nodeList []btrfsvol.LogicalAddr, cmd *cobra.Command, _ []string) error { ctx := cmd.Context() - var nodeList []btrfsvol.LogicalAddr - var err error - if nodeListFilename != "" { - nodeList, err = readJSONFile[[]btrfsvol.LogicalAddr](ctx, nodeListFilename) - } else { - nodeList, err = btrfsutil.ListNodes(ctx, fs) - } - if err != nil { - return err - } - var treeErrCnt int var treeItemCnt map[btrfsitem.Type]int flush := func() { @@ -124,10 +112,5 @@ func init() { return nil }), - } - cmd.Flags().StringVar(&nodeListFilename, "node-list", "", - "Output of 'btrfs-recs inspect [rebuild-mappings] list-nodes' to use for a lost+found tree") - noError(cmd.MarkFlagFilename("node-list")) - - inspectors.AddCommand(cmd) + }) } diff --git a/cmd/btrfs-rec/inspect_rebuildtrees.go b/cmd/btrfs-rec/inspect_rebuildtrees.go index 676533a..1e808a0 100644 --- a/cmd/btrfs-rec/inspect_rebuildtrees.go +++ b/cmd/btrfs-rec/inspect_rebuildtrees.go @@ -17,13 +17,11 @@ import ( "git.lukeshu.com/btrfs-progs-ng/cmd/btrfs-rec/inspect/rebuildtrees" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfsutil" "git.lukeshu.com/btrfs-progs-ng/lib/textui" ) func init() { - var nodeListFilename string - cmd := &cobra.Command{ + inspectors.AddCommand(&cobra.Command{ Use: "rebuild-trees", Long: "" + "Rebuild broken btrees based on missing items that are implied " + @@ -34,20 +32,9 @@ func init() { "If no --node-list is given, then a slow sector-by-sector scan " + "will be used to find all nodes.", Args: cliutil.WrapPositionalArgs(cobra.NoArgs), - RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { + RunE: runWithRawFSAndNodeList(func(fs *btrfs.FS, nodeList []btrfsvol.LogicalAddr, cmd *cobra.Command, args []string) error { ctx := cmd.Context() - var nodeList []btrfsvol.LogicalAddr - var err error - if nodeListFilename != "" { - nodeList, err = readJSONFile[[]btrfsvol.LogicalAddr](ctx, nodeListFilename) - } else { - nodeList, err = btrfsutil.ListNodes(ctx, fs) - } - if err != nil { - return err - } - rebuilder, err := rebuildtrees.NewRebuilder(ctx, fs, nodeList) if err != nil { return err @@ -78,10 +65,5 @@ func init() { return rebuildErr }), - } - cmd.Flags().StringVar(&nodeListFilename, "node-list", "", - "Output of 'btrfs-recs inspect [rebuild-mappings] list-nodes' to use for the node list") - noError(cmd.MarkFlagFilename("node-list")) - - inspectors.AddCommand(cmd) + }) } diff --git a/cmd/btrfs-rec/main.go b/cmd/btrfs-rec/main.go index d221c3d..da3aced 100644 --- a/cmd/btrfs-rec/main.go +++ b/cmd/btrfs-rec/main.go @@ -48,7 +48,10 @@ var ( var globalFlags struct { logLevel textui.LogLevelFlag pvs []string + mappings string + nodeList string + rebuild bool stopProfiling profile.StopFunc @@ -86,12 +89,21 @@ func main() { globalFlags.logLevel.Level = dlog.LogLevelInfo argparser.PersistentFlags().Var(&globalFlags.logLevel, "verbosity", "set the verbosity") - argparser.PersistentFlags().StringArrayVar(&globalFlags.pvs, "pv", nil, "open the file `physical_volume` as part of the filesystem") + argparser.PersistentFlags().StringArrayVar(&globalFlags.pvs, "pv", nil, + "open the file `physical_volume` as part of the filesystem") noError(argparser.MarkPersistentFlagFilename("pv")) - argparser.PersistentFlags().StringVar(&globalFlags.mappings, "mappings", "", "load chunk/dev-extent/blockgroup data from external JSON file `mappings.json`") + argparser.PersistentFlags().StringVar(&globalFlags.mappings, "mappings", "", + "load chunk/dev-extent/blockgroup data from external JSON file `mappings.json`") noError(argparser.MarkPersistentFlagFilename("mappings")) + argparser.PersistentFlags().StringVar(&globalFlags.nodeList, "node-list", "", + "load node list (output of 'btrfs-recs inspect [rebuild-mappings] list-nodes') from external JSON file `nodes.json`") + noError(argparser.MarkPersistentFlagFilename("node-list")) + + argparser.PersistentFlags().BoolVar(&globalFlags.rebuild, "rebuild", false, + "attempt to rebuild broken btrees when reading") + globalFlags.stopProfiling = profile.AddProfileFlags(argparser.PersistentFlags(), "profile.") globalFlags.openFlag = os.O_RDONLY @@ -176,8 +188,51 @@ func runWithRawFS(runE func(*btrfs.FS, *cobra.Command, []string) error) func(*co }) } -func runWithReadableFS(runE func(btrfs.ReadableFS, *cobra.Command, []string) error) func(*cobra.Command, []string) error { +func runWithRawFSAndNodeList(runE func(*btrfs.FS, []btrfsvol.LogicalAddr, *cobra.Command, []string) error) func(*cobra.Command, []string) error { return runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { - return runE(btrfsutil.NewOldRebuiltForrest(fs), cmd, args) + ctx := cmd.Context() + + var nodeList []btrfsvol.LogicalAddr + var err error + if globalFlags.nodeList != "" { + nodeList, err = readJSONFile[[]btrfsvol.LogicalAddr](ctx, globalFlags.nodeList) + } else { + nodeList, err = btrfsutil.ListNodes(ctx, fs) + } + if err != nil { + return err + } + + return runE(fs, nodeList, cmd, args) + }) +} + +func _runWithReadableFS(wantNodeList bool, runE func(btrfs.ReadableFS, []btrfsvol.LogicalAddr, *cobra.Command, []string) error) func(*cobra.Command, []string) error { + inner := func(fs *btrfs.FS, nodeList []btrfsvol.LogicalAddr, cmd *cobra.Command, args []string) error { + var rfs btrfs.ReadableFS = fs + if globalFlags.rebuild { + rfs = btrfsutil.NewOldRebuiltForrest(fs) + } + + return runE(rfs, nodeList, cmd, args) + } + + return func(cmd *cobra.Command, args []string) error { + if wantNodeList { + return runWithRawFSAndNodeList(inner)(cmd, args) + } + return runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { + return inner(fs, nil, cmd, args) + })(cmd, args) + } +} + +func runWithReadableFSAndNodeList(runE func(btrfs.ReadableFS, []btrfsvol.LogicalAddr, *cobra.Command, []string) error) func(*cobra.Command, []string) error { + return _runWithReadableFS(true, runE) +} + +func runWithReadableFS(runE func(btrfs.ReadableFS, *cobra.Command, []string) error) func(*cobra.Command, []string) error { + return _runWithReadableFS(false, func(fs btrfs.ReadableFS, _ []btrfsvol.LogicalAddr, cmd *cobra.Command, args []string) error { + return runE(fs, cmd, args) }) } diff --git a/lib/btrfs/btrfsvol/lvm.go b/lib/btrfs/btrfsvol/lvm.go index 3834345..7ed58a0 100644 --- a/lib/btrfs/btrfsvol/lvm.go +++ b/lib/btrfs/btrfsvol/lvm.go @@ -17,6 +17,7 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/containers" "git.lukeshu.com/btrfs-progs-ng/lib/diskio" + "git.lukeshu.com/btrfs-progs-ng/lib/maps" ) type LogicalVolume[PhysicalVolume diskio.File[PhysicalAddr]] struct { @@ -41,7 +42,7 @@ func (lv *LogicalVolume[PhysicalVolume]) init() { lv.physical2logical = make(map[DeviceID]*containers.RBTree[devextMapping], len(lv.id2pv)) } for devid := range lv.id2pv { - if _, ok := lv.physical2logical[devid]; !ok { + if !maps.HasKey(lv.physical2logical, devid) { lv.physical2logical[devid] = new(containers.RBTree[devextMapping]) } } @@ -120,7 +121,7 @@ func (lv *LogicalVolume[PhysicalVolume]) AddMapping(m Mapping) error { func (lv *LogicalVolume[PhysicalVolume]) addMapping(m Mapping, dryRun bool) error { lv.init() // sanity check - if _, haveDev := lv.id2pv[m.PAddr.Dev]; !haveDev { + if !maps.HasKey(lv.id2pv, m.PAddr.Dev) { return fmt.Errorf("(%p).AddMapping: do not have a physical volume with id=%v", lv, m.PAddr.Dev) } @@ -228,12 +229,12 @@ func (lv *LogicalVolume[PhysicalVolume]) fsck() error { lv.logical2physical.Range(func(node *containers.RBNode[chunkMapping]) bool { chunk := node.Value for _, stripe := range chunk.PAddrs { - if _, devOK := lv.id2pv[stripe.Dev]; !devOK { + if !maps.HasKey(lv.id2pv, stripe.Dev) { err = fmt.Errorf("(%p).fsck: chunk references physical volume %v which does not exist", lv, stripe.Dev) return false } - if _, exists := physical2logical[stripe.Dev]; !exists { + if !maps.HasKey(physical2logical, stripe.Dev) { physical2logical[stripe.Dev] = new(containers.RBTree[devextMapping]) } physical2logical[stripe.Dev].Insert(devextMapping{ diff --git a/lib/btrfsutil/graph.go b/lib/btrfsutil/graph.go index fe7fe70..8e26c08 100644 --- a/lib/btrfsutil/graph.go +++ b/lib/btrfsutil/graph.go @@ -10,8 +10,10 @@ import ( "reflect" "time" + "github.com/datawire/dlib/derror" "github.com/datawire/dlib/dlog" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsprim" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfstree" @@ -71,6 +73,47 @@ func (n GraphNode) String() string { n.Level, n.Generation, n.Owner, len(n.Items)) } +func (n GraphNode) CheckExpectations(g Graph, exp btrfstree.NodeExpectations) error { + var errs derror.MultiError + if exp.LAddr.OK && n.Addr != exp.LAddr.Val { + errs = append(errs, fmt.Errorf("read from laddr=%v but claims to be at laddr=%v", + exp.LAddr.Val, n.Addr)) + } + if exp.Level.OK && n.Level != exp.Level.Val { + errs = append(errs, fmt.Errorf("expected level=%v but claims to be level=%v", + exp.Level.Val, n.Level)) + } + if n.Level > btrfstree.MaxLevel { + errs = append(errs, fmt.Errorf("maximum level=%v but claims to be level=%v", + btrfstree.MaxLevel, n.Level)) + } + if exp.Generation.OK && n.Generation != exp.Generation.Val { + errs = append(errs, fmt.Errorf("expected generation=%v but claims to be generation=%v", + exp.Generation.Val, n.Generation)) + } + if exp.Owner != nil { + if err := exp.Owner(n.Owner, n.Generation); err != nil { + errs = append(errs, err) + } + } + if n.NumItems(g) == 0 { + errs = append(errs, fmt.Errorf("has no items")) + } else { + if minItem := n.MinItem(g); exp.MinItem.OK && exp.MinItem.Val.Compare(minItem) > 0 { + errs = append(errs, fmt.Errorf("expected minItem>=%v but node has minItem=%v", + exp.MinItem.Val, minItem)) + } + if maxItem := n.MaxItem(g); exp.MaxItem.OK && exp.MaxItem.Val.Compare(maxItem) < 0 { + errs = append(errs, fmt.Errorf("expected maxItem<=%v but node has maxItem=%v", + exp.MaxItem.Val, maxItem)) + } + } + if len(errs) > 0 { + return errs + } + return nil +} + type GraphEdge struct { // It is invalid for both 'FromRoot' and 'FromNode' to be // non-zero. If both are zero, then the GraphEdge is from the @@ -225,7 +268,7 @@ func (g Graph) FinalCheck(ctx context.Context, fs btrfstree.NodeSource) error { stats.D = len(g.EdgesTo) progressWriter.Set(stats) for laddr := range g.EdgesTo { - if _, ok := g.Nodes[laddr]; !ok { + if !maps.HasKey(g.Nodes, laddr) { node, err := fs.AcquireNode(ctx, laddr, btrfstree.NodeExpectations{ LAddr: containers.OptionalValue(laddr), }) @@ -283,3 +326,57 @@ func (g Graph) FinalCheck(ctx context.Context, fs btrfstree.NodeSource) error { return nil } + +func ReadGraph(_ctx context.Context, fs *btrfs.FS, nodeList []btrfsvol.LogicalAddr) (Graph, error) { + // read-superblock ///////////////////////////////////////////////////////////// + ctx := dlog.WithField(_ctx, "btrfs.util.read-graph.step", "read-superblock") + dlog.Info(ctx, "Reading superblock...") + sb, err := fs.Superblock() + if err != nil { + return Graph{}, err + } + + // read-roots ////////////////////////////////////////////////////////////////// + ctx = dlog.WithField(_ctx, "btrfs.util.read-graph.step", "read-roots") + graph := NewGraph(ctx, *sb) + + // read-nodes ////////////////////////////////////////////////////////////////// + ctx = dlog.WithField(_ctx, "btrfs.util.read-graph.step", "read-nodes") + dlog.Infof(ctx, "Reading node data from FS...") + var stats textui.Portion[int] + stats.D = len(nodeList) + progressWriter := textui.NewProgress[textui.Portion[int]]( + ctx, + dlog.LogLevelInfo, + textui.Tunable(1*time.Second)) + progressWriter.Set(stats) + for _, laddr := range nodeList { + if err := ctx.Err(); err != nil { + return Graph{}, err + } + node, err := fs.AcquireNode(ctx, laddr, btrfstree.NodeExpectations{ + LAddr: containers.OptionalValue(laddr), + }) + if err != nil { + fs.ReleaseNode(node) + return Graph{}, err + } + graph.InsertNode(node) + fs.ReleaseNode(node) + stats.N++ + progressWriter.Set(stats) + } + if stats.N != stats.D { + panic("should not happen") + } + progressWriter.Done() + dlog.Info(ctx, "... done reading node data") + + // check /////////////////////////////////////////////////////////////////////// + ctx = dlog.WithField(_ctx, "btrfs.util.read-graph.step", "check") + if err := graph.FinalCheck(ctx, fs); err != nil { + return Graph{}, err + } + + return graph, nil +} diff --git a/lib/btrfsutil/listnodes.go b/lib/btrfsutil/listnodes.go index 70b647c..d4572bd 100644 --- a/lib/btrfsutil/listnodes.go +++ b/lib/btrfsutil/listnodes.go @@ -15,45 +15,45 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/textui" ) -type nodeScanner struct { +type nodeLister struct { nodes containers.Set[btrfsvol.LogicalAddr] } -type nodeStats struct { +type nodeListStats struct { numNodes int } -func (s nodeStats) String() string { +func (s nodeListStats) String() string { return textui.Sprintf("found: %d nodes", s.numNodes) } -var _ DeviceScanner[nodeStats, containers.Set[btrfsvol.LogicalAddr]] = (*nodeScanner)(nil) +var _ DeviceScanner[nodeListStats, containers.Set[btrfsvol.LogicalAddr]] = (*nodeLister)(nil) -func newNodeScanner(context.Context, btrfstree.Superblock, btrfsvol.PhysicalAddr, int) DeviceScanner[nodeStats, containers.Set[btrfsvol.LogicalAddr]] { - s := new(nodeScanner) +func newNodeLister(context.Context, btrfstree.Superblock, btrfsvol.PhysicalAddr, int) DeviceScanner[nodeListStats, containers.Set[btrfsvol.LogicalAddr]] { + s := new(nodeLister) s.nodes = make(containers.Set[btrfsvol.LogicalAddr]) return s } -func (s *nodeScanner) ScanStats() nodeStats { - return nodeStats{numNodes: len(s.nodes)} +func (s *nodeLister) ScanStats() nodeListStats { + return nodeListStats{numNodes: len(s.nodes)} } -func (*nodeScanner) ScanSector(context.Context, *btrfs.Device, btrfsvol.PhysicalAddr) error { +func (*nodeLister) ScanSector(context.Context, *btrfs.Device, btrfsvol.PhysicalAddr) error { return nil } -func (s *nodeScanner) ScanNode(_ context.Context, _ btrfsvol.PhysicalAddr, node *btrfstree.Node) error { +func (s *nodeLister) ScanNode(_ context.Context, _ btrfsvol.PhysicalAddr, node *btrfstree.Node) error { s.nodes.Insert(node.Head.Addr) return nil } -func (s *nodeScanner) ScanDone(_ context.Context) (containers.Set[btrfsvol.LogicalAddr], error) { +func (s *nodeLister) ScanDone(_ context.Context) (containers.Set[btrfsvol.LogicalAddr], error) { return s.nodes, nil } func ListNodes(ctx context.Context, fs *btrfs.FS) ([]btrfsvol.LogicalAddr, error) { - perDev, err := ScanDevices[nodeStats, containers.Set[btrfsvol.LogicalAddr]](ctx, fs, newNodeScanner) + perDev, err := ScanDevices[nodeListStats, containers.Set[btrfsvol.LogicalAddr]](ctx, fs, newNodeLister) if err != nil { return nil, err } diff --git a/lib/btrfsutil/old_rebuilt_forrest.go b/lib/btrfsutil/old_rebuilt_forrest.go index c451fb8..0a3106d 100644 --- a/lib/btrfsutil/old_rebuilt_forrest.go +++ b/lib/btrfsutil/old_rebuilt_forrest.go @@ -53,11 +53,11 @@ type oldRebuiltTreeValue struct { Key btrfsprim.Key ItemSize uint32 - Node nodeInfo + Node oldRebuiltNodeInfo Slot int } -type nodeInfo struct { +type oldRebuiltNodeInfo struct { LAddr btrfsvol.LogicalAddr Level uint8 Generation btrfsprim.Generation @@ -187,7 +187,7 @@ func (bt *OldRebuiltForrest) rawTreeWalk(ctx context.Context, treeID btrfsprim.O cacheEntry.ParentUUID = root.ParentUUID cacheEntry.ParentGen = root.ParentGen - var curNode nodeInfo + var curNode oldRebuiltNodeInfo cbs := btrfstree.TreeWalkHandler{ BadNode: func(path btrfstree.Path, node *btrfstree.Node, err error) bool { nodeAddr, nodeExp, _ := path.NodeExpectations(ctx, false) @@ -200,7 +200,7 @@ func (bt *OldRebuiltForrest) rawTreeWalk(ctx context.Context, treeID btrfsprim.O return false }, Node: func(path btrfstree.Path, node *btrfstree.Node) { - curNode = nodeInfo{ + curNode = oldRebuiltNodeInfo{ LAddr: node.Head.Addr, Level: node.Head.Level, Generation: node.Head.Generation, @@ -247,7 +247,7 @@ func (tree oldRebuiltTree) addErrs(fn func(btrfsprim.Key, uint32) int, err error return errs } -func (bt *OldRebuiltForrest) readNode(ctx context.Context, nodeInfo nodeInfo) *btrfstree.Node { +func (bt *OldRebuiltForrest) readNode(ctx context.Context, nodeInfo oldRebuiltNodeInfo) *btrfstree.Node { node, err := bt.AcquireNode(ctx, nodeInfo.LAddr, btrfstree.NodeExpectations{ LAddr: containers.OptionalValue(nodeInfo.LAddr), Level: containers.OptionalValue(nodeInfo.Level), diff --git a/lib/btrfsutil/rebuilt_forrest.go b/lib/btrfsutil/rebuilt_forrest.go index fcfb353..ba88bbc 100644 --- a/lib/btrfsutil/rebuilt_forrest.go +++ b/lib/btrfsutil/rebuilt_forrest.go @@ -16,7 +16,6 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" "git.lukeshu.com/btrfs-progs-ng/lib/containers" "git.lukeshu.com/btrfs-progs-ng/lib/slices" - "git.lukeshu.com/btrfs-progs-ng/lib/textui" ) type RebuiltForrestCallbacks interface { @@ -43,10 +42,11 @@ func (cb noopRebuiltForrestCallbacks) LookupRoot(ctx context.Context, tree btrfs ObjectID: tree, ItemType: btrfsprim.ROOT_ITEM_KEY, } - itemKey, itemPtr, ok := rootTree.RebuiltItems(ctx).Search(func(key btrfsprim.Key, _ ItemPtr) int { + itemKey, itemPtr, ok := rootTree.RebuiltAcquireItems(ctx).Search(func(key btrfsprim.Key, _ ItemPtr) int { key.Offset = 0 return tgt.Compare(key) }) + rootTree.RebuiltReleaseItems() if !ok { return 0, btrfsitem.Root{}, false } @@ -70,7 +70,8 @@ func (cb noopRebuiltForrestCallbacks) LookupUUID(ctx context.Context, uuid btrfs return 0, false } tgt := btrfsitem.UUIDToKey(uuid) - itemPtr, ok := uuidTree.RebuiltItems(ctx).Load(tgt) + itemPtr, ok := uuidTree.RebuiltAcquireItems(ctx).Load(tgt) + uuidTree.RebuiltReleaseItems() if !ok { return 0, false } @@ -118,8 +119,9 @@ func (cb noopRebuiltForrestCallbacks) LookupUUID(ctx context.Context, uuid btrfs // - it provides several RebuiltTree methods that provide advice on // what roots should be added to a tree in order to repair it: // -// .RebuiltItems() and RebuiltPotentialItems() to compare what's -// in the tree and what could be in the tree. +// .RebuiltAcquireItems()/.RebuiltReleaseItems() and +// .RebuiltAcquirePotentialItems()/.RebuiltReleasePotentialItems() +// to compare what's in the tree and what could be in the tree. // // .RebuiltLeafToRoots() to map potential items to things that can // be passed to .RebuiltAddRoot(). @@ -139,11 +141,10 @@ type RebuiltForrest struct { // mutable - treesMu nestedMutex - trees map[btrfsprim.ObjID]*RebuiltTree // must hold .treesMu to access - leafs containers.Cache[btrfsprim.ObjID, map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr]] - incItems containers.Cache[btrfsprim.ObjID, itemIndex] - excItems containers.Cache[btrfsprim.ObjID, itemIndex] + treesMu nestedMutex + trees map[btrfsprim.ObjID]*RebuiltTree // must hold .treesMu to access + + rebuiltSharedCache } // NewRebuiltForrest returns a new RebuiltForrest instance. The @@ -158,19 +159,8 @@ func NewRebuiltForrest(file btrfstree.NodeSource, sb btrfstree.Superblock, graph trees: make(map[btrfsprim.ObjID]*RebuiltTree), } - ret.leafs = containers.NewARCache[btrfsprim.ObjID, map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr]](textui.Tunable(8), - containers.SourceFunc[btrfsprim.ObjID, map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr]]( - func(ctx context.Context, treeID btrfsprim.ObjID, leafs *map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr]) { - *leafs = ret.trees[treeID].uncachedLeafToRoots(ctx) - })) - ret.incItems = containers.NewARCache[btrfsprim.ObjID, itemIndex](textui.Tunable(8), - containers.SourceFunc[btrfsprim.ObjID, itemIndex](func(ctx context.Context, treeID btrfsprim.ObjID, incItems *itemIndex) { - *incItems = ret.trees[treeID].uncachedIncItems(ctx) - })) - ret.excItems = containers.NewARCache[btrfsprim.ObjID, itemIndex](textui.Tunable(8), - containers.SourceFunc[btrfsprim.ObjID, itemIndex](func(ctx context.Context, treeID btrfsprim.ObjID, excItems *itemIndex) { - *excItems = ret.trees[treeID].uncachedExcItems(ctx) - })) + ret.rebuiltSharedCache = makeRebuiltSharedCache(ret) + if ret.cb == nil { ret.cb = noopRebuiltForrestCallbacks{ forrest: ret, @@ -249,11 +239,11 @@ func (ts *RebuiltForrest) addTree(ctx context.Context, treeID btrfsprim.ObjID, s } parentID, ok := ts.cb.LookupUUID(ctx, rootItem.ParentUUID) if !ok { - dlog.Error(ctx, "failed to add tree: lookup UUID") + dlog.Errorf(ctx, "failed to add tree: lookup UUID %v", rootItem.ParentUUID) return false } if !ts.addTree(ctx, parentID, stack) { - dlog.Error(ctx, "failed to add tree: add parent tree") + dlog.Errorf(ctx, "failed to add tree: add parent tree %v", parentID) return false } tree.Parent = ts.trees[parentID] diff --git a/lib/btrfsutil/rebuilt_tree.go b/lib/btrfsutil/rebuilt_tree.go index b996bdb..ffb2e5f 100644 --- a/lib/btrfsutil/rebuilt_tree.go +++ b/lib/btrfsutil/rebuilt_tree.go @@ -37,21 +37,52 @@ type RebuiltTree struct { // derived from tree.Roots, which is why it's OK if they get // evicted. // - // 1. tree.leafToRoots() = tree.forrest.leafs.Acquire(tree.ID) - // 2. tree.RebuiltItems() = tree.forrest.incItems.Acquire(tree.ID) - // 3. tree.RebuiltPotentialItems() = tree.forrest.excItems.Acquire(tree.ID) + // 1. tree.acquireLeafToRoots() = tree.forrest.leafs.Acquire(tree.ID) + // 2. tree.RebuiltAcquireItems() = tree.forrest.incItems.Acquire(tree.ID) + // 3. tree.RebuiltAcquirePotentialItems() = tree.forrest.excItems.Acquire(tree.ID) } -// evictable member 1: .leafToRoots() ////////////////////////////////////////////////////////////////////////////////// +type rebuiltSharedCache struct { + leafs containers.Cache[btrfsprim.ObjID, map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr]] + incItems containers.Cache[btrfsprim.ObjID, containers.SortedMap[btrfsprim.Key, ItemPtr]] + excItems containers.Cache[btrfsprim.ObjID, containers.SortedMap[btrfsprim.Key, ItemPtr]] +} -// leafToRoots returns all leafs (lvl=0) in the filesystem that pass -// .isOwnerOK, whether or not they're in the tree. -func (tree *RebuiltTree) leafToRoots(ctx context.Context) map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr] { - ret := *tree.forrest.leafs.Acquire(ctx, tree.ID) - tree.forrest.leafs.Release(tree.ID) +func makeRebuiltSharedCache(forrest *RebuiltForrest) rebuiltSharedCache { + var ret rebuiltSharedCache + ret.leafs = containers.NewARCache[btrfsprim.ObjID, map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr]]( + textui.Tunable(8), + containers.SourceFunc[btrfsprim.ObjID, map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr]]( + func(ctx context.Context, treeID btrfsprim.ObjID, leafs *map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr]) { + *leafs = forrest.trees[treeID].uncachedLeafToRoots(ctx) + })) + ret.incItems = containers.NewARCache[btrfsprim.ObjID, containers.SortedMap[btrfsprim.Key, ItemPtr]]( + textui.Tunable(8), + containers.SourceFunc[btrfsprim.ObjID, containers.SortedMap[btrfsprim.Key, ItemPtr]]( + func(ctx context.Context, treeID btrfsprim.ObjID, incItems *containers.SortedMap[btrfsprim.Key, ItemPtr]) { + *incItems = forrest.trees[treeID].uncachedIncItems(ctx) + })) + ret.excItems = containers.NewARCache[btrfsprim.ObjID, containers.SortedMap[btrfsprim.Key, ItemPtr]]( + textui.Tunable(8), + containers.SourceFunc[btrfsprim.ObjID, containers.SortedMap[btrfsprim.Key, ItemPtr]]( + func(ctx context.Context, treeID btrfsprim.ObjID, excItems *containers.SortedMap[btrfsprim.Key, ItemPtr]) { + *excItems = forrest.trees[treeID].uncachedExcItems(ctx) + })) return ret } +// evictable member 1: .acquireLeafToRoots() /////////////////////////////////////////////////////////////////////////// + +// acquireLeafToRoots returns all leafs (lvl=0) in the filesystem that +// pass .isOwnerOK, whether or not they're in the tree. +func (tree *RebuiltTree) acquireLeafToRoots(ctx context.Context) map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr] { + return *tree.forrest.leafs.Acquire(ctx, tree.ID) +} + +func (tree *RebuiltTree) releaseLeafToRoots() { + tree.forrest.leafs.Release(tree.ID) +} + func (tree *RebuiltTree) uncachedLeafToRoots(ctx context.Context) map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr] { ctx = dlog.WithField(ctx, "btrfs.util.rebuilt-tree.index-nodes", fmt.Sprintf("tree=%v", tree.ID)) @@ -85,7 +116,7 @@ func (tree *RebuiltTree) indexNode(ctx context.Context, node btrfsvol.LogicalAdd if err := ctx.Err(); err != nil { return } - if _, done := index[node]; done { + if maps.HasKey(index, node) { return } if slices.Contains(node, stack) { @@ -133,72 +164,81 @@ func (tree *RebuiltTree) isOwnerOK(owner btrfsprim.ObjID, gen btrfsprim.Generati } } -// evictable members 2 and 3: .RebuiltItems() and .RebuiltPotentialItems() ///////////////////////////////////////////// +// evictable members 2 and 3: .Rebuilt{Acquire,Release}{Potential,}Items() ///////////////////////////////////////////// -// RebuiltItems returns a map of the items contained in this tree. +// RebuiltAcquireItems returns a map of the items contained in this +// tree. // // Do not mutate the returned map; it is a pointer to the // RebuiltTree's internal map! -func (tree *RebuiltTree) RebuiltItems(ctx context.Context) *containers.SortedMap[btrfsprim.Key, ItemPtr] { - ret := *tree.forrest.incItems.Acquire(ctx, tree.ID) +// +// When done with the map, call .RebuiltReleaseItems(). +func (tree *RebuiltTree) RebuiltAcquireItems(ctx context.Context) *containers.SortedMap[btrfsprim.Key, ItemPtr] { + return tree.forrest.incItems.Acquire(ctx, tree.ID) +} + +// RebuiltReleaseItems releases resources after a call to +// .RebuiltAcquireItems(). +func (tree *RebuiltTree) RebuiltReleaseItems() { tree.forrest.incItems.Release(tree.ID) - return ret } -// RebuiltPotentialItems returns a map of items that could be added to -// this tree with .RebuiltAddRoot(). +// RebuiltAcquirePotentialItems returns a map of items that could be +// added to this tree with .RebuiltAddRoot(). // // Do not mutate the returned map; it is a pointer to the // RebuiltTree's internal map! -func (tree *RebuiltTree) RebuiltPotentialItems(ctx context.Context) *containers.SortedMap[btrfsprim.Key, ItemPtr] { - ret := *tree.forrest.excItems.Acquire(ctx, tree.ID) +// +// When done with the map, call .RebuiltReleasePotentialItems(). +func (tree *RebuiltTree) RebuiltAcquirePotentialItems(ctx context.Context) *containers.SortedMap[btrfsprim.Key, ItemPtr] { + return tree.forrest.excItems.Acquire(ctx, tree.ID) +} + +// RebuiltReleasePotentialItems releases resources after a call to +// .RebuiltAcquirePotentialItems(). +func (tree *RebuiltTree) RebuiltReleasePotentialItems() { tree.forrest.excItems.Release(tree.ID) - return ret } -func (tree *RebuiltTree) uncachedIncItems(ctx context.Context) *containers.SortedMap[btrfsprim.Key, ItemPtr] { +func (tree *RebuiltTree) uncachedIncItems(ctx context.Context) containers.SortedMap[btrfsprim.Key, ItemPtr] { ctx = dlog.WithField(ctx, "btrfs.util.rebuilt-tree.index-inc-items", fmt.Sprintf("tree=%v", tree.ID)) - return tree.items(ctx, tree.Roots.HasAny) + return tree.items(ctx, true) } -func (tree *RebuiltTree) uncachedExcItems(ctx context.Context) *containers.SortedMap[btrfsprim.Key, ItemPtr] { +func (tree *RebuiltTree) uncachedExcItems(ctx context.Context) containers.SortedMap[btrfsprim.Key, ItemPtr] { ctx = dlog.WithField(ctx, "btrfs.util.rebuilt-tree.index-exc-items", fmt.Sprintf("tree=%v", tree.ID)) - return tree.items(ctx, - func(roots containers.Set[btrfsvol.LogicalAddr]) bool { - return !tree.Roots.HasAny(roots) - }) + return tree.items(ctx, false) } -type itemIndex = *containers.SortedMap[btrfsprim.Key, ItemPtr] - -type itemStats struct { +type rebuiltItemStats struct { Leafs textui.Portion[int] NumItems int NumDups int } -func (s itemStats) String() string { +func (s rebuiltItemStats) String() string { return textui.Sprintf("%v (%v items, %v dups)", s.Leafs, s.NumItems, s.NumDups) } -func (tree *RebuiltTree) items(ctx context.Context, leafFn func(roots containers.Set[btrfsvol.LogicalAddr]) bool) *containers.SortedMap[btrfsprim.Key, ItemPtr] { +func (tree *RebuiltTree) items(ctx context.Context, inc bool) containers.SortedMap[btrfsprim.Key, ItemPtr] { tree.mu.RLock() defer tree.mu.RUnlock() var leafs []btrfsvol.LogicalAddr - for leaf, roots := range tree.leafToRoots(ctx) { - if leafFn(roots) { + for leaf, roots := range tree.acquireLeafToRoots(ctx) { + if tree.Roots.HasAny(roots) == inc { leafs = append(leafs, leaf) } } + tree.releaseLeafToRoots() slices.Sort(leafs) - var stats itemStats + var stats rebuiltItemStats stats.Leafs.D = len(leafs) - progressWriter := textui.NewProgress[itemStats](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second)) + progressWriter := textui.NewProgress[rebuiltItemStats](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second)) - index := new(containers.SortedMap[btrfsprim.Key, ItemPtr]) + var index containers.SortedMap[btrfsprim.Key, ItemPtr] for i, leaf := range leafs { stats.Leafs.N = i progressWriter.Set(stats) @@ -220,7 +260,7 @@ func (tree *RebuiltTree) items(ctx context.Context, leafFn func(roots containers } } if stats.Leafs.N > 0 { - stats.Leafs.N = len(leafs) + stats.Leafs.N = stats.Leafs.D progressWriter.Set(stats) progressWriter.Done() } @@ -262,13 +302,13 @@ func (tree *RebuiltTree) RebuiltShouldReplace(oldNode, newNode btrfsvol.LogicalA } } -type rootStats struct { +type rebuiltRootStats struct { Leafs textui.Portion[int] AddedLeafs int AddedItems int } -func (s rootStats) String() string { +func (s rebuiltRootStats) String() string { return textui.Sprintf("%v (added %v leafs, added %v items)", s.Leafs, s.AddedLeafs, s.AddedItems) } @@ -282,11 +322,10 @@ func (tree *RebuiltTree) RebuiltAddRoot(ctx context.Context, rootNode btrfsvol.L ctx = dlog.WithField(ctx, "btrfs.util.rebuilt-tree.add-root", fmt.Sprintf("tree=%v rootNode=%v", tree.ID, rootNode)) dlog.Info(ctx, "adding root...") - leafToRoots := tree.leafToRoots(ctx) - - var stats rootStats + var stats rebuiltRootStats + leafToRoots := tree.acquireLeafToRoots(ctx) stats.Leafs.D = len(leafToRoots) - progressWriter := textui.NewProgress[rootStats](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second)) + progressWriter := textui.NewProgress[rebuiltRootStats](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second)) for i, leaf := range maps.SortedKeys(leafToRoots) { stats.Leafs.N = i progressWriter.Set(stats) @@ -305,6 +344,7 @@ func (tree *RebuiltTree) RebuiltAddRoot(ctx context.Context, rootNode btrfsvol.L } } stats.Leafs.N = len(leafToRoots) + tree.releaseLeafToRoots() progressWriter.Set(stats) progressWriter.Done() @@ -335,7 +375,8 @@ func (tree *RebuiltTree) RebuiltCOWDistance(parentID btrfsprim.ObjID) (dist int, // ReadItem reads an item from a tree. func (tree *RebuiltTree) ReadItem(ctx context.Context, key btrfsprim.Key) btrfsitem.Item { - ptr, ok := tree.RebuiltItems(ctx).Load(key) + ptr, ok := tree.RebuiltAcquireItems(ctx).Load(key) + tree.RebuiltReleaseItems() if !ok { panic(fmt.Errorf("should not happen: btrfsutil.RebuiltTree.ReadItem called for not-included key: %v", key)) } @@ -352,13 +393,14 @@ func (tree *RebuiltTree) RebuiltLeafToRoots(ctx context.Context, leaf btrfsvol.L tree.mu.RLock() defer tree.mu.RUnlock() ret := make(containers.Set[btrfsvol.LogicalAddr]) - for root := range tree.leafToRoots(ctx)[leaf] { + for root := range tree.acquireLeafToRoots(ctx)[leaf] { if tree.Roots.Has(root) { panic(fmt.Errorf("should not happen: (tree=%v).RebuiltLeafToRoots(leaf=%v): tree contains root=%v but not leaf", tree.ID, leaf, root)) } ret.Insert(root) } + tree.releaseLeafToRoots() if len(ret) == 0 { return nil } diff --git a/lib/btrfsutil/scan.go b/lib/btrfsutil/scan.go index 0e268e5..bf848ee 100644 --- a/lib/btrfsutil/scan.go +++ b/lib/btrfsutil/scan.go @@ -33,12 +33,12 @@ type DeviceScanner[Stats comparable, Result any] interface { ScanDone(ctx context.Context) (Result, error) } -type scanStats[T comparable] struct { +type devScanStats[T comparable] struct { portion textui.Portion[btrfsvol.PhysicalAddr] stats T } -func (s scanStats[T]) String() string { +func (s devScanStats[T]) String() string { return textui.Sprintf("scanned %v (%v)", s.portion, s.stats) } @@ -91,8 +91,8 @@ func ScanOneDevice[Stats comparable, Result any](ctx context.Context, dev *btrfs scanner := newScanner(ctx, *sb, numBytes, numSectors) - progressWriter := textui.NewProgress[scanStats[Stats]](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second)) - var stats scanStats[Stats] + progressWriter := textui.NewProgress[devScanStats[Stats]](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second)) + var stats devScanStats[Stats] stats.portion.D = numBytes var minNextNode btrfsvol.PhysicalAddr diff --git a/lib/containers/set.go b/lib/containers/set.go index 074d126..af13d50 100644 --- a/lib/containers/set.go +++ b/lib/containers/set.go @@ -138,20 +138,11 @@ func (o Set[T]) TakeOne() T { } func (o Set[T]) Has(v T) bool { - _, has := o[v] - return has + return maps.HasKey(o, v) } -func (small Set[T]) HasAny(big Set[T]) bool { - if len(big) < len(small) { - small, big = big, small - } - for v := range small { - if _, ok := big[v]; ok { - return true - } - } - return false +func (a Set[T]) HasAny(b Set[T]) bool { + return maps.HaveAnyKeysInCommon(a, b) } func (small Set[T]) Intersection(big Set[T]) Set[T] { @@ -160,7 +151,7 @@ func (small Set[T]) Intersection(big Set[T]) Set[T] { } ret := make(Set[T]) for v := range small { - if _, ok := big[v]; ok { + if maps.HasKey(big, v) { ret.Insert(v) } } diff --git a/lib/maps/maputil.go b/lib/maps/maputil.go index d409e70..63e52a0 100644 --- a/lib/maps/maputil.go +++ b/lib/maps/maputil.go @@ -25,3 +25,20 @@ func SortedKeys[K constraints.Ordered, V any](m map[K]V) []K { slices.Sort(ret) return ret } + +func HasKey[K comparable, V any](m map[K]V, k K) bool { + _, has := m[k] + return has +} + +func HaveAnyKeysInCommon[K comparable, V1, V2 any](small map[K]V1, big map[K]V2) bool { + if len(big) < len(small) { + return HaveAnyKeysInCommon(big, small) + } + for v := range small { + if _, ok := big[v]; ok { + return true + } + } + return false +} diff --git a/lib/textui/log.go b/lib/textui/log.go index 9aff364..e5d3f60 100644 --- a/lib/textui/log.go +++ b/lib/textui/log.go @@ -26,6 +26,8 @@ import ( "git.lukeshu.com/go/typedsync" "github.com/datawire/dlib/dlog" "github.com/spf13/pflag" + + "git.lukeshu.com/btrfs-progs-ng/lib/maps" ) type LogLevelFlag struct { @@ -201,7 +203,7 @@ func (l *logger) log(lvl dlog.LogLevel, writeMsg func(io.Writer)) { fields := make(map[string]any) var fieldKeys []string for f := l; f.parent != nil; f = f.parent { - if _, exists := fields[f.fieldKey]; exists { + if maps.HasKey(fields, f.fieldKey) { continue } fields[f.fieldKey] = f.fieldVal @@ -328,6 +330,10 @@ func fieldOrd(key string) int { case "btrfs.inspect.rebuild-trees.rebuild.want.reason": return -8 + // btrfsutil.Graph ///////////////////////////////////////////////////// + case "btrfs.util.read-graph.step": + return -1 + // btrfsutil.RebuiltForrest //////////////////////////////////////////// case "btrfs.util.rebuilt-forrest.add-tree": return -7 @@ -426,10 +432,14 @@ func writeField(w io.Writer, key string, val any) { name = strings.TrimPrefix(name, "rebuild.") } } - case strings.HasPrefix(name, "util.rebuilt-forrest."): - name = strings.TrimPrefix(name, "util.rebuilt-forrest.") - case strings.HasPrefix(name, "util.rebuilt-tree."): - name = strings.TrimPrefix(name, "util.rebuilt-tree.") + case strings.HasPrefix(name, "util."): + name = strings.TrimPrefix(name, "util.") + switch { + case strings.HasPrefix(name, "rebuilt-forrest."): + name = strings.TrimPrefix(name, "rebuilt-forrest.") + case strings.HasPrefix(name, "rebuilt-tree."): + name = strings.TrimPrefix(name, "rebuilt-tree.") + } } } diff --git a/scripts/main.sh b/scripts/main.sh index dddb9bb..c5fc238 100755 --- a/scripts/main.sh +++ b/scripts/main.sh @@ -31,10 +31,15 @@ export GOMEMLIMIT="$(awk '/^MemTotal:/{ print $2 "KiB" }' </proc/meminfo)" ###################################################################### +# 0: initial scan ########################################## + run-btrfs-rec $gendir/0.scandevices.json \ inspect rebuild-mappings scan run-btrfs-rec $gendir/0.nodes.json \ inspect rebuild-mappings list-nodes $gendir/0.scandevices.json + +# 1-2: rebuild chunk/dev-ext/blockgroup trees ############## + run-btrfs-rec $gendir/1.mappings.json \ inspect rebuild-mappings process $gendir/0.scandevices.json @@ -62,13 +67,20 @@ run-btrfs-rec $gendir/2.mappings.json \ -e '2a{"LAddr":13631488,"PAddr":{"Dev":1,"Addr":13631488},"Size":1},') \ inspect rebuild-mappings process $gendir/0.scandevices.json +# 3: rebuild other trees ################################### + run-btrfs-rec $gendir/3.trees.json \ --mappings=$gendir/2.mappings.json \ - inspect rebuild-trees --node-list=$gendir/0.nodes.json + --node-list=$gendir/0.nodes.json \ + inspect rebuild-trees + +# 4: dump data from the FS ################################# run-btrfs-rec $gendir/4.ls-files.txt \ --mappings=$gendir/2.mappings.json \ + --rebuild \ inspect ls-files run-btrfs-rec $gendir/4.ls-trees.txt \ --mappings=$gendir/2.mappings.json \ - inspect ls-trees --node-list=$gendir/0.nodes.json + --node-list=$gendir/0.nodes.json \ + inspect ls-trees diff --git a/scripts/mount.sh b/scripts/mount.sh index bf84163..4894e94 100755 --- a/scripts/mount.sh +++ b/scripts/mount.sh @@ -15,6 +15,7 @@ mkdir -p "$mountpoint" sudo ./bin/btrfs-rec \ --pv="$image" --mappings="$gendir/2.mappings.json" \ + --rebuild \ inspect mount \ --skip-filesums \ "$mountpoint" |