summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLuke Shumaker <lukeshu@lukeshu.com>2023-04-05 07:49:02 -0600
committerLuke Shumaker <lukeshu@lukeshu.com>2023-04-05 07:49:02 -0600
commit68eb7a16b9759646619a7d9dec2b62fa9d0c30cf (patch)
tree8bb4b70337a299f1dacb3c2858210d4bf6bd4b04
parentb0f290078d531d2dcb5d34e809b0711ce9b6491e (diff)
parentd7dd6dfd7aeb40f06ff4fbe7f906d8feed64b95f (diff)
Merge branch 'lukeshu/misc'
-rw-r--r--cmd/btrfs-rec/inspect/rebuildmappings/process.go2
-rw-r--r--cmd/btrfs-rec/inspect/rebuildtrees/rebuild.go28
-rw-r--r--cmd/btrfs-rec/inspect/rebuildtrees/rebuild_wantcb.go61
-rw-r--r--cmd/btrfs-rec/inspect/rebuildtrees/scan.go30
-rw-r--r--cmd/btrfs-rec/inspect_lstrees.go23
-rw-r--r--cmd/btrfs-rec/inspect_rebuildtrees.go24
-rw-r--r--cmd/btrfs-rec/main.go63
-rw-r--r--lib/btrfs/btrfsvol/lvm.go9
-rw-r--r--lib/btrfsutil/graph.go99
-rw-r--r--lib/btrfsutil/listnodes.go24
-rw-r--r--lib/btrfsutil/old_rebuilt_forrest.go10
-rw-r--r--lib/btrfsutil/rebuilt_forrest.go40
-rw-r--r--lib/btrfsutil/rebuilt_tree.go134
-rw-r--r--lib/btrfsutil/scan.go8
-rw-r--r--lib/containers/set.go17
-rw-r--r--lib/maps/maputil.go17
-rw-r--r--lib/textui/log.go20
-rwxr-xr-xscripts/main.sh16
-rwxr-xr-xscripts/mount.sh1
19 files changed, 409 insertions, 217 deletions
diff --git a/cmd/btrfs-rec/inspect/rebuildmappings/process.go b/cmd/btrfs-rec/inspect/rebuildmappings/process.go
index 7a49cc6..2e694b5 100644
--- a/cmd/btrfs-rec/inspect/rebuildmappings/process.go
+++ b/cmd/btrfs-rec/inspect/rebuildmappings/process.go
@@ -38,7 +38,7 @@ func RebuildMappings(ctx context.Context, fs *btrfs.FS, scanResults ScanDevicesR
devIDs := maps.SortedKeys(scanResults)
devices := fs.LV.PhysicalVolumes()
for _, devID := range devIDs {
- if _, ok := devices[devID]; !ok {
+ if !maps.HasKey(devices, devID) {
return fmt.Errorf("device ID %v mentioned in scan results is not part of the filesystem", devID)
}
devResults := scanResults[devID]
diff --git a/cmd/btrfs-rec/inspect/rebuildtrees/rebuild.go b/cmd/btrfs-rec/inspect/rebuildtrees/rebuild.go
index 0d25ac3..427070a 100644
--- a/cmd/btrfs-rec/inspect/rebuildtrees/rebuild.go
+++ b/cmd/btrfs-rec/inspect/rebuildtrees/rebuild.go
@@ -198,11 +198,13 @@ func (o *rebuilder) processAddedItemQueue(ctx context.Context) error {
ctx := dlog.WithField(ctx, "btrfs.inspect.rebuild-trees.rebuild.settle.item", key)
tree := o.rebuilt.RebuiltTree(ctx, key.TreeID)
- incPtr, ok := tree.RebuiltItems(ctx).Load(key.Key)
+ incPtr, ok := tree.RebuiltAcquireItems(ctx).Load(key.Key)
+ tree.RebuiltReleaseItems()
if !ok {
panic(fmt.Errorf("should not happen: failed to load already-added item: %v", key))
}
- excPtr, ok := tree.RebuiltPotentialItems(ctx).Load(key.Key)
+ excPtr, ok := tree.RebuiltAcquirePotentialItems(ctx).Load(key.Key)
+ tree.RebuiltReleasePotentialItems()
if ok && tree.RebuiltShouldReplace(incPtr.Node, excPtr.Node) {
wantKey := wantWithTree{
TreeID: key.TreeID,
@@ -517,24 +519,12 @@ func (o *rebuilder) resolveTreeAugments(ctx context.Context, treeID btrfsprim.Ob
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
func (queue *treeAugmentQueue) has(wantKey want) bool {
- if queue != nil {
- if queue.zero != nil {
- if _, ok := queue.zero[wantKey]; ok {
- return true
- }
- }
- if queue.single != nil {
- if _, ok := queue.single[wantKey]; ok {
- return true
- }
- }
- if queue.multi != nil {
- if _, ok := queue.multi[wantKey]; ok {
- return true
- }
- }
+ if queue == nil {
+ return false
}
- return false
+ return (queue.zero != nil && maps.HasKey(queue.zero, wantKey)) ||
+ (queue.single != nil && maps.HasKey(queue.single, wantKey)) ||
+ (queue.multi != nil && maps.HasKey(queue.multi, wantKey))
}
func (queue *treeAugmentQueue) store(wantKey want, choices containers.Set[btrfsvol.LogicalAddr]) {
diff --git a/cmd/btrfs-rec/inspect/rebuildtrees/rebuild_wantcb.go b/cmd/btrfs-rec/inspect/rebuildtrees/rebuild_wantcb.go
index eff2a83..0526e93 100644
--- a/cmd/btrfs-rec/inspect/rebuildtrees/rebuild_wantcb.go
+++ b/cmd/btrfs-rec/inspect/rebuildtrees/rebuild_wantcb.go
@@ -43,18 +43,22 @@ func (o graphCallbacks) Want(ctx context.Context, reason string, treeID btrfspri
}
func (o *rebuilder) _want(ctx context.Context, wantKey wantWithTree) (key btrfsprim.Key, ok bool) {
- if o.rebuilt.RebuiltTree(ctx, wantKey.TreeID) == nil {
+ tree := o.rebuilt.RebuiltTree(ctx, wantKey.TreeID)
+ if tree == nil {
o.enqueueRetry(wantKey.TreeID)
return btrfsprim.Key{}, false
}
+ tgt := wantKey.Key.Key()
+
// check if we already have it
- tgt := wantKey.Key.Key()
- if key, _, ok := o.rebuilt.RebuiltTree(ctx, wantKey.TreeID).RebuiltItems(ctx).Search(func(key btrfsprim.Key, _ btrfsutil.ItemPtr) int {
+ key, _, ok = tree.RebuiltAcquireItems(ctx).Search(func(key btrfsprim.Key, _ btrfsutil.ItemPtr) int {
key.Offset = 0
return tgt.Compare(key)
- }); ok {
+ })
+ tree.RebuiltReleaseItems()
+ if ok {
return key, true
}
@@ -64,15 +68,16 @@ func (o *rebuilder) _want(ctx context.Context, wantKey wantWithTree) (key btrfsp
return btrfsprim.Key{}, false
}
wants := make(containers.Set[btrfsvol.LogicalAddr])
- o.rebuilt.RebuiltTree(ctx, wantKey.TreeID).RebuiltPotentialItems(ctx).Subrange(
+ tree.RebuiltAcquirePotentialItems(ctx).Subrange(
func(k btrfsprim.Key, _ btrfsutil.ItemPtr) int {
k.Offset = 0
return tgt.Compare(k)
},
func(_ btrfsprim.Key, v btrfsutil.ItemPtr) bool {
- wants.InsertFrom(o.rebuilt.RebuiltTree(ctx, wantKey.TreeID).RebuiltLeafToRoots(ctx, v.Node))
+ wants.InsertFrom(tree.RebuiltLeafToRoots(ctx, v.Node))
return true
})
+ tree.RebuiltReleasePotentialItems()
o.wantAugment(ctx, wantKey, wants)
return btrfsprim.Key{}, false
}
@@ -93,15 +98,19 @@ func (o graphCallbacks) WantOff(ctx context.Context, reason string, treeID btrfs
}
func (o *rebuilder) _wantOff(ctx context.Context, wantKey wantWithTree) (ok bool) {
- if o.rebuilt.RebuiltTree(ctx, wantKey.TreeID) == nil {
+ tree := o.rebuilt.RebuiltTree(ctx, wantKey.TreeID)
+ if tree == nil {
o.enqueueRetry(wantKey.TreeID)
return false
}
+ tgt := wantKey.Key.Key()
+
// check if we already have it
- tgt := wantKey.Key.Key()
- if _, ok := o.rebuilt.RebuiltTree(ctx, wantKey.TreeID).RebuiltItems(ctx).Load(tgt); ok {
+ _, ok = tree.RebuiltAcquireItems(ctx).Load(tgt)
+ tree.RebuiltReleaseItems()
+ if ok {
return true
}
@@ -111,12 +120,13 @@ func (o *rebuilder) _wantOff(ctx context.Context, wantKey wantWithTree) (ok bool
return false
}
wants := make(containers.Set[btrfsvol.LogicalAddr])
- o.rebuilt.RebuiltTree(ctx, wantKey.TreeID).RebuiltPotentialItems(ctx).Subrange(
+ tree.RebuiltAcquirePotentialItems(ctx).Subrange(
func(k btrfsprim.Key, _ btrfsutil.ItemPtr) int { return tgt.Compare(k) },
func(_ btrfsprim.Key, v btrfsutil.ItemPtr) bool {
- wants.InsertFrom(o.rebuilt.RebuiltTree(ctx, wantKey.TreeID).RebuiltLeafToRoots(ctx, v.Node))
+ wants.InsertFrom(tree.RebuiltLeafToRoots(ctx, v.Node))
return true
})
+ tree.RebuiltReleasePotentialItems()
o.wantAugment(ctx, wantKey, wants)
return false
}
@@ -134,16 +144,18 @@ func (o graphCallbacks) WantDirIndex(ctx context.Context, reason string, treeID
}
ctx = withWant(ctx, logFieldItemWant, reason, wantKey)
- if o.rebuilt.RebuiltTree(ctx, treeID) == nil {
+ tree := o.rebuilt.RebuiltTree(ctx, treeID)
+ if tree == nil {
o.enqueueRetry(treeID)
return
}
+ tgt := wantKey.Key.Key()
+
// check if we already have it
- tgt := wantKey.Key.Key()
found := false
- o.rebuilt.RebuiltTree(ctx, treeID).RebuiltItems(ctx).Subrange(
+ tree.RebuiltAcquireItems(ctx).Subrange(
func(key btrfsprim.Key, _ btrfsutil.ItemPtr) int {
key.Offset = 0
return tgt.Compare(key)
@@ -154,6 +166,7 @@ func (o graphCallbacks) WantDirIndex(ctx context.Context, reason string, treeID
}
return !found
})
+ tree.RebuiltReleaseItems()
if found {
return
}
@@ -164,17 +177,18 @@ func (o graphCallbacks) WantDirIndex(ctx context.Context, reason string, treeID
return
}
wants := make(containers.Set[btrfsvol.LogicalAddr])
- o.rebuilt.RebuiltTree(ctx, treeID).RebuiltPotentialItems(ctx).Subrange(
+ tree.RebuiltAcquirePotentialItems(ctx).Subrange(
func(key btrfsprim.Key, _ btrfsutil.ItemPtr) int {
key.Offset = 0
return tgt.Compare(key)
},
func(_ btrfsprim.Key, ptr btrfsutil.ItemPtr) bool {
if itemName, ok := o.scan.Names[ptr]; ok && bytes.Equal(itemName, name) {
- wants.InsertFrom(o.rebuilt.RebuiltTree(ctx, treeID).RebuiltLeafToRoots(ctx, ptr.Node))
+ wants.InsertFrom(tree.RebuiltLeafToRoots(ctx, ptr.Node))
}
return true
})
+ tree.RebuiltReleasePotentialItems()
o.wantAugment(ctx, wantKey, wants)
}
@@ -259,7 +273,8 @@ func (o graphCallbacks) _wantRange(
ctx = withWant(ctx, logFieldItemWant, reason, wantKey)
wantKey.Key.OffsetType = offsetRange
- if o.rebuilt.RebuiltTree(ctx, treeID) == nil {
+ tree := o.rebuilt.RebuiltTree(ctx, treeID)
+ if tree == nil {
o.enqueueRetry(treeID)
return
}
@@ -275,7 +290,7 @@ func (o graphCallbacks) _wantRange(
})
o._walkRange(
ctx,
- o.rebuilt.RebuiltTree(ctx, treeID).RebuiltItems(ctx),
+ tree.RebuiltAcquireItems(ctx),
treeID, objID, typ, beg, end,
func(runKey btrfsprim.Key, _ btrfsutil.ItemPtr, runBeg, runEnd uint64) {
var overlappingGaps []*containers.RBNode[gap]
@@ -315,12 +330,13 @@ func (o graphCallbacks) _wantRange(
})
}
})
+ tree.RebuiltReleaseItems()
// Step 2: Fill each gap.
if gaps.Len() == 0 {
return
}
- potentialItems := o.rebuilt.RebuiltTree(ctx, treeID).RebuiltPotentialItems(ctx)
+ potentialItems := tree.RebuiltAcquirePotentialItems(ctx)
gaps.Range(func(rbNode *containers.RBNode[gap]) bool {
gap := rbNode.Value
last := gap.Beg
@@ -340,7 +356,7 @@ func (o graphCallbacks) _wantRange(
wantKey.Key.OffsetLow = gap.Beg
wantKey.Key.OffsetHigh = gap.End
wantCtx := withWant(ctx, logFieldItemWant, reason, wantKey)
- o.wantAugment(wantCtx, wantKey, o.rebuilt.RebuiltTree(wantCtx, treeID).RebuiltLeafToRoots(wantCtx, v.Node))
+ o.wantAugment(wantCtx, wantKey, tree.RebuiltLeafToRoots(wantCtx, v.Node))
last = runEnd
})
if last < gap.End {
@@ -352,6 +368,7 @@ func (o graphCallbacks) _wantRange(
}
return true
})
+ tree.RebuiltReleasePotentialItems()
}
// WantCSum implements btrfscheck.GraphCallbacks.
@@ -372,7 +389,9 @@ func (o graphCallbacks) WantCSum(ctx context.Context, reason string, inodeTree,
o.enqueueRetry(inodeTree)
return
}
- inodePtr, ok := o.rebuilt.RebuiltTree(inodeCtx, inodeTree).RebuiltItems(inodeCtx).Load(inodeWant.Key.Key())
+ tree := o.rebuilt.RebuiltTree(inodeCtx, inodeTree)
+ inodePtr, ok := tree.RebuiltAcquireItems(inodeCtx).Load(inodeWant.Key.Key())
+ tree.RebuiltReleaseItems()
if !ok {
panic(fmt.Errorf("should not happen: could not load key: %v", inodeWant))
}
diff --git a/cmd/btrfs-rec/inspect/rebuildtrees/scan.go b/cmd/btrfs-rec/inspect/rebuildtrees/scan.go
index a6f9c7a..f266dab 100644
--- a/cmd/btrfs-rec/inspect/rebuildtrees/scan.go
+++ b/cmd/btrfs-rec/inspect/rebuildtrees/scan.go
@@ -41,31 +41,35 @@ type ScanDevicesResult struct {
Sizes map[btrfsutil.ItemPtr]SizeAndErr // EXTENT_CSUM and EXTENT_DATA
}
-func ScanDevices(ctx context.Context, fs *btrfs.FS, nodeList []btrfsvol.LogicalAddr) (ScanDevicesResult, error) {
+func ScanDevices(_ctx context.Context, fs *btrfs.FS, nodeList []btrfsvol.LogicalAddr) (ScanDevicesResult, error) {
+ // read-superblock /////////////////////////////////////////////////////////////
+ ctx := dlog.WithField(_ctx, "btrfs.inspect.rebuild-trees.read.substep", "read-superblock")
dlog.Info(ctx, "Reading superblock...")
sb, err := fs.Superblock()
if err != nil {
return ScanDevicesResult{}, err
}
- dlog.Infof(ctx, "Reading node data from FS...")
-
- var stats textui.Portion[int]
- stats.D = len(nodeList)
- progressWriter := textui.NewProgress[textui.Portion[int]](
- dlog.WithField(ctx, "btrfs.inspect.rebuild-trees.read.substep", "read-nodes"),
- dlog.LogLevelInfo, textui.Tunable(1*time.Second))
-
+ // read-roots //////////////////////////////////////////////////////////////////
+ ctx = dlog.WithField(_ctx, "btrfs.inspect.rebuild-trees.read.substep", "read-roots")
ret := ScanDevicesResult{
Superblock: *sb,
Graph: btrfsutil.NewGraph(ctx, *sb),
-
Flags: make(map[btrfsutil.ItemPtr]FlagsAndErr),
Names: make(map[btrfsutil.ItemPtr][]byte),
Sizes: make(map[btrfsutil.ItemPtr]SizeAndErr),
}
+ // read-nodes //////////////////////////////////////////////////////////////////
+ ctx = dlog.WithField(_ctx, "btrfs.inspect.rebuild-trees.read.substep", "read-nodes")
+ dlog.Infof(ctx, "Reading node data from FS...")
+ var stats textui.Portion[int]
+ stats.D = len(nodeList)
+ progressWriter := textui.NewProgress[textui.Portion[int]](
+ ctx,
+ dlog.LogLevelInfo,
+ textui.Tunable(1*time.Second))
progressWriter.Set(stats)
for _, laddr := range nodeList {
if err := ctx.Err(); err != nil {
@@ -78,11 +82,8 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, nodeList []btrfsvol.LogicalA
fs.ReleaseNode(node)
return ScanDevicesResult{}, err
}
-
ret.insertNode(node)
-
fs.ReleaseNode(node)
-
stats.N++
progressWriter.Set(stats)
}
@@ -92,7 +93,8 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, nodeList []btrfsvol.LogicalA
progressWriter.Done()
dlog.Info(ctx, "... done reading node data")
- ctx = dlog.WithField(ctx, "btrfs.inspect.rebuild-trees.read.substep", "check")
+ // check ///////////////////////////////////////////////////////////////////////
+ ctx = dlog.WithField(_ctx, "btrfs.inspect.rebuild-trees.read.substep", "check")
if err := ret.Graph.FinalCheck(ctx, fs); err != nil {
return ScanDevicesResult{}, err
}
diff --git a/cmd/btrfs-rec/inspect_lstrees.go b/cmd/btrfs-rec/inspect_lstrees.go
index 24622b4..decb75c 100644
--- a/cmd/btrfs-rec/inspect_lstrees.go
+++ b/cmd/btrfs-rec/inspect_lstrees.go
@@ -25,28 +25,16 @@ import (
)
func init() {
- var nodeListFilename string
- cmd := &cobra.Command{
+ inspectors.AddCommand(&cobra.Command{
Use: "ls-trees",
Short: "A brief view what types of items are in each tree",
Long: "" +
"If no --node-list is given, then a slow sector-by-sector scan " +
"will be used to find all lost+found nodes.",
Args: cliutil.WrapPositionalArgs(cobra.NoArgs),
- RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error {
+ RunE: runWithReadableFSAndNodeList(func(fs btrfs.ReadableFS, nodeList []btrfsvol.LogicalAddr, cmd *cobra.Command, _ []string) error {
ctx := cmd.Context()
- var nodeList []btrfsvol.LogicalAddr
- var err error
- if nodeListFilename != "" {
- nodeList, err = readJSONFile[[]btrfsvol.LogicalAddr](ctx, nodeListFilename)
- } else {
- nodeList, err = btrfsutil.ListNodes(ctx, fs)
- }
- if err != nil {
- return err
- }
-
var treeErrCnt int
var treeItemCnt map[btrfsitem.Type]int
flush := func() {
@@ -124,10 +112,5 @@ func init() {
return nil
}),
- }
- cmd.Flags().StringVar(&nodeListFilename, "node-list", "",
- "Output of 'btrfs-recs inspect [rebuild-mappings] list-nodes' to use for a lost+found tree")
- noError(cmd.MarkFlagFilename("node-list"))
-
- inspectors.AddCommand(cmd)
+ })
}
diff --git a/cmd/btrfs-rec/inspect_rebuildtrees.go b/cmd/btrfs-rec/inspect_rebuildtrees.go
index 676533a..1e808a0 100644
--- a/cmd/btrfs-rec/inspect_rebuildtrees.go
+++ b/cmd/btrfs-rec/inspect_rebuildtrees.go
@@ -17,13 +17,11 @@ import (
"git.lukeshu.com/btrfs-progs-ng/cmd/btrfs-rec/inspect/rebuildtrees"
"git.lukeshu.com/btrfs-progs-ng/lib/btrfs"
"git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol"
- "git.lukeshu.com/btrfs-progs-ng/lib/btrfsutil"
"git.lukeshu.com/btrfs-progs-ng/lib/textui"
)
func init() {
- var nodeListFilename string
- cmd := &cobra.Command{
+ inspectors.AddCommand(&cobra.Command{
Use: "rebuild-trees",
Long: "" +
"Rebuild broken btrees based on missing items that are implied " +
@@ -34,20 +32,9 @@ func init() {
"If no --node-list is given, then a slow sector-by-sector scan " +
"will be used to find all nodes.",
Args: cliutil.WrapPositionalArgs(cobra.NoArgs),
- RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error {
+ RunE: runWithRawFSAndNodeList(func(fs *btrfs.FS, nodeList []btrfsvol.LogicalAddr, cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
- var nodeList []btrfsvol.LogicalAddr
- var err error
- if nodeListFilename != "" {
- nodeList, err = readJSONFile[[]btrfsvol.LogicalAddr](ctx, nodeListFilename)
- } else {
- nodeList, err = btrfsutil.ListNodes(ctx, fs)
- }
- if err != nil {
- return err
- }
-
rebuilder, err := rebuildtrees.NewRebuilder(ctx, fs, nodeList)
if err != nil {
return err
@@ -78,10 +65,5 @@ func init() {
return rebuildErr
}),
- }
- cmd.Flags().StringVar(&nodeListFilename, "node-list", "",
- "Output of 'btrfs-recs inspect [rebuild-mappings] list-nodes' to use for the node list")
- noError(cmd.MarkFlagFilename("node-list"))
-
- inspectors.AddCommand(cmd)
+ })
}
diff --git a/cmd/btrfs-rec/main.go b/cmd/btrfs-rec/main.go
index d221c3d..da3aced 100644
--- a/cmd/btrfs-rec/main.go
+++ b/cmd/btrfs-rec/main.go
@@ -48,7 +48,10 @@ var (
var globalFlags struct {
logLevel textui.LogLevelFlag
pvs []string
+
mappings string
+ nodeList string
+ rebuild bool
stopProfiling profile.StopFunc
@@ -86,12 +89,21 @@ func main() {
globalFlags.logLevel.Level = dlog.LogLevelInfo
argparser.PersistentFlags().Var(&globalFlags.logLevel, "verbosity", "set the verbosity")
- argparser.PersistentFlags().StringArrayVar(&globalFlags.pvs, "pv", nil, "open the file `physical_volume` as part of the filesystem")
+ argparser.PersistentFlags().StringArrayVar(&globalFlags.pvs, "pv", nil,
+ "open the file `physical_volume` as part of the filesystem")
noError(argparser.MarkPersistentFlagFilename("pv"))
- argparser.PersistentFlags().StringVar(&globalFlags.mappings, "mappings", "", "load chunk/dev-extent/blockgroup data from external JSON file `mappings.json`")
+ argparser.PersistentFlags().StringVar(&globalFlags.mappings, "mappings", "",
+ "load chunk/dev-extent/blockgroup data from external JSON file `mappings.json`")
noError(argparser.MarkPersistentFlagFilename("mappings"))
+ argparser.PersistentFlags().StringVar(&globalFlags.nodeList, "node-list", "",
+ "load node list (output of 'btrfs-recs inspect [rebuild-mappings] list-nodes') from external JSON file `nodes.json`")
+ noError(argparser.MarkPersistentFlagFilename("node-list"))
+
+ argparser.PersistentFlags().BoolVar(&globalFlags.rebuild, "rebuild", false,
+ "attempt to rebuild broken btrees when reading")
+
globalFlags.stopProfiling = profile.AddProfileFlags(argparser.PersistentFlags(), "profile.")
globalFlags.openFlag = os.O_RDONLY
@@ -176,8 +188,51 @@ func runWithRawFS(runE func(*btrfs.FS, *cobra.Command, []string) error) func(*co
})
}
-func runWithReadableFS(runE func(btrfs.ReadableFS, *cobra.Command, []string) error) func(*cobra.Command, []string) error {
+func runWithRawFSAndNodeList(runE func(*btrfs.FS, []btrfsvol.LogicalAddr, *cobra.Command, []string) error) func(*cobra.Command, []string) error {
return runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error {
- return runE(btrfsutil.NewOldRebuiltForrest(fs), cmd, args)
+ ctx := cmd.Context()
+
+ var nodeList []btrfsvol.LogicalAddr
+ var err error
+ if globalFlags.nodeList != "" {
+ nodeList, err = readJSONFile[[]btrfsvol.LogicalAddr](ctx, globalFlags.nodeList)
+ } else {
+ nodeList, err = btrfsutil.ListNodes(ctx, fs)
+ }
+ if err != nil {
+ return err
+ }
+
+ return runE(fs, nodeList, cmd, args)
+ })
+}
+
+func _runWithReadableFS(wantNodeList bool, runE func(btrfs.ReadableFS, []btrfsvol.LogicalAddr, *cobra.Command, []string) error) func(*cobra.Command, []string) error {
+ inner := func(fs *btrfs.FS, nodeList []btrfsvol.LogicalAddr, cmd *cobra.Command, args []string) error {
+ var rfs btrfs.ReadableFS = fs
+ if globalFlags.rebuild {
+ rfs = btrfsutil.NewOldRebuiltForrest(fs)
+ }
+
+ return runE(rfs, nodeList, cmd, args)
+ }
+
+ return func(cmd *cobra.Command, args []string) error {
+ if wantNodeList {
+ return runWithRawFSAndNodeList(inner)(cmd, args)
+ }
+ return runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error {
+ return inner(fs, nil, cmd, args)
+ })(cmd, args)
+ }
+}
+
+func runWithReadableFSAndNodeList(runE func(btrfs.ReadableFS, []btrfsvol.LogicalAddr, *cobra.Command, []string) error) func(*cobra.Command, []string) error {
+ return _runWithReadableFS(true, runE)
+}
+
+func runWithReadableFS(runE func(btrfs.ReadableFS, *cobra.Command, []string) error) func(*cobra.Command, []string) error {
+ return _runWithReadableFS(false, func(fs btrfs.ReadableFS, _ []btrfsvol.LogicalAddr, cmd *cobra.Command, args []string) error {
+ return runE(fs, cmd, args)
})
}
diff --git a/lib/btrfs/btrfsvol/lvm.go b/lib/btrfs/btrfsvol/lvm.go
index 3834345..7ed58a0 100644
--- a/lib/btrfs/btrfsvol/lvm.go
+++ b/lib/btrfs/btrfsvol/lvm.go
@@ -17,6 +17,7 @@ import (
"git.lukeshu.com/btrfs-progs-ng/lib/containers"
"git.lukeshu.com/btrfs-progs-ng/lib/diskio"
+ "git.lukeshu.com/btrfs-progs-ng/lib/maps"
)
type LogicalVolume[PhysicalVolume diskio.File[PhysicalAddr]] struct {
@@ -41,7 +42,7 @@ func (lv *LogicalVolume[PhysicalVolume]) init() {
lv.physical2logical = make(map[DeviceID]*containers.RBTree[devextMapping], len(lv.id2pv))
}
for devid := range lv.id2pv {
- if _, ok := lv.physical2logical[devid]; !ok {
+ if !maps.HasKey(lv.physical2logical, devid) {
lv.physical2logical[devid] = new(containers.RBTree[devextMapping])
}
}
@@ -120,7 +121,7 @@ func (lv *LogicalVolume[PhysicalVolume]) AddMapping(m Mapping) error {
func (lv *LogicalVolume[PhysicalVolume]) addMapping(m Mapping, dryRun bool) error {
lv.init()
// sanity check
- if _, haveDev := lv.id2pv[m.PAddr.Dev]; !haveDev {
+ if !maps.HasKey(lv.id2pv, m.PAddr.Dev) {
return fmt.Errorf("(%p).AddMapping: do not have a physical volume with id=%v",
lv, m.PAddr.Dev)
}
@@ -228,12 +229,12 @@ func (lv *LogicalVolume[PhysicalVolume]) fsck() error {
lv.logical2physical.Range(func(node *containers.RBNode[chunkMapping]) bool {
chunk := node.Value
for _, stripe := range chunk.PAddrs {
- if _, devOK := lv.id2pv[stripe.Dev]; !devOK {
+ if !maps.HasKey(lv.id2pv, stripe.Dev) {
err = fmt.Errorf("(%p).fsck: chunk references physical volume %v which does not exist",
lv, stripe.Dev)
return false
}
- if _, exists := physical2logical[stripe.Dev]; !exists {
+ if !maps.HasKey(physical2logical, stripe.Dev) {
physical2logical[stripe.Dev] = new(containers.RBTree[devextMapping])
}
physical2logical[stripe.Dev].Insert(devextMapping{
diff --git a/lib/btrfsutil/graph.go b/lib/btrfsutil/graph.go
index fe7fe70..8e26c08 100644
--- a/lib/btrfsutil/graph.go
+++ b/lib/btrfsutil/graph.go
@@ -10,8 +10,10 @@ import (
"reflect"
"time"
+ "github.com/datawire/dlib/derror"
"github.com/datawire/dlib/dlog"
+ "git.lukeshu.com/btrfs-progs-ng/lib/btrfs"
"git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem"
"git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsprim"
"git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfstree"
@@ -71,6 +73,47 @@ func (n GraphNode) String() string {
n.Level, n.Generation, n.Owner, len(n.Items))
}
+func (n GraphNode) CheckExpectations(g Graph, exp btrfstree.NodeExpectations) error {
+ var errs derror.MultiError
+ if exp.LAddr.OK && n.Addr != exp.LAddr.Val {
+ errs = append(errs, fmt.Errorf("read from laddr=%v but claims to be at laddr=%v",
+ exp.LAddr.Val, n.Addr))
+ }
+ if exp.Level.OK && n.Level != exp.Level.Val {
+ errs = append(errs, fmt.Errorf("expected level=%v but claims to be level=%v",
+ exp.Level.Val, n.Level))
+ }
+ if n.Level > btrfstree.MaxLevel {
+ errs = append(errs, fmt.Errorf("maximum level=%v but claims to be level=%v",
+ btrfstree.MaxLevel, n.Level))
+ }
+ if exp.Generation.OK && n.Generation != exp.Generation.Val {
+ errs = append(errs, fmt.Errorf("expected generation=%v but claims to be generation=%v",
+ exp.Generation.Val, n.Generation))
+ }
+ if exp.Owner != nil {
+ if err := exp.Owner(n.Owner, n.Generation); err != nil {
+ errs = append(errs, err)
+ }
+ }
+ if n.NumItems(g) == 0 {
+ errs = append(errs, fmt.Errorf("has no items"))
+ } else {
+ if minItem := n.MinItem(g); exp.MinItem.OK && exp.MinItem.Val.Compare(minItem) > 0 {
+ errs = append(errs, fmt.Errorf("expected minItem>=%v but node has minItem=%v",
+ exp.MinItem.Val, minItem))
+ }
+ if maxItem := n.MaxItem(g); exp.MaxItem.OK && exp.MaxItem.Val.Compare(maxItem) < 0 {
+ errs = append(errs, fmt.Errorf("expected maxItem<=%v but node has maxItem=%v",
+ exp.MaxItem.Val, maxItem))
+ }
+ }
+ if len(errs) > 0 {
+ return errs
+ }
+ return nil
+}
+
type GraphEdge struct {
// It is invalid for both 'FromRoot' and 'FromNode' to be
// non-zero. If both are zero, then the GraphEdge is from the
@@ -225,7 +268,7 @@ func (g Graph) FinalCheck(ctx context.Context, fs btrfstree.NodeSource) error {
stats.D = len(g.EdgesTo)
progressWriter.Set(stats)
for laddr := range g.EdgesTo {
- if _, ok := g.Nodes[laddr]; !ok {
+ if !maps.HasKey(g.Nodes, laddr) {
node, err := fs.AcquireNode(ctx, laddr, btrfstree.NodeExpectations{
LAddr: containers.OptionalValue(laddr),
})
@@ -283,3 +326,57 @@ func (g Graph) FinalCheck(ctx context.Context, fs btrfstree.NodeSource) error {
return nil
}
+
+func ReadGraph(_ctx context.Context, fs *btrfs.FS, nodeList []btrfsvol.LogicalAddr) (Graph, error) {
+ // read-superblock /////////////////////////////////////////////////////////////
+ ctx := dlog.WithField(_ctx, "btrfs.util.read-graph.step", "read-superblock")
+ dlog.Info(ctx, "Reading superblock...")
+ sb, err := fs.Superblock()
+ if err != nil {
+ return Graph{}, err
+ }
+
+ // read-roots //////////////////////////////////////////////////////////////////
+ ctx = dlog.WithField(_ctx, "btrfs.util.read-graph.step", "read-roots")
+ graph := NewGraph(ctx, *sb)
+
+ // read-nodes //////////////////////////////////////////////////////////////////
+ ctx = dlog.WithField(_ctx, "btrfs.util.read-graph.step", "read-nodes")
+ dlog.Infof(ctx, "Reading node data from FS...")
+ var stats textui.Portion[int]
+ stats.D = len(nodeList)
+ progressWriter := textui.NewProgress[textui.Portion[int]](
+ ctx,
+ dlog.LogLevelInfo,
+ textui.Tunable(1*time.Second))
+ progressWriter.Set(stats)
+ for _, laddr := range nodeList {
+ if err := ctx.Err(); err != nil {
+ return Graph{}, err
+ }
+ node, err := fs.AcquireNode(ctx, laddr, btrfstree.NodeExpectations{
+ LAddr: containers.OptionalValue(laddr),
+ })
+ if err != nil {
+ fs.ReleaseNode(node)
+ return Graph{}, err
+ }
+ graph.InsertNode(node)
+ fs.ReleaseNode(node)
+ stats.N++
+ progressWriter.Set(stats)
+ }
+ if stats.N != stats.D {
+ panic("should not happen")
+ }
+ progressWriter.Done()
+ dlog.Info(ctx, "... done reading node data")
+
+ // check ///////////////////////////////////////////////////////////////////////
+ ctx = dlog.WithField(_ctx, "btrfs.util.read-graph.step", "check")
+ if err := graph.FinalCheck(ctx, fs); err != nil {
+ return Graph{}, err
+ }
+
+ return graph, nil
+}
diff --git a/lib/btrfsutil/listnodes.go b/lib/btrfsutil/listnodes.go
index 70b647c..d4572bd 100644
--- a/lib/btrfsutil/listnodes.go
+++ b/lib/btrfsutil/listnodes.go
@@ -15,45 +15,45 @@ import (
"git.lukeshu.com/btrfs-progs-ng/lib/textui"
)
-type nodeScanner struct {
+type nodeLister struct {
nodes containers.Set[btrfsvol.LogicalAddr]
}
-type nodeStats struct {
+type nodeListStats struct {
numNodes int
}
-func (s nodeStats) String() string {
+func (s nodeListStats) String() string {
return textui.Sprintf("found: %d nodes", s.numNodes)
}
-var _ DeviceScanner[nodeStats, containers.Set[btrfsvol.LogicalAddr]] = (*nodeScanner)(nil)
+var _ DeviceScanner[nodeListStats, containers.Set[btrfsvol.LogicalAddr]] = (*nodeLister)(nil)
-func newNodeScanner(context.Context, btrfstree.Superblock, btrfsvol.PhysicalAddr, int) DeviceScanner[nodeStats, containers.Set[btrfsvol.LogicalAddr]] {
- s := new(nodeScanner)
+func newNodeLister(context.Context, btrfstree.Superblock, btrfsvol.PhysicalAddr, int) DeviceScanner[nodeListStats, containers.Set[btrfsvol.LogicalAddr]] {
+ s := new(nodeLister)
s.nodes = make(containers.Set[btrfsvol.LogicalAddr])
return s
}
-func (s *nodeScanner) ScanStats() nodeStats {
- return nodeStats{numNodes: len(s.nodes)}
+func (s *nodeLister) ScanStats() nodeListStats {
+ return nodeListStats{numNodes: len(s.nodes)}
}
-func (*nodeScanner) ScanSector(context.Context, *btrfs.Device, btrfsvol.PhysicalAddr) error {
+func (*nodeLister) ScanSector(context.Context, *btrfs.Device, btrfsvol.PhysicalAddr) error {
return nil
}
-func (s *nodeScanner) ScanNode(_ context.Context, _ btrfsvol.PhysicalAddr, node *btrfstree.Node) error {
+func (s *nodeLister) ScanNode(_ context.Context, _ btrfsvol.PhysicalAddr, node *btrfstree.Node) error {
s.nodes.Insert(node.Head.Addr)
return nil
}
-func (s *nodeScanner) ScanDone(_ context.Context) (containers.Set[btrfsvol.LogicalAddr], error) {
+func (s *nodeLister) ScanDone(_ context.Context) (containers.Set[btrfsvol.LogicalAddr], error) {
return s.nodes, nil
}
func ListNodes(ctx context.Context, fs *btrfs.FS) ([]btrfsvol.LogicalAddr, error) {
- perDev, err := ScanDevices[nodeStats, containers.Set[btrfsvol.LogicalAddr]](ctx, fs, newNodeScanner)
+ perDev, err := ScanDevices[nodeListStats, containers.Set[btrfsvol.LogicalAddr]](ctx, fs, newNodeLister)
if err != nil {
return nil, err
}
diff --git a/lib/btrfsutil/old_rebuilt_forrest.go b/lib/btrfsutil/old_rebuilt_forrest.go
index c451fb8..0a3106d 100644
--- a/lib/btrfsutil/old_rebuilt_forrest.go
+++ b/lib/btrfsutil/old_rebuilt_forrest.go
@@ -53,11 +53,11 @@ type oldRebuiltTreeValue struct {
Key btrfsprim.Key
ItemSize uint32
- Node nodeInfo
+ Node oldRebuiltNodeInfo
Slot int
}
-type nodeInfo struct {
+type oldRebuiltNodeInfo struct {
LAddr btrfsvol.LogicalAddr
Level uint8
Generation btrfsprim.Generation
@@ -187,7 +187,7 @@ func (bt *OldRebuiltForrest) rawTreeWalk(ctx context.Context, treeID btrfsprim.O
cacheEntry.ParentUUID = root.ParentUUID
cacheEntry.ParentGen = root.ParentGen
- var curNode nodeInfo
+ var curNode oldRebuiltNodeInfo
cbs := btrfstree.TreeWalkHandler{
BadNode: func(path btrfstree.Path, node *btrfstree.Node, err error) bool {
nodeAddr, nodeExp, _ := path.NodeExpectations(ctx, false)
@@ -200,7 +200,7 @@ func (bt *OldRebuiltForrest) rawTreeWalk(ctx context.Context, treeID btrfsprim.O
return false
},
Node: func(path btrfstree.Path, node *btrfstree.Node) {
- curNode = nodeInfo{
+ curNode = oldRebuiltNodeInfo{
LAddr: node.Head.Addr,
Level: node.Head.Level,
Generation: node.Head.Generation,
@@ -247,7 +247,7 @@ func (tree oldRebuiltTree) addErrs(fn func(btrfsprim.Key, uint32) int, err error
return errs
}
-func (bt *OldRebuiltForrest) readNode(ctx context.Context, nodeInfo nodeInfo) *btrfstree.Node {
+func (bt *OldRebuiltForrest) readNode(ctx context.Context, nodeInfo oldRebuiltNodeInfo) *btrfstree.Node {
node, err := bt.AcquireNode(ctx, nodeInfo.LAddr, btrfstree.NodeExpectations{
LAddr: containers.OptionalValue(nodeInfo.LAddr),
Level: containers.OptionalValue(nodeInfo.Level),
diff --git a/lib/btrfsutil/rebuilt_forrest.go b/lib/btrfsutil/rebuilt_forrest.go
index fcfb353..ba88bbc 100644
--- a/lib/btrfsutil/rebuilt_forrest.go
+++ b/lib/btrfsutil/rebuilt_forrest.go
@@ -16,7 +16,6 @@ import (
"git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol"
"git.lukeshu.com/btrfs-progs-ng/lib/containers"
"git.lukeshu.com/btrfs-progs-ng/lib/slices"
- "git.lukeshu.com/btrfs-progs-ng/lib/textui"
)
type RebuiltForrestCallbacks interface {
@@ -43,10 +42,11 @@ func (cb noopRebuiltForrestCallbacks) LookupRoot(ctx context.Context, tree btrfs
ObjectID: tree,
ItemType: btrfsprim.ROOT_ITEM_KEY,
}
- itemKey, itemPtr, ok := rootTree.RebuiltItems(ctx).Search(func(key btrfsprim.Key, _ ItemPtr) int {
+ itemKey, itemPtr, ok := rootTree.RebuiltAcquireItems(ctx).Search(func(key btrfsprim.Key, _ ItemPtr) int {
key.Offset = 0
return tgt.Compare(key)
})
+ rootTree.RebuiltReleaseItems()
if !ok {
return 0, btrfsitem.Root{}, false
}
@@ -70,7 +70,8 @@ func (cb noopRebuiltForrestCallbacks) LookupUUID(ctx context.Context, uuid btrfs
return 0, false
}
tgt := btrfsitem.UUIDToKey(uuid)
- itemPtr, ok := uuidTree.RebuiltItems(ctx).Load(tgt)
+ itemPtr, ok := uuidTree.RebuiltAcquireItems(ctx).Load(tgt)
+ uuidTree.RebuiltReleaseItems()
if !ok {
return 0, false
}
@@ -118,8 +119,9 @@ func (cb noopRebuiltForrestCallbacks) LookupUUID(ctx context.Context, uuid btrfs
// - it provides several RebuiltTree methods that provide advice on
// what roots should be added to a tree in order to repair it:
//
-// .RebuiltItems() and RebuiltPotentialItems() to compare what's
-// in the tree and what could be in the tree.
+// .RebuiltAcquireItems()/.RebuiltReleaseItems() and
+// .RebuiltAcquirePotentialItems()/.RebuiltReleasePotentialItems()
+// to compare what's in the tree and what could be in the tree.
//
// .RebuiltLeafToRoots() to map potential items to things that can
// be passed to .RebuiltAddRoot().
@@ -139,11 +141,10 @@ type RebuiltForrest struct {
// mutable
- treesMu nestedMutex
- trees map[btrfsprim.ObjID]*RebuiltTree // must hold .treesMu to access
- leafs containers.Cache[btrfsprim.ObjID, map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr]]
- incItems containers.Cache[btrfsprim.ObjID, itemIndex]
- excItems containers.Cache[btrfsprim.ObjID, itemIndex]
+ treesMu nestedMutex
+ trees map[btrfsprim.ObjID]*RebuiltTree // must hold .treesMu to access
+
+ rebuiltSharedCache
}
// NewRebuiltForrest returns a new RebuiltForrest instance. The
@@ -158,19 +159,8 @@ func NewRebuiltForrest(file btrfstree.NodeSource, sb btrfstree.Superblock, graph
trees: make(map[btrfsprim.ObjID]*RebuiltTree),
}
- ret.leafs = containers.NewARCache[btrfsprim.ObjID, map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr]](textui.Tunable(8),
- containers.SourceFunc[btrfsprim.ObjID, map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr]](
- func(ctx context.Context, treeID btrfsprim.ObjID, leafs *map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr]) {
- *leafs = ret.trees[treeID].uncachedLeafToRoots(ctx)
- }))
- ret.incItems = containers.NewARCache[btrfsprim.ObjID, itemIndex](textui.Tunable(8),
- containers.SourceFunc[btrfsprim.ObjID, itemIndex](func(ctx context.Context, treeID btrfsprim.ObjID, incItems *itemIndex) {
- *incItems = ret.trees[treeID].uncachedIncItems(ctx)
- }))
- ret.excItems = containers.NewARCache[btrfsprim.ObjID, itemIndex](textui.Tunable(8),
- containers.SourceFunc[btrfsprim.ObjID, itemIndex](func(ctx context.Context, treeID btrfsprim.ObjID, excItems *itemIndex) {
- *excItems = ret.trees[treeID].uncachedExcItems(ctx)
- }))
+ ret.rebuiltSharedCache = makeRebuiltSharedCache(ret)
+
if ret.cb == nil {
ret.cb = noopRebuiltForrestCallbacks{
forrest: ret,
@@ -249,11 +239,11 @@ func (ts *RebuiltForrest) addTree(ctx context.Context, treeID btrfsprim.ObjID, s
}
parentID, ok := ts.cb.LookupUUID(ctx, rootItem.ParentUUID)
if !ok {
- dlog.Error(ctx, "failed to add tree: lookup UUID")
+ dlog.Errorf(ctx, "failed to add tree: lookup UUID %v", rootItem.ParentUUID)
return false
}
if !ts.addTree(ctx, parentID, stack) {
- dlog.Error(ctx, "failed to add tree: add parent tree")
+ dlog.Errorf(ctx, "failed to add tree: add parent tree %v", parentID)
return false
}
tree.Parent = ts.trees[parentID]
diff --git a/lib/btrfsutil/rebuilt_tree.go b/lib/btrfsutil/rebuilt_tree.go
index b996bdb..ffb2e5f 100644
--- a/lib/btrfsutil/rebuilt_tree.go
+++ b/lib/btrfsutil/rebuilt_tree.go
@@ -37,21 +37,52 @@ type RebuiltTree struct {
// derived from tree.Roots, which is why it's OK if they get
// evicted.
//
- // 1. tree.leafToRoots() = tree.forrest.leafs.Acquire(tree.ID)
- // 2. tree.RebuiltItems() = tree.forrest.incItems.Acquire(tree.ID)
- // 3. tree.RebuiltPotentialItems() = tree.forrest.excItems.Acquire(tree.ID)
+ // 1. tree.acquireLeafToRoots() = tree.forrest.leafs.Acquire(tree.ID)
+ // 2. tree.RebuiltAcquireItems() = tree.forrest.incItems.Acquire(tree.ID)
+ // 3. tree.RebuiltAcquirePotentialItems() = tree.forrest.excItems.Acquire(tree.ID)
}
-// evictable member 1: .leafToRoots() //////////////////////////////////////////////////////////////////////////////////
+type rebuiltSharedCache struct {
+ leafs containers.Cache[btrfsprim.ObjID, map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr]]
+ incItems containers.Cache[btrfsprim.ObjID, containers.SortedMap[btrfsprim.Key, ItemPtr]]
+ excItems containers.Cache[btrfsprim.ObjID, containers.SortedMap[btrfsprim.Key, ItemPtr]]
+}
-// leafToRoots returns all leafs (lvl=0) in the filesystem that pass
-// .isOwnerOK, whether or not they're in the tree.
-func (tree *RebuiltTree) leafToRoots(ctx context.Context) map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr] {
- ret := *tree.forrest.leafs.Acquire(ctx, tree.ID)
- tree.forrest.leafs.Release(tree.ID)
+func makeRebuiltSharedCache(forrest *RebuiltForrest) rebuiltSharedCache {
+ var ret rebuiltSharedCache
+ ret.leafs = containers.NewARCache[btrfsprim.ObjID, map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr]](
+ textui.Tunable(8),
+ containers.SourceFunc[btrfsprim.ObjID, map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr]](
+ func(ctx context.Context, treeID btrfsprim.ObjID, leafs *map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr]) {
+ *leafs = forrest.trees[treeID].uncachedLeafToRoots(ctx)
+ }))
+ ret.incItems = containers.NewARCache[btrfsprim.ObjID, containers.SortedMap[btrfsprim.Key, ItemPtr]](
+ textui.Tunable(8),
+ containers.SourceFunc[btrfsprim.ObjID, containers.SortedMap[btrfsprim.Key, ItemPtr]](
+ func(ctx context.Context, treeID btrfsprim.ObjID, incItems *containers.SortedMap[btrfsprim.Key, ItemPtr]) {
+ *incItems = forrest.trees[treeID].uncachedIncItems(ctx)
+ }))
+ ret.excItems = containers.NewARCache[btrfsprim.ObjID, containers.SortedMap[btrfsprim.Key, ItemPtr]](
+ textui.Tunable(8),
+ containers.SourceFunc[btrfsprim.ObjID, containers.SortedMap[btrfsprim.Key, ItemPtr]](
+ func(ctx context.Context, treeID btrfsprim.ObjID, excItems *containers.SortedMap[btrfsprim.Key, ItemPtr]) {
+ *excItems = forrest.trees[treeID].uncachedExcItems(ctx)
+ }))
return ret
}
+// evictable member 1: .acquireLeafToRoots() ///////////////////////////////////////////////////////////////////////////
+
+// acquireLeafToRoots returns all leafs (lvl=0) in the filesystem that
+// pass .isOwnerOK, whether or not they're in the tree.
+func (tree *RebuiltTree) acquireLeafToRoots(ctx context.Context) map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr] {
+ return *tree.forrest.leafs.Acquire(ctx, tree.ID)
+}
+
+func (tree *RebuiltTree) releaseLeafToRoots() {
+ tree.forrest.leafs.Release(tree.ID)
+}
+
func (tree *RebuiltTree) uncachedLeafToRoots(ctx context.Context) map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr] {
ctx = dlog.WithField(ctx, "btrfs.util.rebuilt-tree.index-nodes", fmt.Sprintf("tree=%v", tree.ID))
@@ -85,7 +116,7 @@ func (tree *RebuiltTree) indexNode(ctx context.Context, node btrfsvol.LogicalAdd
if err := ctx.Err(); err != nil {
return
}
- if _, done := index[node]; done {
+ if maps.HasKey(index, node) {
return
}
if slices.Contains(node, stack) {
@@ -133,72 +164,81 @@ func (tree *RebuiltTree) isOwnerOK(owner btrfsprim.ObjID, gen btrfsprim.Generati
}
}
-// evictable members 2 and 3: .RebuiltItems() and .RebuiltPotentialItems() /////////////////////////////////////////////
+// evictable members 2 and 3: .Rebuilt{Acquire,Release}{Potential,}Items() /////////////////////////////////////////////
-// RebuiltItems returns a map of the items contained in this tree.
+// RebuiltAcquireItems returns a map of the items contained in this
+// tree.
//
// Do not mutate the returned map; it is a pointer to the
// RebuiltTree's internal map!
-func (tree *RebuiltTree) RebuiltItems(ctx context.Context) *containers.SortedMap[btrfsprim.Key, ItemPtr] {
- ret := *tree.forrest.incItems.Acquire(ctx, tree.ID)
+//
+// When done with the map, call .RebuiltReleaseItems().
+func (tree *RebuiltTree) RebuiltAcquireItems(ctx context.Context) *containers.SortedMap[btrfsprim.Key, ItemPtr] {
+ return tree.forrest.incItems.Acquire(ctx, tree.ID)
+}
+
+// RebuiltReleaseItems releases resources after a call to
+// .RebuiltAcquireItems().
+func (tree *RebuiltTree) RebuiltReleaseItems() {
tree.forrest.incItems.Release(tree.ID)
- return ret
}
-// RebuiltPotentialItems returns a map of items that could be added to
-// this tree with .RebuiltAddRoot().
+// RebuiltAcquirePotentialItems returns a map of items that could be
+// added to this tree with .RebuiltAddRoot().
//
// Do not mutate the returned map; it is a pointer to the
// RebuiltTree's internal map!
-func (tree *RebuiltTree) RebuiltPotentialItems(ctx context.Context) *containers.SortedMap[btrfsprim.Key, ItemPtr] {
- ret := *tree.forrest.excItems.Acquire(ctx, tree.ID)
+//
+// When done with the map, call .RebuiltReleasePotentialItems().
+func (tree *RebuiltTree) RebuiltAcquirePotentialItems(ctx context.Context) *containers.SortedMap[btrfsprim.Key, ItemPtr] {
+ return tree.forrest.excItems.Acquire(ctx, tree.ID)
+}
+
+// RebuiltReleasePotentialItems releases resources after a call to
+// .RebuiltAcquirePotentialItems().
+func (tree *RebuiltTree) RebuiltReleasePotentialItems() {
tree.forrest.excItems.Release(tree.ID)
- return ret
}
-func (tree *RebuiltTree) uncachedIncItems(ctx context.Context) *containers.SortedMap[btrfsprim.Key, ItemPtr] {
+func (tree *RebuiltTree) uncachedIncItems(ctx context.Context) containers.SortedMap[btrfsprim.Key, ItemPtr] {
ctx = dlog.WithField(ctx, "btrfs.util.rebuilt-tree.index-inc-items", fmt.Sprintf("tree=%v", tree.ID))
- return tree.items(ctx, tree.Roots.HasAny)
+ return tree.items(ctx, true)
}
-func (tree *RebuiltTree) uncachedExcItems(ctx context.Context) *containers.SortedMap[btrfsprim.Key, ItemPtr] {
+func (tree *RebuiltTree) uncachedExcItems(ctx context.Context) containers.SortedMap[btrfsprim.Key, ItemPtr] {
ctx = dlog.WithField(ctx, "btrfs.util.rebuilt-tree.index-exc-items", fmt.Sprintf("tree=%v", tree.ID))
- return tree.items(ctx,
- func(roots containers.Set[btrfsvol.LogicalAddr]) bool {
- return !tree.Roots.HasAny(roots)
- })
+ return tree.items(ctx, false)
}
-type itemIndex = *containers.SortedMap[btrfsprim.Key, ItemPtr]
-
-type itemStats struct {
+type rebuiltItemStats struct {
Leafs textui.Portion[int]
NumItems int
NumDups int
}
-func (s itemStats) String() string {
+func (s rebuiltItemStats) String() string {
return textui.Sprintf("%v (%v items, %v dups)",
s.Leafs, s.NumItems, s.NumDups)
}
-func (tree *RebuiltTree) items(ctx context.Context, leafFn func(roots containers.Set[btrfsvol.LogicalAddr]) bool) *containers.SortedMap[btrfsprim.Key, ItemPtr] {
+func (tree *RebuiltTree) items(ctx context.Context, inc bool) containers.SortedMap[btrfsprim.Key, ItemPtr] {
tree.mu.RLock()
defer tree.mu.RUnlock()
var leafs []btrfsvol.LogicalAddr
- for leaf, roots := range tree.leafToRoots(ctx) {
- if leafFn(roots) {
+ for leaf, roots := range tree.acquireLeafToRoots(ctx) {
+ if tree.Roots.HasAny(roots) == inc {
leafs = append(leafs, leaf)
}
}
+ tree.releaseLeafToRoots()
slices.Sort(leafs)
- var stats itemStats
+ var stats rebuiltItemStats
stats.Leafs.D = len(leafs)
- progressWriter := textui.NewProgress[itemStats](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second))
+ progressWriter := textui.NewProgress[rebuiltItemStats](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second))
- index := new(containers.SortedMap[btrfsprim.Key, ItemPtr])
+ var index containers.SortedMap[btrfsprim.Key, ItemPtr]
for i, leaf := range leafs {
stats.Leafs.N = i
progressWriter.Set(stats)
@@ -220,7 +260,7 @@ func (tree *RebuiltTree) items(ctx context.Context, leafFn func(roots containers
}
}
if stats.Leafs.N > 0 {
- stats.Leafs.N = len(leafs)
+ stats.Leafs.N = stats.Leafs.D
progressWriter.Set(stats)
progressWriter.Done()
}
@@ -262,13 +302,13 @@ func (tree *RebuiltTree) RebuiltShouldReplace(oldNode, newNode btrfsvol.LogicalA
}
}
-type rootStats struct {
+type rebuiltRootStats struct {
Leafs textui.Portion[int]
AddedLeafs int
AddedItems int
}
-func (s rootStats) String() string {
+func (s rebuiltRootStats) String() string {
return textui.Sprintf("%v (added %v leafs, added %v items)",
s.Leafs, s.AddedLeafs, s.AddedItems)
}
@@ -282,11 +322,10 @@ func (tree *RebuiltTree) RebuiltAddRoot(ctx context.Context, rootNode btrfsvol.L
ctx = dlog.WithField(ctx, "btrfs.util.rebuilt-tree.add-root", fmt.Sprintf("tree=%v rootNode=%v", tree.ID, rootNode))
dlog.Info(ctx, "adding root...")
- leafToRoots := tree.leafToRoots(ctx)
-
- var stats rootStats
+ var stats rebuiltRootStats
+ leafToRoots := tree.acquireLeafToRoots(ctx)
stats.Leafs.D = len(leafToRoots)
- progressWriter := textui.NewProgress[rootStats](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second))
+ progressWriter := textui.NewProgress[rebuiltRootStats](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second))
for i, leaf := range maps.SortedKeys(leafToRoots) {
stats.Leafs.N = i
progressWriter.Set(stats)
@@ -305,6 +344,7 @@ func (tree *RebuiltTree) RebuiltAddRoot(ctx context.Context, rootNode btrfsvol.L
}
}
stats.Leafs.N = len(leafToRoots)
+ tree.releaseLeafToRoots()
progressWriter.Set(stats)
progressWriter.Done()
@@ -335,7 +375,8 @@ func (tree *RebuiltTree) RebuiltCOWDistance(parentID btrfsprim.ObjID) (dist int,
// ReadItem reads an item from a tree.
func (tree *RebuiltTree) ReadItem(ctx context.Context, key btrfsprim.Key) btrfsitem.Item {
- ptr, ok := tree.RebuiltItems(ctx).Load(key)
+ ptr, ok := tree.RebuiltAcquireItems(ctx).Load(key)
+ tree.RebuiltReleaseItems()
if !ok {
panic(fmt.Errorf("should not happen: btrfsutil.RebuiltTree.ReadItem called for not-included key: %v", key))
}
@@ -352,13 +393,14 @@ func (tree *RebuiltTree) RebuiltLeafToRoots(ctx context.Context, leaf btrfsvol.L
tree.mu.RLock()
defer tree.mu.RUnlock()
ret := make(containers.Set[btrfsvol.LogicalAddr])
- for root := range tree.leafToRoots(ctx)[leaf] {
+ for root := range tree.acquireLeafToRoots(ctx)[leaf] {
if tree.Roots.Has(root) {
panic(fmt.Errorf("should not happen: (tree=%v).RebuiltLeafToRoots(leaf=%v): tree contains root=%v but not leaf",
tree.ID, leaf, root))
}
ret.Insert(root)
}
+ tree.releaseLeafToRoots()
if len(ret) == 0 {
return nil
}
diff --git a/lib/btrfsutil/scan.go b/lib/btrfsutil/scan.go
index 0e268e5..bf848ee 100644
--- a/lib/btrfsutil/scan.go
+++ b/lib/btrfsutil/scan.go
@@ -33,12 +33,12 @@ type DeviceScanner[Stats comparable, Result any] interface {
ScanDone(ctx context.Context) (Result, error)
}
-type scanStats[T comparable] struct {
+type devScanStats[T comparable] struct {
portion textui.Portion[btrfsvol.PhysicalAddr]
stats T
}
-func (s scanStats[T]) String() string {
+func (s devScanStats[T]) String() string {
return textui.Sprintf("scanned %v (%v)",
s.portion, s.stats)
}
@@ -91,8 +91,8 @@ func ScanOneDevice[Stats comparable, Result any](ctx context.Context, dev *btrfs
scanner := newScanner(ctx, *sb, numBytes, numSectors)
- progressWriter := textui.NewProgress[scanStats[Stats]](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second))
- var stats scanStats[Stats]
+ progressWriter := textui.NewProgress[devScanStats[Stats]](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second))
+ var stats devScanStats[Stats]
stats.portion.D = numBytes
var minNextNode btrfsvol.PhysicalAddr
diff --git a/lib/containers/set.go b/lib/containers/set.go
index 074d126..af13d50 100644
--- a/lib/containers/set.go
+++ b/lib/containers/set.go
@@ -138,20 +138,11 @@ func (o Set[T]) TakeOne() T {
}
func (o Set[T]) Has(v T) bool {
- _, has := o[v]
- return has
+ return maps.HasKey(o, v)
}
-func (small Set[T]) HasAny(big Set[T]) bool {
- if len(big) < len(small) {
- small, big = big, small
- }
- for v := range small {
- if _, ok := big[v]; ok {
- return true
- }
- }
- return false
+func (a Set[T]) HasAny(b Set[T]) bool {
+ return maps.HaveAnyKeysInCommon(a, b)
}
func (small Set[T]) Intersection(big Set[T]) Set[T] {
@@ -160,7 +151,7 @@ func (small Set[T]) Intersection(big Set[T]) Set[T] {
}
ret := make(Set[T])
for v := range small {
- if _, ok := big[v]; ok {
+ if maps.HasKey(big, v) {
ret.Insert(v)
}
}
diff --git a/lib/maps/maputil.go b/lib/maps/maputil.go
index d409e70..63e52a0 100644
--- a/lib/maps/maputil.go
+++ b/lib/maps/maputil.go
@@ -25,3 +25,20 @@ func SortedKeys[K constraints.Ordered, V any](m map[K]V) []K {
slices.Sort(ret)
return ret
}
+
+func HasKey[K comparable, V any](m map[K]V, k K) bool {
+ _, has := m[k]
+ return has
+}
+
+func HaveAnyKeysInCommon[K comparable, V1, V2 any](small map[K]V1, big map[K]V2) bool {
+ if len(big) < len(small) {
+ return HaveAnyKeysInCommon(big, small)
+ }
+ for v := range small {
+ if _, ok := big[v]; ok {
+ return true
+ }
+ }
+ return false
+}
diff --git a/lib/textui/log.go b/lib/textui/log.go
index 9aff364..e5d3f60 100644
--- a/lib/textui/log.go
+++ b/lib/textui/log.go
@@ -26,6 +26,8 @@ import (
"git.lukeshu.com/go/typedsync"
"github.com/datawire/dlib/dlog"
"github.com/spf13/pflag"
+
+ "git.lukeshu.com/btrfs-progs-ng/lib/maps"
)
type LogLevelFlag struct {
@@ -201,7 +203,7 @@ func (l *logger) log(lvl dlog.LogLevel, writeMsg func(io.Writer)) {
fields := make(map[string]any)
var fieldKeys []string
for f := l; f.parent != nil; f = f.parent {
- if _, exists := fields[f.fieldKey]; exists {
+ if maps.HasKey(fields, f.fieldKey) {
continue
}
fields[f.fieldKey] = f.fieldVal
@@ -328,6 +330,10 @@ func fieldOrd(key string) int {
case "btrfs.inspect.rebuild-trees.rebuild.want.reason":
return -8
+ // btrfsutil.Graph /////////////////////////////////////////////////////
+ case "btrfs.util.read-graph.step":
+ return -1
+
// btrfsutil.RebuiltForrest ////////////////////////////////////////////
case "btrfs.util.rebuilt-forrest.add-tree":
return -7
@@ -426,10 +432,14 @@ func writeField(w io.Writer, key string, val any) {
name = strings.TrimPrefix(name, "rebuild.")
}
}
- case strings.HasPrefix(name, "util.rebuilt-forrest."):
- name = strings.TrimPrefix(name, "util.rebuilt-forrest.")
- case strings.HasPrefix(name, "util.rebuilt-tree."):
- name = strings.TrimPrefix(name, "util.rebuilt-tree.")
+ case strings.HasPrefix(name, "util."):
+ name = strings.TrimPrefix(name, "util.")
+ switch {
+ case strings.HasPrefix(name, "rebuilt-forrest."):
+ name = strings.TrimPrefix(name, "rebuilt-forrest.")
+ case strings.HasPrefix(name, "rebuilt-tree."):
+ name = strings.TrimPrefix(name, "rebuilt-tree.")
+ }
}
}
diff --git a/scripts/main.sh b/scripts/main.sh
index dddb9bb..c5fc238 100755
--- a/scripts/main.sh
+++ b/scripts/main.sh
@@ -31,10 +31,15 @@ export GOMEMLIMIT="$(awk '/^MemTotal:/{ print $2 "KiB" }' </proc/meminfo)"
######################################################################
+# 0: initial scan ##########################################
+
run-btrfs-rec $gendir/0.scandevices.json \
inspect rebuild-mappings scan
run-btrfs-rec $gendir/0.nodes.json \
inspect rebuild-mappings list-nodes $gendir/0.scandevices.json
+
+# 1-2: rebuild chunk/dev-ext/blockgroup trees ##############
+
run-btrfs-rec $gendir/1.mappings.json \
inspect rebuild-mappings process $gendir/0.scandevices.json
@@ -62,13 +67,20 @@ run-btrfs-rec $gendir/2.mappings.json \
-e '2a{"LAddr":13631488,"PAddr":{"Dev":1,"Addr":13631488},"Size":1},') \
inspect rebuild-mappings process $gendir/0.scandevices.json
+# 3: rebuild other trees ###################################
+
run-btrfs-rec $gendir/3.trees.json \
--mappings=$gendir/2.mappings.json \
- inspect rebuild-trees --node-list=$gendir/0.nodes.json
+ --node-list=$gendir/0.nodes.json \
+ inspect rebuild-trees
+
+# 4: dump data from the FS #################################
run-btrfs-rec $gendir/4.ls-files.txt \
--mappings=$gendir/2.mappings.json \
+ --rebuild \
inspect ls-files
run-btrfs-rec $gendir/4.ls-trees.txt \
--mappings=$gendir/2.mappings.json \
- inspect ls-trees --node-list=$gendir/0.nodes.json
+ --node-list=$gendir/0.nodes.json \
+ inspect ls-trees
diff --git a/scripts/mount.sh b/scripts/mount.sh
index bf84163..4894e94 100755
--- a/scripts/mount.sh
+++ b/scripts/mount.sh
@@ -15,6 +15,7 @@ mkdir -p "$mountpoint"
sudo ./bin/btrfs-rec \
--pv="$image"
--mappings="$gendir/2.mappings.json" \
+ --rebuild \
inspect mount \
--skip-filesums \
"$mountpoint"