From 77046dcf26687fd74df0f7966d7d469f6b6de717 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 27 Nov 2022 16:41:44 -0700 Subject: rebuildnodes: Refactor the node-graph stuff in to a separate sub-package --- lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go | 135 ++--------------------- 1 file changed, 9 insertions(+), 126 deletions(-) (limited to 'lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go') diff --git a/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go b/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go index c32ae8e..40ace46 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go @@ -6,7 +6,6 @@ package rebuildnodes import ( "context" - "fmt" "github.com/datawire/dlib/dlog" @@ -16,78 +15,14 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfstree" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildnodes/graph" "git.lukeshu.com/btrfs-progs-ng/lib/containers" "git.lukeshu.com/btrfs-progs-ng/lib/maps" ) -type nodeData struct { - Level uint8 - Generation btrfsprim.Generation - Owner btrfsprim.ObjID - NumItems uint32 - MinItem btrfsprim.Key - MaxItem btrfsprim.Key -} - -type kpData struct { - FromTree btrfsprim.ObjID - FromNode btrfsvol.LogicalAddr - FromItem int - - ToNode btrfsvol.LogicalAddr - ToLevel uint8 - ToKey btrfsprim.Key - ToGeneration btrfsprim.Generation -} - -func (kp kpData) String() string { - return fmt.Sprintf(`{t:%v,n:%v}[%d]->{n:%v,l:%v,g:%v,k:(%d,%v,%d)}`, - kp.FromTree, kp.FromNode, kp.FromItem, - kp.ToNode, kp.ToLevel, kp.ToGeneration, - kp.ToKey.ObjectID, - kp.ToKey.ItemType, - kp.ToKey.Offset) -} - -type nodeGraph struct { - Nodes map[btrfsvol.LogicalAddr]nodeData - BadNodes map[btrfsvol.LogicalAddr]error - EdgesFrom map[btrfsvol.LogicalAddr][]*kpData - EdgesTo map[btrfsvol.LogicalAddr][]*kpData -} - -func (g nodeGraph) insertEdge(kp kpData) { - ptr := &kp - if kp.ToNode == 0 { - panic("kp.ToNode should not be zero") - } - g.EdgesFrom[kp.FromNode] = append(g.EdgesFrom[kp.FromNode], ptr) - g.EdgesTo[kp.ToNode] = append(g.EdgesTo[kp.ToNode], ptr) -} - -func (g nodeGraph) insertTreeRoot(sb btrfstree.Superblock, treeID btrfsprim.ObjID) { - treeInfo, err := btrfstree.LookupTreeRoot(nil, sb, treeID) - if err != nil { - // This shouldn't ever happen for treeIDs that are - // mentioned directly in the superblock; which are the - // only trees for which we should call - // .insertTreeRoot(). - panic(fmt.Errorf("LookupTreeRoot(%v): %w", treeID, err)) - } - if treeInfo.RootNode == 0 { - return - } - g.insertEdge(kpData{ - FromTree: treeID, - ToNode: treeInfo.RootNode, - ToLevel: treeInfo.Level, - ToGeneration: treeInfo.Generation, - }) -} - type scanResult struct { uuidMap - nodeGraph + nodeGraph *graph.Graph } func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.ScanDevicesResult) (*scanResult, error) { @@ -101,7 +36,7 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca lastPct := -1 total := countNodes(scanResults) done := 0 - progress := func() { + progress := func(done, total int) { pct := int(100 * float64(done) / float64(total)) if pct != lastPct || done == total { dlog.Infof(ctx, "... %v%% (%v/%v)", @@ -118,31 +53,17 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca SeenTrees: make(containers.Set[btrfsprim.ObjID]), }, - nodeGraph: nodeGraph{ - Nodes: make(map[btrfsvol.LogicalAddr]nodeData), - BadNodes: make(map[btrfsvol.LogicalAddr]error), - EdgesFrom: make(map[btrfsvol.LogicalAddr][]*kpData), - EdgesTo: make(map[btrfsvol.LogicalAddr][]*kpData), - }, + nodeGraph: graph.New(*sb), } // These 4 trees are mentioned directly in the superblock, so // they are always seen. - // - // 1 ret.SeenTrees.Insert(btrfsprim.ROOT_TREE_OBJECTID) - ret.insertTreeRoot(*sb, btrfsprim.ROOT_TREE_OBJECTID) - // 2 ret.SeenTrees.Insert(btrfsprim.CHUNK_TREE_OBJECTID) - ret.insertTreeRoot(*sb, btrfsprim.CHUNK_TREE_OBJECTID) - // 3 ret.SeenTrees.Insert(btrfsprim.TREE_LOG_OBJECTID) - ret.insertTreeRoot(*sb, btrfsprim.TREE_LOG_OBJECTID) - // 4 ret.SeenTrees.Insert(btrfsprim.BLOCK_GROUP_TREE_OBJECTID) - ret.insertTreeRoot(*sb, btrfsprim.BLOCK_GROUP_TREE_OBJECTID) - progress() + progress(done, total) for _, devResults := range scanResults { for laddr := range devResults.FoundNodes { nodeRef, err := btrfstree.ReadNode[btrfsvol.LogicalAddr](fs, *sb, laddr, btrfstree.NodeExpectations{ @@ -168,13 +89,6 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca return nil, err } ret.SeenTrees.Insert(item.Key.ObjectID) - // graph building - ret.insertEdge(kpData{ - FromTree: item.Key.ObjectID, - ToNode: itemBody.ByteNr, - ToLevel: itemBody.Level, - ToGeneration: itemBody.Generation, - }) case btrfsitem.UUIDMap: uuid := btrfsitem.KeyToUUID(item.Key) if err := maybeSet("ObjID2UUID", ret.ObjID2UUID, itemBody.ObjID, uuid); err != nil { @@ -188,28 +102,10 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca ret.SeenTrees.Insert(nodeRef.Data.Head.Owner) // graph building - ret.Nodes[laddr] = nodeData{ - Level: nodeRef.Data.Head.Level, - Generation: nodeRef.Data.Head.Generation, - Owner: nodeRef.Data.Head.Owner, - NumItems: nodeRef.Data.Head.NumItems, - MinItem: func() btrfsprim.Key { k, _ := nodeRef.Data.MinItem(); return k }(), - MaxItem: func() btrfsprim.Key { k, _ := nodeRef.Data.MaxItem(); return k }(), - } - for i, kp := range nodeRef.Data.BodyInternal { - ret.insertEdge(kpData{ - FromTree: nodeRef.Data.Head.Owner, - FromNode: laddr, - FromItem: i, - ToNode: kp.BlockPtr, - ToLevel: nodeRef.Data.Head.Level - 1, - ToKey: kp.Key, - ToGeneration: kp.Generation, - }) - } + ret.nodeGraph.InsertNode(nodeRef) done++ - progress() + progress(done, total) } } @@ -225,21 +121,8 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca dlog.Info(ctx, "... done reading node data") dlog.Infof(ctx, "Checking keypointers for dead-ends...") - total = len(ret.EdgesTo) - done = 0 - progress() - for laddr := range ret.EdgesTo { - if _, ok := ret.Nodes[laddr]; !ok { - _, err := btrfstree.ReadNode[btrfsvol.LogicalAddr](fs, *sb, laddr, btrfstree.NodeExpectations{ - LAddr: containers.Optional[btrfsvol.LogicalAddr]{OK: true, Val: laddr}, - }) - if err == nil { - return nil, fmt.Errorf("node@%v exists but was not in node scan results", laddr) - } - ret.BadNodes[laddr] = err - } - done++ - progress() + if err := ret.nodeGraph.FinalCheck(fs, *sb, progress); err != nil { + return nil, err } dlog.Info(ctx, "... done checking keypointers") -- cgit v1.2.3-2-g168b From 182bab3aa8b8e08d6a35d344c32d93a4293a36f4 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 29 Nov 2022 14:42:30 -0700 Subject: rebuildnodes: Refactor uuidMap in to a separate sub-package --- lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go | 51 +++--------------------- 1 file changed, 6 insertions(+), 45 deletions(-) (limited to 'lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go') diff --git a/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go b/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go index 40ace46..9174035 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go @@ -10,18 +10,17 @@ import ( "github.com/datawire/dlib/dlog" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsprim" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfstree" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect" "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildnodes/graph" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildnodes/uuidmap" "git.lukeshu.com/btrfs-progs-ng/lib/containers" "git.lukeshu.com/btrfs-progs-ng/lib/maps" ) type scanResult struct { - uuidMap + uuidMap *uuidmap.UUIDMap nodeGraph *graph.Graph } @@ -46,23 +45,10 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca } ret := &scanResult{ - uuidMap: uuidMap{ - ObjID2UUID: make(map[btrfsprim.ObjID]btrfsprim.UUID), - UUID2ObjID: make(map[btrfsprim.UUID]btrfsprim.ObjID), - TreeParent: make(map[btrfsprim.ObjID]btrfsprim.UUID), - - SeenTrees: make(containers.Set[btrfsprim.ObjID]), - }, + uuidMap: uuidmap.New(), nodeGraph: graph.New(*sb), } - // These 4 trees are mentioned directly in the superblock, so - // they are always seen. - ret.SeenTrees.Insert(btrfsprim.ROOT_TREE_OBJECTID) - ret.SeenTrees.Insert(btrfsprim.CHUNK_TREE_OBJECTID) - ret.SeenTrees.Insert(btrfsprim.TREE_LOG_OBJECTID) - ret.SeenTrees.Insert(btrfsprim.BLOCK_GROUP_TREE_OBJECTID) - progress(done, total) for _, devResults := range scanResults { for laddr := range devResults.FoundNodes { @@ -73,35 +59,10 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca return nil, err } - // UUID map rebuilding - for _, item := range nodeRef.Data.BodyLeaf { - switch itemBody := item.Body.(type) { - case btrfsitem.Root: - if err := maybeSet("ObjID2UUID", ret.ObjID2UUID, item.Key.ObjectID, itemBody.UUID); err != nil { - return nil, err - } - if itemBody.UUID != (btrfsprim.UUID{}) { - if err := maybeSet("UUID2ObjID", ret.UUID2ObjID, itemBody.UUID, item.Key.ObjectID); err != nil { - return nil, err - } - } - if err := maybeSet("ParentUUID", ret.TreeParent, item.Key.ObjectID, itemBody.ParentUUID); err != nil { - return nil, err - } - ret.SeenTrees.Insert(item.Key.ObjectID) - case btrfsitem.UUIDMap: - uuid := btrfsitem.KeyToUUID(item.Key) - if err := maybeSet("ObjID2UUID", ret.ObjID2UUID, itemBody.ObjID, uuid); err != nil { - return nil, err - } - if err := maybeSet("UUID2ObjID", ret.UUID2ObjID, uuid, itemBody.ObjID); err != nil { - return nil, err - } - } + if err := ret.uuidMap.InsertNode(nodeRef); err != nil { + return nil, err } - ret.SeenTrees.Insert(nodeRef.Data.Head.Owner) - // graph building ret.nodeGraph.InsertNode(nodeRef) done++ @@ -113,7 +74,7 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca panic("should not happen") } - missing := ret.missingRootItems() + missing := ret.uuidMap.MissingRootItems() if len(missing) > 0 { dlog.Errorf(ctx, "... could not find root items for %d trees: %v", len(missing), maps.SortedKeys(missing)) } -- cgit v1.2.3-2-g168b From 94a86a763157327ac969c98e19d7770db477a6a3 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 14 Dec 2022 16:19:07 -0700 Subject: Migrate existing progress logs to textui.NewProgress --- lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go | 26 ++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) (limited to 'lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go') diff --git a/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go b/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go index 9174035..3575534 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go @@ -6,6 +6,8 @@ package rebuildnodes import ( "context" + "fmt" + "time" "github.com/datawire/dlib/dlog" @@ -17,6 +19,7 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildnodes/uuidmap" "git.lukeshu.com/btrfs-progs-ng/lib/containers" "git.lukeshu.com/btrfs-progs-ng/lib/maps" + "git.lukeshu.com/btrfs-progs-ng/lib/textui" ) type scanResult struct { @@ -24,6 +27,16 @@ type scanResult struct { nodeGraph *graph.Graph } +type scanStats struct { + N, D int +} + +func (s scanStats) String() string { + return fmt.Sprintf("... %v%% (%v/%v)", + int(100*float64(s.N)/float64(s.D)), + s.N, s.D) +} + func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.ScanDevicesResult) (*scanResult, error) { dlog.Infof(ctx, "Reading node data from FS...") @@ -32,16 +45,11 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca return nil, err } - lastPct := -1 total := countNodes(scanResults) done := 0 + progressWriter := textui.NewProgress[scanStats](ctx, dlog.LogLevelInfo, 1*time.Second) progress := func(done, total int) { - pct := int(100 * float64(done) / float64(total)) - if pct != lastPct || done == total { - dlog.Infof(ctx, "... %v%% (%v/%v)", - pct, done, total) - lastPct = pct - } + progressWriter.Set(scanStats{N: done, D: total}) } ret := &scanResult{ @@ -69,10 +77,10 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca progress(done, total) } } - if done != total { panic("should not happen") } + progressWriter.Done() missing := ret.uuidMap.MissingRootItems() if len(missing) > 0 { @@ -81,10 +89,12 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca dlog.Info(ctx, "... done reading node data") + progressWriter = textui.NewProgress[scanStats](ctx, dlog.LogLevelInfo, 1*time.Second) dlog.Infof(ctx, "Checking keypointers for dead-ends...") if err := ret.nodeGraph.FinalCheck(fs, *sb, progress); err != nil { return nil, err } + progressWriter.Done() dlog.Info(ctx, "... done checking keypointers") return ret, nil -- cgit v1.2.3-2-g168b