From b261c2a4f5f028c9d83cef208ccc7d829f841bad Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 31 Dec 2022 11:59:09 -0700 Subject: tree-wide: Annotate values that I might want to be tuning --- lib/btrfs/io4_fs.go | 44 +++++++++++++--------- .../btrfsinspect/rebuildmappings/fuzzymatchsums.go | 2 +- .../rebuildnodes/btrees/rebuilt_btrees.go | 6 +-- .../btrfsinspect/rebuildnodes/graph/graph.go | 4 +- .../btrfsinspect/rebuildnodes/keyio/keyio.go | 3 +- .../btrfsinspect/rebuildnodes/rebuild.go | 4 +- lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go | 2 +- lib/btrfsprogs/btrfsinspect/scandevices.go | 2 +- lib/btrfsprogs/btrfsutil/open.go | 5 ++- lib/btrfsprogs/btrfsutil/skinny_paths.go | 3 +- lib/containers/lru.go | 26 ++----------- lib/textui/log_memstats.go | 2 +- lib/textui/tunable.go | 13 +++++++ 13 files changed, 61 insertions(+), 55 deletions(-) create mode 100644 lib/textui/tunable.go (limited to 'lib') diff --git a/lib/btrfs/io4_fs.go b/lib/btrfs/io4_fs.go index 3848ef1..41df3a8 100644 --- a/lib/btrfs/io4_fs.go +++ b/lib/btrfs/io4_fs.go @@ -22,6 +22,7 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/containers" "git.lukeshu.com/btrfs-progs-ng/lib/maps" "git.lukeshu.com/btrfs-progs-ng/lib/slices" + "git.lukeshu.com/btrfs-progs-ng/lib/textui" ) type BareInode struct { @@ -69,32 +70,37 @@ type Subvolume struct { TreeID btrfsprim.ObjID NoChecksums bool - rootOnce sync.Once - rootVal btrfsitem.Root - rootErr error + initOnce sync.Once - bareInodeCache containers.LRUCache[btrfsprim.ObjID, *BareInode] - fullInodeCache containers.LRUCache[btrfsprim.ObjID, *FullInode] - dirCache containers.LRUCache[btrfsprim.ObjID, *Dir] - fileCache containers.LRUCache[btrfsprim.ObjID, *File] + rootVal btrfsitem.Root + rootErr error + + bareInodeCache *containers.LRUCache[btrfsprim.ObjID, *BareInode] + fullInodeCache *containers.LRUCache[btrfsprim.ObjID, *FullInode] + dirCache *containers.LRUCache[btrfsprim.ObjID, *Dir] + fileCache *containers.LRUCache[btrfsprim.ObjID, *File] } func (sv *Subvolume) init() { - sv.rootOnce.Do(func() { + sv.initOnce.Do(func() { root, err := sv.FS.TreeSearch(btrfsprim.ROOT_TREE_OBJECTID, btrfstree.RootItemSearchFn(sv.TreeID)) if err != nil { sv.rootErr = err - return + } else { + switch rootBody := root.Body.(type) { + case btrfsitem.Root: + sv.rootVal = rootBody + case btrfsitem.Error: + sv.rootErr = fmt.Errorf("FS_TREE ROOT_ITEM has malformed body: %w", rootBody.Err) + default: + panic(fmt.Errorf("should not happen: ROOT_ITEM has unexpected item type: %T", rootBody)) + } } - switch rootBody := root.Body.(type) { - case btrfsitem.Root: - sv.rootVal = rootBody - case btrfsitem.Error: - sv.rootErr = fmt.Errorf("FS_TREE ROOT_ITEM has malformed body: %w", rootBody.Err) - default: - panic(fmt.Errorf("should not happen: ROOT_ITEM has unexpected item type: %T", rootBody)) - } + sv.bareInodeCache = containers.NewLRUCache[btrfsprim.ObjID, *BareInode](textui.Tunable(128)) + sv.fullInodeCache = containers.NewLRUCache[btrfsprim.ObjID, *FullInode](textui.Tunable(128)) + sv.dirCache = containers.NewLRUCache[btrfsprim.ObjID, *Dir](textui.Tunable(128)) + sv.fileCache = containers.NewLRUCache[btrfsprim.ObjID, *File](textui.Tunable(128)) }) } @@ -104,6 +110,7 @@ func (sv *Subvolume) GetRootInode() (btrfsprim.ObjID, error) { } func (sv *Subvolume) LoadBareInode(inode btrfsprim.ObjID) (*BareInode, error) { + sv.init() val := sv.bareInodeCache.GetOrElse(inode, func() (val *BareInode) { val = &BareInode{ Inode: inode, @@ -136,6 +143,7 @@ func (sv *Subvolume) LoadBareInode(inode btrfsprim.ObjID) (*BareInode, error) { } func (sv *Subvolume) LoadFullInode(inode btrfsprim.ObjID) (*FullInode, error) { + sv.init() val := sv.fullInodeCache.GetOrElse(inode, func() (val *FullInode) { val = &FullInode{ BareInode: BareInode{ @@ -191,6 +199,7 @@ func (sv *Subvolume) LoadFullInode(inode btrfsprim.ObjID) (*FullInode, error) { } func (sv *Subvolume) LoadDir(inode btrfsprim.ObjID) (*Dir, error) { + sv.init() val := sv.dirCache.GetOrElse(inode, func() (val *Dir) { val = new(Dir) fullInode, err := sv.LoadFullInode(inode) @@ -326,6 +335,7 @@ func (dir *Dir) AbsPath() (string, error) { } func (sv *Subvolume) LoadFile(inode btrfsprim.ObjID) (*File, error) { + sv.init() val := sv.fileCache.GetOrElse(inode, func() (val *File) { val = new(File) fullInode, err := sv.LoadFullInode(inode) diff --git a/lib/btrfsprogs/btrfsinspect/rebuildmappings/fuzzymatchsums.go b/lib/btrfsprogs/btrfsinspect/rebuildmappings/fuzzymatchsums.go index b1be7ba..a8d05eb 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildmappings/fuzzymatchsums.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildmappings/fuzzymatchsums.go @@ -19,7 +19,7 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/textui" ) -const minFuzzyPct = 0.5 +var minFuzzyPct = textui.Tunable(0.5) type fuzzyRecord struct { PAddr btrfsvol.QualifiedPhysicalAddr diff --git a/lib/btrfsprogs/btrfsinspect/rebuildnodes/btrees/rebuilt_btrees.go b/lib/btrfsprogs/btrfsinspect/rebuildnodes/btrees/rebuilt_btrees.go index 33eb352..5e8883a 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildnodes/btrees/rebuilt_btrees.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildnodes/btrees/rebuilt_btrees.go @@ -1,4 +1,4 @@ -// Copyright (C) 2022 Luke Shumaker +// Copyright (C) 2022-2023 Luke Shumaker // // SPDX-License-Identifier: GPL-2.0-or-later @@ -191,7 +191,7 @@ func (ts *RebuiltTrees) AddRoot(ctx context.Context, treeID btrfsprim.ObjID, roo var stats rootStats stats.Leafs.D = len(tree.leafToRoots) - progressWriter := textui.NewProgress[rootStats](ctx, dlog.LogLevelInfo, 1*time.Second) + progressWriter := textui.NewProgress[rootStats](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second)) for i, leaf := range maps.SortedKeys(tree.leafToRoots) { stats.Leafs.N = i progressWriter.Set(stats) @@ -297,7 +297,7 @@ func (tree *rebuiltTree) indexLeafs(ctx context.Context, graph pkggraph.Graph) { var stats textui.Portion[int] stats.D = len(graph.Nodes) - progressWriter := textui.NewProgress[textui.Portion[int]](ctx, dlog.LogLevelInfo, 1*time.Second) + progressWriter := textui.NewProgress[textui.Portion[int]](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second)) progress := func() { stats.N = len(nodeToRoots) progressWriter.Set(stats) diff --git a/lib/btrfsprogs/btrfsinspect/rebuildnodes/graph/graph.go b/lib/btrfsprogs/btrfsinspect/rebuildnodes/graph/graph.go index c4ed675..cf86d74 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildnodes/graph/graph.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildnodes/graph/graph.go @@ -197,7 +197,7 @@ func (g Graph) FinalCheck(ctx context.Context, fs diskio.File[btrfsvol.LogicalAd ctx = dlog.WithField(_ctx, "btrfsinspect.rebuild-nodes.read.substep", "check-keypointers") dlog.Info(_ctx, "Checking keypointers for dead-ends...") - progressWriter := textui.NewProgress[textui.Portion[int]](ctx, dlog.LogLevelInfo, 1*time.Second) + progressWriter := textui.NewProgress[textui.Portion[int]](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second)) stats.D = len(g.EdgesTo) progressWriter.Set(stats) for laddr := range g.EdgesTo { @@ -221,7 +221,7 @@ func (g Graph) FinalCheck(ctx context.Context, fs diskio.File[btrfsvol.LogicalAd dlog.Info(_ctx, "Checking for btree loops...") stats.D = len(g.Nodes) stats.N = 0 - progressWriter = textui.NewProgress[textui.Portion[int]](ctx, dlog.LogLevelInfo, 1*time.Second) + progressWriter = textui.NewProgress[textui.Portion[int]](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second)) progressWriter.Set(stats) visited := make(containers.Set[btrfsvol.LogicalAddr], len(g.Nodes)) numLoops := 0 diff --git a/lib/btrfsprogs/btrfsinspect/rebuildnodes/keyio/keyio.go b/lib/btrfsprogs/btrfsinspect/rebuildnodes/keyio/keyio.go index b1e68f9..24c3dcf 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildnodes/keyio/keyio.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildnodes/keyio/keyio.go @@ -17,6 +17,7 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildnodes/graph" "git.lukeshu.com/btrfs-progs-ng/lib/containers" "git.lukeshu.com/btrfs-progs-ng/lib/diskio" + "git.lukeshu.com/btrfs-progs-ng/lib/textui" ) type ItemPtr struct { @@ -50,7 +51,7 @@ func NewHandle(file diskio.File[btrfsvol.LogicalAddr], sb btrfstree.Superblock) Sizes: make(map[ItemPtr]SizeAndErr), - cache: containers.NewLRUCache[btrfsvol.LogicalAddr, *diskio.Ref[btrfsvol.LogicalAddr, btrfstree.Node]](8), + cache: containers.NewLRUCache[btrfsvol.LogicalAddr, *diskio.Ref[btrfsvol.LogicalAddr, btrfstree.Node]](textui.Tunable(8)), } } diff --git a/lib/btrfsprogs/btrfsinspect/rebuildnodes/rebuild.go b/lib/btrfsprogs/btrfsinspect/rebuildnodes/rebuild.go index 5d0804c..0a3ccdf 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildnodes/rebuild.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildnodes/rebuild.go @@ -126,7 +126,7 @@ func (o *rebuilder) rebuild(_ctx context.Context) error { o.itemQueue = nil var progress textui.Portion[int] progress.D = len(itemQueue) - progressWriter := textui.NewProgress[textui.Portion[int]](stepCtx, dlog.LogLevelInfo, 1*time.Second) + progressWriter := textui.NewProgress[textui.Portion[int]](stepCtx, dlog.LogLevelInfo, textui.Tunable(1*time.Second)) stepCtx = dlog.WithField(stepCtx, "btrfsinspect.rebuild-nodes.rebuild.substep.progress", &progress) for i, key := range itemQueue { itemCtx := dlog.WithField(stepCtx, "btrfsinspect.rebuild-nodes.rebuild.process.item", key) @@ -160,7 +160,7 @@ func (o *rebuilder) rebuild(_ctx context.Context) error { progress.D += len(resolvedAugments[treeID]) } o.augmentQueue = make(map[btrfsprim.ObjID][]map[btrfsvol.LogicalAddr]int) - progressWriter = textui.NewProgress[textui.Portion[int]](stepCtx, dlog.LogLevelInfo, 1*time.Second) + progressWriter = textui.NewProgress[textui.Portion[int]](stepCtx, dlog.LogLevelInfo, textui.Tunable(1*time.Second)) stepCtx = dlog.WithField(stepCtx, "btrfsinspect.rebuild-nodes.rebuild.substep.progress", &progress) for _, treeID := range maps.SortedKeys(resolvedAugments) { treeCtx := dlog.WithField(stepCtx, "btrfsinspect.rebuild-nodes.rebuild.augment.tree", treeID) diff --git a/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go b/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go index 7a112b4..7e96e29 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go @@ -32,7 +32,7 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca stats.D = countNodes(scanResults) progressWriter := textui.NewProgress[textui.Portion[int]]( dlog.WithField(ctx, "btrfsinspect.rebuild-nodes.read.substep", "read-nodes"), - dlog.LogLevelInfo, 1*time.Second) + dlog.LogLevelInfo, textui.Tunable(1*time.Second)) nodeGraph := graph.New(*sb) keyIO := keyio.NewHandle(fs, *sb) diff --git a/lib/btrfsprogs/btrfsinspect/scandevices.go b/lib/btrfsprogs/btrfsinspect/scandevices.go index 4058663..7668a83 100644 --- a/lib/btrfsprogs/btrfsinspect/scandevices.go +++ b/lib/btrfsprogs/btrfsinspect/scandevices.go @@ -126,7 +126,7 @@ func ScanOneDevice(ctx context.Context, dev *btrfs.Device, sb btrfstree.Superblo var sums strings.Builder sums.Grow(numSums * csumSize) - progressWriter := textui.NewProgress[scanStats](ctx, dlog.LogLevelInfo, 1*time.Second) + progressWriter := textui.NewProgress[scanStats](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second)) progress := func(pos btrfsvol.PhysicalAddr) { progressWriter.Set(scanStats{ Portion: textui.Portion[btrfsvol.PhysicalAddr]{ diff --git a/lib/btrfsprogs/btrfsutil/open.go b/lib/btrfsprogs/btrfsutil/open.go index 9491a20..441d4e2 100644 --- a/lib/btrfsprogs/btrfsutil/open.go +++ b/lib/btrfsprogs/btrfsutil/open.go @@ -14,6 +14,7 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" "git.lukeshu.com/btrfs-progs-ng/lib/diskio" + "git.lukeshu.com/btrfs-progs-ng/lib/textui" ) func Open(ctx context.Context, flag int, filenames ...string) (*btrfs.FS, error) { @@ -30,8 +31,8 @@ func Open(ctx context.Context, flag int, filenames ...string) (*btrfs.FS, error) } bufFile := diskio.NewBufferedFile[btrfsvol.PhysicalAddr]( typedFile, - 16384, // block size: 16KiB - 1024, // number of blocks to buffer; total of 16MiB + textui.Tunable[btrfsvol.PhysicalAddr](16384), // block size: 16KiB + textui.Tunable(1024), // number of blocks to buffer; total of 16MiB ) devFile := &btrfs.Device{ File: bufFile, diff --git a/lib/btrfsprogs/btrfsutil/skinny_paths.go b/lib/btrfsprogs/btrfsutil/skinny_paths.go index 2f71968..6a51739 100644 --- a/lib/btrfsprogs/btrfsutil/skinny_paths.go +++ b/lib/btrfsprogs/btrfsutil/skinny_paths.go @@ -11,6 +11,7 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" "git.lukeshu.com/btrfs-progs-ng/lib/containers" "git.lukeshu.com/btrfs-progs-ng/lib/diskio" + "git.lukeshu.com/btrfs-progs-ng/lib/textui" ) type skinnyItem struct { @@ -42,7 +43,7 @@ func (a *SkinnyPathArena) init() { // which is a good number for me. Then I tought it to do a // better job of recovering trees, and so the memory grew, and I // cut it to 64K. Then to 8K. Then grew it to 128K. - a.fatItems = containers.NewLRUCache[skinnyItem, btrfstree.TreePathElem](128 * 1024) + a.fatItems = containers.NewLRUCache[skinnyItem, btrfstree.TreePathElem](textui.Tunable(128 * 1024)) } } diff --git a/lib/containers/lru.go b/lib/containers/lru.go index bfda361..aa372ed 100644 --- a/lib/containers/lru.go +++ b/lib/containers/lru.go @@ -5,45 +5,30 @@ package containers import ( - "sync" - lru "github.com/hashicorp/golang-lru" ) // LRUCache is a least-recently-used(ish) cache. A zero LRUCache is -// usable and has a cache size of 128 items; use NewLRUCache to set a -// different size. +// not usable; it must be initialized with NewLRUCache. type LRUCache[K comparable, V any] struct { - initOnce sync.Once - inner *lru.ARCCache + inner *lru.ARCCache } func NewLRUCache[K comparable, V any](size int) *LRUCache[K, V] { c := new(LRUCache[K, V]) - c.initOnce.Do(func() { - c.inner, _ = lru.NewARC(size) - }) + c.inner, _ = lru.NewARC(size) return c } -func (c *LRUCache[K, V]) init() { - c.initOnce.Do(func() { - c.inner, _ = lru.NewARC(128) - }) -} - func (c *LRUCache[K, V]) Add(key K, value V) { - c.init() c.inner.Add(key, value) } func (c *LRUCache[K, V]) Contains(key K) bool { - c.init() return c.inner.Contains(key) } func (c *LRUCache[K, V]) Get(key K) (value V, ok bool) { - c.init() _value, ok := c.inner.Get(key) if ok { //nolint:forcetypeassert // Typed wrapper around untyped lib. @@ -53,7 +38,6 @@ func (c *LRUCache[K, V]) Get(key K) (value V, ok bool) { } func (c *LRUCache[K, V]) Keys() []K { - c.init() untyped := c.inner.Keys() typed := make([]K, len(untyped)) for i := range untyped { @@ -64,12 +48,10 @@ func (c *LRUCache[K, V]) Keys() []K { } func (c *LRUCache[K, V]) Len() int { - c.init() return c.inner.Len() } func (c *LRUCache[K, V]) Peek(key K) (value V, ok bool) { - c.init() _value, ok := c.inner.Peek(key) if ok { //nolint:forcetypeassert // Typed wrapper around untyped lib. @@ -79,12 +61,10 @@ func (c *LRUCache[K, V]) Peek(key K) (value V, ok bool) { } func (c *LRUCache[K, V]) Purge() { - c.init() c.inner.Purge() } func (c *LRUCache[K, V]) Remove(key K) { - c.init() c.inner.Remove(key) } diff --git a/lib/textui/log_memstats.go b/lib/textui/log_memstats.go index 39733c6..6e3c3a1 100644 --- a/lib/textui/log_memstats.go +++ b/lib/textui/log_memstats.go @@ -19,7 +19,7 @@ type LiveMemUse struct { var _ fmt.Stringer = (*LiveMemUse)(nil) -const liveMemUseUpdateInterval = 1 * time.Second +var liveMemUseUpdateInterval = Tunable(1 * time.Second) func (o *LiveMemUse) String() string { o.mu.Lock() diff --git a/lib/textui/tunable.go b/lib/textui/tunable.go new file mode 100644 index 0000000..7e5f311 --- /dev/null +++ b/lib/textui/tunable.go @@ -0,0 +1,13 @@ +// Copyright (C) 2022 Luke Shumaker +// +// SPDX-License-Identifier: GPL-2.0-or-later + +package textui + +// Tunable annotates a value as something that might want to be tuned +// as the program gets optimized. +// +// TODO(lukeshu): Have Tunable be runtime-configurable. +func Tunable[T any](x T) T { + return x +} -- cgit v1.2.3-2-g168b