From ff4435105d35aff994492517a292e60628bce5bf Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 18 Aug 2022 02:06:53 -0600 Subject: Move ShortSum and friends to the btrfssum package --- lib/btrfsprogs/btrfsinspect/csums.go | 190 --------------------- .../btrfsinspect/rebuildmappings/allsums.go | 18 +- .../btrfsinspect/rebuildmappings/gaps.go | 13 +- .../btrfsinspect/rebuildmappings/scan.go | 37 ++-- lib/btrfsprogs/btrfsinspect/scandevices.go | 15 +- 5 files changed, 41 insertions(+), 232 deletions(-) delete mode 100644 lib/btrfsprogs/btrfsinspect/csums.go (limited to 'lib/btrfsprogs/btrfsinspect') diff --git a/lib/btrfsprogs/btrfsinspect/csums.go b/lib/btrfsprogs/btrfsinspect/csums.go deleted file mode 100644 index e690d24..0000000 --- a/lib/btrfsprogs/btrfsinspect/csums.go +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright (C) 2022 Luke Shumaker -// -// SPDX-License-Identifier: GPL-2.0-or-later - -package btrfsinspect - -import ( - "context" - "fmt" - "io" - "strings" - - "git.lukeshu.com/go/lowmemjson" - - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" - "git.lukeshu.com/btrfs-progs-ng/lib/diskio" -) - -// ShortSum ////////////////////////////////////////////////////////// - -type ShortSum string - -var ( - _ lowmemjson.Encodable = ShortSum("") - _ lowmemjson.Decodable = (*ShortSum)(nil) -) - -func (sum ShortSum) EncodeJSON(w io.Writer) error { - const hextable = "0123456789abcdef" - var buf [2]byte - buf[0] = '"' - if _, err := w.Write(buf[:1]); err != nil { - return err - } - for i := 0; i < len(sum); i++ { - buf[0] = hextable[sum[i]>>4] - buf[1] = hextable[sum[i]&0x0f] - if _, err := w.Write(buf[:]); err != nil { - return err - } - } - buf[0] = '"' - if _, err := w.Write(buf[:1]); err != nil { - return err - } - return nil -} - -func deHex(r rune) (byte, bool) { - if r > 0xff { - return 0, false - } - c := byte(r) - switch { - case '0' <= c && c <= '9': - return c - '0', true - case 'a' <= c && c <= 'f': - return c - 'a' + 10, true - case 'A' <= c && c <= 'F': - return c - 'A' + 10, true - default: - return 0, false - } -} - -func (sum *ShortSum) DecodeJSON(r io.RuneScanner) error { - var out strings.Builder - if c, _, err := r.ReadRune(); err != nil { - return err - } else if c != '"' { - return fmt.Errorf("expected %q, got %q", '"', c) - } - for { - a, _, err := r.ReadRune() - if err != nil { - return err - } - if a == '"' { - break - } - aN, ok := deHex(a) - if !ok { - return fmt.Errorf("expected a hex digit, got %q", a) - } - b, _, err := r.ReadRune() - if err != nil { - return err - } - bN, ok := deHex(b) - if !ok { - return fmt.Errorf("expected a hex digit, got %q", b) - } - out.WriteByte(aN<<4 | bN) - } - *sum = ShortSum(out.String()) - return nil -} - -// SumRun //////////////////////////////////////////////////////////// - -type SumRun[Addr btrfsvol.IntAddr[Addr]] struct { - // How big a ShortSum is in this Run. - ChecksumSize int - // Base address where this run starts. - Addr Addr `json:",omitempty"` - // All of the ShortSums in this run, concatenated together. - Sums ShortSum -} - -func (run SumRun[Addr]) NumSums() int { - return len(run.Sums) / run.ChecksumSize -} - -func (run SumRun[Addr]) Size() btrfsvol.AddrDelta { - return btrfsvol.AddrDelta(run.NumSums()) * btrfsitem.CSumBlockSize -} - -// Get implements diskio.Sequence[int, ShortSum] -func (run SumRun[Addr]) Get(sumIdx int64) (ShortSum, error) { - if sumIdx < 0 || int(sumIdx) >= run.NumSums() { - return "", io.EOF - } - off := int(sumIdx) * run.ChecksumSize - return ShortSum(run.Sums[off : off+run.ChecksumSize]), nil -} - -func (run SumRun[Addr]) SumForAddr(addr Addr) (ShortSum, bool) { - if addr < run.Addr || addr >= run.Addr.Add(run.Size()) { - return "", false - } - off := int((addr-run.Addr)/btrfsitem.CSumBlockSize) * run.ChecksumSize - return ShortSum(run.Sums[off : off+run.ChecksumSize]), true -} - -func (run SumRun[Addr]) Walk(ctx context.Context, fn func(Addr, ShortSum) error) error { - for addr, off := run.Addr, 0; off < len(run.Sums); addr, off = addr+btrfsitem.CSumBlockSize, off+run.ChecksumSize { - if err := ctx.Err(); err != nil { - return err - } - if err := fn(addr, ShortSum(run.Sums[off:off+run.ChecksumSize])); err != nil { - return err - } - } - return nil -} - -// SumRunWithGaps //////////////////////////////////////////////////// - -type SumRunWithGaps[Addr btrfsvol.IntAddr[Addr]] struct { - Addr Addr - Size btrfsvol.AddrDelta - Runs []SumRun[Addr] -} - -func (sg SumRunWithGaps[Addr]) NumSums() int { - return int(sg.Size / btrfsitem.CSumBlockSize) -} - -func (sg SumRunWithGaps[Addr]) PctFull() float64 { - total := sg.NumSums() - var full int - for _, run := range sg.Runs { - full += run.NumSums() - } - return float64(full) / float64(total) -} - -func (sg SumRunWithGaps[Addr]) SumForAddr(addr Addr) (ShortSum, error) { - if addr < sg.Addr || addr >= sg.Addr.Add(sg.Size) { - return "", io.EOF - } - for _, run := range sg.Runs { - if run.Addr > addr { - return "", diskio.ErrWildcard - } - if run.Addr.Add(run.Size()) <= addr { - continue - } - off := int((addr-run.Addr)/btrfsitem.CSumBlockSize) * run.ChecksumSize - return ShortSum(run.Sums[off : off+run.ChecksumSize]), nil - } - return "", diskio.ErrWildcard -} - -// Get implements diskio.Sequence[int, ShortSum] -func (sg SumRunWithGaps[Addr]) Get(sumIdx int64) (ShortSum, error) { - addr := sg.Addr.Add(btrfsvol.AddrDelta(sumIdx) * btrfsitem.CSumBlockSize) - return sg.SumForAddr(addr) -} diff --git a/lib/btrfsprogs/btrfsinspect/rebuildmappings/allsums.go b/lib/btrfsprogs/btrfsinspect/rebuildmappings/allsums.go index 1a8855b..58ba018 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildmappings/allsums.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildmappings/allsums.go @@ -8,16 +8,16 @@ import ( "context" "math" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfssum" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect" ) type AllSums struct { - Logical []btrfsinspect.SumRun[btrfsvol.LogicalAddr] - Physical map[btrfsvol.DeviceID]btrfsinspect.SumRun[btrfsvol.PhysicalAddr] + Logical []btrfssum.SumRun[btrfsvol.LogicalAddr] + Physical map[btrfsvol.DeviceID]btrfssum.SumRun[btrfsvol.PhysicalAddr] } -func (as AllSums) SumForPAddr(paddr btrfsvol.QualifiedPhysicalAddr) (btrfsinspect.ShortSum, bool) { +func (as AllSums) SumForPAddr(paddr btrfsvol.QualifiedPhysicalAddr) (btrfssum.ShortSum, bool) { run, ok := as.Physical[paddr.Dev] if !ok { return "", false @@ -25,20 +25,20 @@ func (as AllSums) SumForPAddr(paddr btrfsvol.QualifiedPhysicalAddr) (btrfsinspec return run.SumForAddr(paddr.Addr) } -func (as AllSums) RunForLAddr(laddr btrfsvol.LogicalAddr) (btrfsinspect.SumRun[btrfsvol.LogicalAddr], btrfsvol.LogicalAddr, bool) { +func (as AllSums) RunForLAddr(laddr btrfsvol.LogicalAddr) (btrfssum.SumRun[btrfsvol.LogicalAddr], btrfsvol.LogicalAddr, bool) { for _, run := range as.Logical { if run.Addr > laddr { - return btrfsinspect.SumRun[btrfsvol.LogicalAddr]{}, run.Addr, false + return btrfssum.SumRun[btrfsvol.LogicalAddr]{}, run.Addr, false } if run.Addr.Add(run.Size()) <= laddr { continue } return run, 0, true } - return btrfsinspect.SumRun[btrfsvol.LogicalAddr]{}, math.MaxInt64, false + return btrfssum.SumRun[btrfsvol.LogicalAddr]{}, math.MaxInt64, false } -func (as AllSums) SumForLAddr(laddr btrfsvol.LogicalAddr) (btrfsinspect.ShortSum, bool) { +func (as AllSums) SumForLAddr(laddr btrfsvol.LogicalAddr) (btrfssum.ShortSum, bool) { run, _, ok := as.RunForLAddr(laddr) if !ok { return "", false @@ -46,7 +46,7 @@ func (as AllSums) SumForLAddr(laddr btrfsvol.LogicalAddr) (btrfsinspect.ShortSum return run.SumForAddr(laddr) } -func (as AllSums) WalkLogical(ctx context.Context, fn func(btrfsvol.LogicalAddr, btrfsinspect.ShortSum) error) error { +func (as AllSums) WalkLogical(ctx context.Context, fn func(btrfsvol.LogicalAddr, btrfssum.ShortSum) error) error { for _, run := range as.Logical { if err := run.Walk(ctx, fn); err != nil { return err diff --git a/lib/btrfsprogs/btrfsinspect/rebuildmappings/gaps.go b/lib/btrfsprogs/btrfsinspect/rebuildmappings/gaps.go index 418dba6..987aed7 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildmappings/gaps.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildmappings/gaps.go @@ -11,9 +11,8 @@ import ( "golang.org/x/exp/constraints" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfssum" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect" "git.lukeshu.com/btrfs-progs-ng/lib/maps" ) @@ -57,17 +56,17 @@ func roundUp[T constraints.Integer](x, multiple T) T { func WalkGaps(ctx context.Context, sums AllSums, gaps map[btrfsvol.DeviceID][]PhysicalGap, - fn func(btrfsvol.DeviceID, btrfsinspect.SumRun[btrfsvol.PhysicalAddr]) error, + fn func(btrfsvol.DeviceID, btrfssum.SumRun[btrfsvol.PhysicalAddr]) error, ) error { for _, devID := range maps.SortedKeys(gaps) { for _, gap := range gaps[devID] { if err := ctx.Err(); err != nil { return err } - begAddr := roundUp(gap.Beg, btrfsitem.CSumBlockSize) - begOff := int(begAddr/btrfsitem.CSumBlockSize) * sums.Physical[devID].ChecksumSize - endOff := int(gap.End/btrfsitem.CSumBlockSize) * sums.Physical[devID].ChecksumSize - if err := fn(devID, btrfsinspect.SumRun[btrfsvol.PhysicalAddr]{ + begAddr := roundUp(gap.Beg, btrfssum.BlockSize) + begOff := int(begAddr/btrfssum.BlockSize) * sums.Physical[devID].ChecksumSize + endOff := int(gap.End/btrfssum.BlockSize) * sums.Physical[devID].ChecksumSize + if err := fn(devID, btrfssum.SumRun[btrfsvol.PhysicalAddr]{ ChecksumSize: sums.Physical[devID].ChecksumSize, Addr: begAddr, Sums: sums.Physical[devID].Sums[begOff:endOff], diff --git a/lib/btrfsprogs/btrfsinspect/rebuildmappings/scan.go b/lib/btrfsprogs/btrfsinspect/rebuildmappings/scan.go index d37d00f..efb1775 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildmappings/scan.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildmappings/scan.go @@ -11,9 +11,8 @@ import ( "github.com/datawire/dlib/dlog" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfssum" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect" "git.lukeshu.com/btrfs-progs-ng/lib/containers" "git.lukeshu.com/btrfs-progs-ng/lib/diskio" "git.lukeshu.com/btrfs-progs-ng/lib/maps" @@ -22,10 +21,10 @@ import ( func ScanForExtents(ctx context.Context, fs *btrfs.FS, blockgroups map[btrfsvol.LogicalAddr]BlockGroup, sums AllSums) error { dlog.Info(ctx, "Pairing up blockgroups and sums...") - bgSums := make(map[btrfsvol.LogicalAddr]btrfsinspect.SumRunWithGaps[btrfsvol.LogicalAddr]) + bgSums := make(map[btrfsvol.LogicalAddr]btrfssum.SumRunWithGaps[btrfsvol.LogicalAddr]) for i, bgLAddr := range maps.SortedKeys(blockgroups) { blockgroup := blockgroups[bgLAddr] - runs := btrfsinspect.SumRunWithGaps[btrfsvol.LogicalAddr]{ + runs := btrfssum.SumRunWithGaps[btrfsvol.LogicalAddr]{ Addr: blockgroup.LAddr, Size: blockgroup.Size, } @@ -35,12 +34,12 @@ func ScanForExtents(ctx context.Context, fs *btrfs.FS, blockgroups map[btrfsvol. laddr = next continue } - off := int((laddr-run.Addr)/btrfsitem.CSumBlockSize) * run.ChecksumSize + off := int((laddr-run.Addr)/btrfssum.BlockSize) * run.ChecksumSize deltaAddr := slices.Min[btrfsvol.AddrDelta]( blockgroup.Size-laddr.Sub(blockgroup.LAddr), - btrfsvol.AddrDelta((len(run.Sums)-off)/run.ChecksumSize)*btrfsitem.CSumBlockSize) - deltaOff := int(deltaAddr/btrfsitem.CSumBlockSize) * run.ChecksumSize - runs.Runs = append(runs.Runs, btrfsinspect.SumRun[btrfsvol.LogicalAddr]{ + btrfsvol.AddrDelta((len(run.Sums)-off)/run.ChecksumSize)*btrfssum.BlockSize) + deltaOff := int(deltaAddr/btrfssum.BlockSize) * run.ChecksumSize + runs.Runs = append(runs.Runs, btrfssum.SumRun[btrfsvol.LogicalAddr]{ ChecksumSize: run.ChecksumSize, Addr: laddr, Sums: run.Sums[off : off+deltaOff], @@ -64,15 +63,15 @@ func ScanForExtents(ctx context.Context, fs *btrfs.FS, blockgroups map[btrfsvol. continue } - if err := WalkGaps(ctx, sums, gaps, func(devID btrfsvol.DeviceID, gap btrfsinspect.SumRun[btrfsvol.PhysicalAddr]) error { - matches, err := diskio.IndexAll[int64, btrfsinspect.ShortSum](gap, bgRun) + if err := WalkGaps(ctx, sums, gaps, func(devID btrfsvol.DeviceID, gap btrfssum.SumRun[btrfsvol.PhysicalAddr]) error { + matches, err := diskio.IndexAll[int64, btrfssum.ShortSum](gap, bgRun) if err != nil { return err } for _, match := range matches { bgMatches[bgLAddr] = append(bgMatches[bgLAddr], btrfsvol.QualifiedPhysicalAddr{ Dev: devID, - Addr: gap.Addr + (btrfsvol.PhysicalAddr(match) * btrfsitem.CSumBlockSize), + Addr: gap.Addr + (btrfsvol.PhysicalAddr(match) * btrfssum.BlockSize), }) } return nil @@ -113,10 +112,10 @@ func ScanForExtents(ctx context.Context, fs *btrfs.FS, blockgroups map[btrfsvol. dlog.Info(ctx, "... done applying") dlog.Info(ctx, "Reverse-indexing remaining unmapped logical sums...") - sum2laddrs := make(map[btrfsinspect.ShortSum][]btrfsvol.LogicalAddr) + sum2laddrs := make(map[btrfssum.ShortSum][]btrfsvol.LogicalAddr) var numUnmappedBlocks int64 - if err := sums.WalkLogical(ctx, func(laddr btrfsvol.LogicalAddr, sum btrfsinspect.ShortSum) error { - var dat [btrfsitem.CSumBlockSize]byte + if err := sums.WalkLogical(ctx, func(laddr btrfsvol.LogicalAddr, sum btrfssum.ShortSum) error { + var dat [btrfssum.BlockSize]byte if _, err := fs.ReadAt(dat[:], laddr); err != nil { if errors.Is(err, btrfsvol.ErrCouldNotMap) { sum2laddrs[sum] = append(sum2laddrs[sum], laddr) @@ -189,12 +188,12 @@ type ExtentMappings struct { } func (em *ExtentMappings) considerMapping(ctx context.Context, laddr btrfsvol.LogicalAddr, paddr btrfsvol.QualifiedPhysicalAddr) (btrfsvol.Mapping, bool) { - blockgroup := LookupBlockGroup(em.InBlockGroups, laddr, btrfsitem.CSumBlockSize) + blockgroup := LookupBlockGroup(em.InBlockGroups, laddr, btrfssum.BlockSize) if blockgroup == nil { return btrfsvol.Mapping{ LAddr: laddr, PAddr: paddr, - Size: btrfsitem.CSumBlockSize, + Size: btrfssum.BlockSize, }, true } mapping := btrfsvol.Mapping{ @@ -214,7 +213,7 @@ func (em *ExtentMappings) considerMapping(ctx context.Context, laddr btrfsvol.Lo return btrfsvol.Mapping{}, false } - for offset := btrfsvol.AddrDelta(0); offset <= mapping.Size; offset += btrfsitem.CSumBlockSize { + for offset := btrfsvol.AddrDelta(0); offset <= mapping.Size; offset += btrfssum.BlockSize { expCSum, ok := em.InSums.SumForLAddr(mapping.LAddr.Add(offset)) if !ok { continue @@ -252,7 +251,7 @@ func (em *ExtentMappings) ScanOneDevice( dlog.Infof(ctx, "... dev[%q] Scanning for extents...", devName) var totalMappings int - _ = WalkGaps(ctx, gaps, btrfsitem.CSumBlockSize, + _ = WalkGaps(ctx, gaps, btrfssum.BlockSize, func(_, _ int64) {}, func(paddr btrfsvol.PhysicalAddr) error { qpaddr := btrfsvol.QualifiedPhysicalAddr{ @@ -276,7 +275,7 @@ func (em *ExtentMappings) ScanOneDevice( lastProgress = pct } } - return WalkGaps(ctx, gaps, btrfsitem.CSumBlockSize, + return WalkGaps(ctx, gaps, btrfssum.BlockSize, func(_, _ int64) { progress() }, diff --git a/lib/btrfsprogs/btrfsinspect/scandevices.go b/lib/btrfsprogs/btrfsinspect/scandevices.go index f4da3dc..33790df 100644 --- a/lib/btrfsprogs/btrfsinspect/scandevices.go +++ b/lib/btrfsprogs/btrfsinspect/scandevices.go @@ -17,6 +17,7 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/binstruct" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfssum" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsutil" ) @@ -52,7 +53,7 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS) (ScanDevicesResult, error) { } type ScanOneDeviceResult struct { - Checksums SumRun[btrfsvol.PhysicalAddr] + Checksums btrfssum.SumRun[btrfsvol.PhysicalAddr] FoundNodes map[btrfsvol.LogicalAddr][]btrfsvol.PhysicalAddr FoundChunks []btrfs.SysChunk FoundBlockGroups []SysBlockGroup @@ -90,14 +91,14 @@ func ScanOneDevice(ctx context.Context, dev *btrfs.Device, sb btrfs.Superblock) return result, fmt.Errorf("node_size(%v) < sector_size(%v)", sb.NodeSize, sb.SectorSize) } - if sb.SectorSize != btrfsitem.CSumBlockSize { + if sb.SectorSize != btrfssum.BlockSize { // TODO: probably handle this? - return result, fmt.Errorf("sector_size(%v) != btrfsitem.CSumBlockSize", + return result, fmt.Errorf("sector_size(%v) != btrfssum.BlockSize", sb.SectorSize) } alg := sb.ChecksumType csumSize := alg.Size() - numSums := int(devSize / btrfsitem.CSumBlockSize) + numSums := int(devSize / btrfssum.BlockSize) var sums strings.Builder sums.Grow(numSums * csumSize) @@ -121,7 +122,7 @@ func ScanOneDevice(ctx context.Context, dev *btrfs.Device, sb btrfs.Superblock) if ctx.Err() != nil { return result, ctx.Err() } - pos := btrfsvol.PhysicalAddr(i * btrfsitem.CSumBlockSize) + pos := btrfsvol.PhysicalAddr(i * btrfssum.BlockSize) progress(pos) sum, err := btrfsutil.ChecksumPhysical(dev, alg, pos) @@ -211,9 +212,9 @@ func ScanOneDevice(ctx context.Context, dev *btrfs.Device, sb btrfs.Superblock) } progress(devSize) - result.Checksums = SumRun[btrfsvol.PhysicalAddr]{ + result.Checksums = btrfssum.SumRun[btrfsvol.PhysicalAddr]{ ChecksumSize: csumSize, - Sums: ShortSum(sums.String()), + Sums: btrfssum.ShortSum(sums.String()), } return result, nil -- cgit v1.2.3-2-g168b