From cdda325c80a202dd6af861b912f4b3fa161dab5e Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 6 Oct 2022 02:07:53 -0600 Subject: check the sums of file reads --- lib/btrfs/btrfssum/shortsum.go | 6 +++ lib/btrfs/csums.go | 64 ++++++++++++++++++++++++++++++ lib/btrfs/io4_fs.go | 51 ++++++++++++++++++++---- lib/btrfsprogs/btrfsinspect/scandevices.go | 3 +- lib/btrfsprogs/btrfsutil/csums.go | 63 ----------------------------- 5 files changed, 115 insertions(+), 72 deletions(-) create mode 100644 lib/btrfs/csums.go delete mode 100644 lib/btrfsprogs/btrfsutil/csums.go diff --git a/lib/btrfs/btrfssum/shortsum.go b/lib/btrfs/btrfssum/shortsum.go index 537e0ce..6fd0c68 100644 --- a/lib/btrfs/btrfssum/shortsum.go +++ b/lib/btrfs/btrfssum/shortsum.go @@ -28,6 +28,12 @@ var ( _ lowmemjson.Decodable = (*ShortSum)(nil) ) +func (sum ShortSum) ToFullSum() CSum { + var ret CSum + copy(ret[:], sum) + return ret +} + func (sum ShortSum) EncodeJSON(w io.Writer) error { const hextable = "0123456789abcdef" var buf [2]byte diff --git a/lib/btrfs/csums.go b/lib/btrfs/csums.go new file mode 100644 index 0000000..bbd19bd --- /dev/null +++ b/lib/btrfs/csums.go @@ -0,0 +1,64 @@ +// Copyright (C) 2022 Luke Shumaker +// +// SPDX-License-Identifier: GPL-2.0-or-later + +package btrfs + +import ( + "fmt" + + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsprim" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfssum" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfstree" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" + "git.lukeshu.com/btrfs-progs-ng/lib/diskio" +) + +func ChecksumLogical(fs diskio.File[btrfsvol.LogicalAddr], alg btrfssum.CSumType, laddr btrfsvol.LogicalAddr) (btrfssum.CSum, error) { + var dat [btrfssum.BlockSize]byte + if _, err := fs.ReadAt(dat[:], laddr); err != nil { + return btrfssum.CSum{}, err + } + return alg.Sum(dat[:]) +} + +func ChecksumPhysical(dev *Device, alg btrfssum.CSumType, paddr btrfsvol.PhysicalAddr) (btrfssum.CSum, error) { + var dat [btrfssum.BlockSize]byte + if _, err := dev.ReadAt(dat[:], paddr); err != nil { + return btrfssum.CSum{}, err + } + return alg.Sum(dat[:]) +} + +func ChecksumQualifiedPhysical(fs *FS, alg btrfssum.CSumType, paddr btrfsvol.QualifiedPhysicalAddr) (btrfssum.CSum, error) { + dev := fs.LV.PhysicalVolumes()[paddr.Dev] + if dev == nil { + return btrfssum.CSum{}, fmt.Errorf("no such device_id=%v", paddr.Dev) + } + return ChecksumPhysical(dev, alg, paddr.Addr) +} + +func LookupCSum(fs btrfstree.TreeOperator, alg btrfssum.CSumType, laddr btrfsvol.LogicalAddr) (btrfssum.SumRun[btrfsvol.LogicalAddr], error) { + item, err := fs.TreeSearch(btrfsprim.CSUM_TREE_OBJECTID, func(key btrfsprim.Key, size uint32) int { + itemBeg := btrfsvol.LogicalAddr(key.Offset) + numSums := int64(size) / int64(alg.Size()) + itemEnd := itemBeg + btrfsvol.LogicalAddr(numSums*btrfssum.BlockSize) + switch { + case itemEnd <= laddr: + return 1 + case laddr < itemBeg: + return -1 + default: + return 0 + } + }) + if err != nil { + return btrfssum.SumRun[btrfsvol.LogicalAddr]{}, err + } + body, ok := item.Body.(btrfsitem.ExtentCSum) + if !ok { + return btrfssum.SumRun[btrfsvol.LogicalAddr]{}, fmt.Errorf("item body is %T not ExtentCSum", item.Body) + } + return body.SumRun, nil +} diff --git a/lib/btrfs/io4_fs.go b/lib/btrfs/io4_fs.go index 82d4c87..29f481c 100644 --- a/lib/btrfs/io4_fs.go +++ b/lib/btrfs/io4_fs.go @@ -16,6 +16,7 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsprim" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfssum" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfstree" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" "git.lukeshu.com/btrfs-progs-ng/lib/containers" @@ -366,8 +367,8 @@ func (ret *File) populate() { func (file *File) ReadAt(dat []byte, off int64) (int, error) { // These stateless maybe-short-reads each do an O(n) extent // lookup, so reading a file is O(n^2), but we expect n to be - // small, so whatev. Turn file.Extents it in to an rbtree if - // it becomes a problem. + // small, so whatev. Turn file.Extents in to an rbtree if it + // becomes a problem. done := 0 for done < len(dat) { n, err := file.maybeShortReadAt(dat[done:], off+int64(done)) @@ -394,15 +395,51 @@ func (file *File) maybeShortReadAt(dat []byte, off int64) (int, error) { continue } offsetWithinExt := off - extent.OffsetWithinFile - readSize := slices.Min(int64(len(dat)), extLen-offsetWithinExt) + readSize := slices.Min(int64(len(dat)), extLen-offsetWithinExt, btrfssum.BlockSize) switch extent.Type { case btrfsitem.FILE_EXTENT_INLINE: return copy(dat, extent.BodyInline[offsetWithinExt:offsetWithinExt+readSize]), nil case btrfsitem.FILE_EXTENT_REG, btrfsitem.FILE_EXTENT_PREALLOC: - return file.SV.FS.ReadAt(dat[:readSize], - extent.BodyExtent.DiskByteNr. - Add(extent.BodyExtent.Offset). - Add(btrfsvol.AddrDelta(offsetWithinExt))) + sb, err := file.SV.FS.Superblock() + if err != nil { + return 0, err + } + var beg btrfsvol.LogicalAddr = extent.BodyExtent.DiskByteNr. + Add(extent.BodyExtent.Offset). + Add(btrfsvol.AddrDelta(offsetWithinExt)) + var block [btrfssum.BlockSize]byte + blockBeg := (beg / btrfssum.BlockSize) * btrfssum.BlockSize + n, err := file.SV.FS.ReadAt(block[:], blockBeg) + if n > int(beg-blockBeg) { + n = copy(dat[:readSize], block[beg-blockBeg:]) + } else { + n = 0 + } + if err != nil { + return 0, err + } + + sumRun, err := LookupCSum(file.SV.FS, sb.ChecksumType, blockBeg) + if err != nil { + return 0, fmt.Errorf("checksum@%v: %w", blockBeg, err) + } + _expSum, ok := sumRun.SumForAddr(blockBeg) + if !ok { + panic(fmt.Errorf("run from LookupCSum(fs, typ, %v) did not contain %v: %#v", + blockBeg, blockBeg, sumRun)) + } + expSum := _expSum.ToFullSum() + + actSum, err := sb.ChecksumType.Sum(block[:]) + if err != nil { + return 0, fmt.Errorf("checksum@%v: %w", blockBeg, err) + } + + if actSum != expSum { + return 0, fmt.Errorf("checksum@%v: actual sum %v != expected sum %v", + blockBeg, actSum, expSum) + } + return n, nil } } if file.InodeItem != nil && off >= file.InodeItem.Size { diff --git a/lib/btrfsprogs/btrfsinspect/scandevices.go b/lib/btrfsprogs/btrfsinspect/scandevices.go index c25a86f..87dcedf 100644 --- a/lib/btrfsprogs/btrfsinspect/scandevices.go +++ b/lib/btrfsprogs/btrfsinspect/scandevices.go @@ -21,7 +21,6 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfssum" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfstree" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsutil" ) type ScanDevicesResult map[btrfsvol.DeviceID]ScanOneDeviceResult @@ -126,7 +125,7 @@ func ScanOneDevice(ctx context.Context, dev *btrfs.Device, sb btrfstree.Superblo pos := btrfsvol.PhysicalAddr(i * btrfssum.BlockSize) progress(pos) - sum, err := btrfsutil.ChecksumPhysical(dev, alg, pos) + sum, err := btrfs.ChecksumPhysical(dev, alg, pos) if err != nil { return result, err } diff --git a/lib/btrfsprogs/btrfsutil/csums.go b/lib/btrfsprogs/btrfsutil/csums.go deleted file mode 100644 index a49f584..0000000 --- a/lib/btrfsprogs/btrfsutil/csums.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (C) 2022 Luke Shumaker -// -// SPDX-License-Identifier: GPL-2.0-or-later - -package btrfsutil - -import ( - "fmt" - - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsprim" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfssum" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" -) - -func ChecksumLogical(fs *btrfs.FS, alg btrfssum.CSumType, laddr btrfsvol.LogicalAddr) (btrfssum.CSum, error) { - var dat [btrfssum.BlockSize]byte - if _, err := fs.ReadAt(dat[:], laddr); err != nil { - return btrfssum.CSum{}, err - } - return alg.Sum(dat[:]) -} - -func ChecksumPhysical(dev *btrfs.Device, alg btrfssum.CSumType, paddr btrfsvol.PhysicalAddr) (btrfssum.CSum, error) { - var dat [btrfssum.BlockSize]byte - if _, err := dev.ReadAt(dat[:], paddr); err != nil { - return btrfssum.CSum{}, err - } - return alg.Sum(dat[:]) -} - -func ChecksumQualifiedPhysical(fs *btrfs.FS, alg btrfssum.CSumType, paddr btrfsvol.QualifiedPhysicalAddr) (btrfssum.CSum, error) { - dev := fs.LV.PhysicalVolumes()[paddr.Dev] - if dev == nil { - return btrfssum.CSum{}, fmt.Errorf("no such device_id=%v", paddr.Dev) - } - return ChecksumPhysical(dev, alg, paddr.Addr) -} - -func LookupCSum(fs *btrfs.FS, alg btrfssum.CSumType, laddr btrfsvol.LogicalAddr) (btrfssum.SumRun[btrfsvol.LogicalAddr], error) { - item, err := fs.TreeSearch(btrfsprim.CSUM_TREE_OBJECTID, func(key btrfsprim.Key, size uint32) int { - itemBeg := btrfsvol.LogicalAddr(key.ObjectID) - numSums := int64(size) / int64(alg.Size()) - itemEnd := itemBeg + btrfsvol.LogicalAddr(numSums*btrfssum.BlockSize) - switch { - case itemEnd <= laddr: - return 1 - case laddr < itemBeg: - return -1 - default: - return 0 - } - }) - if err != nil { - return btrfssum.SumRun[btrfsvol.LogicalAddr]{}, err - } - body, ok := item.Body.(btrfsitem.ExtentCSum) - if !ok { - return btrfssum.SumRun[btrfsvol.LogicalAddr]{}, fmt.Errorf("item body is %T not ExtentCSum", item.Body) - } - return body.SumRun, nil -} -- cgit v1.2.3-2-g168b