copy: fix quota for FsFileCopier

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino 2024-03-12 08:43:23 +01:00
parent f38966c6ac
commit ca2757d41e
No known key found for this signature in database
GPG key ID: 935D2952DEC4EECF
7 changed files with 89 additions and 32 deletions

8
go.mod
View file

@ -3,7 +3,7 @@ module github.com/drakkan/sftpgo/v2
go 1.22 go 1.22
require ( require (
cloud.google.com/go/storage v1.39.0 cloud.google.com/go/storage v1.39.1
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5
@ -175,9 +175,9 @@ require (
golang.org/x/tools v0.19.0 // indirect golang.org/x/tools v0.19.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/appengine v1.6.8 // indirect google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20240311132316-a219d84964c2 // indirect google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect
google.golang.org/grpc v1.62.1 // indirect google.golang.org/grpc v1.62.1 // indirect
google.golang.org/protobuf v1.33.0 // indirect google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect

16
go.sum
View file

@ -9,8 +9,8 @@ cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc=
cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI=
cloud.google.com/go/kms v1.15.7 h1:7caV9K3yIxvlQPAcaFffhlT7d1qpxjB1wHBtjWa13SM= cloud.google.com/go/kms v1.15.7 h1:7caV9K3yIxvlQPAcaFffhlT7d1qpxjB1wHBtjWa13SM=
cloud.google.com/go/kms v1.15.7/go.mod h1:ub54lbsa6tDkUwnu4W7Yt1aAIFLnspgh0kPGToDukeI= cloud.google.com/go/kms v1.15.7/go.mod h1:ub54lbsa6tDkUwnu4W7Yt1aAIFLnspgh0kPGToDukeI=
cloud.google.com/go/storage v1.39.0 h1:brbjUa4hbDHhpQf48tjqMaXEV+f1OGoaTmQau9tmCsA= cloud.google.com/go/storage v1.39.1 h1:MvraqHKhogCOTXTlct/9C3K3+Uy2jBmFYb3/Sp6dVtY=
cloud.google.com/go/storage v1.39.0/go.mod h1:OAEj/WZwUYjA3YHQ10/YcN9ttGuEpLwvaoyBXIPikEk= cloud.google.com/go/storage v1.39.1/go.mod h1:xK6xZmxZmo+fyP7+DEF6FhNc24/JAe95OLyOHCXFH1o=
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU=
@ -535,12 +535,12 @@ google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJ
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20240311132316-a219d84964c2 h1:rrOOzm+NteCjTNqCnDAdYhvKL1G/9N/Lj1GRxJtQEL0= google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 h1:ImUcDPHjTrAqNhlOkSocDLfG9rrNHH7w7uoKWPaWZ8s=
google.golang.org/genproto v0.0.0-20240311132316-a219d84964c2/go.mod h1:yA7a1bW1kwl459Ol0m0lV4hLTfrL/7Bkk4Mj2Ir1mWI= google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7/go.mod h1:/3XmxOjePkvmKrHuBy4zNFw7IzxJXtAgdpXi8Ll990U=
google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ= google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 h1:oqta3O3AnlWbmIE3bFnWbu4bRxZjfbWCp0cKSuZh01E=
google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 h1:9IZDv+/GcI6u+a4jRFRLxQs0RUCfavGfoOgEW6jpkI0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 h1:8EeVk1VKMD+GD/neyEHGmz7pFblqPjHoi+PGQIlLx2s=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=

View file

@ -628,7 +628,14 @@ func (c *BaseConnection) copyFile(virtualSourcePath, virtualTargetPath string, s
if err != nil { if err != nil {
return err return err
} }
return copier.CopyFile(fsSourcePath, fsTargetPath, srcSize) startTime := time.Now()
numFiles, sizeDiff, err := copier.CopyFile(fsSourcePath, fsTargetPath, srcSize)
elapsed := time.Since(startTime).Nanoseconds() / 1000000
updateUserQuotaAfterFileWrite(c, virtualTargetPath, numFiles, sizeDiff)
logger.CommandLog(copyLogSender, fsSourcePath, fsTargetPath, c.User.Username, "", c.ID, c.protocol, -1, -1,
"", "", "", srcSize, c.localAddr, c.remoteAddr, elapsed)
ExecuteActionNotification(c, operationCopy, fsSourcePath, virtualSourcePath, fsTargetPath, virtualTargetPath, "", srcSize, err, elapsed, nil) //nolint:errcheck
return err
} }
} }

View file

@ -651,8 +651,22 @@ func (fs *AzureBlobFs) ResolvePath(virtualPath string) (string, error) {
} }
// CopyFile implements the FsFileCopier interface // CopyFile implements the FsFileCopier interface
func (fs *AzureBlobFs) CopyFile(source, target string, _ int64) error { func (fs *AzureBlobFs) CopyFile(source, target string, srcSize int64) (int, int64, error) {
return fs.copyFileInternal(source, target) numFiles := 1
sizeDiff := srcSize
attrs, err := fs.headObject(target)
if err == nil {
sizeDiff -= util.GetIntFromPointer(attrs.ContentLength)
numFiles = 0
} else {
if !fs.IsNotExist(err) {
return 0, 0, err
}
}
if err := fs.copyFileInternal(source, target); err != nil {
return 0, 0, err
}
return numFiles, sizeDiff, nil
} }
func (fs *AzureBlobFs) headObject(name string) (blob.GetPropertiesResponse, error) { func (fs *AzureBlobFs) headObject(name string) (blob.GetPropertiesResponse, error) {

View file

@ -636,8 +636,25 @@ func (fs *GCSFs) ResolvePath(virtualPath string) (string, error) {
} }
// CopyFile implements the FsFileCopier interface // CopyFile implements the FsFileCopier interface
func (fs *GCSFs) CopyFile(source, target string, _ int64) error { func (fs *GCSFs) CopyFile(source, target string, srcSize int64) (int, int64, error) {
return fs.copyFileInternal(source, target) numFiles := 1
sizeDiff := srcSize
var conditions *storage.Conditions
attrs, err := fs.headObject(target)
if err == nil {
sizeDiff -= attrs.Size
numFiles = 0
conditions = &storage.Conditions{GenerationMatch: attrs.Generation}
} else {
if !fs.IsNotExist(err) {
return 0, 0, err
}
conditions = &storage.Conditions{DoesNotExist: true}
}
if err := fs.copyFileInternal(source, target, conditions); err != nil {
return 0, 0, err
}
return numFiles, sizeDiff, nil
} }
func (fs *GCSFs) resolve(name, prefix, contentType string) (string, bool) { func (fs *GCSFs) resolve(name, prefix, contentType string) (string, bool) {
@ -732,17 +749,21 @@ func (fs *GCSFs) composeObjects(ctx context.Context, dst, partialObject *storage
return err return err
} }
func (fs *GCSFs) copyFileInternal(source, target string) error { func (fs *GCSFs) copyFileInternal(source, target string, conditions *storage.Conditions) error {
src := fs.svc.Bucket(fs.config.Bucket).Object(source) src := fs.svc.Bucket(fs.config.Bucket).Object(source)
dst := fs.svc.Bucket(fs.config.Bucket).Object(target) dst := fs.svc.Bucket(fs.config.Bucket).Object(target)
attrs, statErr := fs.headObject(target) if conditions != nil {
if statErr == nil { dst = dst.If(*conditions)
dst = dst.If(storage.Conditions{GenerationMatch: attrs.Generation})
} else if fs.IsNotExist(statErr) {
dst = dst.If(storage.Conditions{DoesNotExist: true})
} else { } else {
fsLog(fs, logger.LevelWarn, "unable to set precondition for rename, target %q, stat err: %v", attrs, err := fs.headObject(target)
target, statErr) if err == nil {
dst = dst.If(storage.Conditions{GenerationMatch: attrs.Generation})
} else if fs.IsNotExist(err) {
dst = dst.If(storage.Conditions{DoesNotExist: true})
} else {
fsLog(fs, logger.LevelWarn, "unable to set precondition for copy, target %q, stat err: %v",
target, err)
}
} }
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout)) ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout))
@ -790,7 +811,7 @@ func (fs *GCSFs) renameInternal(source, target string, fi os.FileInfo, recursion
} }
} }
} else { } else {
if err := fs.copyFileInternal(source, target); err != nil { if err := fs.copyFileInternal(source, target, nil); err != nil {
return numFiles, filesSize, err return numFiles, filesSize, err
} }
numFiles++ numFiles++

View file

@ -56,8 +56,9 @@ import (
const ( const (
// using this mime type for directories improves compatibility with s3fs-fuse // using this mime type for directories improves compatibility with s3fs-fuse
s3DirMimeType = "application/x-directory" s3DirMimeType = "application/x-directory"
s3TransferBufferSize = 256 * 1024 s3TransferBufferSize = 256 * 1024
s3CopyObjectThreshold = 500 * 1024 * 1024
) )
var ( var (
@ -626,8 +627,22 @@ func (fs *S3Fs) ResolvePath(virtualPath string) (string, error) {
} }
// CopyFile implements the FsFileCopier interface // CopyFile implements the FsFileCopier interface
func (fs *S3Fs) CopyFile(source, target string, srcSize int64) error { func (fs *S3Fs) CopyFile(source, target string, srcSize int64) (int, int64, error) {
return fs.copyFileInternal(source, target, srcSize) numFiles := 1
sizeDiff := srcSize
attrs, err := fs.headObject(target)
if err == nil {
sizeDiff -= util.GetIntFromPointer(attrs.ContentLength)
numFiles = 0
} else {
if !fs.IsNotExist(err) {
return 0, 0, err
}
}
if err := fs.copyFileInternal(source, target, srcSize); err != nil {
return 0, 0, err
}
return numFiles, sizeDiff, nil
} }
func (fs *S3Fs) resolve(name *string, prefix string) (string, bool) { func (fs *S3Fs) resolve(name *string, prefix string) (string, bool) {
@ -666,7 +681,7 @@ func (fs *S3Fs) copyFileInternal(source, target string, fileSize int64) error {
contentType := mime.TypeByExtension(path.Ext(source)) contentType := mime.TypeByExtension(path.Ext(source))
copySource := pathEscape(fs.Join(fs.config.Bucket, source)) copySource := pathEscape(fs.Join(fs.config.Bucket, source))
if fileSize > 500*1024*1024 { if fileSize > s3CopyObjectThreshold {
fsLog(fs, logger.LevelDebug, "renaming file %q with size %d using multipart copy", fsLog(fs, logger.LevelDebug, "renaming file %q with size %d using multipart copy",
source, fileSize) source, fileSize)
err := fs.doMultipartCopy(copySource, target, contentType, fileSize) err := fs.doMultipartCopy(copySource, target, contentType, fileSize)

View file

@ -159,7 +159,7 @@ type FsRealPather interface {
// FsFileCopier is a Fs that implements the CopyFile method. // FsFileCopier is a Fs that implements the CopyFile method.
type FsFileCopier interface { type FsFileCopier interface {
Fs Fs
CopyFile(source, target string, srcSize int64) error CopyFile(source, target string, srcSize int64) (int, int64, error)
} }
// File defines an interface representing a SFTPGo file // File defines an interface representing a SFTPGo file